@remnic/core 1.1.12 → 1.1.13

package/src/bulk-import/pipeline.ts

@@ -0,0 +1,178 @@
+// ---------------------------------------------------------------------------
+// Bulk-import batch processing pipeline
+// ---------------------------------------------------------------------------
+
+import {
+  validateImportTurn,
+  type BulkImportError,
+  type BulkImportOptions,
+  type BulkImportResult,
+  type BulkImportSource,
+  type ImportTurn,
+} from "./types.js";
+
+const DEFAULT_BATCH_SIZE = 20;
+const MIN_BATCH_SIZE = 1;
+const MAX_BATCH_SIZE = 1000;
+
+export interface ProcessBatchResult {
+  memoriesCreated: number;
+  duplicatesSkipped: number;
+  /**
+   * Number of entities created by the batch. Optional so Phase-1 stubs that
+   * only count memories can omit it; when absent it is treated as 0.
+   */
+  entitiesCreated?: number;
+}
+
+export interface ProcessBatchContext {
+  dedup: boolean;
+  trustLevel: "import";
+  namespace?: string;
+}
+
+export type ProcessBatchFn = (
+  turns: ImportTurn[],
+  context: ProcessBatchContext,
+) => Promise<ProcessBatchResult>;
+
+export function validateBatchSize(value: number | undefined): number {
+  if (value === undefined) return DEFAULT_BATCH_SIZE;
+  if (typeof value !== "number" || !Number.isFinite(value)) {
+    throw new Error(
+      `batchSize must be a finite number, received ${String(value)}`,
+    );
+  }
+  if (!Number.isInteger(value)) {
+    throw new Error(
+      `batchSize must be an integer, received ${value}`,
+    );
+  }
+  if (value < MIN_BATCH_SIZE || value > MAX_BATCH_SIZE) {
+    throw new Error(
+      `batchSize must be between ${MIN_BATCH_SIZE} and ${MAX_BATCH_SIZE}, received ${value}`,
+    );
+  }
+  return value;
+}
+
+export function resolveBulkImportContext(
+  options: BulkImportOptions,
+): ProcessBatchContext {
+  if (options.dedup !== undefined && typeof options.dedup !== "boolean") {
+    throw new Error(
+      `dedup must be a boolean when provided, received ${String(options.dedup)}`,
+    );
+  }
+  if (options.trustLevel !== undefined && options.trustLevel !== "import") {
+    throw new Error(
+      `trustLevel must be 'import' when provided, received ${String(options.trustLevel)}`,
+    );
+  }
+  if (options.namespace !== undefined) {
+    if (typeof options.namespace !== "string" || options.namespace.trim().length === 0) {
+      throw new Error(
+        `namespace must be a non-empty string when provided, received ${String(options.namespace)}`,
+      );
+    }
+  }
+
+  return {
+    dedup: options.dedup !== false,
+    trustLevel: options.trustLevel ?? "import",
+    ...(options.namespace !== undefined
+      ? { namespace: options.namespace }
+      : {}),
+  };
+}
+
+/**
+ * Format a batch of turns into a conversation transcript string.
+ */
+export function formatBatchTranscript(turns: ImportTurn[]): string {
+  return turns
+    .map((t) => {
+      const prefix =
+        t.participantName ?? t.participantId ?? t.role;
+      return `[${t.timestamp}] ${prefix}: ${t.content}`;
+    })
+    .join("\n");
+}
+
+/**
+ * Run the bulk-import pipeline over a parsed source.
+ *
+ * Splits turns into batches and delegates each to `processBatch`.
+ * In dryRun mode, validates and counts without calling `processBatch`.
+ */
+export async function runBulkImportPipeline(
+  source: BulkImportSource,
+  options: BulkImportOptions = {},
+  processBatch: ProcessBatchFn,
+): Promise<BulkImportResult> {
+  const batchSize = validateBatchSize(options.batchSize);
+  const dryRun = options.dryRun === true;
+  const batchContext = resolveBulkImportContext(options);
+
+  const result: BulkImportResult = {
+    memoriesCreated: 0,
+    duplicatesSkipped: 0,
+    entitiesCreated: 0,
+    turnsProcessed: 0,
+    batchesProcessed: 0,
+    errors: [],
+  };
+
+  const turns = source.turns;
+
+  if (!turns || turns.length === 0) {
+    return result;
+  }
+
+  // Validate all turns upfront; collect validation errors
+  const validTurns: ImportTurn[] = [];
+  for (let i = 0; i < turns.length; i += 1) {
+    const issues = validateImportTurn(turns[i], i);
+    if (issues.length > 0) {
+      const error: BulkImportError = {
+        batchIndex: -1,
+        message: issues.map((iss) => iss.message).join("; "),
+      };
+      result.errors.push(error);
+    } else {
+      validTurns.push(turns[i]);
+    }
+  }
+
+  if (dryRun) {
+    result.turnsProcessed = validTurns.length;
+    result.batchesProcessed =
+      validTurns.length > 0
+        ? Math.ceil(validTurns.length / batchSize)
+        : 0;
+    return result;
+  }
+
+  // Process in batches
+  let batchIndex = 0;
+  for (let i = 0; i < validTurns.length; i += batchSize) {
+    const batch = validTurns.slice(i, i + batchSize);
+    try {
+      const batchResult = await processBatch(batch, batchContext);
+      result.memoriesCreated += batchResult.memoriesCreated;
+      result.duplicatesSkipped += batchResult.duplicatesSkipped;
+      if (typeof batchResult.entitiesCreated === "number") {
+        result.entitiesCreated += batchResult.entitiesCreated;
+      }
+    } catch (err: unknown) {
+      const message =
+        err instanceof Error ? err.message : String(err);
+      result.errors.push({ batchIndex, message });
+    }
+    result.turnsProcessed += batch.length;
+    result.batchesProcessed += 1;
+    batchIndex += 1;
+  }
+
+  return result;
+}

package/src/bulk-import/registry.test.ts

@@ -0,0 +1,151 @@
+import assert from "node:assert/strict";
+import { describe, it, beforeEach } from "node:test";
+
+import {
+  registerBulkImportSource,
+  getBulkImportSource,
+  listBulkImportSources,
+  clearBulkImportSources,
+} from "./registry.js";
+import type { BulkImportSourceAdapter } from "./types.js";
+
+function makeAdapter(
+  name: string,
+): BulkImportSourceAdapter {
+  return {
+    name,
+    parse: () => ({
+      turns: [],
+      metadata: {
+        source: name,
+        exportDate: "2024-06-15T00:00:00.000Z",
+        messageCount: 0,
+        dateRange: {
+          from: "2024-01-01T00:00:00.000Z",
+          to: "2024-06-15T00:00:00.000Z",
+        },
+      },
+    }),
+  };
+}
+
+describe("bulk-import registry", () => {
+  beforeEach(() => {
+    clearBulkImportSources();
+  });
+
+  it("registers and retrieves a source adapter", () => {
+    const adapter = makeAdapter("weclone-telegram");
+    registerBulkImportSource(adapter);
+    const retrieved = getBulkImportSource("weclone-telegram");
+    assert.equal(retrieved, adapter);
+  });
+
+  it("lists registered adapter names", () => {
+    registerBulkImportSource(makeAdapter("source-a"));
+    registerBulkImportSource(makeAdapter("source-b"));
+    const names = listBulkImportSources();
+    assert.deepEqual(names.sort(), ["source-a", "source-b"]);
+  });
+
+  it("rejects duplicate registration", () => {
+    registerBulkImportSource(makeAdapter("dup-source"));
+    assert.throws(
+      () => registerBulkImportSource(makeAdapter("dup-source")),
+      (err: Error) => {
+        assert.ok(
+          err.message.includes("already registered"),
+          `expected 'already registered' in: ${err.message}`,
+        );
+        return true;
+      },
+    );
+  });
+
+  it("returns undefined for unknown source", () => {
+    const result = getBulkImportSource("nonexistent");
+    assert.equal(result, undefined);
+  });
+
+  it("rejects empty adapter name", () => {
+    assert.throws(
+      () => registerBulkImportSource(makeAdapter("")),
+      (err: Error) => {
+        assert.ok(
+          err.message.includes("non-empty string"),
+          `expected 'non-empty string' in: ${err.message}`,
+        );
+        return true;
+      },
+    );
+  });
+
+  it("rejects whitespace-only adapter name", () => {
+    assert.throws(
+      () => registerBulkImportSource(makeAdapter("   ")),
+      (err: Error) => {
+        assert.ok(
+          err.message.includes("non-empty string"),
+          `expected 'non-empty string' in: ${err.message}`,
+        );
+        return true;
+      },
+    );
+  });
+
+  it("rejects null adapter", () => {
+    assert.throws(
+      () =>
+        registerBulkImportSource(
+          null as unknown as BulkImportSourceAdapter,
+        ),
+      (err: Error) => {
+        assert.ok(err.message.includes("must be an object"));
+        return true;
+      },
+    );
+  });
+
+  it("rejects adapter without parse function", () => {
+    const bad = { name: "no-parse" } as unknown as BulkImportSourceAdapter;
+    assert.throws(
+      () => registerBulkImportSource(bad),
+      (err: Error) => {
+        assert.ok(err.message.includes("parse function"));
+        return true;
+      },
+    );
+  });
+
+  it("clearBulkImportSources removes all adapters", () => {
+    registerBulkImportSource(makeAdapter("x"));
+    assert.equal(listBulkImportSources().length, 1);
+    clearBulkImportSources();
+    assert.equal(listBulkImportSources().length, 0);
+  });
+
+  it("retrieves adapter when lookup name has leading/trailing whitespace", () => {
+    // Registration trims the name on write; lookup must trim as well,
+    // otherwise `get('  weclone  ')` would miss the adapter stored
+    // under the trimmed key `weclone`.
+    const adapter = makeAdapter("weclone");
+    registerBulkImportSource(adapter);
+    assert.equal(getBulkImportSource("weclone"), adapter);
+    assert.equal(getBulkImportSource("  weclone  "), adapter);
+    assert.equal(getBulkImportSource("\tweclone\n"), adapter);
+  });
+
+  it("returns undefined for non-string or empty lookup names", () => {
+    registerBulkImportSource(makeAdapter("weclone"));
+    assert.equal(getBulkImportSource(""), undefined);
+    assert.equal(getBulkImportSource("   "), undefined);
+    assert.equal(
+      getBulkImportSource(null as unknown as string),
+      undefined,
+    );
+    assert.equal(
+      getBulkImportSource(undefined as unknown as string),
+      undefined,
+    );
+  });
+});

package/src/bulk-import/registry.ts

@@ -0,0 +1,72 @@
+// ---------------------------------------------------------------------------
+// Bulk-import source adapter registry
+// ---------------------------------------------------------------------------
+
+import type { BulkImportSourceAdapter } from "./types.js";
+
+const adapters = new Map<string, BulkImportSourceAdapter>();
+
+/**
+ * Register a source adapter. Rejects duplicate names and empty names.
+ */
+export function registerBulkImportSource(
+  adapter: BulkImportSourceAdapter,
+): void {
+  if (!adapter || typeof adapter !== "object") {
+    throw new Error("bulk-import adapter must be an object");
+  }
+  if (
+    !adapter.name ||
+    typeof adapter.name !== "string" ||
+    adapter.name.trim().length === 0
+  ) {
+    throw new Error("bulk-import adapter name must be a non-empty string");
+  }
+  if (typeof adapter.parse !== "function") {
+    throw new Error(
+      `bulk-import adapter '${adapter.name}' must have a parse function`,
+    );
+  }
+  const key = adapter.name.trim();
+  if (adapters.has(key)) {
+    throw new Error(
+      `bulk-import source adapter '${key}' is already registered`,
+    );
+  }
+  // Store adapter with trimmed name so `adapter.name` stays consistent with
+  // the registry key returned by `listBulkImportSources()`.
+  const normalized: BulkImportSourceAdapter =
+    adapter.name === key ? adapter : { ...adapter, name: key };
+  adapters.set(key, normalized);
+}
+
+/**
+ * Retrieve a registered adapter by name.
+ *
+ * Names are trimmed on write (see `registerBulkImportSource`), so we
+ * also trim on read to avoid the asymmetric-lookup footgun where a
+ * whitespace-padded lookup would never find a registered adapter.
+ * Non-string or empty-after-trim inputs return undefined.
+ */
+export function getBulkImportSource(
+  name: string,
+): BulkImportSourceAdapter | undefined {
+  if (typeof name !== "string") return undefined;
+  const key = name.trim();
+  if (key.length === 0) return undefined;
+  return adapters.get(key);
+}
+
+/**
+ * List all registered adapter names.
+ */
+export function listBulkImportSources(): string[] {
+  return [...adapters.keys()];
+}
+
+/**
+ * Clear all registered adapters (for testing).
+ */
+export function clearBulkImportSources(): void {
+  adapters.clear();
+}

package/src/bulk-import/types.test.ts

@@ -0,0 +1,272 @@
+import assert from "node:assert/strict";
+import { describe, it } from "node:test";
+
+import {
+  isImportRole,
+  parseIsoTimestamp,
+  validateImportTurn,
+  type ImportTurn,
+} from "./types.js";
+
+// ---------------------------------------------------------------------------
+// isImportRole
+// ---------------------------------------------------------------------------
+
+describe("isImportRole", () => {
+  it("accepts 'user'", () => {
+    assert.equal(isImportRole("user"), true);
+  });
+
+  it("accepts 'assistant'", () => {
+    assert.equal(isImportRole("assistant"), true);
+  });
+
+  it("accepts 'other'", () => {
+    assert.equal(isImportRole("other"), true);
+  });
+
+  it("rejects unknown string", () => {
+    assert.equal(isImportRole("system"), false);
+  });
+
+  it("rejects non-string", () => {
+    assert.equal(isImportRole(42), false);
+  });
+});
+
+// ---------------------------------------------------------------------------
+// parseIsoTimestamp
+// ---------------------------------------------------------------------------
+
+describe("parseIsoTimestamp", () => {
+  it("parses a valid ISO timestamp with millis", () => {
+    const ts = parseIsoTimestamp("2024-06-15T10:30:00.000Z");
+    assert.equal(typeof ts, "number");
+    assert.ok(ts! > 0);
+  });
+
+  it("parses a valid ISO timestamp without millis", () => {
+    const ts = parseIsoTimestamp("2024-06-15T10:30:00Z");
+    assert.equal(typeof ts, "number");
+    assert.ok(ts! > 0);
+  });
+
+  it("parses a valid ISO timestamp with positive timezone offset", () => {
+    const ts = parseIsoTimestamp("2024-01-15T10:30:00+05:30");
+    assert.equal(typeof ts, "number");
+    assert.ok(ts! > 0);
+  });
+
+  it("parses a valid ISO timestamp with negative timezone offset", () => {
+    const ts = parseIsoTimestamp("2024-01-15T10:30:00-08:00");
+    assert.equal(typeof ts, "number");
+    assert.ok(ts! > 0);
+  });
+
+  it("parses a valid ISO timestamp with millis and timezone offset", () => {
+    const ts = parseIsoTimestamp("2024-01-15T10:30:00.000+05:30");
+    assert.equal(typeof ts, "number");
+    assert.ok(ts! > 0);
+  });
+
+  it("rejects overflowed offset timestamp (Feb 31)", () => {
+    assert.equal(parseIsoTimestamp("2024-02-31T10:00:00+00:00"), null);
+  });
+
+  it("rejects overflowed UTC timestamp (Feb 30)", () => {
+    assert.equal(parseIsoTimestamp("2024-02-30T10:00:00Z"), null);
+  });
+
+  it("rejects offset hour exceeding 14", () => {
+    assert.equal(parseIsoTimestamp("2024-01-15T10:00:00+25:00"), null);
+  });
+
+  it("rejects offset hour of 15", () => {
+    assert.equal(parseIsoTimestamp("2024-01-15T10:00:00+15:00"), null);
+  });
+
+  it("rejects offset minute exceeding 59", () => {
+    assert.equal(parseIsoTimestamp("2024-01-15T10:00:00+05:61"), null);
+  });
+
+  it("rejects overflowed month (month 13)", () => {
+    assert.equal(parseIsoTimestamp("2024-13-15T10:00:00+00:00"), null);
+  });
+
+  it("rejects overflowed day for April (Apr 31)", () => {
+    assert.equal(parseIsoTimestamp("2024-04-31T10:00:00+05:30"), null);
+  });
+
+  it("accepts valid Feb 29 in leap year with offset", () => {
+    const ts = parseIsoTimestamp("2024-02-29T10:00:00+05:30");
+    assert.equal(typeof ts, "number");
+    assert.ok(ts! > 0);
+  });
+
+  it("accepts two-digit fractional seconds", () => {
+    const ts = parseIsoTimestamp("2024-01-15T10:30:00.12Z");
+    assert.equal(typeof ts, "number");
+    assert.ok(ts! > 0);
+  });
+
+  it("accepts six-digit fractional seconds with offset", () => {
+    const ts = parseIsoTimestamp("2024-01-15T10:30:00.123456+00:00");
+    assert.equal(typeof ts, "number");
+    assert.ok(ts! > 0);
+  });
+
+  it("accepts single-digit fractional seconds", () => {
+    const ts = parseIsoTimestamp("2024-01-15T10:30:00.5Z");
+    assert.equal(typeof ts, "number");
+    assert.ok(ts! > 0);
+  });
+
+  it("rejects Feb 29 in non-leap year with offset", () => {
+    assert.equal(parseIsoTimestamp("2023-02-29T10:00:00+05:30"), null);
+  });
+
+  it("returns null for non-ISO string", () => {
+    assert.equal(parseIsoTimestamp("June 15, 2024"), null);
+  });
+
+  it("returns null for empty string", () => {
+    assert.equal(parseIsoTimestamp(""), null);
+  });
+
+  it("returns null for non-string input", () => {
+    assert.equal(parseIsoTimestamp(123 as unknown as string), null);
+  });
+});
+
+// ---------------------------------------------------------------------------
+// validateImportTurn
+// ---------------------------------------------------------------------------
+
+function makeValidTurn(overrides?: Partial<ImportTurn>): ImportTurn {
+  return {
+    role: "user",
+    content: "Hello, world!",
+    timestamp: "2024-06-15T10:30:00.000Z",
+    ...overrides,
+  };
+}
+
+describe("validateImportTurn", () => {
+  it("returns no issues for a valid turn", () => {
+    const issues = validateImportTurn(makeValidTurn());
+    assert.equal(issues.length, 0);
+  });
+
+  it("returns no issues for a valid turn with 'other' role", () => {
+    const issues = validateImportTurn(makeValidTurn({ role: "other" }));
+    assert.equal(issues.length, 0);
+  });
+
+  it("returns no issues for a valid turn with 'assistant' role", () => {
+    const issues = validateImportTurn(
+      makeValidTurn({ role: "assistant" }),
+    );
+    assert.equal(issues.length, 0);
+  });
+
+  it("reports invalid role", () => {
+    const turn = makeValidTurn({
+      role: "system" as unknown as ImportTurn["role"],
+    });
+    const issues = validateImportTurn(turn);
+    assert.equal(issues.length, 1);
+    assert.equal(issues[0].code, "turn.role.invalid");
+    assert.ok(issues[0].message.includes("system"));
+  });
+
+  it("reports empty content", () => {
+    const turn = makeValidTurn({ content: "" });
+    const issues = validateImportTurn(turn);
+    assert.equal(issues.length, 1);
+    assert.equal(issues[0].code, "turn.content.invalid");
+  });
+
+  it("reports whitespace-only content", () => {
+    const turn = makeValidTurn({ content: "   " });
+    const issues = validateImportTurn(turn);
+    assert.equal(issues.length, 1);
+    assert.equal(issues[0].code, "turn.content.invalid");
+  });
+
+  it("reports invalid timestamp", () => {
+    const turn = makeValidTurn({ timestamp: "not-a-date" });
+    const issues = validateImportTurn(turn);
+    assert.equal(issues.length, 1);
+    assert.equal(issues[0].code, "turn.timestamp.invalid");
+    assert.ok(issues[0].message.includes("not-a-date"));
+  });
+
+  it("reports empty timestamp", () => {
+    const turn = makeValidTurn({ timestamp: "" });
+    const issues = validateImportTurn(turn);
+    assert.equal(issues.length, 1);
+    assert.equal(issues[0].code, "turn.timestamp.invalid");
+  });
+
+  it("reports multiple issues at once", () => {
+    const turn = makeValidTurn({
+      role: "invalid" as unknown as ImportTurn["role"],
+      content: "",
+      timestamp: "bad",
+    });
+    const issues = validateImportTurn(turn);
+    assert.equal(issues.length, 3);
+    const codes = issues.map((i) => i.code);
+    assert.ok(codes.includes("turn.role.invalid"));
+    assert.ok(codes.includes("turn.content.invalid"));
+    assert.ok(codes.includes("turn.timestamp.invalid"));
+  });
+
+  it("returns turn.invalid for null input", () => {
+    const issues = validateImportTurn(
+      null as unknown as ImportTurn,
+    );
+    assert.equal(issues.length, 1);
+    assert.equal(issues[0].code, "turn.invalid");
+  });
+
+  it("includes index when provided", () => {
+    const turn = makeValidTurn({ content: "" });
+    const issues = validateImportTurn(turn, 5);
+    assert.equal(issues[0].index, 5);
+  });
+
+  it("accepts timestamp without millis", () => {
+    const turn = makeValidTurn({
+      timestamp: "2024-06-15T10:30:00Z",
+    });
+    const issues = validateImportTurn(turn);
+    assert.equal(issues.length, 0);
+  });
+
+  it("accepts timestamp with positive timezone offset", () => {
+    const turn = makeValidTurn({
+      timestamp: "2024-01-15T10:30:00+05:30",
+    });
+    const issues = validateImportTurn(turn);
+    assert.equal(issues.length, 0);
+  });
+
+  it("accepts timestamp with negative timezone offset", () => {
+    const turn = makeValidTurn({
+      timestamp: "2024-01-15T10:30:00-08:00",
+    });
+    const issues = validateImportTurn(turn);
+    assert.equal(issues.length, 0);
+  });
+
+  it("includes optional fields without issue", () => {
+    const turn = makeValidTurn({
+      participantId: "p1",
+      participantName: "Alice",
+      replyToId: "msg-42",
+    });
+    const issues = validateImportTurn(turn);
+    assert.equal(issues.length, 0);
+  });
+});