getmnemo-migrate 0.1.0

package/dist/cli.js

@@ -0,0 +1,220 @@
+#!/usr/bin/env node
+import { Command, Option } from "commander";
+import cliProgress from "cli-progress";
+import kleur from "kleur";
+import ora from "ora";
+import { ADAPTER_NAMES, createAdapter, } from "./adapters/index.js";
+import { listJobs, readJob, requestCancellation, writeJob, } from "./lib/jobs.js";
+import { migrate, planMigration, reconcileMigration } from "./lib/migrator.js";
+import { MnemoWriter } from "./lib/mnemo-writer.js";
+const VERSION = "0.1.0";
+function parsePositiveInt(value, fallback, name) {
+    if (value === undefined)
+        return fallback;
+    const parsed = Number(value);
+    if (!Number.isInteger(parsed) || parsed < 1)
+        throw new Error(`${name} must be a positive integer`);
+    return parsed;
+}
+function buildAdapterConfig(opts) {
+    return {
+        ...(opts.file !== undefined ? { file: opts.file } : {}),
+        ...(opts.map !== undefined ? { map: opts.map } : {}),
+        ...(opts.user !== undefined ? { user: opts.user } : {}),
+        ...(opts.baseUrl !== undefined ? { baseUrl: opts.baseUrl } : {}),
+        ...(opts.pageSize !== undefined
+            ? { pageSize: parsePositiveInt(opts.pageSize, 100, "--page-size") }
+            : {}),
+    };
+}
+function mergeAdapterConfig(saved, opts) {
+    return { ...saved, ...buildAdapterConfig(opts) };
+}
+function parseLane(value, fallback) {
+    const lane = value ?? fallback;
+    if (lane !== "memory" && lane !== "document") {
+        throw new Error('--lane must be either "memory" or "document"');
+    }
+    return lane;
+}
+function requireContainer(value) {
+    const container = value ?? process.env.GETMNEMO_CONTAINER_TAG;
+    if (!container)
+        throw new Error("--container or GETMNEMO_CONTAINER_TAG is required");
+    return container;
+}
+function getWriter() {
+    const apiKey = process.env.GETMNEMO_API_KEY;
+    const workspaceId = process.env.GETMNEMO_WORKSPACE_ID;
+    if (!apiKey || !workspaceId) {
+        throw new Error("GETMNEMO_API_KEY and GETMNEMO_WORKSPACE_ID must be set");
+    }
+    return new MnemoWriter({
+        apiKey,
+        workspaceId,
+        apiUrl: process.env.GETMNEMO_API_URL,
+    });
+}
+function addAdapterOptions(command, includeDefaults) {
+    command
+        .option("--file <path>", "source file or directory")
+        .option("--map <spec>", "field mapping such as content=body,id=record_id")
+        .option("--user <id>", "source user id (required by some providers)")
+        .option("--base-url <url>", "override provider base URL")
+        .option("--page-size <n>", "source page-size hint", includeDefaults ? "100" : undefined);
+    return command;
+}
+function addRunOptions(command, includeDefaults) {
+    addAdapterOptions(command, includeDefaults)
+        .option("--concurrency <n>", "maximum concurrent writes", includeDefaults ? "5" : undefined)
+        .option("--container <tag>", "destination container tag")
+        .addOption(new Option("--lane <lane>", "memory or document").choices(["memory", "document"]))
+        .option("--no-wait", "return after document jobs are accepted")
+        .option("--wait-timeout <seconds>", "maximum ingestion wait", includeDefaults ? "86400" : undefined);
+    return command;
+}
+function startProgress(source, total) {
+    const bar = new cliProgress.SingleBar({
+        format: `${kleur.cyan("import")} ${source} |{bar}| {value}/{total} handled ({failed} failed) | ETA: {eta_formatted}`,
+        hideCursor: true,
+    }, cliProgress.Presets.shades_classic);
+    bar.start(total ?? 0, 0, { failed: 0 });
+    return bar;
+}
+function printSummary(state) {
+    process.stdout.write(`\n${kleur.green("✓")} job ${kleur.bold(state.id)}: ${state.status}\n` +
+        `  records written: ${state.written}\n` +
+        `  records reused:  ${state.reused}\n` +
+        `  write failures:  ${state.failed}\n` +
+        `  jobs completed:  ${state.remoteCompleted}\n` +
+        `  jobs failed:     ${state.remoteFailed}\n` +
+        `  jobs pending:    ${state.remotePending}\n`);
+}
+export function buildCli() {
+    const program = new Command();
+    program
+        .name("getmnemo-migrate")
+        .description(`${kleur.cyan("Mnemo migrate")} — import external memories and customer exports.`)
+        .version(VERSION, "-v, --version")
+        .showHelpAfterError("(add --help for additional information)");
+    const from = program.command("from <source>").description(`import from ${ADAPTER_NAMES.join(", ")}`);
+    addRunOptions(from, true).action(async (source, opts) => {
+        const adapterConfig = buildAdapterConfig(opts);
+        const adapter = createAdapter(source, adapterConfig);
+        const lane = parseLane(opts.lane, adapter.defaultLane);
+        const containerTag = requireContainer(opts.container);
+        const writer = getWriter();
+        const total = await adapter.count();
+        const bar = startProgress(source, total);
+        let state = await migrate(adapter, writer, {
+            concurrency: parsePositiveInt(opts.concurrency, 5, "--concurrency"),
+            lane,
+            containerTag,
+            adapterConfig,
+            onProgress: (progress) => {
+                if (progress.total !== null && bar.getTotal() !== progress.total)
+                    bar.setTotal(progress.total);
+                bar.update(progress.written + progress.failed, { failed: progress.failed });
+            },
+        });
+        bar.stop();
+        if (lane === "document" && opts.wait && state.status !== "paused") {
+            state = await reconcileMigration(writer, state, parsePositiveInt(opts.waitTimeout, 86_400, "--wait-timeout") * 1000);
+        }
+        printSummary(state);
+    });
+    const plan = program.command("plan").description("preview an import without writing");
+    const planFrom = plan.command("from <source>").description("count and sample source records");
+    addAdapterOptions(planFrom, true)
+        .option("--sample <n>", "sample size", "5")
+        .action(async (source, opts) => {
+        const adapter = createAdapter(source, buildAdapterConfig(opts));
+        const spinner = ora(`Inspecting ${source}...`).start();
+        const result = await planMigration(adapter, parsePositiveInt(opts.sample, 5, "--sample"));
+        spinner.stop();
+        process.stdout.write(`${kleur.cyan("source:")} ${source}\n`);
+        process.stdout.write(`${kleur.cyan("lane:")}   ${adapter.defaultLane}\n`);
+        process.stdout.write(`${kleur.cyan("total:")}  ${result.total ?? "unknown"}\n`);
+        process.stdout.write(`${kleur.cyan("sample:")}\n`);
+        for (const record of result.sample) {
+            const preview = record.content.length > 120 ? `${record.content.slice(0, 119)}…` : record.content;
+            process.stdout.write(`  - ${kleur.dim(record.sourceId ?? "content hash")}  ${preview}\n`);
+        }
+    });
+    const resume = program.command("resume <jobId>").description("resume an interrupted or paused import");
+    addRunOptions(resume, false).action(async (jobId, opts) => {
+        const existing = await readJob(jobId);
+        const adapterConfig = mergeAdapterConfig(existing.adapterConfig, opts);
+        const adapter = createAdapter(existing.source, adapterConfig);
+        const lane = parseLane(opts.lane, existing.lane);
+        const containerTag = opts.container ?? existing.containerTag;
+        const writer = getWriter();
+        const bar = startProgress(existing.source, existing.total ?? null);
+        bar.update(existing.written + existing.failed, { failed: existing.failed });
+        let state = await migrate(adapter, writer, {
+            concurrency: parsePositiveInt(opts.concurrency, 5, "--concurrency"),
+            resumeJobId: jobId,
+            lane,
+            containerTag,
+            adapterConfig,
+            onProgress: (progress) => bar.update(progress.written + progress.failed, { failed: progress.failed }),
+        });
+        bar.stop();
+        if (lane === "document" && opts.wait && state.status !== "paused") {
+            state = await reconcileMigration(writer, state, parsePositiveInt(opts.waitTimeout, 86_400, "--wait-timeout") * 1000);
+        }
+        printSummary(state);
+    });
+    program
+        .command("reconcile <jobId>")
+        .description("refresh all remote ingestion jobs and print the completion report")
+        .option("--wait-timeout <seconds>", "maximum wait", "86400")
+        .action(async (jobId, opts) => {
+        const state = await reconcileMigration(getWriter(), await readJob(jobId), parsePositiveInt(opts.waitTimeout, 86_400, "--wait-timeout") * 1000);
+        printSummary(state);
+    });
+    program
+        .command("status [jobId]")
+        .description("show one import report or list recent imports")
+        .action(async (jobId) => {
+        if (jobId) {
+            process.stdout.write(`${JSON.stringify(await readJob(jobId), null, 2)}\n`);
+            return;
+        }
+        const jobs = await listJobs();
+        if (jobs.length === 0)
+            process.stdout.write("No migration jobs yet.\n");
+        for (const job of jobs.slice(0, 20)) {
+            process.stdout.write(`${job.id}  ${job.status.padEnd(22)} ${job.source.padEnd(12)} ${job.written}/${job.total ?? "?"} written\n`);
+        }
+    });
+    program
+        .command("cancel <jobId>")
+        .description("request a running import to pause after its current bounded batch")
+        .action(async (jobId) => {
+        const state = await readJob(jobId);
+        if (["completed", "completed_with_errors", "failed"].includes(state.status)) {
+            throw new Error(`job ${jobId} is already ${state.status}`);
+        }
+        await requestCancellation(jobId);
+        state.status = "paused";
+        await writeJob(state);
+        process.stdout.write(`${kleur.yellow("!")} pause requested for ${jobId}\n`);
+    });
+    return program;
+}
+async function main() {
+    try {
+        await buildCli().parseAsync(process.argv);
+    }
+    catch (error) {
+        process.stderr.write(kleur.red(`error: ${error instanceof Error ? error.message : String(error)}\n`));
+        process.exitCode = 1;
+    }
+}
+const isMain = import.meta.url === `file://${process.argv[1]}` ||
+    process.argv[1]?.endsWith("cli.js") ||
+    process.argv[1]?.endsWith("cli.ts");
+if (isMain)
+    void main();
+//# sourceMappingURL=cli.js.map

package/dist/cli.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli.js","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,WAAW,CAAC;AAC5C,OAAO,WAAW,MAAM,cAAc,CAAC;AACvC,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,GAAG,MAAM,KAAK,CAAC;AACtB,OAAO,EACL,aAAa,EACb,aAAa,GAGd,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EACL,QAAQ,EACR,OAAO,EACP,mBAAmB,EACnB,QAAQ,GAET,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AAC/E,OAAO,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AAEpD,MAAM,OAAO,GAAG,OAAO,CAAC;AAkBxB,SAAS,gBAAgB,CAAC,KAAyB,EAAE,QAAgB,EAAE,IAAY;IACjF,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,QAAQ,CAAC;IACzC,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAC7B,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,MAAM,GAAG,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,6BAA6B,CAAC,CAAC;IACnG,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,kBAAkB,CAAC,IAAuB;IACjD,OAAO;QACL,GAAG,CAAC,IAAI,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACvD,GAAG,CAAC,IAAI,CAAC,GAAG,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACpD,GAAG,CAAC,IAAI,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACvD,GAAG,CAAC,IAAI,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAChE,GAAG,CAAC,IAAI,CAAC,QAAQ,KAAK,SAAS;YAC7B,CAAC,CAAC,EAAE,QAAQ,EAAE,gBAAgB,CAAC,IAAI,CAAC,QAAQ,EAAE,GAAG,EAAE,aAAa,CAAC,EAAE;YACnE,CAAC,CAAC,EAAE,CAAC;KACR,CAAC;AACJ,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAoB,EAAE,IAAuB;IACvE,OAAO,EAAE,GAAG,KAAK,EAAE,GAAG,kBAAkB,CAAC,IAAI,CAAC,EAAE,CAAC;AACnD,CAAC;AAED,SAAS,SAAS,CAAC,KAAyB,EAAE,QAAoB;IAChE,MAAM,IAAI,GAAG,KAAK,IAAI,QAAQ,CAAC;IAC/B,IAAI,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,UAAU,EAAE,CAAC;QAC7C,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;IAClE,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAyB;IACjD,MAAM,SAAS,GAAG,KAAK,IAAI,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC;IAC9D,IAAI,CAAC,SAAS;QAAE,MAAM,IAAI,KAAK,CAAC,mDAAmD,CAAC,CAAC;IACrF,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,SAAS;IAChB,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC;IAC5C,MAAM,WAAW,GAAG,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC;IACtD,IAAI,CAAC,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;QAC5B,MAAM,IAAI,KAAK,CAAC,wDAAwD,CAAC,CAAC;IAC5E,CAAC;IACD,OAAO,IAAI,WAAW,CAAC;QACrB,MAAM;QACN,WAAW;QACX,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,gBAAgB;KACrC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,iBAAiB,CAAC,OAAgB,EAAE,eAAwB;IACnE,OAAO;SACJ,MAAM,CAAC,eAAe,EAAE,0BAA0B,CAAC;SACnD,MAAM,CAAC,cAAc,EAAE,iDAAiD,CAAC;SACzE,MAAM,CAAC,aAAa,EAAE,6CAA6C,CAAC;SACpE,MAAM,CAAC,kBAAkB,EAAE,4BAA4B,CAAC;SACxD,MAAM,CACL,iBAAiB,EACjB,uBAAuB,EACvB,eAAe,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CACpC,CAAC;IACJ,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,aAAa,CAAC,OAAgB,EAAE,eAAwB;IAC/D,iBAAiB,CAAC,OAAO,EAAE,eAAe,CAAC;SACxC,MAAM,CAAC,mBAAmB,EAAE,2BAA2B,EAAE,eAAe,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC;SAC3F,MAAM,CAAC,mBAAmB,EAAE,2BAA2B,CAAC;SACxD,SAAS,CAAC,IAAI,MAAM,CAAC,eAAe,EAAE,oBAAoB,CAAC,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC,CAAC;SAC5F,MAAM,CAAC,WAAW,EAAE,yCAAyC,CAAC;SAC9D,MAAM,CAAC,0BAA0B,EAAE,wBAAwB,EAAE,eAAe,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IACvG,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,aAAa,CAAC,MAAc,EAAE,KAAoB;IACzD,MAAM,GAAG,GAAG,IAAI,WAAW,CAAC,SAAS,CACnC;QACE,MAAM,EAAE,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,MAAM,2EAA2E;QACpH,UAAU,EAAE,IAAI;KACjB,EACD,WAAW,CAAC,OAAO,CAAC,cAAc,CACnC,CAAC;IACF,GAAG,CAAC,KAAK,CAAC,KAAK,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC,CAAC;IACxC,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,YAAY,CAAC,KAAe;IACnC,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,KAAK,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,KAAK,CAAC,MAAM,IAAI;QACpE,sBAAsB,KAAK,CAAC,OAAO,IAAI;QACvC,sBAAsB,KAAK,CAAC,MAAM,IAAI;QACtC,sBAAsB,KAAK,CAAC,MAAM,IAAI;QACtC,sBAAsB,KAAK,CAAC,eAAe,IAAI;QAC/C,sBAAsB,KAAK,CAAC,YAAY,IAAI;QAC5C,sBAAsB,KAAK,CAAC,aAAa,IAAI,CAChD,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,QAAQ;IACtB,MAAM,OAAO,GAAG,IAAI,OAAO,EAAE,CAAC;IAC9B,OAAO;SACJ,IAAI,CAAC,kBAAkB,CAAC;SACxB,WAAW,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,eAAe,CAAC,mDAAmD,CAAC;SAC9F,OAAO,CAAC,OAAO,EAAE,eAAe,CAAC;SACjC,kBAAkB,CAAC,yCAAyC,CAAC,CAAC;IAEjE,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC,WAAW,CAAC,eAAe,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACrG,aAAa,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,MAAM,CAAC,KAAK,EAAE,MAAc,EAAE,IAAa,EAAE,EAAE;QACvE,MAAM,aAAa,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAC;QAC/C,MAAM,OAAO,GAAG,aAAa,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;QACrD,MAAM,IAAI,GAAG,SAAS,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,WAAW,CAAC,CAAC;QACvD,MAAM,YAAY,GAAG,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACtD,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;QAC3B,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;QACpC,MAAM,GAAG,GAAG,aAAa,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;QACzC,IAAI,KAAK,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,MAAM,EAAE;YACzC,WAAW,EAAE,gBAAgB,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,EAAE,eAAe,CAAC;YACnE,IAAI;YACJ,YAAY;YACZ,aAAa;YACb,UAAU,EAAE,CAAC,QAAQ,EAAE,EAAE;gBACvB,IAAI,QAAQ,CAAC,KAAK,KAAK,IAAI,IAAI,GAAG,CAAC,QAAQ,EAAE,KAAK,QAAQ,CAAC,KAAK;oBAAE,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;gBAC/F,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,GAAG,QAAQ,CAAC,MAAM,EAAE,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;YAC9E,CAAC;SACF,CAAC,CAAC;QACH,GAAG,CAAC,IAAI,EAAE,CAAC;QACX,IAAI,IAAI,KAAK,UAAU,IAAI,IAAI,CAAC,IAAI,IAAI,KAAK,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;YAClE,KAAK,GAAG,MAAM,kBAAkB,CAC9B,MAAM,EACN,KAAK,EACL,gBAAgB,CAAC,IAAI,CAAC,WAAW,EAAE,MAAM,EAAE,gBAAgB,CAAC,GAAG,IAAI,CACpE,CAAC;QACJ,CAAC;QACD,YAAY,CAAC,KAAK,CAAC,CAAC;IACtB,CAAC,CAAC,CAAC;IAEH,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,WAAW,CAAC,mCAAmC,CAAC,CAAC;IACtF,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC,WAAW,CAAC,iCAAiC,CAAC,CAAC;IAC9F,iBAAiB,CAAC,QAAQ,EAAE,IAAI,CAAC;SAC9B,MAAM,CAAC,cAAc,EAAE,aAAa,EAAE,GAAG,CAAC;SAC1C,MAAM,CAAC,KAAK,EAAE,MAAc,EAAE,IAA6C,EAAE,EAAE;QAC9E,MAAM,OAAO,GAAG,aAAa,CAAC,MAAM,EAAE,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAC;QAChE,MAAM,OAAO,GAAG,GAAG,CAAC,cAAc,MAAM,KAAK,CAAC,CAAC,KAAK,EAAE,CAAC;QACvD,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,OAAO,EAAE,gBAAgB,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,UAAU,CAAC,CAAC,CAAC;QAC1F,OAAO,CAAC,IAAI,EAAE,CAAC;QACf,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,MAAM,IAAI,CAAC,CAAC;QAC7D,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,OAAO,CAAC,WAAW,IAAI,CAAC,CAAC;QAC1E,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,MAAM,CAAC,KAAK,IAAI,SAAS,IAAI,CAAC,CAAC;QAChF,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QACnD,KAAK,MAAM,MAAM,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;YACnC,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC;YAClG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,IAAI,cAAc,CAAC,KAAK,OAAO,IAAI,CAAC,CAAC;QAC5F,CAAC;IACH,CAAC,CAAC,CAAC;IAEL,MAAM,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC,WAAW,CAAC,wCAAwC,CAAC,CAAC;IACvG,aAAa,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,MAAM,CAAC,KAAK,EAAE,KAAa,EAAE,IAAa,EAAE,EAAE;QACzE,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,CAAC;QACtC,MAAM,aAAa,GAAG,kBAAkB,CAAC,QAAQ,CAAC,aAAa,EAAE,IAAI,CAAC,CAAC;QACvE,MAAM,OAAO,GAAG,aAAa,CAAC,QAAQ,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;QAC9D,MAAM,IAAI,GAAG,SAAS,CAAC,IAAI,CAAC,IAAI,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC;QACjD,MAAM,YAAY,GAAG,IAAI,CAAC,SAAS,IAAI,QAAQ,CAAC,YAAY,CAAC;QAC7D,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;QAC3B,MAAM,GAAG,GAAG,aAAa,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,KAAK,IAAI,IAAI,CAAC,CAAC;QACnE,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,GAAG,QAAQ,CAAC,MAAM,EAAE,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;QAC5E,IAAI,KAAK,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,MAAM,EAAE;YACzC,WAAW,EAAE,gBAAgB,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,EAAE,eAAe,CAAC;YACnE,WAAW,EAAE,KAAK;YAClB,IAAI;YACJ,YAAY;YACZ,aAAa;YACb,UAAU,EAAE,CAAC,QAAQ,EAAE,EAAE,CACvB,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,GAAG,QAAQ,CAAC,MAAM,EAAE,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,CAAC;SAC9E,CAAC,CAAC;QACH,GAAG,CAAC,IAAI,EAAE,CAAC;QACX,IAAI,IAAI,KAAK,UAAU,IAAI,IAAI,CAAC,IAAI,IAAI,KAAK,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;YAClE,KAAK,GAAG,MAAM,kBAAkB,CAC9B,MAAM,EACN,KAAK,EACL,gBAAgB,CAAC,IAAI,CAAC,WAAW,EAAE,MAAM,EAAE,gBAAgB,CAAC,GAAG,IAAI,CACpE,CAAC;QACJ,CAAC;QACD,YAAY,CAAC,KAAK,CAAC,CAAC;IACtB,CAAC,CAAC,CAAC;IAEH,OAAO;SACJ,OAAO,CAAC,mBAAmB,CAAC;SAC5B,WAAW,CAAC,mEAAmE,CAAC;SAChF,MAAM,CAAC,0BAA0B,EAAE,cAAc,EAAE,OAAO,CAAC;SAC3D,MAAM,CAAC,KAAK,EAAE,KAAa,EAAE,IAA6B,EAAE,EAAE;QAC7D,MAAM,KAAK,GAAG,MAAM,kBAAkB,CACpC,SAAS,EAAE,EACX,MAAM,OAAO,CAAC,KAAK,CAAC,EACpB,gBAAgB,CAAC,IAAI,CAAC,WAAW,EAAE,MAAM,EAAE,gBAAgB,CAAC,GAAG,IAAI,CACpE,CAAC;QACF,YAAY,CAAC,KAAK,CAAC,CAAC;IACtB,CAAC,CAAC,CAAC;IAEL,OAAO;SACJ,OAAO,CAAC,gBAAgB,CAAC;SACzB,WAAW,CAAC,+CAA+C,CAAC;SAC5D,MAAM,CAAC,KAAK,EAAE,KAAc,EAAE,EAAE;QAC/B,IAAI,KAAK,EAAE,CAAC;YACV,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,OAAO,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;YAC3E,OAAO;QACT,CAAC;QACD,MAAM,IAAI,GAAG,MAAM,QAAQ,EAAE,CAAC;QAC9B,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAC;QACxE,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;YACpC,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,GAAG,GAAG,CAAC,EAAE,KAAK,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,GAAG,CAAC,OAAO,IAAI,GAAG,CAAC,KAAK,IAAI,GAAG,YAAY,CAC5G,CAAC;QACJ,CAAC;IACH,CAAC,CAAC,CAAC;IAEL,OAAO;SACJ,OAAO,CAAC,gBAAgB,CAAC;SACzB,WAAW,CAAC,mEAAmE,CAAC;SAChF,MAAM,CAAC,KAAK,EAAE,KAAa,EAAE,EAAE;QAC9B,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,CAAC;QACnC,IAAI,CAAC,WAAW,EAAE,uBAAuB,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC;YAC5E,MAAM,IAAI,KAAK,CAAC,OAAO,KAAK,eAAe,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;QAC7D,CAAC;QACD,MAAM,mBAAmB,CAAC,KAAK,CAAC,CAAC;QACjC,KAAK,CAAC,MAAM,GAAG,QAAQ,CAAC;QACxB,MAAM,QAAQ,CAAC,KAAK,CAAC,CAAC;QACtB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,wBAAwB,KAAK,IAAI,CAAC,CAAC;IAC9E,CAAC,CAAC,CAAC;IAEL,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,KAAK,UAAU,IAAI;IACjB,IAAI,CAAC;QACH,MAAM,QAAQ,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAC5C,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,UAAU,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;QACtG,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;IACvB,CAAC;AACH,CAAC;AAED,MAAM,MAAM,GACV,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,UAAU,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE;IAC/C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,QAAQ,CAAC,QAAQ,CAAC;IACnC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,QAAQ,CAAC,QAAQ,CAAC,CAAC;AAEtC,IAAI,MAAM;IAAE,KAAK,IAAI,EAAE,CAAC"}

package/dist/lib/jobs.d.ts

@@ -0,0 +1,44 @@
+import type { AdapterConfig, ImportLane } from "../adapters/index.js";
+export type JobStatus = "running" | "submitted" | "completed" | "completed_with_errors" | "failed" | "paused";
+export interface JobState {
+    id: string;
+    source: string;
+    lane: ImportLane;
+    containerTag: string;
+    adapterConfig: AdapterConfig;
+    status: JobStatus;
+    startedAt: string;
+    updatedAt: string;
+    finishedAt?: string;
+    total?: number | null;
+    written: number;
+    reused: number;
+    failed: number;
+    remoteCompleted: number;
+    remoteFailed: number;
+    remotePending: number;
+    /** Backward compatibility with jobs written by 0.1.0. */
+    processedIds?: string[];
+    lastError?: string;
+}
+export interface FailedRecord {
+    sourceId: string;
+    error: string;
+    occurredAt: string;
+}
+export declare function jobsDir(): string;
+export declare function jobPath(id: string): string;
+export declare function newJobId(): string;
+export declare function readJob(id: string): Promise<JobState>;
+export declare function writeJob(state: JobState): Promise<void>;
+export declare function appendProcessedId(jobId: string, sourceId: string): Promise<void>;
+export declare function loadProcessedIds(jobId: string, legacy?: string[]): Promise<Set<string>>;
+export declare function appendRemoteJob(jobId: string, remoteJobId: string): Promise<void>;
+export declare function loadRemoteJobs(jobId: string): Promise<string[]>;
+export declare function appendFailure(jobId: string, failure: FailedRecord): Promise<void>;
+export declare function requestCancellation(jobId: string): Promise<void>;
+export declare function clearCancellation(jobId: string): Promise<void>;
+export declare function cancellationRequested(jobId: string): Promise<boolean>;
+export declare function acquireSourceLock(lockKey: string): Promise<() => Promise<void>>;
+export declare function listJobs(): Promise<JobState[]>;
+//# sourceMappingURL=jobs.d.ts.map

package/dist/lib/jobs.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"jobs.d.ts","sourceRoot":"","sources":["../../src/lib/jobs.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,aAAa,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAEtE,MAAM,MAAM,SAAS,GACjB,SAAS,GACT,WAAW,GACX,WAAW,GACX,uBAAuB,GACvB,QAAQ,GACR,QAAQ,CAAC;AAEb,MAAM,WAAW,QAAQ;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,UAAU,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,aAAa,CAAC;IAC7B,MAAM,EAAE,SAAS,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,eAAe,EAAE,MAAM,CAAC;IACxB,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,yDAAyD;IACzD,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,YAAY;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;CACpB;AAMD,wBAAgB,OAAO,IAAI,MAAM,CAEhC;AAED,wBAAgB,OAAO,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM,CAE1C;AAMD,wBAAgB,QAAQ,IAAI,MAAM,CAEjC;AAED,wBAAsB,OAAO,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,CAAC,CAG3D;AAED,wBAAsB,QAAQ,CAAC,KAAK,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,CAO7D;AAOD,wBAAsB,iBAAiB,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAEtF;AAED,wBAAsB,gBAAgB,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,GAAE,MAAM,EAAO,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAQjG;AAED,wBAAsB,eAAe,CAAC,KAAK,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAEvF;AAED,wBAAsB,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAQrE;AAED,wBAAsB,aAAa,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAEvF;AAED,wBAAsB,mBAAmB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAEtE;AAED,wBAAsB,iBAAiB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAMpE;AAED,wBAAsB,qBAAqB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAQ3E;AAED,wBAAsB,iBAAiB,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,CAqCrF;AAED,wBAAsB,QAAQ,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC,CAcpD"}

package/dist/lib/jobs.js

@@ -0,0 +1,146 @@
+import { createHash, randomUUID } from "node:crypto";
+import { promises as fs } from "node:fs";
+import { homedir } from "node:os";
+import { dirname, join } from "node:path";
+function resolveJobsDir() {
+    return process.env.GETMNEMO_MIGRATION_DIR ?? join(homedir(), ".getmnemo", "migrations");
+}
+export function jobsDir() {
+    return resolveJobsDir();
+}
+export function jobPath(id) {
+    return join(resolveJobsDir(), `${id}.json`);
+}
+function artifactPath(id, suffix) {
+    return join(resolveJobsDir(), `${id}.${suffix}`);
+}
+export function newJobId() {
+    return `mig_${randomUUID()}`;
+}
+export async function readJob(id) {
+    const raw = await fs.readFile(jobPath(id), "utf8");
+    return JSON.parse(raw);
+}
+export async function writeJob(state) {
+    const path = jobPath(state.id);
+    await fs.mkdir(dirname(path), { recursive: true, mode: 0o700 });
+    const next = { ...state, updatedAt: new Date().toISOString() };
+    const tmp = `${path}.${process.pid}.${randomUUID()}.tmp`;
+    await fs.writeFile(tmp, JSON.stringify(next, null, 2) + "\n", { mode: 0o600 });
+    await fs.rename(tmp, path);
+}
+async function appendLine(path, line) {
+    await fs.mkdir(dirname(path), { recursive: true, mode: 0o700 });
+    await fs.appendFile(path, line + "\n", { mode: 0o600 });
+}
+export async function appendProcessedId(jobId, sourceId) {
+    await appendLine(artifactPath(jobId, "processed"), sourceId);
+}
+export async function loadProcessedIds(jobId, legacy = []) {
+    try {
+        const raw = await fs.readFile(artifactPath(jobId, "processed"), "utf8");
+        return new Set([...legacy, ...raw.split("\n").filter(Boolean)]);
+    }
+    catch (error) {
+        if (error.code === "ENOENT")
+            return new Set(legacy);
+        throw error;
+    }
+}
+export async function appendRemoteJob(jobId, remoteJobId) {
+    await appendLine(artifactPath(jobId, "remote-jobs"), remoteJobId);
+}
+export async function loadRemoteJobs(jobId) {
+    try {
+        const raw = await fs.readFile(artifactPath(jobId, "remote-jobs"), "utf8");
+        return [...new Set(raw.split("\n").filter(Boolean))];
+    }
+    catch (error) {
+        if (error.code === "ENOENT")
+            return [];
+        throw error;
+    }
+}
+export async function appendFailure(jobId, failure) {
+    await appendLine(artifactPath(jobId, "failures.jsonl"), JSON.stringify(failure));
+}
+export async function requestCancellation(jobId) {
+    await fs.writeFile(artifactPath(jobId, "cancel"), `${new Date().toISOString()}\n`, { mode: 0o600 });
+}
+export async function clearCancellation(jobId) {
+    try {
+        await fs.unlink(artifactPath(jobId, "cancel"));
+    }
+    catch (error) {
+        if (error.code !== "ENOENT")
+            throw error;
+    }
+}
+export async function cancellationRequested(jobId) {
+    try {
+        await fs.access(artifactPath(jobId, "cancel"));
+        return true;
+    }
+    catch (error) {
+        if (error.code === "ENOENT")
+            return false;
+        throw error;
+    }
+}
+export async function acquireSourceLock(lockKey) {
+    const safe = createHash("sha256").update(lockKey).digest("hex").slice(0, 16);
+    const directory = resolveJobsDir();
+    const lockPath = join(directory, `${safe}.lock`);
+    await fs.mkdir(directory, { recursive: true, mode: 0o700 });
+    for (let attempt = 0; attempt < 2; attempt += 1) {
+        try {
+            const handle = await fs.open(lockPath, "wx", 0o600);
+            await handle.writeFile(`${process.pid}\n`);
+            await handle.close();
+            break;
+        }
+        catch (error) {
+            if (error.code !== "EEXIST")
+                throw error;
+            const owner = Number.parseInt(await fs.readFile(lockPath, "utf8"), 10);
+            let ownerIsAlive = Number.isInteger(owner) && owner > 0;
+            if (ownerIsAlive) {
+                try {
+                    process.kill(owner, 0);
+                }
+                catch (probeError) {
+                    ownerIsAlive = probeError.code === "EPERM";
+                }
+            }
+            if (ownerIsAlive || attempt > 0) {
+                throw new Error(`another migration is already running for this source (${lockPath})`);
+            }
+            await fs.unlink(lockPath);
+        }
+    }
+    return async () => {
+        try {
+            await fs.unlink(lockPath);
+        }
+        catch (error) {
+            if (error.code !== "ENOENT")
+                throw error;
+        }
+    };
+}
+export async function listJobs() {
+    const directory = resolveJobsDir();
+    try {
+        const files = await fs.readdir(directory);
+        const jobs = await Promise.all(files
+            .filter((file) => /^mig_.+\.json$/.test(file))
+            .map(async (file) => JSON.parse(await fs.readFile(join(directory, file), "utf8"))));
+        return jobs.sort((a, b) => b.startedAt.localeCompare(a.startedAt));
+    }
+    catch (error) {
+        if (error.code === "ENOENT")
+            return [];
+        throw error;
+    }
+}
+//# sourceMappingURL=jobs.js.map

package/dist/lib/jobs.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"jobs.js","sourceRoot":"","sources":["../../src/lib/jobs.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACrD,OAAO,EAAE,QAAQ,IAAI,EAAE,EAAE,MAAM,SAAS,CAAC;AACzC,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAClC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAuC1C,SAAS,cAAc;IACrB,OAAO,OAAO,CAAC,GAAG,CAAC,sBAAsB,IAAI,IAAI,CAAC,OAAO,EAAE,EAAE,WAAW,EAAE,YAAY,CAAC,CAAC;AAC1F,CAAC;AAED,MAAM,UAAU,OAAO;IACrB,OAAO,cAAc,EAAE,CAAC;AAC1B,CAAC;AAED,MAAM,UAAU,OAAO,CAAC,EAAU;IAChC,OAAO,IAAI,CAAC,cAAc,EAAE,EAAE,GAAG,EAAE,OAAO,CAAC,CAAC;AAC9C,CAAC;AAED,SAAS,YAAY,CAAC,EAAU,EAAE,MAAc;IAC9C,OAAO,IAAI,CAAC,cAAc,EAAE,EAAE,GAAG,EAAE,IAAI,MAAM,EAAE,CAAC,CAAC;AACnD,CAAC;AAED,MAAM,UAAU,QAAQ;IACtB,OAAO,OAAO,UAAU,EAAE,EAAE,CAAC;AAC/B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,EAAU;IACtC,MAAM,GAAG,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC;IACnD,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAa,CAAC;AACrC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,KAAe;IAC5C,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAC/B,MAAM,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IAChE,MAAM,IAAI,GAAa,EAAE,GAAG,KAAK,EAAE,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC;IACzE,MAAM,GAAG,GAAG,GAAG,IAAI,IAAI,OAAO,CAAC,GAAG,IAAI,UAAU,EAAE,MAAM,CAAC;IACzD,MAAM,EAAE,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IAC/E,MAAM,EAAE,CAAC,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;AAC7B,CAAC;AAED,KAAK,UAAU,UAAU,CAAC,IAAY,EAAE,IAAY;IAClD,MAAM,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IAChE,MAAM,EAAE,CAAC,UAAU,CAAC,IAAI,EAAE,IAAI,GAAG,IAAI,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;AAC1D,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,KAAa,EAAE,QAAgB;IACrE,MAAM,UAAU,CAAC,YAAY,CAAC,KAAK,EAAE,WAAW,CAAC,EAAE,QAAQ,CAAC,CAAC;AAC/D,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,KAAa,EAAE,SAAmB,EAAE;IACzE,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,YAAY,CAAC,KAAK,EAAE,WAAW,CAAC,EAAE,MAAM,CAAC,CAAC;QACxE,OAAO,IAAI,GAAG,CAAC,CAAC,GAAG,MAAM,EAAE,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IAClE,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ;YAAE,OAAO,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC;QAC/E,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,KAAa,EAAE,WAAmB;IACtE,MAAM,UAAU,CAAC,YAAY,CAAC,KAAK,EAAE,aAAa,CAAC,EAAE,WAAW,CAAC,CAAC;AACpE,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,KAAa;IAChD,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,YAAY,CAAC,KAAK,EAAE,aAAa,CAAC,EAAE,MAAM,CAAC,CAAC;QAC1E,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IACvD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ;YAAE,OAAO,EAAE,CAAC;QAClE,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,KAAa,EAAE,OAAqB;IACtE,MAAM,UAAU,CAAC,YAAY,CAAC,KAAK,EAAE,gBAAgB,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC;AACnF,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,KAAa;IACrD,MAAM,EAAE,CAAC,SAAS,CAAC,YAAY,CAAC,KAAK,EAAE,QAAQ,CAAC,EAAE,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;AACtG,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,KAAa;IACnD,IAAI,CAAC;QACH,MAAM,EAAE,CAAC,MAAM,CAAC,YAAY,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC;IACjD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ;YAAE,MAAM,KAAK,CAAC;IACtE,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,qBAAqB,CAAC,KAAa;IACvD,IAAI,CAAC;QACH,MAAM,EAAE,CAAC,MAAM,CAAC,YAAY,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC;QAC/C,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ;YAAE,OAAO,KAAK,CAAC;QACrE,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,OAAe;IACrD,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAC7E,MAAM,SAAS,GAAG,cAAc,EAAE,CAAC;IACnC,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,EAAE,GAAG,IAAI,OAAO,CAAC,CAAC;IACjD,MAAM,EAAE,CAAC,KAAK,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IAE5D,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,CAAC,EAAE,CAAC;QAChD,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC;YACpD,MAAM,MAAM,CAAC,SAAS,CAAC,GAAG,OAAO,CAAC,GAAG,IAAI,CAAC,CAAC;YAC3C,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;YACrB,MAAM;QACR,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ;gBAAE,MAAM,KAAK,CAAC;YACpE,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC;YACvE,IAAI,YAAY,GAAG,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,CAAC;YACxD,IAAI,YAAY,EAAE,CAAC;gBACjB,IAAI,CAAC;oBACH,OAAO,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;gBACzB,CAAC;gBAAC,OAAO,UAAU,EAAE,CAAC;oBACpB,YAAY,GAAI,UAAoC,CAAC,IAAI,KAAK,OAAO,CAAC;gBACxE,CAAC;YACH,CAAC;YACD,IAAI,YAAY,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;gBAChC,MAAM,IAAI,KAAK,CAAC,yDAAyD,QAAQ,GAAG,CAAC,CAAC;YACxF,CAAC;YACD,MAAM,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC5B,CAAC;IACH,CAAC;IAED,OAAO,KAAK,IAAI,EAAE;QAChB,IAAI,CAAC;YACH,MAAM,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC5B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ;gBAAE,MAAM,KAAK,CAAC;QACtE,CAAC;IACH,CAAC,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,QAAQ;IAC5B,MAAM,SAAS,GAAG,cAAc,EAAE,CAAC;IACnC,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAC1C,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,GAAG,CAC5B,KAAK;aACF,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;aAC7C,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,CAAa,CAAC,CACjG,CAAC;QACF,OAAO,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC;IACrE,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ;YAAE,OAAO,EAAE,CAAC;QAClE,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC"}

package/dist/lib/migrator.d.ts

@@ -0,0 +1,24 @@
+import type { Adapter, AdapterConfig, ImportLane, SourceRecord } from "../adapters/index.js";
+import { type JobState } from "./jobs.js";
+import type { DestinationWriter } from "./mnemo-writer.js";
+export interface MigrateOptions {
+    concurrency?: number;
+    resumeJobId?: string;
+    lane: ImportLane;
+    containerTag: string;
+    adapterConfig: AdapterConfig;
+    onProgress?: (state: {
+        written: number;
+        reused: number;
+        failed: number;
+        total: number | null;
+    }) => void;
+}
+export interface PlanResult {
+    total: number | null;
+    sample: SourceRecord[];
+}
+export declare function planMigration(adapter: Adapter, sampleSize?: number): Promise<PlanResult>;
+export declare function migrate(adapter: Adapter, writer: DestinationWriter, options: MigrateOptions): Promise<JobState>;
+export declare function reconcileMigration(writer: DestinationWriter, input: JobState, timeoutMs: number): Promise<JobState>;
+//# sourceMappingURL=migrator.d.ts.map

package/dist/lib/migrator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"migrator.d.ts","sourceRoot":"","sources":["../../src/lib/migrator.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,aAAa,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAC7F,OAAO,EAYL,KAAK,QAAQ,EACd,MAAM,WAAW,CAAC;AACnB,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAG3D,MAAM,WAAW,cAAc;IAC7B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,UAAU,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,aAAa,CAAC;IAC7B,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAA;KAAE,KAAK,IAAI,CAAC;CACzG;AAED,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,MAAM,EAAE,YAAY,EAAE,CAAC;CACxB;AAMD,wBAAsB,aAAa,CAAC,OAAO,EAAE,OAAO,EAAE,UAAU,SAAI,GAAG,OAAO,CAAC,UAAU,CAAC,CAQzF;AAiDD,wBAAsB,OAAO,CAC3B,OAAO,EAAE,OAAO,EAChB,MAAM,EAAE,iBAAiB,EACzB,OAAO,EAAE,cAAc,GACtB,OAAO,CAAC,QAAQ,CAAC,CAyFnB;AAED,wBAAsB,kBAAkB,CACtC,MAAM,EAAE,iBAAiB,EACzB,KAAK,EAAE,QAAQ,EACf,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC,QAAQ,CAAC,CAwCnB"}

package/dist/lib/migrator.js

@@ -0,0 +1,188 @@
+import { acquireSourceLock, appendFailure, appendProcessedId, appendRemoteJob, cancellationRequested, clearCancellation, loadProcessedIds, loadRemoteJobs, newJobId, readJob, writeJob, } from "./jobs.js";
+import { sourceIdentity } from "./mnemo-writer.js";
+export async function planMigration(adapter, sampleSize = 5) {
+    const total = await adapter.count();
+    const sample = [];
+    for await (const record of adapter.iterate()) {
+        sample.push(record);
+        if (sample.length >= sampleSize)
+            break;
+    }
+    return { total, sample };
+}
+async function processBatch(records, adapter, writer, options) {
+    return Promise.all(records.map(async (record) => {
+        const sourceId = sourceIdentity(record);
+        try {
+            const result = await writer.write(record, {
+                lane: options.lane,
+                source: adapter.name,
+                containerTag: options.containerTag,
+            });
+            return { kind: "success", sourceId, ...result };
+        }
+        catch (error) {
+            return {
+                kind: "failure",
+                sourceId,
+                error: error instanceof Error ? error.message : String(error),
+            };
+        }
+    }));
+}
+function newState(adapter, options, total) {
+    return {
+        id: newJobId(),
+        source: adapter.name,
+        lane: options.lane,
+        containerTag: options.containerTag,
+        adapterConfig: options.adapterConfig,
+        status: "running",
+        startedAt: new Date().toISOString(),
+        updatedAt: new Date().toISOString(),
+        total,
+        written: 0,
+        reused: 0,
+        failed: 0,
+        remoteCompleted: 0,
+        remoteFailed: 0,
+        remotePending: 0,
+    };
+}
+export async function migrate(adapter, writer, options) {
+    const requestedConcurrency = options.concurrency ?? 5;
+    const concurrency = Number.isFinite(requestedConcurrency)
+        ? Math.max(1, Math.min(25, Math.floor(requestedConcurrency)))
+        : 5;
+    let state = options.resumeJobId
+        ? await readJob(options.resumeJobId)
+        : newState(adapter, options, await adapter.count());
+    if (options.resumeJobId) {
+        if (state.source !== adapter.name)
+            throw new Error("resume source does not match the original job");
+        if (state.lane !== options.lane)
+            throw new Error("resume lane does not match the original job");
+        if (state.containerTag !== options.containerTag) {
+            throw new Error("resume container does not match the original job");
+        }
+        state.status = "running";
+        state.finishedAt = undefined;
+        state.failed = 0;
+        state.lastError = undefined;
+    }
+    const lockKey = [adapter.name, options.containerTag, options.adapterConfig.file ?? "remote"].join(":");
+    const releaseLock = await acquireSourceLock(lockKey);
+    await clearCancellation(state.id);
+    await writeJob(state);
+    const seen = await loadProcessedIds(state.id, state.processedIds ?? []);
+    let batch = [];
+    const commitBatch = async () => {
+        if (batch.length === 0)
+            return;
+        const outcomes = await processBatch(batch, adapter, writer, options);
+        batch = [];
+        for (const outcome of outcomes) {
+            if (outcome.kind === "success") {
+                state.written += 1;
+                if (outcome.reused)
+                    state.reused += 1;
+                seen.add(outcome.sourceId);
+                await appendProcessedId(state.id, outcome.sourceId);
+                for (const remoteJobId of outcome.remoteJobIds) {
+                    await appendRemoteJob(state.id, remoteJobId);
+                    state.remotePending += 1;
+                }
+            }
+            else {
+                state.failed += 1;
+                state.lastError = outcome.error;
+                await appendFailure(state.id, {
+                    sourceId: outcome.sourceId,
+                    error: outcome.error,
+                    occurredAt: new Date().toISOString(),
+                });
+            }
+            options.onProgress?.({
+                written: state.written,
+                reused: state.reused,
+                failed: state.failed,
+                total: state.total ?? null,
+            });
+        }
+        await writeJob(state);
+    };
+    try {
+        for await (const record of adapter.iterate()) {
+            if (await cancellationRequested(state.id)) {
+                await commitBatch();
+                state.status = "paused";
+                await writeJob(state);
+                return state;
+            }
+            const id = sourceIdentity(record);
+            if (seen.has(id))
+                continue;
+            batch.push(record);
+            if (batch.length >= concurrency)
+                await commitBatch();
+        }
+        await commitBatch();
+        state.status = options.lane === "document" ? "submitted" : state.failed > 0 ? "completed_with_errors" : "completed";
+        state.finishedAt = new Date().toISOString();
+        await writeJob(state);
+        return state;
+    }
+    catch (error) {
+        state.status = "failed";
+        state.finishedAt = new Date().toISOString();
+        state.lastError = error instanceof Error ? error.message : String(error);
+        await writeJob(state);
+        throw error;
+    }
+    finally {
+        await releaseLock();
+    }
+}
+export async function reconcileMigration(writer, input, timeoutMs) {
+    const state = { ...input };
+    const pending = new Set(await loadRemoteJobs(state.id));
+    const deadline = Date.now() + timeoutMs;
+    let completed = 0;
+    let failed = 0;
+    while (pending.size > 0 && Date.now() < deadline) {
+        const ids = [...pending];
+        for (let offset = 0; offset < ids.length; offset += 50) {
+            const jobs = await writer.getJobs(ids.slice(offset, offset + 50));
+            for (const job of jobs) {
+                if (job.status === "completed") {
+                    pending.delete(job.id);
+                    completed += 1;
+                }
+                else if (job.status === "failed") {
+                    pending.delete(job.id);
+                    failed += 1;
+                    state.lastError = job.error?.message ?? `ingestion job ${job.id} failed`;
+                }
+            }
+        }
+        state.remoteCompleted = completed;
+        state.remoteFailed = failed;
+        state.remotePending = pending.size;
+        await writeJob(state);
+        if (pending.size > 0)
+            await new Promise((resolve) => setTimeout(resolve, 2_000));
+    }
+    state.remoteCompleted = completed;
+    state.remoteFailed = failed;
+    state.remotePending = pending.size;
+    if (pending.size === 0) {
+        state.status = state.failed + failed > 0 ? "completed_with_errors" : "completed";
+        state.finishedAt = new Date().toISOString();
+    }
+    else {
+        state.status = "submitted";
+    }
+    await writeJob(state);
+    return state;
+}
+//# sourceMappingURL=migrator.js.map