@amemhq/core 1.1.0 → 2.1.0

package/dist/cli-migrate.cjs

@@ -31,6 +31,8 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
 // src/cli-migrate.ts
 var cli_migrate_exports = {};
 __export(cli_migrate_exports, {
+  carried: () => carried,
+  deriveTarget: () => deriveTarget,
   parseArgs: () => parseArgs
 });
 module.exports = __toCommonJS(cli_migrate_exports);
@@ -40,9 +42,10 @@ var pipeline = null;
 var extractor = null;
 var loadedKey = null;
 var cachedDim = null;
-var DEFAULT_EMBEDDING_MODEL = "Xenova/paraphrase-multilingual-MiniLM-L12-v2";
+var DEFAULT_EMBEDDING_MODEL = "Xenova/bge-m3";
+var pinnedModel = null;
 function getEmbeddingModel() {
-  return process.env.AMEM_EMBED_MODEL?.trim() || DEFAULT_EMBEDDING_MODEL;
+  return process.env.AMEM_EMBED_MODEL?.trim() || pinnedModel || DEFAULT_EMBEDDING_MODEL;
 }
 var CLS_POOLED_MODELS = /* @__PURE__ */ new Set([
   "bge-m3",
@@ -69,6 +72,27 @@ function getEmbeddingDevice() {
 function getEmbeddingDtype() {
   return process.env.AMEM_EMBED_DTYPE?.trim() || void 0;
 }
+function applyModelPaths(env) {
+  const cache = process.env.AMEM_MODEL_CACHE?.trim();
+  if (cache) env.cacheDir = cache;
+  const local = process.env.AMEM_MODEL_DIR?.trim();
+  if (local) {
+    env.localModelPath = local;
+    env.allowLocalModels = true;
+  }
+}
+function makeProgressReporter() {
+  const lastPct = /* @__PURE__ */ new Map();
+  return (e) => {
+    if (e.status !== "progress" || !e.file || typeof e.progress !== "number") return;
+    if (!e.file.endsWith(".onnx") && !e.file.endsWith(".onnx_data")) return;
+    const pct = Math.floor(e.progress / 10) * 10;
+    if (lastPct.get(e.file) === pct) return;
+    lastPct.set(e.file, pct);
+    const size = e.total ? ` of ${(e.total / 1e9).toFixed(2)} GB` : "";
+    console.log(`[amem] downloading ${e.file}: ${pct}%${size}`);
+  };
+}
 function extractorKey() {
   return `${getEmbeddingModel()}|${getEmbeddingDevice() ?? ""}|${getEmbeddingDtype() ?? ""}`;
 }
@@ -78,11 +102,13 @@ async function getExtractor() {
   if (!pipeline) {
     const mod = await import("@huggingface/transformers");
     pipeline = mod.pipeline;
+    applyModelPaths(mod.env);
   }
   const device = getEmbeddingDevice();
   const dtype = getEmbeddingDtype();
   extractor = await pipeline("feature-extraction", getEmbeddingModel(), {
     revision: "main",
+    progress_callback: makeProgressReporter(),
     // Omitted entirely when unset, so an unconfigured install gets exactly the
     // library defaults it got before these existed.
     ...device ? { device } : {},
@@ -202,6 +228,47 @@ async function createCollectionRaw(collection, size) {
 async function upsertPointsRaw(collection, points) {
   await qdrant("PUT", `/collections/${collection}/points?wait=true`, { points });
 }
+async function scrollIdsRaw(collection, limit = 1e4) {
+  const ids = /* @__PURE__ */ new Set();
+  let offset = void 0;
+  for (; ; ) {
+    const body = { with_payload: false, with_vector: false, limit };
+    if (offset !== void 0 && offset !== null) body.offset = offset;
+    const res = await qdrant("POST", `/collections/${collection}/points/scroll`, body);
+    for (const p of res.points) ids.add(String(p.id));
+    offset = res.next_page_offset;
+    if (offset === void 0 || offset === null || res.points.length === 0) break;
+  }
+  return ids;
+}
+async function deleteCollectionRaw(collection) {
+  await qdrant("DELETE", `/collections/${collection}`);
+}
+async function snapshotCollectionRaw(collection) {
+  const r = await qdrant("POST", `/collections/${collection}/snapshots`);
+  return { name: r.name, size: r.size ?? 0 };
+}
+async function resolveAliasRaw(alias) {
+  try {
+    const res = await qdrant("GET", `/aliases`);
+    return res.aliases.find((a) => a.alias_name === alias)?.collection_name ?? null;
+  } catch {
+    return null;
+  }
+}
+async function createAliasRaw(alias, collection) {
+  await qdrant("POST", `/collections/aliases`, {
+    actions: [{ create_alias: { collection_name: collection, alias_name: alias } }]
+  });
+}
+async function setAliasRaw(alias, collection) {
+  await qdrant("POST", `/collections/aliases`, {
+    actions: [
+      { delete_alias: { alias_name: alias } },
+      { create_alias: { collection_name: collection, alias_name: alias } }
+    ]
+  });
+}
 function noteToPoint(note) {
   return {
     id: note.id,
@@ -795,8 +862,19 @@ async function migrateCollection(opts) {
   );
   if (dryRun) {
     log("[migrate] dry run \u2014 nothing written. Pass dryRun: false to apply.");
-    return { total: notes.length, missingDerived, refreshed: 0, migrated: 0, sourceDim, targetDim, model, dryRun: true };
+    return {
+      total: notes.length,
+      missingDerived,
+      refreshed: 0,
+      migrated: 0,
+      skipped: 0,
+      sourceDim,
+      targetDim,
+      model,
+      dryRun: true
+    };
   }
+  let alreadyDone = /* @__PURE__ */ new Set();
   const existingTargetDim = await collectionDimRaw(to);
   if (existingTargetDim === null) {
     await createCollectionRaw(to, targetDim);
@@ -805,9 +883,17 @@ async function migrateCollection(opts) {
     if (existingTargetDim !== targetDim) {
       throw new Error(`migrate: target "${to}" exists at ${existingTargetDim}d but the model produces ${targetDim}d`);
     }
-    const existingCount = await countPointsRaw(to);
-    if (existingCount > 0) {
-      throw new Error(`migrate: target "${to}" already holds ${existingCount} point(s); use an empty collection`);
+    const present = await scrollIdsRaw(to);
+    if (present.size > 0) {
+      const sourceIds = new Set(notes.map((n) => n.id));
+      const foreign = [...present].filter((id) => !sourceIds.has(id));
+      if (foreign.length > 0) {
+        throw new Error(
+          `migrate: target "${to}" holds ${foreign.length} point(s) that are not in "${from}" (e.g. ${foreign[0]}). That is not an interrupted migration \u2014 use a different target.`
+        );
+      }
+      alreadyDone = present;
+      log(`[migrate] resuming: ${present.size} of ${notes.length} already in ${to}`);
     }
   }
   let refreshed = 0;
@@ -821,6 +907,7 @@ async function migrateCollection(opts) {
     buffer = [];
   };
   for (const note of notes) {
+    if (alreadyDone.has(note.id)) continue;
     if (refreshFields && missingDerivedFields(note)) {
       try {
         const built = await llmConstructNote(note.content);
@@ -836,7 +923,7 @@ async function migrateCollection(opts) {
     buffer.push(point);
     if (buffer.length >= BATCH) {
       await flush();
-      log(`[migrate] ${migrated}/${notes.length}`);
+      log(`[migrate] ${alreadyDone.size + migrated}/${notes.length}`);
     }
   }
   await flush();
@@ -844,82 +931,200 @@ async function migrateCollection(opts) {
   if (finalCount !== notes.length) {
     warn(`[migrate] target holds ${finalCount} point(s) but the source had ${notes.length} \u2014 check before switching`);
   }
-  log(
-    `[migrate] done: ${migrated} migrated, ${refreshed} re-extracted. "${from}" is untouched \u2014 switch with AMEM_COLLECTION=${to}, and keep the old one until you are satisfied.`
-  );
-  return { total: notes.length, missingDerived, refreshed, migrated, sourceDim, targetDim, model, dryRun: false };
+  log(`[migrate] done: ${migrated} written, ${alreadyDone.size} already present, ${refreshed} re-extracted.`);
+  return {
+    total: notes.length,
+    missingDerived,
+    refreshed,
+    migrated,
+    skipped: alreadyDone.size,
+    sourceDim,
+    targetDim,
+    model,
+    dryRun: false
+  };
+}
+async function switchToMigrated(opts) {
+  const { name, to } = opts;
+  const log = opts.logger?.info ?? ((m) => console.log(m));
+  if (name === to) throw new Error(`switch: "${name}" and "${to}" are the same collection`);
+  let snap;
+  const already = await resolveAliasRaw(name);
+  if (already === to) {
+    log(`[switch] "${name}" already points at "${to}" \u2014 nothing to do`);
+    return { name, to, moved: await countPointsRaw(to) };
+  }
+  const targetCount = await countPointsRaw(to);
+  if (targetCount === 0) throw new Error(`switch: "${to}" is empty \u2014 migrate into it first`);
+  if (already === null) {
+    const sourceCount = await countPointsRaw(name);
+    if (targetCount < sourceCount) {
+      throw new Error(
+        `switch: "${to}" holds ${targetCount} point(s) but "${name}" still holds ${sourceCount}. The migration is not finished \u2014 run it again before switching.`
+      );
+    }
+    log(`[switch] verified ${targetCount} in "${to}" against ${sourceCount} in "${name}"`);
+    if (opts.snapshot !== false) {
+      snap = await snapshotCollectionRaw(name);
+      log(`[switch] snapshotted "${name}" \u2192 ${snap.name} (${(snap.size / 1e6).toFixed(0)} MB)`);
+    }
+    await deleteCollectionRaw(name);
+    log(`[switch] dropped "${name}"`);
+    await createAliasRaw(name, to);
+  } else {
+    await setAliasRaw(name, to);
+  }
+  log(`[switch] "${name}" now resolves to "${to}"`);
+  return { name, to, moved: targetCount, snapshot: snap };
 }
 
 // src/cli-migrate.ts
-var USAGE = `amem-migrate \u2014 rebuild a collection under the current embedding model
+var USAGE = `amem-migrate \u2014 move a memory store onto a different embedding model
 
-  npx --package=@amemhq/core amem-migrate --to <collection> [options]
+  amem-migrate              what state the store is in, and what comes next
+  amem-migrate --apply      do the next step; safe to interrupt and re-run
+  amem-migrate --switch     put the new store behind the old name (irreversible)
 
 Options
-  --to <name>           Target collection. Required. Must not already hold points.
-  --from <name>         Source collection. Defaults to AMEM_COLLECTION.
-  --apply               Actually write. Without this it is a dry run.
-  --no-refresh-fields   Skip re-extracting keywords/tags for notes that never
-                        had them. Faster, and makes no LLM calls at all.
+  --from-collection <name>  the store to migrate. Defaults to AMEM_COLLECTION.
+  --to-collection <name>    where to build it. Derived from the source if omitted.
+  --no-refresh-fields       skip re-extracting keywords for notes that never had
+                            them. Makes the run completely offline.
   -h, --help
 
-The model is whatever AMEM_EMBED_MODEL says, so set that first:
+  AMEM_MODEL_CACHE=<dir>    where model weights are cached. Set the same value
+                            everywhere and the plugin and this command share one
+                            copy instead of downloading it each.
+  AMEM_MODEL_DIR=<dir>      read weights already on disk instead of downloading.
+
+Migrates onto amem's current default unless AMEM_EMBED_MODEL says otherwise:
+
+  AMEM_EMBED_MODEL=Alibaba-NLP/gte-multilingual-base amem-migrate --apply
 
-  AMEM_EMBED_MODEL=Xenova/bge-m3 \\
-    npx --package=@amemhq/core amem-migrate --to amem_notes_v2 --apply
+Stop your agent before --apply, not before this. The bare run only downloads and
+reports, and the download is the long part \u2014 leaving the agent up through it costs
+nothing, while writing underneath a running agent means notes arrive after the
+rebuild has read them and you have to run --apply again.
 
-The source is only ever read. If the result looks wrong, point AMEM_COLLECTION
-back at it and nothing has been lost.`;
+Roughly: the download is your bandwidth (2.27 GB for the default), the rebuild is
+your CPU (about 3 notes a second on an M4), and notes missing keywords cost one
+LLM call each.
+
+Nothing before --switch touches the original. If a run looks wrong, delete the
+target and start again.`;
 function parseArgs(argv) {
   const value = (flag) => {
     const i = argv.indexOf(flag);
     return i === -1 ? void 0 : argv[i + 1];
   };
   return {
-    help: argv.includes("-h") || argv.includes("--help"),
-    to: value("--to"),
-    from: value("--from"),
-    dryRun: !argv.includes("--apply"),
+    // `help` as a bare word too. Without it, typing what most CLIs accept starts
+    // a multi-gigabyte model download instead of printing anything.
+    help: argv.includes("-h") || argv.includes("--help") || argv[0] === "help",
+    apply: argv.includes("--apply"),
+    switchOver: argv.includes("--switch"),
+    from: value("--from-collection"),
+    to: value("--to-collection"),
     refreshFields: !argv.includes("--no-refresh-fields")
   };
 }
+function deriveTarget(source) {
+  const m = source.match(/^(.*)_v(\d+)$/);
+  return m ? `${m[1]}_v${Number(m[2]) + 1}` : `${source}_v2`;
+}
+function carried(args) {
+  return (args.from ? ` --from-collection ${args.from}` : "") + (args.to ? ` --to-collection ${args.to}` : "");
+}
+async function detect(from, to) {
+  const alias = await resolveAliasRaw(from);
+  if (alias !== null) return { kind: "switched", points: await countPointsRaw(from) };
+  const sourceDim = await collectionDimRaw(from);
+  if (sourceDim === null) return { kind: "no-source" };
+  const modelDim = await getEmbeddingDim();
+  if (sourceDim === modelDim) return { kind: "already-current", model: getEmbeddingModel() };
+  const notes = await countPointsRaw(from);
+  const targetDim = await collectionDimRaw(to);
+  if (targetDim === null) return { kind: "not-started", notes };
+  const done = (await scrollIdsRaw(to)).size;
+  return done >= notes ? { kind: "ready-to-switch", notes } : { kind: "partial", done, notes };
+}
 async function main() {
   const args = parseArgs(process.argv.slice(2));
   if (args.help) {
     console.log(USAGE);
     return;
   }
-  const to = args.to;
-  if (!to) {
-    console.error("amem-migrate: --to <collection> is required\n");
-    console.error(USAGE);
-    process.exitCode = 1;
-    return;
-  }
   const from = args.from ?? getCollection();
-  const { dryRun, refreshFields } = args;
+  const to = args.to ?? deriveTarget(from);
+  const flags = carried(args);
+  const phase = await detect(from, to);
+  console.log(`store:  ${from}`);
   console.log(`model:  ${getEmbeddingModel()}`);
-  console.log(`from:   ${from}`);
-  console.log(`to:     ${to}`);
-  console.log(`mode:   ${dryRun ? "dry run \u2014 nothing will be written" : "APPLY"}
-`);
-  const result = await migrateCollection({ from, to, dryRun, refreshFields });
-  console.log(`
-${result.total} notes in source (${result.sourceDim}-dim)`);
-  console.log(`target is ${result.targetDim}-dim`);
-  if (result.missingDerived > 0) {
-    console.log(
-      `${result.missingDerived} note(s) predate the extraction pipeline` + (refreshFields ? `, ${result.refreshed} re-extracted` : " \u2014 skipped (--no-refresh-fields)")
-    );
+  switch (phase.kind) {
+    case "no-source":
+      console.error(`
+No collection named "${from}". Nothing to migrate.`);
+      process.exitCode = 1;
+      return;
+    case "switched":
+      console.log(`
+"${from}" is already an alias \u2014 ${phase.points} notes, nothing to do.`);
+      return;
+    case "already-current":
+      console.log(`
+Already on ${phase.model}. Nothing to migrate.`);
+      return;
+    case "not-started":
+      if (!args.apply) {
+        console.log(`
+${phase.notes} notes to rebuild into "${to}".`);
+        console.log(`Run "amem-migrate${flags} --apply" to start. "${from}" is only read.`);
+        return;
+      }
+      break;
+    case "partial":
+      if (!args.apply) {
+        console.log(`
+${phase.done} of ${phase.notes} rebuilt into "${to}".`);
+        console.log(`Run "amem-migrate${flags} --apply" to carry on from there.`);
+        return;
+      }
+      break;
+    case "ready-to-switch":
+      if (!args.switchOver) {
+        console.log(`
+All ${phase.notes} notes are in "${to}". "${from}" is untouched.`);
+        console.log(`Check it, then run "amem-migrate${flags} --switch" to put "${to}" behind the name "${from}".`);
+        console.log(`That drops "${from}" and cannot be undone.`);
+        return;
+      }
+      {
+        const res = await switchToMigrated({ name: from, to });
+        console.log(`
+Done. Nothing to change in your config \u2014 "${from}" now resolves to "${to}".`);
+        if (res.snapshot) {
+          console.log(`
+The old store is kept as a snapshot, ${(res.snapshot.size / 1e6).toFixed(0)} MB:`);
+          console.log(`  <qdrant storage>/snapshots/${from}/${res.snapshot.name}`);
+          console.log(`Delete that file once the new store has proven itself. Nothing else will.`);
+        }
+      }
+      return;
   }
-  if (dryRun) {
-    console.log(`
-Nothing was written. Re-run with --apply to migrate.`);
+  if (args.switchOver) {
+    console.error(`
+Not finished yet \u2014 run "amem-migrate${flags} --apply" until it is before switching.`);
+    process.exitCode = 1;
     return;
   }
+  const result = await migrateCollection({ from, to, dryRun: false, refreshFields: args.refreshFields });
   console.log(`
-${result.migrated} notes written to "${to}".`);
-  console.log(`Point AMEM_COLLECTION at "${to}" to start using it. "${from}" is untouched.`);
+${result.migrated + result.skipped} of ${result.total} rebuilt.`);
+  if (result.migrated + result.skipped >= result.total) {
+    console.log(`Check "${to}", then run "amem-migrate${flags} --switch".`);
+  } else {
+    console.log(`Run "amem-migrate${flags} --apply" again to carry on.`);
+  }
 }
 main().catch((err) => {
   console.error(`amem-migrate: ${err instanceof Error ? err.message : String(err)}`);
@@ -927,6 +1132,8 @@ main().catch((err) => {
 });
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
+  carried,
+  deriveTarget,
   parseArgs
 });
 //# sourceMappingURL=cli-migrate.cjs.map