@amemhq/core 1.1.0 → 2.0.0

package/dist/cli-migrate.cjs

@@ -31,6 +31,8 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
 // src/cli-migrate.ts
 var cli_migrate_exports = {};
 __export(cli_migrate_exports, {
+  carried: () => carried,
+  deriveTarget: () => deriveTarget,
   parseArgs: () => parseArgs
 });
 module.exports = __toCommonJS(cli_migrate_exports);
@@ -40,9 +42,11 @@ var pipeline = null;
 var extractor = null;
 var loadedKey = null;
 var cachedDim = null;
-var DEFAULT_EMBEDDING_MODEL = "Xenova/paraphrase-multilingual-MiniLM-L12-v2";
+var DEFAULT_EMBEDDING_MODEL = "Xenova/bge-m3";
+var DEFAULT_MODEL_DTYPE = "fp16";
+var pinnedModel = null;
 function getEmbeddingModel() {
-  return process.env.AMEM_EMBED_MODEL?.trim() || DEFAULT_EMBEDDING_MODEL;
+  return process.env.AMEM_EMBED_MODEL?.trim() || pinnedModel || DEFAULT_EMBEDDING_MODEL;
 }
 var CLS_POOLED_MODELS = /* @__PURE__ */ new Set([
   "bge-m3",
@@ -67,7 +71,9 @@ function getEmbeddingDevice() {
   return process.env.AMEM_EMBED_DEVICE?.trim() || void 0;
 }
 function getEmbeddingDtype() {
-  return process.env.AMEM_EMBED_DTYPE?.trim() || void 0;
+  const explicit = process.env.AMEM_EMBED_DTYPE?.trim();
+  if (explicit) return explicit;
+  return getEmbeddingModel() === DEFAULT_EMBEDDING_MODEL ? DEFAULT_MODEL_DTYPE : void 0;
 }
 function extractorKey() {
   return `${getEmbeddingModel()}|${getEmbeddingDevice() ?? ""}|${getEmbeddingDtype() ?? ""}`;
@@ -202,6 +208,43 @@ async function createCollectionRaw(collection, size) {
 async function upsertPointsRaw(collection, points) {
   await qdrant("PUT", `/collections/${collection}/points?wait=true`, { points });
 }
+async function scrollIdsRaw(collection, limit = 1e4) {
+  const ids = /* @__PURE__ */ new Set();
+  let offset = void 0;
+  for (; ; ) {
+    const body = { with_payload: false, with_vector: false, limit };
+    if (offset !== void 0 && offset !== null) body.offset = offset;
+    const res = await qdrant("POST", `/collections/${collection}/points/scroll`, body);
+    for (const p of res.points) ids.add(String(p.id));
+    offset = res.next_page_offset;
+    if (offset === void 0 || offset === null || res.points.length === 0) break;
+  }
+  return ids;
+}
+async function deleteCollectionRaw(collection) {
+  await qdrant("DELETE", `/collections/${collection}`);
+}
+async function resolveAliasRaw(alias) {
+  try {
+    const res = await qdrant("GET", `/aliases`);
+    return res.aliases.find((a) => a.alias_name === alias)?.collection_name ?? null;
+  } catch {
+    return null;
+  }
+}
+async function createAliasRaw(alias, collection) {
+  await qdrant("POST", `/collections/aliases`, {
+    actions: [{ create_alias: { collection_name: collection, alias_name: alias } }]
+  });
+}
+async function setAliasRaw(alias, collection) {
+  await qdrant("POST", `/collections/aliases`, {
+    actions: [
+      { delete_alias: { alias_name: alias } },
+      { create_alias: { collection_name: collection, alias_name: alias } }
+    ]
+  });
+}
 function noteToPoint(note) {
   return {
     id: note.id,
@@ -795,8 +838,19 @@ async function migrateCollection(opts) {
   );
   if (dryRun) {
     log("[migrate] dry run \u2014 nothing written. Pass dryRun: false to apply.");
-    return { total: notes.length, missingDerived, refreshed: 0, migrated: 0, sourceDim, targetDim, model, dryRun: true };
+    return {
+      total: notes.length,
+      missingDerived,
+      refreshed: 0,
+      migrated: 0,
+      skipped: 0,
+      sourceDim,
+      targetDim,
+      model,
+      dryRun: true
+    };
   }
+  let alreadyDone = /* @__PURE__ */ new Set();
   const existingTargetDim = await collectionDimRaw(to);
   if (existingTargetDim === null) {
     await createCollectionRaw(to, targetDim);
@@ -805,9 +859,17 @@ async function migrateCollection(opts) {
     if (existingTargetDim !== targetDim) {
       throw new Error(`migrate: target "${to}" exists at ${existingTargetDim}d but the model produces ${targetDim}d`);
     }
-    const existingCount = await countPointsRaw(to);
-    if (existingCount > 0) {
-      throw new Error(`migrate: target "${to}" already holds ${existingCount} point(s); use an empty collection`);
+    const present = await scrollIdsRaw(to);
+    if (present.size > 0) {
+      const sourceIds = new Set(notes.map((n) => n.id));
+      const foreign = [...present].filter((id) => !sourceIds.has(id));
+      if (foreign.length > 0) {
+        throw new Error(
+          `migrate: target "${to}" holds ${foreign.length} point(s) that are not in "${from}" (e.g. ${foreign[0]}). That is not an interrupted migration \u2014 use a different target.`
+        );
+      }
+      alreadyDone = present;
+      log(`[migrate] resuming: ${present.size} of ${notes.length} already in ${to}`);
     }
   }
   let refreshed = 0;
@@ -821,6 +883,7 @@ async function migrateCollection(opts) {
     buffer = [];
   };
   for (const note of notes) {
+    if (alreadyDone.has(note.id)) continue;
     if (refreshFields && missingDerivedFields(note)) {
       try {
         const built = await llmConstructNote(note.content);
@@ -836,7 +899,7 @@ async function migrateCollection(opts) {
     buffer.push(point);
     if (buffer.length >= BATCH) {
       await flush();
-      log(`[migrate] ${migrated}/${notes.length}`);
+      log(`[migrate] ${alreadyDone.size + migrated}/${notes.length}`);
     }
   }
   await flush();
@@ -844,32 +907,68 @@ async function migrateCollection(opts) {
   if (finalCount !== notes.length) {
     warn(`[migrate] target holds ${finalCount} point(s) but the source had ${notes.length} \u2014 check before switching`);
   }
-  log(
-    `[migrate] done: ${migrated} migrated, ${refreshed} re-extracted. "${from}" is untouched \u2014 switch with AMEM_COLLECTION=${to}, and keep the old one until you are satisfied.`
-  );
-  return { total: notes.length, missingDerived, refreshed, migrated, sourceDim, targetDim, model, dryRun: false };
+  log(`[migrate] done: ${migrated} written, ${alreadyDone.size} already present, ${refreshed} re-extracted.`);
+  return {
+    total: notes.length,
+    missingDerived,
+    refreshed,
+    migrated,
+    skipped: alreadyDone.size,
+    sourceDim,
+    targetDim,
+    model,
+    dryRun: false
+  };
+}
+async function switchToMigrated(opts) {
+  const { name, to } = opts;
+  const log = opts.logger?.info ?? ((m) => console.log(m));
+  if (name === to) throw new Error(`switch: "${name}" and "${to}" are the same collection`);
+  const already = await resolveAliasRaw(name);
+  if (already === to) {
+    log(`[switch] "${name}" already points at "${to}" \u2014 nothing to do`);
+    return { name, to, moved: await countPointsRaw(to) };
+  }
+  const targetCount = await countPointsRaw(to);
+  if (targetCount === 0) throw new Error(`switch: "${to}" is empty \u2014 migrate into it first`);
+  if (already === null) {
+    const sourceCount = await countPointsRaw(name);
+    if (targetCount < sourceCount) {
+      throw new Error(
+        `switch: "${to}" holds ${targetCount} point(s) but "${name}" still holds ${sourceCount}. The migration is not finished \u2014 run it again before switching.`
+      );
+    }
+    log(`[switch] verified ${targetCount} in "${to}" against ${sourceCount} in "${name}"`);
+    await deleteCollectionRaw(name);
+    log(`[switch] dropped "${name}"`);
+    await createAliasRaw(name, to);
+  } else {
+    await setAliasRaw(name, to);
+  }
+  log(`[switch] "${name}" now resolves to "${to}"`);
+  return { name, to, moved: targetCount };
 }
 
 // src/cli-migrate.ts
-var USAGE = `amem-migrate \u2014 rebuild a collection under the current embedding model
+var USAGE = `amem-migrate \u2014 move a memory store onto a different embedding model
 
-  npx --package=@amemhq/core amem-migrate --to <collection> [options]
+  amem-migrate              what state the store is in, and what comes next
+  amem-migrate --apply      do the next step; safe to interrupt and re-run
+  amem-migrate --switch     put the new store behind the old name (irreversible)
 
 Options
-  --to <name>           Target collection. Required. Must not already hold points.
-  --from <name>         Source collection. Defaults to AMEM_COLLECTION.
-  --apply               Actually write. Without this it is a dry run.
-  --no-refresh-fields   Skip re-extracting keywords/tags for notes that never
-                        had them. Faster, and makes no LLM calls at all.
+  --from-collection <name>  the store to migrate. Defaults to AMEM_COLLECTION.
+  --to-collection <name>    where to build it. Derived from the source if omitted.
+  --no-refresh-fields       skip re-extracting keywords for notes that never had
+                            them. Makes the run completely offline.
   -h, --help
 
-The model is whatever AMEM_EMBED_MODEL says, so set that first:
+Migrates onto amem's current default unless AMEM_EMBED_MODEL says otherwise:
 
-  AMEM_EMBED_MODEL=Xenova/bge-m3 \\
-    npx --package=@amemhq/core amem-migrate --to amem_notes_v2 --apply
+  AMEM_EMBED_MODEL=Alibaba-NLP/gte-multilingual-base amem-migrate --apply
 
-The source is only ever read. If the result looks wrong, point AMEM_COLLECTION
-back at it and nothing has been lost.`;
+Nothing before --switch touches the original. If a run looks wrong, delete the
+target and start again.`;
 function parseArgs(argv) {
   const value = (flag) => {
     const i = argv.indexOf(flag);
@@ -877,49 +976,102 @@ function parseArgs(argv) {
   };
   return {
     help: argv.includes("-h") || argv.includes("--help"),
-    to: value("--to"),
-    from: value("--from"),
-    dryRun: !argv.includes("--apply"),
+    apply: argv.includes("--apply"),
+    switchOver: argv.includes("--switch"),
+    from: value("--from-collection"),
+    to: value("--to-collection"),
     refreshFields: !argv.includes("--no-refresh-fields")
   };
 }
+function deriveTarget(source) {
+  const m = source.match(/^(.*)_v(\d+)$/);
+  return m ? `${m[1]}_v${Number(m[2]) + 1}` : `${source}_v2`;
+}
+function carried(args) {
+  return (args.from ? ` --from-collection ${args.from}` : "") + (args.to ? ` --to-collection ${args.to}` : "");
+}
+async function detect(from, to) {
+  const alias = await resolveAliasRaw(from);
+  if (alias !== null) return { kind: "switched", points: await countPointsRaw(from) };
+  const sourceDim = await collectionDimRaw(from);
+  if (sourceDim === null) return { kind: "no-source" };
+  const modelDim = await getEmbeddingDim();
+  if (sourceDim === modelDim) return { kind: "already-current", model: getEmbeddingModel() };
+  const notes = await countPointsRaw(from);
+  const targetDim = await collectionDimRaw(to);
+  if (targetDim === null) return { kind: "not-started", notes };
+  const done = (await scrollIdsRaw(to)).size;
+  return done >= notes ? { kind: "ready-to-switch", notes } : { kind: "partial", done, notes };
+}
 async function main() {
   const args = parseArgs(process.argv.slice(2));
   if (args.help) {
     console.log(USAGE);
     return;
   }
-  const to = args.to;
-  if (!to) {
-    console.error("amem-migrate: --to <collection> is required\n");
-    console.error(USAGE);
-    process.exitCode = 1;
-    return;
-  }
   const from = args.from ?? getCollection();
-  const { dryRun, refreshFields } = args;
+  const to = args.to ?? deriveTarget(from);
+  const flags = carried(args);
+  const phase = await detect(from, to);
+  console.log(`store:  ${from}`);
   console.log(`model:  ${getEmbeddingModel()}`);
-  console.log(`from:   ${from}`);
-  console.log(`to:     ${to}`);
-  console.log(`mode:   ${dryRun ? "dry run \u2014 nothing will be written" : "APPLY"}
-`);
-  const result = await migrateCollection({ from, to, dryRun, refreshFields });
-  console.log(`
-${result.total} notes in source (${result.sourceDim}-dim)`);
-  console.log(`target is ${result.targetDim}-dim`);
-  if (result.missingDerived > 0) {
-    console.log(
-      `${result.missingDerived} note(s) predate the extraction pipeline` + (refreshFields ? `, ${result.refreshed} re-extracted` : " \u2014 skipped (--no-refresh-fields)")
-    );
+  switch (phase.kind) {
+    case "no-source":
+      console.error(`
+No collection named "${from}". Nothing to migrate.`);
+      process.exitCode = 1;
+      return;
+    case "switched":
+      console.log(`
+"${from}" is already an alias \u2014 ${phase.points} notes, nothing to do.`);
+      return;
+    case "already-current":
+      console.log(`
+Already on ${phase.model}. Nothing to migrate.`);
+      return;
+    case "not-started":
+      if (!args.apply) {
+        console.log(`
+${phase.notes} notes to rebuild into "${to}".`);
+        console.log(`Run "amem-migrate${flags} --apply" to start. "${from}" is only read.`);
+        return;
+      }
+      break;
+    case "partial":
+      if (!args.apply) {
+        console.log(`
+${phase.done} of ${phase.notes} rebuilt into "${to}".`);
+        console.log(`Run "amem-migrate${flags} --apply" to carry on from there.`);
+        return;
+      }
+      break;
+    case "ready-to-switch":
+      if (!args.switchOver) {
+        console.log(`
+All ${phase.notes} notes are in "${to}". "${from}" is untouched.`);
+        console.log(`Check it, then run "amem-migrate${flags} --switch" to put "${to}" behind the name "${from}".`);
+        console.log(`That drops "${from}" and cannot be undone.`);
+        return;
+      }
+      await switchToMigrated({ name: from, to });
+      console.log(`
+Done. Nothing to change in your config \u2014 "${from}" now resolves to "${to}".`);
+      return;
   }
-  if (dryRun) {
-    console.log(`
-Nothing was written. Re-run with --apply to migrate.`);
+  if (args.switchOver) {
+    console.error(`
+Not finished yet \u2014 run "amem-migrate${flags} --apply" until it is before switching.`);
+    process.exitCode = 1;
     return;
   }
+  const result = await migrateCollection({ from, to, dryRun: false, refreshFields: args.refreshFields });
   console.log(`
-${result.migrated} notes written to "${to}".`);
-  console.log(`Point AMEM_COLLECTION at "${to}" to start using it. "${from}" is untouched.`);
+${result.migrated + result.skipped} of ${result.total} rebuilt.`);
+  if (result.migrated + result.skipped >= result.total) {
+    console.log(`Check "${to}", then run "amem-migrate${flags} --switch".`);
+  } else {
+    console.log(`Run "amem-migrate${flags} --apply" again to carry on.`);
+  }
 }
 main().catch((err) => {
   console.error(`amem-migrate: ${err instanceof Error ? err.message : String(err)}`);
@@ -927,6 +1079,8 @@ main().catch((err) => {
 });
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
+  carried,
+  deriveTarget,
   parseArgs
 });
 //# sourceMappingURL=cli-migrate.cjs.map