@amemhq/core 2.0.0 → 2.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-K6WZTDM7.js → chunk-3HJV3LLV.js} +41 -7
- package/dist/chunk-3HJV3LLV.js.map +1 -0
- package/dist/cli-migrate.cjs +61 -8
- package/dist/cli-migrate.cjs.map +1 -1
- package/dist/cli-migrate.js +28 -4
- package/dist/cli-migrate.js.map +1 -1
- package/dist/index.cjs +40 -6
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +22 -0
- package/dist/index.d.ts +22 -0
- package/dist/index.js +1 -1
- package/package.json +1 -2
- package/dist/chunk-K6WZTDM7.js.map +0 -1
package/dist/cli-migrate.js
CHANGED
|
@@ -9,7 +9,7 @@ import {
|
|
|
9
9
|
resolveAliasRaw,
|
|
10
10
|
scrollIdsRaw,
|
|
11
11
|
switchToMigrated
|
|
12
|
-
} from "./chunk-
|
|
12
|
+
} from "./chunk-3HJV3LLV.js";
|
|
13
13
|
|
|
14
14
|
// src/cli-migrate.ts
|
|
15
15
|
var USAGE = `amem-migrate \u2014 move a memory store onto a different embedding model
|
|
@@ -25,10 +25,24 @@ Options
|
|
|
25
25
|
them. Makes the run completely offline.
|
|
26
26
|
-h, --help
|
|
27
27
|
|
|
28
|
+
AMEM_MODEL_CACHE=<dir> where model weights are cached. Set the same value
|
|
29
|
+
everywhere and the plugin and this command share one
|
|
30
|
+
copy instead of downloading it each.
|
|
31
|
+
AMEM_MODEL_DIR=<dir> read weights already on disk instead of downloading.
|
|
32
|
+
|
|
28
33
|
Migrates onto amem's current default unless AMEM_EMBED_MODEL says otherwise:
|
|
29
34
|
|
|
30
35
|
AMEM_EMBED_MODEL=Alibaba-NLP/gte-multilingual-base amem-migrate --apply
|
|
31
36
|
|
|
37
|
+
Stop your agent before --apply, not before this. The bare run only downloads and
|
|
38
|
+
reports, and the download is the long part \u2014 leaving the agent up through it costs
|
|
39
|
+
nothing, while writing underneath a running agent means notes arrive after the
|
|
40
|
+
rebuild has read them and you have to run --apply again.
|
|
41
|
+
|
|
42
|
+
Roughly: the download is your bandwidth (2.27 GB for the default), the rebuild is
|
|
43
|
+
your CPU (about 3 notes a second on an M4), and notes missing keywords cost one
|
|
44
|
+
LLM call each.
|
|
45
|
+
|
|
32
46
|
Nothing before --switch touches the original. If a run looks wrong, delete the
|
|
33
47
|
target and start again.`;
|
|
34
48
|
function parseArgs(argv) {
|
|
@@ -37,7 +51,9 @@ function parseArgs(argv) {
|
|
|
37
51
|
return i === -1 ? void 0 : argv[i + 1];
|
|
38
52
|
};
|
|
39
53
|
return {
|
|
40
|
-
help
|
|
54
|
+
// `help` as a bare word too. Without it, typing what most CLIs accept starts
|
|
55
|
+
// a multi-gigabyte model download instead of printing anything.
|
|
56
|
+
help: argv.includes("-h") || argv.includes("--help") || argv[0] === "help",
|
|
41
57
|
apply: argv.includes("--apply"),
|
|
42
58
|
switchOver: argv.includes("--switch"),
|
|
43
59
|
from: value("--from-collection"),
|
|
@@ -115,9 +131,17 @@ All ${phase.notes} notes are in "${to}". "${from}" is untouched.`);
|
|
|
115
131
|
console.log(`That drops "${from}" and cannot be undone.`);
|
|
116
132
|
return;
|
|
117
133
|
}
|
|
118
|
-
|
|
119
|
-
|
|
134
|
+
{
|
|
135
|
+
const res = await switchToMigrated({ name: from, to });
|
|
136
|
+
console.log(`
|
|
120
137
|
Done. Nothing to change in your config \u2014 "${from}" now resolves to "${to}".`);
|
|
138
|
+
if (res.snapshot) {
|
|
139
|
+
console.log(`
|
|
140
|
+
The old store is kept as a snapshot, ${(res.snapshot.size / 1e6).toFixed(0)} MB:`);
|
|
141
|
+
console.log(` <qdrant storage>/snapshots/${from}/${res.snapshot.name}`);
|
|
142
|
+
console.log(`Delete that file once the new store has proven itself. Nothing else will.`);
|
|
143
|
+
}
|
|
144
|
+
}
|
|
121
145
|
return;
|
|
122
146
|
}
|
|
123
147
|
if (args.switchOver) {
|
package/dist/cli-migrate.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../src/cli-migrate.ts"],"sourcesContent":["#!/usr/bin/env node\n/**\n * amem-migrate — move a memory store onto a different embedding model.\n *\n * Written for the person who got here from an error message, not for someone\n * embedding the engine. A developer using `@amemhq/core` directly has\n * `migrateCollection()` and `switchToMigrated()` and can sequence them however\n * their deployment wants; this is the other audience, who installed a plugin and\n * whose memory has stopped working.\n *\n * So it takes no decisions from the caller that it can take itself. It works out\n * which phase the store is in, does the next safe thing, and prints the one\n * command that comes after. The target collection name is derived; nobody has to\n * know Qdrant has collections at all.\n *\n * Exactly one step is irreversible — dropping the old collection to free its name\n * for the alias — and that one has its own flag rather than being the tail of a\n * run that started out read-only.\n */\nimport { migrateCollection, switchToMigrated } from './migrate.js'\nimport { getCollection, collectionDimRaw, countPointsRaw, scrollIdsRaw, resolveAliasRaw } from './storage.js'\nimport { getEmbeddingModel, getEmbeddingDim } from './embedding.js'\n\nconst USAGE = `amem-migrate — move a memory store onto a different embedding model\n\n amem-migrate what state the store is in, and what comes next\n amem-migrate --apply do the next step; safe to interrupt and re-run\n amem-migrate --switch put the new store behind the old name (irreversible)\n\nOptions\n --from-collection <name> the store to migrate. Defaults to AMEM_COLLECTION.\n --to-collection <name> where to build it. Derived from the source if omitted.\n --no-refresh-fields skip re-extracting keywords for notes that never had\n them. Makes the run completely offline.\n -h, --help\n\nMigrates onto amem's current default unless AMEM_EMBED_MODEL says otherwise:\n\n AMEM_EMBED_MODEL=Alibaba-NLP/gte-multilingual-base amem-migrate --apply\n\nNothing before --switch touches the original. If a run looks wrong, delete the\ntarget and start again.`\n\nexport interface MigrateArgs {\n help: boolean\n apply: boolean\n switchOver: boolean\n from?: string\n to?: string\n refreshFields: boolean\n}\n\nexport function parseArgs(argv: string[]): MigrateArgs {\n const value = (flag: string): string | undefined => {\n const i = argv.indexOf(flag)\n return i === -1 ? undefined : argv[i + 1]\n }\n return {\n help: argv.includes('-h') || argv.includes('--help'),\n apply: argv.includes('--apply'),\n switchOver: argv.includes('--switch'),\n from: value('--from-collection'),\n to: value('--to-collection'),\n refreshFields: !argv.includes('--no-refresh-fields'),\n }\n}\n\n/**\n * `amem_notes` → `amem_notes_v2`, and `amem_notes_v2` → `amem_notes_v3`.\n *\n * Derived rather than asked for: the name is an implementation detail of a\n * mechanism the user is not supposed to have to learn, and a wrong guess at it is\n * how you end up with two half-migrated stores.\n */\nexport function deriveTarget(source: string): string {\n const m = source.match(/^(.*)_v(\\d+)$/)\n return m ? `${m[1]}_v${Number(m[2]) + 1}` : `${source}_v2`\n}\n\n/**\n * The collection flags this run was given, so every \"now run …\" line it prints is\n * copy-pasteable as-is.\n *\n * Without it a mode B operator who passed `--from-collection` would be told to run\n * a bare `amem-migrate --apply`, which falls back to AMEM_COLLECTION and migrates\n * a different store. Only the collection flags are carried: forgetting\n * `--no-refresh-fields` costs an LLM call, forgetting these loses the plot.\n */\nexport function carried(args: MigrateArgs): string {\n return (args.from ? ` --from-collection ${args.from}` : '') + (args.to ? ` --to-collection ${args.to}` : '')\n}\n\ntype Phase =\n | { kind: 'no-source' }\n | { kind: 'already-current'; model: string }\n | { kind: 'not-started'; notes: number }\n | { kind: 'partial'; done: number; notes: number }\n | { kind: 'ready-to-switch'; notes: number }\n | { kind: 'switched'; points: number }\n\nasync function detect(from: string, to: string): Promise<Phase> {\n const alias = await resolveAliasRaw(from)\n if (alias !== null) return { kind: 'switched', points: await countPointsRaw(from) }\n\n const sourceDim = await collectionDimRaw(from)\n if (sourceDim === null) return { kind: 'no-source' }\n\n const modelDim = await getEmbeddingDim()\n if (sourceDim === modelDim) return { kind: 'already-current', model: getEmbeddingModel() }\n\n const notes = await countPointsRaw(from)\n const targetDim = await collectionDimRaw(to)\n if (targetDim === null) return { kind: 'not-started', notes }\n\n const done = (await scrollIdsRaw(to)).size\n return done >= notes ? { kind: 'ready-to-switch', notes } : { kind: 'partial', done, notes }\n}\n\nasync function main(): Promise<void> {\n const args = parseArgs(process.argv.slice(2))\n if (args.help) {\n console.log(USAGE)\n return\n }\n\n const from = args.from ?? getCollection()\n const to = args.to ?? deriveTarget(from)\n const flags = carried(args)\n const phase = await detect(from, to)\n\n console.log(`store: ${from}`)\n console.log(`model: ${getEmbeddingModel()}`)\n\n switch (phase.kind) {\n case 'no-source':\n console.error(`\\nNo collection named \"${from}\". Nothing to migrate.`)\n process.exitCode = 1\n return\n\n case 'switched':\n console.log(`\\n\"${from}\" is already an alias — ${phase.points} notes, nothing to do.`)\n return\n\n case 'already-current':\n console.log(`\\nAlready on ${phase.model}. Nothing to migrate.`)\n return\n\n case 'not-started':\n if (!args.apply) {\n console.log(`\\n${phase.notes} notes to rebuild into \"${to}\".`)\n console.log(`Run \"amem-migrate${flags} --apply\" to start. \"${from}\" is only read.`)\n return\n }\n break\n\n case 'partial':\n if (!args.apply) {\n console.log(`\\n${phase.done} of ${phase.notes} rebuilt into \"${to}\".`)\n console.log(`Run \"amem-migrate${flags} --apply\" to carry on from there.`)\n return\n }\n break\n\n case 'ready-to-switch':\n if (!args.switchOver) {\n console.log(`\\nAll ${phase.notes} notes are in \"${to}\". \"${from}\" is untouched.`)\n console.log(`Check it, then run \"amem-migrate${flags} --switch\" to put \"${to}\" behind the name \"${from}\".`)\n console.log(`That drops \"${from}\" and cannot be undone.`)\n return\n }\n await switchToMigrated({ name: from, to })\n console.log(`\\nDone. Nothing to change in your config — \"${from}\" now resolves to \"${to}\".`)\n return\n }\n\n if (args.switchOver) {\n console.error(`\\nNot finished yet — run \"amem-migrate${flags} --apply\" until it is before switching.`)\n process.exitCode = 1\n return\n }\n\n const result = await migrateCollection({ from, to, dryRun: false, refreshFields: args.refreshFields })\n console.log(`\\n${result.migrated + result.skipped} of ${result.total} rebuilt.`)\n if (result.migrated + result.skipped >= result.total) {\n console.log(`Check \"${to}\", then run \"amem-migrate${flags} --switch\".`)\n } else {\n console.log(`Run \"amem-migrate${flags} --apply\" again to carry on.`)\n }\n}\n\nmain().catch((err: unknown) => {\n console.error(`amem-migrate: ${err instanceof Error ? err.message : String(err)}`)\n process.exitCode = 1\n})\n"],"mappings":";;;;;;;;;;;;;;AAuBA,IAAM,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AA6BP,SAAS,UAAU,MAA6B;AACrD,QAAM,QAAQ,CAAC,SAAqC;AAClD,UAAM,IAAI,KAAK,QAAQ,IAAI;AAC3B,WAAO,MAAM,KAAK,SAAY,KAAK,IAAI,CAAC;AAAA,EAC1C;AACA,SAAO;AAAA,IACL,MAAM,KAAK,SAAS,IAAI,KAAK,KAAK,SAAS,QAAQ;AAAA,IACnD,OAAO,KAAK,SAAS,SAAS;AAAA,IAC9B,YAAY,KAAK,SAAS,UAAU;AAAA,IACpC,MAAM,MAAM,mBAAmB;AAAA,IAC/B,IAAI,MAAM,iBAAiB;AAAA,IAC3B,eAAe,CAAC,KAAK,SAAS,qBAAqB;AAAA,EACrD;AACF;AASO,SAAS,aAAa,QAAwB;AACnD,QAAM,IAAI,OAAO,MAAM,eAAe;AACtC,SAAO,IAAI,GAAG,EAAE,CAAC,CAAC,KAAK,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,MAAM;AACvD;AAWO,SAAS,QAAQ,MAA2B;AACjD,UAAQ,KAAK,OAAO,sBAAsB,KAAK,IAAI,KAAK,OAAO,KAAK,KAAK,oBAAoB,KAAK,EAAE,KAAK;AAC3G;AAUA,eAAe,OAAO,MAAc,IAA4B;AAC9D,QAAM,QAAQ,MAAM,gBAAgB,IAAI;AACxC,MAAI,UAAU,KAAM,QAAO,EAAE,MAAM,YAAY,QAAQ,MAAM,eAAe,IAAI,EAAE;AAElF,QAAM,YAAY,MAAM,iBAAiB,IAAI;AAC7C,MAAI,cAAc,KAAM,QAAO,EAAE,MAAM,YAAY;AAEnD,QAAM,WAAW,MAAM,gBAAgB;AACvC,MAAI,cAAc,SAAU,QAAO,EAAE,MAAM,mBAAmB,OAAO,kBAAkB,EAAE;AAEzF,QAAM,QAAQ,MAAM,eAAe,IAAI;AACvC,QAAM,YAAY,MAAM,iBAAiB,EAAE;AAC3C,MAAI,cAAc,KAAM,QAAO,EAAE,MAAM,eAAe,MAAM;AAE5D,QAAM,QAAQ,MAAM,aAAa,EAAE,GAAG;AACtC,SAAO,QAAQ,QAAQ,EAAE,MAAM,mBAAmB,MAAM,IAAI,EAAE,MAAM,WAAW,MAAM,MAAM;AAC7F;AAEA,eAAe,OAAsB;AACnC,QAAM,OAAO,UAAU,QAAQ,KAAK,MAAM,CAAC,CAAC;AAC5C,MAAI,KAAK,MAAM;AACb,YAAQ,IAAI,KAAK;AACjB;AAAA,EACF;AAEA,QAAM,OAAO,KAAK,QAAQ,cAAc;AACxC,QAAM,KAAK,KAAK,MAAM,aAAa,IAAI;AACvC,QAAM,QAAQ,QAAQ,IAAI;AAC1B,QAAM,QAAQ,MAAM,OAAO,MAAM,EAAE;AAEnC,UAAQ,IAAI,WAAW,IAAI,EAAE;AAC7B,UAAQ,IAAI,WAAW,kBAAkB,CAAC,EAAE;AAE5C,UAAQ,MAAM,MAAM;AAAA,IAClB,KAAK;AACH,cAAQ,MAAM;AAAA,uBAA0B,IAAI,wBAAwB;AACpE,cAAQ,WAAW;AACnB;AAAA,IAEF,KAAK;AACH,cAAQ,IAAI;AAAA,GAAM,IAAI,gCAA2B,MAAM,MAAM,wBAAwB;AACrF;AAAA,IAEF,KAAK;AACH,cAAQ,IAAI;AAAA,aAAgB,MAAM,KAAK,uBAAuB;AAC9D;AAAA,IAEF,KAAK;AACH,UAAI,CAAC,KAAK,OAAO;AACf,gBAAQ,IAAI;AAAA,EAAK,MAAM,KAAK,2BAA2B,EAAE,IAAI;AAC7D,gBAAQ,IAAI,oBAAoB,KAAK,wBAAwB,IAAI,iBAAiB;AAClF;AAAA,MACF;AACA;AAAA,IAEF,KAAK;AACH,UAAI,CAAC,KAAK,OAAO;AACf,gBAAQ,IAAI;AAAA,EAAK,MAAM,IAAI,OAAO,MAAM,KAAK,kBAAkB,EAAE,IAAI;AACrE,gBAAQ,IAAI,oBAAoB,KAAK,mCAAmC;AACxE;AAAA,MACF;AACA;AAAA,IAEF,KAAK;AACH,UAAI,CAAC,KAAK,YAAY;AACpB,gBAAQ,IAAI;AAAA,MAAS,MAAM,KAAK,kBAAkB,EAAE,OAAO,IAAI,iBAAiB;AAChF,gBAAQ,IAAI,mCAAmC,KAAK,sBAAsB,EAAE,sBAAsB,IAAI,IAAI;AAC1G,gBAAQ,IAAI,eAAe,IAAI,yBAAyB;AACxD;AAAA,MACF;AACA,YAAM,iBAAiB,EAAE,MAAM,MAAM,GAAG,CAAC;AACzC,cAAQ,IAAI;AAAA,iDAA+C,IAAI,sBAAsB,EAAE,IAAI;AAC3F;AAAA,EACJ;AAEA,MAAI,KAAK,YAAY;AACnB,YAAQ,MAAM;AAAA,2CAAyC,KAAK,yCAAyC;AACrG,YAAQ,WAAW;AACnB;AAAA,EACF;AAEA,QAAM,SAAS,MAAM,kBAAkB,EAAE,MAAM,IAAI,QAAQ,OAAO,eAAe,KAAK,cAAc,CAAC;AACrG,UAAQ,IAAI;AAAA,EAAK,OAAO,WAAW,OAAO,OAAO,OAAO,OAAO,KAAK,WAAW;AAC/E,MAAI,OAAO,WAAW,OAAO,WAAW,OAAO,OAAO;AACpD,YAAQ,IAAI,UAAU,EAAE,4BAA4B,KAAK,aAAa;AAAA,EACxE,OAAO;AACL,YAAQ,IAAI,oBAAoB,KAAK,8BAA8B;AAAA,EACrE;AACF;AAEA,KAAK,EAAE,MAAM,CAAC,QAAiB;AAC7B,UAAQ,MAAM,iBAAiB,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC,EAAE;AACjF,UAAQ,WAAW;AACrB,CAAC;","names":[]}
|
|
1
|
+
{"version":3,"sources":["../src/cli-migrate.ts"],"sourcesContent":["#!/usr/bin/env node\n/**\n * amem-migrate — move a memory store onto a different embedding model.\n *\n * Written for the person who got here from an error message, not for someone\n * embedding the engine. A developer using `@amemhq/core` directly has\n * `migrateCollection()` and `switchToMigrated()` and can sequence them however\n * their deployment wants; this is the other audience, who installed a plugin and\n * whose memory has stopped working.\n *\n * So it takes no decisions from the caller that it can take itself. It works out\n * which phase the store is in, does the next safe thing, and prints the one\n * command that comes after. The target collection name is derived; nobody has to\n * know Qdrant has collections at all.\n *\n * Exactly one step is irreversible — dropping the old collection to free its name\n * for the alias — and that one has its own flag rather than being the tail of a\n * run that started out read-only.\n */\nimport { migrateCollection, switchToMigrated } from './migrate.js'\nimport { getCollection, collectionDimRaw, countPointsRaw, scrollIdsRaw, resolveAliasRaw } from './storage.js'\nimport { getEmbeddingModel, getEmbeddingDim } from './embedding.js'\n\nconst USAGE = `amem-migrate — move a memory store onto a different embedding model\n\n amem-migrate what state the store is in, and what comes next\n amem-migrate --apply do the next step; safe to interrupt and re-run\n amem-migrate --switch put the new store behind the old name (irreversible)\n\nOptions\n --from-collection <name> the store to migrate. Defaults to AMEM_COLLECTION.\n --to-collection <name> where to build it. Derived from the source if omitted.\n --no-refresh-fields skip re-extracting keywords for notes that never had\n them. Makes the run completely offline.\n -h, --help\n\n AMEM_MODEL_CACHE=<dir> where model weights are cached. Set the same value\n everywhere and the plugin and this command share one\n copy instead of downloading it each.\n AMEM_MODEL_DIR=<dir> read weights already on disk instead of downloading.\n\nMigrates onto amem's current default unless AMEM_EMBED_MODEL says otherwise:\n\n AMEM_EMBED_MODEL=Alibaba-NLP/gte-multilingual-base amem-migrate --apply\n\nStop your agent before --apply, not before this. The bare run only downloads and\nreports, and the download is the long part — leaving the agent up through it costs\nnothing, while writing underneath a running agent means notes arrive after the\nrebuild has read them and you have to run --apply again.\n\nRoughly: the download is your bandwidth (2.27 GB for the default), the rebuild is\nyour CPU (about 3 notes a second on an M4), and notes missing keywords cost one\nLLM call each.\n\nNothing before --switch touches the original. If a run looks wrong, delete the\ntarget and start again.`\n\nexport interface MigrateArgs {\n help: boolean\n apply: boolean\n switchOver: boolean\n from?: string\n to?: string\n refreshFields: boolean\n}\n\nexport function parseArgs(argv: string[]): MigrateArgs {\n const value = (flag: string): string | undefined => {\n const i = argv.indexOf(flag)\n return i === -1 ? undefined : argv[i + 1]\n }\n return {\n // `help` as a bare word too. Without it, typing what most CLIs accept starts\n // a multi-gigabyte model download instead of printing anything.\n help: argv.includes('-h') || argv.includes('--help') || argv[0] === 'help',\n apply: argv.includes('--apply'),\n switchOver: argv.includes('--switch'),\n from: value('--from-collection'),\n to: value('--to-collection'),\n refreshFields: !argv.includes('--no-refresh-fields'),\n }\n}\n\n/**\n * `amem_notes` → `amem_notes_v2`, and `amem_notes_v2` → `amem_notes_v3`.\n *\n * Derived rather than asked for: the name is an implementation detail of a\n * mechanism the user is not supposed to have to learn, and a wrong guess at it is\n * how you end up with two half-migrated stores.\n */\nexport function deriveTarget(source: string): string {\n const m = source.match(/^(.*)_v(\\d+)$/)\n return m ? `${m[1]}_v${Number(m[2]) + 1}` : `${source}_v2`\n}\n\n/**\n * The collection flags this run was given, so every \"now run …\" line it prints is\n * copy-pasteable as-is.\n *\n * Without it a mode B operator who passed `--from-collection` would be told to run\n * a bare `amem-migrate --apply`, which falls back to AMEM_COLLECTION and migrates\n * a different store. Only the collection flags are carried: forgetting\n * `--no-refresh-fields` costs an LLM call, forgetting these loses the plot.\n */\nexport function carried(args: MigrateArgs): string {\n return (args.from ? ` --from-collection ${args.from}` : '') + (args.to ? ` --to-collection ${args.to}` : '')\n}\n\ntype Phase =\n | { kind: 'no-source' }\n | { kind: 'already-current'; model: string }\n | { kind: 'not-started'; notes: number }\n | { kind: 'partial'; done: number; notes: number }\n | { kind: 'ready-to-switch'; notes: number }\n | { kind: 'switched'; points: number }\n\nasync function detect(from: string, to: string): Promise<Phase> {\n const alias = await resolveAliasRaw(from)\n if (alias !== null) return { kind: 'switched', points: await countPointsRaw(from) }\n\n const sourceDim = await collectionDimRaw(from)\n if (sourceDim === null) return { kind: 'no-source' }\n\n const modelDim = await getEmbeddingDim()\n if (sourceDim === modelDim) return { kind: 'already-current', model: getEmbeddingModel() }\n\n const notes = await countPointsRaw(from)\n const targetDim = await collectionDimRaw(to)\n if (targetDim === null) return { kind: 'not-started', notes }\n\n const done = (await scrollIdsRaw(to)).size\n return done >= notes ? { kind: 'ready-to-switch', notes } : { kind: 'partial', done, notes }\n}\n\nasync function main(): Promise<void> {\n const args = parseArgs(process.argv.slice(2))\n if (args.help) {\n console.log(USAGE)\n return\n }\n\n const from = args.from ?? getCollection()\n const to = args.to ?? deriveTarget(from)\n const flags = carried(args)\n const phase = await detect(from, to)\n\n console.log(`store: ${from}`)\n console.log(`model: ${getEmbeddingModel()}`)\n\n switch (phase.kind) {\n case 'no-source':\n console.error(`\\nNo collection named \"${from}\". Nothing to migrate.`)\n process.exitCode = 1\n return\n\n case 'switched':\n console.log(`\\n\"${from}\" is already an alias — ${phase.points} notes, nothing to do.`)\n return\n\n case 'already-current':\n console.log(`\\nAlready on ${phase.model}. Nothing to migrate.`)\n return\n\n case 'not-started':\n if (!args.apply) {\n console.log(`\\n${phase.notes} notes to rebuild into \"${to}\".`)\n console.log(`Run \"amem-migrate${flags} --apply\" to start. \"${from}\" is only read.`)\n return\n }\n break\n\n case 'partial':\n if (!args.apply) {\n console.log(`\\n${phase.done} of ${phase.notes} rebuilt into \"${to}\".`)\n console.log(`Run \"amem-migrate${flags} --apply\" to carry on from there.`)\n return\n }\n break\n\n case 'ready-to-switch':\n if (!args.switchOver) {\n console.log(`\\nAll ${phase.notes} notes are in \"${to}\". \"${from}\" is untouched.`)\n console.log(`Check it, then run \"amem-migrate${flags} --switch\" to put \"${to}\" behind the name \"${from}\".`)\n console.log(`That drops \"${from}\" and cannot be undone.`)\n return\n }\n {\n const res = await switchToMigrated({ name: from, to })\n console.log(`\\nDone. Nothing to change in your config — \"${from}\" now resolves to \"${to}\".`)\n if (res.snapshot) {\n console.log(`\\nThe old store is kept as a snapshot, ${(res.snapshot.size / 1e6).toFixed(0)} MB:`)\n console.log(` <qdrant storage>/snapshots/${from}/${res.snapshot.name}`)\n // Not a curl: once \"${from}\" is an alias, the snapshot API resolves it\n // to the new collection and reports none. The file is the only handle.\n console.log(`Delete that file once the new store has proven itself. Nothing else will.`)\n }\n }\n return\n }\n\n if (args.switchOver) {\n console.error(`\\nNot finished yet — run \"amem-migrate${flags} --apply\" until it is before switching.`)\n process.exitCode = 1\n return\n }\n\n const result = await migrateCollection({ from, to, dryRun: false, refreshFields: args.refreshFields })\n console.log(`\\n${result.migrated + result.skipped} of ${result.total} rebuilt.`)\n if (result.migrated + result.skipped >= result.total) {\n console.log(`Check \"${to}\", then run \"amem-migrate${flags} --switch\".`)\n } else {\n console.log(`Run \"amem-migrate${flags} --apply\" again to carry on.`)\n }\n}\n\nmain().catch((err: unknown) => {\n console.error(`amem-migrate: ${err instanceof Error ? err.message : String(err)}`)\n process.exitCode = 1\n})\n"],"mappings":";;;;;;;;;;;;;;AAuBA,IAAM,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AA2CP,SAAS,UAAU,MAA6B;AACrD,QAAM,QAAQ,CAAC,SAAqC;AAClD,UAAM,IAAI,KAAK,QAAQ,IAAI;AAC3B,WAAO,MAAM,KAAK,SAAY,KAAK,IAAI,CAAC;AAAA,EAC1C;AACA,SAAO;AAAA;AAAA;AAAA,IAGL,MAAM,KAAK,SAAS,IAAI,KAAK,KAAK,SAAS,QAAQ,KAAK,KAAK,CAAC,MAAM;AAAA,IACpE,OAAO,KAAK,SAAS,SAAS;AAAA,IAC9B,YAAY,KAAK,SAAS,UAAU;AAAA,IACpC,MAAM,MAAM,mBAAmB;AAAA,IAC/B,IAAI,MAAM,iBAAiB;AAAA,IAC3B,eAAe,CAAC,KAAK,SAAS,qBAAqB;AAAA,EACrD;AACF;AASO,SAAS,aAAa,QAAwB;AACnD,QAAM,IAAI,OAAO,MAAM,eAAe;AACtC,SAAO,IAAI,GAAG,EAAE,CAAC,CAAC,KAAK,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,MAAM;AACvD;AAWO,SAAS,QAAQ,MAA2B;AACjD,UAAQ,KAAK,OAAO,sBAAsB,KAAK,IAAI,KAAK,OAAO,KAAK,KAAK,oBAAoB,KAAK,EAAE,KAAK;AAC3G;AAUA,eAAe,OAAO,MAAc,IAA4B;AAC9D,QAAM,QAAQ,MAAM,gBAAgB,IAAI;AACxC,MAAI,UAAU,KAAM,QAAO,EAAE,MAAM,YAAY,QAAQ,MAAM,eAAe,IAAI,EAAE;AAElF,QAAM,YAAY,MAAM,iBAAiB,IAAI;AAC7C,MAAI,cAAc,KAAM,QAAO,EAAE,MAAM,YAAY;AAEnD,QAAM,WAAW,MAAM,gBAAgB;AACvC,MAAI,cAAc,SAAU,QAAO,EAAE,MAAM,mBAAmB,OAAO,kBAAkB,EAAE;AAEzF,QAAM,QAAQ,MAAM,eAAe,IAAI;AACvC,QAAM,YAAY,MAAM,iBAAiB,EAAE;AAC3C,MAAI,cAAc,KAAM,QAAO,EAAE,MAAM,eAAe,MAAM;AAE5D,QAAM,QAAQ,MAAM,aAAa,EAAE,GAAG;AACtC,SAAO,QAAQ,QAAQ,EAAE,MAAM,mBAAmB,MAAM,IAAI,EAAE,MAAM,WAAW,MAAM,MAAM;AAC7F;AAEA,eAAe,OAAsB;AACnC,QAAM,OAAO,UAAU,QAAQ,KAAK,MAAM,CAAC,CAAC;AAC5C,MAAI,KAAK,MAAM;AACb,YAAQ,IAAI,KAAK;AACjB;AAAA,EACF;AAEA,QAAM,OAAO,KAAK,QAAQ,cAAc;AACxC,QAAM,KAAK,KAAK,MAAM,aAAa,IAAI;AACvC,QAAM,QAAQ,QAAQ,IAAI;AAC1B,QAAM,QAAQ,MAAM,OAAO,MAAM,EAAE;AAEnC,UAAQ,IAAI,WAAW,IAAI,EAAE;AAC7B,UAAQ,IAAI,WAAW,kBAAkB,CAAC,EAAE;AAE5C,UAAQ,MAAM,MAAM;AAAA,IAClB,KAAK;AACH,cAAQ,MAAM;AAAA,uBAA0B,IAAI,wBAAwB;AACpE,cAAQ,WAAW;AACnB;AAAA,IAEF,KAAK;AACH,cAAQ,IAAI;AAAA,GAAM,IAAI,gCAA2B,MAAM,MAAM,wBAAwB;AACrF;AAAA,IAEF,KAAK;AACH,cAAQ,IAAI;AAAA,aAAgB,MAAM,KAAK,uBAAuB;AAC9D;AAAA,IAEF,KAAK;AACH,UAAI,CAAC,KAAK,OAAO;AACf,gBAAQ,IAAI;AAAA,EAAK,MAAM,KAAK,2BAA2B,EAAE,IAAI;AAC7D,gBAAQ,IAAI,oBAAoB,KAAK,wBAAwB,IAAI,iBAAiB;AAClF;AAAA,MACF;AACA;AAAA,IAEF,KAAK;AACH,UAAI,CAAC,KAAK,OAAO;AACf,gBAAQ,IAAI;AAAA,EAAK,MAAM,IAAI,OAAO,MAAM,KAAK,kBAAkB,EAAE,IAAI;AACrE,gBAAQ,IAAI,oBAAoB,KAAK,mCAAmC;AACxE;AAAA,MACF;AACA;AAAA,IAEF,KAAK;AACH,UAAI,CAAC,KAAK,YAAY;AACpB,gBAAQ,IAAI;AAAA,MAAS,MAAM,KAAK,kBAAkB,EAAE,OAAO,IAAI,iBAAiB;AAChF,gBAAQ,IAAI,mCAAmC,KAAK,sBAAsB,EAAE,sBAAsB,IAAI,IAAI;AAC1G,gBAAQ,IAAI,eAAe,IAAI,yBAAyB;AACxD;AAAA,MACF;AACA;AACE,cAAM,MAAM,MAAM,iBAAiB,EAAE,MAAM,MAAM,GAAG,CAAC;AACrD,gBAAQ,IAAI;AAAA,iDAA+C,IAAI,sBAAsB,EAAE,IAAI;AAC3F,YAAI,IAAI,UAAU;AAChB,kBAAQ,IAAI;AAAA,wCAA2C,IAAI,SAAS,OAAO,KAAK,QAAQ,CAAC,CAAC,MAAM;AAChG,kBAAQ,IAAI,gCAAgC,IAAI,IAAI,IAAI,SAAS,IAAI,EAAE;AAGvE,kBAAQ,IAAI,2EAA2E;AAAA,QACzF;AAAA,MACF;AACA;AAAA,EACJ;AAEA,MAAI,KAAK,YAAY;AACnB,YAAQ,MAAM;AAAA,2CAAyC,KAAK,yCAAyC;AACrG,YAAQ,WAAW;AACnB;AAAA,EACF;AAEA,QAAM,SAAS,MAAM,kBAAkB,EAAE,MAAM,IAAI,QAAQ,OAAO,eAAe,KAAK,cAAc,CAAC;AACrG,UAAQ,IAAI;AAAA,EAAK,OAAO,WAAW,OAAO,OAAO,OAAO,OAAO,KAAK,WAAW;AAC/E,MAAI,OAAO,WAAW,OAAO,WAAW,OAAO,OAAO;AACpD,YAAQ,IAAI,UAAU,EAAE,4BAA4B,KAAK,aAAa;AAAA,EACxE,OAAO;AACL,YAAQ,IAAI,oBAAoB,KAAK,8BAA8B;AAAA,EACrE;AACF;AAEA,KAAK,EAAE,MAAM,CAAC,QAAiB;AAC7B,UAAQ,MAAM,iBAAiB,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC,EAAE;AACjF,UAAQ,WAAW;AACrB,CAAC;","names":[]}
|
package/dist/index.cjs
CHANGED
|
@@ -96,7 +96,6 @@ var cachedDim = null;
|
|
|
96
96
|
var DEFAULT_EMBEDDING_MODEL = "Xenova/bge-m3";
|
|
97
97
|
var LEGACY_DEFAULT_EMBEDDING_MODEL = "Xenova/paraphrase-multilingual-MiniLM-L12-v2";
|
|
98
98
|
var LEGACY_DEFAULT_DIM = 384;
|
|
99
|
-
var DEFAULT_MODEL_DTYPE = "fp16";
|
|
100
99
|
var pinnedModel = null;
|
|
101
100
|
function pinEmbeddingModel(model) {
|
|
102
101
|
if (pinnedModel === model) return;
|
|
@@ -134,9 +133,28 @@ function getEmbeddingDevice() {
|
|
|
134
133
|
return process.env.AMEM_EMBED_DEVICE?.trim() || void 0;
|
|
135
134
|
}
|
|
136
135
|
function getEmbeddingDtype() {
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
136
|
+
return process.env.AMEM_EMBED_DTYPE?.trim() || void 0;
|
|
137
|
+
}
|
|
138
|
+
function applyModelPaths(env) {
|
|
139
|
+
const cache = process.env.AMEM_MODEL_CACHE?.trim();
|
|
140
|
+
if (cache) env.cacheDir = cache;
|
|
141
|
+
const local = process.env.AMEM_MODEL_DIR?.trim();
|
|
142
|
+
if (local) {
|
|
143
|
+
env.localModelPath = local;
|
|
144
|
+
env.allowLocalModels = true;
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
function makeProgressReporter() {
|
|
148
|
+
const lastPct = /* @__PURE__ */ new Map();
|
|
149
|
+
return (e) => {
|
|
150
|
+
if (e.status !== "progress" || !e.file || typeof e.progress !== "number") return;
|
|
151
|
+
if (!e.file.endsWith(".onnx") && !e.file.endsWith(".onnx_data")) return;
|
|
152
|
+
const pct = Math.floor(e.progress / 10) * 10;
|
|
153
|
+
if (lastPct.get(e.file) === pct) return;
|
|
154
|
+
lastPct.set(e.file, pct);
|
|
155
|
+
const size = e.total ? ` of ${(e.total / 1e9).toFixed(2)} GB` : "";
|
|
156
|
+
console.log(`[amem] downloading ${e.file}: ${pct}%${size}`);
|
|
157
|
+
};
|
|
140
158
|
}
|
|
141
159
|
function extractorKey() {
|
|
142
160
|
return `${getEmbeddingModel()}|${getEmbeddingDevice() ?? ""}|${getEmbeddingDtype() ?? ""}`;
|
|
@@ -147,11 +165,13 @@ async function getExtractor() {
|
|
|
147
165
|
if (!pipeline) {
|
|
148
166
|
const mod = await import("@huggingface/transformers");
|
|
149
167
|
pipeline = mod.pipeline;
|
|
168
|
+
applyModelPaths(mod.env);
|
|
150
169
|
}
|
|
151
170
|
const device = getEmbeddingDevice();
|
|
152
171
|
const dtype = getEmbeddingDtype();
|
|
153
172
|
extractor = await pipeline("feature-extraction", getEmbeddingModel(), {
|
|
154
173
|
revision: "main",
|
|
174
|
+
progress_callback: makeProgressReporter(),
|
|
155
175
|
// Omitted entirely when unset, so an unconfigured install gets exactly the
|
|
156
176
|
// library defaults it got before these existed.
|
|
157
177
|
...device ? { device } : {},
|
|
@@ -462,6 +482,10 @@ async function scrollIdsRaw(collection, limit = 1e4) {
|
|
|
462
482
|
async function deleteCollectionRaw(collection) {
|
|
463
483
|
await qdrant("DELETE", `/collections/${collection}`);
|
|
464
484
|
}
|
|
485
|
+
async function snapshotCollectionRaw(collection) {
|
|
486
|
+
const r = await qdrant("POST", `/collections/${collection}/snapshots`);
|
|
487
|
+
return { name: r.name, size: r.size ?? 0 };
|
|
488
|
+
}
|
|
465
489
|
async function resolveAliasRaw(alias) {
|
|
466
490
|
try {
|
|
467
491
|
const res = await qdrant("GET", `/aliases`);
|
|
@@ -1926,7 +1950,12 @@ async function searchMemory(query, topK = 5, agentId = "main", opts) {
|
|
|
1926
1950
|
keywords: note.keywords,
|
|
1927
1951
|
links: note.links,
|
|
1928
1952
|
timestamp: note.timestamp,
|
|
1929
|
-
|
|
1953
|
+
// Neither map covers a note that got here on BM25 alone: it was never in
|
|
1954
|
+
// the dense results, and it was not expanded into. That is a real cosine
|
|
1955
|
+
// nobody had measured, not a zero — and reporting 0 made a lexical match
|
|
1956
|
+
// look like the least relevant row in the list. Both vectors are already
|
|
1957
|
+
// in hand, so measuring it is one dot product.
|
|
1958
|
+
similarity: embSimMap.get(id) ?? bfsSimMap.get(id) ?? cosineSimilarity(queryEmbedding, note.embedding),
|
|
1930
1959
|
rrf: rrfMap.get(id) ?? 0,
|
|
1931
1960
|
via,
|
|
1932
1961
|
topics: note.topics ?? [],
|
|
@@ -2539,6 +2568,7 @@ async function switchToMigrated(opts) {
|
|
|
2539
2568
|
const { name, to } = opts;
|
|
2540
2569
|
const log = opts.logger?.info ?? ((m) => console.log(m));
|
|
2541
2570
|
if (name === to) throw new Error(`switch: "${name}" and "${to}" are the same collection`);
|
|
2571
|
+
let snap;
|
|
2542
2572
|
const already = await resolveAliasRaw(name);
|
|
2543
2573
|
if (already === to) {
|
|
2544
2574
|
log(`[switch] "${name}" already points at "${to}" \u2014 nothing to do`);
|
|
@@ -2554,6 +2584,10 @@ async function switchToMigrated(opts) {
|
|
|
2554
2584
|
);
|
|
2555
2585
|
}
|
|
2556
2586
|
log(`[switch] verified ${targetCount} in "${to}" against ${sourceCount} in "${name}"`);
|
|
2587
|
+
if (opts.snapshot !== false) {
|
|
2588
|
+
snap = await snapshotCollectionRaw(name);
|
|
2589
|
+
log(`[switch] snapshotted "${name}" \u2192 ${snap.name} (${(snap.size / 1e6).toFixed(0)} MB)`);
|
|
2590
|
+
}
|
|
2557
2591
|
await deleteCollectionRaw(name);
|
|
2558
2592
|
log(`[switch] dropped "${name}"`);
|
|
2559
2593
|
await createAliasRaw(name, to);
|
|
@@ -2561,7 +2595,7 @@ async function switchToMigrated(opts) {
|
|
|
2561
2595
|
await setAliasRaw(name, to);
|
|
2562
2596
|
}
|
|
2563
2597
|
log(`[switch] "${name}" now resolves to "${to}"`);
|
|
2564
|
-
return { name, to, moved: targetCount };
|
|
2598
|
+
return { name, to, moved: targetCount, snapshot: snap };
|
|
2565
2599
|
}
|
|
2566
2600
|
|
|
2567
2601
|
// src/crud-guard.ts
|