@bubstack/moe-memory 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (157) hide show
  1. package/.mcp.json +18 -0
  2. package/README.md +47 -0
  3. package/agents/search-conversations.md +171 -0
  4. package/dist/LICENSE +25 -0
  5. package/dist/cli.d.ts +25 -0
  6. package/dist/cli.d.ts.map +1 -0
  7. package/dist/cli.js +125 -0
  8. package/dist/cli.js.map +1 -0
  9. package/dist/codex-hook-trust.d.ts +4 -0
  10. package/dist/codex-hook-trust.d.ts.map +1 -0
  11. package/dist/codex-hook-trust.js +114 -0
  12. package/dist/codex-hook-trust.js.map +1 -0
  13. package/dist/codex-support.d.ts +6 -0
  14. package/dist/codex-support.d.ts.map +1 -0
  15. package/dist/codex-support.js +29 -0
  16. package/dist/codex-support.js.map +1 -0
  17. package/dist/constants.d.ts +14 -0
  18. package/dist/constants.d.ts.map +1 -0
  19. package/dist/constants.js +14 -0
  20. package/dist/constants.js.map +1 -0
  21. package/dist/db.d.ts +103 -0
  22. package/dist/db.d.ts.map +1 -0
  23. package/dist/db.js +430 -0
  24. package/dist/db.js.map +1 -0
  25. package/dist/doctor-cli.d.ts +3 -0
  26. package/dist/doctor-cli.d.ts.map +1 -0
  27. package/dist/doctor-cli.js +43 -0
  28. package/dist/doctor-cli.js.map +1 -0
  29. package/dist/doctor.d.ts +17 -0
  30. package/dist/doctor.d.ts.map +1 -0
  31. package/dist/doctor.js +98 -0
  32. package/dist/doctor.js.map +1 -0
  33. package/dist/embedding-migration.d.ts +73 -0
  34. package/dist/embedding-migration.d.ts.map +1 -0
  35. package/dist/embedding-migration.js +120 -0
  36. package/dist/embedding-migration.js.map +1 -0
  37. package/dist/embeddings.d.ts +78 -0
  38. package/dist/embeddings.d.ts.map +1 -0
  39. package/dist/embeddings.js +180 -0
  40. package/dist/embeddings.js.map +1 -0
  41. package/dist/file-lock.d.ts +39 -0
  42. package/dist/file-lock.d.ts.map +1 -0
  43. package/dist/file-lock.js +91 -0
  44. package/dist/file-lock.js.map +1 -0
  45. package/dist/index-cli.d.ts +10 -0
  46. package/dist/index-cli.d.ts.map +1 -0
  47. package/dist/index-cli.js +195 -0
  48. package/dist/index-cli.js.map +1 -0
  49. package/dist/index.d.ts +12 -0
  50. package/dist/index.d.ts.map +1 -0
  51. package/dist/index.js +19 -0
  52. package/dist/index.js.map +1 -0
  53. package/dist/indexer.d.ts +4 -0
  54. package/dist/indexer.d.ts.map +1 -0
  55. package/dist/indexer.js +318 -0
  56. package/dist/indexer.js.map +1 -0
  57. package/dist/install-check.d.ts +48 -0
  58. package/dist/install-check.d.ts.map +1 -0
  59. package/dist/install-check.js +82 -0
  60. package/dist/install-check.js.map +1 -0
  61. package/dist/journal/index.d.ts +5 -0
  62. package/dist/journal/index.d.ts.map +1 -0
  63. package/dist/journal/index.js +5 -0
  64. package/dist/journal/index.js.map +1 -0
  65. package/dist/journal/legacy-sidecars.d.ts +74 -0
  66. package/dist/journal/legacy-sidecars.d.ts.map +1 -0
  67. package/dist/journal/legacy-sidecars.js +151 -0
  68. package/dist/journal/legacy-sidecars.js.map +1 -0
  69. package/dist/journal/markdown.d.ts +91 -0
  70. package/dist/journal/markdown.d.ts.map +1 -0
  71. package/dist/journal/markdown.js +234 -0
  72. package/dist/journal/markdown.js.map +1 -0
  73. package/dist/journal/search.d.ts +91 -0
  74. package/dist/journal/search.d.ts.map +1 -0
  75. package/dist/journal/search.js +203 -0
  76. package/dist/journal/search.js.map +1 -0
  77. package/dist/journal/store.d.ts +96 -0
  78. package/dist/journal/store.d.ts.map +1 -0
  79. package/dist/journal/store.js +270 -0
  80. package/dist/journal/store.js.map +1 -0
  81. package/dist/journal-cli.d.ts +19 -0
  82. package/dist/journal-cli.d.ts.map +1 -0
  83. package/dist/journal-cli.js +153 -0
  84. package/dist/journal-cli.js.map +1 -0
  85. package/dist/logging.d.ts +6 -0
  86. package/dist/logging.d.ts.map +1 -0
  87. package/dist/logging.js +20 -0
  88. package/dist/logging.js.map +1 -0
  89. package/dist/mcp-server.d.ts +34 -0
  90. package/dist/mcp-server.d.ts.map +1 -0
  91. package/dist/mcp-server.js +607 -0
  92. package/dist/mcp-server.js.map +1 -0
  93. package/dist/parser.d.ts +11 -0
  94. package/dist/parser.d.ts.map +1 -0
  95. package/dist/parser.js +445 -0
  96. package/dist/parser.js.map +1 -0
  97. package/dist/paths.d.ts +103 -0
  98. package/dist/paths.d.ts.map +1 -0
  99. package/dist/paths.js +258 -0
  100. package/dist/paths.js.map +1 -0
  101. package/dist/search-cli.d.ts +3 -0
  102. package/dist/search-cli.d.ts.map +1 -0
  103. package/dist/search-cli.js +138 -0
  104. package/dist/search-cli.js.map +1 -0
  105. package/dist/search.d.ts +36 -0
  106. package/dist/search.d.ts.map +1 -0
  107. package/dist/search.js +371 -0
  108. package/dist/search.js.map +1 -0
  109. package/dist/show-cli.d.ts +3 -0
  110. package/dist/show-cli.d.ts.map +1 -0
  111. package/dist/show-cli.js +60 -0
  112. package/dist/show-cli.js.map +1 -0
  113. package/dist/show.d.ts +3 -0
  114. package/dist/show.d.ts.map +1 -0
  115. package/dist/show.js +984 -0
  116. package/dist/show.js.map +1 -0
  117. package/dist/stats-cli.d.ts +3 -0
  118. package/dist/stats-cli.d.ts.map +1 -0
  119. package/dist/stats-cli.js +51 -0
  120. package/dist/stats-cli.js.map +1 -0
  121. package/dist/stats.d.ts +19 -0
  122. package/dist/stats.d.ts.map +1 -0
  123. package/dist/stats.js +116 -0
  124. package/dist/stats.js.map +1 -0
  125. package/dist/summarizer.d.ts +83 -0
  126. package/dist/summarizer.d.ts.map +1 -0
  127. package/dist/summarizer.js +526 -0
  128. package/dist/summarizer.js.map +1 -0
  129. package/dist/summary-sentinel.d.ts +31 -0
  130. package/dist/summary-sentinel.d.ts.map +1 -0
  131. package/dist/summary-sentinel.js +80 -0
  132. package/dist/summary-sentinel.js.map +1 -0
  133. package/dist/sync-cli.d.ts +15 -0
  134. package/dist/sync-cli.d.ts.map +1 -0
  135. package/dist/sync-cli.js +202 -0
  136. package/dist/sync-cli.js.map +1 -0
  137. package/dist/sync.d.ts +35 -0
  138. package/dist/sync.d.ts.map +1 -0
  139. package/dist/sync.js +204 -0
  140. package/dist/sync.js.map +1 -0
  141. package/dist/types.d.ts +125 -0
  142. package/dist/types.d.ts.map +1 -0
  143. package/dist/types.js +32 -0
  144. package/dist/types.js.map +1 -0
  145. package/dist/verify.d.ts +30 -0
  146. package/dist/verify.d.ts.map +1 -0
  147. package/dist/verify.js +174 -0
  148. package/dist/verify.js.map +1 -0
  149. package/dist/version.d.ts +13 -0
  150. package/dist/version.d.ts.map +1 -0
  151. package/dist/version.js +13 -0
  152. package/dist/version.js.map +1 -0
  153. package/hooks/hooks.json +15 -0
  154. package/package.json +71 -0
  155. package/prompts/search-agent.md +160 -0
  156. package/skills/remembering-conversations/MCP-TOOLS.md +147 -0
  157. package/skills/remembering-conversations/SKILL.md +97 -0
package/dist/doctor.js ADDED
@@ -0,0 +1,98 @@
1
+ import { MIN_CODEX_VERSION, parseCodexCliVersion, versionMeetsMinimum } from "./codex-support.js";
2
+ function parseFeatureState(featuresOutput, feature) {
3
+ const line = featuresOutput
4
+ .split(/\r?\n/)
5
+ .map((entry) => entry.trim())
6
+ .find((entry) => entry.startsWith(`${feature} `));
7
+ if (!line) {
8
+ return undefined;
9
+ }
10
+ const lastColumn = line.split(/\s+/).at(-1);
11
+ if (lastColumn === "true")
12
+ return true;
13
+ if (lastColumn === "false")
14
+ return false;
15
+ return undefined;
16
+ }
17
+ function parseMcpState(mcpListOutput) {
18
+ const line = mcpListOutput
19
+ .split(/\r?\n/)
20
+ .map((entry) => entry.trim())
21
+ .find((entry) => entry.startsWith("moe-memory "));
22
+ if (!line) {
23
+ return "missing";
24
+ }
25
+ return line.includes(" enabled") ? "enabled" : "disabled";
26
+ }
27
+ function formatHookTrustState(hookTrustState) {
28
+ switch (hookTrustState) {
29
+ case "trusted":
30
+ return "trusted";
31
+ case "untrusted":
32
+ return "untrusted; open /hooks in Codex, review the Moe Memory hook, and press t to trust it.";
33
+ case "modified":
34
+ return "modified since it was trusted; open /hooks in Codex, review the Moe Memory hook, and press t to trust it again.";
35
+ case "not_found":
36
+ return "not found; confirm the Moe Memory plugin is installed and enabled.";
37
+ case "unknown":
38
+ return "unknown; could not inspect Codex hooks. Open /hooks in Codex to verify trust.";
39
+ }
40
+ }
41
+ export function buildCodexDoctorReport(inputs) {
42
+ const version = parseCodexCliVersion(inputs.codexVersionOutput);
43
+ const versionOk = version !== undefined && versionMeetsMinimum(version);
44
+ const pluginHooksEnabled = parseFeatureState(inputs.featuresOutput, "plugin_hooks");
45
+ const pluginsEnabled = parseFeatureState(inputs.featuresOutput, "plugins");
46
+ const mcpState = parseMcpState(inputs.mcpListOutput);
47
+ const issues = [];
48
+ if (!versionOk) {
49
+ issues.push(`Codex must be upgraded with codex update (minimum ${MIN_CODEX_VERSION}).`);
50
+ }
51
+ if (pluginsEnabled === false) {
52
+ issues.push("Codex plugins are disabled; run codex features enable plugins.");
53
+ }
54
+ if (pluginHooksEnabled !== true) {
55
+ issues.push("Codex plugin hooks are not enabled; run codex features enable plugin_hooks.");
56
+ }
57
+ if (!inputs.sessionsDirExists) {
58
+ issues.push("Codex sessions directory does not exist yet; start at least one Codex session.");
59
+ }
60
+ if (mcpState !== "enabled") {
61
+ issues.push("Moe Memory MCP server is not enabled in codex mcp list.");
62
+ }
63
+ if (inputs.hookTrustState === "untrusted" || inputs.hookTrustState === "modified") {
64
+ issues.push("Moe Memory Codex hook is not trusted; open /hooks in Codex and press t to trust it.");
65
+ }
66
+ else if (inputs.hookTrustState === "not_found") {
67
+ issues.push("Moe Memory Codex hook was not found; confirm the plugin is installed and enabled.");
68
+ }
69
+ else if (inputs.hookTrustState === "unknown") {
70
+ issues.push("Moe Memory Codex hook trust could not be verified.");
71
+ }
72
+ const lines = [
73
+ "Moe Memory Codex Doctor",
74
+ "================================",
75
+ "",
76
+ `Codex version: ${inputs.codexVersionOutput.trim() || "(not found)"} ${versionOk ? `(ok; minimum ${MIN_CODEX_VERSION})` : `(requires minimum ${MIN_CODEX_VERSION})`}`,
77
+ `Codex home: ${inputs.codexHome}`,
78
+ `Codex sessions: ${inputs.sessionsDirExists ? "found" : "missing"}`,
79
+ `Plugins feature: ${pluginsEnabled === true ? "enabled" : pluginsEnabled === false ? "disabled" : "unknown"}`,
80
+ `Plugin hooks feature: ${pluginHooksEnabled === true ? "enabled" : pluginHooksEnabled === false ? "disabled" : "unknown"}`,
81
+ `Moe Memory MCP: ${mcpState}`,
82
+ `Index database: ${inputs.dbPath}`,
83
+ `Hook/background sync log: ${inputs.logPath}`,
84
+ "",
85
+ `Hook trust: ${formatHookTrustState(inputs.hookTrustState)}`,
86
+ ];
87
+ if (issues.length > 0) {
88
+ lines.push("", "Issues:");
89
+ for (const issue of issues) {
90
+ lines.push(`- ${issue}`);
91
+ }
92
+ }
93
+ return {
94
+ ok: issues.length === 0,
95
+ text: `${lines.join("\n")}\n`,
96
+ };
97
+ }
98
+ //# sourceMappingURL=doctor.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"doctor.js","sourceRoot":"","sources":["../src/doctor.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,iBAAiB,EAAE,oBAAoB,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AAkBlG,SAAS,iBAAiB,CAAC,cAAsB,EAAE,OAAe;IAChE,MAAM,IAAI,GAAG,cAAc;SACxB,KAAK,CAAC,OAAO,CAAC;SACd,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;SAC5B,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,UAAU,CAAC,GAAG,OAAO,GAAG,CAAC,CAAC,CAAC;IACpD,IAAI,CAAC,IAAI,EAAE,CAAC;QACV,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAC5C,IAAI,UAAU,KAAK,MAAM;QAAE,OAAO,IAAI,CAAC;IACvC,IAAI,UAAU,KAAK,OAAO;QAAE,OAAO,KAAK,CAAC;IACzC,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,aAAa,CAAC,aAAqB;IAC1C,MAAM,IAAI,GAAG,aAAa;SACvB,KAAK,CAAC,OAAO,CAAC;SACd,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;SAC5B,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC,CAAC;IACpD,IAAI,CAAC,IAAI,EAAE,CAAC;QACV,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,OAAO,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,UAAU,CAAC;AAC5D,CAAC;AAED,SAAS,oBAAoB,CAAC,cAAmC;IAC/D,QAAQ,cAAc,EAAE,CAAC;QACvB,KAAK,SAAS;YACZ,OAAO,SAAS,CAAC;QACnB,KAAK,WAAW;YACd,OAAO,uFAAuF,CAAC;QACjG,KAAK,UAAU;YACb,OAAO,iHAAiH,CAAC;QAC3H,KAAK,WAAW;YACd,OAAO,oEAAoE,CAAC;QAC9E,KAAK,SAAS;YACZ,OAAO,+EAA+E,CAAC;IAC3F,CAAC;AACH,CAAC;AAED,MAAM,UAAU,sBAAsB,CAAC,MAAyB;IAC9D,MAAM,OAAO,GAAG,oBAAoB,CAAC,MAAM,CAAC,kBAAkB,CAAC,CAAC;IAChE,MAAM,SAAS,GAAG,OAAO,KAAK,SAAS,IAAI,mBAAmB,CAAC,OAAO,CAAC,CAAC;IACxE,MAAM,kBAAkB,GAAG,iBAAiB,CAAC,MAAM,CAAC,cAAc,EAAE,cAAc,CAAC,CAAC;IACpF,MAAM,cAAc,GAAG,iBAAiB,CAAC,MAAM,CAAC,cAAc,EAAE,SAAS,CAAC,CAAC;IAC3E,MAAM,QAAQ,GAAG,aAAa,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;IAErD,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,MAAM,CAAC,IAAI,CAAC,qDAAqD,iBAAiB,IAAI,CAAC,CAAC;IAC1F,CAAC;IACD,IAAI,cAAc,KAAK,KAAK,EAAE,CAAC;QAC7B,MAAM,CAAC,IAAI,CAAC,gEAAgE,CAAC,CAAC;IAChF,CAAC;IACD,IAAI,kBAAkB,KAAK,IAAI,EAAE,CAAC;QAChC,MAAM,CAAC,IAAI,CAAC,6EAA6E,CAAC,CAAC;IAC7F,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,iBAAiB,EAAE,CAAC;QAC9B,MAAM,CAAC,IAAI,CAAC,gFAAgF,CAAC,CAAC;IAChG,CAAC;IACD,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;QAC3B,MAAM,CAAC,IAAI,CAAC,yDAAyD,CAAC,CAAC;IACzE,CAAC;IACD,IAAI,MAAM,CAAC,cAAc,KAAK,WAAW,IAAI,MAAM,CAAC,cAAc,KAAK,UAAU,EAAE,CAAC;QAClF,MAAM,CAAC,IAAI,CACT,qFAAqF,CACtF,CAAC;IACJ,CAAC;SAAM,IAAI,MAAM,CAAC,cAAc,KAAK,WAAW,EAAE,CAAC;QACjD,MAAM,CAAC,IAAI,CACT,mFAAmF,CACpF,CAAC;IACJ,CAAC;SAAM,IAAI,MAAM,CAAC,cAAc,KAAK,SAAS,EAAE,CAAC;QAC/C,MAAM,CAAC,IAAI,CAAC,oDAAoD,CAAC,CAAC;IACpE,CAAC;IAED,MAAM,KAAK,GAAG;QACZ,yBAAyB;QACzB,kCAAkC;QAClC,EAAE;QACF,kBAAkB,MAAM,CAAC,kBAAkB,CAAC,IAAI,EAAE,IAAI,aAAa,IAAI,SAAS,CAAC,CAAC,CAAC,gBAAgB,iBAAiB,GAAG,CAAC,CAAC,CAAC,qBAAqB,iBAAiB,GAAG,EAAE;QACrK,eAAe,MAAM,CAAC,SAAS,EAAE;QACjC,mBAAmB,MAAM,CAAC,iBAAiB,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,EAAE;QACnE,oBAAoB,cAAc,KAAK,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,cAAc,KAAK,KAAK,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS,EAAE;QAC7G,yBAAyB,kBAAkB,KAAK,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,kBAAkB,KAAK,KAAK,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS,EAAE;QAC1H,mBAAmB,QAAQ,EAAE;QAC7B,mBAAmB,MAAM,CAAC,MAAM,EAAE;QAClC,6BAA6B,MAAM,CAAC,OAAO,EAAE;QAC7C,EAAE;QACF,eAAe,oBAAoB,CAAC,MAAM,CAAC,cAAc,CAAC,EAAE;KAC7D,CAAC;IAEF,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtB,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,CAAC,CAAC;QAC1B,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,KAAK,CAAC,IAAI,CAAC,KAAK,KAAK,EAAE,CAAC,CAAC;QAC3B,CAAC;IACH,CAAC;IAED,OAAO;QACL,EAAE,EAAE,MAAM,CAAC,MAAM,KAAK,CAAC;QACvB,IAAI,EAAE,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI;KAC9B,CAAC;AACJ,CAAC"}
@@ -0,0 +1,73 @@
1
+ /**
2
+ * Embedding migration.
3
+ *
4
+ * The encoder was upgraded from all-MiniLM-L6-v2 to bge-small-en-v1.5. Existing
5
+ * databases have vec_exchanges rows produced by the old encoder. This module
6
+ * provides the primitives for an incremental, lock-protected, resumable
7
+ * background migration that re-embeds stale rows in batches during sync.
8
+ *
9
+ * EMBEDDING_VERSION — bumped any time the encoder pipeline changes
10
+ * acquire/release lock — file-based with PID-liveness fallback
11
+ * pickStaleBatch — find rows whose embedding_version is behind
12
+ * recordReembedded — atomic update of vec_exchanges + version bump
13
+ *
14
+ * This mechanism is the single strongest argument for building the merged
15
+ * package on episodic-memory's foundation: private-journal-mcp had no version
16
+ * stamp at all, and its startup scan keyed purely on the ABSENCE of a sidecar,
17
+ * so it would never have re-embedded a stale one.
18
+ */
19
+ import type Database from "better-sqlite3";
20
+ import { acquireFileLock, type FileLockHandle, releaseFileLock } from "./file-lock.js";
21
+ /**
22
+ * Bump when anything in the embedding pipeline changes (model, dtype, prefix).
23
+ *
24
+ * 1 → 2 on the merge with private-journal-mcp. Journal entries were embedded
25
+ * upstream with `Xenova/all-MiniLM-L6-v2` and exchanges with
26
+ * `Xenova/bge-small-en-v1.5`. Both are 384-dimensional, so a unified vec0
27
+ * column accepts either vector without complaint and a mixed corpus ranks
28
+ * wrongly with no error — which is exactly the event this constant exists for.
29
+ */
30
+ export declare const EMBEDDING_VERSION = 2;
31
+ /**
32
+ * Lock primitives for the migration are the same as for sync (#97) and any
33
+ * other once-per-machine background task — see src/file-lock.ts for the shape.
34
+ * Re-exported here so existing call sites keep their original import names.
35
+ */
36
+ export type MigrationLockHandle = FileLockHandle;
37
+ export declare const acquireMigrationLock: typeof acquireFileLock;
38
+ export declare const releaseMigrationLock: typeof releaseFileLock;
39
+ export interface StaleRow {
40
+ id: string;
41
+ user_message: string;
42
+ assistant_message: string;
43
+ tools: string | null;
44
+ }
45
+ /**
46
+ * Return up to `limit` rows whose embedding_version is older than
47
+ * EMBEDDING_VERSION, joined with their tool names so the caller can
48
+ * reproduce the production exchange-text format.
49
+ */
50
+ export declare function pickStaleBatch(db: Database.Database, limit: number): StaleRow[];
51
+ /**
52
+ * Replace a row's vec_exchanges embedding and stamp its embedding_version
53
+ * atomically. Wrap each batch's calls in a single transaction at the caller
54
+ * for durability; this function executes its statements in order without
55
+ * starting its own transaction.
56
+ */
57
+ export declare function recordReembedded(db: Database.Database, id: string, embedding: number[]): void;
58
+ /**
59
+ * Count rows whose embedding is older than the current version.
60
+ * Used to decide whether migration is needed and to report progress.
61
+ */
62
+ export declare function countStale(db: Database.Database): number;
63
+ /** Path of the migration lock under the index directory. */
64
+ export declare function getMigrationLockPath(indexDir: string): string;
65
+ /**
66
+ * Run a single migration batch: re-embed up to `batchSize` rows whose
67
+ * embedding_version is behind. Lock-protected; exits silently if another
68
+ * process holds the lock.
69
+ *
70
+ * Returns the number of rows re-embedded (0 if nothing to do or locked out).
71
+ */
72
+ export declare function runMigrationBatch(db: Database.Database, indexDir: string, batchSize: number, embedFn: (user: string, assistant: string, toolNames?: string[]) => Promise<number[]>): Promise<number>;
73
+ //# sourceMappingURL=embedding-migration.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"embedding-migration.d.ts","sourceRoot":"","sources":["../src/embedding-migration.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAGH,OAAO,KAAK,QAAQ,MAAM,gBAAgB,CAAC;AAC3C,OAAO,EAAE,eAAe,EAAE,KAAK,cAAc,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAEvF;;;;;;;;GAQG;AACH,eAAO,MAAM,iBAAiB,IAAI,CAAC;AAEnC;;;;GAIG;AACH,MAAM,MAAM,mBAAmB,GAAG,cAAc,CAAC;AACjD,eAAO,MAAM,oBAAoB,wBAAkB,CAAC;AACpD,eAAO,MAAM,oBAAoB,wBAAkB,CAAC;AAEpD,MAAM,WAAW,QAAQ;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,YAAY,EAAE,MAAM,CAAC;IACrB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;CACtB;AAED;;;;GAIG;AACH,wBAAgB,cAAc,CAAC,EAAE,EAAE,QAAQ,CAAC,QAAQ,EAAE,KAAK,EAAE,MAAM,GAAG,QAAQ,EAAE,CAe/E;AAED;;;;;GAKG;AACH,wBAAgB,gBAAgB,CAAC,EAAE,EAAE,QAAQ,CAAC,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,GAAG,IAAI,CAO7F;AAED;;;GAGG;AACH,wBAAgB,UAAU,CAAC,EAAE,EAAE,QAAQ,CAAC,QAAQ,GAAG,MAAM,CAKxD;AAED,4DAA4D;AAC5D,wBAAgB,oBAAoB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAE7D;AAED;;;;;;GAMG;AACH,wBAAsB,iBAAiB,CACrC,EAAE,EAAE,QAAQ,CAAC,QAAQ,EACrB,QAAQ,EAAE,MAAM,EAChB,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,EAAE,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC,GACpF,OAAO,CAAC,MAAM,CAAC,CAmCjB"}
@@ -0,0 +1,120 @@
1
+ /**
2
+ * Embedding migration.
3
+ *
4
+ * The encoder was upgraded from all-MiniLM-L6-v2 to bge-small-en-v1.5. Existing
5
+ * databases have vec_exchanges rows produced by the old encoder. This module
6
+ * provides the primitives for an incremental, lock-protected, resumable
7
+ * background migration that re-embeds stale rows in batches during sync.
8
+ *
9
+ * EMBEDDING_VERSION — bumped any time the encoder pipeline changes
10
+ * acquire/release lock — file-based with PID-liveness fallback
11
+ * pickStaleBatch — find rows whose embedding_version is behind
12
+ * recordReembedded — atomic update of vec_exchanges + version bump
13
+ *
14
+ * This mechanism is the single strongest argument for building the merged
15
+ * package on episodic-memory's foundation: private-journal-mcp had no version
16
+ * stamp at all, and its startup scan keyed purely on the ABSENCE of a sidecar,
17
+ * so it would never have re-embedded a stale one.
18
+ */
19
+ import path from "node:path";
20
+ import { acquireFileLock, releaseFileLock } from "./file-lock.js";
21
+ /**
22
+ * Bump when anything in the embedding pipeline changes (model, dtype, prefix).
23
+ *
24
+ * 1 → 2 on the merge with private-journal-mcp. Journal entries were embedded
25
+ * upstream with `Xenova/all-MiniLM-L6-v2` and exchanges with
26
+ * `Xenova/bge-small-en-v1.5`. Both are 384-dimensional, so a unified vec0
27
+ * column accepts either vector without complaint and a mixed corpus ranks
28
+ * wrongly with no error — which is exactly the event this constant exists for.
29
+ */
30
+ export const EMBEDDING_VERSION = 2;
31
+ export const acquireMigrationLock = acquireFileLock;
32
+ export const releaseMigrationLock = releaseFileLock;
33
+ /**
34
+ * Return up to `limit` rows whose embedding_version is older than
35
+ * EMBEDDING_VERSION, joined with their tool names so the caller can
36
+ * reproduce the production exchange-text format.
37
+ */
38
+ export function pickStaleBatch(db, limit) {
39
+ return db
40
+ .prepare(`
41
+ SELECT
42
+ e.id,
43
+ e.user_message,
44
+ e.assistant_message,
45
+ GROUP_CONCAT(DISTINCT tc.tool_name) AS tools
46
+ FROM exchanges e
47
+ LEFT JOIN tool_calls tc ON tc.exchange_id = e.id
48
+ WHERE e.embedding_version < ?
49
+ GROUP BY e.id
50
+ LIMIT ?
51
+ `)
52
+ .all(EMBEDDING_VERSION, limit);
53
+ }
54
+ /**
55
+ * Replace a row's vec_exchanges embedding and stamp its embedding_version
56
+ * atomically. Wrap each batch's calls in a single transaction at the caller
57
+ * for durability; this function executes its statements in order without
58
+ * starting its own transaction.
59
+ */
60
+ export function recordReembedded(db, id, embedding) {
61
+ db.prepare("DELETE FROM vec_exchanges WHERE id = ?").run(id);
62
+ db.prepare("INSERT INTO vec_exchanges (id, embedding) VALUES (?, ?)").run(id, Buffer.from(new Float32Array(embedding).buffer));
63
+ db.prepare("UPDATE exchanges SET embedding_version = ? WHERE id = ?").run(EMBEDDING_VERSION, id);
64
+ }
65
+ /**
66
+ * Count rows whose embedding is older than the current version.
67
+ * Used to decide whether migration is needed and to report progress.
68
+ */
69
+ export function countStale(db) {
70
+ const row = db
71
+ .prepare("SELECT COUNT(*) AS c FROM exchanges WHERE embedding_version < ?")
72
+ .get(EMBEDDING_VERSION);
73
+ return row.c;
74
+ }
75
+ /** Path of the migration lock under the index directory. */
76
+ export function getMigrationLockPath(indexDir) {
77
+ return path.join(indexDir, ".embedding-migration.lock");
78
+ }
79
+ /**
80
+ * Run a single migration batch: re-embed up to `batchSize` rows whose
81
+ * embedding_version is behind. Lock-protected; exits silently if another
82
+ * process holds the lock.
83
+ *
84
+ * Returns the number of rows re-embedded (0 if nothing to do or locked out).
85
+ */
86
+ export async function runMigrationBatch(db, indexDir, batchSize, embedFn) {
87
+ const remaining = countStale(db);
88
+ if (remaining === 0)
89
+ return 0;
90
+ const lockPath = getMigrationLockPath(indexDir);
91
+ const lock = acquireMigrationLock(lockPath);
92
+ if (!lock) {
93
+ console.error(`moe-memory: another process is migrating embeddings (${remaining} rows still stale); skipping`);
94
+ return 0;
95
+ }
96
+ try {
97
+ const rows = pickStaleBatch(db, batchSize);
98
+ if (rows.length === 0)
99
+ return 0;
100
+ console.error(`moe-memory: re-embedding batch of ${rows.length} (${remaining} stale total)...`);
101
+ // Compute embeddings outside the transaction (async work),
102
+ // then write atomically (one transaction per batch for durability).
103
+ const embeddings = [];
104
+ for (const row of rows) {
105
+ const tools = row.tools ? row.tools.split(",") : undefined;
106
+ const vec = await embedFn(row.user_message, row.assistant_message, tools);
107
+ embeddings.push({ id: row.id, vec });
108
+ }
109
+ const writeTx = db.transaction((items) => {
110
+ for (const item of items)
111
+ recordReembedded(db, item.id, item.vec);
112
+ });
113
+ writeTx(embeddings);
114
+ return embeddings.length;
115
+ }
116
+ finally {
117
+ releaseMigrationLock(lock);
118
+ }
119
+ }
120
+ //# sourceMappingURL=embedding-migration.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"embedding-migration.js","sourceRoot":"","sources":["../src/embedding-migration.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,eAAe,EAAuB,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAEvF;;;;;;;;GAQG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,CAAC;AAQnC,MAAM,CAAC,MAAM,oBAAoB,GAAG,eAAe,CAAC;AACpD,MAAM,CAAC,MAAM,oBAAoB,GAAG,eAAe,CAAC;AASpD;;;;GAIG;AACH,MAAM,UAAU,cAAc,CAAC,EAAqB,EAAE,KAAa;IACjE,OAAO,EAAE;SACN,OAAO,CAAC;;;;;;;;;;;GAWV,CAAC;SACC,GAAG,CAAC,iBAAiB,EAAE,KAAK,CAAe,CAAC;AACjD,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,gBAAgB,CAAC,EAAqB,EAAE,EAAU,EAAE,SAAmB;IACrF,EAAE,CAAC,OAAO,CAAC,wCAAwC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAC7D,EAAE,CAAC,OAAO,CAAC,yDAAyD,CAAC,CAAC,GAAG,CACvE,EAAE,EACF,MAAM,CAAC,IAAI,CAAC,IAAI,YAAY,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,CAChD,CAAC;IACF,EAAE,CAAC,OAAO,CAAC,yDAAyD,CAAC,CAAC,GAAG,CAAC,iBAAiB,EAAE,EAAE,CAAC,CAAC;AACnG,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,UAAU,CAAC,EAAqB;IAC9C,MAAM,GAAG,GAAG,EAAE;SACX,OAAO,CAAC,iEAAiE,CAAC;SAC1E,GAAG,CAAC,iBAAiB,CAAkB,CAAC;IAC3C,OAAO,GAAG,CAAC,CAAC,CAAC;AACf,CAAC;AAED,4DAA4D;AAC5D,MAAM,UAAU,oBAAoB,CAAC,QAAgB;IACnD,OAAO,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,2BAA2B,CAAC,CAAC;AAC1D,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,EAAqB,EACrB,QAAgB,EAChB,SAAiB,EACjB,OAAqF;IAErF,MAAM,SAAS,GAAG,UAAU,CAAC,EAAE,CAAC,CAAC;IACjC,IAAI,SAAS,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAE9B,MAAM,QAAQ,GAAG,oBAAoB,CAAC,QAAQ,CAAC,CAAC;IAChD,MAAM,IAAI,GAAG,oBAAoB,CAAC,QAAQ,CAAC,CAAC;IAC5C,IAAI,CAAC,IAAI,EAAE,CAAC;QACV,OAAO,CAAC,KAAK,CACX,wDAAwD,SAAS,8BAA8B,CAChG,CAAC;QACF,OAAO,CAAC,CAAC;IACX,CAAC;IAED,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,cAAc,CAAC,EAAE,EAAE,SAAS,CAAC,CAAC;QAC3C,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,CAAC,CAAC;QAEhC,OAAO,CAAC,KAAK,CAAC,qCAAqC,IAAI,CAAC,MAAM,KAAK,SAAS,kBAAkB,CAAC,CAAC;QAEhG,2DAA2D;QAC3D,oEAAoE;QACpE,MAAM,UAAU,GAAyC,EAAE,CAAC;QAC5D,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;YAC3D,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,YAAY,EAAE,GAAG,CAAC,iBAAiB,EAAE,KAAK,CAAC,CAAC;YAC1E,UAAU,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;QACvC,CAAC;QACD,MAAM,OAAO,GAAG,EAAE,CAAC,WAAW,CAAC,CAAC,KAAwB,EAAE,EAAE;YAC1D,KAAK,MAAM,IAAI,IAAI,KAAK;gBAAE,gBAAgB,CAAC,EAAE,EAAE,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;QACpE,CAAC,CAAC,CAAC;QACH,OAAO,CAAC,UAAU,CAAC,CAAC;QACpB,OAAO,UAAU,CAAC,MAAM,CAAC;IAC3B,CAAC;YAAS,CAAC;QACT,oBAAoB,CAAC,IAAI,CAAC,CAAC;IAC7B,CAAC;AACH,CAAC"}
@@ -0,0 +1,78 @@
1
+ /**
2
+ * The one embedding layer.
3
+ *
4
+ * Reconciled from two upstream `embeddings.ts` files that did the same job with
5
+ * incompatible shapes and two releases of the same library — `@xenova/transformers`
6
+ * is the former name of `@huggingface/transformers`.
7
+ *
8
+ * WON episodic-memory: module-level functions over a module-level pipeline,
9
+ * `@huggingface/transformers` ^4, `Xenova/bge-small-en-v1.5` at dtype q8,
10
+ * 2000-char truncation, the asymmetric BGE query prefix, and the two
11
+ * import-time `env` mutations that keep transformers.js off stdout.
12
+ * LOST private-journal-mcp: an `EmbeddingService` singleton with a private
13
+ * constructor and a `private readonly modelName`, on `@xenova/transformers` ^2
14
+ * with `Xenova/all-MiniLM-L6-v2`, no dtype, no truncation, and no
15
+ * query/passage asymmetry. One process, one model, forever — which
16
+ * blocks a per-record-type encoder and blocks swapping the encoder in a
17
+ * test.
18
+ *
19
+ * Three things were carried FORWARD from the losing side, because they were
20
+ * better there:
21
+ *
22
+ * - the memoised init promise, so two concurrent callers load the model once;
23
+ * - the init timeout with retry-on-failure (the promise is cleared so the
24
+ * next call retries rather than awaiting a dead promise);
25
+ * - `resetEmbeddings()`, the seam its two timeout tests need.
26
+ *
27
+ * And one thing was FIXED: `env.cacheDir` is pinned. Upstream set neither
28
+ * `cacheDir` nor a local model path, so the first `initEmbeddings()` fetched the
29
+ * model into whatever transformers.js defaults to — under pnpm a path inside
30
+ * the content-addressed store, shared across the workspace and possibly
31
+ * read-only in a container.
32
+ *
33
+ * ⚠️ Anything that changes model, dtype, prefix, pooling, normalisation or
34
+ * truncation MUST bump EMBEDDING_VERSION in embedding-migration.ts. Two
35
+ * encoders' vectors are dimensionally identical at 384 and semantically
36
+ * incomparable, so a mixed corpus does not error — it just ranks wrongly.
37
+ */
38
+ export { EMBEDDING_DIMENSIONS } from "./constants.js";
39
+ export declare const BGE_QUERY_PREFIX = "Represent this sentence for searching relevant passages: ";
40
+ export declare function initEmbeddings(): Promise<void>;
41
+ /**
42
+ * Drop the loaded model and the init memo. Test seam — the two timeout tests
43
+ * inherited from private-journal-mcp need to make loading fail, then succeed.
44
+ */
45
+ export declare function resetEmbeddings(): void;
46
+ export declare function generateEmbedding(text: string): Promise<number[]>;
47
+ /**
48
+ * Prepend the BGE retrieval prefix to a query string. Idempotent: returns
49
+ * the input unchanged if the prefix is already present.
50
+ */
51
+ export declare function withQueryPrefix(query: string): string;
52
+ /**
53
+ * Generate an embedding for a search QUERY. Adds the model-specific prefix
54
+ * before embedding, which gives a small but consistent recall lift on
55
+ * retrieval tasks. Document/passage embeddings stay unmodified — that's the
56
+ * asymmetric pattern BGE models are trained for.
57
+ *
58
+ * BOTH record types' queries route through here. private-journal-mcp embedded
59
+ * its queries with the same call it used for documents, which under an
60
+ * asymmetric encoder costs recall with no error and no log line.
61
+ */
62
+ export declare function generateQueryEmbedding(query: string): Promise<number[]>;
63
+ /**
64
+ * Document embedding for a conversation exchange (a harvested transcript turn).
65
+ */
66
+ export declare function generateExchangeEmbedding(userMessage: string, assistantMessage: string, toolNames?: string[]): Promise<number[]>;
67
+ /**
68
+ * Document embedding for a journal entry (deliberately written by the user).
69
+ *
70
+ * Same encoder, same normalisation, same truncation as an exchange — that is
71
+ * the whole point of the merge — but named separately because the two record
72
+ * types are separately queryable and a future divergence should be visible
73
+ * here rather than hidden behind a shared call.
74
+ */
75
+ export declare function generateEntryEmbedding(text: string): Promise<number[]>;
76
+ /** The signature both stores accept, so tests can inject a deterministic encoder. */
77
+ export type EmbedFn = (text: string) => Promise<number[]>;
78
+ //# sourceMappingURL=embeddings.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"embeddings.d.ts","sourceRoot":"","sources":["../src/embeddings.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AAKH,OAAO,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAC;AA0BtD,eAAO,MAAM,gBAAgB,8DAA8D,CAAC;AAyD5F,wBAAsB,cAAc,IAAI,OAAO,CAAC,IAAI,CAAC,CAIpD;AAED;;;GAGG;AACH,wBAAgB,eAAe,IAAI,IAAI,CAGtC;AAED,wBAAsB,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAcvE;AAED;;;GAGG;AACH,wBAAgB,eAAe,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAGrD;AAED;;;;;;;;;GASG;AACH,wBAAsB,sBAAsB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAE7E;AAED;;GAEG;AACH,wBAAsB,yBAAyB,CAC7C,WAAW,EAAE,MAAM,EACnB,gBAAgB,EAAE,MAAM,EACxB,SAAS,CAAC,EAAE,MAAM,EAAE,GACnB,OAAO,CAAC,MAAM,EAAE,CAAC,CASnB;AAED;;;;;;;GAOG;AACH,wBAAsB,sBAAsB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAE5E;AAED,qFAAqF;AACrF,MAAM,MAAM,OAAO,GAAG,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC"}
@@ -0,0 +1,180 @@
1
+ /**
2
+ * The one embedding layer.
3
+ *
4
+ * Reconciled from two upstream `embeddings.ts` files that did the same job with
5
+ * incompatible shapes and two releases of the same library — `@xenova/transformers`
6
+ * is the former name of `@huggingface/transformers`.
7
+ *
8
+ * WON episodic-memory: module-level functions over a module-level pipeline,
9
+ * `@huggingface/transformers` ^4, `Xenova/bge-small-en-v1.5` at dtype q8,
10
+ * 2000-char truncation, the asymmetric BGE query prefix, and the two
11
+ * import-time `env` mutations that keep transformers.js off stdout.
12
+ * LOST private-journal-mcp: an `EmbeddingService` singleton with a private
13
+ * constructor and a `private readonly modelName`, on `@xenova/transformers` ^2
14
+ * with `Xenova/all-MiniLM-L6-v2`, no dtype, no truncation, and no
15
+ * query/passage asymmetry. One process, one model, forever — which
16
+ * blocks a per-record-type encoder and blocks swapping the encoder in a
17
+ * test.
18
+ *
19
+ * Three things were carried FORWARD from the losing side, because they were
20
+ * better there:
21
+ *
22
+ * - the memoised init promise, so two concurrent callers load the model once;
23
+ * - the init timeout with retry-on-failure (the promise is cleared so the
24
+ * next call retries rather than awaiting a dead promise);
25
+ * - `resetEmbeddings()`, the seam its two timeout tests need.
26
+ *
27
+ * And one thing was FIXED: `env.cacheDir` is pinned. Upstream set neither
28
+ * `cacheDir` nor a local model path, so the first `initEmbeddings()` fetched the
29
+ * model into whatever transformers.js defaults to — under pnpm a path inside
30
+ * the content-addressed store, shared across the workspace and possibly
31
+ * read-only in a container.
32
+ *
33
+ * ⚠️ Anything that changes model, dtype, prefix, pooling, normalisation or
34
+ * truncation MUST bump EMBEDDING_VERSION in embedding-migration.ts. Two
35
+ * encoders' vectors are dimensionally identical at 384 and semantically
36
+ * incomparable, so a mixed corpus does not error — it just ranks wrongly.
37
+ */
38
+ import { env, pipeline } from "@huggingface/transformers";
39
+ import { getModelCacheDir } from "./paths.js";
40
+ export { EMBEDDING_DIMENSIONS } from "./constants.js";
41
+ // Disable progress callbacks to prevent stdout pollution in MCP context.
42
+ // In MCP, stdout is reserved for JSON-RPC communication. These two are
43
+ // import-time side effects on the library's shared `env`, deliberately: every
44
+ // entry point that reaches the encoder needs them, including the journal half.
45
+ env.allowLocalModels = true;
46
+ env.useBrowserCache = false;
47
+ /**
48
+ * Embedding model configuration.
49
+ *
50
+ * Using BAAI's bge-small-en-v1.5 (via Xenova's ONNX export) instead of the
51
+ * older all-MiniLM-L6-v2 — measured +6.34 R@1 on a 17K-corpus retrieval test
52
+ * against real production data. Same 384 dimensions, so vec_exchanges schema
53
+ * is unchanged.
54
+ *
55
+ * `Xenova/` here is a Hugging Face org namespace inside a model id, not a
56
+ * vendor brand. It is resolved over the network and must not be swept.
57
+ *
58
+ * BGE models recommend prepending a task prefix to QUERY embeddings only
59
+ * (passages/documents go through unmodified). See `withQueryPrefix` and
60
+ * `generateQueryEmbedding` below.
61
+ */
62
+ const MODEL_ID = "Xenova/bge-small-en-v1.5";
63
+ const MODEL_DTYPE = "q8";
64
+ export const BGE_QUERY_PREFIX = "Represent this sentence for searching relevant passages: ";
65
+ /** Longer inputs degrade mean-pooled embeddings; 2000 chars measured best. */
66
+ const MAX_INPUT_CHARS = 2000;
67
+ const DEFAULT_INIT_TIMEOUT_MS = 180_000;
68
+ let embeddingPipeline = null;
69
+ let initPromise = null;
70
+ function initTimeoutMs() {
71
+ const raw = Number(process.env.MOE_MEMORY_MODEL_INIT_TIMEOUT_MS);
72
+ return Number.isFinite(raw) && raw > 0 ? raw : DEFAULT_INIT_TIMEOUT_MS;
73
+ }
74
+ async function loadPipeline() {
75
+ const timeoutAfter = initTimeoutMs();
76
+ let timeoutId;
77
+ const timeout = new Promise((_resolve, reject) => {
78
+ timeoutId = setTimeout(() => reject(new Error(`Embedding model loading timed out after ${timeoutAfter / 1000}s. ` +
79
+ `The model cache is ${getModelCacheDir()}; a stale lock or a failed ` +
80
+ `partial download there is the usual cause. Remove it and retry.`)), timeoutAfter);
81
+ });
82
+ try {
83
+ // Pin the cache so the download lands somewhere writable that moves with
84
+ // MOE_MEMORY_CONFIG_DIR. Set here rather than at import time so nothing
85
+ // creates directories just by loading this module.
86
+ env.cacheDir = getModelCacheDir();
87
+ console.error("Loading embedding model (first run may take time)...");
88
+ embeddingPipeline = await Promise.race([
89
+ pipeline("feature-extraction", MODEL_ID, {
90
+ dtype: MODEL_DTYPE,
91
+ progress_callback: () => { },
92
+ }),
93
+ timeout,
94
+ ]);
95
+ console.error("Embedding model loaded");
96
+ }
97
+ catch (error) {
98
+ // Clear the memo so the next call retries instead of awaiting a dead promise.
99
+ initPromise = null;
100
+ embeddingPipeline = null;
101
+ throw error;
102
+ }
103
+ finally {
104
+ if (timeoutId !== undefined)
105
+ clearTimeout(timeoutId);
106
+ }
107
+ }
108
+ export async function initEmbeddings() {
109
+ if (embeddingPipeline)
110
+ return;
111
+ if (!initPromise)
112
+ initPromise = loadPipeline();
113
+ return initPromise;
114
+ }
115
+ /**
116
+ * Drop the loaded model and the init memo. Test seam — the two timeout tests
117
+ * inherited from private-journal-mcp need to make loading fail, then succeed.
118
+ */
119
+ export function resetEmbeddings() {
120
+ embeddingPipeline = null;
121
+ initPromise = null;
122
+ }
123
+ export async function generateEmbedding(text) {
124
+ await initEmbeddings();
125
+ const pipe = embeddingPipeline;
126
+ if (!pipe)
127
+ throw new Error("Embedding model not initialized");
128
+ // Truncate to avoid token limits (512 tokens max for bge-small).
129
+ const truncated = text.substring(0, MAX_INPUT_CHARS);
130
+ const output = await pipe(truncated, {
131
+ pooling: "mean",
132
+ normalize: true,
133
+ });
134
+ return Array.from(output.data);
135
+ }
136
+ /**
137
+ * Prepend the BGE retrieval prefix to a query string. Idempotent: returns
138
+ * the input unchanged if the prefix is already present.
139
+ */
140
+ export function withQueryPrefix(query) {
141
+ if (query.startsWith(BGE_QUERY_PREFIX))
142
+ return query;
143
+ return BGE_QUERY_PREFIX + query;
144
+ }
145
+ /**
146
+ * Generate an embedding for a search QUERY. Adds the model-specific prefix
147
+ * before embedding, which gives a small but consistent recall lift on
148
+ * retrieval tasks. Document/passage embeddings stay unmodified — that's the
149
+ * asymmetric pattern BGE models are trained for.
150
+ *
151
+ * BOTH record types' queries route through here. private-journal-mcp embedded
152
+ * its queries with the same call it used for documents, which under an
153
+ * asymmetric encoder costs recall with no error and no log line.
154
+ */
155
+ export async function generateQueryEmbedding(query) {
156
+ return generateEmbedding(withQueryPrefix(query));
157
+ }
158
+ /**
159
+ * Document embedding for a conversation exchange (a harvested transcript turn).
160
+ */
161
+ export async function generateExchangeEmbedding(userMessage, assistantMessage, toolNames) {
162
+ // Combine user question, assistant answer, and tools used for better searchability
163
+ let combined = `User: ${userMessage}\n\nAssistant: ${assistantMessage}`;
164
+ if (toolNames && toolNames.length > 0) {
165
+ combined += `\n\nTools: ${toolNames.join(", ")}`;
166
+ }
167
+ return generateEmbedding(combined);
168
+ }
169
+ /**
170
+ * Document embedding for a journal entry (deliberately written by the user).
171
+ *
172
+ * Same encoder, same normalisation, same truncation as an exchange — that is
173
+ * the whole point of the merge — but named separately because the two record
174
+ * types are separately queryable and a future divergence should be visible
175
+ * here rather than hidden behind a shared call.
176
+ */
177
+ export async function generateEntryEmbedding(text) {
178
+ return generateEmbedding(text);
179
+ }
180
+ //# sourceMappingURL=embeddings.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"embeddings.js","sourceRoot":"","sources":["../src/embeddings.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AAEH,OAAO,EAAE,GAAG,EAAkC,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AAC1F,OAAO,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAE9C,OAAO,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAC;AAEtD,yEAAyE;AACzE,uEAAuE;AACvE,8EAA8E;AAC9E,+EAA+E;AAC/E,GAAG,CAAC,gBAAgB,GAAG,IAAI,CAAC;AAC5B,GAAG,CAAC,eAAe,GAAG,KAAK,CAAC;AAE5B;;;;;;;;;;;;;;GAcG;AACH,MAAM,QAAQ,GAAG,0BAA0B,CAAC;AAC5C,MAAM,WAAW,GAAG,IAAI,CAAC;AACzB,MAAM,CAAC,MAAM,gBAAgB,GAAG,2DAA2D,CAAC;AAE5F,8EAA8E;AAC9E,MAAM,eAAe,GAAG,IAAI,CAAC;AAE7B,MAAM,uBAAuB,GAAG,OAAO,CAAC;AAExC,IAAI,iBAAiB,GAAqC,IAAI,CAAC;AAC/D,IAAI,WAAW,GAAyB,IAAI,CAAC;AAE7C,SAAS,aAAa;IACpB,MAAM,GAAG,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,gCAAgC,CAAC,CAAC;IACjE,OAAO,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,uBAAuB,CAAC;AACzE,CAAC;AAED,KAAK,UAAU,YAAY;IACzB,MAAM,YAAY,GAAG,aAAa,EAAE,CAAC;IACrC,IAAI,SAAoD,CAAC;IACzD,MAAM,OAAO,GAAG,IAAI,OAAO,CAAQ,CAAC,QAAQ,EAAE,MAAM,EAAE,EAAE;QACtD,SAAS,GAAG,UAAU,CACpB,GAAG,EAAE,CACH,MAAM,CACJ,IAAI,KAAK,CACP,2CAA2C,YAAY,GAAG,IAAI,KAAK;YACjE,sBAAsB,gBAAgB,EAAE,6BAA6B;YACrE,iEAAiE,CACpE,CACF,EACH,YAAY,CACb,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,IAAI,CAAC;QACH,yEAAyE;QACzE,wEAAwE;QACxE,mDAAmD;QACnD,GAAG,CAAC,QAAQ,GAAG,gBAAgB,EAAE,CAAC;QAElC,OAAO,CAAC,KAAK,CAAC,sDAAsD,CAAC,CAAC;QACtE,iBAAiB,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC;YACrC,QAAQ,CAAC,oBAAoB,EAAE,QAAQ,EAAE;gBACvC,KAAK,EAAE,WAAW;gBAClB,iBAAiB,EAAE,GAAG,EAAE,GAAE,CAAC;aAC5B,CAAC;YACF,OAAO;SACR,CAAC,CAAC;QACH,OAAO,CAAC,KAAK,CAAC,wBAAwB,CAAC,CAAC;IAC1C,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,8EAA8E;QAC9E,WAAW,GAAG,IAAI,CAAC;QACnB,iBAAiB,GAAG,IAAI,CAAC;QACzB,MAAM,KAAK,CAAC;IACd,CAAC;YAAS,CAAC;QACT,IAAI,SAAS,KAAK,SAAS;YAAE,YAAY,CAAC,SAAS,CAAC,CAAC;IACvD,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc;IAClC,IAAI,iBAAiB;QAAE,OAAO;IAC9B,IAAI,CAAC,WAAW;QAAE,WAAW,GAAG,YAAY,EAAE,CAAC;IAC/C,OAAO,WAAW,CAAC;AACrB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,eAAe;IAC7B,iBAAiB,GAAG,IAAI,CAAC;IACzB,WAAW,GAAG,IAAI,CAAC;AACrB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,IAAY;IAClD,MAAM,cAAc,EAAE,CAAC;IACvB,MAAM,IAAI,GAAG,iBAAiB,CAAC;IAC/B,IAAI,CAAC,IAAI;QAAE,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;IAE9D,iEAAiE;IACjE,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,eAAe,CAAC,CAAC;IAErD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,EAAE;QACnC,OAAO,EAAE,MAAM;QACf,SAAS,EAAE,IAAI;KAChB,CAAC,CAAC;IAEH,OAAO,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,IAAoB,CAAC,CAAC;AACjD,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,eAAe,CAAC,KAAa;IAC3C,IAAI,KAAK,CAAC,UAAU,CAAC,gBAAgB,CAAC;QAAE,OAAO,KAAK,CAAC;IACrD,OAAO,gBAAgB,GAAG,KAAK,CAAC;AAClC,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAAC,KAAa;IACxD,OAAO,iBAAiB,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC,CAAC;AACnD,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAC7C,WAAmB,EACnB,gBAAwB,EACxB,SAAoB;IAEpB,mFAAmF;IACnF,IAAI,QAAQ,GAAG,SAAS,WAAW,kBAAkB,gBAAgB,EAAE,CAAC;IAExE,IAAI,SAAS,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtC,QAAQ,IAAI,cAAc,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;IACnD,CAAC;IAED,OAAO,iBAAiB,CAAC,QAAQ,CAAC,CAAC;AACrC,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAAC,IAAY;IACvD,OAAO,iBAAiB,CAAC,IAAI,CAAC,CAAC;AACjC,CAAC"}
@@ -0,0 +1,39 @@
1
+ /**
2
+ * File-based exclusive locks. Thin wrapper around the `proper-lockfile`
3
+ * package, which uses an atomic mkdir + mtime-heartbeat protocol that is
4
+ * race-free under concurrent stale-stealers — the failure mode that pure
5
+ * "openSync(wx) + PID file" implementations cannot fully close without
6
+ * advisory locking (flock/fcntl).
7
+ *
8
+ * Used by long-running background work that must not run more than once at a
9
+ * time on a given machine: the embedding migration (#73) and
10
+ * `sync --background` (#97).
11
+ *
12
+ * On disk for a held lock at `<lockPath>`:
13
+ * <lockPath> — diagnostic file containing the holder's PID
14
+ * <lockPath>.lock/ — proper-lockfile's mutex directory (mkdir-atomic)
15
+ *
16
+ * Stale recovery is mtime-based with a 10-minute threshold: if the holder
17
+ * crashed without releasing, the .lock/ directory's mtime stops advancing,
18
+ * and a contender past the threshold steals it atomically. The protocol's
19
+ * atomicity guarantee is the package's contract — see proper-lockfile's
20
+ * README for the underlying analysis (the same approach used by npm itself).
21
+ *
22
+ * Error policy:
23
+ * - `null` means lock contention.
24
+ * - Unexpected I/O errors (EACCES, ENOSPC, EMFILE, etc.) are thrown so
25
+ * callers can surface disk problems rather than mask them as "locked".
26
+ */
27
+ export interface FileLockHandle {
28
+ path: string;
29
+ release: () => void;
30
+ }
31
+ export declare function acquireFileLock(lockPath: string): FileLockHandle | null;
32
+ export declare function releaseFileLock(handle: FileLockHandle): void;
33
+ /**
34
+ * Read the recorded holder PID from a lock file's diagnostic content.
35
+ * Returns null if the file doesn't exist or doesn't contain a valid PID.
36
+ * The PID is informational only — the real mutex lives in `<lockPath>.lock/`.
37
+ */
38
+ export declare function readLockHolder(lockPath: string): number | null;
39
+ //# sourceMappingURL=file-lock.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"file-lock.d.ts","sourceRoot":"","sources":["../src/file-lock.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AAMH,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,IAAI,CAAC;CACrB;AAUD,wBAAgB,eAAe,CAAC,QAAQ,EAAE,MAAM,GAAG,cAAc,GAAG,IAAI,CA6BvE;AAED,wBAAgB,eAAe,CAAC,MAAM,EAAE,cAAc,GAAG,IAAI,CAQ5D;AAED;;;;GAIG;AACH,wBAAgB,cAAc,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAO9D"}