@remnic/core 9.62.1 → 9.62.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (484) hide show
  1. package/dist/access-admin-ops-surface.d.ts +11 -11
  2. package/dist/access-admin-ops-surface.js +26 -26
  3. package/dist/access-audit.js +2 -2
  4. package/dist/access-authorization-probe.d.ts +11 -11
  5. package/dist/access-authorization-probe.js +27 -27
  6. package/dist/access-boundary.d.ts +11 -11
  7. package/dist/access-boundary.js +26 -26
  8. package/dist/access-cli.js +88 -86
  9. package/dist/access-cli.js.map +1 -1
  10. package/dist/access-coding-context-resolution.d.ts +1 -1
  11. package/dist/access-extraction-force-flush.d.ts +11 -11
  12. package/dist/access-extraction-force-flush.js +26 -26
  13. package/dist/access-health-types.d.ts +1 -1
  14. package/dist/access-http-lcm-compaction.d.ts +11 -11
  15. package/dist/access-http-lifecycle-flush.d.ts +11 -11
  16. package/dist/access-http-offline-stream.d.ts +11 -11
  17. package/dist/access-http-query.js +26 -26
  18. package/dist/access-http.d.ts +12 -12
  19. package/dist/access-http.js +36 -36
  20. package/dist/access-identity-continuity-surface.d.ts +9 -9
  21. package/dist/access-identity-continuity-surface.js +26 -26
  22. package/dist/access-lcm-surface.d.ts +11 -11
  23. package/dist/access-lcm-surface.js +26 -26
  24. package/dist/access-mcp.d.ts +11 -11
  25. package/dist/access-mcp.js +31 -31
  26. package/dist/access-memory-search-fanout.d.ts +2 -2
  27. package/dist/{access-namespace-preflight-EMvicnrb.d.ts → access-namespace-preflight-yO4LMGFH.d.ts} +1 -1
  28. package/dist/access-namespace-preflight.d.ts +3 -3
  29. package/dist/access-namespace-preflight.js +26 -26
  30. package/dist/access-observe-write-surface.d.ts +11 -11
  31. package/dist/access-observe-write-surface.js +26 -26
  32. package/dist/access-offline-manifest.d.ts +9 -9
  33. package/dist/access-offline-manifest.js +2 -2
  34. package/dist/access-operations-batch.js +27 -27
  35. package/dist/access-operations.d.ts +11 -11
  36. package/dist/access-operations.js +30 -30
  37. package/dist/access-recall-concurrency.d.ts +11 -11
  38. package/dist/access-recall-concurrency.js +26 -26
  39. package/dist/access-recall-response.d.ts +11 -11
  40. package/dist/access-recall-response.js +26 -26
  41. package/dist/access-recall-surface.d.ts +11 -11
  42. package/dist/access-recall-surface.js +26 -26
  43. package/dist/{access-service-BeDYQQ62.d.ts → access-service-M4LgO141.d.ts} +7 -7
  44. package/dist/access-service-helpers.d.ts +11 -11
  45. package/dist/access-service.d.ts +11 -11
  46. package/dist/access-service.js +26 -26
  47. package/dist/access-surface-catalog.d.ts +11 -11
  48. package/dist/access-surface-catalog.js +28 -28
  49. package/dist/access-wearables-meetings-surface.d.ts +3 -3
  50. package/dist/action-confidence.d.ts +1 -1
  51. package/dist/active-memory-bridge.d.ts +1 -1
  52. package/dist/active-recall.d.ts +1 -1
  53. package/dist/active-recall.js +1 -1
  54. package/dist/ambient-provenance.d.ts +1 -1
  55. package/dist/artifact-search.d.ts +1 -1
  56. package/dist/{auto-sync-FMWQQB3J.js → auto-sync-Z52VWMNC.js} +3 -3
  57. package/dist/behavior-learner.d.ts +1 -1
  58. package/dist/behavior-learner.js +2 -2
  59. package/dist/behavior-signals.d.ts +1 -1
  60. package/dist/bootstrap.d.ts +9 -9
  61. package/dist/briefing-window.d.ts +2 -2
  62. package/dist/briefing.d.ts +2 -2
  63. package/dist/briefing.js +11 -10
  64. package/dist/buffer-surprise-report.d.ts +1 -1
  65. package/dist/buffer-turn-helpers.d.ts +1 -1
  66. package/dist/buffer.d.ts +2 -2
  67. package/dist/bulk-import/index.d.ts +3 -3
  68. package/dist/calibration.d.ts +1 -1
  69. package/dist/capabilities.d.ts +1 -1
  70. package/dist/{catalog-B2SxZ6LU.d.ts → catalog-CyYTf_ok.d.ts} +1 -1
  71. package/dist/causal-behavior.d.ts +1 -1
  72. package/dist/causal-behavior.js +3 -3
  73. package/dist/causal-chain.js +3 -3
  74. package/dist/causal-consolidation.d.ts +1 -1
  75. package/dist/causal-consolidation.js +14 -14
  76. package/dist/causal-retrieval.js +3 -3
  77. package/dist/causal-trajectory-graph.d.ts +1 -1
  78. package/dist/causal-trajectory.js +2 -2
  79. package/dist/{chunk-B2QHVLJO.js → chunk-256FBCVX.js} +22 -10
  80. package/dist/chunk-256FBCVX.js.map +1 -0
  81. package/dist/{chunk-QP2IZTUR.js → chunk-2TYIAKU5.js} +2 -2
  82. package/dist/{chunk-QNMNNFKD.js → chunk-347ZR6D4.js} +2 -2
  83. package/dist/{chunk-76YNCJ5H.js → chunk-3F3CVVDS.js} +2 -2
  84. package/dist/{chunk-KCFZOSR6.js → chunk-3MA7JVXN.js} +4 -4
  85. package/dist/{chunk-KHQ57QLJ.js → chunk-46WM6JX2.js} +7 -3
  86. package/dist/chunk-46WM6JX2.js.map +1 -0
  87. package/dist/chunk-47DU4SGJ.js +92 -0
  88. package/dist/chunk-47DU4SGJ.js.map +1 -0
  89. package/dist/{chunk-HKYNBEL4.js → chunk-4FFROSFG.js} +37 -14
  90. package/dist/{chunk-HKYNBEL4.js.map → chunk-4FFROSFG.js.map} +1 -1
  91. package/dist/{chunk-TLUSSS2F.js → chunk-4HO3542O.js} +2 -2
  92. package/dist/{chunk-4E67XCT2.js → chunk-4IZ2IT7D.js} +3 -3
  93. package/dist/{chunk-33JBK2XP.js → chunk-5F6HMNMZ.js} +2 -2
  94. package/dist/{chunk-5WQMORU5.js → chunk-5PFGMO33.js} +2 -2
  95. package/dist/{chunk-6HEM6HTQ.js → chunk-6GGTATZF.js} +2 -2
  96. package/dist/{chunk-UIRU3MIJ.js → chunk-7BJYTX6Q.js} +2 -2
  97. package/dist/{chunk-2GRLJMKJ.js → chunk-7DRSWPUV.js} +3 -3
  98. package/dist/{chunk-TVVEYCNW.js → chunk-7K5Q6COX.js} +4 -4
  99. package/dist/{chunk-HXXGZ4KV.js → chunk-ALNZEWGN.js} +2 -2
  100. package/dist/{chunk-5WVUO6QT.js → chunk-AR253JBM.js} +1 -1
  101. package/dist/chunk-AR253JBM.js.map +1 -0
  102. package/dist/{chunk-67MKNKJQ.js → chunk-AYIHDC5S.js} +2 -2
  103. package/dist/{chunk-IXJ5U6ZL.js → chunk-BBHFAVMP.js} +8 -8
  104. package/dist/{chunk-IIRK5V5Z.js → chunk-BHO6RHE5.js} +3 -3
  105. package/dist/{chunk-WANOCU3K.js → chunk-BP6WMYLP.js} +3 -3
  106. package/dist/{chunk-ZJKLGS76.js → chunk-CALURYWR.js} +283 -248
  107. package/dist/chunk-CALURYWR.js.map +1 -0
  108. package/dist/{chunk-JLMH5YMU.js → chunk-CH627RRV.js} +2 -2
  109. package/dist/{chunk-XIABRKF6.js → chunk-CLJ5QS6J.js} +239 -47
  110. package/dist/chunk-CLJ5QS6J.js.map +1 -0
  111. package/dist/{chunk-YLHM5BQQ.js → chunk-DCUMFQYM.js} +171 -8
  112. package/dist/chunk-DCUMFQYM.js.map +1 -0
  113. package/dist/{chunk-ZBJMUXZH.js → chunk-DJ7HMNSL.js} +15 -8
  114. package/dist/chunk-DJ7HMNSL.js.map +1 -0
  115. package/dist/{chunk-WTMG5GAK.js → chunk-DROAPP6N.js} +3 -3
  116. package/dist/{chunk-PBH2Z2NY.js → chunk-DTLF2SLZ.js} +2 -2
  117. package/dist/{chunk-7JAXC526.js → chunk-EFWHCTYJ.js} +2 -2
  118. package/dist/{chunk-3XG4QEA2.js → chunk-FF6E6HOT.js} +39 -39
  119. package/dist/{chunk-NP7FLVZW.js → chunk-G2FCSILA.js} +2 -2
  120. package/dist/{chunk-7PVTOZR4.js → chunk-H23GIXJC.js} +2 -2
  121. package/dist/{chunk-WT6UL3UA.js → chunk-H7PRHUBD.js} +2 -2
  122. package/dist/{chunk-ETCVSZLG.js → chunk-HSBQJRV2.js} +2 -2
  123. package/dist/{chunk-GZRDOVQN.js → chunk-HVKMJAEF.js} +2 -2
  124. package/dist/{chunk-CSOYVTRX.js → chunk-IIPJKCZB.js} +3 -3
  125. package/dist/{chunk-E57E5SCS.js → chunk-IRLH4NM2.js} +3 -3
  126. package/dist/{chunk-4G45LJJP.js → chunk-IZRNVNPV.js} +2 -2
  127. package/dist/{chunk-YSUSUZR4.js → chunk-JCJGYNIS.js} +3 -3
  128. package/dist/{chunk-EU74U4GT.js → chunk-JDI7G6EA.js} +2 -2
  129. package/dist/{chunk-QWPAQLYV.js → chunk-JFLKSD5V.js} +2 -2
  130. package/dist/{chunk-KQS72GCR.js → chunk-LJOIZPJH.js} +2 -2
  131. package/dist/{chunk-U27G4E5G.js → chunk-LLGDVG52.js} +3 -3
  132. package/dist/{chunk-7N7DB4L5.js → chunk-LTJTP5YQ.js} +10 -9
  133. package/dist/chunk-LTJTP5YQ.js.map +1 -0
  134. package/dist/{chunk-XL3TATMJ.js → chunk-OQIRBGUU.js} +21 -12
  135. package/dist/chunk-OQIRBGUU.js.map +1 -0
  136. package/dist/{chunk-JGW2EUCQ.js → chunk-PMRUMGK6.js} +2 -2
  137. package/dist/{chunk-3G6YU5YD.js → chunk-PSNN25DW.js} +2 -2
  138. package/dist/{chunk-LRGSLV7M.js → chunk-PYHOAXTI.js} +4 -4
  139. package/dist/{chunk-75U2FR7U.js → chunk-QCI73Z7C.js} +2 -2
  140. package/dist/{chunk-EHMMRZOZ.js → chunk-QICOU4RY.js} +2 -2
  141. package/dist/{chunk-K4WIUTNH.js → chunk-RFH7U5LT.js} +2 -2
  142. package/dist/{chunk-4SGDYYWV.js → chunk-RPKJHYVE.js} +2 -2
  143. package/dist/{chunk-BHO3IVAN.js → chunk-RYST72N5.js} +12 -1
  144. package/dist/chunk-RYST72N5.js.map +1 -0
  145. package/dist/chunk-T3GIETKV.js +152 -0
  146. package/dist/chunk-T3GIETKV.js.map +1 -0
  147. package/dist/{chunk-DYDACAK2.js → chunk-TGSMU3F2.js} +3 -3
  148. package/dist/{chunk-MJFDJTDC.js → chunk-TIRJV2LW.js} +2 -2
  149. package/dist/{chunk-QRA3XVJX.js → chunk-UDTME32W.js} +2 -2
  150. package/dist/{chunk-LAM6UDFV.js → chunk-WMOJIOIG.js} +2 -2
  151. package/dist/{chunk-LLLY5IBK.js → chunk-WR2YKUIW.js} +2 -2
  152. package/dist/{chunk-KYI3CM42.js → chunk-X6E7TT3A.js} +2 -2
  153. package/dist/{chunk-QN7XQEIT.js → chunk-YIIC5IDU.js} +2 -2
  154. package/dist/{chunk-JDJOAL4B.js → chunk-YSR3DXGC.js} +2 -2
  155. package/dist/{chunk-LBJBNWS2.js → chunk-YTHQHH5U.js} +2 -2
  156. package/dist/{chunk-JF6YSHJM.js → chunk-ZPGHA7EO.js} +12 -12
  157. package/dist/{chunk-VUDALPI5.js → chunk-ZQKXRKMR.js} +19 -5
  158. package/dist/chunk-ZQKXRKMR.js.map +1 -0
  159. package/dist/{chunk-YG56A6XY.js → chunk-ZS4JADYY.js} +3 -3
  160. package/dist/{cli-DB_uakTc.d.ts → cli-Bw9z-KqX.d.ts} +5 -5
  161. package/dist/cli.d.ts +13 -13
  162. package/dist/cli.js +60 -60
  163. package/dist/coding/pre-action-gate.d.ts +1 -1
  164. package/dist/coding/pre-action-gate.js +3 -3
  165. package/dist/compounding/engine.d.ts +2 -2
  166. package/dist/compounding/engine.js +11 -10
  167. package/dist/compounding/preference-consolidator.d.ts +1 -1
  168. package/dist/compression-optimizer.d.ts +1 -1
  169. package/dist/config.d.ts +1 -1
  170. package/dist/config.js +1 -1
  171. package/dist/connectors/codex-materialize-runner.d.ts +1 -1
  172. package/dist/connectors/codex-materialize-runner.js +11 -10
  173. package/dist/connectors/codex-materialize.d.ts +1 -1
  174. package/dist/connectors/index.d.ts +1 -1
  175. package/dist/connectors/index.js +11 -10
  176. package/dist/consolidation-provenance-check.d.ts +2 -2
  177. package/dist/consolidation-undo.d.ts +2 -2
  178. package/dist/contradiction/index.d.ts +2 -2
  179. package/dist/converge-config.d.ts +1 -1
  180. package/dist/convergence-refresh.d.ts +2 -2
  181. package/dist/convergence-refresh.js +13 -12
  182. package/dist/conversation-index/backend.d.ts +1 -1
  183. package/dist/conversation-index/chunker.d.ts +1 -1
  184. package/dist/conversation-index/faiss-adapter.d.ts +1 -1
  185. package/dist/conversation-index/indexer.d.ts +1 -1
  186. package/dist/conversation-index/search.d.ts +1 -1
  187. package/dist/corpus-watermark.d.ts +1 -1
  188. package/dist/corpus-watermark.js +13 -12
  189. package/dist/day-summary.d.ts +1 -1
  190. package/dist/delinearize.d.ts +1 -1
  191. package/dist/dependency-propagation-config.d.ts +1 -1
  192. package/dist/{dependency-propagation-delivery-DcOVpK3y.d.ts → dependency-propagation-delivery-BxhvAppj.d.ts} +1 -1
  193. package/dist/direct-answer-wiring.d.ts +1 -1
  194. package/dist/direct-answer-wiring.js +3 -3
  195. package/dist/direct-answer.d.ts +1 -1
  196. package/dist/direct-answer.js +2 -2
  197. package/dist/embedding-fallback.d.ts +1 -1
  198. package/dist/enrichment/index.d.ts +1 -1
  199. package/dist/entity-origin-fields.d.ts +1 -1
  200. package/dist/entity-retrieval.d.ts +2 -2
  201. package/dist/entity-retrieval.js +11 -10
  202. package/dist/entity-schema.d.ts +1 -1
  203. package/dist/episodic-context.d.ts +92 -0
  204. package/dist/episodic-context.js +16 -0
  205. package/dist/episodic-context.js.map +1 -0
  206. package/dist/explicit-capture.d.ts +9 -9
  207. package/dist/external-wiki-access.d.ts +11 -11
  208. package/dist/external-wiki-access.js +27 -27
  209. package/dist/external-wiki-collection-registration.d.ts +1 -1
  210. package/dist/external-wiki-collection.d.ts +1 -1
  211. package/dist/external-wiki-mcp-tools.d.ts +11 -11
  212. package/dist/extraction-error-classification.d.ts +1 -1
  213. package/dist/extraction-faithfulness.d.ts +1 -1
  214. package/dist/extraction-judge-telemetry.d.ts +1 -1
  215. package/dist/extraction-judge-training.d.ts +1 -1
  216. package/dist/extraction-judge.d.ts +1 -1
  217. package/dist/extraction-liveness.d.ts +1 -1
  218. package/dist/extraction-normalization.d.ts +1 -1
  219. package/dist/extraction-normalization.js +2 -2
  220. package/dist/extraction-prompt.d.ts +1 -1
  221. package/dist/extraction-source-grounding-rules.d.ts +1 -1
  222. package/dist/extraction-source-grounding.d.ts +1 -1
  223. package/dist/extraction.d.ts +1 -1
  224. package/dist/extraction.js +14 -14
  225. package/dist/fallback-llm.d.ts +1 -1
  226. package/dist/{forget-5TWK7CZT.js → forget-SZ2BXDGF.js} +2 -2
  227. package/dist/graph-dashboard-diff.d.ts +1 -1
  228. package/dist/graph-dashboard-key.d.ts +1 -1
  229. package/dist/graph-dashboard-parser.d.ts +1 -1
  230. package/dist/graph-edge-reinforcement.d.ts +1 -1
  231. package/dist/graph-path-reconstruction.d.ts +1 -1
  232. package/dist/graph-path-scoring.d.ts +1 -1
  233. package/dist/graph-snapshot.d.ts +1 -1
  234. package/dist/graph.d.ts +1 -1
  235. package/dist/harmonic-construction.js +2 -2
  236. package/dist/harmonic-retrieval.js +2 -2
  237. package/dist/importance.d.ts +1 -1
  238. package/dist/importers/index.d.ts +1 -1
  239. package/dist/in-flight-reads.d.ts +1 -1
  240. package/dist/index.d.ts +21 -21
  241. package/dist/index.js +100 -98
  242. package/dist/intent.d.ts +1 -1
  243. package/dist/lcm/engine.d.ts +8 -2
  244. package/dist/lcm/engine.js +4 -4
  245. package/dist/lcm/index.d.ts +1 -1
  246. package/dist/lcm/index.js +9 -9
  247. package/dist/lcm/tools.d.ts +1 -1
  248. package/dist/lifecycle.d.ts +1 -1
  249. package/dist/live-connectors-runner.d.ts +1 -1
  250. package/dist/local-llm.d.ts +1 -1
  251. package/dist/local-model-endpoint.d.ts +1 -1
  252. package/dist/maintenance/memory-governance.d.ts +1 -1
  253. package/dist/maintenance/memory-governance.js +11 -10
  254. package/dist/maintenance/rebuild-memory-lifecycle-ledger.d.ts +2 -2
  255. package/dist/maintenance/rebuild-memory-lifecycle-ledger.js +11 -10
  256. package/dist/maintenance/rebuild-memory-projection.d.ts +2 -2
  257. package/dist/maintenance/rebuild-memory-projection.js +12 -11
  258. package/dist/{maintenance-5c1olVtB.d.ts → maintenance-uJb74j64.d.ts} +3 -3
  259. package/dist/mcp-memory-inspector-app.d.ts +11 -11
  260. package/dist/memory-action-policy.d.ts +1 -1
  261. package/dist/memory-cache.d.ts +1 -1
  262. package/dist/memory-lifecycle-ledger-utils.d.ts +1 -1
  263. package/dist/memory-projection-store.d.ts +1 -1
  264. package/dist/memory-provenance.d.ts +1 -1
  265. package/dist/memory-snapshot.d.ts +1 -1
  266. package/dist/memory-worth-outcomes.d.ts +2 -2
  267. package/dist/models-json.d.ts +1 -1
  268. package/dist/namespaces/migrate.d.ts +3 -3
  269. package/dist/namespaces/migrate.js +12 -11
  270. package/dist/namespaces/principal.d.ts +1 -1
  271. package/dist/namespaces/search.d.ts +1 -1
  272. package/dist/namespaces/search.js +6 -5
  273. package/dist/namespaces/storage.d.ts +3 -3
  274. package/dist/namespaces/storage.js +11 -10
  275. package/dist/native-knowledge.d.ts +1 -1
  276. package/dist/offline-sync-impression-drain.d.ts +1 -1
  277. package/dist/operator-doctor-corpus.d.ts +1 -1
  278. package/dist/operator-doctor-corpus.js +14 -13
  279. package/dist/operator-doctor-replica.d.ts +1 -1
  280. package/dist/operator-toolkit.d.ts +3 -3
  281. package/dist/operator-toolkit.js +19 -18
  282. package/dist/orchestration/compression-guideline-coordinator.d.ts +2 -2
  283. package/dist/orchestration/maintenance.d.ts +4 -4
  284. package/dist/orchestration/maintenance.js +16 -15
  285. package/dist/{orchestrator-HPuY6I4i.d.ts → orchestrator-CFi-5JSR.d.ts} +23 -8
  286. package/dist/orchestrator.d.ts +9 -9
  287. package/dist/orchestrator.js +88 -86
  288. package/dist/patterns-cli.d.ts +1 -1
  289. package/dist/{pipeline-CvVe1Q2_.d.ts → pipeline-BdF3ferY.d.ts} +1 -1
  290. package/dist/policy-runtime.d.ts +1 -1
  291. package/dist/policy-runtime.js +3 -3
  292. package/dist/proactive-contention.d.ts +1 -1
  293. package/dist/provenance.d.ts +1 -1
  294. package/dist/{public-http-D0JKFkyf.d.ts → public-http-DK-QwaBu.d.ts} +1 -1
  295. package/dist/{qmd-Bd0Tp76_.d.ts → qmd-D_62bYNl.d.ts} +1 -1
  296. package/dist/qmd-preflight.d.ts +2 -2
  297. package/dist/qmd-recall-cache.d.ts +1 -1
  298. package/dist/qmd.d.ts +2 -2
  299. package/dist/recall-concurrency-config.d.ts +1 -1
  300. package/dist/recall-disclosure-escalation.d.ts +1 -1
  301. package/dist/recall-explain-renderer.d.ts +1 -1
  302. package/dist/recall-explain-renderer.js +3 -3
  303. package/dist/recall-memory-map.d.ts +1 -1
  304. package/dist/recall-planner-llm.d.ts +1 -1
  305. package/dist/recall-query-policy.js +2 -2
  306. package/dist/recall-state.d.ts +1 -1
  307. package/dist/recall-tag-filter.d.ts +1 -1
  308. package/dist/recall-timings.d.ts +1 -1
  309. package/dist/recall-tokenization.d.ts +3 -1
  310. package/dist/recall-tokenization.js +5 -1
  311. package/dist/recall-xray-cli.d.ts +1 -1
  312. package/dist/recall-xray-cli.js +4 -4
  313. package/dist/recall-xray-renderer.d.ts +1 -1
  314. package/dist/recall-xray-renderer.js +3 -3
  315. package/dist/recall-xray.d.ts +1 -1
  316. package/dist/recall-xray.js +2 -2
  317. package/dist/reconcile/cursor.d.ts +1 -1
  318. package/dist/reconcile/manifest.d.ts +10 -1
  319. package/dist/reconcile/manifest.js +1 -1
  320. package/dist/reconcile/plan.d.ts +1 -1
  321. package/dist/replay/normalizers/chatgpt.d.ts +1 -1
  322. package/dist/replay/normalizers/claude.d.ts +1 -1
  323. package/dist/replay/normalizers/openclaw.d.ts +1 -1
  324. package/dist/replay/normalizers/shared.d.ts +1 -1
  325. package/dist/replay/runner.d.ts +1 -1
  326. package/dist/replay/types.d.ts +1 -1
  327. package/dist/replica-divergence.d.ts +1 -1
  328. package/dist/replica-peers-config.d.ts +1 -1
  329. package/dist/resolve-auth-token.d.ts +1 -1
  330. package/dist/resume-bundles.js +4 -4
  331. package/dist/retrieval-agents.d.ts +2 -2
  332. package/dist/retrieval-tiers.d.ts +1 -1
  333. package/dist/routing/engine.d.ts +1 -1
  334. package/dist/routing/store.d.ts +1 -1
  335. package/dist/salvage-envelope.d.ts +1 -1
  336. package/dist/{scope-profiles-ClLtm9gK.d.ts → scope-profiles-CeMv7pP0.d.ts} +1 -1
  337. package/dist/search/embed-helper.d.ts +1 -1
  338. package/dist/search/factory.d.ts +1 -1
  339. package/dist/search/factory.js +5 -4
  340. package/dist/search/index.d.ts +1 -1
  341. package/dist/search/index.js +9 -8
  342. package/dist/search/lancedb-backend.d.ts +14 -1
  343. package/dist/search/lancedb-backend.js +1 -1
  344. package/dist/search/meilisearch-backend.d.ts +1 -1
  345. package/dist/search/noop-backend.d.ts +1 -1
  346. package/dist/search/orama-backend.d.ts +22 -1
  347. package/dist/search/orama-backend.js +2 -1
  348. package/dist/search/port.d.ts +1 -1
  349. package/dist/search/remote-backend.d.ts +1 -1
  350. package/dist/{semantic-consolidation-B1EkeEDj.d.ts → semantic-consolidation-B-85brYj.d.ts} +1 -1
  351. package/dist/semantic-consolidation.d.ts +2 -2
  352. package/dist/semantic-consolidation.js +12 -12
  353. package/dist/semantic-rule-promotion.js +11 -10
  354. package/dist/semantic-rule-verifier.d.ts +1 -1
  355. package/dist/semantic-rule-verifier.js +11 -11
  356. package/dist/{service-Cf3ANQj9.d.ts → service-CoO0HA2E.d.ts} +2 -2
  357. package/dist/session-observer-bands.d.ts +1 -1
  358. package/dist/session-observer-state.d.ts +1 -1
  359. package/dist/shared-context/manager.d.ts +1 -1
  360. package/dist/signal.d.ts +1 -1
  361. package/dist/source-agent-qualifier.d.ts +1 -1
  362. package/dist/source-agent-qualifier.js +11 -10
  363. package/dist/{storage-DS2O_6Qt.d.ts → storage-BVAVitkO.d.ts} +1 -1
  364. package/dist/storage.d.ts +2 -2
  365. package/dist/storage.js +10 -9
  366. package/dist/summarizer.d.ts +1 -1
  367. package/dist/summary-snapshot.d.ts +1 -1
  368. package/dist/support-passport/index.d.ts +14 -14
  369. package/dist/support-passport/index.js +29 -29
  370. package/dist/temporal-supersession.d.ts +2 -2
  371. package/dist/temporal-timeline-recall.d.ts +1 -1
  372. package/dist/temporal-validity.d.ts +1 -1
  373. package/dist/threading.d.ts +1 -1
  374. package/dist/tier-migration.d.ts +2 -2
  375. package/dist/tier-routing.d.ts +1 -1
  376. package/dist/topics.d.ts +1 -1
  377. package/dist/transcript.d.ts +1 -1
  378. package/dist/transfer/import-sqlite.js +2 -2
  379. package/dist/trust-score-stage.d.ts +1 -1
  380. package/dist/trust-score.d.ts +1 -1
  381. package/dist/trust-zones.js +2 -2
  382. package/dist/{types-BOANGhCJ.d.ts → types-DWHOxlZP.d.ts} +3 -0
  383. package/dist/types.d.ts +1 -1
  384. package/dist/types.js +1 -1
  385. package/dist/user-message-cleaning.d.ts +23 -0
  386. package/dist/user-message-cleaning.js +10 -0
  387. package/dist/user-message-cleaning.js.map +1 -0
  388. package/dist/utility-runtime.d.ts +1 -1
  389. package/dist/verified-recall.js +11 -11
  390. package/dist/work-product-ledger.js +2 -2
  391. package/dist/write-envelope.d.ts +1 -1
  392. package/package.json +12 -2
  393. package/src/access-service-offline-file-content.test.ts +1 -0
  394. package/src/config.ts +11 -0
  395. package/src/episodic-context-recall.integration.test.ts +169 -0
  396. package/src/episodic-context.test.ts +266 -0
  397. package/src/episodic-context.ts +274 -0
  398. package/src/lcm/engine.ts +18 -1
  399. package/src/lifecycle/tombstones.test.ts +134 -0
  400. package/src/lifecycle/tombstones.ts +29 -2
  401. package/src/orchestration/episodic-context-section.ts +255 -0
  402. package/src/orchestration/recall-internal-deps.ts +2 -0
  403. package/src/orchestration/recall-internal.ts +40 -25
  404. package/src/orchestration/recall-result-formatter.ts +37 -0
  405. package/src/recall-tokenization.ts +14 -8
  406. package/src/reconcile/manifest.test.ts +133 -4
  407. package/src/reconcile/manifest.ts +43 -7
  408. package/src/search/factory.ts +1 -0
  409. package/src/search/lancedb-backend.ts +24 -16
  410. package/src/search/orama-backend.test.ts +208 -1
  411. package/src/search/orama-backend.ts +133 -18
  412. package/src/search/orama-cjk-tokenizer.test.ts +108 -0
  413. package/src/search/orama-cjk-tokenizer.ts +143 -0
  414. package/src/storage/tombstone-migration-sources.test.ts +116 -0
  415. package/src/storage/tombstone-migration-sources.ts +90 -0
  416. package/src/storage.ts +13 -34
  417. package/src/types.ts +3 -3
  418. package/src/user-message-cleaning.ts +139 -0
  419. package/dist/chunk-5WVUO6QT.js.map +0 -1
  420. package/dist/chunk-7N7DB4L5.js.map +0 -1
  421. package/dist/chunk-B2QHVLJO.js.map +0 -1
  422. package/dist/chunk-BHO3IVAN.js.map +0 -1
  423. package/dist/chunk-KHQ57QLJ.js.map +0 -1
  424. package/dist/chunk-VUDALPI5.js.map +0 -1
  425. package/dist/chunk-XIABRKF6.js.map +0 -1
  426. package/dist/chunk-XL3TATMJ.js.map +0 -1
  427. package/dist/chunk-YLHM5BQQ.js.map +0 -1
  428. package/dist/chunk-ZBJMUXZH.js.map +0 -1
  429. package/dist/chunk-ZJKLGS76.js.map +0 -1
  430. /package/dist/{auto-sync-FMWQQB3J.js.map → auto-sync-Z52VWMNC.js.map} +0 -0
  431. /package/dist/{chunk-QP2IZTUR.js.map → chunk-2TYIAKU5.js.map} +0 -0
  432. /package/dist/{chunk-QNMNNFKD.js.map → chunk-347ZR6D4.js.map} +0 -0
  433. /package/dist/{chunk-76YNCJ5H.js.map → chunk-3F3CVVDS.js.map} +0 -0
  434. /package/dist/{chunk-KCFZOSR6.js.map → chunk-3MA7JVXN.js.map} +0 -0
  435. /package/dist/{chunk-TLUSSS2F.js.map → chunk-4HO3542O.js.map} +0 -0
  436. /package/dist/{chunk-4E67XCT2.js.map → chunk-4IZ2IT7D.js.map} +0 -0
  437. /package/dist/{chunk-33JBK2XP.js.map → chunk-5F6HMNMZ.js.map} +0 -0
  438. /package/dist/{chunk-5WQMORU5.js.map → chunk-5PFGMO33.js.map} +0 -0
  439. /package/dist/{chunk-6HEM6HTQ.js.map → chunk-6GGTATZF.js.map} +0 -0
  440. /package/dist/{chunk-UIRU3MIJ.js.map → chunk-7BJYTX6Q.js.map} +0 -0
  441. /package/dist/{chunk-2GRLJMKJ.js.map → chunk-7DRSWPUV.js.map} +0 -0
  442. /package/dist/{chunk-TVVEYCNW.js.map → chunk-7K5Q6COX.js.map} +0 -0
  443. /package/dist/{chunk-HXXGZ4KV.js.map → chunk-ALNZEWGN.js.map} +0 -0
  444. /package/dist/{chunk-67MKNKJQ.js.map → chunk-AYIHDC5S.js.map} +0 -0
  445. /package/dist/{chunk-IXJ5U6ZL.js.map → chunk-BBHFAVMP.js.map} +0 -0
  446. /package/dist/{chunk-IIRK5V5Z.js.map → chunk-BHO6RHE5.js.map} +0 -0
  447. /package/dist/{chunk-WANOCU3K.js.map → chunk-BP6WMYLP.js.map} +0 -0
  448. /package/dist/{chunk-JLMH5YMU.js.map → chunk-CH627RRV.js.map} +0 -0
  449. /package/dist/{chunk-WTMG5GAK.js.map → chunk-DROAPP6N.js.map} +0 -0
  450. /package/dist/{chunk-PBH2Z2NY.js.map → chunk-DTLF2SLZ.js.map} +0 -0
  451. /package/dist/{chunk-7JAXC526.js.map → chunk-EFWHCTYJ.js.map} +0 -0
  452. /package/dist/{chunk-3XG4QEA2.js.map → chunk-FF6E6HOT.js.map} +0 -0
  453. /package/dist/{chunk-NP7FLVZW.js.map → chunk-G2FCSILA.js.map} +0 -0
  454. /package/dist/{chunk-7PVTOZR4.js.map → chunk-H23GIXJC.js.map} +0 -0
  455. /package/dist/{chunk-WT6UL3UA.js.map → chunk-H7PRHUBD.js.map} +0 -0
  456. /package/dist/{chunk-ETCVSZLG.js.map → chunk-HSBQJRV2.js.map} +0 -0
  457. /package/dist/{chunk-GZRDOVQN.js.map → chunk-HVKMJAEF.js.map} +0 -0
  458. /package/dist/{chunk-CSOYVTRX.js.map → chunk-IIPJKCZB.js.map} +0 -0
  459. /package/dist/{chunk-E57E5SCS.js.map → chunk-IRLH4NM2.js.map} +0 -0
  460. /package/dist/{chunk-4G45LJJP.js.map → chunk-IZRNVNPV.js.map} +0 -0
  461. /package/dist/{chunk-YSUSUZR4.js.map → chunk-JCJGYNIS.js.map} +0 -0
  462. /package/dist/{chunk-EU74U4GT.js.map → chunk-JDI7G6EA.js.map} +0 -0
  463. /package/dist/{chunk-QWPAQLYV.js.map → chunk-JFLKSD5V.js.map} +0 -0
  464. /package/dist/{chunk-KQS72GCR.js.map → chunk-LJOIZPJH.js.map} +0 -0
  465. /package/dist/{chunk-U27G4E5G.js.map → chunk-LLGDVG52.js.map} +0 -0
  466. /package/dist/{chunk-JGW2EUCQ.js.map → chunk-PMRUMGK6.js.map} +0 -0
  467. /package/dist/{chunk-3G6YU5YD.js.map → chunk-PSNN25DW.js.map} +0 -0
  468. /package/dist/{chunk-LRGSLV7M.js.map → chunk-PYHOAXTI.js.map} +0 -0
  469. /package/dist/{chunk-75U2FR7U.js.map → chunk-QCI73Z7C.js.map} +0 -0
  470. /package/dist/{chunk-EHMMRZOZ.js.map → chunk-QICOU4RY.js.map} +0 -0
  471. /package/dist/{chunk-K4WIUTNH.js.map → chunk-RFH7U5LT.js.map} +0 -0
  472. /package/dist/{chunk-4SGDYYWV.js.map → chunk-RPKJHYVE.js.map} +0 -0
  473. /package/dist/{chunk-DYDACAK2.js.map → chunk-TGSMU3F2.js.map} +0 -0
  474. /package/dist/{chunk-MJFDJTDC.js.map → chunk-TIRJV2LW.js.map} +0 -0
  475. /package/dist/{chunk-QRA3XVJX.js.map → chunk-UDTME32W.js.map} +0 -0
  476. /package/dist/{chunk-LAM6UDFV.js.map → chunk-WMOJIOIG.js.map} +0 -0
  477. /package/dist/{chunk-LLLY5IBK.js.map → chunk-WR2YKUIW.js.map} +0 -0
  478. /package/dist/{chunk-KYI3CM42.js.map → chunk-X6E7TT3A.js.map} +0 -0
  479. /package/dist/{chunk-QN7XQEIT.js.map → chunk-YIIC5IDU.js.map} +0 -0
  480. /package/dist/{chunk-JDJOAL4B.js.map → chunk-YSR3DXGC.js.map} +0 -0
  481. /package/dist/{chunk-LBJBNWS2.js.map → chunk-YTHQHH5U.js.map} +0 -0
  482. /package/dist/{chunk-JF6YSHJM.js.map → chunk-ZPGHA7EO.js.map} +0 -0
  483. /package/dist/{chunk-YG56A6XY.js.map → chunk-ZS4JADYY.js.map} +0 -0
  484. /package/dist/{forget-5TWK7CZT.js.map → forget-SZ2BXDGF.js.map} +0 -0
@@ -0,0 +1,108 @@
1
+ import assert from "node:assert/strict";
2
+ import test from "node:test";
3
+
4
+ import * as orama from "@orama/orama";
5
+
6
+ import {
7
+ containsNonLegacyTokenizerChars,
8
+ createCjkCapableTokenizer,
9
+ isSpaceFreeScriptChar,
10
+ ORAMA_CJK_TOKENIZER_LANGUAGE,
11
+ } from "./orama-cjk-tokenizer.js";
12
+
13
+ const stockTokenizer = orama.components.tokenizer.createTokenizer({ language: "english" });
14
+ const tokenizer = createCjkCapableTokenizer(orama);
15
+
16
+ test("CJK tokenizer matches the stock English tokenizer for legacy Latin input", () => {
17
+ for (const input of [
18
+ "The quick brown fox jumps over the lazy dog",
19
+ "Nebula-472 deploy finished; it's live (prod)!",
20
+ "snake_case camelCase 42.7%",
21
+ "",
22
+ ]) {
23
+ assert.deepEqual(tokenizer.tokenize(input), stockTokenizer.tokenize(input), input);
24
+ }
25
+ });
26
+
27
+ test("CJK tokenizer expands Japanese runs into n-grams", () => {
28
+ const tokens = tokenizer.tokenize("東京都庁の所在地");
29
+
30
+ assert.equal(tokens.includes("東京都庁"), true);
31
+ assert.equal(tokens.includes("所在"), true);
32
+ assert.equal(tokens.includes("地"), true);
33
+ assert.equal(tokens.includes("東京都庁の所在地"), true);
34
+ });
35
+
36
+ test("CJK tokenizer reuses the recall segmentation n-gram sizes", () => {
37
+ // A 5-char run yields chars + 2/3/4-grams + the whole run.
38
+ const tokens = tokenizer.tokenize("用户喜欢深");
39
+ const expected = new Set(["用", "户", "喜", "欢", "深"]);
40
+ for (const size of [2, 3, 4]) {
41
+ for (let i = 0; i <= 5 - size; i++) {
42
+ expected.add("用户喜欢深".slice(i, i + size));
43
+ }
44
+ }
45
+ expected.add("用户喜欢深");
46
+ for (const token of expected) {
47
+ assert.equal(tokens.includes(token), true, token);
48
+ }
49
+ });
50
+
51
+ test("CJK tokenizer keeps Latin tokens intact inside mixed-script input", () => {
52
+ const tokens = tokenizer.tokenize("Nebula-472 東京都庁");
53
+
54
+ assert.equal(tokens.includes("nebula-472"), true);
55
+ assert.equal(tokens.includes("東京都庁"), true);
56
+ });
57
+
58
+ test("CJK tokenizer n-grams Thai runs", () => {
59
+ const tokens = tokenizer.tokenize("กรุงเทพมหานคร");
60
+
61
+ assert.equal(tokens.includes("กรุง"), true);
62
+ assert.equal(tokens.includes("งเทพ"), true);
63
+ assert.equal(tokens.includes("กรุงเทพมหานคร"), true);
64
+ });
65
+
66
+ test("CJK tokenizer keeps Hangul and other alphabetic scripts as whole words", () => {
67
+ const tokens = tokenizer.tokenize("사용자 설정 Привет мир");
68
+
69
+ assert.equal(tokens.includes("사용자"), true);
70
+ assert.equal(tokens.includes("설정"), true);
71
+ assert.equal(tokens.includes("사"), false);
72
+ assert.equal(tokens.includes("привет"), true);
73
+ assert.equal(tokens.includes("мир"), true);
74
+ });
75
+
76
+ test("CJK tokenizer persists a versioned language marker", () => {
77
+ assert.equal(tokenizer.language, ORAMA_CJK_TOKENIZER_LANGUAGE);
78
+ assert.equal(typeof tokenizer.tokenize, "function");
79
+ assert.ok(tokenizer.normalizationCache instanceof Map);
80
+ });
81
+
82
+ test("isSpaceFreeScriptChar covers CJK and Thai only", () => {
83
+ assert.equal(isSpaceFreeScriptChar("東"), true);
84
+ assert.equal(isSpaceFreeScriptChar("あ"), true);
85
+ assert.equal(isSpaceFreeScriptChar("ア"), true);
86
+ assert.equal(isSpaceFreeScriptChar("ก"), true);
87
+ assert.equal(isSpaceFreeScriptChar("사"), false);
88
+ assert.equal(isSpaceFreeScriptChar("a"), false);
89
+ });
90
+
91
+ test("CJK tokenizer returns no tokens for non-string input", () => {
92
+ assert.deepEqual(tokenizer.tokenize(undefined as unknown as string), []);
93
+ assert.deepEqual(tokenizer.tokenize(null as unknown as string), []);
94
+ assert.deepEqual(tokenizer.tokenize(42 as unknown as string), []);
95
+ });
96
+
97
+ test("rebuild gate flags only tokenization-changing characters", () => {
98
+ // Word material outside the stock keep-set changes tokens (whole words,
99
+ // n-grams, attached marks) — the gate must flag it.
100
+ assert.equal(containsNonLegacyTokenizerChars("naïve"), true);
101
+ assert.equal(containsNonLegacyTokenizerChars("東京都庁"), true);
102
+ assert.equal(containsNonLegacyTokenizerChars("사용자"), true);
103
+ assert.equal(containsNonLegacyTokenizerChars("cafe\u0301"), true);
104
+ // Separators in BOTH tokenizers — punctuation and symbols — leave the
105
+ // stock token stream unchanged, so no re-index is warranted.
106
+ assert.equal(containsNonLegacyTokenizerChars("plain — text „quoted” · done"), false);
107
+ assert.equal(containsNonLegacyTokenizerChars("Nebula-472 deploy (prod)"), false);
108
+ });
@@ -0,0 +1,143 @@
1
+ import { expandUnsegmentableRecallNGrams, isUnsegmentableRecallChar } from "../recall-tokenization.js";
2
+
3
+ /**
4
+ * Language marker persisted inside Orama index files (`.msp`). Orama persists
5
+ * `tokenizer.language` with the index, so this doubles as the tokenization
6
+ * version: `OramaBackend` detects any other marker on restore and rebuilds
7
+ * the full-text index (issue #2187). Bump the suffix whenever tokenization
8
+ * changes.
9
+ */
10
+ export const ORAMA_CJK_TOKENIZER_LANGUAGE = "english+cjk-v1";
11
+
12
+ /** Minimal structural surface of `@orama/orama` this module consumes. */
13
+ export interface OramaTokenizerComponents {
14
+ components: {
15
+ tokenizer: {
16
+ createTokenizer: (config: { language: string }) => OramaTokenizer;
17
+ };
18
+ };
19
+ }
20
+
21
+ export interface OramaTokenizer {
22
+ language: string;
23
+ normalizationCache: Map<string, string>;
24
+ tokenize: (raw: string, language?: string, prop?: string, withCache?: boolean) => string[];
25
+ }
26
+
27
+ const THAI_SCRIPT_CHAR = /[\p{Script=Thai}]/u;
28
+ const WORD_CHAR = /[\p{L}\p{N}\p{M}]/u;
29
+ const COMBINING_MARK = /\p{M}/u;
30
+ const WHITESPACE_CHAR = /\s/u;
31
+ /**
32
+ * Chars the stock English tokenizer keeps inside a token. Input made only of
33
+ * these (plus whitespace) is delegated to the stock tokenizer byte-for-byte so
34
+ * existing English indexes stay term-compatible.
35
+ */
36
+ const LEGACY_TOKENIZER_CHAR = /[A-Za-zàèéìòóù0-9_'-]/u;
37
+
38
+ /**
39
+ * Space-free scripts: CJK per the shared recall segmentation strategy, plus
40
+ * Thai. Runs in these scripts are indexed as character n-grams so phrase
41
+ * queries match without word boundaries.
42
+ */
43
+ export function isSpaceFreeScriptChar(char: string): boolean {
44
+ return isUnsegmentableRecallChar(char) || THAI_SCRIPT_CHAR.test(char);
45
+ }
46
+
47
+ /**
48
+ * True when tokenizing the value differs from the stock English tokenizer —
49
+ * i.e. the value contains characters outside the stock keep-set that this
50
+ * tokenizer still indexes as token material: word characters (whole-word
51
+ * non-ASCII scripts), combining marks (attached to runs), and space-free
52
+ * scripts (CJK/Thai n-grams). Symbols and punctuation that are separators in
53
+ * BOTH tokenizers do not count. `OramaBackend` uses this to decide whether a
54
+ * stale pre-CJK index needs re-indexing on upgrade.
55
+ */
56
+ export function containsNonLegacyTokenizerChars(value: string): boolean {
57
+ for (const ch of value) {
58
+ if (LEGACY_TOKENIZER_CHAR.test(ch) || WHITESPACE_CHAR.test(ch)) continue;
59
+ if (WORD_CHAR.test(ch) || COMBINING_MARK.test(ch) || isSpaceFreeScriptChar(ch)) {
60
+ return true;
61
+ }
62
+ }
63
+ return false;
64
+ }
65
+
66
+ /**
67
+ * Orama tokenizer component that segments space-free scripts (CJK, Thai).
68
+ *
69
+ * - CJK/Thai runs expand to the same n-gram set the recall query-side
70
+ * tokenizer (`recall-tokenization.ts`) produces, so index-side and
71
+ * query-side tokens agree.
72
+ * - Other non-Latin scripts (Hangul, Cyrillic, Greek, Arabic, ...) are kept
73
+ * as whole words instead of being dropped by the English splitter.
74
+ * - Legacy Latin content tokenizes exactly like the stock English tokenizer;
75
+ * content with other characters keeps every stock token and additionally
76
+ * indexes non-ASCII words whole (e.g. "über" adds "über" beside "ber").
77
+ */
78
+ export function createCjkCapableTokenizer(oramaModule: OramaTokenizerComponents): OramaTokenizer {
79
+ const base = oramaModule.components.tokenizer.createTokenizer({ language: "english" });
80
+
81
+ const tokenize = (raw: string, _language?: string, prop?: string, withCache?: boolean): string[] => {
82
+ if (typeof raw !== "string") return [];
83
+ const normalized = raw.normalize("NFC");
84
+
85
+ if (!containsNonLegacyTokenizerChars(normalized)) {
86
+ return base.tokenize(normalized, "english", prop, withCache);
87
+ }
88
+
89
+ // The stock tokenizer treats every non-legacy char as a separator, so its
90
+ // output over the raw input already carries the legacy-Latin tokens
91
+ // (e.g. the "nebula-472" in "Nebula-472 東京都庁").
92
+ const tokens = base.tokenize(normalized, "english", prop, withCache);
93
+ const seen = new Set(tokens);
94
+
95
+ const pushToken = (token: string) => {
96
+ if (token && !seen.has(token)) {
97
+ seen.add(token);
98
+ tokens.push(token);
99
+ }
100
+ };
101
+
102
+ let spaceFreeRun = "";
103
+ let wordRun = "";
104
+ const flushSpaceFreeRun = () => {
105
+ if (!spaceFreeRun) return;
106
+ for (const token of expandUnsegmentableRecallNGrams(spaceFreeRun)) {
107
+ pushToken(token);
108
+ }
109
+ spaceFreeRun = "";
110
+ };
111
+ const flushWordRun = () => {
112
+ if (!wordRun) return;
113
+ pushToken(wordRun.toLowerCase());
114
+ wordRun = "";
115
+ };
116
+
117
+ for (const ch of normalized) {
118
+ if (isSpaceFreeScriptChar(ch)) {
119
+ flushWordRun();
120
+ spaceFreeRun += ch;
121
+ } else if (COMBINING_MARK.test(ch)) {
122
+ if (spaceFreeRun) spaceFreeRun += ch;
123
+ else if (wordRun) wordRun += ch;
124
+ } else if (WORD_CHAR.test(ch) && !LEGACY_TOKENIZER_CHAR.test(ch)) {
125
+ flushSpaceFreeRun();
126
+ wordRun += ch;
127
+ } else {
128
+ flushSpaceFreeRun();
129
+ flushWordRun();
130
+ }
131
+ }
132
+ flushSpaceFreeRun();
133
+ flushWordRun();
134
+
135
+ return tokens;
136
+ };
137
+
138
+ return {
139
+ language: ORAMA_CJK_TOKENIZER_LANGUAGE,
140
+ normalizationCache: base.normalizationCache,
141
+ tokenize,
142
+ };
143
+ }
@@ -0,0 +1,116 @@
1
+ import * as assert from "node:assert/strict";
2
+ import { mkdtemp, rm, utimes, writeFile } from "node:fs/promises";
3
+ import { tmpdir } from "node:os";
4
+ import path from "node:path";
5
+ import test from "node:test";
6
+
7
+ import { createTombstoneMigrationSourceContents } from "./tombstone-migration-sources.js";
8
+ import type { MemoryFile } from "../types.js";
9
+
10
+ function memory(id: string, content: string): MemoryFile {
11
+ return {
12
+ path: `facts/${id}.md`,
13
+ rawContent: content,
14
+ frontmatter: { id, category: "fact", status: "active" },
15
+ } as unknown as MemoryFile;
16
+ }
17
+
18
+ interface Harness {
19
+ dir: string;
20
+ ledgerPath: string;
21
+ collectCalls: number;
22
+ readsByPaths: string[][];
23
+ gateCollect: Promise<void>;
24
+ releaseCollect: () => void;
25
+ }
26
+
27
+ async function makeHarness(): Promise<Harness> {
28
+ const dir = await mkdtemp(path.join(tmpdir(), "tomb-migration-race-"));
29
+ const ledgerPath = path.join(dir, "tombstones.jsonl");
30
+ await writeFile(ledgerPath, "", "utf8");
31
+ let gateCollect: (() => void) | undefined;
32
+ const gate = new Promise<void>((resolve) => {
33
+ gateCollect = resolve;
34
+ });
35
+ const harness: Harness = {
36
+ dir,
37
+ ledgerPath,
38
+ collectCalls: 0,
39
+ readsByPaths: [],
40
+ gateCollect: gate,
41
+ releaseCollect: () => gateCollect?.(),
42
+ };
43
+ return harness;
44
+ }
45
+
46
+ test("an in-flight fill does not publish a stale cache after the ledger advances", async () => {
47
+ const h = await makeHarness();
48
+ try {
49
+ const generations: string[][] = [["facts/old-a.md"], ["facts/new-a.md", "facts/old-a.md"]];
50
+ const contents = new Map<string, string>([
51
+ ["facts/old-a.md", "old body"],
52
+ ["facts/new-a.md", "new body"],
53
+ ]);
54
+ const sourceContents = createTombstoneMigrationSourceContents({
55
+ tombstonesPath: () => h.ledgerPath,
56
+ collectTombstoneMigrationPaths: async () => {
57
+ const callIndex = h.collectCalls + 1;
58
+ h.collectCalls = callIndex;
59
+ if (callIndex === 1) {
60
+ // First fill stalls while a peer retires a legacy memory; the
61
+ // generation is bound at call entry so the stall cannot upgrade
62
+ // it to the fresh revision.
63
+ await h.gateCollect;
64
+ }
65
+ return generations[Math.min(callIndex - 1, generations.length - 1)]!;
66
+ },
67
+ readParsedMemoriesFromPaths: async (filePaths: string[]) => {
68
+ h.readsByPaths.push([...filePaths]);
69
+ return filePaths
70
+ .filter((filePath) => contents.has(filePath))
71
+ .map((filePath) => memory(path.basename(filePath, ".md"), contents.get(filePath)!));
72
+ },
73
+ storedContentIdentityCandidates: (content: string) => [content],
74
+ });
75
+
76
+ // The first callback requests an id whose path exists only in the new
77
+ // ledger revision, so a stale snapshot would miss it entirely.
78
+ const first = sourceContents(["new-a"]);
79
+ // Advance the ledger mtime so a later callback drops both caches while
80
+ // the first fill is still awaiting its (stale) path snapshot.
81
+ await utimes(h.ledgerPath, new Date(), new Date(Date.now() + 5000));
82
+ const second = sourceContents(["old-a"]);
83
+ h.releaseCollect();
84
+
85
+ // Without the revision guard, the first callback would miss on the
86
+ // stale snapshot, publish a full map built from it, and resolve empty.
87
+ const firstResult = await first;
88
+ assert.deepEqual([...firstResult.keys()], ["new-a"]);
89
+ const secondResult = await second;
90
+ assert.deepEqual([...secondResult.keys()], ["old-a"]);
91
+ assert.equal(h.collectCalls >= 2, true, "restart re-collects paths against the fresh revision");
92
+ } finally {
93
+ await rm(h.dir, { recursive: true, force: true });
94
+ }
95
+ });
96
+
97
+ test("a stable ledger revision reuses the collected paths across callbacks", async () => {
98
+ const h = await makeHarness();
99
+ try {
100
+ const sourceContents = createTombstoneMigrationSourceContents({
101
+ tombstonesPath: () => h.ledgerPath,
102
+ collectTombstoneMigrationPaths: async () => {
103
+ h.collectCalls += 1;
104
+ return ["facts/a.md"];
105
+ },
106
+ readParsedMemoriesFromPaths: async (filePaths: string[]) =>
107
+ filePaths.map((filePath) => memory(path.basename(filePath, ".md"), "body")),
108
+ storedContentIdentityCandidates: (content: string) => [content],
109
+ });
110
+ await sourceContents(["a"]);
111
+ await sourceContents(["a"]);
112
+ assert.equal(h.collectCalls, 1, "unchanged ledger mtime reuses the path snapshot");
113
+ } finally {
114
+ await rm(h.dir, { recursive: true, force: true });
115
+ }
116
+ });
@@ -0,0 +1,90 @@
1
+ import { statSync } from "node:fs";
2
+ import path from "node:path";
3
+ import type { TombstoneStoreOptions } from "../lifecycle/tombstones.js";
4
+ import type { MemoryFile } from "../types.js";
5
+
6
+ /**
7
+ * Deps injected by `StorageManager.buildTombstoneStore` so this module owns no
8
+ * storage-god-file imports (issue #1533 ratchet).
9
+ */
10
+ export interface TombstoneMigrationSourceDeps {
11
+ /** On-disk tombstone ledger path; its mtime scopes both caches below. */
12
+ tombstonesPath: () => string;
13
+ collectTombstoneMigrationPaths: () => Promise<string[]>;
14
+ readParsedMemoriesFromPaths: (filePaths: string[], batchSize?: number) => Promise<MemoryFile[]>;
15
+ storedContentIdentityCandidates: (content: string) => string[];
16
+ }
17
+
18
+ /**
19
+ * Build the `sourceContentsForMemoryIds` callback `TombstoneStore` uses to
20
+ * verify pre-Unicode rows against their retired source memories. The corpus
21
+ * path snapshot and the resolved source-content map are memoized for one
22
+ * ledger revision only (issue #2367): a peer process can create and retire a
23
+ * legacy-format memory after this snapshot, and the staleness reload must
24
+ * resolve the new row against a FRESH path list — otherwise the row stays
25
+ * unverified (withheld from the lookup tiers) until restart.
26
+ */
27
+ export function createTombstoneMigrationSourceContents(
28
+ deps: TombstoneMigrationSourceDeps,
29
+ ): NonNullable<TombstoneStoreOptions["sourceContentsForMemoryIds"]> {
30
+ let sourcePathsPromise: Promise<string[]> | undefined;
31
+ let sourceContentsByIdPromise: Promise<Map<string, readonly string[]>> | undefined;
32
+ let ledgerMtimeMs: number | null = null;
33
+ const dropCachesIfLedgerChanged = (): void => {
34
+ let mtimeMs: number | null;
35
+ try {
36
+ mtimeMs = Math.floor(statSync(deps.tombstonesPath()).mtimeMs);
37
+ } catch {
38
+ return; // ENOENT / stat failure — keep whatever is cached.
39
+ }
40
+ if (mtimeMs === ledgerMtimeMs) return;
41
+ ledgerMtimeMs = mtimeMs;
42
+ sourcePathsPromise = undefined;
43
+ sourceContentsByIdPromise = undefined;
44
+ };
45
+ return async (sourceMemoryIds) => {
46
+ for (;;) {
47
+ dropCachesIfLedgerChanged();
48
+ // In-flight fills are bound to the ledger revision they started
49
+ // against: if the ledger advances while a fill awaits, restart
50
+ // against the fresh revision instead of publishing a cache built
51
+ // from a stale path snapshot (issue #2367 review round 2).
52
+ const revision = ledgerMtimeMs;
53
+ const requested = new Set(sourceMemoryIds);
54
+ const sourcePaths = await (sourcePathsPromise ??= deps.collectTombstoneMigrationPaths());
55
+ if (ledgerMtimeMs !== revision) continue;
56
+ const directPaths = sourcePaths.filter((filePath) =>
57
+ requested.has(path.basename(filePath, ".md"))
58
+ );
59
+ const contents = new Map<string, readonly string[]>();
60
+ for (const memory of await deps.readParsedMemoriesFromPaths(directPaths, 50)) {
61
+ const id = memory.frontmatter.id;
62
+ if (requested.has(id) && !contents.has(id)) {
63
+ contents.set(id, deps.storedContentIdentityCandidates(memory.content));
64
+ }
65
+ }
66
+ // Re-check after the direct-path read too: a concurrent drop during
67
+ // that await must not let this call publish a fill built from the
68
+ // stale snapshot into the freshly cleared slot (issue #2367 round 3).
69
+ if (ledgerMtimeMs !== revision) continue;
70
+ if (contents.size === requested.size) return contents;
71
+ const fill = (sourceContentsByIdPromise ??= (async () => {
72
+ const allContents = new Map<string, readonly string[]>();
73
+ for (const memory of await deps.readParsedMemoriesFromPaths(sourcePaths, 50)) {
74
+ const id = memory.frontmatter.id;
75
+ if (!allContents.has(id)) {
76
+ allContents.set(id, deps.storedContentIdentityCandidates(memory.content));
77
+ }
78
+ }
79
+ return allContents;
80
+ })());
81
+ const allContents = await fill;
82
+ if (ledgerMtimeMs !== revision) continue;
83
+ for (const id of requested) {
84
+ const content = allContents.get(id);
85
+ if (content !== undefined) contents.set(id, content);
86
+ }
87
+ return contents;
88
+ }
89
+ };
90
+ }
package/src/storage.ts CHANGED
@@ -26,6 +26,7 @@ import { resolveSafeStoragePath } from "./storage-paths.js";
26
26
  import { log } from "./logger.js";
27
27
  import { createMemorySnapshot } from "./memory-snapshot.js";
28
28
  import { assertMemoryFrontmatterId, warnProjectionFallback } from "./storage-guards.js";
29
+ import { createTombstoneMigrationSourceContents } from "./storage/tombstone-migration-sources.js";
29
30
  import { MemoryReadStore, readWindowedMemories, type WindowedMemoryReadOptions, type WindowedMemoryReadResult } from "./storage/memory-read-store.js";
30
31
  import { hasSupersessionAudit } from "./storage/supersession-audit.js";
31
32
  import { runCommittedInvalidation } from "./storage/committed-invalidation.js";
@@ -3016,46 +3017,24 @@ export class StorageManager extends TombstoneBlockedCaptureIndexHost {
3016
3017
  }
3017
3018
 
3018
3019
  private buildTombstoneStore(): TombstoneStore {
3019
- let sourcePathsPromise: Promise<string[]> | undefined;
3020
- let sourceContentsByIdPromise: Promise<Map<string, readonly string[]>> | undefined;
3021
3020
  const options: TombstoneStoreOptions = {
3022
3021
  enabled: this.tombstonesConfig.enabled,
3023
3022
  semanticMatch: this.tombstonesConfig.semanticMatch,
3024
3023
  semanticThreshold: this.tombstonesConfig.semanticThreshold,
3025
3024
  hashContent: ContentHashIndex.computeHash,
3026
3025
  normalizeText: ContentHashIndex.normalizeContent,
3027
- sourceContentsForMemoryIds: async (sourceMemoryIds) => {
3028
- const requested = new Set(sourceMemoryIds);
3029
- const sourcePaths = await (sourcePathsPromise ??=
3030
- this.memoryReadStore.collectTombstoneMigrationPaths());
3031
- const directPaths = sourcePaths.filter((filePath) =>
3032
- requested.has(path.basename(filePath, ".md"))
3033
- );
3034
- const contents = new Map<string, readonly string[]>();
3035
- for (const memory of await this.readParsedMemoriesFromPaths(directPaths, 50)) {
3036
- const id = memory.frontmatter.id;
3037
- if (requested.has(id) && !contents.has(id)) {
3038
- contents.set(id, this.storedContentIdentityCandidates(memory.content));
3039
- }
3040
- }
3041
- if (contents.size === requested.size) return contents;
3042
- sourceContentsByIdPromise ??= (async () => {
3043
- const allContents = new Map<string, readonly string[]>();
3044
- for (const memory of await this.readParsedMemoriesFromPaths(sourcePaths, 50)) {
3045
- const id = memory.frontmatter.id;
3046
- if (!allContents.has(id)) {
3047
- allContents.set(id, this.storedContentIdentityCandidates(memory.content));
3048
- }
3049
- }
3050
- return allContents;
3051
- })();
3052
- const allContents = await sourceContentsByIdPromise;
3053
- for (const id of requested) {
3054
- const content = allContents.get(id);
3055
- if (content !== undefined) contents.set(id, content);
3056
- }
3057
- return contents;
3058
- },
3026
+ // Corpus-path + source-content caches are scoped to one ledger revision
3027
+ // so a peer-appended legacy row is verified on the next staleness reload
3028
+ // (issue #2367). Implementation lives in storage/tombstone-migration-sources.ts.
3029
+ sourceContentsForMemoryIds: createTombstoneMigrationSourceContents({
3030
+ tombstonesPath: () => this.tombstonesPath,
3031
+ collectTombstoneMigrationPaths: () =>
3032
+ this.memoryReadStore.collectTombstoneMigrationPaths(),
3033
+ readParsedMemoriesFromPaths: (filePaths, batchSize) =>
3034
+ this.readParsedMemoriesFromPaths(filePaths, batchSize),
3035
+ storedContentIdentityCandidates: (content) =>
3036
+ this.storedContentIdentityCandidates(content),
3037
+ }),
3059
3038
  };
3060
3039
  const io: TombstoneFileIo = {
3061
3040
  read: (filePath) => this.readStorageSecureFile(filePath),
package/src/types.ts CHANGED
@@ -631,7 +631,6 @@ export interface DependencyPropagationConfig {
631
631
  dryRun: boolean;
632
632
  }
633
633
 
634
-
635
634
  export interface HeartbeatConfig {
636
635
  enabled: boolean;
637
636
  journalPath: string;
@@ -1408,6 +1407,7 @@ export interface PluginConfig extends BoundedJsonlStateConfig, SecurityConfig {
1408
1407
  harmonicRetrievalEnabled: boolean;
1409
1408
  abstractionAnchorsEnabled: boolean;
1410
1409
  abstractionNodeStoreDir: string;
1410
+ episodicContextEnabled: boolean;
1411
1411
  // Episodic/semantic split foundation
1412
1412
  verifiedRecallEnabled: boolean;
1413
1413
  semanticRulePromotionEnabled: boolean;
@@ -1870,6 +1870,8 @@ export interface PluginConfig extends BoundedJsonlStateConfig, SecurityConfig {
1870
1870
  oramaEnabled: boolean;
1871
1871
  oramaDbPath?: string;
1872
1872
  oramaEmbeddingDimension?: number;
1873
+ /** Segment space-free scripts (CJK/Thai) in the Orama lexical index (issue #2187). Default `true`. */
1874
+ oramaCjkSegmentation?: boolean;
1873
1875
 
1874
1876
  // QMD daemon mode
1875
1877
  qmdDaemonEnabled: boolean;
@@ -2908,12 +2910,10 @@ export interface MemoryFrontmatter extends SourceConnectorProvenance, OriginMeta
2908
2910
  /** Last time this memory was accessed (ISO 8601) */
2909
2911
  lastAccessed?: string;
2910
2912
  // Memory Worth counters (issue #560)
2911
- //
2912
2913
  // Per-fact outcome counters used to derive a dynamic utility score —
2913
2914
  // `p(success | retrieved)` — as a complement to the static `importance`
2914
2915
  // field. Absent on legacy memories written before #560; readers must treat
2915
2916
  // `undefined` as zero observations (uniform Beta(1,1) prior).
2916
- //
2917
2917
  // Both values must be non-negative integers on write. PR 1 wires only the
2918
2918
  // schema + storage round-trip — no increments, scoring, or filtering yet.
2919
2919
  /** Number of sessions where this memory was retrieved and the outcome was judged a success. */