@remnic/core 9.62.1 → 9.63.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/access-admin-ops-surface.d.ts +11 -11
- package/dist/access-admin-ops-surface.js +26 -26
- package/dist/access-audit.js +2 -2
- package/dist/access-authorization-probe.d.ts +11 -11
- package/dist/access-authorization-probe.js +27 -27
- package/dist/access-boundary.d.ts +11 -11
- package/dist/access-boundary.js +26 -26
- package/dist/access-cli.js +88 -86
- package/dist/access-cli.js.map +1 -1
- package/dist/access-coding-context-resolution.d.ts +1 -1
- package/dist/access-extraction-force-flush.d.ts +11 -11
- package/dist/access-extraction-force-flush.js +26 -26
- package/dist/access-health-types.d.ts +1 -1
- package/dist/access-http-lcm-compaction.d.ts +11 -11
- package/dist/access-http-lifecycle-flush.d.ts +11 -11
- package/dist/access-http-offline-stream.d.ts +11 -11
- package/dist/access-http-query.js +26 -26
- package/dist/access-http.d.ts +12 -12
- package/dist/access-http.js +36 -36
- package/dist/access-identity-continuity-surface.d.ts +9 -9
- package/dist/access-identity-continuity-surface.js +26 -26
- package/dist/access-lcm-surface.d.ts +11 -11
- package/dist/access-lcm-surface.js +26 -26
- package/dist/access-mcp.d.ts +11 -11
- package/dist/access-mcp.js +31 -31
- package/dist/access-memory-search-fanout.d.ts +2 -2
- package/dist/{access-namespace-preflight-EMvicnrb.d.ts → access-namespace-preflight-yO4LMGFH.d.ts} +1 -1
- package/dist/access-namespace-preflight.d.ts +3 -3
- package/dist/access-namespace-preflight.js +26 -26
- package/dist/access-observe-write-surface.d.ts +11 -11
- package/dist/access-observe-write-surface.js +26 -26
- package/dist/access-offline-manifest.d.ts +9 -9
- package/dist/access-offline-manifest.js +2 -2
- package/dist/access-operations-batch.js +27 -27
- package/dist/access-operations.d.ts +11 -11
- package/dist/access-operations.js +30 -30
- package/dist/access-recall-concurrency.d.ts +11 -11
- package/dist/access-recall-concurrency.js +26 -26
- package/dist/access-recall-response.d.ts +11 -11
- package/dist/access-recall-response.js +26 -26
- package/dist/access-recall-surface.d.ts +11 -11
- package/dist/access-recall-surface.js +26 -26
- package/dist/{access-service-BeDYQQ62.d.ts → access-service-M4LgO141.d.ts} +7 -7
- package/dist/access-service-helpers.d.ts +11 -11
- package/dist/access-service.d.ts +11 -11
- package/dist/access-service.js +26 -26
- package/dist/access-surface-catalog.d.ts +11 -11
- package/dist/access-surface-catalog.js +28 -28
- package/dist/access-wearables-meetings-surface.d.ts +3 -3
- package/dist/action-confidence.d.ts +1 -1
- package/dist/active-memory-bridge.d.ts +1 -1
- package/dist/active-recall.d.ts +1 -1
- package/dist/active-recall.js +1 -1
- package/dist/ambient-provenance.d.ts +1 -1
- package/dist/artifact-search.d.ts +1 -1
- package/dist/{auto-sync-FMWQQB3J.js → auto-sync-Z52VWMNC.js} +3 -3
- package/dist/behavior-learner.d.ts +1 -1
- package/dist/behavior-learner.js +2 -2
- package/dist/behavior-signals.d.ts +1 -1
- package/dist/bootstrap.d.ts +9 -9
- package/dist/briefing-window.d.ts +2 -2
- package/dist/briefing.d.ts +2 -2
- package/dist/briefing.js +11 -10
- package/dist/buffer-surprise-report.d.ts +1 -1
- package/dist/buffer-turn-helpers.d.ts +1 -1
- package/dist/buffer.d.ts +2 -2
- package/dist/bulk-import/index.d.ts +3 -3
- package/dist/calibration.d.ts +1 -1
- package/dist/capabilities.d.ts +1 -1
- package/dist/{catalog-B2SxZ6LU.d.ts → catalog-CyYTf_ok.d.ts} +1 -1
- package/dist/causal-behavior.d.ts +1 -1
- package/dist/causal-behavior.js +3 -3
- package/dist/causal-chain.js +3 -3
- package/dist/causal-consolidation.d.ts +1 -1
- package/dist/causal-consolidation.js +14 -14
- package/dist/causal-retrieval.js +3 -3
- package/dist/causal-trajectory-graph.d.ts +1 -1
- package/dist/causal-trajectory.js +2 -2
- package/dist/{chunk-B2QHVLJO.js → chunk-256FBCVX.js} +22 -10
- package/dist/chunk-256FBCVX.js.map +1 -0
- package/dist/{chunk-QP2IZTUR.js → chunk-2TYIAKU5.js} +2 -2
- package/dist/{chunk-QNMNNFKD.js → chunk-347ZR6D4.js} +2 -2
- package/dist/{chunk-76YNCJ5H.js → chunk-3F3CVVDS.js} +2 -2
- package/dist/{chunk-KCFZOSR6.js → chunk-3MA7JVXN.js} +4 -4
- package/dist/{chunk-KHQ57QLJ.js → chunk-46WM6JX2.js} +7 -3
- package/dist/chunk-46WM6JX2.js.map +1 -0
- package/dist/chunk-47DU4SGJ.js +92 -0
- package/dist/chunk-47DU4SGJ.js.map +1 -0
- package/dist/{chunk-HKYNBEL4.js → chunk-4FFROSFG.js} +37 -14
- package/dist/{chunk-HKYNBEL4.js.map → chunk-4FFROSFG.js.map} +1 -1
- package/dist/{chunk-TLUSSS2F.js → chunk-4HO3542O.js} +2 -2
- package/dist/{chunk-4E67XCT2.js → chunk-4IZ2IT7D.js} +3 -3
- package/dist/{chunk-33JBK2XP.js → chunk-5F6HMNMZ.js} +2 -2
- package/dist/{chunk-5WQMORU5.js → chunk-5PFGMO33.js} +2 -2
- package/dist/{chunk-6HEM6HTQ.js → chunk-6GGTATZF.js} +2 -2
- package/dist/{chunk-UIRU3MIJ.js → chunk-7BJYTX6Q.js} +2 -2
- package/dist/{chunk-2GRLJMKJ.js → chunk-7DRSWPUV.js} +3 -3
- package/dist/{chunk-TVVEYCNW.js → chunk-7K5Q6COX.js} +4 -4
- package/dist/{chunk-HXXGZ4KV.js → chunk-ALNZEWGN.js} +2 -2
- package/dist/{chunk-5WVUO6QT.js → chunk-AR253JBM.js} +1 -1
- package/dist/chunk-AR253JBM.js.map +1 -0
- package/dist/{chunk-67MKNKJQ.js → chunk-AYIHDC5S.js} +2 -2
- package/dist/{chunk-IXJ5U6ZL.js → chunk-BBHFAVMP.js} +8 -8
- package/dist/{chunk-IIRK5V5Z.js → chunk-BHO6RHE5.js} +3 -3
- package/dist/{chunk-WANOCU3K.js → chunk-BP6WMYLP.js} +3 -3
- package/dist/{chunk-ZJKLGS76.js → chunk-CALURYWR.js} +283 -248
- package/dist/chunk-CALURYWR.js.map +1 -0
- package/dist/{chunk-JLMH5YMU.js → chunk-CH627RRV.js} +2 -2
- package/dist/{chunk-XIABRKF6.js → chunk-CLJ5QS6J.js} +239 -47
- package/dist/chunk-CLJ5QS6J.js.map +1 -0
- package/dist/{chunk-YLHM5BQQ.js → chunk-DCUMFQYM.js} +171 -8
- package/dist/chunk-DCUMFQYM.js.map +1 -0
- package/dist/{chunk-ZBJMUXZH.js → chunk-DJ7HMNSL.js} +15 -8
- package/dist/chunk-DJ7HMNSL.js.map +1 -0
- package/dist/{chunk-WTMG5GAK.js → chunk-DROAPP6N.js} +3 -3
- package/dist/{chunk-PBH2Z2NY.js → chunk-DTLF2SLZ.js} +2 -2
- package/dist/{chunk-7JAXC526.js → chunk-EFWHCTYJ.js} +2 -2
- package/dist/{chunk-3XG4QEA2.js → chunk-FF6E6HOT.js} +39 -39
- package/dist/{chunk-NP7FLVZW.js → chunk-G2FCSILA.js} +2 -2
- package/dist/{chunk-7PVTOZR4.js → chunk-H23GIXJC.js} +2 -2
- package/dist/{chunk-WT6UL3UA.js → chunk-H7PRHUBD.js} +2 -2
- package/dist/{chunk-ETCVSZLG.js → chunk-HSBQJRV2.js} +2 -2
- package/dist/{chunk-GZRDOVQN.js → chunk-HVKMJAEF.js} +2 -2
- package/dist/{chunk-CSOYVTRX.js → chunk-IIPJKCZB.js} +3 -3
- package/dist/{chunk-E57E5SCS.js → chunk-IRLH4NM2.js} +3 -3
- package/dist/{chunk-4G45LJJP.js → chunk-IZRNVNPV.js} +2 -2
- package/dist/{chunk-YSUSUZR4.js → chunk-JCJGYNIS.js} +3 -3
- package/dist/{chunk-EU74U4GT.js → chunk-JDI7G6EA.js} +2 -2
- package/dist/{chunk-QWPAQLYV.js → chunk-JFLKSD5V.js} +2 -2
- package/dist/{chunk-KQS72GCR.js → chunk-LJOIZPJH.js} +2 -2
- package/dist/{chunk-U27G4E5G.js → chunk-LLGDVG52.js} +3 -3
- package/dist/{chunk-7N7DB4L5.js → chunk-LTJTP5YQ.js} +10 -9
- package/dist/chunk-LTJTP5YQ.js.map +1 -0
- package/dist/{chunk-XL3TATMJ.js → chunk-OQIRBGUU.js} +21 -12
- package/dist/chunk-OQIRBGUU.js.map +1 -0
- package/dist/{chunk-JGW2EUCQ.js → chunk-PMRUMGK6.js} +2 -2
- package/dist/{chunk-3G6YU5YD.js → chunk-PSNN25DW.js} +2 -2
- package/dist/{chunk-LRGSLV7M.js → chunk-PYHOAXTI.js} +4 -4
- package/dist/{chunk-75U2FR7U.js → chunk-QCI73Z7C.js} +2 -2
- package/dist/{chunk-EHMMRZOZ.js → chunk-QICOU4RY.js} +2 -2
- package/dist/{chunk-K4WIUTNH.js → chunk-RFH7U5LT.js} +2 -2
- package/dist/{chunk-4SGDYYWV.js → chunk-RPKJHYVE.js} +2 -2
- package/dist/{chunk-BHO3IVAN.js → chunk-RYST72N5.js} +12 -1
- package/dist/chunk-RYST72N5.js.map +1 -0
- package/dist/chunk-T3GIETKV.js +152 -0
- package/dist/chunk-T3GIETKV.js.map +1 -0
- package/dist/{chunk-DYDACAK2.js → chunk-TGSMU3F2.js} +3 -3
- package/dist/{chunk-MJFDJTDC.js → chunk-TIRJV2LW.js} +2 -2
- package/dist/{chunk-QRA3XVJX.js → chunk-UDTME32W.js} +2 -2
- package/dist/{chunk-LAM6UDFV.js → chunk-WMOJIOIG.js} +2 -2
- package/dist/{chunk-LLLY5IBK.js → chunk-WR2YKUIW.js} +2 -2
- package/dist/{chunk-KYI3CM42.js → chunk-X6E7TT3A.js} +2 -2
- package/dist/{chunk-QN7XQEIT.js → chunk-YIIC5IDU.js} +2 -2
- package/dist/{chunk-JDJOAL4B.js → chunk-YSR3DXGC.js} +2 -2
- package/dist/{chunk-LBJBNWS2.js → chunk-YTHQHH5U.js} +2 -2
- package/dist/{chunk-JF6YSHJM.js → chunk-ZPGHA7EO.js} +12 -12
- package/dist/{chunk-VUDALPI5.js → chunk-ZQKXRKMR.js} +19 -5
- package/dist/chunk-ZQKXRKMR.js.map +1 -0
- package/dist/{chunk-YG56A6XY.js → chunk-ZS4JADYY.js} +3 -3
- package/dist/{cli-DB_uakTc.d.ts → cli-Bw9z-KqX.d.ts} +5 -5
- package/dist/cli.d.ts +13 -13
- package/dist/cli.js +60 -60
- package/dist/coding/pre-action-gate.d.ts +1 -1
- package/dist/coding/pre-action-gate.js +3 -3
- package/dist/compounding/engine.d.ts +2 -2
- package/dist/compounding/engine.js +11 -10
- package/dist/compounding/preference-consolidator.d.ts +1 -1
- package/dist/compression-optimizer.d.ts +1 -1
- package/dist/config.d.ts +1 -1
- package/dist/config.js +1 -1
- package/dist/connectors/codex-materialize-runner.d.ts +1 -1
- package/dist/connectors/codex-materialize-runner.js +11 -10
- package/dist/connectors/codex-materialize.d.ts +1 -1
- package/dist/connectors/index.d.ts +1 -1
- package/dist/connectors/index.js +11 -10
- package/dist/consolidation-provenance-check.d.ts +2 -2
- package/dist/consolidation-undo.d.ts +2 -2
- package/dist/contradiction/index.d.ts +2 -2
- package/dist/converge-config.d.ts +1 -1
- package/dist/convergence-refresh.d.ts +2 -2
- package/dist/convergence-refresh.js +13 -12
- package/dist/conversation-index/backend.d.ts +1 -1
- package/dist/conversation-index/chunker.d.ts +1 -1
- package/dist/conversation-index/faiss-adapter.d.ts +1 -1
- package/dist/conversation-index/indexer.d.ts +1 -1
- package/dist/conversation-index/search.d.ts +1 -1
- package/dist/corpus-watermark.d.ts +1 -1
- package/dist/corpus-watermark.js +13 -12
- package/dist/day-summary.d.ts +1 -1
- package/dist/delinearize.d.ts +1 -1
- package/dist/dependency-propagation-config.d.ts +1 -1
- package/dist/{dependency-propagation-delivery-DcOVpK3y.d.ts → dependency-propagation-delivery-BxhvAppj.d.ts} +1 -1
- package/dist/direct-answer-wiring.d.ts +1 -1
- package/dist/direct-answer-wiring.js +3 -3
- package/dist/direct-answer.d.ts +1 -1
- package/dist/direct-answer.js +2 -2
- package/dist/embedding-fallback.d.ts +1 -1
- package/dist/enrichment/index.d.ts +1 -1
- package/dist/entity-origin-fields.d.ts +1 -1
- package/dist/entity-retrieval.d.ts +2 -2
- package/dist/entity-retrieval.js +11 -10
- package/dist/entity-schema.d.ts +1 -1
- package/dist/episodic-context.d.ts +92 -0
- package/dist/episodic-context.js +16 -0
- package/dist/episodic-context.js.map +1 -0
- package/dist/explicit-capture.d.ts +9 -9
- package/dist/external-wiki-access.d.ts +11 -11
- package/dist/external-wiki-access.js +27 -27
- package/dist/external-wiki-collection-registration.d.ts +1 -1
- package/dist/external-wiki-collection.d.ts +1 -1
- package/dist/external-wiki-mcp-tools.d.ts +11 -11
- package/dist/extraction-error-classification.d.ts +1 -1
- package/dist/extraction-faithfulness.d.ts +1 -1
- package/dist/extraction-judge-telemetry.d.ts +1 -1
- package/dist/extraction-judge-training.d.ts +1 -1
- package/dist/extraction-judge.d.ts +1 -1
- package/dist/extraction-liveness.d.ts +1 -1
- package/dist/extraction-normalization.d.ts +1 -1
- package/dist/extraction-normalization.js +2 -2
- package/dist/extraction-prompt.d.ts +1 -1
- package/dist/extraction-source-grounding-rules.d.ts +1 -1
- package/dist/extraction-source-grounding.d.ts +1 -1
- package/dist/extraction.d.ts +1 -1
- package/dist/extraction.js +14 -14
- package/dist/fallback-llm.d.ts +1 -1
- package/dist/{forget-5TWK7CZT.js → forget-SZ2BXDGF.js} +2 -2
- package/dist/graph-dashboard-diff.d.ts +1 -1
- package/dist/graph-dashboard-key.d.ts +1 -1
- package/dist/graph-dashboard-parser.d.ts +1 -1
- package/dist/graph-edge-reinforcement.d.ts +1 -1
- package/dist/graph-path-reconstruction.d.ts +1 -1
- package/dist/graph-path-scoring.d.ts +1 -1
- package/dist/graph-snapshot.d.ts +1 -1
- package/dist/graph.d.ts +1 -1
- package/dist/harmonic-construction.js +2 -2
- package/dist/harmonic-retrieval.js +2 -2
- package/dist/importance.d.ts +1 -1
- package/dist/importers/index.d.ts +1 -1
- package/dist/in-flight-reads.d.ts +1 -1
- package/dist/index.d.ts +21 -21
- package/dist/index.js +100 -98
- package/dist/intent.d.ts +1 -1
- package/dist/lcm/engine.d.ts +8 -2
- package/dist/lcm/engine.js +4 -4
- package/dist/lcm/index.d.ts +1 -1
- package/dist/lcm/index.js +9 -9
- package/dist/lcm/tools.d.ts +1 -1
- package/dist/lifecycle.d.ts +1 -1
- package/dist/live-connectors-runner.d.ts +1 -1
- package/dist/local-llm.d.ts +1 -1
- package/dist/local-model-endpoint.d.ts +1 -1
- package/dist/maintenance/memory-governance.d.ts +1 -1
- package/dist/maintenance/memory-governance.js +11 -10
- package/dist/maintenance/rebuild-memory-lifecycle-ledger.d.ts +2 -2
- package/dist/maintenance/rebuild-memory-lifecycle-ledger.js +11 -10
- package/dist/maintenance/rebuild-memory-projection.d.ts +2 -2
- package/dist/maintenance/rebuild-memory-projection.js +12 -11
- package/dist/{maintenance-5c1olVtB.d.ts → maintenance-uJb74j64.d.ts} +3 -3
- package/dist/mcp-memory-inspector-app.d.ts +11 -11
- package/dist/memory-action-policy.d.ts +1 -1
- package/dist/memory-cache.d.ts +1 -1
- package/dist/memory-lifecycle-ledger-utils.d.ts +1 -1
- package/dist/memory-projection-store.d.ts +1 -1
- package/dist/memory-provenance.d.ts +1 -1
- package/dist/memory-snapshot.d.ts +1 -1
- package/dist/memory-worth-outcomes.d.ts +2 -2
- package/dist/models-json.d.ts +1 -1
- package/dist/namespaces/migrate.d.ts +3 -3
- package/dist/namespaces/migrate.js +12 -11
- package/dist/namespaces/principal.d.ts +1 -1
- package/dist/namespaces/search.d.ts +1 -1
- package/dist/namespaces/search.js +6 -5
- package/dist/namespaces/storage.d.ts +3 -3
- package/dist/namespaces/storage.js +11 -10
- package/dist/native-knowledge.d.ts +1 -1
- package/dist/offline-sync-impression-drain.d.ts +1 -1
- package/dist/operator-doctor-corpus.d.ts +1 -1
- package/dist/operator-doctor-corpus.js +14 -13
- package/dist/operator-doctor-replica.d.ts +1 -1
- package/dist/operator-toolkit.d.ts +3 -3
- package/dist/operator-toolkit.js +19 -18
- package/dist/orchestration/compression-guideline-coordinator.d.ts +2 -2
- package/dist/orchestration/maintenance.d.ts +4 -4
- package/dist/orchestration/maintenance.js +16 -15
- package/dist/{orchestrator-HPuY6I4i.d.ts → orchestrator-CFi-5JSR.d.ts} +23 -8
- package/dist/orchestrator.d.ts +9 -9
- package/dist/orchestrator.js +88 -86
- package/dist/patterns-cli.d.ts +1 -1
- package/dist/{pipeline-CvVe1Q2_.d.ts → pipeline-BdF3ferY.d.ts} +1 -1
- package/dist/policy-runtime.d.ts +1 -1
- package/dist/policy-runtime.js +3 -3
- package/dist/proactive-contention.d.ts +1 -1
- package/dist/provenance.d.ts +1 -1
- package/dist/{public-http-D0JKFkyf.d.ts → public-http-DK-QwaBu.d.ts} +1 -1
- package/dist/{qmd-Bd0Tp76_.d.ts → qmd-D_62bYNl.d.ts} +1 -1
- package/dist/qmd-preflight.d.ts +2 -2
- package/dist/qmd-recall-cache.d.ts +1 -1
- package/dist/qmd.d.ts +2 -2
- package/dist/recall-concurrency-config.d.ts +1 -1
- package/dist/recall-disclosure-escalation.d.ts +1 -1
- package/dist/recall-explain-renderer.d.ts +1 -1
- package/dist/recall-explain-renderer.js +3 -3
- package/dist/recall-memory-map.d.ts +1 -1
- package/dist/recall-planner-llm.d.ts +1 -1
- package/dist/recall-query-policy.js +2 -2
- package/dist/recall-state.d.ts +1 -1
- package/dist/recall-tag-filter.d.ts +1 -1
- package/dist/recall-timings.d.ts +1 -1
- package/dist/recall-tokenization.d.ts +3 -1
- package/dist/recall-tokenization.js +5 -1
- package/dist/recall-xray-cli.d.ts +1 -1
- package/dist/recall-xray-cli.js +4 -4
- package/dist/recall-xray-renderer.d.ts +1 -1
- package/dist/recall-xray-renderer.js +3 -3
- package/dist/recall-xray.d.ts +1 -1
- package/dist/recall-xray.js +2 -2
- package/dist/reconcile/cursor.d.ts +1 -1
- package/dist/reconcile/manifest.d.ts +10 -1
- package/dist/reconcile/manifest.js +1 -1
- package/dist/reconcile/plan.d.ts +1 -1
- package/dist/replay/normalizers/chatgpt.d.ts +1 -1
- package/dist/replay/normalizers/claude.d.ts +1 -1
- package/dist/replay/normalizers/openclaw.d.ts +1 -1
- package/dist/replay/normalizers/shared.d.ts +1 -1
- package/dist/replay/runner.d.ts +1 -1
- package/dist/replay/types.d.ts +1 -1
- package/dist/replica-divergence.d.ts +1 -1
- package/dist/replica-peers-config.d.ts +1 -1
- package/dist/resolve-auth-token.d.ts +1 -1
- package/dist/resume-bundles.js +4 -4
- package/dist/retrieval-agents.d.ts +2 -2
- package/dist/retrieval-tiers.d.ts +1 -1
- package/dist/routing/engine.d.ts +1 -1
- package/dist/routing/store.d.ts +1 -1
- package/dist/salvage-envelope.d.ts +1 -1
- package/dist/{scope-profiles-ClLtm9gK.d.ts → scope-profiles-CeMv7pP0.d.ts} +1 -1
- package/dist/search/embed-helper.d.ts +1 -1
- package/dist/search/factory.d.ts +1 -1
- package/dist/search/factory.js +5 -4
- package/dist/search/index.d.ts +1 -1
- package/dist/search/index.js +9 -8
- package/dist/search/lancedb-backend.d.ts +14 -1
- package/dist/search/lancedb-backend.js +1 -1
- package/dist/search/meilisearch-backend.d.ts +1 -1
- package/dist/search/noop-backend.d.ts +1 -1
- package/dist/search/orama-backend.d.ts +22 -1
- package/dist/search/orama-backend.js +2 -1
- package/dist/search/port.d.ts +1 -1
- package/dist/search/remote-backend.d.ts +1 -1
- package/dist/{semantic-consolidation-B1EkeEDj.d.ts → semantic-consolidation-B-85brYj.d.ts} +1 -1
- package/dist/semantic-consolidation.d.ts +2 -2
- package/dist/semantic-consolidation.js +12 -12
- package/dist/semantic-rule-promotion.js +11 -10
- package/dist/semantic-rule-verifier.d.ts +1 -1
- package/dist/semantic-rule-verifier.js +11 -11
- package/dist/{service-Cf3ANQj9.d.ts → service-CoO0HA2E.d.ts} +2 -2
- package/dist/session-observer-bands.d.ts +1 -1
- package/dist/session-observer-state.d.ts +1 -1
- package/dist/shared-context/manager.d.ts +1 -1
- package/dist/signal.d.ts +1 -1
- package/dist/source-agent-qualifier.d.ts +1 -1
- package/dist/source-agent-qualifier.js +11 -10
- package/dist/{storage-DS2O_6Qt.d.ts → storage-BVAVitkO.d.ts} +1 -1
- package/dist/storage.d.ts +2 -2
- package/dist/storage.js +10 -9
- package/dist/summarizer.d.ts +1 -1
- package/dist/summary-snapshot.d.ts +1 -1
- package/dist/support-passport/index.d.ts +14 -14
- package/dist/support-passport/index.js +29 -29
- package/dist/temporal-supersession.d.ts +2 -2
- package/dist/temporal-timeline-recall.d.ts +1 -1
- package/dist/temporal-validity.d.ts +1 -1
- package/dist/threading.d.ts +1 -1
- package/dist/tier-migration.d.ts +2 -2
- package/dist/tier-routing.d.ts +1 -1
- package/dist/topics.d.ts +1 -1
- package/dist/transcript.d.ts +1 -1
- package/dist/transfer/import-sqlite.js +2 -2
- package/dist/trust-score-stage.d.ts +1 -1
- package/dist/trust-score.d.ts +1 -1
- package/dist/trust-zones.js +2 -2
- package/dist/{types-BOANGhCJ.d.ts → types-DWHOxlZP.d.ts} +3 -0
- package/dist/types.d.ts +1 -1
- package/dist/types.js +1 -1
- package/dist/user-message-cleaning.d.ts +23 -0
- package/dist/user-message-cleaning.js +10 -0
- package/dist/user-message-cleaning.js.map +1 -0
- package/dist/utility-runtime.d.ts +1 -1
- package/dist/verified-recall.js +11 -11
- package/dist/work-product-ledger.js +2 -2
- package/dist/write-envelope.d.ts +1 -1
- package/package.json +12 -2
- package/src/access-service-offline-file-content.test.ts +1 -0
- package/src/config.ts +11 -0
- package/src/episodic-context-recall.integration.test.ts +169 -0
- package/src/episodic-context.test.ts +266 -0
- package/src/episodic-context.ts +274 -0
- package/src/lcm/engine.ts +18 -1
- package/src/lifecycle/tombstones.test.ts +134 -0
- package/src/lifecycle/tombstones.ts +29 -2
- package/src/orchestration/episodic-context-section.ts +255 -0
- package/src/orchestration/recall-internal-deps.ts +2 -0
- package/src/orchestration/recall-internal.ts +40 -25
- package/src/orchestration/recall-result-formatter.ts +37 -0
- package/src/recall-tokenization.ts +14 -8
- package/src/reconcile/manifest.test.ts +133 -4
- package/src/reconcile/manifest.ts +43 -7
- package/src/search/factory.ts +1 -0
- package/src/search/lancedb-backend.ts +24 -16
- package/src/search/orama-backend.test.ts +208 -1
- package/src/search/orama-backend.ts +133 -18
- package/src/search/orama-cjk-tokenizer.test.ts +108 -0
- package/src/search/orama-cjk-tokenizer.ts +143 -0
- package/src/storage/tombstone-migration-sources.test.ts +116 -0
- package/src/storage/tombstone-migration-sources.ts +90 -0
- package/src/storage.ts +13 -34
- package/src/types.ts +3 -3
- package/src/user-message-cleaning.ts +139 -0
- package/dist/chunk-5WVUO6QT.js.map +0 -1
- package/dist/chunk-7N7DB4L5.js.map +0 -1
- package/dist/chunk-B2QHVLJO.js.map +0 -1
- package/dist/chunk-BHO3IVAN.js.map +0 -1
- package/dist/chunk-KHQ57QLJ.js.map +0 -1
- package/dist/chunk-VUDALPI5.js.map +0 -1
- package/dist/chunk-XIABRKF6.js.map +0 -1
- package/dist/chunk-XL3TATMJ.js.map +0 -1
- package/dist/chunk-YLHM5BQQ.js.map +0 -1
- package/dist/chunk-ZBJMUXZH.js.map +0 -1
- package/dist/chunk-ZJKLGS76.js.map +0 -1
- /package/dist/{auto-sync-FMWQQB3J.js.map → auto-sync-Z52VWMNC.js.map} +0 -0
- /package/dist/{chunk-QP2IZTUR.js.map → chunk-2TYIAKU5.js.map} +0 -0
- /package/dist/{chunk-QNMNNFKD.js.map → chunk-347ZR6D4.js.map} +0 -0
- /package/dist/{chunk-76YNCJ5H.js.map → chunk-3F3CVVDS.js.map} +0 -0
- /package/dist/{chunk-KCFZOSR6.js.map → chunk-3MA7JVXN.js.map} +0 -0
- /package/dist/{chunk-TLUSSS2F.js.map → chunk-4HO3542O.js.map} +0 -0
- /package/dist/{chunk-4E67XCT2.js.map → chunk-4IZ2IT7D.js.map} +0 -0
- /package/dist/{chunk-33JBK2XP.js.map → chunk-5F6HMNMZ.js.map} +0 -0
- /package/dist/{chunk-5WQMORU5.js.map → chunk-5PFGMO33.js.map} +0 -0
- /package/dist/{chunk-6HEM6HTQ.js.map → chunk-6GGTATZF.js.map} +0 -0
- /package/dist/{chunk-UIRU3MIJ.js.map → chunk-7BJYTX6Q.js.map} +0 -0
- /package/dist/{chunk-2GRLJMKJ.js.map → chunk-7DRSWPUV.js.map} +0 -0
- /package/dist/{chunk-TVVEYCNW.js.map → chunk-7K5Q6COX.js.map} +0 -0
- /package/dist/{chunk-HXXGZ4KV.js.map → chunk-ALNZEWGN.js.map} +0 -0
- /package/dist/{chunk-67MKNKJQ.js.map → chunk-AYIHDC5S.js.map} +0 -0
- /package/dist/{chunk-IXJ5U6ZL.js.map → chunk-BBHFAVMP.js.map} +0 -0
- /package/dist/{chunk-IIRK5V5Z.js.map → chunk-BHO6RHE5.js.map} +0 -0
- /package/dist/{chunk-WANOCU3K.js.map → chunk-BP6WMYLP.js.map} +0 -0
- /package/dist/{chunk-JLMH5YMU.js.map → chunk-CH627RRV.js.map} +0 -0
- /package/dist/{chunk-WTMG5GAK.js.map → chunk-DROAPP6N.js.map} +0 -0
- /package/dist/{chunk-PBH2Z2NY.js.map → chunk-DTLF2SLZ.js.map} +0 -0
- /package/dist/{chunk-7JAXC526.js.map → chunk-EFWHCTYJ.js.map} +0 -0
- /package/dist/{chunk-3XG4QEA2.js.map → chunk-FF6E6HOT.js.map} +0 -0
- /package/dist/{chunk-NP7FLVZW.js.map → chunk-G2FCSILA.js.map} +0 -0
- /package/dist/{chunk-7PVTOZR4.js.map → chunk-H23GIXJC.js.map} +0 -0
- /package/dist/{chunk-WT6UL3UA.js.map → chunk-H7PRHUBD.js.map} +0 -0
- /package/dist/{chunk-ETCVSZLG.js.map → chunk-HSBQJRV2.js.map} +0 -0
- /package/dist/{chunk-GZRDOVQN.js.map → chunk-HVKMJAEF.js.map} +0 -0
- /package/dist/{chunk-CSOYVTRX.js.map → chunk-IIPJKCZB.js.map} +0 -0
- /package/dist/{chunk-E57E5SCS.js.map → chunk-IRLH4NM2.js.map} +0 -0
- /package/dist/{chunk-4G45LJJP.js.map → chunk-IZRNVNPV.js.map} +0 -0
- /package/dist/{chunk-YSUSUZR4.js.map → chunk-JCJGYNIS.js.map} +0 -0
- /package/dist/{chunk-EU74U4GT.js.map → chunk-JDI7G6EA.js.map} +0 -0
- /package/dist/{chunk-QWPAQLYV.js.map → chunk-JFLKSD5V.js.map} +0 -0
- /package/dist/{chunk-KQS72GCR.js.map → chunk-LJOIZPJH.js.map} +0 -0
- /package/dist/{chunk-U27G4E5G.js.map → chunk-LLGDVG52.js.map} +0 -0
- /package/dist/{chunk-JGW2EUCQ.js.map → chunk-PMRUMGK6.js.map} +0 -0
- /package/dist/{chunk-3G6YU5YD.js.map → chunk-PSNN25DW.js.map} +0 -0
- /package/dist/{chunk-LRGSLV7M.js.map → chunk-PYHOAXTI.js.map} +0 -0
- /package/dist/{chunk-75U2FR7U.js.map → chunk-QCI73Z7C.js.map} +0 -0
- /package/dist/{chunk-EHMMRZOZ.js.map → chunk-QICOU4RY.js.map} +0 -0
- /package/dist/{chunk-K4WIUTNH.js.map → chunk-RFH7U5LT.js.map} +0 -0
- /package/dist/{chunk-4SGDYYWV.js.map → chunk-RPKJHYVE.js.map} +0 -0
- /package/dist/{chunk-DYDACAK2.js.map → chunk-TGSMU3F2.js.map} +0 -0
- /package/dist/{chunk-MJFDJTDC.js.map → chunk-TIRJV2LW.js.map} +0 -0
- /package/dist/{chunk-QRA3XVJX.js.map → chunk-UDTME32W.js.map} +0 -0
- /package/dist/{chunk-LAM6UDFV.js.map → chunk-WMOJIOIG.js.map} +0 -0
- /package/dist/{chunk-LLLY5IBK.js.map → chunk-WR2YKUIW.js.map} +0 -0
- /package/dist/{chunk-KYI3CM42.js.map → chunk-X6E7TT3A.js.map} +0 -0
- /package/dist/{chunk-QN7XQEIT.js.map → chunk-YIIC5IDU.js.map} +0 -0
- /package/dist/{chunk-JDJOAL4B.js.map → chunk-YSR3DXGC.js.map} +0 -0
- /package/dist/{chunk-LBJBNWS2.js.map → chunk-YTHQHH5U.js.map} +0 -0
- /package/dist/{chunk-JF6YSHJM.js.map → chunk-ZPGHA7EO.js.map} +0 -0
- /package/dist/{chunk-YG56A6XY.js.map → chunk-ZS4JADYY.js.map} +0 -0
- /package/dist/{forget-5TWK7CZT.js.map → forget-SZ2BXDGF.js.map} +0 -0
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
import assert from "node:assert/strict";
|
|
2
|
+
import test from "node:test";
|
|
3
|
+
|
|
4
|
+
import * as orama from "@orama/orama";
|
|
5
|
+
|
|
6
|
+
import {
|
|
7
|
+
containsNonLegacyTokenizerChars,
|
|
8
|
+
createCjkCapableTokenizer,
|
|
9
|
+
isSpaceFreeScriptChar,
|
|
10
|
+
ORAMA_CJK_TOKENIZER_LANGUAGE,
|
|
11
|
+
} from "./orama-cjk-tokenizer.js";
|
|
12
|
+
|
|
13
|
+
const stockTokenizer = orama.components.tokenizer.createTokenizer({ language: "english" });
|
|
14
|
+
const tokenizer = createCjkCapableTokenizer(orama);
|
|
15
|
+
|
|
16
|
+
test("CJK tokenizer matches the stock English tokenizer for legacy Latin input", () => {
|
|
17
|
+
for (const input of [
|
|
18
|
+
"The quick brown fox jumps over the lazy dog",
|
|
19
|
+
"Nebula-472 deploy finished; it's live (prod)!",
|
|
20
|
+
"snake_case camelCase 42.7%",
|
|
21
|
+
"",
|
|
22
|
+
]) {
|
|
23
|
+
assert.deepEqual(tokenizer.tokenize(input), stockTokenizer.tokenize(input), input);
|
|
24
|
+
}
|
|
25
|
+
});
|
|
26
|
+
|
|
27
|
+
test("CJK tokenizer expands Japanese runs into n-grams", () => {
|
|
28
|
+
const tokens = tokenizer.tokenize("東京都庁の所在地");
|
|
29
|
+
|
|
30
|
+
assert.equal(tokens.includes("東京都庁"), true);
|
|
31
|
+
assert.equal(tokens.includes("所在"), true);
|
|
32
|
+
assert.equal(tokens.includes("地"), true);
|
|
33
|
+
assert.equal(tokens.includes("東京都庁の所在地"), true);
|
|
34
|
+
});
|
|
35
|
+
|
|
36
|
+
test("CJK tokenizer reuses the recall segmentation n-gram sizes", () => {
|
|
37
|
+
// A 5-char run yields chars + 2/3/4-grams + the whole run.
|
|
38
|
+
const tokens = tokenizer.tokenize("用户喜欢深");
|
|
39
|
+
const expected = new Set(["用", "户", "喜", "欢", "深"]);
|
|
40
|
+
for (const size of [2, 3, 4]) {
|
|
41
|
+
for (let i = 0; i <= 5 - size; i++) {
|
|
42
|
+
expected.add("用户喜欢深".slice(i, i + size));
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
expected.add("用户喜欢深");
|
|
46
|
+
for (const token of expected) {
|
|
47
|
+
assert.equal(tokens.includes(token), true, token);
|
|
48
|
+
}
|
|
49
|
+
});
|
|
50
|
+
|
|
51
|
+
test("CJK tokenizer keeps Latin tokens intact inside mixed-script input", () => {
|
|
52
|
+
const tokens = tokenizer.tokenize("Nebula-472 東京都庁");
|
|
53
|
+
|
|
54
|
+
assert.equal(tokens.includes("nebula-472"), true);
|
|
55
|
+
assert.equal(tokens.includes("東京都庁"), true);
|
|
56
|
+
});
|
|
57
|
+
|
|
58
|
+
test("CJK tokenizer n-grams Thai runs", () => {
|
|
59
|
+
const tokens = tokenizer.tokenize("กรุงเทพมหานคร");
|
|
60
|
+
|
|
61
|
+
assert.equal(tokens.includes("กรุง"), true);
|
|
62
|
+
assert.equal(tokens.includes("งเทพ"), true);
|
|
63
|
+
assert.equal(tokens.includes("กรุงเทพมหานคร"), true);
|
|
64
|
+
});
|
|
65
|
+
|
|
66
|
+
test("CJK tokenizer keeps Hangul and other alphabetic scripts as whole words", () => {
|
|
67
|
+
const tokens = tokenizer.tokenize("사용자 설정 Привет мир");
|
|
68
|
+
|
|
69
|
+
assert.equal(tokens.includes("사용자"), true);
|
|
70
|
+
assert.equal(tokens.includes("설정"), true);
|
|
71
|
+
assert.equal(tokens.includes("사"), false);
|
|
72
|
+
assert.equal(tokens.includes("привет"), true);
|
|
73
|
+
assert.equal(tokens.includes("мир"), true);
|
|
74
|
+
});
|
|
75
|
+
|
|
76
|
+
test("CJK tokenizer persists a versioned language marker", () => {
|
|
77
|
+
assert.equal(tokenizer.language, ORAMA_CJK_TOKENIZER_LANGUAGE);
|
|
78
|
+
assert.equal(typeof tokenizer.tokenize, "function");
|
|
79
|
+
assert.ok(tokenizer.normalizationCache instanceof Map);
|
|
80
|
+
});
|
|
81
|
+
|
|
82
|
+
test("isSpaceFreeScriptChar covers CJK and Thai only", () => {
|
|
83
|
+
assert.equal(isSpaceFreeScriptChar("東"), true);
|
|
84
|
+
assert.equal(isSpaceFreeScriptChar("あ"), true);
|
|
85
|
+
assert.equal(isSpaceFreeScriptChar("ア"), true);
|
|
86
|
+
assert.equal(isSpaceFreeScriptChar("ก"), true);
|
|
87
|
+
assert.equal(isSpaceFreeScriptChar("사"), false);
|
|
88
|
+
assert.equal(isSpaceFreeScriptChar("a"), false);
|
|
89
|
+
});
|
|
90
|
+
|
|
91
|
+
test("CJK tokenizer returns no tokens for non-string input", () => {
|
|
92
|
+
assert.deepEqual(tokenizer.tokenize(undefined as unknown as string), []);
|
|
93
|
+
assert.deepEqual(tokenizer.tokenize(null as unknown as string), []);
|
|
94
|
+
assert.deepEqual(tokenizer.tokenize(42 as unknown as string), []);
|
|
95
|
+
});
|
|
96
|
+
|
|
97
|
+
test("rebuild gate flags only tokenization-changing characters", () => {
|
|
98
|
+
// Word material outside the stock keep-set changes tokens (whole words,
|
|
99
|
+
// n-grams, attached marks) — the gate must flag it.
|
|
100
|
+
assert.equal(containsNonLegacyTokenizerChars("naïve"), true);
|
|
101
|
+
assert.equal(containsNonLegacyTokenizerChars("東京都庁"), true);
|
|
102
|
+
assert.equal(containsNonLegacyTokenizerChars("사용자"), true);
|
|
103
|
+
assert.equal(containsNonLegacyTokenizerChars("cafe\u0301"), true);
|
|
104
|
+
// Separators in BOTH tokenizers — punctuation and symbols — leave the
|
|
105
|
+
// stock token stream unchanged, so no re-index is warranted.
|
|
106
|
+
assert.equal(containsNonLegacyTokenizerChars("plain — text „quoted” · done"), false);
|
|
107
|
+
assert.equal(containsNonLegacyTokenizerChars("Nebula-472 deploy (prod)"), false);
|
|
108
|
+
});
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
import { expandUnsegmentableRecallNGrams, isUnsegmentableRecallChar } from "../recall-tokenization.js";
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Language marker persisted inside Orama index files (`.msp`). Orama persists
|
|
5
|
+
* `tokenizer.language` with the index, so this doubles as the tokenization
|
|
6
|
+
* version: `OramaBackend` detects any other marker on restore and rebuilds
|
|
7
|
+
* the full-text index (issue #2187). Bump the suffix whenever tokenization
|
|
8
|
+
* changes.
|
|
9
|
+
*/
|
|
10
|
+
export const ORAMA_CJK_TOKENIZER_LANGUAGE = "english+cjk-v1";
|
|
11
|
+
|
|
12
|
+
/** Minimal structural surface of `@orama/orama` this module consumes. */
|
|
13
|
+
export interface OramaTokenizerComponents {
|
|
14
|
+
components: {
|
|
15
|
+
tokenizer: {
|
|
16
|
+
createTokenizer: (config: { language: string }) => OramaTokenizer;
|
|
17
|
+
};
|
|
18
|
+
};
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
export interface OramaTokenizer {
|
|
22
|
+
language: string;
|
|
23
|
+
normalizationCache: Map<string, string>;
|
|
24
|
+
tokenize: (raw: string, language?: string, prop?: string, withCache?: boolean) => string[];
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
const THAI_SCRIPT_CHAR = /[\p{Script=Thai}]/u;
|
|
28
|
+
const WORD_CHAR = /[\p{L}\p{N}\p{M}]/u;
|
|
29
|
+
const COMBINING_MARK = /\p{M}/u;
|
|
30
|
+
const WHITESPACE_CHAR = /\s/u;
|
|
31
|
+
/**
|
|
32
|
+
* Chars the stock English tokenizer keeps inside a token. Input made only of
|
|
33
|
+
* these (plus whitespace) is delegated to the stock tokenizer byte-for-byte so
|
|
34
|
+
* existing English indexes stay term-compatible.
|
|
35
|
+
*/
|
|
36
|
+
const LEGACY_TOKENIZER_CHAR = /[A-Za-zàèéìòóù0-9_'-]/u;
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* Space-free scripts: CJK per the shared recall segmentation strategy, plus
|
|
40
|
+
* Thai. Runs in these scripts are indexed as character n-grams so phrase
|
|
41
|
+
* queries match without word boundaries.
|
|
42
|
+
*/
|
|
43
|
+
export function isSpaceFreeScriptChar(char: string): boolean {
|
|
44
|
+
return isUnsegmentableRecallChar(char) || THAI_SCRIPT_CHAR.test(char);
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* True when tokenizing the value differs from the stock English tokenizer —
|
|
49
|
+
* i.e. the value contains characters outside the stock keep-set that this
|
|
50
|
+
* tokenizer still indexes as token material: word characters (whole-word
|
|
51
|
+
* non-ASCII scripts), combining marks (attached to runs), and space-free
|
|
52
|
+
* scripts (CJK/Thai n-grams). Symbols and punctuation that are separators in
|
|
53
|
+
* BOTH tokenizers do not count. `OramaBackend` uses this to decide whether a
|
|
54
|
+
* stale pre-CJK index needs re-indexing on upgrade.
|
|
55
|
+
*/
|
|
56
|
+
export function containsNonLegacyTokenizerChars(value: string): boolean {
|
|
57
|
+
for (const ch of value) {
|
|
58
|
+
if (LEGACY_TOKENIZER_CHAR.test(ch) || WHITESPACE_CHAR.test(ch)) continue;
|
|
59
|
+
if (WORD_CHAR.test(ch) || COMBINING_MARK.test(ch) || isSpaceFreeScriptChar(ch)) {
|
|
60
|
+
return true;
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
return false;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* Orama tokenizer component that segments space-free scripts (CJK, Thai).
|
|
68
|
+
*
|
|
69
|
+
* - CJK/Thai runs expand to the same n-gram set the recall query-side
|
|
70
|
+
* tokenizer (`recall-tokenization.ts`) produces, so index-side and
|
|
71
|
+
* query-side tokens agree.
|
|
72
|
+
* - Other non-Latin scripts (Hangul, Cyrillic, Greek, Arabic, ...) are kept
|
|
73
|
+
* as whole words instead of being dropped by the English splitter.
|
|
74
|
+
* - Legacy Latin content tokenizes exactly like the stock English tokenizer;
|
|
75
|
+
* content with other characters keeps every stock token and additionally
|
|
76
|
+
* indexes non-ASCII words whole (e.g. "über" adds "über" beside "ber").
|
|
77
|
+
*/
|
|
78
|
+
export function createCjkCapableTokenizer(oramaModule: OramaTokenizerComponents): OramaTokenizer {
|
|
79
|
+
const base = oramaModule.components.tokenizer.createTokenizer({ language: "english" });
|
|
80
|
+
|
|
81
|
+
const tokenize = (raw: string, _language?: string, prop?: string, withCache?: boolean): string[] => {
|
|
82
|
+
if (typeof raw !== "string") return [];
|
|
83
|
+
const normalized = raw.normalize("NFC");
|
|
84
|
+
|
|
85
|
+
if (!containsNonLegacyTokenizerChars(normalized)) {
|
|
86
|
+
return base.tokenize(normalized, "english", prop, withCache);
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
// The stock tokenizer treats every non-legacy char as a separator, so its
|
|
90
|
+
// output over the raw input already carries the legacy-Latin tokens
|
|
91
|
+
// (e.g. the "nebula-472" in "Nebula-472 東京都庁").
|
|
92
|
+
const tokens = base.tokenize(normalized, "english", prop, withCache);
|
|
93
|
+
const seen = new Set(tokens);
|
|
94
|
+
|
|
95
|
+
const pushToken = (token: string) => {
|
|
96
|
+
if (token && !seen.has(token)) {
|
|
97
|
+
seen.add(token);
|
|
98
|
+
tokens.push(token);
|
|
99
|
+
}
|
|
100
|
+
};
|
|
101
|
+
|
|
102
|
+
let spaceFreeRun = "";
|
|
103
|
+
let wordRun = "";
|
|
104
|
+
const flushSpaceFreeRun = () => {
|
|
105
|
+
if (!spaceFreeRun) return;
|
|
106
|
+
for (const token of expandUnsegmentableRecallNGrams(spaceFreeRun)) {
|
|
107
|
+
pushToken(token);
|
|
108
|
+
}
|
|
109
|
+
spaceFreeRun = "";
|
|
110
|
+
};
|
|
111
|
+
const flushWordRun = () => {
|
|
112
|
+
if (!wordRun) return;
|
|
113
|
+
pushToken(wordRun.toLowerCase());
|
|
114
|
+
wordRun = "";
|
|
115
|
+
};
|
|
116
|
+
|
|
117
|
+
for (const ch of normalized) {
|
|
118
|
+
if (isSpaceFreeScriptChar(ch)) {
|
|
119
|
+
flushWordRun();
|
|
120
|
+
spaceFreeRun += ch;
|
|
121
|
+
} else if (COMBINING_MARK.test(ch)) {
|
|
122
|
+
if (spaceFreeRun) spaceFreeRun += ch;
|
|
123
|
+
else if (wordRun) wordRun += ch;
|
|
124
|
+
} else if (WORD_CHAR.test(ch) && !LEGACY_TOKENIZER_CHAR.test(ch)) {
|
|
125
|
+
flushSpaceFreeRun();
|
|
126
|
+
wordRun += ch;
|
|
127
|
+
} else {
|
|
128
|
+
flushSpaceFreeRun();
|
|
129
|
+
flushWordRun();
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
flushSpaceFreeRun();
|
|
133
|
+
flushWordRun();
|
|
134
|
+
|
|
135
|
+
return tokens;
|
|
136
|
+
};
|
|
137
|
+
|
|
138
|
+
return {
|
|
139
|
+
language: ORAMA_CJK_TOKENIZER_LANGUAGE,
|
|
140
|
+
normalizationCache: base.normalizationCache,
|
|
141
|
+
tokenize,
|
|
142
|
+
};
|
|
143
|
+
}
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
import * as assert from "node:assert/strict";
|
|
2
|
+
import { mkdtemp, rm, utimes, writeFile } from "node:fs/promises";
|
|
3
|
+
import { tmpdir } from "node:os";
|
|
4
|
+
import path from "node:path";
|
|
5
|
+
import test from "node:test";
|
|
6
|
+
|
|
7
|
+
import { createTombstoneMigrationSourceContents } from "./tombstone-migration-sources.js";
|
|
8
|
+
import type { MemoryFile } from "../types.js";
|
|
9
|
+
|
|
10
|
+
function memory(id: string, content: string): MemoryFile {
|
|
11
|
+
return {
|
|
12
|
+
path: `facts/${id}.md`,
|
|
13
|
+
rawContent: content,
|
|
14
|
+
frontmatter: { id, category: "fact", status: "active" },
|
|
15
|
+
} as unknown as MemoryFile;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
interface Harness {
|
|
19
|
+
dir: string;
|
|
20
|
+
ledgerPath: string;
|
|
21
|
+
collectCalls: number;
|
|
22
|
+
readsByPaths: string[][];
|
|
23
|
+
gateCollect: Promise<void>;
|
|
24
|
+
releaseCollect: () => void;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
async function makeHarness(): Promise<Harness> {
|
|
28
|
+
const dir = await mkdtemp(path.join(tmpdir(), "tomb-migration-race-"));
|
|
29
|
+
const ledgerPath = path.join(dir, "tombstones.jsonl");
|
|
30
|
+
await writeFile(ledgerPath, "", "utf8");
|
|
31
|
+
let gateCollect: (() => void) | undefined;
|
|
32
|
+
const gate = new Promise<void>((resolve) => {
|
|
33
|
+
gateCollect = resolve;
|
|
34
|
+
});
|
|
35
|
+
const harness: Harness = {
|
|
36
|
+
dir,
|
|
37
|
+
ledgerPath,
|
|
38
|
+
collectCalls: 0,
|
|
39
|
+
readsByPaths: [],
|
|
40
|
+
gateCollect: gate,
|
|
41
|
+
releaseCollect: () => gateCollect?.(),
|
|
42
|
+
};
|
|
43
|
+
return harness;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
test("an in-flight fill does not publish a stale cache after the ledger advances", async () => {
|
|
47
|
+
const h = await makeHarness();
|
|
48
|
+
try {
|
|
49
|
+
const generations: string[][] = [["facts/old-a.md"], ["facts/new-a.md", "facts/old-a.md"]];
|
|
50
|
+
const contents = new Map<string, string>([
|
|
51
|
+
["facts/old-a.md", "old body"],
|
|
52
|
+
["facts/new-a.md", "new body"],
|
|
53
|
+
]);
|
|
54
|
+
const sourceContents = createTombstoneMigrationSourceContents({
|
|
55
|
+
tombstonesPath: () => h.ledgerPath,
|
|
56
|
+
collectTombstoneMigrationPaths: async () => {
|
|
57
|
+
const callIndex = h.collectCalls + 1;
|
|
58
|
+
h.collectCalls = callIndex;
|
|
59
|
+
if (callIndex === 1) {
|
|
60
|
+
// First fill stalls while a peer retires a legacy memory; the
|
|
61
|
+
// generation is bound at call entry so the stall cannot upgrade
|
|
62
|
+
// it to the fresh revision.
|
|
63
|
+
await h.gateCollect;
|
|
64
|
+
}
|
|
65
|
+
return generations[Math.min(callIndex - 1, generations.length - 1)]!;
|
|
66
|
+
},
|
|
67
|
+
readParsedMemoriesFromPaths: async (filePaths: string[]) => {
|
|
68
|
+
h.readsByPaths.push([...filePaths]);
|
|
69
|
+
return filePaths
|
|
70
|
+
.filter((filePath) => contents.has(filePath))
|
|
71
|
+
.map((filePath) => memory(path.basename(filePath, ".md"), contents.get(filePath)!));
|
|
72
|
+
},
|
|
73
|
+
storedContentIdentityCandidates: (content: string) => [content],
|
|
74
|
+
});
|
|
75
|
+
|
|
76
|
+
// The first callback requests an id whose path exists only in the new
|
|
77
|
+
// ledger revision, so a stale snapshot would miss it entirely.
|
|
78
|
+
const first = sourceContents(["new-a"]);
|
|
79
|
+
// Advance the ledger mtime so a later callback drops both caches while
|
|
80
|
+
// the first fill is still awaiting its (stale) path snapshot.
|
|
81
|
+
await utimes(h.ledgerPath, new Date(), new Date(Date.now() + 5000));
|
|
82
|
+
const second = sourceContents(["old-a"]);
|
|
83
|
+
h.releaseCollect();
|
|
84
|
+
|
|
85
|
+
// Without the revision guard, the first callback would miss on the
|
|
86
|
+
// stale snapshot, publish a full map built from it, and resolve empty.
|
|
87
|
+
const firstResult = await first;
|
|
88
|
+
assert.deepEqual([...firstResult.keys()], ["new-a"]);
|
|
89
|
+
const secondResult = await second;
|
|
90
|
+
assert.deepEqual([...secondResult.keys()], ["old-a"]);
|
|
91
|
+
assert.equal(h.collectCalls >= 2, true, "restart re-collects paths against the fresh revision");
|
|
92
|
+
} finally {
|
|
93
|
+
await rm(h.dir, { recursive: true, force: true });
|
|
94
|
+
}
|
|
95
|
+
});
|
|
96
|
+
|
|
97
|
+
test("a stable ledger revision reuses the collected paths across callbacks", async () => {
|
|
98
|
+
const h = await makeHarness();
|
|
99
|
+
try {
|
|
100
|
+
const sourceContents = createTombstoneMigrationSourceContents({
|
|
101
|
+
tombstonesPath: () => h.ledgerPath,
|
|
102
|
+
collectTombstoneMigrationPaths: async () => {
|
|
103
|
+
h.collectCalls += 1;
|
|
104
|
+
return ["facts/a.md"];
|
|
105
|
+
},
|
|
106
|
+
readParsedMemoriesFromPaths: async (filePaths: string[]) =>
|
|
107
|
+
filePaths.map((filePath) => memory(path.basename(filePath, ".md"), "body")),
|
|
108
|
+
storedContentIdentityCandidates: (content: string) => [content],
|
|
109
|
+
});
|
|
110
|
+
await sourceContents(["a"]);
|
|
111
|
+
await sourceContents(["a"]);
|
|
112
|
+
assert.equal(h.collectCalls, 1, "unchanged ledger mtime reuses the path snapshot");
|
|
113
|
+
} finally {
|
|
114
|
+
await rm(h.dir, { recursive: true, force: true });
|
|
115
|
+
}
|
|
116
|
+
});
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
import { statSync } from "node:fs";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import type { TombstoneStoreOptions } from "../lifecycle/tombstones.js";
|
|
4
|
+
import type { MemoryFile } from "../types.js";
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Deps injected by `StorageManager.buildTombstoneStore` so this module owns no
|
|
8
|
+
* storage-god-file imports (issue #1533 ratchet).
|
|
9
|
+
*/
|
|
10
|
+
export interface TombstoneMigrationSourceDeps {
|
|
11
|
+
/** On-disk tombstone ledger path; its mtime scopes both caches below. */
|
|
12
|
+
tombstonesPath: () => string;
|
|
13
|
+
collectTombstoneMigrationPaths: () => Promise<string[]>;
|
|
14
|
+
readParsedMemoriesFromPaths: (filePaths: string[], batchSize?: number) => Promise<MemoryFile[]>;
|
|
15
|
+
storedContentIdentityCandidates: (content: string) => string[];
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* Build the `sourceContentsForMemoryIds` callback `TombstoneStore` uses to
|
|
20
|
+
* verify pre-Unicode rows against their retired source memories. The corpus
|
|
21
|
+
* path snapshot and the resolved source-content map are memoized for one
|
|
22
|
+
* ledger revision only (issue #2367): a peer process can create and retire a
|
|
23
|
+
* legacy-format memory after this snapshot, and the staleness reload must
|
|
24
|
+
* resolve the new row against a FRESH path list — otherwise the row stays
|
|
25
|
+
* unverified (withheld from the lookup tiers) until restart.
|
|
26
|
+
*/
|
|
27
|
+
export function createTombstoneMigrationSourceContents(
|
|
28
|
+
deps: TombstoneMigrationSourceDeps,
|
|
29
|
+
): NonNullable<TombstoneStoreOptions["sourceContentsForMemoryIds"]> {
|
|
30
|
+
let sourcePathsPromise: Promise<string[]> | undefined;
|
|
31
|
+
let sourceContentsByIdPromise: Promise<Map<string, readonly string[]>> | undefined;
|
|
32
|
+
let ledgerMtimeMs: number | null = null;
|
|
33
|
+
const dropCachesIfLedgerChanged = (): void => {
|
|
34
|
+
let mtimeMs: number | null;
|
|
35
|
+
try {
|
|
36
|
+
mtimeMs = Math.floor(statSync(deps.tombstonesPath()).mtimeMs);
|
|
37
|
+
} catch {
|
|
38
|
+
return; // ENOENT / stat failure — keep whatever is cached.
|
|
39
|
+
}
|
|
40
|
+
if (mtimeMs === ledgerMtimeMs) return;
|
|
41
|
+
ledgerMtimeMs = mtimeMs;
|
|
42
|
+
sourcePathsPromise = undefined;
|
|
43
|
+
sourceContentsByIdPromise = undefined;
|
|
44
|
+
};
|
|
45
|
+
return async (sourceMemoryIds) => {
|
|
46
|
+
for (;;) {
|
|
47
|
+
dropCachesIfLedgerChanged();
|
|
48
|
+
// In-flight fills are bound to the ledger revision they started
|
|
49
|
+
// against: if the ledger advances while a fill awaits, restart
|
|
50
|
+
// against the fresh revision instead of publishing a cache built
|
|
51
|
+
// from a stale path snapshot (issue #2367 review round 2).
|
|
52
|
+
const revision = ledgerMtimeMs;
|
|
53
|
+
const requested = new Set(sourceMemoryIds);
|
|
54
|
+
const sourcePaths = await (sourcePathsPromise ??= deps.collectTombstoneMigrationPaths());
|
|
55
|
+
if (ledgerMtimeMs !== revision) continue;
|
|
56
|
+
const directPaths = sourcePaths.filter((filePath) =>
|
|
57
|
+
requested.has(path.basename(filePath, ".md"))
|
|
58
|
+
);
|
|
59
|
+
const contents = new Map<string, readonly string[]>();
|
|
60
|
+
for (const memory of await deps.readParsedMemoriesFromPaths(directPaths, 50)) {
|
|
61
|
+
const id = memory.frontmatter.id;
|
|
62
|
+
if (requested.has(id) && !contents.has(id)) {
|
|
63
|
+
contents.set(id, deps.storedContentIdentityCandidates(memory.content));
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
// Re-check after the direct-path read too: a concurrent drop during
|
|
67
|
+
// that await must not let this call publish a fill built from the
|
|
68
|
+
// stale snapshot into the freshly cleared slot (issue #2367 round 3).
|
|
69
|
+
if (ledgerMtimeMs !== revision) continue;
|
|
70
|
+
if (contents.size === requested.size) return contents;
|
|
71
|
+
const fill = (sourceContentsByIdPromise ??= (async () => {
|
|
72
|
+
const allContents = new Map<string, readonly string[]>();
|
|
73
|
+
for (const memory of await deps.readParsedMemoriesFromPaths(sourcePaths, 50)) {
|
|
74
|
+
const id = memory.frontmatter.id;
|
|
75
|
+
if (!allContents.has(id)) {
|
|
76
|
+
allContents.set(id, deps.storedContentIdentityCandidates(memory.content));
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
return allContents;
|
|
80
|
+
})());
|
|
81
|
+
const allContents = await fill;
|
|
82
|
+
if (ledgerMtimeMs !== revision) continue;
|
|
83
|
+
for (const id of requested) {
|
|
84
|
+
const content = allContents.get(id);
|
|
85
|
+
if (content !== undefined) contents.set(id, content);
|
|
86
|
+
}
|
|
87
|
+
return contents;
|
|
88
|
+
}
|
|
89
|
+
};
|
|
90
|
+
}
|
package/src/storage.ts
CHANGED
|
@@ -26,6 +26,7 @@ import { resolveSafeStoragePath } from "./storage-paths.js";
|
|
|
26
26
|
import { log } from "./logger.js";
|
|
27
27
|
import { createMemorySnapshot } from "./memory-snapshot.js";
|
|
28
28
|
import { assertMemoryFrontmatterId, warnProjectionFallback } from "./storage-guards.js";
|
|
29
|
+
import { createTombstoneMigrationSourceContents } from "./storage/tombstone-migration-sources.js";
|
|
29
30
|
import { MemoryReadStore, readWindowedMemories, type WindowedMemoryReadOptions, type WindowedMemoryReadResult } from "./storage/memory-read-store.js";
|
|
30
31
|
import { hasSupersessionAudit } from "./storage/supersession-audit.js";
|
|
31
32
|
import { runCommittedInvalidation } from "./storage/committed-invalidation.js";
|
|
@@ -3016,46 +3017,24 @@ export class StorageManager extends TombstoneBlockedCaptureIndexHost {
|
|
|
3016
3017
|
}
|
|
3017
3018
|
|
|
3018
3019
|
private buildTombstoneStore(): TombstoneStore {
|
|
3019
|
-
let sourcePathsPromise: Promise<string[]> | undefined;
|
|
3020
|
-
let sourceContentsByIdPromise: Promise<Map<string, readonly string[]>> | undefined;
|
|
3021
3020
|
const options: TombstoneStoreOptions = {
|
|
3022
3021
|
enabled: this.tombstonesConfig.enabled,
|
|
3023
3022
|
semanticMatch: this.tombstonesConfig.semanticMatch,
|
|
3024
3023
|
semanticThreshold: this.tombstonesConfig.semanticThreshold,
|
|
3025
3024
|
hashContent: ContentHashIndex.computeHash,
|
|
3026
3025
|
normalizeText: ContentHashIndex.normalizeContent,
|
|
3027
|
-
|
|
3028
|
-
|
|
3029
|
-
|
|
3030
|
-
|
|
3031
|
-
|
|
3032
|
-
|
|
3033
|
-
|
|
3034
|
-
|
|
3035
|
-
|
|
3036
|
-
|
|
3037
|
-
|
|
3038
|
-
|
|
3039
|
-
}
|
|
3040
|
-
}
|
|
3041
|
-
if (contents.size === requested.size) return contents;
|
|
3042
|
-
sourceContentsByIdPromise ??= (async () => {
|
|
3043
|
-
const allContents = new Map<string, readonly string[]>();
|
|
3044
|
-
for (const memory of await this.readParsedMemoriesFromPaths(sourcePaths, 50)) {
|
|
3045
|
-
const id = memory.frontmatter.id;
|
|
3046
|
-
if (!allContents.has(id)) {
|
|
3047
|
-
allContents.set(id, this.storedContentIdentityCandidates(memory.content));
|
|
3048
|
-
}
|
|
3049
|
-
}
|
|
3050
|
-
return allContents;
|
|
3051
|
-
})();
|
|
3052
|
-
const allContents = await sourceContentsByIdPromise;
|
|
3053
|
-
for (const id of requested) {
|
|
3054
|
-
const content = allContents.get(id);
|
|
3055
|
-
if (content !== undefined) contents.set(id, content);
|
|
3056
|
-
}
|
|
3057
|
-
return contents;
|
|
3058
|
-
},
|
|
3026
|
+
// Corpus-path + source-content caches are scoped to one ledger revision
|
|
3027
|
+
// so a peer-appended legacy row is verified on the next staleness reload
|
|
3028
|
+
// (issue #2367). Implementation lives in storage/tombstone-migration-sources.ts.
|
|
3029
|
+
sourceContentsForMemoryIds: createTombstoneMigrationSourceContents({
|
|
3030
|
+
tombstonesPath: () => this.tombstonesPath,
|
|
3031
|
+
collectTombstoneMigrationPaths: () =>
|
|
3032
|
+
this.memoryReadStore.collectTombstoneMigrationPaths(),
|
|
3033
|
+
readParsedMemoriesFromPaths: (filePaths, batchSize) =>
|
|
3034
|
+
this.readParsedMemoriesFromPaths(filePaths, batchSize),
|
|
3035
|
+
storedContentIdentityCandidates: (content) =>
|
|
3036
|
+
this.storedContentIdentityCandidates(content),
|
|
3037
|
+
}),
|
|
3059
3038
|
};
|
|
3060
3039
|
const io: TombstoneFileIo = {
|
|
3061
3040
|
read: (filePath) => this.readStorageSecureFile(filePath),
|
package/src/types.ts
CHANGED
|
@@ -631,7 +631,6 @@ export interface DependencyPropagationConfig {
|
|
|
631
631
|
dryRun: boolean;
|
|
632
632
|
}
|
|
633
633
|
|
|
634
|
-
|
|
635
634
|
export interface HeartbeatConfig {
|
|
636
635
|
enabled: boolean;
|
|
637
636
|
journalPath: string;
|
|
@@ -1408,6 +1407,7 @@ export interface PluginConfig extends BoundedJsonlStateConfig, SecurityConfig {
|
|
|
1408
1407
|
harmonicRetrievalEnabled: boolean;
|
|
1409
1408
|
abstractionAnchorsEnabled: boolean;
|
|
1410
1409
|
abstractionNodeStoreDir: string;
|
|
1410
|
+
episodicContextEnabled: boolean;
|
|
1411
1411
|
// Episodic/semantic split foundation
|
|
1412
1412
|
verifiedRecallEnabled: boolean;
|
|
1413
1413
|
semanticRulePromotionEnabled: boolean;
|
|
@@ -1870,6 +1870,8 @@ export interface PluginConfig extends BoundedJsonlStateConfig, SecurityConfig {
|
|
|
1870
1870
|
oramaEnabled: boolean;
|
|
1871
1871
|
oramaDbPath?: string;
|
|
1872
1872
|
oramaEmbeddingDimension?: number;
|
|
1873
|
+
/** Segment space-free scripts (CJK/Thai) in the Orama lexical index (issue #2187). Default `true`. */
|
|
1874
|
+
oramaCjkSegmentation?: boolean;
|
|
1873
1875
|
|
|
1874
1876
|
// QMD daemon mode
|
|
1875
1877
|
qmdDaemonEnabled: boolean;
|
|
@@ -2908,12 +2910,10 @@ export interface MemoryFrontmatter extends SourceConnectorProvenance, OriginMeta
|
|
|
2908
2910
|
/** Last time this memory was accessed (ISO 8601) */
|
|
2909
2911
|
lastAccessed?: string;
|
|
2910
2912
|
// Memory Worth counters (issue #560)
|
|
2911
|
-
//
|
|
2912
2913
|
// Per-fact outcome counters used to derive a dynamic utility score —
|
|
2913
2914
|
// `p(success | retrieved)` — as a complement to the static `importance`
|
|
2914
2915
|
// field. Absent on legacy memories written before #560; readers must treat
|
|
2915
2916
|
// `undefined` as zero observations (uniform Beta(1,1) prior).
|
|
2916
|
-
//
|
|
2917
2917
|
// Both values must be non-negative integers on write. PR 1 wires only the
|
|
2918
2918
|
// schema + storage round-trip — no increments, scoring, or filtering yet.
|
|
2919
2919
|
/** Number of sessions where this memory was retrieved and the outcome was judged a success. */
|