@remnic/core 1.0.2 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +1 -0
- package/dist/abort-error.d.ts +32 -0
- package/dist/abort-error.js +11 -0
- package/dist/access-cli.d.ts +13 -3
- package/dist/access-cli.js +96 -80
- package/dist/access-cli.js.map +1 -1
- package/dist/access-http.d.ts +12 -4
- package/dist/access-http.js +25 -18
- package/dist/access-mcp.d.ts +32 -4
- package/dist/access-mcp.js +16 -1
- package/dist/access-schema.d.ts +28 -28
- package/dist/access-schema.js +1 -1
- package/dist/access-service-HmO1Trrx.d.ts +732 -0
- package/dist/access-service.d.ts +15 -601
- package/dist/access-service.js +21 -15
- package/dist/active-memory-bridge.d.ts +66 -0
- package/dist/active-memory-bridge.js +11 -0
- package/dist/active-memory-bridge.js.map +1 -0
- package/dist/active-recall.d.ts +96 -0
- package/dist/active-recall.js +308 -0
- package/dist/active-recall.js.map +1 -0
- package/dist/behavior-learner.js +1 -1
- package/dist/bootstrap.d.ts +6 -3
- package/dist/bootstrap.js +2 -2
- package/dist/boxes.js +2 -2
- package/dist/briefing.d.ts +169 -0
- package/dist/briefing.js +52 -0
- package/dist/briefing.js.map +1 -0
- package/dist/buffer.d.ts +19 -5
- package/dist/buffer.js +2 -2
- package/dist/calibration.js +6 -6
- package/dist/causal-behavior.js +5 -5
- package/dist/causal-chain.js +3 -3
- package/dist/causal-consolidation.d.ts +22 -2
- package/dist/causal-consolidation.js +36 -9
- package/dist/causal-consolidation.js.map +1 -1
- package/dist/causal-retrieval.js +6 -6
- package/dist/causal-trajectory-graph.js +1 -1
- package/dist/causal-trajectory.d.ts +14 -1
- package/dist/causal-trajectory.js +5 -1
- package/dist/{chunk-KWBU5S5U.js → chunk-2ODBA7MQ.js} +9 -3
- package/dist/chunk-2ODBA7MQ.js.map +1 -0
- package/dist/{chunk-ZJLY4QSU.js → chunk-37UIFYWO.js} +130 -6
- package/dist/chunk-37UIFYWO.js.map +1 -0
- package/dist/chunk-3PG3H5TD.js +7 -0
- package/dist/chunk-3PG3H5TD.js.map +1 -0
- package/dist/{chunk-NTTLPF7F.js → chunk-3QFQGRHO.js} +5 -5
- package/dist/{chunk-QDOSNLB4.js → chunk-3QHL5ABG.js} +17 -15
- package/dist/chunk-3QHL5ABG.js.map +1 -0
- package/dist/{chunk-6UJQNRIO.js → chunk-3SV6CQHO.js} +92 -33
- package/dist/chunk-3SV6CQHO.js.map +1 -0
- package/dist/{chunk-U4PV25RD.js → chunk-3WHVNEN7.js} +1 -1
- package/dist/chunk-3WHVNEN7.js.map +1 -0
- package/dist/{chunk-XUHI52HK.js → chunk-44ICJRF3.js} +98 -10
- package/dist/chunk-44ICJRF3.js.map +1 -0
- package/dist/{chunk-HG2NKWR2.js → chunk-47UU5PU2.js} +49 -10
- package/dist/chunk-47UU5PU2.js.map +1 -0
- package/dist/chunk-4DJQYKMN.js +187 -0
- package/dist/chunk-4DJQYKMN.js.map +1 -0
- package/dist/chunk-4KAN3GZ3.js +225 -0
- package/dist/chunk-4KAN3GZ3.js.map +1 -0
- package/dist/chunk-4LACOVZX.js +813 -0
- package/dist/chunk-4LACOVZX.js.map +1 -0
- package/dist/{chunk-ORZMT74A.js → chunk-4NRAJUDS.js} +11 -1
- package/dist/chunk-4NRAJUDS.js.map +1 -0
- package/dist/{chunk-B7LOFDVE.js → chunk-4WMCPJWX.js} +8 -3
- package/dist/chunk-4WMCPJWX.js.map +1 -0
- package/dist/{chunk-G3AG3KZN.js → chunk-5IZL4DCV.js} +2 -2
- package/dist/{chunk-BRK4ODMI.js → chunk-5NPGSAVB.js} +2 -2
- package/dist/{chunk-QANCTXQF.js → chunk-6LX5ORAS.js} +3 -3
- package/dist/chunk-6MKAMLQL.js +16 -0
- package/dist/chunk-6MKAMLQL.js.map +1 -0
- package/dist/{chunk-ESSMF2FR.js → chunk-6PFRXT4K.js} +15 -6
- package/dist/chunk-6PFRXT4K.js.map +1 -0
- package/dist/{chunk-UIYZ5T3I.js → chunk-6UJ47TVX.js} +8 -8
- package/dist/chunk-6ZH4TU6I.js +245 -0
- package/dist/chunk-6ZH4TU6I.js.map +1 -0
- package/dist/{chunk-L5RPWGFK.js → chunk-7DHTMOND.js} +2 -2
- package/dist/{chunk-L7WO3MZ4.js → chunk-7ECD5ATE.js} +2 -2
- package/dist/{chunk-Q6FETXJA.js → chunk-7SEAZFFB.js} +2 -2
- package/dist/{chunk-V4YC4LUK.js → chunk-7WQ6SLIE.js} +175 -63
- package/dist/chunk-7WQ6SLIE.js.map +1 -0
- package/dist/chunk-ALXMCZEU.js +332 -0
- package/dist/chunk-ALXMCZEU.js.map +1 -0
- package/dist/{chunk-TVVVQQAK.js → chunk-BLKTA7MM.js} +58 -24
- package/dist/chunk-BLKTA7MM.js.map +1 -0
- package/dist/{chunk-SCHEKPYH.js → chunk-C2EFFULQ.js} +1 -1
- package/dist/{chunk-GJR6D6KC.js → chunk-D654IBA6.js} +2 -2
- package/dist/{chunk-OTFNI3OO.js → chunk-DEPL3635.js} +1828 -401
- package/dist/chunk-DEPL3635.js.map +1 -0
- package/dist/{chunk-UYSKNO6E.js → chunk-DHHP2Z4X.js} +15 -4
- package/dist/chunk-DHHP2Z4X.js.map +1 -0
- package/dist/{chunk-UV2FO7J4.js → chunk-E6K4NIEU.js} +2 -2
- package/dist/{chunk-T4WRIV2C.js → chunk-EABGC2TL.js} +2 -2
- package/dist/chunk-EJI5XIBB.js +232 -0
- package/dist/chunk-EJI5XIBB.js.map +1 -0
- package/dist/{chunk-ONRU4L2N.js → chunk-FEMOX5AD.js} +2 -2
- package/dist/{chunk-IFFFR3MR.js → chunk-FSFEQI74.js} +3 -3
- package/dist/chunk-G4SK7DSQ.js +121 -0
- package/dist/chunk-G4SK7DSQ.js.map +1 -0
- package/dist/{chunk-WWIQTB2Y.js → chunk-GGD5W7TB.js} +9 -2
- package/dist/chunk-GGD5W7TB.js.map +1 -0
- package/dist/{chunk-QWUUMMIK.js → chunk-GV6NLQ4X.js} +1355 -80
- package/dist/chunk-GV6NLQ4X.js.map +1 -0
- package/dist/{chunk-2PO5ZRKV.js → chunk-GZCUW5IC.js} +16 -3
- package/dist/chunk-GZCUW5IC.js.map +1 -0
- package/dist/{chunk-AAI7JARD.js → chunk-HMDCOMYU.js} +8 -11
- package/dist/chunk-HMDCOMYU.js.map +1 -0
- package/dist/chunk-IQT3XTKW.js +121 -0
- package/dist/chunk-IQT3XTKW.js.map +1 -0
- package/dist/{chunk-J3BT33K7.js → chunk-ITRLGI2T.js} +5 -5
- package/dist/{chunk-BDFZXRSO.js → chunk-J4IYOZZ5.js} +15 -2
- package/dist/chunk-J4IYOZZ5.js.map +1 -0
- package/dist/{chunk-J47FNDR7.js → chunk-JIU55F3X.js} +7 -7
- package/dist/{chunk-MDDAA2AO.js → chunk-JL2PU6AI.js} +17 -6
- package/dist/chunk-JL2PU6AI.js.map +1 -0
- package/dist/{chunk-ZKYI7UVO.js → chunk-JR4ZC3G4.js} +2 -2
- package/dist/{chunk-UCYSTFZR.js → chunk-JRNQ3RNA.js} +2 -2
- package/dist/{chunk-GPGBSNKM.js → chunk-K4FLSOR5.js} +2 -2
- package/dist/chunk-KVE7R4CG.js +320 -0
- package/dist/chunk-KVE7R4CG.js.map +1 -0
- package/dist/chunk-LAYN4LDC.js +267 -0
- package/dist/chunk-LAYN4LDC.js.map +1 -0
- package/dist/{chunk-ISY75RLM.js → chunk-MBJHSA7F.js} +344 -7
- package/dist/chunk-MBJHSA7F.js.map +1 -0
- package/dist/{chunk-PGK3VUHN.js → chunk-MTLYEMJB.js} +3 -2
- package/dist/chunk-MTLYEMJB.js.map +1 -0
- package/dist/{chunk-QY2BHY5O.js → chunk-MVTHXUBX.js} +297 -34
- package/dist/chunk-MVTHXUBX.js.map +1 -0
- package/dist/{chunk-LP47L3ZX.js → chunk-N42IWANG.js} +5 -5
- package/dist/{chunk-YNI4S5WT.js → chunk-N53K2EXC.js} +2 -2
- package/dist/{chunk-763GUIOU.js → chunk-NBNN5GOB.js} +2 -2
- package/dist/{chunk-CXWFUJR2.js → chunk-NQEVYWX6.js} +195 -5
- package/dist/chunk-NQEVYWX6.js.map +1 -0
- package/dist/{chunk-KL4CP4SB.js → chunk-O5ETUNBT.js} +17 -5
- package/dist/chunk-O5ETUNBT.js.map +1 -0
- package/dist/{chunk-OOSWAUYB.js → chunk-ODWDQNRE.js} +2 -2
- package/dist/chunk-OIT5QGG4.js +80 -0
- package/dist/chunk-OIT5QGG4.js.map +1 -0
- package/dist/{chunk-HLBYLYRD.js → chunk-PAORGQRI.js} +70 -13
- package/dist/chunk-PAORGQRI.js.map +1 -0
- package/dist/chunk-PVGDJXVK.js +21 -0
- package/dist/chunk-PVGDJXVK.js.map +1 -0
- package/dist/{chunk-OTAVQCSF.js → chunk-PYXS46O7.js} +2 -2
- package/dist/chunk-QDW3E4RD.js +108 -0
- package/dist/chunk-QDW3E4RD.js.map +1 -0
- package/dist/{chunk-YNCQ7E4M.js → chunk-QDYXG4CS.js} +4 -3
- package/dist/chunk-QDYXG4CS.js.map +1 -0
- package/dist/{chunk-HLXVTBF3.js → chunk-QNJMBKFK.js} +3 -2
- package/dist/chunk-QNJMBKFK.js.map +1 -0
- package/dist/{chunk-4A24LIM2.js → chunk-S75M5ZRK.js} +2 -2
- package/dist/chunk-SYUK3VLY.js +789 -0
- package/dist/chunk-SYUK3VLY.js.map +1 -0
- package/dist/{chunk-QCCCQT3O.js → chunk-TBBDFYXW.js} +2 -2
- package/dist/chunk-TBBDFYXW.js.map +1 -0
- package/dist/chunk-U66YHYC7.js +31 -0
- package/dist/chunk-U66YHYC7.js.map +1 -0
- package/dist/{chunk-MWGVGUIS.js → chunk-UEYA6UC7.js} +36 -4
- package/dist/chunk-UEYA6UC7.js.map +1 -0
- package/dist/{chunk-M5KEYE5E.js → chunk-URB2WSKZ.js} +2 -2
- package/dist/chunk-UVJFDP7P.js +202 -0
- package/dist/chunk-UVJFDP7P.js.map +1 -0
- package/dist/chunk-W6SL7OFG.js +180 -0
- package/dist/chunk-W6SL7OFG.js.map +1 -0
- package/dist/chunk-WBSAYXVI.js +7945 -0
- package/dist/chunk-WBSAYXVI.js.map +1 -0
- package/dist/{chunk-M5ZBBBJI.js → chunk-XZ2TIKGC.js} +39 -9
- package/dist/chunk-XZ2TIKGC.js.map +1 -0
- package/dist/chunk-Y4FHOFJ2.js +140 -0
- package/dist/chunk-Y4FHOFJ2.js.map +1 -0
- package/dist/chunk-YDBIWGNI.js +298 -0
- package/dist/chunk-YDBIWGNI.js.map +1 -0
- package/dist/chunk-YNB73F22.js +137 -0
- package/dist/chunk-YNB73F22.js.map +1 -0
- package/dist/{chunk-IZME7KW2.js → chunk-ZVBB3T7V.js} +31 -12
- package/dist/chunk-ZVBB3T7V.js.map +1 -0
- package/dist/chunking.js +1 -1
- package/dist/citations.d.ts +67 -0
- package/dist/citations.js +13 -0
- package/dist/citations.js.map +1 -0
- package/dist/cli-BneVIEvh.d.ts +1240 -0
- package/dist/cli.d.ts +32 -1147
- package/dist/cli.js +150 -7092
- package/dist/cli.js.map +1 -1
- package/dist/codex-materialize-CQlLTzke.d.ts +139 -0
- package/dist/codex-thread-key.d.ts +3 -0
- package/dist/codex-thread-key.js +7 -0
- package/dist/codex-thread-key.js.map +1 -0
- package/dist/config.js +3 -2
- package/dist/connectors/codex/instructions.md +160 -0
- package/dist/connectors/codex/resources/namespace-cheatsheet.md +48 -0
- package/dist/contradiction-review-WIUBAR52.js +21 -0
- package/dist/contradiction-review-WIUBAR52.js.map +1 -0
- package/dist/contradiction-scan-GR33PONM.js +376 -0
- package/dist/contradiction-scan-GR33PONM.js.map +1 -0
- package/dist/day-summary.d.ts +7 -2
- package/dist/day-summary.js +5 -2
- package/dist/direct-answer-wiring.d.ts +77 -0
- package/dist/direct-answer-wiring.js +75 -0
- package/dist/direct-answer-wiring.js.map +1 -0
- package/dist/direct-answer.d.ts +106 -0
- package/dist/direct-answer.js +10 -0
- package/dist/direct-answer.js.map +1 -0
- package/dist/embedding-fallback.d.ts +96 -2
- package/dist/embedding-fallback.js +6 -4
- package/dist/{engine-2A6J4XEX.js → engine-5TIQBYZR.js} +10 -7
- package/dist/engine-5TIQBYZR.js.map +1 -0
- package/dist/entity-retrieval.d.ts +3 -2
- package/dist/entity-retrieval.js +10 -7
- package/dist/entity-schema.d.ts +11 -0
- package/dist/entity-schema.js +19 -0
- package/dist/entity-schema.js.map +1 -0
- package/dist/explicit-capture.d.ts +6 -3
- package/dist/explicit-capture.js +2 -2
- package/dist/extraction-judge.d.ts +66 -0
- package/dist/extraction-judge.js +18 -0
- package/dist/extraction-judge.js.map +1 -0
- package/dist/extraction.d.ts +1 -0
- package/dist/extraction.js +12 -10
- package/dist/fallback-llm.d.ts +11 -2
- package/dist/fallback-llm.js +4 -4
- package/dist/graph.js +1 -1
- package/dist/harmonic-retrieval.js +2 -1
- package/dist/importance.d.ts +11 -1
- package/dist/importance.js +3 -1
- package/dist/index.d.ts +1027 -9
- package/dist/index.js +3303 -349
- package/dist/index.js.map +1 -1
- package/dist/intent.d.ts +2 -1
- package/dist/intent.js +3 -1
- package/dist/lifecycle.js +1 -1
- package/dist/local-llm.d.ts +10 -3
- package/dist/local-llm.js +2 -2
- package/dist/logger.d.ts +1 -1
- package/dist/logger.js +1 -1
- package/dist/memory-cache.d.ts +2 -2
- package/dist/memory-cache.js +1 -1
- package/dist/{memory-projection-store-NxMkbocT.d.ts → memory-projection-store-DeSXPh1j.d.ts} +1 -1
- package/dist/memory-projection-store.d.ts +1 -1
- package/dist/model-registry.js +2 -2
- package/dist/models-json.js +2 -2
- package/dist/native-knowledge.js +2 -2
- package/dist/negative.js +2 -2
- package/dist/operator-toolkit.js +20 -15
- package/dist/{orchestrator-zTa-Qo-1.d.ts → orchestrator-DRYA6_lW.d.ts} +273 -9
- package/dist/orchestrator.d.ts +6 -3
- package/dist/orchestrator.js +76 -63
- package/dist/page-versioning.d.ts +77 -0
- package/dist/page-versioning.js +15 -0
- package/dist/page-versioning.js.map +1 -0
- package/dist/plugin-id.d.ts +37 -0
- package/dist/plugin-id.js +11 -0
- package/dist/plugin-id.js.map +1 -0
- package/dist/policy-runtime.js +2 -2
- package/dist/profiling.js +2 -2
- package/dist/qmd.d.ts +5 -2
- package/dist/qmd.js +4 -3
- package/dist/recall-audit.d.ts +20 -0
- package/dist/recall-audit.js +50 -0
- package/dist/recall-audit.js.map +1 -0
- package/dist/recall-mmr.d.ts +152 -0
- package/dist/recall-mmr.js +17 -0
- package/dist/recall-mmr.js.map +1 -0
- package/dist/recall-qos.js +2 -2
- package/dist/recall-state.d.ts +28 -1
- package/dist/recall-state.js +2 -2
- package/dist/relevance.js +2 -2
- package/dist/resolution-QBTDHTG7.js +100 -0
- package/dist/resolution-QBTDHTG7.js.map +1 -0
- package/dist/resolve-provider-secret.d.ts +24 -1
- package/dist/resolve-provider-secret.js +4 -2
- package/dist/resume-bundles.js +6 -5
- package/dist/retrieval-agents.js +2 -2
- package/dist/retrieval.js +2 -2
- package/dist/schemas.d.ts +412 -54
- package/dist/schemas.js +3 -1
- package/dist/sdk-compat.d.ts +2 -0
- package/dist/sdk-compat.js +6 -3
- package/dist/sdk-compat.js.map +1 -1
- package/dist/semantic-chunking.d.ts +87 -0
- package/dist/semantic-chunking.js +20 -0
- package/dist/semantic-chunking.js.map +1 -0
- package/dist/semantic-consolidation-DrvSYRdB.d.ts +119 -0
- package/dist/semantic-consolidation.d.ts +4 -42
- package/dist/semantic-consolidation.js +23 -2
- package/dist/semantic-rule-promotion.js +9 -6
- package/dist/semantic-rule-verifier.js +10 -7
- package/dist/session-observer-state.js +2 -2
- package/dist/session-toggles.d.ts +22 -0
- package/dist/session-toggles.js +116 -0
- package/dist/session-toggles.js.map +1 -0
- package/dist/skills-registry.d.ts +47 -0
- package/dist/skills-registry.js +48 -0
- package/dist/skills-registry.js.map +1 -0
- package/dist/source-attribution.d.ts +169 -0
- package/dist/source-attribution.js +27 -0
- package/dist/source-attribution.js.map +1 -0
- package/dist/storage.d.ts +171 -10
- package/dist/storage.js +16 -5
- package/dist/summarizer.js +7 -7
- package/dist/temporal-supersession.d.ts +127 -0
- package/dist/temporal-supersession.js +20 -0
- package/dist/temporal-supersession.js.map +1 -0
- package/dist/threading.js +2 -2
- package/dist/tier-migration.d.ts +2 -1
- package/dist/tier-routing.js +2 -2
- package/dist/tokens.d.ts +21 -1
- package/dist/tokens.js +5 -1
- package/dist/transcript.js +2 -2
- package/dist/types-DJhqDJUV.d.ts +50 -0
- package/dist/types.d.ts +529 -3
- package/dist/types.js +1 -1
- package/dist/utility-learner.js +2 -2
- package/dist/utility-runtime.js +3 -3
- package/dist/verified-recall.js +11 -8
- package/dist/whitespace.d.ts +4 -0
- package/dist/whitespace.js +9 -0
- package/dist/whitespace.js.map +1 -0
- package/package.json +14 -8
- package/dist/chunk-2CJCWDMR.js +0 -87
- package/dist/chunk-2CJCWDMR.js.map +0 -1
- package/dist/chunk-2PO5ZRKV.js.map +0 -1
- package/dist/chunk-6UJQNRIO.js.map +0 -1
- package/dist/chunk-AAI7JARD.js.map +0 -1
- package/dist/chunk-B7LOFDVE.js.map +0 -1
- package/dist/chunk-BDFZXRSO.js.map +0 -1
- package/dist/chunk-CXWFUJR2.js.map +0 -1
- package/dist/chunk-DORBM6OB.js +0 -81
- package/dist/chunk-DORBM6OB.js.map +0 -1
- package/dist/chunk-ESSMF2FR.js.map +0 -1
- package/dist/chunk-HG2NKWR2.js.map +0 -1
- package/dist/chunk-HLBYLYRD.js.map +0 -1
- package/dist/chunk-HLXVTBF3.js.map +0 -1
- package/dist/chunk-ISY75RLM.js.map +0 -1
- package/dist/chunk-IZME7KW2.js.map +0 -1
- package/dist/chunk-KL4CP4SB.js.map +0 -1
- package/dist/chunk-KWBU5S5U.js.map +0 -1
- package/dist/chunk-M5ZBBBJI.js.map +0 -1
- package/dist/chunk-MDDAA2AO.js.map +0 -1
- package/dist/chunk-MWGVGUIS.js.map +0 -1
- package/dist/chunk-ORZMT74A.js.map +0 -1
- package/dist/chunk-OTFNI3OO.js.map +0 -1
- package/dist/chunk-PGK3VUHN.js.map +0 -1
- package/dist/chunk-QCCCQT3O.js.map +0 -1
- package/dist/chunk-QDOSNLB4.js.map +0 -1
- package/dist/chunk-QPKFPHOO.js +0 -178
- package/dist/chunk-QPKFPHOO.js.map +0 -1
- package/dist/chunk-QWUUMMIK.js.map +0 -1
- package/dist/chunk-QY2BHY5O.js.map +0 -1
- package/dist/chunk-TVVVQQAK.js.map +0 -1
- package/dist/chunk-U4PV25RD.js.map +0 -1
- package/dist/chunk-UYSKNO6E.js.map +0 -1
- package/dist/chunk-V4YC4LUK.js.map +0 -1
- package/dist/chunk-WWIQTB2Y.js.map +0 -1
- package/dist/chunk-XUHI52HK.js.map +0 -1
- package/dist/chunk-YNCQ7E4M.js.map +0 -1
- package/dist/chunk-ZJLY4QSU.js.map +0 -1
- /package/dist/{engine-2A6J4XEX.js.map → abort-error.js.map} +0 -0
- /package/dist/{chunk-NTTLPF7F.js.map → chunk-3QFQGRHO.js.map} +0 -0
- /package/dist/{chunk-G3AG3KZN.js.map → chunk-5IZL4DCV.js.map} +0 -0
- /package/dist/{chunk-BRK4ODMI.js.map → chunk-5NPGSAVB.js.map} +0 -0
- /package/dist/{chunk-QANCTXQF.js.map → chunk-6LX5ORAS.js.map} +0 -0
- /package/dist/{chunk-UIYZ5T3I.js.map → chunk-6UJ47TVX.js.map} +0 -0
- /package/dist/{chunk-L5RPWGFK.js.map → chunk-7DHTMOND.js.map} +0 -0
- /package/dist/{chunk-L7WO3MZ4.js.map → chunk-7ECD5ATE.js.map} +0 -0
- /package/dist/{chunk-Q6FETXJA.js.map → chunk-7SEAZFFB.js.map} +0 -0
- /package/dist/{chunk-SCHEKPYH.js.map → chunk-C2EFFULQ.js.map} +0 -0
- /package/dist/{chunk-GJR6D6KC.js.map → chunk-D654IBA6.js.map} +0 -0
- /package/dist/{chunk-UV2FO7J4.js.map → chunk-E6K4NIEU.js.map} +0 -0
- /package/dist/{chunk-T4WRIV2C.js.map → chunk-EABGC2TL.js.map} +0 -0
- /package/dist/{chunk-ONRU4L2N.js.map → chunk-FEMOX5AD.js.map} +0 -0
- /package/dist/{chunk-IFFFR3MR.js.map → chunk-FSFEQI74.js.map} +0 -0
- /package/dist/{chunk-J3BT33K7.js.map → chunk-ITRLGI2T.js.map} +0 -0
- /package/dist/{chunk-J47FNDR7.js.map → chunk-JIU55F3X.js.map} +0 -0
- /package/dist/{chunk-ZKYI7UVO.js.map → chunk-JR4ZC3G4.js.map} +0 -0
- /package/dist/{chunk-UCYSTFZR.js.map → chunk-JRNQ3RNA.js.map} +0 -0
- /package/dist/{chunk-GPGBSNKM.js.map → chunk-K4FLSOR5.js.map} +0 -0
- /package/dist/{chunk-LP47L3ZX.js.map → chunk-N42IWANG.js.map} +0 -0
- /package/dist/{chunk-YNI4S5WT.js.map → chunk-N53K2EXC.js.map} +0 -0
- /package/dist/{chunk-763GUIOU.js.map → chunk-NBNN5GOB.js.map} +0 -0
- /package/dist/{chunk-OOSWAUYB.js.map → chunk-ODWDQNRE.js.map} +0 -0
- /package/dist/{chunk-OTAVQCSF.js.map → chunk-PYXS46O7.js.map} +0 -0
- /package/dist/{chunk-4A24LIM2.js.map → chunk-S75M5ZRK.js.map} +0 -0
- /package/dist/{chunk-M5KEYE5E.js.map → chunk-URB2WSKZ.js.map} +0 -0
|
@@ -2,36 +2,54 @@ import {
|
|
|
2
2
|
CompoundingEngine,
|
|
3
3
|
SharedContextManager,
|
|
4
4
|
defaultTierMigrationCycleBudget
|
|
5
|
-
} from "./chunk-
|
|
5
|
+
} from "./chunk-DHHP2Z4X.js";
|
|
6
|
+
import {
|
|
7
|
+
extractTopics
|
|
8
|
+
} from "./chunk-UHGBNIOS.js";
|
|
6
9
|
import {
|
|
7
10
|
applyUtilityPromotionRuntimePolicy,
|
|
8
11
|
applyUtilityRankingRuntimeDelta,
|
|
9
12
|
loadUtilityRuntimeValues
|
|
10
|
-
} from "./chunk-
|
|
13
|
+
} from "./chunk-FSFEQI74.js";
|
|
14
|
+
import {
|
|
15
|
+
HourlySummarizer
|
|
16
|
+
} from "./chunk-N42IWANG.js";
|
|
17
|
+
import {
|
|
18
|
+
applyTemporalSupersession,
|
|
19
|
+
normalizeSupersessionKey,
|
|
20
|
+
shouldFilterSupersededFromRecall
|
|
21
|
+
} from "./chunk-W6SL7OFG.js";
|
|
11
22
|
import {
|
|
12
23
|
TierMigrationExecutor
|
|
13
24
|
} from "./chunk-Z5AAYHUC.js";
|
|
14
25
|
import {
|
|
15
26
|
decideTierTransition
|
|
16
|
-
} from "./chunk-
|
|
27
|
+
} from "./chunk-S75M5ZRK.js";
|
|
17
28
|
import {
|
|
18
29
|
TmtBuilder
|
|
19
30
|
} from "./chunk-TPB3I2AC.js";
|
|
20
|
-
import {
|
|
21
|
-
extractTopics
|
|
22
|
-
} from "./chunk-UHGBNIOS.js";
|
|
23
31
|
import {
|
|
24
32
|
SessionObserverState
|
|
25
|
-
} from "./chunk-
|
|
33
|
+
} from "./chunk-JR4ZC3G4.js";
|
|
26
34
|
import {
|
|
27
|
-
|
|
28
|
-
} from "./chunk-
|
|
35
|
+
semanticChunkContent
|
|
36
|
+
} from "./chunk-KVE7R4CG.js";
|
|
37
|
+
import {
|
|
38
|
+
findUnresolvedEntityRefs
|
|
39
|
+
} from "./chunk-X7XN6YU4.js";
|
|
40
|
+
import {
|
|
41
|
+
RelevanceStore
|
|
42
|
+
} from "./chunk-5NPGSAVB.js";
|
|
43
|
+
import {
|
|
44
|
+
RerankCache,
|
|
45
|
+
rerankLocalOrNoop
|
|
46
|
+
} from "./chunk-C7VW7C3F.js";
|
|
29
47
|
import {
|
|
30
48
|
mergeWithAgentResults,
|
|
31
49
|
runDirectAgent,
|
|
32
50
|
runTemporalAgent,
|
|
33
51
|
shouldRunAgent
|
|
34
|
-
} from "./chunk-
|
|
52
|
+
} from "./chunk-K4FLSOR5.js";
|
|
35
53
|
import {
|
|
36
54
|
clearIndexes,
|
|
37
55
|
deindexMemory,
|
|
@@ -46,101 +64,89 @@ import {
|
|
|
46
64
|
} from "./chunk-V3RXWQIE.js";
|
|
47
65
|
import {
|
|
48
66
|
applyRuntimeRetrievalPolicy
|
|
49
|
-
} from "./chunk-
|
|
50
|
-
import {
|
|
51
|
-
buildConsolidationPrompt,
|
|
52
|
-
findSimilarClusters,
|
|
53
|
-
parseConsolidationResponse
|
|
54
|
-
} from "./chunk-2CJCWDMR.js";
|
|
55
|
-
import {
|
|
56
|
-
LastRecallStore,
|
|
57
|
-
TierMigrationStatusStore,
|
|
58
|
-
clampGraphRecallExpandedEntries
|
|
59
|
-
} from "./chunk-HG2NKWR2.js";
|
|
60
|
-
import {
|
|
61
|
-
findUnresolvedEntityRefs
|
|
62
|
-
} from "./chunk-X7XN6YU4.js";
|
|
63
|
-
import {
|
|
64
|
-
RelevanceStore
|
|
65
|
-
} from "./chunk-BRK4ODMI.js";
|
|
66
|
-
import {
|
|
67
|
-
RerankCache,
|
|
68
|
-
rerankLocalOrNoop
|
|
69
|
-
} from "./chunk-C7VW7C3F.js";
|
|
67
|
+
} from "./chunk-5IZL4DCV.js";
|
|
70
68
|
import {
|
|
71
69
|
buildQmdRecallCacheKey,
|
|
72
70
|
getCachedQmdRecall,
|
|
73
71
|
setCachedQmdRecall
|
|
74
72
|
} from "./chunk-YCN4BVDK.js";
|
|
73
|
+
import {
|
|
74
|
+
reorderRecallResultsWithMmr
|
|
75
|
+
} from "./chunk-YDBIWGNI.js";
|
|
75
76
|
import {
|
|
76
77
|
createRecallSectionMetricRecorder
|
|
77
|
-
} from "./chunk-
|
|
78
|
+
} from "./chunk-7DHTMOND.js";
|
|
79
|
+
import {
|
|
80
|
+
LastRecallStore,
|
|
81
|
+
TierMigrationStatusStore,
|
|
82
|
+
clampGraphRecallExpandedEntries
|
|
83
|
+
} from "./chunk-47UU5PU2.js";
|
|
78
84
|
import {
|
|
79
85
|
NegativeExampleStore
|
|
80
|
-
} from "./chunk-
|
|
86
|
+
} from "./chunk-D654IBA6.js";
|
|
81
87
|
import {
|
|
82
88
|
evaluateMemoryActionPolicy
|
|
83
89
|
} from "./chunk-H63EDPFJ.js";
|
|
90
|
+
import {
|
|
91
|
+
classifyMemoryKind
|
|
92
|
+
} from "./chunk-YAZNBMNF.js";
|
|
84
93
|
import {
|
|
85
94
|
hasBroadGraphIntent,
|
|
86
95
|
inferIntentFromText,
|
|
87
96
|
intentCompatibilityScore,
|
|
97
|
+
isTaskInitiationIntent,
|
|
88
98
|
planRecallMode
|
|
89
|
-
} from "./chunk-
|
|
99
|
+
} from "./chunk-GGD5W7TB.js";
|
|
100
|
+
import {
|
|
101
|
+
createVerdictCache,
|
|
102
|
+
judgeFactDurability,
|
|
103
|
+
validateProcedureExtraction
|
|
104
|
+
} from "./chunk-LAYN4LDC.js";
|
|
90
105
|
import {
|
|
91
106
|
ExtractionEngine
|
|
92
|
-
} from "./chunk-
|
|
107
|
+
} from "./chunk-3SV6CQHO.js";
|
|
93
108
|
import {
|
|
94
109
|
parseMemoryActionEligibilityContext
|
|
95
|
-
} from "./chunk-
|
|
110
|
+
} from "./chunk-UEYA6UC7.js";
|
|
96
111
|
import {
|
|
97
112
|
ProfilingCollector
|
|
98
|
-
} from "./chunk-
|
|
113
|
+
} from "./chunk-NBNN5GOB.js";
|
|
99
114
|
import {
|
|
100
115
|
ModelRegistry
|
|
101
|
-
} from "./chunk-
|
|
116
|
+
} from "./chunk-FEMOX5AD.js";
|
|
102
117
|
import {
|
|
103
118
|
LocalLlmClient
|
|
104
|
-
} from "./chunk-
|
|
105
|
-
import {
|
|
106
|
-
classifyMemoryKind
|
|
107
|
-
} from "./chunk-YAZNBMNF.js";
|
|
119
|
+
} from "./chunk-JL2PU6AI.js";
|
|
108
120
|
import {
|
|
109
121
|
formatDaySummaryMemories
|
|
110
|
-
} from "./chunk-
|
|
122
|
+
} from "./chunk-GZCUW5IC.js";
|
|
111
123
|
import {
|
|
112
124
|
EmbeddingFallback
|
|
113
|
-
} from "./chunk-
|
|
125
|
+
} from "./chunk-ALXMCZEU.js";
|
|
114
126
|
import {
|
|
115
127
|
buildEntityRecallSection,
|
|
116
128
|
entityRecentTranscriptLookbackHours,
|
|
117
129
|
readRecentEntityTranscriptEntries
|
|
118
|
-
} from "./chunk-
|
|
130
|
+
} from "./chunk-7WQ6SLIE.js";
|
|
119
131
|
import {
|
|
120
132
|
RoutingRulesStore,
|
|
121
133
|
normalizeReplaySessionKey
|
|
122
|
-
} from "./chunk-
|
|
134
|
+
} from "./chunk-PAORGQRI.js";
|
|
123
135
|
import {
|
|
124
136
|
searchVerifiedEpisodes
|
|
125
|
-
} from "./chunk-
|
|
137
|
+
} from "./chunk-6UJ47TVX.js";
|
|
126
138
|
import {
|
|
127
139
|
ThreadingManager
|
|
128
|
-
} from "./chunk-
|
|
140
|
+
} from "./chunk-JRNQ3RNA.js";
|
|
129
141
|
import {
|
|
130
142
|
searchVerifiedSemanticRules
|
|
131
|
-
} from "./chunk-
|
|
143
|
+
} from "./chunk-JIU55F3X.js";
|
|
132
144
|
import {
|
|
133
145
|
searchWorkProductLedgerEntries
|
|
134
146
|
} from "./chunk-CULXMQJH.js";
|
|
135
147
|
import {
|
|
136
148
|
TranscriptManager
|
|
137
|
-
} from "./chunk-
|
|
138
|
-
import {
|
|
139
|
-
PolicyRuntimeManager
|
|
140
|
-
} from "./chunk-T4WRIV2C.js";
|
|
141
|
-
import {
|
|
142
|
-
searchObjectiveStateSnapshots
|
|
143
|
-
} from "./chunk-LOBRX7VD.js";
|
|
149
|
+
} from "./chunk-E6K4NIEU.js";
|
|
144
150
|
import {
|
|
145
151
|
NamespaceSearchRouter,
|
|
146
152
|
NamespaceStorageRouter,
|
|
@@ -148,33 +154,34 @@ import {
|
|
|
148
154
|
createConversationIndexRuntime,
|
|
149
155
|
createSearchBackend,
|
|
150
156
|
writeConversationChunks
|
|
151
|
-
} from "./chunk-
|
|
157
|
+
} from "./chunk-ZVBB3T7V.js";
|
|
152
158
|
import {
|
|
153
159
|
parseQmdExplain
|
|
154
|
-
} from "./chunk-
|
|
160
|
+
} from "./chunk-BLKTA7MM.js";
|
|
155
161
|
import {
|
|
156
|
-
|
|
157
|
-
} from "./chunk-
|
|
162
|
+
PolicyRuntimeManager
|
|
163
|
+
} from "./chunk-EABGC2TL.js";
|
|
164
|
+
import {
|
|
165
|
+
searchObjectiveStateSnapshots
|
|
166
|
+
} from "./chunk-LOBRX7VD.js";
|
|
158
167
|
import {
|
|
159
168
|
searchHarmonicRetrieval
|
|
160
|
-
} from "./chunk-
|
|
169
|
+
} from "./chunk-HMDCOMYU.js";
|
|
170
|
+
import {
|
|
171
|
+
isAboveImportanceThreshold,
|
|
172
|
+
scoreImportance
|
|
173
|
+
} from "./chunk-J4IYOZZ5.js";
|
|
161
174
|
import {
|
|
162
175
|
collectNativeKnowledgeChunks,
|
|
163
176
|
formatNativeKnowledgeSection,
|
|
164
177
|
searchNativeKnowledge
|
|
165
|
-
} from "./chunk-
|
|
178
|
+
} from "./chunk-7SEAZFFB.js";
|
|
166
179
|
import {
|
|
167
180
|
recordEvalShadowRecall
|
|
168
181
|
} from "./chunk-K6WK37A6.js";
|
|
169
182
|
import {
|
|
170
|
-
|
|
171
|
-
} from "./chunk-
|
|
172
|
-
import {
|
|
173
|
-
searchCausalTrajectories
|
|
174
|
-
} from "./chunk-ORZMT74A.js";
|
|
175
|
-
import {
|
|
176
|
-
chunkContent
|
|
177
|
-
} from "./chunk-B7LOFDVE.js";
|
|
183
|
+
CODEX_THREAD_KEY_PREFIX
|
|
184
|
+
} from "./chunk-3PG3H5TD.js";
|
|
178
185
|
import {
|
|
179
186
|
applyCommitmentLedgerLifecycle
|
|
180
187
|
} from "./chunk-FYIYMQ5N.js";
|
|
@@ -184,28 +191,64 @@ import {
|
|
|
184
191
|
refineCompressionGuidelineCandidateSemantically,
|
|
185
192
|
renderCompressionGuidelinesMarkdown
|
|
186
193
|
} from "./chunk-2NMMFZ5T.js";
|
|
194
|
+
import {
|
|
195
|
+
buildConsolidationPrompt,
|
|
196
|
+
buildExtensionsBlockForConsolidation,
|
|
197
|
+
findSimilarClusters,
|
|
198
|
+
materializeAfterSemanticConsolidation,
|
|
199
|
+
parseConsolidationResponse
|
|
200
|
+
} from "./chunk-SYUK3VLY.js";
|
|
201
|
+
import {
|
|
202
|
+
FallbackLlmClient
|
|
203
|
+
} from "./chunk-44ICJRF3.js";
|
|
204
|
+
import {
|
|
205
|
+
GraphIndex
|
|
206
|
+
} from "./chunk-C2EFFULQ.js";
|
|
207
|
+
import {
|
|
208
|
+
chunkContent
|
|
209
|
+
} from "./chunk-4WMCPJWX.js";
|
|
210
|
+
import {
|
|
211
|
+
buildRecallQueryPolicy
|
|
212
|
+
} from "./chunk-6HZ6AO2P.js";
|
|
187
213
|
import {
|
|
188
214
|
buildBehaviorSignalsForMemory,
|
|
189
215
|
dedupeBehaviorSignalsByMemoryAndHash
|
|
190
216
|
} from "./chunk-JWPLJLDU.js";
|
|
191
217
|
import {
|
|
192
218
|
BootstrapEngine
|
|
193
|
-
} from "./chunk-
|
|
219
|
+
} from "./chunk-N53K2EXC.js";
|
|
220
|
+
import {
|
|
221
|
+
BoxBuilder
|
|
222
|
+
} from "./chunk-URB2WSKZ.js";
|
|
194
223
|
import {
|
|
195
224
|
SmartBuffer
|
|
196
|
-
} from "./chunk-
|
|
225
|
+
} from "./chunk-UVJFDP7P.js";
|
|
197
226
|
import {
|
|
198
227
|
isDisagreementPrompt
|
|
199
228
|
} from "./chunk-XYIK4LF6.js";
|
|
200
229
|
import {
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
} from "./chunk-M5KEYE5E.js";
|
|
230
|
+
abortError,
|
|
231
|
+
isAbortError,
|
|
232
|
+
throwIfAborted
|
|
233
|
+
} from "./chunk-PVGDJXVK.js";
|
|
206
234
|
import {
|
|
207
235
|
resolveHomeDir
|
|
208
236
|
} from "./chunk-MARWOCVP.js";
|
|
237
|
+
import {
|
|
238
|
+
searchTrustZoneRecords
|
|
239
|
+
} from "./chunk-EQINRHYR.js";
|
|
240
|
+
import {
|
|
241
|
+
shouldSkipImplicitExtraction
|
|
242
|
+
} from "./chunk-QDYXG4CS.js";
|
|
243
|
+
import {
|
|
244
|
+
selectRouteRule
|
|
245
|
+
} from "./chunk-QNJMBKFK.js";
|
|
246
|
+
import {
|
|
247
|
+
buildProcedurePersistBody
|
|
248
|
+
} from "./chunk-QDW3E4RD.js";
|
|
249
|
+
import {
|
|
250
|
+
searchCausalTrajectories
|
|
251
|
+
} from "./chunk-4NRAJUDS.js";
|
|
209
252
|
import {
|
|
210
253
|
canReadNamespace,
|
|
211
254
|
defaultNamespaceForPrincipal,
|
|
@@ -213,16 +256,26 @@ import {
|
|
|
213
256
|
resolvePrincipal
|
|
214
257
|
} from "./chunk-N5AKDXAI.js";
|
|
215
258
|
import {
|
|
216
|
-
|
|
217
|
-
|
|
259
|
+
decideLifecycleTransition,
|
|
260
|
+
resolveLifecycleState
|
|
261
|
+
} from "./chunk-TBBDFYXW.js";
|
|
218
262
|
import {
|
|
219
263
|
ContentHashIndex,
|
|
220
264
|
StorageManager,
|
|
221
|
-
|
|
222
|
-
|
|
265
|
+
compareEntityTimestamps,
|
|
266
|
+
fingerprintEntityStructuredFacts,
|
|
267
|
+
normalizeAttributePairs,
|
|
268
|
+
normalizeEntityName,
|
|
269
|
+
parseEntityFile
|
|
270
|
+
} from "./chunk-GV6NLQ4X.js";
|
|
223
271
|
import {
|
|
224
272
|
confidenceTier
|
|
225
|
-
} from "./chunk-
|
|
273
|
+
} from "./chunk-3WHVNEN7.js";
|
|
274
|
+
import {
|
|
275
|
+
attachCitation,
|
|
276
|
+
hasCitationForTemplate,
|
|
277
|
+
stripCitationForTemplate
|
|
278
|
+
} from "./chunk-4KAN3GZ3.js";
|
|
226
279
|
import {
|
|
227
280
|
openBetterSqlite3
|
|
228
281
|
} from "./chunk-BOUYNNYD.js";
|
|
@@ -231,26 +284,16 @@ import {
|
|
|
231
284
|
rotateMarkdownFileToArchive
|
|
232
285
|
} from "./chunk-DM2T26WE.js";
|
|
233
286
|
import {
|
|
234
|
-
|
|
235
|
-
} from "./chunk-
|
|
236
|
-
import {
|
|
237
|
-
selectRouteRule
|
|
238
|
-
} from "./chunk-HLXVTBF3.js";
|
|
287
|
+
sanitizeMemoryContent
|
|
288
|
+
} from "./chunk-M62O4P4T.js";
|
|
239
289
|
import {
|
|
240
290
|
log
|
|
241
|
-
} from "./chunk-
|
|
242
|
-
import {
|
|
243
|
-
buildRecallQueryPolicy
|
|
244
|
-
} from "./chunk-6HZ6AO2P.js";
|
|
245
|
-
import {
|
|
246
|
-
decideLifecycleTransition,
|
|
247
|
-
resolveLifecycleState
|
|
248
|
-
} from "./chunk-QCCCQT3O.js";
|
|
291
|
+
} from "./chunk-2ODBA7MQ.js";
|
|
249
292
|
|
|
250
293
|
// src/orchestrator.ts
|
|
251
294
|
import path5 from "path";
|
|
252
295
|
import os from "os";
|
|
253
|
-
import { createHash as createHash2 } from "crypto";
|
|
296
|
+
import { createHash as createHash2, randomBytes } from "crypto";
|
|
254
297
|
import { existsSync as existsSync2 } from "fs";
|
|
255
298
|
import {
|
|
256
299
|
mkdir as mkdir4,
|
|
@@ -804,11 +847,66 @@ async function migrateFromEngram(options) {
|
|
|
804
847
|
}
|
|
805
848
|
}
|
|
806
849
|
|
|
850
|
+
// src/procedural/procedure-recall.ts
|
|
851
|
+
function tokenOverlapScore(prompt, memoryText) {
|
|
852
|
+
const norm = (s) => s.toLowerCase().split(/[^a-z0-9]+/).filter((t) => t.length > 2);
|
|
853
|
+
const promptTokens = new Set(norm(prompt));
|
|
854
|
+
const memTokens = new Set(norm(memoryText));
|
|
855
|
+
if (promptTokens.size === 0 || memTokens.size === 0) return 0;
|
|
856
|
+
let inter = 0;
|
|
857
|
+
for (const t of promptTokens) {
|
|
858
|
+
if (memTokens.has(t)) inter++;
|
|
859
|
+
}
|
|
860
|
+
const union = /* @__PURE__ */ new Set([...promptTokens, ...memTokens]);
|
|
861
|
+
return inter / Math.max(1, union.size);
|
|
862
|
+
}
|
|
863
|
+
function scoreProcedureForPrompt(m, prompt, queryIntent) {
|
|
864
|
+
const memText = `${m.content}
|
|
865
|
+
${(m.frontmatter.tags ?? []).join(" ")}`;
|
|
866
|
+
const jaccard = tokenOverlapScore(prompt, memText);
|
|
867
|
+
const memIntent = inferIntentFromText(m.content.slice(0, 2e3));
|
|
868
|
+
const intentScore = intentCompatibilityScore(queryIntent, memIntent);
|
|
869
|
+
return jaccard * 0.55 + intentScore * 0.45;
|
|
870
|
+
}
|
|
871
|
+
async function buildProcedureRecallSection(storage, prompt, config) {
|
|
872
|
+
if (config.procedural?.enabled !== true) return null;
|
|
873
|
+
const trimmed = typeof prompt === "string" ? prompt.trim() : "";
|
|
874
|
+
if (!trimmed) return null;
|
|
875
|
+
const queryIntent = inferIntentFromText(trimmed);
|
|
876
|
+
if (!isTaskInitiationIntent(queryIntent)) return null;
|
|
877
|
+
const maxN = Math.min(
|
|
878
|
+
10,
|
|
879
|
+
Math.max(
|
|
880
|
+
1,
|
|
881
|
+
typeof config.procedural.recallMaxProcedures === "number" && Number.isFinite(config.procedural.recallMaxProcedures) ? Math.floor(config.procedural.recallMaxProcedures) : 3
|
|
882
|
+
)
|
|
883
|
+
);
|
|
884
|
+
const all = await storage.readAllMemories();
|
|
885
|
+
const scored = all.filter(
|
|
886
|
+
(m) => m.frontmatter.category === "procedure" && m.frontmatter.status !== "pending_review" && m.frontmatter.status !== "rejected" && m.frontmatter.status !== "quarantined" && m.frontmatter.status !== "superseded" && m.frontmatter.status !== "archived"
|
|
887
|
+
).map((m) => ({ m, score: scoreProcedureForPrompt(m, trimmed, queryIntent) })).filter((x) => x.score > 0.04).sort((a, b) => b.score - a.score).slice(0, maxN);
|
|
888
|
+
if (scored.length === 0) return null;
|
|
889
|
+
const blocks = scored.map(({ m, score }) => {
|
|
890
|
+
const id = m.frontmatter.id;
|
|
891
|
+
const flat = m.content.replace(/\s+/g, " ").trim();
|
|
892
|
+
const preview = flat.slice(0, 320);
|
|
893
|
+
const suffix = flat.length > 320 ? "\u2026" : "";
|
|
894
|
+
return `### ${id} (match ${score.toFixed(2)})
|
|
895
|
+
|
|
896
|
+
${preview}${suffix}`;
|
|
897
|
+
});
|
|
898
|
+
return `## Relevant procedures
|
|
899
|
+
|
|
900
|
+
${blocks.join("\n\n")}`;
|
|
901
|
+
}
|
|
902
|
+
|
|
807
903
|
// src/maintenance/memory-governance-cron.ts
|
|
808
904
|
import { mkdir as mkdir2, readFile as readFile2, rename, rm as rm2, stat as stat2, writeFile as writeFile2 } from "fs/promises";
|
|
809
905
|
import path2 from "path";
|
|
810
906
|
var DAY_SUMMARY_CRON_ID = "engram-day-summary";
|
|
811
907
|
var GOVERNANCE_CRON_ID = "engram-nightly-governance";
|
|
908
|
+
var PROCEDURAL_MINING_CRON_ID = "engram-procedural-mining";
|
|
909
|
+
var CONTRADICTION_SCAN_CRON_ID = "engram-contradiction-scan";
|
|
812
910
|
async function acquireCronJobsLock(jobsPath) {
|
|
813
911
|
const lockPath2 = `${jobsPath}.lock`;
|
|
814
912
|
const start = Date.now();
|
|
@@ -921,6 +1019,105 @@ async function ensureNightlyGovernanceCron(jobsPath, options) {
|
|
|
921
1019
|
delivery: { mode: "none" }
|
|
922
1020
|
}));
|
|
923
1021
|
}
|
|
1022
|
+
async function ensureProceduralMiningCron(jobsPath, options) {
|
|
1023
|
+
const scheduleExpr = typeof options.scheduleExpr === "string" && options.scheduleExpr.trim().length > 0 ? options.scheduleExpr.trim() : "17 3 * * *";
|
|
1024
|
+
const agentId = typeof options.agentId === "string" && options.agentId.trim().length > 0 ? options.agentId.trim() : "main";
|
|
1025
|
+
return ensureCronJob(jobsPath, PROCEDURAL_MINING_CRON_ID, () => ({
|
|
1026
|
+
id: PROCEDURAL_MINING_CRON_ID,
|
|
1027
|
+
agentId,
|
|
1028
|
+
name: "Remnic Procedural Mining (nightly)",
|
|
1029
|
+
enabled: true,
|
|
1030
|
+
schedule: {
|
|
1031
|
+
kind: "cron",
|
|
1032
|
+
expr: scheduleExpr,
|
|
1033
|
+
tz: options.timezone
|
|
1034
|
+
},
|
|
1035
|
+
sessionTarget: "isolated",
|
|
1036
|
+
wakeMode: "now",
|
|
1037
|
+
payload: {
|
|
1038
|
+
kind: "agentTurn",
|
|
1039
|
+
timeoutSeconds: 900,
|
|
1040
|
+
thinking: "off",
|
|
1041
|
+
message: "You are OpenClaw automation. Call tool `engram.procedure_mining_run` with empty params. If successful output exactly NO_REPLY. On error output one concise line. Do NOT use message tool."
|
|
1042
|
+
},
|
|
1043
|
+
delivery: { mode: "none" }
|
|
1044
|
+
}));
|
|
1045
|
+
}
|
|
1046
|
+
async function ensureContradictionScanCron(jobsPath, options) {
|
|
1047
|
+
const scheduleExpr = typeof options.scheduleExpr === "string" && options.scheduleExpr.trim().length > 0 ? options.scheduleExpr.trim() : "37 3 * * *";
|
|
1048
|
+
const agentId = typeof options.agentId === "string" && options.agentId.trim().length > 0 ? options.agentId.trim() : "main";
|
|
1049
|
+
return ensureCronJob(jobsPath, CONTRADICTION_SCAN_CRON_ID, () => ({
|
|
1050
|
+
id: CONTRADICTION_SCAN_CRON_ID,
|
|
1051
|
+
agentId,
|
|
1052
|
+
name: "Remnic Contradiction Scan (nightly)",
|
|
1053
|
+
enabled: true,
|
|
1054
|
+
schedule: {
|
|
1055
|
+
kind: "cron",
|
|
1056
|
+
expr: scheduleExpr,
|
|
1057
|
+
tz: options.timezone
|
|
1058
|
+
},
|
|
1059
|
+
sessionTarget: "isolated",
|
|
1060
|
+
wakeMode: "now",
|
|
1061
|
+
payload: {
|
|
1062
|
+
kind: "agentTurn",
|
|
1063
|
+
timeoutSeconds: 900,
|
|
1064
|
+
thinking: "off",
|
|
1065
|
+
message: "You are OpenClaw automation. Call tool `engram.contradiction_scan_run` with empty params. If successful output exactly NO_REPLY. On error output one concise line. Do NOT use message tool."
|
|
1066
|
+
},
|
|
1067
|
+
delivery: { mode: "none" }
|
|
1068
|
+
}));
|
|
1069
|
+
}
|
|
1070
|
+
|
|
1071
|
+
// src/dedup/semantic.ts
|
|
1072
|
+
async function decideSemanticDedup(content, lookup, options) {
|
|
1073
|
+
if (!options.enabled) {
|
|
1074
|
+
return { action: "keep", reason: "disabled" };
|
|
1075
|
+
}
|
|
1076
|
+
if (options.candidates === 0) {
|
|
1077
|
+
return { action: "keep", reason: "disabled" };
|
|
1078
|
+
}
|
|
1079
|
+
const trimmed = typeof content === "string" ? content.trim() : "";
|
|
1080
|
+
if (!trimmed) {
|
|
1081
|
+
return { action: "keep", reason: "no_near_duplicate" };
|
|
1082
|
+
}
|
|
1083
|
+
const candidates = Math.max(1, Math.floor(options.candidates));
|
|
1084
|
+
let hits = [];
|
|
1085
|
+
try {
|
|
1086
|
+
hits = await lookup(trimmed, candidates);
|
|
1087
|
+
} catch {
|
|
1088
|
+
return { action: "keep", reason: "backend_unavailable" };
|
|
1089
|
+
}
|
|
1090
|
+
if (!Array.isArray(hits) || hits.length === 0) {
|
|
1091
|
+
return { action: "keep", reason: "no_candidates" };
|
|
1092
|
+
}
|
|
1093
|
+
let top;
|
|
1094
|
+
for (const hit of hits) {
|
|
1095
|
+
if (!hit || typeof hit.score !== "number" || !Number.isFinite(hit.score)) {
|
|
1096
|
+
continue;
|
|
1097
|
+
}
|
|
1098
|
+
if (!top || hit.score > top.score) {
|
|
1099
|
+
top = hit;
|
|
1100
|
+
}
|
|
1101
|
+
}
|
|
1102
|
+
if (!top) {
|
|
1103
|
+
return { action: "keep", reason: "no_near_duplicate" };
|
|
1104
|
+
}
|
|
1105
|
+
if (top.score >= options.threshold) {
|
|
1106
|
+
return {
|
|
1107
|
+
action: "skip",
|
|
1108
|
+
reason: "near_duplicate",
|
|
1109
|
+
topScore: top.score,
|
|
1110
|
+
topId: top.id,
|
|
1111
|
+
topPath: top.path
|
|
1112
|
+
};
|
|
1113
|
+
}
|
|
1114
|
+
return {
|
|
1115
|
+
action: "keep",
|
|
1116
|
+
reason: "no_near_duplicate",
|
|
1117
|
+
topScore: top.score,
|
|
1118
|
+
topId: top.id
|
|
1119
|
+
};
|
|
1120
|
+
}
|
|
924
1121
|
|
|
925
1122
|
// src/lcm/schema.ts
|
|
926
1123
|
import path3 from "path";
|
|
@@ -2233,15 +2430,72 @@ async function cleanupConversationChunks(rootDir, retentionDays) {
|
|
|
2233
2430
|
}
|
|
2234
2431
|
|
|
2235
2432
|
// src/orchestrator.ts
|
|
2236
|
-
function
|
|
2237
|
-
const
|
|
2238
|
-
|
|
2239
|
-
|
|
2433
|
+
function dedupeEntitySynthesisEvidenceEntries(entries) {
|
|
2434
|
+
const dedupedEvidenceEntries = [];
|
|
2435
|
+
const evidenceByFact = /* @__PURE__ */ new Map();
|
|
2436
|
+
for (const entry of entries) {
|
|
2437
|
+
const normalizedFact = entry.text.trim();
|
|
2438
|
+
if (!normalizedFact) continue;
|
|
2439
|
+
const existing = evidenceByFact.get(normalizedFact);
|
|
2440
|
+
if (!existing) {
|
|
2441
|
+
evidenceByFact.set(normalizedFact, { newest: entry, oldest: entry });
|
|
2442
|
+
continue;
|
|
2443
|
+
}
|
|
2444
|
+
if (compareEntityTimestamps(entry.timestamp, existing.newest.timestamp) > 0) {
|
|
2445
|
+
existing.newest = entry;
|
|
2446
|
+
}
|
|
2447
|
+
if (compareEntityTimestamps(entry.timestamp, existing.oldest.timestamp) < 0) {
|
|
2448
|
+
existing.oldest = entry;
|
|
2449
|
+
}
|
|
2450
|
+
}
|
|
2451
|
+
for (const { newest, oldest } of evidenceByFact.values()) {
|
|
2452
|
+
dedupedEvidenceEntries.push(newest);
|
|
2453
|
+
const newestKey = [
|
|
2454
|
+
newest.timestamp,
|
|
2455
|
+
newest.source ?? "",
|
|
2456
|
+
newest.sessionKey ?? "",
|
|
2457
|
+
newest.principal ?? "",
|
|
2458
|
+
newest.text
|
|
2459
|
+
].join("\0");
|
|
2460
|
+
const oldestKey = [
|
|
2461
|
+
oldest.timestamp,
|
|
2462
|
+
oldest.source ?? "",
|
|
2463
|
+
oldest.sessionKey ?? "",
|
|
2464
|
+
oldest.principal ?? "",
|
|
2465
|
+
oldest.text
|
|
2466
|
+
].join("\0");
|
|
2467
|
+
if (oldestKey !== newestKey) {
|
|
2468
|
+
dedupedEvidenceEntries.push(oldest);
|
|
2469
|
+
}
|
|
2470
|
+
}
|
|
2471
|
+
return dedupedEvidenceEntries;
|
|
2240
2472
|
}
|
|
2473
|
+
function flattenStructuredSectionEvidence(sections) {
|
|
2474
|
+
return (sections ?? []).flatMap(
|
|
2475
|
+
(section) => section.facts.map((fact) => fact.trim()).filter((fact) => fact.length > 0).map((fact) => ({
|
|
2476
|
+
timestamp: "",
|
|
2477
|
+
text: fact,
|
|
2478
|
+
source: `section:${section.title}`
|
|
2479
|
+
}))
|
|
2480
|
+
);
|
|
2481
|
+
}
|
|
2482
|
+
function fingerprintEntitySynthesisEvidence(entity) {
|
|
2483
|
+
const fingerprint = createHash2("sha256");
|
|
2484
|
+
const timelineEntries = entity.timeline.map((entry) => [
|
|
2485
|
+
entry.timestamp,
|
|
2486
|
+
entry.source ?? "",
|
|
2487
|
+
entry.sessionKey ?? "",
|
|
2488
|
+
entry.principal ?? "",
|
|
2489
|
+
entry.text
|
|
2490
|
+
].join("\0")).sort();
|
|
2491
|
+
fingerprint.update(timelineEntries.join(""));
|
|
2492
|
+
fingerprint.update("");
|
|
2493
|
+
fingerprint.update(fingerprintEntityStructuredFacts(entity) ?? "");
|
|
2494
|
+
return fingerprint.digest("hex");
|
|
2495
|
+
}
|
|
2496
|
+
var abortRecallError = abortError;
|
|
2241
2497
|
function throwIfRecallAborted(signal, message = "recall aborted") {
|
|
2242
|
-
|
|
2243
|
-
throw abortRecallError(message);
|
|
2244
|
-
}
|
|
2498
|
+
throwIfAborted(signal, message);
|
|
2245
2499
|
}
|
|
2246
2500
|
async function raceRecallAbort(promise, signal, message = "recall aborted") {
|
|
2247
2501
|
throwIfRecallAborted(signal, message);
|
|
@@ -2501,7 +2755,8 @@ function parseMemoryIntentSnapshot(value) {
|
|
|
2501
2755
|
actionType: typeof candidate.actionType === "string" ? candidate.actionType : "unknown",
|
|
2502
2756
|
entityTypes: Array.isArray(candidate.entityTypes) ? candidate.entityTypes.filter(
|
|
2503
2757
|
(item) => typeof item === "string"
|
|
2504
|
-
) : []
|
|
2758
|
+
) : [],
|
|
2759
|
+
taskInitiation: candidate.taskInitiation === true
|
|
2505
2760
|
};
|
|
2506
2761
|
}
|
|
2507
2762
|
function buildQmdIntentHint(intent) {
|
|
@@ -2515,6 +2770,9 @@ function buildQmdIntentHint(intent) {
|
|
|
2515
2770
|
if (intent.entityTypes.length > 0) {
|
|
2516
2771
|
parts.push(`entities:${intent.entityTypes.join(",")}`);
|
|
2517
2772
|
}
|
|
2773
|
+
if (intent.taskInitiation === true) {
|
|
2774
|
+
parts.push("task_initiation");
|
|
2775
|
+
}
|
|
2518
2776
|
return parts.length > 0 ? parts.join(" ") : void 0;
|
|
2519
2777
|
}
|
|
2520
2778
|
function parseQmdRecallResults(value) {
|
|
@@ -2625,6 +2883,7 @@ var Orchestrator = class _Orchestrator {
|
|
|
2625
2883
|
summarizer;
|
|
2626
2884
|
localLlm;
|
|
2627
2885
|
fastLlm;
|
|
2886
|
+
judgeVerdictCache;
|
|
2628
2887
|
fastGatewayLlm;
|
|
2629
2888
|
modelRegistry;
|
|
2630
2889
|
relevance;
|
|
@@ -2669,6 +2928,7 @@ var Orchestrator = class _Orchestrator {
|
|
|
2669
2928
|
nonZeroExtractionsSinceConsolidation = 0;
|
|
2670
2929
|
lastConsolidationRunAtMs = 0;
|
|
2671
2930
|
consolidationInFlight = false;
|
|
2931
|
+
consolidationObservers = /* @__PURE__ */ new Set();
|
|
2672
2932
|
qmdMaintenanceTimer = null;
|
|
2673
2933
|
qmdMaintenancePending = false;
|
|
2674
2934
|
qmdMaintenanceInFlight = false;
|
|
@@ -2688,6 +2948,41 @@ var Orchestrator = class _Orchestrator {
|
|
|
2688
2948
|
// Initialization gate: recall() awaits this before proceeding
|
|
2689
2949
|
initPromise = null;
|
|
2690
2950
|
resolveInit = null;
|
|
2951
|
+
/**
|
|
2952
|
+
* Resolves when deferred initialization (QMD probe, warmup, caches, cron)
|
|
2953
|
+
* completes. CLI and http-serve callers that need `qmd.isAvailable()` to
|
|
2954
|
+
* reflect reality should `await orchestrator.deferredReady` after
|
|
2955
|
+
* `initialize()`. Gateway callers can ignore it — recall() degrades
|
|
2956
|
+
* gracefully when QMD isn't ready yet.
|
|
2957
|
+
*
|
|
2958
|
+
* Also resolves (without error) when `initialize()` throws before reaching
|
|
2959
|
+
* the deferred-init phase, so callers never hang on a permanently-pending
|
|
2960
|
+
* promise.
|
|
2961
|
+
*
|
|
2962
|
+
* Host adapters that need to tie deferred init to their stop() lifecycle
|
|
2963
|
+
* should `await orchestrator.deferredReady` before proceeding with teardown
|
|
2964
|
+
* to prevent background QMD/warmup/cron tasks from racing with shutdown.
|
|
2965
|
+
*/
|
|
2966
|
+
deferredReady = Promise.resolve();
|
|
2967
|
+
resolveDeferredReady = null;
|
|
2968
|
+
deferredInitAbort = null;
|
|
2969
|
+
/**
|
|
2970
|
+
* Whether the deferred init's QMD startup sync completed successfully.
|
|
2971
|
+
* When false after deferredReady resolves, the server retry loop should
|
|
2972
|
+
* attempt startupSearchSync() even if `qmd.isAvailable()` is true —
|
|
2973
|
+
* availability only means probe succeeded, not that the index is current.
|
|
2974
|
+
*/
|
|
2975
|
+
deferredSyncSucceeded = false;
|
|
2976
|
+
/**
|
|
2977
|
+
* Abort deferred initialization so background QMD sync/warmup stops
|
|
2978
|
+
* promptly on shutdown. Safe to call multiple times or before init.
|
|
2979
|
+
*/
|
|
2980
|
+
abortDeferredInit() {
|
|
2981
|
+
if (this.deferredInitAbort) {
|
|
2982
|
+
this.deferredInitAbort.abort();
|
|
2983
|
+
this.deferredInitAbort = null;
|
|
2984
|
+
}
|
|
2985
|
+
}
|
|
2691
2986
|
/** Set per-session workspace for the next recall() call (compaction reset). @internal */
|
|
2692
2987
|
setRecallWorkspaceOverride(sessionKey, dir) {
|
|
2693
2988
|
this._recallWorkspaceOverrides.set(sessionKey, dir);
|
|
@@ -2796,7 +3091,13 @@ var Orchestrator = class _Orchestrator {
|
|
|
2796
3091
|
config,
|
|
2797
3092
|
this.storageRouter
|
|
2798
3093
|
);
|
|
2799
|
-
this.storage = new StorageManager(config.memoryDir);
|
|
3094
|
+
this.storage = new StorageManager(config.memoryDir, config.entitySchemas);
|
|
3095
|
+
this.storage.citationTemplate = config.inlineSourceAttributionFormat;
|
|
3096
|
+
this.storage.setVersioningConfig({
|
|
3097
|
+
enabled: config.versioningEnabled,
|
|
3098
|
+
maxVersionsPerPage: config.versioningMaxPerPage,
|
|
3099
|
+
sidecarDir: config.versioningSidecarDir
|
|
3100
|
+
});
|
|
2800
3101
|
this.qmd = createSearchBackend(config);
|
|
2801
3102
|
const conversationIndexRuntime = createConversationIndexRuntime(config, {
|
|
2802
3103
|
getQmd: () => this.conversationQmd,
|
|
@@ -2832,7 +3133,9 @@ var Orchestrator = class _Orchestrator {
|
|
|
2832
3133
|
this.modelRegistry,
|
|
2833
3134
|
this.transcript
|
|
2834
3135
|
);
|
|
3136
|
+
this.judgeVerdictCache = createVerdictCache();
|
|
2835
3137
|
this.localLlm = new LocalLlmClient(config, this.modelRegistry);
|
|
3138
|
+
this.localLlm.disableThinking = config.localLlmDisableThinking;
|
|
2836
3139
|
this.fastLlm = config.localLlmFastEnabled ? (() => {
|
|
2837
3140
|
const client = new LocalLlmClient(
|
|
2838
3141
|
{
|
|
@@ -3071,100 +3374,173 @@ var Orchestrator = class _Orchestrator {
|
|
|
3071
3374
|
return this.fastLlm;
|
|
3072
3375
|
}
|
|
3073
3376
|
async initialize() {
|
|
3074
|
-
|
|
3075
|
-
|
|
3076
|
-
logger: (message) => log.info(message)
|
|
3377
|
+
this.deferredReady = new Promise((resolve) => {
|
|
3378
|
+
this.resolveDeferredReady = resolve;
|
|
3077
3379
|
});
|
|
3078
|
-
|
|
3079
|
-
|
|
3080
|
-
|
|
3081
|
-
|
|
3082
|
-
|
|
3083
|
-
|
|
3084
|
-
|
|
3085
|
-
|
|
3086
|
-
|
|
3087
|
-
|
|
3088
|
-
|
|
3089
|
-
|
|
3380
|
+
try {
|
|
3381
|
+
await migrateFromEngram({
|
|
3382
|
+
quiet: true,
|
|
3383
|
+
logger: (message) => log.info(message)
|
|
3384
|
+
});
|
|
3385
|
+
await this.storage.ensureDirectories();
|
|
3386
|
+
await this.storage.loadAliases();
|
|
3387
|
+
if (this.config.namespacesEnabled) {
|
|
3388
|
+
const namespaces = /* @__PURE__ */ new Set([
|
|
3389
|
+
this.config.defaultNamespace,
|
|
3390
|
+
this.config.sharedNamespace,
|
|
3391
|
+
...this.config.namespacePolicies.map((p) => p.name)
|
|
3392
|
+
]);
|
|
3393
|
+
for (const ns of namespaces) {
|
|
3394
|
+
const sm = await this.storageRouter.storageFor(ns);
|
|
3395
|
+
await sm.ensureDirectories();
|
|
3396
|
+
await sm.loadAliases().catch(() => void 0);
|
|
3397
|
+
}
|
|
3398
|
+
}
|
|
3399
|
+
await this.relevance.load();
|
|
3400
|
+
await this.negatives.load();
|
|
3401
|
+
await this.lastRecall.load();
|
|
3402
|
+
await this.tierMigrationStatus.load();
|
|
3403
|
+
await this.sessionObserver.load();
|
|
3404
|
+
this.runtimePolicyValues = await this.policyRuntime.loadRuntimeValues();
|
|
3405
|
+
this.utilityRuntimeValues = await loadUtilityRuntimeValues({
|
|
3406
|
+
memoryDir: this.config.memoryDir,
|
|
3407
|
+
memoryUtilityLearningEnabled: this.config.memoryUtilityLearningEnabled,
|
|
3408
|
+
promotionByOutcomeEnabled: this.config.promotionByOutcomeEnabled
|
|
3409
|
+
});
|
|
3410
|
+
if (this.config.factDeduplicationEnabled) {
|
|
3411
|
+
const stateDir = path5.join(this.config.memoryDir, "state");
|
|
3412
|
+
this.contentHashIndex = new ContentHashIndex(stateDir);
|
|
3413
|
+
await this.contentHashIndex.load();
|
|
3414
|
+
log.info(
|
|
3415
|
+
`content-hash dedup: loaded ${this.contentHashIndex.size} hashes`
|
|
3416
|
+
);
|
|
3090
3417
|
}
|
|
3091
|
-
|
|
3092
|
-
|
|
3093
|
-
|
|
3094
|
-
|
|
3095
|
-
|
|
3096
|
-
|
|
3097
|
-
|
|
3098
|
-
|
|
3099
|
-
|
|
3100
|
-
|
|
3101
|
-
|
|
3102
|
-
|
|
3103
|
-
|
|
3104
|
-
const stateDir = path5.join(this.config.memoryDir, "state");
|
|
3105
|
-
this.contentHashIndex = new ContentHashIndex(stateDir);
|
|
3106
|
-
await this.contentHashIndex.load();
|
|
3107
|
-
log.info(
|
|
3108
|
-
`content-hash dedup: loaded ${this.contentHashIndex.size} hashes`
|
|
3109
|
-
);
|
|
3110
|
-
}
|
|
3111
|
-
await this.transcript.initialize();
|
|
3112
|
-
await this.summarizer.initialize();
|
|
3113
|
-
if (this.sharedContext) {
|
|
3114
|
-
await this.sharedContext.ensureStructure();
|
|
3115
|
-
}
|
|
3116
|
-
if (this.compounding) {
|
|
3117
|
-
await this.compounding.ensureDirs();
|
|
3118
|
-
}
|
|
3119
|
-
if (this.resolveInit) {
|
|
3120
|
-
this.resolveInit();
|
|
3121
|
-
this.resolveInit = null;
|
|
3122
|
-
log.info("init gate opened (essential state loaded)");
|
|
3123
|
-
}
|
|
3124
|
-
{
|
|
3125
|
-
const available = await this.qmd.probe();
|
|
3126
|
-
if (available) {
|
|
3127
|
-
log.info(`Search backend: available ${this.qmd.debugStatus()}`);
|
|
3128
|
-
const namespaces = this.config.namespacesEnabled ? this.configuredNamespaces() : [this.config.defaultNamespace];
|
|
3129
|
-
const states = await Promise.all(
|
|
3130
|
-
namespaces.map(async (namespace) => ({
|
|
3131
|
-
namespace,
|
|
3132
|
-
state: this.config.namespacesEnabled ? await this.namespaceSearchRouter.ensureNamespaceCollection(
|
|
3133
|
-
namespace
|
|
3134
|
-
) : await this.qmd.ensureCollection(this.config.memoryDir)
|
|
3135
|
-
}))
|
|
3418
|
+
await this.transcript.initialize();
|
|
3419
|
+
await this.summarizer.initialize();
|
|
3420
|
+
if (this.sharedContext) {
|
|
3421
|
+
await this.sharedContext.ensureStructure();
|
|
3422
|
+
}
|
|
3423
|
+
if (this.compounding) {
|
|
3424
|
+
await this.compounding.ensureDirs();
|
|
3425
|
+
}
|
|
3426
|
+
try {
|
|
3427
|
+
await this.buffer.load();
|
|
3428
|
+
} catch (bufErr) {
|
|
3429
|
+
log.error(
|
|
3430
|
+
`buffer.load() failed (init gate will still open): ${bufErr}`
|
|
3136
3431
|
);
|
|
3137
|
-
|
|
3138
|
-
|
|
3139
|
-
|
|
3140
|
-
|
|
3141
|
-
this.
|
|
3142
|
-
|
|
3143
|
-
|
|
3144
|
-
|
|
3145
|
-
|
|
3146
|
-
|
|
3147
|
-
|
|
3148
|
-
|
|
3149
|
-
|
|
3150
|
-
|
|
3151
|
-
|
|
3152
|
-
|
|
3432
|
+
this.buffer.resetToEmpty();
|
|
3433
|
+
}
|
|
3434
|
+
if (this.config.compactionResetEnabled) {
|
|
3435
|
+
try {
|
|
3436
|
+
const wsDir = this.config.workspaceDir || defaultWorkspaceDir();
|
|
3437
|
+
const files = await readdir3(wsDir).catch(() => []);
|
|
3438
|
+
for (const f of files) {
|
|
3439
|
+
if (!f.startsWith(".compaction-reset-signal-")) continue;
|
|
3440
|
+
const fp = path5.join(wsDir, f);
|
|
3441
|
+
const s = await stat3(fp).catch(() => null);
|
|
3442
|
+
if (s && Date.now() - s.mtimeMs >= COMPACTION_SIGNAL_MAX_AGE_MS) {
|
|
3443
|
+
await unlink2(fp).catch(() => {
|
|
3444
|
+
});
|
|
3445
|
+
log.debug(`initialize: removed stale compaction signal ${f}`);
|
|
3446
|
+
}
|
|
3447
|
+
}
|
|
3448
|
+
} catch (err) {
|
|
3449
|
+
log.debug("initialize: stale signal sweep failed:", err);
|
|
3153
3450
|
}
|
|
3154
|
-
|
|
3155
|
-
|
|
3156
|
-
|
|
3451
|
+
}
|
|
3452
|
+
try {
|
|
3453
|
+
const available = await this.qmd.probe();
|
|
3454
|
+
if (available) {
|
|
3455
|
+
log.info(`Search backend: available ${this.qmd.debugStatus()}`);
|
|
3456
|
+
const namespaces = this.config.namespacesEnabled ? this.configuredNamespaces() : [this.config.defaultNamespace];
|
|
3457
|
+
const states = await Promise.all(
|
|
3458
|
+
namespaces.map(async (namespace) => ({
|
|
3459
|
+
namespace,
|
|
3460
|
+
state: this.config.namespacesEnabled ? await this.namespaceSearchRouter.ensureNamespaceCollection(
|
|
3461
|
+
namespace
|
|
3462
|
+
) : await this.qmd.ensureCollection(this.config.memoryDir)
|
|
3463
|
+
}))
|
|
3464
|
+
);
|
|
3465
|
+
const defaultState = states.find(
|
|
3466
|
+
(entry) => entry.namespace === this.config.defaultNamespace
|
|
3467
|
+
)?.state ?? "unknown";
|
|
3468
|
+
if (defaultState === "missing") {
|
|
3469
|
+
this.qmd = new NoopSearchBackend();
|
|
3470
|
+
log.warn(
|
|
3471
|
+
"Search collection missing for Remnic memory store; disabling search retrieval for this runtime (fallback retrieval remains enabled)"
|
|
3472
|
+
);
|
|
3473
|
+
} else if (defaultState === "unknown") {
|
|
3157
3474
|
log.warn(
|
|
3158
|
-
|
|
3475
|
+
"Search collection check unavailable; keeping search retrieval enabled for fail-open behavior"
|
|
3476
|
+
);
|
|
3477
|
+
} else if (defaultState === "skipped") {
|
|
3478
|
+
log.debug(
|
|
3479
|
+
"Search collection check skipped (remote or daemon-only mode)"
|
|
3159
3480
|
);
|
|
3160
3481
|
}
|
|
3482
|
+
for (const entry of states) {
|
|
3483
|
+
if (entry.namespace === this.config.defaultNamespace) continue;
|
|
3484
|
+
if (entry.state === "missing") {
|
|
3485
|
+
log.warn(
|
|
3486
|
+
`Search collection missing for namespace '${entry.namespace}'; namespace retrieval will fail open to non-search paths`
|
|
3487
|
+
);
|
|
3488
|
+
}
|
|
3489
|
+
}
|
|
3490
|
+
} else if (this.qmd instanceof NoopSearchBackend) {
|
|
3491
|
+
log.debug(`Search backend: noop (search intentionally disabled)`);
|
|
3492
|
+
} else {
|
|
3493
|
+
log.warn(`Search backend: not available ${this.qmd.debugStatus()}`);
|
|
3161
3494
|
}
|
|
3162
|
-
}
|
|
3163
|
-
log.
|
|
3164
|
-
}
|
|
3165
|
-
|
|
3495
|
+
} catch (err) {
|
|
3496
|
+
log.error(`QMD probe/collection check failed (non-fatal): ${err}`);
|
|
3497
|
+
}
|
|
3498
|
+
if (this.resolveInit) {
|
|
3499
|
+
this.resolveInit();
|
|
3500
|
+
this.resolveInit = null;
|
|
3501
|
+
log.info("init gate opened (essential state + QMD state loaded)");
|
|
3502
|
+
}
|
|
3503
|
+
const resolveDeferred = this.resolveDeferredReady;
|
|
3504
|
+
this.resolveDeferredReady = null;
|
|
3505
|
+
this.deferredInitAbort = new AbortController();
|
|
3506
|
+
this.deferredInitialize(this.deferredInitAbort.signal).catch((err) => {
|
|
3507
|
+
log.error(`deferred initialization failed (non-fatal): ${err}`);
|
|
3508
|
+
}).finally(() => {
|
|
3509
|
+
resolveDeferred?.();
|
|
3510
|
+
});
|
|
3511
|
+
} catch (err) {
|
|
3512
|
+
if (this.resolveInit) {
|
|
3513
|
+
this.resolveInit();
|
|
3514
|
+
this.resolveInit = null;
|
|
3515
|
+
}
|
|
3516
|
+
if (this.resolveDeferredReady) {
|
|
3517
|
+
this.resolveDeferredReady();
|
|
3518
|
+
this.resolveDeferredReady = null;
|
|
3519
|
+
}
|
|
3520
|
+
throw err;
|
|
3521
|
+
}
|
|
3522
|
+
}
|
|
3523
|
+
async deferredInitialize(signal) {
|
|
3524
|
+
if (this.qmd.isAvailable() && this.config.qmdMaintenanceEnabled) {
|
|
3525
|
+
try {
|
|
3526
|
+
log.info("QMD startup sync: updating index to match current disk state");
|
|
3527
|
+
if (this.config.namespacesEnabled) {
|
|
3528
|
+
await this.namespaceSearchRouter.updateNamespaces(
|
|
3529
|
+
this.configuredNamespaces()
|
|
3530
|
+
);
|
|
3531
|
+
} else {
|
|
3532
|
+
await this.qmd.update();
|
|
3533
|
+
}
|
|
3534
|
+
log.info("QMD startup sync: complete");
|
|
3535
|
+
this.deferredSyncSucceeded = true;
|
|
3536
|
+
} catch (err) {
|
|
3537
|
+
log.warn(`QMD startup sync failed (non-fatal): ${err}`);
|
|
3166
3538
|
}
|
|
3539
|
+
} else if (!this.qmd.isAvailable()) {
|
|
3540
|
+
} else {
|
|
3541
|
+
this.deferredSyncSucceeded = true;
|
|
3167
3542
|
}
|
|
3543
|
+
if (signal.aborted) return;
|
|
3168
3544
|
const warmupPromises = [];
|
|
3169
3545
|
if (this.qmd.isAvailable()) {
|
|
3170
3546
|
const warmupNs = this.config.defaultNamespace;
|
|
@@ -3189,68 +3565,180 @@ var Orchestrator = class _Orchestrator {
|
|
|
3189
3565
|
);
|
|
3190
3566
|
}
|
|
3191
3567
|
await Promise.all(warmupPromises);
|
|
3568
|
+
if (signal.aborted) return;
|
|
3569
|
+
const cacheWarmups = [];
|
|
3192
3570
|
if (this.config.knowledgeIndexEnabled) {
|
|
3193
|
-
(
|
|
3194
|
-
|
|
3195
|
-
|
|
3196
|
-
|
|
3197
|
-
|
|
3198
|
-
|
|
3199
|
-
|
|
3200
|
-
|
|
3201
|
-
|
|
3202
|
-
|
|
3203
|
-
|
|
3204
|
-
|
|
3205
|
-
|
|
3206
|
-
|
|
3207
|
-
});
|
|
3571
|
+
cacheWarmups.push(
|
|
3572
|
+
(async () => {
|
|
3573
|
+
try {
|
|
3574
|
+
const t0 = Date.now();
|
|
3575
|
+
await this.storage.buildKnowledgeIndex(this.config);
|
|
3576
|
+
log.info(`Knowledge Index warmup: complete in ${Date.now() - t0}ms`);
|
|
3577
|
+
} catch (err) {
|
|
3578
|
+
log.debug(`Knowledge Index warmup failed (non-fatal): ${err}`);
|
|
3579
|
+
}
|
|
3580
|
+
})()
|
|
3581
|
+
);
|
|
3582
|
+
}
|
|
3583
|
+
cacheWarmups.push(this.storage.readAllMemories().then(() => {
|
|
3584
|
+
}).catch(() => {
|
|
3585
|
+
}));
|
|
3586
|
+
cacheWarmups.push(this.storage.readAllEntityFiles().then(() => {
|
|
3587
|
+
}).catch(() => {
|
|
3588
|
+
}));
|
|
3589
|
+
await Promise.all(cacheWarmups);
|
|
3590
|
+
if (signal.aborted) return;
|
|
3208
3591
|
if (this.config.conversationIndexEnabled && this.conversationIndexBackend) {
|
|
3209
|
-
|
|
3210
|
-
|
|
3592
|
+
try {
|
|
3593
|
+
const init = await this.conversationIndexBackend.initialize();
|
|
3594
|
+
if (!init.enabled) {
|
|
3595
|
+
this.config.conversationIndexEnabled = false;
|
|
3596
|
+
}
|
|
3597
|
+
if (init.logLevel === "info") {
|
|
3598
|
+
log.info(init.message);
|
|
3599
|
+
} else if (init.logLevel === "warn") {
|
|
3600
|
+
log.warn(init.message);
|
|
3601
|
+
} else {
|
|
3602
|
+
log.debug(init.message);
|
|
3603
|
+
}
|
|
3604
|
+
} catch (err) {
|
|
3605
|
+
log.error(`Conversation index initialization failed (non-fatal): ${err}`);
|
|
3211
3606
|
this.config.conversationIndexEnabled = false;
|
|
3212
3607
|
}
|
|
3213
|
-
if (init.logLevel === "info") {
|
|
3214
|
-
log.info(init.message);
|
|
3215
|
-
} else if (init.logLevel === "warn") {
|
|
3216
|
-
log.warn(init.message);
|
|
3217
|
-
} else {
|
|
3218
|
-
log.debug(init.message);
|
|
3219
|
-
}
|
|
3220
3608
|
}
|
|
3221
|
-
|
|
3609
|
+
if (signal.aborted) return;
|
|
3222
3610
|
if (this.config.localLlmEnabled) {
|
|
3223
|
-
await this.validateLocalLlmModel();
|
|
3224
|
-
}
|
|
3225
|
-
if (this.config.compactionResetEnabled) {
|
|
3226
3611
|
try {
|
|
3227
|
-
|
|
3228
|
-
const files = await readdir3(wsDir).catch(() => []);
|
|
3229
|
-
for (const f of files) {
|
|
3230
|
-
if (!f.startsWith(".compaction-reset-signal-")) continue;
|
|
3231
|
-
const fp = path5.join(wsDir, f);
|
|
3232
|
-
const s = await stat3(fp).catch(() => null);
|
|
3233
|
-
if (s && Date.now() - s.mtimeMs >= COMPACTION_SIGNAL_MAX_AGE_MS) {
|
|
3234
|
-
await unlink2(fp).catch(() => {
|
|
3235
|
-
});
|
|
3236
|
-
log.debug(`initialize: removed stale compaction signal ${f}`);
|
|
3237
|
-
}
|
|
3238
|
-
}
|
|
3612
|
+
await this.validateLocalLlmModel();
|
|
3239
3613
|
} catch (err) {
|
|
3240
|
-
log.
|
|
3614
|
+
log.error(`Local LLM validation failed (non-fatal): ${err}`);
|
|
3241
3615
|
}
|
|
3242
3616
|
}
|
|
3243
|
-
|
|
3617
|
+
if (signal.aborted) return;
|
|
3244
3618
|
if (this.config.daySummaryEnabled) {
|
|
3245
|
-
|
|
3619
|
+
try {
|
|
3620
|
+
await this.autoRegisterDaySummaryCron();
|
|
3621
|
+
} catch (err) {
|
|
3246
3622
|
log.debug(`day-summary cron auto-register failed (non-fatal): ${err}`);
|
|
3247
|
-
}
|
|
3623
|
+
}
|
|
3248
3624
|
}
|
|
3249
3625
|
if (this.config.nightlyGovernanceCronAutoRegister) {
|
|
3250
|
-
|
|
3626
|
+
try {
|
|
3627
|
+
await this.autoRegisterNightlyGovernanceCron();
|
|
3628
|
+
} catch (err) {
|
|
3251
3629
|
log.debug(`nightly governance cron auto-register failed (non-fatal): ${err}`);
|
|
3252
|
-
}
|
|
3630
|
+
}
|
|
3253
3631
|
}
|
|
3632
|
+
if (this.config.procedural?.proceduralMiningCronAutoRegister) {
|
|
3633
|
+
try {
|
|
3634
|
+
await this.autoRegisterProceduralMiningCron();
|
|
3635
|
+
} catch (err) {
|
|
3636
|
+
log.debug(`procedural mining cron auto-register failed (non-fatal): ${err}`);
|
|
3637
|
+
}
|
|
3638
|
+
}
|
|
3639
|
+
if (this.config.contradictionScan?.enabled) {
|
|
3640
|
+
try {
|
|
3641
|
+
await this.autoRegisterContradictionScanCron();
|
|
3642
|
+
} catch (err) {
|
|
3643
|
+
log.debug(`contradiction scan cron auto-register failed (non-fatal): ${err}`);
|
|
3644
|
+
}
|
|
3645
|
+
}
|
|
3646
|
+
log.info("orchestrator initialized (full \u2014 deferred steps complete)");
|
|
3647
|
+
}
|
|
3648
|
+
/**
|
|
3649
|
+
* Namespace-aware startup search sync. Re-probes QMD, ensures collections
|
|
3650
|
+
* (namespace-aware when namespacesEnabled), runs update, and warms up search.
|
|
3651
|
+
* Designed for server retry paths that run after the deferred init completes
|
|
3652
|
+
* when QMD was not available during initial startup.
|
|
3653
|
+
*
|
|
3654
|
+
* Accepts an optional AbortSignal so callers can interrupt the sync during
|
|
3655
|
+
* shutdown. The signal is checked between phases and forwarded into the QMD
|
|
3656
|
+
* update and warmup search calls so a long-running `qmd update` subprocess
|
|
3657
|
+
* is killed promptly rather than left in flight after `httpServer.stop()`.
|
|
3658
|
+
*
|
|
3659
|
+
* Returns true if the sync succeeded (QMD now available), false otherwise.
|
|
3660
|
+
*/
|
|
3661
|
+
async startupSearchSync(signal) {
|
|
3662
|
+
if (signal?.aborted) return false;
|
|
3663
|
+
const available = await this.qmd.probe();
|
|
3664
|
+
if (!available) return false;
|
|
3665
|
+
if (signal?.aborted) {
|
|
3666
|
+
log.debug("startupSearchSync: aborted after probe");
|
|
3667
|
+
return false;
|
|
3668
|
+
}
|
|
3669
|
+
log.info(`startupSearchSync: backend now available ${this.qmd.debugStatus()}`);
|
|
3670
|
+
if (this.config.namespacesEnabled) {
|
|
3671
|
+
this.namespaceSearchRouter.clearCache();
|
|
3672
|
+
}
|
|
3673
|
+
const namespaces = this.config.namespacesEnabled ? this.configuredNamespaces() : [this.config.defaultNamespace];
|
|
3674
|
+
const states = await Promise.all(
|
|
3675
|
+
namespaces.map(async (namespace) => ({
|
|
3676
|
+
namespace,
|
|
3677
|
+
state: this.config.namespacesEnabled ? await this.namespaceSearchRouter.ensureNamespaceCollection(namespace) : await this.qmd.ensureCollection(this.config.memoryDir)
|
|
3678
|
+
}))
|
|
3679
|
+
);
|
|
3680
|
+
if (signal?.aborted) {
|
|
3681
|
+
log.debug("startupSearchSync: aborted after ensureCollection");
|
|
3682
|
+
return false;
|
|
3683
|
+
}
|
|
3684
|
+
const defaultState = states.find((e) => e.namespace === this.config.defaultNamespace)?.state ?? "unknown";
|
|
3685
|
+
if (defaultState === "missing") {
|
|
3686
|
+
if ("available" in this.qmd) {
|
|
3687
|
+
this.qmd.available = false;
|
|
3688
|
+
}
|
|
3689
|
+
this.qmd = new NoopSearchBackend();
|
|
3690
|
+
log.warn("startupSearchSync: search collection missing; disabling search (fallback retrieval remains enabled)");
|
|
3691
|
+
return false;
|
|
3692
|
+
}
|
|
3693
|
+
if (this.config.qmdMaintenanceEnabled) {
|
|
3694
|
+
try {
|
|
3695
|
+
const failTsBefore = "lastUpdateFailedAtMs" in this.qmd ? this.qmd.lastUpdateFailedAtMs : null;
|
|
3696
|
+
const hasRunTs = "lastUpdateRanAtMs" in this.qmd;
|
|
3697
|
+
if ("resetUpdateThrottles" in this.qmd) {
|
|
3698
|
+
this.qmd.resetUpdateThrottles();
|
|
3699
|
+
}
|
|
3700
|
+
log.info("startupSearchSync: updating index to match current disk state");
|
|
3701
|
+
let namespacesUpdated = 0;
|
|
3702
|
+
if (this.config.namespacesEnabled) {
|
|
3703
|
+
namespacesUpdated = await this.namespaceSearchRouter.updateNamespaces(namespaces);
|
|
3704
|
+
} else {
|
|
3705
|
+
await this.qmd.update(signal);
|
|
3706
|
+
}
|
|
3707
|
+
if (signal?.aborted) {
|
|
3708
|
+
log.debug("startupSearchSync: aborted after update");
|
|
3709
|
+
return false;
|
|
3710
|
+
}
|
|
3711
|
+
const failTsAfter = "lastUpdateFailedAtMs" in this.qmd ? this.qmd.lastUpdateFailedAtMs : null;
|
|
3712
|
+
const runTsAfter = hasRunTs ? this.qmd.lastUpdateRanAtMs : null;
|
|
3713
|
+
if (failTsAfter !== null && failTsAfter !== failTsBefore) {
|
|
3714
|
+
log.warn("startupSearchSync: update silently failed (detected via fail timestamp)");
|
|
3715
|
+
return false;
|
|
3716
|
+
}
|
|
3717
|
+
if (this.config.namespacesEnabled) {
|
|
3718
|
+
if (namespacesUpdated === 0) {
|
|
3719
|
+
log.warn("startupSearchSync: no namespace backends were eligible for update (all unavailable or collections missing)");
|
|
3720
|
+
return false;
|
|
3721
|
+
}
|
|
3722
|
+
log.info(`startupSearchSync: namespace updates succeeded (${namespacesUpdated}/${namespaces.length} namespaces updated)`);
|
|
3723
|
+
} else if (hasRunTs && runTsAfter === null) {
|
|
3724
|
+
log.warn("startupSearchSync: update was throttled/skipped (run timestamp is null after reset + update)");
|
|
3725
|
+
return false;
|
|
3726
|
+
}
|
|
3727
|
+
log.info("startupSearchSync: sync complete");
|
|
3728
|
+
} catch (err) {
|
|
3729
|
+
log.warn(`startupSearchSync: update failed: ${err}`);
|
|
3730
|
+
return false;
|
|
3731
|
+
}
|
|
3732
|
+
}
|
|
3733
|
+
if (!signal?.aborted) {
|
|
3734
|
+
try {
|
|
3735
|
+
await this.qmd.search("warmup", this.config.defaultNamespace, 1, void 0, { signal });
|
|
3736
|
+
log.info("startupSearchSync: warmup complete");
|
|
3737
|
+
} catch (err) {
|
|
3738
|
+
log.debug(`startupSearchSync: warmup search failed (non-fatal): ${err}`);
|
|
3739
|
+
}
|
|
3740
|
+
}
|
|
3741
|
+
return true;
|
|
3254
3742
|
}
|
|
3255
3743
|
/**
|
|
3256
3744
|
* Auto-register the engram-day-summary cron job in OpenClaw if it doesn't exist.
|
|
@@ -3302,6 +3790,46 @@ var Orchestrator = class _Orchestrator {
|
|
|
3302
3790
|
log.debug(`nightly governance cron auto-register error: ${err}`);
|
|
3303
3791
|
}
|
|
3304
3792
|
}
|
|
3793
|
+
async autoRegisterProceduralMiningCron() {
|
|
3794
|
+
const home = resolveHomeDir();
|
|
3795
|
+
const jobsPath = path5.join(home, ".openclaw", "cron", "jobs.json");
|
|
3796
|
+
try {
|
|
3797
|
+
if (!existsSync2(jobsPath)) {
|
|
3798
|
+
log.debug("procedural mining cron: jobs.json not found, skipping auto-register");
|
|
3799
|
+
return;
|
|
3800
|
+
}
|
|
3801
|
+
const created = await ensureProceduralMiningCron(jobsPath, {
|
|
3802
|
+
timezone: Intl.DateTimeFormat().resolvedOptions().timeZone
|
|
3803
|
+
});
|
|
3804
|
+
if (created.created) {
|
|
3805
|
+
log.info(`procedural mining cron auto-registered (${created.jobId})`);
|
|
3806
|
+
} else {
|
|
3807
|
+
log.debug("procedural mining cron already exists, skipping auto-register");
|
|
3808
|
+
}
|
|
3809
|
+
} catch (err) {
|
|
3810
|
+
log.debug(`procedural mining cron auto-register error: ${err}`);
|
|
3811
|
+
}
|
|
3812
|
+
}
|
|
3813
|
+
async autoRegisterContradictionScanCron() {
|
|
3814
|
+
const home = resolveHomeDir();
|
|
3815
|
+
const jobsPath = path5.join(home, ".openclaw", "cron", "jobs.json");
|
|
3816
|
+
try {
|
|
3817
|
+
if (!existsSync2(jobsPath)) {
|
|
3818
|
+
log.debug("contradiction scan cron: jobs.json not found, skipping auto-register");
|
|
3819
|
+
return;
|
|
3820
|
+
}
|
|
3821
|
+
const created = await ensureContradictionScanCron(jobsPath, {
|
|
3822
|
+
timezone: Intl.DateTimeFormat().resolvedOptions().timeZone
|
|
3823
|
+
});
|
|
3824
|
+
if (created.created) {
|
|
3825
|
+
log.info(`contradiction scan cron auto-registered (${created.jobId})`);
|
|
3826
|
+
} else {
|
|
3827
|
+
log.debug("contradiction scan cron already exists, skipping auto-register");
|
|
3828
|
+
}
|
|
3829
|
+
} catch (err) {
|
|
3830
|
+
log.debug(`contradiction scan cron auto-register error: ${err}`);
|
|
3831
|
+
}
|
|
3832
|
+
}
|
|
3305
3833
|
async applyBehaviorRuntimePolicy(state) {
|
|
3306
3834
|
const result = await this.policyRuntime.applyFromBehaviorState(state);
|
|
3307
3835
|
this.runtimePolicyValues = await this.policyRuntime.loadRuntimeValues();
|
|
@@ -3378,6 +3906,16 @@ var Orchestrator = class _Orchestrator {
|
|
|
3378
3906
|
async runConsolidationNow() {
|
|
3379
3907
|
return this.runConsolidation();
|
|
3380
3908
|
}
|
|
3909
|
+
async reindexMemoryById(id, options) {
|
|
3910
|
+
await this.indexPersistedMemory(options?.storage ?? this.storage, id);
|
|
3911
|
+
this.requestQmdMaintenance();
|
|
3912
|
+
}
|
|
3913
|
+
registerConsolidationObserver(observer) {
|
|
3914
|
+
this.consolidationObservers.add(observer);
|
|
3915
|
+
return () => {
|
|
3916
|
+
this.consolidationObservers.delete(observer);
|
|
3917
|
+
};
|
|
3918
|
+
}
|
|
3381
3919
|
async runSemanticConsolidationNow(options) {
|
|
3382
3920
|
return this.runSemanticConsolidation({ ...options, force: true });
|
|
3383
3921
|
}
|
|
@@ -3433,9 +3971,18 @@ var Orchestrator = class _Orchestrator {
|
|
|
3433
3971
|
);
|
|
3434
3972
|
return result;
|
|
3435
3973
|
}
|
|
3974
|
+
let extensionsBlock = "";
|
|
3975
|
+
try {
|
|
3976
|
+
extensionsBlock = await buildExtensionsBlockForConsolidation(this.config);
|
|
3977
|
+
} catch (err) {
|
|
3978
|
+
log.warn(`[semantic-consolidation] extension discovery failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`);
|
|
3979
|
+
}
|
|
3436
3980
|
for (const cluster of clusters) {
|
|
3437
3981
|
try {
|
|
3438
|
-
|
|
3982
|
+
let prompt = buildConsolidationPrompt(cluster);
|
|
3983
|
+
if (extensionsBlock.length > 0) {
|
|
3984
|
+
prompt += "\n\n" + extensionsBlock;
|
|
3985
|
+
}
|
|
3439
3986
|
const messages = [
|
|
3440
3987
|
{
|
|
3441
3988
|
role: "system",
|
|
@@ -3496,7 +4043,14 @@ var Orchestrator = class _Orchestrator {
|
|
|
3496
4043
|
});
|
|
3497
4044
|
if (archiveResult) {
|
|
3498
4045
|
if (this.contentHashIndex) {
|
|
3499
|
-
|
|
4046
|
+
if (m.frontmatter.contentHash) {
|
|
4047
|
+
this.contentHashIndex.removeByHash(m.frontmatter.contentHash);
|
|
4048
|
+
} else {
|
|
4049
|
+
log.warn(
|
|
4050
|
+
`[semantic-consolidation] removing hash for legacy memory ${m.frontmatter.id ?? "(unknown)"} via content fallback \u2014 no contentHash in frontmatter`
|
|
4051
|
+
);
|
|
4052
|
+
this.contentHashIndex.remove(m.content);
|
|
4053
|
+
}
|
|
3500
4054
|
}
|
|
3501
4055
|
await this.embeddingFallback.removeFromIndex(m.frontmatter.id);
|
|
3502
4056
|
if (this.config.queryAwareIndexingEnabled && m.path && m.frontmatter?.created) {
|
|
@@ -3530,6 +4084,16 @@ var Orchestrator = class _Orchestrator {
|
|
|
3530
4084
|
log.info(
|
|
3531
4085
|
`[semantic-consolidation] complete: clusters=${result.clustersFound}, consolidated=${result.memoriesConsolidated}, archived=${result.memoriesArchived}, errors=${result.errors}`
|
|
3532
4086
|
);
|
|
4087
|
+
try {
|
|
4088
|
+
await materializeAfterSemanticConsolidation({
|
|
4089
|
+
config: this.config,
|
|
4090
|
+
memoryDir: this.config.memoryDir
|
|
4091
|
+
});
|
|
4092
|
+
} catch (err) {
|
|
4093
|
+
log.warn(
|
|
4094
|
+
`[semantic-consolidation] Codex materialize post-hook failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`
|
|
4095
|
+
);
|
|
4096
|
+
}
|
|
3533
4097
|
return result;
|
|
3534
4098
|
}
|
|
3535
4099
|
async waitForExtractionIdle(timeoutMs = 6e4) {
|
|
@@ -3558,6 +4122,116 @@ var Orchestrator = class _Orchestrator {
|
|
|
3558
4122
|
const ns = namespace && namespace.length > 0 ? namespace : this.config.defaultNamespace;
|
|
3559
4123
|
return this.storageRouter.storageFor(ns);
|
|
3560
4124
|
}
|
|
4125
|
+
async processEntitySynthesisQueue(namespace, maxEntities = 5) {
|
|
4126
|
+
if (!this.config.entitySummaryEnabled || maxEntities <= 0 || this.config.entitySynthesisMaxTokens <= 0) return 0;
|
|
4127
|
+
const storage = await this.getStorage(namespace);
|
|
4128
|
+
const queued = await storage.refreshEntitySynthesisQueue();
|
|
4129
|
+
let processed = 0;
|
|
4130
|
+
let attempted = 0;
|
|
4131
|
+
for (const entityName of queued) {
|
|
4132
|
+
if (attempted >= maxEntities) break;
|
|
4133
|
+
attempted += 1;
|
|
4134
|
+
try {
|
|
4135
|
+
const raw = await storage.readEntity(entityName);
|
|
4136
|
+
if (!raw) continue;
|
|
4137
|
+
const entity = parseEntityFile(raw, this.config.entitySchemas);
|
|
4138
|
+
const previousSynthesis = entity.synthesis || entity.summary || "";
|
|
4139
|
+
const sortedTimelineEntries = entity.timeline.slice().sort((left, right) => compareEntityTimestamps(right.timestamp, left.timestamp));
|
|
4140
|
+
const newerTimelineEntries = sortedTimelineEntries.filter(
|
|
4141
|
+
(entry) => !entity.synthesisUpdatedAt || compareEntityTimestamps(entry.timestamp, entity.synthesisUpdatedAt) > 0
|
|
4142
|
+
);
|
|
4143
|
+
const appendedTimelineEntries = entity.synthesisTimelineCount === void 0 ? [] : entity.timeline.slice(Math.max(0, entity.synthesisTimelineCount));
|
|
4144
|
+
const structuredEvidenceEntries = flattenStructuredSectionEvidence(entity.structuredSections);
|
|
4145
|
+
const structuredEvidenceCount = structuredEvidenceEntries.length;
|
|
4146
|
+
const structuredEvidenceDigest = fingerprintEntityStructuredFacts(entity);
|
|
4147
|
+
const structuredEvidenceDrifted = structuredEvidenceDigest !== (entity.synthesisStructuredFactDigest?.trim() || void 0);
|
|
4148
|
+
const appendedStructuredEvidenceEntries = entity.synthesisStructuredFactCount === void 0 || structuredEvidenceDrifted ? structuredEvidenceEntries : structuredEvidenceEntries.slice(Math.max(0, entity.synthesisStructuredFactCount));
|
|
4149
|
+
const candidateEvidenceEntries = [
|
|
4150
|
+
...newerTimelineEntries,
|
|
4151
|
+
...appendedTimelineEntries,
|
|
4152
|
+
...appendedStructuredEvidenceEntries
|
|
4153
|
+
].slice().sort((left, right) => compareEntityTimestamps(right.timestamp, left.timestamp));
|
|
4154
|
+
const dedupedEvidenceEntries = dedupeEntitySynthesisEvidenceEntries(
|
|
4155
|
+
candidateEvidenceEntries.length > 0 ? candidateEvidenceEntries : [...sortedTimelineEntries, ...structuredEvidenceEntries]
|
|
4156
|
+
);
|
|
4157
|
+
const chronologicalEvidenceEntries = dedupedEvidenceEntries.slice().sort((left, right) => compareEntityTimestamps(left.timestamp, right.timestamp));
|
|
4158
|
+
if (chronologicalEvidenceEntries.length === 0) continue;
|
|
4159
|
+
const latestEvidenceTimestamp = chronologicalEvidenceEntries.slice().reverse().map((entry) => entry.timestamp?.trim() || void 0).find((timestamp) => Boolean(timestamp));
|
|
4160
|
+
const previousSynthesisUpdatedAt = entity.synthesisUpdatedAt?.trim() || void 0;
|
|
4161
|
+
const nextSynthesisUpdatedAt = compareEntityTimestamps(
|
|
4162
|
+
latestEvidenceTimestamp,
|
|
4163
|
+
previousSynthesisUpdatedAt
|
|
4164
|
+
) >= 0 ? latestEvidenceTimestamp : previousSynthesisUpdatedAt;
|
|
4165
|
+
const evidenceBatches = [];
|
|
4166
|
+
for (let index = 0; index < chronologicalEvidenceEntries.length; index += 8) {
|
|
4167
|
+
evidenceBatches.push(chronologicalEvidenceEntries.slice(index, index + 8));
|
|
4168
|
+
}
|
|
4169
|
+
let nextSynthesis = previousSynthesis;
|
|
4170
|
+
let batchFailed = false;
|
|
4171
|
+
for (const evidenceEntries of evidenceBatches) {
|
|
4172
|
+
const evidenceText = evidenceEntries.map((entry) => {
|
|
4173
|
+
const sectionTitle = entry.source?.startsWith("section:") ? entry.source.slice("section:".length) : "";
|
|
4174
|
+
const metadata = [
|
|
4175
|
+
`timestamp=${entry.timestamp}`,
|
|
4176
|
+
sectionTitle ? `section=${sectionTitle}` : entry.source ? `source=${entry.source}` : "",
|
|
4177
|
+
entry.sessionKey ? `session=${entry.sessionKey}` : "",
|
|
4178
|
+
entry.principal ? `principal=${entry.principal}` : ""
|
|
4179
|
+
].filter(Boolean).join(", ");
|
|
4180
|
+
return `- ${metadata}: ${entry.text}`;
|
|
4181
|
+
}).join("\n");
|
|
4182
|
+
const response = await this.fastChatCompletion(
|
|
4183
|
+
[
|
|
4184
|
+
{
|
|
4185
|
+
role: "system",
|
|
4186
|
+
content: "Rewrite the entity synthesis as compact current truth. Preserve uncertainty when evidence conflicts. Return plain text only."
|
|
4187
|
+
},
|
|
4188
|
+
{
|
|
4189
|
+
role: "user",
|
|
4190
|
+
content: [
|
|
4191
|
+
`Entity: ${entity.name} (${entity.type})`,
|
|
4192
|
+
nextSynthesis ? `Previous synthesis:
|
|
4193
|
+
${nextSynthesis}` : "Previous synthesis: none",
|
|
4194
|
+
`New evidence:
|
|
4195
|
+
${evidenceText}`
|
|
4196
|
+
].join("\n\n")
|
|
4197
|
+
}
|
|
4198
|
+
],
|
|
4199
|
+
{
|
|
4200
|
+
temperature: 0.2,
|
|
4201
|
+
maxTokens: this.config.entitySynthesisMaxTokens,
|
|
4202
|
+
operation: "entity_summary",
|
|
4203
|
+
priority: "background"
|
|
4204
|
+
}
|
|
4205
|
+
);
|
|
4206
|
+
const synthesis = response?.content?.trim().replace(/^["']|["']$/g, "");
|
|
4207
|
+
const maxSynthesisChars = Math.max(2e3, this.config.entitySynthesisMaxTokens * 8);
|
|
4208
|
+
if (!synthesis || synthesis.length < 10 || synthesis.length > maxSynthesisChars) {
|
|
4209
|
+
batchFailed = true;
|
|
4210
|
+
break;
|
|
4211
|
+
}
|
|
4212
|
+
nextSynthesis = synthesis;
|
|
4213
|
+
}
|
|
4214
|
+
if (batchFailed || nextSynthesis.length === 0) continue;
|
|
4215
|
+
const latestRaw = await storage.readEntity(entityName);
|
|
4216
|
+
if (!latestRaw) continue;
|
|
4217
|
+
const latestEntity = parseEntityFile(latestRaw, this.config.entitySchemas);
|
|
4218
|
+
if (fingerprintEntitySynthesisEvidence(latestEntity) !== fingerprintEntitySynthesisEvidence(entity)) {
|
|
4219
|
+
continue;
|
|
4220
|
+
}
|
|
4221
|
+
await storage.updateEntitySynthesis(entityName, nextSynthesis, {
|
|
4222
|
+
entityUpdatedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
4223
|
+
synthesisStructuredFactDigest: structuredEvidenceDigest,
|
|
4224
|
+
synthesisStructuredFactCount: structuredEvidenceCount,
|
|
4225
|
+
synthesisTimelineCount: entity.timeline.length,
|
|
4226
|
+
updatedAt: nextSynthesisUpdatedAt
|
|
4227
|
+
});
|
|
4228
|
+
processed += 1;
|
|
4229
|
+
} catch (err) {
|
|
4230
|
+
log.debug(`entity synthesis refresh failed for ${entityName}: ${err}`);
|
|
4231
|
+
}
|
|
4232
|
+
}
|
|
4233
|
+
return processed;
|
|
4234
|
+
}
|
|
3561
4235
|
async generateDaySummary(memories) {
|
|
3562
4236
|
if (this.initPromise) {
|
|
3563
4237
|
let initGateTimeoutHandle;
|
|
@@ -6555,6 +7229,22 @@ ${formatted}`;
|
|
|
6555
7229
|
});
|
|
6556
7230
|
return section;
|
|
6557
7231
|
})();
|
|
7232
|
+
const procedureRecallPromise = (async () => {
|
|
7233
|
+
if (this.config.procedural?.enabled !== true) return null;
|
|
7234
|
+
if (!this.isRecallSectionEnabled("procedure-recall", true)) return null;
|
|
7235
|
+
try {
|
|
7236
|
+
return await buildProcedureRecallSection(
|
|
7237
|
+
profileStorage,
|
|
7238
|
+
retrievalQuery,
|
|
7239
|
+
this.config
|
|
7240
|
+
);
|
|
7241
|
+
} catch (err) {
|
|
7242
|
+
log.debug(
|
|
7243
|
+
`procedure-recall: failed open: ${err instanceof Error ? err.message : String(err)}`
|
|
7244
|
+
);
|
|
7245
|
+
return null;
|
|
7246
|
+
}
|
|
7247
|
+
})();
|
|
6558
7248
|
const compoundingPromise = observeEnrichmentPromise(
|
|
6559
7249
|
(async () => {
|
|
6560
7250
|
const t0 = Date.now();
|
|
@@ -6619,6 +7309,7 @@ ${formatted}`;
|
|
|
6619
7309
|
causalTrajectorySection,
|
|
6620
7310
|
cmcCausalChainsSection,
|
|
6621
7311
|
calibrationSection,
|
|
7312
|
+
procedureRecallSection,
|
|
6622
7313
|
trustZoneSection,
|
|
6623
7314
|
verifiedRecallSection,
|
|
6624
7315
|
verifiedRulesSection,
|
|
@@ -6640,6 +7331,7 @@ ${formatted}`;
|
|
|
6640
7331
|
["causalTraj", causalTrajectoryPromise],
|
|
6641
7332
|
["cmc", cmcRetrievalPromise],
|
|
6642
7333
|
["calibration", calibrationPromise],
|
|
7334
|
+
["procedureRecall", procedureRecallPromise],
|
|
6643
7335
|
["trustZone", trustZonePromise],
|
|
6644
7336
|
["verifiedRecall", verifiedRecallPromise],
|
|
6645
7337
|
["verifiedRules", verifiedRulesPromise],
|
|
@@ -6745,6 +7437,13 @@ ${profile}`
|
|
|
6745
7437
|
calibrationSection
|
|
6746
7438
|
);
|
|
6747
7439
|
}
|
|
7440
|
+
if (procedureRecallSection) {
|
|
7441
|
+
this.appendRecallSection(
|
|
7442
|
+
sectionBuckets,
|
|
7443
|
+
"procedure-recall",
|
|
7444
|
+
procedureRecallSection
|
|
7445
|
+
);
|
|
7446
|
+
}
|
|
6748
7447
|
if (identityContinuity) {
|
|
6749
7448
|
this.appendRecallSection(
|
|
6750
7449
|
sectionBuckets,
|
|
@@ -7059,7 +7758,11 @@ ${tmtNode.summary}`
|
|
|
7059
7758
|
confidenceGateRejected = true;
|
|
7060
7759
|
}
|
|
7061
7760
|
}
|
|
7062
|
-
memoryResults =
|
|
7761
|
+
memoryResults = this.diversifyAndLimitRecallResults(
|
|
7762
|
+
"memories",
|
|
7763
|
+
memoryResults,
|
|
7764
|
+
recallResultLimit
|
|
7765
|
+
);
|
|
7063
7766
|
if (this.config.memoryReconstructionEnabled && memoryResults.length > 0) {
|
|
7064
7767
|
try {
|
|
7065
7768
|
const snippets = memoryResults.map((r) => r.snippet);
|
|
@@ -7138,11 +7841,16 @@ ${tmtNode.summary}`
|
|
|
7138
7841
|
limit: embeddingFetchLimit
|
|
7139
7842
|
}
|
|
7140
7843
|
);
|
|
7141
|
-
const
|
|
7844
|
+
const boostedScoped = await this.boostSearchResults(
|
|
7142
7845
|
scopedCandidates,
|
|
7143
7846
|
recallNamespaces,
|
|
7144
7847
|
retrievalQuery
|
|
7145
|
-
)
|
|
7848
|
+
);
|
|
7849
|
+
const scoped = this.diversifyAndLimitRecallResults(
|
|
7850
|
+
"memories",
|
|
7851
|
+
boostedScoped,
|
|
7852
|
+
recallResultLimit
|
|
7853
|
+
);
|
|
7146
7854
|
if (scoped.length > 0) {
|
|
7147
7855
|
if (shouldPersistGraphSnapshot) {
|
|
7148
7856
|
graphSnapshotFinalResults = this.buildGraphRecallRankedResults(
|
|
@@ -7253,11 +7961,16 @@ ${tmtNode.summary}`
|
|
|
7253
7961
|
limit: embeddingFetchLimit
|
|
7254
7962
|
}
|
|
7255
7963
|
);
|
|
7256
|
-
const
|
|
7964
|
+
const boostedScoped = await this.boostSearchResults(
|
|
7257
7965
|
scopedCandidates,
|
|
7258
7966
|
recallNamespaces,
|
|
7259
7967
|
retrievalQuery
|
|
7260
|
-
)
|
|
7968
|
+
);
|
|
7969
|
+
const scoped = this.diversifyAndLimitRecallResults(
|
|
7970
|
+
"memories",
|
|
7971
|
+
boostedScoped,
|
|
7972
|
+
recallResultLimit
|
|
7973
|
+
);
|
|
7261
7974
|
if (scoped.length > 0) {
|
|
7262
7975
|
if (shouldPersistGraphSnapshot) {
|
|
7263
7976
|
graphSnapshotFinalResults = this.buildGraphRecallRankedResults(
|
|
@@ -7285,8 +7998,20 @@ ${tmtNode.summary}`
|
|
|
7285
7998
|
} else {
|
|
7286
7999
|
const memories = await this.readAllMemoriesForNamespaces(recallNamespaces);
|
|
7287
8000
|
if (memories.length > 0) {
|
|
8001
|
+
const supersessionOptions = {
|
|
8002
|
+
enabled: this.config.temporalSupersessionEnabled,
|
|
8003
|
+
includeInRecall: this.config.temporalSupersessionIncludeInRecall
|
|
8004
|
+
};
|
|
7288
8005
|
const activeMemories = memories.filter(
|
|
7289
|
-
(m) =>
|
|
8006
|
+
(m) => {
|
|
8007
|
+
if (isArtifactMemoryPath(m.path)) return false;
|
|
8008
|
+
const status = m.frontmatter.status;
|
|
8009
|
+
if (!status || status === "active") return true;
|
|
8010
|
+
if (status === "superseded") {
|
|
8011
|
+
return !shouldFilterSupersededFromRecall(m.frontmatter, supersessionOptions);
|
|
8012
|
+
}
|
|
8013
|
+
return false;
|
|
8014
|
+
}
|
|
7290
8015
|
);
|
|
7291
8016
|
const queryAwareScopedMemories = queryAwarePrefilter.candidatePaths ? activeMemories.filter(
|
|
7292
8017
|
(memory) => queryAwarePrefilter.candidatePaths?.has(memory.path)
|
|
@@ -7334,12 +8059,17 @@ ${tmtNode.summary}`
|
|
|
7334
8059
|
score: 1 - i / Math.max(recentSorted.length, 1)
|
|
7335
8060
|
})
|
|
7336
8061
|
);
|
|
7337
|
-
const
|
|
8062
|
+
const boostedRecent = (await this.boostSearchResults(
|
|
7338
8063
|
recentAsResults,
|
|
7339
8064
|
recallNamespaces,
|
|
7340
8065
|
retrievalQuery,
|
|
7341
8066
|
preloadedMap
|
|
7342
|
-
)).sort((a, b) => b.score - a.score)
|
|
8067
|
+
)).sort((a, b) => b.score - a.score);
|
|
8068
|
+
const recent = this.diversifyAndLimitRecallResults(
|
|
8069
|
+
"memories",
|
|
8070
|
+
boostedRecent,
|
|
8071
|
+
recallResultLimit
|
|
8072
|
+
);
|
|
7343
8073
|
if (recent.length > 0) {
|
|
7344
8074
|
if (shouldPersistGraphSnapshot) {
|
|
7345
8075
|
graphSnapshotFinalResults = this.buildGraphRecallRankedResults(
|
|
@@ -7663,7 +8393,7 @@ _Context: ${topQuestion.context}_`
|
|
|
7663
8393
|
closeProfileTrace();
|
|
7664
8394
|
}
|
|
7665
8395
|
}
|
|
7666
|
-
async processTurn(role, content, sessionKey) {
|
|
8396
|
+
async processTurn(role, content, sessionKey, options = {}) {
|
|
7667
8397
|
if (role !== "user" && role !== "assistant") {
|
|
7668
8398
|
log.debug(`processTurn: ignoring unsupported role=${String(role)}`);
|
|
7669
8399
|
return;
|
|
@@ -7674,15 +8404,42 @@ _Context: ${topQuestion.context}_`
|
|
|
7674
8404
|
);
|
|
7675
8405
|
return;
|
|
7676
8406
|
}
|
|
8407
|
+
const bufferKey = typeof options.bufferKey === "string" && options.bufferKey.length > 0 ? options.bufferKey : typeof sessionKey === "string" && sessionKey.length > 0 ? sessionKey : "default";
|
|
7677
8408
|
const turn = {
|
|
7678
8409
|
role,
|
|
7679
8410
|
content,
|
|
7680
8411
|
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
7681
|
-
sessionKey
|
|
8412
|
+
sessionKey,
|
|
8413
|
+
logicalSessionKey: options.logicalSessionKey ?? bufferKey,
|
|
8414
|
+
providerThreadId: options.providerThreadId ?? null,
|
|
8415
|
+
turnFingerprint: options.turnFingerprint,
|
|
8416
|
+
persistProcessedFingerprint: options.persistProcessedFingerprint === true
|
|
7682
8417
|
};
|
|
7683
|
-
const decision = await this.buffer.addTurn(turn);
|
|
8418
|
+
const decision = await this.buffer.addTurn(bufferKey, turn);
|
|
7684
8419
|
if (decision === "keep_buffering") return;
|
|
7685
|
-
await this.queueBufferedExtraction(
|
|
8420
|
+
await this.queueBufferedExtraction(
|
|
8421
|
+
this.buffer.getTurns(bufferKey),
|
|
8422
|
+
"trigger_mode",
|
|
8423
|
+
{ bufferKey }
|
|
8424
|
+
);
|
|
8425
|
+
}
|
|
8426
|
+
async flushSession(sessionKey, options) {
|
|
8427
|
+
const explicitBufferKey = typeof options.bufferKey === "string" && options.bufferKey.length > 0 ? options.bufferKey : null;
|
|
8428
|
+
const discoveredBufferKeys = explicitBufferKey || typeof sessionKey !== "string" || sessionKey.length === 0 || typeof this.buffer.findBufferKeysForSession !== "function" ? [] : await this.buffer.findBufferKeysForSession(sessionKey);
|
|
8429
|
+
const bufferKeys = explicitBufferKey ? [explicitBufferKey] : discoveredBufferKeys.length > 0 ? discoveredBufferKeys : typeof sessionKey === "string" && sessionKey.length > 0 ? [sessionKey] : ["default"];
|
|
8430
|
+
for (const bufferKey of bufferKeys) {
|
|
8431
|
+
const turns = this.buffer.getTurns(bufferKey);
|
|
8432
|
+
if (turns.length === 0) continue;
|
|
8433
|
+
await new Promise((resolve, reject) => {
|
|
8434
|
+
void this.queueBufferedExtraction(turns, "trigger_mode", {
|
|
8435
|
+
bufferKey,
|
|
8436
|
+
clearBufferAfterExtraction: true,
|
|
8437
|
+
skipDedupeCheck: true,
|
|
8438
|
+
abortSignal: options.abortSignal,
|
|
8439
|
+
onTaskSettled: (error) => error ? reject(error) : resolve()
|
|
8440
|
+
}).catch(reject);
|
|
8441
|
+
});
|
|
8442
|
+
}
|
|
7686
8443
|
}
|
|
7687
8444
|
async ingestReplayBatch(turns, options = {}) {
|
|
7688
8445
|
if (!Array.isArray(turns) || turns.length === 0) return;
|
|
@@ -7706,7 +8463,7 @@ _Context: ${topQuestion.context}_`
|
|
|
7706
8463
|
bySession.set(key, list);
|
|
7707
8464
|
}
|
|
7708
8465
|
const replayTasks = [];
|
|
7709
|
-
for (const sessionTurns of bySession.
|
|
8466
|
+
for (const [key, sessionTurns] of bySession.entries()) {
|
|
7710
8467
|
if (sessionTurns.length === 0) continue;
|
|
7711
8468
|
replayTasks.push(
|
|
7712
8469
|
new Promise((resolve, reject) => {
|
|
@@ -7714,6 +8471,7 @@ _Context: ${topQuestion.context}_`
|
|
|
7714
8471
|
skipDedupeCheck: true,
|
|
7715
8472
|
clearBufferAfterExtraction: false,
|
|
7716
8473
|
skipCharThreshold: true,
|
|
8474
|
+
bufferKey: key,
|
|
7717
8475
|
extractionDeadlineMs: options.deadlineMs,
|
|
7718
8476
|
onTaskSettled: (err) => err ? reject(err) : resolve()
|
|
7719
8477
|
}).catch(reject);
|
|
@@ -7730,25 +8488,113 @@ _Context: ${topQuestion.context}_`
|
|
|
7730
8488
|
}
|
|
7731
8489
|
}
|
|
7732
8490
|
}
|
|
7733
|
-
|
|
8491
|
+
/**
|
|
8492
|
+
* Return the namespace that `ingestBulkImportBatch` writes into (#460).
|
|
8493
|
+
*
|
|
8494
|
+
* Exposed so host CLIs can snapshot the same storage root that extraction
|
|
8495
|
+
* actually writes to, avoiding the "CLI counts files at namespace A while
|
|
8496
|
+
* writes land in namespace B" footgun that a naïve
|
|
8497
|
+
* `config.defaultNamespace` snapshot could hit when a namespace policy
|
|
8498
|
+
* named `"default"` also exists.
|
|
8499
|
+
*
|
|
8500
|
+
* Today bulk-import is pinned to `config.defaultNamespace`; future
|
|
8501
|
+
* per-invocation namespace routing would thread an explicit target here
|
|
8502
|
+
* and through `ingestBulkImportBatch`.
|
|
8503
|
+
*/
|
|
8504
|
+
bulkImportWriteNamespace() {
|
|
8505
|
+
return this.config.defaultNamespace;
|
|
8506
|
+
}
|
|
8507
|
+
/**
|
|
8508
|
+
* Ingest a batch of bulk-import turns (#460). Like ingestReplayBatch, this
|
|
8509
|
+
* normalizes user/assistant turns into the extraction buffer and awaits
|
|
8510
|
+
* settlement, but it intentionally bypasses the captureMode="explicit"
|
|
8511
|
+
* gate because bulk-import is itself an explicit user action — the user
|
|
8512
|
+
* ran `bulk-import --source <name> --file ...` and would be surprised to
|
|
8513
|
+
* see the command silently no-op when capture is otherwise restricted.
|
|
8514
|
+
*
|
|
8515
|
+
* Turns with role="other" are skipped (not supported by the extraction
|
|
8516
|
+
* pipeline).
|
|
8517
|
+
*
|
|
8518
|
+
* Two design decisions worth calling out:
|
|
8519
|
+
*
|
|
8520
|
+
* - **sessionKey is truthy and per-batch-unique.**
|
|
8521
|
+
* `ThreadingManager.shouldStartNewThread` only applies the session-key
|
|
8522
|
+
* boundary check when `turn.sessionKey` is truthy (threading.ts:82);
|
|
8523
|
+
* with an empty string, imported turns could attach to the current
|
|
8524
|
+
* live thread or merge across unrelated import batches. A unique
|
|
8525
|
+
* `bulk-import:batch:<timestamp>-<rand>` key forces a fresh thread per
|
|
8526
|
+
* batch without matching common prefix/map rules in
|
|
8527
|
+
* `principalFromSessionKeyRules`. (Catch-all regex rules could still
|
|
8528
|
+
* remap the principal, but that only affects metadata provenance —
|
|
8529
|
+
* see the next point for why write routing is unaffected.)
|
|
8530
|
+
*
|
|
8531
|
+
* - **writeNamespaceOverride pins the storage target.**
|
|
8532
|
+
* We pass `writeNamespaceOverride: this.bulkImportWriteNamespace()` to
|
|
8533
|
+
* `queueBufferedExtraction`, which tells `runExtraction` to skip
|
|
8534
|
+
* `defaultNamespaceForPrincipal` and write directly into the
|
|
8535
|
+
* orchestrator's declared bulk-import write namespace. This keeps
|
|
8536
|
+
* writes deterministic even when namespace policies named `"default"`
|
|
8537
|
+
* exist alongside a different `config.defaultNamespace`, and also
|
|
8538
|
+
* guards against regex-catch-all principal rules steering bulk-import
|
|
8539
|
+
* into an unexpected tenant.
|
|
8540
|
+
*
|
|
8541
|
+
* Per-invocation namespace routing (letting callers target a namespace
|
|
8542
|
+
* other than `bulkImportWriteNamespace()`) is a separate feature tracked
|
|
8543
|
+
* as a follow-up — the hook is the `writeNamespaceOverride` option, but
|
|
8544
|
+
* the CLI surface does not yet expose a `--namespace` flag.
|
|
8545
|
+
*/
|
|
8546
|
+
async ingestBulkImportBatch(turns, options = {}) {
|
|
8547
|
+
if (!Array.isArray(turns) || turns.length === 0) return;
|
|
8548
|
+
const sessionKey = `bulk-import:batch:${Date.now().toString(36)}-` + randomBytes(6).toString("hex");
|
|
8549
|
+
const sessionTurns = [];
|
|
8550
|
+
for (const turn of turns) {
|
|
8551
|
+
if (turn.role !== "user" && turn.role !== "assistant") continue;
|
|
8552
|
+
sessionTurns.push({
|
|
8553
|
+
role: turn.role,
|
|
8554
|
+
content: turn.content,
|
|
8555
|
+
timestamp: turn.timestamp,
|
|
8556
|
+
sessionKey
|
|
8557
|
+
});
|
|
8558
|
+
}
|
|
8559
|
+
if (sessionTurns.length === 0) return;
|
|
8560
|
+
await new Promise((resolve, reject) => {
|
|
8561
|
+
void this.queueBufferedExtraction(sessionTurns, "trigger_mode", {
|
|
8562
|
+
skipDedupeCheck: true,
|
|
8563
|
+
clearBufferAfterExtraction: false,
|
|
8564
|
+
skipCharThreshold: true,
|
|
8565
|
+
bufferKey: sessionKey,
|
|
8566
|
+
extractionDeadlineMs: options.deadlineMs,
|
|
8567
|
+
writeNamespaceOverride: this.bulkImportWriteNamespace(),
|
|
8568
|
+
onTaskSettled: (err) => err ? reject(err) : resolve()
|
|
8569
|
+
}).catch(reject);
|
|
8570
|
+
});
|
|
8571
|
+
}
|
|
8572
|
+
async observeSessionHeartbeat(sessionKey, options = {}) {
|
|
7734
8573
|
if (this.config.sessionObserverEnabled !== true) return;
|
|
7735
8574
|
if (!sessionKey || sessionKey.length === 0) return;
|
|
8575
|
+
const bufferKey = typeof options.bufferKey === "string" && options.bufferKey.length > 0 ? options.bufferKey : sessionKey;
|
|
7736
8576
|
const previous = this.heartbeatObserverChains.get(sessionKey) ?? Promise.resolve();
|
|
7737
8577
|
const next = previous.catch(() => void 0).then(async () => {
|
|
7738
|
-
const turns = this.buffer.getTurns();
|
|
8578
|
+
const turns = this.buffer.getTurns(bufferKey);
|
|
7739
8579
|
if (turns.length === 0) return;
|
|
7740
|
-
const
|
|
7741
|
-
|
|
8580
|
+
const normalizedSessionKey = normalizeReplaySessionKey(sessionKey);
|
|
8581
|
+
const allowSharedSessionBuffer = bufferKey.startsWith(
|
|
8582
|
+
CODEX_THREAD_KEY_PREFIX
|
|
7742
8583
|
);
|
|
7743
|
-
if (
|
|
8584
|
+
if (!allowSharedSessionBuffer && turns.some(
|
|
8585
|
+
(turn) => turn.sessionKey && normalizeReplaySessionKey(turn.sessionKey) !== normalizedSessionKey
|
|
8586
|
+
)) {
|
|
7744
8587
|
log.debug(
|
|
7745
|
-
`heartbeat observer skipped: mixed
|
|
8588
|
+
`heartbeat observer skipped: mixed-session buffer contents for ${bufferKey}`
|
|
7746
8589
|
);
|
|
7747
8590
|
return;
|
|
7748
8591
|
}
|
|
7749
|
-
if (!this.shouldQueueExtraction(turns, {
|
|
8592
|
+
if (!this.shouldQueueExtraction(turns, {
|
|
8593
|
+
commit: false,
|
|
8594
|
+
bufferKey
|
|
8595
|
+
})) {
|
|
7750
8596
|
log.debug(
|
|
7751
|
-
`heartbeat observer skipped: extraction dedupe for ${
|
|
8597
|
+
`heartbeat observer skipped: extraction dedupe for ${bufferKey}`
|
|
7752
8598
|
);
|
|
7753
8599
|
return;
|
|
7754
8600
|
}
|
|
@@ -7762,7 +8608,9 @@ _Context: ${topQuestion.context}_`
|
|
|
7762
8608
|
log.debug(
|
|
7763
8609
|
`heartbeat observer trigger: session=${sessionKey} deltaBytes=${decision.deltaBytes} deltaTokens=${decision.deltaTokens}`
|
|
7764
8610
|
);
|
|
7765
|
-
await this.queueBufferedExtraction(turns, "heartbeat_observer"
|
|
8611
|
+
await this.queueBufferedExtraction(turns, "heartbeat_observer", {
|
|
8612
|
+
bufferKey
|
|
8613
|
+
});
|
|
7766
8614
|
});
|
|
7767
8615
|
this.heartbeatObserverChains.set(sessionKey, next);
|
|
7768
8616
|
try {
|
|
@@ -7774,7 +8622,8 @@ _Context: ${topQuestion.context}_`
|
|
|
7774
8622
|
}
|
|
7775
8623
|
}
|
|
7776
8624
|
async queueBufferedExtraction(turnsToExtract, reason, options = {}) {
|
|
7777
|
-
|
|
8625
|
+
const bufferKey = options.bufferKey ?? turnsToExtract[0]?.sessionKey ?? "default";
|
|
8626
|
+
if (!options.skipDedupeCheck && !this.shouldQueueExtraction(turnsToExtract, { bufferKey })) {
|
|
7778
8627
|
log.debug(`extraction dedupe skip: preserving buffer (${reason})`);
|
|
7779
8628
|
options.onTaskSettled?.();
|
|
7780
8629
|
return;
|
|
@@ -7784,7 +8633,10 @@ _Context: ${topQuestion.context}_`
|
|
|
7784
8633
|
await this.runExtraction(turnsToExtract, {
|
|
7785
8634
|
clearBufferAfterExtraction: options.clearBufferAfterExtraction ?? true,
|
|
7786
8635
|
skipCharThreshold: options.skipCharThreshold ?? false,
|
|
7787
|
-
deadlineMs: options.extractionDeadlineMs
|
|
8636
|
+
deadlineMs: options.extractionDeadlineMs,
|
|
8637
|
+
bufferKey,
|
|
8638
|
+
abortSignal: options.abortSignal,
|
|
8639
|
+
writeNamespaceOverride: options.writeNamespaceOverride
|
|
7788
8640
|
});
|
|
7789
8641
|
options.onTaskSettled?.();
|
|
7790
8642
|
} catch (err) {
|
|
@@ -7795,20 +8647,33 @@ _Context: ${topQuestion.context}_`
|
|
|
7795
8647
|
if (!this.queueProcessing) {
|
|
7796
8648
|
this.queueProcessing = true;
|
|
7797
8649
|
this.processQueue().catch((err) => {
|
|
7798
|
-
|
|
8650
|
+
this.logExtractionQueueFailure(err, "processor");
|
|
7799
8651
|
this.queueProcessing = false;
|
|
7800
8652
|
});
|
|
7801
8653
|
}
|
|
7802
8654
|
log.debug(`queued extraction from ${reason}`);
|
|
7803
8655
|
}
|
|
8656
|
+
normalizeExtractionFingerprintTurns(turns) {
|
|
8657
|
+
if (!Array.isArray(turns) || turns.length === 0) return [];
|
|
8658
|
+
return turns.filter((turn) => turn.role === "user" || turn.role === "assistant").map((turn) => {
|
|
8659
|
+
if (typeof turn.turnFingerprint === "string" && turn.turnFingerprint.length > 0) {
|
|
8660
|
+
return `fp:${turn.turnFingerprint}`;
|
|
8661
|
+
}
|
|
8662
|
+
return `${turn.role}:${(turn.content ?? "").replace(/\s+/g, " ").trim().slice(0, this.config.extractionMaxTurnChars)}`;
|
|
8663
|
+
}).filter((value) => value.length > 0);
|
|
8664
|
+
}
|
|
8665
|
+
buildExtractionFingerprint(turns, bufferKey) {
|
|
8666
|
+
const normalized = this.normalizeExtractionFingerprintTurns(turns).join("\n");
|
|
8667
|
+
if (!normalized) return null;
|
|
8668
|
+
return createHash2("sha256").update(`${bufferKey}
|
|
8669
|
+
${normalized}`).digest("hex");
|
|
8670
|
+
}
|
|
7804
8671
|
shouldQueueExtraction(turns, options = {}) {
|
|
7805
8672
|
if (!this.config.extractionDedupeEnabled) return true;
|
|
7806
8673
|
if (!Array.isArray(turns) || turns.length === 0) return false;
|
|
7807
|
-
const
|
|
7808
|
-
|
|
7809
|
-
|
|
7810
|
-
if (!normalized) return false;
|
|
7811
|
-
const fingerprint = createHash2("sha256").update(normalized).digest("hex");
|
|
8674
|
+
const bufferKey = options.bufferKey ?? turns[0]?.sessionKey ?? "default";
|
|
8675
|
+
const fingerprint = this.buildExtractionFingerprint(turns, bufferKey);
|
|
8676
|
+
if (!fingerprint) return false;
|
|
7812
8677
|
const now = Date.now();
|
|
7813
8678
|
const seenAt = this.recentExtractionFingerprints.get(fingerprint);
|
|
7814
8679
|
if (seenAt && now - seenAt < this.config.extractionDedupeWindowMs) {
|
|
@@ -7840,25 +8705,58 @@ _Context: ${topQuestion.context}_`
|
|
|
7840
8705
|
try {
|
|
7841
8706
|
await task();
|
|
7842
8707
|
} catch (err) {
|
|
7843
|
-
|
|
8708
|
+
this.logExtractionQueueFailure(err, "task");
|
|
7844
8709
|
}
|
|
7845
8710
|
}
|
|
7846
8711
|
}
|
|
7847
8712
|
this.queueProcessing = false;
|
|
7848
8713
|
}
|
|
8714
|
+
/**
|
|
8715
|
+
* Classify + log a failure from either the per-task catch inside
|
|
8716
|
+
* `processQueue()` or the outer `processQueue().catch(...)` in
|
|
8717
|
+
* `queueBufferedExtraction()`. Issue #549: `throwIfRecallAborted`
|
|
8718
|
+
* (used throughout `runExtraction`) raises an Error whose `name` is
|
|
8719
|
+
* `"AbortError"`. That path fires when `before_reset` aborts a
|
|
8720
|
+
* queued task to avoid duplicate extraction — it is intentional
|
|
8721
|
+
* cancellation, not a failure. Downgrading the log to debug
|
|
8722
|
+
* prevents spurious `error`-level lines that routinely appear
|
|
8723
|
+
* right next to a successful `persisted: N facts, M entities` log
|
|
8724
|
+
* and that confuse operators into thinking extraction is broken.
|
|
8725
|
+
* Genuine extraction failures (network, parse, I/O) still log at
|
|
8726
|
+
* `error`.
|
|
8727
|
+
*
|
|
8728
|
+
* Source differentiates the two call sites so the log message
|
|
8729
|
+
* names the right layer (`task` vs `processor`).
|
|
8730
|
+
*/
|
|
8731
|
+
logExtractionQueueFailure(err, source) {
|
|
8732
|
+
const aborted = source === "task" ? "background extraction task aborted (session transition)" : "background extraction queue processor aborted (session transition)";
|
|
8733
|
+
const failed = source === "task" ? "background extraction task failed" : "background extraction queue processor failed";
|
|
8734
|
+
if (isAbortError(err)) {
|
|
8735
|
+
log.debug(aborted);
|
|
8736
|
+
} else {
|
|
8737
|
+
log.error(failed, err);
|
|
8738
|
+
}
|
|
8739
|
+
}
|
|
7849
8740
|
async runExtraction(turns, options = {}) {
|
|
7850
8741
|
log.debug(`running extraction on ${turns.length} turns`);
|
|
7851
8742
|
const clearBufferAfterExtraction = options.clearBufferAfterExtraction ?? true;
|
|
7852
8743
|
const skipCharThreshold = options.skipCharThreshold ?? false;
|
|
7853
8744
|
const deadlineMs = typeof options.deadlineMs === "number" && Number.isFinite(options.deadlineMs) ? options.deadlineMs : void 0;
|
|
8745
|
+
const bufferKey = options.bufferKey ?? turns[0]?.sessionKey ?? "default";
|
|
7854
8746
|
const throwIfDeadlineExceeded = (stage) => {
|
|
7855
8747
|
if (typeof deadlineMs === "number" && Date.now() > deadlineMs) {
|
|
7856
8748
|
throw new Error(`replay extraction deadline exceeded (${stage})`);
|
|
7857
8749
|
}
|
|
7858
8750
|
};
|
|
7859
|
-
const
|
|
8751
|
+
const throwIfAborted2 = (stage) => {
|
|
8752
|
+
throwIfRecallAborted(options.abortSignal, `extraction aborted (${stage})`);
|
|
8753
|
+
};
|
|
8754
|
+
const clearBuffer = async (options2) => {
|
|
8755
|
+
if (options2?.ignoreAbort !== true) {
|
|
8756
|
+
throwIfAborted2("before_clear_buffer");
|
|
8757
|
+
}
|
|
7860
8758
|
if (clearBufferAfterExtraction) {
|
|
7861
|
-
await this.buffer.clearAfterExtraction();
|
|
8759
|
+
await this.buffer.clearAfterExtraction(bufferKey);
|
|
7862
8760
|
}
|
|
7863
8761
|
};
|
|
7864
8762
|
const sessionKey = turns[0]?.sessionKey ?? "";
|
|
@@ -7874,6 +8772,7 @@ _Context: ${topQuestion.context}_`
|
|
|
7874
8772
|
content: t.content.trim().slice(0, this.config.extractionMaxTurnChars)
|
|
7875
8773
|
})).filter((t) => t.content.length > 0);
|
|
7876
8774
|
throwIfDeadlineExceeded("before_extract");
|
|
8775
|
+
throwIfAborted2("before_extract");
|
|
7877
8776
|
const userTurns = normalizedTurns.filter((t) => t.role === "user");
|
|
7878
8777
|
const totalChars = normalizedTurns.reduce(
|
|
7879
8778
|
(sum, t) => sum + t.content.length,
|
|
@@ -7889,14 +8788,36 @@ _Context: ${topQuestion.context}_`
|
|
|
7889
8788
|
return;
|
|
7890
8789
|
}
|
|
7891
8790
|
const principal = resolvePrincipal(sessionKey, this.config);
|
|
7892
|
-
const selfNamespace = defaultNamespaceForPrincipal(principal, this.config);
|
|
8791
|
+
const selfNamespace = typeof options.writeNamespaceOverride === "string" && options.writeNamespaceOverride.length > 0 ? options.writeNamespaceOverride : defaultNamespaceForPrincipal(principal, this.config);
|
|
7893
8792
|
const storage = await this.storageRouter.storageFor(selfNamespace);
|
|
7894
|
-
const
|
|
7895
|
-
|
|
8793
|
+
const shouldPersistProcessedFingerprint = normalizedTurns.some(
|
|
8794
|
+
(turn) => turn.persistProcessedFingerprint === true
|
|
8795
|
+
);
|
|
8796
|
+
const extractionFingerprint = this.buildExtractionFingerprint(
|
|
7896
8797
|
normalizedTurns,
|
|
7897
|
-
|
|
8798
|
+
bufferKey
|
|
8799
|
+
);
|
|
8800
|
+
let meta = extractionFingerprint && shouldPersistProcessedFingerprint ? await storage.loadMeta() : null;
|
|
8801
|
+
if (extractionFingerprint && shouldPersistProcessedFingerprint && (meta?.processedExtractionFingerprints ?? []).some(
|
|
8802
|
+
(entry) => entry.fingerprint === extractionFingerprint
|
|
8803
|
+
)) {
|
|
8804
|
+
log.debug(
|
|
8805
|
+
`runExtraction: skipping already-processed extraction fingerprint for ${bufferKey}`
|
|
8806
|
+
);
|
|
8807
|
+
await clearBuffer();
|
|
8808
|
+
return;
|
|
8809
|
+
}
|
|
8810
|
+
const existingEntities = await storage.listEntityNames();
|
|
8811
|
+
const result = await raceRecallAbort(
|
|
8812
|
+
this.extraction.extract(
|
|
8813
|
+
normalizedTurns,
|
|
8814
|
+
existingEntities
|
|
8815
|
+
),
|
|
8816
|
+
options.abortSignal,
|
|
8817
|
+
"extraction aborted (during_extract)"
|
|
7898
8818
|
);
|
|
7899
8819
|
throwIfDeadlineExceeded("before_persist");
|
|
8820
|
+
throwIfAborted2("before_persist");
|
|
7900
8821
|
if (!result) {
|
|
7901
8822
|
log.warn("runExtraction: extraction returned null/undefined");
|
|
7902
8823
|
await clearBuffer();
|
|
@@ -7932,9 +8853,35 @@ _Context: ${topQuestion.context}_`
|
|
|
7932
8853
|
const persistedIds = await this.persistExtraction(
|
|
7933
8854
|
result,
|
|
7934
8855
|
storage,
|
|
7935
|
-
threadIdForExtraction
|
|
8856
|
+
threadIdForExtraction,
|
|
8857
|
+
{ sessionKey, principal }
|
|
7936
8858
|
);
|
|
7937
|
-
await
|
|
8859
|
+
meta ??= await storage.loadMeta();
|
|
8860
|
+
if (extractionFingerprint && shouldPersistProcessedFingerprint) {
|
|
8861
|
+
try {
|
|
8862
|
+
await this.recordProcessedExtractionFingerprint(
|
|
8863
|
+
storage,
|
|
8864
|
+
extractionFingerprint,
|
|
8865
|
+
meta
|
|
8866
|
+
);
|
|
8867
|
+
} catch (error) {
|
|
8868
|
+
log.warn(
|
|
8869
|
+
"runExtraction: failed to persist processed extraction fingerprint; continuing with buffer clear",
|
|
8870
|
+
error
|
|
8871
|
+
);
|
|
8872
|
+
}
|
|
8873
|
+
}
|
|
8874
|
+
meta.extractionCount += 1;
|
|
8875
|
+
meta.lastExtractionAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
8876
|
+
meta.totalMemories += Array.isArray(result?.facts) ? result.facts.length : 0;
|
|
8877
|
+
meta.totalEntities += Array.isArray(result?.entities) ? result.entities.length : 0;
|
|
8878
|
+
let postPersistMetaError;
|
|
8879
|
+
try {
|
|
8880
|
+
await storage.saveMeta(meta);
|
|
8881
|
+
} catch (error) {
|
|
8882
|
+
postPersistMetaError = error;
|
|
8883
|
+
}
|
|
8884
|
+
await clearBuffer({ ignoreAbort: true });
|
|
7938
8885
|
if (this.config.memoryBoxesEnabled && persistedIds.length > 0) {
|
|
7939
8886
|
const extractionTopics = deriveTopicsFromExtraction(result);
|
|
7940
8887
|
const firstUserTurn = turns.find((t) => t.role === "user");
|
|
@@ -7971,14 +8918,26 @@ _Context: ${topQuestion.context}_`
|
|
|
7971
8918
|
const nonZeroExtraction = result.facts.length > 0 || result.entities.length > 0 || result.questions.length > 0 || result.profileUpdates.length > 0;
|
|
7972
8919
|
if (nonZeroExtraction) this.nonZeroExtractionsSinceConsolidation += 1;
|
|
7973
8920
|
this.maybeScheduleConsolidation(nonZeroExtraction);
|
|
7974
|
-
const meta = await storage.loadMeta();
|
|
7975
|
-
meta.extractionCount += 1;
|
|
7976
|
-
meta.lastExtractionAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
7977
|
-
meta.totalMemories += Array.isArray(result?.facts) ? result.facts.length : 0;
|
|
7978
|
-
meta.totalEntities += Array.isArray(result?.entities) ? result.entities.length : 0;
|
|
7979
|
-
await storage.saveMeta(meta);
|
|
7980
8921
|
this.requestQmdMaintenance();
|
|
7981
8922
|
await this.runTierMigrationCycle(storage, "extraction");
|
|
8923
|
+
if (postPersistMetaError) {
|
|
8924
|
+
throw postPersistMetaError;
|
|
8925
|
+
}
|
|
8926
|
+
}
|
|
8927
|
+
async recordProcessedExtractionFingerprint(storage, fingerprint, preloadedMeta) {
|
|
8928
|
+
const meta = preloadedMeta ?? await storage.loadMeta();
|
|
8929
|
+
const observedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
8930
|
+
const seen = new Map(
|
|
8931
|
+
(meta.processedExtractionFingerprints ?? []).map((entry) => [
|
|
8932
|
+
entry.fingerprint,
|
|
8933
|
+
entry.observedAt
|
|
8934
|
+
])
|
|
8935
|
+
);
|
|
8936
|
+
seen.set(fingerprint, observedAt);
|
|
8937
|
+
meta.processedExtractionFingerprints = Array.from(seen.entries()).map(([value, at]) => ({ fingerprint: value, observedAt: at })).sort((left, right) => left.observedAt.localeCompare(right.observedAt)).slice(-500);
|
|
8938
|
+
if (!preloadedMeta) {
|
|
8939
|
+
await storage.saveMeta(meta);
|
|
8940
|
+
}
|
|
7982
8941
|
}
|
|
7983
8942
|
async runTierMigrationCycle(storage, trigger, options) {
|
|
7984
8943
|
const dryRun = options?.dryRun === true;
|
|
@@ -8222,7 +9181,22 @@ _Context: ${topQuestion.context}_`
|
|
|
8222
9181
|
}
|
|
8223
9182
|
}
|
|
8224
9183
|
}
|
|
8225
|
-
async persistExtraction(result, storage, threadIdForExtraction) {
|
|
9184
|
+
async persistExtraction(result, storage, threadIdForExtraction, sourceContext) {
|
|
9185
|
+
const citationEnabled = this.config.inlineSourceAttributionEnabled === true;
|
|
9186
|
+
const citationTemplate = this.config.inlineSourceAttributionFormat;
|
|
9187
|
+
const citationContextBase = citationEnabled ? {
|
|
9188
|
+
agent: sourceContext?.principal,
|
|
9189
|
+
session: sourceContext?.sessionKey
|
|
9190
|
+
} : {};
|
|
9191
|
+
const applyInlineCitation = (content) => {
|
|
9192
|
+
if (!citationEnabled) return content;
|
|
9193
|
+
if (typeof content !== "string" || content.length === 0) return content;
|
|
9194
|
+
const citationContext = {
|
|
9195
|
+
...citationContextBase,
|
|
9196
|
+
ts: (/* @__PURE__ */ new Date()).toISOString()
|
|
9197
|
+
};
|
|
9198
|
+
return attachCitation(content, citationContext, citationTemplate);
|
|
9199
|
+
};
|
|
8226
9200
|
const persistedIds = [];
|
|
8227
9201
|
const persistedIdsByStorage = /* @__PURE__ */ new Map();
|
|
8228
9202
|
const trackPersistedId = (targetStorage, id, options = {}) => {
|
|
@@ -8238,6 +9212,8 @@ _Context: ${topQuestion.context}_`
|
|
|
8238
9212
|
persistedIdsByStorage.set(key, { storage: targetStorage, ids: [id] });
|
|
8239
9213
|
};
|
|
8240
9214
|
let dedupedCount = 0;
|
|
9215
|
+
let importanceGatedCount = 0;
|
|
9216
|
+
let batchBackendUnavailable = false;
|
|
8241
9217
|
const behaviorSignalsByStorage = /* @__PURE__ */ new Map();
|
|
8242
9218
|
const trackBehaviorSignals = (targetStorage, events) => {
|
|
8243
9219
|
if (events.length === 0) return;
|
|
@@ -8284,16 +9260,70 @@ _Context: ${topQuestion.context}_`
|
|
|
8284
9260
|
const sharedStorage = await this.storageRouter.storageFor(
|
|
8285
9261
|
this.config.sharedNamespace
|
|
8286
9262
|
);
|
|
8287
|
-
|
|
8288
|
-
|
|
9263
|
+
const rawContent = citationEnabled && hasCitationForTemplate(options.content, citationTemplate) ? stripCitationForTemplate(options.content, citationTemplate) : options.content;
|
|
9264
|
+
const citedContent = applyInlineCitation(rawContent);
|
|
9265
|
+
const sanitizedBase = sanitizeMemoryContent(rawContent);
|
|
9266
|
+
const dedupContent = options.category === "fact" && options.structuredAttributes && Object.keys(options.structuredAttributes).length > 0 ? `${sanitizedBase.text}
|
|
9267
|
+
[Attributes: ${normalizeAttributePairs(options.structuredAttributes)}]` : sanitizedBase.text;
|
|
9268
|
+
if (options.category === "fact" && await sharedStorage.hasFactContentHash(dedupContent)) {
|
|
9269
|
+
if (this.config.temporalSupersessionEnabled && options.entityRef && options.structuredAttributes && Object.keys(options.structuredAttributes).length > 0) {
|
|
9270
|
+
let hashDedupMatchingFact;
|
|
9271
|
+
let hashDedupLookupComplete = false;
|
|
9272
|
+
try {
|
|
9273
|
+
const normalizedIncoming = ContentHashIndex.normalizeContent(dedupContent);
|
|
9274
|
+
const allShared = await sharedStorage.readAllMemories();
|
|
9275
|
+
const incomingEntityNorm = normalizeSupersessionKey(options.entityRef);
|
|
9276
|
+
hashDedupMatchingFact = allShared.find((m) => {
|
|
9277
|
+
if (m.frontmatter.category !== "fact") return false;
|
|
9278
|
+
if ((m.frontmatter.status ?? "active") !== "active") return false;
|
|
9279
|
+
if (!m.frontmatter.entityRef) return false;
|
|
9280
|
+
if (normalizeSupersessionKey(m.frontmatter.entityRef) !== incomingEntityNorm) {
|
|
9281
|
+
log.debug(
|
|
9282
|
+
`persistExtraction: hash-dedup skipping cross-entity match (incoming="${incomingEntityNorm}" candidate="${normalizeSupersessionKey(m.frontmatter.entityRef)}")`
|
|
9283
|
+
);
|
|
9284
|
+
return false;
|
|
9285
|
+
}
|
|
9286
|
+
return ContentHashIndex.normalizeContent(m.content ?? "") === normalizedIncoming;
|
|
9287
|
+
});
|
|
9288
|
+
hashDedupLookupComplete = true;
|
|
9289
|
+
if (hashDedupMatchingFact) {
|
|
9290
|
+
await applyTemporalSupersession({
|
|
9291
|
+
storage: sharedStorage,
|
|
9292
|
+
newMemoryId: hashDedupMatchingFact.frontmatter.id,
|
|
9293
|
+
entityRef: options.entityRef,
|
|
9294
|
+
structuredAttributes: options.structuredAttributes,
|
|
9295
|
+
createdAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
9296
|
+
enabled: true,
|
|
9297
|
+
useCallerTimestamp: true
|
|
9298
|
+
});
|
|
9299
|
+
return;
|
|
9300
|
+
}
|
|
9301
|
+
log.debug(
|
|
9302
|
+
`persistExtraction: hash-dedup found no active same-entity shared fact for ${options.sourceMemoryId}; falling through to write`
|
|
9303
|
+
);
|
|
9304
|
+
} catch (hashDedupSupersessionErr) {
|
|
9305
|
+
log.warn(
|
|
9306
|
+
`persistExtraction: shared-namespace supersession on hash-dedup path failed open for ${options.sourceMemoryId}: ${hashDedupSupersessionErr}`
|
|
9307
|
+
);
|
|
9308
|
+
if (hashDedupLookupComplete && hashDedupMatchingFact) {
|
|
9309
|
+
return;
|
|
9310
|
+
}
|
|
9311
|
+
log.debug(
|
|
9312
|
+
`persistExtraction: hash-dedup catch: lookup incomplete or no candidate found for ${options.sourceMemoryId}; falling through to write`
|
|
9313
|
+
);
|
|
9314
|
+
}
|
|
9315
|
+
} else {
|
|
9316
|
+
return;
|
|
9317
|
+
}
|
|
8289
9318
|
}
|
|
8290
9319
|
const promotedId = await sharedStorage.writeMemory(
|
|
8291
9320
|
options.category,
|
|
8292
|
-
|
|
9321
|
+
citedContent,
|
|
8293
9322
|
{
|
|
8294
9323
|
confidence: options.confidence,
|
|
8295
9324
|
tags: [...options.tags, "shared-promotion"],
|
|
8296
9325
|
entityRef: options.entityRef,
|
|
9326
|
+
structuredAttributes: options.structuredAttributes,
|
|
8297
9327
|
source: `${options.source}-shared-promotion`,
|
|
8298
9328
|
importance: options.importance,
|
|
8299
9329
|
lineage: [options.sourceMemoryId],
|
|
@@ -8301,9 +9331,30 @@ _Context: ${topQuestion.context}_`
|
|
|
8301
9331
|
intentGoal: options.intentGoal,
|
|
8302
9332
|
intentActionType: options.intentActionType,
|
|
8303
9333
|
intentEntityTypes: options.intentEntityTypes,
|
|
8304
|
-
memoryKind: options.memoryKind
|
|
9334
|
+
memoryKind: options.memoryKind,
|
|
9335
|
+
// Index the RAW content hash so hasFactContentHash(rawContent)
|
|
9336
|
+
// returns true on subsequent extractions. Without this, the index
|
|
9337
|
+
// would record the hash of citedContent (which changes every call
|
|
9338
|
+
// due to an updated timestamp), causing duplicate promotions.
|
|
9339
|
+
contentHashSource: rawContent
|
|
8305
9340
|
}
|
|
8306
9341
|
);
|
|
9342
|
+
if (this.config.temporalSupersessionEnabled && options.entityRef && options.structuredAttributes && Object.keys(options.structuredAttributes).length > 0) {
|
|
9343
|
+
try {
|
|
9344
|
+
await applyTemporalSupersession({
|
|
9345
|
+
storage: sharedStorage,
|
|
9346
|
+
newMemoryId: promotedId,
|
|
9347
|
+
entityRef: options.entityRef,
|
|
9348
|
+
structuredAttributes: options.structuredAttributes,
|
|
9349
|
+
createdAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
9350
|
+
enabled: true
|
|
9351
|
+
});
|
|
9352
|
+
} catch (sharedSupersessionErr) {
|
|
9353
|
+
log.warn(
|
|
9354
|
+
`persistExtraction: shared-namespace temporal supersession failed open for promoted ${promotedId}: ${sharedSupersessionErr}`
|
|
9355
|
+
);
|
|
9356
|
+
}
|
|
9357
|
+
}
|
|
8307
9358
|
trackPersistedId(sharedStorage, promotedId, {
|
|
8308
9359
|
includeReturnedIds: false
|
|
8309
9360
|
});
|
|
@@ -8391,7 +9442,86 @@ _Context: ${topQuestion.context}_`
|
|
|
8391
9442
|
}
|
|
8392
9443
|
const routeRules = await this.loadRoutingRules();
|
|
8393
9444
|
const routeOptions = this.routeEngineOptions();
|
|
9445
|
+
const preRoutedCategories = new Array(facts.length);
|
|
9446
|
+
if (routeRules.length > 0) {
|
|
9447
|
+
for (let fi = 0; fi < facts.length; fi++) {
|
|
9448
|
+
const f = facts[fi];
|
|
9449
|
+
if (!f || typeof f.content !== "string" || !f.content.trim() || typeof f.category !== "string" || !f.category.trim()) {
|
|
9450
|
+
continue;
|
|
9451
|
+
}
|
|
9452
|
+
try {
|
|
9453
|
+
const tags = Array.isArray(f.tags) ? f.tags : [];
|
|
9454
|
+
const routeText = `${f.category} ${tags.join(" ")} ${f.content}`;
|
|
9455
|
+
const selected = selectRouteRule(routeText, routeRules, routeOptions);
|
|
9456
|
+
if (selected?.target.category) {
|
|
9457
|
+
preRoutedCategories[fi] = selected.target.category;
|
|
9458
|
+
}
|
|
9459
|
+
} catch {
|
|
9460
|
+
}
|
|
9461
|
+
}
|
|
9462
|
+
}
|
|
9463
|
+
let judgeVerdictsByFactIndex = null;
|
|
9464
|
+
let judgeGatedCount = 0;
|
|
9465
|
+
if (this.config.extractionJudgeEnabled) {
|
|
9466
|
+
try {
|
|
9467
|
+
const judgeCandidates = [];
|
|
9468
|
+
const candidateToFactIndex = [];
|
|
9469
|
+
for (let fi = 0; fi < facts.length; fi++) {
|
|
9470
|
+
const f = facts[fi];
|
|
9471
|
+
if (!f || typeof f.content !== "string" || !f.content.trim() || typeof f.category !== "string" || !f.category.trim()) {
|
|
9472
|
+
continue;
|
|
9473
|
+
}
|
|
9474
|
+
const judgeCategory = preRoutedCategories[fi] ?? f.category;
|
|
9475
|
+
if (judgeCategory === "procedure") {
|
|
9476
|
+
continue;
|
|
9477
|
+
}
|
|
9478
|
+
const tags = Array.isArray(f.tags) ? f.tags : [];
|
|
9479
|
+
const imp = scoreImportance(
|
|
9480
|
+
f.content,
|
|
9481
|
+
judgeCategory,
|
|
9482
|
+
tags
|
|
9483
|
+
);
|
|
9484
|
+
if (!isAboveImportanceThreshold(
|
|
9485
|
+
imp.level,
|
|
9486
|
+
this.config.extractionMinImportanceLevel
|
|
9487
|
+
)) {
|
|
9488
|
+
continue;
|
|
9489
|
+
}
|
|
9490
|
+
judgeCandidates.push({
|
|
9491
|
+
text: f.content,
|
|
9492
|
+
category: judgeCategory,
|
|
9493
|
+
confidence: typeof f.confidence === "number" ? f.confidence : 0.7,
|
|
9494
|
+
tags,
|
|
9495
|
+
importanceLevel: imp.level
|
|
9496
|
+
});
|
|
9497
|
+
candidateToFactIndex.push(fi);
|
|
9498
|
+
}
|
|
9499
|
+
const judgeResult = await judgeFactDurability(
|
|
9500
|
+
judgeCandidates,
|
|
9501
|
+
this.config,
|
|
9502
|
+
this.localLlm,
|
|
9503
|
+
new FallbackLlmClient(this.config.gatewayConfig),
|
|
9504
|
+
this.judgeVerdictCache
|
|
9505
|
+
);
|
|
9506
|
+
judgeVerdictsByFactIndex = /* @__PURE__ */ new Map();
|
|
9507
|
+
for (const [candidateIdx, verdict] of judgeResult.verdicts) {
|
|
9508
|
+
const factIdx = candidateToFactIndex[candidateIdx];
|
|
9509
|
+
if (factIdx !== void 0) {
|
|
9510
|
+
judgeVerdictsByFactIndex.set(factIdx, verdict);
|
|
9511
|
+
}
|
|
9512
|
+
}
|
|
9513
|
+
log.info(
|
|
9514
|
+
`extraction-judge: ${judgeResult.verdicts.size}/${judgeCandidates.length} facts evaluated, ${judgeResult.cached} cached, ${judgeResult.judged} judged, ${judgeResult.elapsed}ms`
|
|
9515
|
+
);
|
|
9516
|
+
} catch (err) {
|
|
9517
|
+
log.warn(
|
|
9518
|
+
`extraction-judge: pipeline error, proceeding without filtering (fail-open): ${err instanceof Error ? err.message : String(err)}`
|
|
9519
|
+
);
|
|
9520
|
+
}
|
|
9521
|
+
}
|
|
9522
|
+
let factLoopIndex = -1;
|
|
8394
9523
|
for (const fact of facts) {
|
|
9524
|
+
factLoopIndex++;
|
|
8395
9525
|
if (!fact || typeof fact.content !== "string" || !fact.content.trim()) {
|
|
8396
9526
|
continue;
|
|
8397
9527
|
}
|
|
@@ -8400,13 +9530,6 @@ _Context: ${topQuestion.context}_`
|
|
|
8400
9530
|
}
|
|
8401
9531
|
fact.tags = Array.isArray(fact.tags) ? fact.tags.filter((t) => typeof t === "string") : [];
|
|
8402
9532
|
fact.confidence = typeof fact.confidence === "number" ? fact.confidence : 0.7;
|
|
8403
|
-
if (this.contentHashIndex && this.contentHashIndex.has(fact.content)) {
|
|
8404
|
-
log.debug(
|
|
8405
|
-
`dedup: skipping duplicate fact "${fact.content.slice(0, 60)}\u2026"`
|
|
8406
|
-
);
|
|
8407
|
-
dedupedCount++;
|
|
8408
|
-
continue;
|
|
8409
|
-
}
|
|
8410
9533
|
let writeCategory = fact.category;
|
|
8411
9534
|
let targetStorage = storage;
|
|
8412
9535
|
let routedRuleId;
|
|
@@ -8431,33 +9554,187 @@ _Context: ${topQuestion.context}_`
|
|
|
8431
9554
|
);
|
|
8432
9555
|
}
|
|
8433
9556
|
}
|
|
9557
|
+
const canonicalContentForHash = citationEnabled && hasCitationForTemplate(fact.content, citationTemplate) ? stripCitationForTemplate(fact.content, citationTemplate) : fact.content;
|
|
9558
|
+
const contentHashDedupKey = writeCategory === "procedure" ? buildProcedurePersistBody(fact.content, fact.procedureSteps) : canonicalContentForHash;
|
|
9559
|
+
if (this.contentHashIndex && this.contentHashIndex.has(contentHashDedupKey)) {
|
|
9560
|
+
log.debug(
|
|
9561
|
+
`dedup: skipping duplicate fact "${fact.content.slice(0, 60)}\u2026"`
|
|
9562
|
+
);
|
|
9563
|
+
dedupedCount++;
|
|
9564
|
+
continue;
|
|
9565
|
+
}
|
|
8434
9566
|
const importance = scoreImportance(
|
|
8435
9567
|
fact.content,
|
|
8436
9568
|
writeCategory,
|
|
8437
9569
|
fact.tags
|
|
8438
9570
|
);
|
|
9571
|
+
if (writeCategory === "procedure" && this.config.procedural?.enabled !== true) {
|
|
9572
|
+
log.debug("persistExtraction: skip procedure memory (procedural.enabled is false)");
|
|
9573
|
+
continue;
|
|
9574
|
+
}
|
|
9575
|
+
if (!isAboveImportanceThreshold(
|
|
9576
|
+
importance.level,
|
|
9577
|
+
this.config.extractionMinImportanceLevel
|
|
9578
|
+
)) {
|
|
9579
|
+
importanceGatedCount++;
|
|
9580
|
+
const snippet = fact.content.slice(0, 60).replace(/\s+/g, " ").trim();
|
|
9581
|
+
log.debug(`extraction: skip trivial "${snippet}"`);
|
|
9582
|
+
log.debug(
|
|
9583
|
+
`metric:importance_gated level=${importance.level} threshold=${this.config.extractionMinImportanceLevel} category=${writeCategory} count=${importanceGatedCount}`
|
|
9584
|
+
);
|
|
9585
|
+
continue;
|
|
9586
|
+
}
|
|
9587
|
+
if (judgeVerdictsByFactIndex) {
|
|
9588
|
+
const verdict = judgeVerdictsByFactIndex.get(factLoopIndex);
|
|
9589
|
+
if (verdict && !verdict.durable) {
|
|
9590
|
+
if (this.config.extractionJudgeShadow) {
|
|
9591
|
+
log.info(
|
|
9592
|
+
`extraction-judge[shadow]: would reject "${fact.content.slice(0, 60)}\u2026" reason="${verdict.reason}"`
|
|
9593
|
+
);
|
|
9594
|
+
} else {
|
|
9595
|
+
judgeGatedCount++;
|
|
9596
|
+
log.debug(
|
|
9597
|
+
`extraction-judge: rejected "${fact.content.slice(0, 60)}\u2026" reason="${verdict.reason}"`
|
|
9598
|
+
);
|
|
9599
|
+
continue;
|
|
9600
|
+
}
|
|
9601
|
+
}
|
|
9602
|
+
}
|
|
9603
|
+
if (writeCategory === "procedure") {
|
|
9604
|
+
const procGate = validateProcedureExtraction({
|
|
9605
|
+
content: fact.content,
|
|
9606
|
+
procedureSteps: fact.procedureSteps
|
|
9607
|
+
});
|
|
9608
|
+
if (!procGate.durable) {
|
|
9609
|
+
log.debug(
|
|
9610
|
+
`extraction-procedure-gate: rejected "${fact.content.slice(0, 60)}\u2026" reason="${procGate.reason}"`
|
|
9611
|
+
);
|
|
9612
|
+
continue;
|
|
9613
|
+
}
|
|
9614
|
+
}
|
|
9615
|
+
let pendingSemanticSkip = null;
|
|
9616
|
+
if (this.config.semanticDedupEnabled) {
|
|
9617
|
+
let semanticDecision;
|
|
9618
|
+
if (batchBackendUnavailable) {
|
|
9619
|
+
semanticDecision = { action: "keep", reason: "backend_unavailable" };
|
|
9620
|
+
} else {
|
|
9621
|
+
try {
|
|
9622
|
+
const lookupStorage = targetStorage;
|
|
9623
|
+
semanticDecision = await decideSemanticDedup(
|
|
9624
|
+
fact.content,
|
|
9625
|
+
(content, limit) => this.semanticDedupLookup(content, limit, lookupStorage),
|
|
9626
|
+
{
|
|
9627
|
+
enabled: true,
|
|
9628
|
+
threshold: this.config.semanticDedupThreshold,
|
|
9629
|
+
candidates: this.config.semanticDedupCandidates
|
|
9630
|
+
}
|
|
9631
|
+
);
|
|
9632
|
+
} catch (err) {
|
|
9633
|
+
log.warn(
|
|
9634
|
+
`semantic dedup decision failed; failing open and writing fact: ${err}`
|
|
9635
|
+
);
|
|
9636
|
+
semanticDecision = {
|
|
9637
|
+
action: "keep",
|
|
9638
|
+
reason: "backend_unavailable"
|
|
9639
|
+
};
|
|
9640
|
+
}
|
|
9641
|
+
if (semanticDecision.reason === "backend_unavailable") {
|
|
9642
|
+
batchBackendUnavailable = true;
|
|
9643
|
+
}
|
|
9644
|
+
}
|
|
9645
|
+
if (semanticDecision.action === "skip") {
|
|
9646
|
+
pendingSemanticSkip = semanticDecision;
|
|
9647
|
+
}
|
|
9648
|
+
}
|
|
8439
9649
|
const inferredIntent = this.config.intentRoutingEnabled ? inferIntentFromText(
|
|
8440
9650
|
`${writeCategory} ${fact.tags.join(" ")} ${fact.content}`
|
|
8441
9651
|
) : null;
|
|
8442
9652
|
const extractionWriteSource = fact.source === "proactive" ? "extraction-proactive" : "extraction";
|
|
8443
|
-
|
|
8444
|
-
|
|
9653
|
+
let supersedes;
|
|
9654
|
+
let links = [];
|
|
9655
|
+
let contradictionDetected = false;
|
|
9656
|
+
if (this.config.contradictionDetectionEnabled && this.qmd.isAvailable()) {
|
|
9657
|
+
const targetNamespace = this.namespaceFromStorageDir(targetStorage.dir);
|
|
9658
|
+
const contradiction = await this.checkForContradiction(
|
|
9659
|
+
fact.content,
|
|
9660
|
+
writeCategory,
|
|
9661
|
+
targetNamespace
|
|
9662
|
+
);
|
|
9663
|
+
if (contradiction) {
|
|
9664
|
+
contradictionDetected = true;
|
|
9665
|
+
if (this.config.contradictionAutoResolve) {
|
|
9666
|
+
supersedes = contradiction.supersededId;
|
|
9667
|
+
}
|
|
9668
|
+
links.push({
|
|
9669
|
+
targetId: contradiction.supersededId,
|
|
9670
|
+
linkType: "contradicts",
|
|
9671
|
+
strength: contradiction.confidence,
|
|
9672
|
+
reason: contradiction.reason
|
|
9673
|
+
});
|
|
9674
|
+
if (this.config.contradictionAutoResolve && this.config.queryAwareIndexingEnabled && contradiction.supersededPath) {
|
|
9675
|
+
deindexMemory(
|
|
9676
|
+
this.config.memoryDir,
|
|
9677
|
+
contradiction.supersededPath,
|
|
9678
|
+
contradiction.supersededCreated,
|
|
9679
|
+
contradiction.supersededTags
|
|
9680
|
+
);
|
|
9681
|
+
}
|
|
9682
|
+
}
|
|
9683
|
+
}
|
|
9684
|
+
const isCorrection = writeCategory === "correction";
|
|
9685
|
+
if (pendingSemanticSkip && !contradictionDetected && !isCorrection) {
|
|
9686
|
+
log.debug(
|
|
9687
|
+
`dedup: skipping semantic near-duplicate fact "${fact.content.slice(0, 60).replace(/\s+/g, " ")}\u2026" score=${pendingSemanticSkip.topScore.toFixed(
|
|
9688
|
+
3
|
|
9689
|
+
)} neighbor=${pendingSemanticSkip.topId}`
|
|
9690
|
+
);
|
|
9691
|
+
dedupedCount++;
|
|
9692
|
+
continue;
|
|
9693
|
+
}
|
|
9694
|
+
if (this.config.chunkingEnabled && writeCategory !== "procedure") {
|
|
9695
|
+
let chunkResult;
|
|
9696
|
+
if (this.config.semanticChunkingEnabled) {
|
|
9697
|
+
try {
|
|
9698
|
+
const embedFn = this.embeddingFallback.embedTexts.bind(this.embeddingFallback);
|
|
9699
|
+
const semanticResult = await semanticChunkContent(
|
|
9700
|
+
fact.content,
|
|
9701
|
+
embedFn,
|
|
9702
|
+
this.config.semanticChunkingConfig
|
|
9703
|
+
);
|
|
9704
|
+
chunkResult = semanticResult;
|
|
9705
|
+
} catch (err) {
|
|
9706
|
+
if (this.config.semanticChunkingConfig?.fallbackToRecursive === false) {
|
|
9707
|
+
throw err;
|
|
9708
|
+
}
|
|
9709
|
+
log.debug(
|
|
9710
|
+
`semantic chunking failed, falling back to recursive chunker: ${err}`
|
|
9711
|
+
);
|
|
9712
|
+
chunkResult = chunkContent(fact.content, chunkingConfig);
|
|
9713
|
+
}
|
|
9714
|
+
} else {
|
|
9715
|
+
chunkResult = chunkContent(fact.content, chunkingConfig);
|
|
9716
|
+
}
|
|
8445
9717
|
if (chunkResult.chunked && chunkResult.chunks.length > 1) {
|
|
8446
9718
|
const memoryKind2 = this.config.episodeNoteModeEnabled ? classifyMemoryKind(fact.content, fact.tags ?? [], writeCategory) : void 0;
|
|
9719
|
+
const rawChunkedContent = citationEnabled && hasCitationForTemplate(fact.content, citationTemplate) ? stripCitationForTemplate(fact.content, citationTemplate) : fact.content;
|
|
9720
|
+
const citedChunkedContent = applyInlineCitation(rawChunkedContent);
|
|
8447
9721
|
const parentId = await targetStorage.writeMemory(
|
|
8448
9722
|
writeCategory,
|
|
8449
|
-
|
|
9723
|
+
citedChunkedContent,
|
|
8450
9724
|
{
|
|
8451
9725
|
confidence: fact.confidence,
|
|
8452
9726
|
tags: [...fact.tags, "chunked"],
|
|
8453
9727
|
entityRef: fact.entityRef,
|
|
8454
9728
|
source: extractionWriteSource,
|
|
8455
9729
|
importance,
|
|
9730
|
+
supersedes,
|
|
9731
|
+
links: links.length > 0 ? links : void 0,
|
|
8456
9732
|
intentGoal: inferredIntent?.goal,
|
|
8457
9733
|
intentActionType: inferredIntent?.actionType,
|
|
8458
9734
|
intentEntityTypes: inferredIntent?.entityTypes,
|
|
8459
9735
|
memoryKind: memoryKind2,
|
|
8460
|
-
structuredAttributes: fact.structuredAttributes
|
|
9736
|
+
structuredAttributes: fact.structuredAttributes,
|
|
9737
|
+
contentHashSource: rawChunkedContent
|
|
8461
9738
|
}
|
|
8462
9739
|
);
|
|
8463
9740
|
for (const chunk of chunkResult.chunks) {
|
|
@@ -8472,7 +9749,9 @@ _Context: ${topQuestion.context}_`
|
|
|
8472
9749
|
chunk.index,
|
|
8473
9750
|
chunkResult.chunks.length,
|
|
8474
9751
|
writeCategory,
|
|
8475
|
-
chunk
|
|
9752
|
+
// Each chunk carries its own inline citation so provenance
|
|
9753
|
+
// survives when a single chunk is quoted in isolation.
|
|
9754
|
+
applyInlineCitation(chunk.content),
|
|
8476
9755
|
{
|
|
8477
9756
|
confidence: fact.confidence,
|
|
8478
9757
|
tags: fact.tags,
|
|
@@ -8499,6 +9778,19 @@ _Context: ${topQuestion.context}_`
|
|
|
8499
9778
|
threadEpisodeIdsForGraph.push(parentId);
|
|
8500
9779
|
}
|
|
8501
9780
|
await this.indexPersistedMemory(targetStorage, parentId);
|
|
9781
|
+
try {
|
|
9782
|
+
const supersessionEntityRef = typeof fact.entityRef === "string" ? fact.entityRef : void 0;
|
|
9783
|
+
await applyTemporalSupersession({
|
|
9784
|
+
storage: targetStorage,
|
|
9785
|
+
newMemoryId: parentId,
|
|
9786
|
+
entityRef: supersessionEntityRef,
|
|
9787
|
+
structuredAttributes: fact.structuredAttributes,
|
|
9788
|
+
createdAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
9789
|
+
enabled: this.config.temporalSupersessionEnabled
|
|
9790
|
+
});
|
|
9791
|
+
} catch (err) {
|
|
9792
|
+
log.warn(`temporal-supersession (chunked): unexpected error: ${err}`);
|
|
9793
|
+
}
|
|
8502
9794
|
await promoteMemoryToShared({
|
|
8503
9795
|
sourceStorage: targetStorage,
|
|
8504
9796
|
category: writeCategory,
|
|
@@ -8506,6 +9798,7 @@ _Context: ${topQuestion.context}_`
|
|
|
8506
9798
|
confidence: fact.confidence,
|
|
8507
9799
|
tags: fact.tags,
|
|
8508
9800
|
entityRef: fact.entityRef,
|
|
9801
|
+
structuredAttributes: fact.structuredAttributes,
|
|
8509
9802
|
sourceMemoryId: parentId,
|
|
8510
9803
|
importance,
|
|
8511
9804
|
intentGoal: inferredIntent?.goal,
|
|
@@ -8515,14 +9808,15 @@ _Context: ${topQuestion.context}_`
|
|
|
8515
9808
|
source: extractionWriteSource
|
|
8516
9809
|
});
|
|
8517
9810
|
if (this.contentHashIndex) {
|
|
8518
|
-
|
|
9811
|
+
const canonicalChunkedContent = citationEnabled && hasCitationForTemplate(fact.content, citationTemplate) ? stripCitationForTemplate(fact.content, citationTemplate) : fact.content;
|
|
9812
|
+
this.contentHashIndex.add(canonicalChunkedContent);
|
|
8519
9813
|
}
|
|
8520
9814
|
for (const chunk of chunkResult.chunks) {
|
|
8521
9815
|
const chunkId = `${parentId}-chunk-${chunk.index}`;
|
|
8522
9816
|
await this.indexPersistedMemory(targetStorage, chunkId);
|
|
8523
9817
|
}
|
|
8524
9818
|
if (this.config.verbatimArtifactsEnabled && this.config.verbatimArtifactCategories.includes(writeCategory) && fact.confidence >= this.config.verbatimArtifactsMinConfidence) {
|
|
8525
|
-
await targetStorage.writeArtifact(
|
|
9819
|
+
await targetStorage.writeArtifact(citedChunkedContent, {
|
|
8526
9820
|
confidence: fact.confidence,
|
|
8527
9821
|
tags: [...fact.tags, "artifact", "chunked-parent"],
|
|
8528
9822
|
artifactType: this.artifactTypeForCategory(writeCategory),
|
|
@@ -8581,33 +9875,6 @@ _Context: ${topQuestion.context}_`
|
|
|
8581
9875
|
continue;
|
|
8582
9876
|
}
|
|
8583
9877
|
}
|
|
8584
|
-
let supersedes;
|
|
8585
|
-
let links = [];
|
|
8586
|
-
if (this.config.contradictionDetectionEnabled && this.qmd.isAvailable()) {
|
|
8587
|
-
const targetNamespace = this.namespaceFromStorageDir(targetStorage.dir);
|
|
8588
|
-
const contradiction = await this.checkForContradiction(
|
|
8589
|
-
fact.content,
|
|
8590
|
-
writeCategory,
|
|
8591
|
-
targetNamespace
|
|
8592
|
-
);
|
|
8593
|
-
if (contradiction) {
|
|
8594
|
-
supersedes = contradiction.supersededId;
|
|
8595
|
-
links.push({
|
|
8596
|
-
targetId: contradiction.supersededId,
|
|
8597
|
-
linkType: "contradicts",
|
|
8598
|
-
strength: contradiction.confidence,
|
|
8599
|
-
reason: contradiction.reason
|
|
8600
|
-
});
|
|
8601
|
-
if (this.config.queryAwareIndexingEnabled && contradiction.supersededPath) {
|
|
8602
|
-
deindexMemory(
|
|
8603
|
-
this.config.memoryDir,
|
|
8604
|
-
contradiction.supersededPath,
|
|
8605
|
-
contradiction.supersededCreated,
|
|
8606
|
-
contradiction.supersededTags
|
|
8607
|
-
);
|
|
8608
|
-
}
|
|
8609
|
-
}
|
|
8610
|
-
}
|
|
8611
9878
|
if (this.config.memoryLinkingEnabled && this.qmd.isAvailable()) {
|
|
8612
9879
|
const targetNamespace = this.namespaceFromStorageDir(targetStorage.dir);
|
|
8613
9880
|
const suggestedLinks = await this.suggestLinksForMemory(
|
|
@@ -8619,10 +9886,12 @@ _Context: ${topQuestion.context}_`
|
|
|
8619
9886
|
links.push(...suggestedLinks);
|
|
8620
9887
|
}
|
|
8621
9888
|
}
|
|
8622
|
-
const memoryKind = this.config.episodeNoteModeEnabled ? classifyMemoryKind(fact.content, fact.tags ?? [], writeCategory) : void 0;
|
|
9889
|
+
const memoryKind = writeCategory === "procedure" ? void 0 : this.config.episodeNoteModeEnabled ? classifyMemoryKind(fact.content, fact.tags ?? [], writeCategory) : void 0;
|
|
9890
|
+
const rawPersistBody = writeCategory === "procedure" ? buildProcedurePersistBody(fact.content, fact.procedureSteps) : fact.content;
|
|
9891
|
+
const citedFactContent = applyInlineCitation(rawPersistBody);
|
|
8623
9892
|
const memoryId = await targetStorage.writeMemory(
|
|
8624
9893
|
writeCategory,
|
|
8625
|
-
|
|
9894
|
+
citedFactContent,
|
|
8626
9895
|
{
|
|
8627
9896
|
confidence: fact.confidence,
|
|
8628
9897
|
tags: fact.tags,
|
|
@@ -8635,7 +9904,8 @@ _Context: ${topQuestion.context}_`
|
|
|
8635
9904
|
intentActionType: inferredIntent?.actionType,
|
|
8636
9905
|
intentEntityTypes: inferredIntent?.entityTypes,
|
|
8637
9906
|
memoryKind,
|
|
8638
|
-
structuredAttributes: fact.structuredAttributes
|
|
9907
|
+
structuredAttributes: fact.structuredAttributes,
|
|
9908
|
+
contentHashSource: writeCategory === "fact" ? fact.content : void 0
|
|
8639
9909
|
}
|
|
8640
9910
|
);
|
|
8641
9911
|
if (routedRuleId) {
|
|
@@ -8643,6 +9913,19 @@ _Context: ${topQuestion.context}_`
|
|
|
8643
9913
|
`routing applied for memory ${memoryId}: rule=${routedRuleId} category=${writeCategory} storage=${targetStorage.dir}`
|
|
8644
9914
|
);
|
|
8645
9915
|
}
|
|
9916
|
+
try {
|
|
9917
|
+
const supersessionEntityRef = typeof fact.entityRef === "string" ? fact.entityRef : void 0;
|
|
9918
|
+
await applyTemporalSupersession({
|
|
9919
|
+
storage: targetStorage,
|
|
9920
|
+
newMemoryId: memoryId,
|
|
9921
|
+
entityRef: supersessionEntityRef,
|
|
9922
|
+
structuredAttributes: fact.structuredAttributes,
|
|
9923
|
+
createdAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
9924
|
+
enabled: this.config.temporalSupersessionEnabled
|
|
9925
|
+
});
|
|
9926
|
+
} catch (err) {
|
|
9927
|
+
log.warn(`temporal-supersession: unexpected error: ${err}`);
|
|
9928
|
+
}
|
|
8646
9929
|
trackBehaviorSignals(
|
|
8647
9930
|
targetStorage,
|
|
8648
9931
|
buildBehaviorSignalsForMemory({
|
|
@@ -8666,6 +9949,7 @@ _Context: ${topQuestion.context}_`
|
|
|
8666
9949
|
confidence: fact.confidence,
|
|
8667
9950
|
tags: fact.tags,
|
|
8668
9951
|
entityRef: typeof fact.entityRef === "string" ? fact.entityRef : void 0,
|
|
9952
|
+
structuredAttributes: fact.structuredAttributes,
|
|
8669
9953
|
sourceMemoryId: memoryId,
|
|
8670
9954
|
importance,
|
|
8671
9955
|
intentGoal: inferredIntent?.goal,
|
|
@@ -8710,7 +9994,7 @@ _Context: ${topQuestion.context}_`
|
|
|
8710
9994
|
}
|
|
8711
9995
|
}
|
|
8712
9996
|
if (this.config.verbatimArtifactsEnabled && this.config.verbatimArtifactCategories.includes(writeCategory) && fact.confidence >= this.config.verbatimArtifactsMinConfidence) {
|
|
8713
|
-
await targetStorage.writeArtifact(
|
|
9997
|
+
await targetStorage.writeArtifact(citedFactContent, {
|
|
8714
9998
|
confidence: fact.confidence,
|
|
8715
9999
|
tags: [...fact.tags, "artifact"],
|
|
8716
10000
|
artifactType: this.artifactTypeForCategory(writeCategory),
|
|
@@ -8721,7 +10005,9 @@ _Context: ${topQuestion.context}_`
|
|
|
8721
10005
|
});
|
|
8722
10006
|
}
|
|
8723
10007
|
if (this.contentHashIndex) {
|
|
8724
|
-
|
|
10008
|
+
const canonicalFactContent = citationEnabled && hasCitationForTemplate(fact.content, citationTemplate) ? stripCitationForTemplate(fact.content, citationTemplate) : fact.content;
|
|
10009
|
+
const hashRegisterKey = writeCategory === "procedure" ? buildProcedurePersistBody(fact.content, fact.procedureSteps) : canonicalFactContent;
|
|
10010
|
+
this.contentHashIndex.add(hashRegisterKey);
|
|
8725
10011
|
}
|
|
8726
10012
|
}
|
|
8727
10013
|
for (const entity of entities) {
|
|
@@ -8732,7 +10018,12 @@ _Context: ${topQuestion.context}_`
|
|
|
8732
10018
|
continue;
|
|
8733
10019
|
}
|
|
8734
10020
|
const safeFacts = Array.isArray(entity?.facts) ? entity.facts.filter((f) => typeof f === "string") : [];
|
|
8735
|
-
const id = await storage.writeEntity(name, type, safeFacts
|
|
10021
|
+
const id = await storage.writeEntity(name, type, safeFacts, {
|
|
10022
|
+
source: typeof entity?.source === "string" ? entity.source : "extraction",
|
|
10023
|
+
sessionKey: sourceContext?.sessionKey,
|
|
10024
|
+
principal: sourceContext?.principal,
|
|
10025
|
+
structuredSections: Array.isArray(entity?.structuredSections) ? entity.structuredSections : void 0
|
|
10026
|
+
});
|
|
8736
10027
|
if (id) trackPersistedId(storage, id);
|
|
8737
10028
|
} catch (err) {
|
|
8738
10029
|
log.warn(`persistExtraction: entity write failed: ${err}`);
|
|
@@ -8801,8 +10092,10 @@ _Context: ${topQuestion.context}_`
|
|
|
8801
10092
|
);
|
|
8802
10093
|
}
|
|
8803
10094
|
const dedupSuffix = dedupedCount > 0 ? ` (${dedupedCount} deduped)` : "";
|
|
10095
|
+
const gatedSuffix = importanceGatedCount > 0 ? ` (${importanceGatedCount} gated)` : "";
|
|
10096
|
+
const judgeSuffix = judgeGatedCount > 0 ? ` (${judgeGatedCount} judge-rejected)` : "";
|
|
8804
10097
|
log.info(
|
|
8805
|
-
`persisted: ${facts.length - dedupedCount} facts${dedupSuffix}, ${entities.length} entities, ${questions.length} questions, ${profileUpdates.length} profile updates`
|
|
10098
|
+
`persisted: ${facts.length - dedupedCount - importanceGatedCount - judgeGatedCount} facts${dedupSuffix}${gatedSuffix}${judgeSuffix}, ${entities.length} entities, ${questions.length} questions, ${profileUpdates.length} profile updates`
|
|
8806
10099
|
);
|
|
8807
10100
|
void (async () => {
|
|
8808
10101
|
if (persistedIdsByStorage.size === 0) {
|
|
@@ -9023,7 +10316,10 @@ _Context: ${topQuestion.context}_`
|
|
|
9023
10316
|
}
|
|
9024
10317
|
for (const entity of result.entityUpdates) {
|
|
9025
10318
|
const safeFacts = Array.isArray(entity?.facts) ? entity.facts.filter((f) => typeof f === "string") : [];
|
|
9026
|
-
await this.storage.writeEntity(entity.name, entity.type, safeFacts
|
|
10319
|
+
await this.storage.writeEntity(entity.name, entity.type, safeFacts, {
|
|
10320
|
+
source: "consolidation",
|
|
10321
|
+
structuredSections: Array.isArray(entity?.structuredSections) ? entity.structuredSections : void 0
|
|
10322
|
+
});
|
|
9027
10323
|
}
|
|
9028
10324
|
const entitiesMerged = await this.storage.mergeFragmentedEntities();
|
|
9029
10325
|
if (entitiesMerged > 0) {
|
|
@@ -9031,53 +10327,15 @@ _Context: ${topQuestion.context}_`
|
|
|
9031
10327
|
}
|
|
9032
10328
|
if (this.config.entitySummaryEnabled) {
|
|
9033
10329
|
try {
|
|
9034
|
-
const
|
|
9035
|
-
|
|
9036
|
-
|
|
10330
|
+
const synthesized = await this.processEntitySynthesisQueue(
|
|
10331
|
+
this.config.defaultNamespace,
|
|
10332
|
+
5
|
|
9037
10333
|
);
|
|
9038
|
-
|
|
9039
|
-
|
|
9040
|
-
for (const entity of toSummarize) {
|
|
9041
|
-
try {
|
|
9042
|
-
const factsText = entity.facts.slice(0, 10).join("; ");
|
|
9043
|
-
const prompt = `Summarize this entity in one sentence. Entity: ${entity.name} (${entity.type}). Facts: ${factsText}`;
|
|
9044
|
-
const response = await this.fastChatCompletion(
|
|
9045
|
-
[
|
|
9046
|
-
{
|
|
9047
|
-
role: "system",
|
|
9048
|
-
content: "Respond with a single concise sentence summarizing the entity. No JSON, just plain text."
|
|
9049
|
-
},
|
|
9050
|
-
{ role: "user", content: prompt }
|
|
9051
|
-
],
|
|
9052
|
-
{
|
|
9053
|
-
temperature: 0.3,
|
|
9054
|
-
maxTokens: 100,
|
|
9055
|
-
operation: "entity_summary",
|
|
9056
|
-
priority: "background"
|
|
9057
|
-
}
|
|
9058
|
-
);
|
|
9059
|
-
if (response?.content) {
|
|
9060
|
-
const summary = response.content.trim().replace(/^["']|["']$/g, "");
|
|
9061
|
-
if (summary.length > 10 && summary.length < 500) {
|
|
9062
|
-
const entityFileName = normalizeEntityName(
|
|
9063
|
-
entity.name,
|
|
9064
|
-
entity.type
|
|
9065
|
-
);
|
|
9066
|
-
await this.storage.updateEntitySummary(entityFileName, summary);
|
|
9067
|
-
summarized++;
|
|
9068
|
-
}
|
|
9069
|
-
}
|
|
9070
|
-
} catch (err) {
|
|
9071
|
-
log.debug(
|
|
9072
|
-
`entity summary generation failed for ${entity.name}: ${err}`
|
|
9073
|
-
);
|
|
9074
|
-
}
|
|
9075
|
-
}
|
|
9076
|
-
if (summarized > 0) {
|
|
9077
|
-
log.info(`generated ${summarized} entity summaries`);
|
|
10334
|
+
if (synthesized > 0) {
|
|
10335
|
+
log.info(`refreshed ${synthesized} entity syntheses`);
|
|
9078
10336
|
}
|
|
9079
10337
|
} catch (err) {
|
|
9080
|
-
log.debug(`entity
|
|
10338
|
+
log.debug(`entity synthesis pass failed: ${err}`);
|
|
9081
10339
|
}
|
|
9082
10340
|
}
|
|
9083
10341
|
const deletedCommitments = await this.storage.cleanExpiredCommitments(
|
|
@@ -9262,6 +10520,24 @@ ${texts.map((t, i) => `[${i + 1}] ${t}`).join("\n\n")}`;
|
|
|
9262
10520
|
log.warn(`tmt: consolidation hook failed (ignored): ${err}`);
|
|
9263
10521
|
}
|
|
9264
10522
|
}
|
|
10523
|
+
if (this.consolidationObservers.size > 0) {
|
|
10524
|
+
const observation = {
|
|
10525
|
+
runAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
10526
|
+
recentMemories: recent,
|
|
10527
|
+
existingMemories: older.slice(-50),
|
|
10528
|
+
profile,
|
|
10529
|
+
result,
|
|
10530
|
+
merged,
|
|
10531
|
+
invalidated
|
|
10532
|
+
};
|
|
10533
|
+
for (const observer of this.consolidationObservers) {
|
|
10534
|
+
try {
|
|
10535
|
+
await observer(observation);
|
|
10536
|
+
} catch (err) {
|
|
10537
|
+
log.warn(`consolidation observer failed (ignored): ${err}`);
|
|
10538
|
+
}
|
|
10539
|
+
}
|
|
10540
|
+
}
|
|
9265
10541
|
log.info("consolidation complete");
|
|
9266
10542
|
return { memoriesProcessed: allMemories.length, merged, invalidated };
|
|
9267
10543
|
}
|
|
@@ -9666,7 +10942,14 @@ ${texts.map((t, i) => `[${i + 1}] ${t}`).join("\n\n")}`;
|
|
|
9666
10942
|
const result = await this.storage.archiveMemory(memory);
|
|
9667
10943
|
if (result) {
|
|
9668
10944
|
if (this.contentHashIndex) {
|
|
9669
|
-
|
|
10945
|
+
if (memory.frontmatter.contentHash) {
|
|
10946
|
+
this.contentHashIndex.removeByHash(memory.frontmatter.contentHash);
|
|
10947
|
+
} else {
|
|
10948
|
+
log.warn(
|
|
10949
|
+
`[fact-archival] removing hash for legacy memory ${memory.frontmatter.id ?? "(unknown)"} via content fallback \u2014 no contentHash in frontmatter`
|
|
10950
|
+
);
|
|
10951
|
+
this.contentHashIndex.remove(memory.content);
|
|
10952
|
+
}
|
|
9670
10953
|
}
|
|
9671
10954
|
await this.embeddingFallback.removeFromIndex(memory.frontmatter.id);
|
|
9672
10955
|
if (this.config.queryAwareIndexingEnabled && memory.path && memory.frontmatter?.created) {
|
|
@@ -10115,14 +11398,59 @@ ${lines.join("\n\n")}`;
|
|
|
10115
11398
|
});
|
|
10116
11399
|
}
|
|
10117
11400
|
publishRecallResults(options) {
|
|
11401
|
+
const sectionId = "memories";
|
|
10118
11402
|
const memoryIds = this.extractMemoryIdsFromResults(options.results);
|
|
10119
11403
|
this.trackMemoryAccess(memoryIds);
|
|
10120
11404
|
this.appendRecallSection(
|
|
10121
11405
|
options.sectionBuckets,
|
|
10122
|
-
|
|
11406
|
+
sectionId,
|
|
10123
11407
|
this.formatQmdResults(options.title, options.results)
|
|
10124
11408
|
);
|
|
10125
11409
|
}
|
|
11410
|
+
/**
|
|
11411
|
+
* Apply MMR over the pre-truncation recall candidate pool and then slice
|
|
11412
|
+
* the result to `limit`. This is the single place in the pipeline where
|
|
11413
|
+
* MMR runs, and it must be called *before* callers throw away candidates
|
|
11414
|
+
* that would otherwise sit below the final cutoff. Running MMR post-slice
|
|
11415
|
+
* is a no-op in the cases we care about — diverse candidates just below
|
|
11416
|
+
* the cutoff are already gone and can never be promoted.
|
|
11417
|
+
*
|
|
11418
|
+
* Callers must pass the full candidate pool (post-rerank, pre-slice).
|
|
11419
|
+
*/
|
|
11420
|
+
diversifyAndLimitRecallResults(sectionId, results, limit) {
|
|
11421
|
+
const safeLimit = typeof limit === "number" && Number.isFinite(limit) ? Math.max(0, Math.floor(limit)) : 0;
|
|
11422
|
+
if (!Array.isArray(results) || results.length === 0) return [];
|
|
11423
|
+
if (safeLimit === 0) return [];
|
|
11424
|
+
const diversified = this.applyMmrToQmdResults(sectionId, results);
|
|
11425
|
+
return diversified.slice(0, safeLimit);
|
|
11426
|
+
}
|
|
11427
|
+
/**
|
|
11428
|
+
* Apply Maximal Marginal Relevance to a section's ordered candidate list.
|
|
11429
|
+
*
|
|
11430
|
+
* Operates per-section so one redundant cluster cannot dominate a section,
|
|
11431
|
+
* and so one section's MMR pass cannot starve other sections. Returns the
|
|
11432
|
+
* input unchanged when disabled, when there are fewer than 2 candidates, or
|
|
11433
|
+
* when no budget information is available.
|
|
11434
|
+
*/
|
|
11435
|
+
applyMmrToQmdResults(sectionId, results) {
|
|
11436
|
+
if (this.config.recallMmrEnabled === false) return results;
|
|
11437
|
+
if (!Array.isArray(results) || results.length < 2) return results;
|
|
11438
|
+
const configuredTopN = this.config.recallMmrTopN;
|
|
11439
|
+
const topN = typeof configuredTopN === "number" && Number.isFinite(configuredTopN) ? Math.max(0, Math.floor(configuredTopN)) : 40;
|
|
11440
|
+
if (topN === 0) return results;
|
|
11441
|
+
const lambda = this.config.recallMmrLambda ?? 0.7;
|
|
11442
|
+
const { reordered, diversity } = reorderRecallResultsWithMmr(results, {
|
|
11443
|
+
lambda,
|
|
11444
|
+
topN
|
|
11445
|
+
});
|
|
11446
|
+
try {
|
|
11447
|
+
log.info(
|
|
11448
|
+
`recall_mmr: section=${sectionId} kept=${diversity.kept}/${diversity.considered} headReorderCount=${diversity.headReorderCount} avgSimBefore=${diversity.avgPairwiseSimBefore.toFixed(3)} avgSimAfter=${diversity.avgPairwiseSimAfter.toFixed(3)} lambda=${lambda.toFixed(2)}`
|
|
11449
|
+
);
|
|
11450
|
+
} catch {
|
|
11451
|
+
}
|
|
11452
|
+
return reordered;
|
|
11453
|
+
}
|
|
10126
11454
|
buildLastRecallBudgetSummary(options) {
|
|
10127
11455
|
return {
|
|
10128
11456
|
requestedTopK: options.requestedTopK,
|
|
@@ -10149,6 +11477,71 @@ ${lines.join("\n\n")}`;
|
|
|
10149
11477
|
}
|
|
10150
11478
|
return [...used];
|
|
10151
11479
|
}
|
|
11480
|
+
/**
|
|
11481
|
+
* Issue #373 — nearest-neighbor lookup for the write-time semantic dedup
|
|
11482
|
+
* guard. Returns the top-K embedding hits against the currently indexed
|
|
11483
|
+
* memories, or an empty array when the embedding backend is unavailable.
|
|
11484
|
+
* Intentionally does NOT throw; `decideSemanticDedup` treats both "empty"
|
|
11485
|
+
* and "error" outcomes as fail-open (keep the candidate).
|
|
11486
|
+
*
|
|
11487
|
+
* PR #399 P1 fix: when namespaces are enabled the lookup must be scoped
|
|
11488
|
+
* to the SAME namespace as the fact being written. Otherwise a
|
|
11489
|
+
* high-similarity memory from another namespace can suppress a write in
|
|
11490
|
+
* the target namespace — cross-tenant data loss. Callers pass the target
|
|
11491
|
+
* storage so we can translate its root directory into the correct index
|
|
11492
|
+
* path prefix (and, for the legacy default-namespace layout at
|
|
11493
|
+
* `memoryDir` root, an exclusion list for `namespaces/*`).
|
|
11494
|
+
*/
|
|
11495
|
+
async semanticDedupLookup(content, limit, targetStorage) {
|
|
11496
|
+
if (!this.config.embeddingFallbackEnabled) {
|
|
11497
|
+
throw new Error("semantic dedup: embedding backend not configured");
|
|
11498
|
+
}
|
|
11499
|
+
if (!await this.embeddingFallback.isAvailable()) {
|
|
11500
|
+
log.debug("semantic dedup: embedding backend unavailable, skipping");
|
|
11501
|
+
throw new Error("semantic dedup: embedding backend unavailable");
|
|
11502
|
+
}
|
|
11503
|
+
const scope = this.semanticDedupScopeFor(targetStorage);
|
|
11504
|
+
const hits = await this.embeddingFallback.search(content, limit, { ...scope, throwOnTimeout: true });
|
|
11505
|
+
if (!Array.isArray(hits) || hits.length === 0) return [];
|
|
11506
|
+
return hits.map((hit) => ({
|
|
11507
|
+
id: hit.id,
|
|
11508
|
+
score: hit.score,
|
|
11509
|
+
path: hit.path
|
|
11510
|
+
}));
|
|
11511
|
+
}
|
|
11512
|
+
/**
|
|
11513
|
+
* Resolve the namespace-scoped filter to pass into
|
|
11514
|
+
* `EmbeddingFallback.search()` for semantic dedup. Returns an empty
|
|
11515
|
+
* object (no filter) when namespaces are disabled, preserving the
|
|
11516
|
+
* pre-PR #399 behavior for single-tenant installs.
|
|
11517
|
+
*
|
|
11518
|
+
* Index entries are stored as paths relative to `config.memoryDir`, so:
|
|
11519
|
+
* - A non-default namespace `ns` lives under `namespaces/<ns>/…` and
|
|
11520
|
+
* we include exactly that prefix.
|
|
11521
|
+
* - The default namespace may live at `memoryDir` root (legacy) or at
|
|
11522
|
+
* `memoryDir/namespaces/<default>/…` (migrated). When it lives at
|
|
11523
|
+
* root we include everything but EXCLUDE all `namespaces/…` entries
|
|
11524
|
+
* so facts from non-default namespaces can't cross-match.
|
|
11525
|
+
*/
|
|
11526
|
+
semanticDedupScopeFor(targetStorage) {
|
|
11527
|
+
if (!this.config.namespacesEnabled) return {};
|
|
11528
|
+
const memoryDir = path5.resolve(this.config.memoryDir);
|
|
11529
|
+
const storageDir = path5.resolve(targetStorage.dir);
|
|
11530
|
+
if (storageDir === memoryDir) {
|
|
11531
|
+
return { pathExcludePrefixes: ["namespaces/"] };
|
|
11532
|
+
}
|
|
11533
|
+
let rel = path5.relative(memoryDir, storageDir);
|
|
11534
|
+
if (!rel || rel.startsWith("..")) {
|
|
11535
|
+
log.debug(
|
|
11536
|
+
`semantic dedup: target storage dir ${storageDir} is outside memoryDir ${memoryDir}; scoping lookup to absolute path prefix`
|
|
11537
|
+
);
|
|
11538
|
+
const absPrefix = storageDir.replace(/\\/g, "/");
|
|
11539
|
+
return { pathPrefix: absPrefix.endsWith("/") ? absPrefix : `${absPrefix}/` };
|
|
11540
|
+
}
|
|
11541
|
+
rel = rel.replace(/\\/g, "/");
|
|
11542
|
+
if (!rel.endsWith("/")) rel = `${rel}/`;
|
|
11543
|
+
return { pathPrefix: rel };
|
|
11544
|
+
}
|
|
10152
11545
|
async searchEmbeddingFallback(query, limit) {
|
|
10153
11546
|
if (!this.config.embeddingFallbackEnabled) return [];
|
|
10154
11547
|
if (!await this.embeddingFallback.isAvailable()) return [];
|
|
@@ -10333,7 +11726,11 @@ ${lines.join("\n\n")}`;
|
|
|
10333
11726
|
"rerankProvider=cloud is reserved/experimental in v2.2.0; skipping rerank"
|
|
10334
11727
|
);
|
|
10335
11728
|
}
|
|
10336
|
-
return
|
|
11729
|
+
return this.diversifyAndLimitRecallResults(
|
|
11730
|
+
"memories",
|
|
11731
|
+
results,
|
|
11732
|
+
options.recallResultLimit
|
|
11733
|
+
);
|
|
10337
11734
|
}
|
|
10338
11735
|
// ---------------------------------------------------------------------------
|
|
10339
11736
|
// Access Tracking (Phase 1A)
|
|
@@ -10449,6 +11846,8 @@ ${lines.join("\n\n")}`;
|
|
|
10449
11846
|
}
|
|
10450
11847
|
}
|
|
10451
11848
|
let lifecycleFilteredCount = 0;
|
|
11849
|
+
let temporalSupersededFilteredCount = 0;
|
|
11850
|
+
let dedicatedSurfaceFilteredCount = 0;
|
|
10452
11851
|
const boosted = [];
|
|
10453
11852
|
const recencyWeight = this.effectiveRecencyWeight();
|
|
10454
11853
|
for (const r of results) {
|
|
@@ -10462,6 +11861,17 @@ ${lines.join("\n\n")}`;
|
|
|
10462
11861
|
lifecycleFilteredCount += 1;
|
|
10463
11862
|
continue;
|
|
10464
11863
|
}
|
|
11864
|
+
if (shouldFilterSupersededFromRecall(memory.frontmatter, {
|
|
11865
|
+
enabled: this.config.temporalSupersessionEnabled,
|
|
11866
|
+
includeInRecall: this.config.temporalSupersessionIncludeInRecall
|
|
11867
|
+
})) {
|
|
11868
|
+
temporalSupersededFilteredCount += 1;
|
|
11869
|
+
continue;
|
|
11870
|
+
}
|
|
11871
|
+
if (options?.allowDedicatedSurface !== true && (memory.frontmatter.memoryKind === "dream" || memory.frontmatter.memoryKind === "procedural")) {
|
|
11872
|
+
dedicatedSurfaceFilteredCount += 1;
|
|
11873
|
+
continue;
|
|
11874
|
+
}
|
|
10465
11875
|
if (recencyWeight > 0) {
|
|
10466
11876
|
const createdAt = new Date(memory.frontmatter.created).getTime();
|
|
10467
11877
|
const ageMs = now - createdAt;
|
|
@@ -10561,6 +11971,16 @@ ${lines.join("\n\n")}`;
|
|
|
10561
11971
|
`lifecycle retrieval filter removed ${lifecycleFilteredCount} stale/archived candidates`
|
|
10562
11972
|
);
|
|
10563
11973
|
}
|
|
11974
|
+
if (temporalSupersededFilteredCount > 0) {
|
|
11975
|
+
log.debug(
|
|
11976
|
+
`temporal supersession filter removed ${temporalSupersededFilteredCount} superseded candidates`
|
|
11977
|
+
);
|
|
11978
|
+
}
|
|
11979
|
+
if (dedicatedSurfaceFilteredCount > 0) {
|
|
11980
|
+
log.debug(
|
|
11981
|
+
`dedicated surface filter removed ${dedicatedSurfaceFilteredCount} dream/procedural candidates from generic recall`
|
|
11982
|
+
);
|
|
11983
|
+
}
|
|
10564
11984
|
return boosted.sort((a, b) => b.score - a.score);
|
|
10565
11985
|
}
|
|
10566
11986
|
/**
|
|
@@ -10623,27 +12043,31 @@ ${lines.join("\n\n")}`;
|
|
|
10623
12043
|
);
|
|
10624
12044
|
if (!verification) continue;
|
|
10625
12045
|
if (verification.isContradiction && verification.confidence >= this.config.contradictionMinConfidence) {
|
|
12046
|
+
if (verification.whichIsNewer === "first") {
|
|
12047
|
+
log.info(
|
|
12048
|
+
`detected contradiction (confidence: ${verification.confidence}): ${existingMemory.frontmatter.id} vs new memory \u2014 existing is newer, incoming fact is stale`
|
|
12049
|
+
);
|
|
12050
|
+
continue;
|
|
12051
|
+
}
|
|
10626
12052
|
if (this.config.contradictionAutoResolve) {
|
|
10627
|
-
|
|
10628
|
-
|
|
10629
|
-
|
|
10630
|
-
|
|
10631
|
-
|
|
10632
|
-
|
|
10633
|
-
);
|
|
10634
|
-
return {
|
|
10635
|
-
supersededId: existingMemory.frontmatter.id,
|
|
10636
|
-
confidence: verification.confidence,
|
|
10637
|
-
reason: verification.reasoning,
|
|
10638
|
-
supersededPath: existingMemory.path,
|
|
10639
|
-
supersededCreated: existingMemory.frontmatter.created,
|
|
10640
|
-
supersededTags: existingMemory.frontmatter.tags ?? []
|
|
10641
|
-
};
|
|
10642
|
-
}
|
|
12053
|
+
await resultStorage.supersedeMemory(
|
|
12054
|
+
existingMemory.frontmatter.id,
|
|
12055
|
+
"pending-new",
|
|
12056
|
+
// Will be updated after the new memory is written
|
|
12057
|
+
verification.reasoning
|
|
12058
|
+
);
|
|
10643
12059
|
}
|
|
10644
12060
|
log.info(
|
|
10645
|
-
`detected contradiction (confidence: ${verification.confidence}): ${existingMemory.frontmatter.id} vs new memory`
|
|
12061
|
+
`detected contradiction (confidence: ${verification.confidence}): ${existingMemory.frontmatter.id} vs new memory${this.config.contradictionAutoResolve ? " (auto-resolved)" : " (queued for manual review)"}`
|
|
10646
12062
|
);
|
|
12063
|
+
return {
|
|
12064
|
+
supersededId: existingMemory.frontmatter.id,
|
|
12065
|
+
confidence: verification.confidence,
|
|
12066
|
+
reason: verification.reasoning,
|
|
12067
|
+
supersededPath: existingMemory.path,
|
|
12068
|
+
supersededCreated: existingMemory.frontmatter.created,
|
|
12069
|
+
supersededTags: existingMemory.frontmatter.tags ?? []
|
|
12070
|
+
};
|
|
10647
12071
|
}
|
|
10648
12072
|
}
|
|
10649
12073
|
return null;
|
|
@@ -10731,6 +12155,9 @@ ${lines.join("\n\n")}`;
|
|
|
10731
12155
|
export {
|
|
10732
12156
|
rollbackFromEngramMigration,
|
|
10733
12157
|
migrateFromEngram,
|
|
12158
|
+
buildProcedureRecallSection,
|
|
12159
|
+
decideSemanticDedup,
|
|
12160
|
+
dedupeEntitySynthesisEvidenceEntries,
|
|
10734
12161
|
defaultWorkspaceDir,
|
|
10735
12162
|
sanitizeSessionKeyForFilename,
|
|
10736
12163
|
isArtifactMemoryPath,
|
|
@@ -10758,4 +12185,4 @@ export {
|
|
|
10758
12185
|
resolvePersistedMemoryRelativePath,
|
|
10759
12186
|
Orchestrator
|
|
10760
12187
|
};
|
|
10761
|
-
//# sourceMappingURL=chunk-
|
|
12188
|
+
//# sourceMappingURL=chunk-DEPL3635.js.map
|