@remnic/core 1.0.2 → 1.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +1 -0
- package/dist/access-cli.d.ts +13 -3
- package/dist/access-cli.js +90 -75
- package/dist/access-cli.js.map +1 -1
- package/dist/access-http.d.ts +10 -3
- package/dist/access-http.js +25 -18
- package/dist/access-mcp.d.ts +30 -3
- package/dist/access-mcp.js +16 -1
- package/dist/access-schema.d.ts +12 -12
- package/dist/access-schema.js +1 -1
- package/dist/access-service.d.ts +65 -4
- package/dist/access-service.js +21 -15
- package/dist/active-memory-bridge.d.ts +66 -0
- package/dist/active-memory-bridge.js +11 -0
- package/dist/active-recall.d.ts +96 -0
- package/dist/active-recall.js +308 -0
- package/dist/active-recall.js.map +1 -0
- package/dist/behavior-learner.js +1 -1
- package/dist/bootstrap.d.ts +6 -3
- package/dist/bootstrap.js +2 -2
- package/dist/boxes.js +2 -2
- package/dist/briefing.d.ts +169 -0
- package/dist/briefing.js +52 -0
- package/dist/briefing.js.map +1 -0
- package/dist/buffer.d.ts +19 -5
- package/dist/buffer.js +2 -2
- package/dist/calibration.js +6 -6
- package/dist/causal-behavior.js +5 -5
- package/dist/causal-chain.js +3 -3
- package/dist/causal-consolidation.d.ts +22 -2
- package/dist/causal-consolidation.js +36 -9
- package/dist/causal-consolidation.js.map +1 -1
- package/dist/causal-retrieval.js +6 -6
- package/dist/causal-trajectory-graph.js +1 -1
- package/dist/causal-trajectory.d.ts +14 -1
- package/dist/causal-trajectory.js +5 -1
- package/dist/{chunk-KWBU5S5U.js → chunk-2ODBA7MQ.js} +9 -3
- package/dist/chunk-2ODBA7MQ.js.map +1 -0
- package/dist/{chunk-6UJQNRIO.js → chunk-2VFW5K5U.js} +93 -36
- package/dist/chunk-2VFW5K5U.js.map +1 -0
- package/dist/chunk-3PG3H5TD.js +7 -0
- package/dist/chunk-3PG3H5TD.js.map +1 -0
- package/dist/{chunk-NTTLPF7F.js → chunk-3QFQGRHO.js} +5 -5
- package/dist/chunk-4DJQYKMN.js +187 -0
- package/dist/chunk-4DJQYKMN.js.map +1 -0
- package/dist/chunk-4KAN3GZ3.js +225 -0
- package/dist/chunk-4KAN3GZ3.js.map +1 -0
- package/dist/{chunk-ORZMT74A.js → chunk-4NRAJUDS.js} +11 -1
- package/dist/chunk-4NRAJUDS.js.map +1 -0
- package/dist/{chunk-B7LOFDVE.js → chunk-4WMCPJWX.js} +8 -3
- package/dist/chunk-4WMCPJWX.js.map +1 -0
- package/dist/{chunk-G3AG3KZN.js → chunk-5IZL4DCV.js} +2 -2
- package/dist/{chunk-BRK4ODMI.js → chunk-5NPGSAVB.js} +2 -2
- package/dist/chunk-6MKAMLQL.js +16 -0
- package/dist/chunk-6MKAMLQL.js.map +1 -0
- package/dist/{chunk-ESSMF2FR.js → chunk-6PFRXT4K.js} +15 -6
- package/dist/chunk-6PFRXT4K.js.map +1 -0
- package/dist/chunk-6ZH4TU6I.js +245 -0
- package/dist/chunk-6ZH4TU6I.js.map +1 -0
- package/dist/{chunk-V4YC4LUK.js → chunk-74JR4N5J.js} +175 -63
- package/dist/chunk-74JR4N5J.js.map +1 -0
- package/dist/{chunk-L5RPWGFK.js → chunk-7DHTMOND.js} +2 -2
- package/dist/{chunk-TVVVQQAK.js → chunk-7PA4OZEU.js} +53 -11
- package/dist/chunk-7PA4OZEU.js.map +1 -0
- package/dist/{chunk-Q6FETXJA.js → chunk-7SEAZFFB.js} +2 -2
- package/dist/chunk-ALXMCZEU.js +332 -0
- package/dist/chunk-ALXMCZEU.js.map +1 -0
- package/dist/{chunk-QANCTXQF.js → chunk-AYPYCLR7.js} +3 -3
- package/dist/{chunk-WWIQTB2Y.js → chunk-BKQJBXXX.js} +9 -2
- package/dist/chunk-BKQJBXXX.js.map +1 -0
- package/dist/{chunk-LP47L3ZX.js → chunk-BTY5RRRF.js} +7 -7
- package/dist/{chunk-SCHEKPYH.js → chunk-C2EFFULQ.js} +1 -1
- package/dist/{chunk-GJR6D6KC.js → chunk-D654IBA6.js} +2 -2
- package/dist/{chunk-UV2FO7J4.js → chunk-E6K4NIEU.js} +2 -2
- package/dist/{chunk-T4WRIV2C.js → chunk-EABGC2TL.js} +2 -2
- package/dist/chunk-ECKDIK5F.js +813 -0
- package/dist/chunk-ECKDIK5F.js.map +1 -0
- package/dist/chunk-EJI5XIBB.js +232 -0
- package/dist/chunk-EJI5XIBB.js.map +1 -0
- package/dist/{chunk-ONRU4L2N.js → chunk-FEMOX5AD.js} +2 -2
- package/dist/{chunk-IFFFR3MR.js → chunk-FSFEQI74.js} +3 -3
- package/dist/chunk-G4SK7DSQ.js +121 -0
- package/dist/chunk-G4SK7DSQ.js.map +1 -0
- package/dist/{chunk-UIYZ5T3I.js → chunk-GJQPH5G3.js} +8 -8
- package/dist/{chunk-2PO5ZRKV.js → chunk-GZCUW5IC.js} +16 -3
- package/dist/chunk-GZCUW5IC.js.map +1 -0
- package/dist/{chunk-IZME7KW2.js → chunk-HITJFT7E.js} +24 -10
- package/dist/{chunk-IZME7KW2.js.map → chunk-HITJFT7E.js.map} +1 -1
- package/dist/chunk-IQT3XTKW.js +121 -0
- package/dist/chunk-IQT3XTKW.js.map +1 -0
- package/dist/{chunk-BDFZXRSO.js → chunk-J4IYOZZ5.js} +15 -2
- package/dist/chunk-J4IYOZZ5.js.map +1 -0
- package/dist/{chunk-ZKYI7UVO.js → chunk-JR4ZC3G4.js} +2 -2
- package/dist/{chunk-UCYSTFZR.js → chunk-JRNQ3RNA.js} +2 -2
- package/dist/{chunk-UYSKNO6E.js → chunk-JROGC36Y.js} +15 -4
- package/dist/chunk-JROGC36Y.js.map +1 -0
- package/dist/{chunk-GPGBSNKM.js → chunk-K4FLSOR5.js} +2 -2
- package/dist/{chunk-M5ZBBBJI.js → chunk-KEG4GNGI.js} +2 -2
- package/dist/chunk-KVE7R4CG.js +320 -0
- package/dist/chunk-KVE7R4CG.js.map +1 -0
- package/dist/{chunk-L7WO3MZ4.js → chunk-KWP7T3DP.js} +2 -2
- package/dist/chunk-LAYN4LDC.js +267 -0
- package/dist/chunk-LAYN4LDC.js.map +1 -0
- package/dist/{chunk-PGK3VUHN.js → chunk-MTLYEMJB.js} +3 -2
- package/dist/chunk-MTLYEMJB.js.map +1 -0
- package/dist/{chunk-J47FNDR7.js → chunk-MYQWXITD.js} +7 -7
- package/dist/{chunk-YNI4S5WT.js → chunk-N53K2EXC.js} +2 -2
- package/dist/{chunk-763GUIOU.js → chunk-NBNN5GOB.js} +2 -2
- package/dist/{chunk-CXWFUJR2.js → chunk-NSB3WSYS.js} +125 -6
- package/dist/chunk-NSB3WSYS.js.map +1 -0
- package/dist/{chunk-KL4CP4SB.js → chunk-O5ETUNBT.js} +17 -5
- package/dist/chunk-O5ETUNBT.js.map +1 -0
- package/dist/{chunk-OOSWAUYB.js → chunk-ODWDQNRE.js} +2 -2
- package/dist/{chunk-ISY75RLM.js → chunk-OJFGVJS6.js} +288 -7
- package/dist/chunk-OJFGVJS6.js.map +1 -0
- package/dist/{chunk-HLBYLYRD.js → chunk-PAORGQRI.js} +70 -13
- package/dist/chunk-PAORGQRI.js.map +1 -0
- package/dist/{chunk-ZJLY4QSU.js → chunk-PMB3WGDL.js} +69 -6
- package/dist/chunk-PMB3WGDL.js.map +1 -0
- package/dist/{chunk-J3BT33K7.js → chunk-POBPGDWI.js} +5 -5
- package/dist/{chunk-QWUUMMIK.js → chunk-POMSFKTB.js} +1351 -76
- package/dist/chunk-POMSFKTB.js.map +1 -0
- package/dist/{chunk-OTAVQCSF.js → chunk-PYXS46O7.js} +2 -2
- package/dist/chunk-QDW3E4RD.js +108 -0
- package/dist/chunk-QDW3E4RD.js.map +1 -0
- package/dist/{chunk-YNCQ7E4M.js → chunk-QDYXG4CS.js} +4 -3
- package/dist/chunk-QDYXG4CS.js.map +1 -0
- package/dist/{chunk-XUHI52HK.js → chunk-QKAH5B6E.js} +4 -4
- package/dist/{chunk-HLXVTBF3.js → chunk-QNJMBKFK.js} +3 -2
- package/dist/chunk-QNJMBKFK.js.map +1 -0
- package/dist/chunk-RCICHSHL.js +789 -0
- package/dist/chunk-RCICHSHL.js.map +1 -0
- package/dist/{chunk-HG2NKWR2.js → chunk-S4LX5EBI.js} +2 -2
- package/dist/{chunk-4A24LIM2.js → chunk-S75M5ZRK.js} +2 -2
- package/dist/{chunk-QCCCQT3O.js → chunk-TBBDFYXW.js} +2 -2
- package/dist/chunk-TBBDFYXW.js.map +1 -0
- package/dist/{chunk-U4PV25RD.js → chunk-U2IQTSBY.js} +1 -1
- package/dist/chunk-U2IQTSBY.js.map +1 -0
- package/dist/chunk-U66YHYC7.js +31 -0
- package/dist/chunk-U66YHYC7.js.map +1 -0
- package/dist/{chunk-MWGVGUIS.js → chunk-UEYA6UC7.js} +36 -4
- package/dist/chunk-UEYA6UC7.js.map +1 -0
- package/dist/{chunk-MDDAA2AO.js → chunk-UPMD5XND.js} +2 -2
- package/dist/{chunk-M5KEYE5E.js → chunk-URB2WSKZ.js} +2 -2
- package/dist/chunk-UVJFDP7P.js +202 -0
- package/dist/chunk-UVJFDP7P.js.map +1 -0
- package/dist/{chunk-QY2BHY5O.js → chunk-V7XCAHIB.js} +265 -25
- package/dist/chunk-V7XCAHIB.js.map +1 -0
- package/dist/chunk-W6SL7OFG.js +180 -0
- package/dist/chunk-W6SL7OFG.js.map +1 -0
- package/dist/{chunk-QDOSNLB4.js → chunk-X4WESCKA.js} +17 -15
- package/dist/chunk-X4WESCKA.js.map +1 -0
- package/dist/{chunk-OTFNI3OO.js → chunk-XMGSSBFX.js} +1738 -383
- package/dist/chunk-XMGSSBFX.js.map +1 -0
- package/dist/chunk-YDBIWGNI.js +298 -0
- package/dist/chunk-YDBIWGNI.js.map +1 -0
- package/dist/chunk-YFYL2SIJ.js +7857 -0
- package/dist/chunk-YFYL2SIJ.js.map +1 -0
- package/dist/chunking.js +1 -1
- package/dist/citations.d.ts +67 -0
- package/dist/citations.js +13 -0
- package/dist/citations.js.map +1 -0
- package/dist/cli-DwIBnp2g.d.ts +1240 -0
- package/dist/cli.d.ts +31 -1147
- package/dist/cli.js +149 -7092
- package/dist/cli.js.map +1 -1
- package/dist/codex-materialize-CQlLTzke.d.ts +139 -0
- package/dist/codex-thread-key.d.ts +3 -0
- package/dist/codex-thread-key.js +7 -0
- package/dist/codex-thread-key.js.map +1 -0
- package/dist/config.js +3 -2
- package/dist/connectors/codex/instructions.md +160 -0
- package/dist/connectors/codex/resources/namespace-cheatsheet.md +48 -0
- package/dist/day-summary.d.ts +7 -2
- package/dist/day-summary.js +5 -2
- package/dist/embedding-fallback.d.ts +96 -2
- package/dist/embedding-fallback.js +6 -4
- package/dist/{engine-2A6J4XEX.js → engine-X7X3AAG3.js} +10 -7
- package/dist/engine-X7X3AAG3.js.map +1 -0
- package/dist/entity-retrieval.d.ts +3 -2
- package/dist/entity-retrieval.js +10 -7
- package/dist/entity-schema.d.ts +11 -0
- package/dist/entity-schema.js +19 -0
- package/dist/entity-schema.js.map +1 -0
- package/dist/explicit-capture.d.ts +6 -3
- package/dist/explicit-capture.js +2 -2
- package/dist/extraction-judge.d.ts +66 -0
- package/dist/extraction-judge.js +18 -0
- package/dist/extraction-judge.js.map +1 -0
- package/dist/extraction.d.ts +1 -0
- package/dist/extraction.js +12 -10
- package/dist/fallback-llm.js +4 -4
- package/dist/graph.js +1 -1
- package/dist/importance.d.ts +11 -1
- package/dist/importance.js +3 -1
- package/dist/index.d.ts +1140 -8
- package/dist/index.js +3350 -333
- package/dist/index.js.map +1 -1
- package/dist/intent.d.ts +2 -1
- package/dist/intent.js +3 -1
- package/dist/lifecycle.js +1 -1
- package/dist/local-llm.js +2 -2
- package/dist/logger.d.ts +1 -1
- package/dist/logger.js +1 -1
- package/dist/memory-cache.d.ts +2 -2
- package/dist/memory-cache.js +1 -1
- package/dist/{memory-projection-store-NxMkbocT.d.ts → memory-projection-store-DeSXPh1j.d.ts} +1 -1
- package/dist/memory-projection-store.d.ts +1 -1
- package/dist/model-registry.js +2 -2
- package/dist/models-json.js +2 -2
- package/dist/native-knowledge.js +2 -2
- package/dist/negative.js +2 -2
- package/dist/operator-toolkit.js +20 -16
- package/dist/{orchestrator-zTa-Qo-1.d.ts → orchestrator-B9kwlCep.d.ts} +252 -7
- package/dist/orchestrator.d.ts +6 -3
- package/dist/orchestrator.js +70 -58
- package/dist/page-versioning.d.ts +77 -0
- package/dist/page-versioning.js +15 -0
- package/dist/page-versioning.js.map +1 -0
- package/dist/plugin-id.d.ts +37 -0
- package/dist/plugin-id.js +11 -0
- package/dist/plugin-id.js.map +1 -0
- package/dist/policy-runtime.js +2 -2
- package/dist/profiling.js +2 -2
- package/dist/qmd.d.ts +5 -2
- package/dist/qmd.js +3 -3
- package/dist/recall-audit.d.ts +20 -0
- package/dist/recall-audit.js +50 -0
- package/dist/recall-audit.js.map +1 -0
- package/dist/recall-mmr.d.ts +152 -0
- package/dist/recall-mmr.js +17 -0
- package/dist/recall-mmr.js.map +1 -0
- package/dist/recall-qos.js +2 -2
- package/dist/recall-state.js +2 -2
- package/dist/relevance.js +2 -2
- package/dist/resolve-provider-secret.js +2 -2
- package/dist/resume-bundles.js +5 -4
- package/dist/retrieval-agents.js +2 -2
- package/dist/retrieval.js +2 -2
- package/dist/schemas.d.ts +398 -40
- package/dist/schemas.js +3 -1
- package/dist/sdk-compat.d.ts +2 -0
- package/dist/sdk-compat.js +6 -3
- package/dist/sdk-compat.js.map +1 -1
- package/dist/semantic-chunking.d.ts +87 -0
- package/dist/semantic-chunking.js +20 -0
- package/dist/semantic-chunking.js.map +1 -0
- package/dist/semantic-consolidation-DrvSYRdB.d.ts +119 -0
- package/dist/semantic-consolidation.d.ts +4 -42
- package/dist/semantic-consolidation.js +23 -2
- package/dist/semantic-rule-promotion.js +9 -6
- package/dist/semantic-rule-verifier.js +10 -7
- package/dist/session-observer-state.js +2 -2
- package/dist/session-toggles.d.ts +22 -0
- package/dist/session-toggles.js +116 -0
- package/dist/session-toggles.js.map +1 -0
- package/dist/skills-registry.d.ts +47 -0
- package/dist/skills-registry.js +48 -0
- package/dist/skills-registry.js.map +1 -0
- package/dist/source-attribution.d.ts +169 -0
- package/dist/source-attribution.js +27 -0
- package/dist/source-attribution.js.map +1 -0
- package/dist/storage.d.ts +171 -10
- package/dist/storage.js +16 -5
- package/dist/summarizer.js +7 -7
- package/dist/temporal-supersession.d.ts +127 -0
- package/dist/temporal-supersession.js +20 -0
- package/dist/temporal-supersession.js.map +1 -0
- package/dist/threading.js +2 -2
- package/dist/tier-migration.d.ts +2 -1
- package/dist/tier-routing.js +2 -2
- package/dist/tokens.d.ts +21 -1
- package/dist/tokens.js +5 -1
- package/dist/transcript.js +2 -2
- package/dist/types.d.ts +497 -3
- package/dist/types.js +1 -1
- package/dist/utility-learner.js +2 -2
- package/dist/utility-runtime.js +3 -3
- package/dist/verified-recall.js +11 -8
- package/dist/whitespace.d.ts +4 -0
- package/dist/whitespace.js +9 -0
- package/dist/whitespace.js.map +1 -0
- package/package.json +14 -8
- package/dist/chunk-2CJCWDMR.js +0 -87
- package/dist/chunk-2CJCWDMR.js.map +0 -1
- package/dist/chunk-2PO5ZRKV.js.map +0 -1
- package/dist/chunk-6UJQNRIO.js.map +0 -1
- package/dist/chunk-B7LOFDVE.js.map +0 -1
- package/dist/chunk-BDFZXRSO.js.map +0 -1
- package/dist/chunk-CXWFUJR2.js.map +0 -1
- package/dist/chunk-DORBM6OB.js +0 -81
- package/dist/chunk-DORBM6OB.js.map +0 -1
- package/dist/chunk-ESSMF2FR.js.map +0 -1
- package/dist/chunk-HLBYLYRD.js.map +0 -1
- package/dist/chunk-HLXVTBF3.js.map +0 -1
- package/dist/chunk-ISY75RLM.js.map +0 -1
- package/dist/chunk-KL4CP4SB.js.map +0 -1
- package/dist/chunk-KWBU5S5U.js.map +0 -1
- package/dist/chunk-MWGVGUIS.js.map +0 -1
- package/dist/chunk-ORZMT74A.js.map +0 -1
- package/dist/chunk-OTFNI3OO.js.map +0 -1
- package/dist/chunk-PGK3VUHN.js.map +0 -1
- package/dist/chunk-QCCCQT3O.js.map +0 -1
- package/dist/chunk-QDOSNLB4.js.map +0 -1
- package/dist/chunk-QPKFPHOO.js +0 -178
- package/dist/chunk-QPKFPHOO.js.map +0 -1
- package/dist/chunk-QWUUMMIK.js.map +0 -1
- package/dist/chunk-QY2BHY5O.js.map +0 -1
- package/dist/chunk-TVVVQQAK.js.map +0 -1
- package/dist/chunk-U4PV25RD.js.map +0 -1
- package/dist/chunk-UYSKNO6E.js.map +0 -1
- package/dist/chunk-V4YC4LUK.js.map +0 -1
- package/dist/chunk-WWIQTB2Y.js.map +0 -1
- package/dist/chunk-YNCQ7E4M.js.map +0 -1
- package/dist/chunk-ZJLY4QSU.js.map +0 -1
- /package/dist/{engine-2A6J4XEX.js.map → active-memory-bridge.js.map} +0 -0
- /package/dist/{chunk-NTTLPF7F.js.map → chunk-3QFQGRHO.js.map} +0 -0
- /package/dist/{chunk-G3AG3KZN.js.map → chunk-5IZL4DCV.js.map} +0 -0
- /package/dist/{chunk-BRK4ODMI.js.map → chunk-5NPGSAVB.js.map} +0 -0
- /package/dist/{chunk-L5RPWGFK.js.map → chunk-7DHTMOND.js.map} +0 -0
- /package/dist/{chunk-Q6FETXJA.js.map → chunk-7SEAZFFB.js.map} +0 -0
- /package/dist/{chunk-QANCTXQF.js.map → chunk-AYPYCLR7.js.map} +0 -0
- /package/dist/{chunk-LP47L3ZX.js.map → chunk-BTY5RRRF.js.map} +0 -0
- /package/dist/{chunk-SCHEKPYH.js.map → chunk-C2EFFULQ.js.map} +0 -0
- /package/dist/{chunk-GJR6D6KC.js.map → chunk-D654IBA6.js.map} +0 -0
- /package/dist/{chunk-UV2FO7J4.js.map → chunk-E6K4NIEU.js.map} +0 -0
- /package/dist/{chunk-T4WRIV2C.js.map → chunk-EABGC2TL.js.map} +0 -0
- /package/dist/{chunk-ONRU4L2N.js.map → chunk-FEMOX5AD.js.map} +0 -0
- /package/dist/{chunk-IFFFR3MR.js.map → chunk-FSFEQI74.js.map} +0 -0
- /package/dist/{chunk-UIYZ5T3I.js.map → chunk-GJQPH5G3.js.map} +0 -0
- /package/dist/{chunk-ZKYI7UVO.js.map → chunk-JR4ZC3G4.js.map} +0 -0
- /package/dist/{chunk-UCYSTFZR.js.map → chunk-JRNQ3RNA.js.map} +0 -0
- /package/dist/{chunk-GPGBSNKM.js.map → chunk-K4FLSOR5.js.map} +0 -0
- /package/dist/{chunk-M5ZBBBJI.js.map → chunk-KEG4GNGI.js.map} +0 -0
- /package/dist/{chunk-L7WO3MZ4.js.map → chunk-KWP7T3DP.js.map} +0 -0
- /package/dist/{chunk-J47FNDR7.js.map → chunk-MYQWXITD.js.map} +0 -0
- /package/dist/{chunk-YNI4S5WT.js.map → chunk-N53K2EXC.js.map} +0 -0
- /package/dist/{chunk-763GUIOU.js.map → chunk-NBNN5GOB.js.map} +0 -0
- /package/dist/{chunk-OOSWAUYB.js.map → chunk-ODWDQNRE.js.map} +0 -0
- /package/dist/{chunk-J3BT33K7.js.map → chunk-POBPGDWI.js.map} +0 -0
- /package/dist/{chunk-OTAVQCSF.js.map → chunk-PYXS46O7.js.map} +0 -0
- /package/dist/{chunk-XUHI52HK.js.map → chunk-QKAH5B6E.js.map} +0 -0
- /package/dist/{chunk-HG2NKWR2.js.map → chunk-S4LX5EBI.js.map} +0 -0
- /package/dist/{chunk-4A24LIM2.js.map → chunk-S75M5ZRK.js.map} +0 -0
- /package/dist/{chunk-MDDAA2AO.js.map → chunk-UPMD5XND.js.map} +0 -0
- /package/dist/{chunk-M5KEYE5E.js.map → chunk-URB2WSKZ.js.map} +0 -0
|
@@ -2,36 +2,48 @@ import {
|
|
|
2
2
|
CompoundingEngine,
|
|
3
3
|
SharedContextManager,
|
|
4
4
|
defaultTierMigrationCycleBudget
|
|
5
|
-
} from "./chunk-
|
|
5
|
+
} from "./chunk-JROGC36Y.js";
|
|
6
6
|
import {
|
|
7
7
|
applyUtilityPromotionRuntimePolicy,
|
|
8
8
|
applyUtilityRankingRuntimeDelta,
|
|
9
9
|
loadUtilityRuntimeValues
|
|
10
|
-
} from "./chunk-
|
|
10
|
+
} from "./chunk-FSFEQI74.js";
|
|
11
|
+
import {
|
|
12
|
+
applyTemporalSupersession,
|
|
13
|
+
normalizeSupersessionKey,
|
|
14
|
+
shouldFilterSupersededFromRecall
|
|
15
|
+
} from "./chunk-W6SL7OFG.js";
|
|
11
16
|
import {
|
|
12
17
|
TierMigrationExecutor
|
|
13
18
|
} from "./chunk-Z5AAYHUC.js";
|
|
14
19
|
import {
|
|
15
20
|
decideTierTransition
|
|
16
|
-
} from "./chunk-
|
|
21
|
+
} from "./chunk-S75M5ZRK.js";
|
|
17
22
|
import {
|
|
18
23
|
TmtBuilder
|
|
19
24
|
} from "./chunk-TPB3I2AC.js";
|
|
20
25
|
import {
|
|
21
26
|
extractTopics
|
|
22
27
|
} from "./chunk-UHGBNIOS.js";
|
|
28
|
+
import {
|
|
29
|
+
HourlySummarizer
|
|
30
|
+
} from "./chunk-BTY5RRRF.js";
|
|
31
|
+
import {
|
|
32
|
+
semanticChunkContent
|
|
33
|
+
} from "./chunk-KVE7R4CG.js";
|
|
23
34
|
import {
|
|
24
35
|
SessionObserverState
|
|
25
|
-
} from "./chunk-
|
|
36
|
+
} from "./chunk-JR4ZC3G4.js";
|
|
26
37
|
import {
|
|
27
|
-
|
|
28
|
-
|
|
38
|
+
RerankCache,
|
|
39
|
+
rerankLocalOrNoop
|
|
40
|
+
} from "./chunk-C7VW7C3F.js";
|
|
29
41
|
import {
|
|
30
42
|
mergeWithAgentResults,
|
|
31
43
|
runDirectAgent,
|
|
32
44
|
runTemporalAgent,
|
|
33
45
|
shouldRunAgent
|
|
34
|
-
} from "./chunk-
|
|
46
|
+
} from "./chunk-K4FLSOR5.js";
|
|
35
47
|
import {
|
|
36
48
|
clearIndexes,
|
|
37
49
|
deindexMemory,
|
|
@@ -46,38 +58,32 @@ import {
|
|
|
46
58
|
} from "./chunk-V3RXWQIE.js";
|
|
47
59
|
import {
|
|
48
60
|
applyRuntimeRetrievalPolicy
|
|
49
|
-
} from "./chunk-
|
|
61
|
+
} from "./chunk-5IZL4DCV.js";
|
|
50
62
|
import {
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
63
|
+
reorderRecallResultsWithMmr
|
|
64
|
+
} from "./chunk-YDBIWGNI.js";
|
|
65
|
+
import {
|
|
66
|
+
createRecallSectionMetricRecorder
|
|
67
|
+
} from "./chunk-7DHTMOND.js";
|
|
55
68
|
import {
|
|
56
69
|
LastRecallStore,
|
|
57
70
|
TierMigrationStatusStore,
|
|
58
71
|
clampGraphRecallExpandedEntries
|
|
59
|
-
} from "./chunk-
|
|
72
|
+
} from "./chunk-S4LX5EBI.js";
|
|
60
73
|
import {
|
|
61
74
|
findUnresolvedEntityRefs
|
|
62
75
|
} from "./chunk-X7XN6YU4.js";
|
|
63
76
|
import {
|
|
64
77
|
RelevanceStore
|
|
65
|
-
} from "./chunk-
|
|
66
|
-
import {
|
|
67
|
-
RerankCache,
|
|
68
|
-
rerankLocalOrNoop
|
|
69
|
-
} from "./chunk-C7VW7C3F.js";
|
|
78
|
+
} from "./chunk-5NPGSAVB.js";
|
|
70
79
|
import {
|
|
71
80
|
buildQmdRecallCacheKey,
|
|
72
81
|
getCachedQmdRecall,
|
|
73
82
|
setCachedQmdRecall
|
|
74
83
|
} from "./chunk-YCN4BVDK.js";
|
|
75
|
-
import {
|
|
76
|
-
createRecallSectionMetricRecorder
|
|
77
|
-
} from "./chunk-L5RPWGFK.js";
|
|
78
84
|
import {
|
|
79
85
|
NegativeExampleStore
|
|
80
|
-
} from "./chunk-
|
|
86
|
+
} from "./chunk-D654IBA6.js";
|
|
81
87
|
import {
|
|
82
88
|
evaluateMemoryActionPolicy
|
|
83
89
|
} from "./chunk-H63EDPFJ.js";
|
|
@@ -85,59 +91,62 @@ import {
|
|
|
85
91
|
hasBroadGraphIntent,
|
|
86
92
|
inferIntentFromText,
|
|
87
93
|
intentCompatibilityScore,
|
|
94
|
+
isTaskInitiationIntent,
|
|
88
95
|
planRecallMode
|
|
89
|
-
} from "./chunk-
|
|
90
|
-
import {
|
|
91
|
-
ExtractionEngine
|
|
92
|
-
} from "./chunk-6UJQNRIO.js";
|
|
93
|
-
import {
|
|
94
|
-
parseMemoryActionEligibilityContext
|
|
95
|
-
} from "./chunk-MWGVGUIS.js";
|
|
96
|
-
import {
|
|
97
|
-
ProfilingCollector
|
|
98
|
-
} from "./chunk-763GUIOU.js";
|
|
99
|
-
import {
|
|
100
|
-
ModelRegistry
|
|
101
|
-
} from "./chunk-ONRU4L2N.js";
|
|
102
|
-
import {
|
|
103
|
-
LocalLlmClient
|
|
104
|
-
} from "./chunk-MDDAA2AO.js";
|
|
96
|
+
} from "./chunk-BKQJBXXX.js";
|
|
105
97
|
import {
|
|
106
98
|
classifyMemoryKind
|
|
107
99
|
} from "./chunk-YAZNBMNF.js";
|
|
108
|
-
import {
|
|
109
|
-
formatDaySummaryMemories
|
|
110
|
-
} from "./chunk-2PO5ZRKV.js";
|
|
111
100
|
import {
|
|
112
101
|
EmbeddingFallback
|
|
113
|
-
} from "./chunk-
|
|
102
|
+
} from "./chunk-ALXMCZEU.js";
|
|
114
103
|
import {
|
|
115
104
|
buildEntityRecallSection,
|
|
116
105
|
entityRecentTranscriptLookbackHours,
|
|
117
106
|
readRecentEntityTranscriptEntries
|
|
118
|
-
} from "./chunk-
|
|
107
|
+
} from "./chunk-74JR4N5J.js";
|
|
108
|
+
import {
|
|
109
|
+
createVerdictCache,
|
|
110
|
+
judgeFactDurability,
|
|
111
|
+
validateProcedureExtraction
|
|
112
|
+
} from "./chunk-LAYN4LDC.js";
|
|
113
|
+
import {
|
|
114
|
+
ExtractionEngine
|
|
115
|
+
} from "./chunk-2VFW5K5U.js";
|
|
116
|
+
import {
|
|
117
|
+
parseMemoryActionEligibilityContext
|
|
118
|
+
} from "./chunk-UEYA6UC7.js";
|
|
119
|
+
import {
|
|
120
|
+
ProfilingCollector
|
|
121
|
+
} from "./chunk-NBNN5GOB.js";
|
|
122
|
+
import {
|
|
123
|
+
LocalLlmClient
|
|
124
|
+
} from "./chunk-UPMD5XND.js";
|
|
125
|
+
import {
|
|
126
|
+
ModelRegistry
|
|
127
|
+
} from "./chunk-FEMOX5AD.js";
|
|
119
128
|
import {
|
|
120
129
|
RoutingRulesStore,
|
|
121
130
|
normalizeReplaySessionKey
|
|
122
|
-
} from "./chunk-
|
|
131
|
+
} from "./chunk-PAORGQRI.js";
|
|
123
132
|
import {
|
|
124
133
|
searchVerifiedEpisodes
|
|
125
|
-
} from "./chunk-
|
|
134
|
+
} from "./chunk-GJQPH5G3.js";
|
|
126
135
|
import {
|
|
127
136
|
ThreadingManager
|
|
128
|
-
} from "./chunk-
|
|
137
|
+
} from "./chunk-JRNQ3RNA.js";
|
|
129
138
|
import {
|
|
130
139
|
searchVerifiedSemanticRules
|
|
131
|
-
} from "./chunk-
|
|
140
|
+
} from "./chunk-MYQWXITD.js";
|
|
132
141
|
import {
|
|
133
142
|
searchWorkProductLedgerEntries
|
|
134
143
|
} from "./chunk-CULXMQJH.js";
|
|
135
144
|
import {
|
|
136
145
|
TranscriptManager
|
|
137
|
-
} from "./chunk-
|
|
146
|
+
} from "./chunk-E6K4NIEU.js";
|
|
138
147
|
import {
|
|
139
148
|
PolicyRuntimeManager
|
|
140
|
-
} from "./chunk-
|
|
149
|
+
} from "./chunk-EABGC2TL.js";
|
|
141
150
|
import {
|
|
142
151
|
searchObjectiveStateSnapshots
|
|
143
152
|
} from "./chunk-LOBRX7VD.js";
|
|
@@ -148,13 +157,14 @@ import {
|
|
|
148
157
|
createConversationIndexRuntime,
|
|
149
158
|
createSearchBackend,
|
|
150
159
|
writeConversationChunks
|
|
151
|
-
} from "./chunk-
|
|
160
|
+
} from "./chunk-HITJFT7E.js";
|
|
152
161
|
import {
|
|
153
162
|
parseQmdExplain
|
|
154
|
-
} from "./chunk-
|
|
163
|
+
} from "./chunk-7PA4OZEU.js";
|
|
155
164
|
import {
|
|
165
|
+
isAboveImportanceThreshold,
|
|
156
166
|
scoreImportance
|
|
157
|
-
} from "./chunk-
|
|
167
|
+
} from "./chunk-J4IYOZZ5.js";
|
|
158
168
|
import {
|
|
159
169
|
searchHarmonicRetrieval
|
|
160
170
|
} from "./chunk-AAI7JARD.js";
|
|
@@ -162,19 +172,13 @@ import {
|
|
|
162
172
|
collectNativeKnowledgeChunks,
|
|
163
173
|
formatNativeKnowledgeSection,
|
|
164
174
|
searchNativeKnowledge
|
|
165
|
-
} from "./chunk-
|
|
175
|
+
} from "./chunk-7SEAZFFB.js";
|
|
166
176
|
import {
|
|
167
177
|
recordEvalShadowRecall
|
|
168
178
|
} from "./chunk-K6WK37A6.js";
|
|
169
179
|
import {
|
|
170
|
-
|
|
171
|
-
} from "./chunk-
|
|
172
|
-
import {
|
|
173
|
-
searchCausalTrajectories
|
|
174
|
-
} from "./chunk-ORZMT74A.js";
|
|
175
|
-
import {
|
|
176
|
-
chunkContent
|
|
177
|
-
} from "./chunk-B7LOFDVE.js";
|
|
180
|
+
CODEX_THREAD_KEY_PREFIX
|
|
181
|
+
} from "./chunk-3PG3H5TD.js";
|
|
178
182
|
import {
|
|
179
183
|
applyCommitmentLedgerLifecycle
|
|
180
184
|
} from "./chunk-FYIYMQ5N.js";
|
|
@@ -184,45 +188,83 @@ import {
|
|
|
184
188
|
refineCompressionGuidelineCandidateSemantically,
|
|
185
189
|
renderCompressionGuidelinesMarkdown
|
|
186
190
|
} from "./chunk-2NMMFZ5T.js";
|
|
191
|
+
import {
|
|
192
|
+
formatDaySummaryMemories
|
|
193
|
+
} from "./chunk-GZCUW5IC.js";
|
|
194
|
+
import {
|
|
195
|
+
buildConsolidationPrompt,
|
|
196
|
+
buildExtensionsBlockForConsolidation,
|
|
197
|
+
findSimilarClusters,
|
|
198
|
+
materializeAfterSemanticConsolidation,
|
|
199
|
+
parseConsolidationResponse
|
|
200
|
+
} from "./chunk-RCICHSHL.js";
|
|
201
|
+
import {
|
|
202
|
+
GraphIndex
|
|
203
|
+
} from "./chunk-C2EFFULQ.js";
|
|
204
|
+
import {
|
|
205
|
+
chunkContent
|
|
206
|
+
} from "./chunk-4WMCPJWX.js";
|
|
207
|
+
import {
|
|
208
|
+
buildRecallQueryPolicy
|
|
209
|
+
} from "./chunk-6HZ6AO2P.js";
|
|
187
210
|
import {
|
|
188
211
|
buildBehaviorSignalsForMemory,
|
|
189
212
|
dedupeBehaviorSignalsByMemoryAndHash
|
|
190
213
|
} from "./chunk-JWPLJLDU.js";
|
|
191
214
|
import {
|
|
192
215
|
BootstrapEngine
|
|
193
|
-
} from "./chunk-
|
|
216
|
+
} from "./chunk-N53K2EXC.js";
|
|
217
|
+
import {
|
|
218
|
+
BoxBuilder
|
|
219
|
+
} from "./chunk-URB2WSKZ.js";
|
|
194
220
|
import {
|
|
195
221
|
SmartBuffer
|
|
196
|
-
} from "./chunk-
|
|
222
|
+
} from "./chunk-UVJFDP7P.js";
|
|
197
223
|
import {
|
|
198
224
|
isDisagreementPrompt
|
|
199
225
|
} from "./chunk-XYIK4LF6.js";
|
|
200
226
|
import {
|
|
201
227
|
FallbackLlmClient
|
|
202
|
-
} from "./chunk-
|
|
203
|
-
import {
|
|
204
|
-
BoxBuilder
|
|
205
|
-
} from "./chunk-M5KEYE5E.js";
|
|
228
|
+
} from "./chunk-QKAH5B6E.js";
|
|
206
229
|
import {
|
|
207
230
|
resolveHomeDir
|
|
208
231
|
} from "./chunk-MARWOCVP.js";
|
|
209
|
-
import {
|
|
210
|
-
canReadNamespace,
|
|
211
|
-
defaultNamespaceForPrincipal,
|
|
212
|
-
recallNamespacesForPrincipal,
|
|
213
|
-
resolvePrincipal
|
|
214
|
-
} from "./chunk-N5AKDXAI.js";
|
|
215
232
|
import {
|
|
216
233
|
searchTrustZoneRecords
|
|
217
234
|
} from "./chunk-EQINRHYR.js";
|
|
235
|
+
import {
|
|
236
|
+
shouldSkipImplicitExtraction
|
|
237
|
+
} from "./chunk-QDYXG4CS.js";
|
|
238
|
+
import {
|
|
239
|
+
selectRouteRule
|
|
240
|
+
} from "./chunk-QNJMBKFK.js";
|
|
241
|
+
import {
|
|
242
|
+
buildProcedurePersistBody
|
|
243
|
+
} from "./chunk-QDW3E4RD.js";
|
|
244
|
+
import {
|
|
245
|
+
searchCausalTrajectories
|
|
246
|
+
} from "./chunk-4NRAJUDS.js";
|
|
247
|
+
import {
|
|
248
|
+
decideLifecycleTransition,
|
|
249
|
+
resolveLifecycleState
|
|
250
|
+
} from "./chunk-TBBDFYXW.js";
|
|
218
251
|
import {
|
|
219
252
|
ContentHashIndex,
|
|
220
253
|
StorageManager,
|
|
221
|
-
|
|
222
|
-
|
|
254
|
+
compareEntityTimestamps,
|
|
255
|
+
fingerprintEntityStructuredFacts,
|
|
256
|
+
normalizeAttributePairs,
|
|
257
|
+
normalizeEntityName,
|
|
258
|
+
parseEntityFile
|
|
259
|
+
} from "./chunk-POMSFKTB.js";
|
|
223
260
|
import {
|
|
224
261
|
confidenceTier
|
|
225
|
-
} from "./chunk-
|
|
262
|
+
} from "./chunk-U2IQTSBY.js";
|
|
263
|
+
import {
|
|
264
|
+
attachCitation,
|
|
265
|
+
hasCitationForTemplate,
|
|
266
|
+
stripCitationForTemplate
|
|
267
|
+
} from "./chunk-4KAN3GZ3.js";
|
|
226
268
|
import {
|
|
227
269
|
openBetterSqlite3
|
|
228
270
|
} from "./chunk-BOUYNNYD.js";
|
|
@@ -231,26 +273,22 @@ import {
|
|
|
231
273
|
rotateMarkdownFileToArchive
|
|
232
274
|
} from "./chunk-DM2T26WE.js";
|
|
233
275
|
import {
|
|
234
|
-
|
|
235
|
-
} from "./chunk-
|
|
236
|
-
import {
|
|
237
|
-
selectRouteRule
|
|
238
|
-
} from "./chunk-HLXVTBF3.js";
|
|
276
|
+
sanitizeMemoryContent
|
|
277
|
+
} from "./chunk-M62O4P4T.js";
|
|
239
278
|
import {
|
|
240
279
|
log
|
|
241
|
-
} from "./chunk-
|
|
280
|
+
} from "./chunk-2ODBA7MQ.js";
|
|
242
281
|
import {
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
} from "./chunk-QCCCQT3O.js";
|
|
282
|
+
canReadNamespace,
|
|
283
|
+
defaultNamespaceForPrincipal,
|
|
284
|
+
recallNamespacesForPrincipal,
|
|
285
|
+
resolvePrincipal
|
|
286
|
+
} from "./chunk-N5AKDXAI.js";
|
|
249
287
|
|
|
250
288
|
// src/orchestrator.ts
|
|
251
289
|
import path5 from "path";
|
|
252
290
|
import os from "os";
|
|
253
|
-
import { createHash as createHash2 } from "crypto";
|
|
291
|
+
import { createHash as createHash2, randomBytes } from "crypto";
|
|
254
292
|
import { existsSync as existsSync2 } from "fs";
|
|
255
293
|
import {
|
|
256
294
|
mkdir as mkdir4,
|
|
@@ -804,11 +842,65 @@ async function migrateFromEngram(options) {
|
|
|
804
842
|
}
|
|
805
843
|
}
|
|
806
844
|
|
|
845
|
+
// src/procedural/procedure-recall.ts
|
|
846
|
+
function tokenOverlapScore(prompt, memoryText) {
|
|
847
|
+
const norm = (s) => s.toLowerCase().split(/[^a-z0-9]+/).filter((t) => t.length > 2);
|
|
848
|
+
const promptTokens = new Set(norm(prompt));
|
|
849
|
+
const memTokens = new Set(norm(memoryText));
|
|
850
|
+
if (promptTokens.size === 0 || memTokens.size === 0) return 0;
|
|
851
|
+
let inter = 0;
|
|
852
|
+
for (const t of promptTokens) {
|
|
853
|
+
if (memTokens.has(t)) inter++;
|
|
854
|
+
}
|
|
855
|
+
const union = /* @__PURE__ */ new Set([...promptTokens, ...memTokens]);
|
|
856
|
+
return inter / Math.max(1, union.size);
|
|
857
|
+
}
|
|
858
|
+
function scoreProcedureForPrompt(m, prompt, queryIntent) {
|
|
859
|
+
const memText = `${m.content}
|
|
860
|
+
${(m.frontmatter.tags ?? []).join(" ")}`;
|
|
861
|
+
const jaccard = tokenOverlapScore(prompt, memText);
|
|
862
|
+
const memIntent = inferIntentFromText(m.content.slice(0, 2e3));
|
|
863
|
+
const intentScore = intentCompatibilityScore(queryIntent, memIntent);
|
|
864
|
+
return jaccard * 0.55 + intentScore * 0.45;
|
|
865
|
+
}
|
|
866
|
+
async function buildProcedureRecallSection(storage, prompt, config) {
|
|
867
|
+
if (config.procedural?.enabled !== true) return null;
|
|
868
|
+
const trimmed = typeof prompt === "string" ? prompt.trim() : "";
|
|
869
|
+
if (!trimmed) return null;
|
|
870
|
+
const queryIntent = inferIntentFromText(trimmed);
|
|
871
|
+
if (!isTaskInitiationIntent(queryIntent)) return null;
|
|
872
|
+
const maxN = Math.min(
|
|
873
|
+
10,
|
|
874
|
+
Math.max(
|
|
875
|
+
1,
|
|
876
|
+
typeof config.procedural.recallMaxProcedures === "number" && Number.isFinite(config.procedural.recallMaxProcedures) ? Math.floor(config.procedural.recallMaxProcedures) : 3
|
|
877
|
+
)
|
|
878
|
+
);
|
|
879
|
+
const all = await storage.readAllMemories();
|
|
880
|
+
const scored = all.filter(
|
|
881
|
+
(m) => m.frontmatter.category === "procedure" && m.frontmatter.status !== "pending_review" && m.frontmatter.status !== "rejected" && m.frontmatter.status !== "quarantined" && m.frontmatter.status !== "superseded" && m.frontmatter.status !== "archived"
|
|
882
|
+
).map((m) => ({ m, score: scoreProcedureForPrompt(m, trimmed, queryIntent) })).filter((x) => x.score > 0.04).sort((a, b) => b.score - a.score).slice(0, maxN);
|
|
883
|
+
if (scored.length === 0) return null;
|
|
884
|
+
const blocks = scored.map(({ m, score }) => {
|
|
885
|
+
const id = m.frontmatter.id;
|
|
886
|
+
const flat = m.content.replace(/\s+/g, " ").trim();
|
|
887
|
+
const preview = flat.slice(0, 320);
|
|
888
|
+
const suffix = flat.length > 320 ? "\u2026" : "";
|
|
889
|
+
return `### ${id} (match ${score.toFixed(2)})
|
|
890
|
+
|
|
891
|
+
${preview}${suffix}`;
|
|
892
|
+
});
|
|
893
|
+
return `## Relevant procedures
|
|
894
|
+
|
|
895
|
+
${blocks.join("\n\n")}`;
|
|
896
|
+
}
|
|
897
|
+
|
|
807
898
|
// src/maintenance/memory-governance-cron.ts
|
|
808
899
|
import { mkdir as mkdir2, readFile as readFile2, rename, rm as rm2, stat as stat2, writeFile as writeFile2 } from "fs/promises";
|
|
809
900
|
import path2 from "path";
|
|
810
901
|
var DAY_SUMMARY_CRON_ID = "engram-day-summary";
|
|
811
902
|
var GOVERNANCE_CRON_ID = "engram-nightly-governance";
|
|
903
|
+
var PROCEDURAL_MINING_CRON_ID = "engram-procedural-mining";
|
|
812
904
|
async function acquireCronJobsLock(jobsPath) {
|
|
813
905
|
const lockPath2 = `${jobsPath}.lock`;
|
|
814
906
|
const start = Date.now();
|
|
@@ -921,6 +1013,81 @@ async function ensureNightlyGovernanceCron(jobsPath, options) {
|
|
|
921
1013
|
delivery: { mode: "none" }
|
|
922
1014
|
}));
|
|
923
1015
|
}
|
|
1016
|
+
async function ensureProceduralMiningCron(jobsPath, options) {
|
|
1017
|
+
const scheduleExpr = typeof options.scheduleExpr === "string" && options.scheduleExpr.trim().length > 0 ? options.scheduleExpr.trim() : "17 3 * * *";
|
|
1018
|
+
const agentId = typeof options.agentId === "string" && options.agentId.trim().length > 0 ? options.agentId.trim() : "main";
|
|
1019
|
+
return ensureCronJob(jobsPath, PROCEDURAL_MINING_CRON_ID, () => ({
|
|
1020
|
+
id: PROCEDURAL_MINING_CRON_ID,
|
|
1021
|
+
agentId,
|
|
1022
|
+
name: "Remnic Procedural Mining (nightly)",
|
|
1023
|
+
enabled: true,
|
|
1024
|
+
schedule: {
|
|
1025
|
+
kind: "cron",
|
|
1026
|
+
expr: scheduleExpr,
|
|
1027
|
+
tz: options.timezone
|
|
1028
|
+
},
|
|
1029
|
+
sessionTarget: "isolated",
|
|
1030
|
+
wakeMode: "now",
|
|
1031
|
+
payload: {
|
|
1032
|
+
kind: "agentTurn",
|
|
1033
|
+
timeoutSeconds: 900,
|
|
1034
|
+
thinking: "off",
|
|
1035
|
+
message: "You are OpenClaw automation. Call tool `engram.procedure_mining_run` with empty params. If successful output exactly NO_REPLY. On error output one concise line. Do NOT use message tool."
|
|
1036
|
+
},
|
|
1037
|
+
delivery: { mode: "none" }
|
|
1038
|
+
}));
|
|
1039
|
+
}
|
|
1040
|
+
|
|
1041
|
+
// src/dedup/semantic.ts
|
|
1042
|
+
async function decideSemanticDedup(content, lookup, options) {
|
|
1043
|
+
if (!options.enabled) {
|
|
1044
|
+
return { action: "keep", reason: "disabled" };
|
|
1045
|
+
}
|
|
1046
|
+
if (options.candidates === 0) {
|
|
1047
|
+
return { action: "keep", reason: "disabled" };
|
|
1048
|
+
}
|
|
1049
|
+
const trimmed = typeof content === "string" ? content.trim() : "";
|
|
1050
|
+
if (!trimmed) {
|
|
1051
|
+
return { action: "keep", reason: "no_near_duplicate" };
|
|
1052
|
+
}
|
|
1053
|
+
const candidates = Math.max(1, Math.floor(options.candidates));
|
|
1054
|
+
let hits = [];
|
|
1055
|
+
try {
|
|
1056
|
+
hits = await lookup(trimmed, candidates);
|
|
1057
|
+
} catch {
|
|
1058
|
+
return { action: "keep", reason: "backend_unavailable" };
|
|
1059
|
+
}
|
|
1060
|
+
if (!Array.isArray(hits) || hits.length === 0) {
|
|
1061
|
+
return { action: "keep", reason: "no_candidates" };
|
|
1062
|
+
}
|
|
1063
|
+
let top;
|
|
1064
|
+
for (const hit of hits) {
|
|
1065
|
+
if (!hit || typeof hit.score !== "number" || !Number.isFinite(hit.score)) {
|
|
1066
|
+
continue;
|
|
1067
|
+
}
|
|
1068
|
+
if (!top || hit.score > top.score) {
|
|
1069
|
+
top = hit;
|
|
1070
|
+
}
|
|
1071
|
+
}
|
|
1072
|
+
if (!top) {
|
|
1073
|
+
return { action: "keep", reason: "no_near_duplicate" };
|
|
1074
|
+
}
|
|
1075
|
+
if (top.score >= options.threshold) {
|
|
1076
|
+
return {
|
|
1077
|
+
action: "skip",
|
|
1078
|
+
reason: "near_duplicate",
|
|
1079
|
+
topScore: top.score,
|
|
1080
|
+
topId: top.id,
|
|
1081
|
+
topPath: top.path
|
|
1082
|
+
};
|
|
1083
|
+
}
|
|
1084
|
+
return {
|
|
1085
|
+
action: "keep",
|
|
1086
|
+
reason: "no_near_duplicate",
|
|
1087
|
+
topScore: top.score,
|
|
1088
|
+
topId: top.id
|
|
1089
|
+
};
|
|
1090
|
+
}
|
|
924
1091
|
|
|
925
1092
|
// src/lcm/schema.ts
|
|
926
1093
|
import path3 from "path";
|
|
@@ -2233,6 +2400,69 @@ async function cleanupConversationChunks(rootDir, retentionDays) {
|
|
|
2233
2400
|
}
|
|
2234
2401
|
|
|
2235
2402
|
// src/orchestrator.ts
|
|
2403
|
+
function dedupeEntitySynthesisEvidenceEntries(entries) {
|
|
2404
|
+
const dedupedEvidenceEntries = [];
|
|
2405
|
+
const evidenceByFact = /* @__PURE__ */ new Map();
|
|
2406
|
+
for (const entry of entries) {
|
|
2407
|
+
const normalizedFact = entry.text.trim();
|
|
2408
|
+
if (!normalizedFact) continue;
|
|
2409
|
+
const existing = evidenceByFact.get(normalizedFact);
|
|
2410
|
+
if (!existing) {
|
|
2411
|
+
evidenceByFact.set(normalizedFact, { newest: entry, oldest: entry });
|
|
2412
|
+
continue;
|
|
2413
|
+
}
|
|
2414
|
+
if (compareEntityTimestamps(entry.timestamp, existing.newest.timestamp) > 0) {
|
|
2415
|
+
existing.newest = entry;
|
|
2416
|
+
}
|
|
2417
|
+
if (compareEntityTimestamps(entry.timestamp, existing.oldest.timestamp) < 0) {
|
|
2418
|
+
existing.oldest = entry;
|
|
2419
|
+
}
|
|
2420
|
+
}
|
|
2421
|
+
for (const { newest, oldest } of evidenceByFact.values()) {
|
|
2422
|
+
dedupedEvidenceEntries.push(newest);
|
|
2423
|
+
const newestKey = [
|
|
2424
|
+
newest.timestamp,
|
|
2425
|
+
newest.source ?? "",
|
|
2426
|
+
newest.sessionKey ?? "",
|
|
2427
|
+
newest.principal ?? "",
|
|
2428
|
+
newest.text
|
|
2429
|
+
].join("\0");
|
|
2430
|
+
const oldestKey = [
|
|
2431
|
+
oldest.timestamp,
|
|
2432
|
+
oldest.source ?? "",
|
|
2433
|
+
oldest.sessionKey ?? "",
|
|
2434
|
+
oldest.principal ?? "",
|
|
2435
|
+
oldest.text
|
|
2436
|
+
].join("\0");
|
|
2437
|
+
if (oldestKey !== newestKey) {
|
|
2438
|
+
dedupedEvidenceEntries.push(oldest);
|
|
2439
|
+
}
|
|
2440
|
+
}
|
|
2441
|
+
return dedupedEvidenceEntries;
|
|
2442
|
+
}
|
|
2443
|
+
function flattenStructuredSectionEvidence(sections) {
|
|
2444
|
+
return (sections ?? []).flatMap(
|
|
2445
|
+
(section) => section.facts.map((fact) => fact.trim()).filter((fact) => fact.length > 0).map((fact) => ({
|
|
2446
|
+
timestamp: "",
|
|
2447
|
+
text: fact,
|
|
2448
|
+
source: `section:${section.title}`
|
|
2449
|
+
}))
|
|
2450
|
+
);
|
|
2451
|
+
}
|
|
2452
|
+
function fingerprintEntitySynthesisEvidence(entity) {
|
|
2453
|
+
const fingerprint = createHash2("sha256");
|
|
2454
|
+
const timelineEntries = entity.timeline.map((entry) => [
|
|
2455
|
+
entry.timestamp,
|
|
2456
|
+
entry.source ?? "",
|
|
2457
|
+
entry.sessionKey ?? "",
|
|
2458
|
+
entry.principal ?? "",
|
|
2459
|
+
entry.text
|
|
2460
|
+
].join("\0")).sort();
|
|
2461
|
+
fingerprint.update(timelineEntries.join(""));
|
|
2462
|
+
fingerprint.update("");
|
|
2463
|
+
fingerprint.update(fingerprintEntityStructuredFacts(entity) ?? "");
|
|
2464
|
+
return fingerprint.digest("hex");
|
|
2465
|
+
}
|
|
2236
2466
|
function abortRecallError(message) {
|
|
2237
2467
|
const err = new Error(message);
|
|
2238
2468
|
Object.defineProperty(err, "name", { value: "AbortError" });
|
|
@@ -2501,7 +2731,8 @@ function parseMemoryIntentSnapshot(value) {
|
|
|
2501
2731
|
actionType: typeof candidate.actionType === "string" ? candidate.actionType : "unknown",
|
|
2502
2732
|
entityTypes: Array.isArray(candidate.entityTypes) ? candidate.entityTypes.filter(
|
|
2503
2733
|
(item) => typeof item === "string"
|
|
2504
|
-
) : []
|
|
2734
|
+
) : [],
|
|
2735
|
+
taskInitiation: candidate.taskInitiation === true
|
|
2505
2736
|
};
|
|
2506
2737
|
}
|
|
2507
2738
|
function buildQmdIntentHint(intent) {
|
|
@@ -2515,6 +2746,9 @@ function buildQmdIntentHint(intent) {
|
|
|
2515
2746
|
if (intent.entityTypes.length > 0) {
|
|
2516
2747
|
parts.push(`entities:${intent.entityTypes.join(",")}`);
|
|
2517
2748
|
}
|
|
2749
|
+
if (intent.taskInitiation === true) {
|
|
2750
|
+
parts.push("task_initiation");
|
|
2751
|
+
}
|
|
2518
2752
|
return parts.length > 0 ? parts.join(" ") : void 0;
|
|
2519
2753
|
}
|
|
2520
2754
|
function parseQmdRecallResults(value) {
|
|
@@ -2625,6 +2859,7 @@ var Orchestrator = class _Orchestrator {
|
|
|
2625
2859
|
summarizer;
|
|
2626
2860
|
localLlm;
|
|
2627
2861
|
fastLlm;
|
|
2862
|
+
judgeVerdictCache;
|
|
2628
2863
|
fastGatewayLlm;
|
|
2629
2864
|
modelRegistry;
|
|
2630
2865
|
relevance;
|
|
@@ -2669,6 +2904,7 @@ var Orchestrator = class _Orchestrator {
|
|
|
2669
2904
|
nonZeroExtractionsSinceConsolidation = 0;
|
|
2670
2905
|
lastConsolidationRunAtMs = 0;
|
|
2671
2906
|
consolidationInFlight = false;
|
|
2907
|
+
consolidationObservers = /* @__PURE__ */ new Set();
|
|
2672
2908
|
qmdMaintenanceTimer = null;
|
|
2673
2909
|
qmdMaintenancePending = false;
|
|
2674
2910
|
qmdMaintenanceInFlight = false;
|
|
@@ -2688,6 +2924,41 @@ var Orchestrator = class _Orchestrator {
|
|
|
2688
2924
|
// Initialization gate: recall() awaits this before proceeding
|
|
2689
2925
|
initPromise = null;
|
|
2690
2926
|
resolveInit = null;
|
|
2927
|
+
/**
|
|
2928
|
+
* Resolves when deferred initialization (QMD probe, warmup, caches, cron)
|
|
2929
|
+
* completes. CLI and http-serve callers that need `qmd.isAvailable()` to
|
|
2930
|
+
* reflect reality should `await orchestrator.deferredReady` after
|
|
2931
|
+
* `initialize()`. Gateway callers can ignore it — recall() degrades
|
|
2932
|
+
* gracefully when QMD isn't ready yet.
|
|
2933
|
+
*
|
|
2934
|
+
* Also resolves (without error) when `initialize()` throws before reaching
|
|
2935
|
+
* the deferred-init phase, so callers never hang on a permanently-pending
|
|
2936
|
+
* promise.
|
|
2937
|
+
*
|
|
2938
|
+
* Host adapters that need to tie deferred init to their stop() lifecycle
|
|
2939
|
+
* should `await orchestrator.deferredReady` before proceeding with teardown
|
|
2940
|
+
* to prevent background QMD/warmup/cron tasks from racing with shutdown.
|
|
2941
|
+
*/
|
|
2942
|
+
deferredReady = Promise.resolve();
|
|
2943
|
+
resolveDeferredReady = null;
|
|
2944
|
+
deferredInitAbort = null;
|
|
2945
|
+
/**
|
|
2946
|
+
* Whether the deferred init's QMD startup sync completed successfully.
|
|
2947
|
+
* When false after deferredReady resolves, the server retry loop should
|
|
2948
|
+
* attempt startupSearchSync() even if `qmd.isAvailable()` is true —
|
|
2949
|
+
* availability only means probe succeeded, not that the index is current.
|
|
2950
|
+
*/
|
|
2951
|
+
deferredSyncSucceeded = false;
|
|
2952
|
+
/**
|
|
2953
|
+
* Abort deferred initialization so background QMD sync/warmup stops
|
|
2954
|
+
* promptly on shutdown. Safe to call multiple times or before init.
|
|
2955
|
+
*/
|
|
2956
|
+
abortDeferredInit() {
|
|
2957
|
+
if (this.deferredInitAbort) {
|
|
2958
|
+
this.deferredInitAbort.abort();
|
|
2959
|
+
this.deferredInitAbort = null;
|
|
2960
|
+
}
|
|
2961
|
+
}
|
|
2691
2962
|
/** Set per-session workspace for the next recall() call (compaction reset). @internal */
|
|
2692
2963
|
setRecallWorkspaceOverride(sessionKey, dir) {
|
|
2693
2964
|
this._recallWorkspaceOverrides.set(sessionKey, dir);
|
|
@@ -2796,7 +3067,13 @@ var Orchestrator = class _Orchestrator {
|
|
|
2796
3067
|
config,
|
|
2797
3068
|
this.storageRouter
|
|
2798
3069
|
);
|
|
2799
|
-
this.storage = new StorageManager(config.memoryDir);
|
|
3070
|
+
this.storage = new StorageManager(config.memoryDir, config.entitySchemas);
|
|
3071
|
+
this.storage.citationTemplate = config.inlineSourceAttributionFormat;
|
|
3072
|
+
this.storage.setVersioningConfig({
|
|
3073
|
+
enabled: config.versioningEnabled,
|
|
3074
|
+
maxVersionsPerPage: config.versioningMaxPerPage,
|
|
3075
|
+
sidecarDir: config.versioningSidecarDir
|
|
3076
|
+
});
|
|
2800
3077
|
this.qmd = createSearchBackend(config);
|
|
2801
3078
|
const conversationIndexRuntime = createConversationIndexRuntime(config, {
|
|
2802
3079
|
getQmd: () => this.conversationQmd,
|
|
@@ -2832,6 +3109,7 @@ var Orchestrator = class _Orchestrator {
|
|
|
2832
3109
|
this.modelRegistry,
|
|
2833
3110
|
this.transcript
|
|
2834
3111
|
);
|
|
3112
|
+
this.judgeVerdictCache = createVerdictCache();
|
|
2835
3113
|
this.localLlm = new LocalLlmClient(config, this.modelRegistry);
|
|
2836
3114
|
this.fastLlm = config.localLlmFastEnabled ? (() => {
|
|
2837
3115
|
const client = new LocalLlmClient(
|
|
@@ -3071,100 +3349,173 @@ var Orchestrator = class _Orchestrator {
|
|
|
3071
3349
|
return this.fastLlm;
|
|
3072
3350
|
}
|
|
3073
3351
|
async initialize() {
|
|
3074
|
-
|
|
3075
|
-
|
|
3076
|
-
logger: (message) => log.info(message)
|
|
3352
|
+
this.deferredReady = new Promise((resolve) => {
|
|
3353
|
+
this.resolveDeferredReady = resolve;
|
|
3077
3354
|
});
|
|
3078
|
-
|
|
3079
|
-
|
|
3080
|
-
|
|
3081
|
-
|
|
3082
|
-
|
|
3083
|
-
|
|
3084
|
-
|
|
3085
|
-
|
|
3086
|
-
|
|
3087
|
-
|
|
3088
|
-
|
|
3089
|
-
|
|
3355
|
+
try {
|
|
3356
|
+
await migrateFromEngram({
|
|
3357
|
+
quiet: true,
|
|
3358
|
+
logger: (message) => log.info(message)
|
|
3359
|
+
});
|
|
3360
|
+
await this.storage.ensureDirectories();
|
|
3361
|
+
await this.storage.loadAliases();
|
|
3362
|
+
if (this.config.namespacesEnabled) {
|
|
3363
|
+
const namespaces = /* @__PURE__ */ new Set([
|
|
3364
|
+
this.config.defaultNamespace,
|
|
3365
|
+
this.config.sharedNamespace,
|
|
3366
|
+
...this.config.namespacePolicies.map((p) => p.name)
|
|
3367
|
+
]);
|
|
3368
|
+
for (const ns of namespaces) {
|
|
3369
|
+
const sm = await this.storageRouter.storageFor(ns);
|
|
3370
|
+
await sm.ensureDirectories();
|
|
3371
|
+
await sm.loadAliases().catch(() => void 0);
|
|
3372
|
+
}
|
|
3373
|
+
}
|
|
3374
|
+
await this.relevance.load();
|
|
3375
|
+
await this.negatives.load();
|
|
3376
|
+
await this.lastRecall.load();
|
|
3377
|
+
await this.tierMigrationStatus.load();
|
|
3378
|
+
await this.sessionObserver.load();
|
|
3379
|
+
this.runtimePolicyValues = await this.policyRuntime.loadRuntimeValues();
|
|
3380
|
+
this.utilityRuntimeValues = await loadUtilityRuntimeValues({
|
|
3381
|
+
memoryDir: this.config.memoryDir,
|
|
3382
|
+
memoryUtilityLearningEnabled: this.config.memoryUtilityLearningEnabled,
|
|
3383
|
+
promotionByOutcomeEnabled: this.config.promotionByOutcomeEnabled
|
|
3384
|
+
});
|
|
3385
|
+
if (this.config.factDeduplicationEnabled) {
|
|
3386
|
+
const stateDir = path5.join(this.config.memoryDir, "state");
|
|
3387
|
+
this.contentHashIndex = new ContentHashIndex(stateDir);
|
|
3388
|
+
await this.contentHashIndex.load();
|
|
3389
|
+
log.info(
|
|
3390
|
+
`content-hash dedup: loaded ${this.contentHashIndex.size} hashes`
|
|
3391
|
+
);
|
|
3090
3392
|
}
|
|
3091
|
-
|
|
3092
|
-
|
|
3093
|
-
|
|
3094
|
-
|
|
3095
|
-
|
|
3096
|
-
|
|
3097
|
-
|
|
3098
|
-
|
|
3099
|
-
|
|
3100
|
-
|
|
3101
|
-
|
|
3102
|
-
|
|
3103
|
-
|
|
3104
|
-
const stateDir = path5.join(this.config.memoryDir, "state");
|
|
3105
|
-
this.contentHashIndex = new ContentHashIndex(stateDir);
|
|
3106
|
-
await this.contentHashIndex.load();
|
|
3107
|
-
log.info(
|
|
3108
|
-
`content-hash dedup: loaded ${this.contentHashIndex.size} hashes`
|
|
3109
|
-
);
|
|
3110
|
-
}
|
|
3111
|
-
await this.transcript.initialize();
|
|
3112
|
-
await this.summarizer.initialize();
|
|
3113
|
-
if (this.sharedContext) {
|
|
3114
|
-
await this.sharedContext.ensureStructure();
|
|
3115
|
-
}
|
|
3116
|
-
if (this.compounding) {
|
|
3117
|
-
await this.compounding.ensureDirs();
|
|
3118
|
-
}
|
|
3119
|
-
if (this.resolveInit) {
|
|
3120
|
-
this.resolveInit();
|
|
3121
|
-
this.resolveInit = null;
|
|
3122
|
-
log.info("init gate opened (essential state loaded)");
|
|
3123
|
-
}
|
|
3124
|
-
{
|
|
3125
|
-
const available = await this.qmd.probe();
|
|
3126
|
-
if (available) {
|
|
3127
|
-
log.info(`Search backend: available ${this.qmd.debugStatus()}`);
|
|
3128
|
-
const namespaces = this.config.namespacesEnabled ? this.configuredNamespaces() : [this.config.defaultNamespace];
|
|
3129
|
-
const states = await Promise.all(
|
|
3130
|
-
namespaces.map(async (namespace) => ({
|
|
3131
|
-
namespace,
|
|
3132
|
-
state: this.config.namespacesEnabled ? await this.namespaceSearchRouter.ensureNamespaceCollection(
|
|
3133
|
-
namespace
|
|
3134
|
-
) : await this.qmd.ensureCollection(this.config.memoryDir)
|
|
3135
|
-
}))
|
|
3393
|
+
await this.transcript.initialize();
|
|
3394
|
+
await this.summarizer.initialize();
|
|
3395
|
+
if (this.sharedContext) {
|
|
3396
|
+
await this.sharedContext.ensureStructure();
|
|
3397
|
+
}
|
|
3398
|
+
if (this.compounding) {
|
|
3399
|
+
await this.compounding.ensureDirs();
|
|
3400
|
+
}
|
|
3401
|
+
try {
|
|
3402
|
+
await this.buffer.load();
|
|
3403
|
+
} catch (bufErr) {
|
|
3404
|
+
log.error(
|
|
3405
|
+
`buffer.load() failed (init gate will still open): ${bufErr}`
|
|
3136
3406
|
);
|
|
3137
|
-
|
|
3138
|
-
|
|
3139
|
-
|
|
3140
|
-
|
|
3141
|
-
this.
|
|
3142
|
-
|
|
3143
|
-
|
|
3144
|
-
|
|
3145
|
-
|
|
3146
|
-
|
|
3147
|
-
|
|
3148
|
-
|
|
3149
|
-
|
|
3150
|
-
|
|
3151
|
-
|
|
3152
|
-
|
|
3407
|
+
this.buffer.resetToEmpty();
|
|
3408
|
+
}
|
|
3409
|
+
if (this.config.compactionResetEnabled) {
|
|
3410
|
+
try {
|
|
3411
|
+
const wsDir = this.config.workspaceDir || defaultWorkspaceDir();
|
|
3412
|
+
const files = await readdir3(wsDir).catch(() => []);
|
|
3413
|
+
for (const f of files) {
|
|
3414
|
+
if (!f.startsWith(".compaction-reset-signal-")) continue;
|
|
3415
|
+
const fp = path5.join(wsDir, f);
|
|
3416
|
+
const s = await stat3(fp).catch(() => null);
|
|
3417
|
+
if (s && Date.now() - s.mtimeMs >= COMPACTION_SIGNAL_MAX_AGE_MS) {
|
|
3418
|
+
await unlink2(fp).catch(() => {
|
|
3419
|
+
});
|
|
3420
|
+
log.debug(`initialize: removed stale compaction signal ${f}`);
|
|
3421
|
+
}
|
|
3422
|
+
}
|
|
3423
|
+
} catch (err) {
|
|
3424
|
+
log.debug("initialize: stale signal sweep failed:", err);
|
|
3153
3425
|
}
|
|
3154
|
-
|
|
3155
|
-
|
|
3156
|
-
|
|
3426
|
+
}
|
|
3427
|
+
try {
|
|
3428
|
+
const available = await this.qmd.probe();
|
|
3429
|
+
if (available) {
|
|
3430
|
+
log.info(`Search backend: available ${this.qmd.debugStatus()}`);
|
|
3431
|
+
const namespaces = this.config.namespacesEnabled ? this.configuredNamespaces() : [this.config.defaultNamespace];
|
|
3432
|
+
const states = await Promise.all(
|
|
3433
|
+
namespaces.map(async (namespace) => ({
|
|
3434
|
+
namespace,
|
|
3435
|
+
state: this.config.namespacesEnabled ? await this.namespaceSearchRouter.ensureNamespaceCollection(
|
|
3436
|
+
namespace
|
|
3437
|
+
) : await this.qmd.ensureCollection(this.config.memoryDir)
|
|
3438
|
+
}))
|
|
3439
|
+
);
|
|
3440
|
+
const defaultState = states.find(
|
|
3441
|
+
(entry) => entry.namespace === this.config.defaultNamespace
|
|
3442
|
+
)?.state ?? "unknown";
|
|
3443
|
+
if (defaultState === "missing") {
|
|
3444
|
+
this.qmd = new NoopSearchBackend();
|
|
3445
|
+
log.warn(
|
|
3446
|
+
"Search collection missing for Remnic memory store; disabling search retrieval for this runtime (fallback retrieval remains enabled)"
|
|
3447
|
+
);
|
|
3448
|
+
} else if (defaultState === "unknown") {
|
|
3157
3449
|
log.warn(
|
|
3158
|
-
|
|
3450
|
+
"Search collection check unavailable; keeping search retrieval enabled for fail-open behavior"
|
|
3159
3451
|
);
|
|
3452
|
+
} else if (defaultState === "skipped") {
|
|
3453
|
+
log.debug(
|
|
3454
|
+
"Search collection check skipped (remote or daemon-only mode)"
|
|
3455
|
+
);
|
|
3456
|
+
}
|
|
3457
|
+
for (const entry of states) {
|
|
3458
|
+
if (entry.namespace === this.config.defaultNamespace) continue;
|
|
3459
|
+
if (entry.state === "missing") {
|
|
3460
|
+
log.warn(
|
|
3461
|
+
`Search collection missing for namespace '${entry.namespace}'; namespace retrieval will fail open to non-search paths`
|
|
3462
|
+
);
|
|
3463
|
+
}
|
|
3160
3464
|
}
|
|
3465
|
+
} else if (this.qmd instanceof NoopSearchBackend) {
|
|
3466
|
+
log.debug(`Search backend: noop (search intentionally disabled)`);
|
|
3467
|
+
} else {
|
|
3468
|
+
log.warn(`Search backend: not available ${this.qmd.debugStatus()}`);
|
|
3161
3469
|
}
|
|
3162
|
-
}
|
|
3163
|
-
log.
|
|
3164
|
-
}
|
|
3165
|
-
|
|
3470
|
+
} catch (err) {
|
|
3471
|
+
log.error(`QMD probe/collection check failed (non-fatal): ${err}`);
|
|
3472
|
+
}
|
|
3473
|
+
if (this.resolveInit) {
|
|
3474
|
+
this.resolveInit();
|
|
3475
|
+
this.resolveInit = null;
|
|
3476
|
+
log.info("init gate opened (essential state + QMD state loaded)");
|
|
3477
|
+
}
|
|
3478
|
+
const resolveDeferred = this.resolveDeferredReady;
|
|
3479
|
+
this.resolveDeferredReady = null;
|
|
3480
|
+
this.deferredInitAbort = new AbortController();
|
|
3481
|
+
this.deferredInitialize(this.deferredInitAbort.signal).catch((err) => {
|
|
3482
|
+
log.error(`deferred initialization failed (non-fatal): ${err}`);
|
|
3483
|
+
}).finally(() => {
|
|
3484
|
+
resolveDeferred?.();
|
|
3485
|
+
});
|
|
3486
|
+
} catch (err) {
|
|
3487
|
+
if (this.resolveInit) {
|
|
3488
|
+
this.resolveInit();
|
|
3489
|
+
this.resolveInit = null;
|
|
3490
|
+
}
|
|
3491
|
+
if (this.resolveDeferredReady) {
|
|
3492
|
+
this.resolveDeferredReady();
|
|
3493
|
+
this.resolveDeferredReady = null;
|
|
3494
|
+
}
|
|
3495
|
+
throw err;
|
|
3496
|
+
}
|
|
3497
|
+
}
|
|
3498
|
+
async deferredInitialize(signal) {
|
|
3499
|
+
if (this.qmd.isAvailable() && this.config.qmdMaintenanceEnabled) {
|
|
3500
|
+
try {
|
|
3501
|
+
log.info("QMD startup sync: updating index to match current disk state");
|
|
3502
|
+
if (this.config.namespacesEnabled) {
|
|
3503
|
+
await this.namespaceSearchRouter.updateNamespaces(
|
|
3504
|
+
this.configuredNamespaces()
|
|
3505
|
+
);
|
|
3506
|
+
} else {
|
|
3507
|
+
await this.qmd.update();
|
|
3508
|
+
}
|
|
3509
|
+
log.info("QMD startup sync: complete");
|
|
3510
|
+
this.deferredSyncSucceeded = true;
|
|
3511
|
+
} catch (err) {
|
|
3512
|
+
log.warn(`QMD startup sync failed (non-fatal): ${err}`);
|
|
3166
3513
|
}
|
|
3514
|
+
} else if (!this.qmd.isAvailable()) {
|
|
3515
|
+
} else {
|
|
3516
|
+
this.deferredSyncSucceeded = true;
|
|
3167
3517
|
}
|
|
3518
|
+
if (signal.aborted) return;
|
|
3168
3519
|
const warmupPromises = [];
|
|
3169
3520
|
if (this.qmd.isAvailable()) {
|
|
3170
3521
|
const warmupNs = this.config.defaultNamespace;
|
|
@@ -3189,68 +3540,173 @@ var Orchestrator = class _Orchestrator {
|
|
|
3189
3540
|
);
|
|
3190
3541
|
}
|
|
3191
3542
|
await Promise.all(warmupPromises);
|
|
3543
|
+
if (signal.aborted) return;
|
|
3544
|
+
const cacheWarmups = [];
|
|
3192
3545
|
if (this.config.knowledgeIndexEnabled) {
|
|
3193
|
-
(
|
|
3194
|
-
|
|
3195
|
-
|
|
3196
|
-
|
|
3197
|
-
|
|
3198
|
-
|
|
3199
|
-
|
|
3200
|
-
|
|
3201
|
-
|
|
3202
|
-
|
|
3546
|
+
cacheWarmups.push(
|
|
3547
|
+
(async () => {
|
|
3548
|
+
try {
|
|
3549
|
+
const t0 = Date.now();
|
|
3550
|
+
await this.storage.buildKnowledgeIndex(this.config);
|
|
3551
|
+
log.info(`Knowledge Index warmup: complete in ${Date.now() - t0}ms`);
|
|
3552
|
+
} catch (err) {
|
|
3553
|
+
log.debug(`Knowledge Index warmup failed (non-fatal): ${err}`);
|
|
3554
|
+
}
|
|
3555
|
+
})()
|
|
3556
|
+
);
|
|
3203
3557
|
}
|
|
3204
|
-
this.storage.readAllMemories().
|
|
3205
|
-
})
|
|
3206
|
-
|
|
3207
|
-
|
|
3558
|
+
cacheWarmups.push(this.storage.readAllMemories().then(() => {
|
|
3559
|
+
}).catch(() => {
|
|
3560
|
+
}));
|
|
3561
|
+
cacheWarmups.push(this.storage.readAllEntityFiles().then(() => {
|
|
3562
|
+
}).catch(() => {
|
|
3563
|
+
}));
|
|
3564
|
+
await Promise.all(cacheWarmups);
|
|
3565
|
+
if (signal.aborted) return;
|
|
3208
3566
|
if (this.config.conversationIndexEnabled && this.conversationIndexBackend) {
|
|
3209
|
-
|
|
3210
|
-
|
|
3567
|
+
try {
|
|
3568
|
+
const init = await this.conversationIndexBackend.initialize();
|
|
3569
|
+
if (!init.enabled) {
|
|
3570
|
+
this.config.conversationIndexEnabled = false;
|
|
3571
|
+
}
|
|
3572
|
+
if (init.logLevel === "info") {
|
|
3573
|
+
log.info(init.message);
|
|
3574
|
+
} else if (init.logLevel === "warn") {
|
|
3575
|
+
log.warn(init.message);
|
|
3576
|
+
} else {
|
|
3577
|
+
log.debug(init.message);
|
|
3578
|
+
}
|
|
3579
|
+
} catch (err) {
|
|
3580
|
+
log.error(`Conversation index initialization failed (non-fatal): ${err}`);
|
|
3211
3581
|
this.config.conversationIndexEnabled = false;
|
|
3212
3582
|
}
|
|
3213
|
-
if (init.logLevel === "info") {
|
|
3214
|
-
log.info(init.message);
|
|
3215
|
-
} else if (init.logLevel === "warn") {
|
|
3216
|
-
log.warn(init.message);
|
|
3217
|
-
} else {
|
|
3218
|
-
log.debug(init.message);
|
|
3219
|
-
}
|
|
3220
3583
|
}
|
|
3221
|
-
|
|
3584
|
+
if (signal.aborted) return;
|
|
3222
3585
|
if (this.config.localLlmEnabled) {
|
|
3223
|
-
await this.validateLocalLlmModel();
|
|
3224
|
-
}
|
|
3225
|
-
if (this.config.compactionResetEnabled) {
|
|
3226
3586
|
try {
|
|
3227
|
-
|
|
3228
|
-
const files = await readdir3(wsDir).catch(() => []);
|
|
3229
|
-
for (const f of files) {
|
|
3230
|
-
if (!f.startsWith(".compaction-reset-signal-")) continue;
|
|
3231
|
-
const fp = path5.join(wsDir, f);
|
|
3232
|
-
const s = await stat3(fp).catch(() => null);
|
|
3233
|
-
if (s && Date.now() - s.mtimeMs >= COMPACTION_SIGNAL_MAX_AGE_MS) {
|
|
3234
|
-
await unlink2(fp).catch(() => {
|
|
3235
|
-
});
|
|
3236
|
-
log.debug(`initialize: removed stale compaction signal ${f}`);
|
|
3237
|
-
}
|
|
3238
|
-
}
|
|
3587
|
+
await this.validateLocalLlmModel();
|
|
3239
3588
|
} catch (err) {
|
|
3240
|
-
log.
|
|
3589
|
+
log.error(`Local LLM validation failed (non-fatal): ${err}`);
|
|
3241
3590
|
}
|
|
3242
3591
|
}
|
|
3243
|
-
|
|
3592
|
+
if (signal.aborted) return;
|
|
3244
3593
|
if (this.config.daySummaryEnabled) {
|
|
3245
|
-
|
|
3594
|
+
try {
|
|
3595
|
+
await this.autoRegisterDaySummaryCron();
|
|
3596
|
+
} catch (err) {
|
|
3246
3597
|
log.debug(`day-summary cron auto-register failed (non-fatal): ${err}`);
|
|
3247
|
-
}
|
|
3598
|
+
}
|
|
3248
3599
|
}
|
|
3249
3600
|
if (this.config.nightlyGovernanceCronAutoRegister) {
|
|
3250
|
-
|
|
3601
|
+
try {
|
|
3602
|
+
await this.autoRegisterNightlyGovernanceCron();
|
|
3603
|
+
} catch (err) {
|
|
3251
3604
|
log.debug(`nightly governance cron auto-register failed (non-fatal): ${err}`);
|
|
3252
|
-
}
|
|
3605
|
+
}
|
|
3606
|
+
}
|
|
3607
|
+
if (this.config.procedural?.proceduralMiningCronAutoRegister) {
|
|
3608
|
+
try {
|
|
3609
|
+
await this.autoRegisterProceduralMiningCron();
|
|
3610
|
+
} catch (err) {
|
|
3611
|
+
log.debug(`procedural mining cron auto-register failed (non-fatal): ${err}`);
|
|
3612
|
+
}
|
|
3253
3613
|
}
|
|
3614
|
+
log.info("orchestrator initialized (full \u2014 deferred steps complete)");
|
|
3615
|
+
}
|
|
3616
|
+
/**
|
|
3617
|
+
* Namespace-aware startup search sync. Re-probes QMD, ensures collections
|
|
3618
|
+
* (namespace-aware when namespacesEnabled), runs update, and warms up search.
|
|
3619
|
+
* Designed for server retry paths that run after the deferred init completes
|
|
3620
|
+
* when QMD was not available during initial startup.
|
|
3621
|
+
*
|
|
3622
|
+
* Accepts an optional AbortSignal so callers can interrupt the sync during
|
|
3623
|
+
* shutdown. The signal is checked between phases and forwarded into the QMD
|
|
3624
|
+
* update and warmup search calls so a long-running `qmd update` subprocess
|
|
3625
|
+
* is killed promptly rather than left in flight after `httpServer.stop()`.
|
|
3626
|
+
*
|
|
3627
|
+
* Returns true if the sync succeeded (QMD now available), false otherwise.
|
|
3628
|
+
*/
|
|
3629
|
+
async startupSearchSync(signal) {
|
|
3630
|
+
if (signal?.aborted) return false;
|
|
3631
|
+
const available = await this.qmd.probe();
|
|
3632
|
+
if (!available) return false;
|
|
3633
|
+
if (signal?.aborted) {
|
|
3634
|
+
log.debug("startupSearchSync: aborted after probe");
|
|
3635
|
+
return false;
|
|
3636
|
+
}
|
|
3637
|
+
log.info(`startupSearchSync: backend now available ${this.qmd.debugStatus()}`);
|
|
3638
|
+
if (this.config.namespacesEnabled) {
|
|
3639
|
+
this.namespaceSearchRouter.clearCache();
|
|
3640
|
+
}
|
|
3641
|
+
const namespaces = this.config.namespacesEnabled ? this.configuredNamespaces() : [this.config.defaultNamespace];
|
|
3642
|
+
const states = await Promise.all(
|
|
3643
|
+
namespaces.map(async (namespace) => ({
|
|
3644
|
+
namespace,
|
|
3645
|
+
state: this.config.namespacesEnabled ? await this.namespaceSearchRouter.ensureNamespaceCollection(namespace) : await this.qmd.ensureCollection(this.config.memoryDir)
|
|
3646
|
+
}))
|
|
3647
|
+
);
|
|
3648
|
+
if (signal?.aborted) {
|
|
3649
|
+
log.debug("startupSearchSync: aborted after ensureCollection");
|
|
3650
|
+
return false;
|
|
3651
|
+
}
|
|
3652
|
+
const defaultState = states.find((e) => e.namespace === this.config.defaultNamespace)?.state ?? "unknown";
|
|
3653
|
+
if (defaultState === "missing") {
|
|
3654
|
+
if ("available" in this.qmd) {
|
|
3655
|
+
this.qmd.available = false;
|
|
3656
|
+
}
|
|
3657
|
+
this.qmd = new NoopSearchBackend();
|
|
3658
|
+
log.warn("startupSearchSync: search collection missing; disabling search (fallback retrieval remains enabled)");
|
|
3659
|
+
return false;
|
|
3660
|
+
}
|
|
3661
|
+
if (this.config.qmdMaintenanceEnabled) {
|
|
3662
|
+
try {
|
|
3663
|
+
const failTsBefore = "lastUpdateFailedAtMs" in this.qmd ? this.qmd.lastUpdateFailedAtMs : null;
|
|
3664
|
+
const hasRunTs = "lastUpdateRanAtMs" in this.qmd;
|
|
3665
|
+
if ("resetUpdateThrottles" in this.qmd) {
|
|
3666
|
+
this.qmd.resetUpdateThrottles();
|
|
3667
|
+
}
|
|
3668
|
+
log.info("startupSearchSync: updating index to match current disk state");
|
|
3669
|
+
let namespacesUpdated = 0;
|
|
3670
|
+
if (this.config.namespacesEnabled) {
|
|
3671
|
+
namespacesUpdated = await this.namespaceSearchRouter.updateNamespaces(namespaces);
|
|
3672
|
+
} else {
|
|
3673
|
+
await this.qmd.update(signal);
|
|
3674
|
+
}
|
|
3675
|
+
if (signal?.aborted) {
|
|
3676
|
+
log.debug("startupSearchSync: aborted after update");
|
|
3677
|
+
return false;
|
|
3678
|
+
}
|
|
3679
|
+
const failTsAfter = "lastUpdateFailedAtMs" in this.qmd ? this.qmd.lastUpdateFailedAtMs : null;
|
|
3680
|
+
const runTsAfter = hasRunTs ? this.qmd.lastUpdateRanAtMs : null;
|
|
3681
|
+
if (failTsAfter !== null && failTsAfter !== failTsBefore) {
|
|
3682
|
+
log.warn("startupSearchSync: update silently failed (detected via fail timestamp)");
|
|
3683
|
+
return false;
|
|
3684
|
+
}
|
|
3685
|
+
if (this.config.namespacesEnabled) {
|
|
3686
|
+
if (namespacesUpdated === 0) {
|
|
3687
|
+
log.warn("startupSearchSync: no namespace backends were eligible for update (all unavailable or collections missing)");
|
|
3688
|
+
return false;
|
|
3689
|
+
}
|
|
3690
|
+
log.info(`startupSearchSync: namespace updates succeeded (${namespacesUpdated}/${namespaces.length} namespaces updated)`);
|
|
3691
|
+
} else if (hasRunTs && runTsAfter === null) {
|
|
3692
|
+
log.warn("startupSearchSync: update was throttled/skipped (run timestamp is null after reset + update)");
|
|
3693
|
+
return false;
|
|
3694
|
+
}
|
|
3695
|
+
log.info("startupSearchSync: sync complete");
|
|
3696
|
+
} catch (err) {
|
|
3697
|
+
log.warn(`startupSearchSync: update failed: ${err}`);
|
|
3698
|
+
return false;
|
|
3699
|
+
}
|
|
3700
|
+
}
|
|
3701
|
+
if (!signal?.aborted) {
|
|
3702
|
+
try {
|
|
3703
|
+
await this.qmd.search("warmup", this.config.defaultNamespace, 1, void 0, { signal });
|
|
3704
|
+
log.info("startupSearchSync: warmup complete");
|
|
3705
|
+
} catch (err) {
|
|
3706
|
+
log.debug(`startupSearchSync: warmup search failed (non-fatal): ${err}`);
|
|
3707
|
+
}
|
|
3708
|
+
}
|
|
3709
|
+
return true;
|
|
3254
3710
|
}
|
|
3255
3711
|
/**
|
|
3256
3712
|
* Auto-register the engram-day-summary cron job in OpenClaw if it doesn't exist.
|
|
@@ -3302,6 +3758,26 @@ var Orchestrator = class _Orchestrator {
|
|
|
3302
3758
|
log.debug(`nightly governance cron auto-register error: ${err}`);
|
|
3303
3759
|
}
|
|
3304
3760
|
}
|
|
3761
|
+
async autoRegisterProceduralMiningCron() {
|
|
3762
|
+
const home = resolveHomeDir();
|
|
3763
|
+
const jobsPath = path5.join(home, ".openclaw", "cron", "jobs.json");
|
|
3764
|
+
try {
|
|
3765
|
+
if (!existsSync2(jobsPath)) {
|
|
3766
|
+
log.debug("procedural mining cron: jobs.json not found, skipping auto-register");
|
|
3767
|
+
return;
|
|
3768
|
+
}
|
|
3769
|
+
const created = await ensureProceduralMiningCron(jobsPath, {
|
|
3770
|
+
timezone: Intl.DateTimeFormat().resolvedOptions().timeZone
|
|
3771
|
+
});
|
|
3772
|
+
if (created.created) {
|
|
3773
|
+
log.info(`procedural mining cron auto-registered (${created.jobId})`);
|
|
3774
|
+
} else {
|
|
3775
|
+
log.debug("procedural mining cron already exists, skipping auto-register");
|
|
3776
|
+
}
|
|
3777
|
+
} catch (err) {
|
|
3778
|
+
log.debug(`procedural mining cron auto-register error: ${err}`);
|
|
3779
|
+
}
|
|
3780
|
+
}
|
|
3305
3781
|
async applyBehaviorRuntimePolicy(state) {
|
|
3306
3782
|
const result = await this.policyRuntime.applyFromBehaviorState(state);
|
|
3307
3783
|
this.runtimePolicyValues = await this.policyRuntime.loadRuntimeValues();
|
|
@@ -3378,6 +3854,16 @@ var Orchestrator = class _Orchestrator {
|
|
|
3378
3854
|
async runConsolidationNow() {
|
|
3379
3855
|
return this.runConsolidation();
|
|
3380
3856
|
}
|
|
3857
|
+
async reindexMemoryById(id, options) {
|
|
3858
|
+
await this.indexPersistedMemory(options?.storage ?? this.storage, id);
|
|
3859
|
+
this.requestQmdMaintenance();
|
|
3860
|
+
}
|
|
3861
|
+
registerConsolidationObserver(observer) {
|
|
3862
|
+
this.consolidationObservers.add(observer);
|
|
3863
|
+
return () => {
|
|
3864
|
+
this.consolidationObservers.delete(observer);
|
|
3865
|
+
};
|
|
3866
|
+
}
|
|
3381
3867
|
async runSemanticConsolidationNow(options) {
|
|
3382
3868
|
return this.runSemanticConsolidation({ ...options, force: true });
|
|
3383
3869
|
}
|
|
@@ -3433,9 +3919,18 @@ var Orchestrator = class _Orchestrator {
|
|
|
3433
3919
|
);
|
|
3434
3920
|
return result;
|
|
3435
3921
|
}
|
|
3922
|
+
let extensionsBlock = "";
|
|
3923
|
+
try {
|
|
3924
|
+
extensionsBlock = await buildExtensionsBlockForConsolidation(this.config);
|
|
3925
|
+
} catch (err) {
|
|
3926
|
+
log.warn(`[semantic-consolidation] extension discovery failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`);
|
|
3927
|
+
}
|
|
3436
3928
|
for (const cluster of clusters) {
|
|
3437
3929
|
try {
|
|
3438
|
-
|
|
3930
|
+
let prompt = buildConsolidationPrompt(cluster);
|
|
3931
|
+
if (extensionsBlock.length > 0) {
|
|
3932
|
+
prompt += "\n\n" + extensionsBlock;
|
|
3933
|
+
}
|
|
3439
3934
|
const messages = [
|
|
3440
3935
|
{
|
|
3441
3936
|
role: "system",
|
|
@@ -3496,7 +3991,14 @@ var Orchestrator = class _Orchestrator {
|
|
|
3496
3991
|
});
|
|
3497
3992
|
if (archiveResult) {
|
|
3498
3993
|
if (this.contentHashIndex) {
|
|
3499
|
-
|
|
3994
|
+
if (m.frontmatter.contentHash) {
|
|
3995
|
+
this.contentHashIndex.removeByHash(m.frontmatter.contentHash);
|
|
3996
|
+
} else {
|
|
3997
|
+
log.warn(
|
|
3998
|
+
`[semantic-consolidation] removing hash for legacy memory ${m.frontmatter.id ?? "(unknown)"} via content fallback \u2014 no contentHash in frontmatter`
|
|
3999
|
+
);
|
|
4000
|
+
this.contentHashIndex.remove(m.content);
|
|
4001
|
+
}
|
|
3500
4002
|
}
|
|
3501
4003
|
await this.embeddingFallback.removeFromIndex(m.frontmatter.id);
|
|
3502
4004
|
if (this.config.queryAwareIndexingEnabled && m.path && m.frontmatter?.created) {
|
|
@@ -3530,6 +4032,16 @@ var Orchestrator = class _Orchestrator {
|
|
|
3530
4032
|
log.info(
|
|
3531
4033
|
`[semantic-consolidation] complete: clusters=${result.clustersFound}, consolidated=${result.memoriesConsolidated}, archived=${result.memoriesArchived}, errors=${result.errors}`
|
|
3532
4034
|
);
|
|
4035
|
+
try {
|
|
4036
|
+
await materializeAfterSemanticConsolidation({
|
|
4037
|
+
config: this.config,
|
|
4038
|
+
memoryDir: this.config.memoryDir
|
|
4039
|
+
});
|
|
4040
|
+
} catch (err) {
|
|
4041
|
+
log.warn(
|
|
4042
|
+
`[semantic-consolidation] Codex materialize post-hook failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`
|
|
4043
|
+
);
|
|
4044
|
+
}
|
|
3533
4045
|
return result;
|
|
3534
4046
|
}
|
|
3535
4047
|
async waitForExtractionIdle(timeoutMs = 6e4) {
|
|
@@ -3558,6 +4070,116 @@ var Orchestrator = class _Orchestrator {
|
|
|
3558
4070
|
const ns = namespace && namespace.length > 0 ? namespace : this.config.defaultNamespace;
|
|
3559
4071
|
return this.storageRouter.storageFor(ns);
|
|
3560
4072
|
}
|
|
4073
|
+
async processEntitySynthesisQueue(namespace, maxEntities = 5) {
|
|
4074
|
+
if (!this.config.entitySummaryEnabled || maxEntities <= 0 || this.config.entitySynthesisMaxTokens <= 0) return 0;
|
|
4075
|
+
const storage = await this.getStorage(namespace);
|
|
4076
|
+
const queued = await storage.refreshEntitySynthesisQueue();
|
|
4077
|
+
let processed = 0;
|
|
4078
|
+
let attempted = 0;
|
|
4079
|
+
for (const entityName of queued) {
|
|
4080
|
+
if (attempted >= maxEntities) break;
|
|
4081
|
+
attempted += 1;
|
|
4082
|
+
try {
|
|
4083
|
+
const raw = await storage.readEntity(entityName);
|
|
4084
|
+
if (!raw) continue;
|
|
4085
|
+
const entity = parseEntityFile(raw, this.config.entitySchemas);
|
|
4086
|
+
const previousSynthesis = entity.synthesis || entity.summary || "";
|
|
4087
|
+
const sortedTimelineEntries = entity.timeline.slice().sort((left, right) => compareEntityTimestamps(right.timestamp, left.timestamp));
|
|
4088
|
+
const newerTimelineEntries = sortedTimelineEntries.filter(
|
|
4089
|
+
(entry) => !entity.synthesisUpdatedAt || compareEntityTimestamps(entry.timestamp, entity.synthesisUpdatedAt) > 0
|
|
4090
|
+
);
|
|
4091
|
+
const appendedTimelineEntries = entity.synthesisTimelineCount === void 0 ? [] : entity.timeline.slice(Math.max(0, entity.synthesisTimelineCount));
|
|
4092
|
+
const structuredEvidenceEntries = flattenStructuredSectionEvidence(entity.structuredSections);
|
|
4093
|
+
const structuredEvidenceCount = structuredEvidenceEntries.length;
|
|
4094
|
+
const structuredEvidenceDigest = fingerprintEntityStructuredFacts(entity);
|
|
4095
|
+
const structuredEvidenceDrifted = structuredEvidenceDigest !== (entity.synthesisStructuredFactDigest?.trim() || void 0);
|
|
4096
|
+
const appendedStructuredEvidenceEntries = entity.synthesisStructuredFactCount === void 0 || structuredEvidenceDrifted ? structuredEvidenceEntries : structuredEvidenceEntries.slice(Math.max(0, entity.synthesisStructuredFactCount));
|
|
4097
|
+
const candidateEvidenceEntries = [
|
|
4098
|
+
...newerTimelineEntries,
|
|
4099
|
+
...appendedTimelineEntries,
|
|
4100
|
+
...appendedStructuredEvidenceEntries
|
|
4101
|
+
].slice().sort((left, right) => compareEntityTimestamps(right.timestamp, left.timestamp));
|
|
4102
|
+
const dedupedEvidenceEntries = dedupeEntitySynthesisEvidenceEntries(
|
|
4103
|
+
candidateEvidenceEntries.length > 0 ? candidateEvidenceEntries : [...sortedTimelineEntries, ...structuredEvidenceEntries]
|
|
4104
|
+
);
|
|
4105
|
+
const chronologicalEvidenceEntries = dedupedEvidenceEntries.slice().sort((left, right) => compareEntityTimestamps(left.timestamp, right.timestamp));
|
|
4106
|
+
if (chronologicalEvidenceEntries.length === 0) continue;
|
|
4107
|
+
const latestEvidenceTimestamp = chronologicalEvidenceEntries.slice().reverse().map((entry) => entry.timestamp?.trim() || void 0).find((timestamp) => Boolean(timestamp));
|
|
4108
|
+
const previousSynthesisUpdatedAt = entity.synthesisUpdatedAt?.trim() || void 0;
|
|
4109
|
+
const nextSynthesisUpdatedAt = compareEntityTimestamps(
|
|
4110
|
+
latestEvidenceTimestamp,
|
|
4111
|
+
previousSynthesisUpdatedAt
|
|
4112
|
+
) >= 0 ? latestEvidenceTimestamp : previousSynthesisUpdatedAt;
|
|
4113
|
+
const evidenceBatches = [];
|
|
4114
|
+
for (let index = 0; index < chronologicalEvidenceEntries.length; index += 8) {
|
|
4115
|
+
evidenceBatches.push(chronologicalEvidenceEntries.slice(index, index + 8));
|
|
4116
|
+
}
|
|
4117
|
+
let nextSynthesis = previousSynthesis;
|
|
4118
|
+
let batchFailed = false;
|
|
4119
|
+
for (const evidenceEntries of evidenceBatches) {
|
|
4120
|
+
const evidenceText = evidenceEntries.map((entry) => {
|
|
4121
|
+
const sectionTitle = entry.source?.startsWith("section:") ? entry.source.slice("section:".length) : "";
|
|
4122
|
+
const metadata = [
|
|
4123
|
+
`timestamp=${entry.timestamp}`,
|
|
4124
|
+
sectionTitle ? `section=${sectionTitle}` : entry.source ? `source=${entry.source}` : "",
|
|
4125
|
+
entry.sessionKey ? `session=${entry.sessionKey}` : "",
|
|
4126
|
+
entry.principal ? `principal=${entry.principal}` : ""
|
|
4127
|
+
].filter(Boolean).join(", ");
|
|
4128
|
+
return `- ${metadata}: ${entry.text}`;
|
|
4129
|
+
}).join("\n");
|
|
4130
|
+
const response = await this.fastChatCompletion(
|
|
4131
|
+
[
|
|
4132
|
+
{
|
|
4133
|
+
role: "system",
|
|
4134
|
+
content: "Rewrite the entity synthesis as compact current truth. Preserve uncertainty when evidence conflicts. Return plain text only."
|
|
4135
|
+
},
|
|
4136
|
+
{
|
|
4137
|
+
role: "user",
|
|
4138
|
+
content: [
|
|
4139
|
+
`Entity: ${entity.name} (${entity.type})`,
|
|
4140
|
+
nextSynthesis ? `Previous synthesis:
|
|
4141
|
+
${nextSynthesis}` : "Previous synthesis: none",
|
|
4142
|
+
`New evidence:
|
|
4143
|
+
${evidenceText}`
|
|
4144
|
+
].join("\n\n")
|
|
4145
|
+
}
|
|
4146
|
+
],
|
|
4147
|
+
{
|
|
4148
|
+
temperature: 0.2,
|
|
4149
|
+
maxTokens: this.config.entitySynthesisMaxTokens,
|
|
4150
|
+
operation: "entity_summary",
|
|
4151
|
+
priority: "background"
|
|
4152
|
+
}
|
|
4153
|
+
);
|
|
4154
|
+
const synthesis = response?.content?.trim().replace(/^["']|["']$/g, "");
|
|
4155
|
+
const maxSynthesisChars = Math.max(2e3, this.config.entitySynthesisMaxTokens * 8);
|
|
4156
|
+
if (!synthesis || synthesis.length < 10 || synthesis.length > maxSynthesisChars) {
|
|
4157
|
+
batchFailed = true;
|
|
4158
|
+
break;
|
|
4159
|
+
}
|
|
4160
|
+
nextSynthesis = synthesis;
|
|
4161
|
+
}
|
|
4162
|
+
if (batchFailed || nextSynthesis.length === 0) continue;
|
|
4163
|
+
const latestRaw = await storage.readEntity(entityName);
|
|
4164
|
+
if (!latestRaw) continue;
|
|
4165
|
+
const latestEntity = parseEntityFile(latestRaw, this.config.entitySchemas);
|
|
4166
|
+
if (fingerprintEntitySynthesisEvidence(latestEntity) !== fingerprintEntitySynthesisEvidence(entity)) {
|
|
4167
|
+
continue;
|
|
4168
|
+
}
|
|
4169
|
+
await storage.updateEntitySynthesis(entityName, nextSynthesis, {
|
|
4170
|
+
entityUpdatedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
4171
|
+
synthesisStructuredFactDigest: structuredEvidenceDigest,
|
|
4172
|
+
synthesisStructuredFactCount: structuredEvidenceCount,
|
|
4173
|
+
synthesisTimelineCount: entity.timeline.length,
|
|
4174
|
+
updatedAt: nextSynthesisUpdatedAt
|
|
4175
|
+
});
|
|
4176
|
+
processed += 1;
|
|
4177
|
+
} catch (err) {
|
|
4178
|
+
log.debug(`entity synthesis refresh failed for ${entityName}: ${err}`);
|
|
4179
|
+
}
|
|
4180
|
+
}
|
|
4181
|
+
return processed;
|
|
4182
|
+
}
|
|
3561
4183
|
async generateDaySummary(memories) {
|
|
3562
4184
|
if (this.initPromise) {
|
|
3563
4185
|
let initGateTimeoutHandle;
|
|
@@ -6555,6 +7177,22 @@ ${formatted}`;
|
|
|
6555
7177
|
});
|
|
6556
7178
|
return section;
|
|
6557
7179
|
})();
|
|
7180
|
+
const procedureRecallPromise = (async () => {
|
|
7181
|
+
if (this.config.procedural?.enabled !== true) return null;
|
|
7182
|
+
if (!this.isRecallSectionEnabled("procedure-recall", true)) return null;
|
|
7183
|
+
try {
|
|
7184
|
+
return await buildProcedureRecallSection(
|
|
7185
|
+
this.storage,
|
|
7186
|
+
retrievalQuery,
|
|
7187
|
+
this.config
|
|
7188
|
+
);
|
|
7189
|
+
} catch (err) {
|
|
7190
|
+
log.debug(
|
|
7191
|
+
`procedure-recall: failed open: ${err instanceof Error ? err.message : String(err)}`
|
|
7192
|
+
);
|
|
7193
|
+
return null;
|
|
7194
|
+
}
|
|
7195
|
+
})();
|
|
6558
7196
|
const compoundingPromise = observeEnrichmentPromise(
|
|
6559
7197
|
(async () => {
|
|
6560
7198
|
const t0 = Date.now();
|
|
@@ -6619,6 +7257,7 @@ ${formatted}`;
|
|
|
6619
7257
|
causalTrajectorySection,
|
|
6620
7258
|
cmcCausalChainsSection,
|
|
6621
7259
|
calibrationSection,
|
|
7260
|
+
procedureRecallSection,
|
|
6622
7261
|
trustZoneSection,
|
|
6623
7262
|
verifiedRecallSection,
|
|
6624
7263
|
verifiedRulesSection,
|
|
@@ -6640,6 +7279,7 @@ ${formatted}`;
|
|
|
6640
7279
|
["causalTraj", causalTrajectoryPromise],
|
|
6641
7280
|
["cmc", cmcRetrievalPromise],
|
|
6642
7281
|
["calibration", calibrationPromise],
|
|
7282
|
+
["procedureRecall", procedureRecallPromise],
|
|
6643
7283
|
["trustZone", trustZonePromise],
|
|
6644
7284
|
["verifiedRecall", verifiedRecallPromise],
|
|
6645
7285
|
["verifiedRules", verifiedRulesPromise],
|
|
@@ -6745,6 +7385,13 @@ ${profile}`
|
|
|
6745
7385
|
calibrationSection
|
|
6746
7386
|
);
|
|
6747
7387
|
}
|
|
7388
|
+
if (procedureRecallSection) {
|
|
7389
|
+
this.appendRecallSection(
|
|
7390
|
+
sectionBuckets,
|
|
7391
|
+
"procedure-recall",
|
|
7392
|
+
procedureRecallSection
|
|
7393
|
+
);
|
|
7394
|
+
}
|
|
6748
7395
|
if (identityContinuity) {
|
|
6749
7396
|
this.appendRecallSection(
|
|
6750
7397
|
sectionBuckets,
|
|
@@ -7059,7 +7706,11 @@ ${tmtNode.summary}`
|
|
|
7059
7706
|
confidenceGateRejected = true;
|
|
7060
7707
|
}
|
|
7061
7708
|
}
|
|
7062
|
-
memoryResults =
|
|
7709
|
+
memoryResults = this.diversifyAndLimitRecallResults(
|
|
7710
|
+
"memories",
|
|
7711
|
+
memoryResults,
|
|
7712
|
+
recallResultLimit
|
|
7713
|
+
);
|
|
7063
7714
|
if (this.config.memoryReconstructionEnabled && memoryResults.length > 0) {
|
|
7064
7715
|
try {
|
|
7065
7716
|
const snippets = memoryResults.map((r) => r.snippet);
|
|
@@ -7138,11 +7789,16 @@ ${tmtNode.summary}`
|
|
|
7138
7789
|
limit: embeddingFetchLimit
|
|
7139
7790
|
}
|
|
7140
7791
|
);
|
|
7141
|
-
const
|
|
7792
|
+
const boostedScoped = await this.boostSearchResults(
|
|
7142
7793
|
scopedCandidates,
|
|
7143
7794
|
recallNamespaces,
|
|
7144
7795
|
retrievalQuery
|
|
7145
|
-
)
|
|
7796
|
+
);
|
|
7797
|
+
const scoped = this.diversifyAndLimitRecallResults(
|
|
7798
|
+
"memories",
|
|
7799
|
+
boostedScoped,
|
|
7800
|
+
recallResultLimit
|
|
7801
|
+
);
|
|
7146
7802
|
if (scoped.length > 0) {
|
|
7147
7803
|
if (shouldPersistGraphSnapshot) {
|
|
7148
7804
|
graphSnapshotFinalResults = this.buildGraphRecallRankedResults(
|
|
@@ -7253,11 +7909,16 @@ ${tmtNode.summary}`
|
|
|
7253
7909
|
limit: embeddingFetchLimit
|
|
7254
7910
|
}
|
|
7255
7911
|
);
|
|
7256
|
-
const
|
|
7912
|
+
const boostedScoped = await this.boostSearchResults(
|
|
7257
7913
|
scopedCandidates,
|
|
7258
7914
|
recallNamespaces,
|
|
7259
7915
|
retrievalQuery
|
|
7260
|
-
)
|
|
7916
|
+
);
|
|
7917
|
+
const scoped = this.diversifyAndLimitRecallResults(
|
|
7918
|
+
"memories",
|
|
7919
|
+
boostedScoped,
|
|
7920
|
+
recallResultLimit
|
|
7921
|
+
);
|
|
7261
7922
|
if (scoped.length > 0) {
|
|
7262
7923
|
if (shouldPersistGraphSnapshot) {
|
|
7263
7924
|
graphSnapshotFinalResults = this.buildGraphRecallRankedResults(
|
|
@@ -7285,8 +7946,20 @@ ${tmtNode.summary}`
|
|
|
7285
7946
|
} else {
|
|
7286
7947
|
const memories = await this.readAllMemoriesForNamespaces(recallNamespaces);
|
|
7287
7948
|
if (memories.length > 0) {
|
|
7949
|
+
const supersessionOptions = {
|
|
7950
|
+
enabled: this.config.temporalSupersessionEnabled,
|
|
7951
|
+
includeInRecall: this.config.temporalSupersessionIncludeInRecall
|
|
7952
|
+
};
|
|
7288
7953
|
const activeMemories = memories.filter(
|
|
7289
|
-
(m) =>
|
|
7954
|
+
(m) => {
|
|
7955
|
+
if (isArtifactMemoryPath(m.path)) return false;
|
|
7956
|
+
const status = m.frontmatter.status;
|
|
7957
|
+
if (!status || status === "active") return true;
|
|
7958
|
+
if (status === "superseded") {
|
|
7959
|
+
return !shouldFilterSupersededFromRecall(m.frontmatter, supersessionOptions);
|
|
7960
|
+
}
|
|
7961
|
+
return false;
|
|
7962
|
+
}
|
|
7290
7963
|
);
|
|
7291
7964
|
const queryAwareScopedMemories = queryAwarePrefilter.candidatePaths ? activeMemories.filter(
|
|
7292
7965
|
(memory) => queryAwarePrefilter.candidatePaths?.has(memory.path)
|
|
@@ -7334,12 +8007,17 @@ ${tmtNode.summary}`
|
|
|
7334
8007
|
score: 1 - i / Math.max(recentSorted.length, 1)
|
|
7335
8008
|
})
|
|
7336
8009
|
);
|
|
7337
|
-
const
|
|
8010
|
+
const boostedRecent = (await this.boostSearchResults(
|
|
7338
8011
|
recentAsResults,
|
|
7339
8012
|
recallNamespaces,
|
|
7340
8013
|
retrievalQuery,
|
|
7341
8014
|
preloadedMap
|
|
7342
|
-
)).sort((a, b) => b.score - a.score)
|
|
8015
|
+
)).sort((a, b) => b.score - a.score);
|
|
8016
|
+
const recent = this.diversifyAndLimitRecallResults(
|
|
8017
|
+
"memories",
|
|
8018
|
+
boostedRecent,
|
|
8019
|
+
recallResultLimit
|
|
8020
|
+
);
|
|
7343
8021
|
if (recent.length > 0) {
|
|
7344
8022
|
if (shouldPersistGraphSnapshot) {
|
|
7345
8023
|
graphSnapshotFinalResults = this.buildGraphRecallRankedResults(
|
|
@@ -7663,7 +8341,7 @@ _Context: ${topQuestion.context}_`
|
|
|
7663
8341
|
closeProfileTrace();
|
|
7664
8342
|
}
|
|
7665
8343
|
}
|
|
7666
|
-
async processTurn(role, content, sessionKey) {
|
|
8344
|
+
async processTurn(role, content, sessionKey, options = {}) {
|
|
7667
8345
|
if (role !== "user" && role !== "assistant") {
|
|
7668
8346
|
log.debug(`processTurn: ignoring unsupported role=${String(role)}`);
|
|
7669
8347
|
return;
|
|
@@ -7674,15 +8352,42 @@ _Context: ${topQuestion.context}_`
|
|
|
7674
8352
|
);
|
|
7675
8353
|
return;
|
|
7676
8354
|
}
|
|
8355
|
+
const bufferKey = typeof options.bufferKey === "string" && options.bufferKey.length > 0 ? options.bufferKey : typeof sessionKey === "string" && sessionKey.length > 0 ? sessionKey : "default";
|
|
7677
8356
|
const turn = {
|
|
7678
8357
|
role,
|
|
7679
8358
|
content,
|
|
7680
8359
|
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
7681
|
-
sessionKey
|
|
8360
|
+
sessionKey,
|
|
8361
|
+
logicalSessionKey: options.logicalSessionKey ?? bufferKey,
|
|
8362
|
+
providerThreadId: options.providerThreadId ?? null,
|
|
8363
|
+
turnFingerprint: options.turnFingerprint,
|
|
8364
|
+
persistProcessedFingerprint: options.persistProcessedFingerprint === true
|
|
7682
8365
|
};
|
|
7683
|
-
const decision = await this.buffer.addTurn(turn);
|
|
8366
|
+
const decision = await this.buffer.addTurn(bufferKey, turn);
|
|
7684
8367
|
if (decision === "keep_buffering") return;
|
|
7685
|
-
await this.queueBufferedExtraction(
|
|
8368
|
+
await this.queueBufferedExtraction(
|
|
8369
|
+
this.buffer.getTurns(bufferKey),
|
|
8370
|
+
"trigger_mode",
|
|
8371
|
+
{ bufferKey }
|
|
8372
|
+
);
|
|
8373
|
+
}
|
|
8374
|
+
async flushSession(sessionKey, options) {
|
|
8375
|
+
const explicitBufferKey = typeof options.bufferKey === "string" && options.bufferKey.length > 0 ? options.bufferKey : null;
|
|
8376
|
+
const discoveredBufferKeys = explicitBufferKey || typeof sessionKey !== "string" || sessionKey.length === 0 || typeof this.buffer.findBufferKeysForSession !== "function" ? [] : await this.buffer.findBufferKeysForSession(sessionKey);
|
|
8377
|
+
const bufferKeys = explicitBufferKey ? [explicitBufferKey] : discoveredBufferKeys.length > 0 ? discoveredBufferKeys : typeof sessionKey === "string" && sessionKey.length > 0 ? [sessionKey] : ["default"];
|
|
8378
|
+
for (const bufferKey of bufferKeys) {
|
|
8379
|
+
const turns = this.buffer.getTurns(bufferKey);
|
|
8380
|
+
if (turns.length === 0) continue;
|
|
8381
|
+
await new Promise((resolve, reject) => {
|
|
8382
|
+
void this.queueBufferedExtraction(turns, "trigger_mode", {
|
|
8383
|
+
bufferKey,
|
|
8384
|
+
clearBufferAfterExtraction: true,
|
|
8385
|
+
skipDedupeCheck: true,
|
|
8386
|
+
abortSignal: options.abortSignal,
|
|
8387
|
+
onTaskSettled: (error) => error ? reject(error) : resolve()
|
|
8388
|
+
}).catch(reject);
|
|
8389
|
+
});
|
|
8390
|
+
}
|
|
7686
8391
|
}
|
|
7687
8392
|
async ingestReplayBatch(turns, options = {}) {
|
|
7688
8393
|
if (!Array.isArray(turns) || turns.length === 0) return;
|
|
@@ -7706,7 +8411,7 @@ _Context: ${topQuestion.context}_`
|
|
|
7706
8411
|
bySession.set(key, list);
|
|
7707
8412
|
}
|
|
7708
8413
|
const replayTasks = [];
|
|
7709
|
-
for (const sessionTurns of bySession.
|
|
8414
|
+
for (const [key, sessionTurns] of bySession.entries()) {
|
|
7710
8415
|
if (sessionTurns.length === 0) continue;
|
|
7711
8416
|
replayTasks.push(
|
|
7712
8417
|
new Promise((resolve, reject) => {
|
|
@@ -7714,6 +8419,7 @@ _Context: ${topQuestion.context}_`
|
|
|
7714
8419
|
skipDedupeCheck: true,
|
|
7715
8420
|
clearBufferAfterExtraction: false,
|
|
7716
8421
|
skipCharThreshold: true,
|
|
8422
|
+
bufferKey: key,
|
|
7717
8423
|
extractionDeadlineMs: options.deadlineMs,
|
|
7718
8424
|
onTaskSettled: (err) => err ? reject(err) : resolve()
|
|
7719
8425
|
}).catch(reject);
|
|
@@ -7730,25 +8436,113 @@ _Context: ${topQuestion.context}_`
|
|
|
7730
8436
|
}
|
|
7731
8437
|
}
|
|
7732
8438
|
}
|
|
7733
|
-
|
|
8439
|
+
/**
|
|
8440
|
+
* Return the namespace that `ingestBulkImportBatch` writes into (#460).
|
|
8441
|
+
*
|
|
8442
|
+
* Exposed so host CLIs can snapshot the same storage root that extraction
|
|
8443
|
+
* actually writes to, avoiding the "CLI counts files at namespace A while
|
|
8444
|
+
* writes land in namespace B" footgun that a naïve
|
|
8445
|
+
* `config.defaultNamespace` snapshot could hit when a namespace policy
|
|
8446
|
+
* named `"default"` also exists.
|
|
8447
|
+
*
|
|
8448
|
+
* Today bulk-import is pinned to `config.defaultNamespace`; future
|
|
8449
|
+
* per-invocation namespace routing would thread an explicit target here
|
|
8450
|
+
* and through `ingestBulkImportBatch`.
|
|
8451
|
+
*/
|
|
8452
|
+
bulkImportWriteNamespace() {
|
|
8453
|
+
return this.config.defaultNamespace;
|
|
8454
|
+
}
|
|
8455
|
+
/**
|
|
8456
|
+
* Ingest a batch of bulk-import turns (#460). Like ingestReplayBatch, this
|
|
8457
|
+
* normalizes user/assistant turns into the extraction buffer and awaits
|
|
8458
|
+
* settlement, but it intentionally bypasses the captureMode="explicit"
|
|
8459
|
+
* gate because bulk-import is itself an explicit user action — the user
|
|
8460
|
+
* ran `bulk-import --source <name> --file ...` and would be surprised to
|
|
8461
|
+
* see the command silently no-op when capture is otherwise restricted.
|
|
8462
|
+
*
|
|
8463
|
+
* Turns with role="other" are skipped (not supported by the extraction
|
|
8464
|
+
* pipeline).
|
|
8465
|
+
*
|
|
8466
|
+
* Two design decisions worth calling out:
|
|
8467
|
+
*
|
|
8468
|
+
* - **sessionKey is truthy and per-batch-unique.**
|
|
8469
|
+
* `ThreadingManager.shouldStartNewThread` only applies the session-key
|
|
8470
|
+
* boundary check when `turn.sessionKey` is truthy (threading.ts:82);
|
|
8471
|
+
* with an empty string, imported turns could attach to the current
|
|
8472
|
+
* live thread or merge across unrelated import batches. A unique
|
|
8473
|
+
* `bulk-import:batch:<timestamp>-<rand>` key forces a fresh thread per
|
|
8474
|
+
* batch without matching common prefix/map rules in
|
|
8475
|
+
* `principalFromSessionKeyRules`. (Catch-all regex rules could still
|
|
8476
|
+
* remap the principal, but that only affects metadata provenance —
|
|
8477
|
+
* see the next point for why write routing is unaffected.)
|
|
8478
|
+
*
|
|
8479
|
+
* - **writeNamespaceOverride pins the storage target.**
|
|
8480
|
+
* We pass `writeNamespaceOverride: this.bulkImportWriteNamespace()` to
|
|
8481
|
+
* `queueBufferedExtraction`, which tells `runExtraction` to skip
|
|
8482
|
+
* `defaultNamespaceForPrincipal` and write directly into the
|
|
8483
|
+
* orchestrator's declared bulk-import write namespace. This keeps
|
|
8484
|
+
* writes deterministic even when namespace policies named `"default"`
|
|
8485
|
+
* exist alongside a different `config.defaultNamespace`, and also
|
|
8486
|
+
* guards against regex-catch-all principal rules steering bulk-import
|
|
8487
|
+
* into an unexpected tenant.
|
|
8488
|
+
*
|
|
8489
|
+
* Per-invocation namespace routing (letting callers target a namespace
|
|
8490
|
+
* other than `bulkImportWriteNamespace()`) is a separate feature tracked
|
|
8491
|
+
* as a follow-up — the hook is the `writeNamespaceOverride` option, but
|
|
8492
|
+
* the CLI surface does not yet expose a `--namespace` flag.
|
|
8493
|
+
*/
|
|
8494
|
+
async ingestBulkImportBatch(turns, options = {}) {
|
|
8495
|
+
if (!Array.isArray(turns) || turns.length === 0) return;
|
|
8496
|
+
const sessionKey = `bulk-import:batch:${Date.now().toString(36)}-` + randomBytes(6).toString("hex");
|
|
8497
|
+
const sessionTurns = [];
|
|
8498
|
+
for (const turn of turns) {
|
|
8499
|
+
if (turn.role !== "user" && turn.role !== "assistant") continue;
|
|
8500
|
+
sessionTurns.push({
|
|
8501
|
+
role: turn.role,
|
|
8502
|
+
content: turn.content,
|
|
8503
|
+
timestamp: turn.timestamp,
|
|
8504
|
+
sessionKey
|
|
8505
|
+
});
|
|
8506
|
+
}
|
|
8507
|
+
if (sessionTurns.length === 0) return;
|
|
8508
|
+
await new Promise((resolve, reject) => {
|
|
8509
|
+
void this.queueBufferedExtraction(sessionTurns, "trigger_mode", {
|
|
8510
|
+
skipDedupeCheck: true,
|
|
8511
|
+
clearBufferAfterExtraction: false,
|
|
8512
|
+
skipCharThreshold: true,
|
|
8513
|
+
bufferKey: sessionKey,
|
|
8514
|
+
extractionDeadlineMs: options.deadlineMs,
|
|
8515
|
+
writeNamespaceOverride: this.bulkImportWriteNamespace(),
|
|
8516
|
+
onTaskSettled: (err) => err ? reject(err) : resolve()
|
|
8517
|
+
}).catch(reject);
|
|
8518
|
+
});
|
|
8519
|
+
}
|
|
8520
|
+
async observeSessionHeartbeat(sessionKey, options = {}) {
|
|
7734
8521
|
if (this.config.sessionObserverEnabled !== true) return;
|
|
7735
8522
|
if (!sessionKey || sessionKey.length === 0) return;
|
|
8523
|
+
const bufferKey = typeof options.bufferKey === "string" && options.bufferKey.length > 0 ? options.bufferKey : sessionKey;
|
|
7736
8524
|
const previous = this.heartbeatObserverChains.get(sessionKey) ?? Promise.resolve();
|
|
7737
8525
|
const next = previous.catch(() => void 0).then(async () => {
|
|
7738
|
-
const turns = this.buffer.getTurns();
|
|
8526
|
+
const turns = this.buffer.getTurns(bufferKey);
|
|
7739
8527
|
if (turns.length === 0) return;
|
|
7740
|
-
const
|
|
7741
|
-
|
|
8528
|
+
const normalizedSessionKey = normalizeReplaySessionKey(sessionKey);
|
|
8529
|
+
const allowSharedSessionBuffer = bufferKey.startsWith(
|
|
8530
|
+
CODEX_THREAD_KEY_PREFIX
|
|
7742
8531
|
);
|
|
7743
|
-
if (
|
|
8532
|
+
if (!allowSharedSessionBuffer && turns.some(
|
|
8533
|
+
(turn) => turn.sessionKey && normalizeReplaySessionKey(turn.sessionKey) !== normalizedSessionKey
|
|
8534
|
+
)) {
|
|
7744
8535
|
log.debug(
|
|
7745
|
-
`heartbeat observer skipped: mixed
|
|
8536
|
+
`heartbeat observer skipped: mixed-session buffer contents for ${bufferKey}`
|
|
7746
8537
|
);
|
|
7747
8538
|
return;
|
|
7748
8539
|
}
|
|
7749
|
-
if (!this.shouldQueueExtraction(turns, {
|
|
8540
|
+
if (!this.shouldQueueExtraction(turns, {
|
|
8541
|
+
commit: false,
|
|
8542
|
+
bufferKey
|
|
8543
|
+
})) {
|
|
7750
8544
|
log.debug(
|
|
7751
|
-
`heartbeat observer skipped: extraction dedupe for ${
|
|
8545
|
+
`heartbeat observer skipped: extraction dedupe for ${bufferKey}`
|
|
7752
8546
|
);
|
|
7753
8547
|
return;
|
|
7754
8548
|
}
|
|
@@ -7762,7 +8556,9 @@ _Context: ${topQuestion.context}_`
|
|
|
7762
8556
|
log.debug(
|
|
7763
8557
|
`heartbeat observer trigger: session=${sessionKey} deltaBytes=${decision.deltaBytes} deltaTokens=${decision.deltaTokens}`
|
|
7764
8558
|
);
|
|
7765
|
-
await this.queueBufferedExtraction(turns, "heartbeat_observer"
|
|
8559
|
+
await this.queueBufferedExtraction(turns, "heartbeat_observer", {
|
|
8560
|
+
bufferKey
|
|
8561
|
+
});
|
|
7766
8562
|
});
|
|
7767
8563
|
this.heartbeatObserverChains.set(sessionKey, next);
|
|
7768
8564
|
try {
|
|
@@ -7774,7 +8570,8 @@ _Context: ${topQuestion.context}_`
|
|
|
7774
8570
|
}
|
|
7775
8571
|
}
|
|
7776
8572
|
async queueBufferedExtraction(turnsToExtract, reason, options = {}) {
|
|
7777
|
-
|
|
8573
|
+
const bufferKey = options.bufferKey ?? turnsToExtract[0]?.sessionKey ?? "default";
|
|
8574
|
+
if (!options.skipDedupeCheck && !this.shouldQueueExtraction(turnsToExtract, { bufferKey })) {
|
|
7778
8575
|
log.debug(`extraction dedupe skip: preserving buffer (${reason})`);
|
|
7779
8576
|
options.onTaskSettled?.();
|
|
7780
8577
|
return;
|
|
@@ -7784,7 +8581,10 @@ _Context: ${topQuestion.context}_`
|
|
|
7784
8581
|
await this.runExtraction(turnsToExtract, {
|
|
7785
8582
|
clearBufferAfterExtraction: options.clearBufferAfterExtraction ?? true,
|
|
7786
8583
|
skipCharThreshold: options.skipCharThreshold ?? false,
|
|
7787
|
-
deadlineMs: options.extractionDeadlineMs
|
|
8584
|
+
deadlineMs: options.extractionDeadlineMs,
|
|
8585
|
+
bufferKey,
|
|
8586
|
+
abortSignal: options.abortSignal,
|
|
8587
|
+
writeNamespaceOverride: options.writeNamespaceOverride
|
|
7788
8588
|
});
|
|
7789
8589
|
options.onTaskSettled?.();
|
|
7790
8590
|
} catch (err) {
|
|
@@ -7801,14 +8601,27 @@ _Context: ${topQuestion.context}_`
|
|
|
7801
8601
|
}
|
|
7802
8602
|
log.debug(`queued extraction from ${reason}`);
|
|
7803
8603
|
}
|
|
8604
|
+
normalizeExtractionFingerprintTurns(turns) {
|
|
8605
|
+
if (!Array.isArray(turns) || turns.length === 0) return [];
|
|
8606
|
+
return turns.filter((turn) => turn.role === "user" || turn.role === "assistant").map((turn) => {
|
|
8607
|
+
if (typeof turn.turnFingerprint === "string" && turn.turnFingerprint.length > 0) {
|
|
8608
|
+
return `fp:${turn.turnFingerprint}`;
|
|
8609
|
+
}
|
|
8610
|
+
return `${turn.role}:${(turn.content ?? "").replace(/\s+/g, " ").trim().slice(0, this.config.extractionMaxTurnChars)}`;
|
|
8611
|
+
}).filter((value) => value.length > 0);
|
|
8612
|
+
}
|
|
8613
|
+
buildExtractionFingerprint(turns, bufferKey) {
|
|
8614
|
+
const normalized = this.normalizeExtractionFingerprintTurns(turns).join("\n");
|
|
8615
|
+
if (!normalized) return null;
|
|
8616
|
+
return createHash2("sha256").update(`${bufferKey}
|
|
8617
|
+
${normalized}`).digest("hex");
|
|
8618
|
+
}
|
|
7804
8619
|
shouldQueueExtraction(turns, options = {}) {
|
|
7805
8620
|
if (!this.config.extractionDedupeEnabled) return true;
|
|
7806
8621
|
if (!Array.isArray(turns) || turns.length === 0) return false;
|
|
7807
|
-
const
|
|
7808
|
-
|
|
7809
|
-
|
|
7810
|
-
if (!normalized) return false;
|
|
7811
|
-
const fingerprint = createHash2("sha256").update(normalized).digest("hex");
|
|
8622
|
+
const bufferKey = options.bufferKey ?? turns[0]?.sessionKey ?? "default";
|
|
8623
|
+
const fingerprint = this.buildExtractionFingerprint(turns, bufferKey);
|
|
8624
|
+
if (!fingerprint) return false;
|
|
7812
8625
|
const now = Date.now();
|
|
7813
8626
|
const seenAt = this.recentExtractionFingerprints.get(fingerprint);
|
|
7814
8627
|
if (seenAt && now - seenAt < this.config.extractionDedupeWindowMs) {
|
|
@@ -7851,14 +8664,21 @@ _Context: ${topQuestion.context}_`
|
|
|
7851
8664
|
const clearBufferAfterExtraction = options.clearBufferAfterExtraction ?? true;
|
|
7852
8665
|
const skipCharThreshold = options.skipCharThreshold ?? false;
|
|
7853
8666
|
const deadlineMs = typeof options.deadlineMs === "number" && Number.isFinite(options.deadlineMs) ? options.deadlineMs : void 0;
|
|
8667
|
+
const bufferKey = options.bufferKey ?? turns[0]?.sessionKey ?? "default";
|
|
7854
8668
|
const throwIfDeadlineExceeded = (stage) => {
|
|
7855
8669
|
if (typeof deadlineMs === "number" && Date.now() > deadlineMs) {
|
|
7856
8670
|
throw new Error(`replay extraction deadline exceeded (${stage})`);
|
|
7857
8671
|
}
|
|
7858
8672
|
};
|
|
7859
|
-
const
|
|
8673
|
+
const throwIfAborted = (stage) => {
|
|
8674
|
+
throwIfRecallAborted(options.abortSignal, `extraction aborted (${stage})`);
|
|
8675
|
+
};
|
|
8676
|
+
const clearBuffer = async (options2) => {
|
|
8677
|
+
if (options2?.ignoreAbort !== true) {
|
|
8678
|
+
throwIfAborted("before_clear_buffer");
|
|
8679
|
+
}
|
|
7860
8680
|
if (clearBufferAfterExtraction) {
|
|
7861
|
-
await this.buffer.clearAfterExtraction();
|
|
8681
|
+
await this.buffer.clearAfterExtraction(bufferKey);
|
|
7862
8682
|
}
|
|
7863
8683
|
};
|
|
7864
8684
|
const sessionKey = turns[0]?.sessionKey ?? "";
|
|
@@ -7874,6 +8694,7 @@ _Context: ${topQuestion.context}_`
|
|
|
7874
8694
|
content: t.content.trim().slice(0, this.config.extractionMaxTurnChars)
|
|
7875
8695
|
})).filter((t) => t.content.length > 0);
|
|
7876
8696
|
throwIfDeadlineExceeded("before_extract");
|
|
8697
|
+
throwIfAborted("before_extract");
|
|
7877
8698
|
const userTurns = normalizedTurns.filter((t) => t.role === "user");
|
|
7878
8699
|
const totalChars = normalizedTurns.reduce(
|
|
7879
8700
|
(sum, t) => sum + t.content.length,
|
|
@@ -7889,14 +8710,36 @@ _Context: ${topQuestion.context}_`
|
|
|
7889
8710
|
return;
|
|
7890
8711
|
}
|
|
7891
8712
|
const principal = resolvePrincipal(sessionKey, this.config);
|
|
7892
|
-
const selfNamespace = defaultNamespaceForPrincipal(principal, this.config);
|
|
8713
|
+
const selfNamespace = typeof options.writeNamespaceOverride === "string" && options.writeNamespaceOverride.length > 0 ? options.writeNamespaceOverride : defaultNamespaceForPrincipal(principal, this.config);
|
|
7893
8714
|
const storage = await this.storageRouter.storageFor(selfNamespace);
|
|
7894
|
-
const
|
|
7895
|
-
|
|
8715
|
+
const shouldPersistProcessedFingerprint = normalizedTurns.some(
|
|
8716
|
+
(turn) => turn.persistProcessedFingerprint === true
|
|
8717
|
+
);
|
|
8718
|
+
const extractionFingerprint = this.buildExtractionFingerprint(
|
|
7896
8719
|
normalizedTurns,
|
|
7897
|
-
|
|
8720
|
+
bufferKey
|
|
8721
|
+
);
|
|
8722
|
+
let meta = extractionFingerprint && shouldPersistProcessedFingerprint ? await storage.loadMeta() : null;
|
|
8723
|
+
if (extractionFingerprint && shouldPersistProcessedFingerprint && (meta?.processedExtractionFingerprints ?? []).some(
|
|
8724
|
+
(entry) => entry.fingerprint === extractionFingerprint
|
|
8725
|
+
)) {
|
|
8726
|
+
log.debug(
|
|
8727
|
+
`runExtraction: skipping already-processed extraction fingerprint for ${bufferKey}`
|
|
8728
|
+
);
|
|
8729
|
+
await clearBuffer();
|
|
8730
|
+
return;
|
|
8731
|
+
}
|
|
8732
|
+
const existingEntities = await storage.listEntityNames();
|
|
8733
|
+
const result = await raceRecallAbort(
|
|
8734
|
+
this.extraction.extract(
|
|
8735
|
+
normalizedTurns,
|
|
8736
|
+
existingEntities
|
|
8737
|
+
),
|
|
8738
|
+
options.abortSignal,
|
|
8739
|
+
"extraction aborted (during_extract)"
|
|
7898
8740
|
);
|
|
7899
8741
|
throwIfDeadlineExceeded("before_persist");
|
|
8742
|
+
throwIfAborted("before_persist");
|
|
7900
8743
|
if (!result) {
|
|
7901
8744
|
log.warn("runExtraction: extraction returned null/undefined");
|
|
7902
8745
|
await clearBuffer();
|
|
@@ -7932,9 +8775,35 @@ _Context: ${topQuestion.context}_`
|
|
|
7932
8775
|
const persistedIds = await this.persistExtraction(
|
|
7933
8776
|
result,
|
|
7934
8777
|
storage,
|
|
7935
|
-
threadIdForExtraction
|
|
8778
|
+
threadIdForExtraction,
|
|
8779
|
+
{ sessionKey, principal }
|
|
7936
8780
|
);
|
|
7937
|
-
await
|
|
8781
|
+
meta ??= await storage.loadMeta();
|
|
8782
|
+
if (extractionFingerprint && shouldPersistProcessedFingerprint) {
|
|
8783
|
+
try {
|
|
8784
|
+
await this.recordProcessedExtractionFingerprint(
|
|
8785
|
+
storage,
|
|
8786
|
+
extractionFingerprint,
|
|
8787
|
+
meta
|
|
8788
|
+
);
|
|
8789
|
+
} catch (error) {
|
|
8790
|
+
log.warn(
|
|
8791
|
+
"runExtraction: failed to persist processed extraction fingerprint; continuing with buffer clear",
|
|
8792
|
+
error
|
|
8793
|
+
);
|
|
8794
|
+
}
|
|
8795
|
+
}
|
|
8796
|
+
meta.extractionCount += 1;
|
|
8797
|
+
meta.lastExtractionAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
8798
|
+
meta.totalMemories += Array.isArray(result?.facts) ? result.facts.length : 0;
|
|
8799
|
+
meta.totalEntities += Array.isArray(result?.entities) ? result.entities.length : 0;
|
|
8800
|
+
let postPersistMetaError;
|
|
8801
|
+
try {
|
|
8802
|
+
await storage.saveMeta(meta);
|
|
8803
|
+
} catch (error) {
|
|
8804
|
+
postPersistMetaError = error;
|
|
8805
|
+
}
|
|
8806
|
+
await clearBuffer({ ignoreAbort: true });
|
|
7938
8807
|
if (this.config.memoryBoxesEnabled && persistedIds.length > 0) {
|
|
7939
8808
|
const extractionTopics = deriveTopicsFromExtraction(result);
|
|
7940
8809
|
const firstUserTurn = turns.find((t) => t.role === "user");
|
|
@@ -7971,14 +8840,26 @@ _Context: ${topQuestion.context}_`
|
|
|
7971
8840
|
const nonZeroExtraction = result.facts.length > 0 || result.entities.length > 0 || result.questions.length > 0 || result.profileUpdates.length > 0;
|
|
7972
8841
|
if (nonZeroExtraction) this.nonZeroExtractionsSinceConsolidation += 1;
|
|
7973
8842
|
this.maybeScheduleConsolidation(nonZeroExtraction);
|
|
7974
|
-
const meta = await storage.loadMeta();
|
|
7975
|
-
meta.extractionCount += 1;
|
|
7976
|
-
meta.lastExtractionAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
7977
|
-
meta.totalMemories += Array.isArray(result?.facts) ? result.facts.length : 0;
|
|
7978
|
-
meta.totalEntities += Array.isArray(result?.entities) ? result.entities.length : 0;
|
|
7979
|
-
await storage.saveMeta(meta);
|
|
7980
8843
|
this.requestQmdMaintenance();
|
|
7981
8844
|
await this.runTierMigrationCycle(storage, "extraction");
|
|
8845
|
+
if (postPersistMetaError) {
|
|
8846
|
+
throw postPersistMetaError;
|
|
8847
|
+
}
|
|
8848
|
+
}
|
|
8849
|
+
async recordProcessedExtractionFingerprint(storage, fingerprint, preloadedMeta) {
|
|
8850
|
+
const meta = preloadedMeta ?? await storage.loadMeta();
|
|
8851
|
+
const observedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
8852
|
+
const seen = new Map(
|
|
8853
|
+
(meta.processedExtractionFingerprints ?? []).map((entry) => [
|
|
8854
|
+
entry.fingerprint,
|
|
8855
|
+
entry.observedAt
|
|
8856
|
+
])
|
|
8857
|
+
);
|
|
8858
|
+
seen.set(fingerprint, observedAt);
|
|
8859
|
+
meta.processedExtractionFingerprints = Array.from(seen.entries()).map(([value, at]) => ({ fingerprint: value, observedAt: at })).sort((left, right) => left.observedAt.localeCompare(right.observedAt)).slice(-500);
|
|
8860
|
+
if (!preloadedMeta) {
|
|
8861
|
+
await storage.saveMeta(meta);
|
|
8862
|
+
}
|
|
7982
8863
|
}
|
|
7983
8864
|
async runTierMigrationCycle(storage, trigger, options) {
|
|
7984
8865
|
const dryRun = options?.dryRun === true;
|
|
@@ -8222,7 +9103,22 @@ _Context: ${topQuestion.context}_`
|
|
|
8222
9103
|
}
|
|
8223
9104
|
}
|
|
8224
9105
|
}
|
|
8225
|
-
async persistExtraction(result, storage, threadIdForExtraction) {
|
|
9106
|
+
async persistExtraction(result, storage, threadIdForExtraction, sourceContext) {
|
|
9107
|
+
const citationEnabled = this.config.inlineSourceAttributionEnabled === true;
|
|
9108
|
+
const citationTemplate = this.config.inlineSourceAttributionFormat;
|
|
9109
|
+
const citationContextBase = citationEnabled ? {
|
|
9110
|
+
agent: sourceContext?.principal,
|
|
9111
|
+
session: sourceContext?.sessionKey
|
|
9112
|
+
} : {};
|
|
9113
|
+
const applyInlineCitation = (content) => {
|
|
9114
|
+
if (!citationEnabled) return content;
|
|
9115
|
+
if (typeof content !== "string" || content.length === 0) return content;
|
|
9116
|
+
const citationContext = {
|
|
9117
|
+
...citationContextBase,
|
|
9118
|
+
ts: (/* @__PURE__ */ new Date()).toISOString()
|
|
9119
|
+
};
|
|
9120
|
+
return attachCitation(content, citationContext, citationTemplate);
|
|
9121
|
+
};
|
|
8226
9122
|
const persistedIds = [];
|
|
8227
9123
|
const persistedIdsByStorage = /* @__PURE__ */ new Map();
|
|
8228
9124
|
const trackPersistedId = (targetStorage, id, options = {}) => {
|
|
@@ -8238,6 +9134,8 @@ _Context: ${topQuestion.context}_`
|
|
|
8238
9134
|
persistedIdsByStorage.set(key, { storage: targetStorage, ids: [id] });
|
|
8239
9135
|
};
|
|
8240
9136
|
let dedupedCount = 0;
|
|
9137
|
+
let importanceGatedCount = 0;
|
|
9138
|
+
let batchBackendUnavailable = false;
|
|
8241
9139
|
const behaviorSignalsByStorage = /* @__PURE__ */ new Map();
|
|
8242
9140
|
const trackBehaviorSignals = (targetStorage, events) => {
|
|
8243
9141
|
if (events.length === 0) return;
|
|
@@ -8284,16 +9182,70 @@ _Context: ${topQuestion.context}_`
|
|
|
8284
9182
|
const sharedStorage = await this.storageRouter.storageFor(
|
|
8285
9183
|
this.config.sharedNamespace
|
|
8286
9184
|
);
|
|
8287
|
-
|
|
8288
|
-
|
|
9185
|
+
const rawContent = citationEnabled && hasCitationForTemplate(options.content, citationTemplate) ? stripCitationForTemplate(options.content, citationTemplate) : options.content;
|
|
9186
|
+
const citedContent = applyInlineCitation(rawContent);
|
|
9187
|
+
const sanitizedBase = sanitizeMemoryContent(rawContent);
|
|
9188
|
+
const dedupContent = options.category === "fact" && options.structuredAttributes && Object.keys(options.structuredAttributes).length > 0 ? `${sanitizedBase.text}
|
|
9189
|
+
[Attributes: ${normalizeAttributePairs(options.structuredAttributes)}]` : sanitizedBase.text;
|
|
9190
|
+
if (options.category === "fact" && await sharedStorage.hasFactContentHash(dedupContent)) {
|
|
9191
|
+
if (this.config.temporalSupersessionEnabled && options.entityRef && options.structuredAttributes && Object.keys(options.structuredAttributes).length > 0) {
|
|
9192
|
+
let hashDedupMatchingFact;
|
|
9193
|
+
let hashDedupLookupComplete = false;
|
|
9194
|
+
try {
|
|
9195
|
+
const normalizedIncoming = ContentHashIndex.normalizeContent(dedupContent);
|
|
9196
|
+
const allShared = await sharedStorage.readAllMemories();
|
|
9197
|
+
const incomingEntityNorm = normalizeSupersessionKey(options.entityRef);
|
|
9198
|
+
hashDedupMatchingFact = allShared.find((m) => {
|
|
9199
|
+
if (m.frontmatter.category !== "fact") return false;
|
|
9200
|
+
if ((m.frontmatter.status ?? "active") !== "active") return false;
|
|
9201
|
+
if (!m.frontmatter.entityRef) return false;
|
|
9202
|
+
if (normalizeSupersessionKey(m.frontmatter.entityRef) !== incomingEntityNorm) {
|
|
9203
|
+
log.debug(
|
|
9204
|
+
`persistExtraction: hash-dedup skipping cross-entity match (incoming="${incomingEntityNorm}" candidate="${normalizeSupersessionKey(m.frontmatter.entityRef)}")`
|
|
9205
|
+
);
|
|
9206
|
+
return false;
|
|
9207
|
+
}
|
|
9208
|
+
return ContentHashIndex.normalizeContent(m.content ?? "") === normalizedIncoming;
|
|
9209
|
+
});
|
|
9210
|
+
hashDedupLookupComplete = true;
|
|
9211
|
+
if (hashDedupMatchingFact) {
|
|
9212
|
+
await applyTemporalSupersession({
|
|
9213
|
+
storage: sharedStorage,
|
|
9214
|
+
newMemoryId: hashDedupMatchingFact.frontmatter.id,
|
|
9215
|
+
entityRef: options.entityRef,
|
|
9216
|
+
structuredAttributes: options.structuredAttributes,
|
|
9217
|
+
createdAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
9218
|
+
enabled: true,
|
|
9219
|
+
useCallerTimestamp: true
|
|
9220
|
+
});
|
|
9221
|
+
return;
|
|
9222
|
+
}
|
|
9223
|
+
log.debug(
|
|
9224
|
+
`persistExtraction: hash-dedup found no active same-entity shared fact for ${options.sourceMemoryId}; falling through to write`
|
|
9225
|
+
);
|
|
9226
|
+
} catch (hashDedupSupersessionErr) {
|
|
9227
|
+
log.warn(
|
|
9228
|
+
`persistExtraction: shared-namespace supersession on hash-dedup path failed open for ${options.sourceMemoryId}: ${hashDedupSupersessionErr}`
|
|
9229
|
+
);
|
|
9230
|
+
if (hashDedupLookupComplete && hashDedupMatchingFact) {
|
|
9231
|
+
return;
|
|
9232
|
+
}
|
|
9233
|
+
log.debug(
|
|
9234
|
+
`persistExtraction: hash-dedup catch: lookup incomplete or no candidate found for ${options.sourceMemoryId}; falling through to write`
|
|
9235
|
+
);
|
|
9236
|
+
}
|
|
9237
|
+
} else {
|
|
9238
|
+
return;
|
|
9239
|
+
}
|
|
8289
9240
|
}
|
|
8290
9241
|
const promotedId = await sharedStorage.writeMemory(
|
|
8291
9242
|
options.category,
|
|
8292
|
-
|
|
9243
|
+
citedContent,
|
|
8293
9244
|
{
|
|
8294
9245
|
confidence: options.confidence,
|
|
8295
9246
|
tags: [...options.tags, "shared-promotion"],
|
|
8296
9247
|
entityRef: options.entityRef,
|
|
9248
|
+
structuredAttributes: options.structuredAttributes,
|
|
8297
9249
|
source: `${options.source}-shared-promotion`,
|
|
8298
9250
|
importance: options.importance,
|
|
8299
9251
|
lineage: [options.sourceMemoryId],
|
|
@@ -8301,9 +9253,30 @@ _Context: ${topQuestion.context}_`
|
|
|
8301
9253
|
intentGoal: options.intentGoal,
|
|
8302
9254
|
intentActionType: options.intentActionType,
|
|
8303
9255
|
intentEntityTypes: options.intentEntityTypes,
|
|
8304
|
-
memoryKind: options.memoryKind
|
|
9256
|
+
memoryKind: options.memoryKind,
|
|
9257
|
+
// Index the RAW content hash so hasFactContentHash(rawContent)
|
|
9258
|
+
// returns true on subsequent extractions. Without this, the index
|
|
9259
|
+
// would record the hash of citedContent (which changes every call
|
|
9260
|
+
// due to an updated timestamp), causing duplicate promotions.
|
|
9261
|
+
contentHashSource: rawContent
|
|
8305
9262
|
}
|
|
8306
9263
|
);
|
|
9264
|
+
if (this.config.temporalSupersessionEnabled && options.entityRef && options.structuredAttributes && Object.keys(options.structuredAttributes).length > 0) {
|
|
9265
|
+
try {
|
|
9266
|
+
await applyTemporalSupersession({
|
|
9267
|
+
storage: sharedStorage,
|
|
9268
|
+
newMemoryId: promotedId,
|
|
9269
|
+
entityRef: options.entityRef,
|
|
9270
|
+
structuredAttributes: options.structuredAttributes,
|
|
9271
|
+
createdAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
9272
|
+
enabled: true
|
|
9273
|
+
});
|
|
9274
|
+
} catch (sharedSupersessionErr) {
|
|
9275
|
+
log.warn(
|
|
9276
|
+
`persistExtraction: shared-namespace temporal supersession failed open for promoted ${promotedId}: ${sharedSupersessionErr}`
|
|
9277
|
+
);
|
|
9278
|
+
}
|
|
9279
|
+
}
|
|
8307
9280
|
trackPersistedId(sharedStorage, promotedId, {
|
|
8308
9281
|
includeReturnedIds: false
|
|
8309
9282
|
});
|
|
@@ -8391,7 +9364,86 @@ _Context: ${topQuestion.context}_`
|
|
|
8391
9364
|
}
|
|
8392
9365
|
const routeRules = await this.loadRoutingRules();
|
|
8393
9366
|
const routeOptions = this.routeEngineOptions();
|
|
9367
|
+
const preRoutedCategories = new Array(facts.length);
|
|
9368
|
+
if (routeRules.length > 0) {
|
|
9369
|
+
for (let fi = 0; fi < facts.length; fi++) {
|
|
9370
|
+
const f = facts[fi];
|
|
9371
|
+
if (!f || typeof f.content !== "string" || !f.content.trim() || typeof f.category !== "string" || !f.category.trim()) {
|
|
9372
|
+
continue;
|
|
9373
|
+
}
|
|
9374
|
+
try {
|
|
9375
|
+
const tags = Array.isArray(f.tags) ? f.tags : [];
|
|
9376
|
+
const routeText = `${f.category} ${tags.join(" ")} ${f.content}`;
|
|
9377
|
+
const selected = selectRouteRule(routeText, routeRules, routeOptions);
|
|
9378
|
+
if (selected?.target.category) {
|
|
9379
|
+
preRoutedCategories[fi] = selected.target.category;
|
|
9380
|
+
}
|
|
9381
|
+
} catch {
|
|
9382
|
+
}
|
|
9383
|
+
}
|
|
9384
|
+
}
|
|
9385
|
+
let judgeVerdictsByFactIndex = null;
|
|
9386
|
+
let judgeGatedCount = 0;
|
|
9387
|
+
if (this.config.extractionJudgeEnabled) {
|
|
9388
|
+
try {
|
|
9389
|
+
const judgeCandidates = [];
|
|
9390
|
+
const candidateToFactIndex = [];
|
|
9391
|
+
for (let fi = 0; fi < facts.length; fi++) {
|
|
9392
|
+
const f = facts[fi];
|
|
9393
|
+
if (!f || typeof f.content !== "string" || !f.content.trim() || typeof f.category !== "string" || !f.category.trim()) {
|
|
9394
|
+
continue;
|
|
9395
|
+
}
|
|
9396
|
+
const judgeCategory = preRoutedCategories[fi] ?? f.category;
|
|
9397
|
+
if (judgeCategory === "procedure") {
|
|
9398
|
+
continue;
|
|
9399
|
+
}
|
|
9400
|
+
const tags = Array.isArray(f.tags) ? f.tags : [];
|
|
9401
|
+
const imp = scoreImportance(
|
|
9402
|
+
f.content,
|
|
9403
|
+
judgeCategory,
|
|
9404
|
+
tags
|
|
9405
|
+
);
|
|
9406
|
+
if (!isAboveImportanceThreshold(
|
|
9407
|
+
imp.level,
|
|
9408
|
+
this.config.extractionMinImportanceLevel
|
|
9409
|
+
)) {
|
|
9410
|
+
continue;
|
|
9411
|
+
}
|
|
9412
|
+
judgeCandidates.push({
|
|
9413
|
+
text: f.content,
|
|
9414
|
+
category: judgeCategory,
|
|
9415
|
+
confidence: typeof f.confidence === "number" ? f.confidence : 0.7,
|
|
9416
|
+
tags,
|
|
9417
|
+
importanceLevel: imp.level
|
|
9418
|
+
});
|
|
9419
|
+
candidateToFactIndex.push(fi);
|
|
9420
|
+
}
|
|
9421
|
+
const judgeResult = await judgeFactDurability(
|
|
9422
|
+
judgeCandidates,
|
|
9423
|
+
this.config,
|
|
9424
|
+
this.localLlm,
|
|
9425
|
+
new FallbackLlmClient(this.config.gatewayConfig),
|
|
9426
|
+
this.judgeVerdictCache
|
|
9427
|
+
);
|
|
9428
|
+
judgeVerdictsByFactIndex = /* @__PURE__ */ new Map();
|
|
9429
|
+
for (const [candidateIdx, verdict] of judgeResult.verdicts) {
|
|
9430
|
+
const factIdx = candidateToFactIndex[candidateIdx];
|
|
9431
|
+
if (factIdx !== void 0) {
|
|
9432
|
+
judgeVerdictsByFactIndex.set(factIdx, verdict);
|
|
9433
|
+
}
|
|
9434
|
+
}
|
|
9435
|
+
log.info(
|
|
9436
|
+
`extraction-judge: ${judgeResult.verdicts.size}/${judgeCandidates.length} facts evaluated, ${judgeResult.cached} cached, ${judgeResult.judged} judged, ${judgeResult.elapsed}ms`
|
|
9437
|
+
);
|
|
9438
|
+
} catch (err) {
|
|
9439
|
+
log.warn(
|
|
9440
|
+
`extraction-judge: pipeline error, proceeding without filtering (fail-open): ${err instanceof Error ? err.message : String(err)}`
|
|
9441
|
+
);
|
|
9442
|
+
}
|
|
9443
|
+
}
|
|
9444
|
+
let factLoopIndex = -1;
|
|
8394
9445
|
for (const fact of facts) {
|
|
9446
|
+
factLoopIndex++;
|
|
8395
9447
|
if (!fact || typeof fact.content !== "string" || !fact.content.trim()) {
|
|
8396
9448
|
continue;
|
|
8397
9449
|
}
|
|
@@ -8400,12 +9452,15 @@ _Context: ${topQuestion.context}_`
|
|
|
8400
9452
|
}
|
|
8401
9453
|
fact.tags = Array.isArray(fact.tags) ? fact.tags.filter((t) => typeof t === "string") : [];
|
|
8402
9454
|
fact.confidence = typeof fact.confidence === "number" ? fact.confidence : 0.7;
|
|
8403
|
-
if (this.contentHashIndex
|
|
8404
|
-
|
|
8405
|
-
|
|
8406
|
-
|
|
8407
|
-
|
|
8408
|
-
|
|
9455
|
+
if (this.contentHashIndex) {
|
|
9456
|
+
const canonicalContent = citationEnabled && hasCitationForTemplate(fact.content, citationTemplate) ? stripCitationForTemplate(fact.content, citationTemplate) : fact.content;
|
|
9457
|
+
if (this.contentHashIndex.has(canonicalContent)) {
|
|
9458
|
+
log.debug(
|
|
9459
|
+
`dedup: skipping duplicate fact "${fact.content.slice(0, 60)}\u2026"`
|
|
9460
|
+
);
|
|
9461
|
+
dedupedCount++;
|
|
9462
|
+
continue;
|
|
9463
|
+
}
|
|
8409
9464
|
}
|
|
8410
9465
|
let writeCategory = fact.category;
|
|
8411
9466
|
let targetStorage = storage;
|
|
@@ -8436,28 +9491,179 @@ _Context: ${topQuestion.context}_`
|
|
|
8436
9491
|
writeCategory,
|
|
8437
9492
|
fact.tags
|
|
8438
9493
|
);
|
|
9494
|
+
if (writeCategory === "procedure" && this.config.procedural?.enabled !== true) {
|
|
9495
|
+
log.debug("persistExtraction: skip procedure memory (procedural.enabled is false)");
|
|
9496
|
+
continue;
|
|
9497
|
+
}
|
|
9498
|
+
if (!isAboveImportanceThreshold(
|
|
9499
|
+
importance.level,
|
|
9500
|
+
this.config.extractionMinImportanceLevel
|
|
9501
|
+
)) {
|
|
9502
|
+
importanceGatedCount++;
|
|
9503
|
+
const snippet = fact.content.slice(0, 60).replace(/\s+/g, " ").trim();
|
|
9504
|
+
log.debug(`extraction: skip trivial "${snippet}"`);
|
|
9505
|
+
log.debug(
|
|
9506
|
+
`metric:importance_gated level=${importance.level} threshold=${this.config.extractionMinImportanceLevel} category=${writeCategory} count=${importanceGatedCount}`
|
|
9507
|
+
);
|
|
9508
|
+
continue;
|
|
9509
|
+
}
|
|
9510
|
+
if (judgeVerdictsByFactIndex) {
|
|
9511
|
+
const verdict = judgeVerdictsByFactIndex.get(factLoopIndex);
|
|
9512
|
+
if (verdict && !verdict.durable) {
|
|
9513
|
+
if (this.config.extractionJudgeShadow) {
|
|
9514
|
+
log.info(
|
|
9515
|
+
`extraction-judge[shadow]: would reject "${fact.content.slice(0, 60)}\u2026" reason="${verdict.reason}"`
|
|
9516
|
+
);
|
|
9517
|
+
} else {
|
|
9518
|
+
judgeGatedCount++;
|
|
9519
|
+
log.debug(
|
|
9520
|
+
`extraction-judge: rejected "${fact.content.slice(0, 60)}\u2026" reason="${verdict.reason}"`
|
|
9521
|
+
);
|
|
9522
|
+
continue;
|
|
9523
|
+
}
|
|
9524
|
+
}
|
|
9525
|
+
}
|
|
9526
|
+
if (writeCategory === "procedure") {
|
|
9527
|
+
const procGate = validateProcedureExtraction({
|
|
9528
|
+
content: fact.content,
|
|
9529
|
+
procedureSteps: fact.procedureSteps
|
|
9530
|
+
});
|
|
9531
|
+
if (!procGate.durable) {
|
|
9532
|
+
if (this.config.extractionJudgeShadow) {
|
|
9533
|
+
log.info(
|
|
9534
|
+
`extraction-procedure-gate[shadow]: would reject "${fact.content.slice(0, 60)}\u2026" reason="${procGate.reason}"`
|
|
9535
|
+
);
|
|
9536
|
+
} else {
|
|
9537
|
+
log.debug(
|
|
9538
|
+
`extraction-procedure-gate: rejected "${fact.content.slice(0, 60)}\u2026" reason="${procGate.reason}"`
|
|
9539
|
+
);
|
|
9540
|
+
continue;
|
|
9541
|
+
}
|
|
9542
|
+
}
|
|
9543
|
+
}
|
|
9544
|
+
let pendingSemanticSkip = null;
|
|
9545
|
+
if (this.config.semanticDedupEnabled) {
|
|
9546
|
+
let semanticDecision;
|
|
9547
|
+
if (batchBackendUnavailable) {
|
|
9548
|
+
semanticDecision = { action: "keep", reason: "backend_unavailable" };
|
|
9549
|
+
} else {
|
|
9550
|
+
try {
|
|
9551
|
+
const lookupStorage = targetStorage;
|
|
9552
|
+
semanticDecision = await decideSemanticDedup(
|
|
9553
|
+
fact.content,
|
|
9554
|
+
(content, limit) => this.semanticDedupLookup(content, limit, lookupStorage),
|
|
9555
|
+
{
|
|
9556
|
+
enabled: true,
|
|
9557
|
+
threshold: this.config.semanticDedupThreshold,
|
|
9558
|
+
candidates: this.config.semanticDedupCandidates
|
|
9559
|
+
}
|
|
9560
|
+
);
|
|
9561
|
+
} catch (err) {
|
|
9562
|
+
log.warn(
|
|
9563
|
+
`semantic dedup decision failed; failing open and writing fact: ${err}`
|
|
9564
|
+
);
|
|
9565
|
+
semanticDecision = {
|
|
9566
|
+
action: "keep",
|
|
9567
|
+
reason: "backend_unavailable"
|
|
9568
|
+
};
|
|
9569
|
+
}
|
|
9570
|
+
if (semanticDecision.reason === "backend_unavailable") {
|
|
9571
|
+
batchBackendUnavailable = true;
|
|
9572
|
+
}
|
|
9573
|
+
}
|
|
9574
|
+
if (semanticDecision.action === "skip") {
|
|
9575
|
+
pendingSemanticSkip = semanticDecision;
|
|
9576
|
+
}
|
|
9577
|
+
}
|
|
8439
9578
|
const inferredIntent = this.config.intentRoutingEnabled ? inferIntentFromText(
|
|
8440
9579
|
`${writeCategory} ${fact.tags.join(" ")} ${fact.content}`
|
|
8441
9580
|
) : null;
|
|
8442
9581
|
const extractionWriteSource = fact.source === "proactive" ? "extraction-proactive" : "extraction";
|
|
8443
|
-
|
|
8444
|
-
|
|
9582
|
+
let supersedes;
|
|
9583
|
+
let links = [];
|
|
9584
|
+
let contradictionDetected = false;
|
|
9585
|
+
if (this.config.contradictionDetectionEnabled && this.qmd.isAvailable()) {
|
|
9586
|
+
const targetNamespace = this.namespaceFromStorageDir(targetStorage.dir);
|
|
9587
|
+
const contradiction = await this.checkForContradiction(
|
|
9588
|
+
fact.content,
|
|
9589
|
+
writeCategory,
|
|
9590
|
+
targetNamespace
|
|
9591
|
+
);
|
|
9592
|
+
if (contradiction) {
|
|
9593
|
+
contradictionDetected = true;
|
|
9594
|
+
if (this.config.contradictionAutoResolve) {
|
|
9595
|
+
supersedes = contradiction.supersededId;
|
|
9596
|
+
}
|
|
9597
|
+
links.push({
|
|
9598
|
+
targetId: contradiction.supersededId,
|
|
9599
|
+
linkType: "contradicts",
|
|
9600
|
+
strength: contradiction.confidence,
|
|
9601
|
+
reason: contradiction.reason
|
|
9602
|
+
});
|
|
9603
|
+
if (this.config.contradictionAutoResolve && this.config.queryAwareIndexingEnabled && contradiction.supersededPath) {
|
|
9604
|
+
deindexMemory(
|
|
9605
|
+
this.config.memoryDir,
|
|
9606
|
+
contradiction.supersededPath,
|
|
9607
|
+
contradiction.supersededCreated,
|
|
9608
|
+
contradiction.supersededTags
|
|
9609
|
+
);
|
|
9610
|
+
}
|
|
9611
|
+
}
|
|
9612
|
+
}
|
|
9613
|
+
const isCorrection = writeCategory === "correction";
|
|
9614
|
+
if (pendingSemanticSkip && !contradictionDetected && !isCorrection) {
|
|
9615
|
+
log.debug(
|
|
9616
|
+
`dedup: skipping semantic near-duplicate fact "${fact.content.slice(0, 60).replace(/\s+/g, " ")}\u2026" score=${pendingSemanticSkip.topScore.toFixed(
|
|
9617
|
+
3
|
|
9618
|
+
)} neighbor=${pendingSemanticSkip.topId}`
|
|
9619
|
+
);
|
|
9620
|
+
dedupedCount++;
|
|
9621
|
+
continue;
|
|
9622
|
+
}
|
|
9623
|
+
if (this.config.chunkingEnabled && writeCategory !== "procedure") {
|
|
9624
|
+
let chunkResult;
|
|
9625
|
+
if (this.config.semanticChunkingEnabled) {
|
|
9626
|
+
try {
|
|
9627
|
+
const embedFn = this.embeddingFallback.embedTexts.bind(this.embeddingFallback);
|
|
9628
|
+
const semanticResult = await semanticChunkContent(
|
|
9629
|
+
fact.content,
|
|
9630
|
+
embedFn,
|
|
9631
|
+
this.config.semanticChunkingConfig
|
|
9632
|
+
);
|
|
9633
|
+
chunkResult = semanticResult;
|
|
9634
|
+
} catch (err) {
|
|
9635
|
+
if (this.config.semanticChunkingConfig?.fallbackToRecursive === false) {
|
|
9636
|
+
throw err;
|
|
9637
|
+
}
|
|
9638
|
+
log.debug(
|
|
9639
|
+
`semantic chunking failed, falling back to recursive chunker: ${err}`
|
|
9640
|
+
);
|
|
9641
|
+
chunkResult = chunkContent(fact.content, chunkingConfig);
|
|
9642
|
+
}
|
|
9643
|
+
} else {
|
|
9644
|
+
chunkResult = chunkContent(fact.content, chunkingConfig);
|
|
9645
|
+
}
|
|
8445
9646
|
if (chunkResult.chunked && chunkResult.chunks.length > 1) {
|
|
8446
9647
|
const memoryKind2 = this.config.episodeNoteModeEnabled ? classifyMemoryKind(fact.content, fact.tags ?? [], writeCategory) : void 0;
|
|
9648
|
+
const rawChunkedContent = citationEnabled && hasCitationForTemplate(fact.content, citationTemplate) ? stripCitationForTemplate(fact.content, citationTemplate) : fact.content;
|
|
9649
|
+
const citedChunkedContent = applyInlineCitation(rawChunkedContent);
|
|
8447
9650
|
const parentId = await targetStorage.writeMemory(
|
|
8448
9651
|
writeCategory,
|
|
8449
|
-
|
|
9652
|
+
citedChunkedContent,
|
|
8450
9653
|
{
|
|
8451
9654
|
confidence: fact.confidence,
|
|
8452
9655
|
tags: [...fact.tags, "chunked"],
|
|
8453
9656
|
entityRef: fact.entityRef,
|
|
8454
9657
|
source: extractionWriteSource,
|
|
8455
9658
|
importance,
|
|
9659
|
+
supersedes,
|
|
9660
|
+
links: links.length > 0 ? links : void 0,
|
|
8456
9661
|
intentGoal: inferredIntent?.goal,
|
|
8457
9662
|
intentActionType: inferredIntent?.actionType,
|
|
8458
9663
|
intentEntityTypes: inferredIntent?.entityTypes,
|
|
8459
9664
|
memoryKind: memoryKind2,
|
|
8460
|
-
structuredAttributes: fact.structuredAttributes
|
|
9665
|
+
structuredAttributes: fact.structuredAttributes,
|
|
9666
|
+
contentHashSource: rawChunkedContent
|
|
8461
9667
|
}
|
|
8462
9668
|
);
|
|
8463
9669
|
for (const chunk of chunkResult.chunks) {
|
|
@@ -8472,7 +9678,9 @@ _Context: ${topQuestion.context}_`
|
|
|
8472
9678
|
chunk.index,
|
|
8473
9679
|
chunkResult.chunks.length,
|
|
8474
9680
|
writeCategory,
|
|
8475
|
-
chunk
|
|
9681
|
+
// Each chunk carries its own inline citation so provenance
|
|
9682
|
+
// survives when a single chunk is quoted in isolation.
|
|
9683
|
+
applyInlineCitation(chunk.content),
|
|
8476
9684
|
{
|
|
8477
9685
|
confidence: fact.confidence,
|
|
8478
9686
|
tags: fact.tags,
|
|
@@ -8499,6 +9707,19 @@ _Context: ${topQuestion.context}_`
|
|
|
8499
9707
|
threadEpisodeIdsForGraph.push(parentId);
|
|
8500
9708
|
}
|
|
8501
9709
|
await this.indexPersistedMemory(targetStorage, parentId);
|
|
9710
|
+
try {
|
|
9711
|
+
const supersessionEntityRef = typeof fact.entityRef === "string" ? fact.entityRef : void 0;
|
|
9712
|
+
await applyTemporalSupersession({
|
|
9713
|
+
storage: targetStorage,
|
|
9714
|
+
newMemoryId: parentId,
|
|
9715
|
+
entityRef: supersessionEntityRef,
|
|
9716
|
+
structuredAttributes: fact.structuredAttributes,
|
|
9717
|
+
createdAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
9718
|
+
enabled: this.config.temporalSupersessionEnabled
|
|
9719
|
+
});
|
|
9720
|
+
} catch (err) {
|
|
9721
|
+
log.warn(`temporal-supersession (chunked): unexpected error: ${err}`);
|
|
9722
|
+
}
|
|
8502
9723
|
await promoteMemoryToShared({
|
|
8503
9724
|
sourceStorage: targetStorage,
|
|
8504
9725
|
category: writeCategory,
|
|
@@ -8506,6 +9727,7 @@ _Context: ${topQuestion.context}_`
|
|
|
8506
9727
|
confidence: fact.confidence,
|
|
8507
9728
|
tags: fact.tags,
|
|
8508
9729
|
entityRef: fact.entityRef,
|
|
9730
|
+
structuredAttributes: fact.structuredAttributes,
|
|
8509
9731
|
sourceMemoryId: parentId,
|
|
8510
9732
|
importance,
|
|
8511
9733
|
intentGoal: inferredIntent?.goal,
|
|
@@ -8515,14 +9737,15 @@ _Context: ${topQuestion.context}_`
|
|
|
8515
9737
|
source: extractionWriteSource
|
|
8516
9738
|
});
|
|
8517
9739
|
if (this.contentHashIndex) {
|
|
8518
|
-
|
|
9740
|
+
const canonicalChunkedContent = citationEnabled && hasCitationForTemplate(fact.content, citationTemplate) ? stripCitationForTemplate(fact.content, citationTemplate) : fact.content;
|
|
9741
|
+
this.contentHashIndex.add(canonicalChunkedContent);
|
|
8519
9742
|
}
|
|
8520
9743
|
for (const chunk of chunkResult.chunks) {
|
|
8521
9744
|
const chunkId = `${parentId}-chunk-${chunk.index}`;
|
|
8522
9745
|
await this.indexPersistedMemory(targetStorage, chunkId);
|
|
8523
9746
|
}
|
|
8524
9747
|
if (this.config.verbatimArtifactsEnabled && this.config.verbatimArtifactCategories.includes(writeCategory) && fact.confidence >= this.config.verbatimArtifactsMinConfidence) {
|
|
8525
|
-
await targetStorage.writeArtifact(
|
|
9748
|
+
await targetStorage.writeArtifact(citedChunkedContent, {
|
|
8526
9749
|
confidence: fact.confidence,
|
|
8527
9750
|
tags: [...fact.tags, "artifact", "chunked-parent"],
|
|
8528
9751
|
artifactType: this.artifactTypeForCategory(writeCategory),
|
|
@@ -8581,33 +9804,6 @@ _Context: ${topQuestion.context}_`
|
|
|
8581
9804
|
continue;
|
|
8582
9805
|
}
|
|
8583
9806
|
}
|
|
8584
|
-
let supersedes;
|
|
8585
|
-
let links = [];
|
|
8586
|
-
if (this.config.contradictionDetectionEnabled && this.qmd.isAvailable()) {
|
|
8587
|
-
const targetNamespace = this.namespaceFromStorageDir(targetStorage.dir);
|
|
8588
|
-
const contradiction = await this.checkForContradiction(
|
|
8589
|
-
fact.content,
|
|
8590
|
-
writeCategory,
|
|
8591
|
-
targetNamespace
|
|
8592
|
-
);
|
|
8593
|
-
if (contradiction) {
|
|
8594
|
-
supersedes = contradiction.supersededId;
|
|
8595
|
-
links.push({
|
|
8596
|
-
targetId: contradiction.supersededId,
|
|
8597
|
-
linkType: "contradicts",
|
|
8598
|
-
strength: contradiction.confidence,
|
|
8599
|
-
reason: contradiction.reason
|
|
8600
|
-
});
|
|
8601
|
-
if (this.config.queryAwareIndexingEnabled && contradiction.supersededPath) {
|
|
8602
|
-
deindexMemory(
|
|
8603
|
-
this.config.memoryDir,
|
|
8604
|
-
contradiction.supersededPath,
|
|
8605
|
-
contradiction.supersededCreated,
|
|
8606
|
-
contradiction.supersededTags
|
|
8607
|
-
);
|
|
8608
|
-
}
|
|
8609
|
-
}
|
|
8610
|
-
}
|
|
8611
9807
|
if (this.config.memoryLinkingEnabled && this.qmd.isAvailable()) {
|
|
8612
9808
|
const targetNamespace = this.namespaceFromStorageDir(targetStorage.dir);
|
|
8613
9809
|
const suggestedLinks = await this.suggestLinksForMemory(
|
|
@@ -8619,10 +9815,12 @@ _Context: ${topQuestion.context}_`
|
|
|
8619
9815
|
links.push(...suggestedLinks);
|
|
8620
9816
|
}
|
|
8621
9817
|
}
|
|
8622
|
-
const memoryKind = this.config.episodeNoteModeEnabled ? classifyMemoryKind(fact.content, fact.tags ?? [], writeCategory) : void 0;
|
|
9818
|
+
const memoryKind = writeCategory === "procedure" ? void 0 : this.config.episodeNoteModeEnabled ? classifyMemoryKind(fact.content, fact.tags ?? [], writeCategory) : void 0;
|
|
9819
|
+
const rawPersistBody = writeCategory === "procedure" ? buildProcedurePersistBody(fact.content, fact.procedureSteps) : fact.content;
|
|
9820
|
+
const citedFactContent = applyInlineCitation(rawPersistBody);
|
|
8623
9821
|
const memoryId = await targetStorage.writeMemory(
|
|
8624
9822
|
writeCategory,
|
|
8625
|
-
|
|
9823
|
+
citedFactContent,
|
|
8626
9824
|
{
|
|
8627
9825
|
confidence: fact.confidence,
|
|
8628
9826
|
tags: fact.tags,
|
|
@@ -8635,7 +9833,8 @@ _Context: ${topQuestion.context}_`
|
|
|
8635
9833
|
intentActionType: inferredIntent?.actionType,
|
|
8636
9834
|
intentEntityTypes: inferredIntent?.entityTypes,
|
|
8637
9835
|
memoryKind,
|
|
8638
|
-
structuredAttributes: fact.structuredAttributes
|
|
9836
|
+
structuredAttributes: fact.structuredAttributes,
|
|
9837
|
+
contentHashSource: writeCategory === "fact" ? fact.content : void 0
|
|
8639
9838
|
}
|
|
8640
9839
|
);
|
|
8641
9840
|
if (routedRuleId) {
|
|
@@ -8643,6 +9842,19 @@ _Context: ${topQuestion.context}_`
|
|
|
8643
9842
|
`routing applied for memory ${memoryId}: rule=${routedRuleId} category=${writeCategory} storage=${targetStorage.dir}`
|
|
8644
9843
|
);
|
|
8645
9844
|
}
|
|
9845
|
+
try {
|
|
9846
|
+
const supersessionEntityRef = typeof fact.entityRef === "string" ? fact.entityRef : void 0;
|
|
9847
|
+
await applyTemporalSupersession({
|
|
9848
|
+
storage: targetStorage,
|
|
9849
|
+
newMemoryId: memoryId,
|
|
9850
|
+
entityRef: supersessionEntityRef,
|
|
9851
|
+
structuredAttributes: fact.structuredAttributes,
|
|
9852
|
+
createdAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
9853
|
+
enabled: this.config.temporalSupersessionEnabled
|
|
9854
|
+
});
|
|
9855
|
+
} catch (err) {
|
|
9856
|
+
log.warn(`temporal-supersession: unexpected error: ${err}`);
|
|
9857
|
+
}
|
|
8646
9858
|
trackBehaviorSignals(
|
|
8647
9859
|
targetStorage,
|
|
8648
9860
|
buildBehaviorSignalsForMemory({
|
|
@@ -8666,6 +9878,7 @@ _Context: ${topQuestion.context}_`
|
|
|
8666
9878
|
confidence: fact.confidence,
|
|
8667
9879
|
tags: fact.tags,
|
|
8668
9880
|
entityRef: typeof fact.entityRef === "string" ? fact.entityRef : void 0,
|
|
9881
|
+
structuredAttributes: fact.structuredAttributes,
|
|
8669
9882
|
sourceMemoryId: memoryId,
|
|
8670
9883
|
importance,
|
|
8671
9884
|
intentGoal: inferredIntent?.goal,
|
|
@@ -8710,7 +9923,7 @@ _Context: ${topQuestion.context}_`
|
|
|
8710
9923
|
}
|
|
8711
9924
|
}
|
|
8712
9925
|
if (this.config.verbatimArtifactsEnabled && this.config.verbatimArtifactCategories.includes(writeCategory) && fact.confidence >= this.config.verbatimArtifactsMinConfidence) {
|
|
8713
|
-
await targetStorage.writeArtifact(
|
|
9926
|
+
await targetStorage.writeArtifact(citedFactContent, {
|
|
8714
9927
|
confidence: fact.confidence,
|
|
8715
9928
|
tags: [...fact.tags, "artifact"],
|
|
8716
9929
|
artifactType: this.artifactTypeForCategory(writeCategory),
|
|
@@ -8721,7 +9934,8 @@ _Context: ${topQuestion.context}_`
|
|
|
8721
9934
|
});
|
|
8722
9935
|
}
|
|
8723
9936
|
if (this.contentHashIndex) {
|
|
8724
|
-
|
|
9937
|
+
const canonicalFactContent = citationEnabled && hasCitationForTemplate(fact.content, citationTemplate) ? stripCitationForTemplate(fact.content, citationTemplate) : fact.content;
|
|
9938
|
+
this.contentHashIndex.add(canonicalFactContent);
|
|
8725
9939
|
}
|
|
8726
9940
|
}
|
|
8727
9941
|
for (const entity of entities) {
|
|
@@ -8732,7 +9946,12 @@ _Context: ${topQuestion.context}_`
|
|
|
8732
9946
|
continue;
|
|
8733
9947
|
}
|
|
8734
9948
|
const safeFacts = Array.isArray(entity?.facts) ? entity.facts.filter((f) => typeof f === "string") : [];
|
|
8735
|
-
const id = await storage.writeEntity(name, type, safeFacts
|
|
9949
|
+
const id = await storage.writeEntity(name, type, safeFacts, {
|
|
9950
|
+
source: typeof entity?.source === "string" ? entity.source : "extraction",
|
|
9951
|
+
sessionKey: sourceContext?.sessionKey,
|
|
9952
|
+
principal: sourceContext?.principal,
|
|
9953
|
+
structuredSections: Array.isArray(entity?.structuredSections) ? entity.structuredSections : void 0
|
|
9954
|
+
});
|
|
8736
9955
|
if (id) trackPersistedId(storage, id);
|
|
8737
9956
|
} catch (err) {
|
|
8738
9957
|
log.warn(`persistExtraction: entity write failed: ${err}`);
|
|
@@ -8801,8 +10020,10 @@ _Context: ${topQuestion.context}_`
|
|
|
8801
10020
|
);
|
|
8802
10021
|
}
|
|
8803
10022
|
const dedupSuffix = dedupedCount > 0 ? ` (${dedupedCount} deduped)` : "";
|
|
10023
|
+
const gatedSuffix = importanceGatedCount > 0 ? ` (${importanceGatedCount} gated)` : "";
|
|
10024
|
+
const judgeSuffix = judgeGatedCount > 0 ? ` (${judgeGatedCount} judge-rejected)` : "";
|
|
8804
10025
|
log.info(
|
|
8805
|
-
`persisted: ${facts.length - dedupedCount} facts${dedupSuffix}, ${entities.length} entities, ${questions.length} questions, ${profileUpdates.length} profile updates`
|
|
10026
|
+
`persisted: ${facts.length - dedupedCount - importanceGatedCount - judgeGatedCount} facts${dedupSuffix}${gatedSuffix}${judgeSuffix}, ${entities.length} entities, ${questions.length} questions, ${profileUpdates.length} profile updates`
|
|
8806
10027
|
);
|
|
8807
10028
|
void (async () => {
|
|
8808
10029
|
if (persistedIdsByStorage.size === 0) {
|
|
@@ -9023,7 +10244,10 @@ _Context: ${topQuestion.context}_`
|
|
|
9023
10244
|
}
|
|
9024
10245
|
for (const entity of result.entityUpdates) {
|
|
9025
10246
|
const safeFacts = Array.isArray(entity?.facts) ? entity.facts.filter((f) => typeof f === "string") : [];
|
|
9026
|
-
await this.storage.writeEntity(entity.name, entity.type, safeFacts
|
|
10247
|
+
await this.storage.writeEntity(entity.name, entity.type, safeFacts, {
|
|
10248
|
+
source: "consolidation",
|
|
10249
|
+
structuredSections: Array.isArray(entity?.structuredSections) ? entity.structuredSections : void 0
|
|
10250
|
+
});
|
|
9027
10251
|
}
|
|
9028
10252
|
const entitiesMerged = await this.storage.mergeFragmentedEntities();
|
|
9029
10253
|
if (entitiesMerged > 0) {
|
|
@@ -9031,53 +10255,15 @@ _Context: ${topQuestion.context}_`
|
|
|
9031
10255
|
}
|
|
9032
10256
|
if (this.config.entitySummaryEnabled) {
|
|
9033
10257
|
try {
|
|
9034
|
-
const
|
|
9035
|
-
|
|
9036
|
-
|
|
10258
|
+
const synthesized = await this.processEntitySynthesisQueue(
|
|
10259
|
+
this.config.defaultNamespace,
|
|
10260
|
+
5
|
|
9037
10261
|
);
|
|
9038
|
-
|
|
9039
|
-
|
|
9040
|
-
for (const entity of toSummarize) {
|
|
9041
|
-
try {
|
|
9042
|
-
const factsText = entity.facts.slice(0, 10).join("; ");
|
|
9043
|
-
const prompt = `Summarize this entity in one sentence. Entity: ${entity.name} (${entity.type}). Facts: ${factsText}`;
|
|
9044
|
-
const response = await this.fastChatCompletion(
|
|
9045
|
-
[
|
|
9046
|
-
{
|
|
9047
|
-
role: "system",
|
|
9048
|
-
content: "Respond with a single concise sentence summarizing the entity. No JSON, just plain text."
|
|
9049
|
-
},
|
|
9050
|
-
{ role: "user", content: prompt }
|
|
9051
|
-
],
|
|
9052
|
-
{
|
|
9053
|
-
temperature: 0.3,
|
|
9054
|
-
maxTokens: 100,
|
|
9055
|
-
operation: "entity_summary",
|
|
9056
|
-
priority: "background"
|
|
9057
|
-
}
|
|
9058
|
-
);
|
|
9059
|
-
if (response?.content) {
|
|
9060
|
-
const summary = response.content.trim().replace(/^["']|["']$/g, "");
|
|
9061
|
-
if (summary.length > 10 && summary.length < 500) {
|
|
9062
|
-
const entityFileName = normalizeEntityName(
|
|
9063
|
-
entity.name,
|
|
9064
|
-
entity.type
|
|
9065
|
-
);
|
|
9066
|
-
await this.storage.updateEntitySummary(entityFileName, summary);
|
|
9067
|
-
summarized++;
|
|
9068
|
-
}
|
|
9069
|
-
}
|
|
9070
|
-
} catch (err) {
|
|
9071
|
-
log.debug(
|
|
9072
|
-
`entity summary generation failed for ${entity.name}: ${err}`
|
|
9073
|
-
);
|
|
9074
|
-
}
|
|
9075
|
-
}
|
|
9076
|
-
if (summarized > 0) {
|
|
9077
|
-
log.info(`generated ${summarized} entity summaries`);
|
|
10262
|
+
if (synthesized > 0) {
|
|
10263
|
+
log.info(`refreshed ${synthesized} entity syntheses`);
|
|
9078
10264
|
}
|
|
9079
10265
|
} catch (err) {
|
|
9080
|
-
log.debug(`entity
|
|
10266
|
+
log.debug(`entity synthesis pass failed: ${err}`);
|
|
9081
10267
|
}
|
|
9082
10268
|
}
|
|
9083
10269
|
const deletedCommitments = await this.storage.cleanExpiredCommitments(
|
|
@@ -9262,6 +10448,24 @@ ${texts.map((t, i) => `[${i + 1}] ${t}`).join("\n\n")}`;
|
|
|
9262
10448
|
log.warn(`tmt: consolidation hook failed (ignored): ${err}`);
|
|
9263
10449
|
}
|
|
9264
10450
|
}
|
|
10451
|
+
if (this.consolidationObservers.size > 0) {
|
|
10452
|
+
const observation = {
|
|
10453
|
+
runAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
10454
|
+
recentMemories: recent,
|
|
10455
|
+
existingMemories: older.slice(-50),
|
|
10456
|
+
profile,
|
|
10457
|
+
result,
|
|
10458
|
+
merged,
|
|
10459
|
+
invalidated
|
|
10460
|
+
};
|
|
10461
|
+
for (const observer of this.consolidationObservers) {
|
|
10462
|
+
try {
|
|
10463
|
+
await observer(observation);
|
|
10464
|
+
} catch (err) {
|
|
10465
|
+
log.warn(`consolidation observer failed (ignored): ${err}`);
|
|
10466
|
+
}
|
|
10467
|
+
}
|
|
10468
|
+
}
|
|
9265
10469
|
log.info("consolidation complete");
|
|
9266
10470
|
return { memoriesProcessed: allMemories.length, merged, invalidated };
|
|
9267
10471
|
}
|
|
@@ -9666,7 +10870,14 @@ ${texts.map((t, i) => `[${i + 1}] ${t}`).join("\n\n")}`;
|
|
|
9666
10870
|
const result = await this.storage.archiveMemory(memory);
|
|
9667
10871
|
if (result) {
|
|
9668
10872
|
if (this.contentHashIndex) {
|
|
9669
|
-
|
|
10873
|
+
if (memory.frontmatter.contentHash) {
|
|
10874
|
+
this.contentHashIndex.removeByHash(memory.frontmatter.contentHash);
|
|
10875
|
+
} else {
|
|
10876
|
+
log.warn(
|
|
10877
|
+
`[fact-archival] removing hash for legacy memory ${memory.frontmatter.id ?? "(unknown)"} via content fallback \u2014 no contentHash in frontmatter`
|
|
10878
|
+
);
|
|
10879
|
+
this.contentHashIndex.remove(memory.content);
|
|
10880
|
+
}
|
|
9670
10881
|
}
|
|
9671
10882
|
await this.embeddingFallback.removeFromIndex(memory.frontmatter.id);
|
|
9672
10883
|
if (this.config.queryAwareIndexingEnabled && memory.path && memory.frontmatter?.created) {
|
|
@@ -10115,14 +11326,59 @@ ${lines.join("\n\n")}`;
|
|
|
10115
11326
|
});
|
|
10116
11327
|
}
|
|
10117
11328
|
publishRecallResults(options) {
|
|
11329
|
+
const sectionId = "memories";
|
|
10118
11330
|
const memoryIds = this.extractMemoryIdsFromResults(options.results);
|
|
10119
11331
|
this.trackMemoryAccess(memoryIds);
|
|
10120
11332
|
this.appendRecallSection(
|
|
10121
11333
|
options.sectionBuckets,
|
|
10122
|
-
|
|
11334
|
+
sectionId,
|
|
10123
11335
|
this.formatQmdResults(options.title, options.results)
|
|
10124
11336
|
);
|
|
10125
11337
|
}
|
|
11338
|
+
/**
|
|
11339
|
+
* Apply MMR over the pre-truncation recall candidate pool and then slice
|
|
11340
|
+
* the result to `limit`. This is the single place in the pipeline where
|
|
11341
|
+
* MMR runs, and it must be called *before* callers throw away candidates
|
|
11342
|
+
* that would otherwise sit below the final cutoff. Running MMR post-slice
|
|
11343
|
+
* is a no-op in the cases we care about — diverse candidates just below
|
|
11344
|
+
* the cutoff are already gone and can never be promoted.
|
|
11345
|
+
*
|
|
11346
|
+
* Callers must pass the full candidate pool (post-rerank, pre-slice).
|
|
11347
|
+
*/
|
|
11348
|
+
diversifyAndLimitRecallResults(sectionId, results, limit) {
|
|
11349
|
+
const safeLimit = typeof limit === "number" && Number.isFinite(limit) ? Math.max(0, Math.floor(limit)) : 0;
|
|
11350
|
+
if (!Array.isArray(results) || results.length === 0) return [];
|
|
11351
|
+
if (safeLimit === 0) return [];
|
|
11352
|
+
const diversified = this.applyMmrToQmdResults(sectionId, results);
|
|
11353
|
+
return diversified.slice(0, safeLimit);
|
|
11354
|
+
}
|
|
11355
|
+
/**
|
|
11356
|
+
* Apply Maximal Marginal Relevance to a section's ordered candidate list.
|
|
11357
|
+
*
|
|
11358
|
+
* Operates per-section so one redundant cluster cannot dominate a section,
|
|
11359
|
+
* and so one section's MMR pass cannot starve other sections. Returns the
|
|
11360
|
+
* input unchanged when disabled, when there are fewer than 2 candidates, or
|
|
11361
|
+
* when no budget information is available.
|
|
11362
|
+
*/
|
|
11363
|
+
applyMmrToQmdResults(sectionId, results) {
|
|
11364
|
+
if (this.config.recallMmrEnabled === false) return results;
|
|
11365
|
+
if (!Array.isArray(results) || results.length < 2) return results;
|
|
11366
|
+
const configuredTopN = this.config.recallMmrTopN;
|
|
11367
|
+
const topN = typeof configuredTopN === "number" && Number.isFinite(configuredTopN) ? Math.max(0, Math.floor(configuredTopN)) : 40;
|
|
11368
|
+
if (topN === 0) return results;
|
|
11369
|
+
const lambda = this.config.recallMmrLambda ?? 0.7;
|
|
11370
|
+
const { reordered, diversity } = reorderRecallResultsWithMmr(results, {
|
|
11371
|
+
lambda,
|
|
11372
|
+
topN
|
|
11373
|
+
});
|
|
11374
|
+
try {
|
|
11375
|
+
log.info(
|
|
11376
|
+
`recall_mmr: section=${sectionId} kept=${diversity.kept}/${diversity.considered} headReorderCount=${diversity.headReorderCount} avgSimBefore=${diversity.avgPairwiseSimBefore.toFixed(3)} avgSimAfter=${diversity.avgPairwiseSimAfter.toFixed(3)} lambda=${lambda.toFixed(2)}`
|
|
11377
|
+
);
|
|
11378
|
+
} catch {
|
|
11379
|
+
}
|
|
11380
|
+
return reordered;
|
|
11381
|
+
}
|
|
10126
11382
|
buildLastRecallBudgetSummary(options) {
|
|
10127
11383
|
return {
|
|
10128
11384
|
requestedTopK: options.requestedTopK,
|
|
@@ -10149,6 +11405,71 @@ ${lines.join("\n\n")}`;
|
|
|
10149
11405
|
}
|
|
10150
11406
|
return [...used];
|
|
10151
11407
|
}
|
|
11408
|
+
/**
|
|
11409
|
+
* Issue #373 — nearest-neighbor lookup for the write-time semantic dedup
|
|
11410
|
+
* guard. Returns the top-K embedding hits against the currently indexed
|
|
11411
|
+
* memories, or an empty array when the embedding backend is unavailable.
|
|
11412
|
+
* Intentionally does NOT throw; `decideSemanticDedup` treats both "empty"
|
|
11413
|
+
* and "error" outcomes as fail-open (keep the candidate).
|
|
11414
|
+
*
|
|
11415
|
+
* PR #399 P1 fix: when namespaces are enabled the lookup must be scoped
|
|
11416
|
+
* to the SAME namespace as the fact being written. Otherwise a
|
|
11417
|
+
* high-similarity memory from another namespace can suppress a write in
|
|
11418
|
+
* the target namespace — cross-tenant data loss. Callers pass the target
|
|
11419
|
+
* storage so we can translate its root directory into the correct index
|
|
11420
|
+
* path prefix (and, for the legacy default-namespace layout at
|
|
11421
|
+
* `memoryDir` root, an exclusion list for `namespaces/*`).
|
|
11422
|
+
*/
|
|
11423
|
+
async semanticDedupLookup(content, limit, targetStorage) {
|
|
11424
|
+
if (!this.config.embeddingFallbackEnabled) {
|
|
11425
|
+
throw new Error("semantic dedup: embedding backend not configured");
|
|
11426
|
+
}
|
|
11427
|
+
if (!await this.embeddingFallback.isAvailable()) {
|
|
11428
|
+
log.debug("semantic dedup: embedding backend unavailable, skipping");
|
|
11429
|
+
throw new Error("semantic dedup: embedding backend unavailable");
|
|
11430
|
+
}
|
|
11431
|
+
const scope = this.semanticDedupScopeFor(targetStorage);
|
|
11432
|
+
const hits = await this.embeddingFallback.search(content, limit, { ...scope, throwOnTimeout: true });
|
|
11433
|
+
if (!Array.isArray(hits) || hits.length === 0) return [];
|
|
11434
|
+
return hits.map((hit) => ({
|
|
11435
|
+
id: hit.id,
|
|
11436
|
+
score: hit.score,
|
|
11437
|
+
path: hit.path
|
|
11438
|
+
}));
|
|
11439
|
+
}
|
|
11440
|
+
/**
|
|
11441
|
+
* Resolve the namespace-scoped filter to pass into
|
|
11442
|
+
* `EmbeddingFallback.search()` for semantic dedup. Returns an empty
|
|
11443
|
+
* object (no filter) when namespaces are disabled, preserving the
|
|
11444
|
+
* pre-PR #399 behavior for single-tenant installs.
|
|
11445
|
+
*
|
|
11446
|
+
* Index entries are stored as paths relative to `config.memoryDir`, so:
|
|
11447
|
+
* - A non-default namespace `ns` lives under `namespaces/<ns>/…` and
|
|
11448
|
+
* we include exactly that prefix.
|
|
11449
|
+
* - The default namespace may live at `memoryDir` root (legacy) or at
|
|
11450
|
+
* `memoryDir/namespaces/<default>/…` (migrated). When it lives at
|
|
11451
|
+
* root we include everything but EXCLUDE all `namespaces/…` entries
|
|
11452
|
+
* so facts from non-default namespaces can't cross-match.
|
|
11453
|
+
*/
|
|
11454
|
+
semanticDedupScopeFor(targetStorage) {
|
|
11455
|
+
if (!this.config.namespacesEnabled) return {};
|
|
11456
|
+
const memoryDir = path5.resolve(this.config.memoryDir);
|
|
11457
|
+
const storageDir = path5.resolve(targetStorage.dir);
|
|
11458
|
+
if (storageDir === memoryDir) {
|
|
11459
|
+
return { pathExcludePrefixes: ["namespaces/"] };
|
|
11460
|
+
}
|
|
11461
|
+
let rel = path5.relative(memoryDir, storageDir);
|
|
11462
|
+
if (!rel || rel.startsWith("..")) {
|
|
11463
|
+
log.debug(
|
|
11464
|
+
`semantic dedup: target storage dir ${storageDir} is outside memoryDir ${memoryDir}; scoping lookup to absolute path prefix`
|
|
11465
|
+
);
|
|
11466
|
+
const absPrefix = storageDir.replace(/\\/g, "/");
|
|
11467
|
+
return { pathPrefix: absPrefix.endsWith("/") ? absPrefix : `${absPrefix}/` };
|
|
11468
|
+
}
|
|
11469
|
+
rel = rel.replace(/\\/g, "/");
|
|
11470
|
+
if (!rel.endsWith("/")) rel = `${rel}/`;
|
|
11471
|
+
return { pathPrefix: rel };
|
|
11472
|
+
}
|
|
10152
11473
|
async searchEmbeddingFallback(query, limit) {
|
|
10153
11474
|
if (!this.config.embeddingFallbackEnabled) return [];
|
|
10154
11475
|
if (!await this.embeddingFallback.isAvailable()) return [];
|
|
@@ -10333,7 +11654,11 @@ ${lines.join("\n\n")}`;
|
|
|
10333
11654
|
"rerankProvider=cloud is reserved/experimental in v2.2.0; skipping rerank"
|
|
10334
11655
|
);
|
|
10335
11656
|
}
|
|
10336
|
-
return
|
|
11657
|
+
return this.diversifyAndLimitRecallResults(
|
|
11658
|
+
"memories",
|
|
11659
|
+
results,
|
|
11660
|
+
options.recallResultLimit
|
|
11661
|
+
);
|
|
10337
11662
|
}
|
|
10338
11663
|
// ---------------------------------------------------------------------------
|
|
10339
11664
|
// Access Tracking (Phase 1A)
|
|
@@ -10449,6 +11774,8 @@ ${lines.join("\n\n")}`;
|
|
|
10449
11774
|
}
|
|
10450
11775
|
}
|
|
10451
11776
|
let lifecycleFilteredCount = 0;
|
|
11777
|
+
let temporalSupersededFilteredCount = 0;
|
|
11778
|
+
let dedicatedSurfaceFilteredCount = 0;
|
|
10452
11779
|
const boosted = [];
|
|
10453
11780
|
const recencyWeight = this.effectiveRecencyWeight();
|
|
10454
11781
|
for (const r of results) {
|
|
@@ -10462,6 +11789,17 @@ ${lines.join("\n\n")}`;
|
|
|
10462
11789
|
lifecycleFilteredCount += 1;
|
|
10463
11790
|
continue;
|
|
10464
11791
|
}
|
|
11792
|
+
if (shouldFilterSupersededFromRecall(memory.frontmatter, {
|
|
11793
|
+
enabled: this.config.temporalSupersessionEnabled,
|
|
11794
|
+
includeInRecall: this.config.temporalSupersessionIncludeInRecall
|
|
11795
|
+
})) {
|
|
11796
|
+
temporalSupersededFilteredCount += 1;
|
|
11797
|
+
continue;
|
|
11798
|
+
}
|
|
11799
|
+
if (options?.allowDedicatedSurface !== true && (memory.frontmatter.memoryKind === "dream" || memory.frontmatter.memoryKind === "procedural")) {
|
|
11800
|
+
dedicatedSurfaceFilteredCount += 1;
|
|
11801
|
+
continue;
|
|
11802
|
+
}
|
|
10465
11803
|
if (recencyWeight > 0) {
|
|
10466
11804
|
const createdAt = new Date(memory.frontmatter.created).getTime();
|
|
10467
11805
|
const ageMs = now - createdAt;
|
|
@@ -10561,6 +11899,16 @@ ${lines.join("\n\n")}`;
|
|
|
10561
11899
|
`lifecycle retrieval filter removed ${lifecycleFilteredCount} stale/archived candidates`
|
|
10562
11900
|
);
|
|
10563
11901
|
}
|
|
11902
|
+
if (temporalSupersededFilteredCount > 0) {
|
|
11903
|
+
log.debug(
|
|
11904
|
+
`temporal supersession filter removed ${temporalSupersededFilteredCount} superseded candidates`
|
|
11905
|
+
);
|
|
11906
|
+
}
|
|
11907
|
+
if (dedicatedSurfaceFilteredCount > 0) {
|
|
11908
|
+
log.debug(
|
|
11909
|
+
`dedicated surface filter removed ${dedicatedSurfaceFilteredCount} dream/procedural candidates from generic recall`
|
|
11910
|
+
);
|
|
11911
|
+
}
|
|
10564
11912
|
return boosted.sort((a, b) => b.score - a.score);
|
|
10565
11913
|
}
|
|
10566
11914
|
/**
|
|
@@ -10623,27 +11971,31 @@ ${lines.join("\n\n")}`;
|
|
|
10623
11971
|
);
|
|
10624
11972
|
if (!verification) continue;
|
|
10625
11973
|
if (verification.isContradiction && verification.confidence >= this.config.contradictionMinConfidence) {
|
|
11974
|
+
if (verification.whichIsNewer === "first") {
|
|
11975
|
+
log.info(
|
|
11976
|
+
`detected contradiction (confidence: ${verification.confidence}): ${existingMemory.frontmatter.id} vs new memory \u2014 existing is newer, incoming fact is stale`
|
|
11977
|
+
);
|
|
11978
|
+
continue;
|
|
11979
|
+
}
|
|
10626
11980
|
if (this.config.contradictionAutoResolve) {
|
|
10627
|
-
|
|
10628
|
-
|
|
10629
|
-
|
|
10630
|
-
|
|
10631
|
-
|
|
10632
|
-
|
|
10633
|
-
);
|
|
10634
|
-
return {
|
|
10635
|
-
supersededId: existingMemory.frontmatter.id,
|
|
10636
|
-
confidence: verification.confidence,
|
|
10637
|
-
reason: verification.reasoning,
|
|
10638
|
-
supersededPath: existingMemory.path,
|
|
10639
|
-
supersededCreated: existingMemory.frontmatter.created,
|
|
10640
|
-
supersededTags: existingMemory.frontmatter.tags ?? []
|
|
10641
|
-
};
|
|
10642
|
-
}
|
|
11981
|
+
await resultStorage.supersedeMemory(
|
|
11982
|
+
existingMemory.frontmatter.id,
|
|
11983
|
+
"pending-new",
|
|
11984
|
+
// Will be updated after the new memory is written
|
|
11985
|
+
verification.reasoning
|
|
11986
|
+
);
|
|
10643
11987
|
}
|
|
10644
11988
|
log.info(
|
|
10645
|
-
`detected contradiction (confidence: ${verification.confidence}): ${existingMemory.frontmatter.id} vs new memory`
|
|
11989
|
+
`detected contradiction (confidence: ${verification.confidence}): ${existingMemory.frontmatter.id} vs new memory${this.config.contradictionAutoResolve ? " (auto-resolved)" : " (queued for manual review)"}`
|
|
10646
11990
|
);
|
|
11991
|
+
return {
|
|
11992
|
+
supersededId: existingMemory.frontmatter.id,
|
|
11993
|
+
confidence: verification.confidence,
|
|
11994
|
+
reason: verification.reasoning,
|
|
11995
|
+
supersededPath: existingMemory.path,
|
|
11996
|
+
supersededCreated: existingMemory.frontmatter.created,
|
|
11997
|
+
supersededTags: existingMemory.frontmatter.tags ?? []
|
|
11998
|
+
};
|
|
10647
11999
|
}
|
|
10648
12000
|
}
|
|
10649
12001
|
return null;
|
|
@@ -10731,6 +12083,9 @@ ${lines.join("\n\n")}`;
|
|
|
10731
12083
|
export {
|
|
10732
12084
|
rollbackFromEngramMigration,
|
|
10733
12085
|
migrateFromEngram,
|
|
12086
|
+
buildProcedureRecallSection,
|
|
12087
|
+
decideSemanticDedup,
|
|
12088
|
+
dedupeEntitySynthesisEvidenceEntries,
|
|
10734
12089
|
defaultWorkspaceDir,
|
|
10735
12090
|
sanitizeSessionKeyForFilename,
|
|
10736
12091
|
isArtifactMemoryPath,
|
|
@@ -10758,4 +12113,4 @@ export {
|
|
|
10758
12113
|
resolvePersistedMemoryRelativePath,
|
|
10759
12114
|
Orchestrator
|
|
10760
12115
|
};
|
|
10761
|
-
//# sourceMappingURL=chunk-
|
|
12116
|
+
//# sourceMappingURL=chunk-XMGSSBFX.js.map
|