@fgv/ts-agent-memory 5.1.0-47 → 5.1.0-49
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/packlets/ingest/orchestrator.js +11 -1
- package/dist/packlets/ingest/orchestrator.js.map +1 -1
- package/dist/packlets/observe/memoryObservationStore.js +6 -0
- package/dist/packlets/observe/memoryObservationStore.js.map +1 -1
- package/dist/packlets/observe/observer.js.map +1 -1
- package/dist/packlets/retrieve/linkTraversalRetriever.js +1 -1
- package/dist/packlets/retrieve/linkTraversalRetriever.js.map +1 -1
- package/dist/packlets/retrieve/recencyRetriever.js +1 -1
- package/dist/packlets/retrieve/recencyRetriever.js.map +1 -1
- package/dist/packlets/retrieve/retriever.js +11 -5
- package/dist/packlets/retrieve/retriever.js.map +1 -1
- package/dist/packlets/retrieve/structuredFilterRetriever.js +27 -5
- package/dist/packlets/retrieve/structuredFilterRetriever.js.map +1 -1
- package/dist/packlets/retrieve/tagRetriever.js +1 -1
- package/dist/packlets/retrieve/tagRetriever.js.map +1 -1
- package/dist/packlets/retrieve/temporalRetrievers.js +1 -1
- package/dist/packlets/retrieve/temporalRetrievers.js.map +1 -1
- package/dist/packlets/store/fileTreeMemoryStore.js +152 -132
- package/dist/packlets/store/fileTreeMemoryStore.js.map +1 -1
- package/dist/packlets/store/vectorMaintenance.js +251 -0
- package/dist/packlets/store/vectorMaintenance.js.map +1 -0
- package/dist/packlets/types/envelope.js.map +1 -1
- package/dist/packlets/vector/inMemoryCosineIndex.js +81 -18
- package/dist/packlets/vector/inMemoryCosineIndex.js.map +1 -1
- package/dist/packlets/vector/inMemoryFragmentCosineIndex.js +7 -0
- package/dist/packlets/vector/inMemoryFragmentCosineIndex.js.map +1 -1
- package/dist/packlets/vector/vectorIndex.js.map +1 -1
- package/dist/ts-agent-memory.d.ts +431 -79
- package/lib/packlets/ingest/orchestrator.d.ts.map +1 -1
- package/lib/packlets/ingest/orchestrator.js +11 -1
- package/lib/packlets/ingest/orchestrator.js.map +1 -1
- package/lib/packlets/observe/memoryObservationStore.d.ts.map +1 -1
- package/lib/packlets/observe/memoryObservationStore.js +6 -0
- package/lib/packlets/observe/memoryObservationStore.js.map +1 -1
- package/lib/packlets/observe/observer.d.ts +61 -0
- package/lib/packlets/observe/observer.d.ts.map +1 -1
- package/lib/packlets/observe/observer.js.map +1 -1
- package/lib/packlets/retrieve/linkTraversalRetriever.d.ts +1 -1
- package/lib/packlets/retrieve/linkTraversalRetriever.js +1 -1
- package/lib/packlets/retrieve/linkTraversalRetriever.js.map +1 -1
- package/lib/packlets/retrieve/recencyRetriever.d.ts +1 -1
- package/lib/packlets/retrieve/recencyRetriever.js +1 -1
- package/lib/packlets/retrieve/recencyRetriever.js.map +1 -1
- package/lib/packlets/retrieve/retriever.d.ts +26 -7
- package/lib/packlets/retrieve/retriever.d.ts.map +1 -1
- package/lib/packlets/retrieve/retriever.js +11 -5
- package/lib/packlets/retrieve/retriever.js.map +1 -1
- package/lib/packlets/retrieve/structuredFilterRetriever.d.ts +26 -4
- package/lib/packlets/retrieve/structuredFilterRetriever.d.ts.map +1 -1
- package/lib/packlets/retrieve/structuredFilterRetriever.js +27 -5
- package/lib/packlets/retrieve/structuredFilterRetriever.js.map +1 -1
- package/lib/packlets/retrieve/tagRetriever.d.ts +1 -1
- package/lib/packlets/retrieve/tagRetriever.js +1 -1
- package/lib/packlets/retrieve/tagRetriever.js.map +1 -1
- package/lib/packlets/retrieve/temporalRetrievers.js +1 -1
- package/lib/packlets/retrieve/temporalRetrievers.js.map +1 -1
- package/lib/packlets/store/fileTreeMemoryStore.d.ts +135 -59
- package/lib/packlets/store/fileTreeMemoryStore.d.ts.map +1 -1
- package/lib/packlets/store/fileTreeMemoryStore.js +151 -131
- package/lib/packlets/store/fileTreeMemoryStore.js.map +1 -1
- package/lib/packlets/store/vectorMaintenance.d.ts +149 -0
- package/lib/packlets/store/vectorMaintenance.d.ts.map +1 -0
- package/lib/packlets/store/vectorMaintenance.js +255 -0
- package/lib/packlets/store/vectorMaintenance.js.map +1 -0
- package/lib/packlets/types/envelope.d.ts +11 -0
- package/lib/packlets/types/envelope.d.ts.map +1 -1
- package/lib/packlets/types/envelope.js.map +1 -1
- package/lib/packlets/vector/inMemoryCosineIndex.d.ts +30 -7
- package/lib/packlets/vector/inMemoryCosineIndex.d.ts.map +1 -1
- package/lib/packlets/vector/inMemoryCosineIndex.js +80 -17
- package/lib/packlets/vector/inMemoryCosineIndex.js.map +1 -1
- package/lib/packlets/vector/inMemoryFragmentCosineIndex.d.ts +7 -0
- package/lib/packlets/vector/inMemoryFragmentCosineIndex.d.ts.map +1 -1
- package/lib/packlets/vector/inMemoryFragmentCosineIndex.js +7 -0
- package/lib/packlets/vector/inMemoryFragmentCosineIndex.js.map +1 -1
- package/lib/packlets/vector/vectorIndex.d.ts +129 -1
- package/lib/packlets/vector/vectorIndex.d.ts.map +1 -1
- package/lib/packlets/vector/vectorIndex.js.map +1 -1
- package/package.json +17 -7
- package/.rush/temp/5667e36f1c0cc503a5c20a3ebff76e7b9a702d41.tar.log +0 -468
- package/.rush/temp/chunked-rush-logs/ts-agent-memory.build.chunks.jsonl +0 -9
- package/.rush/temp/operation/build/all.log +0 -9
- package/.rush/temp/operation/build/log-chunks.jsonl +0 -9
- package/.rush/temp/operation/build/state.json +0 -3
- package/.rush/temp/shrinkwrap-deps.json +0 -689
- package/config/api-extractor.json +0 -343
- package/config/jest.config.json +0 -14
- package/config/rig.json +0 -4
- package/dist/test/unit/converters/antagonistRoundTrip.test.js +0 -95
- package/dist/test/unit/converters/antagonistRoundTrip.test.js.map +0 -1
- package/dist/test/unit/converters/bodyConverterRegistry.test.js +0 -72
- package/dist/test/unit/converters/bodyConverterRegistry.test.js.map +0 -1
- package/dist/test/unit/converters/envelopeConverter.test.js +0 -313
- package/dist/test/unit/converters/envelopeConverter.test.js.map +0 -1
- package/dist/test/unit/index/memoryIndex.test.js +0 -214
- package/dist/test/unit/index/memoryIndex.test.js.map +0 -1
- package/dist/test/unit/ingest/antagonistCycleAndParity.test.js +0 -277
- package/dist/test/unit/ingest/antagonistCycleAndParity.test.js.map +0 -1
- package/dist/test/unit/ingest/cycleGuard.test.js +0 -81
- package/dist/test/unit/ingest/cycleGuard.test.js.map +0 -1
- package/dist/test/unit/ingest/dedupScope.test.js +0 -434
- package/dist/test/unit/ingest/dedupScope.test.js.map +0 -1
- package/dist/test/unit/ingest/orchestrator.test.js +0 -1058
- package/dist/test/unit/ingest/orchestrator.test.js.map +0 -1
- package/dist/test/unit/observe/memoryObservationStore.test.js +0 -118
- package/dist/test/unit/observe/memoryObservationStore.test.js.map +0 -1
- package/dist/test/unit/retrieve/fragmentSemanticRetriever.test.js +0 -116
- package/dist/test/unit/retrieve/fragmentSemanticRetriever.test.js.map +0 -1
- package/dist/test/unit/retrieve/linkTraversalRetriever.test.js +0 -257
- package/dist/test/unit/retrieve/linkTraversalRetriever.test.js.map +0 -1
- package/dist/test/unit/retrieve/retrievers.test.js +0 -768
- package/dist/test/unit/retrieve/retrievers.test.js.map +0 -1
- package/dist/test/unit/retrieve/temporalRetrievers.test.js +0 -182
- package/dist/test/unit/retrieve/temporalRetrievers.test.js.map +0 -1
- package/dist/test/unit/store/antagonistTemporalBoundary.test.js +0 -120
- package/dist/test/unit/store/antagonistTemporalBoundary.test.js.map +0 -1
- package/dist/test/unit/store/embedOnWrite.test.js +0 -317
- package/dist/test/unit/store/embedOnWrite.test.js.map +0 -1
- package/dist/test/unit/store/fileTreeMemoryStore.test.js +0 -778
- package/dist/test/unit/store/fileTreeMemoryStore.test.js.map +0 -1
- package/dist/test/unit/store/fragmentEmbedOnWrite.test.js +0 -255
- package/dist/test/unit/store/fragmentEmbedOnWrite.test.js.map +0 -1
- package/dist/test/unit/store/indexInjection.test.js +0 -432
- package/dist/test/unit/store/indexInjection.test.js.map +0 -1
- package/dist/test/unit/store/lenientOpen.test.js +0 -248
- package/dist/test/unit/store/lenientOpen.test.js.map +0 -1
- package/dist/test/unit/store/listScoped.test.js +0 -109
- package/dist/test/unit/store/listScoped.test.js.map +0 -1
- package/dist/test/unit/store/observations.test.js +0 -239
- package/dist/test/unit/store/observations.test.js.map +0 -1
- package/dist/test/unit/store/rankAxis.test.js +0 -254
- package/dist/test/unit/store/rankAxis.test.js.map +0 -1
- package/dist/test/unit/store/scopeEncoding.test.js +0 -24
- package/dist/test/unit/store/scopeEncoding.test.js.map +0 -1
- package/dist/test/unit/store/temporalStore.test.js +0 -398
- package/dist/test/unit/store/temporalStore.test.js.map +0 -1
- package/dist/test/unit/tools/memoryTools.test.js +0 -842
- package/dist/test/unit/tools/memoryTools.test.js.map +0 -1
- package/dist/test/unit/types/identityCodec.test.js +0 -187
- package/dist/test/unit/types/identityCodec.test.js.map +0 -1
- package/dist/test/unit/types/ids.test.js +0 -84
- package/dist/test/unit/types/ids.test.js.map +0 -1
- package/dist/test/unit/types/temporalCodec.test.js +0 -203
- package/dist/test/unit/types/temporalCodec.test.js.map +0 -1
- package/dist/test/unit/types/temporalPolicy.test.js +0 -83
- package/dist/test/unit/types/temporalPolicy.test.js.map +0 -1
- package/dist/test/unit/types/writePolicy.test.js +0 -331
- package/dist/test/unit/types/writePolicy.test.js.map +0 -1
- package/dist/test/unit/vector/fragmentConverters.test.js +0 -58
- package/dist/test/unit/vector/fragmentConverters.test.js.map +0 -1
- package/dist/test/unit/vector/inMemoryCosineIndex.test.js +0 -252
- package/dist/test/unit/vector/inMemoryCosineIndex.test.js.map +0 -1
- package/dist/test/unit/vector/inMemoryFragmentCosineIndex.test.js +0 -362
- package/dist/test/unit/vector/inMemoryFragmentCosineIndex.test.js.map +0 -1
- package/dist/test/unit/vector/vectorIndex.test.js +0 -51
- package/dist/test/unit/vector/vectorIndex.test.js.map +0 -1
- package/eslint.config.js +0 -15
- package/etc/ts-agent-memory.api.md +0 -940
- package/lib/test/unit/converters/antagonistRoundTrip.test.d.ts +0 -10
- package/lib/test/unit/converters/antagonistRoundTrip.test.d.ts.map +0 -1
- package/lib/test/unit/converters/antagonistRoundTrip.test.js +0 -97
- package/lib/test/unit/converters/antagonistRoundTrip.test.js.map +0 -1
- package/lib/test/unit/converters/bodyConverterRegistry.test.d.ts +0 -2
- package/lib/test/unit/converters/bodyConverterRegistry.test.d.ts.map +0 -1
- package/lib/test/unit/converters/bodyConverterRegistry.test.js +0 -74
- package/lib/test/unit/converters/bodyConverterRegistry.test.js.map +0 -1
- package/lib/test/unit/converters/envelopeConverter.test.d.ts +0 -2
- package/lib/test/unit/converters/envelopeConverter.test.d.ts.map +0 -1
- package/lib/test/unit/converters/envelopeConverter.test.js +0 -315
- package/lib/test/unit/converters/envelopeConverter.test.js.map +0 -1
- package/lib/test/unit/index/memoryIndex.test.d.ts +0 -2
- package/lib/test/unit/index/memoryIndex.test.d.ts.map +0 -1
- package/lib/test/unit/index/memoryIndex.test.js +0 -216
- package/lib/test/unit/index/memoryIndex.test.js.map +0 -1
- package/lib/test/unit/ingest/antagonistCycleAndParity.test.d.ts +0 -9
- package/lib/test/unit/ingest/antagonistCycleAndParity.test.d.ts.map +0 -1
- package/lib/test/unit/ingest/antagonistCycleAndParity.test.js +0 -279
- package/lib/test/unit/ingest/antagonistCycleAndParity.test.js.map +0 -1
- package/lib/test/unit/ingest/cycleGuard.test.d.ts +0 -2
- package/lib/test/unit/ingest/cycleGuard.test.d.ts.map +0 -1
- package/lib/test/unit/ingest/cycleGuard.test.js +0 -83
- package/lib/test/unit/ingest/cycleGuard.test.js.map +0 -1
- package/lib/test/unit/ingest/dedupScope.test.d.ts +0 -2
- package/lib/test/unit/ingest/dedupScope.test.d.ts.map +0 -1
- package/lib/test/unit/ingest/dedupScope.test.js +0 -436
- package/lib/test/unit/ingest/dedupScope.test.js.map +0 -1
- package/lib/test/unit/ingest/orchestrator.test.d.ts +0 -2
- package/lib/test/unit/ingest/orchestrator.test.d.ts.map +0 -1
- package/lib/test/unit/ingest/orchestrator.test.js +0 -1060
- package/lib/test/unit/ingest/orchestrator.test.js.map +0 -1
- package/lib/test/unit/observe/memoryObservationStore.test.d.ts +0 -2
- package/lib/test/unit/observe/memoryObservationStore.test.d.ts.map +0 -1
- package/lib/test/unit/observe/memoryObservationStore.test.js +0 -120
- package/lib/test/unit/observe/memoryObservationStore.test.js.map +0 -1
- package/lib/test/unit/retrieve/fragmentSemanticRetriever.test.d.ts +0 -2
- package/lib/test/unit/retrieve/fragmentSemanticRetriever.test.d.ts.map +0 -1
- package/lib/test/unit/retrieve/fragmentSemanticRetriever.test.js +0 -118
- package/lib/test/unit/retrieve/fragmentSemanticRetriever.test.js.map +0 -1
- package/lib/test/unit/retrieve/linkTraversalRetriever.test.d.ts +0 -2
- package/lib/test/unit/retrieve/linkTraversalRetriever.test.d.ts.map +0 -1
- package/lib/test/unit/retrieve/linkTraversalRetriever.test.js +0 -259
- package/lib/test/unit/retrieve/linkTraversalRetriever.test.js.map +0 -1
- package/lib/test/unit/retrieve/retrievers.test.d.ts +0 -2
- package/lib/test/unit/retrieve/retrievers.test.d.ts.map +0 -1
- package/lib/test/unit/retrieve/retrievers.test.js +0 -770
- package/lib/test/unit/retrieve/retrievers.test.js.map +0 -1
- package/lib/test/unit/retrieve/temporalRetrievers.test.d.ts +0 -2
- package/lib/test/unit/retrieve/temporalRetrievers.test.d.ts.map +0 -1
- package/lib/test/unit/retrieve/temporalRetrievers.test.js +0 -184
- package/lib/test/unit/retrieve/temporalRetrievers.test.js.map +0 -1
- package/lib/test/unit/store/antagonistTemporalBoundary.test.d.ts +0 -9
- package/lib/test/unit/store/antagonistTemporalBoundary.test.d.ts.map +0 -1
- package/lib/test/unit/store/antagonistTemporalBoundary.test.js +0 -122
- package/lib/test/unit/store/antagonistTemporalBoundary.test.js.map +0 -1
- package/lib/test/unit/store/embedOnWrite.test.d.ts +0 -2
- package/lib/test/unit/store/embedOnWrite.test.d.ts.map +0 -1
- package/lib/test/unit/store/embedOnWrite.test.js +0 -319
- package/lib/test/unit/store/embedOnWrite.test.js.map +0 -1
- package/lib/test/unit/store/fileTreeMemoryStore.test.d.ts +0 -2
- package/lib/test/unit/store/fileTreeMemoryStore.test.d.ts.map +0 -1
- package/lib/test/unit/store/fileTreeMemoryStore.test.js +0 -780
- package/lib/test/unit/store/fileTreeMemoryStore.test.js.map +0 -1
- package/lib/test/unit/store/fragmentEmbedOnWrite.test.d.ts +0 -2
- package/lib/test/unit/store/fragmentEmbedOnWrite.test.d.ts.map +0 -1
- package/lib/test/unit/store/fragmentEmbedOnWrite.test.js +0 -257
- package/lib/test/unit/store/fragmentEmbedOnWrite.test.js.map +0 -1
- package/lib/test/unit/store/indexInjection.test.d.ts +0 -2
- package/lib/test/unit/store/indexInjection.test.d.ts.map +0 -1
- package/lib/test/unit/store/indexInjection.test.js +0 -434
- package/lib/test/unit/store/indexInjection.test.js.map +0 -1
- package/lib/test/unit/store/lenientOpen.test.d.ts +0 -2
- package/lib/test/unit/store/lenientOpen.test.d.ts.map +0 -1
- package/lib/test/unit/store/lenientOpen.test.js +0 -250
- package/lib/test/unit/store/lenientOpen.test.js.map +0 -1
- package/lib/test/unit/store/listScoped.test.d.ts +0 -2
- package/lib/test/unit/store/listScoped.test.d.ts.map +0 -1
- package/lib/test/unit/store/listScoped.test.js +0 -111
- package/lib/test/unit/store/listScoped.test.js.map +0 -1
- package/lib/test/unit/store/observations.test.d.ts +0 -2
- package/lib/test/unit/store/observations.test.d.ts.map +0 -1
- package/lib/test/unit/store/observations.test.js +0 -241
- package/lib/test/unit/store/observations.test.js.map +0 -1
- package/lib/test/unit/store/rankAxis.test.d.ts +0 -2
- package/lib/test/unit/store/rankAxis.test.d.ts.map +0 -1
- package/lib/test/unit/store/rankAxis.test.js +0 -256
- package/lib/test/unit/store/rankAxis.test.js.map +0 -1
- package/lib/test/unit/store/scopeEncoding.test.d.ts +0 -2
- package/lib/test/unit/store/scopeEncoding.test.d.ts.map +0 -1
- package/lib/test/unit/store/scopeEncoding.test.js +0 -26
- package/lib/test/unit/store/scopeEncoding.test.js.map +0 -1
- package/lib/test/unit/store/temporalStore.test.d.ts +0 -2
- package/lib/test/unit/store/temporalStore.test.d.ts.map +0 -1
- package/lib/test/unit/store/temporalStore.test.js +0 -400
- package/lib/test/unit/store/temporalStore.test.js.map +0 -1
- package/lib/test/unit/tools/memoryTools.test.d.ts +0 -2
- package/lib/test/unit/tools/memoryTools.test.d.ts.map +0 -1
- package/lib/test/unit/tools/memoryTools.test.js +0 -844
- package/lib/test/unit/tools/memoryTools.test.js.map +0 -1
- package/lib/test/unit/types/identityCodec.test.d.ts +0 -2
- package/lib/test/unit/types/identityCodec.test.d.ts.map +0 -1
- package/lib/test/unit/types/identityCodec.test.js +0 -189
- package/lib/test/unit/types/identityCodec.test.js.map +0 -1
- package/lib/test/unit/types/ids.test.d.ts +0 -2
- package/lib/test/unit/types/ids.test.d.ts.map +0 -1
- package/lib/test/unit/types/ids.test.js +0 -86
- package/lib/test/unit/types/ids.test.js.map +0 -1
- package/lib/test/unit/types/temporalCodec.test.d.ts +0 -2
- package/lib/test/unit/types/temporalCodec.test.d.ts.map +0 -1
- package/lib/test/unit/types/temporalCodec.test.js +0 -205
- package/lib/test/unit/types/temporalCodec.test.js.map +0 -1
- package/lib/test/unit/types/temporalPolicy.test.d.ts +0 -2
- package/lib/test/unit/types/temporalPolicy.test.d.ts.map +0 -1
- package/lib/test/unit/types/temporalPolicy.test.js +0 -85
- package/lib/test/unit/types/temporalPolicy.test.js.map +0 -1
- package/lib/test/unit/types/writePolicy.test.d.ts +0 -2
- package/lib/test/unit/types/writePolicy.test.d.ts.map +0 -1
- package/lib/test/unit/types/writePolicy.test.js +0 -333
- package/lib/test/unit/types/writePolicy.test.js.map +0 -1
- package/lib/test/unit/vector/fragmentConverters.test.d.ts +0 -2
- package/lib/test/unit/vector/fragmentConverters.test.d.ts.map +0 -1
- package/lib/test/unit/vector/fragmentConverters.test.js +0 -60
- package/lib/test/unit/vector/fragmentConverters.test.js.map +0 -1
- package/lib/test/unit/vector/inMemoryCosineIndex.test.d.ts +0 -2
- package/lib/test/unit/vector/inMemoryCosineIndex.test.d.ts.map +0 -1
- package/lib/test/unit/vector/inMemoryCosineIndex.test.js +0 -254
- package/lib/test/unit/vector/inMemoryCosineIndex.test.js.map +0 -1
- package/lib/test/unit/vector/inMemoryFragmentCosineIndex.test.d.ts +0 -2
- package/lib/test/unit/vector/inMemoryFragmentCosineIndex.test.d.ts.map +0 -1
- package/lib/test/unit/vector/inMemoryFragmentCosineIndex.test.js +0 -364
- package/lib/test/unit/vector/inMemoryFragmentCosineIndex.test.js.map +0 -1
- package/lib/test/unit/vector/vectorIndex.test.d.ts +0 -2
- package/lib/test/unit/vector/vectorIndex.test.d.ts.map +0 -1
- package/lib/test/unit/vector/vectorIndex.test.js +0 -53
- package/lib/test/unit/vector/vectorIndex.test.js.map +0 -1
- package/rush-logs/ts-agent-memory.build.cache.log +0 -3
- package/rush-logs/ts-agent-memory.build.log +0 -9
- package/src/index.ts +0 -14
- package/src/packlets/converters/bodyConverterRegistry.ts +0 -105
- package/src/packlets/converters/envelopeConverter.ts +0 -233
- package/src/packlets/converters/index.ts +0 -7
- package/src/packlets/index/index.ts +0 -6
- package/src/packlets/index/memoryIndex.ts +0 -332
- package/src/packlets/ingest/cycleGuard.ts +0 -153
- package/src/packlets/ingest/hostStages.ts +0 -111
- package/src/packlets/ingest/index.ts +0 -9
- package/src/packlets/ingest/model.ts +0 -195
- package/src/packlets/ingest/orchestrator.ts +0 -968
- package/src/packlets/observe/index.ts +0 -7
- package/src/packlets/observe/memoryObservationStore.ts +0 -153
- package/src/packlets/observe/observer.ts +0 -119
- package/src/packlets/retrieve/fragmentSemanticRetriever.ts +0 -138
- package/src/packlets/retrieve/hybridRetriever.ts +0 -194
- package/src/packlets/retrieve/index.ts +0 -14
- package/src/packlets/retrieve/linkTraversalRetriever.ts +0 -158
- package/src/packlets/retrieve/recencyRetriever.ts +0 -54
- package/src/packlets/retrieve/retriever.ts +0 -291
- package/src/packlets/retrieve/semanticRetriever.ts +0 -152
- package/src/packlets/retrieve/structuredFilterRetriever.ts +0 -58
- package/src/packlets/retrieve/tagRetriever.ts +0 -58
- package/src/packlets/retrieve/temporalRetrievers.ts +0 -210
- package/src/packlets/store/fileTreeMemoryStore.ts +0 -1872
- package/src/packlets/store/index.ts +0 -7
- package/src/packlets/store/scopeEncoding.ts +0 -36
- package/src/packlets/tools/index.ts +0 -6
- package/src/packlets/tools/memoryTools.ts +0 -706
- package/src/packlets/types/envelope.ts +0 -196
- package/src/packlets/types/filenameSafety.ts +0 -57
- package/src/packlets/types/identityCodec.ts +0 -447
- package/src/packlets/types/ids.ts +0 -124
- package/src/packlets/types/index.ts +0 -11
- package/src/packlets/types/temporal.ts +0 -96
- package/src/packlets/types/writePolicy.ts +0 -612
- package/src/packlets/vector/fragmentConverters.ts +0 -55
- package/src/packlets/vector/inMemoryCosineIndex.ts +0 -196
- package/src/packlets/vector/inMemoryFragmentCosineIndex.ts +0 -289
- package/src/packlets/vector/index.ts +0 -9
- package/src/packlets/vector/vectorIndex.ts +0 -273
- package/src/test/unit/converters/antagonistRoundTrip.test.ts +0 -110
- package/src/test/unit/converters/bodyConverterRegistry.test.ts +0 -89
- package/src/test/unit/converters/envelopeConverter.test.ts +0 -418
- package/src/test/unit/index/memoryIndex.test.ts +0 -272
- package/src/test/unit/ingest/antagonistCycleAndParity.test.ts +0 -367
- package/src/test/unit/ingest/cycleGuard.test.ts +0 -110
- package/src/test/unit/ingest/dedupScope.test.ts +0 -542
- package/src/test/unit/ingest/orchestrator.test.ts +0 -1355
- package/src/test/unit/observe/memoryObservationStore.test.ts +0 -158
- package/src/test/unit/retrieve/fragmentSemanticRetriever.test.ts +0 -163
- package/src/test/unit/retrieve/linkTraversalRetriever.test.ts +0 -329
- package/src/test/unit/retrieve/retrievers.test.ts +0 -1018
- package/src/test/unit/retrieve/temporalRetrievers.test.ts +0 -226
- package/src/test/unit/store/antagonistTemporalBoundary.test.ts +0 -158
- package/src/test/unit/store/embedOnWrite.test.ts +0 -418
- package/src/test/unit/store/fileTreeMemoryStore.test.ts +0 -1041
- package/src/test/unit/store/fragmentEmbedOnWrite.test.ts +0 -349
- package/src/test/unit/store/indexInjection.test.ts +0 -550
- package/src/test/unit/store/lenientOpen.test.ts +0 -292
- package/src/test/unit/store/listScoped.test.ts +0 -138
- package/src/test/unit/store/observations.test.ts +0 -290
- package/src/test/unit/store/rankAxis.test.ts +0 -349
- package/src/test/unit/store/scopeEncoding.test.ts +0 -37
- package/src/test/unit/store/temporalStore.test.ts +0 -469
- package/src/test/unit/tools/memoryTools.test.ts +0 -1121
- package/src/test/unit/types/identityCodec.test.ts +0 -266
- package/src/test/unit/types/ids.test.ts +0 -94
- package/src/test/unit/types/temporalCodec.test.ts +0 -259
- package/src/test/unit/types/temporalPolicy.test.ts +0 -124
- package/src/test/unit/types/writePolicy.test.ts +0 -446
- package/src/test/unit/vector/fragmentConverters.test.ts +0 -80
- package/src/test/unit/vector/inMemoryCosineIndex.test.ts +0 -318
- package/src/test/unit/vector/inMemoryFragmentCosineIndex.test.ts +0 -477
- package/src/test/unit/vector/vectorIndex.test.ts +0 -64
- package/temp/build/lint/_eslint-5eVG3S6w.json +0 -306
- package/temp/build/typescript/ts_8nwakTlr.json +0 -1
- package/temp/ts-agent-memory.api.json +0 -21030
- package/temp/ts-agent-memory.api.md +0 -940
- package/tsconfig.json +0 -8
|
@@ -1,968 +0,0 @@
|
|
|
1
|
-
/*
|
|
2
|
-
* Copyright (c) 2026 Erik Fortune
|
|
3
|
-
* SPDX-License-Identifier: MIT
|
|
4
|
-
*/
|
|
5
|
-
|
|
6
|
-
import { Hash, Logging, Result, fail, mapResults, succeed } from '@fgv/ts-utils';
|
|
7
|
-
import {
|
|
8
|
-
Convert,
|
|
9
|
-
DedupScope,
|
|
10
|
-
EntityId,
|
|
11
|
-
IEdge,
|
|
12
|
-
IEdgeTarget,
|
|
13
|
-
IIdentityCodec,
|
|
14
|
-
IIdentityCodecResult,
|
|
15
|
-
IMemoryEnvelope,
|
|
16
|
-
IMemoryRecord,
|
|
17
|
-
IProvenance,
|
|
18
|
-
Kind,
|
|
19
|
-
LinkType,
|
|
20
|
-
MemoryId,
|
|
21
|
-
Tag,
|
|
22
|
-
edgeTargetKey,
|
|
23
|
-
isTemporalRecord,
|
|
24
|
-
isVersionCurrent
|
|
25
|
-
} from '../types';
|
|
26
|
-
import { IBodyConverterRegistry } from '../converters';
|
|
27
|
-
import { IMemoryStore } from '../store';
|
|
28
|
-
import { IVectorIndex, MemoryEmbedder } from '../vector';
|
|
29
|
-
import { ICycleGuardEdge, assertNoCycles } from './cycleGuard';
|
|
30
|
-
import {
|
|
31
|
-
IEntityResolver,
|
|
32
|
-
IFactExtractor,
|
|
33
|
-
IMemoryClassifier,
|
|
34
|
-
IRelationCandidate,
|
|
35
|
-
IRelationExtractor
|
|
36
|
-
} from './hostStages';
|
|
37
|
-
import {
|
|
38
|
-
ICandidateEdge,
|
|
39
|
-
ICandidateRecord,
|
|
40
|
-
IEntityResolutionCandidate,
|
|
41
|
-
IIngestItem,
|
|
42
|
-
IIngestItemResult,
|
|
43
|
-
IIngestedRecordResult,
|
|
44
|
-
IMemoryClassification,
|
|
45
|
-
IngestDisposition,
|
|
46
|
-
ResolutionVerdict
|
|
47
|
-
} from './model';
|
|
48
|
-
|
|
49
|
-
/**
|
|
50
|
-
* The provenance source stamped on every record the ingest pipeline writes.
|
|
51
|
-
* @public
|
|
52
|
-
*/
|
|
53
|
-
export const HOST_INGEST_PROVENANCE_SOURCE: string = 'host-ingest';
|
|
54
|
-
|
|
55
|
-
/**
|
|
56
|
-
* The link type whose presence on a temporal-kind candidate fires the
|
|
57
|
-
* contradicts→temporal-versioned interlock.
|
|
58
|
-
* @public
|
|
59
|
-
*/
|
|
60
|
-
export const CONTRADICTS_LINK_TYPE: LinkType = 'contradicts' as LinkType;
|
|
61
|
-
|
|
62
|
-
/**
|
|
63
|
-
* The default stage-4 layer-2 cosine similarity threshold (design note §2). A
|
|
64
|
-
* near-duplicate candidate is surfaced to the {@link IEntityResolver} only when
|
|
65
|
-
* its nearest neighbor scores `>= 0.85`.
|
|
66
|
-
* @public
|
|
67
|
-
*/
|
|
68
|
-
export const DEFAULT_SIMILARITY_THRESHOLD: number = 0.85;
|
|
69
|
-
|
|
70
|
-
/**
|
|
71
|
-
* The default top-K for stage-4 layer-2 similarity candidate generation.
|
|
72
|
-
* @public
|
|
73
|
-
*/
|
|
74
|
-
export const DEFAULT_SIMILARITY_TOP_K: number = 5;
|
|
75
|
-
|
|
76
|
-
/**
|
|
77
|
-
* How the write-time cycle guard behaves: `'reject'` (default — a cycle-inducing
|
|
78
|
-
* edge fails the ingest) or `'off'` (no acyclicity constraint, for deployments
|
|
79
|
-
* whose link graphs are legitimately cyclic — e.g. mutual associative links).
|
|
80
|
-
* @public
|
|
81
|
-
*/
|
|
82
|
-
export type CycleGuardMode = 'reject' | 'off';
|
|
83
|
-
|
|
84
|
-
/**
|
|
85
|
-
* The fgv-owned six-stage ingest orchestrator. Composes the host's staged
|
|
86
|
-
* classify / extract / (optional) resolve / relate machinery around fgv's owned
|
|
87
|
-
* validation boundary, dedup, edge + cycle safety, provenance stamping, and the
|
|
88
|
-
* contradicts→temporal interlock.
|
|
89
|
-
* @public
|
|
90
|
-
*/
|
|
91
|
-
export interface IMemoryIngestOrchestrator {
|
|
92
|
-
/**
|
|
93
|
-
* Ingest a SINGLE item end-to-end (the first-class per-turn streaming path).
|
|
94
|
-
*/
|
|
95
|
-
ingestItem(item: IIngestItem): Promise<Result<IIngestItemResult>>;
|
|
96
|
-
|
|
97
|
-
/**
|
|
98
|
-
* Ingest a batch of items. A convenience loop over {@link
|
|
99
|
-
* IMemoryIngestOrchestrator.ingestItem | ingestItem} — items are processed in
|
|
100
|
-
* order and the first failure aborts the batch.
|
|
101
|
-
*/
|
|
102
|
-
ingestBatch(items: ReadonlyArray<IIngestItem>): Promise<Result<ReadonlyArray<IIngestItemResult>>>;
|
|
103
|
-
}
|
|
104
|
-
|
|
105
|
-
/**
|
|
106
|
-
* Parameters for {@link MemoryIngestOrchestrator.create}.
|
|
107
|
-
* @public
|
|
108
|
-
*/
|
|
109
|
-
export interface IMemoryIngestOrchestratorCreateParams {
|
|
110
|
-
/** The store every write bottoms out in (stage 6). */
|
|
111
|
-
readonly store: IMemoryStore;
|
|
112
|
-
/** Per-kind body converter registry — the stage-3 typed validation boundary. */
|
|
113
|
-
readonly registry: IBodyConverterRegistry;
|
|
114
|
-
/** Per-kind identity codecs (maps a candidate `entityId` to its storage address). */
|
|
115
|
-
readonly codecs?: ReadonlyMap<Kind, IIdentityCodec>;
|
|
116
|
-
/** Default identity codec for kinds without an explicit entry. */
|
|
117
|
-
readonly defaultCodec?: IIdentityCodec;
|
|
118
|
-
/** Stage 2 — the host's classifier. */
|
|
119
|
-
readonly classifier: IMemoryClassifier;
|
|
120
|
-
/** Stage 3 — the host's fact extractor. */
|
|
121
|
-
readonly extractor: IFactExtractor;
|
|
122
|
-
/** Stage 5 — the host's relation extractor. */
|
|
123
|
-
readonly relationExtractor: IRelationExtractor;
|
|
124
|
-
/**
|
|
125
|
-
* Stage 4 (optional, OQ-13) — the host's entity resolver. Absent → stage-4
|
|
126
|
-
* dedup is exact-`{ kind, body }`-hash only (the deterministic-identity path).
|
|
127
|
-
*/
|
|
128
|
-
readonly entityResolver?: IEntityResolver;
|
|
129
|
-
/**
|
|
130
|
-
* Optional vector index for stage-4 layer-2 similarity candidate-gen. Wired
|
|
131
|
-
* together with {@link IMemoryIngestOrchestratorCreateParams.embed | embed} AND
|
|
132
|
-
* {@link IMemoryIngestOrchestratorCreateParams.entityResolver | entityResolver};
|
|
133
|
-
* absent (or either co-requisite absent) → layer-2 is skipped and dedup is
|
|
134
|
-
* exact-only.
|
|
135
|
-
*/
|
|
136
|
-
readonly vectorIndex?: IVectorIndex;
|
|
137
|
-
/** Optional embedder used to embed a candidate for layer-2 similarity search. */
|
|
138
|
-
readonly embed?: MemoryEmbedder;
|
|
139
|
-
/**
|
|
140
|
-
* Stage-4 layer-2 cosine threshold. Defaults to {@link
|
|
141
|
-
* DEFAULT_SIMILARITY_THRESHOLD} (`0.85`).
|
|
142
|
-
*/
|
|
143
|
-
readonly similarityThreshold?: number;
|
|
144
|
-
/**
|
|
145
|
-
* Stage-4 layer-2 top-K. Defaults to {@link DEFAULT_SIMILARITY_TOP_K} (`5`).
|
|
146
|
-
*/
|
|
147
|
-
readonly similarityTopK?: number;
|
|
148
|
-
/** Write-time cycle guard mode. Defaults to `'reject'`. */
|
|
149
|
-
readonly cycleGuard?: CycleGuardMode;
|
|
150
|
-
/** Diagnostic logger (defaults to a no-op). */
|
|
151
|
-
readonly logger?: Logging.ILogger;
|
|
152
|
-
}
|
|
153
|
-
|
|
154
|
-
/** The fully-wired stage-4 layer-2 dependencies (present only when all three are supplied). */
|
|
155
|
-
interface ISimilarityWiring {
|
|
156
|
-
readonly resolver: IEntityResolver;
|
|
157
|
-
readonly vectorIndex: IVectorIndex;
|
|
158
|
-
readonly embed: MemoryEmbedder;
|
|
159
|
-
}
|
|
160
|
-
|
|
161
|
-
/** A snapshot record paired with its resolved scope-qualified `(scope, id)` address. */
|
|
162
|
-
interface IScopedRecord {
|
|
163
|
-
readonly address: IEdgeTarget;
|
|
164
|
-
readonly record: IMemoryRecord<unknown>;
|
|
165
|
-
}
|
|
166
|
-
|
|
167
|
-
/** Internal per-candidate plan threaded through the pipeline. */
|
|
168
|
-
interface ICandidatePlan {
|
|
169
|
-
readonly candidate: ICandidateRecord;
|
|
170
|
-
/** The write target's storage address (re-addressed to the target for merge-into). */
|
|
171
|
-
readonly writeAddress: IIdentityCodecResult;
|
|
172
|
-
/** The write target's entity id (re-addressed for merge-into). */
|
|
173
|
-
readonly writeEntityId: EntityId;
|
|
174
|
-
/** The reference id used in stage-5 edges (the write target's `idStem`). */
|
|
175
|
-
readonly refId: MemoryId;
|
|
176
|
-
/**
|
|
177
|
-
* The scope-qualified reference used in stage-5 edges: the write target's
|
|
178
|
-
* `(scope, idStem)`. Stage-5 sources / dedup all key on this scoped address so
|
|
179
|
-
* two candidates that share a stem across scopes never collide.
|
|
180
|
-
*/
|
|
181
|
-
readonly refTarget: IEdgeTarget;
|
|
182
|
-
/** The resolution verdict; carries the target id on its target-bearing arms. */
|
|
183
|
-
readonly verdict: ResolutionVerdict;
|
|
184
|
-
/**
|
|
185
|
-
* The pre-ingest target record for a `merge-into` write — its existing
|
|
186
|
-
* `tags` / `links` / `provenance` are UNIONed with the candidate's at stage 6
|
|
187
|
-
* so a wholesale-replace merge never wipes them.
|
|
188
|
-
*/
|
|
189
|
-
readonly mergeTarget?: IMemoryRecord<unknown>;
|
|
190
|
-
}
|
|
191
|
-
|
|
192
|
-
/**
|
|
193
|
-
* Default {@link IMemoryIngestOrchestrator}.
|
|
194
|
-
* @public
|
|
195
|
-
*/
|
|
196
|
-
export class MemoryIngestOrchestrator implements IMemoryIngestOrchestrator {
|
|
197
|
-
private readonly _store: IMemoryStore;
|
|
198
|
-
private readonly _registry: IBodyConverterRegistry;
|
|
199
|
-
private readonly _codecs: ReadonlyMap<Kind, IIdentityCodec>;
|
|
200
|
-
private readonly _defaultCodec: IIdentityCodec | undefined;
|
|
201
|
-
private readonly _classifier: IMemoryClassifier;
|
|
202
|
-
private readonly _extractor: IFactExtractor;
|
|
203
|
-
private readonly _relationExtractor: IRelationExtractor;
|
|
204
|
-
private readonly _similarity: ISimilarityWiring | undefined;
|
|
205
|
-
private readonly _similarityThreshold: number;
|
|
206
|
-
private readonly _similarityTopK: number;
|
|
207
|
-
private readonly _cycleGuard: CycleGuardMode;
|
|
208
|
-
private readonly _logger: Logging.ILogger;
|
|
209
|
-
private readonly _hasher: Hash.Crc32Normalizer;
|
|
210
|
-
|
|
211
|
-
private constructor(params: IMemoryIngestOrchestratorCreateParams) {
|
|
212
|
-
this._store = params.store;
|
|
213
|
-
this._registry = params.registry;
|
|
214
|
-
this._codecs = params.codecs ?? new Map<Kind, IIdentityCodec>();
|
|
215
|
-
this._defaultCodec = params.defaultCodec;
|
|
216
|
-
this._classifier = params.classifier;
|
|
217
|
-
this._extractor = params.extractor;
|
|
218
|
-
this._relationExtractor = params.relationExtractor;
|
|
219
|
-
this._similarity =
|
|
220
|
-
params.entityResolver !== undefined && params.vectorIndex !== undefined && params.embed !== undefined
|
|
221
|
-
? { resolver: params.entityResolver, vectorIndex: params.vectorIndex, embed: params.embed }
|
|
222
|
-
: undefined;
|
|
223
|
-
this._similarityThreshold = params.similarityThreshold ?? DEFAULT_SIMILARITY_THRESHOLD;
|
|
224
|
-
this._similarityTopK = params.similarityTopK ?? DEFAULT_SIMILARITY_TOP_K;
|
|
225
|
-
this._cycleGuard = params.cycleGuard ?? 'reject';
|
|
226
|
-
this._logger = params.logger ?? new Logging.NoOpLogger();
|
|
227
|
-
this._hasher = new Hash.Crc32Normalizer();
|
|
228
|
-
}
|
|
229
|
-
|
|
230
|
-
/** Family-convention factory. */
|
|
231
|
-
public static create(params: IMemoryIngestOrchestratorCreateParams): Result<MemoryIngestOrchestrator> {
|
|
232
|
-
return succeed(new MemoryIngestOrchestrator(params));
|
|
233
|
-
}
|
|
234
|
-
|
|
235
|
-
/** {@inheritDoc IMemoryIngestOrchestrator.ingestItem} */
|
|
236
|
-
public async ingestItem(item: IIngestItem): Promise<Result<IIngestItemResult>> {
|
|
237
|
-
return (await this._classify(item))
|
|
238
|
-
.thenOnSuccess((classification) => this._extract(item, classification))
|
|
239
|
-
.thenOnSuccess((candidates) => this._processCandidates(item, candidates));
|
|
240
|
-
}
|
|
241
|
-
|
|
242
|
-
/** {@inheritDoc IMemoryIngestOrchestrator.ingestBatch} */
|
|
243
|
-
public async ingestBatch(
|
|
244
|
-
items: ReadonlyArray<IIngestItem>
|
|
245
|
-
): Promise<Result<ReadonlyArray<IIngestItemResult>>> {
|
|
246
|
-
const results: IIngestItemResult[] = [];
|
|
247
|
-
for (const item of items) {
|
|
248
|
-
const result: Result<IIngestItemResult> = await this.ingestItem(item);
|
|
249
|
-
if (result.isFailure()) {
|
|
250
|
-
return fail(result.message);
|
|
251
|
-
}
|
|
252
|
-
results.push(result.value);
|
|
253
|
-
}
|
|
254
|
-
return succeed(results);
|
|
255
|
-
}
|
|
256
|
-
|
|
257
|
-
/** Stage 2 — classify (host), normalizing a rejected promise into a Failure. */
|
|
258
|
-
private async _classify(item: IIngestItem): Promise<Result<IMemoryClassification>> {
|
|
259
|
-
return this._capture(() => this._classifier.classify(item), `ingest '${item.id}': classify`);
|
|
260
|
-
}
|
|
261
|
-
|
|
262
|
-
/** Stage 3 — extract (host), normalizing a rejected promise into a Failure. */
|
|
263
|
-
private async _extract(
|
|
264
|
-
item: IIngestItem,
|
|
265
|
-
classification: IMemoryClassification
|
|
266
|
-
): Promise<Result<ReadonlyArray<ICandidateRecord>>> {
|
|
267
|
-
return this._capture(() => this._extractor.extract(item, classification), `ingest '${item.id}': extract`);
|
|
268
|
-
}
|
|
269
|
-
|
|
270
|
-
/**
|
|
271
|
-
* Stages 3b-6 over the extracted candidates: validate bodies, resolve/dedup
|
|
272
|
-
* (stage 4), relate + cycle guard (stage 5), and load-with-provenance (stage 6).
|
|
273
|
-
*/
|
|
274
|
-
private async _processCandidates(
|
|
275
|
-
item: IIngestItem,
|
|
276
|
-
candidates: ReadonlyArray<ICandidateRecord>
|
|
277
|
-
): Promise<Result<IIngestItemResult>> {
|
|
278
|
-
// Snapshot the store once; stage-4 resolution and the cycle guard reason over
|
|
279
|
-
// the pre-ingest state.
|
|
280
|
-
const snapshotResult: Result<ReadonlyArray<IMemoryRecord<unknown>>> = await this._store.list();
|
|
281
|
-
if (snapshotResult.isFailure()) {
|
|
282
|
-
return fail(`ingest '${item.id}': failed to snapshot store: ${snapshotResult.message}`);
|
|
283
|
-
}
|
|
284
|
-
const snapshot: ReadonlyArray<IMemoryRecord<unknown>> = snapshotResult.value;
|
|
285
|
-
// Resolve every snapshot record's scope-qualified address ONCE, then index it
|
|
286
|
-
// by the canonical scoped key. The single `byKey` view drives BOTH the edge
|
|
287
|
-
// path (validation + cycle guard) AND the verdict/similarity target lookups —
|
|
288
|
-
// so a filename stem reused across scopes never aliases on any path.
|
|
289
|
-
const scopedResult: Result<ReadonlyArray<IScopedRecord>> = this._scopeRecords(snapshot);
|
|
290
|
-
if (scopedResult.isFailure()) {
|
|
291
|
-
return fail(`ingest '${item.id}': ${scopedResult.message}`);
|
|
292
|
-
}
|
|
293
|
-
const scoped: ReadonlyArray<IScopedRecord> = scopedResult.value;
|
|
294
|
-
const byKey: ReadonlyMap<string, IMemoryRecord<unknown>> = new Map<string, IMemoryRecord<unknown>>(
|
|
295
|
-
scoped.map((s) => [edgeTargetKey(s.address), s.record])
|
|
296
|
-
);
|
|
297
|
-
|
|
298
|
-
// Stage 3b + 4: validate each body and resolve a verdict/plan.
|
|
299
|
-
const plans: ICandidatePlan[] = [];
|
|
300
|
-
for (const candidate of candidates) {
|
|
301
|
-
const planResult: Result<ICandidatePlan> = await this._planCandidate(item, candidate, snapshot, byKey);
|
|
302
|
-
if (planResult.isFailure()) {
|
|
303
|
-
return fail(planResult.message);
|
|
304
|
-
}
|
|
305
|
-
plans.push(planResult.value);
|
|
306
|
-
}
|
|
307
|
-
|
|
308
|
-
// Stage 5: relate over the writable candidates, validate + cycle-guard.
|
|
309
|
-
const writablePlans: ICandidatePlan[] = plans.filter((plan) => plan.verdict.verdict !== 'duplicate-of');
|
|
310
|
-
const edgesResult: Result<ReadonlyArray<ICandidateEdge>> = await this._relate(
|
|
311
|
-
item,
|
|
312
|
-
writablePlans,
|
|
313
|
-
MemoryIngestOrchestrator._collapseRedirects(plans),
|
|
314
|
-
scoped,
|
|
315
|
-
byKey
|
|
316
|
-
);
|
|
317
|
-
if (edgesResult.isFailure()) {
|
|
318
|
-
return fail(edgesResult.message);
|
|
319
|
-
}
|
|
320
|
-
const edges: ReadonlyArray<ICandidateEdge> = edgesResult.value;
|
|
321
|
-
|
|
322
|
-
// Stage 6: load-with-provenance, in extraction order.
|
|
323
|
-
const records: IIngestedRecordResult[] = [];
|
|
324
|
-
for (const plan of plans) {
|
|
325
|
-
const outcome: Result<IIngestedRecordResult> = await this._loadCandidate(item, plan, edges);
|
|
326
|
-
if (outcome.isFailure()) {
|
|
327
|
-
return fail(outcome.message);
|
|
328
|
-
}
|
|
329
|
-
records.push(outcome.value);
|
|
330
|
-
}
|
|
331
|
-
return succeed({ item, records });
|
|
332
|
-
}
|
|
333
|
-
|
|
334
|
-
/** Stage 3b + 4 for one candidate: validate body, resolve address, resolve verdict. */
|
|
335
|
-
private async _planCandidate(
|
|
336
|
-
item: IIngestItem,
|
|
337
|
-
candidate: ICandidateRecord,
|
|
338
|
-
snapshot: ReadonlyArray<IMemoryRecord<unknown>>,
|
|
339
|
-
byKey: ReadonlyMap<string, IMemoryRecord<unknown>>
|
|
340
|
-
): Promise<Result<ICandidatePlan>> {
|
|
341
|
-
const kind: Kind = candidate.envelope.kind;
|
|
342
|
-
// Stage 3b: the typed validation boundary — no unchecked host body reaches the store.
|
|
343
|
-
const bodyResult: Result<string> = this._registry
|
|
344
|
-
.convert(kind, candidate.body)
|
|
345
|
-
.withErrorFormat((msg) => `ingest '${item.id}': candidate body for kind '${kind}' is invalid: ${msg}`)
|
|
346
|
-
.onSuccess(() => MemoryIngestOrchestrator._asStringBody(item, candidate));
|
|
347
|
-
if (bodyResult.isFailure()) {
|
|
348
|
-
return fail(bodyResult.message);
|
|
349
|
-
}
|
|
350
|
-
const body: string = bodyResult.value;
|
|
351
|
-
|
|
352
|
-
const addrResult: Result<IIdentityCodecResult> = this._resolveAddress(candidate.envelope.entityId, kind);
|
|
353
|
-
if (addrResult.isFailure()) {
|
|
354
|
-
return fail(`ingest '${item.id}': ${addrResult.message}`);
|
|
355
|
-
}
|
|
356
|
-
const addr: IIdentityCodecResult = addrResult.value;
|
|
357
|
-
|
|
358
|
-
return (await this._resolveVerdict(candidate, kind, body, addr, snapshot, byKey)).onSuccess((verdict) =>
|
|
359
|
-
this._planFromVerdict(item, candidate, addr, verdict, byKey)
|
|
360
|
-
);
|
|
361
|
-
}
|
|
362
|
-
|
|
363
|
-
/**
|
|
364
|
-
* Turn a resolved verdict into a candidate plan. A `new` verdict writes under the
|
|
365
|
-
* candidate's own address. Every TARGET-bearing verdict
|
|
366
|
-
* (`duplicate-of` / `supersede` / `merge-into`) is validated uniformly: the
|
|
367
|
-
* target must be a real store record (fgv owns validation — a non-compliant host
|
|
368
|
-
* resolver never smuggles a bogus id through) AND its kind must equal the
|
|
369
|
-
* candidate's kind (a cross-kind target would write to the wrong scope). Only
|
|
370
|
-
* `merge-into` re-addresses the write to the target's entity.
|
|
371
|
-
*/
|
|
372
|
-
private _planFromVerdict(
|
|
373
|
-
item: IIngestItem,
|
|
374
|
-
candidate: ICandidateRecord,
|
|
375
|
-
addr: IIdentityCodecResult,
|
|
376
|
-
verdict: ResolutionVerdict,
|
|
377
|
-
byKey: ReadonlyMap<string, IMemoryRecord<unknown>>
|
|
378
|
-
): Result<ICandidatePlan> {
|
|
379
|
-
if (verdict.verdict === 'new') {
|
|
380
|
-
return Convert.memoryId.convert(addr.idStem).onSuccess((refId) =>
|
|
381
|
-
succeed({
|
|
382
|
-
candidate,
|
|
383
|
-
writeAddress: addr,
|
|
384
|
-
writeEntityId: candidate.envelope.entityId,
|
|
385
|
-
refId,
|
|
386
|
-
refTarget: { scope: addr.scope, id: refId },
|
|
387
|
-
verdict
|
|
388
|
-
})
|
|
389
|
-
);
|
|
390
|
-
}
|
|
391
|
-
// duplicate-of | supersede | merge-into: the target must exist and share the
|
|
392
|
-
// candidate's kind. Lookup is on the canonical scoped key, so a stem reused
|
|
393
|
-
// across scopes binds only the record the verdict actually named.
|
|
394
|
-
const target: IMemoryRecord<unknown> | undefined = byKey.get(edgeTargetKey(verdict.target));
|
|
395
|
-
if (target === undefined) {
|
|
396
|
-
return fail(
|
|
397
|
-
`ingest '${item.id}': ${verdict.verdict} target '${MemoryIngestOrchestrator._formatTarget(
|
|
398
|
-
verdict.target
|
|
399
|
-
)}' does not exist in the store`
|
|
400
|
-
);
|
|
401
|
-
}
|
|
402
|
-
if (target.envelope.kind !== candidate.envelope.kind) {
|
|
403
|
-
return fail(
|
|
404
|
-
`ingest '${item.id}': ${verdict.verdict} target '${MemoryIngestOrchestrator._formatTarget(
|
|
405
|
-
verdict.target
|
|
406
|
-
)}' is kind '${target.envelope.kind}' but the candidate is kind '${candidate.envelope.kind}'`
|
|
407
|
-
);
|
|
408
|
-
}
|
|
409
|
-
if (verdict.verdict === 'merge-into') {
|
|
410
|
-
// Re-address the write to the target's entity, carrying the target record so
|
|
411
|
-
// stage 6 can UNION its existing tags/links (never overwrite them).
|
|
412
|
-
const targetEntityId: EntityId = target.envelope.entityId;
|
|
413
|
-
// The merge target is a snapshot record, and `_scopeRecords` already proved
|
|
414
|
-
// every snapshot record's address resolves before planning runs — so this
|
|
415
|
-
// re-resolve (needed for the full codec result: scope + idStem + isVersioned)
|
|
416
|
-
// cannot fail. No error-context wrap is warranted; a failure here would be a
|
|
417
|
-
// logic contradiction, not a user-facing condition.
|
|
418
|
-
return this._resolveAddress(targetEntityId, target.envelope.kind).onSuccess((targetAddr) =>
|
|
419
|
-
Convert.memoryId.convert(targetAddr.idStem).onSuccess((refId) =>
|
|
420
|
-
succeed({
|
|
421
|
-
candidate,
|
|
422
|
-
writeAddress: targetAddr,
|
|
423
|
-
writeEntityId: targetEntityId,
|
|
424
|
-
refId,
|
|
425
|
-
refTarget: { scope: targetAddr.scope, id: refId },
|
|
426
|
-
verdict,
|
|
427
|
-
mergeTarget: target
|
|
428
|
-
})
|
|
429
|
-
)
|
|
430
|
-
);
|
|
431
|
-
}
|
|
432
|
-
// duplicate-of | supersede: write under the candidate's own address.
|
|
433
|
-
return Convert.memoryId.convert(addr.idStem).onSuccess((refId) =>
|
|
434
|
-
succeed({
|
|
435
|
-
candidate,
|
|
436
|
-
writeAddress: addr,
|
|
437
|
-
writeEntityId: candidate.envelope.entityId,
|
|
438
|
-
refId,
|
|
439
|
-
refTarget: { scope: addr.scope, id: refId },
|
|
440
|
-
verdict
|
|
441
|
-
})
|
|
442
|
-
);
|
|
443
|
-
}
|
|
444
|
-
|
|
445
|
-
/**
|
|
446
|
-
* Stage 4 — resolve a dedup verdict. Layer 1: an exact `{ kind, body }` match in
|
|
447
|
-
* the candidate's scope is a `duplicate-of`, at the granularity the kind's
|
|
448
|
-
* {@link DedupScope} declares — scope-wide for `'content'`, restricted to the
|
|
449
|
-
* candidate's own entity for `'entity'` (design note §1). Layer 2 (only when
|
|
450
|
-
* a resolver + vector index + embedder are all wired): embed the candidate,
|
|
451
|
-
* surface over-threshold neighbors, and dispatch to the {@link IEntityResolver}.
|
|
452
|
-
* Otherwise the verdict is `new` (the exact-only fall-back path).
|
|
453
|
-
*/
|
|
454
|
-
private async _resolveVerdict(
|
|
455
|
-
candidate: ICandidateRecord,
|
|
456
|
-
kind: Kind,
|
|
457
|
-
body: string,
|
|
458
|
-
addr: IIdentityCodecResult,
|
|
459
|
-
snapshot: ReadonlyArray<IMemoryRecord<unknown>>,
|
|
460
|
-
byKey: ReadonlyMap<string, IMemoryRecord<unknown>>
|
|
461
|
-
): Promise<Result<ResolutionVerdict>> {
|
|
462
|
-
return this._findExactMatch(kind, body, addr, snapshot).thenOnSuccess(async (matchId) => {
|
|
463
|
-
if (matchId !== undefined) {
|
|
464
|
-
// The exact-match cohort is filtered to `addr.scope`, so the match lives
|
|
465
|
-
// under that scope — its scope-qualified target is `(addr.scope, matchId)`.
|
|
466
|
-
return succeed({ verdict: 'duplicate-of', target: { scope: addr.scope, id: matchId } });
|
|
467
|
-
}
|
|
468
|
-
const layer2: ISimilarityWiring | undefined = this._similarity;
|
|
469
|
-
if (layer2 === undefined) {
|
|
470
|
-
// No layer-2: exact-only fall-back (the deterministic-identity host path).
|
|
471
|
-
return succeed({ verdict: 'new' });
|
|
472
|
-
}
|
|
473
|
-
return this._resolveViaSimilarity(candidate, addr, body, layer2, byKey);
|
|
474
|
-
});
|
|
475
|
-
}
|
|
476
|
-
|
|
477
|
-
/** Layer-2 similarity candidate-gen + resolver dispatch. */
|
|
478
|
-
private async _resolveViaSimilarity(
|
|
479
|
-
candidate: ICandidateRecord,
|
|
480
|
-
addr: IIdentityCodecResult,
|
|
481
|
-
body: string,
|
|
482
|
-
wiring: ISimilarityWiring,
|
|
483
|
-
byKey: ReadonlyMap<string, IMemoryRecord<unknown>>
|
|
484
|
-
): Promise<Result<ResolutionVerdict>> {
|
|
485
|
-
return MemoryIngestOrchestrator._provisionalRecord(candidate, addr.idStem, body).thenOnSuccess(
|
|
486
|
-
(provisional) => this._resolveViaSimilarityEmbedded(candidate, addr, wiring, byKey, provisional)
|
|
487
|
-
);
|
|
488
|
-
}
|
|
489
|
-
|
|
490
|
-
/** Layer-2 continuation once the candidate has a provisional record to embed. */
|
|
491
|
-
private async _resolveViaSimilarityEmbedded(
|
|
492
|
-
candidate: ICandidateRecord,
|
|
493
|
-
addr: IIdentityCodecResult,
|
|
494
|
-
wiring: ISimilarityWiring,
|
|
495
|
-
byKey: ReadonlyMap<string, IMemoryRecord<unknown>>,
|
|
496
|
-
provisional: IMemoryRecord<unknown>
|
|
497
|
-
): Promise<Result<ResolutionVerdict>> {
|
|
498
|
-
const embedded: Result<Float32Array> = await this._capture(
|
|
499
|
-
() => wiring.embed(provisional),
|
|
500
|
-
`ingest '${candidate.envelope.entityId}': embed candidate`
|
|
501
|
-
);
|
|
502
|
-
if (embedded.isFailure()) {
|
|
503
|
-
return fail(embedded.message);
|
|
504
|
-
}
|
|
505
|
-
const queried = await this._capture(
|
|
506
|
-
() => wiring.vectorIndex.query(embedded.value, this._similarityTopK),
|
|
507
|
-
`ingest '${candidate.envelope.entityId}': similarity query`
|
|
508
|
-
);
|
|
509
|
-
if (queried.isFailure()) {
|
|
510
|
-
return fail(queried.message);
|
|
511
|
-
}
|
|
512
|
-
const similar: IEntityResolutionCandidate[] = [];
|
|
513
|
-
for (const hit of queried.value) {
|
|
514
|
-
// Skip below-threshold hits and the candidate's OWN scope-qualified address
|
|
515
|
-
// (a re-ingest of the same entity must not dedup against its prior version).
|
|
516
|
-
// Both the scope AND the id must match to be "self" — a same-stem record in
|
|
517
|
-
// another scope is a legitimate distinct neighbor. This field-equality is
|
|
518
|
-
// exactly equivalent to comparing `edgeTargetKey(hit.target)` against the
|
|
519
|
-
// candidate's own key (edgeTargetKey is pure and injective on `(scope, id)`),
|
|
520
|
-
// so this one site reads the components directly rather than re-keying.
|
|
521
|
-
const isSelf: boolean = hit.target.scope === addr.scope && hit.target.id === addr.idStem;
|
|
522
|
-
if (hit.score < this._similarityThreshold || isSelf) {
|
|
523
|
-
continue;
|
|
524
|
-
}
|
|
525
|
-
const record: IMemoryRecord<unknown> | undefined = byKey.get(edgeTargetKey(hit.target));
|
|
526
|
-
if (record !== undefined) {
|
|
527
|
-
similar.push({ target: hit.target, record, score: hit.score });
|
|
528
|
-
}
|
|
529
|
-
}
|
|
530
|
-
if (similar.length === 0) {
|
|
531
|
-
return succeed({ verdict: 'new' });
|
|
532
|
-
}
|
|
533
|
-
return this._capture(
|
|
534
|
-
() => wiring.resolver.resolve(candidate, similar),
|
|
535
|
-
`ingest '${candidate.envelope.entityId}': resolve`
|
|
536
|
-
);
|
|
537
|
-
}
|
|
538
|
-
|
|
539
|
-
/**
|
|
540
|
-
* Find an existing record whose `{ kind, body }` hash matches the candidate's
|
|
541
|
-
* (layer-1 exact dedup, design note §1). Invalidated temporal versions are
|
|
542
|
-
* excluded — only a live (non-temporal or current) record deduplicates a
|
|
543
|
-
* candidate.
|
|
544
|
-
*
|
|
545
|
-
* @remarks
|
|
546
|
-
* The cohort's granularity is the kind's DECLARED {@link DedupScope}, read
|
|
547
|
-
* through {@link IMemoryStore.dedupScopeFor} so this layer and the store's own
|
|
548
|
-
* write path can never disagree:
|
|
549
|
-
*
|
|
550
|
-
* - `'content'` — every live same-kind record in the candidate's scope,
|
|
551
|
-
* regardless of entity. A cross-id body collision IS a duplicate (the
|
|
552
|
-
* knowledge family; unchanged behavior).
|
|
553
|
-
* - `'entity'` — only records at the candidate's OWN entity address. Two
|
|
554
|
-
* distinct entities with byte-identical bodies never collapse into one
|
|
555
|
-
* another, which is what the declaration has always meant on the direct-put
|
|
556
|
-
* path and now means here too.
|
|
557
|
-
*
|
|
558
|
-
* The same-id collapse itself remains the store's job — this layer only decides
|
|
559
|
-
* whether a cross-id body collision is eligible to be a `duplicate-of` at all.
|
|
560
|
-
*/
|
|
561
|
-
private _findExactMatch(
|
|
562
|
-
kind: Kind,
|
|
563
|
-
body: string,
|
|
564
|
-
addr: IIdentityCodecResult,
|
|
565
|
-
snapshot: ReadonlyArray<IMemoryRecord<unknown>>
|
|
566
|
-
): Result<MemoryId | undefined> {
|
|
567
|
-
const dedupScope: DedupScope = this._store.dedupScopeFor(kind);
|
|
568
|
-
// Same-kind, same-scope, LIVE (non-temporal or current) records are the exact
|
|
569
|
-
// cohort, narrowed to the candidate's own entity address under `'entity'`
|
|
570
|
-
// granularity. `_resolveAddress(...).orDefault()` collapses an unresolved
|
|
571
|
-
// codec to `undefined` — a non-matching address — with no explicit failure
|
|
572
|
-
// branch, exactly as the scope-only filter did before.
|
|
573
|
-
const cohort: ReadonlyArray<IMemoryRecord<unknown>> = snapshot.filter((record) => {
|
|
574
|
-
if (record.envelope.kind !== kind || (isTemporalRecord(record) && !isVersionCurrent(record))) {
|
|
575
|
-
return false;
|
|
576
|
-
}
|
|
577
|
-
const recordAddr: IIdentityCodecResult | undefined = this._resolveAddress(
|
|
578
|
-
record.envelope.entityId,
|
|
579
|
-
record.envelope.kind
|
|
580
|
-
).orDefault();
|
|
581
|
-
if (recordAddr === undefined || recordAddr.scope !== addr.scope) {
|
|
582
|
-
return false;
|
|
583
|
-
}
|
|
584
|
-
return dedupScope === 'content' || recordAddr.idStem === addr.idStem;
|
|
585
|
-
});
|
|
586
|
-
return this._exactKey(kind, body).onSuccess((key) =>
|
|
587
|
-
mapResults(
|
|
588
|
-
cohort.map((record) =>
|
|
589
|
-
MemoryIngestOrchestrator._recordBodyString(record).onSuccess((recordBody) =>
|
|
590
|
-
this._exactKey(record.envelope.kind, recordBody).onSuccess((recordKey) =>
|
|
591
|
-
succeed({ id: record.envelope.id, key: recordKey })
|
|
592
|
-
)
|
|
593
|
-
)
|
|
594
|
-
)
|
|
595
|
-
).onSuccess((keyed) => succeed(keyed.find((entry) => entry.key === key)?.id))
|
|
596
|
-
);
|
|
597
|
-
}
|
|
598
|
-
|
|
599
|
-
/**
|
|
600
|
-
* The body of a persisted record, required to be a string (the store persists
|
|
601
|
-
* only string bodies). Fails loudly rather than blind-casting an `unknown` body
|
|
602
|
-
* into the exact-dedup hash — a non-string existing body is a store-integrity
|
|
603
|
-
* fault, surfaced with context, not a silent miscompute.
|
|
604
|
-
*/
|
|
605
|
-
private static _recordBodyString(record: IMemoryRecord<unknown>): Result<string> {
|
|
606
|
-
if (typeof record.body !== 'string') {
|
|
607
|
-
return fail(
|
|
608
|
-
`ingest: stored record '${record.envelope.id}' has a non-string body (got ${typeof record.body})`
|
|
609
|
-
);
|
|
610
|
-
}
|
|
611
|
-
return succeed(record.body);
|
|
612
|
-
}
|
|
613
|
-
|
|
614
|
-
/** The stage-4 exact-dedup key over `{ kind, body }` (design note §1). */
|
|
615
|
-
private _exactKey(kind: Kind, body: string): Result<string> {
|
|
616
|
-
return this._hasher.computeHash({ kind, body });
|
|
617
|
-
}
|
|
618
|
-
|
|
619
|
-
/** Stage 5 — relate (host), redirect collapsed targets, validate edges, and run the write-time cycle guard. */
|
|
620
|
-
private async _relate(
|
|
621
|
-
item: IIngestItem,
|
|
622
|
-
writablePlans: ReadonlyArray<ICandidatePlan>,
|
|
623
|
-
redirects: ReadonlyMap<string, IEdgeTarget>,
|
|
624
|
-
scoped: ReadonlyArray<IScopedRecord>,
|
|
625
|
-
byKey: ReadonlyMap<string, IMemoryRecord<unknown>>
|
|
626
|
-
): Promise<Result<ReadonlyArray<ICandidateEdge>>> {
|
|
627
|
-
const relationCandidates: IRelationCandidate[] = writablePlans.map((plan) => ({
|
|
628
|
-
candidate: plan.candidate,
|
|
629
|
-
id: plan.refTarget
|
|
630
|
-
}));
|
|
631
|
-
const proposed: Result<ReadonlyArray<ICandidateEdge>> = await this._capture(
|
|
632
|
-
() => this._relationExtractor.relate({ item, candidates: relationCandidates }),
|
|
633
|
-
`ingest '${item.id}': relate`
|
|
634
|
-
);
|
|
635
|
-
if (proposed.isFailure()) {
|
|
636
|
-
return proposed;
|
|
637
|
-
}
|
|
638
|
-
// Redirect BEFORE validation and BEFORE the cycle guard: an edge naming a
|
|
639
|
-
// collapsed candidate must be judged — and persisted — against the record it
|
|
640
|
-
// collapsed into, not against the address that no longer gets written.
|
|
641
|
-
const edges: ReadonlyArray<ICandidateEdge> = MemoryIngestOrchestrator._redirectEdges(
|
|
642
|
-
proposed.value,
|
|
643
|
-
redirects
|
|
644
|
-
);
|
|
645
|
-
// refIds and the existing-record view (`byKey`, shared with the verdict path)
|
|
646
|
-
// both key on the canonical scoped address, so a stem reused across scopes
|
|
647
|
-
// never aliases.
|
|
648
|
-
const refIds: ReadonlySet<string> = new Set<string>(
|
|
649
|
-
writablePlans.map((plan) => edgeTargetKey(plan.refTarget))
|
|
650
|
-
);
|
|
651
|
-
const validation: Result<true> = this._validateEdges(item, edges, refIds, byKey);
|
|
652
|
-
if (validation.isFailure()) {
|
|
653
|
-
return fail(validation.message);
|
|
654
|
-
}
|
|
655
|
-
if (this._cycleGuard === 'reject') {
|
|
656
|
-
const guard: Result<true> = assertNoCycles(
|
|
657
|
-
MemoryIngestOrchestrator._existingEdges(scoped),
|
|
658
|
-
edges.map((e) => ({ source: e.source, target: e.edge.target, type: e.edge.type }))
|
|
659
|
-
);
|
|
660
|
-
if (guard.isFailure()) {
|
|
661
|
-
return fail(`ingest '${item.id}': ${guard.message}`);
|
|
662
|
-
}
|
|
663
|
-
}
|
|
664
|
-
return succeed(edges);
|
|
665
|
-
}
|
|
666
|
-
|
|
667
|
-
/**
|
|
668
|
-
* The stage-5 edge-target redirect map for this pass: every `duplicate-of`
|
|
669
|
-
* candidate's OWN scoped address, mapped to the address of the record its
|
|
670
|
-
* verdict collapsed it into (design note §3).
|
|
671
|
-
*
|
|
672
|
-
* @remarks
|
|
673
|
-
* A `duplicate-of` verdict means "this candidate IS that record". The candidate
|
|
674
|
-
* is therefore not written, and its address never becomes a live reference — so
|
|
675
|
-
* a sibling edge built against it in the same pass would resolve to neither a
|
|
676
|
-
* written candidate nor an existing record and would fail the WHOLE ingest item.
|
|
677
|
-
* That is a second-order hazard of collapsing, not a defect in the edge: the
|
|
678
|
-
* host related two candidates it was correctly told about, and one of them
|
|
679
|
-
* turned out to already exist.
|
|
680
|
-
*
|
|
681
|
-
* Redirecting is the honest repair, and it is independent of {@link DedupScope}
|
|
682
|
-
* — it applies equally to a `'content'` kind, where the collapse is exactly
|
|
683
|
-
* right and the ingest still must not fail.
|
|
684
|
-
*
|
|
685
|
-
* The map cannot chain: {@link MemoryIngestOrchestrator._planFromVerdict}
|
|
686
|
-
* requires every target-bearing verdict's target to already exist in the store
|
|
687
|
-
* snapshot, so a redirect destination is always a persisted record and never
|
|
688
|
-
* another collapsed candidate. One pass is sufficient by construction.
|
|
689
|
-
*/
|
|
690
|
-
private static _collapseRedirects(plans: ReadonlyArray<ICandidatePlan>): ReadonlyMap<string, IEdgeTarget> {
|
|
691
|
-
const redirects: Map<string, IEdgeTarget> = new Map<string, IEdgeTarget>();
|
|
692
|
-
for (const plan of plans) {
|
|
693
|
-
if (plan.verdict.verdict === 'duplicate-of') {
|
|
694
|
-
redirects.set(edgeTargetKey(plan.refTarget), plan.verdict.target);
|
|
695
|
-
}
|
|
696
|
-
}
|
|
697
|
-
return redirects;
|
|
698
|
-
}
|
|
699
|
-
|
|
700
|
-
/**
|
|
701
|
-
* Rewrite each edge whose TARGET names a collapsed candidate to point at the
|
|
702
|
-
* record that candidate collapsed into.
|
|
703
|
-
*
|
|
704
|
-
* @remarks
|
|
705
|
-
* Only the target is redirected. An edge whose SOURCE is a collapsed candidate
|
|
706
|
-
* stays untouched and is still rejected by {@link
|
|
707
|
-
* MemoryIngestOrchestrator._validateEdges}: sources are the records an edge is
|
|
708
|
-
* written ONTO, a collapsed candidate is never written, and the relation
|
|
709
|
-
* extractor is only ever offered writable candidates as edge sources in the
|
|
710
|
-
* first place. Silently relocating such an edge onto an existing record would
|
|
711
|
-
* attribute a link the host never asked for — the loud failure is correct there.
|
|
712
|
-
*/
|
|
713
|
-
private static _redirectEdges(
|
|
714
|
-
edges: ReadonlyArray<ICandidateEdge>,
|
|
715
|
-
redirects: ReadonlyMap<string, IEdgeTarget>
|
|
716
|
-
): ReadonlyArray<ICandidateEdge> {
|
|
717
|
-
if (redirects.size === 0) {
|
|
718
|
-
return edges;
|
|
719
|
-
}
|
|
720
|
-
return edges.map((e) => {
|
|
721
|
-
const to: IEdgeTarget | undefined = redirects.get(edgeTargetKey(e.edge.target));
|
|
722
|
-
return to === undefined ? e : { ...e, edge: { ...e.edge, target: to } };
|
|
723
|
-
});
|
|
724
|
-
}
|
|
725
|
-
|
|
726
|
-
/**
|
|
727
|
-
* Validate stage-5 edges: each `source` must be a candidate being written; each
|
|
728
|
-
* `target` must resolve to a sibling candidate or an existing store record. All
|
|
729
|
-
* matching is on the canonical scope-qualified address.
|
|
730
|
-
*/
|
|
731
|
-
private _validateEdges(
|
|
732
|
-
item: IIngestItem,
|
|
733
|
-
edges: ReadonlyArray<ICandidateEdge>,
|
|
734
|
-
refIds: ReadonlySet<string>,
|
|
735
|
-
byKey: ReadonlyMap<string, IMemoryRecord<unknown>>
|
|
736
|
-
): Result<true> {
|
|
737
|
-
return mapResults(
|
|
738
|
-
edges.map((edge) => {
|
|
739
|
-
if (!refIds.has(edgeTargetKey(edge.source))) {
|
|
740
|
-
return fail(
|
|
741
|
-
`ingest '${item.id}': edge source '${MemoryIngestOrchestrator._formatTarget(
|
|
742
|
-
edge.source
|
|
743
|
-
)}' is not a candidate being written`
|
|
744
|
-
);
|
|
745
|
-
}
|
|
746
|
-
const targetKey: string = edgeTargetKey(edge.edge.target);
|
|
747
|
-
if (!refIds.has(targetKey) && byKey.get(targetKey) === undefined) {
|
|
748
|
-
return fail(
|
|
749
|
-
`ingest '${item.id}': edge target '${MemoryIngestOrchestrator._formatTarget(
|
|
750
|
-
edge.edge.target
|
|
751
|
-
)}' resolves to neither a sibling candidate nor an existing record`
|
|
752
|
-
);
|
|
753
|
-
}
|
|
754
|
-
return succeed(true);
|
|
755
|
-
})
|
|
756
|
-
).onSuccess(() => succeed(true));
|
|
757
|
-
}
|
|
758
|
-
|
|
759
|
-
/** Stage 6 — stamp provenance + edges, admit through the store, record the outcome. */
|
|
760
|
-
private async _loadCandidate(
|
|
761
|
-
item: IIngestItem,
|
|
762
|
-
plan: ICandidatePlan,
|
|
763
|
-
allEdges: ReadonlyArray<ICandidateEdge>
|
|
764
|
-
): Promise<Result<IIngestedRecordResult>> {
|
|
765
|
-
const refKey: string = edgeTargetKey(plan.refTarget);
|
|
766
|
-
const myEdges: ReadonlyArray<ICandidateEdge> = allEdges.filter((e) => edgeTargetKey(e.source) === refKey);
|
|
767
|
-
if (plan.verdict.verdict === 'duplicate-of') {
|
|
768
|
-
// No write: the existing target satisfied the candidate. The target id is a
|
|
769
|
-
// typed field of the narrowed `duplicate-of` verdict — no cast, no guard.
|
|
770
|
-
return succeed({
|
|
771
|
-
candidate: plan.candidate,
|
|
772
|
-
resolution: plan.verdict,
|
|
773
|
-
disposition: 'deduped',
|
|
774
|
-
id: plan.verdict.target.id,
|
|
775
|
-
edges: myEdges
|
|
776
|
-
});
|
|
777
|
-
}
|
|
778
|
-
return this._buildRecord(item, plan, myEdges).thenOnSuccess(async (record) =>
|
|
779
|
-
(await this._store.put(record))
|
|
780
|
-
.withErrorFormat((msg) => `ingest '${item.id}': write failed: ${msg}`)
|
|
781
|
-
.onSuccess((persisted) => {
|
|
782
|
-
const disposition: IngestDisposition = plan.verdict.verdict === 'merge-into' ? 'merged' : 'written';
|
|
783
|
-
const interlock: 'temporal-versioned' | undefined =
|
|
784
|
-
plan.writeAddress.isVersioned && myEdges.some((e) => e.edge.type === CONTRADICTS_LINK_TYPE)
|
|
785
|
-
? 'temporal-versioned'
|
|
786
|
-
: undefined;
|
|
787
|
-
return succeed({
|
|
788
|
-
candidate: plan.candidate,
|
|
789
|
-
resolution: plan.verdict,
|
|
790
|
-
disposition,
|
|
791
|
-
id: persisted.envelope.id,
|
|
792
|
-
record: persisted,
|
|
793
|
-
edges: myEdges,
|
|
794
|
-
...(interlock !== undefined ? { interlock } : {})
|
|
795
|
-
});
|
|
796
|
-
})
|
|
797
|
-
);
|
|
798
|
-
}
|
|
799
|
-
|
|
800
|
-
/** Build the fully-stamped record to persist (provenance + edges + placeholder txn fields). */
|
|
801
|
-
private _buildRecord(
|
|
802
|
-
item: IIngestItem,
|
|
803
|
-
plan: ICandidatePlan,
|
|
804
|
-
myEdges: ReadonlyArray<ICandidateEdge>
|
|
805
|
-
): Result<IMemoryRecord<unknown>> {
|
|
806
|
-
const base: Omit<IMemoryEnvelope, 'id' | 'seq' | 'contentHash' | 'created' | 'updated'> =
|
|
807
|
-
plan.candidate.envelope;
|
|
808
|
-
// For merge-into the store's `applyUpdate` REPLACES array fields wholesale, so
|
|
809
|
-
// the persisted envelope must already carry the UNION of the target's existing
|
|
810
|
-
// tags/links/provenance and the candidate's — otherwise the merge silently
|
|
811
|
-
// wipes the target's prior links/tags.
|
|
812
|
-
const target: IMemoryRecord<unknown> | undefined = plan.mergeTarget;
|
|
813
|
-
const priorLinks: ReadonlyArray<IEdge> = target !== undefined ? target.envelope.links : [];
|
|
814
|
-
const priorTags: ReadonlyArray<Tag> = target !== undefined ? target.envelope.tags : [];
|
|
815
|
-
const priorProvenance: IProvenance = target !== undefined ? target.envelope.provenance : { source: '' };
|
|
816
|
-
return Convert.memoryId.convert(plan.writeAddress.idStem).onSuccess((id) =>
|
|
817
|
-
// Dedup the combined links by canonical edge key: the target's prior links, the
|
|
818
|
-
// candidate's own links, and the stage-5 edges (two candidates that resolve
|
|
819
|
-
// merge-into the SAME target share a `refId` — hence the same `myEdges` — and a
|
|
820
|
-
// relation extractor may repeat an edge) each appear exactly once.
|
|
821
|
-
this._dedupEdges([...priorLinks, ...base.links, ...myEdges.map((e) => e.edge)]).onSuccess((links) => {
|
|
822
|
-
const provenance: IProvenance = {
|
|
823
|
-
// Preserve the target's provenance extension keys, overlay the candidate's,
|
|
824
|
-
// then stamp the host-ingest source + derivedFrom (target present only for
|
|
825
|
-
// merge-into; otherwise the seed is inert).
|
|
826
|
-
...(target !== undefined ? priorProvenance : {}),
|
|
827
|
-
...base.provenance,
|
|
828
|
-
source: HOST_INGEST_PROVENANCE_SOURCE,
|
|
829
|
-
...(item.sourceId !== undefined ? { derivedFrom: item.sourceId } : {})
|
|
830
|
-
};
|
|
831
|
-
const envelope: IMemoryEnvelope = {
|
|
832
|
-
...base,
|
|
833
|
-
id,
|
|
834
|
-
entityId: plan.writeEntityId,
|
|
835
|
-
seq: 0,
|
|
836
|
-
contentHash: '',
|
|
837
|
-
created: 0,
|
|
838
|
-
updated: 0,
|
|
839
|
-
provenance,
|
|
840
|
-
tags: MemoryIngestOrchestrator._unionTags(priorTags, base.tags),
|
|
841
|
-
links
|
|
842
|
-
};
|
|
843
|
-
return succeed({ envelope, body: plan.candidate.body });
|
|
844
|
-
})
|
|
845
|
-
);
|
|
846
|
-
}
|
|
847
|
-
|
|
848
|
-
/** Union two tag lists, de-duplicated, preserving first-occurrence order. */
|
|
849
|
-
private static _unionTags(prior: ReadonlyArray<Tag>, incoming: ReadonlyArray<Tag>): ReadonlyArray<Tag> {
|
|
850
|
-
const seen: Set<string> = new Set<string>();
|
|
851
|
-
const out: Tag[] = [];
|
|
852
|
-
for (const tag of [...prior, ...incoming]) {
|
|
853
|
-
if (!seen.has(tag)) {
|
|
854
|
-
seen.add(tag);
|
|
855
|
-
out.push(tag);
|
|
856
|
-
}
|
|
857
|
-
}
|
|
858
|
-
return out;
|
|
859
|
-
}
|
|
860
|
-
|
|
861
|
-
/** De-duplicate a link list by canonical edge hash, preserving first-occurrence order. */
|
|
862
|
-
private _dedupEdges(edges: ReadonlyArray<IEdge>): Result<ReadonlyArray<IEdge>> {
|
|
863
|
-
const seen: Set<string> = new Set<string>();
|
|
864
|
-
const deduped: IEdge[] = [];
|
|
865
|
-
return mapResults(
|
|
866
|
-
edges.map((edge) =>
|
|
867
|
-
this._hasher.computeHash(edge).onSuccess((key) => {
|
|
868
|
-
if (!seen.has(key)) {
|
|
869
|
-
seen.add(key);
|
|
870
|
-
deduped.push(edge);
|
|
871
|
-
}
|
|
872
|
-
return succeed(key);
|
|
873
|
-
})
|
|
874
|
-
)
|
|
875
|
-
).onSuccess(() => succeed(deduped));
|
|
876
|
-
}
|
|
877
|
-
|
|
878
|
-
/** Resolve a `(kind, entityId)` to its storage address via the registered codec. */
|
|
879
|
-
private _resolveAddress(entityId: EntityId, kind: Kind): Result<IIdentityCodecResult> {
|
|
880
|
-
const codec: IIdentityCodec | undefined = this._codecs.get(kind) ?? this._defaultCodec;
|
|
881
|
-
if (codec === undefined) {
|
|
882
|
-
return fail(`no identity codec registered for kind '${kind}'`);
|
|
883
|
-
}
|
|
884
|
-
return codec.encode(entityId);
|
|
885
|
-
}
|
|
886
|
-
|
|
887
|
-
/** Run a host hook, normalizing a thrown/rejected hook into a Failure (never throws across the seam). */
|
|
888
|
-
private async _capture<T>(op: () => Promise<Result<T>>, label: string): Promise<Result<T>> {
|
|
889
|
-
try {
|
|
890
|
-
return await op();
|
|
891
|
-
} catch (err) {
|
|
892
|
-
const message: string = `${label} threw: ${String(err)}`;
|
|
893
|
-
this._logger.warn(message);
|
|
894
|
-
return fail(message);
|
|
895
|
-
}
|
|
896
|
-
}
|
|
897
|
-
|
|
898
|
-
/**
|
|
899
|
-
* Every existing outbound edge in the snapshot, as cycle-guard edges. The
|
|
900
|
-
* source is the record's own scope-qualified address (already resolved by
|
|
901
|
-
* {@link MemoryIngestOrchestrator._scopeRecords}); the target is the edge's
|
|
902
|
-
* own scoped target. Both ends are scoped so the guard never conflates a stem
|
|
903
|
-
* shared across scopes into one graph node.
|
|
904
|
-
*/
|
|
905
|
-
private static _existingEdges(scoped: ReadonlyArray<IScopedRecord>): ReadonlyArray<ICycleGuardEdge> {
|
|
906
|
-
const edges: ICycleGuardEdge[] = [];
|
|
907
|
-
for (const s of scoped) {
|
|
908
|
-
for (const edge of s.record.envelope.links) {
|
|
909
|
-
edges.push({ source: s.address, target: edge.target, type: edge.type });
|
|
910
|
-
}
|
|
911
|
-
}
|
|
912
|
-
return edges;
|
|
913
|
-
}
|
|
914
|
-
|
|
915
|
-
/**
|
|
916
|
-
* Resolve every snapshot record's scope-qualified `(scope, id)` address via its
|
|
917
|
-
* registered codec. Fails loudly if a record's kind has no resolvable codec —
|
|
918
|
-
* the edge path cannot place an un-scopeable record in the graph, and a missing
|
|
919
|
-
* codec for a stored kind is a real misconfiguration, not something to paper over.
|
|
920
|
-
*/
|
|
921
|
-
private _scopeRecords(
|
|
922
|
-
records: ReadonlyArray<IMemoryRecord<unknown>>
|
|
923
|
-
): Result<ReadonlyArray<IScopedRecord>> {
|
|
924
|
-
return mapResults(
|
|
925
|
-
records.map((record) =>
|
|
926
|
-
this._resolveAddress(record.envelope.entityId, record.envelope.kind)
|
|
927
|
-
.withErrorFormat((msg) => `cannot resolve scope for stored record '${record.envelope.id}': ${msg}`)
|
|
928
|
-
.onSuccess((addr) => succeed({ address: { scope: addr.scope, id: record.envelope.id }, record }))
|
|
929
|
-
)
|
|
930
|
-
);
|
|
931
|
-
}
|
|
932
|
-
|
|
933
|
-
/** Human-readable `scope/id` rendering of a scoped target for edge-validation diagnostics. */
|
|
934
|
-
private static _formatTarget(target: IEdgeTarget): string {
|
|
935
|
-
return `${target.scope}/${target.id}`;
|
|
936
|
-
}
|
|
937
|
-
|
|
938
|
-
/** A provisional record for embedding a candidate (placeholder txn-time fields). */
|
|
939
|
-
private static _provisionalRecord(
|
|
940
|
-
candidate: ICandidateRecord,
|
|
941
|
-
idStem: string,
|
|
942
|
-
body: string
|
|
943
|
-
): Result<IMemoryRecord<unknown>> {
|
|
944
|
-
return Convert.memoryId.convert(idStem).onSuccess((id) => {
|
|
945
|
-
const envelope: IMemoryEnvelope = {
|
|
946
|
-
...candidate.envelope,
|
|
947
|
-
id,
|
|
948
|
-
seq: 0,
|
|
949
|
-
contentHash: '',
|
|
950
|
-
created: 0,
|
|
951
|
-
updated: 0
|
|
952
|
-
};
|
|
953
|
-
return succeed({ envelope, body });
|
|
954
|
-
});
|
|
955
|
-
}
|
|
956
|
-
|
|
957
|
-
/** Require a candidate body to be a string (the store persists only string bodies). */
|
|
958
|
-
private static _asStringBody(item: IIngestItem, candidate: ICandidateRecord): Result<string> {
|
|
959
|
-
if (typeof candidate.body !== 'string') {
|
|
960
|
-
return fail(
|
|
961
|
-
`ingest '${item.id}': candidate body for kind '${
|
|
962
|
-
candidate.envelope.kind
|
|
963
|
-
}' must be a string (got ${typeof candidate.body})`
|
|
964
|
-
);
|
|
965
|
-
}
|
|
966
|
-
return succeed(candidate.body);
|
|
967
|
-
}
|
|
968
|
-
}
|