@snoai/memory 0.0.0-stage → 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +176 -0
- package/README.md +33 -2
- package/config/b-profile-section-registry.json +118 -0
- package/config/llm/compiled/2026-05-19T230932Z/manifest.json +25 -0
- package/config/llm/compiled/2026-05-19T230932Z/registry-root.json +67 -0
- package/config/llm/compiled/2026-05-19T230932Z/resolved/mem_claw/openai_gpt_5_nano.json +5 -0
- package/config/llm/compiled/2026-05-19T230932Z/resolved/mem_claw/openrouter_auto.json +5 -0
- package/config/llm/compiled/2026-05-19T230932Z/resolved/mem_claw/sno_ai_extract.json +5 -0
- package/config/llm/compiled/2026-05-19T230932Z/source/mem_claw/openai_gpt_5_nano.mda +24 -0
- package/config/llm/compiled/2026-05-19T230932Z/source/mem_claw/openrouter_auto.mda +24 -0
- package/config/llm/compiled/2026-05-19T230932Z/source/mem_claw/sno_ai_extract.mda +24 -0
- package/config/llm/compiled/2026-07-21T001311Z/manifest.json +43 -0
- package/config/llm/compiled/2026-07-21T001311Z/registry-root.json +97 -0
- package/config/llm/compiled/2026-07-21T001311Z/resolved/mem_claw/openai_gpt_5_nano.json +5 -0
- package/config/llm/compiled/2026-07-21T001311Z/resolved/mem_claw/openrouter_auto.json +5 -0
- package/config/llm/compiled/2026-07-21T001311Z/resolved/mem_claw/sno_ai_extract.json +5 -0
- package/config/llm/compiled/2026-07-21T001311Z/resolved/mem_claw/sno_conflict_verdict.json +10 -0
- package/config/llm/compiled/2026-07-21T001311Z/resolved/mem_claw/sno_extract_chat.json +10 -0
- package/config/llm/compiled/2026-07-21T001311Z/resolved/mem_claw/sno_extract_profile.json +10 -0
- package/config/llm/compiled/2026-07-21T001311Z/source/mem_claw/openai_gpt_5_nano.mda +24 -0
- package/config/llm/compiled/2026-07-21T001311Z/source/mem_claw/openrouter_auto.mda +24 -0
- package/config/llm/compiled/2026-07-21T001311Z/source/mem_claw/sno_ai_extract.mda +24 -0
- package/config/llm/compiled/2026-07-21T001311Z/source/mem_claw/sno_conflict_verdict.mda +27 -0
- package/config/llm/compiled/2026-07-21T001311Z/source/mem_claw/sno_extract_chat.mda +27 -0
- package/config/llm/compiled/2026-07-21T001311Z/source/mem_claw/sno_extract_profile.mda +27 -0
- package/config/llm/compiled/2026-08-29T060335Z/manifest.json +43 -0
- package/config/llm/compiled/2026-08-29T060335Z/registry-root.json +97 -0
- package/config/llm/compiled/2026-08-29T060335Z/resolved/mem_claw/openai_gpt_5_nano.json +5 -0
- package/config/llm/compiled/2026-08-29T060335Z/resolved/mem_claw/openrouter_auto.json +5 -0
- package/config/llm/compiled/2026-08-29T060335Z/resolved/mem_claw/sno_ai_extract.json +5 -0
- package/config/llm/compiled/2026-08-29T060335Z/resolved/mem_claw/sno_conflict_verdict.json +10 -0
- package/config/llm/compiled/2026-08-29T060335Z/resolved/mem_claw/sno_extract_chat.json +10 -0
- package/config/llm/compiled/2026-08-29T060335Z/resolved/mem_claw/sno_extract_profile.json +10 -0
- package/config/llm/compiled/2026-08-29T060335Z/source/mem_claw/openai_gpt_5_nano.mda +24 -0
- package/config/llm/compiled/2026-08-29T060335Z/source/mem_claw/openrouter_auto.mda +24 -0
- package/config/llm/compiled/2026-08-29T060335Z/source/mem_claw/sno_ai_extract.mda +24 -0
- package/config/llm/compiled/2026-08-29T060335Z/source/mem_claw/sno_conflict_verdict.mda +27 -0
- package/config/llm/compiled/2026-08-29T060335Z/source/mem_claw/sno_extract_chat.mda +27 -0
- package/config/llm/compiled/2026-08-29T060335Z/source/mem_claw/sno_extract_profile.mda +27 -0
- package/config/llm/compiled/2026-09-04T004828Z/manifest.json +43 -0
- package/config/llm/compiled/2026-09-04T004828Z/registry-root.json +97 -0
- package/config/llm/compiled/2026-09-04T004828Z/resolved/mem_claw/openai_gpt_5_nano.json +5 -0
- package/config/llm/compiled/2026-09-04T004828Z/resolved/mem_claw/openrouter_auto.json +5 -0
- package/config/llm/compiled/2026-09-04T004828Z/resolved/mem_claw/sno_ai_extract.json +5 -0
- package/config/llm/compiled/2026-09-04T004828Z/resolved/mem_claw/sno_conflict_verdict.json +10 -0
- package/config/llm/compiled/2026-09-04T004828Z/resolved/mem_claw/sno_extract_chat.json +10 -0
- package/config/llm/compiled/2026-09-04T004828Z/resolved/mem_claw/sno_extract_profile.json +10 -0
- package/config/llm/compiled/2026-09-04T004828Z/source/mem_claw/openai_gpt_5_nano.mda +24 -0
- package/config/llm/compiled/2026-09-04T004828Z/source/mem_claw/openrouter_auto.mda +24 -0
- package/config/llm/compiled/2026-09-04T004828Z/source/mem_claw/sno_ai_extract.mda +24 -0
- package/config/llm/compiled/2026-09-04T004828Z/source/mem_claw/sno_conflict_verdict.mda +27 -0
- package/config/llm/compiled/2026-09-04T004828Z/source/mem_claw/sno_extract_chat.mda +27 -0
- package/config/llm/compiled/2026-09-04T004828Z/source/mem_claw/sno_extract_profile.mda +27 -0
- package/config/llm/current.json +4 -0
- package/config/llm/source/mem_claw/openai_gpt_5_nano.mda +24 -0
- package/config/llm/source/mem_claw/openrouter_auto.mda +24 -0
- package/config/llm/source/mem_claw/sno_ai_extract.mda +24 -0
- package/config/llm/source/mem_claw/sno_conflict_verdict.mda +27 -0
- package/config/llm/source/mem_claw/sno_extract_chat.mda +27 -0
- package/config/llm/source/mem_claw/sno_extract_profile.mda +27 -0
- package/config/llm.trust.json +37 -0
- package/dist/_types-CUKHSRkS.d.ts +103 -0
- package/dist/access-tracker-BbEvEocb.d.ts +110 -0
- package/dist/adapter-B7Jox9-6.d.ts +69 -0
- package/dist/adapter-XaRHZWZ2.js +333 -0
- package/dist/agent-llm-port-5oXgBpzJ.d.ts +53 -0
- package/dist/all-resources-BQK2ugjB.js +6379 -0
- package/dist/all-resources-Dij6l5h0.d.ts +18 -0
- package/dist/ambient-learning-text-normalizer-CxjATmMp.js +170 -0
- package/dist/ambient-learning-text-normalizer-_fH3LeuK.d.ts +10 -0
- package/dist/atomic-extraction-reply-BJ9XV_9s.js +863 -0
- package/dist/atomic-extraction-skill-CcXmuCqL.js +56 -0
- package/dist/atomic-memory-extraction-C5rmogoY.js +3605 -0
- package/dist/atomic-memory-extraction-d9EG-4ti.d.ts +327 -0
- package/dist/b-profile-section-dictionary-provider-BHkoNqgk.js +1812 -0
- package/dist/b-profile-section-dictionary-provider-BxbksLj3.d.ts +62 -0
- package/dist/best-effort-BPZSzKk0.d.ts +9 -0
- package/dist/calendar-instruction-CzqZbAPG.js +194 -0
- package/dist/capture-policy-detector-CxbHPkMX.d.ts +53 -0
- package/dist/capture-policy-detector-DaMfy9CZ.js +357 -0
- package/dist/classifier-DnbbtwTS.d.ts +272 -0
- package/dist/client.d.ts +34 -0
- package/dist/client.js +306 -0
- package/dist/coding-skin.d.ts +51 -0
- package/dist/coding-skin.js +55 -0
- package/dist/config-CL90qYva.js +2576 -0
- package/dist/connection-3IVXpuPE.d.ts +391 -0
- package/dist/connection-C9YD5sgm.js +297 -0
- package/dist/daily-log-generator-CH_KuWHs.js +469 -0
- package/dist/daily-log-generator-CsQPHl4A.d.ts +73 -0
- package/dist/daily-spend-estimator-DiwzBlo1.d.ts +8 -0
- package/dist/data-bootstrap-Bl3lthzE.d.ts +30 -0
- package/dist/data-bootstrap-P8sie9WA.js +172 -0
- package/dist/data-paths-C7sNdqvP.d.ts +22 -0
- package/dist/derived-suppression-cache-CI0durYx.d.ts +36 -0
- package/dist/diagnostic-encoder.d.ts +6 -0
- package/dist/diagnostic-encoder.js +29 -0
- package/dist/embedding-provider-client-Q_hJonK9.js +294 -0
- package/dist/embedding-provider-client-vynZMuit.d.ts +94 -0
- package/dist/entry-metadata-parser-BIVSMCIk.js +20 -0
- package/dist/error-D-wb-rZw.js +23 -0
- package/dist/error-signals-D8_3vYu4.d.ts +28 -0
- package/dist/error-signals-lMH2uAdV.js +134 -0
- package/dist/errors-CeVRxf9V.d.ts +49 -0
- package/dist/extraction-text-sanitizer-B_AhIecT.d.ts +32 -0
- package/dist/i18n-text-DbD6Qify.d.ts +13 -0
- package/dist/index-0aqS8pVF.d.ts +351 -0
- package/dist/index-BO4RU_mc.d.ts +650 -0
- package/dist/index-DL8AVj0Z.d.ts +571 -0
- package/dist/index.d.ts +65 -0
- package/dist/index.js +1201 -0
- package/dist/intent-analyzer-B5XVrWwe.d.ts +28 -0
- package/dist/internal/config/index.d.ts +2 -0
- package/dist/internal/config/index.js +581 -0
- package/dist/internal/config/plugin-config-schema.d.ts +2 -0
- package/dist/internal/config/plugin-config-schema.js +2 -0
- package/dist/internal/config/product-mode.d.ts +2 -0
- package/dist/internal/config/product-mode.js +36 -0
- package/dist/internal/config/settings.d.ts +2 -0
- package/dist/internal/config/settings.js +265 -0
- package/dist/internal/config/skill-categories.d.ts +4 -0
- package/dist/internal/config/skill-categories.js +68 -0
- package/dist/internal/config/skin-defaults.d.ts +6 -0
- package/dist/internal/config/skin-defaults.js +6 -0
- package/dist/internal/contract/discovery.d.ts +10 -0
- package/dist/internal/contract/discovery.js +53 -0
- package/dist/internal/contract/profile.d.ts +2 -0
- package/dist/internal/contract/profile.js +44 -0
- package/dist/internal/contract/routes.d.ts +2 -0
- package/dist/internal/contract/routes.js +62 -0
- package/dist/internal/engine/bindings/memory-cli-shared.d.ts +47 -0
- package/dist/internal/engine/bindings/memory-cli-shared.js +84 -0
- package/dist/internal/engine/bindings/memory-forget-tool.d.ts +5 -0
- package/dist/internal/engine/bindings/memory-forget-tool.js +66 -0
- package/dist/internal/engine/bindings/memory-list-tool.d.ts +5 -0
- package/dist/internal/engine/bindings/memory-list-tool.js +57 -0
- package/dist/internal/engine/bindings/memory-recall-tool.d.ts +10 -0
- package/dist/internal/engine/bindings/memory-recall-tool.js +283 -0
- package/dist/internal/engine/bindings/memory-reflection-resolve-tool.d.ts +5 -0
- package/dist/internal/engine/bindings/memory-reflection-resolve-tool.js +150 -0
- package/dist/internal/engine/bindings/memory-stats-tool.d.ts +5 -0
- package/dist/internal/engine/bindings/memory-stats-tool.js +43 -0
- package/dist/internal/engine/bindings/memory-store-tool.d.ts +5 -0
- package/dist/internal/engine/bindings/memory-store-tool.js +165 -0
- package/dist/internal/engine/bindings/memory-tool-access.d.ts +16 -0
- package/dist/internal/engine/bindings/memory-tool-access.js +46 -0
- package/dist/internal/engine/bindings/memory-tool-formatting.d.ts +62 -0
- package/dist/internal/engine/bindings/memory-tool-formatting.js +106 -0
- package/dist/internal/engine/bindings/memory-tool-results.d.ts +8 -0
- package/dist/internal/engine/bindings/memory-tool-results.js +41 -0
- package/dist/internal/engine/bindings/memory-tool-schemas.d.ts +109 -0
- package/dist/internal/engine/bindings/memory-tool-schemas.js +75 -0
- package/dist/internal/engine/bindings/memory-update-tool.d.ts +5 -0
- package/dist/internal/engine/bindings/memory-update-tool.js +127 -0
- package/dist/internal/engine/bindings/sno-station-mem-ambient-learning-hook.d.ts +14 -0
- package/dist/internal/engine/bindings/sno-station-mem-ambient-learning-hook.js +248 -0
- package/dist/internal/engine/bindings/sno-station-mem-auto-recall-hook.d.ts +12 -0
- package/dist/internal/engine/bindings/sno-station-mem-auto-recall-hook.js +324 -0
- package/dist/internal/engine/bindings/sno-station-mem-hook-types.d.ts +56 -0
- package/dist/internal/engine/bindings/sno-station-mem-hook-types.js +1 -0
- package/dist/internal/engine/bindings/sno-station-mem-insight-distill-factory.d.ts +14 -0
- package/dist/internal/engine/bindings/sno-station-mem-insight-distill-factory.js +119 -0
- package/dist/internal/engine/bindings/sno-station-mem-message-transcript.d.ts +52 -0
- package/dist/internal/engine/bindings/sno-station-mem-message-transcript.js +145 -0
- package/dist/internal/engine/bindings/sno-station-mem-reset-hook.d.ts +10 -0
- package/dist/internal/engine/bindings/sno-station-mem-reset-hook.js +9 -0
- package/dist/internal/engine/bindings/sno-station-mem-runtime-mode.d.ts +36 -0
- package/dist/internal/engine/bindings/sno-station-mem-runtime-mode.js +104 -0
- package/dist/internal/engine/bindings/sno-station-mem-session-state.d.ts +10 -0
- package/dist/internal/engine/bindings/sno-station-mem-session-state.js +27 -0
- package/dist/internal/engine/eval/trace.d.ts +2 -0
- package/dist/internal/engine/eval/trace.js +151 -0
- package/dist/internal/engine/extraction/ambient-learning-text-normalizer.d.ts +2 -0
- package/dist/internal/engine/extraction/ambient-learning-text-normalizer.js +2 -0
- package/dist/internal/engine/extraction/atomic-memory-extraction.d.ts +2 -0
- package/dist/internal/engine/extraction/atomic-memory-extraction.js +2 -0
- package/dist/internal/engine/extraction/b-profile-section-dictionary-provider.d.ts +2 -0
- package/dist/internal/engine/extraction/b-profile-section-dictionary-provider.js +2 -0
- package/dist/internal/engine/extraction/b-profile-section-rekey.d.ts +49 -0
- package/dist/internal/engine/extraction/b-profile-section-rekey.js +730 -0
- package/dist/internal/engine/extraction/capture-policy-detector.d.ts +2 -0
- package/dist/internal/engine/extraction/capture-policy-detector.js +2 -0
- package/dist/internal/engine/extraction/embedding-provider-client.d.ts +2 -0
- package/dist/internal/engine/extraction/embedding-provider-client.js +2 -0
- package/dist/internal/engine/extraction/extraction-text-sanitizer.d.ts +2 -0
- package/dist/internal/engine/extraction/extraction-text-sanitizer.js +114 -0
- package/dist/internal/engine/extraction/memory-metadata-codec.d.ts +3 -0
- package/dist/internal/engine/extraction/memory-metadata-codec.js +2 -0
- package/dist/internal/engine/extraction/memory-temporality-classifier.d.ts +2 -0
- package/dist/internal/engine/extraction/memory-temporality-classifier.js +2 -0
- package/dist/internal/engine/i18n/all-resources.d.ts +2 -0
- package/dist/internal/engine/i18n/all-resources.js +2 -0
- package/dist/internal/engine/i18n/locales.d.ts +2 -0
- package/dist/internal/engine/i18n/locales.js +19 -0
- package/dist/internal/engine/i18n/res/_types.d.ts +2 -0
- package/dist/internal/engine/i18n/res/_types.js +11 -0
- package/dist/internal/engine/observability/adapter.d.ts +2 -0
- package/dist/internal/engine/observability/adapter.js +2 -0
- package/dist/internal/engine/observability/best-effort.d.ts +2 -0
- package/dist/internal/engine/observability/best-effort.js +41 -0
- package/dist/internal/engine/observability/early-diagnostics.js +2 -0
- package/dist/internal/engine/observability/memory-snapshot.d.ts +2 -0
- package/dist/internal/engine/observability/memory-snapshot.js +44 -0
- package/dist/internal/engine/observability/observable-embedder.d.ts +2 -0
- package/dist/internal/engine/observability/observable-embedder.js +43 -0
- package/dist/internal/engine/observability/observable-llm-client.d.ts +2 -0
- package/dist/internal/engine/observability/observable-llm-client.js +98 -0
- package/dist/internal/engine/observability/observable-memory-store.d.ts +2 -0
- package/dist/internal/engine/observability/observable-memory-store.js +701 -0
- package/dist/internal/engine/observability/observable-retriever.d.ts +2 -0
- package/dist/internal/engine/observability/observable-retriever.js +95 -0
- package/dist/internal/engine/observability/runtime-diagnostics.d.ts +13 -0
- package/dist/internal/engine/observability/runtime-diagnostics.js +96 -0
- package/dist/internal/engine/observability/session-registry.d.ts +2 -0
- package/dist/internal/engine/observability/session-registry.js +58 -0
- package/dist/internal/engine/observability/version-metadata.d.ts +7 -0
- package/dist/internal/engine/observability/version-metadata.js +59 -0
- package/dist/internal/engine/operations/daily-spend-estimator.d.ts +2 -0
- package/dist/internal/engine/operations/daily-spend-estimator.js +97 -0
- package/dist/internal/engine/operations/learning-file-maintenance.d.ts +2 -0
- package/dist/internal/engine/operations/learning-file-maintenance.js +255 -0
- package/dist/internal/engine/operations/memory-tier-promoter.d.ts +2 -0
- package/dist/internal/engine/operations/memory-tier-promoter.js +95 -0
- package/dist/internal/engine/operations/runtime-audit-log.d.ts +3 -0
- package/dist/internal/engine/operations/runtime-audit-log.js +3 -0
- package/dist/internal/engine/provider/canonical-memory-corpus.d.ts +38 -0
- package/dist/internal/engine/provider/canonical-memory-corpus.js +253 -0
- package/dist/internal/engine/provider/provider-registration.d.ts +39 -0
- package/dist/internal/engine/provider/provider-registration.js +2 -0
- package/dist/internal/engine/provider/provider-row-renderer.d.ts +7 -0
- package/dist/internal/engine/provider/provider-row-renderer.js +53 -0
- package/dist/internal/engine/provider/provider-types.d.ts +2 -0
- package/dist/internal/engine/provider/provider-types.js +1 -0
- package/dist/internal/engine/reflection/daily-log-generator.d.ts +2 -0
- package/dist/internal/engine/reflection/daily-log-generator.js +2 -0
- package/dist/internal/engine/reflection/reflection-command-hooks.d.ts +2 -0
- package/dist/internal/engine/reflection/reflection-command-hooks.js +2 -0
- package/dist/internal/engine/reflection/reflection-deps.d.ts +2 -0
- package/dist/internal/engine/reflection/reflection-deps.js +1 -0
- package/dist/internal/engine/reflection/reflection-embedded-generator.d.ts +2 -0
- package/dist/internal/engine/reflection/reflection-embedded-generator.js +2 -0
- package/dist/internal/engine/reflection/reflection-injection-hooks.d.ts +2 -0
- package/dist/internal/engine/reflection/reflection-injection-hooks.js +219 -0
- package/dist/internal/engine/reflection/reflection-lifecycle-hooks.d.ts +2 -0
- package/dist/internal/engine/reflection/reflection-lifecycle-hooks.js +63 -0
- package/dist/internal/engine/reflection/strategy-hook-runner.d.ts +4 -0
- package/dist/internal/engine/reflection/strategy-hook-runner.js +3 -0
- package/dist/internal/engine/rem/classifier.d.ts +2 -0
- package/dist/internal/engine/rem/classifier.js +180 -0
- package/dist/internal/engine/rem/index.d.ts +4 -0
- package/dist/internal/engine/rem/index.js +2551 -0
- package/dist/internal/engine/rem/types.d.ts +2 -0
- package/dist/internal/engine/rem/types.js +40 -0
- package/dist/internal/engine/retrieval/access-tracker.d.ts +2 -0
- package/dist/internal/engine/retrieval/access-tracker.js +314 -0
- package/dist/internal/engine/retrieval/intent-analyzer.d.ts +2 -0
- package/dist/internal/engine/retrieval/intent-analyzer.js +27 -0
- package/dist/internal/engine/retrieval/retrieval-gate.d.ts +2 -0
- package/dist/internal/engine/retrieval/retrieval-gate.js +47 -0
- package/dist/internal/engine/retrieval/retriever.d.ts +2 -0
- package/dist/internal/engine/retrieval/retriever.js +2 -0
- package/dist/internal/engine/security/memory-scope-policy.d.ts +2 -0
- package/dist/internal/engine/security/memory-scope-policy.js +2 -0
- package/dist/internal/engine/security/redact.d.ts +2 -0
- package/dist/internal/engine/security/redact.js +34 -0
- package/dist/internal/engine/security/scopes.d.ts +3 -0
- package/dist/internal/engine/security/scopes.js +3 -0
- package/dist/internal/engine/shared/errors.d.ts +2 -0
- package/dist/internal/engine/shared/errors.js +66 -0
- package/dist/internal/engine/shared/i18n-text.d.ts +2 -0
- package/dist/internal/engine/shared/i18n-text.js +58 -0
- package/dist/internal/engine/shared/iso-date-time.d.ts +16 -0
- package/dist/internal/engine/shared/iso-date-time.js +40 -0
- package/dist/internal/engine/shared/lru.d.ts +2 -0
- package/dist/internal/engine/shared/lru.js +30 -0
- package/dist/internal/engine/shared/memory-kind-policy.d.ts +2 -0
- package/dist/internal/engine/shared/memory-kind-policy.js +64 -0
- package/dist/internal/engine/shared/paths.d.ts +3 -0
- package/dist/internal/engine/shared/paths.js +2 -0
- package/dist/internal/engine/shared/types.d.ts +3 -0
- package/dist/internal/engine/shared/types.js +25 -0
- package/dist/internal/engine/shared/utils.d.ts +2 -0
- package/dist/internal/engine/shared/utils.js +175 -0
- package/dist/internal/engine/telemetry/memory-telemetry-api.d.ts +134 -0
- package/dist/internal/engine/telemetry/memory-telemetry-api.js +730 -0
- package/dist/internal/engine/telemetry/memory-telemetry-observability.d.ts +24 -0
- package/dist/internal/engine/telemetry/memory-telemetry-observability.js +133 -0
- package/dist/internal/engine/telemetry/memory-telemetry-outbox.d.ts +2 -0
- package/dist/internal/engine/telemetry/memory-telemetry-outbox.js +443 -0
- package/dist/internal/engine/telemetry/memory-telemetry-types.d.ts +2 -0
- package/dist/internal/engine/telemetry/memory-telemetry-types.js +21 -0
- package/dist/internal/engine/telemetry/observe-ledger.d.ts +30 -0
- package/dist/internal/engine/telemetry/observe-ledger.js +111 -0
- package/dist/internal/model/agent-llm-port.d.ts +2 -0
- package/dist/internal/model/agent-llm-port.js +1 -0
- package/dist/internal/model/llm-client.d.ts +3 -0
- package/dist/internal/model/llm-client.js +3 -0
- package/dist/internal/model/llm-failure.d.ts +26 -0
- package/dist/internal/model/llm-failure.js +63 -0
- package/dist/internal/model/llm-mode-routing.d.ts +22 -0
- package/dist/internal/model/llm-mode-routing.js +37 -0
- package/dist/internal/store/atomic-memory-cutover-sql.d.ts +13 -0
- package/dist/internal/store/atomic-memory-cutover-sql.js +237 -0
- package/dist/internal/store/connection.d.ts +2 -0
- package/dist/internal/store/connection.js +2 -0
- package/dist/internal/store/data-bootstrap.d.ts +2 -0
- package/dist/internal/store/data-bootstrap.js +2 -0
- package/dist/internal/store/data-paths.d.ts +2 -0
- package/dist/internal/store/data-paths.js +65 -0
- package/dist/internal/store/maintenance.d.ts +75 -0
- package/dist/internal/store/maintenance.js +1401 -0
- package/dist/internal/store/memory-kinds-cutover-migrator.d.ts +28 -0
- package/dist/internal/store/memory-kinds-cutover-migrator.js +415 -0
- package/dist/internal/store/migrations.d.ts +27 -0
- package/dist/internal/store/migrations.js +201 -0
- package/dist/internal/store/sqlite-runtime.d.ts +2 -0
- package/dist/internal/store/sqlite-runtime.js +109 -0
- package/dist/internal/store/store.d.ts +3 -0
- package/dist/internal/store/store.js +2 -0
- package/dist/learning-file-maintenance-9b-Ut6Kv.d.ts +30 -0
- package/dist/llm-client-BSYUAcUb.js +1374 -0
- package/dist/llm-client-DOTSGzhe.d.ts +51 -0
- package/dist/locales-JxGmkG6t.d.ts +7 -0
- package/dist/log-site-catalog.generated-BMKWj-Nx.js +2321 -0
- package/dist/log-site-catalog.json +1 -0
- package/dist/lru-DyoaTyIs.d.ts +14 -0
- package/dist/memdump.d.ts +52 -0
- package/dist/memdump.js +163 -0
- package/dist/memory-entry-projector-DHo8jH10.js +1015 -0
- package/dist/memory-kind-policy-BiuRjSJc.d.ts +23 -0
- package/dist/memory-metadata-codec-DL1sbJiN.js +577 -0
- package/dist/memory-metadata-codec-DUvsYe_X.d.ts +68 -0
- package/dist/memory-runtime-DqUS4cWi.js +19459 -0
- package/dist/memory-scope-policy-1gg5QlWZ.d.ts +67 -0
- package/dist/memory-scope-policy-DICSCURA.js +283 -0
- package/dist/memory-snapshot-CqY1BUUA.d.ts +16 -0
- package/dist/memory-source-order-YMRoeDLd.js +134 -0
- package/dist/memory-store-base-CQ_bHlYA.d.ts +1094 -0
- package/dist/memory-store-shared-DqpWJXZZ.js +158 -0
- package/dist/memory-store-write-validation-B_hDG2zf.js +243 -0
- package/dist/memory-telemetry-incidents-BWA9kcZx.js +8 -0
- package/dist/memory-telemetry-metadata-CA7W8tK6.js +77 -0
- package/dist/memory-telemetry-outbox-ddTdImt7.d.ts +74 -0
- package/dist/memory-telemetry-receipts-MAJecvXQ.d.ts +34 -0
- package/dist/memory-telemetry-receipts-YsxbwOAf.js +91 -0
- package/dist/memory-telemetry-types-CufVmrCs.d.ts +14 -0
- package/dist/memory-temporality-classifier-BUjXG5o7.js +184 -0
- package/dist/memory-temporality-classifier-Brht5jyK.d.ts +43 -0
- package/dist/memory-tier-promoter-CqGxCkue.d.ts +43 -0
- package/dist/memory-tool-dependencies-CatGzI9R.d.ts +14 -0
- package/dist/memory-tool-dependencies-DjUoelnm.js +19 -0
- package/dist/model-call-table-CspopkBg.js +563 -0
- package/dist/model-reply-text-CKopStP2.js +334 -0
- package/dist/observable-embedder-CNNe9gkg.d.ts +17 -0
- package/dist/observable-llm-client-CRuKlefQ.d.ts +21 -0
- package/dist/observable-memory-store-CnGDpOY5.d.ts +73 -0
- package/dist/observable-retriever-Cooc62Qw.d.ts +23 -0
- package/dist/operation-cancellation-CTKWwNHk.js +81 -0
- package/dist/paths-qVF-5uBq.d.ts +1 -0
- package/dist/plugin-config-schema-C0UUbOmG.d.ts +413 -0
- package/dist/plugin-config-schema-YJLyIIcJ.js +361 -0
- package/dist/profile-TWPJ_myr.d.ts +13 -0
- package/dist/profile-section-writer-BhGKPBjw.js +2149 -0
- package/dist/provider-registration-CF1tg8Xy.js +721 -0
- package/dist/provider-runtime-types-DFhtdA6d.d.ts +134 -0
- package/dist/provider-types-CZGm6v88.d.ts +23 -0
- package/dist/redact-BTIZcHQL.d.ts +5 -0
- package/dist/reflection-command-hooks-ClUl-iv6.js +644 -0
- package/dist/reflection-command-hooks-G1-dI2Av.d.ts +32 -0
- package/dist/reflection-deps-Ccm2Jf5W.d.ts +19 -0
- package/dist/reflection-embedded-generator-BRvqV5N-.js +130 -0
- package/dist/reflection-embedded-generator-kfQfKEYV.d.ts +14 -0
- package/dist/reflection-injection-hooks-CS_uhjoD.d.ts +43 -0
- package/dist/reflection-lifecycle-hooks-CRSpUdLi.d.ts +27 -0
- package/dist/relation-dictionary-Cev1kyLe.js +4248 -0
- package/dist/release-download.d.ts +8 -0
- package/dist/release-download.js +32 -0
- package/dist/relevant-memories-context-m8nuFZcM.js +23 -0
- package/dist/rem-batch-executor-Dg2HgROM.js +2 -0
- package/dist/rem-batch-executor-DuC4nYyf.js +31640 -0
- package/dist/rem-consumer-retrieval-CR0dtcHR.js +246 -0
- package/dist/rem-trigger-Hn6od8IV.js +730 -0
- package/dist/resolver-Dy-hnjCR.js +103 -0
- package/dist/results-C3ZLroMb.js +140 -0
- package/dist/retrieval-gate-Bx2Zg7jD.d.ts +31 -0
- package/dist/retriever-Dxss5fRv.d.ts +469 -0
- package/dist/retriever-ZoLvPRHq.js +1757 -0
- package/dist/rolldown-runtime-D7D4PA-g.js +13 -0
- package/dist/routes-DLHT__Ay.js +60 -0
- package/dist/runtime-audit-log-BCeujlEp.js +430 -0
- package/dist/runtime-audit-log-zpQiR9HT.d.ts +149 -0
- package/dist/scopes-CSOn6zdd.js +71 -0
- package/dist/scopes-CtrY-YQ3.d.ts +23 -0
- package/dist/server-CSvxchqg.js +1412 -0
- package/dist/session-registry-DBoE7AvZ.d.ts +14 -0
- package/dist/sidecar/main.d.ts +1 -0
- package/dist/sidecar/main.js +56 -0
- package/dist/signed-registry-constants-Bv0fgZ8Q.d.ts +12 -0
- package/dist/signed-registry-constants-yq3DlG3B.js +82 -0
- package/dist/sno-station-mem-runtime-dependencies-CoWqUMbK.js +30 -0
- package/dist/sno-station-mem-runtime-dependencies-DHESIuPn.d.ts +31 -0
- package/dist/sqlite-runtime-L_ywirTm.d.ts +58 -0
- package/dist/store-C8sUTssE.d.ts +27 -0
- package/dist/store-nPql26YM.js +5830 -0
- package/dist/strategy-hook-runner-Bxy851Xx.d.ts +16 -0
- package/dist/strategy-hook-runner-Djify5Rw.js +110 -0
- package/dist/token-counter-BD1f48Km.js +73 -0
- package/dist/trace-C1JxMfs-.d.ts +86 -0
- package/dist/types-CWdhXLxr.d.ts +477 -0
- package/dist/types-pVomjzqR.d.ts +132 -0
- package/dist/unplaced-candidate-migration-CbQHzcEM.js +940 -0
- package/dist/utils-Ctb9aevJ.d.ts +34 -0
- package/drizzle/0000_initial.sql +10 -0
- package/drizzle/0001_virtual_tables.sql +34 -0
- package/drizzle/0002_add_indexes.sql +11 -0
- package/drizzle/0003_content_hash_lookup_index.sql +2 -0
- package/drizzle/0004_chunk_swap.sql +60 -0
- package/drizzle/0005_drop_legacy_vec_fts.sql +10 -0
- package/drizzle/0006_fts_simple.sql +45 -0
- package/drizzle/0007_restore_simple_fts.sql +40 -0
- package/drizzle/0008_widen_memory_hash_unique_index.sql +4 -0
- package/drizzle/0009_memory_telemetry.sql +109 -0
- package/drizzle/0010_provider_mappings.sql +27 -0
- package/drizzle/0011_db_optimization.sql +82 -0
- package/drizzle/0012_b_profile_lanes.sql +9 -0
- package/drizzle/0013_extraction_timestamps.sql +6 -0
- package/drizzle/0014_extraction_timestamp_source.sql +3 -0
- package/drizzle/0015_task_lifecycle_state.sql +101 -0
- package/drizzle/0016_task_lifecycle_legacy_migration.sql +144 -0
- package/drizzle/0017_unplaced_candidates.sql +15 -0
- package/drizzle/0018_rem_batch_chassis.sql +71 -0
- package/drizzle/0019_profile_recovery_entries.sql +17 -0
- package/drizzle/0020_rem_scan_retry_progress.down.sql +14 -0
- package/drizzle/0020_rem_scan_retry_progress.sql +24 -0
- package/drizzle/0021_rem_two_facet_storage.down.sql +14 -0
- package/drizzle/0021_rem_two_facet_storage.sql +42 -0
- package/drizzle/0022_rem_facet_chunk_versions.down.sql +4 -0
- package/drizzle/0022_rem_facet_chunk_versions.sql +4 -0
- package/drizzle/0023_rem_defect_ledger.sql +40 -0
- package/drizzle/0024_rem_unbuilt_foundations.down.sql +13 -0
- package/drizzle/0024_rem_unbuilt_foundations.sql +65 -0
- package/drizzle/0025_rem_journal_operation_constraint.sql +28 -0
- package/drizzle/0026_rem_journal_detail.sql +1 -0
- package/drizzle/0027_legacy_nodix_namespace.sql +3 -0
- package/drizzle/0028_memory_timezone.sql +23 -0
- package/drizzle/0029_rem_write_hash_symmetry.sql +17 -0
- package/drizzle/0030_atomic_memory_additive.sql +105 -0
- package/drizzle/0031_atomic_extraction_ledger.sql +23 -0
- package/drizzle/0032_atomic_memory_entities.sql +12 -0
- package/drizzle/0033_todo_own_store.sql +99 -0
- package/drizzle/meta/0000_snapshot.json +87 -0
- package/drizzle/meta/_journal.json +244 -0
- package/generated/rem-operations.json +1 -0
- package/package.json +126 -4
- package/scripts/download-simple-tokenizer.mjs +254 -0
- package/settings.default.json +243 -0
- package/skills/extract-atomic-memory/SKILL.md +298 -0
- package/skills/extract-atomic-memory/references/account-for-turns.md +9 -0
- package/skills/extract-atomic-memory/references/calendar-meaning.md +64 -0
- package/skills/extract-atomic-memory/references/capture.md +120 -0
- package/skills/extract-atomic-memory/references/enrichment.md +92 -0
- package/skills/extract-atomic-memory/references/missing-durable-half.md +6 -0
- package/skills/extract-atomic-memory/references/progress-classification.md +21 -0
- package/skills/extract-atomic-memory/references/resolve-subject.md +19 -0
- package/skills/extract-atomic-memory/references/surrounding-context.md +10 -0
- package/skills/extract-atomic-memory/references/user-subject-guard.md +29 -0
- package/skills/judge-group-closure/SKILL.md +25 -0
- package/skills/judge-rem-memory-updates/SKILL.md +56 -0
- package/skills/judge-rem-memory-updates/agents/openai.yaml +4 -0
- package/skills/judge-retired-position/SKILL.md +25 -0
- package/skills/judge-task-lifecycle/SKILL.md +45 -0
- package/skills/key-state-attribute/SKILL.md +16 -0
- package/skills/resolve-entity-identity/SKILL.md +24 -0
- package/skills/resolve-relative-time/SKILL.md +22 -0
- package/sqlite-extensions/LICENSE-simple-tokenizer +28 -0
- package/sqlite-extensions/LICENSE-sqlite-vec +21 -0
- package/sqlite-extensions/linux-arm64/vec0.so +0 -0
- package/sqlite-extensions/linux-x64/vec0.so +0 -0
|
@@ -0,0 +1,1757 @@
|
|
|
1
|
+
import { CHUNKING_VERSION, DEFAULT_BM25_WEIGHT, DEFAULT_RECALL_LIFECYCLE, DEFAULT_RERANK_MODEL, DEFAULT_RERANK_TIMEOUT_MS, DEFAULT_VECTOR_WEIGHT, IMPORTANCE_WEIGHT_BASE, LIGHTWEIGHT_COSINE_WEIGHT, LIGHTWEIGHT_FUSION_WEIGHT, LIGHTWEIGHT_RERANK_PENALTY, MMR_LAMBDA, RECENCY_WEIGHT, RERANK_BLEND_CROSS, RERANK_BLEND_VECTOR, TIME_DECAY_FLOOR, experimentCandidatePoolOverride, experimentMmrDisabled } from "./internal/config/index.js";
|
|
2
|
+
import "./internal/engine/shared/types.js";
|
|
3
|
+
import { RetrievalError } from "./internal/engine/shared/errors.js";
|
|
4
|
+
import { o as isMemoryExpired, s as parseInsightMetadata } from "./memory-metadata-codec-DL1sbJiN.js";
|
|
5
|
+
import { clamp01 } from "./internal/engine/shared/utils.js";
|
|
6
|
+
import { i as memoryOperationSignal } from "./operation-cancellation-CTKWwNHk.js";
|
|
7
|
+
import { classifyLlmFailure } from "./internal/model/llm-failure.js";
|
|
8
|
+
import { appendQaTrace, computeConfigHash, isTraceEnabled } from "./internal/engine/eval/trace.js";
|
|
9
|
+
import { computeEffectiveHalfLife, parseAccessMetadata } from "./internal/engine/retrieval/access-tracker.js";
|
|
10
|
+
import { createLogger as createLogger$1, currentLogContext, privateLogReference, withLogContext } from "@snoai/utils/logger";
|
|
11
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
12
|
+
//#region src/engine/retrieval/retrieval-trace.ts
|
|
13
|
+
var StageLifecycle = class StageLifecycle {
|
|
14
|
+
name;
|
|
15
|
+
inputOrder;
|
|
16
|
+
inputSet;
|
|
17
|
+
openedAt;
|
|
18
|
+
constructor(name, entryIds, openedAt) {
|
|
19
|
+
this.name = name;
|
|
20
|
+
this.inputOrder = Array.from(new Set(entryIds));
|
|
21
|
+
this.inputSet = new Set(this.inputOrder);
|
|
22
|
+
this.openedAt = openedAt;
|
|
23
|
+
}
|
|
24
|
+
static open(name, entryIds, openedAt) {
|
|
25
|
+
return new StageLifecycle(name, entryIds, openedAt);
|
|
26
|
+
}
|
|
27
|
+
closeUnchanged(closedAt) {
|
|
28
|
+
return this.closeWith(this.inputOrder, void 0, closedAt);
|
|
29
|
+
}
|
|
30
|
+
closeWith(survivingIds, scores, closedAt, metadata) {
|
|
31
|
+
const base = {
|
|
32
|
+
name: this.name,
|
|
33
|
+
inputCount: this.inputSet.size,
|
|
34
|
+
outputCount: survivingIds.length,
|
|
35
|
+
droppedIds: this.findDroppedIds(survivingIds),
|
|
36
|
+
outputIds: [...survivingIds],
|
|
37
|
+
scoreRange: getScoreRange(scores),
|
|
38
|
+
durationMs: closedAt - this.openedAt
|
|
39
|
+
};
|
|
40
|
+
return metadata ? {
|
|
41
|
+
...base,
|
|
42
|
+
metadata
|
|
43
|
+
} : base;
|
|
44
|
+
}
|
|
45
|
+
findDroppedIds(survivingIds) {
|
|
46
|
+
const survivors = new Set(survivingIds);
|
|
47
|
+
return this.inputOrder.filter((id) => !survivors.has(id));
|
|
48
|
+
}
|
|
49
|
+
};
|
|
50
|
+
function getScoreRange(scores) {
|
|
51
|
+
if (!scores || scores.length === 0) return null;
|
|
52
|
+
let min = scores[0] ?? 0;
|
|
53
|
+
let max = min;
|
|
54
|
+
for (const score of scores.slice(1)) {
|
|
55
|
+
if (score < min) min = score;
|
|
56
|
+
if (score > max) max = score;
|
|
57
|
+
}
|
|
58
|
+
return [min, max];
|
|
59
|
+
}
|
|
60
|
+
function finalOutputCount(stages) {
|
|
61
|
+
return stages.at(-1)?.outputCount ?? 0;
|
|
62
|
+
}
|
|
63
|
+
function formatScoreRange(scoreRange) {
|
|
64
|
+
return scoreRange ? ` scores=[${scoreRange[0].toFixed(3)}, ${scoreRange[1].toFixed(3)}]` : "";
|
|
65
|
+
}
|
|
66
|
+
function formatDroppedIds(droppedIds) {
|
|
67
|
+
if (droppedIds.length === 0) return null;
|
|
68
|
+
if (droppedIds.length <= 5) return ` dropped: ${droppedIds.join(", ")}`;
|
|
69
|
+
return ` dropped: ${droppedIds.slice(0, 5).join(", ")} (+${droppedIds.length - 5} more)`;
|
|
70
|
+
}
|
|
71
|
+
var TraceCollector = class {
|
|
72
|
+
startedAtMs;
|
|
73
|
+
startedMonotonicMs;
|
|
74
|
+
results = [];
|
|
75
|
+
lifecycle = [];
|
|
76
|
+
lifecycleEnabled;
|
|
77
|
+
activeStage = null;
|
|
78
|
+
/**
|
|
79
|
+
* Initializes retrieval tracing collaborators while keeping runtime work in explicit methods.
|
|
80
|
+
*/
|
|
81
|
+
constructor(options) {
|
|
82
|
+
this.startedAtMs = Date.now();
|
|
83
|
+
this.startedMonotonicMs = performance.now();
|
|
84
|
+
this.lifecycleEnabled = options?.lifecycleEnabled === true;
|
|
85
|
+
}
|
|
86
|
+
/**
|
|
87
|
+
* Begin tracking a pipeline stage.
|
|
88
|
+
* @param name - Stage identifier (e.g. "vector_search")
|
|
89
|
+
* @param entryIds - IDs of entries entering this stage
|
|
90
|
+
*/
|
|
91
|
+
startStage(name, entryIds) {
|
|
92
|
+
this.closeActiveStageAsUnchanged();
|
|
93
|
+
this.activeStage = StageLifecycle.open(name, entryIds, performance.now());
|
|
94
|
+
}
|
|
95
|
+
/**
|
|
96
|
+
* End the current stage.
|
|
97
|
+
* @param survivingIds - IDs of entries that survived this stage
|
|
98
|
+
* @param scores - Optional scores for surviving entries (parallel to survivingIds)
|
|
99
|
+
* @param metadata - Optional structured annotations attached to this stage
|
|
100
|
+
*/
|
|
101
|
+
endStage(survivingIds, scores, metadata) {
|
|
102
|
+
const stage = this.activeStage;
|
|
103
|
+
if (!stage) return;
|
|
104
|
+
this.results.push(stage.closeWith(survivingIds, scores, performance.now(), metadata));
|
|
105
|
+
this.activeStage = null;
|
|
106
|
+
}
|
|
107
|
+
/**
|
|
108
|
+
* Finalize the trace and produce the complete RetrievalTrace object.
|
|
109
|
+
*/
|
|
110
|
+
finalize(query, mode) {
|
|
111
|
+
this.closeActiveStageAsUnchanged();
|
|
112
|
+
const base = {
|
|
113
|
+
query,
|
|
114
|
+
mode,
|
|
115
|
+
startedAt: this.startedAtMs,
|
|
116
|
+
stages: this.results,
|
|
117
|
+
finalCount: finalOutputCount(this.results),
|
|
118
|
+
totalMs: performance.now() - this.startedMonotonicMs
|
|
119
|
+
};
|
|
120
|
+
if (this.lifecycleEnabled && this.lifecycle.length > 0) return {
|
|
121
|
+
...base,
|
|
122
|
+
lifecycleEvents: this.lifecycle
|
|
123
|
+
};
|
|
124
|
+
return base;
|
|
125
|
+
}
|
|
126
|
+
/** Append a retention-scorer outcome to the lifecycle stream. */
|
|
127
|
+
writeRetentionTrace(event) {
|
|
128
|
+
if (!this.lifecycleEnabled) return;
|
|
129
|
+
this.lifecycle.push({
|
|
130
|
+
kind: "retention-score-computed",
|
|
131
|
+
at: Date.now(),
|
|
132
|
+
...event
|
|
133
|
+
});
|
|
134
|
+
}
|
|
135
|
+
/** Append an access-tracker update to the lifecycle stream. */
|
|
136
|
+
writeAccessTrace(event) {
|
|
137
|
+
if (!this.lifecycleEnabled) return;
|
|
138
|
+
this.lifecycle.push({
|
|
139
|
+
kind: "access-tracker-update",
|
|
140
|
+
at: Date.now(),
|
|
141
|
+
...event
|
|
142
|
+
});
|
|
143
|
+
}
|
|
144
|
+
/** Append a tier-promotion evaluation outcome to the lifecycle stream. */
|
|
145
|
+
writeTierTrace(event) {
|
|
146
|
+
if (!this.lifecycleEnabled) return;
|
|
147
|
+
this.lifecycle.push({
|
|
148
|
+
kind: "tier-promotion-evaluated",
|
|
149
|
+
at: Date.now(),
|
|
150
|
+
...event
|
|
151
|
+
});
|
|
152
|
+
}
|
|
153
|
+
/** Access collected lifecycle events (read-only). */
|
|
154
|
+
get lifecycleEvents() {
|
|
155
|
+
return this.lifecycle;
|
|
156
|
+
}
|
|
157
|
+
/**
|
|
158
|
+
* Produce a human-readable summary of the trace.
|
|
159
|
+
*/
|
|
160
|
+
summarize() {
|
|
161
|
+
const lines = [`Retrieval trace (${this.results.length} stages):`];
|
|
162
|
+
for (const stage of this.results) {
|
|
163
|
+
lines.push(` ${stage.name}: ${stage.inputCount} -> ${stage.outputCount} (-${stage.droppedIds.length}) ${stage.durationMs}ms${formatScoreRange(stage.scoreRange)}`);
|
|
164
|
+
const droppedLine = formatDroppedIds(stage.droppedIds);
|
|
165
|
+
if (droppedLine) lines.push(droppedLine);
|
|
166
|
+
}
|
|
167
|
+
lines.push(` total: ${performance.now() - this.startedMonotonicMs}ms, final count: ${finalOutputCount(this.results)}`);
|
|
168
|
+
return lines.join("\n");
|
|
169
|
+
}
|
|
170
|
+
/** Access collected stages (read-only). */
|
|
171
|
+
get stages() {
|
|
172
|
+
return this.results;
|
|
173
|
+
}
|
|
174
|
+
closeActiveStageAsUnchanged() {
|
|
175
|
+
const stage = this.activeStage;
|
|
176
|
+
if (!stage) return;
|
|
177
|
+
this.results.push(stage.closeUnchanged(performance.now()));
|
|
178
|
+
this.activeStage = null;
|
|
179
|
+
}
|
|
180
|
+
};
|
|
181
|
+
//#endregion
|
|
182
|
+
//#region src/engine/retrieval/retrieval-config.ts
|
|
183
|
+
/** @file retrieval-config.ts
|
|
184
|
+
* @purpose Defines retriever runtime configuration and request context contracts.
|
|
185
|
+
* @boundary Types and defaults only; no ranking execution.
|
|
186
|
+
*/
|
|
187
|
+
/**
|
|
188
|
+
* The serving-path validity default. This is the one named code
|
|
189
|
+
* point that supplies it: reading the default here is what makes the
|
|
190
|
+
* invalidated-row filter a property of the serving path rather than a parameter
|
|
191
|
+
* every caller has to remember, and the omission is no longer silent.
|
|
192
|
+
*
|
|
193
|
+
* An explicit value always wins — including `0`, the deliberate history read,
|
|
194
|
+
* where the store predicate becomes `invalidated_at > 0` and drops nothing.
|
|
195
|
+
*
|
|
196
|
+
* Scope is the retriever, never the store layer: six write-side consumers (dedup,
|
|
197
|
+
* reflection loops, observability, the REM batch executor) call the store search
|
|
198
|
+
* APIs directly and legitimately read invalidated rows. Do not move this
|
|
199
|
+
* down a layer.
|
|
200
|
+
*/
|
|
201
|
+
function withServingValidityDefault(context) {
|
|
202
|
+
if (context.excludeInvalidatedBefore !== void 0 && context.includeRefused !== void 0) return context;
|
|
203
|
+
return {
|
|
204
|
+
...context,
|
|
205
|
+
excludeInvalidatedBefore: context.excludeInvalidatedBefore ?? Date.now(),
|
|
206
|
+
includeRefused: context.includeRefused ?? true
|
|
207
|
+
};
|
|
208
|
+
}
|
|
209
|
+
const DEFAULT_RETRIEVAL_CONFIG = {
|
|
210
|
+
temporalWeighting: false,
|
|
211
|
+
mmrWindowOnly: false,
|
|
212
|
+
mode: "precision-recall",
|
|
213
|
+
vectorWeight: DEFAULT_VECTOR_WEIGHT,
|
|
214
|
+
bm25Weight: DEFAULT_BM25_WEIGHT,
|
|
215
|
+
minScore: 0,
|
|
216
|
+
rerank: "cross-encoder",
|
|
217
|
+
candidatePoolSize: 64,
|
|
218
|
+
recencyHalfLifeDays: 14,
|
|
219
|
+
recencyWeight: RECENCY_WEIGHT,
|
|
220
|
+
temporalExpiry: false,
|
|
221
|
+
temporalDecay: true,
|
|
222
|
+
rerankModel: DEFAULT_RERANK_MODEL,
|
|
223
|
+
rerankTimeoutMs: DEFAULT_RERANK_TIMEOUT_MS,
|
|
224
|
+
rerankProvider: "voyage",
|
|
225
|
+
lengthNormAnchor: 0,
|
|
226
|
+
hardMinScore: 0,
|
|
227
|
+
timeDecayHalfLifeDays: 60,
|
|
228
|
+
rerankBlendVector: RERANK_BLEND_VECTOR,
|
|
229
|
+
rerankBlendCross: RERANK_BLEND_CROSS,
|
|
230
|
+
lightweightFusionWeight: LIGHTWEIGHT_FUSION_WEIGHT,
|
|
231
|
+
lightweightCosineWeight: LIGHTWEIGHT_COSINE_WEIGHT,
|
|
232
|
+
importanceWeightBase: IMPORTANCE_WEIGHT_BASE,
|
|
233
|
+
timeDecayFloor: TIME_DECAY_FLOOR,
|
|
234
|
+
mmrLambda: MMR_LAMBDA,
|
|
235
|
+
recallLifecycle: DEFAULT_RECALL_LIFECYCLE
|
|
236
|
+
};
|
|
237
|
+
//#endregion
|
|
238
|
+
//#region src/engine/retrieval/retriever-core.ts
|
|
239
|
+
/**
|
|
240
|
+
* Enumerates every branch in `rerank()` that fell back to the pre-rerank
|
|
241
|
+
* candidate ordering instead of applying the cross-encoder score. Aggregated
|
|
242
|
+
* via {@link RetrievalStatsCollector} and surfaced as trace stage metadata
|
|
243
|
+
* so callers can attribute degraded ranking without parsing log lines.
|
|
244
|
+
*
|
|
245
|
+
* - `missing_api_key`: cross-encoder configured but no rerankApiKey set;
|
|
246
|
+
* call routes through the lightweight cosine path instead.
|
|
247
|
+
* - `no_endpoint`: provider has no known endpoint in `RERANK_DEFAULT_ENDPOINTS`
|
|
248
|
+
* and `rerankEndpoint` was not overridden; rerank was skipped.
|
|
249
|
+
* - `http_error`: provider responded with a non-OK status that did not map
|
|
250
|
+
* to a fatal RetrievalError (i.e. degraded but not propagated).
|
|
251
|
+
* - `invalid_response`: provider HTTP 2xx but response shape did not parse.
|
|
252
|
+
* - `timeout`: the per-call AbortSignal fired before the provider responded.
|
|
253
|
+
* - `request_error`: any other thrown error inside the fetch path.
|
|
254
|
+
* - `query_over_window`: the query alone fills the reranker's per-pair token window, so no
|
|
255
|
+
* document could be sent with it. Nothing was requested.
|
|
256
|
+
*/
|
|
257
|
+
const RERANK_FALLBACK_REASONS = [
|
|
258
|
+
"missing_api_key",
|
|
259
|
+
"no_endpoint",
|
|
260
|
+
"http_error",
|
|
261
|
+
"invalid_response",
|
|
262
|
+
"timeout",
|
|
263
|
+
"request_error",
|
|
264
|
+
"query_over_window"
|
|
265
|
+
];
|
|
266
|
+
var MemoryRetriever = class {
|
|
267
|
+
store;
|
|
268
|
+
embedder;
|
|
269
|
+
config;
|
|
270
|
+
_statsCollector;
|
|
271
|
+
_accessTracker;
|
|
272
|
+
_cachedConfigHash;
|
|
273
|
+
_retentionScorer;
|
|
274
|
+
_tierPromoter;
|
|
275
|
+
_recallLifecycle;
|
|
276
|
+
retrieve(_context) {
|
|
277
|
+
throw new Error("MemoryRetriever implementation modules were not loaded");
|
|
278
|
+
}
|
|
279
|
+
retrieveWithTrace(_context) {
|
|
280
|
+
throw new Error("MemoryRetriever implementation modules were not loaded");
|
|
281
|
+
}
|
|
282
|
+
/**
|
|
283
|
+
* Initializes precision recall retrieval ranking collaborators while keeping runtime work in explicit
|
|
284
|
+
* methods.
|
|
285
|
+
*/
|
|
286
|
+
constructor(store, embedder, _logger, config = DEFAULT_RETRIEVAL_CONFIG) {
|
|
287
|
+
this.store = store;
|
|
288
|
+
this.embedder = embedder;
|
|
289
|
+
this.config = config;
|
|
290
|
+
}
|
|
291
|
+
/** Enable aggregate retrieval statistics collection. */
|
|
292
|
+
setStatsCollector(collector) {
|
|
293
|
+
this._statsCollector = collector;
|
|
294
|
+
}
|
|
295
|
+
/** Get the stats collector (if set). */
|
|
296
|
+
getStatsCollector() {
|
|
297
|
+
return this._statsCollector;
|
|
298
|
+
}
|
|
299
|
+
/** Set the access tracker for recording manual retrieval accesses. */
|
|
300
|
+
setAccessTracker(tracker) {
|
|
301
|
+
this._accessTracker = tracker;
|
|
302
|
+
}
|
|
303
|
+
/**
|
|
304
|
+
* Install the Phase 0 §5 tier-promoter dependency. Pure dependency injection;
|
|
305
|
+
* the post-retrieval evaluation hook only fires when both this setter has
|
|
306
|
+
* been called and `recallLifecycle.tierPromoter` is true.
|
|
307
|
+
*/
|
|
308
|
+
setTierPromoter(promoter) {
|
|
309
|
+
this._tierPromoter = promoter;
|
|
310
|
+
}
|
|
311
|
+
/**
|
|
312
|
+
* Resolve the `recallLifecycle` block once at composition time so the hot
|
|
313
|
+
* retrieval path reads it without re-parsing config. All gates default
|
|
314
|
+
* false in `recallLifecycleSchema`, preserving the no-behavior-change
|
|
315
|
+
* Phase 0 contract for callers that never call this setter.
|
|
316
|
+
*/
|
|
317
|
+
setRecallLifecycle(config) {
|
|
318
|
+
this._recallLifecycle = config;
|
|
319
|
+
}
|
|
320
|
+
/** Returns the immutable retrieval config for diagnostics and tests. */
|
|
321
|
+
getConfig() {
|
|
322
|
+
return this.config;
|
|
323
|
+
}
|
|
324
|
+
};
|
|
325
|
+
/** Creates the retriever with store, embedding, ranking config, and optional caller logger slot. */
|
|
326
|
+
function createRetriever(store, embedder, logger, config = DEFAULT_RETRIEVAL_CONFIG) {
|
|
327
|
+
return new MemoryRetriever(store, embedder, logger, config);
|
|
328
|
+
}
|
|
329
|
+
//#endregion
|
|
330
|
+
//#region src/engine/retrieval/retriever-query-tools.ts
|
|
331
|
+
Object.assign(MemoryRetriever.prototype, {
|
|
332
|
+
throwIfAborted(context) {
|
|
333
|
+
if (!context.signal?.aborted) return;
|
|
334
|
+
const reason = context.signal.reason;
|
|
335
|
+
throw reason instanceof Error ? reason : /* @__PURE__ */ new Error("retrieval aborted");
|
|
336
|
+
},
|
|
337
|
+
filterExpired(results) {
|
|
338
|
+
if (!this.config.temporalExpiry) return results;
|
|
339
|
+
return results.filter((r) => !this.isEntryExpired(r.entry));
|
|
340
|
+
},
|
|
341
|
+
filterExpiredCandidates(results) {
|
|
342
|
+
if (!this.config.temporalExpiry) return results;
|
|
343
|
+
return results.filter((r) => !this.isEntryExpired(r.entry));
|
|
344
|
+
},
|
|
345
|
+
isEntryExpired(entry) {
|
|
346
|
+
const metadata = parseInsightMetadata(entry.metadata, entry);
|
|
347
|
+
return isMemoryExpired(metadata);
|
|
348
|
+
}
|
|
349
|
+
});
|
|
350
|
+
//#endregion
|
|
351
|
+
//#region src/engine/retrieval/retrieval-scoring-utils.ts
|
|
352
|
+
const log = createLogger$1("sno-station-mem:retriever");
|
|
353
|
+
function dotProduct(a, b) {
|
|
354
|
+
let sum = 0;
|
|
355
|
+
const len = a.length;
|
|
356
|
+
for (let i = 0; i < len; i += 1) sum += (a[i] ?? 0) * (b[i] ?? 0);
|
|
357
|
+
return sum;
|
|
358
|
+
}
|
|
359
|
+
function collectParallelStage(results, key, fallbackIds) {
|
|
360
|
+
const omitted = [];
|
|
361
|
+
const values = [];
|
|
362
|
+
for (const [index, result] of results.entries()) {
|
|
363
|
+
const value = result[key];
|
|
364
|
+
if (typeof value !== "number" || !Number.isFinite(value)) omitted.push(fallbackIds[index] ?? "");
|
|
365
|
+
else values.push(value);
|
|
366
|
+
}
|
|
367
|
+
if (omitted.length === 0) return {
|
|
368
|
+
values,
|
|
369
|
+
omitted
|
|
370
|
+
};
|
|
371
|
+
return {
|
|
372
|
+
values: null,
|
|
373
|
+
omitted
|
|
374
|
+
};
|
|
375
|
+
}
|
|
376
|
+
//#endregion
|
|
377
|
+
//#region src/engine/retrieval/retriever-search-modes.ts
|
|
378
|
+
Object.assign(MemoryRetriever.prototype, {
|
|
379
|
+
async aggregationComplete(context, trace) {
|
|
380
|
+
trace?.startStage("aggregation_evidence", []);
|
|
381
|
+
const results = await this.store.searchAggregationEvidence({
|
|
382
|
+
limit: 640,
|
|
383
|
+
projectIdFilter: context.scopeFilter,
|
|
384
|
+
category: context.category,
|
|
385
|
+
includeRefused: context.includeRefused,
|
|
386
|
+
includeRetired: context.includeRetired,
|
|
387
|
+
aggregation: context.aggregation,
|
|
388
|
+
facetPolicy: context.facetPolicy,
|
|
389
|
+
...context.excludeInvalidatedBefore !== void 0 && { excludeInvalidatedBefore: context.excludeInvalidatedBefore },
|
|
390
|
+
...context.excludeSuperseded === void 0 ? {} : { excludeSuperseded: context.excludeSuperseded }
|
|
391
|
+
});
|
|
392
|
+
this.throwIfAborted(context);
|
|
393
|
+
const mapped = results.map((result) => ({
|
|
394
|
+
entry: result.entry,
|
|
395
|
+
score: result.score,
|
|
396
|
+
sources: {},
|
|
397
|
+
chunkId: result.chunkId,
|
|
398
|
+
chunkIndex: result.chunkIndex,
|
|
399
|
+
bestChunkScore: result.bestChunkScore,
|
|
400
|
+
denseScore: result.score,
|
|
401
|
+
snippet: result.snippet,
|
|
402
|
+
eventIdentity: result.eventIdentity,
|
|
403
|
+
scopeRowCount: result.scopeRowCount,
|
|
404
|
+
aggregationIncomplete: result.aggregationIncomplete
|
|
405
|
+
}));
|
|
406
|
+
trace?.endStage(mapped.map((result) => result.entry.id), mapped.map((result) => result.score));
|
|
407
|
+
log.info("aggregation evidence served", {
|
|
408
|
+
rows: mapped.length,
|
|
409
|
+
scopeRows: mapped[0]?.scopeRowCount ?? 0,
|
|
410
|
+
readingBudget: context.limit,
|
|
411
|
+
bounded: true,
|
|
412
|
+
scopes: context.scopeFilter?.length ?? 0,
|
|
413
|
+
...context.category ? { category: context.category } : {}
|
|
414
|
+
}, {
|
|
415
|
+
event_name: "memory.retriever_search_modes.diagnostic",
|
|
416
|
+
file: "packages/memory/src/engine/retrieval/retriever-search-modes.ts",
|
|
417
|
+
function: "aggregationComplete",
|
|
418
|
+
site_id: "retrieval.retriever-search-modes.aggregationComplete.5a3b7f9337"
|
|
419
|
+
});
|
|
420
|
+
return mapped;
|
|
421
|
+
},
|
|
422
|
+
async vectorOnly(context, trace) {
|
|
423
|
+
const vector = await this.embedder.embed(context.query);
|
|
424
|
+
this.throwIfAborted(context);
|
|
425
|
+
const searchLimit = this.config.temporalExpiry ? Math.max(this.config.candidatePoolSize, context.limit * 2) : context.limit;
|
|
426
|
+
trace?.startStage("vector_search", []);
|
|
427
|
+
const results = await this.store.searchSemantic(vector, {
|
|
428
|
+
limit: searchLimit,
|
|
429
|
+
projectIdFilter: context.scopeFilter,
|
|
430
|
+
category: context.category,
|
|
431
|
+
includeRefused: context.includeRefused,
|
|
432
|
+
includeRetired: context.includeRetired,
|
|
433
|
+
facetPolicy: context.facetPolicy,
|
|
434
|
+
...context.excludeInvalidatedBefore !== void 0 && { excludeInvalidatedBefore: context.excludeInvalidatedBefore },
|
|
435
|
+
...context.excludeSuperseded === void 0 ? {} : { excludeSuperseded: context.excludeSuperseded }
|
|
436
|
+
});
|
|
437
|
+
this.throwIfAborted(context);
|
|
438
|
+
let mapped = results.map((result, index) => ({
|
|
439
|
+
entry: result.entry,
|
|
440
|
+
score: result.score,
|
|
441
|
+
sources: { vector: {
|
|
442
|
+
score: result.score,
|
|
443
|
+
rank: index + 1
|
|
444
|
+
} },
|
|
445
|
+
chunkId: result.chunkId,
|
|
446
|
+
chunkIndex: result.chunkIndex,
|
|
447
|
+
bestChunkScore: result.bestChunkScore,
|
|
448
|
+
snippet: result.snippet,
|
|
449
|
+
denseScore: result.score
|
|
450
|
+
}));
|
|
451
|
+
trace?.endStage(mapped.map((r) => r.entry.id), mapped.map((r) => r.score));
|
|
452
|
+
trace?.startStage("expiry_filter", mapped.map((result) => result.entry.id));
|
|
453
|
+
mapped = this.filterExpired(mapped);
|
|
454
|
+
trace?.endStage(mapped.map((result) => result.entry.id));
|
|
455
|
+
mapped = this.applyScoringPipeline(mapped, trace, context.limit);
|
|
456
|
+
trace?.startStage("min_score_filter", mapped.map((result) => result.entry.id));
|
|
457
|
+
mapped = mapped.filter((result) => result.score >= (context.minScore ?? this.config.minScore));
|
|
458
|
+
trace?.endStage(mapped.map((result) => result.entry.id), mapped.map((result) => result.score));
|
|
459
|
+
trace?.startStage("limit_slice", mapped.map((result) => result.entry.id));
|
|
460
|
+
const served = mapped.slice(0, context.limit);
|
|
461
|
+
trace?.endStage(served.map((result) => result.entry.id), served.map((result) => result.score));
|
|
462
|
+
return served;
|
|
463
|
+
},
|
|
464
|
+
async precisionRecall(context, trace) {
|
|
465
|
+
trace?.startStage("embed_query", []);
|
|
466
|
+
const queryVector = await this.embedder.embed(context.query);
|
|
467
|
+
trace?.endStage([]);
|
|
468
|
+
this.throwIfAborted(context);
|
|
469
|
+
const bm25Query = context.query;
|
|
470
|
+
const sharedSearchOptions = {
|
|
471
|
+
limit: experimentCandidatePoolOverride() ?? Math.min(Math.max(this.config.candidatePoolSize, context.limit * 2), 2048),
|
|
472
|
+
projectIdFilter: context.scopeFilter,
|
|
473
|
+
category: context.category,
|
|
474
|
+
includeRefused: context.includeRefused,
|
|
475
|
+
includeRetired: context.includeRetired,
|
|
476
|
+
facetPolicy: context.facetPolicy,
|
|
477
|
+
...context.excludeInvalidatedBefore !== void 0 && { excludeInvalidatedBefore: context.excludeInvalidatedBefore },
|
|
478
|
+
...context.excludeSuperseded === void 0 ? {} : { excludeSuperseded: context.excludeSuperseded }
|
|
479
|
+
};
|
|
480
|
+
trace?.startStage("parallel_search", []);
|
|
481
|
+
const [vectorSettled, keywordSettled] = await Promise.allSettled([this.store.searchSemantic(queryVector, sharedSearchOptions), this.store.searchKeyword(bm25Query, sharedSearchOptions)]);
|
|
482
|
+
this.throwIfAborted(context);
|
|
483
|
+
const errorMessage = (reason) => reason instanceof Error ? reason.message : String(reason);
|
|
484
|
+
const vectorError = vectorSettled.status === "rejected" ? errorMessage(vectorSettled.reason) : void 0;
|
|
485
|
+
const keywordError = keywordSettled.status === "rejected" ? errorMessage(keywordSettled.reason) : void 0;
|
|
486
|
+
const branchMetadata = {
|
|
487
|
+
vectorStatus: vectorSettled.status,
|
|
488
|
+
vectorCount: vectorSettled.status === "fulfilled" ? vectorSettled.value.length : 0,
|
|
489
|
+
keywordStatus: keywordSettled.status,
|
|
490
|
+
keywordCount: keywordSettled.status === "fulfilled" ? keywordSettled.value.length : 0,
|
|
491
|
+
...vectorError === void 0 ? {} : { vectorError },
|
|
492
|
+
...keywordError === void 0 ? {} : { keywordError }
|
|
493
|
+
};
|
|
494
|
+
if (vectorSettled.status === "rejected" && keywordSettled.status === "rejected") {
|
|
495
|
+
trace?.endStage([], [], branchMetadata);
|
|
496
|
+
throw new RetrievalError("Both vector and keyword search failed", vectorSettled.reason);
|
|
497
|
+
}
|
|
498
|
+
let vectorResults = vectorSettled.status === "fulfilled" ? vectorSettled.value : [];
|
|
499
|
+
if (vectorSettled.status === "rejected") log.warn("vector search branch failed, using keyword-only", { error: vectorError }, {
|
|
500
|
+
event_name: "memory.retriever_search_modes.diagnostic",
|
|
501
|
+
file: "packages/memory/src/engine/retrieval/retriever-search-modes.ts",
|
|
502
|
+
function: "precisionRecall",
|
|
503
|
+
site_id: "retrieval.retriever-search-modes.precisionRecall.97b8770f50"
|
|
504
|
+
});
|
|
505
|
+
let keywordResults = keywordSettled.status === "fulfilled" ? keywordSettled.value : [];
|
|
506
|
+
if (keywordSettled.status === "rejected") log.warn("keyword search branch failed, using vector-only", { error: keywordError }, {
|
|
507
|
+
event_name: "memory.retriever_search_modes.diagnostic",
|
|
508
|
+
file: "packages/memory/src/engine/retrieval/retriever-search-modes.ts",
|
|
509
|
+
function: "precisionRecall",
|
|
510
|
+
site_id: "retrieval.retriever-search-modes.precisionRecall.c20ade60c4"
|
|
511
|
+
});
|
|
512
|
+
if (trace) {
|
|
513
|
+
const allIds = [...vectorResults.map((r) => r.entry.id), ...keywordResults.map((r) => r.entry.id)];
|
|
514
|
+
trace.endStage(allIds, [...vectorResults.map((r) => r.score), ...keywordResults.map((r) => r.score)], branchMetadata);
|
|
515
|
+
}
|
|
516
|
+
const beforeExpiry = [.../* @__PURE__ */ new Set([...vectorResults.map((result) => result.entry.id), ...keywordResults.map((result) => result.entry.id)])];
|
|
517
|
+
trace?.startStage("expiry_filter", beforeExpiry);
|
|
518
|
+
vectorResults = this.filterExpiredCandidates(vectorResults);
|
|
519
|
+
keywordResults = this.filterExpiredCandidates(keywordResults);
|
|
520
|
+
trace?.endStage([.../* @__PURE__ */ new Set([...vectorResults.map((result) => result.entry.id), ...keywordResults.map((result) => result.entry.id)])]);
|
|
521
|
+
trace?.startStage("rrf_fusion", [...vectorResults.map((r) => r.entry.id), ...keywordResults.map((r) => r.entry.id)]);
|
|
522
|
+
let fused = this.rrfFuse(vectorResults, keywordResults);
|
|
523
|
+
trace?.endStage(fused.map((r) => r.entry.id), fused.map((r) => r.score));
|
|
524
|
+
log.debug("precision recall search", {
|
|
525
|
+
vectorCount: vectorResults.length,
|
|
526
|
+
keywordCount: keywordResults.length,
|
|
527
|
+
fusedCount: fused.length,
|
|
528
|
+
queryExpanded: bm25Query !== context.query
|
|
529
|
+
}, {
|
|
530
|
+
event_name: "memory.retriever_search_modes.diagnostic",
|
|
531
|
+
file: "packages/memory/src/engine/retrieval/retriever-search-modes.ts",
|
|
532
|
+
function: "precisionRecall",
|
|
533
|
+
site_id: "retrieval.retriever-search-modes.precisionRecall.aaf0b2eb17"
|
|
534
|
+
});
|
|
535
|
+
trace?.startStage("rerank", fused.map((r) => r.entry.id));
|
|
536
|
+
const rerankOutcome = await this.rerank(context.query, fused, queryVector);
|
|
537
|
+
fused = rerankOutcome.candidates;
|
|
538
|
+
this.throwIfAborted(context);
|
|
539
|
+
const rerankStageMetadata = rerankOutcome.fallback ? {
|
|
540
|
+
...rerankOutcome.stats,
|
|
541
|
+
rerankFallbackReason: rerankOutcome.fallback.reason,
|
|
542
|
+
rerankFallbackProvider: rerankOutcome.fallback.provider
|
|
543
|
+
} : rerankOutcome.stats;
|
|
544
|
+
trace?.endStage(fused.map((r) => r.entry.id), fused.map((r) => r.score), rerankStageMetadata);
|
|
545
|
+
fused = this.applyScoringPipeline(fused, trace, context.limit);
|
|
546
|
+
trace?.startStage("min_score_filter", fused.map((result) => result.entry.id));
|
|
547
|
+
fused = fused.filter((result) => result.score >= (context.minScore ?? this.config.minScore));
|
|
548
|
+
trace?.endStage(fused.map((result) => result.entry.id), fused.map((result) => result.score));
|
|
549
|
+
const identifiers = context.query.match(/[\w./-]*\d[\w./-]*/g)?.filter((token) => token.length >= 6) ?? [];
|
|
550
|
+
const exact = fused.filter((result) => identifiers.some((token) => result.entry.text.includes(token)));
|
|
551
|
+
if (exact.length > 0) fused = [...exact, ...fused.filter((result) => !exact.includes(result))];
|
|
552
|
+
trace?.startStage("limit_slice", fused.map((result) => result.entry.id));
|
|
553
|
+
const served = fused.slice(0, context.limit);
|
|
554
|
+
trace?.endStage(served.map((result) => result.entry.id), served.map((result) => result.score));
|
|
555
|
+
return served;
|
|
556
|
+
},
|
|
557
|
+
rrfFuse(vector, keyword) {
|
|
558
|
+
const vectorMap = /* @__PURE__ */ new Map();
|
|
559
|
+
const keywordMap = /* @__PURE__ */ new Map();
|
|
560
|
+
for (const [index, result] of vector.entries()) vectorMap.set(result.entry.id, {
|
|
561
|
+
...result,
|
|
562
|
+
rank: index + 1
|
|
563
|
+
});
|
|
564
|
+
for (const [index, result] of keyword.entries()) keywordMap.set(result.entry.id, {
|
|
565
|
+
...result,
|
|
566
|
+
rank: index + 1
|
|
567
|
+
});
|
|
568
|
+
const allIds = /* @__PURE__ */ new Set([...vectorMap.keys(), ...keywordMap.keys()]);
|
|
569
|
+
const fused = [];
|
|
570
|
+
for (const id of allIds) {
|
|
571
|
+
const vectorHit = vectorMap.get(id);
|
|
572
|
+
const keywordHit = keywordMap.get(id);
|
|
573
|
+
const base = vectorHit ?? keywordHit;
|
|
574
|
+
if (!base) continue;
|
|
575
|
+
const vectorScore = vectorHit?.score ?? 0;
|
|
576
|
+
const keywordScore = keywordHit?.score ?? 0;
|
|
577
|
+
const weightedFusion = vectorScore * this.config.vectorWeight + keywordScore * this.config.bm25Weight;
|
|
578
|
+
const score = vectorHit ? clamp01(weightedFusion, .1) : clamp01(keywordScore, .1);
|
|
579
|
+
const vectorBest = vectorHit?.bestChunkScore ?? -1;
|
|
580
|
+
const keywordBest = keywordHit?.bestChunkScore ?? -1;
|
|
581
|
+
const anchor = vectorHit && keywordHit ? vectorBest >= keywordBest ? vectorHit : keywordHit : vectorHit ?? keywordHit;
|
|
582
|
+
fused.push({
|
|
583
|
+
entry: base.entry,
|
|
584
|
+
score,
|
|
585
|
+
sources: {
|
|
586
|
+
vector: vectorHit ? {
|
|
587
|
+
score: vectorHit.score,
|
|
588
|
+
rank: vectorHit.rank ?? 1
|
|
589
|
+
} : void 0,
|
|
590
|
+
bm25: keywordHit ? {
|
|
591
|
+
score: keywordHit.score,
|
|
592
|
+
rank: keywordHit.rank ?? 1
|
|
593
|
+
} : void 0,
|
|
594
|
+
fused: { score }
|
|
595
|
+
},
|
|
596
|
+
chunkId: anchor?.chunkId,
|
|
597
|
+
chunkIndex: anchor?.chunkIndex,
|
|
598
|
+
bestChunkScore: anchor?.bestChunkScore,
|
|
599
|
+
snippet: anchor?.snippet,
|
|
600
|
+
denseScore: vectorHit?.score,
|
|
601
|
+
bm25Score: keywordHit?.score,
|
|
602
|
+
fusedScore: score
|
|
603
|
+
});
|
|
604
|
+
}
|
|
605
|
+
return fused.sort((a, b) => {
|
|
606
|
+
const scoreDiff = b.score - a.score;
|
|
607
|
+
if (scoreDiff !== 0) return scoreDiff;
|
|
608
|
+
return a.entry.id.localeCompare(b.entry.id);
|
|
609
|
+
});
|
|
610
|
+
}
|
|
611
|
+
});
|
|
612
|
+
//#endregion
|
|
613
|
+
//#region src/engine/retrieval/retrieval-rerank-provider.ts
|
|
614
|
+
/** @file retrieval-rerank-provider.ts
|
|
615
|
+
* @purpose Adapts external rerank provider request and response formats.
|
|
616
|
+
* @boundary HTTP payload shaping and response parsing only.
|
|
617
|
+
*/
|
|
618
|
+
const RERANK_DEFAULT_ENDPOINTS = {
|
|
619
|
+
voyage: "https://api.voyageai.com/v1/rerank",
|
|
620
|
+
jina: "https://api.jina.ai/v1/rerank",
|
|
621
|
+
pinecone: "https://api.pinecone.io/rerank",
|
|
622
|
+
dashscope: "https://dashscope.aliyuncs.com/api/v1/services/rerank"
|
|
623
|
+
};
|
|
624
|
+
function buildRerankHttpError(status) {
|
|
625
|
+
if (classifyLlmFailure({ status }).category === "auth") return new RetrievalError(`Rerank API failed with status ${status}`);
|
|
626
|
+
}
|
|
627
|
+
function buildRerankRequest(provider, apiKey, model, query, candidates, topN) {
|
|
628
|
+
switch (provider) {
|
|
629
|
+
case "tei": return {
|
|
630
|
+
headers: {
|
|
631
|
+
"Content-Type": "application/json",
|
|
632
|
+
Authorization: `Bearer ${apiKey}`
|
|
633
|
+
},
|
|
634
|
+
body: {
|
|
635
|
+
query,
|
|
636
|
+
texts: candidates,
|
|
637
|
+
raw_scores: true
|
|
638
|
+
}
|
|
639
|
+
};
|
|
640
|
+
case "dashscope": return {
|
|
641
|
+
headers: {
|
|
642
|
+
"Content-Type": "application/json",
|
|
643
|
+
Authorization: `Bearer ${apiKey}`
|
|
644
|
+
},
|
|
645
|
+
body: {
|
|
646
|
+
model,
|
|
647
|
+
input: {
|
|
648
|
+
query,
|
|
649
|
+
documents: candidates
|
|
650
|
+
}
|
|
651
|
+
}
|
|
652
|
+
};
|
|
653
|
+
case "pinecone": return {
|
|
654
|
+
headers: {
|
|
655
|
+
"Content-Type": "application/json",
|
|
656
|
+
"Api-Key": apiKey,
|
|
657
|
+
"X-Pinecone-API-Version": "2024-10"
|
|
658
|
+
},
|
|
659
|
+
body: {
|
|
660
|
+
model,
|
|
661
|
+
query,
|
|
662
|
+
documents: candidates.map((text) => ({ text })),
|
|
663
|
+
top_n: topN,
|
|
664
|
+
rank_fields: ["text"]
|
|
665
|
+
}
|
|
666
|
+
};
|
|
667
|
+
case "voyage": return {
|
|
668
|
+
headers: {
|
|
669
|
+
"Content-Type": "application/json",
|
|
670
|
+
Authorization: `Bearer ${apiKey}`
|
|
671
|
+
},
|
|
672
|
+
body: {
|
|
673
|
+
model,
|
|
674
|
+
query,
|
|
675
|
+
documents: candidates,
|
|
676
|
+
top_k: topN
|
|
677
|
+
}
|
|
678
|
+
};
|
|
679
|
+
default: return {
|
|
680
|
+
headers: {
|
|
681
|
+
"Content-Type": "application/json",
|
|
682
|
+
Authorization: `Bearer ${apiKey}`
|
|
683
|
+
},
|
|
684
|
+
body: {
|
|
685
|
+
model,
|
|
686
|
+
query,
|
|
687
|
+
documents: candidates,
|
|
688
|
+
top_n: topN
|
|
689
|
+
}
|
|
690
|
+
};
|
|
691
|
+
}
|
|
692
|
+
}
|
|
693
|
+
function parseRerankResponse(provider, data) {
|
|
694
|
+
/** Parses items into the normalized shape used by precision recall retrieval ranking. */
|
|
695
|
+
const parseItems = (items, scoreKeys) => {
|
|
696
|
+
if (!Array.isArray(items)) return void 0;
|
|
697
|
+
const parsed = [];
|
|
698
|
+
for (const raw of items) {
|
|
699
|
+
const index = typeof raw?.index === "number" ? raw.index : Number(raw?.index);
|
|
700
|
+
if (!Number.isFinite(index)) continue;
|
|
701
|
+
let score;
|
|
702
|
+
for (const key of scoreKeys) {
|
|
703
|
+
const value = raw?.[key];
|
|
704
|
+
const n = typeof value === "number" ? value : Number(value);
|
|
705
|
+
if (Number.isFinite(n)) {
|
|
706
|
+
score = n;
|
|
707
|
+
break;
|
|
708
|
+
}
|
|
709
|
+
}
|
|
710
|
+
if (score === void 0) continue;
|
|
711
|
+
parsed.push({
|
|
712
|
+
index,
|
|
713
|
+
score
|
|
714
|
+
});
|
|
715
|
+
}
|
|
716
|
+
return parsed.length > 0 ? parsed : void 0;
|
|
717
|
+
};
|
|
718
|
+
const objectData = data && typeof data === "object" && !Array.isArray(data) ? data : void 0;
|
|
719
|
+
switch (provider) {
|
|
720
|
+
case "tei": return parseItems(data, ["score", "relevance_score"]) ?? parseItems(objectData?.results, ["score", "relevance_score"]) ?? parseItems(objectData?.data, ["score", "relevance_score"]);
|
|
721
|
+
case "dashscope": {
|
|
722
|
+
const output = objectData?.output;
|
|
723
|
+
if (output) return parseItems(output.results, ["relevance_score", "score"]);
|
|
724
|
+
return parseItems(objectData?.results, ["relevance_score", "score"]);
|
|
725
|
+
}
|
|
726
|
+
case "pinecone": return parseItems(objectData?.data, ["score", "relevance_score"]) ?? parseItems(objectData?.results, ["score", "relevance_score"]);
|
|
727
|
+
case "voyage": return parseItems(objectData?.data, ["relevance_score", "score"]) ?? parseItems(objectData?.results, ["relevance_score", "score"]);
|
|
728
|
+
default: return parseItems(objectData?.results, ["relevance_score", "score"]) ?? parseItems(objectData?.data, ["relevance_score", "score"]);
|
|
729
|
+
}
|
|
730
|
+
}
|
|
731
|
+
//#endregion
|
|
732
|
+
//#region src/engine/retrieval/retriever-rerank.ts
|
|
733
|
+
/** @file retriever-rerank.ts
|
|
734
|
+
* @purpose Runs remote and local reranking.
|
|
735
|
+
* @boundary Prototype-mounted MemoryRetriever methods; no constructor state ownership.
|
|
736
|
+
*/
|
|
737
|
+
const RERANK_TRANSIENT_ATTEMPTS = 3;
|
|
738
|
+
const RERANK_TRANSIENT_BACKOFF_MS = 200;
|
|
739
|
+
const RERANK_TRANSIENT_STATUSES = /* @__PURE__ */ new Set([
|
|
740
|
+
429,
|
|
741
|
+
502,
|
|
742
|
+
503,
|
|
743
|
+
504
|
|
744
|
+
]);
|
|
745
|
+
const RERANK_RETRY_AFTER_MAX_MS = 5e3;
|
|
746
|
+
function isRerankTimeout(error) {
|
|
747
|
+
return error instanceof Error && (error.name === "TimeoutError" || error.name === "AbortError");
|
|
748
|
+
}
|
|
749
|
+
/**
|
|
750
|
+
* The reranker's refusal when a pair is over its window, as the endpoint states it:
|
|
751
|
+
* `{"detail":{"code":"input_token_limit_exceeded","max_input_tokens":512,
|
|
752
|
+
* "candidates":[{"index":0,"input_tokens":728,"excess_tokens":216}]}}` (measured 2026-09-14).
|
|
753
|
+
*
|
|
754
|
+
* It names every offending text, so the batch can be repaired and sent again instead of
|
|
755
|
+
* losing the scores of the 49 texts that were fine. The counts are the SERVER's, taken with
|
|
756
|
+
* the ranking model's tokenizer — which is not the embedder's — so they are the only exact
|
|
757
|
+
* measure available here of how far over the window a text is.
|
|
758
|
+
*/
|
|
759
|
+
const RERANK_OVER_WINDOW_CODE = "input_token_limit_exceeded";
|
|
760
|
+
function parseOverWindowRefusal(body) {
|
|
761
|
+
let parsed;
|
|
762
|
+
try {
|
|
763
|
+
parsed = JSON.parse(body);
|
|
764
|
+
} catch {
|
|
765
|
+
return;
|
|
766
|
+
}
|
|
767
|
+
if (typeof parsed !== "object" || parsed === null) return void 0;
|
|
768
|
+
const detail = parsed.detail;
|
|
769
|
+
if (typeof detail !== "object" || detail === null) return void 0;
|
|
770
|
+
const shape = detail;
|
|
771
|
+
if (shape.code !== RERANK_OVER_WINDOW_CODE) return void 0;
|
|
772
|
+
if (typeof shape.max_input_tokens !== "number" || !Array.isArray(shape.candidates)) return;
|
|
773
|
+
const overLong = [];
|
|
774
|
+
for (const candidate of shape.candidates) {
|
|
775
|
+
if (typeof candidate !== "object" || candidate === null) continue;
|
|
776
|
+
const named = candidate;
|
|
777
|
+
if (typeof named.index !== "number" || typeof named.input_tokens !== "number") continue;
|
|
778
|
+
overLong.push({
|
|
779
|
+
index: named.index,
|
|
780
|
+
inputTokens: named.input_tokens
|
|
781
|
+
});
|
|
782
|
+
}
|
|
783
|
+
return overLong.length === 0 ? void 0 : {
|
|
784
|
+
maxInputTokens: shape.max_input_tokens,
|
|
785
|
+
overLong
|
|
786
|
+
};
|
|
787
|
+
}
|
|
788
|
+
function isTransientRerankError(error) {
|
|
789
|
+
if (isRerankTimeout(error)) return true;
|
|
790
|
+
return error instanceof TypeError && error.cause !== void 0;
|
|
791
|
+
}
|
|
792
|
+
async function fetchRerankWithRetry(send) {
|
|
793
|
+
let lastError;
|
|
794
|
+
for (let attempt = 1; attempt <= RERANK_TRANSIENT_ATTEMPTS; attempt += 1) {
|
|
795
|
+
let retryDelayMs = RERANK_TRANSIENT_BACKOFF_MS * attempt;
|
|
796
|
+
try {
|
|
797
|
+
const response = await send();
|
|
798
|
+
if (response.ok) return {
|
|
799
|
+
response,
|
|
800
|
+
data: await response.json()
|
|
801
|
+
};
|
|
802
|
+
if (!RERANK_TRANSIENT_STATUSES.has(response.status) || attempt === RERANK_TRANSIENT_ATTEMPTS) return { response };
|
|
803
|
+
if (response.status === 429) {
|
|
804
|
+
const retryAfter = response.headers.get("retry-after");
|
|
805
|
+
if (retryAfter !== null && /^\d+$/.test(retryAfter)) retryDelayMs = Math.min(Number.parseInt(retryAfter, 10) * 1e3, RERANK_RETRY_AFTER_MAX_MS);
|
|
806
|
+
}
|
|
807
|
+
await response.body?.cancel();
|
|
808
|
+
lastError = /* @__PURE__ */ new Error(`rerank API responded ${response.status}`);
|
|
809
|
+
} catch (error) {
|
|
810
|
+
if (!isTransientRerankError(error) || attempt === RERANK_TRANSIENT_ATTEMPTS) throw error;
|
|
811
|
+
lastError = error;
|
|
812
|
+
}
|
|
813
|
+
await new Promise((resolve) => setTimeout(resolve, retryDelayMs));
|
|
814
|
+
}
|
|
815
|
+
throw lastError;
|
|
816
|
+
}
|
|
817
|
+
Object.assign(MemoryRetriever.prototype, {
|
|
818
|
+
async rerank(query, candidates, queryVector) {
|
|
819
|
+
if (candidates.length === 0) return { candidates };
|
|
820
|
+
if (this.config.rerank === "none") return { candidates };
|
|
821
|
+
const provider = this.config.rerankProvider ?? "none";
|
|
822
|
+
if (this.config.rerank === "cross-encoder" && !this.config.rerankApiKey) throw new RetrievalError(`rerank "cross-encoder" requires retrieval.rerankApiKey (provider: ${provider}); set the key, or set retrieval.rerank to "lightweight" to choose the local ranker`);
|
|
823
|
+
if (this.config.rerank === "lightweight" || !this.config.rerankApiKey) return { candidates: this.rerankLightweight(candidates, queryVector) };
|
|
824
|
+
const rerankEndpoint = this.config.rerankEndpoint ?? RERANK_DEFAULT_ENDPOINTS[provider];
|
|
825
|
+
if (!rerankEndpoint) {
|
|
826
|
+
log.warn("rerank skipped: no endpoint for provider", { provider }, {
|
|
827
|
+
event_name: "memory.retriever_rerank.diagnostic",
|
|
828
|
+
file: "packages/memory/src/engine/retrieval/retriever-rerank.ts",
|
|
829
|
+
function: "rerank",
|
|
830
|
+
site_id: "retrieval.retriever-rerank.rerank.6fbc6b3a22"
|
|
831
|
+
});
|
|
832
|
+
return {
|
|
833
|
+
candidates,
|
|
834
|
+
fallback: {
|
|
835
|
+
reason: "no_endpoint",
|
|
836
|
+
provider
|
|
837
|
+
}
|
|
838
|
+
};
|
|
839
|
+
}
|
|
840
|
+
const rerankTimeoutMs = this.config.rerankTimeoutMs ?? 15e3;
|
|
841
|
+
const rerankApiKey = this.config.rerankApiKey;
|
|
842
|
+
const maxCandidates = this.config.rerankMaxCandidates;
|
|
843
|
+
const overCap = maxCandidates !== void 0 && candidates.length > maxCandidates;
|
|
844
|
+
const toRerank = overCap ? candidates.slice(0, maxCandidates) : candidates;
|
|
845
|
+
const beyondCap = overCap ? candidates.slice(maxCandidates) : [];
|
|
846
|
+
const batchSize = 50;
|
|
847
|
+
const queryTokens = this.embedder.countTokens(query);
|
|
848
|
+
const documentTokenBudget = 509 - queryTokens;
|
|
849
|
+
if (documentTokenBudget <= 0) {
|
|
850
|
+
log.warn("rerank skipped: the query alone fills the reranker token window", {
|
|
851
|
+
queryTokens,
|
|
852
|
+
window: 512,
|
|
853
|
+
templateTokens: 3
|
|
854
|
+
}, {
|
|
855
|
+
event_name: "memory.retriever_rerank.diagnostic",
|
|
856
|
+
file: "packages/memory/src/engine/retrieval/retriever-rerank.ts",
|
|
857
|
+
function: "rerank",
|
|
858
|
+
site_id: "retrieval.retriever-rerank.rerank.query_over_window"
|
|
859
|
+
});
|
|
860
|
+
return {
|
|
861
|
+
candidates,
|
|
862
|
+
fallback: {
|
|
863
|
+
reason: "query_over_window",
|
|
864
|
+
provider
|
|
865
|
+
}
|
|
866
|
+
};
|
|
867
|
+
}
|
|
868
|
+
let truncatedCandidates = 0;
|
|
869
|
+
const rerankTexts = toRerank.map((c) => {
|
|
870
|
+
const bounded = this.embedder.truncateToTokens(c.entry.text, documentTokenBudget);
|
|
871
|
+
if (bounded !== c.entry.text) truncatedCandidates += 1;
|
|
872
|
+
return bounded;
|
|
873
|
+
});
|
|
874
|
+
if (truncatedCandidates > 0) log.warn("rerank candidates were cut to the document budget the query left", {
|
|
875
|
+
truncatedCandidates,
|
|
876
|
+
documentTokenBudget,
|
|
877
|
+
queryTokens
|
|
878
|
+
}, {
|
|
879
|
+
event_name: "memory.retriever_rerank.diagnostic",
|
|
880
|
+
file: "packages/memory/src/engine/retrieval/retriever-rerank.ts",
|
|
881
|
+
function: "rerank",
|
|
882
|
+
site_id: "retrieval.retriever-rerank.rerank.candidate_over_ceiling"
|
|
883
|
+
});
|
|
884
|
+
try {
|
|
885
|
+
const runBatch = async (start) => {
|
|
886
|
+
let batchTexts = rerankTexts.slice(start, start + batchSize);
|
|
887
|
+
for (let pass = 0;; pass += 1) {
|
|
888
|
+
const { headers, body } = buildRerankRequest(provider, rerankApiKey, this.config.rerankModel ?? "rerank-2", query, batchTexts, batchTexts.length);
|
|
889
|
+
const { response, data } = await fetchRerankWithRetry(() => fetch(rerankEndpoint, {
|
|
890
|
+
method: "POST",
|
|
891
|
+
headers,
|
|
892
|
+
body: JSON.stringify(body),
|
|
893
|
+
signal: AbortSignal.timeout(rerankTimeoutMs)
|
|
894
|
+
}));
|
|
895
|
+
if (!response.ok) {
|
|
896
|
+
const retryAfter = response.headers.get("retry-after");
|
|
897
|
+
const fatalError = buildRerankHttpError(response.status);
|
|
898
|
+
if (fatalError) throw fatalError;
|
|
899
|
+
const errorBody = await response.text().catch(() => "<unreadable body>");
|
|
900
|
+
const refusal = pass === 0 ? parseOverWindowRefusal(errorBody) : void 0;
|
|
901
|
+
if (refusal) {
|
|
902
|
+
const allowed = refusal.maxInputTokens - 3;
|
|
903
|
+
batchTexts = batchTexts.map((text, index) => {
|
|
904
|
+
const named = refusal.overLong.find((c) => c.index === index);
|
|
905
|
+
if (!named) return text;
|
|
906
|
+
const localTokens = this.embedder.countTokens(text);
|
|
907
|
+
const localPairTokens = queryTokens + localTokens;
|
|
908
|
+
const target = Math.max(1, Math.floor(allowed * localPairTokens / named.inputTokens) - queryTokens);
|
|
909
|
+
return this.embedder.truncateToTokens(text, target);
|
|
910
|
+
});
|
|
911
|
+
log.warn("rerank batch refused over its token window; resending the named texts cut to fit", {
|
|
912
|
+
status: response.status,
|
|
913
|
+
overLongCount: refusal.overLong.length,
|
|
914
|
+
maxInputTokens: refusal.maxInputTokens,
|
|
915
|
+
sentToRerankCount: batchTexts.length,
|
|
916
|
+
batchStart: start
|
|
917
|
+
}, {
|
|
918
|
+
event_name: "memory.retriever_rerank.diagnostic",
|
|
919
|
+
file: "packages/memory/src/engine/retrieval/retriever-rerank.ts",
|
|
920
|
+
function: "runBatch",
|
|
921
|
+
site_id: "retrieval.retriever-rerank.runBatch.over_window_repair"
|
|
922
|
+
});
|
|
923
|
+
continue;
|
|
924
|
+
}
|
|
925
|
+
log.warn("rerank API failed, using pre-rerank results", {
|
|
926
|
+
status: response.status,
|
|
927
|
+
endpoint_hash: createHash("sha256").update(rerankEndpoint).digest("hex"),
|
|
928
|
+
retryAfter,
|
|
929
|
+
sentToRerankCount: batchTexts.length,
|
|
930
|
+
batchStart: start,
|
|
931
|
+
body_length: errorBody.length
|
|
932
|
+
}, {
|
|
933
|
+
event_name: "memory.retriever_rerank.diagnostic",
|
|
934
|
+
file: "packages/memory/src/engine/retrieval/retriever-rerank.ts",
|
|
935
|
+
function: "runBatch",
|
|
936
|
+
site_id: "retrieval.retriever-rerank.runBatch.76d7d1fac6"
|
|
937
|
+
});
|
|
938
|
+
return "http_error";
|
|
939
|
+
}
|
|
940
|
+
const parsed = parseRerankResponse(provider, data);
|
|
941
|
+
if (!parsed) {
|
|
942
|
+
log.warn("rerank API returned unparseable response", {
|
|
943
|
+
provider,
|
|
944
|
+
endpoint_hash: createHash("sha256").update(rerankEndpoint).digest("hex"),
|
|
945
|
+
batchStart: start
|
|
946
|
+
}, {
|
|
947
|
+
event_name: "memory.retriever_rerank.diagnostic",
|
|
948
|
+
file: "packages/memory/src/engine/retrieval/retriever-rerank.ts",
|
|
949
|
+
function: "runBatch",
|
|
950
|
+
site_id: "retrieval.retriever-rerank.runBatch.4429b8bd8a"
|
|
951
|
+
});
|
|
952
|
+
return "invalid_response";
|
|
953
|
+
}
|
|
954
|
+
const batchItems = [];
|
|
955
|
+
for (const item of parsed) {
|
|
956
|
+
if (item.index < 0 || item.index >= batchTexts.length) continue;
|
|
957
|
+
batchItems.push({
|
|
958
|
+
index: start + item.index,
|
|
959
|
+
score: item.score
|
|
960
|
+
});
|
|
961
|
+
}
|
|
962
|
+
return batchItems;
|
|
963
|
+
}
|
|
964
|
+
};
|
|
965
|
+
const batchStarts = [];
|
|
966
|
+
for (let start = 0; start < rerankTexts.length; start += batchSize) batchStarts.push(start);
|
|
967
|
+
const items = [];
|
|
968
|
+
for (let wave = 0; wave < batchStarts.length; wave += 4) {
|
|
969
|
+
const waveStarts = batchStarts.slice(wave, wave + 4);
|
|
970
|
+
const settled = await Promise.allSettled(waveStarts.map(runBatch));
|
|
971
|
+
for (const outcome of settled) if (outcome.status === "rejected" && outcome.reason instanceof RetrievalError) throw outcome.reason;
|
|
972
|
+
for (const outcome of settled) if (outcome.status === "rejected") throw outcome.reason;
|
|
973
|
+
for (const outcome of settled) if (outcome.status === "fulfilled" && typeof outcome.value === "string") return {
|
|
974
|
+
candidates,
|
|
975
|
+
fallback: {
|
|
976
|
+
reason: outcome.value,
|
|
977
|
+
provider
|
|
978
|
+
}
|
|
979
|
+
};
|
|
980
|
+
for (const outcome of settled) if (outcome.status === "fulfilled" && typeof outcome.value !== "string") items.push(...outcome.value);
|
|
981
|
+
}
|
|
982
|
+
const blendCross = this.config.rerankBlendCross ?? .7;
|
|
983
|
+
const blendVector = this.config.rerankBlendVector ?? .3;
|
|
984
|
+
const returnedIndices = new Set(items.map((r) => r.index));
|
|
985
|
+
const reranked = [];
|
|
986
|
+
const seenIndices = /* @__PURE__ */ new Set();
|
|
987
|
+
for (const item of items) {
|
|
988
|
+
if (seenIndices.has(item.index)) continue;
|
|
989
|
+
seenIndices.add(item.index);
|
|
990
|
+
const candidate = toRerank[item.index];
|
|
991
|
+
if (!candidate) continue;
|
|
992
|
+
const sourceScore = this.getRerankSourceScore(candidate);
|
|
993
|
+
let crossScore = item.score;
|
|
994
|
+
if (provider === "tei") crossScore = 1 / (1 + Math.exp(-item.score));
|
|
995
|
+
const blendedScore = clamp01(crossScore * blendCross + sourceScore * blendVector, 0);
|
|
996
|
+
reranked.push({
|
|
997
|
+
...candidate,
|
|
998
|
+
score: blendedScore,
|
|
999
|
+
sources: {
|
|
1000
|
+
...candidate.sources,
|
|
1001
|
+
reranked: { score: item.score }
|
|
1002
|
+
}
|
|
1003
|
+
});
|
|
1004
|
+
}
|
|
1005
|
+
const unreturned = [...toRerank.filter((_, idx) => !returnedIndices.has(idx)), ...beyondCap].map((r) => ({
|
|
1006
|
+
...r,
|
|
1007
|
+
score: clamp01(this.getRerankSourceScore(r) * LIGHTWEIGHT_RERANK_PENALTY, 0)
|
|
1008
|
+
}));
|
|
1009
|
+
const merged = [...reranked, ...unreturned].sort((a, b) => b.score - a.score);
|
|
1010
|
+
log.debug("reranked", {
|
|
1011
|
+
totalCandidateCount: candidates.length,
|
|
1012
|
+
sentToRerankCount: toRerank.length,
|
|
1013
|
+
returnedCount: reranked.length,
|
|
1014
|
+
provider
|
|
1015
|
+
}, {
|
|
1016
|
+
event_name: "memory.retriever_rerank.diagnostic",
|
|
1017
|
+
file: "packages/memory/src/engine/retrieval/retriever-rerank.ts",
|
|
1018
|
+
function: "rerank",
|
|
1019
|
+
site_id: "retrieval.retriever-rerank.rerank.7847c6248a"
|
|
1020
|
+
});
|
|
1021
|
+
return {
|
|
1022
|
+
candidates: merged.length > 0 ? merged : candidates,
|
|
1023
|
+
stats: {
|
|
1024
|
+
rerankSentCount: toRerank.length,
|
|
1025
|
+
rerankReturnedCount: reranked.length,
|
|
1026
|
+
rerankBeyondCapCount: beyondCap.length
|
|
1027
|
+
}
|
|
1028
|
+
};
|
|
1029
|
+
} catch (error) {
|
|
1030
|
+
if (error instanceof RetrievalError) throw error;
|
|
1031
|
+
let reason;
|
|
1032
|
+
if (isRerankTimeout(error)) {
|
|
1033
|
+
reason = "timeout";
|
|
1034
|
+
log.warn("rerank API timed out, using pre-rerank results", {
|
|
1035
|
+
timeoutMs: rerankTimeoutMs,
|
|
1036
|
+
attempts: RERANK_TRANSIENT_ATTEMPTS
|
|
1037
|
+
}, {
|
|
1038
|
+
event_name: "memory.retriever_rerank.diagnostic",
|
|
1039
|
+
file: "packages/memory/src/engine/retrieval/retriever-rerank.ts",
|
|
1040
|
+
function: "rerank",
|
|
1041
|
+
site_id: "retrieval.retriever-rerank.rerank.4af9c17c75"
|
|
1042
|
+
});
|
|
1043
|
+
} else {
|
|
1044
|
+
reason = "request_error";
|
|
1045
|
+
log.warn("rerank error, using pre-rerank results", {
|
|
1046
|
+
error,
|
|
1047
|
+
attempts: RERANK_TRANSIENT_ATTEMPTS
|
|
1048
|
+
}, {
|
|
1049
|
+
event_name: "memory.retriever_rerank.diagnostic",
|
|
1050
|
+
file: "packages/memory/src/engine/retrieval/retriever-rerank.ts",
|
|
1051
|
+
function: "rerank",
|
|
1052
|
+
site_id: "retrieval.retriever-rerank.rerank.2c5fc67dbf"
|
|
1053
|
+
});
|
|
1054
|
+
}
|
|
1055
|
+
return {
|
|
1056
|
+
candidates,
|
|
1057
|
+
fallback: {
|
|
1058
|
+
reason,
|
|
1059
|
+
provider
|
|
1060
|
+
}
|
|
1061
|
+
};
|
|
1062
|
+
}
|
|
1063
|
+
},
|
|
1064
|
+
rerankLightweight(candidates, queryVector) {
|
|
1065
|
+
if (queryVector.length === 0) return candidates;
|
|
1066
|
+
if (candidates.length === 0) return candidates;
|
|
1067
|
+
const chunkIds = candidates.map((c) => c.chunkId).filter((id) => id !== void 0);
|
|
1068
|
+
const vectorMap = chunkIds.length > 0 ? this.store.getVectorsByIds(chunkIds) : /* @__PURE__ */ new Map();
|
|
1069
|
+
const wF = this.config.lightweightFusionWeight ?? .7;
|
|
1070
|
+
const wC = this.config.lightweightCosineWeight ?? .3;
|
|
1071
|
+
let blendedCount = 0;
|
|
1072
|
+
let dimMismatchCount = 0;
|
|
1073
|
+
const reranked = candidates.map((candidate) => {
|
|
1074
|
+
const fusedScore = candidate.fusedScore ?? candidate.score;
|
|
1075
|
+
if (!candidate.chunkId) return {
|
|
1076
|
+
...candidate,
|
|
1077
|
+
rerankScore: fusedScore
|
|
1078
|
+
};
|
|
1079
|
+
const chunkVector = vectorMap.get(candidate.chunkId);
|
|
1080
|
+
if (!chunkVector) return {
|
|
1081
|
+
...candidate,
|
|
1082
|
+
rerankScore: fusedScore
|
|
1083
|
+
};
|
|
1084
|
+
if (chunkVector.length !== queryVector.length) {
|
|
1085
|
+
dimMismatchCount += 1;
|
|
1086
|
+
return {
|
|
1087
|
+
...candidate,
|
|
1088
|
+
rerankScore: fusedScore
|
|
1089
|
+
};
|
|
1090
|
+
}
|
|
1091
|
+
const cosine = clamp01(dotProduct(queryVector, chunkVector), 0);
|
|
1092
|
+
const blended = fusedScore * wF + cosine * wC;
|
|
1093
|
+
const finalScore = clamp01(blended, 0);
|
|
1094
|
+
blendedCount += 1;
|
|
1095
|
+
return {
|
|
1096
|
+
...candidate,
|
|
1097
|
+
score: finalScore,
|
|
1098
|
+
rerankScore: finalScore,
|
|
1099
|
+
sources: {
|
|
1100
|
+
...candidate.sources,
|
|
1101
|
+
reranked: { score: finalScore }
|
|
1102
|
+
}
|
|
1103
|
+
};
|
|
1104
|
+
});
|
|
1105
|
+
if (dimMismatchCount > 0) log.warn("lightweight rerank: chunk vector dim mismatch on some candidates", {
|
|
1106
|
+
dimMismatchCount,
|
|
1107
|
+
queryDim: queryVector.length
|
|
1108
|
+
}, {
|
|
1109
|
+
event_name: "memory.retriever_rerank.diagnostic",
|
|
1110
|
+
file: "packages/memory/src/engine/retrieval/retriever-rerank.ts",
|
|
1111
|
+
function: "rerankLightweight",
|
|
1112
|
+
site_id: "retrieval.retriever-rerank.rerankLightweight.70eb2031f1"
|
|
1113
|
+
});
|
|
1114
|
+
log.debug("lightweight rerank applied", {
|
|
1115
|
+
candidateCount: candidates.length,
|
|
1116
|
+
blendedCount,
|
|
1117
|
+
fusionWeight: wF,
|
|
1118
|
+
cosineWeight: wC
|
|
1119
|
+
}, {
|
|
1120
|
+
event_name: "memory.retriever_rerank.diagnostic",
|
|
1121
|
+
file: "packages/memory/src/engine/retrieval/retriever-rerank.ts",
|
|
1122
|
+
function: "rerankLightweight",
|
|
1123
|
+
site_id: "retrieval.retriever-rerank.rerankLightweight.7e8d0eee29"
|
|
1124
|
+
});
|
|
1125
|
+
return reranked.sort((a, b) => {
|
|
1126
|
+
const diff = b.score - a.score;
|
|
1127
|
+
if (diff !== 0) return diff;
|
|
1128
|
+
return a.entry.id.localeCompare(b.entry.id);
|
|
1129
|
+
});
|
|
1130
|
+
},
|
|
1131
|
+
getRerankSourceScore(result) {
|
|
1132
|
+
return clamp01(Math.max(result.sources.bm25?.score ?? 0, result.sources.vector?.score ?? 0), 0);
|
|
1133
|
+
}
|
|
1134
|
+
});
|
|
1135
|
+
//#endregion
|
|
1136
|
+
//#region src/engine/operations/selective-forgetting-scorer.ts
|
|
1137
|
+
/** @file selective-forgetting-scorer.ts
|
|
1138
|
+
* @purpose Computes Selective Forgetting scores from age, access history, and intrinsic value.
|
|
1139
|
+
* @boundary Access history, importance, timestamps, and retrieval scoring inputs.
|
|
1140
|
+
* @see access-tracker.ts, retriever.ts, store.ts.
|
|
1141
|
+
*/
|
|
1142
|
+
/**
|
|
1143
|
+
* Selective Forgetting scorer — Weibull stretched-exponential retention model
|
|
1144
|
+
*
|
|
1145
|
+
* Composite score = recencyWeight * recency + frequencyWeight * frequency + intrinsicWeight * intrinsic
|
|
1146
|
+
*
|
|
1147
|
+
* - Recency: Weibull decay with importance-modulated half-life and tier-specific beta
|
|
1148
|
+
* - Frequency: Logarithmic saturation with time-weighted access pattern bonus
|
|
1149
|
+
* - Intrinsic: importance × confidence
|
|
1150
|
+
*/
|
|
1151
|
+
const MS_PER_DAY = 864e5;
|
|
1152
|
+
const DEFAULT_DECAY_CONFIG = {
|
|
1153
|
+
temporalDecay: true,
|
|
1154
|
+
recencyHalfLifeDays: 180,
|
|
1155
|
+
recencyWeight: .4,
|
|
1156
|
+
frequencyWeight: .3,
|
|
1157
|
+
intrinsicWeight: .3,
|
|
1158
|
+
staleThreshold: .3,
|
|
1159
|
+
searchBoostMin: .85,
|
|
1160
|
+
importanceModulation: 1.5,
|
|
1161
|
+
betaCore: .8,
|
|
1162
|
+
betaWorking: 1,
|
|
1163
|
+
betaPeripheral: 1.3,
|
|
1164
|
+
coreDecayFloor: .9,
|
|
1165
|
+
workingDecayFloor: .7,
|
|
1166
|
+
peripheralDecayFloor: .5
|
|
1167
|
+
};
|
|
1168
|
+
/** Creates the Selective Forgetting scorer that blends intrinsic value, age, tier, and temporal hints. */
|
|
1169
|
+
function createRetentionScorer(config = DEFAULT_DECAY_CONFIG) {
|
|
1170
|
+
const { temporalDecay, recencyHalfLifeDays: halfLife, recencyWeight: rw, frequencyWeight: fw, intrinsicWeight: iw, staleThreshold, importanceModulation: mu, betaCore, betaWorking, betaPeripheral } = config;
|
|
1171
|
+
function betaForTier(tier) {
|
|
1172
|
+
switch (tier) {
|
|
1173
|
+
case "core": return betaCore;
|
|
1174
|
+
case "working": return betaWorking;
|
|
1175
|
+
case "peripheral": return betaPeripheral;
|
|
1176
|
+
}
|
|
1177
|
+
}
|
|
1178
|
+
function resolveTemporalType(memory) {
|
|
1179
|
+
if (memory.temporalType) return memory.temporalType;
|
|
1180
|
+
if (typeof memory.metadata !== "string" || memory.metadata.length === 0) return;
|
|
1181
|
+
return parseInsightMetadata(memory.metadata).memory_temporal_type;
|
|
1182
|
+
}
|
|
1183
|
+
function activeTimestamp(memory) {
|
|
1184
|
+
return memory.accessCount > 0 ? memory.lastAccessedAt : memory.createdAt;
|
|
1185
|
+
}
|
|
1186
|
+
function createScoringContext(memory, now) {
|
|
1187
|
+
const lastActive = activeTimestamp(memory);
|
|
1188
|
+
const temporalType = resolveTemporalType(memory);
|
|
1189
|
+
const baseHalfLifeDays = temporalDecay && temporalType === "dynamic" ? halfLife / 3 : halfLife;
|
|
1190
|
+
return {
|
|
1191
|
+
memory,
|
|
1192
|
+
now,
|
|
1193
|
+
lastActiveAt: lastActive,
|
|
1194
|
+
daysSinceActive: Math.max(0, (now - lastActive) / MS_PER_DAY),
|
|
1195
|
+
temporalType,
|
|
1196
|
+
tierBeta: betaForTier(memory.tier),
|
|
1197
|
+
baseHalfLifeDays
|
|
1198
|
+
};
|
|
1199
|
+
}
|
|
1200
|
+
function evaluateRecency(context) {
|
|
1201
|
+
const effectiveHalfLife = context.baseHalfLifeDays * Math.exp(mu * context.memory.importance);
|
|
1202
|
+
return Math.exp(-Math.LN2 * (context.daysSinceActive / effectiveHalfLife) ** context.tierBeta);
|
|
1203
|
+
}
|
|
1204
|
+
function evaluateFrequency(context) {
|
|
1205
|
+
const { memory } = context;
|
|
1206
|
+
const base = 1 - Math.exp(-memory.accessCount / 5);
|
|
1207
|
+
if (memory.accessCount <= 1) return base;
|
|
1208
|
+
const avgGapDays = Math.max(1, (context.lastActiveAt - memory.createdAt) / MS_PER_DAY) / Math.max(memory.accessCount - 1, 1);
|
|
1209
|
+
return base * (.5 + .5 * Math.exp(-avgGapDays / 30));
|
|
1210
|
+
}
|
|
1211
|
+
function evaluateIntrinsic(context) {
|
|
1212
|
+
return context.memory.importance * context.memory.confidence;
|
|
1213
|
+
}
|
|
1214
|
+
function evaluateComponents(context) {
|
|
1215
|
+
return {
|
|
1216
|
+
recency: evaluateRecency(context),
|
|
1217
|
+
frequency: evaluateFrequency(context),
|
|
1218
|
+
intrinsic: evaluateIntrinsic(context)
|
|
1219
|
+
};
|
|
1220
|
+
}
|
|
1221
|
+
function combineComponents(components) {
|
|
1222
|
+
return rw * components.recency + fw * components.frequency + iw * components.intrinsic;
|
|
1223
|
+
}
|
|
1224
|
+
function scoreOne(memory, now) {
|
|
1225
|
+
const components = evaluateComponents(createScoringContext(memory, now));
|
|
1226
|
+
return {
|
|
1227
|
+
memoryId: memory.id,
|
|
1228
|
+
...components,
|
|
1229
|
+
composite: combineComponents(components)
|
|
1230
|
+
};
|
|
1231
|
+
}
|
|
1232
|
+
return {
|
|
1233
|
+
score(memory, now = Date.now()) {
|
|
1234
|
+
return scoreOne(memory, now);
|
|
1235
|
+
},
|
|
1236
|
+
scoreAll(memories, now = Date.now()) {
|
|
1237
|
+
return memories.map((m) => scoreOne(m, now));
|
|
1238
|
+
},
|
|
1239
|
+
getStaleMemories(memories, now = Date.now()) {
|
|
1240
|
+
return memories.map((m) => scoreOne(m, now)).filter((s) => s.composite < staleThreshold).sort((a, b) => a.composite - b.composite);
|
|
1241
|
+
}
|
|
1242
|
+
};
|
|
1243
|
+
}
|
|
1244
|
+
//#endregion
|
|
1245
|
+
//#region src/engine/retrieval/retriever-scoring-pipeline.ts
|
|
1246
|
+
/** @file retriever-scoring-pipeline.ts
|
|
1247
|
+
* @purpose Applies post-retrieval scoring, decay, noise, and diversity transforms.
|
|
1248
|
+
* @boundary Prototype-mounted MemoryRetriever methods; no constructor state ownership.
|
|
1249
|
+
*/
|
|
1250
|
+
const SEARCH_BOOST_MIN = DEFAULT_DECAY_CONFIG.searchBoostMin;
|
|
1251
|
+
function getTierFloor(tier) {
|
|
1252
|
+
switch (tier) {
|
|
1253
|
+
case "core": return DEFAULT_DECAY_CONFIG.coreDecayFloor;
|
|
1254
|
+
case "working": return DEFAULT_DECAY_CONFIG.workingDecayFloor;
|
|
1255
|
+
case "peripheral": return DEFAULT_DECAY_CONFIG.peripheralDecayFloor;
|
|
1256
|
+
}
|
|
1257
|
+
}
|
|
1258
|
+
/**
|
|
1259
|
+
* Multiplier formula + retention-
|
|
1260
|
+
* scoring spec §62. Bounded to `[SEARCH_BOOST_MIN, 1.0]`; never amplifies,
|
|
1261
|
+
* only suppresses, so existing scores are an upper bound after the stage.
|
|
1262
|
+
*
|
|
1263
|
+
* - `bare`: linear interpolation `searchBoostMin + (1 - searchBoostMin) * composite`.
|
|
1264
|
+
* - `withFloor`: same shape but the interpolated input is
|
|
1265
|
+
* `max(getTierFloor(tier), composite, recency)` so highly-decayed core
|
|
1266
|
+
* memories cannot fall below the tier floor.
|
|
1267
|
+
*/
|
|
1268
|
+
function boostMultiplier(score, mode, tier) {
|
|
1269
|
+
if (mode === "bare") {
|
|
1270
|
+
const linear = SEARCH_BOOST_MIN + (1 - SEARCH_BOOST_MIN) * score.composite;
|
|
1271
|
+
return Math.min(1, Math.max(SEARCH_BOOST_MIN, linear));
|
|
1272
|
+
}
|
|
1273
|
+
const candidate = Math.max(getTierFloor(tier), score.composite, score.recency);
|
|
1274
|
+
const linear = SEARCH_BOOST_MIN + (1 - SEARCH_BOOST_MIN) * candidate;
|
|
1275
|
+
return Math.min(1, Math.max(SEARCH_BOOST_MIN, linear));
|
|
1276
|
+
}
|
|
1277
|
+
/**
|
|
1278
|
+
* Adapt a `RetrievalResult.entry` into the `DecayableMemory` shape the
|
|
1279
|
+
* Retention Scorer consumes. `MemoryEntry` lacks `tier`, `confidence`,
|
|
1280
|
+
* `accessCount`, `lastAccessedAt`, and `createdAt` as native columns; all
|
|
1281
|
+
* five come out of the same metadata JSON the rest of the scoring pipeline
|
|
1282
|
+
* already parses via `parseInsightMetadata` + `parseAccessMetadata`.
|
|
1283
|
+
*
|
|
1284
|
+
* `createdAt` maps to `entry.timestamp` because `MemoryStore.store` writes
|
|
1285
|
+
* the caller-provided `timestamp` (or `Date.now()`) into both the entry row
|
|
1286
|
+
* and the embedding's creation time — there is no separate `created_at`.
|
|
1287
|
+
*/
|
|
1288
|
+
function toDecayableMemory(entry) {
|
|
1289
|
+
const insight = parseInsightMetadata(entry.metadata, entry);
|
|
1290
|
+
const access = parseAccessMetadata(entry.metadata);
|
|
1291
|
+
const tier = insight.tier ?? "peripheral";
|
|
1292
|
+
const memory = {
|
|
1293
|
+
id: entry.id,
|
|
1294
|
+
importance: entry.importance,
|
|
1295
|
+
confidence: insight.confidence,
|
|
1296
|
+
tier,
|
|
1297
|
+
accessCount: access.accessCount,
|
|
1298
|
+
createdAt: entry.timestamp,
|
|
1299
|
+
lastAccessedAt: access.lastAccessedAt > 0 ? access.lastAccessedAt : entry.timestamp
|
|
1300
|
+
};
|
|
1301
|
+
if (insight.memory_temporal_type !== void 0) memory.temporalType = insight.memory_temporal_type;
|
|
1302
|
+
if (entry.metadata !== void 0) memory.metadata = entry.metadata;
|
|
1303
|
+
return memory;
|
|
1304
|
+
}
|
|
1305
|
+
/**
|
|
1306
|
+
* Run one stage of the scoring pipeline and record what it did.
|
|
1307
|
+
*
|
|
1308
|
+
* Every stage mutates scores or removes candidates, and until 2026-08-26 not one of them left
|
|
1309
|
+
* a trace — so a memory that reached the model and lost, and a memory some stage quietly
|
|
1310
|
+
* dropped, were indistinguishable after the fact. `skipReason` matters just as much: a stage
|
|
1311
|
+
* disabled by config returns its input untouched, which looks identical to a stage that ran
|
|
1312
|
+
* and changed nothing. Ported from the upstream reference's `stageCounts` + `buildDropSummary`
|
|
1313
|
+
* (memory-memory-lancedb-pro/src/retriever.ts:655, :303).
|
|
1314
|
+
*/
|
|
1315
|
+
function tracedStage(trace, name, skipReason, run, input) {
|
|
1316
|
+
if (!trace) return run(input);
|
|
1317
|
+
trace.startStage(name, input.map((result) => result.entry.id));
|
|
1318
|
+
const output = run(input);
|
|
1319
|
+
trace.endStage(output.map((result) => result.entry.id), output.map((result) => result.score), skipReason === void 0 ? void 0 : { skipped: skipReason });
|
|
1320
|
+
return output;
|
|
1321
|
+
}
|
|
1322
|
+
Object.assign(MemoryRetriever.prototype, {
|
|
1323
|
+
applyScoringPipeline(results, trace, limit = results.length) {
|
|
1324
|
+
const stage = (name, skipReason, run, input) => tracedStage(trace, name, skipReason, run, input);
|
|
1325
|
+
const temporalOff = !this.config.temporalWeighting;
|
|
1326
|
+
let scored = stage("recency_boost", temporalOff ? "temporalWeighting" : this.config.recencyHalfLifeDays <= 0 ? "recencyHalfLifeDays" : this.config.recencyWeight <= 0 ? "recencyWeight" : void 0, (input) => temporalOff ? input : this.applyRecencyBoost(input), results);
|
|
1327
|
+
scored = stage("importance_weight", void 0, (input) => this.applyImportanceWeight(input), scored);
|
|
1328
|
+
scored = stage("length_normalization", this.config.lengthNormAnchor <= 0 ? "lengthNormAnchor" : void 0, (input) => this.applyLengthNormalization(input), scored);
|
|
1329
|
+
scored = stage("time_decay", temporalOff ? "temporalWeighting" : this.config.timeDecayHalfLifeDays <= 0 ? "timeDecayHalfLifeDays" : void 0, (input) => temporalOff ? input : this.applyTimeDecay(input), scored);
|
|
1330
|
+
scored = stage("retention_boost", temporalOff ? "temporalWeighting" : this.config.recallLifecycle?.retentionScorer === true ? void 0 : "recallLifecycle.retentionScorer", (input) => temporalOff ? input : this.applyRetentionBoost(input), scored);
|
|
1331
|
+
scored = stage("hard_min_score", void 0, (input) => input.filter((result) => result.score >= this.config.hardMinScore), scored);
|
|
1332
|
+
const mmrOff = experimentMmrDisabled();
|
|
1333
|
+
scored = stage("mmr_diversity", mmrOff ? "experiment.disableMmr" : void 0, (input) => {
|
|
1334
|
+
if (mmrOff) return input;
|
|
1335
|
+
if (!this.config.mmrWindowOnly) return this.applyMmrDiversity(input);
|
|
1336
|
+
const windowIds = new Set(results.slice(0, limit).map((result) => result.entry.id));
|
|
1337
|
+
const window = input.filter((result) => windowIds.has(result.entry.id));
|
|
1338
|
+
const tail = input.filter((result) => !windowIds.has(result.entry.id));
|
|
1339
|
+
return [...this.applyMmrDiversity(window), ...tail];
|
|
1340
|
+
}, scored);
|
|
1341
|
+
return scored;
|
|
1342
|
+
},
|
|
1343
|
+
applyRecencyBoost(results) {
|
|
1344
|
+
if (this.config.recencyHalfLifeDays <= 0 || this.config.recencyWeight <= 0) return results;
|
|
1345
|
+
const now = Date.now();
|
|
1346
|
+
const halfLifeMs = this.config.recencyHalfLifeDays * 24 * 60 * 60 * 1e3;
|
|
1347
|
+
return results.map((result) => {
|
|
1348
|
+
const ageMs = Math.max(0, now - result.entry.timestamp);
|
|
1349
|
+
const decay = Math.exp(-Math.log(2) * ageMs / halfLifeMs);
|
|
1350
|
+
return {
|
|
1351
|
+
...result,
|
|
1352
|
+
score: result.score + decay * this.config.recencyWeight
|
|
1353
|
+
};
|
|
1354
|
+
});
|
|
1355
|
+
},
|
|
1356
|
+
applyImportanceWeight(results) {
|
|
1357
|
+
const base = this.config.importanceWeightBase ?? .7;
|
|
1358
|
+
return results.map((result) => ({
|
|
1359
|
+
...result,
|
|
1360
|
+
score: result.score * (base + (1 - base) * result.entry.importance)
|
|
1361
|
+
}));
|
|
1362
|
+
},
|
|
1363
|
+
applyLengthNormalization(results) {
|
|
1364
|
+
if (this.config.lengthNormAnchor <= 0) return results;
|
|
1365
|
+
return results.map((result) => {
|
|
1366
|
+
const length = result.entry.text.length;
|
|
1367
|
+
const ratio = Math.max(1, length / this.config.lengthNormAnchor);
|
|
1368
|
+
const normalization = 1 / (1 + .5 * Math.log2(ratio));
|
|
1369
|
+
return {
|
|
1370
|
+
...result,
|
|
1371
|
+
score: result.score * normalization
|
|
1372
|
+
};
|
|
1373
|
+
});
|
|
1374
|
+
},
|
|
1375
|
+
applyTimeDecay(results) {
|
|
1376
|
+
if (this.config.timeDecayHalfLifeDays <= 0) return results;
|
|
1377
|
+
const floor = this.config.timeDecayFloor ?? .6;
|
|
1378
|
+
const now = Date.now();
|
|
1379
|
+
const baseHalfLifeDays = this.config.timeDecayHalfLifeDays;
|
|
1380
|
+
const reinforcementFactor = this.config.reinforcementFactor ?? .5;
|
|
1381
|
+
const maxMultiplier = this.config.maxHalfLifeMultiplier ?? 3;
|
|
1382
|
+
return results.map((result) => {
|
|
1383
|
+
const access = parseAccessMetadata(result.entry.metadata);
|
|
1384
|
+
const meta = parseInsightMetadata(result.entry.metadata, result.entry);
|
|
1385
|
+
const adjustedBase = this.config.temporalDecay && meta.memory_temporal_type === "dynamic" ? baseHalfLifeDays / 3 : baseHalfLifeDays;
|
|
1386
|
+
const halfLifeMs = computeEffectiveHalfLife(adjustedBase, access.accessCount, access.lastAccessedAt, reinforcementFactor, maxMultiplier, now) * 24 * 60 * 60 * 1e3;
|
|
1387
|
+
const ageMs = Math.max(0, now - result.entry.timestamp);
|
|
1388
|
+
const factor = floor + (1 - floor) * Math.exp(-Math.log(2) * ageMs / halfLifeMs);
|
|
1389
|
+
return {
|
|
1390
|
+
...result,
|
|
1391
|
+
score: result.score * factor
|
|
1392
|
+
};
|
|
1393
|
+
});
|
|
1394
|
+
},
|
|
1395
|
+
applyRetentionBoost(results) {
|
|
1396
|
+
const lifecycle = this._recallLifecycle;
|
|
1397
|
+
if (lifecycle?.retentionScorer !== true) return results;
|
|
1398
|
+
if (this._retentionScorer === void 0) this._retentionScorer = createRetentionScorer();
|
|
1399
|
+
const scorer = this._retentionScorer;
|
|
1400
|
+
const mode = lifecycle.tierFloorMode;
|
|
1401
|
+
const now = Date.now();
|
|
1402
|
+
return results.map((result) => {
|
|
1403
|
+
const decayable = toDecayableMemory(result.entry);
|
|
1404
|
+
const multiplier = boostMultiplier(scorer.score(decayable, now), mode, decayable.tier);
|
|
1405
|
+
return {
|
|
1406
|
+
...result,
|
|
1407
|
+
score: result.score * multiplier
|
|
1408
|
+
};
|
|
1409
|
+
});
|
|
1410
|
+
},
|
|
1411
|
+
applyMmrDiversity(results) {
|
|
1412
|
+
if (results.length === 0) return results;
|
|
1413
|
+
const lambda = this.config.mmrLambda ?? .7;
|
|
1414
|
+
const chunkIds = results.map((r) => r.chunkId).filter((id) => id !== void 0);
|
|
1415
|
+
const vectorMap = chunkIds.length > 0 ? this.store.getVectorsByIds(chunkIds) : /* @__PURE__ */ new Map();
|
|
1416
|
+
const sorted = [...results].sort((a, b) => {
|
|
1417
|
+
const diff = b.score - a.score;
|
|
1418
|
+
if (diff !== 0) return diff;
|
|
1419
|
+
return a.entry.id.localeCompare(b.entry.id);
|
|
1420
|
+
});
|
|
1421
|
+
const maxScore = Math.max(sorted[0]?.score ?? 0, 1e-9);
|
|
1422
|
+
const n = sorted.length;
|
|
1423
|
+
const vectors = new Array(n);
|
|
1424
|
+
for (let i = 0; i < n; i += 1) {
|
|
1425
|
+
const id = sorted[i]?.chunkId;
|
|
1426
|
+
vectors[i] = id ? vectorMap.get(id) : void 0;
|
|
1427
|
+
}
|
|
1428
|
+
const comparableIndices = vectors.flatMap((vector, index) => vector ? [index] : []);
|
|
1429
|
+
const firstComparableIndex = comparableIndices[0];
|
|
1430
|
+
const dimension = firstComparableIndex === void 0 ? void 0 : vectors[firstComparableIndex]?.length;
|
|
1431
|
+
const dimensionsMatch = dimension !== void 0 && comparableIndices.every((index) => vectors[index]?.length === dimension);
|
|
1432
|
+
if (firstComparableIndex === void 0 || !dimensionsMatch || comparableIndices.length < 2) return sorted.map((result) => ({
|
|
1433
|
+
...result,
|
|
1434
|
+
mmrScore: result.score / maxScore
|
|
1435
|
+
}));
|
|
1436
|
+
const maxSim = new Float64Array(n);
|
|
1437
|
+
const taken = new Uint8Array(n);
|
|
1438
|
+
const selected = [];
|
|
1439
|
+
const seed = sorted[firstComparableIndex];
|
|
1440
|
+
if (!seed) return selected;
|
|
1441
|
+
taken[firstComparableIndex] = 1;
|
|
1442
|
+
selected.push({
|
|
1443
|
+
...seed,
|
|
1444
|
+
mmrScore: seed.score / maxScore
|
|
1445
|
+
});
|
|
1446
|
+
let lastAdmitted = firstComparableIndex;
|
|
1447
|
+
while (selected.length < comparableIndices.length) {
|
|
1448
|
+
const justAddedVec = vectors[lastAdmitted];
|
|
1449
|
+
if (justAddedVec) for (const i of comparableIndices) {
|
|
1450
|
+
if (taken[i] === 1) continue;
|
|
1451
|
+
const v = vectors[i];
|
|
1452
|
+
if (!v) continue;
|
|
1453
|
+
const sim = clamp01(dotProduct(v, justAddedVec), 0);
|
|
1454
|
+
if (sim > (maxSim[i] ?? 0)) maxSim[i] = sim;
|
|
1455
|
+
}
|
|
1456
|
+
let bestIndex = -1;
|
|
1457
|
+
let bestMmrScore = Number.NEGATIVE_INFINITY;
|
|
1458
|
+
for (const i of comparableIndices) {
|
|
1459
|
+
if (taken[i] === 1) continue;
|
|
1460
|
+
const candidate = sorted[i];
|
|
1461
|
+
if (!candidate) continue;
|
|
1462
|
+
const mmrScore = lambda * (candidate.score / maxScore) - (1 - lambda) * (maxSim[i] ?? 0);
|
|
1463
|
+
if (mmrScore > bestMmrScore) {
|
|
1464
|
+
bestMmrScore = mmrScore;
|
|
1465
|
+
bestIndex = i;
|
|
1466
|
+
}
|
|
1467
|
+
}
|
|
1468
|
+
if (bestIndex === -1) break;
|
|
1469
|
+
const next = sorted[bestIndex];
|
|
1470
|
+
if (!next) break;
|
|
1471
|
+
taken[bestIndex] = 1;
|
|
1472
|
+
selected.push({
|
|
1473
|
+
...next,
|
|
1474
|
+
mmrScore: bestMmrScore
|
|
1475
|
+
});
|
|
1476
|
+
lastAdmitted = bestIndex;
|
|
1477
|
+
}
|
|
1478
|
+
let selectedIndex = 0;
|
|
1479
|
+
return sorted.map((result, index) => {
|
|
1480
|
+
if (!vectors[index]) return {
|
|
1481
|
+
...result,
|
|
1482
|
+
mmrScore: result.score / maxScore
|
|
1483
|
+
};
|
|
1484
|
+
const diversified = selected[selectedIndex];
|
|
1485
|
+
selectedIndex += 1;
|
|
1486
|
+
return diversified ?? {
|
|
1487
|
+
...result,
|
|
1488
|
+
mmrScore: result.score / maxScore
|
|
1489
|
+
};
|
|
1490
|
+
});
|
|
1491
|
+
}
|
|
1492
|
+
});
|
|
1493
|
+
//#endregion
|
|
1494
|
+
//#region src/engine/retrieval/retriever-execution.ts
|
|
1495
|
+
/** @file retriever-execution.ts
|
|
1496
|
+
* @purpose Runs retrieval entrypoints, traces, and stats recording.
|
|
1497
|
+
* @boundary Prototype-mounted MemoryRetriever methods; no constructor state ownership.
|
|
1498
|
+
*/
|
|
1499
|
+
Object.assign(MemoryRetriever.prototype, {
|
|
1500
|
+
async retrieve(input) {
|
|
1501
|
+
return withLogContext({
|
|
1502
|
+
operation_id: currentLogContext().operation_id ?? randomUUID(),
|
|
1503
|
+
...input.external_reference !== void 0 ? {
|
|
1504
|
+
external_reference: input.external_reference,
|
|
1505
|
+
external_reference_visibility: input.external_reference_visibility
|
|
1506
|
+
} : {}
|
|
1507
|
+
}, async () => {
|
|
1508
|
+
const started = performance.now();
|
|
1509
|
+
const context = withServingValidityDefault({
|
|
1510
|
+
...input,
|
|
1511
|
+
signal: memoryOperationSignal(input.signal)
|
|
1512
|
+
});
|
|
1513
|
+
log.info("retrieval started", {
|
|
1514
|
+
queryLength: context.query.length,
|
|
1515
|
+
limit: context.limit,
|
|
1516
|
+
mode: this.config.mode,
|
|
1517
|
+
source: context.source
|
|
1518
|
+
}, {
|
|
1519
|
+
event_name: "memory.retrieval.started",
|
|
1520
|
+
file: "packages/memory/src/engine/retrieval/retriever-execution.ts",
|
|
1521
|
+
function: "retrieve",
|
|
1522
|
+
site_id: "memory.retrieval.started"
|
|
1523
|
+
});
|
|
1524
|
+
const traceStartMs = Date.now();
|
|
1525
|
+
try {
|
|
1526
|
+
this.throwIfAborted(context);
|
|
1527
|
+
const trace = new TraceCollector();
|
|
1528
|
+
const isExplicitAggregation = (context.allowAggregation === true || context.source === void 0) && context.aggregation !== void 0;
|
|
1529
|
+
let results;
|
|
1530
|
+
if (isExplicitAggregation) results = await this.aggregationComplete(context, trace);
|
|
1531
|
+
else if (this.config.mode === "vector" || !this.store.hasFtsSupport) results = await this.vectorOnly(context, trace);
|
|
1532
|
+
else results = await this.precisionRecall(context, trace);
|
|
1533
|
+
const beforeFinalFilter = results.length;
|
|
1534
|
+
results = results.filter((result) => this.store.isMemoryOnFactSurface(result.entry.id));
|
|
1535
|
+
this.throwIfAborted(context);
|
|
1536
|
+
if (!isExplicitAggregation && this._accessTracker && (this._recallLifecycle?.autoRecallAccessTracking === true || context.source === "manual") && results.length > 0) this._accessTracker.recordAccess(results.map((r) => r.entry.id));
|
|
1537
|
+
if (!isExplicitAggregation && this._recallLifecycle?.tierPromoter && this._tierPromoter && results.length > 0) await this.evaluateTopKTierTransitions(results);
|
|
1538
|
+
const mode = isExplicitAggregation ? "aggregation" : this.config.mode === "vector" || !this.store.hasFtsSupport ? "vector" : "precision-recall";
|
|
1539
|
+
const finalTrace = trace?.finalize(context.query, mode);
|
|
1540
|
+
if (finalTrace && this._statsCollector) this._statsCollector.recordQuery(finalTrace, context.source ?? "unknown");
|
|
1541
|
+
if (isTraceEnabled()) {
|
|
1542
|
+
if (this._cachedConfigHash === void 0) this._cachedConfigHash = computeConfigHash();
|
|
1543
|
+
const fallbackIds = results.map((r, i) => r.chunkId ?? `__no_chunk_id__:${r.entry.id}#${i}`);
|
|
1544
|
+
const dense = collectParallelStage(results, "denseScore", fallbackIds);
|
|
1545
|
+
const bm25 = collectParallelStage(results, "bm25Score", fallbackIds);
|
|
1546
|
+
const fused = collectParallelStage(results, "fusedScore", fallbackIds);
|
|
1547
|
+
const rerank = collectParallelStage(results, "rerankScore", fallbackIds);
|
|
1548
|
+
const mmr = collectParallelStage(results, "mmrScore", fallbackIds);
|
|
1549
|
+
const omitted = {};
|
|
1550
|
+
if (dense.omitted.length > 0) omitted.denseScores = dense.omitted;
|
|
1551
|
+
if (bm25.omitted.length > 0) omitted.bm25Scores = bm25.omitted;
|
|
1552
|
+
if (fused.omitted.length > 0) omitted.fusedScores = fused.omitted;
|
|
1553
|
+
if (rerank.omitted.length > 0) omitted.rerankScores = rerank.omitted;
|
|
1554
|
+
if (mmr.omitted.length > 0) omitted.mmrScores = mmr.omitted;
|
|
1555
|
+
const traceMetadata = Object.keys(omitted).length > 0 ? { omittedScoreArrays: omitted } : void 0;
|
|
1556
|
+
const stageRecord = finalTrace?.stages.map((stage) => ({
|
|
1557
|
+
name: stage.name,
|
|
1558
|
+
inputCount: stage.inputCount,
|
|
1559
|
+
outputCount: stage.outputCount,
|
|
1560
|
+
droppedIds: stage.droppedIds,
|
|
1561
|
+
outputIds: stage.outputIds,
|
|
1562
|
+
scoreRange: stage.scoreRange,
|
|
1563
|
+
durationMs: stage.durationMs,
|
|
1564
|
+
...typeof stage.metadata?.skipped === "string" && { skipped: stage.metadata.skipped }
|
|
1565
|
+
}));
|
|
1566
|
+
appendQaTrace({
|
|
1567
|
+
query: context.query,
|
|
1568
|
+
...stageRecord && stageRecord.length > 0 && { stages: stageRecord },
|
|
1569
|
+
retrievedChunkIds: results.map((r) => r.chunkId ?? ""),
|
|
1570
|
+
retrievedParentMemoryIds: results.map((r) => r.entry.id),
|
|
1571
|
+
...dense.values && { denseScores: dense.values },
|
|
1572
|
+
...bm25.values && { bm25Scores: bm25.values },
|
|
1573
|
+
...fused.values && { fusedScores: fused.values },
|
|
1574
|
+
...rerank.values && { rerankScores: rerank.values },
|
|
1575
|
+
...mmr.values && { mmrScores: mmr.values },
|
|
1576
|
+
finalMemoryScores: results.map((r) => r.score),
|
|
1577
|
+
embedderProvider: this.embedder.providerKind,
|
|
1578
|
+
embedderModel: this.embedder.model,
|
|
1579
|
+
embedderDim: this.embedder.dimensions,
|
|
1580
|
+
chunkingVersion: CHUNKING_VERSION,
|
|
1581
|
+
configHash: this._cachedConfigHash,
|
|
1582
|
+
timestampMs: traceStartMs,
|
|
1583
|
+
latencyMs: performance.now() - started,
|
|
1584
|
+
...traceMetadata && { traceMetadata }
|
|
1585
|
+
});
|
|
1586
|
+
}
|
|
1587
|
+
log.info("Memory retrieval completed", {
|
|
1588
|
+
store_reference: privateLogReference(this.store.dbPath),
|
|
1589
|
+
outcome: results.length ? "success" : "empty_success",
|
|
1590
|
+
duration_ms: performance.now() - started,
|
|
1591
|
+
mode,
|
|
1592
|
+
scope_count: context.scopeFilter?.length ?? "unavailable",
|
|
1593
|
+
result_count: results.length,
|
|
1594
|
+
final_filter_removed_count: beforeFinalFilter - results.length,
|
|
1595
|
+
config_hash: this._cachedConfigHash ?? computeConfigHash(),
|
|
1596
|
+
fts_available: this.store.hasFtsSupport,
|
|
1597
|
+
stages: finalTrace?.stages.map(({ name, inputCount, outputCount, scoreRange, durationMs, metadata }) => ({
|
|
1598
|
+
name,
|
|
1599
|
+
inputCount,
|
|
1600
|
+
outputCount,
|
|
1601
|
+
scoreRange,
|
|
1602
|
+
durationMs,
|
|
1603
|
+
metadata
|
|
1604
|
+
})),
|
|
1605
|
+
trace_available: isTraceEnabled()
|
|
1606
|
+
}, {
|
|
1607
|
+
event_name: "memory.retrieval.completed",
|
|
1608
|
+
file: "packages/memory/src/engine/retrieval/retriever-execution.ts",
|
|
1609
|
+
function: "retrieve",
|
|
1610
|
+
site_id: "memory.retrieval.completed"
|
|
1611
|
+
});
|
|
1612
|
+
return results;
|
|
1613
|
+
} catch (error) {
|
|
1614
|
+
log.error("Memory retrieval failed", {
|
|
1615
|
+
outcome: "failed",
|
|
1616
|
+
error,
|
|
1617
|
+
duration_ms: performance.now() - started
|
|
1618
|
+
}, {
|
|
1619
|
+
event_name: "memory.retrieval.completed",
|
|
1620
|
+
file: "packages/memory/src/engine/retrieval/retriever-execution.ts",
|
|
1621
|
+
function: "retrieve",
|
|
1622
|
+
site_id: "memory.retrieval.failed"
|
|
1623
|
+
});
|
|
1624
|
+
throw new RetrievalError("Failed to retrieve memories", error);
|
|
1625
|
+
}
|
|
1626
|
+
});
|
|
1627
|
+
},
|
|
1628
|
+
async retrieveWithTrace(input) {
|
|
1629
|
+
return withLogContext({
|
|
1630
|
+
operation_id: currentLogContext().operation_id ?? randomUUID(),
|
|
1631
|
+
...input.external_reference !== void 0 ? {
|
|
1632
|
+
external_reference: input.external_reference,
|
|
1633
|
+
external_reference_visibility: input.external_reference_visibility
|
|
1634
|
+
} : {}
|
|
1635
|
+
}, async () => {
|
|
1636
|
+
const started = performance.now();
|
|
1637
|
+
const context = withServingValidityDefault({
|
|
1638
|
+
...input,
|
|
1639
|
+
signal: memoryOperationSignal(input.signal)
|
|
1640
|
+
});
|
|
1641
|
+
try {
|
|
1642
|
+
this.throwIfAborted(context);
|
|
1643
|
+
const trace = new TraceCollector();
|
|
1644
|
+
const isExplicitAggregation = (context.allowAggregation === true || context.source === void 0) && context.aggregation !== void 0;
|
|
1645
|
+
let results;
|
|
1646
|
+
if (isExplicitAggregation) results = await this.aggregationComplete(context, trace);
|
|
1647
|
+
else if (this.config.mode === "vector" || !this.store.hasFtsSupport) results = await this.vectorOnly(context, trace);
|
|
1648
|
+
else results = await this.precisionRecall(context, trace);
|
|
1649
|
+
results = results.filter((result) => this.store.isMemoryOnFactSurface(result.entry.id));
|
|
1650
|
+
this.throwIfAborted(context);
|
|
1651
|
+
const mode = isExplicitAggregation ? "aggregation" : this.config.mode === "vector" || !this.store.hasFtsSupport ? "vector" : "precision-recall";
|
|
1652
|
+
const finalTrace = trace.finalize(context.query, mode);
|
|
1653
|
+
if (!isExplicitAggregation && this._accessTracker && (this._recallLifecycle?.autoRecallAccessTracking === true || context.source === "manual") && results.length > 0) this._accessTracker.recordAccess(results.map((r) => r.entry.id));
|
|
1654
|
+
if (!isExplicitAggregation && this._recallLifecycle?.tierPromoter && this._tierPromoter && results.length > 0) await this.evaluateTopKTierTransitions(results);
|
|
1655
|
+
if (this._statsCollector) this._statsCollector.recordQuery(finalTrace, context.source ?? "debug");
|
|
1656
|
+
log.info("Traced memory retrieval completed", {
|
|
1657
|
+
outcome: results.length ? "success" : "empty_success",
|
|
1658
|
+
store_reference: privateLogReference(this.store.dbPath),
|
|
1659
|
+
duration_ms: performance.now() - started,
|
|
1660
|
+
mode,
|
|
1661
|
+
result_count: results.length,
|
|
1662
|
+
stages: finalTrace.stages.map(({ name, inputCount, outputCount, scoreRange, durationMs, metadata }) => ({
|
|
1663
|
+
name,
|
|
1664
|
+
inputCount,
|
|
1665
|
+
outputCount,
|
|
1666
|
+
scoreRange,
|
|
1667
|
+
durationMs,
|
|
1668
|
+
metadata
|
|
1669
|
+
}))
|
|
1670
|
+
}, {
|
|
1671
|
+
event_name: "memory.retrieval.completed",
|
|
1672
|
+
file: "packages/memory/src/engine/retrieval/retriever-execution.ts",
|
|
1673
|
+
function: "retrieveWithTrace",
|
|
1674
|
+
site_id: "memory.retrieval.traced.completed"
|
|
1675
|
+
});
|
|
1676
|
+
return {
|
|
1677
|
+
results,
|
|
1678
|
+
trace: finalTrace
|
|
1679
|
+
};
|
|
1680
|
+
} catch (error) {
|
|
1681
|
+
log.error("Traced memory retrieval failed", {
|
|
1682
|
+
outcome: "failed",
|
|
1683
|
+
error,
|
|
1684
|
+
duration_ms: performance.now() - started
|
|
1685
|
+
}, {
|
|
1686
|
+
event_name: "memory.retrieval.completed",
|
|
1687
|
+
file: "packages/memory/src/engine/retrieval/retriever-execution.ts",
|
|
1688
|
+
function: "retrieveWithTrace",
|
|
1689
|
+
site_id: "memory.retrieval.traced.failed"
|
|
1690
|
+
});
|
|
1691
|
+
throw new RetrievalError("Failed to retrieve memories", error);
|
|
1692
|
+
}
|
|
1693
|
+
});
|
|
1694
|
+
},
|
|
1695
|
+
/**
|
|
1696
|
+
* Phase 0 §5: evaluate tier promotion/demotion for the top-K retrieved
|
|
1697
|
+
* memories and persist any non-null transition via `store.updateTier`.
|
|
1698
|
+
*
|
|
1699
|
+
* Per-item try/catch isolates evaluator and persistence failures so one
|
|
1700
|
+
* bad memory cannot block the rest. The retention scorer is instantiated
|
|
1701
|
+
* lazily on the retriever instance — synthesizing `DecayableMemory` from
|
|
1702
|
+
* `entry.metadata` keeps this hot-path read-only against the store.
|
|
1703
|
+
*
|
|
1704
|
+
* Callers gate on `recallLifecycle.tierPromoter`; this method assumes the
|
|
1705
|
+
* flag is on and `_tierPromoter` is set.
|
|
1706
|
+
*/
|
|
1707
|
+
async evaluateTopKTierTransitions(results) {
|
|
1708
|
+
const promoter = this._tierPromoter;
|
|
1709
|
+
if (!promoter) return;
|
|
1710
|
+
const topK = this._recallLifecycle?.tierPromotionTopK ?? results.length;
|
|
1711
|
+
if (!this._retentionScorer) this._retentionScorer = createRetentionScorer();
|
|
1712
|
+
const scorer = this._retentionScorer;
|
|
1713
|
+
const now = Date.now();
|
|
1714
|
+
const slice = results.slice(0, Math.max(0, topK));
|
|
1715
|
+
for (const r of slice) try {
|
|
1716
|
+
const entry = r.entry;
|
|
1717
|
+
const access = parseAccessMetadata(entry.metadata);
|
|
1718
|
+
const insight = parseInsightMetadata(entry.metadata, entry);
|
|
1719
|
+
const tier = insight.tier ?? "peripheral";
|
|
1720
|
+
const confidence = insight.confidence ?? .7;
|
|
1721
|
+
const decayable = {
|
|
1722
|
+
id: entry.id,
|
|
1723
|
+
importance: entry.importance,
|
|
1724
|
+
confidence,
|
|
1725
|
+
tier,
|
|
1726
|
+
accessCount: access.accessCount,
|
|
1727
|
+
createdAt: entry.timestamp,
|
|
1728
|
+
lastAccessedAt: access.lastAccessedAt || entry.timestamp,
|
|
1729
|
+
metadata: entry.metadata,
|
|
1730
|
+
...insight.memory_temporal_type && { temporalType: insight.memory_temporal_type }
|
|
1731
|
+
};
|
|
1732
|
+
const score = scorer.score(decayable, now);
|
|
1733
|
+
const tierableMemory = {
|
|
1734
|
+
id: entry.id,
|
|
1735
|
+
tier,
|
|
1736
|
+
importance: entry.importance,
|
|
1737
|
+
accessCount: access.accessCount,
|
|
1738
|
+
timestamp: entry.timestamp
|
|
1739
|
+
};
|
|
1740
|
+
const transition = promoter.evaluate(tierableMemory, score, now);
|
|
1741
|
+
if (!transition || transition.toTier === tier) continue;
|
|
1742
|
+
await this.store.updateTier(entry.id, transition.toTier, { writerAuthority: "offline-family" });
|
|
1743
|
+
} catch (error) {
|
|
1744
|
+
log.warn("tier evaluation failed for memory", {
|
|
1745
|
+
memoryId: r.entry.id,
|
|
1746
|
+
error
|
|
1747
|
+
}, {
|
|
1748
|
+
event_name: "memory.retrieval.tier.failed",
|
|
1749
|
+
file: "packages/memory/src/engine/retrieval/retriever-execution.ts",
|
|
1750
|
+
function: "evaluateTopKTierTransitions",
|
|
1751
|
+
site_id: "memory.retrieval.tier.failed"
|
|
1752
|
+
});
|
|
1753
|
+
}
|
|
1754
|
+
}
|
|
1755
|
+
});
|
|
1756
|
+
//#endregion
|
|
1757
|
+
export { DEFAULT_RETRIEVAL_CONFIG as a, createRetriever as i, MemoryRetriever as n, withServingValidityDefault as o, RERANK_FALLBACK_REASONS as r, collectParallelStage as t };
|