@astragenie/astramemory-local 0.7.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +341 -0
- package/README.md +419 -0
- package/dist/backup/retention.d.ts +15 -0
- package/dist/backup/retention.js +62 -0
- package/dist/backup/retention.js.map +1 -0
- package/dist/backup/snapshot.d.ts +21 -0
- package/dist/backup/snapshot.js +55 -0
- package/dist/backup/snapshot.js.map +1 -0
- package/dist/backup/verify.d.ts +23 -0
- package/dist/backup/verify.js +77 -0
- package/dist/backup/verify.js.map +1 -0
- package/dist/budget/tracker.d.ts +58 -0
- package/dist/budget/tracker.js +102 -0
- package/dist/budget/tracker.js.map +1 -0
- package/dist/capture/codex.d.ts +63 -0
- package/dist/capture/codex.js +0 -0
- package/dist/capture/codex.js.map +1 -0
- package/dist/cli/backup.d.ts +1 -0
- package/dist/cli/backup.js +112 -0
- package/dist/cli/backup.js.map +1 -0
- package/dist/cli/budget.d.ts +7 -0
- package/dist/cli/budget.js +44 -0
- package/dist/cli/budget.js.map +1 -0
- package/dist/cli/capture.d.ts +10 -0
- package/dist/cli/capture.js +113 -0
- package/dist/cli/capture.js.map +1 -0
- package/dist/cli/consolidate.d.ts +16 -0
- package/dist/cli/consolidate.js +146 -0
- package/dist/cli/consolidate.js.map +1 -0
- package/dist/cli/doctor.d.ts +1 -0
- package/dist/cli/doctor.js +54 -0
- package/dist/cli/doctor.js.map +1 -0
- package/dist/cli/entity-backfill.d.ts +10 -0
- package/dist/cli/entity-backfill.js +46 -0
- package/dist/cli/entity-backfill.js.map +1 -0
- package/dist/cli/hook-install.d.ts +45 -0
- package/dist/cli/hook-install.js +77 -0
- package/dist/cli/hook-install.js.map +1 -0
- package/dist/cli/index.d.ts +2 -0
- package/dist/cli/index.js +312 -0
- package/dist/cli/index.js.map +1 -0
- package/dist/cli/init.d.ts +16 -0
- package/dist/cli/init.js +431 -0
- package/dist/cli/init.js.map +1 -0
- package/dist/cli/mcp-stdio.d.ts +18 -0
- package/dist/cli/mcp-stdio.js +67 -0
- package/dist/cli/mcp-stdio.js.map +1 -0
- package/dist/cli/memory.d.ts +15 -0
- package/dist/cli/memory.js +52 -0
- package/dist/cli/memory.js.map +1 -0
- package/dist/cli/open-runtime-db.d.ts +15 -0
- package/dist/cli/open-runtime-db.js +37 -0
- package/dist/cli/open-runtime-db.js.map +1 -0
- package/dist/cli/pair.d.ts +29 -0
- package/dist/cli/pair.js +64 -0
- package/dist/cli/pair.js.map +1 -0
- package/dist/cli/providers.d.ts +10 -0
- package/dist/cli/providers.js +97 -0
- package/dist/cli/providers.js.map +1 -0
- package/dist/cli/queue-purge.d.ts +5 -0
- package/dist/cli/queue-purge.js +92 -0
- package/dist/cli/queue-purge.js.map +1 -0
- package/dist/cli/queue.d.ts +29 -0
- package/dist/cli/queue.js +73 -0
- package/dist/cli/queue.js.map +1 -0
- package/dist/cli/rebuild.d.ts +15 -0
- package/dist/cli/rebuild.js +70 -0
- package/dist/cli/rebuild.js.map +1 -0
- package/dist/cli/reembed-dim.d.ts +21 -0
- package/dist/cli/reembed-dim.js +199 -0
- package/dist/cli/reembed-dim.js.map +1 -0
- package/dist/cli/reinstall.d.ts +1 -0
- package/dist/cli/reinstall.js +205 -0
- package/dist/cli/reinstall.js.map +1 -0
- package/dist/cli/restore.d.ts +1 -0
- package/dist/cli/restore.js +167 -0
- package/dist/cli/restore.js.map +1 -0
- package/dist/cli/retag.d.ts +14 -0
- package/dist/cli/retag.js +62 -0
- package/dist/cli/retag.js.map +1 -0
- package/dist/cli/search.d.ts +66 -0
- package/dist/cli/search.js +174 -0
- package/dist/cli/search.js.map +1 -0
- package/dist/cli/serve.d.ts +9 -0
- package/dist/cli/serve.js +364 -0
- package/dist/cli/serve.js.map +1 -0
- package/dist/cli/service.d.ts +1 -0
- package/dist/cli/service.js +121 -0
- package/dist/cli/service.js.map +1 -0
- package/dist/cli/sync.d.ts +15 -0
- package/dist/cli/sync.js +61 -0
- package/dist/cli/sync.js.map +1 -0
- package/dist/cli/token.d.ts +24 -0
- package/dist/cli/token.js +77 -0
- package/dist/cli/token.js.map +1 -0
- package/dist/cli/wait-health.d.ts +4 -0
- package/dist/cli/wait-health.js +23 -0
- package/dist/cli/wait-health.js.map +1 -0
- package/dist/config/config.d.ts +127 -0
- package/dist/config/config.js +38 -0
- package/dist/config/config.js.map +1 -0
- package/dist/config/datadir.d.ts +30 -0
- package/dist/config/datadir.js +65 -0
- package/dist/config/datadir.js.map +1 -0
- package/dist/config/loader.d.ts +23 -0
- package/dist/config/loader.js +102 -0
- package/dist/config/loader.js.map +1 -0
- package/dist/config/migrate-dirs.d.ts +36 -0
- package/dist/config/migrate-dirs.js +132 -0
- package/dist/config/migrate-dirs.js.map +1 -0
- package/dist/config/persist-envs.d.ts +23 -0
- package/dist/config/persist-envs.js +119 -0
- package/dist/config/persist-envs.js.map +1 -0
- package/dist/config/resolve-runtime.d.ts +19 -0
- package/dist/config/resolve-runtime.js +53 -0
- package/dist/config/resolve-runtime.js.map +1 -0
- package/dist/config/secrets.d.ts +28 -0
- package/dist/config/secrets.js +38 -0
- package/dist/config/secrets.js.map +1 -0
- package/dist/config/sync-settings.d.ts +16 -0
- package/dist/config/sync-settings.js +34 -0
- package/dist/config/sync-settings.js.map +1 -0
- package/dist/config/writer.d.ts +19 -0
- package/dist/config/writer.js +121 -0
- package/dist/config/writer.js.map +1 -0
- package/dist/consolidate/consolidate.d.ts +80 -0
- package/dist/consolidate/consolidate.js +0 -0
- package/dist/consolidate/consolidate.js.map +1 -0
- package/dist/consolidate/proposals.d.ts +35 -0
- package/dist/consolidate/proposals.js +66 -0
- package/dist/consolidate/proposals.js.map +1 -0
- package/dist/contracts/atom-wire.d.ts +48 -0
- package/dist/contracts/atom-wire.js +55 -0
- package/dist/contracts/atom-wire.js.map +1 -0
- package/dist/contracts/embed.d.ts +41 -0
- package/dist/contracts/embed.js +20 -0
- package/dist/contracts/embed.js.map +1 -0
- package/dist/contracts/index.d.ts +5 -0
- package/dist/contracts/index.js +6 -0
- package/dist/contracts/index.js.map +1 -0
- package/dist/contracts/job.d.ts +113 -0
- package/dist/contracts/job.js +32 -0
- package/dist/contracts/job.js.map +1 -0
- package/dist/contracts/llm.d.ts +30 -0
- package/dist/contracts/llm.js +2 -0
- package/dist/contracts/llm.js.map +1 -0
- package/dist/contracts/memory.d.ts +47 -0
- package/dist/contracts/memory.js +5 -0
- package/dist/contracts/memory.js.map +1 -0
- package/dist/contracts/vector.d.ts +29 -0
- package/dist/contracts/vector.js +2 -0
- package/dist/contracts/vector.js.map +1 -0
- package/dist/distill/flatten-turns.d.ts +1 -0
- package/dist/distill/flatten-turns.js +50 -0
- package/dist/distill/flatten-turns.js.map +1 -0
- package/dist/distill/pipeline.d.ts +45 -0
- package/dist/distill/pipeline.js +113 -0
- package/dist/distill/pipeline.js.map +1 -0
- package/dist/distill/prompts/extract.d.ts +122 -0
- package/dist/distill/prompts/extract.js +67 -0
- package/dist/distill/prompts/extract.js.map +1 -0
- package/dist/distill/stages/01-cleanup.d.ts +9 -0
- package/dist/distill/stages/01-cleanup.js +67 -0
- package/dist/distill/stages/01-cleanup.js.map +1 -0
- package/dist/distill/stages/02-normalize.d.ts +9 -0
- package/dist/distill/stages/02-normalize.js +76 -0
- package/dist/distill/stages/02-normalize.js.map +1 -0
- package/dist/distill/stages/03-chunk.d.ts +22 -0
- package/dist/distill/stages/03-chunk.js +138 -0
- package/dist/distill/stages/03-chunk.js.map +1 -0
- package/dist/distill/stages/04-compact.d.ts +28 -0
- package/dist/distill/stages/04-compact.js +69 -0
- package/dist/distill/stages/04-compact.js.map +1 -0
- package/dist/distill/stages/05-extract.d.ts +35 -0
- package/dist/distill/stages/05-extract.js +101 -0
- package/dist/distill/stages/05-extract.js.map +1 -0
- package/dist/distill/stages/06-reduce.d.ts +16 -0
- package/dist/distill/stages/06-reduce.js +30 -0
- package/dist/distill/stages/06-reduce.js.map +1 -0
- package/dist/distill/stages/07-memory-normalize.d.ts +27 -0
- package/dist/distill/stages/07-memory-normalize.js +65 -0
- package/dist/distill/stages/07-memory-normalize.js.map +1 -0
- package/dist/distill/stages/08-embed-index.d.ts +31 -0
- package/dist/distill/stages/08-embed-index.js +82 -0
- package/dist/distill/stages/08-embed-index.js.map +1 -0
- package/dist/doctor/checks.d.ts +77 -0
- package/dist/doctor/checks.js +626 -0
- package/dist/doctor/checks.js.map +1 -0
- package/dist/doctor/hardening-checks.d.ts +9 -0
- package/dist/doctor/hardening-checks.js +182 -0
- package/dist/doctor/hardening-checks.js.map +1 -0
- package/dist/doctor/probes/embed-probe.d.ts +19 -0
- package/dist/doctor/probes/embed-probe.js +47 -0
- package/dist/doctor/probes/embed-probe.js.map +1 -0
- package/dist/doctor/probes/llm-chat-probe.d.ts +11 -0
- package/dist/doctor/probes/llm-chat-probe.js +41 -0
- package/dist/doctor/probes/llm-chat-probe.js.map +1 -0
- package/dist/doctor/probes/plugin-coexistence.d.ts +14 -0
- package/dist/doctor/probes/plugin-coexistence.js +60 -0
- package/dist/doctor/probes/plugin-coexistence.js.map +1 -0
- package/dist/doctor/runner.d.ts +17 -0
- package/dist/doctor/runner.js +53 -0
- package/dist/doctor/runner.js.map +1 -0
- package/dist/doctor/types.d.ts +12 -0
- package/dist/doctor/types.js +2 -0
- package/dist/doctor/types.js.map +1 -0
- package/dist/entity/backfill.d.ts +30 -0
- package/dist/entity/backfill.js +55 -0
- package/dist/entity/backfill.js.map +1 -0
- package/dist/entity/extract-entities.d.ts +27 -0
- package/dist/entity/extract-entities.js +86 -0
- package/dist/entity/extract-entities.js.map +1 -0
- package/dist/entity/normalize.d.ts +17 -0
- package/dist/entity/normalize.js +20 -0
- package/dist/entity/normalize.js.map +1 -0
- package/dist/eval/harness.d.ts +96 -0
- package/dist/eval/harness.js +119 -0
- package/dist/eval/harness.js.map +1 -0
- package/dist/eval/metrics.d.ts +23 -0
- package/dist/eval/metrics.js +44 -0
- package/dist/eval/metrics.js.map +1 -0
- package/dist/log/correlation.d.ts +24 -0
- package/dist/log/correlation.js +33 -0
- package/dist/log/correlation.js.map +1 -0
- package/dist/log/logger.d.ts +38 -0
- package/dist/log/logger.js +129 -0
- package/dist/log/logger.js.map +1 -0
- package/dist/log/scrub.d.ts +33 -0
- package/dist/log/scrub.js +91 -0
- package/dist/log/scrub.js.map +1 -0
- package/dist/mcp/server.d.ts +36 -0
- package/dist/mcp/server.js +553 -0
- package/dist/mcp/server.js.map +1 -0
- package/dist/memory-tool/adapter.d.ts +73 -0
- package/dist/memory-tool/adapter.js +269 -0
- package/dist/memory-tool/adapter.js.map +1 -0
- package/dist/pipeline/errors.d.ts +21 -0
- package/dist/pipeline/errors.js +34 -0
- package/dist/pipeline/errors.js.map +1 -0
- package/dist/pipeline/failure-classifier.d.ts +13 -0
- package/dist/pipeline/failure-classifier.js +72 -0
- package/dist/pipeline/failure-classifier.js.map +1 -0
- package/dist/pipeline/handler-ctx-ext.d.ts +23 -0
- package/dist/pipeline/handler-ctx-ext.js +19 -0
- package/dist/pipeline/handler-ctx-ext.js.map +1 -0
- package/dist/pipeline/handler.d.ts +20 -0
- package/dist/pipeline/handler.js +2 -0
- package/dist/pipeline/handler.js.map +1 -0
- package/dist/pipeline/handlers/cleanup.d.ts +14 -0
- package/dist/pipeline/handlers/cleanup.js +47 -0
- package/dist/pipeline/handlers/cleanup.js.map +1 -0
- package/dist/pipeline/handlers/consolidate.d.ts +8 -0
- package/dist/pipeline/handlers/consolidate.js +23 -0
- package/dist/pipeline/handlers/consolidate.js.map +1 -0
- package/dist/pipeline/handlers/distill-events.d.ts +15 -0
- package/dist/pipeline/handlers/distill-events.js +134 -0
- package/dist/pipeline/handlers/distill-events.js.map +1 -0
- package/dist/pipeline/handlers/distill.d.ts +17 -0
- package/dist/pipeline/handlers/distill.js +110 -0
- package/dist/pipeline/handlers/distill.js.map +1 -0
- package/dist/pipeline/handlers/reembed.d.ts +10 -0
- package/dist/pipeline/handlers/reembed.js +34 -0
- package/dist/pipeline/handlers/reembed.js.map +1 -0
- package/dist/pipeline/job-repo.d.ts +86 -0
- package/dist/pipeline/job-repo.js +168 -0
- package/dist/pipeline/job-repo.js.map +1 -0
- package/dist/pipeline/mock-providers.d.ts +49 -0
- package/dist/pipeline/mock-providers.js +175 -0
- package/dist/pipeline/mock-providers.js.map +1 -0
- package/dist/pipeline/registry.d.ts +15 -0
- package/dist/pipeline/registry.js +20 -0
- package/dist/pipeline/registry.js.map +1 -0
- package/dist/pipeline/worker.d.ts +41 -0
- package/dist/pipeline/worker.js +167 -0
- package/dist/pipeline/worker.js.map +1 -0
- package/dist/providers/embed/azure-openai.d.ts +25 -0
- package/dist/providers/embed/azure-openai.js +138 -0
- package/dist/providers/embed/azure-openai.js.map +1 -0
- package/dist/providers/embed/ollama.d.ts +17 -0
- package/dist/providers/embed/ollama.js +106 -0
- package/dist/providers/embed/ollama.js.map +1 -0
- package/dist/providers/index.d.ts +19 -0
- package/dist/providers/index.js +72 -0
- package/dist/providers/index.js.map +1 -0
- package/dist/providers/llm/azure-openai.d.ts +20 -0
- package/dist/providers/llm/azure-openai.js +135 -0
- package/dist/providers/llm/azure-openai.js.map +1 -0
- package/dist/providers/llm/ollama.d.ts +13 -0
- package/dist/providers/llm/ollama.js +113 -0
- package/dist/providers/llm/ollama.js.map +1 -0
- package/dist/providers/llm/pricing.d.ts +21 -0
- package/dist/providers/llm/pricing.js +22 -0
- package/dist/providers/llm/pricing.js.map +1 -0
- package/dist/recall/pack.d.ts +32 -0
- package/dist/recall/pack.js +90 -0
- package/dist/recall/pack.js.map +1 -0
- package/dist/recall/policy.d.ts +39 -0
- package/dist/recall/policy.js +96 -0
- package/dist/recall/policy.js.map +1 -0
- package/dist/redact/detectors.d.ts +20 -0
- package/dist/redact/detectors.js +85 -0
- package/dist/redact/detectors.js.map +1 -0
- package/dist/redact/entropy.d.ts +24 -0
- package/dist/redact/entropy.js +77 -0
- package/dist/redact/entropy.js.map +1 -0
- package/dist/redact/index.d.ts +47 -0
- package/dist/redact/index.js +165 -0
- package/dist/redact/index.js.map +1 -0
- package/dist/search/fuse.d.ts +108 -0
- package/dist/search/fuse.js +135 -0
- package/dist/search/fuse.js.map +1 -0
- package/dist/search/query.d.ts +28 -0
- package/dist/search/query.js +70 -0
- package/dist/search/query.js.map +1 -0
- package/dist/search/search.d.ts +164 -0
- package/dist/search/search.js +310 -0
- package/dist/search/search.js.map +1 -0
- package/dist/server/app.d.ts +17 -0
- package/dist/server/app.js +133 -0
- package/dist/server/app.js.map +1 -0
- package/dist/server/health-state.d.ts +29 -0
- package/dist/server/health-state.js +28 -0
- package/dist/server/health-state.js.map +1 -0
- package/dist/server/lib/network.d.ts +12 -0
- package/dist/server/lib/network.js +16 -0
- package/dist/server/lib/network.js.map +1 -0
- package/dist/server/lib/score-contract.d.ts +36 -0
- package/dist/server/lib/score-contract.js +54 -0
- package/dist/server/lib/score-contract.js.map +1 -0
- package/dist/server/lib/stable-stringify.d.ts +10 -0
- package/dist/server/lib/stable-stringify.js +27 -0
- package/dist/server/lib/stable-stringify.js.map +1 -0
- package/dist/server/lib/wire-meta.d.ts +7 -0
- package/dist/server/lib/wire-meta.js +29 -0
- package/dist/server/lib/wire-meta.js.map +1 -0
- package/dist/server/queries/dashboard.d.ts +142 -0
- package/dist/server/queries/dashboard.js +166 -0
- package/dist/server/queries/dashboard.js.map +1 -0
- package/dist/server/routes/consolidation.d.ts +14 -0
- package/dist/server/routes/consolidation.js +67 -0
- package/dist/server/routes/consolidation.js.map +1 -0
- package/dist/server/routes/dashboard-api-html.d.ts +15 -0
- package/dist/server/routes/dashboard-api-html.js +144 -0
- package/dist/server/routes/dashboard-api-html.js.map +1 -0
- package/dist/server/routes/dashboard-consolidation-html.d.ts +26 -0
- package/dist/server/routes/dashboard-consolidation-html.js +202 -0
- package/dist/server/routes/dashboard-consolidation-html.js.map +1 -0
- package/dist/server/routes/dashboard-html.d.ts +15 -0
- package/dist/server/routes/dashboard-html.js +365 -0
- package/dist/server/routes/dashboard-html.js.map +1 -0
- package/dist/server/routes/dashboard-jobs-html.d.ts +18 -0
- package/dist/server/routes/dashboard-jobs-html.js +186 -0
- package/dist/server/routes/dashboard-jobs-html.js.map +1 -0
- package/dist/server/routes/dashboard-search-html.d.ts +18 -0
- package/dist/server/routes/dashboard-search-html.js +189 -0
- package/dist/server/routes/dashboard-search-html.js.map +1 -0
- package/dist/server/routes/dashboard.d.ts +19 -0
- package/dist/server/routes/dashboard.js +68 -0
- package/dist/server/routes/dashboard.js.map +1 -0
- package/dist/server/routes/digest.d.ts +9 -0
- package/dist/server/routes/digest.js +37 -0
- package/dist/server/routes/digest.js.map +1 -0
- package/dist/server/routes/entities.d.ts +12 -0
- package/dist/server/routes/entities.js +46 -0
- package/dist/server/routes/entities.js.map +1 -0
- package/dist/server/routes/health.d.ts +14 -0
- package/dist/server/routes/health.js +100 -0
- package/dist/server/routes/health.js.map +1 -0
- package/dist/server/routes/ingest.d.ts +209 -0
- package/dist/server/routes/ingest.js +454 -0
- package/dist/server/routes/ingest.js.map +1 -0
- package/dist/server/routes/lifecycle.d.ts +21 -0
- package/dist/server/routes/lifecycle.js +132 -0
- package/dist/server/routes/lifecycle.js.map +1 -0
- package/dist/server/routes/mcp.d.ts +15 -0
- package/dist/server/routes/mcp.js +36 -0
- package/dist/server/routes/mcp.js.map +1 -0
- package/dist/server/routes/memory-tool.d.ts +14 -0
- package/dist/server/routes/memory-tool.js +28 -0
- package/dist/server/routes/memory-tool.js.map +1 -0
- package/dist/server/routes/memory.d.ts +7 -0
- package/dist/server/routes/memory.js +19 -0
- package/dist/server/routes/memory.js.map +1 -0
- package/dist/server/routes/recall.d.ts +15 -0
- package/dist/server/routes/recall.js +74 -0
- package/dist/server/routes/recall.js.map +1 -0
- package/dist/server/routes/search.d.ts +12 -0
- package/dist/server/routes/search.js +203 -0
- package/dist/server/routes/search.js.map +1 -0
- package/dist/server/routes/version.d.ts +2 -0
- package/dist/server/routes/version.js +11 -0
- package/dist/server/routes/version.js.map +1 -0
- package/dist/server/routes/why.d.ts +9 -0
- package/dist/server/routes/why.js +38 -0
- package/dist/server/routes/why.js.map +1 -0
- package/dist/service/index.d.ts +10 -0
- package/dist/service/index.js +25 -0
- package/dist/service/index.js.map +1 -0
- package/dist/service/install-flow.d.ts +18 -0
- package/dist/service/install-flow.js +47 -0
- package/dist/service/install-flow.js.map +1 -0
- package/dist/service/instance-lock.d.ts +26 -0
- package/dist/service/instance-lock.js +150 -0
- package/dist/service/instance-lock.js.map +1 -0
- package/dist/service/launchd.d.ts +11 -0
- package/dist/service/launchd.js +196 -0
- package/dist/service/launchd.js.map +1 -0
- package/dist/service/schtasks.d.ts +31 -0
- package/dist/service/schtasks.js +274 -0
- package/dist/service/schtasks.js.map +1 -0
- package/dist/service/shim.d.ts +21 -0
- package/dist/service/shim.js +80 -0
- package/dist/service/shim.js.map +1 -0
- package/dist/service/systemd.d.ts +11 -0
- package/dist/service/systemd.js +150 -0
- package/dist/service/systemd.js.map +1 -0
- package/dist/service/task-xml.d.ts +36 -0
- package/dist/service/task-xml.js +91 -0
- package/dist/service/task-xml.js.map +1 -0
- package/dist/service/types.d.ts +47 -0
- package/dist/service/types.js +2 -0
- package/dist/service/types.js.map +1 -0
- package/dist/storage/archival.d.ts +29 -0
- package/dist/storage/archival.js +47 -0
- package/dist/storage/archival.js.map +1 -0
- package/dist/storage/bearer-keystore.d.ts +34 -0
- package/dist/storage/bearer-keystore.js +75 -0
- package/dist/storage/bearer-keystore.js.map +1 -0
- package/dist/storage/db.d.ts +37 -0
- package/dist/storage/db.js +92 -0
- package/dist/storage/db.js.map +1 -0
- package/dist/storage/entities.d.ts +71 -0
- package/dist/storage/entities.js +141 -0
- package/dist/storage/entities.js.map +1 -0
- package/dist/storage/ingest-idempotency.d.ts +26 -0
- package/dist/storage/ingest-idempotency.js +29 -0
- package/dist/storage/ingest-idempotency.js.map +1 -0
- package/dist/storage/keystore.d.ts +64 -0
- package/dist/storage/keystore.js +194 -0
- package/dist/storage/keystore.js.map +1 -0
- package/dist/storage/memories.d.ts +51 -0
- package/dist/storage/memories.js +67 -0
- package/dist/storage/memories.js.map +1 -0
- package/dist/storage/memory-events.d.ts +145 -0
- package/dist/storage/memory-events.js +287 -0
- package/dist/storage/memory-events.js.map +1 -0
- package/dist/storage/migrate-encrypt.d.ts +16 -0
- package/dist/storage/migrate-encrypt.js +121 -0
- package/dist/storage/migrate-encrypt.js.map +1 -0
- package/dist/storage/migrate.d.ts +27 -0
- package/dist/storage/migrate.js +105 -0
- package/dist/storage/migrate.js.map +1 -0
- package/dist/storage/redaction-log.d.ts +18 -0
- package/dist/storage/redaction-log.js +27 -0
- package/dist/storage/redaction-log.js.map +1 -0
- package/dist/storage/usefulness.d.ts +115 -0
- package/dist/storage/usefulness.js +203 -0
- package/dist/storage/usefulness.js.map +1 -0
- package/dist/sync/conflict-resolve.d.ts +26 -0
- package/dist/sync/conflict-resolve.js +139 -0
- package/dist/sync/conflict-resolve.js.map +1 -0
- package/dist/sync/puller.d.ts +115 -0
- package/dist/sync/puller.js +173 -0
- package/dist/sync/puller.js.map +1 -0
- package/dist/sync/shipper.d.ts +112 -0
- package/dist/sync/shipper.js +189 -0
- package/dist/sync/shipper.js.map +1 -0
- package/dist/tag-hygiene/backfill.d.ts +50 -0
- package/dist/tag-hygiene/backfill.js +117 -0
- package/dist/tag-hygiene/backfill.js.map +1 -0
- package/dist/tag-hygiene/derive-repo.d.ts +9 -0
- package/dist/tag-hygiene/derive-repo.js +19 -0
- package/dist/tag-hygiene/derive-repo.js.map +1 -0
- package/dist/tag-hygiene/tier2-infer.d.ts +28 -0
- package/dist/tag-hygiene/tier2-infer.js +72 -0
- package/dist/tag-hygiene/tier2-infer.js.map +1 -0
- package/dist/vector/sqlite-vec.d.ts +16 -0
- package/dist/vector/sqlite-vec.js +49 -0
- package/dist/vector/sqlite-vec.js.map +1 -0
- package/migrations/001-init.sql +117 -0
- package/migrations/002-wire-v1.sql +16 -0
- package/migrations/003-expand-memory-types.sql +81 -0
- package/migrations/004-provenance.sql +4 -0
- package/migrations/005-security.sql +12 -0
- package/migrations/006-atom-v3.sql +28 -0
- package/migrations/007-memory-events.sql +30 -0
- package/migrations/008-consolidation.sql +31 -0
- package/migrations/009-tag-hygiene.sql +13 -0
- package/migrations/010-sync-pull.sql +53 -0
- package/migrations/011-embed-dim-migration.sql +28 -0
- package/migrations/012-entities.sql +36 -0
- package/migrations/013-archival.sql +50 -0
- package/package.json +50 -0
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* migrate-encrypt.ts — plaintext -> encrypted DB auto-migration (SEC-7, AC-2).
|
|
3
|
+
*
|
|
4
|
+
* On daemon startup, a pre-existing v0.3.x plaintext `memory.sqlite` must be
|
|
5
|
+
* transparently upgraded to an encrypted (SQLCipher) file before openDb()
|
|
6
|
+
* applies `PRAGMA key` to it. Detection is a raw byte-header check (no need
|
|
7
|
+
* to even open the file to know it's plaintext): every unencrypted SQLite
|
|
8
|
+
* file begins with the 16-byte magic string `"SQLite format 3\0"`.
|
|
9
|
+
*
|
|
10
|
+
* Migration path (`PRAGMA rekey`, the SQLite3MultipleCiphers mechanism —
|
|
11
|
+
* this driver does NOT ship SQLCipher's `sqlcipher_export()` function):
|
|
12
|
+
* 1. Open the plaintext file with the cipher driver WITHOUT a key
|
|
13
|
+
* (the driver reads plaintext SQLite files natively when no key is set),
|
|
14
|
+
* `PRAGMA wal_checkpoint(TRUNCATE)` to fold any -wal sidecar into the
|
|
15
|
+
* main file, record row counts, close.
|
|
16
|
+
* 2. Copy the (now checkpoint-complete) file to a tmp path and run
|
|
17
|
+
* `PRAGMA rekey = '<key>'` on the COPY — in-place encryption of every
|
|
18
|
+
* page. The original stays untouched until verification passes.
|
|
19
|
+
* 3. Verify: the tmp header is no longer plaintext magic AND reopening tmp
|
|
20
|
+
* with the key reproduces the `memories` + `transcripts` row counts.
|
|
21
|
+
* Abort (throw, delete tmp, original untouched) on any mismatch.
|
|
22
|
+
* 4. Copy the plaintext original to `<dbPath>.pre-encryption.bak` (kept
|
|
23
|
+
* per SEC-7 until the operator is confident the migration is safe —
|
|
24
|
+
* this module does not delete it), then atomically rename the tmp file
|
|
25
|
+
* over `dbPath`.
|
|
26
|
+
*
|
|
27
|
+
* Idempotent: a second call against an already-encrypted file (header is no
|
|
28
|
+
* longer the plaintext magic) is a no-op that returns 'already-encrypted'.
|
|
29
|
+
*/
|
|
30
|
+
import { readFileSync, existsSync, copyFileSync, renameSync, rmSync } from 'node:fs';
|
|
31
|
+
import CipherDatabase from 'better-sqlite3-multiple-ciphers';
|
|
32
|
+
const PLAINTEXT_MAGIC = 'SQLite format 3\0';
|
|
33
|
+
/** Exported for key-aware CLI opens (open-runtime-db.ts): a plaintext file
|
|
34
|
+
* must be opened WITHOUT a key — PRAGMA key on plaintext reads as garbage. */
|
|
35
|
+
export function isPlaintextSqlite(dbPath) {
|
|
36
|
+
const header = readFileSync(dbPath).subarray(0, 16).toString('latin1');
|
|
37
|
+
return header === PLAINTEXT_MAGIC;
|
|
38
|
+
}
|
|
39
|
+
function escapeSqlString(value) {
|
|
40
|
+
return value.replace(/'/g, "''");
|
|
41
|
+
}
|
|
42
|
+
function tableCounts(db) {
|
|
43
|
+
const memories = db.prepare('SELECT COUNT(*) AS n FROM memories').get().n;
|
|
44
|
+
const transcripts = db.prepare('SELECT COUNT(*) AS n FROM transcripts').get().n;
|
|
45
|
+
return { memories, transcripts };
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* Detects a plaintext DB at `dbPath` and migrates it to encrypted form with
|
|
49
|
+
* `key`. Returns:
|
|
50
|
+
* - 'missing' — no file at dbPath (fresh install; openDb() with
|
|
51
|
+
* a key will create an encrypted file directly).
|
|
52
|
+
* - 'already-encrypted' — file exists and is not plaintext (already
|
|
53
|
+
* migrated, or created encrypted from the start).
|
|
54
|
+
* - 'migrated' — a plaintext DB was found and successfully
|
|
55
|
+
* converted; `<dbPath>.pre-encryption.bak` now
|
|
56
|
+
* holds the pre-migration plaintext copy.
|
|
57
|
+
*/
|
|
58
|
+
export function encryptIfPlaintext(dbPath, key) {
|
|
59
|
+
if (!existsSync(dbPath))
|
|
60
|
+
return 'missing';
|
|
61
|
+
if (!isPlaintextSqlite(dbPath))
|
|
62
|
+
return 'already-encrypted';
|
|
63
|
+
const tmpPath = `${dbPath}.encrypting-tmp`;
|
|
64
|
+
if (existsSync(tmpPath)) {
|
|
65
|
+
// Leftover from a crashed/interrupted prior attempt — start clean.
|
|
66
|
+
rmSync(tmpPath, { force: true });
|
|
67
|
+
}
|
|
68
|
+
// 1. Checkpoint + count on the ORIGINAL, then close it before copying.
|
|
69
|
+
const plain = new CipherDatabase(dbPath);
|
|
70
|
+
let beforeCounts;
|
|
71
|
+
try {
|
|
72
|
+
try {
|
|
73
|
+
plain.pragma('wal_checkpoint(TRUNCATE)');
|
|
74
|
+
}
|
|
75
|
+
catch {
|
|
76
|
+
// Non-fatal: DB may not be in WAL mode yet (e.g. first-ever open).
|
|
77
|
+
}
|
|
78
|
+
beforeCounts = tableCounts(plain);
|
|
79
|
+
}
|
|
80
|
+
finally {
|
|
81
|
+
plain.close();
|
|
82
|
+
}
|
|
83
|
+
// 2. Encrypt a COPY in place via PRAGMA rekey (plaintext -> keyed).
|
|
84
|
+
copyFileSync(dbPath, tmpPath);
|
|
85
|
+
const rekeying = new CipherDatabase(tmpPath);
|
|
86
|
+
try {
|
|
87
|
+
rekeying.pragma(`rekey='${escapeSqlString(key)}'`);
|
|
88
|
+
}
|
|
89
|
+
finally {
|
|
90
|
+
rekeying.close();
|
|
91
|
+
}
|
|
92
|
+
// 3. Verify the copy is genuinely encrypted and complete before touching
|
|
93
|
+
// the original file.
|
|
94
|
+
if (isPlaintextSqlite(tmpPath)) {
|
|
95
|
+
rmSync(tmpPath, { force: true });
|
|
96
|
+
throw new Error(`encryptIfPlaintext: PRAGMA rekey left ${tmpPath} with a plaintext header; ` +
|
|
97
|
+
`aborting — plaintext DB at ${dbPath} was left untouched`);
|
|
98
|
+
}
|
|
99
|
+
const verify = new CipherDatabase(tmpPath);
|
|
100
|
+
let afterCounts;
|
|
101
|
+
try {
|
|
102
|
+
verify.pragma(`key='${escapeSqlString(key)}'`);
|
|
103
|
+
afterCounts = tableCounts(verify);
|
|
104
|
+
}
|
|
105
|
+
finally {
|
|
106
|
+
verify.close();
|
|
107
|
+
}
|
|
108
|
+
if (afterCounts.memories !== beforeCounts.memories || afterCounts.transcripts !== beforeCounts.transcripts) {
|
|
109
|
+
rmSync(tmpPath, { force: true });
|
|
110
|
+
throw new Error(`encryptIfPlaintext: row count mismatch after rekey ` +
|
|
111
|
+
`(memories ${beforeCounts.memories} -> ${afterCounts.memories}, ` +
|
|
112
|
+
`transcripts ${beforeCounts.transcripts} -> ${afterCounts.transcripts}); ` +
|
|
113
|
+
`aborting — plaintext DB at ${dbPath} was left untouched`);
|
|
114
|
+
}
|
|
115
|
+
// Preserve the plaintext original (SEC-7), then atomically swap it out.
|
|
116
|
+
const bakPath = `${dbPath}.pre-encryption.bak`;
|
|
117
|
+
copyFileSync(dbPath, bakPath);
|
|
118
|
+
renameSync(tmpPath, dbPath);
|
|
119
|
+
return 'migrated';
|
|
120
|
+
}
|
|
121
|
+
//# sourceMappingURL=migrate-encrypt.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"migrate-encrypt.js","sourceRoot":"","sources":["../../src/storage/migrate-encrypt.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACrF,OAAO,cAAc,MAAM,iCAAiC,CAAC;AAG7D,MAAM,eAAe,GAAG,mBAAmB,CAAC;AAI5C;+EAC+E;AAC/E,MAAM,UAAU,iBAAiB,CAAC,MAAc;IAC9C,MAAM,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACvE,OAAO,MAAM,KAAK,eAAe,CAAC;AACpC,CAAC;AAED,SAAS,eAAe,CAAC,KAAa;IACpC,OAAO,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;AACnC,CAAC;AAED,SAAS,WAAW,CAAC,EAAM;IACzB,MAAM,QAAQ,GAAI,EAAE,CAAC,OAAO,CAAC,oCAAoC,CAAC,CAAC,GAAG,EAAoB,CAAC,CAAC,CAAC;IAC7F,MAAM,WAAW,GAAI,EAAE,CAAC,OAAO,CAAC,uCAAuC,CAAC,CAAC,GAAG,EAAoB,CAAC,CAAC,CAAC;IACnG,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,CAAC;AACnC,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,UAAU,kBAAkB,CAAC,MAAc,EAAE,GAAW;IAC5D,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC;QAAE,OAAO,SAAS,CAAC;IAC1C,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC;QAAE,OAAO,mBAAmB,CAAC;IAE3D,MAAM,OAAO,GAAG,GAAG,MAAM,iBAAiB,CAAC;IAC3C,IAAI,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;QACxB,mEAAmE;QACnE,MAAM,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IACnC,CAAC;IAED,uEAAuE;IACvE,MAAM,KAAK,GAAG,IAAI,cAAc,CAAC,MAAM,CAAC,CAAC;IACzC,IAAI,YAAuD,CAAC;IAC5D,IAAI,CAAC;QACH,IAAI,CAAC;YACH,KAAK,CAAC,MAAM,CAAC,0BAA0B,CAAC,CAAC;QAC3C,CAAC;QAAC,MAAM,CAAC;YACP,mEAAmE;QACrE,CAAC;QACD,YAAY,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;IACpC,CAAC;YAAS,CAAC;QACT,KAAK,CAAC,KAAK,EAAE,CAAC;IAChB,CAAC;IAED,oEAAoE;IACpE,YAAY,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC9B,MAAM,QAAQ,GAAG,IAAI,cAAc,CAAC,OAAO,CAAC,CAAC;IAC7C,IAAI,CAAC;QACH,QAAQ,CAAC,MAAM,CAAC,UAAU,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACrD,CAAC;YAAS,CAAC;QACT,QAAQ,CAAC,KAAK,EAAE,CAAC;IACnB,CAAC;IAED,yEAAyE;IACzE,wBAAwB;IACxB,IAAI,iBAAiB,CAAC,OAAO,CAAC,EAAE,CAAC;QAC/B,MAAM,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACjC,MAAM,IAAI,KAAK,CACb,yCAAyC,OAAO,4BAA4B;YAC5E,8BAA8B,MAAM,qBAAqB,CAC1D,CAAC;IACJ,CAAC;IACD,MAAM,MAAM,GAAG,IAAI,cAAc,CAAC,OAAO,CAAC,CAAC;IAC3C,IAAI,WAAsD,CAAC;IAC3D,IAAI,CAAC;QACH,MAAM,CAAC,MAAM,CAAC,QAAQ,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC/C,WAAW,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC;IACpC,CAAC;YAAS,CAAC;QACT,MAAM,CAAC,KAAK,EAAE,CAAC;IACjB,CAAC;IAED,IAAI,WAAW,CAAC,QAAQ,KAAK,YAAY,CAAC,QAAQ,IAAI,WAAW,CAAC,WAAW,KAAK,YAAY,CAAC,WAAW,EAAE,CAAC;QAC3G,MAAM,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACjC,MAAM,IAAI,KAAK,CACb,qDAAqD;YACrD,aAAa,YAAY,CAAC,QAAQ,OAAO,WAAW,CAAC,QAAQ,IAAI;YACjE,eAAe,YAAY,CAAC,WAAW,OAAO,WAAW,CAAC,WAAW,KAAK;YAC1E,8BAA8B,MAAM,qBAAqB,CAC1D,CAAC;IACJ,CAAC;IAED,wEAAwE;IACxE,MAAM,OAAO,GAAG,GAAG,MAAM,qBAAqB,CAAC;IAC/C,YAAY,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC9B,UAAU,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IAE5B,OAAO,UAAU,CAAC;AACpB,CAAC"}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import type { DB } from './db.js';
|
|
2
|
+
/**
|
|
3
|
+
* Run pending migrations. `dim` (default DEFAULT_EMBED_DIM = 1024) is
|
|
4
|
+
* baked into any NOT-yet-applied migration file containing the
|
|
5
|
+
* __EMBED_DIM__ placeholder (currently just 001-init's `memories_vec
|
|
6
|
+
* FLOAT[__EMBED_DIM__]`) — i.e. it only matters for a genuinely fresh DB.
|
|
7
|
+
* Already-applied migrations are skipped entirely (schema_version tracks
|
|
8
|
+
* them), so passing a different `dim` against an already-migrated DB is a
|
|
9
|
+
* harmless no-op — the historical FLOAT[N] baked in at creation time is
|
|
10
|
+
* immutable short of the resumable re-embed procedure (migration 011,
|
|
11
|
+
* `astramem-local reembed-dim`, docs/embed-dim-migration.md).
|
|
12
|
+
*/
|
|
13
|
+
export declare function migrate(db: DB, dim?: number): void;
|
|
14
|
+
/**
|
|
15
|
+
* Read the dim actually baked into `memories_vec` at CREATE TABLE time
|
|
16
|
+
* (FEAT-409 AC-3) by parsing its persisted DDL out of sqlite_master —
|
|
17
|
+
* the source of truth for what the schema really is, independent of
|
|
18
|
+
* whatever config.embedding.dim currently says. Boot preflight compares
|
|
19
|
+
* the PROVIDER's dim against this (not just provider self-consistency),
|
|
20
|
+
* so a config/provider dim that drifted from the schema (e.g. someone
|
|
21
|
+
* edited config.yaml without running `reembed-dim`) fails loud at boot
|
|
22
|
+
* instead of surfacing as a raw sqlite-vec dimension error deep in the
|
|
23
|
+
* first ingest. Returns null when the table doesn't exist yet (fresh DB
|
|
24
|
+
* about to be created by migrate()) or its DDL doesn't match the
|
|
25
|
+
* expected `FLOAT[N]` shape.
|
|
26
|
+
*/
|
|
27
|
+
export declare function getVecSchemaDim(db: DB): number | null;
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
import { readFileSync, readdirSync } from 'node:fs';
|
|
2
|
+
import { join, dirname } from 'node:path';
|
|
3
|
+
import { fileURLToPath } from 'node:url';
|
|
4
|
+
import { SCHEMA_VERSION } from '../server/lib/wire-meta.js';
|
|
5
|
+
import { DEFAULT_EMBED_DIM } from '../contracts/embed.js';
|
|
6
|
+
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
7
|
+
const MIGRATIONS_DIR = join(__dirname, '..', '..', 'migrations');
|
|
8
|
+
/** Placeholder substituted into migration SQL for the vec schema dim
|
|
9
|
+
* (FEAT-409). Only migrations/001-init.sql uses it today; substitution is
|
|
10
|
+
* a no-op for any migration file that doesn't contain the token. Applying
|
|
11
|
+
* it unconditionally to every not-yet-applied file (rather than special-
|
|
12
|
+
* casing 001) keeps future dim-sensitive migrations trivial to add. */
|
|
13
|
+
const EMBED_DIM_PLACEHOLDER = '__EMBED_DIM__';
|
|
14
|
+
/**
|
|
15
|
+
* Run pending migrations. `dim` (default DEFAULT_EMBED_DIM = 1024) is
|
|
16
|
+
* baked into any NOT-yet-applied migration file containing the
|
|
17
|
+
* __EMBED_DIM__ placeholder (currently just 001-init's `memories_vec
|
|
18
|
+
* FLOAT[__EMBED_DIM__]`) — i.e. it only matters for a genuinely fresh DB.
|
|
19
|
+
* Already-applied migrations are skipped entirely (schema_version tracks
|
|
20
|
+
* them), so passing a different `dim` against an already-migrated DB is a
|
|
21
|
+
* harmless no-op — the historical FLOAT[N] baked in at creation time is
|
|
22
|
+
* immutable short of the resumable re-embed procedure (migration 011,
|
|
23
|
+
* `astramem-local reembed-dim`, docs/embed-dim-migration.md).
|
|
24
|
+
*/
|
|
25
|
+
export function migrate(db, dim = DEFAULT_EMBED_DIM) {
|
|
26
|
+
if (!Number.isInteger(dim) || dim <= 0) {
|
|
27
|
+
throw new Error(`invalid embedding dim: ${dim} (must be a positive integer)`);
|
|
28
|
+
}
|
|
29
|
+
db.exec(`
|
|
30
|
+
CREATE TABLE IF NOT EXISTS schema_version (
|
|
31
|
+
version INTEGER PRIMARY KEY,
|
|
32
|
+
applied_at INTEGER NOT NULL
|
|
33
|
+
)
|
|
34
|
+
`);
|
|
35
|
+
const applied = new Set(db.prepare('SELECT version FROM schema_version').all()
|
|
36
|
+
.map(r => r.version));
|
|
37
|
+
const files = readdirSync(MIGRATIONS_DIR)
|
|
38
|
+
.filter(f => f.endsWith('.sql'))
|
|
39
|
+
.sort();
|
|
40
|
+
for (const f of files) {
|
|
41
|
+
const versionStr = f.split('-')[0];
|
|
42
|
+
const version = parseInt(versionStr ?? '', 10);
|
|
43
|
+
if (Number.isNaN(version))
|
|
44
|
+
throw new Error(`bad migration name: ${f}`);
|
|
45
|
+
if (applied.has(version))
|
|
46
|
+
continue;
|
|
47
|
+
const rawSql = readFileSync(join(MIGRATIONS_DIR, f), 'utf8');
|
|
48
|
+
const sql = rawSql.split(EMBED_DIM_PLACEHOLDER).join(String(dim));
|
|
49
|
+
const tx = db.transaction(() => {
|
|
50
|
+
db.exec(sql);
|
|
51
|
+
db.prepare('INSERT INTO schema_version (version, applied_at) VALUES (?, ?)').run(version, Date.now());
|
|
52
|
+
});
|
|
53
|
+
try {
|
|
54
|
+
tx();
|
|
55
|
+
}
|
|
56
|
+
catch (err) {
|
|
57
|
+
// FEAT-420 AC-3: a half-migrated datadir (e.g. the schema_version row
|
|
58
|
+
// for the last-applied migration was lost while the underlying schema
|
|
59
|
+
// change already landed) makes migrate() try to re-apply SQL against
|
|
60
|
+
// an already-migrated table — SQLite throws a low-level error (e.g.
|
|
61
|
+
// "duplicate column name") that IS fail-loud but not actionable on its
|
|
62
|
+
// own. Wrap it so the operator gets a diagnosis, not just a raw DDL
|
|
63
|
+
// error. No change to migration order/content/skip logic above.
|
|
64
|
+
throw new Error(`Migration ${f} (version ${version}) failed to apply — the database schema may already ` +
|
|
65
|
+
`partially reflect this migration while schema_version does not record it (a half-migrated ` +
|
|
66
|
+
`or corrupted datadir). Restore from a backup rather than continuing. Original error: ` +
|
|
67
|
+
`${err instanceof Error ? err.message : String(err)}`);
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
// Boot-time schema-version drift guard: the SCHEMA_VERSION constant in
|
|
71
|
+
// wire-meta.ts MUST match the highest migration applied to the DB.
|
|
72
|
+
// If they diverge a migration was added without bumping the constant (or
|
|
73
|
+
// vice-versa). Fail loudly rather than silently serve wrong metadata.
|
|
74
|
+
const dbMaxRow = db
|
|
75
|
+
.prepare('SELECT MAX(version) AS max_version FROM schema_version')
|
|
76
|
+
.get();
|
|
77
|
+
const dbMax = dbMaxRow.max_version ?? 0;
|
|
78
|
+
if (dbMax !== SCHEMA_VERSION) {
|
|
79
|
+
throw new Error(`Schema-version constant drift: wire-meta says ${SCHEMA_VERSION}, DB says ${dbMax}. ` +
|
|
80
|
+
`Bump wire-meta.SCHEMA_VERSION when adding a migration.`);
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
/**
|
|
84
|
+
* Read the dim actually baked into `memories_vec` at CREATE TABLE time
|
|
85
|
+
* (FEAT-409 AC-3) by parsing its persisted DDL out of sqlite_master —
|
|
86
|
+
* the source of truth for what the schema really is, independent of
|
|
87
|
+
* whatever config.embedding.dim currently says. Boot preflight compares
|
|
88
|
+
* the PROVIDER's dim against this (not just provider self-consistency),
|
|
89
|
+
* so a config/provider dim that drifted from the schema (e.g. someone
|
|
90
|
+
* edited config.yaml without running `reembed-dim`) fails loud at boot
|
|
91
|
+
* instead of surfacing as a raw sqlite-vec dimension error deep in the
|
|
92
|
+
* first ingest. Returns null when the table doesn't exist yet (fresh DB
|
|
93
|
+
* about to be created by migrate()) or its DDL doesn't match the
|
|
94
|
+
* expected `FLOAT[N]` shape.
|
|
95
|
+
*/
|
|
96
|
+
export function getVecSchemaDim(db) {
|
|
97
|
+
const row = db
|
|
98
|
+
.prepare(`SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'memories_vec'`)
|
|
99
|
+
.get();
|
|
100
|
+
if (!row?.sql)
|
|
101
|
+
return null;
|
|
102
|
+
const match = row.sql.match(/FLOAT\[(\d+)\]/);
|
|
103
|
+
return match ? Number(match[1]) : null;
|
|
104
|
+
}
|
|
105
|
+
//# sourceMappingURL=migrate.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"migrate.js","sourceRoot":"","sources":["../../src/storage/migrate.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AACpD,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAEzC,OAAO,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AAC5D,OAAO,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAC;AAE1D,MAAM,SAAS,GAAG,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AAC1D,MAAM,cAAc,GAAG,IAAI,CAAC,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,YAAY,CAAC,CAAC;AAEjE;;;;uEAIuE;AACvE,MAAM,qBAAqB,GAAG,eAAe,CAAC;AAE9C;;;;;;;;;;GAUG;AACH,MAAM,UAAU,OAAO,CAAC,EAAM,EAAE,MAAc,iBAAiB;IAC7D,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC;QACvC,MAAM,IAAI,KAAK,CAAC,0BAA0B,GAAG,+BAA+B,CAAC,CAAC;IAChF,CAAC;IAED,EAAE,CAAC,IAAI,CAAC;;;;;GAKP,CAAC,CAAC;IAEH,MAAM,OAAO,GAAG,IAAI,GAAG,CACpB,EAAE,CAAC,OAAO,CAAC,oCAAoC,CAAC,CAAC,GAAG,EAA0B;SAC5E,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CACvB,CAAC;IAEF,MAAM,KAAK,GAAG,WAAW,CAAC,cAAc,CAAC;SACtC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;SAC/B,IAAI,EAAE,CAAC;IAEV,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,MAAM,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACnC,MAAM,OAAO,GAAG,QAAQ,CAAC,UAAU,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;QAC/C,IAAI,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,uBAAuB,CAAC,EAAE,CAAC,CAAC;QACvE,IAAI,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC;YAAE,SAAS;QACnC,MAAM,MAAM,GAAG,YAAY,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;QAC7D,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;QAClE,MAAM,EAAE,GAAG,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE;YAC7B,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACb,EAAE,CAAC,OAAO,CAAC,gEAAgE,CAAC,CAAC,GAAG,CAAC,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;QACxG,CAAC,CAAC,CAAC;QACH,IAAI,CAAC;YACH,EAAE,EAAE,CAAC;QACP,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,sEAAsE;YACtE,sEAAsE;YACtE,qEAAqE;YACrE,oEAAoE;YACpE,uEAAuE;YACvE,oEAAoE;YACpE,gEAAgE;YAChE,MAAM,IAAI,KAAK,CACb,aAAa,CAAC,aAAa,OAAO,sDAAsD;gBACxF,4FAA4F;gBAC5F,uFAAuF;gBACvF,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CACtD,CAAC;QACJ,CAAC;IACH,CAAC;IAED,uEAAuE;IACvE,mEAAmE;IACnE,yEAAyE;IACzE,sEAAsE;IACtE,MAAM,QAAQ,GAAG,EAAE;SAChB,OAAO,CAAC,wDAAwD,CAAC;SACjE,GAAG,EAAoC,CAAC;IAC3C,MAAM,KAAK,GAAG,QAAQ,CAAC,WAAW,IAAI,CAAC,CAAC;IACxC,IAAI,KAAK,KAAK,cAAc,EAAE,CAAC;QAC7B,MAAM,IAAI,KAAK,CACb,iDAAiD,cAAc,aAAa,KAAK,IAAI;YACrF,wDAAwD,CACzD,CAAC;IACJ,CAAC;AACH,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,eAAe,CAAC,EAAM;IACpC,MAAM,GAAG,GAAG,EAAE;SACX,OAAO,CAAC,8EAA8E,CAAC;SACvF,GAAG,EAAiC,CAAC;IACxC,IAAI,CAAC,GAAG,EAAE,GAAG;QAAE,OAAO,IAAI,CAAC;IAC3B,MAAM,KAAK,GAAG,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC;IAC9C,OAAO,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AACzC,CAAC"}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Persistence for stage-0 redaction counters (SEC-6). Counts + types only —
|
|
3
|
+
* never the raw secret value. One row per (type, ingest) — see
|
|
4
|
+
* migrations/005-security.sql for the `redaction_log` table.
|
|
5
|
+
*/
|
|
6
|
+
import type { DB } from './db.js';
|
|
7
|
+
import type { RedactionEvent } from '../redact/index.js';
|
|
8
|
+
export interface RedactionCountRow {
|
|
9
|
+
type: string;
|
|
10
|
+
count: number;
|
|
11
|
+
}
|
|
12
|
+
/** Collapse a flat event list into one row per type (count = occurrences). */
|
|
13
|
+
export declare function aggregateRedactionEvents(events: RedactionEvent[]): RedactionCountRow[];
|
|
14
|
+
/**
|
|
15
|
+
* Write one `redaction_log` row per distinct type found in `events`. No-op
|
|
16
|
+
* when `events` is empty (nothing to record, no wasted write).
|
|
17
|
+
*/
|
|
18
|
+
export declare function recordRedactionEvents(db: DB, events: RedactionEvent[], sessionId: string | null): void;
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Persistence for stage-0 redaction counters (SEC-6). Counts + types only —
|
|
3
|
+
* never the raw secret value. One row per (type, ingest) — see
|
|
4
|
+
* migrations/005-security.sql for the `redaction_log` table.
|
|
5
|
+
*/
|
|
6
|
+
/** Collapse a flat event list into one row per type (count = occurrences). */
|
|
7
|
+
export function aggregateRedactionEvents(events) {
|
|
8
|
+
const counts = new Map();
|
|
9
|
+
for (const e of events) {
|
|
10
|
+
counts.set(e.type, (counts.get(e.type) ?? 0) + 1);
|
|
11
|
+
}
|
|
12
|
+
return [...counts.entries()].map(([type, count]) => ({ type, count }));
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* Write one `redaction_log` row per distinct type found in `events`. No-op
|
|
16
|
+
* when `events` is empty (nothing to record, no wasted write).
|
|
17
|
+
*/
|
|
18
|
+
export function recordRedactionEvents(db, events, sessionId) {
|
|
19
|
+
if (events.length === 0)
|
|
20
|
+
return;
|
|
21
|
+
const now = Date.now();
|
|
22
|
+
const stmt = db.prepare('INSERT INTO redaction_log (type, count, session_id, created_at) VALUES (?, ?, ?, ?)');
|
|
23
|
+
for (const row of aggregateRedactionEvents(events)) {
|
|
24
|
+
stmt.run(row.type, row.count, sessionId, now);
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
//# sourceMappingURL=redaction-log.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"redaction-log.js","sourceRoot":"","sources":["../../src/storage/redaction-log.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAUH,8EAA8E;AAC9E,MAAM,UAAU,wBAAwB,CAAC,MAAwB;IAC/D,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;IACzC,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;QACvB,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACpD,CAAC;IACD,OAAO,CAAC,GAAG,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC;AACzE,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,qBAAqB,CAAC,EAAM,EAAE,MAAwB,EAAE,SAAwB;IAC9F,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO;IAChC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,MAAM,IAAI,GAAG,EAAE,CAAC,OAAO,CACrB,qFAAqF,CACtF,CAAC;IACF,KAAK,MAAM,GAAG,IAAI,wBAAwB,CAAC,MAAM,CAAC,EAAE,CAAC;QACnD,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,KAAK,EAAE,SAAS,EAAE,GAAG,CAAC,CAAC;IAChD,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Recall-usefulness event capture + metric (ADR-010, v1: measure only — the
|
|
3
|
+
* events captured here do NOT feed ranking yet, that's v1.x).
|
|
4
|
+
*
|
|
5
|
+
* Rides the existing memory_events log ('usefulness' event_type, already
|
|
6
|
+
* present in the migration 007 CHECK constraint) — no new migration. Three
|
|
7
|
+
* payload families, distinguished by payload.family:
|
|
8
|
+
*
|
|
9
|
+
* - recall_served — one event per atom returned from a search/recall
|
|
10
|
+
* surface (REST /search, /recall, /recall/pack; MCP
|
|
11
|
+
* search_memory/recall_memory).
|
|
12
|
+
* - recall_used — explicit or heuristic signal that a served atom
|
|
13
|
+
* mattered. v1 ships the explicit channel only
|
|
14
|
+
* (mark_memory_used MCP tool / POST /memory/:id/used).
|
|
15
|
+
* - memory_corrected — negative signal: the atom was invalidated,
|
|
16
|
+
* superseded, demoted, or edited. Appended directly
|
|
17
|
+
* inside MemoryEventRepo.invalidate/supersede's own
|
|
18
|
+
* transaction (src/storage/memory-events.ts) — this
|
|
19
|
+
* module documents the payload shape those call
|
|
20
|
+
* sites use, but does not append on their behalf, to
|
|
21
|
+
* avoid a circular import between the two modules.
|
|
22
|
+
*
|
|
23
|
+
* Privacy: the raw query text is NEVER stored — only a truncated sha256 hex
|
|
24
|
+
* digest (hashQuery). Usefulness events inherit the atom's scope (ADR-009)
|
|
25
|
+
* like every other memory_events row.
|
|
26
|
+
*/
|
|
27
|
+
import type { DB } from './db.js';
|
|
28
|
+
export type UsefulnessSurface = 'mcp' | 'rest' | 'cli';
|
|
29
|
+
export type UsefulnessSignal = 'explicit' | 'heuristic';
|
|
30
|
+
export type CorrectionAction = 'invalidated' | 'superseded' | 'demoted' | 'edited';
|
|
31
|
+
/** sha256 hex digest of the raw query, truncated to 16 chars — never store the query text itself. */
|
|
32
|
+
export declare function hashQuery(query: string): string;
|
|
33
|
+
export interface RecordRecallServedInput {
|
|
34
|
+
/** Precomputed query hash. Provide this OR `query` (which gets hashed here). */
|
|
35
|
+
queryHash?: string;
|
|
36
|
+
/** Raw query/recall-input text — hashed via hashQuery, never persisted as-is. */
|
|
37
|
+
query?: string;
|
|
38
|
+
atomIds: string[];
|
|
39
|
+
/** Optional per-atom scores, aligned by index with atomIds. */
|
|
40
|
+
scores?: number[];
|
|
41
|
+
surface: UsefulnessSurface;
|
|
42
|
+
/** Free-form label for the recall mode (e.g. 'search', 'recall', 'pack'). */
|
|
43
|
+
mode?: string;
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* Append one 'usefulness'/'recall_served' event per served atom, in a single
|
|
47
|
+
* transaction. Cheap by design: one prepared-statement loop, no per-atom
|
|
48
|
+
* round trip beyond the INSERT itself. No-ops on an empty atomIds list.
|
|
49
|
+
*/
|
|
50
|
+
export declare function recordRecallServed(db: DB, input: RecordRecallServedInput): void;
|
|
51
|
+
export interface RecordRecallUsedInput {
|
|
52
|
+
atomId: string;
|
|
53
|
+
surface: UsefulnessSurface;
|
|
54
|
+
signal: UsefulnessSignal;
|
|
55
|
+
}
|
|
56
|
+
/** Append one 'usefulness'/'recall_used' event — the atom mattered. */
|
|
57
|
+
export declare function recordRecallUsed(db: DB, input: RecordRecallUsedInput): void;
|
|
58
|
+
export interface RecordMemoryCorrectedInput {
|
|
59
|
+
atomId: string;
|
|
60
|
+
action: CorrectionAction;
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* Append one 'usefulness'/'memory_corrected' event — a negative signal.
|
|
64
|
+
* Exposed for direct callers (e.g. future demote/edit flows); the
|
|
65
|
+
* invalidate/supersede lifecycle ops append this shape inline instead of
|
|
66
|
+
* calling this function, to keep memory-events.ts free of a dependency on
|
|
67
|
+
* this module (see file header).
|
|
68
|
+
*/
|
|
69
|
+
export declare function recordMemoryCorrected(db: DB, input: RecordMemoryCorrectedInput): void;
|
|
70
|
+
/**
|
|
71
|
+
* Per-atom usefulness score for ranking (ADR-010 v1.x). Laplace-smoothed:
|
|
72
|
+
* (used + 1) / (used + corrected + 2) — an atom with no signal scores a
|
|
73
|
+
* neutral 0.5, one explicit recall_used lifts it to 0.67, one correction
|
|
74
|
+
* drops it to 0.33. Every requested id gets an entry so callers never have
|
|
75
|
+
* to special-case missing atoms.
|
|
76
|
+
*/
|
|
77
|
+
export declare function usefulnessScores(db: DB, atomIds: string[]): Map<string, number>;
|
|
78
|
+
export interface UsefulnessRateOpts {
|
|
79
|
+
/** Epoch-ms lower bound on created_at. Defaults to 0 (no lower bound). */
|
|
80
|
+
sinceMs?: number;
|
|
81
|
+
surface?: UsefulnessSurface;
|
|
82
|
+
}
|
|
83
|
+
export interface UsefulnessRate {
|
|
84
|
+
served: number;
|
|
85
|
+
used: number;
|
|
86
|
+
/** used / served; null when served === 0 (avoids divide-by-zero / misleading 0%). */
|
|
87
|
+
rate: number | null;
|
|
88
|
+
}
|
|
89
|
+
/**
|
|
90
|
+
* recall-usefulness rate = distinct atoms used / distinct atoms served, in
|
|
91
|
+
* the given window (ADR-010's seed metric). Distinct-atom counting avoids
|
|
92
|
+
* double-counting an atom served or used more than once in the window.
|
|
93
|
+
*/
|
|
94
|
+
export declare function usefulnessRate(db: DB, opts?: UsefulnessRateOpts): UsefulnessRate;
|
|
95
|
+
/**
|
|
96
|
+
* Per-atom usefulness rate over a trailing window, for archival eligibility
|
|
97
|
+
* (FEAT-404 AC-2/AC-3 — "rate_7d"). Deliberately distinct from
|
|
98
|
+
* usefulnessRate() above: that DB-wide health-dashboard metric returns
|
|
99
|
+
* `rate: null` when served === 0 to avoid a misleading 0% on the /health
|
|
100
|
+
* surface. Archival eligibility instead needs one comparable number per
|
|
101
|
+
* atom, and "zero recall_served events in the window" is itself evidence
|
|
102
|
+
* the atom isn't earning its keep — so it maps to rate 0 here, not null,
|
|
103
|
+
* making an unsignaled atom exactly as archival-eligible as one with 0%
|
|
104
|
+
* explicit usage. Every requested id gets an entry (default 0) so callers
|
|
105
|
+
* never special-case a missing atom.
|
|
106
|
+
*/
|
|
107
|
+
export declare function perAtomUsefulnessRate7d(db: DB, atomIds: string[], sinceMs: number): Map<string, number>;
|
|
108
|
+
export interface UsefulnessByType {
|
|
109
|
+
type: string;
|
|
110
|
+
served: number;
|
|
111
|
+
used: number;
|
|
112
|
+
rate: number | null;
|
|
113
|
+
}
|
|
114
|
+
/** Same metric, grouped by the served/used atom's current memories.type. */
|
|
115
|
+
export declare function usefulnessByType(db: DB, opts?: UsefulnessRateOpts): UsefulnessByType[];
|
|
@@ -0,0 +1,203 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Recall-usefulness event capture + metric (ADR-010, v1: measure only — the
|
|
3
|
+
* events captured here do NOT feed ranking yet, that's v1.x).
|
|
4
|
+
*
|
|
5
|
+
* Rides the existing memory_events log ('usefulness' event_type, already
|
|
6
|
+
* present in the migration 007 CHECK constraint) — no new migration. Three
|
|
7
|
+
* payload families, distinguished by payload.family:
|
|
8
|
+
*
|
|
9
|
+
* - recall_served — one event per atom returned from a search/recall
|
|
10
|
+
* surface (REST /search, /recall, /recall/pack; MCP
|
|
11
|
+
* search_memory/recall_memory).
|
|
12
|
+
* - recall_used — explicit or heuristic signal that a served atom
|
|
13
|
+
* mattered. v1 ships the explicit channel only
|
|
14
|
+
* (mark_memory_used MCP tool / POST /memory/:id/used).
|
|
15
|
+
* - memory_corrected — negative signal: the atom was invalidated,
|
|
16
|
+
* superseded, demoted, or edited. Appended directly
|
|
17
|
+
* inside MemoryEventRepo.invalidate/supersede's own
|
|
18
|
+
* transaction (src/storage/memory-events.ts) — this
|
|
19
|
+
* module documents the payload shape those call
|
|
20
|
+
* sites use, but does not append on their behalf, to
|
|
21
|
+
* avoid a circular import between the two modules.
|
|
22
|
+
*
|
|
23
|
+
* Privacy: the raw query text is NEVER stored — only a truncated sha256 hex
|
|
24
|
+
* digest (hashQuery). Usefulness events inherit the atom's scope (ADR-009)
|
|
25
|
+
* like every other memory_events row.
|
|
26
|
+
*/
|
|
27
|
+
import { createHash } from 'node:crypto';
|
|
28
|
+
import { MemoryEventRepo } from './memory-events.js';
|
|
29
|
+
/** sha256 hex digest of the raw query, truncated to 16 chars — never store the query text itself. */
|
|
30
|
+
export function hashQuery(query) {
|
|
31
|
+
return createHash('sha256').update(query).digest('hex').slice(0, 16);
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* Append one 'usefulness'/'recall_served' event per served atom, in a single
|
|
35
|
+
* transaction. Cheap by design: one prepared-statement loop, no per-atom
|
|
36
|
+
* round trip beyond the INSERT itself. No-ops on an empty atomIds list.
|
|
37
|
+
*/
|
|
38
|
+
export function recordRecallServed(db, input) {
|
|
39
|
+
if (input.atomIds.length === 0)
|
|
40
|
+
return;
|
|
41
|
+
const queryHash = input.queryHash ?? (input.query !== undefined ? hashQuery(input.query) : undefined);
|
|
42
|
+
if (!queryHash)
|
|
43
|
+
throw new Error('recordRecallServed requires queryHash or query');
|
|
44
|
+
const events = new MemoryEventRepo(db);
|
|
45
|
+
const tx = db.transaction(() => {
|
|
46
|
+
input.atomIds.forEach((atomId, i) => {
|
|
47
|
+
events.append({
|
|
48
|
+
event_type: 'usefulness',
|
|
49
|
+
atom_id: atomId,
|
|
50
|
+
payload: {
|
|
51
|
+
family: 'recall_served',
|
|
52
|
+
query_hash: queryHash,
|
|
53
|
+
score: input.scores?.[i] ?? null,
|
|
54
|
+
surface: input.surface,
|
|
55
|
+
mode: input.mode ?? null,
|
|
56
|
+
},
|
|
57
|
+
});
|
|
58
|
+
});
|
|
59
|
+
});
|
|
60
|
+
tx();
|
|
61
|
+
}
|
|
62
|
+
/** Append one 'usefulness'/'recall_used' event — the atom mattered. */
|
|
63
|
+
export function recordRecallUsed(db, input) {
|
|
64
|
+
new MemoryEventRepo(db).append({
|
|
65
|
+
event_type: 'usefulness',
|
|
66
|
+
atom_id: input.atomId,
|
|
67
|
+
payload: { family: 'recall_used', surface: input.surface, signal: input.signal },
|
|
68
|
+
});
|
|
69
|
+
}
|
|
70
|
+
/**
|
|
71
|
+
* Append one 'usefulness'/'memory_corrected' event — a negative signal.
|
|
72
|
+
* Exposed for direct callers (e.g. future demote/edit flows); the
|
|
73
|
+
* invalidate/supersede lifecycle ops append this shape inline instead of
|
|
74
|
+
* calling this function, to keep memory-events.ts free of a dependency on
|
|
75
|
+
* this module (see file header).
|
|
76
|
+
*/
|
|
77
|
+
export function recordMemoryCorrected(db, input) {
|
|
78
|
+
new MemoryEventRepo(db).append({
|
|
79
|
+
event_type: 'usefulness',
|
|
80
|
+
atom_id: input.atomId,
|
|
81
|
+
payload: { family: 'memory_corrected', action: input.action },
|
|
82
|
+
});
|
|
83
|
+
}
|
|
84
|
+
/**
|
|
85
|
+
* Per-atom usefulness score for ranking (ADR-010 v1.x). Laplace-smoothed:
|
|
86
|
+
* (used + 1) / (used + corrected + 2) — an atom with no signal scores a
|
|
87
|
+
* neutral 0.5, one explicit recall_used lifts it to 0.67, one correction
|
|
88
|
+
* drops it to 0.33. Every requested id gets an entry so callers never have
|
|
89
|
+
* to special-case missing atoms.
|
|
90
|
+
*/
|
|
91
|
+
export function usefulnessScores(db, atomIds) {
|
|
92
|
+
const scores = new Map(atomIds.map(id => [id, 0.5]));
|
|
93
|
+
if (atomIds.length === 0)
|
|
94
|
+
return scores;
|
|
95
|
+
const ph = atomIds.map(() => '?').join(',');
|
|
96
|
+
const rows = db.prepare(`
|
|
97
|
+
SELECT atom_id,
|
|
98
|
+
SUM(CASE WHEN json_extract(payload_json, '$.family') = 'recall_used' THEN 1 ELSE 0 END) AS used,
|
|
99
|
+
SUM(CASE WHEN json_extract(payload_json, '$.family') = 'memory_corrected' THEN 1 ELSE 0 END) AS corrected
|
|
100
|
+
FROM memory_events
|
|
101
|
+
WHERE event_type = 'usefulness' AND atom_id IN (${ph})
|
|
102
|
+
GROUP BY atom_id
|
|
103
|
+
`).all(...atomIds);
|
|
104
|
+
for (const r of rows) {
|
|
105
|
+
scores.set(r.atom_id, (r.used + 1) / (r.used + r.corrected + 2));
|
|
106
|
+
}
|
|
107
|
+
return scores;
|
|
108
|
+
}
|
|
109
|
+
function surfaceFilter(opts, column) {
|
|
110
|
+
if (!opts.surface)
|
|
111
|
+
return { clause: '', params: [] };
|
|
112
|
+
return { clause: `AND json_extract(${column}, '$.surface') = ?`, params: [opts.surface] };
|
|
113
|
+
}
|
|
114
|
+
/**
|
|
115
|
+
* recall-usefulness rate = distinct atoms used / distinct atoms served, in
|
|
116
|
+
* the given window (ADR-010's seed metric). Distinct-atom counting avoids
|
|
117
|
+
* double-counting an atom served or used more than once in the window.
|
|
118
|
+
*/
|
|
119
|
+
export function usefulnessRate(db, opts = {}) {
|
|
120
|
+
const since = opts.sinceMs ?? 0;
|
|
121
|
+
const { clause, params } = surfaceFilter(opts, 'payload_json');
|
|
122
|
+
const served = db.prepare(`
|
|
123
|
+
SELECT COUNT(DISTINCT atom_id) AS n
|
|
124
|
+
FROM memory_events
|
|
125
|
+
WHERE event_type = 'usefulness'
|
|
126
|
+
AND json_extract(payload_json, '$.family') = 'recall_served'
|
|
127
|
+
AND created_at >= ?
|
|
128
|
+
${clause}
|
|
129
|
+
`).get(since, ...params).n;
|
|
130
|
+
const used = db.prepare(`
|
|
131
|
+
SELECT COUNT(DISTINCT atom_id) AS n
|
|
132
|
+
FROM memory_events
|
|
133
|
+
WHERE event_type = 'usefulness'
|
|
134
|
+
AND json_extract(payload_json, '$.family') = 'recall_used'
|
|
135
|
+
AND created_at >= ?
|
|
136
|
+
${clause}
|
|
137
|
+
`).get(since, ...params).n;
|
|
138
|
+
return { served, used, rate: served > 0 ? used / served : null };
|
|
139
|
+
}
|
|
140
|
+
/**
|
|
141
|
+
* Per-atom usefulness rate over a trailing window, for archival eligibility
|
|
142
|
+
* (FEAT-404 AC-2/AC-3 — "rate_7d"). Deliberately distinct from
|
|
143
|
+
* usefulnessRate() above: that DB-wide health-dashboard metric returns
|
|
144
|
+
* `rate: null` when served === 0 to avoid a misleading 0% on the /health
|
|
145
|
+
* surface. Archival eligibility instead needs one comparable number per
|
|
146
|
+
* atom, and "zero recall_served events in the window" is itself evidence
|
|
147
|
+
* the atom isn't earning its keep — so it maps to rate 0 here, not null,
|
|
148
|
+
* making an unsignaled atom exactly as archival-eligible as one with 0%
|
|
149
|
+
* explicit usage. Every requested id gets an entry (default 0) so callers
|
|
150
|
+
* never special-case a missing atom.
|
|
151
|
+
*/
|
|
152
|
+
export function perAtomUsefulnessRate7d(db, atomIds, sinceMs) {
|
|
153
|
+
const rates = new Map(atomIds.map(id => [id, 0]));
|
|
154
|
+
if (atomIds.length === 0)
|
|
155
|
+
return rates;
|
|
156
|
+
const ph = atomIds.map(() => '?').join(',');
|
|
157
|
+
const rows = db.prepare(`
|
|
158
|
+
SELECT atom_id,
|
|
159
|
+
SUM(CASE WHEN json_extract(payload_json, '$.family') = 'recall_served' THEN 1 ELSE 0 END) AS served,
|
|
160
|
+
SUM(CASE WHEN json_extract(payload_json, '$.family') = 'recall_used' THEN 1 ELSE 0 END) AS used
|
|
161
|
+
FROM memory_events
|
|
162
|
+
WHERE event_type = 'usefulness' AND atom_id IN (${ph}) AND created_at >= ?
|
|
163
|
+
GROUP BY atom_id
|
|
164
|
+
`).all(...atomIds, sinceMs);
|
|
165
|
+
for (const r of rows) {
|
|
166
|
+
rates.set(r.atom_id, r.served > 0 ? r.used / r.served : 0);
|
|
167
|
+
}
|
|
168
|
+
return rates;
|
|
169
|
+
}
|
|
170
|
+
/** Same metric, grouped by the served/used atom's current memories.type. */
|
|
171
|
+
export function usefulnessByType(db, opts = {}) {
|
|
172
|
+
const since = opts.sinceMs ?? 0;
|
|
173
|
+
const { clause, params } = surfaceFilter(opts, 'e.payload_json');
|
|
174
|
+
const servedRows = db.prepare(`
|
|
175
|
+
SELECT m.type AS type, COUNT(DISTINCT e.atom_id) AS n
|
|
176
|
+
FROM memory_events e
|
|
177
|
+
JOIN memories m ON m.id = e.atom_id
|
|
178
|
+
WHERE e.event_type = 'usefulness'
|
|
179
|
+
AND json_extract(e.payload_json, '$.family') = 'recall_served'
|
|
180
|
+
AND e.created_at >= ?
|
|
181
|
+
${clause}
|
|
182
|
+
GROUP BY m.type
|
|
183
|
+
`).all(since, ...params);
|
|
184
|
+
const usedRows = db.prepare(`
|
|
185
|
+
SELECT m.type AS type, COUNT(DISTINCT e.atom_id) AS n
|
|
186
|
+
FROM memory_events e
|
|
187
|
+
JOIN memories m ON m.id = e.atom_id
|
|
188
|
+
WHERE e.event_type = 'usefulness'
|
|
189
|
+
AND json_extract(e.payload_json, '$.family') = 'recall_used'
|
|
190
|
+
AND e.created_at >= ?
|
|
191
|
+
${clause}
|
|
192
|
+
GROUP BY m.type
|
|
193
|
+
`).all(since, ...params);
|
|
194
|
+
const servedMap = new Map(servedRows.map(r => [r.type, r.n]));
|
|
195
|
+
const usedMap = new Map(usedRows.map(r => [r.type, r.n]));
|
|
196
|
+
const types = Array.from(new Set([...servedMap.keys(), ...usedMap.keys()])).sort();
|
|
197
|
+
return types.map(type => {
|
|
198
|
+
const served = servedMap.get(type) ?? 0;
|
|
199
|
+
const used = usedMap.get(type) ?? 0;
|
|
200
|
+
return { type, served, used, rate: served > 0 ? used / served : null };
|
|
201
|
+
});
|
|
202
|
+
}
|
|
203
|
+
//# sourceMappingURL=usefulness.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"usefulness.js","sourceRoot":"","sources":["../../src/storage/usefulness.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAEzC,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAMrD,qGAAqG;AACrG,MAAM,UAAU,SAAS,CAAC,KAAa;IACrC,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AACvE,CAAC;AAeD;;;;GAIG;AACH,MAAM,UAAU,kBAAkB,CAAC,EAAM,EAAE,KAA8B;IACvE,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO;IACvC,MAAM,SAAS,GAAG,KAAK,CAAC,SAAS,IAAI,CAAC,KAAK,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IACtG,IAAI,CAAC,SAAS;QAAE,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;IAElF,MAAM,MAAM,GAAG,IAAI,eAAe,CAAC,EAAE,CAAC,CAAC;IACvC,MAAM,EAAE,GAAG,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE;QAC7B,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YAClC,MAAM,CAAC,MAAM,CAAC;gBACZ,UAAU,EAAE,YAAY;gBACxB,OAAO,EAAE,MAAM;gBACf,OAAO,EAAE;oBACP,MAAM,EAAE,eAAe;oBACvB,UAAU,EAAE,SAAS;oBACrB,KAAK,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,IAAI,IAAI;oBAChC,OAAO,EAAE,KAAK,CAAC,OAAO;oBACtB,IAAI,EAAE,KAAK,CAAC,IAAI,IAAI,IAAI;iBACzB;aACF,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IACH,EAAE,EAAE,CAAC;AACP,CAAC;AAQD,uEAAuE;AACvE,MAAM,UAAU,gBAAgB,CAAC,EAAM,EAAE,KAA4B;IACnE,IAAI,eAAe,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC;QAC7B,UAAU,EAAE,YAAY;QACxB,OAAO,EAAE,KAAK,CAAC,MAAM;QACrB,OAAO,EAAE,EAAE,MAAM,EAAE,aAAa,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE;KACjF,CAAC,CAAC;AACL,CAAC;AAOD;;;;;;GAMG;AACH,MAAM,UAAU,qBAAqB,CAAC,EAAM,EAAE,KAAiC;IAC7E,IAAI,eAAe,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC;QAC7B,UAAU,EAAE,YAAY;QACxB,OAAO,EAAE,KAAK,CAAC,MAAM;QACrB,OAAO,EAAE,EAAE,MAAM,EAAE,kBAAkB,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE;KAC9D,CAAC,CAAC;AACL,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,gBAAgB,CAAC,EAAM,EAAE,OAAiB;IACxD,MAAM,MAAM,GAAG,IAAI,GAAG,CAAiB,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;IACrE,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,MAAM,CAAC;IAExC,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC5C,MAAM,IAAI,GAAG,EAAE,CAAC,OAAO,CAAC;;;;;sDAK4B,EAAE;;GAErD,CAAC,CAAC,GAAG,CAAC,GAAG,OAAO,CAAgE,CAAC;IAElF,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;QACrB,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC;IACnE,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAeD,SAAS,aAAa,CAAC,IAAwB,EAAE,MAAc;IAC7D,IAAI,CAAC,IAAI,CAAC,OAAO;QAAE,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;IACrD,OAAO,EAAE,MAAM,EAAE,oBAAoB,MAAM,oBAAoB,EAAE,MAAM,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;AAC5F,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,cAAc,CAAC,EAAM,EAAE,OAA2B,EAAE;IAClE,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,IAAI,CAAC,CAAC;IAChC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,aAAa,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC;IAE/D,MAAM,MAAM,GAAI,EAAE,CAAC,OAAO,CAAC;;;;;;QAMrB,MAAM;GACX,CAAC,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,MAAM,CAAmB,CAAC,CAAC,CAAC;IAE7C,MAAM,IAAI,GAAI,EAAE,CAAC,OAAO,CAAC;;;;;;QAMnB,MAAM;GACX,CAAC,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,MAAM,CAAmB,CAAC,CAAC,CAAC;IAE7C,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;AACnE,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,uBAAuB,CAAC,EAAM,EAAE,OAAiB,EAAE,OAAe;IAChF,MAAM,KAAK,GAAG,IAAI,GAAG,CAAiB,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAClE,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAEvC,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC5C,MAAM,IAAI,GAAG,EAAE,CAAC,OAAO,CAAC;;;;;sDAK4B,EAAE;;GAErD,CAAC,CAAC,GAAG,CAAC,GAAG,OAAO,EAAE,OAAO,CAA6D,CAAC;IAExF,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;QACrB,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7D,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AASD,4EAA4E;AAC5E,MAAM,UAAU,gBAAgB,CAAC,EAAM,EAAE,OAA2B,EAAE;IACpE,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,IAAI,CAAC,CAAC;IAChC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,aAAa,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAAC;IAEjE,MAAM,UAAU,GAAG,EAAE,CAAC,OAAO,CAAC;;;;;;;QAOxB,MAAM;;GAEX,CAAC,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,MAAM,CAAuC,CAAC;IAE/D,MAAM,QAAQ,GAAG,EAAE,CAAC,OAAO,CAAC;;;;;;;QAOtB,MAAM;;GAEX,CAAC,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,MAAM,CAAuC,CAAC;IAE/D,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC9D,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1D,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC,GAAG,SAAS,CAAC,IAAI,EAAE,EAAE,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAEnF,OAAO,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE;QACtB,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACxC,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACpC,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IACzE,CAAC,CAAC,CAAC;AACL,CAAC"}
|