nlm-memory 0.20.0 → 0.20.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -10
- package/dist/cli/digest.js +21 -0
- package/dist/cli/digest.js.map +1 -1
- package/dist/cli/init.d.ts +12 -0
- package/dist/cli/init.js +49 -0
- package/dist/cli/init.js.map +1 -0
- package/dist/cli/nlm.d.ts +13 -1
- package/dist/cli/nlm.js +766 -55
- package/dist/cli/nlm.js.map +1 -1
- package/dist/core/actions/overlay.d.ts +20 -6
- package/dist/core/actions/overlay.js +32 -18
- package/dist/core/actions/overlay.js.map +1 -1
- package/dist/core/adapters/claude-code.js +6 -0
- package/dist/core/adapters/claude-code.js.map +1 -1
- package/dist/core/classifier/prompt.js +3 -3
- package/dist/core/digest/compose.d.ts +10 -0
- package/dist/core/digest/compose.js +7 -1
- package/dist/core/digest/compose.js.map +1 -1
- package/dist/core/embedding/embed-backfill.d.ts +13 -2
- package/dist/core/embedding/embed-backfill.js +181 -43
- package/dist/core/embedding/embed-backfill.js.map +1 -1
- package/dist/core/embedding/embedding-config.d.ts +17 -0
- package/dist/core/embedding/embedding-config.js +12 -0
- package/dist/core/embedding/embedding-config.js.map +1 -0
- package/dist/core/embedding/pg-embed-backfill.d.ts +31 -0
- package/dist/core/embedding/pg-embed-backfill.js +264 -0
- package/dist/core/embedding/pg-embed-backfill.js.map +1 -0
- package/dist/core/entities/dedup-suggest.d.ts +25 -0
- package/dist/core/entities/dedup-suggest.js +107 -0
- package/dist/core/entities/dedup-suggest.js.map +1 -0
- package/dist/core/eval/classifier-fixture-eval.d.ts +29 -0
- package/dist/core/eval/classifier-fixture-eval.js +217 -0
- package/dist/core/eval/classifier-fixture-eval.js.map +1 -0
- package/dist/core/eval/extraction-scoring.d.ts +63 -0
- package/dist/core/eval/extraction-scoring.js +79 -0
- package/dist/core/eval/extraction-scoring.js.map +1 -0
- package/dist/core/eval/run-eval.d.ts +40 -0
- package/dist/core/eval/run-eval.js +34 -0
- package/dist/core/eval/run-eval.js.map +1 -0
- package/dist/core/exemplars/capture-from-session.d.ts +1 -0
- package/dist/core/exemplars/capture-from-session.js +6 -2
- package/dist/core/exemplars/capture-from-session.js.map +1 -1
- package/dist/core/exemplars/diff-parse.d.ts +23 -0
- package/dist/core/exemplars/diff-parse.js +73 -0
- package/dist/core/exemplars/diff-parse.js.map +1 -0
- package/dist/core/exemplars/embed-backfill.d.ts +36 -0
- package/dist/core/exemplars/embed-backfill.js +85 -0
- package/dist/core/exemplars/embed-backfill.js.map +1 -0
- package/dist/core/exemplars/embed-text.d.ts +15 -0
- package/dist/core/exemplars/embed-text.js +18 -0
- package/dist/core/exemplars/embed-text.js.map +1 -0
- package/dist/core/exemplars/extract-exemplar.js +2 -65
- package/dist/core/exemplars/extract-exemplar.js.map +1 -1
- package/dist/core/exemplars/recall-code.js +4 -0
- package/dist/core/exemplars/recall-code.js.map +1 -1
- package/dist/core/exemplars/warm-embedder.d.ts +14 -0
- package/dist/core/exemplars/warm-embedder.js +18 -0
- package/dist/core/exemplars/warm-embedder.js.map +1 -0
- package/dist/core/health/corpus-state.d.ts +8 -0
- package/dist/core/health/corpus-state.js +11 -0
- package/dist/core/health/corpus-state.js.map +1 -0
- package/dist/core/health/embed-inflight.d.ts +9 -0
- package/dist/core/health/embed-inflight.js +30 -0
- package/dist/core/health/embed-inflight.js.map +1 -0
- package/dist/core/health/embedding-lane-state.d.ts +6 -0
- package/dist/core/health/embedding-lane-state.js +18 -0
- package/dist/core/health/embedding-lane-state.js.map +1 -0
- package/dist/core/health/warmup-state.d.ts +12 -0
- package/dist/core/health/warmup-state.js +12 -0
- package/dist/core/health/warmup-state.js.map +1 -0
- package/dist/core/hook/gate.d.ts +13 -6
- package/dist/core/hook/gate.js +47 -6
- package/dist/core/hook/gate.js.map +1 -1
- package/dist/core/hook/hook-log.d.ts +5 -0
- package/dist/core/hook/hook-log.js.map +1 -1
- package/dist/core/hook/memo.d.ts +12 -0
- package/dist/core/hook/memo.js +41 -1
- package/dist/core/hook/memo.js.map +1 -1
- package/dist/core/hook/pointer-block.d.ts +1 -0
- package/dist/core/hook/pointer-block.js +23 -2
- package/dist/core/hook/pointer-block.js.map +1 -1
- package/dist/core/hook/query-extract.d.ts +2 -1
- package/dist/core/hook/query-extract.js +10 -1
- package/dist/core/hook/query-extract.js.map +1 -1
- package/dist/core/hook/resolve-conversation-by-query.d.ts +28 -0
- package/dist/core/hook/resolve-conversation-by-query.js +98 -0
- package/dist/core/hook/resolve-conversation-by-query.js.map +1 -0
- package/dist/core/hook/select.d.ts +8 -0
- package/dist/core/hook/select.js +9 -2
- package/dist/core/hook/select.js.map +1 -1
- package/dist/core/hook/transcript.js +16 -2
- package/dist/core/hook/transcript.js.map +1 -1
- package/dist/core/ingest/ingest-session.d.ts +5 -0
- package/dist/core/ingest/ingest-session.js +3 -0
- package/dist/core/ingest/ingest-session.js.map +1 -1
- package/dist/core/ingest/reclassify-oversized.d.ts +5 -0
- package/dist/core/ingest/reclassify-oversized.js +3 -0
- package/dist/core/ingest/reclassify-oversized.js.map +1 -1
- package/dist/core/ingest/reprocess.d.ts +96 -0
- package/dist/core/ingest/reprocess.js +275 -0
- package/dist/core/ingest/reprocess.js.map +1 -0
- package/dist/core/integrity/check-invariants.d.ts +6 -1
- package/dist/core/integrity/check-invariants.js +209 -3
- package/dist/core/integrity/check-invariants.js.map +1 -1
- package/dist/core/metrics/corpus-stats.d.ts +49 -0
- package/dist/core/metrics/corpus-stats.js +111 -0
- package/dist/core/metrics/corpus-stats.js.map +1 -0
- package/dist/core/metrics/re-derivation.d.ts +48 -0
- package/dist/core/metrics/re-derivation.js +87 -0
- package/dist/core/metrics/re-derivation.js.map +1 -0
- package/dist/core/recall/citation-log.d.ts +8 -0
- package/dist/core/recall/citation-log.js +17 -0
- package/dist/core/recall/citation-log.js.map +1 -1
- package/dist/core/recall/query-intent.d.ts +7 -0
- package/dist/core/recall/query-intent.js +25 -0
- package/dist/core/recall/query-intent.js.map +1 -0
- package/dist/core/recall/query-log.d.ts +4 -0
- package/dist/core/recall/query-log.js +4 -0
- package/dist/core/recall/query-log.js.map +1 -1
- package/dist/core/recall/recall-service.d.ts +37 -1
- package/dist/core/recall/recall-service.js +106 -63
- package/dist/core/recall/recall-service.js.map +1 -1
- package/dist/core/recall/related-exemplars.d.ts +1 -0
- package/dist/core/recall/related-exemplars.js +4 -1
- package/dist/core/recall/related-exemplars.js.map +1 -1
- package/dist/core/recall/related-facts.js +2 -2
- package/dist/core/recall/related-facts.js.map +1 -1
- package/dist/core/recall/reranker.d.ts +2 -1
- package/dist/core/recall/reranker.js +3 -3
- package/dist/core/recall/reranker.js.map +1 -1
- package/dist/core/recall-facts/fact-query-log.d.ts +7 -0
- package/dist/core/recall-facts/fact-query-log.js +2 -0
- package/dist/core/recall-facts/fact-query-log.js.map +1 -1
- package/dist/core/recall-facts/fact-recall-service.d.ts +9 -1
- package/dist/core/recall-facts/fact-recall-service.js +21 -8
- package/dist/core/recall-facts/fact-recall-service.js.map +1 -1
- package/dist/core/scheduler/scan-once.d.ts +6 -6
- package/dist/core/scheduler/scan-once.js +13 -49
- package/dist/core/scheduler/scan-once.js.map +1 -1
- package/dist/core/scheduler/scheduler.d.ts +10 -0
- package/dist/core/scheduler/scheduler.js +75 -10
- package/dist/core/scheduler/scheduler.js.map +1 -1
- package/dist/core/signals/code-signal.d.ts +49 -0
- package/dist/core/signals/code-signal.js +53 -0
- package/dist/core/signals/code-signal.js.map +1 -0
- package/dist/core/storage/db-restore.js +3 -1
- package/dist/core/storage/db-restore.js.map +1 -1
- package/dist/core/storage/fact-batch.d.ts +9 -0
- package/dist/core/storage/fact-batch.js +14 -0
- package/dist/core/storage/fact-batch.js.map +1 -0
- package/dist/core/storage/pg-code-exemplar-store.d.ts +1 -0
- package/dist/core/storage/pg-code-exemplar-store.js +7 -0
- package/dist/core/storage/pg-code-exemplar-store.js.map +1 -1
- package/dist/core/storage/pg-embedding-config.d.ts +29 -0
- package/dist/core/storage/pg-embedding-config.js +53 -0
- package/dist/core/storage/pg-embedding-config.js.map +1 -0
- package/dist/core/storage/pg-entity-store.d.ts +12 -0
- package/dist/core/storage/pg-entity-store.js +79 -0
- package/dist/core/storage/pg-entity-store.js.map +1 -0
- package/dist/core/storage/pg-fact-ingest.d.ts +15 -0
- package/dist/core/storage/pg-fact-ingest.js +34 -0
- package/dist/core/storage/pg-fact-ingest.js.map +1 -0
- package/dist/core/storage/pg-fact-store.d.ts +3 -0
- package/dist/core/storage/pg-fact-store.js +26 -30
- package/dist/core/storage/pg-fact-store.js.map +1 -1
- package/dist/core/storage/pg-migrate.d.ts +18 -0
- package/dist/core/storage/pg-migrate.js +73 -0
- package/dist/core/storage/pg-migrate.js.map +1 -0
- package/dist/core/storage/pg-session-store.d.ts +9 -0
- package/dist/core/storage/pg-session-store.js +204 -76
- package/dist/core/storage/pg-session-store.js.map +1 -1
- package/dist/core/storage/pg-storage.d.ts +9 -7
- package/dist/core/storage/pg-storage.js +20 -50
- package/dist/core/storage/pg-storage.js.map +1 -1
- package/dist/core/storage/pg-workstream-store.d.ts +24 -0
- package/dist/core/storage/pg-workstream-store.js +96 -0
- package/dist/core/storage/pg-workstream-store.js.map +1 -0
- package/dist/core/storage/sqlite-code-exemplar-store.d.ts +1 -0
- package/dist/core/storage/sqlite-code-exemplar-store.js +11 -0
- package/dist/core/storage/sqlite-code-exemplar-store.js.map +1 -1
- package/dist/core/storage/sqlite-embedding-config.d.ts +10 -0
- package/dist/core/storage/sqlite-embedding-config.js +27 -0
- package/dist/core/storage/sqlite-embedding-config.js.map +1 -0
- package/dist/core/storage/sqlite-entity-store.d.ts +13 -0
- package/dist/core/storage/sqlite-entity-store.js +96 -0
- package/dist/core/storage/sqlite-entity-store.js.map +1 -0
- package/dist/core/storage/sqlite-fact-store.d.ts +15 -7
- package/dist/core/storage/sqlite-fact-store.js +76 -25
- package/dist/core/storage/sqlite-fact-store.js.map +1 -1
- package/dist/core/storage/sqlite-session-store.d.ts +20 -5
- package/dist/core/storage/sqlite-session-store.js +183 -68
- package/dist/core/storage/sqlite-session-store.js.map +1 -1
- package/dist/core/storage/sqlite-storage.d.ts +8 -8
- package/dist/core/storage/sqlite-storage.js +21 -32
- package/dist/core/storage/sqlite-storage.js.map +1 -1
- package/dist/core/storage/sqlite-workstream-store.d.ts +24 -0
- package/dist/core/storage/sqlite-workstream-store.js +98 -0
- package/dist/core/storage/sqlite-workstream-store.js.map +1 -0
- package/dist/core/work-digest/active-spans.d.ts +8 -0
- package/dist/core/work-digest/active-spans.js +29 -0
- package/dist/core/work-digest/active-spans.js.map +1 -0
- package/dist/core/work-digest/attribute.d.ts +15 -0
- package/dist/core/work-digest/attribute.js +76 -0
- package/dist/core/work-digest/attribute.js.map +1 -0
- package/dist/core/work-digest/build-work-digest.d.ts +12 -0
- package/dist/core/work-digest/build-work-digest.js +69 -0
- package/dist/core/work-digest/build-work-digest.js.map +1 -0
- package/dist/core/work-digest/compose-work-digest.d.ts +3 -0
- package/dist/core/work-digest/compose-work-digest.js +52 -0
- package/dist/core/work-digest/compose-work-digest.js.map +1 -0
- package/dist/core/work-digest/merge-active.d.ts +10 -0
- package/dist/core/work-digest/merge-active.js +22 -0
- package/dist/core/work-digest/merge-active.js.map +1 -0
- package/dist/core/work-digest/read-transcript-timestamps.d.ts +7 -0
- package/dist/core/work-digest/read-transcript-timestamps.js +40 -0
- package/dist/core/work-digest/read-transcript-timestamps.js.map +1 -0
- package/dist/core/work-digest/topics.d.ts +18 -0
- package/dist/core/work-digest/topics.js +26 -0
- package/dist/core/work-digest/topics.js.map +1 -0
- package/dist/core/work-digest/types.d.ts +45 -0
- package/dist/core/work-digest/types.js +2 -0
- package/dist/core/work-digest/types.js.map +1 -0
- package/dist/core/workstream/bind.d.ts +25 -0
- package/dist/core/workstream/bind.js +23 -0
- package/dist/core/workstream/bind.js.map +1 -0
- package/dist/core/workstream/compose-recall.d.ts +2 -0
- package/dist/core/workstream/compose-recall.js +39 -0
- package/dist/core/workstream/compose-recall.js.map +1 -0
- package/dist/core/workstream/merge-suggest.d.ts +19 -0
- package/dist/core/workstream/merge-suggest.js +61 -0
- package/dist/core/workstream/merge-suggest.js.map +1 -0
- package/dist/core/workstream/model.d.ts +20 -0
- package/dist/core/workstream/model.js +9 -0
- package/dist/core/workstream/model.js.map +1 -0
- package/dist/core/workstream/name-match.d.ts +8 -0
- package/dist/core/workstream/name-match.js +18 -0
- package/dist/core/workstream/name-match.js.map +1 -0
- package/dist/core/workstream/resolve.d.ts +10 -0
- package/dist/core/workstream/resolve.js +21 -0
- package/dist/core/workstream/resolve.js.map +1 -0
- package/dist/core/workstream/rollup.d.ts +12 -0
- package/dist/core/workstream/rollup.js +17 -0
- package/dist/core/workstream/rollup.js.map +1 -0
- package/dist/core/workstream/work-topics.d.ts +10 -0
- package/dist/core/workstream/work-topics.js +55 -0
- package/dist/core/workstream/work-topics.js.map +1 -0
- package/dist/hook/hook-helpers.d.ts +4 -0
- package/dist/hook/hook-helpers.js +36 -0
- package/dist/hook/hook-helpers.js.map +1 -0
- package/dist/hook/pre-compact-hook.d.ts +23 -0
- package/dist/hook/pre-compact-hook.js +66 -0
- package/dist/hook/pre-compact-hook.js.map +1 -0
- package/dist/hook/prompt-recall-hook.d.ts +37 -0
- package/dist/hook/prompt-recall-hook.js +120 -20
- package/dist/hook/prompt-recall-hook.js.map +1 -1
- package/dist/hook/recall-gate.d.ts +29 -0
- package/dist/hook/recall-gate.js +50 -0
- package/dist/hook/recall-gate.js.map +1 -0
- package/dist/hook/recall-over-http.d.ts +1 -1
- package/dist/hook/recall-over-http.js +11 -11
- package/dist/hook/recall-over-http.js.map +1 -1
- package/dist/hook/recent-context.d.ts +25 -0
- package/dist/hook/recent-context.js +104 -0
- package/dist/hook/recent-context.js.map +1 -0
- package/dist/hook/score-floor.d.ts +27 -0
- package/dist/hook/score-floor.js +42 -0
- package/dist/hook/score-floor.js.map +1 -0
- package/dist/hook/session-end-hook.js +8 -29
- package/dist/hook/session-end-hook.js.map +1 -1
- package/dist/hook/session-start-hook.js +24 -59
- package/dist/hook/session-start-hook.js.map +1 -1
- package/dist/hook/stop-hook.js +22 -59
- package/dist/hook/stop-hook.js.map +1 -1
- package/dist/hook/subagent-start-hook.d.ts +28 -0
- package/dist/hook/subagent-start-hook.js +81 -0
- package/dist/hook/subagent-start-hook.js.map +1 -0
- package/dist/http/app.js +28 -8
- package/dist/http/app.js.map +1 -1
- package/dist/install/setup.js +2 -0
- package/dist/install/setup.js.map +1 -1
- package/dist/llm/build-classifier.d.ts +22 -0
- package/dist/llm/build-classifier.js +42 -0
- package/dist/llm/build-classifier.js.map +1 -0
- package/dist/llm/build-embedder.d.ts +11 -0
- package/dist/llm/build-embedder.js +42 -0
- package/dist/llm/build-embedder.js.map +1 -0
- package/dist/llm/bundled-embedder-client.d.ts +37 -0
- package/dist/llm/bundled-embedder-client.js +90 -0
- package/dist/llm/bundled-embedder-client.js.map +1 -0
- package/dist/llm/classifier-box.d.ts +13 -5
- package/dist/llm/classifier-box.js +28 -6
- package/dist/llm/classifier-box.js.map +1 -1
- package/dist/llm/classifier-egress.d.ts +18 -0
- package/dist/llm/classifier-egress.js +59 -0
- package/dist/llm/classifier-egress.js.map +1 -0
- package/dist/llm/client-shared.d.ts +4 -0
- package/dist/llm/client-shared.js +39 -0
- package/dist/llm/client-shared.js.map +1 -0
- package/dist/llm/deepseek-client.d.ts +20 -0
- package/dist/llm/deepseek-client.js +59 -45
- package/dist/llm/deepseek-client.js.map +1 -1
- package/dist/llm/embedder-info.d.ts +15 -0
- package/dist/llm/embedder-info.js +27 -0
- package/dist/llm/embedder-info.js.map +1 -0
- package/dist/llm/model-quirks.d.ts +4 -0
- package/dist/llm/model-quirks.js +7 -0
- package/dist/llm/model-quirks.js.map +1 -0
- package/dist/llm/naming.d.ts +5 -0
- package/dist/llm/naming.js +20 -0
- package/dist/llm/naming.js.map +1 -0
- package/dist/llm/ollama-client.d.ts +7 -8
- package/dist/llm/ollama-client.js +59 -51
- package/dist/llm/ollama-client.js.map +1 -1
- package/dist/llm/ollama-code-embedder.d.ts +8 -3
- package/dist/llm/ollama-code-embedder.js +14 -24
- package/dist/llm/ollama-code-embedder.js.map +1 -1
- package/dist/llm/openai-code-embedder-client.d.ts +25 -0
- package/dist/llm/openai-code-embedder-client.js +52 -0
- package/dist/llm/openai-code-embedder-client.js.map +1 -0
- package/dist/llm/openai-embedder-client.d.ts +41 -0
- package/dist/llm/openai-embedder-client.js +83 -0
- package/dist/llm/openai-embedder-client.js.map +1 -0
- package/dist/mcp/server.d.ts +47 -2
- package/dist/mcp/server.js +303 -18
- package/dist/mcp/server.js.map +1 -1
- package/dist/ports/code-embedder.d.ts +6 -1
- package/dist/ports/code-exemplar-store.d.ts +5 -0
- package/dist/ports/entity-store.d.ts +35 -0
- package/dist/ports/entity-store.js +7 -0
- package/dist/ports/entity-store.js.map +1 -0
- package/dist/ports/fact-store.d.ts +0 -0
- package/dist/ports/fact-store.js.map +1 -1
- package/dist/ports/llm-client.d.ts +9 -1
- package/dist/ports/session-store.d.ts +12 -0
- package/dist/ports/storage.d.ts +7 -30
- package/dist/ports/storage.js +3 -7
- package/dist/ports/storage.js.map +1 -1
- package/dist/ports/transcript-adapter.d.ts +2 -0
- package/dist/ports/workstream-store.d.ts +21 -0
- package/dist/ports/workstream-store.js +2 -0
- package/dist/ports/workstream-store.js.map +1 -0
- package/dist/shared/types.d.ts +23 -2
- package/fixtures/classifier-gold/README.md +94 -0
- package/fixtures/classifier-gold/reference.json +318 -0
- package/fixtures/classifier-gold/transcripts/01-bug-fix-null-pointer-api.txt +75 -0
- package/fixtures/classifier-gold/transcripts/02-bug-fix-race-condition-queue.txt +77 -0
- package/fixtures/classifier-gold/transcripts/03-bug-fix-jwt-clock-skew.txt +86 -0
- package/fixtures/classifier-gold/transcripts/04-feature-webhook-retry.txt +95 -0
- package/fixtures/classifier-gold/transcripts/05-feature-csv-export.txt +86 -0
- package/fixtures/classifier-gold/transcripts/06-feature-rate-limiting.txt +61 -0
- package/fixtures/classifier-gold/transcripts/07-refactor-db-abstraction.txt +72 -0
- package/fixtures/classifier-gold/transcripts/08-refactor-config-loading.txt +89 -0
- package/fixtures/classifier-gold/transcripts/09-ops-docker-staging.txt +94 -0
- package/fixtures/classifier-gold/transcripts/10-ops-nginx-ssl.txt +121 -0
- package/fixtures/classifier-gold/transcripts/11-ops-postgres-migration.txt +74 -0
- package/fixtures/classifier-gold/transcripts/12-ops-cron-debugging.txt +73 -0
- package/fixtures/classifier-gold/transcripts/13-research-vector-db-comparison.txt +59 -0
- package/fixtures/classifier-gold/transcripts/14-research-oauth-provider.txt +74 -0
- package/fixtures/classifier-gold/transcripts/15-writing-api-docs.txt +154 -0
- package/fixtures/classifier-gold/transcripts/16-meeting-sprint-planning.txt +69 -0
- package/fixtures/classifier-gold/transcripts/17-trivial-list-files.txt +14 -0
- package/fixtures/classifier-gold/transcripts/18-trivial-version-check.txt +21 -0
- package/fixtures/classifier-gold/transcripts/19-trivial-syntax-help.txt +29 -0
- package/fixtures/classifier-gold/transcripts/20-trivial-read-package-json.txt +41 -0
- package/migrations/000_initial_schema.sql +1 -1
- package/migrations/024_backfill_exemplar_repo_basename.sql +32 -0
- package/migrations/025_workstreams.sql +30 -0
- package/migrations/026_embedding_config.sql +12 -0
- package/migrations/027_drop_legacy_session_embeddings.sql +7 -0
- package/migrations/028_classifier_provenance.sql +5 -0
- package/migrations/pg/001_initial.sql +34 -0
- package/migrations/pg/019_split_replaces.sql +16 -7
- package/migrations/pg/025_workstreams.sql +34 -0
- package/migrations/pg/026_embedding_config.sql +12 -0
- package/migrations/pg/028_classifier_provenance.sql +5 -0
- package/migrations/pg/029_entity_variants.sql +9 -0
- package/nlm/index.js +395 -37
- package/package.json +12 -2
- package/plugin/.codex-plugin/plugin.json +1 -1
- package/plugin/scripts/prompt-recall-hook.mjs +402 -41
- package/plugin/scripts/session-start-hook.mjs +286 -60
- package/plugin/scripts/stop-hook.mjs +85 -65
- package/plugin-hermes-agent/__init__.py +2 -1
- package/templates/agent-contract/README.md +24 -0
- package/templates/agent-contract/claude-code.md +40 -0
- package/templates/agent-contract/generic.md +38 -0
- package/dist/core/embedding/embed-normalize.d.ts +0 -28
- package/dist/core/embedding/embed-normalize.js +0 -98
- package/dist/core/embedding/embed-normalize.js.map +0 -1
- package/dist/core/storage/pg-tx-context.d.ts +0 -52
- package/dist/core/storage/pg-tx-context.js +0 -149
- package/dist/core/storage/pg-tx-context.js.map +0 -1
package/dist/cli/nlm.js
CHANGED
|
@@ -22,11 +22,12 @@
|
|
|
22
22
|
* nlm disconnect claude-code — remove MCP block from ~/.mcp.json
|
|
23
23
|
* nlm disconnect codex — remove Codex plugin
|
|
24
24
|
* nlm digest — print a daily-activity digest (or --telegram to post)
|
|
25
|
+
* nlm init - print (or write) the agent recall contract snippet
|
|
25
26
|
*/
|
|
26
|
-
import { fileURLToPath } from "node:url";
|
|
27
|
+
import { fileURLToPath, pathToFileURL } from "node:url";
|
|
27
28
|
import { dirname, resolve, join } from "node:path";
|
|
28
29
|
import { homedir } from "node:os";
|
|
29
|
-
import { mkdirSync, readFileSync, writeFileSync, existsSync, rmSync, copyFileSync } from "node:fs";
|
|
30
|
+
import { mkdirSync, readFileSync, writeFileSync, existsSync, rmSync, copyFileSync, realpathSync, appendFileSync } from "node:fs";
|
|
30
31
|
import { execFileSync, spawn, spawnSync } from "node:child_process";
|
|
31
32
|
import { Command } from "commander";
|
|
32
33
|
import pkg from "../../package.json" with { type: "json" };
|
|
@@ -40,12 +41,14 @@ import { PgStorage } from "../core/storage/pg-storage.js";
|
|
|
40
41
|
import { applyPendingRestore, stageRestore } from "../core/storage/db-restore.js";
|
|
41
42
|
import { listBackupDates, resolveBackup, runRollingBackup } from "../core/storage/backup-rotation.js";
|
|
42
43
|
import { createApp } from "../http/app.js";
|
|
43
|
-
import { createMcpServer } from "../mcp/server.js";
|
|
44
|
-
import {
|
|
45
|
-
import {
|
|
46
|
-
import {
|
|
47
|
-
import {
|
|
44
|
+
import { createMcpServer, listMergeSuggestionsHandler, mergeWorkstreamsHandler, rebindSessionHandler, recallWorkstreamHandler, renameWorkstreamHandler, retireWorkstreamHandler } from "../mcp/server.js";
|
|
45
|
+
import { classifierEgressNotice } from "../llm/classifier-egress.js";
|
|
46
|
+
import { buildClassifier } from "../llm/build-classifier.js";
|
|
47
|
+
import { OllamaCodeEmbedder, DEFAULT_CODE_EMBED_MODEL } from "../llm/ollama-code-embedder.js";
|
|
48
|
+
import { OpenAICodeEmbedderClient } from "../llm/openai-code-embedder-client.js";
|
|
49
|
+
import { resolveEmbedderInfo } from "../llm/embedder-info.js";
|
|
48
50
|
import { autoloadEnv } from "../llm/env-autoload.js";
|
|
51
|
+
import { buildEmbedder as _buildEmbedder } from "../llm/build-embedder.js";
|
|
49
52
|
import { addHook, buildHookCommand, removeHook } from "../core/hook/claude-settings.js";
|
|
50
53
|
import { codexBinaryAvailable, connectCodex, disconnectCodex, repairCodex, pluginScriptsDir, } from "../install/codex.js";
|
|
51
54
|
import { connectClaudeCode, disconnectClaudeCode, installClaudeCodeHooks, mcpConfigPath } from "../install/claude-code.js";
|
|
@@ -56,6 +59,7 @@ import { embeddingModelPresent, ensureMcpToken, ollamaModelPresent } from "../in
|
|
|
56
59
|
import { connectCursor, disconnectCursor } from "../install/cursor.js";
|
|
57
60
|
import { describeRemove, describeUpsert, installCursorRules, installOpencodeRules, installWindsurfRules, uninstallCursorRules, uninstallOpencodeRules, uninstallWindsurfRules, } from "../install/rules-install.js";
|
|
58
61
|
import { runSupersedeCommand } from "./supersede.js";
|
|
62
|
+
import { runInitCommand } from "./init.js";
|
|
59
63
|
import { getUpdateStatus } from "../core/update-check/check.js";
|
|
60
64
|
import { connectHermes, disconnectHermes, hermesConfigPath } from "../install/hermes.js";
|
|
61
65
|
import { connectHermesAgent, disconnectHermesAgent, hermesAgentPluginDir } from "../install/hermes-agent.js";
|
|
@@ -64,8 +68,16 @@ import { connectPi, disconnectPi, piSettingsPath } from "../install/pi.js";
|
|
|
64
68
|
import { runSetup } from "../install/setup.js";
|
|
65
69
|
import { runParity } from "./classify-parity.js";
|
|
66
70
|
import { reembedCorpus } from "../core/embedding/embed-backfill.js";
|
|
71
|
+
import { reembedCorpusPg } from "../core/embedding/pg-embed-backfill.js";
|
|
72
|
+
import { backfillExemplarEmbeddings } from "../core/exemplars/embed-backfill.js";
|
|
73
|
+
import { warmCodeEmbedder } from "../core/exemplars/warm-embedder.js";
|
|
74
|
+
import { markWarm } from "../core/health/warmup-state.js";
|
|
75
|
+
import { setLaneHealth } from "../core/health/embedding-lane-state.js";
|
|
76
|
+
import { setCorpusSnapshot, corpusSnapshot } from "../core/health/corpus-state.js";
|
|
77
|
+
import { computeCorpusStats, sqliteCorpusStatsDeps, parseCorpusThresholds, thresholdState } from "../core/metrics/corpus-stats.js";
|
|
78
|
+
import { computeReDerivationRate, sqliteReDerivationDeps } from "../core/metrics/re-derivation.js";
|
|
79
|
+
import { reconcileLane } from "../core/embedding/embedding-config.js";
|
|
67
80
|
import { backfillFacts } from "../core/facts/backfill-facts.js";
|
|
68
|
-
import { normalizeEmbeddings } from "../core/embedding/embed-normalize.js";
|
|
69
81
|
import { ScanScheduler } from "../core/scheduler/scheduler.js";
|
|
70
82
|
import { MemoSweepScheduler } from "../core/hook/memo-sweep.js";
|
|
71
83
|
import { isAgentLoaded, isBenignBootoutError } from "./launchctl-helpers.js";
|
|
@@ -75,10 +87,17 @@ import { applyEnvAssignment } from "./config-env.js";
|
|
|
75
87
|
import { adapterFromSource } from "../core/adapters/from-source.js";
|
|
76
88
|
import { runDigest } from "./digest.js";
|
|
77
89
|
import { installScope } from "../core/signals/install-scope.js";
|
|
90
|
+
import { buildWorkDigest } from "../core/work-digest/build-work-digest.js";
|
|
91
|
+
import { composeWorkDigest } from "../core/work-digest/compose-work-digest.js";
|
|
92
|
+
import { defaultTopicProvider, aliasTopicProvider } from "../core/work-digest/topics.js";
|
|
78
93
|
import { runChecksOnSqlite, runChecksOnPg, applyFixOnSqlite, applyFixOnPg } from "../core/integrity/check-invariants.js";
|
|
94
|
+
import { normalizeLabel } from "../core/workstream/model.js";
|
|
95
|
+
import { resolveWorkstreamId } from "../core/workstream/resolve.js";
|
|
96
|
+
import { suggestMerges } from "../core/entities/dedup-suggest.js";
|
|
79
97
|
const __filename = fileURLToPath(import.meta.url);
|
|
80
98
|
const __dirname = dirname(__filename);
|
|
81
99
|
const MIGRATIONS_DIR = resolve(__dirname, "../../migrations");
|
|
100
|
+
const CLASSIFIER_FIXTURES_DIR = resolve(__dirname, "../../fixtures/classifier-gold");
|
|
82
101
|
const PG_MIGRATIONS_DIR = join(fileURLToPath(new URL(".", import.meta.url)), "../../migrations/pg");
|
|
83
102
|
const UI_DIST = resolve(__dirname, "../../dist/ui");
|
|
84
103
|
const DEFAULT_DB_PATH = resolve(homedir(), ".nlm/canonical.sqlite");
|
|
@@ -98,19 +117,121 @@ function port() {
|
|
|
98
117
|
function ollamaUrl() {
|
|
99
118
|
return process.env["NLM_OLLAMA_URL"] ?? "http://localhost:11434";
|
|
100
119
|
}
|
|
101
|
-
function
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
const
|
|
109
|
-
|
|
120
|
+
export function resolveDigestDate(date) {
|
|
121
|
+
if (date !== undefined) {
|
|
122
|
+
if (!/^\d{4}-\d{2}-\d{2}$/.test(date)) {
|
|
123
|
+
throw new Error(`invalid --date "${date}"; expected YYYY-MM-DD`);
|
|
124
|
+
}
|
|
125
|
+
return date;
|
|
126
|
+
}
|
|
127
|
+
const d = new Date();
|
|
128
|
+
const mo = String(d.getMonth() + 1).padStart(2, "0");
|
|
129
|
+
const da = String(d.getDate()).padStart(2, "0");
|
|
130
|
+
return `${d.getFullYear()}-${mo}-${da}`;
|
|
131
|
+
}
|
|
132
|
+
function workDigestEnv() {
|
|
133
|
+
const idle = Number.parseInt(process.env["NLM_WORK_IDLE_THRESHOLD_MIN"] ?? "5", 10);
|
|
134
|
+
const deep = Number.parseInt(process.env["NLM_WORK_DEEP_BLOCK_MIN"] ?? "25", 10);
|
|
135
|
+
return {
|
|
136
|
+
idleThresholdMin: Number.isFinite(idle) && idle > 0 ? idle : 5,
|
|
137
|
+
deepBlockMin: Number.isFinite(deep) && deep > 0 ? deep : 25,
|
|
138
|
+
};
|
|
139
|
+
}
|
|
140
|
+
const TREND_INTERVAL_MS = 7 * 24 * 60 * 60 * 1000;
|
|
141
|
+
export function shouldAppendTrend(lastLineTs, now) {
|
|
142
|
+
if (lastLineTs === null)
|
|
143
|
+
return true;
|
|
144
|
+
const parsed = Date.parse(lastLineTs);
|
|
145
|
+
if (!Number.isFinite(parsed))
|
|
146
|
+
return true;
|
|
147
|
+
return now - parsed >= TREND_INTERVAL_MS;
|
|
148
|
+
}
|
|
149
|
+
function loadTopicProvider() {
|
|
150
|
+
try {
|
|
151
|
+
const raw = readFileSync(join(homedir(), ".nlm", "work-topics.json"), "utf8");
|
|
152
|
+
const map = JSON.parse(raw);
|
|
153
|
+
return aliasTopicProvider(map);
|
|
154
|
+
}
|
|
155
|
+
catch {
|
|
156
|
+
return defaultTopicProvider;
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
/** Build the recall/document embedder. Delegates to the shared factory in
|
|
160
|
+
* src/llm/build-embedder.ts — see that file for provider routing details. */
|
|
161
|
+
function buildEmbedder() {
|
|
162
|
+
return _buildEmbedder();
|
|
163
|
+
}
|
|
164
|
+
/** Build the code-lane embedder. Follows the same destination as the prose
|
|
165
|
+
* embedder (NLM_EMBED_PROVIDER / NLM_EMBED_BASE_URL) but with its own model
|
|
166
|
+
* (NLM_CODE_EMBED_MODEL, e.g. text-embedding-coderankembed on LM Studio).
|
|
167
|
+
* Switching providers on existing exemplars requires `nlm embed-backfill
|
|
168
|
+
* --exemplars`. */
|
|
169
|
+
function buildCodeEmbedder() {
|
|
170
|
+
const provider = (process.env["NLM_EMBED_PROVIDER"] ?? "ollama").toLowerCase();
|
|
171
|
+
if (provider === "openai") {
|
|
110
172
|
autoloadEnv();
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
173
|
+
const baseUrl = process.env["NLM_EMBED_BASE_URL"];
|
|
174
|
+
if (!baseUrl) {
|
|
175
|
+
throw new Error("NLM_EMBED_PROVIDER=openai requires NLM_EMBED_BASE_URL for the code embedder too, " +
|
|
176
|
+
"e.g. http://localhost:1234/v1 for LM Studio.");
|
|
177
|
+
}
|
|
178
|
+
return new OpenAICodeEmbedderClient({
|
|
179
|
+
baseUrl,
|
|
180
|
+
...(process.env["NLM_CODE_EMBED_MODEL"] ? { model: process.env["NLM_CODE_EMBED_MODEL"] } : {}),
|
|
181
|
+
...(process.env["NLM_EMBED_API_KEY"] ? { apiKey: process.env["NLM_EMBED_API_KEY"] } : {}),
|
|
182
|
+
});
|
|
183
|
+
}
|
|
184
|
+
// The bundled provider has no code-embedding engine (CodeRankEmbed has no ONNX equivalent
|
|
185
|
+
// in slice 1). Fall through to the Ollama default: the startup probe will fail soft and
|
|
186
|
+
// the code-exemplar lane stays unknown, which is the intended degradation.
|
|
187
|
+
return new OllamaCodeEmbedder({ baseUrl: ollamaUrl() });
|
|
188
|
+
}
|
|
189
|
+
async function reconcileEmbeddingLanes(storage, embedder, codeEmbedder) {
|
|
190
|
+
const configStore = storage.embeddingConfig;
|
|
191
|
+
if (!configStore)
|
|
192
|
+
return;
|
|
193
|
+
const nowIso = new Date().toISOString();
|
|
194
|
+
try {
|
|
195
|
+
const result = await embedder.embed("nlm dimension probe", "query");
|
|
196
|
+
const dim = result.vector.length;
|
|
197
|
+
const info = resolveEmbedderInfo(process.env, dim);
|
|
198
|
+
const runtime = { lane: "prose", provider: info.provider, model: info.model, dim };
|
|
199
|
+
const rec = reconcileLane(configStore, runtime, nowIso);
|
|
200
|
+
if (rec.state === "stale") {
|
|
201
|
+
const stored = rec.stored;
|
|
202
|
+
console.error(`nlm-memory: prose embedding lane stale: stored ${stored.provider}/${stored.model}@${stored.dim}` +
|
|
203
|
+
` vs runtime ${info.provider}/${info.model}@${dim}; run \`nlm embed-backfill\` to re-embed`);
|
|
204
|
+
setLaneHealth("prose", "stale");
|
|
205
|
+
}
|
|
206
|
+
else {
|
|
207
|
+
setLaneHealth("prose", "ok");
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
catch {
|
|
211
|
+
// probe failure; prose lane stays unknown
|
|
212
|
+
}
|
|
213
|
+
if (process.env["NLM_CODE_EXEMPLARS_ENABLED"] !== "1")
|
|
214
|
+
return;
|
|
215
|
+
try {
|
|
216
|
+
const result = await codeEmbedder.embed("nlm dimension probe", "query");
|
|
217
|
+
const dim = result.dim;
|
|
218
|
+
const provider = (process.env["NLM_EMBED_PROVIDER"] ?? "ollama").toLowerCase();
|
|
219
|
+
const model = process.env["NLM_CODE_EMBED_MODEL"] ?? DEFAULT_CODE_EMBED_MODEL;
|
|
220
|
+
const runtime = { lane: "code", provider, model, dim };
|
|
221
|
+
const rec = reconcileLane(configStore, runtime, nowIso);
|
|
222
|
+
if (rec.state === "stale") {
|
|
223
|
+
const stored = rec.stored;
|
|
224
|
+
console.error(`nlm-memory: code embedding lane stale: stored ${stored.provider}/${stored.model}@${stored.dim}` +
|
|
225
|
+
` vs runtime ${provider}/${model}@${dim}; run \`nlm embed-backfill --exemplars\` to re-embed`);
|
|
226
|
+
setLaneHealth("code", "stale");
|
|
227
|
+
}
|
|
228
|
+
else {
|
|
229
|
+
setLaneHealth("code", "ok");
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
catch {
|
|
233
|
+
// probe failure; code lane stays unknown
|
|
234
|
+
}
|
|
114
235
|
}
|
|
115
236
|
async function buildAdapters(sources) {
|
|
116
237
|
// Sources table is the source of truth. Each enabled row maps to one
|
|
@@ -172,21 +293,56 @@ async function buildStack() {
|
|
|
172
293
|
// seedDefaults (not on ProviderRegistryPort), so this narrows to the SQLite case.
|
|
173
294
|
if (providers instanceof ProviderRegistry)
|
|
174
295
|
await providers.seedDefaults();
|
|
175
|
-
// Recall only uses embed().
|
|
176
|
-
//
|
|
177
|
-
|
|
296
|
+
// Recall only uses embed(). Default embedder is local Ollama; NLM_EMBED_*
|
|
297
|
+
// can point it at any OpenAI-compatible endpoint. Classifier is wired
|
|
298
|
+
// separately for Phase D ingest.
|
|
299
|
+
const embedder = buildEmbedder();
|
|
178
300
|
const classifier = buildClassifier();
|
|
301
|
+
const wsStore = storage.workstreams;
|
|
179
302
|
const recall = new RecallService({
|
|
180
303
|
store,
|
|
181
304
|
llm: embedder,
|
|
182
305
|
factStore: facts,
|
|
183
306
|
exemplarStore: storage.exemplars,
|
|
184
|
-
codeEmbedder:
|
|
307
|
+
codeEmbedder: buildCodeEmbedder(),
|
|
185
308
|
installScope: scope,
|
|
309
|
+
resolveWorkstreamMembers: async (idOrLabel) => {
|
|
310
|
+
const all = await wsStore.listAll();
|
|
311
|
+
const byId = new Map(all.map((w) => [w.id, { id: w.id, mergedInto: w.mergedInto }]));
|
|
312
|
+
const target = all.find((w) => w.id === idOrLabel)
|
|
313
|
+
?? all.find((w) => normalizeLabel(w.label) === normalizeLabel(idOrLabel));
|
|
314
|
+
if (!target)
|
|
315
|
+
return [];
|
|
316
|
+
const survivor = resolveWorkstreamId(target.id, byId);
|
|
317
|
+
return all.filter((w) => resolveWorkstreamId(w.id, byId) === survivor).map((w) => w.id);
|
|
318
|
+
},
|
|
186
319
|
});
|
|
187
320
|
const factRecall = new FactRecallService({ factStore: facts, llm: embedder });
|
|
188
321
|
return { storage, store, facts, signals, scope, sources, providers, recall, factRecall, embedder, classifier };
|
|
189
322
|
}
|
|
323
|
+
export async function runClassifierEvalCommand(deps) {
|
|
324
|
+
const { runClassifierFixtureEval } = await import("../core/eval/classifier-fixture-eval.js");
|
|
325
|
+
const result = await runClassifierFixtureEval(deps.classify, deps.fixturesDir, deps.limit !== undefined ? { limit: deps.limit } : undefined);
|
|
326
|
+
if (deps.json) {
|
|
327
|
+
deps.stdout(JSON.stringify({ ...result, provider: deps.provider, model: deps.model }, null, 2) + "\n");
|
|
328
|
+
return;
|
|
329
|
+
}
|
|
330
|
+
const a = result.aggregate;
|
|
331
|
+
const pct = (n) => `${(n * 100).toFixed(1)}%`;
|
|
332
|
+
deps.stdout(`lane: ${deps.provider}/${deps.model}\n`);
|
|
333
|
+
deps.stdout(`n=${result.perTranscript.length}\n`);
|
|
334
|
+
deps.stdout(`schema-valid: ${pct(a.schemaValidRate)}\n`);
|
|
335
|
+
deps.stdout(`label-accuracy: ${pct(a.labelAccuracy)}\n`);
|
|
336
|
+
deps.stdout(`entity-F1: ${pct(a.entityF1)}\n`);
|
|
337
|
+
deps.stdout(`decision-F1: ${pct(a.decisionF1)}\n`);
|
|
338
|
+
deps.stdout(`conf-calibration: ${pct(a.confidenceCalibrationRate)}\n`);
|
|
339
|
+
deps.stdout(`p50-latency: ${a.p50LatencyMs}ms\n`);
|
|
340
|
+
deps.stdout(`p95-latency: ${a.p95LatencyMs}ms\n`);
|
|
341
|
+
if (a.timedOutOrUnreachable > 0) {
|
|
342
|
+
deps.stdout(`timed-out: ${a.timedOutOrUnreachable} fixture(s) scored as failures (LLM unreachable/timeout)\n`);
|
|
343
|
+
}
|
|
344
|
+
deps.stdout(`see: docs/classifier-tiers.md\n`);
|
|
345
|
+
}
|
|
190
346
|
const program = new Command();
|
|
191
347
|
program
|
|
192
348
|
.name("nlm")
|
|
@@ -208,6 +364,7 @@ program
|
|
|
208
364
|
const { storage, store, facts, signals, scope, sources, providers, recall, factRecall, embedder, classifier } = await buildStack();
|
|
209
365
|
const { existsSync } = await import("node:fs");
|
|
210
366
|
const hasMcpToken = Boolean(process.env["NLM_MCP_TOKEN"]);
|
|
367
|
+
const codeEmb = buildCodeEmbedder();
|
|
211
368
|
const app = createApp({
|
|
212
369
|
recall,
|
|
213
370
|
store,
|
|
@@ -223,6 +380,7 @@ program
|
|
|
223
380
|
embedder,
|
|
224
381
|
store,
|
|
225
382
|
...(facts ? { factStore: facts } : {}),
|
|
383
|
+
classifierDescriptor: { provider: classifier.provider, model: classifier.model },
|
|
226
384
|
},
|
|
227
385
|
signalStore: signals,
|
|
228
386
|
installScope: scope,
|
|
@@ -230,8 +388,8 @@ program
|
|
|
230
388
|
// NLM_CODE_EXEMPLARS_ENABLED flag gates capture (POST /api/signal) and
|
|
231
389
|
// the /api/exemplar + /api/recall-code routes at request time.
|
|
232
390
|
exemplarStore: storage.exemplars,
|
|
233
|
-
codeEmbedder:
|
|
234
|
-
embedderInfo:
|
|
391
|
+
codeEmbedder: codeEmb,
|
|
392
|
+
embedderInfo: resolveEmbedderInfo(),
|
|
235
393
|
...(existsSync(UI_DIST) ? { uiDist: UI_DIST } : {}),
|
|
236
394
|
// Wire POST /mcp only when NLM_MCP_TOKEN is present. Absent = route never
|
|
237
395
|
// mounts, zero attack surface. Present = token-gated Streamable-HTTP MCP
|
|
@@ -246,12 +404,26 @@ program
|
|
|
246
404
|
// Parity with the stdio `nlm mcp` server: remote/container MCP
|
|
247
405
|
// clients hitting POST /mcp get recall_code too when the flag is on.
|
|
248
406
|
exemplarStore: storage.exemplars,
|
|
249
|
-
codeEmbedder:
|
|
407
|
+
codeEmbedder: codeEmb,
|
|
250
408
|
installScope: scope,
|
|
409
|
+
workDigest: { store, topicProvider: loadTopicProvider(), workstreams: storage.workstreams, ...workDigestEnv() },
|
|
410
|
+
workstreams: { store: storage.workstreams, sessions: store, facts: facts, exemplars: storage.exemplars },
|
|
251
411
|
},
|
|
252
412
|
}
|
|
253
413
|
: {}),
|
|
254
414
|
});
|
|
415
|
+
warmCodeEmbedder(codeEmb);
|
|
416
|
+
void reconcileEmbeddingLanes(storage, embedder, codeEmb).catch(() => { });
|
|
417
|
+
void embedder
|
|
418
|
+
.embed("warmup init", "query")
|
|
419
|
+
.then(() => markWarm("textEmbedder"))
|
|
420
|
+
.catch(() => { });
|
|
421
|
+
setImmediate(() => {
|
|
422
|
+
void recall
|
|
423
|
+
.search({ query: "warmup init", mode: "keyword", limit: 1 })
|
|
424
|
+
.then(() => markWarm("fts5"))
|
|
425
|
+
.catch(() => { });
|
|
426
|
+
});
|
|
255
427
|
const p = port();
|
|
256
428
|
serve({ fetch: app.fetch, port: p, hostname: "127.0.0.1" }, (info) => {
|
|
257
429
|
console.error(`nlm-memory http listening on http://localhost:${info.port}`);
|
|
@@ -260,6 +432,14 @@ program
|
|
|
260
432
|
}
|
|
261
433
|
console.error(` db: ${dbPath()}`);
|
|
262
434
|
console.error(` ollama: ${ollamaUrl()}`);
|
|
435
|
+
const classifier = (process.env["NLM_CLASSIFIER"] ?? "ollama").toLowerCase();
|
|
436
|
+
const egress = classifierEgressNotice(classifier, process.env["NLM_CLASSIFIER_BASE_URL"]);
|
|
437
|
+
const classifyTarget = classifier === "openai" && process.env["NLM_CLASSIFIER_BASE_URL"]
|
|
438
|
+
? `${classifier} (${process.env["NLM_CLASSIFIER_BASE_URL"]})`
|
|
439
|
+
: classifier;
|
|
440
|
+
console.error(` classify: ${classifyTarget} [${egress ? "cloud egress" : "local"}]`);
|
|
441
|
+
if (egress)
|
|
442
|
+
console.error(` notice: ${egress}`);
|
|
263
443
|
// Passive update notice. Fire-and-forget so a slow npm registry
|
|
264
444
|
// round-trip can't delay the startup banner; surfaced only when
|
|
265
445
|
// strictly behind. See src/core/update-check/check.ts for the
|
|
@@ -306,6 +486,61 @@ program
|
|
|
306
486
|
void signals.pruneOlderThan(cutoff).catch(() => { });
|
|
307
487
|
}, SIGNAL_PRUNE_INTERVAL_MS);
|
|
308
488
|
signalPruneTimer.unref();
|
|
489
|
+
// Corpus monitor: 24h stats job + weekly re-derivation trend. SQLite only.
|
|
490
|
+
// First run is delayed ~60s so warmup finishes first; then repeats every 24h.
|
|
491
|
+
if (!(storage instanceof PgStorage)) {
|
|
492
|
+
const CORPUS_MONITOR_INTERVAL_MS = 24 * 60 * 60 * 1000;
|
|
493
|
+
const CORPUS_MONITOR_INITIAL_DELAY_MS = 60 * 1000;
|
|
494
|
+
const nlmDataDir = join(homedir(), ".nlm");
|
|
495
|
+
const corpusStatsPath = join(nlmDataDir, "corpus-stats.jsonl");
|
|
496
|
+
const rederivTrendPath = join(nlmDataDir, "re-derivation-trend.jsonl");
|
|
497
|
+
const runCorpusMonitor = async () => {
|
|
498
|
+
try {
|
|
499
|
+
const sqliteStorage = storage;
|
|
500
|
+
const rawDb = sqliteStorage.rawDb();
|
|
501
|
+
const deps = sqliteCorpusStatsDeps(rawDb, dbPath());
|
|
502
|
+
const stats = await computeCorpusStats(deps);
|
|
503
|
+
const thresholds = parseCorpusThresholds(process.env);
|
|
504
|
+
const state = thresholdState(stats.dbBytes, thresholds);
|
|
505
|
+
const ts = new Date().toISOString();
|
|
506
|
+
// hapaxEntities derives from the denormalized entities.session_count column,
|
|
507
|
+
// which drifts (~6% today) until the entity-merge primitive recomputes counts
|
|
508
|
+
// exactly. This is a monitoring approximation, not ground truth.
|
|
509
|
+
const snap = { ...stats, state, lastComputedAt: ts };
|
|
510
|
+
setCorpusSnapshot(snap);
|
|
511
|
+
mkdirSync(nlmDataDir, { recursive: true });
|
|
512
|
+
appendFileSync(corpusStatsPath, JSON.stringify({ ts, ...stats, state }) + "\n", "utf8");
|
|
513
|
+
if (state === "warn" || state === "alert") {
|
|
514
|
+
console.error(`[corpus-monitor] ${state}: db is ${stats.dbBytes} bytes`);
|
|
515
|
+
}
|
|
516
|
+
const now = Date.now();
|
|
517
|
+
let lastTrendTs = null;
|
|
518
|
+
try {
|
|
519
|
+
const trendContent = readFileSync(rederivTrendPath, "utf8");
|
|
520
|
+
const lines = trendContent.split("\n").filter(Boolean);
|
|
521
|
+
const lastLine = lines[lines.length - 1];
|
|
522
|
+
if (lastLine) {
|
|
523
|
+
const parsed = JSON.parse(lastLine);
|
|
524
|
+
lastTrendTs = typeof parsed.ts === "string" ? parsed.ts : null;
|
|
525
|
+
}
|
|
526
|
+
}
|
|
527
|
+
catch {
|
|
528
|
+
// file absent or unreadable; treat as no prior entry
|
|
529
|
+
}
|
|
530
|
+
if (shouldAppendTrend(lastTrendTs, now)) {
|
|
531
|
+
const rdDeps = sqliteReDerivationDeps(rawDb);
|
|
532
|
+
const report = await computeReDerivationRate(rdDeps, 42);
|
|
533
|
+
appendFileSync(rederivTrendPath, JSON.stringify({ ts, windowDays: 42, rate: report.rate, pairCount: report.pairs.length, eligible: report.eligible }) + "\n", "utf8");
|
|
534
|
+
}
|
|
535
|
+
}
|
|
536
|
+
catch (e) {
|
|
537
|
+
console.error(`[corpus-monitor] error: ${e instanceof Error ? e.message : String(e)}`);
|
|
538
|
+
}
|
|
539
|
+
};
|
|
540
|
+
const corpusMonitorTimer = setInterval(() => { void runCorpusMonitor(); }, CORPUS_MONITOR_INTERVAL_MS);
|
|
541
|
+
corpusMonitorTimer.unref();
|
|
542
|
+
setTimeout(() => { void runCorpusMonitor(); }, CORPUS_MONITOR_INITIAL_DELAY_MS).unref();
|
|
543
|
+
}
|
|
309
544
|
// Memo sweep runs independently of the transcript scheduler — it's the
|
|
310
545
|
// backstop for SessionEnd hook unreliability (crashes, kill -9, IDE
|
|
311
546
|
// force-close don't fire SessionEnd, so memo files would otherwise
|
|
@@ -329,7 +564,9 @@ program
|
|
|
329
564
|
installScope: scope,
|
|
330
565
|
intervalMs: opts.intervalMin * 60_000,
|
|
331
566
|
exemplarStore: storage.exemplars,
|
|
332
|
-
codeEmbedder:
|
|
567
|
+
codeEmbedder: buildCodeEmbedder(),
|
|
568
|
+
workstreams: storage.workstreams,
|
|
569
|
+
classifierDescriptor: { provider: classifier.provider, model: classifier.model },
|
|
333
570
|
});
|
|
334
571
|
scheduler.start();
|
|
335
572
|
console.error(` scheduler: ${adapters.map((a) => a.name).join(", ")} every ${opts.intervalMin}m`);
|
|
@@ -368,6 +605,7 @@ program
|
|
|
368
605
|
.option("-k, --kind <kind>", "filter by marker kind (decision|open)")
|
|
369
606
|
.option("-m, --mode <mode>", "recall mode: keyword, semantic, or hybrid (default: keyword)", "keyword")
|
|
370
607
|
.option("-l, --limit <n>", "max results", (v) => Number.parseInt(v, 10), 10)
|
|
608
|
+
.option("-w, --workstream <idOrLabel>", "filter by workstream (id or label; merge chains resolve)")
|
|
371
609
|
.option("--json", "emit the raw JSON result instead of rendered lines")
|
|
372
610
|
.action(async (query, opts) => {
|
|
373
611
|
const { storage, recall } = await buildStack();
|
|
@@ -381,6 +619,7 @@ program
|
|
|
381
619
|
includeSuperseded: true,
|
|
382
620
|
...(opts.entity ? { entity: opts.entity } : {}),
|
|
383
621
|
...(opts.kind ? { kind: opts.kind } : {}),
|
|
622
|
+
...(opts.workstream ? { workstream: opts.workstream } : {}),
|
|
384
623
|
});
|
|
385
624
|
if (opts.json) {
|
|
386
625
|
process.stdout.write(JSON.stringify(result, null, 2) + "\n");
|
|
@@ -398,6 +637,149 @@ program
|
|
|
398
637
|
await storage.close();
|
|
399
638
|
}
|
|
400
639
|
});
|
|
640
|
+
program
|
|
641
|
+
.command("recall-workstream")
|
|
642
|
+
.description("Recall a workstream's accumulated context (id or label)")
|
|
643
|
+
.argument("<idOrLabel>", "workstream id or label")
|
|
644
|
+
.action(async (idOrLabel) => {
|
|
645
|
+
const { storage, store } = await buildStack();
|
|
646
|
+
try {
|
|
647
|
+
const r = await recallWorkstreamHandler({ recall: {}, store, workstreams: { store: storage.workstreams, sessions: store, facts: storage.facts, exemplars: storage.exemplars } }, { idOrLabel });
|
|
648
|
+
process.stdout.write(r.content[0].text + "\n");
|
|
649
|
+
}
|
|
650
|
+
finally {
|
|
651
|
+
await storage.close();
|
|
652
|
+
}
|
|
653
|
+
});
|
|
654
|
+
program
|
|
655
|
+
.command("rebind-session")
|
|
656
|
+
.description("Rebind a session to a workstream (operator correction)")
|
|
657
|
+
.argument("<sessionId>", "session id")
|
|
658
|
+
.argument("<workstream>", "target workstream id or label")
|
|
659
|
+
.action(async (sessionId, workstream) => {
|
|
660
|
+
const { storage, store } = await buildStack();
|
|
661
|
+
try {
|
|
662
|
+
const r = await rebindSessionHandler({ recall: {}, store, workstreams: { store: storage.workstreams, sessions: store, facts: storage.facts, exemplars: storage.exemplars } }, { sessionId, workstream });
|
|
663
|
+
process.stdout.write(r.content[0].text + "\n");
|
|
664
|
+
}
|
|
665
|
+
finally {
|
|
666
|
+
await storage.close();
|
|
667
|
+
}
|
|
668
|
+
});
|
|
669
|
+
program
|
|
670
|
+
.command("merge-workstreams")
|
|
671
|
+
.description("Merge a duplicate workstream into the one to keep")
|
|
672
|
+
.argument("<from>", "duplicate workstream id or label")
|
|
673
|
+
.argument("<into>", "survivor workstream id or label")
|
|
674
|
+
.action(async (from, into) => {
|
|
675
|
+
const { storage, store } = await buildStack();
|
|
676
|
+
try {
|
|
677
|
+
const r = await mergeWorkstreamsHandler({ recall: {}, store, workstreams: { store: storage.workstreams, sessions: store, facts: storage.facts, exemplars: storage.exemplars } }, { from, into });
|
|
678
|
+
process.stdout.write(r.content[0].text + "\n");
|
|
679
|
+
}
|
|
680
|
+
finally {
|
|
681
|
+
await storage.close();
|
|
682
|
+
}
|
|
683
|
+
});
|
|
684
|
+
program
|
|
685
|
+
.command("rename-workstream")
|
|
686
|
+
.description("Rename a workstream")
|
|
687
|
+
.argument("<idOrLabel>", "workstream id or current label")
|
|
688
|
+
.argument("<label>", "new label")
|
|
689
|
+
.action(async (idOrLabel, label) => {
|
|
690
|
+
const { storage, store } = await buildStack();
|
|
691
|
+
try {
|
|
692
|
+
const r = await renameWorkstreamHandler({ recall: {}, store, workstreams: { store: storage.workstreams, sessions: store, facts: storage.facts, exemplars: storage.exemplars } }, { idOrLabel, label });
|
|
693
|
+
process.stdout.write(r.content[0].text + "\n");
|
|
694
|
+
}
|
|
695
|
+
finally {
|
|
696
|
+
await storage.close();
|
|
697
|
+
}
|
|
698
|
+
});
|
|
699
|
+
program
|
|
700
|
+
.command("retire-workstream")
|
|
701
|
+
.description("Retire (mark dead) a workstream")
|
|
702
|
+
.argument("<idOrLabel>", "workstream id or label")
|
|
703
|
+
.action(async (idOrLabel) => {
|
|
704
|
+
const { storage, store } = await buildStack();
|
|
705
|
+
try {
|
|
706
|
+
const r = await retireWorkstreamHandler({ recall: {}, store, workstreams: { store: storage.workstreams, sessions: store, facts: storage.facts, exemplars: storage.exemplars } }, { idOrLabel });
|
|
707
|
+
process.stdout.write(r.content[0].text + "\n");
|
|
708
|
+
}
|
|
709
|
+
finally {
|
|
710
|
+
await storage.close();
|
|
711
|
+
}
|
|
712
|
+
});
|
|
713
|
+
program
|
|
714
|
+
.command("merge-suggestions")
|
|
715
|
+
.description("List likely-duplicate workstreams to merge")
|
|
716
|
+
.option("-m, --min-score <n>", "minimum similarity score 0..1", "0.5")
|
|
717
|
+
.action(async (opts) => {
|
|
718
|
+
const minScore = Number(opts.minScore);
|
|
719
|
+
if (!Number.isFinite(minScore)) {
|
|
720
|
+
process.stderr.write(`nlm: invalid --min-score "${opts.minScore}" (expected a number 0..1)\n`);
|
|
721
|
+
process.exitCode = 1;
|
|
722
|
+
return;
|
|
723
|
+
}
|
|
724
|
+
const { storage, store } = await buildStack();
|
|
725
|
+
try {
|
|
726
|
+
const r = await listMergeSuggestionsHandler({ recall: {}, store, workstreams: { store: storage.workstreams, sessions: store, facts: storage.facts, exemplars: storage.exemplars } }, { minScore });
|
|
727
|
+
process.stdout.write(r.content[0].text + "\n");
|
|
728
|
+
}
|
|
729
|
+
finally {
|
|
730
|
+
await storage.close();
|
|
731
|
+
}
|
|
732
|
+
});
|
|
733
|
+
program
|
|
734
|
+
.command("eval")
|
|
735
|
+
.description("Evaluate recall quality (--queries) or classifier extraction quality (--classifier)")
|
|
736
|
+
.option("--queries <file>", "JSON file: [{ query, expectedIds }]")
|
|
737
|
+
.option("--classifier", "benchmark the configured classifier lane against shipped gold fixtures")
|
|
738
|
+
.option("--limit <n>", "run only the first N fixtures (classifier mode)", (v) => Number.parseInt(v, 10))
|
|
739
|
+
.option("--mode <mode>", "keyword | semantic | hybrid (queries mode)", "keyword")
|
|
740
|
+
.option("--json", "emit JSON instead of a table")
|
|
741
|
+
.action(async (opts) => {
|
|
742
|
+
if (opts.classifier) {
|
|
743
|
+
const stack = await buildStack();
|
|
744
|
+
try {
|
|
745
|
+
const limitVal = opts.limit;
|
|
746
|
+
await runClassifierEvalCommand({
|
|
747
|
+
classify: (t) => stack.classifier.classify(t),
|
|
748
|
+
provider: stack.classifier.provider,
|
|
749
|
+
model: stack.classifier.model,
|
|
750
|
+
fixturesDir: CLASSIFIER_FIXTURES_DIR,
|
|
751
|
+
...(limitVal !== undefined ? { limit: limitVal } : {}),
|
|
752
|
+
json: Boolean(opts.json),
|
|
753
|
+
stdout: (s) => { process.stdout.write(s); },
|
|
754
|
+
});
|
|
755
|
+
}
|
|
756
|
+
finally {
|
|
757
|
+
await stack.storage.close();
|
|
758
|
+
}
|
|
759
|
+
return;
|
|
760
|
+
}
|
|
761
|
+
if (!opts.queries) {
|
|
762
|
+
process.stderr.write("error: required option '--queries <file>' not specified\n");
|
|
763
|
+
process.exitCode = 1;
|
|
764
|
+
return;
|
|
765
|
+
}
|
|
766
|
+
const { readFile } = await import("node:fs/promises");
|
|
767
|
+
const queries = JSON.parse(await readFile(opts.queries, "utf8"));
|
|
768
|
+
const { runEval } = await import("../core/eval/run-eval.js");
|
|
769
|
+
const { storage, recall } = await buildStack();
|
|
770
|
+
try {
|
|
771
|
+
const report = await runEval({ recall }, queries, { mode: opts.mode, k: 5 });
|
|
772
|
+
if (opts.json) {
|
|
773
|
+
process.stdout.write(JSON.stringify(report, null, 2) + "\n");
|
|
774
|
+
return;
|
|
775
|
+
}
|
|
776
|
+
process.stdout.write(`mode=${report.mode} n=${report.n} R@1=${(report.rAt1 * 100).toFixed(1)}% ` +
|
|
777
|
+
`R@5=${(report.rAt5 * 100).toFixed(1)}% MRR=${report.mrr.toFixed(3)}\n`);
|
|
778
|
+
}
|
|
779
|
+
finally {
|
|
780
|
+
await storage.close();
|
|
781
|
+
}
|
|
782
|
+
});
|
|
401
783
|
program
|
|
402
784
|
.command("recall-code")
|
|
403
785
|
.description("Semantic search over code exemplars (requires NLM_CODE_EXEMPLARS_ENABLED=1)")
|
|
@@ -413,11 +795,10 @@ program
|
|
|
413
795
|
process.exit(1);
|
|
414
796
|
}
|
|
415
797
|
const { storage } = await buildStack();
|
|
416
|
-
const { OllamaCodeEmbedder } = await import("../llm/ollama-code-embedder.js");
|
|
417
798
|
const { recallCode } = await import("../core/exemplars/recall-code.js");
|
|
418
799
|
const { installScope: getScope } = await import("../core/signals/install-scope.js");
|
|
419
800
|
try {
|
|
420
|
-
const embedder =
|
|
801
|
+
const embedder = buildCodeEmbedder();
|
|
421
802
|
const result = await recallCode({
|
|
422
803
|
query,
|
|
423
804
|
installScope: getScope(),
|
|
@@ -450,6 +831,48 @@ program
|
|
|
450
831
|
await storage.close();
|
|
451
832
|
}
|
|
452
833
|
});
|
|
834
|
+
program
|
|
835
|
+
.command("code-signal")
|
|
836
|
+
.description("Emit a PATH-(b) code signal for a commit (deterministic pass/fail exemplar). Best-effort: never blocks a commit.")
|
|
837
|
+
.requiredOption("--repo-path <dir>", "path to the git repo whose commit to capture")
|
|
838
|
+
.requiredOption("--sha <sha>", "commit sha to extract the diff from")
|
|
839
|
+
.requiredOption("--test-exit <n>", "test gate exit code (0 -> pass, non-zero -> fail)", (v) => Number.parseInt(v, 10))
|
|
840
|
+
.option("--task <s>", "task description (defaults to the changed funcname/file)")
|
|
841
|
+
.option("--model <s>", "model that produced the change")
|
|
842
|
+
.option("--repo <logical>", "logical repo name (defaults to the repo-path basename)")
|
|
843
|
+
.option("--dry-run", "print the payload and do not POST")
|
|
844
|
+
.action(async (opts) => {
|
|
845
|
+
const { buildCodeSignalPayload, formatCodeSignalResult } = await import("../core/signals/code-signal.js");
|
|
846
|
+
const payload = buildCodeSignalPayload({
|
|
847
|
+
repoPath: opts.repoPath,
|
|
848
|
+
sha: opts.sha,
|
|
849
|
+
testExit: opts.testExit,
|
|
850
|
+
...(opts.task ? { task: opts.task } : {}),
|
|
851
|
+
...(opts.model ? { model: opts.model } : {}),
|
|
852
|
+
...(opts.repo ? { repo: opts.repo } : {}),
|
|
853
|
+
});
|
|
854
|
+
if (opts.dryRun) {
|
|
855
|
+
process.stdout.write(JSON.stringify(payload, null, 2) + "\n");
|
|
856
|
+
return;
|
|
857
|
+
}
|
|
858
|
+
const url = `http://localhost:${port()}/api/signal`;
|
|
859
|
+
try {
|
|
860
|
+
const res = await fetch(url, {
|
|
861
|
+
method: "POST",
|
|
862
|
+
headers: { "content-type": "application/json" },
|
|
863
|
+
body: JSON.stringify(payload),
|
|
864
|
+
});
|
|
865
|
+
if (res.status !== 202) {
|
|
866
|
+
process.stderr.write(`code-signal: daemon returned ${res.status} (not 202); skipping\n`);
|
|
867
|
+
return;
|
|
868
|
+
}
|
|
869
|
+
const accepted = (await res.json());
|
|
870
|
+
process.stdout.write(formatCodeSignalResult(payload.outcome, accepted.id ?? "unknown") + "\n");
|
|
871
|
+
}
|
|
872
|
+
catch {
|
|
873
|
+
process.stderr.write(`code-signal: daemon unreachable at ${url}; skipping\n`);
|
|
874
|
+
}
|
|
875
|
+
});
|
|
453
876
|
program
|
|
454
877
|
.command("misses")
|
|
455
878
|
.description("Show sessions the agent explicitly fetched but the hook never surfaced (recall miss log)")
|
|
@@ -532,6 +955,43 @@ program
|
|
|
532
955
|
}
|
|
533
956
|
}
|
|
534
957
|
});
|
|
958
|
+
program
|
|
959
|
+
.command("metrics")
|
|
960
|
+
.description("Read-only retrieval-quality metrics")
|
|
961
|
+
.argument("<name>", "metric name: re-derivation")
|
|
962
|
+
.option("--window <days>", "lookback window in days", (v) => Number.parseInt(v, 10), 90)
|
|
963
|
+
.option("--json", "emit JSON instead of human-readable output")
|
|
964
|
+
.action(async (name, opts) => {
|
|
965
|
+
if (name !== "re-derivation") {
|
|
966
|
+
console.error(`unknown metric '${name}' (available: re-derivation)`);
|
|
967
|
+
process.exitCode = 1;
|
|
968
|
+
return;
|
|
969
|
+
}
|
|
970
|
+
const { computeReDerivationRate, sqliteReDerivationDeps } = await import("../core/metrics/re-derivation.js");
|
|
971
|
+
const { storage, store } = await buildStack();
|
|
972
|
+
try {
|
|
973
|
+
if (typeof store.rawDb !== "function") {
|
|
974
|
+
console.error("re-derivation metric requires the SQLite backend");
|
|
975
|
+
process.exitCode = 1;
|
|
976
|
+
return;
|
|
977
|
+
}
|
|
978
|
+
const deps = sqliteReDerivationDeps(store.rawDb());
|
|
979
|
+
const report = await computeReDerivationRate(deps, opts.window);
|
|
980
|
+
if (opts.json) {
|
|
981
|
+
process.stdout.write(JSON.stringify(report, null, 2) + "\n");
|
|
982
|
+
return;
|
|
983
|
+
}
|
|
984
|
+
const pct = (report.rate * 100).toFixed(1);
|
|
985
|
+
console.log(`re_derivation_rate (last ${opts.window} day(s)): ${pct}% ` +
|
|
986
|
+
`(${report.pairs.length} re-derived pair${report.pairs.length === 1 ? "" : "s"})`);
|
|
987
|
+
for (const p of report.pairs) {
|
|
988
|
+
console.log(` ${p.a} <-> ${p.b} jaccard=${p.jaccard} shared=${p.sharedEntities.join(", ")}`);
|
|
989
|
+
}
|
|
990
|
+
}
|
|
991
|
+
finally {
|
|
992
|
+
await storage.close();
|
|
993
|
+
}
|
|
994
|
+
});
|
|
535
995
|
program
|
|
536
996
|
.command("supersede")
|
|
537
997
|
.description("Retroactively mark a session as superseded by a newer one")
|
|
@@ -572,12 +1032,39 @@ program
|
|
|
572
1032
|
.description("Re-embed every session into session_embedding_chunks (chunk + max-pool)")
|
|
573
1033
|
.option("-l, --limit <n>", "session cap (default: all)", (v) => Number.parseInt(v, 10))
|
|
574
1034
|
.option("--state <path>", "resume state file (default ~/.nlm/embed_reembed.state)")
|
|
1035
|
+
.option("--exemplars", "instead: embed code_exemplars rows missing a vector (repairs dropped capture embeds)")
|
|
1036
|
+
.option("--dry-run", "report dim mismatch and what would be dropped without writing")
|
|
575
1037
|
.option("-v, --verbose", "per-session progress on stderr")
|
|
576
1038
|
.action(async (opts) => {
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
1039
|
+
if (opts.exemplars) {
|
|
1040
|
+
const { storage } = await buildStack();
|
|
1041
|
+
try {
|
|
1042
|
+
const report = await backfillExemplarEmbeddings({
|
|
1043
|
+
dbPath: dbPath(),
|
|
1044
|
+
embedder: buildCodeEmbedder(),
|
|
1045
|
+
store: storage.exemplars,
|
|
1046
|
+
...(opts.limit ? { limit: opts.limit } : {}),
|
|
1047
|
+
...(opts.verbose
|
|
1048
|
+
? {
|
|
1049
|
+
onProgress: (i, n, id, status) => {
|
|
1050
|
+
process.stderr.write(` [${i}/${n}] ${id} ${status}\n`);
|
|
1051
|
+
},
|
|
1052
|
+
}
|
|
1053
|
+
: {}),
|
|
1054
|
+
});
|
|
1055
|
+
process.stdout.write(JSON.stringify(report, null, 2) + "\n");
|
|
1056
|
+
}
|
|
1057
|
+
finally {
|
|
1058
|
+
await storage.close();
|
|
1059
|
+
}
|
|
1060
|
+
return;
|
|
1061
|
+
}
|
|
1062
|
+
const embedder = buildEmbedder();
|
|
1063
|
+
const pgUrl = process.env["NLM_PG_URL"];
|
|
1064
|
+
const sharedOpts = {
|
|
580
1065
|
embedder,
|
|
1066
|
+
embedderProvider: resolveEmbedderInfo(process.env).provider,
|
|
1067
|
+
...(opts.dryRun ? { dryRun: true } : {}),
|
|
581
1068
|
...(opts.state ? { statePath: opts.state } : {}),
|
|
582
1069
|
...(opts.limit ? { limit: opts.limit } : {}),
|
|
583
1070
|
...(opts.verbose
|
|
@@ -587,7 +1074,10 @@ program
|
|
|
587
1074
|
},
|
|
588
1075
|
}
|
|
589
1076
|
: {}),
|
|
590
|
-
}
|
|
1077
|
+
};
|
|
1078
|
+
const report = pgUrl
|
|
1079
|
+
? await reembedCorpusPg({ pgUrl, ...sharedOpts })
|
|
1080
|
+
: await reembedCorpus({ dbPath: dbPath(), ...sharedOpts });
|
|
591
1081
|
process.stdout.write(JSON.stringify(report, null, 2) + "\n");
|
|
592
1082
|
});
|
|
593
1083
|
program
|
|
@@ -649,6 +1139,7 @@ program
|
|
|
649
1139
|
factStore: stack.storage.facts,
|
|
650
1140
|
embedder: stack.embedder,
|
|
651
1141
|
classifier: stack.classifier,
|
|
1142
|
+
classifierDescriptor: { provider: stack.classifier.provider, model: stack.classifier.model },
|
|
652
1143
|
adapters,
|
|
653
1144
|
}, { ...(opts.limit ? { limit: opts.limit } : {}), dryRun: Boolean(opts.dryRun) });
|
|
654
1145
|
console.log(`reclassify-oversized: attempted=${r.attempted} ingested=${r.ingested} ` +
|
|
@@ -660,19 +1151,59 @@ program
|
|
|
660
1151
|
}
|
|
661
1152
|
});
|
|
662
1153
|
program
|
|
663
|
-
.command("
|
|
664
|
-
.description("
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
.option("--
|
|
668
|
-
.
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
1154
|
+
.command("reprocess")
|
|
1155
|
+
.description("Retroactive: re-classify sessions that lack classification or were classified by a weaker lane.\n" +
|
|
1156
|
+
"Workstream binding (workstream_id) is NOT changed by this command.\n" +
|
|
1157
|
+
"Below-floor results (<0.4 confidence) update the session row but preserve its prior facts.")
|
|
1158
|
+
.option("-l, --limit <n>", "max sessions to process this run", (v) => Number.parseInt(v, 10))
|
|
1159
|
+
.option("--min-confidence <f>", "also reprocess same-model sessions below this confidence", (v) => Number.parseFloat(v))
|
|
1160
|
+
.option("--only-null", "select only sessions with no classifier stamp yet (safe lane-switch resume: sessions already stamped by a stronger model are left untouched)")
|
|
1161
|
+
.option("--exclude-model <tag>", "exclude sessions stamped by this model tag (repeatable; use once per model to preserve corpus quality when switching classifier lanes)", (v, prev) => [...prev, v], [])
|
|
1162
|
+
.option("--state <path>", "resume state file (default ~/.nlm/reprocess.state)")
|
|
1163
|
+
.option("--dry-run", "print cohort report without writing")
|
|
1164
|
+
.option("-v, --verbose", "per-session progress on stderr")
|
|
1165
|
+
.option("--force-embed", "proceed even when the stored prose embedding lane mismatches the runtime embedder")
|
|
1166
|
+
.action(async (opts) => {
|
|
1167
|
+
const { reprocess: runReprocess } = await import("../core/ingest/reprocess.js");
|
|
1168
|
+
const stack = await buildStack();
|
|
1169
|
+
try {
|
|
1170
|
+
if (!(stack.storage instanceof SqliteStorage)) {
|
|
1171
|
+
console.error("reprocess: only supported with SQLite storage (NLM_PG_URL must not be set)");
|
|
1172
|
+
await stack.storage.close();
|
|
1173
|
+
process.exit(1);
|
|
1174
|
+
}
|
|
1175
|
+
const embedderInfo = resolveEmbedderInfo();
|
|
1176
|
+
const report = await runReprocess({
|
|
1177
|
+
db: stack.storage.rawDb(),
|
|
1178
|
+
store: stack.storage.sessions,
|
|
1179
|
+
factStore: stack.storage.facts,
|
|
1180
|
+
embedder: stack.embedder,
|
|
1181
|
+
classifier: stack.classifier,
|
|
1182
|
+
classifierDescriptor: { provider: stack.classifier.provider, model: stack.classifier.model },
|
|
1183
|
+
embeddingConfig: stack.storage.embeddingConfig,
|
|
1184
|
+
embedderDescriptor: { provider: embedderInfo.provider, model: embedderInfo.model },
|
|
1185
|
+
}, {
|
|
1186
|
+
...(opts.limit ? { limit: opts.limit } : {}),
|
|
1187
|
+
...(opts.minConfidence !== undefined ? { minConfidence: opts.minConfidence } : {}),
|
|
1188
|
+
...(opts.onlyNull ? { onlyNull: true } : {}),
|
|
1189
|
+
...(opts.excludeModel && opts.excludeModel.length > 0 ? { excludeModels: opts.excludeModel } : {}),
|
|
1190
|
+
...(opts.state ? { statePath: opts.state } : {}),
|
|
1191
|
+
dryRun: Boolean(opts.dryRun),
|
|
1192
|
+
verbose: Boolean(opts.verbose),
|
|
1193
|
+
forceEmbed: Boolean(opts.forceEmbed),
|
|
1194
|
+
...(opts.verbose
|
|
1195
|
+
? {
|
|
1196
|
+
onProgress: (i, n, sid, status) => {
|
|
1197
|
+
process.stderr.write(` [${i}/${n}] ${sid} ${status}\n`);
|
|
1198
|
+
},
|
|
1199
|
+
}
|
|
1200
|
+
: {}),
|
|
1201
|
+
});
|
|
1202
|
+
process.stdout.write(JSON.stringify(report, null, 2) + "\n");
|
|
1203
|
+
}
|
|
1204
|
+
finally {
|
|
1205
|
+
await stack.storage.close();
|
|
1206
|
+
}
|
|
676
1207
|
});
|
|
677
1208
|
program
|
|
678
1209
|
.command("mcp")
|
|
@@ -685,8 +1216,10 @@ program
|
|
|
685
1216
|
factStore: facts,
|
|
686
1217
|
factRecall,
|
|
687
1218
|
exemplarStore: storage.exemplars,
|
|
688
|
-
codeEmbedder:
|
|
1219
|
+
codeEmbedder: buildCodeEmbedder(),
|
|
689
1220
|
installScope: scope,
|
|
1221
|
+
workDigest: { store, topicProvider: loadTopicProvider(), workstreams: storage.workstreams, ...workDigestEnv() },
|
|
1222
|
+
workstreams: { store: storage.workstreams, sessions: store, facts: facts, exemplars: storage.exemplars },
|
|
690
1223
|
});
|
|
691
1224
|
const transport = new StdioServerTransport();
|
|
692
1225
|
await server.connect(transport);
|
|
@@ -1615,6 +2148,21 @@ program
|
|
|
1615
2148
|
process.exit(1);
|
|
1616
2149
|
}
|
|
1617
2150
|
});
|
|
2151
|
+
program
|
|
2152
|
+
.command("work-digest")
|
|
2153
|
+
.description("Print the operator's agent-assisted work recap for a day")
|
|
2154
|
+
.option("-d, --date <date>", "day to summarize, YYYY-MM-DD (default: today)")
|
|
2155
|
+
.action(async (opts) => {
|
|
2156
|
+
const date = resolveDigestDate(opts.date);
|
|
2157
|
+
const { storage, store } = await buildStack();
|
|
2158
|
+
try {
|
|
2159
|
+
const digest = await buildWorkDigest({ store, topicProvider: loadTopicProvider(), workstreams: storage.workstreams, ...workDigestEnv() }, date);
|
|
2160
|
+
console.log(composeWorkDigest(digest));
|
|
2161
|
+
}
|
|
2162
|
+
finally {
|
|
2163
|
+
await storage.close();
|
|
2164
|
+
}
|
|
2165
|
+
});
|
|
1618
2166
|
async function gatherInstallProbe() {
|
|
1619
2167
|
autoloadEnv();
|
|
1620
2168
|
let reachable = false;
|
|
@@ -1678,7 +2226,7 @@ async function gatherInstallProbe() {
|
|
|
1678
2226
|
program
|
|
1679
2227
|
.command("doctor")
|
|
1680
2228
|
.description("Check database integrity invariants and install/runtime health, optionally repair safe violations")
|
|
1681
|
-
.option("--fix", "repair mechanically safe violations: delete self-loop edges (I1), restore orphaned superseded/replaced sessions to closed (I2)")
|
|
2229
|
+
.option("--fix", "repair mechanically safe violations: delete self-loop edges (I1), restore orphaned superseded/replaced sessions to closed (I2), delete ghost fact embeddings (I7)")
|
|
1682
2230
|
.action(async (opts) => {
|
|
1683
2231
|
const storage = await buildStorage(dbPath());
|
|
1684
2232
|
let violations;
|
|
@@ -1697,7 +2245,10 @@ program
|
|
|
1697
2245
|
if (fixReport.restoredToClosed > 0) {
|
|
1698
2246
|
console.log(` fixed I2: restored ${fixReport.restoredToClosed} session(s) to closed`);
|
|
1699
2247
|
}
|
|
1700
|
-
if (fixReport.
|
|
2248
|
+
if (fixReport.deletedGhostEmbeddings > 0) {
|
|
2249
|
+
console.log(` fixed I7: deleted ${fixReport.deletedGhostEmbeddings} ghost embedding(s)`);
|
|
2250
|
+
}
|
|
2251
|
+
if (fixReport.deletedSelfLoops === 0 && fixReport.restoredToClosed === 0 && fixReport.deletedGhostEmbeddings === 0) {
|
|
1701
2252
|
console.log(" --fix: nothing to repair");
|
|
1702
2253
|
}
|
|
1703
2254
|
}
|
|
@@ -1711,7 +2262,7 @@ program
|
|
|
1711
2262
|
finally {
|
|
1712
2263
|
await storage.close();
|
|
1713
2264
|
}
|
|
1714
|
-
const ALL_CHECKS = ["I1", "I2", "I3", "I4", "I5a", "I5b", "I6"];
|
|
2265
|
+
const ALL_CHECKS = ["I1", "I2", "I3", "I4", "I5a", "I5b", "I6", "I7"];
|
|
1715
2266
|
const byId = new Map(violations.map((v) => [v.id, v]));
|
|
1716
2267
|
let anyFail = false;
|
|
1717
2268
|
console.log("Database integrity:");
|
|
@@ -1726,6 +2277,12 @@ program
|
|
|
1726
2277
|
console.log(`PASS ${id}`);
|
|
1727
2278
|
}
|
|
1728
2279
|
}
|
|
2280
|
+
// I5a-mv is informational: multi-valued predicate groups legitimately exempt
|
|
2281
|
+
// from I5a. Not a violation; printed separately so the count is visible.
|
|
2282
|
+
const i5aMv = byId.get("I5a-mv");
|
|
2283
|
+
if (i5aMv) {
|
|
2284
|
+
console.log(`INFO I5a-mv count=${i5aMv.count} ${i5aMv.description}`);
|
|
2285
|
+
}
|
|
1729
2286
|
console.log("\nInstall & runtime health:");
|
|
1730
2287
|
if (printHealthChecks(evaluateInstallHealth(await gatherInstallProbe())))
|
|
1731
2288
|
anyFail = true;
|
|
@@ -1746,9 +2303,11 @@ function printHealthChecks(checks) {
|
|
|
1746
2303
|
async function dbIntegrityCheck() {
|
|
1747
2304
|
const storage = await buildStorage(dbPath());
|
|
1748
2305
|
try {
|
|
1749
|
-
const
|
|
2306
|
+
const all = storage instanceof PgStorage
|
|
1750
2307
|
? await runChecksOnPg(storage.pgPool())
|
|
1751
2308
|
: runChecksOnSqlite(storage.rawDb());
|
|
2309
|
+
// I5a-mv is informational (exempted multi-valued groups), not a true violation.
|
|
2310
|
+
const violations = all.filter((v) => v.id !== "I5a-mv");
|
|
1752
2311
|
if (violations.length === 0)
|
|
1753
2312
|
return { id: "db-integrity", status: "ok", detail: "all invariants hold" };
|
|
1754
2313
|
return {
|
|
@@ -1800,6 +2359,35 @@ program
|
|
|
1800
2359
|
if (anyFail)
|
|
1801
2360
|
process.exit(1);
|
|
1802
2361
|
});
|
|
2362
|
+
program
|
|
2363
|
+
.command("health")
|
|
2364
|
+
.description("Print corpus monitoring snapshot from the running daemon (null before first 24h run)")
|
|
2365
|
+
.action(async () => {
|
|
2366
|
+
let corpus = null;
|
|
2367
|
+
try {
|
|
2368
|
+
const ctrl = new AbortController();
|
|
2369
|
+
const timer = setTimeout(() => ctrl.abort(), 1_500);
|
|
2370
|
+
const res = await fetch(`http://localhost:${port()}/api/health`, { signal: ctrl.signal });
|
|
2371
|
+
clearTimeout(timer);
|
|
2372
|
+
if (res.ok) {
|
|
2373
|
+
const body = (await res.json());
|
|
2374
|
+
corpus = body.corpus ?? null;
|
|
2375
|
+
}
|
|
2376
|
+
}
|
|
2377
|
+
catch {
|
|
2378
|
+
// daemon not reachable
|
|
2379
|
+
}
|
|
2380
|
+
if (corpus) {
|
|
2381
|
+
console.log(`corpus.state: ${corpus.state}`);
|
|
2382
|
+
console.log(`corpus.dbBytes: ${corpus.dbBytes}`);
|
|
2383
|
+
console.log(`corpus.sessions: ${corpus.sessions}`);
|
|
2384
|
+
console.log(`corpus.entities: ${corpus.entities}`);
|
|
2385
|
+
console.log(`corpus.lastComputedAt: ${corpus.lastComputedAt}`);
|
|
2386
|
+
}
|
|
2387
|
+
else {
|
|
2388
|
+
console.log("corpus: (not yet available; daemon runs the first check ~60s after start)");
|
|
2389
|
+
}
|
|
2390
|
+
});
|
|
1803
2391
|
program
|
|
1804
2392
|
.command("backup")
|
|
1805
2393
|
.description("Write a dated snapshot to ~/.nlm/backups/ and prune ones past the retention window")
|
|
@@ -1862,8 +2450,131 @@ program
|
|
|
1862
2450
|
console.log(`Staged ${opts.from} (${validation.sessions} sessions, schema v${validation.schemaVersion}).`);
|
|
1863
2451
|
console.log("Run `nlm restart` to apply — the current DB is archived aside, not deleted.");
|
|
1864
2452
|
});
|
|
1865
|
-
program
|
|
1866
|
-
|
|
1867
|
-
|
|
2453
|
+
program
|
|
2454
|
+
.command("init")
|
|
2455
|
+
.description("Print (or write) the agent recall contract snippet")
|
|
2456
|
+
.requiredOption("--agent <name>", "agent variant: claude-code or generic")
|
|
2457
|
+
.option("--write <path>", "append the contract to this file instead of printing to stdout")
|
|
2458
|
+
.option("--force", "when --write: replace the existing block instead of refusing")
|
|
2459
|
+
.action((opts) => {
|
|
2460
|
+
runInitCommand({
|
|
2461
|
+
agent: opts.agent,
|
|
2462
|
+
write: opts.write,
|
|
2463
|
+
force: Boolean(opts.force),
|
|
2464
|
+
stdout: (s) => { process.stdout.write(s); },
|
|
2465
|
+
stderr: (s) => { process.stderr.write(s); },
|
|
2466
|
+
});
|
|
2467
|
+
});
|
|
2468
|
+
const entitiesCmd = program.command("entities").description("Entity table operations");
|
|
2469
|
+
entitiesCmd
|
|
2470
|
+
.command("dedup")
|
|
2471
|
+
.description("Suggest (or apply) conservative entity merge pairs. Default: dry run.")
|
|
2472
|
+
.option("--apply-safe", "execute safe-class merges via the merge primitive", false)
|
|
2473
|
+
.option("--interactive", "step through likely-class pairs with y/n on stdin", false)
|
|
2474
|
+
.action(async (opts) => {
|
|
2475
|
+
const storage = await buildStorage(dbPath());
|
|
2476
|
+
let rows;
|
|
2477
|
+
try {
|
|
2478
|
+
if (storage instanceof SqliteStorage) {
|
|
2479
|
+
rows = storage
|
|
2480
|
+
.rawDb()
|
|
2481
|
+
.prepare("SELECT canonical, session_count, status FROM entities ORDER BY canonical")
|
|
2482
|
+
.all();
|
|
2483
|
+
}
|
|
2484
|
+
else {
|
|
2485
|
+
const res = await storage
|
|
2486
|
+
.pgPool()
|
|
2487
|
+
.query("SELECT canonical, session_count, status FROM entities ORDER BY canonical");
|
|
2488
|
+
rows = res.rows;
|
|
2489
|
+
}
|
|
2490
|
+
}
|
|
2491
|
+
catch (err) {
|
|
2492
|
+
await storage.close();
|
|
2493
|
+
console.error("nlm entities dedup: failed to enumerate entities:", err);
|
|
2494
|
+
process.exit(1);
|
|
2495
|
+
}
|
|
2496
|
+
const inputs = rows.map((r) => ({
|
|
2497
|
+
canonical: r.canonical,
|
|
2498
|
+
sessionCount: r.session_count,
|
|
2499
|
+
status: r.status,
|
|
2500
|
+
}));
|
|
2501
|
+
const suggestions = suggestMerges(inputs);
|
|
2502
|
+
const safe = suggestions.filter((s) => s.cls === "safe");
|
|
2503
|
+
const likely = suggestions.filter((s) => s.cls === "likely");
|
|
2504
|
+
let merged = 0;
|
|
2505
|
+
let failed = 0;
|
|
2506
|
+
let skipped = 0;
|
|
2507
|
+
if (!opts.applySafe && !opts.interactive) {
|
|
2508
|
+
console.log(`safe (${safe.length}):`);
|
|
2509
|
+
for (const s of safe)
|
|
2510
|
+
console.log(` ${s.source} -> ${s.target}`);
|
|
2511
|
+
console.log(`likely (${likely.length}):`);
|
|
2512
|
+
for (const s of likely)
|
|
2513
|
+
console.log(` ${s.source} -> ${s.target}`);
|
|
2514
|
+
console.log(`\ntotal: safe=${safe.length} likely=${likely.length} (dry run)`);
|
|
2515
|
+
await storage.close();
|
|
2516
|
+
return;
|
|
2517
|
+
}
|
|
2518
|
+
if (opts.applySafe) {
|
|
2519
|
+
for (const s of safe) {
|
|
2520
|
+
try {
|
|
2521
|
+
await storage.entities.merge(s.source, s.target);
|
|
2522
|
+
console.log(`merged: ${s.source} -> ${s.target}`);
|
|
2523
|
+
merged++;
|
|
2524
|
+
}
|
|
2525
|
+
catch (err) {
|
|
2526
|
+
console.error(`failed: ${s.source} -> ${s.target}: ${err}`);
|
|
2527
|
+
failed++;
|
|
2528
|
+
}
|
|
2529
|
+
}
|
|
2530
|
+
}
|
|
2531
|
+
else {
|
|
2532
|
+
for (const s of safe) {
|
|
2533
|
+
console.log(`safe (skipped): ${s.source} -> ${s.target}`);
|
|
2534
|
+
skipped++;
|
|
2535
|
+
}
|
|
2536
|
+
}
|
|
2537
|
+
if (opts.interactive && likely.length > 0) {
|
|
2538
|
+
const { createInterface } = await import("node:readline");
|
|
2539
|
+
const rl = createInterface({ input: process.stdin, output: process.stdout });
|
|
2540
|
+
const ask = (prompt) => new Promise((resolve) => rl.question(prompt, resolve));
|
|
2541
|
+
for (const s of likely) {
|
|
2542
|
+
const answer = await ask(`merge ${s.source} -> ${s.target}? [y/N] `);
|
|
2543
|
+
if (answer.trim().toLowerCase() === "y") {
|
|
2544
|
+
try {
|
|
2545
|
+
await storage.entities.merge(s.source, s.target);
|
|
2546
|
+
console.log(`merged: ${s.source} -> ${s.target}`);
|
|
2547
|
+
merged++;
|
|
2548
|
+
}
|
|
2549
|
+
catch (err) {
|
|
2550
|
+
console.error(`failed: ${s.source} -> ${s.target}: ${err}`);
|
|
2551
|
+
failed++;
|
|
2552
|
+
}
|
|
2553
|
+
}
|
|
2554
|
+
else {
|
|
2555
|
+
skipped++;
|
|
2556
|
+
}
|
|
2557
|
+
}
|
|
2558
|
+
rl.close();
|
|
2559
|
+
}
|
|
2560
|
+
else {
|
|
2561
|
+
skipped += likely.length;
|
|
2562
|
+
}
|
|
2563
|
+
await storage.close();
|
|
2564
|
+
const remaining = suggestions.length - merged - failed;
|
|
2565
|
+
console.log(`\nsummary: merged=${merged} failed=${failed} suggested-remaining=${remaining}`);
|
|
2566
|
+
if (failed > 0)
|
|
2567
|
+
process.exit(1);
|
|
1868
2568
|
});
|
|
2569
|
+
// Entrypoint guard: only parse argv when this file IS the executed script.
|
|
2570
|
+
// realpathSync resolves the npm bin symlink so the global `nlm` shim still
|
|
2571
|
+
// matches; without the guard, importing any helper from this module inside
|
|
2572
|
+
// vitest ran the CLI against the test runner's argv and exited the worker.
|
|
2573
|
+
const entryPath = process.argv[1];
|
|
2574
|
+
if (entryPath && import.meta.url === pathToFileURL(realpathSync(entryPath)).href) {
|
|
2575
|
+
program.parseAsync().catch((e) => {
|
|
2576
|
+
console.error("nlm: fatal", e);
|
|
2577
|
+
process.exit(1);
|
|
2578
|
+
});
|
|
2579
|
+
}
|
|
1869
2580
|
//# sourceMappingURL=nlm.js.map
|