akm-cli 0.9.0-rc.0 → 0.9.0-rc.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +1283 -22
- package/README.md +62 -37
- package/SECURITY.md +46 -31
- package/dist/akm +162 -38
- package/dist/akm-migrate +44 -0
- package/dist/assets/backends/schtasks-template.xml +2 -1
- package/dist/assets/hints/cli-hints-full.md +268 -118
- package/dist/assets/hints/cli-hints-short.md +87 -24
- package/dist/assets/{profiles → improve-strategies}/catchup.json +3 -1
- package/dist/assets/{profiles → improve-strategies}/consolidate.json +3 -1
- package/dist/assets/{profiles → improve-strategies}/default.json +6 -7
- package/dist/assets/improve-strategies/frequent.json +15 -0
- package/dist/assets/{profiles → improve-strategies}/graph-refresh.json +4 -2
- package/dist/assets/{profiles → improve-strategies}/memory-focus.json +4 -1
- package/dist/assets/{profiles → improve-strategies}/proactive-maintenance.json +5 -5
- package/dist/assets/{profiles → improve-strategies}/quick.json +4 -2
- package/dist/assets/improve-strategies/reflect-distill.json +30 -0
- package/dist/assets/{profiles → improve-strategies}/thorough.json +1 -1
- package/dist/assets/prompts/consolidate-system.md +5 -5
- package/dist/assets/prompts/extract-session.md +2 -6
- package/dist/assets/prompts/memory-infer-user.md +2 -3
- package/dist/assets/prompts/reflect-llm-framed-contract.md +11 -0
- package/dist/assets/prompts/reflect-llm-schema-contract.md +3 -0
- package/dist/assets/prompts/reflect-output-repair.md +3 -0
- package/dist/assets/prompts/workflow-unit-preamble.md +26 -0
- package/dist/assets/stash-skeleton/README.md +38 -10
- package/dist/assets/stash-skeleton/facts/conventions/assets/agent.md +8 -0
- package/dist/assets/stash-skeleton/facts/conventions/assets/command.md +8 -0
- package/dist/assets/stash-skeleton/facts/conventions/assets/fact.md +14 -1
- package/dist/assets/stash-skeleton/facts/conventions/assets/knowledge.md +13 -1
- package/dist/assets/stash-skeleton/facts/conventions/assets/lesson.md +9 -1
- package/dist/assets/stash-skeleton/facts/conventions/assets/memory.md +11 -0
- package/dist/assets/stash-skeleton/facts/conventions/assets/script.md +9 -0
- package/dist/assets/stash-skeleton/facts/conventions/assets/skill.md +9 -0
- package/dist/assets/stash-skeleton/facts/conventions/assets/workflow.md +8 -0
- package/dist/assets/stash-skeleton/facts/conventions/backlinks.md +100 -0
- package/dist/assets/stash-skeleton/facts/conventions/domains.md +64 -0
- package/dist/assets/stash-skeleton/facts/conventions/organization.md +136 -0
- package/dist/assets/tasks/core/extract.yml +3 -2
- package/dist/assets/tasks/core/improve.yml +2 -1
- package/dist/assets/tasks/core/index-refresh.yml +1 -0
- package/dist/assets/tasks/core/sync.yml +1 -0
- package/dist/assets/tasks/core/version-check.yml +2 -1
- package/dist/assets/tasks/improve/akm-graph-refresh-weekly.yml +5 -0
- package/dist/assets/tasks/improve/akm-improve-catchup.yml +8 -0
- package/dist/assets/tasks/improve/akm-improve-consolidate.yml +5 -0
- package/dist/assets/tasks/improve/akm-improve-frequent.yml +5 -0
- package/dist/assets/tasks/improve/akm-improve-nightly.yml +5 -0
- package/dist/assets/templates/html/health.html +5 -4
- package/dist/assets/workflows/workflow-template.md +31 -15
- package/dist/cli/invocation.js +279 -0
- package/dist/cli/parse-args.js +5 -90
- package/dist/cli/retired-commands.js +78 -0
- package/dist/cli/shared.js +158 -48
- package/dist/cli-node.mjs +2 -1
- package/dist/cli.js +747 -293
- package/dist/commands/agent/agent-dispatch.js +19 -18
- package/dist/commands/agent/agent-support.js +0 -24
- package/dist/commands/agent/contribute-cli.js +43 -97
- package/dist/commands/completions.js +80 -23
- package/dist/commands/config-cli.js +44 -281
- package/dist/commands/env/env-binding.js +99 -0
- package/dist/commands/env/env-cli.js +84 -224
- package/dist/commands/env/env.js +12 -163
- package/dist/commands/env/marker-path.js +6 -0
- package/dist/commands/env/secret-cli.js +45 -61
- package/dist/commands/env/secret.js +32 -62
- package/dist/commands/feedback-cli.js +179 -85
- package/dist/commands/health/accept-rate.js +58 -0
- package/dist/commands/health/advisories.js +7 -8
- package/dist/commands/health/checks.js +279 -94
- package/dist/commands/health/html-report.js +197 -578
- package/dist/commands/health/improve-metrics.js +277 -246
- package/dist/commands/health/llm-usage.js +19 -19
- package/dist/commands/health/md-report.js +16 -7
- package/dist/commands/health/metrics.js +67 -32
- package/dist/commands/health/renderers.js +47 -0
- package/dist/commands/health/report-view-model.js +508 -0
- package/dist/commands/health/stash-exposure.js +1 -1
- package/dist/commands/health/surfaces.js +16 -56
- package/dist/commands/health/task-runs.js +3 -67
- package/dist/{migrate-storage-node.mjs → commands/health/types-checks.js} +1 -5
- package/dist/commands/health/types-improve.js +29 -0
- package/dist/{output/text/save.js → commands/health/types-metrics.js} +1 -2
- package/dist/commands/health/types-result.js +7 -0
- package/dist/commands/health/types-runs.js +4 -0
- package/dist/commands/health/types-session-log.js +4 -0
- package/dist/commands/health/types-windows.js +4 -0
- package/dist/commands/health/types.js +26 -21
- package/dist/commands/health/windows.js +2 -3
- package/dist/commands/health.js +296 -167
- package/dist/commands/improve/anti-collapse.js +5 -5
- package/dist/commands/improve/autonomy-gate.js +68 -0
- package/dist/commands/improve/collapse-detector.js +65 -52
- package/dist/commands/improve/consolidate/chunking.js +9 -7
- package/dist/commands/improve/consolidate/eligibility.js +1 -23
- package/dist/commands/improve/consolidate/merge.js +4 -0
- package/dist/commands/improve/consolidate.js +454 -1354
- package/dist/commands/improve/content-hash.js +39 -0
- package/dist/commands/improve/distill/content-repair.js +4 -10
- package/dist/commands/improve/distill/promote-memory.js +89 -64
- package/dist/commands/improve/distill/quality-gate.js +118 -42
- package/dist/commands/improve/distill-guards.js +1 -1
- package/dist/commands/improve/distill-promotion-policy.js +33 -888
- package/dist/commands/improve/distill.js +607 -363
- package/dist/commands/improve/eligibility.js +165 -79
- package/dist/commands/improve/extract-cli.js +35 -126
- package/dist/commands/improve/extract-prompt.js +6 -35
- package/dist/commands/improve/extract.js +640 -391
- package/dist/commands/improve/feedback-valence.js +2 -12
- package/dist/commands/improve/improve-cli.js +134 -135
- package/dist/commands/improve/improve-result-file.js +30 -50
- package/dist/commands/improve/improve-run-types.js +4 -0
- package/dist/commands/improve/improve-strategies.js +135 -0
- package/dist/commands/improve/improve.js +904 -701
- package/dist/commands/improve/locks.js +64 -111
- package/dist/commands/improve/loop-stages.js +1110 -923
- package/dist/commands/improve/memory/derived-ref.js +124 -0
- package/dist/commands/improve/memory/memory-belief.js +79 -7
- package/dist/commands/improve/memory/memory-contradiction-detect.js +49 -52
- package/dist/commands/improve/memory/memory-improve.js +25 -37
- package/dist/commands/improve/outcome-loop.js +25 -88
- package/dist/commands/improve/preparation.js +1034 -813
- package/dist/commands/improve/proactive-maintenance.js +34 -9
- package/dist/commands/improve/proposal-envelope.js +31 -0
- package/dist/commands/improve/reflect.js +983 -794
- package/dist/commands/improve/run-context.js +119 -0
- package/dist/commands/improve/salience.js +24 -127
- package/dist/commands/improve/session-asset.js +7 -3
- package/dist/commands/improve/shared.js +14 -34
- package/dist/commands/improve/source-identity.js +28 -0
- package/dist/commands/improve/triage.js +20 -17
- package/dist/commands/lint/base-linter.js +340 -313
- package/dist/commands/lint/env-key-rules.js +31 -47
- package/dist/commands/lint/index.js +185 -30
- package/dist/commands/{events.js → log.js} +28 -38
- package/dist/commands/migrate-cli.js +54 -0
- package/dist/commands/migration-tool.js +55 -0
- package/dist/commands/observability-cli.js +70 -208
- package/dist/commands/proposal/diff-format.js +50 -0
- package/dist/commands/proposal/drain-policies.js +0 -6
- package/dist/commands/proposal/drain.js +91 -40
- package/dist/commands/proposal/proposal-cli.js +134 -132
- package/dist/commands/proposal/proposal-types.js +56 -0
- package/dist/commands/proposal/proposal.js +83 -65
- package/dist/commands/proposal/propose-cli.js +88 -0
- package/dist/commands/proposal/propose.js +105 -88
- package/dist/commands/proposal/repository.js +1303 -278
- package/dist/commands/proposal/validators/proposal-quality-validators.js +16 -6
- package/dist/commands/proposal/validators/proposal-validators.js +61 -12
- package/dist/commands/proposal/validators/proposals.js +6 -8
- package/dist/commands/read/curate.js +78 -73
- package/dist/commands/read/knowledge.js +510 -13
- package/dist/commands/read/registry-search.js +2 -2
- package/dist/commands/read/remember-cli.js +84 -15
- package/dist/commands/read/search-cli.js +203 -96
- package/dist/commands/read/search.js +126 -94
- package/dist/commands/read/show.js +226 -250
- package/dist/commands/registry-cli.js +34 -60
- package/dist/commands/remember.js +18 -57
- package/dist/commands/sources/add-cli.js +104 -49
- package/dist/commands/sources/bundle-cli.js +166 -0
- package/dist/commands/sources/bundle-config-ops.js +63 -0
- package/dist/commands/sources/info.js +27 -15
- package/dist/commands/sources/init.js +30 -40
- package/dist/commands/sources/installed-stashes.js +469 -172
- package/dist/commands/sources/migration-help.js +7 -4
- package/dist/commands/sources/schema-repair.js +10 -9
- package/dist/commands/sources/self-update.js +182 -121
- package/dist/commands/sources/source-add.js +169 -178
- package/dist/commands/sources/source-clone.js +144 -41
- package/dist/commands/sources/source-manage.js +94 -59
- package/dist/commands/sources/sources-cli.js +64 -205
- package/dist/commands/sources/stash-cli.js +91 -54
- package/dist/commands/sources/stash-skeleton.js +1 -1
- package/dist/commands/tasks/tasks-cli.js +106 -104
- package/dist/commands/tasks/tasks.js +445 -262
- package/dist/commands/workflow-cli.js +232 -121
- package/dist/core/action-contributors.js +1 -1
- package/dist/core/activation-policy.js +49 -0
- package/dist/core/adapter/adapters/agent-skills-adapter.js +181 -0
- package/dist/core/adapter/adapters/akm-adapter.js +528 -0
- package/dist/core/adapter/adapters/akm-lint.js +392 -0
- package/dist/core/adapter/adapters/akm-metadata.js +387 -0
- package/dist/core/adapter/adapters/akm-task-adapter.js +149 -0
- package/dist/core/adapter/adapters/akm-workflow-adapter.js +180 -0
- package/dist/core/adapter/adapters/claude-adapter.js +61 -0
- package/dist/core/adapter/adapters/dotenv-adapter.js +187 -0
- package/dist/core/adapter/adapters/generic-files-adapter.js +119 -0
- package/dist/core/adapter/adapters/index.js +80 -0
- package/dist/core/adapter/adapters/llm-wiki-adapter.js +419 -0
- package/dist/core/adapter/adapters/okf-adapter.js +391 -0
- package/dist/core/adapter/adapters/opencode-adapter.js +68 -0
- package/dist/core/adapter/adapters/shared.js +286 -0
- package/dist/core/adapter/adapters/tool-dir-shared.js +217 -0
- package/dist/core/adapter/adapters/website-snapshot-adapter.js +155 -0
- package/dist/core/adapter/bundle-adapter.js +4 -0
- package/dist/core/adapter/detect-adapter.js +17 -0
- package/dist/core/adapter/recognize-match.js +44 -0
- package/dist/core/adapter/registry.js +56 -0
- package/dist/core/adapter/types.js +4 -0
- package/dist/core/asset/akm-markdown.js +30 -0
- package/dist/core/asset/asset-placement.js +243 -0
- package/dist/core/asset/asset-ref.js +110 -79
- package/dist/core/asset/asset-serialize.js +20 -0
- package/dist/core/asset/frontmatter.js +28 -12
- package/dist/core/asset/markdown.js +40 -51
- package/dist/core/asset/resolve-ref.js +274 -0
- package/dist/core/asset/stash-meta.js +2 -2
- package/dist/core/bundle-id.js +51 -0
- package/dist/core/common.js +281 -86
- package/dist/core/config/config-io.js +42 -128
- package/dist/core/config/config-schema.js +233 -834
- package/dist/core/config/config-sources.js +162 -39
- package/dist/core/config/config-types.js +16 -11
- package/dist/core/config/config-version.js +29 -0
- package/dist/core/config/config-walker.js +126 -37
- package/dist/core/config/config.js +154 -331
- package/dist/core/config/deep-merge.js +41 -0
- package/dist/core/config/engine-semantics.js +28 -0
- package/dist/core/config/experimental.js +21 -0
- package/dist/core/config/schema/embedding.js +38 -0
- package/dist/core/config/schema/engines.js +116 -0
- package/dist/core/config/schema/experimental.js +47 -0
- package/dist/core/config/schema/feedback.js +31 -0
- package/dist/core/config/schema/improve-processes.js +389 -0
- package/dist/core/config/schema/improve.js +94 -0
- package/dist/core/config/schema/index-config.js +176 -0
- package/dist/core/config/schema/output.js +18 -0
- package/dist/core/config/schema/primitives.js +94 -0
- package/dist/core/config/schema/search.js +30 -0
- package/dist/core/config/schema/setup.js +18 -0
- package/dist/core/config/schema/sources-bundles.js +169 -0
- package/dist/core/config/schema/workflow.js +29 -0
- package/dist/core/env-secret-ref.js +155 -20
- package/dist/core/errors.js +17 -15
- package/dist/core/events-types.js +4 -0
- package/dist/core/events.js +46 -128
- package/dist/core/extra-params.js +62 -0
- package/dist/core/file-change.js +17 -0
- package/dist/core/file-lock.js +202 -57
- package/dist/core/fs-txn.js +392 -0
- package/dist/core/git-message.js +59 -0
- package/dist/core/improve-result.js +167 -0
- package/dist/core/json-schema.js +142 -0
- package/dist/core/lesson-lint.js +1 -17
- package/dist/core/logs-db.js +1 -1
- package/dist/core/maintenance-barrier.js +135 -0
- package/dist/core/migration-operation.js +44 -0
- package/dist/core/mutation-target.js +78 -0
- package/dist/core/paths.js +22 -25
- package/dist/core/platform.js +10 -0
- package/dist/core/recognition-util.js +128 -0
- package/dist/core/redaction.js +392 -0
- package/dist/core/standards/resolve-standards-context.js +36 -65
- package/dist/core/standards/resolve-stash-standards.js +2 -2
- package/dist/core/standards/resolve-type-conventions.js +5 -5
- package/dist/core/state/migrations.js +242 -11
- package/dist/core/state-db.js +98 -10
- package/dist/core/structured.js +1 -1
- package/dist/core/subprocess.js +303 -0
- package/dist/core/text-truncation.js +9 -5
- package/dist/core/time.js +20 -0
- package/dist/core/type-presentation.js +130 -0
- package/dist/core/warn.js +0 -3
- package/dist/core/write-source.js +834 -118
- package/dist/indexer/bundle-identity-guard.js +92 -0
- package/dist/indexer/db/graph-db.js +1 -25
- package/dist/indexer/db/llm-cache.js +1 -1
- package/dist/indexer/ensure-index.js +30 -9
- package/dist/indexer/graph/graph-boost.js +9 -30
- package/dist/indexer/graph/graph-extraction.js +41 -27
- package/dist/indexer/graph/graph-types.js +4 -0
- package/dist/indexer/index-writer-lock.js +93 -49
- package/dist/indexer/index-written-assets.js +100 -53
- package/dist/indexer/indexer.js +746 -329
- package/dist/indexer/init.js +18 -25
- package/dist/indexer/installations.js +142 -0
- package/dist/indexer/passes/dir-staleness.js +18 -10
- package/dist/indexer/passes/memory-inference.js +25 -15
- package/dist/indexer/passes/metadata.js +412 -243
- package/dist/indexer/scan/doc-to-entry.js +160 -0
- package/dist/indexer/scan/drain-dir.js +134 -0
- package/dist/indexer/search/db-search.js +292 -108
- package/dist/indexer/search/fts-query.js +64 -0
- package/dist/indexer/search/ranking-contributors.js +145 -25
- package/dist/indexer/search/ranking-types.js +4 -0
- package/dist/indexer/search/ranking.js +28 -71
- package/dist/indexer/search/search-attribution.js +67 -0
- package/dist/indexer/search/search-fields.js +18 -3
- package/dist/indexer/search/search-hit-enrichers.js +30 -40
- package/dist/indexer/search/search-source.js +157 -111
- package/dist/indexer/search/semantic-status.js +4 -1
- package/dist/indexer/usage/usage-events.js +10 -30
- package/dist/indexer/walk/file-context.js +3 -45
- package/dist/indexer/walk/matchers.js +42 -34
- package/dist/indexer/walk/path-resolver.js +11 -5
- package/dist/indexer/walk/walker.js +42 -14
- package/dist/integrations/agent/builder-shared.js +7 -0
- package/dist/integrations/agent/builders.js +5 -56
- package/dist/integrations/agent/config.js +3 -143
- package/dist/integrations/agent/detect.js +17 -2
- package/dist/integrations/agent/engine-resolution.js +231 -0
- package/dist/integrations/agent/index.js +1 -2
- package/dist/integrations/agent/model-aliases.js +16 -2
- package/dist/integrations/agent/profiles.js +36 -62
- package/dist/integrations/agent/prompts.js +46 -18
- package/dist/integrations/agent/runner-dispatch.js +93 -4
- package/dist/integrations/agent/runner.js +76 -208
- package/dist/integrations/agent/spawn.js +88 -196
- package/dist/integrations/harnesses/aider/agent-builder.js +114 -0
- package/dist/integrations/harnesses/aider/index.js +48 -0
- package/dist/integrations/harnesses/aider/result-extractor.js +53 -0
- package/dist/integrations/harnesses/amazonq/agent-builder.js +147 -0
- package/dist/integrations/harnesses/amazonq/index.js +45 -0
- package/dist/integrations/harnesses/amazonq/result-extractor.js +48 -0
- package/dist/integrations/harnesses/claude/agent-builder.js +46 -8
- package/dist/integrations/harnesses/claude/config-import.js +1 -3
- package/dist/integrations/harnesses/claude/index.js +24 -35
- package/dist/integrations/harnesses/claude/result-extractor.js +52 -0
- package/dist/integrations/harnesses/claude/session-log.js +27 -75
- package/dist/integrations/harnesses/codex/agent-builder.js +138 -0
- package/dist/integrations/harnesses/codex/index.js +52 -0
- package/dist/integrations/harnesses/codex/result-extractor.js +73 -0
- package/dist/integrations/harnesses/copilot/agent-builder.js +122 -0
- package/dist/integrations/harnesses/copilot/index.js +48 -0
- package/dist/integrations/harnesses/copilot/result-extractor.js +151 -0
- package/dist/integrations/harnesses/gemini/agent-builder.js +120 -0
- package/dist/integrations/harnesses/gemini/index.js +48 -0
- package/dist/integrations/harnesses/gemini/result-extractor.js +121 -0
- package/dist/integrations/harnesses/ids.js +24 -0
- package/dist/integrations/harnesses/index.js +54 -34
- package/dist/integrations/harnesses/opencode/agent-builder.js +23 -5
- package/dist/integrations/harnesses/opencode/config-import.js +1 -3
- package/dist/integrations/harnesses/opencode/index.js +14 -32
- package/dist/integrations/harnesses/opencode/session-log.js +67 -125
- package/dist/integrations/harnesses/opencode-sdk/harness.js +51 -0
- package/dist/integrations/harnesses/opencode-sdk/sdk-runner.js +681 -108
- package/dist/integrations/harnesses/openhands/agent-builder.js +128 -0
- package/dist/integrations/harnesses/openhands/index.js +48 -0
- package/dist/integrations/harnesses/openhands/result-extractor.js +103 -0
- package/dist/integrations/harnesses/pi/agent-builder.js +97 -0
- package/dist/integrations/harnesses/pi/index.js +45 -0
- package/dist/integrations/harnesses/pi/result-extractor.js +135 -0
- package/dist/integrations/harnesses/shared.js +17 -0
- package/dist/integrations/harnesses/types.js +43 -32
- package/dist/integrations/lockfile.js +211 -24
- package/dist/integrations/session-logs/index.js +36 -39
- package/dist/integrations/session-logs/provider-base.js +113 -0
- package/dist/llm/client.js +182 -110
- package/dist/llm/embedders/deterministic.js +2 -2
- package/dist/llm/embedders/remote.js +21 -9
- package/dist/llm/feature-gate.js +17 -57
- package/dist/llm/graph-extract.js +12 -13
- package/dist/llm/index-passes.js +8 -42
- package/dist/llm/memory-infer.js +144 -1
- package/dist/llm/metadata-enhance.js +45 -30
- package/dist/llm/structured-call.js +16 -8
- package/dist/llm/usage-persist.js +30 -5
- package/dist/llm/usage-telemetry.js +59 -6
- package/dist/output/cli-hints.js +1 -2
- package/dist/output/command-registry.js +27 -0
- package/dist/output/context.js +22 -7
- package/dist/output/format-exempt.js +80 -0
- package/dist/output/generic-render.js +251 -0
- package/dist/output/html-render.js +11 -16
- package/dist/output/render-registry.js +57 -0
- package/dist/output/renderers.js +14 -279
- package/dist/output/shapes/curate.js +10 -1
- package/dist/output/shapes/events.js +12 -7
- package/dist/output/shapes/helpers.js +58 -84
- package/dist/output/shapes/passthrough.js +11 -39
- package/dist/output/shapes/proposal/producer.js +15 -7
- package/dist/output/shapes/registry.js +12 -6
- package/dist/output/shapes.js +0 -9
- package/dist/output/text/{init.js → bundle-create.js} +3 -1
- package/dist/output/text/bundle-show.js +7 -0
- package/dist/output/text/command-format.js +562 -0
- package/dist/output/text/env.js +1 -3
- package/dist/output/text/events.js +8 -7
- package/dist/output/text/helpers.js +15 -1164
- package/dist/output/text/proposal/producer.js +4 -2
- package/dist/output/text/proposal-format.js +202 -0
- package/dist/output/text/registry-commands.js +1 -2
- package/dist/output/text/registry.js +12 -6
- package/dist/output/text/show-directives.js +117 -0
- package/dist/output/text/show-format.js +103 -0
- package/dist/output/text/sync.js +5 -0
- package/dist/output/text/workflow-format.js +332 -0
- package/dist/output/text/workflow.js +3 -2
- package/dist/output/text.js +10 -19
- package/dist/registry/factory.js +4 -6
- package/dist/registry/origin-resolve.js +16 -27
- package/dist/registry/providers/skills-sh.js +3 -3
- package/dist/registry/providers/static-index.js +15 -25
- package/dist/registry/resolve.js +43 -94
- package/dist/registry/semver.js +43 -0
- package/dist/runtime.js +81 -12
- package/dist/scripts/akm-migrate.js +35529 -0
- package/dist/setup/detect.js +5 -7
- package/dist/setup/detected-engines.js +136 -0
- package/dist/setup/engine-config.js +100 -0
- package/dist/setup/registry-stash-loader.js +3 -3
- package/dist/setup/semantic-assets.js +12 -9
- package/dist/setup/setup.js +444 -208
- package/dist/setup/steps/connection-shared.js +120 -0
- package/dist/setup/steps/connection.js +108 -305
- package/dist/setup/steps/platforms.js +13 -12
- package/dist/setup/steps/semantic.js +15 -3
- package/dist/setup/steps/sources.js +21 -15
- package/dist/setup/steps/stashdir.js +6 -4
- package/dist/setup/steps/tasks.js +236 -119
- package/dist/setup/steps.js +3 -2
- package/dist/sources/freshness.js +39 -0
- package/dist/sources/provider-factory.js +11 -17
- package/dist/sources/providers/filesystem.js +2 -3
- package/dist/sources/providers/git-install.js +278 -34
- package/dist/sources/providers/git-provider.js +54 -56
- package/dist/sources/providers/git-stash.js +420 -91
- package/dist/sources/providers/git.js +2 -2
- package/dist/sources/providers/npm.js +16 -19
- package/dist/sources/providers/provider-utils.js +47 -22
- package/dist/sources/providers/sync-from-ref.js +3 -9
- package/dist/sources/providers/website.js +2 -2
- package/dist/sources/resolve.js +11 -10
- package/dist/sources/snapshot-fetchers/types.js +4 -0
- package/dist/sources/{website-ingest.js → snapshot-fetchers/website-ingest.js} +110 -41
- package/dist/storage/database.js +60 -4
- package/dist/storage/engines/sqlite-migrations.js +156 -5
- package/dist/storage/locations.js +1 -2
- package/dist/storage/repositories/canaries-repository.js +1 -1
- package/dist/storage/repositories/events-repository.js +51 -11
- package/dist/storage/repositories/improve-runs-repository.js +6 -32
- package/dist/storage/repositories/index-connection.js +79 -0
- package/dist/storage/repositories/index-db.js +4 -3
- package/dist/storage/repositories/index-entries-repository.js +863 -0
- package/dist/{indexer/db/entry-mapper.js → storage/repositories/index-entry-mapper.js} +19 -2
- package/dist/storage/repositories/index-entry-types.js +4 -0
- package/dist/storage/repositories/index-fts-repository.js +167 -0
- package/dist/storage/repositories/index-llm-cache-repository.js +108 -0
- package/dist/storage/repositories/index-meta-repository.js +49 -0
- package/dist/{indexer/db/schema.js → storage/repositories/index-schema.js} +226 -100
- package/dist/storage/repositories/index-sql.js +12 -0
- package/dist/storage/repositories/index-utility-repository.js +356 -0
- package/dist/storage/repositories/index-vec-repository.js +250 -0
- package/dist/storage/repositories/outcome-repository.js +119 -0
- package/dist/storage/repositories/proposals-repository.js +317 -75
- package/dist/storage/repositories/registry-cache.js +1 -1
- package/dist/storage/repositories/salience-repository.js +172 -0
- package/dist/storage/repositories/task-history-repository.js +110 -3
- package/dist/storage/repositories/workflow-runs-repository.js +240 -19
- package/dist/tasks/backends/cron.js +169 -46
- package/dist/tasks/backends/exec-utils.js +76 -3
- package/dist/tasks/backends/index.js +6 -9
- package/dist/tasks/backends/launchd.js +292 -55
- package/dist/tasks/backends/schtasks.js +557 -70
- package/dist/tasks/backends/types.js +4 -0
- package/dist/tasks/command-executable.js +93 -0
- package/dist/tasks/embedded.js +56 -38
- package/dist/tasks/parser.js +156 -64
- package/dist/tasks/resolve-akm-bin.js +144 -51
- package/dist/tasks/runner.js +377 -209
- package/dist/tasks/schedule.js +108 -19
- package/dist/tasks/scheduler-invocation.js +296 -0
- package/dist/tasks/schema.js +1 -1
- package/dist/tasks/task-id.js +35 -0
- package/dist/tasks/validator.js +30 -16
- package/dist/text-import-hook.mjs +1 -1
- package/dist/workflows/authoring/authoring.js +104 -43
- package/dist/workflows/authoring/scope-key.js +1 -1
- package/dist/workflows/cli.js +0 -16
- package/dist/workflows/concurrency-policy.js +15 -0
- package/dist/workflows/exec/brief.js +450 -0
- package/dist/workflows/exec/frozen-judge.js +47 -0
- package/dist/workflows/exec/native-executor.js +1038 -0
- package/dist/workflows/exec/param-secrets.js +115 -0
- package/dist/workflows/exec/report.js +1460 -0
- package/dist/workflows/exec/run-workflow.js +602 -0
- package/dist/workflows/exec/scheduler.js +71 -0
- package/dist/workflows/exec/step-work.js +1190 -0
- package/dist/workflows/exec/unit-writer.js +23 -0
- package/dist/workflows/exec/workflow-engine-gate.js +67 -0
- package/dist/workflows/exec/worktree.js +171 -0
- package/dist/workflows/ir/compile.js +246 -0
- package/dist/workflows/ir/freeze.js +233 -0
- package/dist/workflows/ir/params.js +54 -0
- package/dist/workflows/ir/plan-hash.js +68 -0
- package/dist/workflows/ir/schema.js +540 -0
- package/dist/workflows/parser.js +878 -304
- package/dist/workflows/program/expressions.js +181 -0
- package/dist/workflows/program/schema.js +51 -0
- package/dist/workflows/renderer.js +100 -45
- package/dist/workflows/resource-limits.js +22 -0
- package/dist/workflows/runtime/agent-identity.js +59 -14
- package/dist/workflows/runtime/checkin.js +1 -1
- package/dist/workflows/runtime/plan-classifier.js +131 -0
- package/dist/workflows/runtime/runs.js +376 -119
- package/dist/workflows/runtime/unit-checkin.js +45 -0
- package/dist/workflows/runtime/unit-phases.js +20 -0
- package/dist/workflows/runtime/workflow-asset-loader.js +241 -40
- package/dist/workflows/schema.js +1 -11
- package/dist/workflows/validate-summary.js +2 -3
- package/dist/workflows/validator.js +52 -30
- package/docs/README.md +42 -78
- package/docs/migration/README.md +8 -0
- package/docs/migration/release-notes/0.6.0.md +1 -1
- package/docs/migration/release-notes/0.7.0.md +9 -8
- package/docs/migration/release-notes/0.9.0.md +158 -14
- package/docs/migration/v0.7-to-v0.8.md +46 -47
- package/docs/migration/v0.8-to-v0.9.md +844 -0
- package/docs/reference/README.md +12 -0
- package/docs/reference/data-and-telemetry.md +333 -0
- package/package.json +21 -17
- package/schemas/akm-asset-envelope.json +93 -0
- package/schemas/akm-config.json +4636 -0
- package/schemas/akm-task.json +87 -0
- package/schemas/akm-workflow.json +373 -0
- package/dist/akm-migrate-storage +0 -38
- package/dist/assets/help/help-accept.md +0 -12
- package/dist/assets/help/help-improve.md +0 -84
- package/dist/assets/help/help-proposals.md +0 -17
- package/dist/assets/help/help-propose.md +0 -17
- package/dist/assets/help/help-reject.md +0 -11
- package/dist/assets/profiles/frequent.json +0 -13
- package/dist/assets/profiles/recombine-only.json +0 -21
- package/dist/assets/profiles/reflect-distill.json +0 -30
- package/dist/assets/profiles/synthesize.json +0 -15
- package/dist/assets/prompts/procedural-system.md +0 -44
- package/dist/assets/prompts/recombine-system.md +0 -40
- package/dist/assets/prompts/staleness-detect-system.md +0 -6
- package/dist/assets/tasks/core/backup.yml +0 -4
- package/dist/assets/tasks/graph-refresh-weekly.yml +0 -10
- package/dist/assets/templates/html/default.html +0 -78
- package/dist/assets/templates/html/vendor/echarts.min.js +0 -45
- package/dist/assets/wiki/index-template.md +0 -12
- package/dist/assets/wiki/ingest-workflow-template.md +0 -83
- package/dist/assets/wiki/log-template.md +0 -8
- package/dist/assets/wiki/schema-template.md +0 -61
- package/dist/cli/config-migrate.js +0 -150
- package/dist/cli/config-validate.js +0 -39
- package/dist/commands/graph/graph-cli.js +0 -124
- package/dist/commands/graph/graph.js +0 -487
- package/dist/commands/improve/calibration.js +0 -161
- package/dist/commands/improve/dedup.js +0 -482
- package/dist/commands/improve/extract-watch.js +0 -140
- package/dist/commands/improve/hot-probation.js +0 -45
- package/dist/commands/improve/improve-auto-accept.js +0 -276
- package/dist/commands/improve/improve-profiles.js +0 -168
- package/dist/commands/improve/procedural.js +0 -398
- package/dist/commands/improve/recombine.js +0 -818
- package/dist/commands/improve/schema-similarity-gate.js +0 -168
- package/dist/commands/lint/agent-linter.js +0 -44
- package/dist/commands/lint/command-linter.js +0 -44
- package/dist/commands/lint/default-linter.js +0 -16
- package/dist/commands/lint/fact-linter.js +0 -39
- package/dist/commands/lint/knowledge-linter.js +0 -16
- package/dist/commands/lint/memory-linter.js +0 -61
- package/dist/commands/lint/registry.js +0 -41
- package/dist/commands/lint/skill-linter.js +0 -45
- package/dist/commands/lint/task-linter.js +0 -50
- package/dist/commands/lint/workflow-linter.js +0 -81
- package/dist/commands/proposal/legacy-import.js +0 -115
- package/dist/commands/sources/history.js +0 -196
- package/dist/commands/tasks/default-tasks.js +0 -186
- package/dist/commands/wiki-cli.js +0 -292
- package/dist/core/asset/asset-registry.js +0 -76
- package/dist/core/asset/asset-spec.js +0 -259
- package/dist/core/config/config-migration.js +0 -602
- package/dist/core/deep-merge.js +0 -38
- package/dist/core/eval/rank-metrics.js +0 -113
- package/dist/core/ripgrep/install.js +0 -163
- package/dist/core/ripgrep/resolve.js +0 -81
- package/dist/indexer/db/db.js +0 -1413
- package/dist/indexer/manifest.js +0 -170
- package/dist/indexer/passes/metadata-contributors.js +0 -31
- package/dist/indexer/usage/unmigrated-vaults-guard.js +0 -94
- package/dist/integrations/harnesses/opencode-sdk/index.js +0 -49
- package/dist/llm/call-ai.js +0 -62
- package/dist/llm/memory-infer-impl.js +0 -138
- package/dist/output/shapes/distill.js +0 -14
- package/dist/output/shapes/history.js +0 -11
- package/dist/output/text/distill.js +0 -6
- package/dist/output/text/enable-disable.js +0 -8
- package/dist/output/text/history.js +0 -6
- package/dist/output/text/wiki.js +0 -16
- package/dist/registry/build-index.js +0 -386
- package/dist/scripts/migrate-storage.js +0 -19108
- package/dist/scripts/migrations/import-fs-improve-runs-to-db.js +0 -9411
- package/dist/scripts/migrations/v16-to-v17.js +0 -141
- package/dist/setup/legacy-config.js +0 -106
- package/dist/storage/repositories/consolidation-repository.js +0 -38
- package/dist/storage/repositories/recombine-repository.js +0 -213
- package/dist/wiki/wiki-templates.js +0 -15
- package/dist/wiki/wiki.js +0 -1012
- package/dist/workflows/db.js +0 -215
- package/docs/data-and-telemetry.md +0 -226
- /package/dist/sources/{wiki-fetchers → snapshot-fetchers}/registry.js +0 -0
- /package/dist/sources/{wiki-fetchers → snapshot-fetchers}/youtube.js +0 -0
|
@@ -1,28 +1,32 @@
|
|
|
1
1
|
// This Source Code Form is subject to the terms of the Mozilla Public
|
|
2
2
|
// License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
3
3
|
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
4
|
+
import fs from "node:fs";
|
|
4
5
|
import path from "node:path";
|
|
5
|
-
import {
|
|
6
|
+
import { parseRefInput } from "../../core/asset/resolve-ref.js";
|
|
6
7
|
import { daysToMs } from "../../core/common.js";
|
|
7
|
-
import { loadConfig } from "../../core/config/config.js";
|
|
8
|
+
import { DEFAULT_GRAPH_EXTRACTION_BATCH_SIZE, loadConfig } from "../../core/config/config.js";
|
|
8
9
|
import { UsageError } from "../../core/errors.js";
|
|
9
10
|
import { appendEvent } from "../../core/events.js";
|
|
10
11
|
import { openLogsDatabase, purgeOldTaskLogs } from "../../core/logs-db.js";
|
|
11
12
|
import { getDbPath } from "../../core/paths.js";
|
|
12
13
|
import { withStateDb } from "../../core/state-db.js";
|
|
13
|
-
import { info
|
|
14
|
-
import {
|
|
15
|
-
import { runGraphExtractionPass } from "../../indexer/graph/graph-extraction.js";
|
|
14
|
+
import { info } from "../../core/warn.js";
|
|
15
|
+
import { DEFAULT_GRAPH_EXTRACTION_INCLUDE_TYPES, runGraphExtractionPass, } from "../../indexer/graph/graph-extraction.js";
|
|
16
16
|
import { withIndexWriterLease } from "../../indexer/index-writer-lock.js";
|
|
17
17
|
import { collectPendingMemories, runMemoryInferencePass, } from "../../indexer/passes/memory-inference.js";
|
|
18
18
|
import { getWritableStashDirs, resolveSourceEntries } from "../../indexer/search/search-source.js";
|
|
19
|
-
import {
|
|
19
|
+
import { materializeLlmRunnerConnection } from "../../integrations/agent/runner.js";
|
|
20
20
|
import { isProcessEnabled } from "../../llm/feature-gate.js";
|
|
21
21
|
import { withLlmStage } from "../../llm/usage-telemetry.js";
|
|
22
22
|
import { purgeOldCycleMetrics } from "../../storage/repositories/canaries-repository.js";
|
|
23
23
|
import { purgeOldEvents } from "../../storage/repositories/events-repository.js";
|
|
24
24
|
import { purgeOldImproveRuns } from "../../storage/repositories/improve-runs-repository.js";
|
|
25
|
-
import {
|
|
25
|
+
import { closeDatabase, openIndexDatabase } from "../../storage/repositories/index-connection.js";
|
|
26
|
+
import { getEntryByRef } from "../../storage/repositories/index-entries-repository.js";
|
|
27
|
+
import { clearAssetOutcomeMissing, countAssetOutcomeMissing, deleteAssetOutcomeMissingBefore, listAssetOutcomeMissingState, stampAssetOutcomeMissing, } from "../../storage/repositories/outcome-repository.js";
|
|
28
|
+
import { clearAssetSalienceMissing, countAssetSalienceMissing, deleteAssetSalienceMissingBefore, listAssetSalienceMissingState, stampAssetSalienceMissing, } from "../../storage/repositories/salience-repository.js";
|
|
29
|
+
import { expireStaleProposals, listProposals, purgeOrphanProposals } from "../proposal/repository.js";
|
|
26
30
|
import { checkDeadUrls } from "../url-checker.js";
|
|
27
31
|
import { DEFAULT_RETENTION_DAYS as CYCLE_METRICS_RETENTION_DAYS, runCollapseDetector } from "./collapse-detector.js";
|
|
28
32
|
import { deriveLessonRef } from "./distill.js";
|
|
@@ -30,570 +34,527 @@ import { deriveKnowledgeRef } from "./distill-promotion-policy.js";
|
|
|
30
34
|
// Eligibility / candidate-selection predicates live in ./eligibility.
|
|
31
35
|
import { findAssetFilePath, isDistillCandidateRef } from "./eligibility.js";
|
|
32
36
|
import { writeEvalCase } from "./eval-cases.js";
|
|
33
|
-
import {
|
|
34
|
-
import { resolveProcessEnabled, shouldSkipRef } from "./improve-profiles.js";
|
|
35
|
-
// The pre-loop preparation pipeline lives in ./preparation.
|
|
36
|
-
import { maybeAutoTuneThreshold } from "./preparation.js";
|
|
37
|
-
import { akmProcedural } from "./procedural.js";
|
|
38
|
-
import { akmRecombine } from "./recombine.js";
|
|
37
|
+
import { shouldSkipRef } from "./improve-strategies.js";
|
|
39
38
|
import { recordNoOp, resetConsecutiveNoOps } from "./salience.js";
|
|
40
39
|
import { errMessage, refSlug } from "./shared.js";
|
|
40
|
+
import { bareImproveRef, durableImproveRef } from "./source-identity.js";
|
|
41
41
|
// ── improve loop / post-loop / maintenance stages ───────────────────
|
|
42
42
|
// The cycle stages run by akmImprove, extracted from improve.ts.
|
|
43
|
-
|
|
44
|
-
|
|
43
|
+
/** O-5 / #378: rolling per-originator error-window cap. */
|
|
44
|
+
const RECENT_ERRORS_CAP = 3;
|
|
45
|
+
/** O-5 / #378: push a per-originator error into the rolling window. */
|
|
46
|
+
function pushRecentError(recentErrors, originator, msg) {
|
|
47
|
+
if (!recentErrors[originator])
|
|
48
|
+
recentErrors[originator] = [];
|
|
49
|
+
recentErrors[originator].push(msg);
|
|
50
|
+
if (recentErrors[originator].length > RECENT_ERRORS_CAP)
|
|
51
|
+
recentErrors[originator].shift();
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* Build the per-run loop environment from the run context: the derived guards
|
|
55
|
+
* and the pending-proposal preload.
|
|
56
|
+
*/
|
|
57
|
+
export function prepareImproveLoopEnv(args) {
|
|
58
|
+
const { ctx, scope, options, reflectFn, distillFn, loopRefs, signalBearingSet, distillCooledRefs, distillOnlyRefs, recentErrors, rejectedProposalsByRef, startMs, budgetMs, improveProfile, resolvedPlan, } = args;
|
|
59
|
+
// WI-9.10: the legacy dual context's optional eventsCtx/budgetSignal are
|
|
60
|
+
// now RunContext's required eventsCtx / optional signal — renamed local
|
|
61
|
+
// aliases so the rest of this function (and the ImproveLoopEnv object
|
|
62
|
+
// literal below) is unchanged. `primaryStashDir` deliberately comes from
|
|
63
|
+
// the state's honest optional field, NOT `ctx.stashDir`: the rare
|
|
64
|
+
// unresolvable-primary path must keep skipping the `if (primaryStashDir)`
|
|
65
|
+
// guards below (see the field's doc in ./improve-run-types).
|
|
66
|
+
const primaryStashDir = args.primaryStashDir;
|
|
67
|
+
const eventsCtx = ctx.eventsCtx;
|
|
68
|
+
const budgetSignal = ctx.signal;
|
|
45
69
|
// O-1 (#364): compute remaining budget at call time so each sub-call
|
|
46
70
|
// receives only its fair share of the wall-clock budget.
|
|
47
71
|
const remainingBudgetMs = () => Math.max(0, budgetMs - (Date.now() - startMs));
|
|
48
|
-
|
|
72
|
+
// Build a Set for O(1) membership test — these refs skip the reflect call (Bug D2).
|
|
73
|
+
const distillOnlyRefSet = new Set(distillOnlyRefs.map((r) => r.ref));
|
|
49
74
|
// requirePlannedRefs guard: when the distill profile sets this flag, skip
|
|
50
75
|
// distill for distill-only refs if the reflect phase produced no planned refs.
|
|
51
76
|
// Prevents the distill loop from generating hundreds of distill-skipped events
|
|
52
77
|
// on quiet passes (all refs on reflect cooldown, no new signal to distill).
|
|
53
78
|
const requirePlannedRefs = improveProfile?.processes?.distill?.requirePlannedRefs === true;
|
|
54
|
-
const
|
|
55
|
-
const hasReflectEligibleRefs = loopRefs.some((r) => !_distillOnlyRefNames.has(r.ref));
|
|
79
|
+
const hasReflectEligibleRefs = loopRefs.some((r) => !distillOnlyRefSet.has(r.ref));
|
|
56
80
|
const skipDistillDueToRequirePlannedRefs = requirePlannedRefs && !hasReflectEligibleRefs;
|
|
57
|
-
// R-2 / #389: Self-Consistency multi-sample voting helpers.
|
|
58
|
-
// Wang et al. arXiv:2203.11171 — N=3 samples beat single-shot on reasoning tasks.
|
|
59
|
-
const SC_THRESHOLD = options.selfConsistencyThreshold ?? 0.7;
|
|
60
|
-
const SC_N = Math.min(Math.max(2, options.selfConsistencyN ?? 3), 5);
|
|
61
|
-
/**
|
|
62
|
-
* Compute Jaccard token overlap between two strings.
|
|
63
|
-
* Tokenizes by whitespace; returns 0 when both are empty.
|
|
64
|
-
*/
|
|
65
|
-
function jaccardSimilarity(a, b) {
|
|
66
|
-
const tokensA = new Set(a.split(/\s+/).filter(Boolean));
|
|
67
|
-
const tokensB = new Set(b.split(/\s+/).filter(Boolean));
|
|
68
|
-
if (tokensA.size === 0 && tokensB.size === 0)
|
|
69
|
-
return 1;
|
|
70
|
-
let intersection = 0;
|
|
71
|
-
for (const t of tokensA) {
|
|
72
|
-
if (tokensB.has(t))
|
|
73
|
-
intersection++;
|
|
74
|
-
}
|
|
75
|
-
const union = tokensA.size + tokensB.size - intersection;
|
|
76
|
-
return union > 0 ? intersection / union : 0;
|
|
77
|
-
}
|
|
78
|
-
/**
|
|
79
|
-
* Given N reflect results, return the one with the highest average Jaccard
|
|
80
|
-
* similarity to all other successful results (majority-vote winner).
|
|
81
|
-
* Falls back to the first successful result when N < 2.
|
|
82
|
-
*/
|
|
83
|
-
function pickMajorityVote(results) {
|
|
84
|
-
const successful = results.filter((r) => r.ok);
|
|
85
|
-
if (successful.length === 0)
|
|
86
|
-
return (results[0] ?? {
|
|
87
|
-
schemaVersion: 1,
|
|
88
|
-
ok: false,
|
|
89
|
-
reason: "non_zero_exit",
|
|
90
|
-
error: "all samples failed",
|
|
91
|
-
exitCode: null,
|
|
92
|
-
});
|
|
93
|
-
if (successful.length === 1)
|
|
94
|
-
return successful[0];
|
|
95
|
-
let bestIdx = 0;
|
|
96
|
-
let bestScore = -1;
|
|
97
|
-
for (let i = 0; i < successful.length; i++) {
|
|
98
|
-
let totalSim = 0;
|
|
99
|
-
for (let j = 0; j < successful.length; j++) {
|
|
100
|
-
if (i === j)
|
|
101
|
-
continue;
|
|
102
|
-
totalSim += jaccardSimilarity(successful[i].proposal.payload.content ?? "", successful[j].proposal.payload.content ?? "");
|
|
103
|
-
}
|
|
104
|
-
const avgSim = totalSim / (successful.length - 1);
|
|
105
|
-
if (avgSim > bestScore) {
|
|
106
|
-
bestScore = avgSim;
|
|
107
|
-
bestIdx = i;
|
|
108
|
-
}
|
|
109
|
-
}
|
|
110
|
-
return successful[bestIdx] ?? successful[0];
|
|
111
|
-
}
|
|
112
|
-
// O-5 / #378: helper to push per-originator errors into the rolling window.
|
|
113
|
-
function pushRecentError(originator, msg) {
|
|
114
|
-
if (!recentErrors[originator])
|
|
115
|
-
recentErrors[originator] = [];
|
|
116
|
-
recentErrors[originator].push(msg);
|
|
117
|
-
if (recentErrors[originator].length > RECENT_ERRORS_CAP)
|
|
118
|
-
recentErrors[originator].shift();
|
|
119
|
-
}
|
|
120
|
-
// Build a Set for O(1) membership test — these refs skip the reflect call (Bug D2).
|
|
121
|
-
const distillOnlyRefSet = new Set(distillOnlyRefs.map((r) => r.ref));
|
|
122
|
-
let completedCount = 0;
|
|
123
|
-
let reflectsWithErrorContext = 0;
|
|
124
|
-
const memoryRefsForInference = new Set();
|
|
125
81
|
// Pre-load all pending proposals once instead of querying per asset in the loop.
|
|
126
82
|
const dedupeStashDirForProposals = primaryStashDir ?? options.stashDir;
|
|
127
83
|
const pendingProposalRefSet = new Set(dedupeStashDirForProposals
|
|
128
84
|
? listProposals(dedupeStashDirForProposals, { status: "pending" }).map((p) => p.ref)
|
|
129
85
|
: []);
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
candidateCount: loopRefs.length,
|
|
142
|
-
});
|
|
143
|
-
const distillGateCfg = makeGateConfig("distill", {
|
|
144
|
-
globalThreshold: options.autoAccept,
|
|
145
|
-
dryRun: options.dryRun ?? false,
|
|
146
|
-
stashDir: primaryStashDir,
|
|
147
|
-
config: options.config ?? loadConfig(),
|
|
86
|
+
return {
|
|
87
|
+
scope,
|
|
88
|
+
options,
|
|
89
|
+
primaryStashDir,
|
|
90
|
+
reflectFn,
|
|
91
|
+
distillFn,
|
|
92
|
+
signalBearingSet,
|
|
93
|
+
distillCooledRefs,
|
|
94
|
+
distillOnlyRefSet,
|
|
95
|
+
recentErrors,
|
|
96
|
+
rejectedProposalsByRef,
|
|
148
97
|
eventsCtx,
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
98
|
+
improveProfile,
|
|
99
|
+
resolvedPlan,
|
|
100
|
+
budgetSignal,
|
|
101
|
+
skipDistillDueToRequirePlannedRefs,
|
|
102
|
+
pendingProposalRefSet,
|
|
103
|
+
remainingBudgetMs,
|
|
104
|
+
};
|
|
105
|
+
}
|
|
106
|
+
/**
|
|
107
|
+
* One improve-loop iteration for a single planned ref: the reflect pass, then
|
|
108
|
+
* the distill pass, with the per-ref error classification (B7) around both.
|
|
109
|
+
* `continue` in the old inline loop body is an early `return` inside the
|
|
110
|
+
* passes; the orchestrator folds the returned tally and owns the run counters.
|
|
111
|
+
*/
|
|
112
|
+
export async function processImproveLoopRef(planned, env) {
|
|
113
|
+
const tally = {
|
|
114
|
+
actions: [],
|
|
115
|
+
reflectsWithErrorContext: 0,
|
|
116
|
+
recentErrorPushes: [],
|
|
117
|
+
memoryRefsForInference: [],
|
|
118
|
+
};
|
|
119
|
+
try {
|
|
120
|
+
// Bug D2: distillOnlyRefs skip the reflect call but still run the distill path.
|
|
121
|
+
// Bug D1: in-loop distill-cooldown check removed — distill-cooled candidates
|
|
122
|
+
// have their synthetic actions emitted in runImprovePreparationStage.
|
|
123
|
+
const isDistillOnly = env.distillOnlyRefSet.has(planned.ref);
|
|
124
|
+
const parsedPlannedRef = parseRefInput(planned.ref);
|
|
125
|
+
await runLoopReflectPass(planned, isDistillOnly, env, tally);
|
|
126
|
+
// isDistillOnly refs: no reflect action emitted — proceed directly to the distill pass.
|
|
127
|
+
await runLoopDistillPass(planned, parsedPlannedRef, isDistillOnly, env, tally);
|
|
128
|
+
}
|
|
129
|
+
catch (err) {
|
|
130
|
+
// B7: UsageError thrown by akmDistill on validation_failed should be recorded
|
|
131
|
+
// as mode:"distill" with outcome:"validation_failed", NOT as a generic error.
|
|
132
|
+
// The distill_invoked event was already emitted inside akmDistill before the throw.
|
|
133
|
+
if (err instanceof UsageError) {
|
|
134
|
+
tally.actions.push({
|
|
135
|
+
ref: planned.ref,
|
|
136
|
+
mode: "distill",
|
|
137
|
+
result: { ok: false, outcome: "validation_failed", error: err.message },
|
|
138
|
+
});
|
|
139
|
+
}
|
|
140
|
+
else {
|
|
141
|
+
tally.actions.push({
|
|
142
|
+
ref: planned.ref,
|
|
143
|
+
mode: "error",
|
|
144
|
+
result: { ok: false, error: errMessage(err) },
|
|
145
|
+
});
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
return tally;
|
|
149
|
+
}
|
|
150
|
+
/**
|
|
151
|
+
* Reflect half of one loop iteration: type/profile gates, the reflect call with
|
|
152
|
+
* recent-error avoidPatterns, outcome classification (cooldown / guard-reject /
|
|
153
|
+
* type-refused / noise-gate), and plasticity counters. Records onto the per-ref
|
|
154
|
+
* tally only.
|
|
155
|
+
*/
|
|
156
|
+
async function runLoopReflectPass(planned, isDistillOnly, env, tally) {
|
|
157
|
+
const { options, primaryStashDir, reflectFn, eventsCtx, improveProfile, resolvedPlan, budgetSignal } = env;
|
|
158
|
+
// B6: derived memories are machine-generated; skip reflect to avoid noisy proposals.
|
|
159
|
+
// shouldDistillMemoryRef already returns false for .derived refs, so the distill
|
|
160
|
+
// path is also a no-op for them — we just avoid unnecessary agent spawns.
|
|
161
|
+
// D2: distillOnlyRefs also skip the reflect call (reflect-cooled, distill path only).
|
|
162
|
+
if (!isDistillOnly && !planned.ref.endsWith(".derived")) {
|
|
163
|
+
// Type guard: skip reflect for unsupported types (script, env, task, etc.)
|
|
164
|
+
// and raw wiki directories, driven by the active improve profile.
|
|
165
|
+
const reflectSkip = shouldSkipRef(planned.ref, "reflect", improveProfile);
|
|
166
|
+
if (reflectSkip.skip) {
|
|
167
|
+
tally.actions.push({
|
|
168
|
+
ref: planned.ref,
|
|
169
|
+
mode: "reflect-skipped",
|
|
170
|
+
result: { ok: true, reason: reflectSkip.reason },
|
|
171
|
+
});
|
|
172
|
+
}
|
|
173
|
+
else {
|
|
174
|
+
// O-5 / #378: only inject reflect-originator errors into the reflect call.
|
|
175
|
+
// Cross-task errors (e.g. schema-repair) must NOT contaminate reflect prompts.
|
|
176
|
+
const reflectErrors = env.recentErrors.reflect ?? [];
|
|
177
|
+
if (reflectErrors.length > 0)
|
|
178
|
+
tally.reflectsWithErrorContext++;
|
|
179
|
+
// O-1 (#364): pass remaining budget as timeoutMs so the agent spawn is
|
|
180
|
+
// bounded by the wall-clock deadline rather than the default per-profile timeout.
|
|
181
|
+
const reflectBudgetMs = env.remainingBudgetMs();
|
|
182
|
+
// Use the runner frozen in the invocation plan; no leaf re-resolution.
|
|
183
|
+
const reflectProfileRunner = resolvedPlan.processes.reflect.runner;
|
|
184
|
+
const reflectCallArgs = {
|
|
185
|
+
ref: planned.ref,
|
|
186
|
+
// Carry the resolved item_ref so reflect uses the same durable key for
|
|
187
|
+
// events and state reads.
|
|
188
|
+
...(planned.itemRef ? { itemRef: planned.itemRef } : {}),
|
|
189
|
+
task: options.task,
|
|
190
|
+
// Active strategy supplies non-engine process tuning.
|
|
191
|
+
...(improveProfile ? { improveProfile } : {}),
|
|
192
|
+
config: options.config,
|
|
193
|
+
...(primaryStashDir ? { stashDir: primaryStashDir } : {}),
|
|
194
|
+
...(options.sourceName && primaryStashDir
|
|
195
|
+
? { target: { source: options.sourceName, root: primaryStashDir } }
|
|
196
|
+
: {}),
|
|
197
|
+
...(reflectErrors.length > 0 ? { avoidPatterns: [...reflectErrors] } : {}),
|
|
198
|
+
eventSource: "improve",
|
|
199
|
+
// #639 — resolve the low-value filter from the ACTIVE improve profile
|
|
200
|
+
// (default off when unset), so the running strategy decides.
|
|
201
|
+
lowValueFilter: improveProfile.processes?.reflect?.lowValueFilter?.enabled === true,
|
|
202
|
+
...(reflectBudgetMs > 0 ? { timeoutMs: reflectBudgetMs } : {}),
|
|
203
|
+
signal: budgetSignal,
|
|
204
|
+
runner: reflectProfileRunner ?? null,
|
|
205
|
+
// R25: reflect's event emits reuse the run's long-lived state.db handle.
|
|
206
|
+
eventsCtx: env.eventsCtx,
|
|
207
|
+
// Attribution: carry the eligibility lane so reflect stamps it on
|
|
208
|
+
// the reflect_invoked event and the persisted proposal.
|
|
209
|
+
...(planned.eligibilitySource ? { eligibilitySource: planned.eligibilitySource } : {}),
|
|
210
|
+
};
|
|
211
|
+
const reflectResult = await withLlmStage("reflect", () => reflectFn(reflectCallArgs), {
|
|
212
|
+
engine: resolvedPlan.processes.reflect.runner?.engine,
|
|
213
|
+
process: "reflect",
|
|
214
|
+
});
|
|
215
|
+
const isCooldown = !reflectResult.ok && reflectResult.reason === "cooldown";
|
|
216
|
+
// Content-policy guard hits (reflect size-rail rejections) are NOT
|
|
217
|
+
// LLM faults — the agent responded fine, the downstream guard
|
|
218
|
+
// blocked the output. Route them to a distinct `reflect-guard-rejected`
|
|
219
|
+
// mode so health metrics can split deterministic guard hits out of
|
|
220
|
+
// true LLM failures. See
|
|
221
|
+
// `/tmp/akm-health-investigations/metrics-taxonomy-review.md` §1a.
|
|
222
|
+
const isGuardReject = !reflectResult.ok && reflectResult.reason === "content_policy_reject";
|
|
223
|
+
// Type-guard rejection (reflect refused a script/env/task ref) is
|
|
224
|
+
// also NOT an LLM failure — the LLM is never invoked. Route to the
|
|
225
|
+
// existing `reflect-skipped` bucket so it does not inflate the
|
|
226
|
+
// failure-rate numerator. ~9% of `reflect-failed` events in the
|
|
227
|
+
// user's stack were this case; see review §1a row "Reflect refused
|
|
228
|
+
// asset type".
|
|
229
|
+
const isTypeRefused = !reflectResult.ok && reflectResult.reason === "unsupported_type";
|
|
230
|
+
// Noise-gate suppression (#580): the candidate edit was an empty
|
|
231
|
+
// diff or a cosmetic-only reformat of the current asset. Like
|
|
232
|
+
// `unsupported_type`, this is a deterministic skip — not an LLM
|
|
233
|
+
// fault — so it routes to the `reflect-skipped` bucket and stays
|
|
234
|
+
// out of recentErrors/avoidPatterns.
|
|
235
|
+
const isNoChange = !reflectResult.ok && reflectResult.reason === "no_change";
|
|
236
|
+
tally.actions.push({
|
|
237
|
+
ref: planned.ref,
|
|
238
|
+
mode: reflectResult.ok
|
|
239
|
+
? "reflect"
|
|
240
|
+
: isCooldown
|
|
241
|
+
? "reflect-cooldown"
|
|
242
|
+
: isGuardReject
|
|
243
|
+
? "reflect-guard-rejected"
|
|
244
|
+
: isTypeRefused || isNoChange
|
|
245
|
+
? "reflect-skipped"
|
|
246
|
+
: "reflect-failed",
|
|
247
|
+
result: reflectResult,
|
|
248
|
+
});
|
|
249
|
+
// Cooldown skips, guard rejects, type-refused skips, and noise-gate
|
|
250
|
+
// skips are not failures — do not pollute recentErrors with them
|
|
251
|
+
// (those get injected as `avoidPatterns` into the next reflect
|
|
252
|
+
// prompt). Guard rejects ARE worth showing the LLM as a learn-signal
|
|
253
|
+
// so the next iteration sees "your last expansion was too large";
|
|
254
|
+
// type-refused and no-change are deterministic and add no learning
|
|
255
|
+
// signal.
|
|
256
|
+
if (!reflectResult.ok && !isCooldown && !isTypeRefused && !isNoChange) {
|
|
257
|
+
const errMsg = reflectResult.error ?? reflectResult.reason ?? "unknown reflect error";
|
|
258
|
+
tally.recentErrorPushes.push({ originator: "reflect", message: errMsg });
|
|
259
|
+
}
|
|
260
|
+
// improve_reflect_outcome — per-asset metric for tuning the reflect path.
|
|
156
261
|
appendEvent({
|
|
157
|
-
eventType: "
|
|
262
|
+
eventType: "improve_reflect_outcome",
|
|
158
263
|
ref: planned.ref,
|
|
159
264
|
metadata: {
|
|
160
|
-
|
|
161
|
-
|
|
265
|
+
ok: reflectResult.ok,
|
|
266
|
+
durationMs: reflectResult.ok ? reflectResult.durationMs : undefined,
|
|
267
|
+
engine: reflectResult.engine,
|
|
268
|
+
reason: reflectResult.ok ? undefined : reflectResult.reason,
|
|
162
269
|
},
|
|
163
270
|
}, eventsCtx);
|
|
164
|
-
//
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
271
|
+
// Plasticity counter (plan §WS-1 step 8): record no-ops so the
|
|
272
|
+
// WS-1 selection comparator (effectiveScore, ~line 3073) can dampen
|
|
273
|
+
// repeatedly-silent assets during consolidation-selection.
|
|
274
|
+
// A no_change reflect means the LLM was invoked but found nothing to
|
|
275
|
+
// improve — the asset is stable. Track it. A successful reflect means
|
|
276
|
+
// the asset changed; reset the counter so the dampener lifts.
|
|
277
|
+
// Use the same item_ref-or-conceptId salience key as preparation/distill.
|
|
278
|
+
const plasticityKey = planned.itemRef ?? durableImproveRef(planned.ref);
|
|
279
|
+
if (isNoChange && eventsCtx?.db) {
|
|
280
|
+
try {
|
|
281
|
+
recordNoOp(eventsCtx.db, plasticityKey);
|
|
282
|
+
}
|
|
283
|
+
catch {
|
|
284
|
+
// best-effort: plasticity counter failure never blocks the run
|
|
285
|
+
}
|
|
171
286
|
}
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
}
|
|
179
|
-
try {
|
|
180
|
-
// Bug D2: distillOnlyRefs skip the reflect call but still run the distill path.
|
|
181
|
-
// Bug D1: in-loop distill-cooldown check removed — distill-cooled candidates
|
|
182
|
-
// have their synthetic actions emitted in runImprovePreparationStage.
|
|
183
|
-
const isDistillOnly = distillOnlyRefSet.has(planned.ref);
|
|
184
|
-
const parsedPlannedRef = parseAssetRef(planned.ref);
|
|
185
|
-
// B6: derived memories are machine-generated; skip reflect to avoid noisy proposals.
|
|
186
|
-
// shouldDistillMemoryRef already returns false for .derived refs, so the distill
|
|
187
|
-
// path is also a no-op for them — we just avoid unnecessary agent spawns.
|
|
188
|
-
// D2: distillOnlyRefs also skip the reflect call (reflect-cooled, distill path only).
|
|
189
|
-
if (!isDistillOnly && !planned.ref.endsWith(".derived")) {
|
|
190
|
-
// Type guard: skip reflect for unsupported types (script, env, task, etc.)
|
|
191
|
-
// and raw wiki directories, driven by the active improve profile.
|
|
192
|
-
const reflectSkip = shouldSkipRef(planned.ref, "reflect", improveProfile);
|
|
193
|
-
if (reflectSkip.skip) {
|
|
194
|
-
actions.push({
|
|
195
|
-
ref: planned.ref,
|
|
196
|
-
mode: "reflect-skipped",
|
|
197
|
-
result: { ok: true, reason: reflectSkip.reason },
|
|
198
|
-
});
|
|
287
|
+
else if (reflectResult.ok && eventsCtx?.db) {
|
|
288
|
+
try {
|
|
289
|
+
resetConsecutiveNoOps(eventsCtx.db, plasticityKey);
|
|
290
|
+
}
|
|
291
|
+
catch {
|
|
292
|
+
// best-effort
|
|
199
293
|
}
|
|
200
|
-
else {
|
|
201
|
-
// O-5 / #378: only inject reflect-originator errors into the reflect call.
|
|
202
|
-
// Cross-task errors (e.g. schema-repair) must NOT contaminate reflect prompts.
|
|
203
|
-
const reflectErrors = recentErrors.reflect ?? [];
|
|
204
|
-
if (reflectErrors.length > 0)
|
|
205
|
-
reflectsWithErrorContext++;
|
|
206
|
-
// O-1 (#364): pass remaining budget as timeoutMs so the agent spawn is
|
|
207
|
-
// bounded by the wall-clock deadline rather than the default per-profile timeout.
|
|
208
|
-
const reflectBudgetMs = remainingBudgetMs();
|
|
209
|
-
// Wire profile.processes.reflect.{mode, profile, timeoutMs} into the reflect
|
|
210
|
-
// dispatch when present. Falls back to akmReflect's own config-based resolution
|
|
211
|
-
// (profiles.improve.<name>.processes.reflect → defaults.llm) when the profile
|
|
212
|
-
// does not specify.
|
|
213
|
-
const reflectProfileRunner = resolveImproveProcessRunnerFromProfile(improveProfile.processes?.reflect, options.config ?? loadConfig());
|
|
214
|
-
const reflectCallArgs = {
|
|
215
|
-
ref: planned.ref,
|
|
216
|
-
task: options.task,
|
|
217
|
-
// Active profile so reflect's per-process reads honor `--profile`.
|
|
218
|
-
...(improveProfile ? { improveProfile } : {}),
|
|
219
|
-
...(options.stashDir ? { stashDir: options.stashDir } : {}),
|
|
220
|
-
...(reflectErrors.length > 0 ? { avoidPatterns: [...reflectErrors] } : {}),
|
|
221
|
-
agentProcess: options.agentProcess ?? "reflect",
|
|
222
|
-
eventSource: "improve",
|
|
223
|
-
// #639 — resolve the low-value filter from the ACTIVE improve profile
|
|
224
|
-
// (default off when unset), so the running profile decides instead of
|
|
225
|
-
// a hardcoded profiles.improve.default path.
|
|
226
|
-
lowValueFilter: improveProfile.processes?.reflect?.lowValueFilter?.enabled === true,
|
|
227
|
-
...(reflectBudgetMs > 0 ? { timeoutMs: reflectBudgetMs } : {}),
|
|
228
|
-
...(reflectProfileRunner ? { runner: reflectProfileRunner } : {}),
|
|
229
|
-
// Attribution: carry the eligibility lane so reflect stamps it on
|
|
230
|
-
// the reflect_invoked event and the persisted proposal.
|
|
231
|
-
...(planned.eligibilitySource ? { eligibilitySource: planned.eligibilitySource } : {}),
|
|
232
|
-
};
|
|
233
|
-
// R-2 / #389: Self-consistency multi-sample voting for high-utility refs.
|
|
234
|
-
// Self-Consistency arXiv:2203.11171 — N=3 samples beat single-shot quality.
|
|
235
|
-
const refUtility = utilityMap.get(planned.ref) ?? 0;
|
|
236
|
-
const useConsistency = refUtility >= SC_THRESHOLD && SC_N >= 2;
|
|
237
|
-
let reflectResult;
|
|
238
|
-
if (useConsistency) {
|
|
239
|
-
const samples = [];
|
|
240
|
-
for (let s = 0; s < SC_N; s++) {
|
|
241
|
-
if (remainingBudgetMs() <= 0)
|
|
242
|
-
break;
|
|
243
|
-
// draftMode: skip DB write so each sample doesn't create a proposal.
|
|
244
|
-
samples.push(await withLlmStage("reflect", () => reflectFn({ ...reflectCallArgs, draftMode: true })));
|
|
245
|
-
}
|
|
246
|
-
const winner = pickMajorityVote(samples.length > 0
|
|
247
|
-
? samples
|
|
248
|
-
: [await withLlmStage("reflect", () => reflectFn({ ...reflectCallArgs, draftMode: true }))]);
|
|
249
|
-
// Persist only the majority-vote winner as a single real proposal.
|
|
250
|
-
if (winner.ok && primaryStashDir) {
|
|
251
|
-
const persistResult = createProposal(primaryStashDir, {
|
|
252
|
-
ref: winner.proposal.ref,
|
|
253
|
-
source: "reflect",
|
|
254
|
-
sourceRun: `reflect-sc-${Date.now()}`,
|
|
255
|
-
payload: winner.proposal.payload,
|
|
256
|
-
// Attribution: the self-consistency path persists the winner here
|
|
257
|
-
// (draftMode skips reflect's own createProposal), so stamp the lane.
|
|
258
|
-
...(planned.eligibilitySource ? { eligibilitySource: planned.eligibilitySource } : {}),
|
|
259
|
-
});
|
|
260
|
-
reflectResult = isProposalSkipped(persistResult)
|
|
261
|
-
? {
|
|
262
|
-
schemaVersion: 1,
|
|
263
|
-
ok: false,
|
|
264
|
-
reason: "cooldown",
|
|
265
|
-
error: `SC proposal skipped: ${persistResult.message}`,
|
|
266
|
-
ref: winner.ref,
|
|
267
|
-
exitCode: null,
|
|
268
|
-
}
|
|
269
|
-
: { ...winner, proposal: persistResult };
|
|
270
|
-
}
|
|
271
|
-
else {
|
|
272
|
-
reflectResult = winner;
|
|
273
|
-
}
|
|
274
|
-
}
|
|
275
|
-
else {
|
|
276
|
-
reflectResult = await withLlmStage("reflect", () => reflectFn(reflectCallArgs));
|
|
277
|
-
}
|
|
278
|
-
const isCooldown = !reflectResult.ok && reflectResult.reason === "cooldown";
|
|
279
|
-
// Content-policy guard hits (reflect size-rail rejections) are NOT
|
|
280
|
-
// LLM faults — the agent responded fine, the downstream guard
|
|
281
|
-
// blocked the output. Route them to a distinct `reflect-guard-rejected`
|
|
282
|
-
// mode so health metrics can split deterministic guard hits out of
|
|
283
|
-
// true LLM failures. See
|
|
284
|
-
// `/tmp/akm-health-investigations/metrics-taxonomy-review.md` §1a.
|
|
285
|
-
const isGuardReject = !reflectResult.ok && reflectResult.reason === "content_policy_reject";
|
|
286
|
-
// Type-guard rejection (reflect refused a script/env/task ref) is
|
|
287
|
-
// also NOT an LLM failure — the LLM is never invoked. Route to the
|
|
288
|
-
// existing `reflect-skipped` bucket so it does not inflate the
|
|
289
|
-
// failure-rate numerator. ~9% of `reflect-failed` events in the
|
|
290
|
-
// user's stack were this case; see review §1a row "Reflect refused
|
|
291
|
-
// asset type".
|
|
292
|
-
const isTypeRefused = !reflectResult.ok && reflectResult.reason === "unsupported_type";
|
|
293
|
-
// Noise-gate suppression (#580): the candidate edit was an empty
|
|
294
|
-
// diff or a cosmetic-only reformat of the current asset. Like
|
|
295
|
-
// `unsupported_type`, this is a deterministic skip — not an LLM
|
|
296
|
-
// fault — so it routes to the `reflect-skipped` bucket and stays
|
|
297
|
-
// out of recentErrors/avoidPatterns.
|
|
298
|
-
const isNoChange = !reflectResult.ok && reflectResult.reason === "no_change";
|
|
299
|
-
actions.push({
|
|
300
|
-
ref: planned.ref,
|
|
301
|
-
mode: reflectResult.ok
|
|
302
|
-
? "reflect"
|
|
303
|
-
: isCooldown
|
|
304
|
-
? "reflect-cooldown"
|
|
305
|
-
: isGuardReject
|
|
306
|
-
? "reflect-guard-rejected"
|
|
307
|
-
: isTypeRefused || isNoChange
|
|
308
|
-
? "reflect-skipped"
|
|
309
|
-
: "reflect-failed",
|
|
310
|
-
result: reflectResult,
|
|
311
|
-
});
|
|
312
|
-
// Cooldown skips, guard rejects, type-refused skips, and noise-gate
|
|
313
|
-
// skips are not failures — do not pollute recentErrors with them
|
|
314
|
-
// (those get injected as `avoidPatterns` into the next reflect
|
|
315
|
-
// prompt). Guard rejects ARE worth showing the LLM as a learn-signal
|
|
316
|
-
// so the next iteration sees "your last expansion was too large";
|
|
317
|
-
// type-refused and no-change are deterministic and add no learning
|
|
318
|
-
// signal.
|
|
319
|
-
if (!reflectResult.ok && !isCooldown && !isTypeRefused && !isNoChange) {
|
|
320
|
-
const errMsg = reflectResult.error ?? reflectResult.reason ?? "unknown reflect error";
|
|
321
|
-
pushRecentError("reflect", errMsg);
|
|
322
|
-
}
|
|
323
|
-
// improve_reflect_outcome — per-asset metric for tuning the reflect path.
|
|
324
|
-
appendEvent({
|
|
325
|
-
eventType: "improve_reflect_outcome",
|
|
326
|
-
ref: planned.ref,
|
|
327
|
-
metadata: {
|
|
328
|
-
ok: reflectResult.ok,
|
|
329
|
-
durationMs: reflectResult.ok ? reflectResult.durationMs : undefined,
|
|
330
|
-
agentProfile: reflectResult.ok ? reflectResult.agentProfile : undefined,
|
|
331
|
-
reason: reflectResult.ok ? undefined : reflectResult.reason,
|
|
332
|
-
},
|
|
333
|
-
}, eventsCtx);
|
|
334
|
-
// Plasticity counter (plan §WS-1 step 8): record no-ops so the
|
|
335
|
-
// WS-1 selection comparator (effectiveScore, ~line 3073) can dampen
|
|
336
|
-
// repeatedly-silent assets during consolidation-selection.
|
|
337
|
-
// A no_change reflect means the LLM was invoked but found nothing to
|
|
338
|
-
// improve — the asset is stable. Track it. A successful reflect means
|
|
339
|
-
// the asset changed; reset the counter so the dampener lifts.
|
|
340
|
-
if (isNoChange && eventsCtx?.db) {
|
|
341
|
-
try {
|
|
342
|
-
recordNoOp(eventsCtx.db, planned.ref);
|
|
343
|
-
}
|
|
344
|
-
catch {
|
|
345
|
-
// best-effort: plasticity counter failure never blocks the run
|
|
346
|
-
}
|
|
347
|
-
}
|
|
348
|
-
else if (reflectResult.ok && eventsCtx?.db) {
|
|
349
|
-
try {
|
|
350
|
-
resetConsecutiveNoOps(eventsCtx.db, planned.ref);
|
|
351
|
-
}
|
|
352
|
-
catch {
|
|
353
|
-
// best-effort
|
|
354
|
-
}
|
|
355
|
-
}
|
|
356
|
-
if (reflectResult.ok) {
|
|
357
|
-
const reflectGr = await runAutoAcceptGate([{ proposalId: reflectResult.proposal.id, confidence: reflectResult.proposal.confidence }], reflectGateCfg);
|
|
358
|
-
gateAutoAcceptedCount += reflectGr.promoted.length;
|
|
359
|
-
gateAutoAcceptFailedCount += reflectGr.failed.length;
|
|
360
|
-
}
|
|
361
|
-
} // end else (reflect type/profile check)
|
|
362
294
|
}
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
295
|
+
} // end else (reflect type/profile check)
|
|
296
|
+
}
|
|
297
|
+
else if (!isDistillOnly && planned.ref.endsWith(".derived")) {
|
|
298
|
+
// B6: .derived refs skip reflect; record synthetic skip action.
|
|
299
|
+
tally.actions.push({
|
|
300
|
+
ref: planned.ref,
|
|
301
|
+
mode: "distill-skipped",
|
|
302
|
+
result: { ok: true, reason: "derived-memory-reflect-skipped" },
|
|
303
|
+
});
|
|
304
|
+
appendEvent({
|
|
305
|
+
eventType: "improve_skipped",
|
|
306
|
+
ref: planned.ref,
|
|
307
|
+
metadata: { reason: "derived_memory_reflect_skipped" },
|
|
308
|
+
}, eventsCtx);
|
|
309
|
+
}
|
|
310
|
+
}
|
|
311
|
+
/**
|
|
312
|
+
* Distill half of one loop iteration: the profile / requirePlannedRefs /
|
|
313
|
+
* candidate-type / weak-signal / cooldown gates, then the pending-proposal and
|
|
314
|
+
* reject-grace dedup checks, then {@link invokeDistillAndRecord}. Each gate
|
|
315
|
+
* that was a `continue` in the old inline loop body is an early `return` here.
|
|
316
|
+
*/
|
|
317
|
+
async function runLoopDistillPass(planned, parsedPlannedRef, isDistillOnly, env, tally) {
|
|
318
|
+
const { options, primaryStashDir, eventsCtx, improveProfile } = env;
|
|
319
|
+
const hasRecentFeedbackSignal = env.signalBearingSet.has(planned.ref);
|
|
320
|
+
const explicitRefScope = env.scope.mode === "ref";
|
|
321
|
+
// Profile gate: apply the full type-filter / raw-wiki / disabled rules to
|
|
322
|
+
// distill so callers who configure `profile.processes.distill.allowedTypes`
|
|
323
|
+
// or land on raw-wiki refs get a recorded skip action instead of silently
|
|
324
|
+
// proceeding.
|
|
325
|
+
const distillSkip = shouldSkipRef(planned.ref, "distill", improveProfile);
|
|
326
|
+
if (distillSkip.skip) {
|
|
327
|
+
tally.actions.push({
|
|
328
|
+
ref: planned.ref,
|
|
329
|
+
mode: "distill-skipped",
|
|
330
|
+
result: { ok: true, reason: distillSkip.reason },
|
|
331
|
+
});
|
|
332
|
+
return;
|
|
333
|
+
}
|
|
334
|
+
// requirePlannedRefs guard: skip distill for distill-only refs when no
|
|
335
|
+
// reflect-eligible refs were planned this run, preventing mass skip events.
|
|
336
|
+
if (env.skipDistillDueToRequirePlannedRefs && isDistillOnly) {
|
|
337
|
+
tally.actions.push({
|
|
338
|
+
ref: planned.ref,
|
|
339
|
+
mode: "distill-skipped",
|
|
340
|
+
result: { ok: true, reason: "require_planned_refs" },
|
|
341
|
+
});
|
|
342
|
+
return;
|
|
343
|
+
}
|
|
344
|
+
// See `isDistillCandidateRef` — excludes `lesson:*` (and anything else in
|
|
345
|
+
// DISTILL_REFUSED_INPUT_TYPES) so distill never gets queued for an input
|
|
346
|
+
// it will refuse.
|
|
347
|
+
const shouldAttemptDistill = isDistillCandidateRef(planned.ref, options.stashDir);
|
|
348
|
+
const skipMemoryDistillForWeakSignal = !isDistillOnly && parsedPlannedRef.type === "memory" && !hasRecentFeedbackSignal && !explicitRefScope;
|
|
349
|
+
// distillCooledRefs guard: pre-filter emitted synthetic actions for distill-candidate
|
|
350
|
+
// refs; non-candidate refs in the set are blocked here.
|
|
351
|
+
// O-2 (#365): bypass the distill cooldown when the user explicitly targeted
|
|
352
|
+
// this ref via --scope — their intent overrides unattended-run policies.
|
|
353
|
+
if (shouldAttemptDistill &&
|
|
354
|
+
!skipMemoryDistillForWeakSignal &&
|
|
355
|
+
(!env.distillCooledRefs.has(planned.ref) || explicitRefScope)) {
|
|
356
|
+
// TODO(refactor): single call site needs both lesson+knowledge refs for proposal dedup. If a third target ref type is added, extract deriveAllTargetRefs(inputRef): string[].
|
|
357
|
+
const lessonRef = deriveLessonRef(planned.ref);
|
|
358
|
+
const knowledgeRef = deriveKnowledgeRef(planned.ref);
|
|
359
|
+
const dedupeStashDir = primaryStashDir ?? options.stashDir;
|
|
360
|
+
if (dedupeStashDir) {
|
|
361
|
+
// B2: check both lesson ref and knowledge ref since auto-promoted memories
|
|
362
|
+
// create knowledge: proposals, not lesson: proposals.
|
|
363
|
+
const hasExistingPending = env.pendingProposalRefSet.has(lessonRef) || env.pendingProposalRefSet.has(knowledgeRef);
|
|
364
|
+
if (hasExistingPending) {
|
|
365
|
+
tally.actions.push({
|
|
366
366
|
ref: planned.ref,
|
|
367
367
|
mode: "distill-skipped",
|
|
368
|
-
result: { ok: true, reason: "
|
|
368
|
+
result: { ok: true, reason: "pending proposal exists" },
|
|
369
369
|
});
|
|
370
370
|
appendEvent({
|
|
371
371
|
eventType: "improve_skipped",
|
|
372
372
|
ref: planned.ref,
|
|
373
|
-
metadata: { reason: "
|
|
373
|
+
metadata: { reason: "pending_proposal_exists" },
|
|
374
374
|
}, eventsCtx);
|
|
375
|
+
return;
|
|
375
376
|
}
|
|
376
|
-
//
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
//
|
|
380
|
-
//
|
|
381
|
-
//
|
|
382
|
-
//
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
}
|
|
394
|
-
// requirePlannedRefs guard: skip distill for distill-only refs when no
|
|
395
|
-
// reflect-eligible refs were planned this run, preventing mass skip events.
|
|
396
|
-
if (skipDistillDueToRequirePlannedRefs && isDistillOnly) {
|
|
397
|
-
actions.push({
|
|
398
|
-
ref: planned.ref,
|
|
399
|
-
mode: "distill-skipped",
|
|
400
|
-
result: { ok: true, reason: "require_planned_refs" },
|
|
401
|
-
});
|
|
402
|
-
completedCount++;
|
|
403
|
-
info(`[improve] ${completedCount}/${loopRefs.length} ${planned.ref}`);
|
|
404
|
-
continue;
|
|
405
|
-
}
|
|
406
|
-
// See `isDistillCandidateRef` — excludes `lesson:*` (and anything else in
|
|
407
|
-
// DISTILL_REFUSED_INPUT_TYPES) so distill never gets queued for an input
|
|
408
|
-
// it will refuse.
|
|
409
|
-
const shouldAttemptDistill = isDistillCandidateRef(planned.ref, options.stashDir);
|
|
410
|
-
const skipMemoryDistillForWeakSignal = !isDistillOnly && parsedPlannedRef.type === "memory" && !hasRecentFeedbackSignal && !explicitRefScope;
|
|
411
|
-
// distillCooledRefs guard: pre-filter emitted synthetic actions for distill-candidate
|
|
412
|
-
// refs; non-candidate refs in the set are blocked here.
|
|
413
|
-
// O-2 (#365): bypass the distill cooldown when the user explicitly targeted
|
|
414
|
-
// this ref via --scope — their intent overrides unattended-run policies.
|
|
415
|
-
if (shouldAttemptDistill &&
|
|
416
|
-
!skipMemoryDistillForWeakSignal &&
|
|
417
|
-
(!distillCooledRefs.has(planned.ref) || explicitRefScope)) {
|
|
418
|
-
// TODO(refactor): single call site needs both lesson+knowledge refs for proposal dedup. If a third target ref type is added, extract deriveAllTargetRefs(inputRef): string[].
|
|
419
|
-
const lessonRef = deriveLessonRef(planned.ref);
|
|
420
|
-
const knowledgeRef = deriveKnowledgeRef(planned.ref);
|
|
421
|
-
const dedupeStashDir = primaryStashDir ?? options.stashDir;
|
|
422
|
-
if (dedupeStashDir) {
|
|
423
|
-
// B2: check both lesson ref and knowledge ref since auto-promoted memories
|
|
424
|
-
// create knowledge: proposals, not lesson: proposals.
|
|
425
|
-
const hasExistingPending = pendingProposalRefSet.has(lessonRef) || pendingProposalRefSet.has(knowledgeRef);
|
|
426
|
-
if (hasExistingPending) {
|
|
427
|
-
actions.push({
|
|
428
|
-
ref: planned.ref,
|
|
429
|
-
mode: "distill-skipped",
|
|
430
|
-
result: { ok: true, reason: "pending proposal exists" },
|
|
431
|
-
});
|
|
432
|
-
appendEvent({
|
|
433
|
-
eventType: "improve_skipped",
|
|
434
|
-
ref: planned.ref,
|
|
435
|
-
metadata: { reason: "pending_proposal_exists" },
|
|
436
|
-
}, eventsCtx);
|
|
437
|
-
completedCount++;
|
|
438
|
-
info(`[improve] ${completedCount}/${loopRefs.length} ${planned.ref}`);
|
|
439
|
-
continue;
|
|
440
|
-
}
|
|
441
|
-
// D-2 (#370): reject-aware cooldown for distill. When the reviewer
|
|
442
|
-
// recently rejected a distilled lesson or knowledge proposal for this
|
|
443
|
-
// asset, skip re-distillation for a 1-day grace window. Prevents the
|
|
444
|
-
// same rejected proposal from being regenerated immediately. The
|
|
445
|
-
// window is fixed (the 0.8.0 redesign moved per-ref cooldowns to
|
|
446
|
-
// signal-delta gates and dropped --distill-cooldown-days; a short
|
|
447
|
-
// reject grace is preserved here so a fresh rejection isn't
|
|
448
|
-
// overridden by the same run).
|
|
449
|
-
// References: ExpeL arXiv:2308.10144, STaR arXiv:2203.14465.
|
|
450
|
-
const DISTILL_REJECT_COOLDOWN_MS = daysToMs(1);
|
|
451
|
-
const recentlyRejectedLesson = !explicitRefScope && // O-2: bypass when --scope <ref> is explicit
|
|
452
|
-
(rejectedProposalsByRef.has(lessonRef) || rejectedProposalsByRef.has(knowledgeRef));
|
|
453
|
-
if (recentlyRejectedLesson) {
|
|
454
|
-
const rejectedEntry = rejectedProposalsByRef.get(lessonRef) ?? rejectedProposalsByRef.get(knowledgeRef);
|
|
455
|
-
const rejectedAgeMs = rejectedEntry ? Date.now() - new Date(rejectedEntry.ts).getTime() : 0;
|
|
456
|
-
if (rejectedAgeMs < DISTILL_REJECT_COOLDOWN_MS) {
|
|
457
|
-
actions.push({
|
|
458
|
-
ref: planned.ref,
|
|
459
|
-
mode: "distill-skipped",
|
|
460
|
-
result: { ok: true, reason: "distill reject grace window" },
|
|
461
|
-
});
|
|
462
|
-
appendEvent({
|
|
463
|
-
eventType: "improve_skipped",
|
|
464
|
-
ref: planned.ref,
|
|
465
|
-
metadata: {
|
|
466
|
-
reason: "distill_reject_grace_window",
|
|
467
|
-
},
|
|
468
|
-
}, eventsCtx);
|
|
469
|
-
completedCount++;
|
|
470
|
-
info(`[improve] ${completedCount}/${loopRefs.length} ${planned.ref}`);
|
|
471
|
-
continue;
|
|
472
|
-
}
|
|
473
|
-
}
|
|
474
|
-
}
|
|
475
|
-
const distillResult = await withLlmStage("distill", () => distillFn({
|
|
476
|
-
ref: planned.ref,
|
|
477
|
-
...(parsedPlannedRef.type === "memory" ? { proposalKind: "auto" } : {}),
|
|
478
|
-
...(options.stashDir ? { stashDir: options.stashDir } : {}),
|
|
479
|
-
// Active profile so distill's per-process reads honor `--profile`.
|
|
480
|
-
...(improveProfile ? { improveProfile } : {}),
|
|
481
|
-
// Attribution: carry the eligibility lane so distill stamps it on the
|
|
482
|
-
// distill_invoked event and the persisted proposal.
|
|
483
|
-
...(planned.eligibilitySource ? { eligibilitySource: planned.eligibilitySource } : {}),
|
|
484
|
-
}));
|
|
485
|
-
actions.push({ ref: planned.ref, mode: "distill", result: distillResult });
|
|
486
|
-
if (distillResult.outcome === "queued" && distillResult.proposal) {
|
|
487
|
-
const distillGr = await runAutoAcceptGate([{ proposalId: distillResult.proposal.id, confidence: distillResult.proposal.confidence }], distillGateCfg);
|
|
488
|
-
gateAutoAcceptedCount += distillGr.promoted.length;
|
|
489
|
-
gateAutoAcceptFailedCount += distillGr.failed.length;
|
|
490
|
-
}
|
|
491
|
-
if (parsedPlannedRef.type === "memory") {
|
|
492
|
-
const promotedToKnowledge = distillResult.outcome === "queued" && distillResult.proposalKind === "knowledge";
|
|
493
|
-
if (!promotedToKnowledge)
|
|
494
|
-
memoryRefsForInference.add(planned.ref);
|
|
495
|
-
}
|
|
496
|
-
// Plasticity counter (plan §WS-1 step 8) for the distill path.
|
|
497
|
-
// quality_rejected: the LLM ran but produced output that didn't pass the
|
|
498
|
-
// quality gate — the asset is not yielding useful distill output.
|
|
499
|
-
// queued: a proposal was produced; reset the no-op counter.
|
|
500
|
-
if (eventsCtx?.db) {
|
|
501
|
-
try {
|
|
502
|
-
if (distillResult.outcome === "quality_rejected" || distillResult.outcome === "skipped") {
|
|
503
|
-
recordNoOp(eventsCtx.db, planned.ref);
|
|
504
|
-
}
|
|
505
|
-
else if (distillResult.outcome === "queued") {
|
|
506
|
-
resetConsecutiveNoOps(eventsCtx.db, planned.ref);
|
|
507
|
-
}
|
|
508
|
-
}
|
|
509
|
-
catch {
|
|
510
|
-
// best-effort: plasticity counter failure never blocks the run
|
|
511
|
-
}
|
|
512
|
-
}
|
|
513
|
-
if (distillResult.outcome === "quality_rejected" && primaryStashDir) {
|
|
514
|
-
const slug = refSlug(planned.ref);
|
|
515
|
-
writeEvalCase(primaryStashDir, {
|
|
377
|
+
// D-2 (#370): reject-aware cooldown for distill. When the reviewer
|
|
378
|
+
// recently rejected a distilled lesson or knowledge proposal for this
|
|
379
|
+
// asset, skip re-distillation for a 1-day grace window. Prevents the
|
|
380
|
+
// same rejected proposal from being regenerated immediately. The
|
|
381
|
+
// window is fixed (the 0.8.0 redesign moved per-ref cooldowns to
|
|
382
|
+
// signal-delta gates and dropped --distill-cooldown-days; a short
|
|
383
|
+
// reject grace is preserved here so a fresh rejection isn't
|
|
384
|
+
// overridden by the same run).
|
|
385
|
+
// References: ExpeL arXiv:2308.10144, STaR arXiv:2203.14465.
|
|
386
|
+
const DISTILL_REJECT_COOLDOWN_MS = daysToMs(1);
|
|
387
|
+
const recentlyRejectedLesson = !explicitRefScope && // O-2: bypass when --scope <ref> is explicit
|
|
388
|
+
(env.rejectedProposalsByRef.has(lessonRef) || env.rejectedProposalsByRef.has(knowledgeRef));
|
|
389
|
+
if (recentlyRejectedLesson) {
|
|
390
|
+
const rejectedEntry = env.rejectedProposalsByRef.get(lessonRef) ?? env.rejectedProposalsByRef.get(knowledgeRef);
|
|
391
|
+
const rejectedAgeMs = rejectedEntry ? Date.now() - new Date(rejectedEntry.ts).getTime() : 0;
|
|
392
|
+
if (rejectedAgeMs < DISTILL_REJECT_COOLDOWN_MS) {
|
|
393
|
+
tally.actions.push({
|
|
516
394
|
ref: planned.ref,
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
rejectedAt: Date.now(),
|
|
520
|
-
source: "distill_quality_rejected",
|
|
521
|
-
slug: `${slug}-${Date.now()}`,
|
|
395
|
+
mode: "distill-skipped",
|
|
396
|
+
result: { ok: true, reason: "distill reject grace window" },
|
|
522
397
|
});
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
const rejectedProposalEvent = rejectedProposalsByRef.get(planned.ref);
|
|
526
|
-
if (rejectedProposalEvent && primaryStashDir) {
|
|
527
|
-
const slug = refSlug(planned.ref);
|
|
528
|
-
writeEvalCase(primaryStashDir, {
|
|
398
|
+
appendEvent({
|
|
399
|
+
eventType: "improve_skipped",
|
|
529
400
|
ref: planned.ref,
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
});
|
|
401
|
+
metadata: {
|
|
402
|
+
reason: "distill_reject_grace_window",
|
|
403
|
+
},
|
|
404
|
+
}, eventsCtx);
|
|
405
|
+
return;
|
|
536
406
|
}
|
|
537
407
|
}
|
|
538
|
-
else if (skipMemoryDistillForWeakSignal) {
|
|
539
|
-
actions.push({
|
|
540
|
-
ref: planned.ref,
|
|
541
|
-
mode: "distill-skipped",
|
|
542
|
-
result: { ok: true, reason: "memory requires recent feedback signal" },
|
|
543
|
-
});
|
|
544
|
-
appendEvent({
|
|
545
|
-
eventType: "improve_skipped",
|
|
546
|
-
ref: planned.ref,
|
|
547
|
-
metadata: { reason: "memory_distill_requires_feedback" },
|
|
548
|
-
}, eventsCtx);
|
|
549
|
-
}
|
|
550
408
|
}
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
409
|
+
await invokeDistillAndRecord(planned, parsedPlannedRef, env, tally);
|
|
410
|
+
}
|
|
411
|
+
else if (skipMemoryDistillForWeakSignal) {
|
|
412
|
+
tally.actions.push({
|
|
413
|
+
ref: planned.ref,
|
|
414
|
+
mode: "distill-skipped",
|
|
415
|
+
result: { ok: true, reason: "memory requires recent feedback signal" },
|
|
416
|
+
});
|
|
417
|
+
appendEvent({
|
|
418
|
+
eventType: "improve_skipped",
|
|
419
|
+
ref: planned.ref,
|
|
420
|
+
metadata: { reason: "memory_distill_requires_feedback" },
|
|
421
|
+
}, eventsCtx);
|
|
422
|
+
}
|
|
423
|
+
}
|
|
424
|
+
/**
|
|
425
|
+
* The distill invocation for one ref that passed every gate: the `distillFn`
|
|
426
|
+
* call, memory-inference queueing, plasticity counters, and the
|
|
427
|
+
* quality-rejected / proposal-rejected eval-case writes.
|
|
428
|
+
*/
|
|
429
|
+
async function invokeDistillAndRecord(planned, parsedPlannedRef, env, tally) {
|
|
430
|
+
const { options, primaryStashDir, distillFn, eventsCtx, improveProfile, resolvedPlan, budgetSignal } = env;
|
|
431
|
+
const distillResult = await withLlmStage("distill", () => distillFn({
|
|
432
|
+
ref: planned.ref,
|
|
433
|
+
// Carry the resolved item_ref so distill matches preparation's state key.
|
|
434
|
+
...(planned.itemRef ? { itemRef: planned.itemRef } : {}),
|
|
435
|
+
...(parsedPlannedRef.type === "memory" ? { proposalKind: "auto" } : {}),
|
|
436
|
+
...(primaryStashDir ? { stashDir: primaryStashDir } : {}),
|
|
437
|
+
// Active profile so distill's per-process reads honor `--profile`.
|
|
438
|
+
...(improveProfile ? { improveProfile } : {}),
|
|
439
|
+
config: options.config,
|
|
440
|
+
llmConfig: resolvedPlan.processes.distill.runner
|
|
441
|
+
? materializeLlmRunnerConnection(resolvedPlan.processes.distill.runner)
|
|
442
|
+
: null,
|
|
443
|
+
signal: budgetSignal,
|
|
444
|
+
// R25: distill's event emits reuse the run's long-lived state.db handle.
|
|
445
|
+
eventsCtx,
|
|
446
|
+
// Attribution: carry the eligibility lane so distill stamps it on the
|
|
447
|
+
// distill_invoked event and the persisted proposal.
|
|
448
|
+
...(planned.eligibilitySource ? { eligibilitySource: planned.eligibilitySource } : {}),
|
|
449
|
+
}), { engine: resolvedPlan.processes.distill.runner?.engine, process: "distill" });
|
|
450
|
+
tally.actions.push({ ref: planned.ref, mode: "distill", result: distillResult });
|
|
451
|
+
if (parsedPlannedRef.type === "memory") {
|
|
452
|
+
const promotedToKnowledge = distillResult.outcome === "queued" && distillResult.proposalKind === "knowledge";
|
|
453
|
+
if (!promotedToKnowledge)
|
|
454
|
+
tally.memoryRefsForInference.push(planned.ref);
|
|
455
|
+
}
|
|
456
|
+
// Plasticity counter (plan §WS-1 step 8) for the distill path.
|
|
457
|
+
// quality_rejected: the LLM ran but produced output that didn't pass the
|
|
458
|
+
// quality gate — the asset is not yielding useful distill output.
|
|
459
|
+
// queued: a proposal was produced; reset the no-op counter.
|
|
460
|
+
if (eventsCtx?.db) {
|
|
461
|
+
// Use the same item_ref-or-conceptId key as the distill/preparation writers.
|
|
462
|
+
const plasticityKey = planned.itemRef ?? durableImproveRef(planned.ref);
|
|
463
|
+
try {
|
|
464
|
+
if (distillResult.outcome === "quality_rejected" || distillResult.outcome === "skipped") {
|
|
465
|
+
recordNoOp(eventsCtx.db, plasticityKey);
|
|
561
466
|
}
|
|
562
|
-
else {
|
|
563
|
-
|
|
564
|
-
ref: planned.ref,
|
|
565
|
-
mode: "error",
|
|
566
|
-
result: { ok: false, error: errMessage(err) },
|
|
567
|
-
});
|
|
467
|
+
else if (distillResult.outcome === "queued") {
|
|
468
|
+
resetConsecutiveNoOps(eventsCtx.db, plasticityKey);
|
|
568
469
|
}
|
|
569
470
|
}
|
|
570
|
-
|
|
571
|
-
|
|
471
|
+
catch {
|
|
472
|
+
// best-effort: plasticity counter failure never blocks the run
|
|
473
|
+
}
|
|
572
474
|
}
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
};
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
475
|
+
if (distillResult.outcome === "quality_rejected" && primaryStashDir) {
|
|
476
|
+
const slug = refSlug(planned.ref);
|
|
477
|
+
writeEvalCase(primaryStashDir, {
|
|
478
|
+
ref: planned.ref,
|
|
479
|
+
failureReason: distillResult.reason ?? "quality gate rejected",
|
|
480
|
+
assetType: parseRefInput(planned.ref).type ?? "unknown",
|
|
481
|
+
rejectedAt: Date.now(),
|
|
482
|
+
source: "distill_quality_rejected",
|
|
483
|
+
slug: `${slug}-${Date.now()}`,
|
|
484
|
+
});
|
|
485
|
+
}
|
|
486
|
+
// D6: use pre-loaded map instead of per-iteration DB query
|
|
487
|
+
const rejectedProposalEvent = env.rejectedProposalsByRef.get(planned.ref);
|
|
488
|
+
if (rejectedProposalEvent && primaryStashDir) {
|
|
489
|
+
const slug = refSlug(planned.ref);
|
|
490
|
+
writeEvalCase(primaryStashDir, {
|
|
491
|
+
ref: planned.ref,
|
|
492
|
+
failureReason: rejectedProposalEvent.metadata?.reason ?? "proposal rejected",
|
|
493
|
+
assetType: parseRefInput(planned.ref).type ?? "unknown",
|
|
494
|
+
rejectedAt: new Date(rejectedProposalEvent.ts).getTime(),
|
|
495
|
+
source: "proposal_rejected",
|
|
496
|
+
slug: `${slug}-rejected`,
|
|
497
|
+
});
|
|
498
|
+
}
|
|
499
|
+
}
|
|
500
|
+
/**
|
|
501
|
+
* Wall-clock budget exhausted mid-loop (O-1 / #364): emit the improve_skipped
|
|
502
|
+
* events for the current and remaining refs (B11) and return the terminal
|
|
503
|
+
* error action for the orchestrator to record before breaking out of the loop.
|
|
504
|
+
*/
|
|
505
|
+
function recordBudgetExhausted(args) {
|
|
506
|
+
const { planned, loopRefs, completedCount, startMs, eventsCtx } = args;
|
|
507
|
+
const remaining = loopRefs.length - completedCount;
|
|
508
|
+
info(`[improve] budget exhausted after ${Math.round((Date.now() - startMs) / 60000)}min — ${remaining} assets skipped`);
|
|
509
|
+
appendEvent({
|
|
510
|
+
eventType: "improve_skipped",
|
|
511
|
+
ref: planned.ref,
|
|
512
|
+
metadata: {
|
|
513
|
+
reason: "budget_exhausted",
|
|
514
|
+
remaining,
|
|
515
|
+
},
|
|
516
|
+
}, eventsCtx);
|
|
517
|
+
// B11: Emit improve_skipped for all remaining assets that will not be processed.
|
|
518
|
+
for (const remainingRef of loopRefs.slice(completedCount + 1)) {
|
|
519
|
+
appendEvent({
|
|
520
|
+
eventType: "improve_skipped",
|
|
521
|
+
ref: remainingRef.ref,
|
|
522
|
+
metadata: { reason: "budget_exhausted_batch", remaining: loopRefs.length - completedCount - 1 },
|
|
523
|
+
}, eventsCtx);
|
|
524
|
+
}
|
|
525
|
+
return {
|
|
526
|
+
ref: planned.ref,
|
|
527
|
+
mode: "error",
|
|
528
|
+
result: { ok: false, error: "timeout: improve wall-clock budget exhausted" },
|
|
529
|
+
};
|
|
530
|
+
}
|
|
531
|
+
export async function runImproveLoopStage(args) {
|
|
532
|
+
const { ctx, loopRefs, actions, recentErrors, startMs, budgetMs } = args;
|
|
533
|
+
const eventsCtx = ctx.eventsCtx;
|
|
534
|
+
const env = prepareImproveLoopEnv(args);
|
|
535
|
+
let completedCount = 0;
|
|
536
|
+
let reflectsWithErrorContext = 0;
|
|
537
|
+
const memoryRefsForInference = new Set();
|
|
538
|
+
for (const planned of loopRefs) {
|
|
539
|
+
if (Date.now() - startMs >= budgetMs) {
|
|
540
|
+
actions.push(recordBudgetExhausted({ planned, loopRefs, completedCount, startMs, eventsCtx }));
|
|
541
|
+
break;
|
|
591
542
|
}
|
|
543
|
+
const tally = await processImproveLoopRef(planned, env);
|
|
544
|
+
// Fold the per-ref tally into run-level state — the passes never touch it.
|
|
545
|
+
actions.push(...tally.actions);
|
|
546
|
+
for (const push of tally.recentErrorPushes)
|
|
547
|
+
pushRecentError(recentErrors, push.originator, push.message);
|
|
548
|
+
reflectsWithErrorContext += tally.reflectsWithErrorContext;
|
|
549
|
+
for (const ref of tally.memoryRefsForInference)
|
|
550
|
+
memoryRefsForInference.add(ref);
|
|
551
|
+
completedCount++;
|
|
552
|
+
info(`[improve] ${completedCount}/${loopRefs.length} ${planned.ref}`);
|
|
592
553
|
}
|
|
593
|
-
return { reflectsWithErrorContext, memoryRefsForInference
|
|
554
|
+
return { reflectsWithErrorContext, memoryRefsForInference };
|
|
594
555
|
}
|
|
595
556
|
export async function runImprovePostLoopStage(args) {
|
|
596
|
-
const { scope, options, primaryStashDir, actionableRefs, appliedCleanup, cleanupWarnings, memoryRefsForInference, reindexFn, eventsCtx, budgetSignal, improveProfile, consolidationRan, } = args;
|
|
557
|
+
const { scope, options, primaryStashDir, actionableRefs, appliedCleanup, cleanupWarnings, memoryRefsForInference, reindexFn, eventsCtx, budgetSignal, improveProfile, resolvedPlan, consolidationRan, } = args;
|
|
597
558
|
const allWarnings = [...cleanupWarnings, ...(appliedCleanup?.warnings ?? [])];
|
|
598
559
|
info("[improve] post-loop maintenance starting");
|
|
599
560
|
const maintenanceResult = await runImproveMaintenancePasses({
|
|
@@ -608,6 +569,7 @@ export async function runImprovePostLoopStage(args) {
|
|
|
608
569
|
budgetSignal,
|
|
609
570
|
eventsCtx,
|
|
610
571
|
improveProfile,
|
|
572
|
+
resolvedPlan,
|
|
611
573
|
});
|
|
612
574
|
let deadUrls;
|
|
613
575
|
if (scope.mode === "all" && primaryStashDir && actionableRefs.length > 0) {
|
|
@@ -615,14 +577,28 @@ export async function runImprovePostLoopStage(args) {
|
|
|
615
577
|
const knowledgeEntries = actionableRefs
|
|
616
578
|
.filter((r) => {
|
|
617
579
|
try {
|
|
618
|
-
return
|
|
580
|
+
return parseRefInput(r.ref).type === "knowledge";
|
|
619
581
|
}
|
|
620
582
|
catch {
|
|
621
583
|
return false;
|
|
622
584
|
}
|
|
623
585
|
})
|
|
624
586
|
.slice(0, 10)
|
|
625
|
-
.map((r) =>
|
|
587
|
+
.map((r) => {
|
|
588
|
+
// The URL scan needs the document body; filePath is pre-resolved on
|
|
589
|
+
// eligible refs at planning time (#591). Best-effort — an unreadable
|
|
590
|
+
// or unresolved file contributes no URLs, same as before.
|
|
591
|
+
let body = "";
|
|
592
|
+
if (r.filePath) {
|
|
593
|
+
try {
|
|
594
|
+
body = fs.readFileSync(r.filePath, "utf8");
|
|
595
|
+
}
|
|
596
|
+
catch {
|
|
597
|
+
// best-effort
|
|
598
|
+
}
|
|
599
|
+
}
|
|
600
|
+
return { ref: r.ref, body };
|
|
601
|
+
});
|
|
626
602
|
if (knowledgeEntries.length > 0) {
|
|
627
603
|
info(`[improve] checking URLs in ${knowledgeEntries.length} knowledge refs`);
|
|
628
604
|
deadUrls = await checkDeadUrls(primaryStashDir, knowledgeEntries);
|
|
@@ -633,86 +609,17 @@ export async function runImprovePostLoopStage(args) {
|
|
|
633
609
|
// best-effort
|
|
634
610
|
}
|
|
635
611
|
}
|
|
636
|
-
// #609 — recombine / synthesize pass. Whole-corpus cross-episodic
|
|
637
|
-
// generalization. Runs in the post-loop stage under consolidate.lock (it
|
|
638
|
-
// reads the consolidated corpus and writes proposals). Opt-in: gated on the
|
|
639
|
-
// `recombine` process being enabled, whole-stash / type scope (never `ref`),
|
|
640
|
-
// and not a dry run. Mirrors the proactiveMaintenance opt-in wiring.
|
|
641
|
-
let recombination;
|
|
642
|
-
if (primaryStashDir &&
|
|
643
|
-
improveProfile &&
|
|
644
|
-
resolveProcessEnabled("recombine", improveProfile) &&
|
|
645
|
-
scope.mode !== "ref" &&
|
|
646
|
-
!options.dryRun) {
|
|
647
|
-
const recombineFn = options.recombineFn ?? akmRecombine;
|
|
648
|
-
try {
|
|
649
|
-
recombination = await recombineFn({
|
|
650
|
-
stashDir: primaryStashDir,
|
|
651
|
-
config: options.config ?? loadConfig(),
|
|
652
|
-
improveProfile,
|
|
653
|
-
...(options.runId ? { sourceRun: options.runId } : {}),
|
|
654
|
-
...(budgetSignal ? { signal: budgetSignal } : {}),
|
|
655
|
-
eligibilitySource: "recombine",
|
|
656
|
-
...(eventsCtx ? { ctx: eventsCtx } : {}),
|
|
657
|
-
minClusterSize: improveProfile.processes?.recombine?.minClusterSize,
|
|
658
|
-
maxClustersPerRun: improveProfile.processes?.recombine?.maxClustersPerRun,
|
|
659
|
-
relatednessSource: improveProfile.processes?.recombine?.relatednessSource,
|
|
660
|
-
confirmThreshold: improveProfile.processes?.recombine?.confirmThreshold,
|
|
661
|
-
// #632 — clustering-tuning knobs. UNSET = pre-#632 behaviour.
|
|
662
|
-
maxClusterSize: improveProfile.processes?.recombine?.maxClusterSize,
|
|
663
|
-
excludeTags: improveProfile.processes?.recombine?.excludeTags,
|
|
664
|
-
excludeEntities: improveProfile.processes?.recombine?.excludeEntities,
|
|
665
|
-
});
|
|
666
|
-
}
|
|
667
|
-
catch (e) {
|
|
668
|
-
allWarnings.push(`recombine: ${String(e)}`);
|
|
669
|
-
}
|
|
670
|
-
}
|
|
671
|
-
// #615 — procedural-compilation pass. Detects recurring successful ordered
|
|
672
|
-
// action sequences and compiles them into workflow proposals. Opt-in: gated
|
|
673
|
-
// on the `procedural` process being enabled, whole-stash / type scope (never
|
|
674
|
-
// `ref`), and not a dry run. Mirrors the recombine opt-in wiring.
|
|
675
|
-
let proceduralCompilation;
|
|
676
|
-
if (primaryStashDir &&
|
|
677
|
-
improveProfile &&
|
|
678
|
-
resolveProcessEnabled("procedural", improveProfile) &&
|
|
679
|
-
scope.mode !== "ref" &&
|
|
680
|
-
!options.dryRun) {
|
|
681
|
-
const proceduralFn = options.proceduralFn ?? akmProcedural;
|
|
682
|
-
try {
|
|
683
|
-
proceduralCompilation = await proceduralFn({
|
|
684
|
-
stashDir: primaryStashDir,
|
|
685
|
-
config: options.config ?? loadConfig(),
|
|
686
|
-
...(improveProfile ? { improveProfile } : {}),
|
|
687
|
-
...(options.runId ? { sourceRun: options.runId } : {}),
|
|
688
|
-
...(budgetSignal ? { signal: budgetSignal } : {}),
|
|
689
|
-
eligibilitySource: "procedural",
|
|
690
|
-
...(eventsCtx ? { ctx: eventsCtx } : {}),
|
|
691
|
-
minRecurrence: improveProfile.processes?.procedural?.minRecurrence,
|
|
692
|
-
maxProposalsPerRun: improveProfile.processes?.procedural?.maxProposalsPerRun,
|
|
693
|
-
});
|
|
694
|
-
}
|
|
695
|
-
catch (e) {
|
|
696
|
-
allWarnings.push(`procedural: ${String(e)}`);
|
|
697
|
-
}
|
|
698
|
-
}
|
|
699
612
|
// ── R5: collapse/churn detector ────────────────────────────────────────────
|
|
700
|
-
// One snapshot per QUALIFYING cycle: consolidate processed work
|
|
701
|
-
//
|
|
702
|
-
// the post-merge index; one call site covers both passes. Deterministic,
|
|
613
|
+
// One snapshot per QUALIFYING cycle: consolidate processed work. Runs AFTER
|
|
614
|
+
// the maintenance reindex so FTS sees the post-merge index. Deterministic,
|
|
703
615
|
// observe-only, fail-open (the orchestrator catches everything) — and inert
|
|
704
616
|
// on the ~9-in-10 default-profile runs that touch no merges.
|
|
705
617
|
let cycleMetrics;
|
|
706
|
-
|
|
707
|
-
if (!options.dryRun && (consolidationRan || recombineWorked)) {
|
|
618
|
+
if (!options.dryRun && consolidationRan) {
|
|
708
619
|
cycleMetrics = runCollapseDetector({
|
|
709
620
|
runId: options.runId ?? "improve-adhoc",
|
|
710
621
|
...(improveProfile ? { improveProfile } : {}),
|
|
711
|
-
pass:
|
|
712
|
-
// prep+loop gate accepts, PLUS recombine's confirmed-lesson promotions —
|
|
713
|
-
// recombine churn is the historically observed failure mode and its
|
|
714
|
-
// promotions never flow through the prep/loop gates.
|
|
715
|
-
acceptedActions: (args.acceptedActions ?? 0) + (recombination?.lessonsPromoted ?? 0),
|
|
622
|
+
pass: "consolidate",
|
|
716
623
|
mergeFloorViolations: args.consolidationMergeFloorViolations ?? 0,
|
|
717
624
|
config: options.config ?? loadConfig(),
|
|
718
625
|
...(eventsCtx ? { eventsCtx } : {}),
|
|
@@ -722,8 +629,6 @@ export async function runImprovePostLoopStage(args) {
|
|
|
722
629
|
allWarnings,
|
|
723
630
|
deadUrls,
|
|
724
631
|
...(cycleMetrics ? { cycleMetrics } : {}),
|
|
725
|
-
...(recombination ? { recombination } : {}),
|
|
726
|
-
...(proceduralCompilation ? { proceduralCompilation } : {}),
|
|
727
632
|
...(maintenanceResult.memoryInference ? { memoryInference: maintenanceResult.memoryInference } : {}),
|
|
728
633
|
...(maintenanceResult.graphExtraction ? { graphExtraction: maintenanceResult.graphExtraction } : {}),
|
|
729
634
|
...(maintenanceResult.actions && maintenanceResult.actions.length > 0
|
|
@@ -733,368 +638,650 @@ export async function runImprovePostLoopStage(args) {
|
|
|
733
638
|
graphExtractionDurationMs: maintenanceResult.graphExtractionDurationMs,
|
|
734
639
|
orphansPurged: maintenanceResult.orphansPurged,
|
|
735
640
|
proposalsExpired: maintenanceResult.proposalsExpired,
|
|
736
|
-
// Consolidation's auto-accept gate counts now accrue in the preparation
|
|
737
|
-
// stage (#551); post-loop no longer runs an auto-accept gate of its own.
|
|
738
|
-
gateAutoAcceptedCount: 0,
|
|
739
|
-
gateAutoAcceptFailedCount: 0,
|
|
740
641
|
};
|
|
741
642
|
}
|
|
742
|
-
// TODO(refactor): mutates the passed-in `allWarnings` array as a hidden side channel. Return warnings in ImproveMaintenanceResult and merge in caller — invasive signature change deferred to next refactor pass.
|
|
743
|
-
// Exported for tests (#584/#585 DB-locking regression coverage); production
|
|
744
|
-
// callers reach it only through akmImprove → runImprovePostLoopStage.
|
|
745
|
-
// TODO(refactor): mutates the passed-in `allWarnings` array as a hidden side channel. Return warnings in ImproveMaintenanceResult and merge in caller — invasive signature change deferred to next refactor pass.
|
|
746
643
|
// Exported for tests (#584/#585 DB-locking regression coverage); production
|
|
747
644
|
// callers reach it only through akmImprove → runImprovePostLoopStage.
|
|
748
645
|
export async function runImproveMaintenancePasses(args) {
|
|
749
|
-
const { options, primaryStashDir, memoryRefsForInference, allWarnings, reindexFn,
|
|
646
|
+
const { options, primaryStashDir, memoryRefsForInference, allWarnings, reindexFn, budgetSignal, eventsCtx } = args;
|
|
750
647
|
if (!primaryStashDir)
|
|
751
648
|
return { memoryInferenceDurationMs: 0, graphExtractionDurationMs: 0 };
|
|
649
|
+
if (budgetSignal?.aborted)
|
|
650
|
+
return { memoryInferenceDurationMs: 0, graphExtractionDurationMs: 0 };
|
|
752
651
|
const config = options.config ?? loadConfig();
|
|
753
652
|
const sources = resolveSourceEntries(options.stashDir, config);
|
|
754
653
|
const memoryInferenceFn = options.memoryInferenceFn ?? runMemoryInferencePass;
|
|
755
654
|
const graphExtractionFn = options.graphExtractionFn ?? runGraphExtractionPass;
|
|
756
|
-
let db;
|
|
757
|
-
let memoryInference;
|
|
758
|
-
let graphExtraction;
|
|
759
|
-
let reindexedAfterInference = false;
|
|
760
|
-
const actions = [];
|
|
761
|
-
let memoryInferenceDurationMs = 0;
|
|
762
|
-
let graphExtractionDurationMs = 0;
|
|
763
|
-
let orphansPurged = 0;
|
|
764
|
-
let proposalsExpired = 0;
|
|
765
655
|
const openIndexDb = () => openIndexDatabase(getDbPath(), config.embedding?.dimension ? { embeddingDim: config.embedding.dimension } : undefined);
|
|
766
|
-
|
|
767
|
-
//
|
|
768
|
-
//
|
|
769
|
-
//
|
|
770
|
-
// state that graph extraction below relies on. The
|
|
771
|
-
// reopen runs in `finally` so a failed reindex still leaves a usable handle.
|
|
656
|
+
const dbCell = {};
|
|
657
|
+
// #584: see the MaintenanceCtx.reindexWithIndexDbReleased doc — close before
|
|
658
|
+
// every reindex, reopen in `finally` so a failed reindex still leaves a
|
|
659
|
+
// usable handle in the cell.
|
|
772
660
|
const reindexWithIndexDbReleased = async (stashDir) => {
|
|
773
|
-
if (
|
|
774
|
-
closeDatabase(
|
|
775
|
-
|
|
661
|
+
if (dbCell.current) {
|
|
662
|
+
closeDatabase(dbCell.current);
|
|
663
|
+
dbCell.current = undefined;
|
|
776
664
|
}
|
|
777
665
|
try {
|
|
778
|
-
await reindexFn({ stashDir });
|
|
666
|
+
await reindexFn({ stashDir, signal: budgetSignal });
|
|
779
667
|
}
|
|
780
668
|
finally {
|
|
781
|
-
|
|
669
|
+
dbCell.current = openIndexDb();
|
|
782
670
|
}
|
|
783
671
|
};
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
672
|
+
const ctx = {
|
|
673
|
+
config,
|
|
674
|
+
sources,
|
|
675
|
+
primaryStashDir,
|
|
676
|
+
eventsCtx,
|
|
677
|
+
budgetSignal,
|
|
678
|
+
improveProfile: args.improveProfile,
|
|
679
|
+
resolvedPlan: args.resolvedPlan,
|
|
680
|
+
memoryInferenceFn,
|
|
681
|
+
graphExtractionFn,
|
|
682
|
+
reindexWithIndexDbReleased,
|
|
683
|
+
};
|
|
684
|
+
const collected = await withIndexWriterLease({ purpose: "improve-maintenance", signal: budgetSignal }, () => runMaintenancePassesUnderLease(ctx, dbCell, {
|
|
685
|
+
actionableRefs: args.actionableRefs,
|
|
686
|
+
memoryRefsForInference,
|
|
687
|
+
consolidationRan: args.consolidationRan,
|
|
688
|
+
allWarnings,
|
|
689
|
+
openIndexDb,
|
|
690
|
+
}));
|
|
691
|
+
return {
|
|
692
|
+
...(collected.memoryInference ? { memoryInference: collected.memoryInference } : {}),
|
|
693
|
+
...(collected.graphExtraction ? { graphExtraction: collected.graphExtraction } : {}),
|
|
694
|
+
...(collected.actions.length > 0 ? { actions: collected.actions } : {}),
|
|
695
|
+
memoryInferenceDurationMs: collected.memoryInferenceDurationMs,
|
|
696
|
+
graphExtractionDurationMs: collected.graphExtractionDurationMs,
|
|
697
|
+
orphansPurged: collected.orphansPurged,
|
|
698
|
+
proposalsExpired: collected.proposalsExpired,
|
|
699
|
+
};
|
|
700
|
+
}
|
|
701
|
+
/**
|
|
702
|
+
* The maintenance sequence run under the index-writer lease (formerly the
|
|
703
|
+
* ~389-line anonymous `withIndexWriterLease` callback): memory inference →
|
|
704
|
+
* reindex-after-inference → graph extraction → proposal hygiene (orphan purge,
|
|
705
|
+
* expiration) → retention purges. Each pass returns its results and warnings;
|
|
706
|
+
* this orchestrator folds warnings into the caller's `allWarnings` sink at the
|
|
707
|
+
* same points the inline code pushed them.
|
|
708
|
+
*/
|
|
709
|
+
async function runMaintenancePassesUnderLease(ctx, dbCell, args) {
|
|
710
|
+
const { allWarnings } = args;
|
|
711
|
+
const actions = [];
|
|
712
|
+
let reindexedAfterInference = false;
|
|
713
|
+
try {
|
|
714
|
+
dbCell.current = args.openIndexDb();
|
|
715
|
+
const inference = await runMemoryInferenceMaintenancePass(ctx, dbCell, args.memoryRefsForInference);
|
|
716
|
+
if (inference.action)
|
|
717
|
+
actions.push(inference.action);
|
|
718
|
+
allWarnings.push(...inference.warnings);
|
|
719
|
+
const memoryInference = inference.memoryInference;
|
|
720
|
+
if (memoryInference && (memoryInference.splitParents > 0 || memoryInference.writtenFacts > 0)) {
|
|
721
|
+
info("[improve] reindexing after memory inference writes");
|
|
722
|
+
try {
|
|
723
|
+
await ctx.reindexWithIndexDbReleased(ctx.primaryStashDir);
|
|
724
|
+
reindexedAfterInference = true;
|
|
725
|
+
info("[improve] reindex after memory inference complete");
|
|
817
726
|
}
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
info(hintRefs > 0
|
|
821
|
-
? `[improve] memory inference starting (${hintRefs} hint refs touched this run; pass discovers all pending)`
|
|
822
|
-
: "[improve] memory inference starting (discovering pending parents)");
|
|
823
|
-
const inferenceStart = Date.now();
|
|
824
|
-
try {
|
|
825
|
-
// O-1 (#364): pass budget signal so a hung inference call is cancelled.
|
|
826
|
-
memoryInference = await withLlmStage("memory-inference", () => memoryInferenceFn({
|
|
827
|
-
config,
|
|
828
|
-
sources,
|
|
829
|
-
signal: budgetSignal,
|
|
830
|
-
db,
|
|
831
|
-
reEnrich: false,
|
|
832
|
-
onProgress: (event) => {
|
|
833
|
-
const current = event.currentRef ? ` ${event.currentRef}` : "";
|
|
834
|
-
info(`[improve] memory inference ${event.processed}/${event.total}${current} (written ${event.writtenFacts}, skipped ${event.skippedNoFacts})`);
|
|
835
|
-
},
|
|
836
|
-
}));
|
|
837
|
-
memoryInferenceDurationMs = Date.now() - inferenceStart;
|
|
838
|
-
actions.push({ ref: "memory:_inference", mode: "memory-inference", result: memoryInference });
|
|
839
|
-
info(`[improve] memory inference complete (${memoryInference.writtenFacts} facts written from ${memoryInference.splitParents} parents)`);
|
|
840
|
-
}
|
|
841
|
-
catch (err) {
|
|
842
|
-
memoryInferenceDurationMs = Date.now() - inferenceStart;
|
|
843
|
-
allWarnings.push(`memory inference failed: ${errMessage(err)}`);
|
|
844
|
-
}
|
|
727
|
+
catch (err) {
|
|
728
|
+
allWarnings.push(`reindex after memory inference failed: ${errMessage(err)}`);
|
|
845
729
|
}
|
|
846
|
-
|
|
847
|
-
|
|
730
|
+
}
|
|
731
|
+
const graph = await runGraphExtractionMaintenancePass(ctx, dbCell, {
|
|
732
|
+
actionableRefs: args.actionableRefs,
|
|
733
|
+
memoryRefsForInference: args.memoryRefsForInference,
|
|
734
|
+
consolidationRan: args.consolidationRan,
|
|
735
|
+
reindexedAfterInference,
|
|
736
|
+
});
|
|
737
|
+
if (graph.action)
|
|
738
|
+
actions.push(graph.action);
|
|
739
|
+
allWarnings.push(...graph.warnings);
|
|
740
|
+
const orphan = runOrphanProposalPurgePass(ctx);
|
|
741
|
+
allWarnings.push(...orphan.warnings);
|
|
742
|
+
// #733: orphan-state GC — stamps/clears/(optionally) collects
|
|
743
|
+
// asset_salience/asset_outcome rows whose ref no longer resolves in
|
|
744
|
+
// index.db. Needs the SAME already-open index.db handle (dbCell.current)
|
|
745
|
+
// the passes above share.
|
|
746
|
+
const stateGc = runOrphanStateGcPass(ctx, dbCell);
|
|
747
|
+
allWarnings.push(...stateGc.warnings);
|
|
748
|
+
const expiration = runProposalExpirationPass(ctx);
|
|
749
|
+
allWarnings.push(...expiration.warnings);
|
|
750
|
+
allWarnings.push(...runRetentionPurgePass(ctx).warnings);
|
|
751
|
+
return {
|
|
752
|
+
memoryInference,
|
|
753
|
+
graphExtraction: graph.graphExtraction,
|
|
754
|
+
actions,
|
|
755
|
+
memoryInferenceDurationMs: inference.durationMs,
|
|
756
|
+
graphExtractionDurationMs: graph.durationMs,
|
|
757
|
+
orphansPurged: orphan.orphansPurged,
|
|
758
|
+
proposalsExpired: expiration.proposalsExpired,
|
|
759
|
+
};
|
|
760
|
+
}
|
|
761
|
+
finally {
|
|
762
|
+
if (dbCell.current)
|
|
763
|
+
closeDatabase(dbCell.current);
|
|
764
|
+
}
|
|
765
|
+
}
|
|
766
|
+
/**
|
|
767
|
+
* Memory inference candidate-discovery (post-Item 9 fix from
|
|
768
|
+
* memories/akm-improve-critical-review-2026-05-20). Previously this pass
|
|
769
|
+
* was gated on memoryRefsForInference.size > 0 AND passed those refs as a
|
|
770
|
+
* candidateRefs filter. But memoryRefsForInference is populated from refs
|
|
771
|
+
* distilled THIS RUN — by the time that happens, those parents are
|
|
772
|
+
* already split (`inferenceProcessed: true`) and `isPendingMemory` excludes
|
|
773
|
+
* them. The genuinely-pending parents in the stash never entered the
|
|
774
|
+
* filter. Result: 0/0/0 for 25 consecutive runs.
|
|
775
|
+
*
|
|
776
|
+
* Fix: always run the pass when the feature is enabled; let the pass's
|
|
777
|
+
* own `collectPendingMemories` + `isPendingMemory` predicate find
|
|
778
|
+
* candidates from the filesystem-of-truth. The this-run set is still
|
|
779
|
+
* logged as a hint but no longer used as a filter.
|
|
780
|
+
*/
|
|
781
|
+
export async function runMemoryInferenceMaintenancePass(ctx, dbCell, memoryRefsForInference) {
|
|
782
|
+
const { config, sources, primaryStashDir, budgetSignal, improveProfile, resolvedPlan, memoryInferenceFn } = ctx;
|
|
783
|
+
const warnings = [];
|
|
784
|
+
let memoryInference;
|
|
785
|
+
let durationMs = 0;
|
|
786
|
+
let action;
|
|
787
|
+
const memoryInferenceDisabledByProfile = improveProfile?.processes?.memoryInference?.enabled === false;
|
|
788
|
+
const minPendingCount = improveProfile?.processes?.memoryInference?.minPendingCount;
|
|
789
|
+
const pendingBelowMinCount = (() => {
|
|
790
|
+
if (!primaryStashDir || minPendingCount === undefined || minPendingCount <= 0)
|
|
791
|
+
return false;
|
|
792
|
+
const pending = collectPendingMemories(primaryStashDir).length;
|
|
793
|
+
if (pending < minPendingCount) {
|
|
794
|
+
info(`[improve] memory inference skipped (${pending} pending < minPendingCount ${minPendingCount})`);
|
|
795
|
+
return true;
|
|
796
|
+
}
|
|
797
|
+
return false;
|
|
798
|
+
})();
|
|
799
|
+
if (memoryInferenceDisabledByProfile) {
|
|
800
|
+
info("[improve] memory inference skipped (disabled by improve profile)");
|
|
801
|
+
}
|
|
802
|
+
else if (pendingBelowMinCount) {
|
|
803
|
+
// skipped — message already emitted above
|
|
804
|
+
}
|
|
805
|
+
else {
|
|
806
|
+
const hintRefs = memoryRefsForInference.size;
|
|
807
|
+
info(hintRefs > 0
|
|
808
|
+
? `[improve] memory inference starting (${hintRefs} hint refs touched this run; pass discovers all pending)`
|
|
809
|
+
: "[improve] memory inference starting (discovering pending parents)");
|
|
810
|
+
const inferenceStart = Date.now();
|
|
811
|
+
try {
|
|
812
|
+
// O-1 (#364): pass budget signal so a hung inference call is cancelled.
|
|
813
|
+
memoryInference = await withLlmStage("memory-inference", () => memoryInferenceFn({
|
|
814
|
+
config,
|
|
815
|
+
...(resolvedPlan
|
|
816
|
+
? {
|
|
817
|
+
llmConfig: resolvedPlan.processes.memoryInference.runner
|
|
818
|
+
? materializeLlmRunnerConnection(resolvedPlan.processes.memoryInference.runner)
|
|
819
|
+
: null,
|
|
820
|
+
}
|
|
821
|
+
: {}),
|
|
822
|
+
sources,
|
|
823
|
+
signal: budgetSignal,
|
|
824
|
+
db: dbCell.current,
|
|
825
|
+
reEnrich: false,
|
|
826
|
+
onProgress: (event) => {
|
|
827
|
+
const current = event.currentRef ? ` ${event.currentRef}` : "";
|
|
828
|
+
info(`[improve] memory inference ${event.processed}/${event.total}${current} (written ${event.writtenFacts}, skipped ${event.skippedNoFacts})`);
|
|
829
|
+
},
|
|
830
|
+
}), { engine: resolvedPlan?.processes.memoryInference.runner?.engine, process: "memoryInference" });
|
|
831
|
+
durationMs = Date.now() - inferenceStart;
|
|
832
|
+
// Synthetic sentinel ref (ref-grammar decision D-R3): a colon-free
|
|
833
|
+
// `<domain>/_<marker>` label on the event row, never parsed as an asset
|
|
834
|
+
// ref. The domain is the asset stash-subdir for asset-scoped sentinels
|
|
835
|
+
// (`memories/…`) and the subsystem name for maintenance/artifact sentinels
|
|
836
|
+
// (`graph/…`, `events/…`, `proposals/…`, `health/…`, …). Readers match the
|
|
837
|
+
// event by `eventType`, never by this string.
|
|
838
|
+
action = { ref: "memories/_inference", mode: "memory-inference", result: memoryInference };
|
|
839
|
+
info(`[improve] memory inference complete (${memoryInference.writtenFacts} facts written from ${memoryInference.splitParents} parents)`);
|
|
840
|
+
}
|
|
841
|
+
catch (err) {
|
|
842
|
+
durationMs = Date.now() - inferenceStart;
|
|
843
|
+
warnings.push(`memory inference failed: ${errMessage(err)}`);
|
|
844
|
+
}
|
|
845
|
+
}
|
|
846
|
+
return { memoryInference, durationMs, action, warnings };
|
|
847
|
+
}
|
|
848
|
+
/**
|
|
849
|
+
* Graph-extraction maintenance pass.
|
|
850
|
+
*
|
|
851
|
+
* INVARIANT: graph extraction normally runs only on files touched by
|
|
852
|
+
* actionable refs (candidatePaths). Full-corpus scans are opt-in via
|
|
853
|
+
* profile.processes.graphExtraction.fullScan = true (used by the
|
|
854
|
+
* `graph-refresh` built-in profile and its weekly scheduled task).
|
|
855
|
+
* The empty-Set fallback is intentional when no refs were touched —
|
|
856
|
+
* the extractor's filter rejects every file and returns empty, keeping
|
|
857
|
+
* the pass invoked so the action is recorded and tests stay exercised.
|
|
858
|
+
*/
|
|
859
|
+
export async function runGraphExtractionMaintenancePass(ctx, dbCell, args) {
|
|
860
|
+
const { config, sources, primaryStashDir, budgetSignal, improveProfile, resolvedPlan, graphExtractionFn } = ctx;
|
|
861
|
+
const warnings = [];
|
|
862
|
+
let graphExtraction;
|
|
863
|
+
let durationMs = 0;
|
|
864
|
+
let action;
|
|
865
|
+
let reindexedAfterInference = args.reindexedAfterInference;
|
|
866
|
+
const graphEnabled = resolvedPlan ? true : isProcessEnabled("index", "graph_extraction", config);
|
|
867
|
+
const graphExtractionDisabledByProfile = improveProfile?.processes?.graphExtraction?.enabled === false;
|
|
868
|
+
const graphExtractionFullScan = improveProfile?.processes?.graphExtraction?.fullScan === true;
|
|
869
|
+
// #624 P2: optional incremental high-signal-first cap. Unset = process all
|
|
870
|
+
// eligible (byte-identical to today; no ranking/slice).
|
|
871
|
+
const graphExtractionTopN = improveProfile?.processes?.graphExtraction?.topN;
|
|
872
|
+
const graphExtractionIncludeTypes = improveProfile?.processes?.graphExtraction?.includeTypes ?? [
|
|
873
|
+
...DEFAULT_GRAPH_EXTRACTION_INCLUDE_TYPES,
|
|
874
|
+
];
|
|
875
|
+
const graphExtractionBatchSize = improveProfile?.processes?.graphExtraction?.batchSize ?? DEFAULT_GRAPH_EXTRACTION_BATCH_SIZE;
|
|
876
|
+
// Build the set of refs actually touched this run.
|
|
877
|
+
const touchedRefs = new Set();
|
|
878
|
+
for (const r of args.actionableRefs)
|
|
879
|
+
touchedRefs.add(r.ref);
|
|
880
|
+
for (const r of args.memoryRefsForInference)
|
|
881
|
+
touchedRefs.add(r);
|
|
882
|
+
if (graphExtractionDisabledByProfile) {
|
|
883
|
+
info("[improve] graph extraction skipped (disabled by improve profile)");
|
|
884
|
+
}
|
|
885
|
+
else if (sources.length > 0 && graphEnabled) {
|
|
886
|
+
info(`[improve] graph extraction starting${graphExtractionFullScan ? " (full-corpus scan)" : ""}`);
|
|
887
|
+
const extractionStart = Date.now();
|
|
888
|
+
try {
|
|
889
|
+
// D9: if consolidation ran but memory inference did not reindex, force a reindex
|
|
890
|
+
// so graph extraction sees current DB state after consolidation writes.
|
|
891
|
+
if (args.consolidationRan && !reindexedAfterInference) {
|
|
892
|
+
info("[improve] reindexing after consolidation (graph extraction needs current state)");
|
|
848
893
|
try {
|
|
849
|
-
await reindexWithIndexDbReleased(primaryStashDir);
|
|
894
|
+
await ctx.reindexWithIndexDbReleased(primaryStashDir);
|
|
850
895
|
reindexedAfterInference = true;
|
|
851
|
-
info("[improve] reindex after
|
|
896
|
+
info("[improve] reindex after consolidation complete");
|
|
852
897
|
}
|
|
853
898
|
catch (err) {
|
|
854
|
-
|
|
899
|
+
warnings.push(`reindex after consolidation failed: ${errMessage(err)}`);
|
|
855
900
|
}
|
|
856
901
|
}
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
//
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
// actionable refs (candidatePaths). Full-corpus scans are opt-in via
|
|
871
|
-
// profile.processes.graphExtraction.fullScan = true (used by the
|
|
872
|
-
// `graph-refresh` built-in profile and its weekly scheduled task).
|
|
873
|
-
// The empty-Set fallback is intentional when no refs were touched —
|
|
874
|
-
// the extractor's filter rejects every file and returns empty, keeping
|
|
875
|
-
// the pass invoked so the action is recorded and tests stay exercised.
|
|
876
|
-
if (graphExtractionDisabledByProfile) {
|
|
877
|
-
info("[improve] graph extraction skipped (disabled by improve profile)");
|
|
878
|
-
}
|
|
879
|
-
else if (sources.length > 0 && graphEnabled) {
|
|
880
|
-
info(`[improve] graph extraction starting${graphExtractionFullScan ? " (full-corpus scan)" : ""}`);
|
|
881
|
-
const extractionStart = Date.now();
|
|
882
|
-
try {
|
|
883
|
-
// D9: if consolidation ran but memory inference did not reindex, force a reindex
|
|
884
|
-
// so graph extraction sees current DB state after consolidation writes.
|
|
885
|
-
if (consolidationRan && !reindexedAfterInference) {
|
|
886
|
-
info("[improve] reindexing after consolidation (graph extraction needs current state)");
|
|
887
|
-
try {
|
|
888
|
-
await reindexWithIndexDbReleased(primaryStashDir);
|
|
889
|
-
reindexedAfterInference = true;
|
|
890
|
-
info("[improve] reindex after consolidation complete");
|
|
891
|
-
}
|
|
892
|
-
catch (err) {
|
|
893
|
-
allWarnings.push(`reindex after consolidation failed: ${errMessage(err)}`);
|
|
894
|
-
}
|
|
895
|
-
}
|
|
896
|
-
// #584: no close/reopen needed here — reindexWithIndexDbReleased
|
|
897
|
-
// already swapped in a fresh post-reindex handle.
|
|
898
|
-
// Resolve touched refs to absolute file paths. Skipped for fullScan
|
|
899
|
-
// (candidatePaths stays undefined → extractor processes all files).
|
|
900
|
-
let candidatePaths;
|
|
901
|
-
if (!graphExtractionFullScan) {
|
|
902
|
-
candidatePaths = new Set();
|
|
903
|
-
if (primaryStashDir && touchedRefs.size > 0) {
|
|
904
|
-
const writableDirSet = new Set(getWritableStashDirs(primaryStashDir).map((d) => path.resolve(d)));
|
|
905
|
-
const resolved = await Promise.all([...touchedRefs].map((ref) => findAssetFilePath(ref, primaryStashDir, writableDirSet).catch(() => null)));
|
|
906
|
-
for (const p of resolved) {
|
|
907
|
-
if (typeof p === "string" && p.length > 0)
|
|
908
|
-
candidatePaths.add(p);
|
|
909
|
-
}
|
|
910
|
-
}
|
|
902
|
+
// #584: no close/reopen needed here — reindexWithIndexDbReleased
|
|
903
|
+
// already swapped in a fresh post-reindex handle.
|
|
904
|
+
// Resolve touched refs to absolute file paths. Skipped for fullScan
|
|
905
|
+
// (candidatePaths stays undefined → extractor processes all files).
|
|
906
|
+
let candidatePaths;
|
|
907
|
+
if (!graphExtractionFullScan) {
|
|
908
|
+
candidatePaths = new Set();
|
|
909
|
+
if (primaryStashDir && touchedRefs.size > 0) {
|
|
910
|
+
const writableDirSet = new Set(getWritableStashDirs(primaryStashDir).map((d) => path.resolve(d)));
|
|
911
|
+
const resolved = await Promise.all([...touchedRefs].map((ref) => findAssetFilePath(ref, primaryStashDir, writableDirSet).catch(() => null)));
|
|
912
|
+
for (const p of resolved) {
|
|
913
|
+
if (typeof p === "string" && p.length > 0)
|
|
914
|
+
candidatePaths.add(p);
|
|
911
915
|
}
|
|
912
|
-
const progressHandler = (event) => {
|
|
913
|
-
const current = event.currentPath ? ` ${path.basename(event.currentPath)}` : "";
|
|
914
|
-
info(`[improve] graph extraction ${event.processed}/${event.total}${current} (extracted ${event.extracted}, entities ${event.totalEntities}, relations ${event.totalRelations})`);
|
|
915
|
-
};
|
|
916
|
-
// O-1 (#364): pass budget signal so a hung graph extraction call is cancelled.
|
|
917
|
-
graphExtraction = await withLlmStage("graph-extraction", () => graphExtractionFn({
|
|
918
|
-
config,
|
|
919
|
-
sources,
|
|
920
|
-
signal: budgetSignal,
|
|
921
|
-
db,
|
|
922
|
-
reEnrich: false,
|
|
923
|
-
onProgress: progressHandler,
|
|
924
|
-
options: { candidatePaths, ...(graphExtractionTopN != null ? { topN: graphExtractionTopN } : {}) },
|
|
925
|
-
}));
|
|
926
|
-
graphExtractionDurationMs = Date.now() - extractionStart;
|
|
927
|
-
actions.push({ ref: "graph:_artifact", mode: "graph-extraction", result: graphExtraction });
|
|
928
|
-
info(`[improve] graph extraction complete (${graphExtraction.quality.extractedFiles} files, ${graphExtraction.quality.entityCount} entities, ${graphExtraction.quality.relationCount} relations)`);
|
|
929
916
|
}
|
|
930
|
-
catch (err) {
|
|
931
|
-
graphExtractionDurationMs = Date.now() - extractionStart;
|
|
932
|
-
allWarnings.push(`graph extraction failed: ${errMessage(err)}`);
|
|
933
|
-
}
|
|
934
|
-
}
|
|
935
|
-
else if (sources.length > 0 && !graphEnabled) {
|
|
936
|
-
info("[improve] graph extraction skipped (features.index.graph_extraction is disabled)");
|
|
937
917
|
}
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
918
|
+
const progressHandler = (event) => {
|
|
919
|
+
const current = event.currentPath ? ` ${path.basename(event.currentPath)}` : "";
|
|
920
|
+
info(`[improve] graph extraction ${event.processed}/${event.total}${current} (extracted ${event.extracted}, entities ${event.totalEntities}, relations ${event.totalRelations})`);
|
|
921
|
+
};
|
|
922
|
+
// O-1 (#364): pass budget signal so a hung graph extraction call is cancelled.
|
|
923
|
+
graphExtraction = await withLlmStage("graph-extraction", () => graphExtractionFn({
|
|
924
|
+
config,
|
|
925
|
+
...(resolvedPlan
|
|
926
|
+
? {
|
|
927
|
+
llmConfig: resolvedPlan.processes.graphExtraction.runner
|
|
928
|
+
? materializeLlmRunnerConnection(resolvedPlan.processes.graphExtraction.runner)
|
|
929
|
+
: null,
|
|
947
930
|
}
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
}
|
|
931
|
+
: {}),
|
|
932
|
+
sources,
|
|
933
|
+
signal: budgetSignal,
|
|
934
|
+
db: dbCell.current,
|
|
935
|
+
reEnrich: false,
|
|
936
|
+
onProgress: progressHandler,
|
|
937
|
+
options: {
|
|
938
|
+
candidatePaths,
|
|
939
|
+
includeTypes: graphExtractionIncludeTypes,
|
|
940
|
+
batchSize: graphExtractionBatchSize,
|
|
941
|
+
...(graphExtractionTopN != null ? { topN: graphExtractionTopN } : {}),
|
|
942
|
+
},
|
|
943
|
+
}), { engine: resolvedPlan?.processes.graphExtraction.runner?.engine, process: "graphExtraction" });
|
|
944
|
+
durationMs = Date.now() - extractionStart;
|
|
945
|
+
// Synthetic sentinel ref (D-R3): `graph` has no asset stash-subdir, so the
|
|
946
|
+
// colon-free `graph/_artifact` names the subsystem, per the sentinel
|
|
947
|
+
// convention documented at the memory-inference writer above.
|
|
948
|
+
action = { ref: "graph/_artifact", mode: "graph-extraction", result: graphExtraction };
|
|
949
|
+
info(`[improve] graph extraction complete (${graphExtraction.quality.extractedFiles} files, ${graphExtraction.quality.entityCount} entities, ${graphExtraction.quality.relationCount} relations)`);
|
|
950
|
+
}
|
|
951
|
+
catch (err) {
|
|
952
|
+
durationMs = Date.now() - extractionStart;
|
|
953
|
+
warnings.push(`graph extraction failed: ${errMessage(err)}`);
|
|
954
|
+
}
|
|
955
|
+
}
|
|
956
|
+
else if (sources.length > 0 && !graphEnabled) {
|
|
957
|
+
info("[improve] graph extraction skipped (features.index.graph_extraction is disabled)");
|
|
958
|
+
}
|
|
959
|
+
return { graphExtraction, durationMs, action, warnings };
|
|
960
|
+
}
|
|
961
|
+
/**
|
|
962
|
+
* Orphan proposal purge — reject pending reflect proposals whose target
|
|
963
|
+
* asset no longer exists on disk. Runs after graph extraction so newly
|
|
964
|
+
* promoted assets from accept flows during this run are already present.
|
|
965
|
+
*/
|
|
966
|
+
function runOrphanProposalPurgePass(ctx) {
|
|
967
|
+
const { primaryStashDir, sources, eventsCtx } = ctx;
|
|
968
|
+
const warnings = [];
|
|
969
|
+
let orphansPurged = 0;
|
|
970
|
+
try {
|
|
971
|
+
const purgeResult = purgeOrphanProposals(primaryStashDir, sources.map((s) => s.path));
|
|
972
|
+
orphansPurged = purgeResult.rejected;
|
|
973
|
+
if (purgeResult.rejected > 0) {
|
|
974
|
+
info(`[improve] orphan purge: ${purgeResult.rejected}/${purgeResult.checked} orphaned proposals rejected (${purgeResult.durationMs}ms)`);
|
|
975
|
+
}
|
|
976
|
+
appendEvent({
|
|
977
|
+
eventType: "proposal_orphan_purge",
|
|
978
|
+
ref: "proposals/_orphan-purge",
|
|
979
|
+
metadata: {
|
|
980
|
+
checked: purgeResult.checked,
|
|
981
|
+
rejected: purgeResult.rejected,
|
|
982
|
+
durationMs: purgeResult.durationMs,
|
|
983
|
+
byType: purgeResult.byType,
|
|
984
|
+
orphans: purgeResult.orphans.map((o) => o.ref),
|
|
985
|
+
},
|
|
986
|
+
}, eventsCtx);
|
|
987
|
+
}
|
|
988
|
+
catch (err) {
|
|
989
|
+
warnings.push(`orphan purge failed: ${errMessage(err)}`);
|
|
990
|
+
}
|
|
991
|
+
return { orphansPurged, warnings };
|
|
992
|
+
}
|
|
993
|
+
/**
|
|
994
|
+
* Phase 6B (Advantage D6b): expire pending proposals that have aged past
|
|
995
|
+
* the retention window. Runs AFTER orphan purge so we never double-archive
|
|
996
|
+
* a proposal that orphan-purge already moved. `expireStaleProposals` emits
|
|
997
|
+
* its own per-proposal `proposal_expired` events; we additionally emit a
|
|
998
|
+
* single roll-up event here for parity with the orphan-purge surface.
|
|
999
|
+
*/
|
|
1000
|
+
function runProposalExpirationPass(ctx) {
|
|
1001
|
+
const { primaryStashDir, config, eventsCtx } = ctx;
|
|
1002
|
+
const warnings = [];
|
|
1003
|
+
let proposalsExpired = 0;
|
|
1004
|
+
try {
|
|
1005
|
+
const expireResult = expireStaleProposals(primaryStashDir, config);
|
|
1006
|
+
proposalsExpired = expireResult.expired;
|
|
1007
|
+
if (expireResult.expired > 0) {
|
|
1008
|
+
info(`[improve] expiration: ${expireResult.expired}/${expireResult.checked} pending proposals expired ` +
|
|
1009
|
+
`(retention=${expireResult.retentionDays}d, ${expireResult.durationMs}ms)`);
|
|
1010
|
+
}
|
|
1011
|
+
appendEvent({
|
|
1012
|
+
eventType: "proposal_expiration_pass",
|
|
1013
|
+
ref: "proposals/_expiration",
|
|
1014
|
+
metadata: {
|
|
1015
|
+
checked: expireResult.checked,
|
|
1016
|
+
expired: expireResult.expired,
|
|
1017
|
+
durationMs: expireResult.durationMs,
|
|
1018
|
+
retentionDays: expireResult.retentionDays,
|
|
1019
|
+
expiredProposals: expireResult.expiredProposals,
|
|
1020
|
+
},
|
|
1021
|
+
}, eventsCtx);
|
|
1022
|
+
}
|
|
1023
|
+
catch (err) {
|
|
1024
|
+
warnings.push(`proposal expiration failed: ${errMessage(err)}`);
|
|
1025
|
+
}
|
|
1026
|
+
return { proposalsExpired, warnings };
|
|
1027
|
+
}
|
|
1028
|
+
/**
|
|
1029
|
+
* Fix #2 (observability 0.8.0): trim the events table in state.db so it
|
|
1030
|
+
* doesn't grow unbounded. `akm health` writes a `health_probe` row on every
|
|
1031
|
+
* invocation, and every command surface emits at least one event besides —
|
|
1032
|
+
* without this trim, state.db is a permanent append-only log. Config key
|
|
1033
|
+
* `improve.eventRetentionDays` (default 90, set 0 to disable) controls the
|
|
1034
|
+
* window. The purge runs against state.db (a different SQLite file from
|
|
1035
|
+
* the index handle the other passes use).
|
|
1036
|
+
*/
|
|
1037
|
+
export function runRetentionPurgePass(ctx) {
|
|
1038
|
+
const { config, eventsCtx } = ctx;
|
|
1039
|
+
const warnings = [];
|
|
1040
|
+
const retentionDays = typeof config.improve?.eventRetentionDays === "number" ? config.improve.eventRetentionDays : 90;
|
|
1041
|
+
if (retentionDays > 0) {
|
|
1042
|
+
// #585: reuse the long-lived eventsCtx.db connection when akmImprove
|
|
1043
|
+
// opened one — opening a second state.db write connection while
|
|
1044
|
+
// eventsDb is still live made two simultaneous writers contend on the
|
|
1045
|
+
// same WAL file ("database is locked"). Only the eventsCtx.dbPath
|
|
1046
|
+
// fallback path (state.db failed to open up-front) opens — and then
|
|
1047
|
+
// owns and closes — its own handle. C2 still holds: the fallback uses
|
|
1048
|
+
// the boundary-pinned path, never a live `process.env` re-read.
|
|
1049
|
+
try {
|
|
1050
|
+
withStateDb((stateDb) => {
|
|
1051
|
+
const purgedCount = purgeOldEvents(stateDb, retentionDays);
|
|
1052
|
+
if (purgedCount > 0) {
|
|
1053
|
+
info(`[improve] events purge: ${purgedCount} event(s) older than ${retentionDays}d removed from state.db`);
|
|
959
1054
|
}
|
|
960
|
-
|
|
961
|
-
|
|
1055
|
+
appendEvent({
|
|
1056
|
+
eventType: "events_purged",
|
|
1057
|
+
ref: "events/_purge",
|
|
1058
|
+
metadata: { purgedCount, retentionDays },
|
|
1059
|
+
}, eventsCtx);
|
|
1060
|
+
// improve_runs uses the same retention window as events — both are
|
|
1061
|
+
// observability/audit data, both grow append-only, both have a
|
|
1062
|
+
// dedicated purge helper. Mirroring the events purge here means a
|
|
1063
|
+
// single retention knob (improve.eventRetentionDays) governs both.
|
|
1064
|
+
const improveRunsPurged = purgeOldImproveRuns(stateDb, retentionDays);
|
|
1065
|
+
if (improveRunsPurged > 0) {
|
|
1066
|
+
info(`[improve] improve_runs purge: ${improveRunsPurged} run(s) older than ${retentionDays}d removed from state.db`);
|
|
962
1067
|
}
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
}
|
|
1068
|
+
appendEvent({
|
|
1069
|
+
eventType: "improve_runs_purged",
|
|
1070
|
+
ref: "improve_runs/_purge",
|
|
1071
|
+
metadata: { purgedCount: improveRunsPurged, retentionDays },
|
|
1072
|
+
}, eventsCtx);
|
|
1073
|
+
// R5: improve_cycle_metrics has its OWN retention window
|
|
1074
|
+
// (default 365d — a slow collapse needs a longer trend than
|
|
1075
|
+
// the 90d events window). canary_queries rows are never purged.
|
|
1076
|
+
const cycleRetention = config.improve?.collapseDetector?.retentionDays ?? CYCLE_METRICS_RETENTION_DAYS;
|
|
1077
|
+
const cycleMetricsPurged = purgeOldCycleMetrics(stateDb, cycleRetention);
|
|
1078
|
+
if (cycleMetricsPurged > 0) {
|
|
1079
|
+
info(`[improve] cycle-metrics purge: ${cycleMetricsPurged} row(s) older than ${cycleRetention}d removed from state.db`);
|
|
975
1080
|
appendEvent({
|
|
976
|
-
|
|
977
|
-
ref
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
durationMs: expireResult.durationMs,
|
|
982
|
-
retentionDays: expireResult.retentionDays,
|
|
983
|
-
expiredProposals: expireResult.expiredProposals,
|
|
984
|
-
},
|
|
1081
|
+
// Dedicated type (mirrors improve_runs_purged) so consumers
|
|
1082
|
+
// never have to disambiguate purge targets via the ref string.
|
|
1083
|
+
eventType: "improve_cycle_metrics_purged",
|
|
1084
|
+
ref: "improve_cycle_metrics/_purge",
|
|
1085
|
+
metadata: { purgedCount: cycleMetricsPurged, retentionDays: cycleRetention },
|
|
985
1086
|
}, eventsCtx);
|
|
986
1087
|
}
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
}
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
// eventsDb is still live made two simultaneous writers contend on the
|
|
1004
|
-
// same WAL file ("database is locked"). Only the eventsCtx.dbPath
|
|
1005
|
-
// fallback path (state.db failed to open up-front) opens — and then
|
|
1006
|
-
// owns and closes — its own handle. C2 still holds: the fallback uses
|
|
1007
|
-
// the boundary-pinned path, never a live `process.env` re-read.
|
|
1008
|
-
try {
|
|
1009
|
-
withStateDb((stateDb) => {
|
|
1010
|
-
const purgedCount = purgeOldEvents(stateDb, retentionDays);
|
|
1011
|
-
if (purgedCount > 0) {
|
|
1012
|
-
info(`[improve] events purge: ${purgedCount} event(s) older than ${retentionDays}d removed from state.db`);
|
|
1013
|
-
}
|
|
1014
|
-
appendEvent({
|
|
1015
|
-
eventType: "events_purged",
|
|
1016
|
-
ref: "events:_purge",
|
|
1017
|
-
metadata: { purgedCount, retentionDays },
|
|
1018
|
-
}, eventsCtx);
|
|
1019
|
-
// improve_runs uses the same retention window as events — both are
|
|
1020
|
-
// observability/audit data, both grow append-only, both have a
|
|
1021
|
-
// dedicated purge helper. Mirroring the events purge here means a
|
|
1022
|
-
// single retention knob (improve.eventRetentionDays) governs both.
|
|
1023
|
-
const improveRunsPurged = purgeOldImproveRuns(stateDb, retentionDays);
|
|
1024
|
-
if (improveRunsPurged > 0) {
|
|
1025
|
-
info(`[improve] improve_runs purge: ${improveRunsPurged} run(s) older than ${retentionDays}d removed from state.db`);
|
|
1026
|
-
}
|
|
1027
|
-
appendEvent({
|
|
1028
|
-
eventType: "improve_runs_purged",
|
|
1029
|
-
ref: "improve_runs:_purge",
|
|
1030
|
-
metadata: { purgedCount: improveRunsPurged, retentionDays },
|
|
1031
|
-
}, eventsCtx);
|
|
1032
|
-
// R5: improve_cycle_metrics has its OWN retention window
|
|
1033
|
-
// (default 365d — a slow collapse needs a longer trend than
|
|
1034
|
-
// the 90d events window). canary_queries rows are never purged.
|
|
1035
|
-
const cycleRetention = config.improve?.collapseDetector?.retentionDays ?? CYCLE_METRICS_RETENTION_DAYS;
|
|
1036
|
-
const cycleMetricsPurged = purgeOldCycleMetrics(stateDb, cycleRetention);
|
|
1037
|
-
if (cycleMetricsPurged > 0) {
|
|
1038
|
-
info(`[improve] cycle-metrics purge: ${cycleMetricsPurged} row(s) older than ${cycleRetention}d removed from state.db`);
|
|
1039
|
-
appendEvent({
|
|
1040
|
-
// Dedicated type (mirrors improve_runs_purged) so consumers
|
|
1041
|
-
// never have to disambiguate purge targets via the ref string.
|
|
1042
|
-
eventType: "improve_cycle_metrics_purged",
|
|
1043
|
-
ref: "improve_cycle_metrics:_purge",
|
|
1044
|
-
metadata: { purgedCount: cycleMetricsPurged, retentionDays: cycleRetention },
|
|
1045
|
-
}, eventsCtx);
|
|
1046
|
-
}
|
|
1047
|
-
}, { path: eventsCtx?.dbPath, borrowed: eventsCtx?.db });
|
|
1048
|
-
}
|
|
1049
|
-
catch (err) {
|
|
1050
|
-
allWarnings.push(`events purge failed: ${errMessage(err)}`);
|
|
1051
|
-
}
|
|
1052
|
-
// task_logs in logs.db (#579) shares the same retention window as
|
|
1053
|
-
// events/improve_runs — all three are observability data governed by
|
|
1054
|
-
// the single improve.eventRetentionDays knob. Separate try/finally
|
|
1055
|
-
// because logs.db is a different file: a locked/missing logs.db must
|
|
1056
|
-
// not block the state.db purges above.
|
|
1057
|
-
let logsDb;
|
|
1058
|
-
try {
|
|
1059
|
-
logsDb = openLogsDatabase();
|
|
1060
|
-
const taskLogsPurged = purgeOldTaskLogs(logsDb, retentionDays);
|
|
1061
|
-
if (taskLogsPurged > 0) {
|
|
1062
|
-
info(`[improve] task_logs purge: ${taskLogsPurged} log line(s) older than ${retentionDays}d removed from logs.db`);
|
|
1063
|
-
}
|
|
1064
|
-
appendEvent({
|
|
1065
|
-
eventType: "task_logs_purged",
|
|
1066
|
-
ref: "task_logs:_purge",
|
|
1067
|
-
metadata: { purgedCount: taskLogsPurged, retentionDays },
|
|
1068
|
-
}, eventsCtx);
|
|
1069
|
-
}
|
|
1070
|
-
catch (err) {
|
|
1071
|
-
allWarnings.push(`task_logs purge failed: ${errMessage(err)}`);
|
|
1072
|
-
}
|
|
1073
|
-
finally {
|
|
1074
|
-
if (logsDb) {
|
|
1075
|
-
try {
|
|
1076
|
-
logsDb.close();
|
|
1077
|
-
}
|
|
1078
|
-
catch {
|
|
1079
|
-
// best-effort
|
|
1080
|
-
}
|
|
1081
|
-
}
|
|
1082
|
-
}
|
|
1083
|
-
}
|
|
1088
|
+
}, { path: eventsCtx?.dbPath, borrowed: eventsCtx?.db });
|
|
1089
|
+
}
|
|
1090
|
+
catch (err) {
|
|
1091
|
+
warnings.push(`events purge failed: ${errMessage(err)}`);
|
|
1092
|
+
}
|
|
1093
|
+
// task_logs in logs.db (#579) shares the same retention window as
|
|
1094
|
+
// events/improve_runs — all three are observability data governed by
|
|
1095
|
+
// the single improve.eventRetentionDays knob. Separate try/finally
|
|
1096
|
+
// because logs.db is a different file: a locked/missing logs.db must
|
|
1097
|
+
// not block the state.db purges above.
|
|
1098
|
+
let logsDb;
|
|
1099
|
+
try {
|
|
1100
|
+
logsDb = openLogsDatabase();
|
|
1101
|
+
const taskLogsPurged = purgeOldTaskLogs(logsDb, retentionDays);
|
|
1102
|
+
if (taskLogsPurged > 0) {
|
|
1103
|
+
info(`[improve] task_logs purge: ${taskLogsPurged} log line(s) older than ${retentionDays}d removed from logs.db`);
|
|
1084
1104
|
}
|
|
1105
|
+
appendEvent({
|
|
1106
|
+
eventType: "task_logs_purged",
|
|
1107
|
+
ref: "task_logs/_purge",
|
|
1108
|
+
metadata: { purgedCount: taskLogsPurged, retentionDays },
|
|
1109
|
+
}, eventsCtx);
|
|
1110
|
+
}
|
|
1111
|
+
catch (err) {
|
|
1112
|
+
warnings.push(`task_logs purge failed: ${errMessage(err)}`);
|
|
1085
1113
|
}
|
|
1086
1114
|
finally {
|
|
1087
|
-
if (
|
|
1088
|
-
|
|
1115
|
+
if (logsDb) {
|
|
1116
|
+
try {
|
|
1117
|
+
logsDb.close();
|
|
1118
|
+
}
|
|
1119
|
+
catch {
|
|
1120
|
+
// best-effort
|
|
1121
|
+
}
|
|
1122
|
+
}
|
|
1089
1123
|
}
|
|
1090
|
-
}
|
|
1091
|
-
return {
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
|
|
1097
|
-
|
|
1098
|
-
|
|
1099
|
-
|
|
1124
|
+
}
|
|
1125
|
+
return { warnings };
|
|
1126
|
+
}
|
|
1127
|
+
// ── #733 — orphan-state GC pass (Workstream C) ──────────────────────────────
|
|
1128
|
+
//
|
|
1129
|
+
// Deliberately lean: one maintenance pass, one additive migration (021), one
|
|
1130
|
+
// event type (asset_state_gc), one config gate (improve.stateGc.collect,
|
|
1131
|
+
// default false). See docs/architecture/specs/0.9.0-close-out-plan.md
|
|
1132
|
+
// Workstream C for the full design rationale. No quarantine archive, no
|
|
1133
|
+
// circuit breaker, no health-advisory plumbing, no new tables.
|
|
1134
|
+
/**
|
|
1135
|
+
* Grace window (ms) before an unresolved `asset_salience` / `asset_outcome`
|
|
1136
|
+
* row becomes delete-eligible — only when `improve.stateGc.collect` is true.
|
|
1137
|
+
* A named constant, not a config knob (owner ruling — see the close-out
|
|
1138
|
+
* plan's Workstream C). Mirrors `TXN_SWEEP_GRACE_MS` (src/core/fs-txn.ts:298).
|
|
1139
|
+
*/
|
|
1140
|
+
export const STATE_GC_GRACE_MS = daysToMs(7);
|
|
1141
|
+
/**
|
|
1142
|
+
* Resolve one state-table's stored `asset_ref` against the live index.
|
|
1143
|
+
*
|
|
1144
|
+
* "ref not present in entries.item_ref" is the authoritative-deletion
|
|
1145
|
+
* predicate (see the pass doc comment below), so this is a thin wrapper
|
|
1146
|
+
* around the same single-ref probe the rest of improve uses
|
|
1147
|
+
* (`getEntryByRef`, index-entries-repository.ts) — which already resolves
|
|
1148
|
+
* both storage spellings a write can produce (`salienceWriteKey`/
|
|
1149
|
+
* `outcomeWriteKey` = `itemRef ?? ref`): an exact bundle-qualified item_ref,
|
|
1150
|
+
* or a bare conceptId matched by suffix across all bundles.
|
|
1151
|
+
*
|
|
1152
|
+
* On top of that, falls back to the BARE conceptId form (`bareImproveRef` —
|
|
1153
|
+
* the same primitive `preparation.ts`'s `normalizeStoredKey` map is built
|
|
1154
|
+
* from via `improveStateReadRefs`) when the stored ref carries a bundle
|
|
1155
|
+
* prefix that no longer matches exactly. This is the legacy-spelling
|
|
1156
|
+
* normalization trap: a naive `asset_ref NOT IN (SELECT item_ref FROM
|
|
1157
|
+
* entries)` would treat a live asset whose row predates bundle-qualification
|
|
1158
|
+
* (or whose bundle prefix is stale) as an orphan and delete it. Preferring
|
|
1159
|
+
* "never delete a live row" over "never miss a genuinely dead one" mirrors
|
|
1160
|
+
* `getEntryByRef`'s own bare-conceptId suffix-match trade-off.
|
|
1161
|
+
*/
|
|
1162
|
+
function isStateRefLive(indexDb, storedRef) {
|
|
1163
|
+
if (getEntryByRef(indexDb, storedRef) !== null)
|
|
1164
|
+
return true;
|
|
1165
|
+
const bare = bareImproveRef(storedRef);
|
|
1166
|
+
return bare !== storedRef && getEntryByRef(indexDb, bare) !== null;
|
|
1167
|
+
}
|
|
1168
|
+
/**
|
|
1169
|
+
* Sweep ONE state table: stamp refs that just went unresolved, clear refs
|
|
1170
|
+
* that resolved again, and — only when `collect` is true — delete rows whose
|
|
1171
|
+
* `missing_since` is older than {@link STATE_GC_GRACE_MS}. `pending` is a
|
|
1172
|
+
* point-in-time snapshot taken AFTER stamp/clear/delete (re-queried, not
|
|
1173
|
+
* accumulated), so it reflects the current backlog rather than this run's
|
|
1174
|
+
* delta — "every run emits the counts either way, so live data accumulates
|
|
1175
|
+
* proof" (close-out plan, Workstream C).
|
|
1176
|
+
*/
|
|
1177
|
+
function gcOneStateTable(args) {
|
|
1178
|
+
const { refRows, indexDb, now, collect, stamp, clear, deleteOlderThan, countPending } = args;
|
|
1179
|
+
const toStamp = [];
|
|
1180
|
+
const toClear = [];
|
|
1181
|
+
for (const row of refRows) {
|
|
1182
|
+
const live = isStateRefLive(indexDb, row.asset_ref);
|
|
1183
|
+
if (!live && row.missing_since == null)
|
|
1184
|
+
toStamp.push(row.asset_ref);
|
|
1185
|
+
else if (live && row.missing_since != null)
|
|
1186
|
+
toClear.push(row.asset_ref);
|
|
1187
|
+
}
|
|
1188
|
+
if (toStamp.length > 0)
|
|
1189
|
+
stamp(toStamp, now);
|
|
1190
|
+
if (toClear.length > 0)
|
|
1191
|
+
clear(toClear);
|
|
1192
|
+
const collected = collect ? deleteOlderThan(now - STATE_GC_GRACE_MS) : 0;
|
|
1193
|
+
const pending = countPending();
|
|
1194
|
+
return { pending, collected };
|
|
1195
|
+
}
|
|
1196
|
+
/**
|
|
1197
|
+
* Orphan-state GC — #733 (Workstream C). For each of the two per-asset state
|
|
1198
|
+
* tables (`asset_salience`, `asset_outcome`), stamps `missing_since` on refs
|
|
1199
|
+
* that no longer resolve against `entries.item_ref` in index.db, clears the
|
|
1200
|
+
* stamp on refs that resolve again, and — only when `improve.stateGc.collect`
|
|
1201
|
+
* is true — deletes rows whose stamp is older than {@link STATE_GC_GRACE_MS}.
|
|
1202
|
+
*
|
|
1203
|
+
* "ref not present in entries.item_ref" IS the authoritative-deletion
|
|
1204
|
+
* predicate: "absent ≠ deleted" is inherited from the indexer, not
|
|
1205
|
+
* re-implemented here — an incomplete or failed source scan preserves that
|
|
1206
|
+
* source's last-known-good `entries` rows (indexer.ts ~1195-1199), and
|
|
1207
|
+
* mass-wipe is already gated upstream (`preserveExistingIndex` +
|
|
1208
|
+
* `fullDelete && scanComplete`). A temporarily unreachable source therefore
|
|
1209
|
+
* never surfaces candidates; no separate scan-status tracking is needed.
|
|
1210
|
+
*
|
|
1211
|
+
* Runs under the SAME index-writer lease / borrowed-state.db-connection
|
|
1212
|
+
* discipline as the neighboring maintenance passes: `dbCell.current` supplies
|
|
1213
|
+
* the already-open index.db handle (#584), and state.db access goes through
|
|
1214
|
+
* `withStateDb(..., { borrowed: eventsCtx?.db })` so this never opens a
|
|
1215
|
+
* second live writer alongside a long-lived `eventsCtx.db` connection — see
|
|
1216
|
+
* the #585 comment on {@link runRetentionPurgePass}'s events-purge call for
|
|
1217
|
+
* why that matters ("database is locked").
|
|
1218
|
+
*
|
|
1219
|
+
* Exported for direct test coverage (tests/integration/commands/improve/
|
|
1220
|
+
* state-gc.test.ts), mirroring the `runMemoryInferenceMaintenancePass` /
|
|
1221
|
+
* `runGraphExtractionMaintenancePass` / `runRetentionPurgePass` precedent;
|
|
1222
|
+
* production callers reach it only through `runMaintenancePassesUnderLease`.
|
|
1223
|
+
*/
|
|
1224
|
+
export function runOrphanStateGcPass(ctx, dbCell) {
|
|
1225
|
+
const { eventsCtx, config } = ctx;
|
|
1226
|
+
const warnings = [];
|
|
1227
|
+
const indexDb = dbCell.current;
|
|
1228
|
+
if (!indexDb) {
|
|
1229
|
+
warnings.push("orphan state GC skipped: no index.db handle available");
|
|
1230
|
+
return { pending: 0, collected: 0, warnings };
|
|
1231
|
+
}
|
|
1232
|
+
const collect = config.improve?.stateGc?.collect === true;
|
|
1233
|
+
const now = Date.now();
|
|
1234
|
+
let pending = 0;
|
|
1235
|
+
let collected = 0;
|
|
1236
|
+
try {
|
|
1237
|
+
withStateDb((stateDb) => {
|
|
1238
|
+
const salienceResult = gcOneStateTable({
|
|
1239
|
+
refRows: listAssetSalienceMissingState(stateDb),
|
|
1240
|
+
indexDb,
|
|
1241
|
+
now,
|
|
1242
|
+
collect,
|
|
1243
|
+
stamp: (refs, ts) => stampAssetSalienceMissing(stateDb, refs, ts),
|
|
1244
|
+
clear: (refs) => clearAssetSalienceMissing(stateDb, refs),
|
|
1245
|
+
deleteOlderThan: (cutoffMs) => deleteAssetSalienceMissingBefore(stateDb, cutoffMs),
|
|
1246
|
+
countPending: () => countAssetSalienceMissing(stateDb),
|
|
1247
|
+
});
|
|
1248
|
+
const outcomeResult = gcOneStateTable({
|
|
1249
|
+
refRows: listAssetOutcomeMissingState(stateDb),
|
|
1250
|
+
indexDb,
|
|
1251
|
+
now,
|
|
1252
|
+
collect,
|
|
1253
|
+
stamp: (refs, ts) => stampAssetOutcomeMissing(stateDb, refs, ts),
|
|
1254
|
+
clear: (refs) => clearAssetOutcomeMissing(stateDb, refs),
|
|
1255
|
+
deleteOlderThan: (cutoffMs) => deleteAssetOutcomeMissingBefore(stateDb, cutoffMs),
|
|
1256
|
+
countPending: () => countAssetOutcomeMissing(stateDb),
|
|
1257
|
+
});
|
|
1258
|
+
// Table-name-shaped keys ("salience"/"outcome", not the guarded
|
|
1259
|
+
// "asset_salience"/"asset_outcome" table names) — this file sits
|
|
1260
|
+
// outside src/storage/repositories/**, where the state-table-sql
|
|
1261
|
+
// lint rule (#672) forbids naming those tables even in a log string
|
|
1262
|
+
// or an object key, not just in raw SQL.
|
|
1263
|
+
const byTable = { salience: salienceResult, outcome: outcomeResult };
|
|
1264
|
+
pending = salienceResult.pending + outcomeResult.pending;
|
|
1265
|
+
collected = salienceResult.collected + outcomeResult.collected;
|
|
1266
|
+
if (pending > 0 || collected > 0) {
|
|
1267
|
+
info(`[improve] orphan state GC: ${pending} pending, ${collected} collected ` +
|
|
1268
|
+
`(salience ${salienceResult.pending}/${salienceResult.collected}, ` +
|
|
1269
|
+
`outcome ${outcomeResult.pending}/${outcomeResult.collected})`);
|
|
1270
|
+
// #733 — asset_state_gc reports the current per-table backlog
|
|
1271
|
+
// snapshot (`pending`) plus this run's deletions (`collected`);
|
|
1272
|
+
// emitted only when there is something to report (mirrors the
|
|
1273
|
+
// rekey script's no-op-stays-silent precedent) so a perpetually
|
|
1274
|
+
// clean stash never accumulates events.
|
|
1275
|
+
appendEvent({
|
|
1276
|
+
eventType: "asset_state_gc",
|
|
1277
|
+
ref: "asset_state/_gc",
|
|
1278
|
+
metadata: { pending, collected, byTable },
|
|
1279
|
+
}, eventsCtx);
|
|
1280
|
+
}
|
|
1281
|
+
}, { path: eventsCtx?.dbPath, borrowed: eventsCtx?.db });
|
|
1282
|
+
}
|
|
1283
|
+
catch (err) {
|
|
1284
|
+
warnings.push(`orphan state GC failed: ${errMessage(err)}`);
|
|
1285
|
+
}
|
|
1286
|
+
return { pending, collected, warnings };
|
|
1100
1287
|
}
|