akm-cli 0.9.16 → 0.9.17-alpha.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +2101 -0
- package/STABILITY.md +11 -10
- package/dist/akm +124 -193
- package/dist/akm-migrate +38 -19
- package/dist/assets/hints/cli-hints-full.md +6 -7
- package/dist/assets/improve-strategies/catchup.json +0 -3
- package/dist/assets/improve-strategies/consolidate.json +0 -1
- package/dist/assets/improve-strategies/default.json +1 -2
- package/dist/assets/improve-strategies/proactive-maintenance.json +1 -2
- package/dist/assets/improve-strategies/quick.json +1 -2
- package/dist/assets/improve-strategies/reflect-distill.json +1 -2
- package/dist/assets/improve-strategies/thorough.json +0 -3
- package/dist/assets/prompts/consolidate-pair.md +20 -0
- package/dist/assets/prompts/consolidate-system.md +4 -11
- package/dist/assets/prompts/retrieval-relevance-judge.md +6 -0
- package/dist/assets/stash-skeleton/facts/conventions/backlinks.md +20 -20
- package/dist/assets/stash-skeleton/facts/conventions/domains.md +2 -2
- package/dist/assets/templates/html/health.html +3 -5
- package/dist/cli/retired-commands.js +1 -1
- package/dist/cli/shared.js +6 -2
- package/dist/cli/unknown-flags.js +24 -1
- package/dist/cli.js +68 -10
- package/dist/commands/agent/agent-dispatch.js +1 -1
- package/dist/commands/command/command-execution.js +24 -62
- package/dist/commands/feedback-cli.js +0 -1
- package/dist/commands/health/accept-rate.js +6 -0
- package/dist/commands/health/archive-usage.js +92 -0
- package/dist/commands/health/checks.js +83 -74
- package/dist/commands/health/config-skew.js +38 -0
- package/dist/commands/health/data-dir-usage.js +25 -13
- package/dist/commands/health/egress.js +54 -0
- package/dist/commands/health/html-report.js +1 -42
- package/dist/commands/health/improve-metrics.js +136 -591
- package/dist/commands/health/md-report.js +1 -6
- package/dist/commands/health/plugin-staleness.js +53 -3
- package/dist/commands/health/renderers.js +12 -4
- package/dist/commands/health/report-view-model.js +14 -120
- package/dist/commands/health/types-improve.js +4 -19
- package/dist/commands/health/windows.js +64 -74
- package/dist/commands/health.js +145 -143
- package/dist/commands/improve/consolidate/chunking.js +26 -117
- package/dist/commands/improve/consolidate/continuity-check.js +137 -0
- package/dist/commands/improve/consolidate/pair-pass.js +791 -0
- package/dist/commands/improve/consolidate/sanitize.js +54 -149
- package/dist/commands/improve/consolidate.js +589 -1127
- package/dist/commands/improve/content-hash.js +16 -24
- package/dist/commands/improve/distill/content-repair.js +18 -100
- package/dist/commands/improve/distill-guards.js +20 -81
- package/dist/commands/improve/distill-promotion-policy.js +23 -243
- package/dist/commands/improve/distill.js +608 -1041
- package/dist/commands/improve/eligibility.js +126 -390
- package/dist/commands/improve/execution.js +8 -10
- package/dist/commands/improve/extract-prompt.js +1 -2
- package/dist/commands/improve/extract.js +487 -1046
- package/dist/commands/improve/feedback-valence.js +0 -25
- package/dist/commands/improve/improve-cli.js +75 -169
- package/dist/commands/improve/improve-result-file.js +10 -66
- package/dist/commands/improve/improve-strategies.js +52 -4
- package/dist/commands/improve/improve-usage-report.js +18 -64
- package/dist/commands/improve/improve.js +480 -1074
- package/dist/commands/improve/ledger.js +119 -0
- package/dist/commands/improve/locks.js +2 -8
- package/dist/commands/improve/loop-stages.js +415 -1073
- package/dist/commands/improve/memory/derived-ref.js +12 -77
- package/dist/commands/improve/memory/memory-belief.js +16 -118
- package/dist/commands/improve/memory/memory-improve.js +266 -14
- package/dist/commands/improve/outcome-loop.js +28 -156
- package/dist/commands/improve/planner.js +5 -15
- package/dist/commands/improve/preparation.js +779 -2319
- package/dist/commands/improve/proactive-maintenance.js +34 -101
- package/dist/commands/improve/reflect-noise.js +104 -280
- package/dist/commands/improve/reflect.js +642 -1353
- package/dist/commands/improve/retrieval-gate.js +127 -0
- package/dist/commands/improve/retrieval-scope.js +92 -0
- package/dist/commands/improve/salience.js +41 -240
- package/dist/commands/improve/session-asset.js +19 -100
- package/dist/commands/improve/stage.js +322 -0
- package/dist/commands/lint/base-linter.js +37 -15
- package/dist/commands/proposal/drain.js +261 -578
- package/dist/commands/proposal/proposal-cli.js +19 -20
- package/dist/commands/proposal/proposal-types.js +31 -24
- package/dist/commands/proposal/proposal.js +38 -8
- package/dist/commands/proposal/propose.js +134 -160
- package/dist/commands/proposal/repository.js +1097 -1394
- package/dist/commands/proposal/validators/proposal-quality-validators.js +71 -174
- package/dist/commands/proposal/validators/proposal-validators.js +1 -1
- package/dist/commands/proposal/validators/proposals.js +22 -89
- package/dist/commands/read/curate.js +105 -462
- package/dist/commands/read/knowledge.js +3 -2
- package/dist/commands/read/search-cli.js +16 -33
- package/dist/commands/read/search.js +17 -23
- package/dist/commands/read/show.js +57 -108
- package/dist/commands/sources/bundle-cli.js +25 -2
- package/dist/commands/sources/bundle-config-ops.js +4 -0
- package/dist/commands/sources/dangerous-env-audit.js +1 -2
- package/dist/commands/sources/info.js +127 -29
- package/dist/commands/sources/installed-stashes.js +197 -746
- package/dist/commands/sources/schema-repair.js +98 -129
- package/dist/commands/sources/source-add.js +62 -12
- package/dist/commands/sources/source-manage.js +9 -2
- package/dist/commands/sources/stash-cli.js +24 -4
- package/dist/commands/tasks/explain.js +10 -13
- package/dist/commands/tasks/tasks-cli.js +12 -13
- package/dist/commands/tasks/tasks.js +350 -936
- package/dist/commands/tasks/validate.js +26 -24
- package/dist/commands/workflow/plan.js +22 -29
- package/dist/commands/workflow-cli.js +4 -4
- package/dist/core/adapter/adapters/akm-adapter.js +2 -1
- package/dist/core/adapter/adapters/akm-lint.js +2 -3
- package/dist/core/adapter/adapters/akm-metadata.js +42 -12
- package/dist/core/adapter/adapters/akm-task-adapter.js +29 -8
- package/dist/core/adapter/adapters/akm-workflow-adapter.js +1 -1
- package/dist/core/adapter/execution-source.js +17 -29
- package/dist/core/asset/asset-placement.js +4 -13
- package/dist/core/asset/frontmatter.js +106 -1
- package/dist/core/asset/resolve-ref.js +1 -1
- package/dist/core/bundle-id.js +42 -5
- package/dist/core/bundle-rename.js +285 -0
- package/dist/core/config/config-io.js +1 -2
- package/dist/core/config/config-schema.js +9 -34
- package/dist/core/config/config-walker.js +1 -1
- package/dist/core/config/config.js +184 -111
- package/dist/core/config/engine-semantics.js +0 -2
- package/dist/core/config/legacy-source-shape-shim.js +38 -9
- package/dist/core/config/schema/embedding.js +20 -5
- package/dist/core/config/schema/engines.js +5 -0
- package/dist/core/config/schema/execution.js +1 -1
- package/dist/core/config/schema/experimental.js +1 -1
- package/dist/core/config/schema/improve-processes.js +54 -125
- package/dist/core/config/schema/improve.js +4 -42
- package/dist/core/config/schema/index-config.js +9 -48
- package/dist/core/config/schema/scheduler.js +12 -12
- package/dist/core/config/schema/search.js +6 -22
- package/dist/core/env-secret-ref.js +0 -1
- package/dist/core/errors.js +8 -9
- package/dist/core/file-change.js +13 -5
- package/dist/core/file-lock.js +76 -173
- package/dist/core/improve-result.js +35 -7
- package/dist/core/improve-types.js +0 -1
- package/dist/core/logs-db.js +2 -2
- package/dist/core/loopback.js +7 -12
- package/dist/core/non-task-input.js +20 -0
- package/dist/core/parse.js +13 -16
- package/dist/core/paths.js +0 -24
- package/dist/core/redaction.js +109 -2
- package/dist/core/run-lock.js +2 -5
- package/dist/core/spawn-env.js +1 -1
- package/dist/core/state/migrations.js +123 -61
- package/dist/core/state-db-scope.js +2 -4
- package/dist/core/state-db.js +126 -692
- package/dist/core/time.js +0 -20
- package/dist/core/type-presentation.js +1 -9
- package/dist/core/write-source.js +294 -1005
- package/dist/execution/input-contract.js +1 -1
- package/dist/execution/resolved-request.js +135 -689
- package/dist/execution/source.js +63 -257
- package/dist/execution/target-ref.js +1 -1
- package/dist/indexer/bundle-identity-guard.js +2 -2
- package/dist/indexer/db/llm-cache.js +2 -2
- package/dist/indexer/ensure-index.js +77 -73
- package/dist/indexer/index-rebuild-lock.js +3 -11
- package/dist/indexer/index-writer-lock.js +8 -17
- package/dist/indexer/index-written-assets.js +141 -154
- package/dist/indexer/indexer.js +400 -1124
- package/dist/indexer/links/declared-links.js +90 -0
- package/dist/indexer/materialize-embeddings.js +60 -397
- package/dist/indexer/passes/memory-inference.js +96 -90
- package/dist/indexer/passes/metadata.js +132 -219
- package/dist/indexer/read-preflight.js +0 -7
- package/dist/indexer/scan/doc-to-entry.js +2 -3
- package/dist/indexer/scan/drain-dir.js +1 -1
- package/dist/indexer/search/db-search.js +190 -590
- package/dist/indexer/search/fts-query.js +30 -41
- package/dist/indexer/search/ranking.js +28 -154
- package/dist/indexer/search/search-attribution.js +12 -32
- package/dist/indexer/search/search-fields.js +11 -15
- package/dist/indexer/search/search-hit-enrichers.js +54 -85
- package/dist/indexer/search/search-source.js +1 -4
- package/dist/indexer/usage/usage-events.js +36 -7
- package/dist/indexer/walk/walker.js +3 -4
- package/dist/integrations/agent/engine-fallback.js +23 -40
- package/dist/integrations/agent/engine-resolution.js +93 -183
- package/dist/integrations/agent/execution.js +507 -0
- package/dist/integrations/agent/model-map.js +28 -156
- package/dist/integrations/agent/request-lowering.js +66 -141
- package/dist/integrations/agent/runner-dispatch.js +143 -321
- package/dist/integrations/agent/runner.js +54 -14
- package/dist/integrations/lockfile.js +53 -101
- package/dist/llm/client.js +18 -6
- package/dist/llm/embedders/deterministic.js +2 -3
- package/dist/llm/embedders/profile.js +71 -0
- package/dist/llm/embedders/remote.js +11 -17
- package/dist/llm/feature-gate.js +0 -8
- package/dist/llm/index-passes.js +3 -5
- package/dist/llm/memory-infer.js +1 -2
- package/dist/llm/structured-call.js +5 -24
- package/dist/output/generic-render.js +23 -11
- package/dist/output/html-render.js +13 -10
- package/dist/output/render-registry.js +3 -32
- package/dist/output/shapes/helpers.js +25 -38
- package/dist/output/shapes/passthrough.js +1 -9
- package/dist/{indexer/graph/graph-types.js → output/text/bundle-rename.js} +4 -1
- package/dist/output/text/command-format.js +69 -31
- package/dist/output/text/helpers.js +1 -1
- package/dist/output/text/migrate.js +5 -14
- package/dist/output/text/proposal-format.js +48 -3
- package/dist/output/text/show-format.js +13 -17
- package/dist/output/text/workflow-format.js +0 -32
- package/dist/output/text.js +2 -0
- package/dist/registry/factory.js +4 -19
- package/dist/registry/network.js +66 -220
- package/dist/registry/providers/index.js +0 -2
- package/dist/registry/providers/skills-sh.js +3 -14
- package/dist/registry/providers/static-index.js +24 -26
- package/dist/registry/resolve.js +55 -131
- package/dist/scripts/akm-migrate-node.js +42948 -92369
- package/dist/scripts/akm-migrate.js +42935 -92354
- package/dist/setup/registry-stash-loader.js +4 -13
- package/dist/setup/semantic-assets.js +3 -44
- package/dist/setup/setup.js +1 -1
- package/dist/setup/steps/connection.js +5 -6
- package/dist/setup/steps/platforms.js +2 -2
- package/dist/setup/steps/tasks.js +25 -15
- package/dist/sources/provider-factory.js +17 -18
- package/dist/sources/providers/filesystem.js +2 -3
- package/dist/sources/providers/git-install.js +7 -1
- package/dist/sources/providers/git-provider.js +0 -3
- package/dist/sources/providers/git-stash.js +83 -21
- package/dist/sources/providers/npm.js +2 -4
- package/dist/sources/providers/provider-utils.js +5 -10
- package/dist/sources/providers/website.js +0 -2
- package/dist/sources/snapshot-fetchers/website-ingest.js +1 -1
- package/dist/sources/website-url.js +2 -2
- package/dist/storage/database.js +9 -35
- package/dist/storage/repositories/improve-ledger-repository.js +209 -0
- package/dist/storage/repositories/index-connection.js +39 -72
- package/dist/storage/repositories/index-entries-repository.js +131 -129
- package/dist/storage/repositories/index-entry-mapper.js +1 -2
- package/dist/storage/repositories/index-entry-schema.js +101 -268
- package/dist/storage/repositories/index-fts-repository.js +86 -256
- package/dist/storage/repositories/index-links-repository.js +143 -0
- package/dist/storage/repositories/index-llm-cache-repository.js +7 -9
- package/dist/storage/repositories/index-meta-repository.js +6 -4
- package/dist/storage/repositories/index-schema.js +257 -325
- package/dist/storage/repositories/index-utility-repository.js +8 -29
- package/dist/storage/repositories/index-vec-repository.js +133 -414
- package/dist/storage/repositories/outcome-repository.js +2 -1
- package/dist/storage/repositories/proposals-repository.js +104 -1
- package/dist/storage/repositories/registry-index-cache-repository.js +100 -0
- package/dist/storage/repositories/salience-repository.js +1 -19
- package/dist/storage/repositories/task-history-repository.js +26 -4
- package/dist/storage/repositories/workflow-runs-repository.js +53 -244
- package/dist/storage/sqlite-migrations.js +136 -0
- package/dist/storage/sqlite-pragmas.js +11 -9
- package/dist/storage/sqlite-transaction.js +170 -0
- package/dist/storage/state-db-integrity.js +130 -0
- package/dist/tasks/activation-config.js +134 -62
- package/dist/tasks/backends/cron.js +191 -302
- package/dist/tasks/backends/exec-utils.js +2 -5
- package/dist/tasks/backends/launchd.js +141 -748
- package/dist/tasks/backends/schtasks.js +119 -623
- package/dist/tasks/prepare/prepare-support.js +5 -15
- package/dist/tasks/prepare/prepare.js +0 -2
- package/dist/tasks/resolve-akm-bin.js +20 -79
- package/dist/tasks/run/attempt-lifecycle.js +0 -1
- package/dist/tasks/run/load-task.js +1 -1
- package/dist/tasks/scheduler-binding.js +20 -238
- package/dist/tasks/scheduler-invocation.js +136 -244
- package/dist/tasks/scheduler-lock.js +53 -0
- package/dist/tasks/scheduler-sync.js +368 -679
- package/dist/tasks/source/parse-task-source.js +55 -9
- package/dist/tasks/source/task-source-v3-frozen.js +3 -4
- package/dist/tasks/source/task-to-v4.js +464 -88
- package/dist/workflows/authoring/authoring.js +3 -12
- package/dist/workflows/compile.js +211 -0
- package/dist/workflows/concurrency-policy.js +13 -74
- package/dist/workflows/exec/child-invocation.js +3 -17
- package/dist/workflows/exec/child-workflow.js +32 -141
- package/dist/workflows/exec/dispatch-redaction.js +13 -53
- package/dist/workflows/exec/environment.js +98 -0
- package/dist/workflows/exec/exec-unit.js +33 -140
- package/dist/workflows/exec/frozen-judge.js +7 -59
- package/dist/workflows/exec/native-executor.js +82 -341
- package/dist/workflows/exec/param-secrets.js +29 -47
- package/dist/workflows/exec/run-workflow.js +154 -387
- package/dist/workflows/exec/scheduler.js +9 -36
- package/dist/workflows/exec/step-work.js +127 -430
- package/dist/workflows/exec/unit-dispatch.js +11 -63
- package/dist/workflows/exec/unit-writer.js +8 -52
- package/dist/workflows/exec/worktree.js +39 -273
- package/dist/workflows/freeze/child-output-references.js +4 -15
- package/dist/workflows/freeze/environment.js +99 -92
- package/dist/workflows/freeze/freeze.js +172 -0
- package/dist/workflows/freeze/step-values.js +19 -21
- package/dist/workflows/freeze/targets/child-workflow.js +23 -92
- package/dist/workflows/freeze/targets/command.js +10 -33
- package/dist/workflows/freeze/targets/script.js +5 -12
- package/dist/workflows/freeze/targets/shell.js +3 -6
- package/dist/workflows/freeze/targets/task.js +25 -80
- package/dist/workflows/freeze/task-bindings.js +20 -67
- package/dist/workflows/{source-ir/github-yaml.js → github-yaml.js} +88 -206
- package/dist/workflows/ir/params.js +6 -51
- package/dist/workflows/ir/plan-hash.js +2 -34
- package/dist/workflows/parser.js +140 -43
- package/dist/{commands/improve/consolidate/types.js → workflows/plan.js} +2 -1
- package/dist/workflows/renderer.js +36 -69
- package/dist/workflows/resource-limits.js +12 -120
- package/dist/workflows/runtime/agent-identity.js +8 -40
- package/dist/workflows/runtime/run-outputs.js +3 -6
- package/dist/workflows/runtime/run-plan.js +316 -0
- package/dist/workflows/runtime/runs.js +48 -200
- package/dist/workflows/runtime/workflow-asset-loader.js +24 -57
- package/dist/workflows/{source-ir/semantics.js → source-semantics.js} +16 -20
- package/dist/workflows/validate-summary.js +2 -7
- package/docs/integration/bundling-akm.md +49 -42
- package/docs/migration/README.md +1 -0
- package/docs/migration/release-notes/0.9.17.md +43 -0
- package/docs/migration/v0.9.1-to-v0.9.2.md +23 -7
- package/docs/reference/cli.md +232 -135
- package/docs/reference/configuration.md +71 -57
- package/docs/reference/data-and-telemetry.md +20 -21
- package/docs/reference/tasks.md +105 -39
- package/docs/reference/workflow-schema.md +14 -18
- package/docs/reference/workflows.md +6 -9
- package/package.json +1 -1
- package/schemas/akm-config.json +115 -738
- package/schemas/akm-workflow.json +1 -0
- package/dist/assets/improve-strategies/graph-refresh.json +0 -15
- package/dist/assets/prompts/contradiction-judge.md +0 -33
- package/dist/assets/prompts/graph-extract-system.md +0 -1
- package/dist/assets/prompts/graph-extract-user-prompt.md +0 -35
- package/dist/assets/prompts/metadata-enhance-system.md +0 -1
- package/dist/assets/tasks/improve/akm-graph-refresh-weekly.yml +0 -4
- package/dist/commands/health/advisories.js +0 -150
- package/dist/commands/health/metrics.js +0 -329
- package/dist/commands/health/surfaces.js +0 -102
- package/dist/commands/improve/anti-collapse.js +0 -83
- package/dist/commands/improve/collapse-detector.js +0 -432
- package/dist/commands/improve/consolidate/eligibility.js +0 -48
- package/dist/commands/improve/consolidate/merge.js +0 -149
- package/dist/commands/improve/distill/promote-memory.js +0 -291
- package/dist/commands/improve/distill/quality-gate.js +0 -337
- package/dist/commands/improve/eval-cases.js +0 -52
- package/dist/commands/improve/memory/memory-contradiction-detect.js +0 -291
- package/dist/commands/improve/proposal-envelope.js +0 -31
- package/dist/commands/improve/run-context.js +0 -123
- package/dist/commands/improve/shared.js +0 -31
- package/dist/commands/improve/source-identity.js +0 -28
- package/dist/commands/improve/triage.js +0 -96
- package/dist/commands/proposal/drain-policies.js +0 -151
- package/dist/commands/sources/update-transaction.js +0 -220
- package/dist/core/action-contributors.js +0 -28
- package/dist/core/config/config-version-shim.js +0 -101
- package/dist/core/fs-txn.js +0 -405
- package/dist/core/lexical-score.js +0 -25
- package/dist/core/maintenance-barrier.js +0 -167
- package/dist/execution/executable-identity.js +0 -105
- package/dist/execution/guarded-source.js +0 -427
- package/dist/indexer/db/graph-db.js +0 -444
- package/dist/indexer/graph/graph-boost.js +0 -427
- package/dist/indexer/graph/graph-dedup.js +0 -95
- package/dist/indexer/graph/graph-extraction.js +0 -1108
- package/dist/indexer/search/name-match.js +0 -35
- package/dist/indexer/search/ranking-contributors.js +0 -515
- package/dist/indexer/search/ranking-types.js +0 -4
- package/dist/indexer/walk/project-context.js +0 -192
- package/dist/integrations/agent/execution-cascade.js +0 -566
- package/dist/integrations/agent/execution-definitions.js +0 -202
- package/dist/integrations/agent/execution-lowering.js +0 -841
- package/dist/integrations/agent/execution-preparation.js +0 -98
- package/dist/integrations/agent/inline-execution.js +0 -74
- package/dist/llm/graph-extract.js +0 -728
- package/dist/llm/metadata-enhance.js +0 -96
- package/dist/registry/create-provider-registry.js +0 -29
- package/dist/registry/pinned-request-helper.js +0 -247
- package/dist/registry/pinned-transport.js +0 -717
- package/dist/sources/providers/index.js +0 -14
- package/dist/storage/engines/sqlite-migrations.js +0 -271
- package/dist/storage/repositories/canaries-repository.js +0 -107
- package/dist/storage/repositories/embedding-salvage-repository.js +0 -184
- package/dist/storage/repositories/registry-cache.js +0 -113
- package/dist/tasks/scheduler-sync-preview.js +0 -52
- package/dist/tasks/source/task-to-v3.js +0 -507
- package/dist/workflows/freeze/resolve-steps.js +0 -86
- package/dist/workflows/freeze/source-freeze.js +0 -64
- package/dist/workflows/ir/compile.js +0 -321
- package/dist/workflows/ir/environment-v4.js +0 -330
- package/dist/workflows/ir/freeze-v4.js +0 -153
- package/dist/workflows/ir/schema-v4.js +0 -745
- package/dist/workflows/ir/schema.js +0 -354
- package/dist/workflows/program/schema.js +0 -77
- package/dist/workflows/runtime/checkin.js +0 -57
- package/dist/workflows/runtime/plan-classifier.js +0 -196
- package/dist/workflows/runtime/unit-checkin.js +0 -45
- package/dist/workflows/runtime/unit-phases.js +0 -20
- package/dist/workflows/schema.js +0 -4
- package/dist/workflows/source-ir/compile.js +0 -200
- package/dist/workflows/source-ir/program.js +0 -50
- package/dist/workflows/source-ir/result.js +0 -26
- package/dist/workflows/source-ir/schema.js +0 -786
- package/dist/workflows/source-ir/triggers.js +0 -79
- package/dist/workflows/source-ir/uses.js +0 -40
- package/dist/workflows/validator.js +0 -60
|
@@ -1,114 +1,95 @@
|
|
|
1
1
|
// This Source Code Form is subject to the terms of the Mozilla Public
|
|
2
2
|
// License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
3
3
|
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
4
|
+
/** The improve loop (reflect + distill per ref), the post-loop checks and the maintenance passes. */
|
|
4
5
|
import fs from "node:fs";
|
|
5
|
-
import path from "node:path";
|
|
6
6
|
import { parseRefInput } from "../../core/asset/resolve-ref.js";
|
|
7
7
|
import { daysToMs } from "../../core/common.js";
|
|
8
|
-
import {
|
|
8
|
+
import { loadConfig } from "../../core/config/config.js";
|
|
9
9
|
import { UsageError } from "../../core/errors.js";
|
|
10
10
|
import { appendEvent } from "../../core/events.js";
|
|
11
11
|
import { openLogsDatabase, purgeOldTaskLogs } from "../../core/logs-db.js";
|
|
12
12
|
import { getDbPath, getTaskLogDir } from "../../core/paths.js";
|
|
13
13
|
import { withStateDb } from "../../core/state-db.js";
|
|
14
14
|
import { info } from "../../core/warn.js";
|
|
15
|
-
import {
|
|
16
|
-
import { deriveWritableBundleIds } from "../../indexer/installations.js";
|
|
15
|
+
import { indexWrittenAssets } from "../../indexer/index-written-assets.js";
|
|
17
16
|
import { collectPendingMemories, runMemoryInferencePass, } from "../../indexer/passes/memory-inference.js";
|
|
18
17
|
import { resolveSourceEntries } from "../../indexer/search/search-source.js";
|
|
19
|
-
import { isProcessEnabled } from "../../llm/feature-gate.js";
|
|
20
|
-
import { withLlmStage } from "../../llm/usage-telemetry.js";
|
|
21
|
-
import { purgeOldCycleMetrics } from "../../storage/repositories/canaries-repository.js";
|
|
22
18
|
import { purgeOldEvents } from "../../storage/repositories/events-repository.js";
|
|
23
19
|
import { purgeOldImproveRuns } from "../../storage/repositories/improve-runs-repository.js";
|
|
24
20
|
import { closeDatabase, openIndexDatabase } from "../../storage/repositories/index-connection.js";
|
|
25
|
-
import {
|
|
21
|
+
import { getLiveRefSnapshot, isRefLiveInSnapshot, } from "../../storage/repositories/index-entries-repository.js";
|
|
26
22
|
import { clearAssetOutcomeMissing, countAssetOutcomeMissing, deleteAssetOutcomeMissingBefore, listAssetOutcomeMissingState, stampAssetOutcomeMissing, } from "../../storage/repositories/outcome-repository.js";
|
|
27
23
|
import { clearAssetSalienceMissing, countAssetSalienceMissing, deleteAssetSalienceMissingBefore, listAssetSalienceMissingState, stampAssetSalienceMissing, } from "../../storage/repositories/salience-repository.js";
|
|
24
|
+
import { readFreelistInfo, STATE_DB_VACUUMED_EVENT, vacuumIfReclaimable } from "../../storage/state-db-integrity.js";
|
|
28
25
|
import { purgeOldTaskLogFiles } from "../../tasks/run/task-log.js";
|
|
29
|
-
import { expireStaleProposals,
|
|
26
|
+
import { expireStaleProposals, purgeOrphanProposals } from "../proposal/repository.js";
|
|
30
27
|
import { checkDeadUrls } from "../url-checker.js";
|
|
31
|
-
import {
|
|
32
|
-
import { deriveLessonRef } from "./distill.js";
|
|
33
|
-
import { deriveKnowledgeRef } from "./distill-promotion-policy.js";
|
|
34
|
-
// Eligibility / candidate-selection predicates live in ./eligibility.
|
|
35
|
-
import { findAssetFilePath, isDistillCandidateRef } from "./eligibility.js";
|
|
36
|
-
import { writeEvalCase } from "./eval-cases.js";
|
|
28
|
+
import { isDistillCandidateRef } from "./eligibility.js";
|
|
37
29
|
import { shouldSkipRef } from "./improve-strategies.js";
|
|
30
|
+
import { recordLedgerAttempt, stateKey, stripBundle } from "./ledger.js";
|
|
31
|
+
import { pushRecentError } from "./preparation.js";
|
|
38
32
|
import { recordNoOp, resetConsecutiveNoOps } from "./salience.js";
|
|
39
|
-
import {
|
|
40
|
-
import { bareImproveRef, durableImproveRef } from "./source-identity.js";
|
|
41
|
-
// ── improve loop / post-loop / maintenance stages ───────────────────
|
|
42
|
-
// The cycle stages run by akmImprove, extracted from improve.ts.
|
|
43
|
-
/** O-5 / #378: rolling per-originator error-window cap. */
|
|
44
|
-
const RECENT_ERRORS_CAP = 3;
|
|
45
|
-
/** O-5 / #378: push a per-originator error into the rolling window. */
|
|
46
|
-
function pushRecentError(recentErrors, originator, msg) {
|
|
47
|
-
if (!recentErrors[originator])
|
|
48
|
-
recentErrors[originator] = [];
|
|
49
|
-
recentErrors[originator].push(msg);
|
|
50
|
-
if (recentErrors[originator].length > RECENT_ERRORS_CAP)
|
|
51
|
-
recentErrors[originator].shift();
|
|
52
|
-
}
|
|
53
|
-
/**
|
|
54
|
-
* Build the per-run loop environment from the run context: the derived guards
|
|
55
|
-
* and the pending-proposal preload.
|
|
56
|
-
*/
|
|
33
|
+
import { attributeStage, errMessage } from "./stage.js";
|
|
57
34
|
export function prepareImproveLoopEnv(args) {
|
|
58
|
-
const
|
|
59
|
-
|
|
60
|
-
// now RunContext's required eventsCtx / optional signal — renamed local
|
|
61
|
-
// aliases so the rest of this function (and the ImproveLoopEnv object
|
|
62
|
-
// literal below) is unchanged. `primaryStashDir` deliberately comes from
|
|
63
|
-
// the state's honest optional field, NOT `ctx.stashDir`: the rare
|
|
64
|
-
// unresolvable-primary path must keep skipping the `if (primaryStashDir)`
|
|
65
|
-
// guards below (see the field's doc in ./improve-run-types).
|
|
66
|
-
const primaryStashDir = args.primaryStashDir;
|
|
67
|
-
const eventsCtx = ctx.eventsCtx;
|
|
68
|
-
const budgetSignal = ctx.signal;
|
|
69
|
-
// O-1 (#364): compute remaining budget at call time so each sub-call
|
|
70
|
-
// receives only its fair share of the wall-clock budget.
|
|
71
|
-
const remainingBudgetMs = () => Math.max(0, budgetMs - (Date.now() - startMs));
|
|
72
|
-
// Build a Set for O(1) membership test — these refs skip the reflect call (Bug D2).
|
|
73
|
-
const distillOnlyRefSet = new Set(distillOnlyRefs.map((r) => r.ref));
|
|
74
|
-
// requirePlannedRefs guard: when the distill profile sets this flag, skip
|
|
75
|
-
// distill for distill-only refs if the reflect phase produced no planned refs.
|
|
76
|
-
// Prevents the distill loop from generating hundreds of distill-skipped events
|
|
77
|
-
// on quiet passes (all refs on reflect cooldown, no new signal to distill).
|
|
78
|
-
const requirePlannedRefs = improveProfile?.processes?.distill?.requirePlannedRefs === true;
|
|
79
|
-
const hasReflectEligibleRefs = loopRefs.some((r) => !distillOnlyRefSet.has(r.ref));
|
|
80
|
-
const skipDistillDueToRequirePlannedRefs = requirePlannedRefs && !hasReflectEligibleRefs;
|
|
81
|
-
// Pre-load all pending proposals once instead of querying per asset in the loop.
|
|
82
|
-
const dedupeStashDirForProposals = primaryStashDir ?? options.stashDir;
|
|
83
|
-
const pendingProposalRefSet = new Set(dedupeStashDirForProposals
|
|
84
|
-
? listProposals(dedupeStashDirForProposals, { status: "pending" }).map((p) => p.ref)
|
|
85
|
-
: []);
|
|
35
|
+
const distillOnlyRefSet = new Set(args.distillOnlyRefs.map((r) => r.ref));
|
|
36
|
+
const requirePlannedRefs = args.improveProfile?.processes?.distill?.requirePlannedRefs === true;
|
|
86
37
|
return {
|
|
87
|
-
scope,
|
|
88
|
-
options,
|
|
89
|
-
primaryStashDir,
|
|
90
|
-
reflectFn,
|
|
91
|
-
distillFn,
|
|
92
|
-
signalBearingSet,
|
|
93
|
-
distillCooledRefs,
|
|
38
|
+
scope: args.scope,
|
|
39
|
+
options: args.options,
|
|
40
|
+
primaryStashDir: args.primaryStashDir,
|
|
41
|
+
reflectFn: args.reflectFn,
|
|
42
|
+
distillFn: args.distillFn,
|
|
43
|
+
signalBearingSet: args.signalBearingSet,
|
|
44
|
+
distillCooledRefs: args.distillCooledRefs,
|
|
94
45
|
distillOnlyRefSet,
|
|
95
|
-
recentErrors,
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
pendingProposalRefSet,
|
|
103
|
-
remainingBudgetMs,
|
|
46
|
+
recentErrors: args.recentErrors,
|
|
47
|
+
eventsCtx: args.eventsCtx,
|
|
48
|
+
improveProfile: args.improveProfile,
|
|
49
|
+
resolvedPlan: args.resolvedPlan,
|
|
50
|
+
budgetSignal: args.budgetSignal,
|
|
51
|
+
skipDistillDueToRequirePlannedRefs: requirePlannedRefs && args.loopRefs.every((r) => distillOnlyRefSet.has(r.ref)),
|
|
52
|
+
remainingBudgetMs: () => Math.max(0, args.budgetMs - (Date.now() - args.startMs)),
|
|
104
53
|
};
|
|
105
54
|
}
|
|
106
55
|
/**
|
|
107
|
-
*
|
|
108
|
-
*
|
|
109
|
-
* `continue` in the old inline loop body is an early `return` inside the
|
|
110
|
-
* passes; the orchestrator folds the returned tally and owns the run counters.
|
|
56
|
+
* Record a loop attempt in the improve ledger. Proposals and quality
|
|
57
|
+
* rejections record themselves; the loop records what they cannot see.
|
|
111
58
|
*/
|
|
59
|
+
function recordLoopAttempt(planned, env, source, outcome, detail) {
|
|
60
|
+
const stashDir = env.primaryStashDir ?? env.options.stashDir;
|
|
61
|
+
if (!stashDir || env.options.dryRun)
|
|
62
|
+
return;
|
|
63
|
+
recordLedgerAttempt({ eventsCtx: env.eventsCtx }, {
|
|
64
|
+
stashDir,
|
|
65
|
+
ref: stateKey(planned.ref, planned.itemRef),
|
|
66
|
+
source,
|
|
67
|
+
outcome,
|
|
68
|
+
...(detail !== undefined ? { detail } : {}),
|
|
69
|
+
});
|
|
70
|
+
}
|
|
71
|
+
/** Plasticity counter: repeated no-ops dampen an asset's selection score; a change lifts it. */
|
|
72
|
+
function recordPlasticity(env, planned, outcome) {
|
|
73
|
+
const db = env.eventsCtx?.db;
|
|
74
|
+
if (!db || !outcome)
|
|
75
|
+
return;
|
|
76
|
+
const key = stateKey(planned.ref, planned.itemRef);
|
|
77
|
+
try {
|
|
78
|
+
if (outcome === "noop")
|
|
79
|
+
recordNoOp(db, key);
|
|
80
|
+
else
|
|
81
|
+
resetConsecutiveNoOps(db, key);
|
|
82
|
+
}
|
|
83
|
+
catch {
|
|
84
|
+
// best-effort
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
function recordSkip(tally, ref, reason, event) {
|
|
88
|
+
tally.actions.push({ ref, mode: "distill-skipped", result: { ok: true, reason } });
|
|
89
|
+
if (event)
|
|
90
|
+
appendEvent({ eventType: "improve_skipped", ref, metadata: { reason: event.reason } }, event.env.eventsCtx);
|
|
91
|
+
}
|
|
92
|
+
/** One loop iteration: reflect, then distill. A distill UsageError is a validation failure. */
|
|
112
93
|
export async function processImproveLoopRef(planned, env) {
|
|
113
94
|
const tally = {
|
|
114
95
|
actions: [],
|
|
@@ -117,20 +98,15 @@ export async function processImproveLoopRef(planned, env) {
|
|
|
117
98
|
memoryRefsForInference: [],
|
|
118
99
|
};
|
|
119
100
|
try {
|
|
120
|
-
// Bug D2: distillOnlyRefs skip the reflect call but still run the distill path.
|
|
121
|
-
// Bug D1: in-loop distill-cooldown check removed — distill-cooled candidates
|
|
122
|
-
// have their synthetic actions emitted in runImprovePreparationStage.
|
|
123
101
|
const isDistillOnly = env.distillOnlyRefSet.has(planned.ref);
|
|
124
|
-
const
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
await runLoopDistillPass(planned,
|
|
102
|
+
const parsed = parseRefInput(planned.ref);
|
|
103
|
+
if (!isDistillOnly)
|
|
104
|
+
await runLoopReflectPass(planned, env, tally);
|
|
105
|
+
await runLoopDistillPass(planned, parsed.type, isDistillOnly, env, tally);
|
|
128
106
|
}
|
|
129
107
|
catch (err) {
|
|
130
|
-
// B7: UsageError thrown by akmDistill on validation_failed should be recorded
|
|
131
|
-
// as mode:"distill" with outcome:"validation_failed", NOT as a generic error.
|
|
132
|
-
// The distill_invoked event was already emitted inside akmDistill before the throw.
|
|
133
108
|
if (err instanceof UsageError) {
|
|
109
|
+
recordLoopAttempt(planned, env, "distill", "failed", err.message);
|
|
134
110
|
tally.actions.push({
|
|
135
111
|
ref: planned.ref,
|
|
136
112
|
mode: "distill",
|
|
@@ -138,448 +114,177 @@ export async function processImproveLoopRef(planned, env) {
|
|
|
138
114
|
});
|
|
139
115
|
}
|
|
140
116
|
else {
|
|
141
|
-
tally.actions.push({
|
|
142
|
-
ref: planned.ref,
|
|
143
|
-
mode: "error",
|
|
144
|
-
result: { ok: false, error: errMessage(err) },
|
|
145
|
-
});
|
|
117
|
+
tally.actions.push({ ref: planned.ref, mode: "error", result: { ok: false, error: errMessage(err) } });
|
|
146
118
|
}
|
|
147
119
|
}
|
|
148
120
|
return tally;
|
|
149
121
|
}
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
*/
|
|
156
|
-
async function runLoopReflectPass(planned, isDistillOnly, env, tally) {
|
|
157
|
-
const { options, primaryStashDir, reflectFn, eventsCtx, improveProfile, resolvedPlan, budgetSignal } = env;
|
|
158
|
-
// B6: derived memories are machine-generated; skip reflect to avoid noisy proposals.
|
|
159
|
-
// shouldDistillMemoryRef already returns false for .derived refs, so the distill
|
|
160
|
-
// path is also a no-op for them — we just avoid unnecessary agent spawns.
|
|
161
|
-
// D2: distillOnlyRefs also skip the reflect call (reflect-cooled, distill path only).
|
|
162
|
-
if (!isDistillOnly && !planned.ref.endsWith(".derived")) {
|
|
163
|
-
// Type guard: skip reflect for unsupported types (script, env, task, etc.)
|
|
164
|
-
// and raw wiki directories, driven by the active improve profile.
|
|
165
|
-
const reflectSkip = shouldSkipRef(planned.ref, "reflect", improveProfile);
|
|
166
|
-
if (reflectSkip.skip) {
|
|
167
|
-
tally.actions.push({
|
|
168
|
-
ref: planned.ref,
|
|
169
|
-
mode: "reflect-skipped",
|
|
170
|
-
result: { ok: true, reason: reflectSkip.reason },
|
|
171
|
-
});
|
|
172
|
-
}
|
|
173
|
-
else {
|
|
174
|
-
// O-5 / #378: only inject reflect-originator errors into the reflect call.
|
|
175
|
-
// Cross-task errors (e.g. schema-repair) must NOT contaminate reflect prompts.
|
|
176
|
-
const reflectErrors = env.recentErrors.reflect ?? [];
|
|
177
|
-
if (reflectErrors.length > 0)
|
|
178
|
-
tally.reflectsWithErrorContext++;
|
|
179
|
-
// O-1 (#364): pass remaining budget as timeoutMs so the agent spawn is
|
|
180
|
-
// bounded by the wall-clock deadline rather than the default per-profile timeout.
|
|
181
|
-
const reflectBudgetMs = env.remainingBudgetMs();
|
|
182
|
-
// Re-enter canonical named-engine lowering with the config snapshot and
|
|
183
|
-
// process profile frozen into the invocation plan. The loop never injects
|
|
184
|
-
// a RunnerSpec seam or observes later caller-config mutations.
|
|
185
|
-
const reflectCallArgs = {
|
|
186
|
-
ref: planned.ref,
|
|
187
|
-
// Carry the resolved item_ref so reflect uses the same durable key for
|
|
188
|
-
// events and state reads.
|
|
189
|
-
...(planned.itemRef ? { itemRef: planned.itemRef } : {}),
|
|
190
|
-
task: options.task,
|
|
191
|
-
// Active strategy supplies non-engine process tuning.
|
|
192
|
-
...(improveProfile ? { improveProfile } : {}),
|
|
193
|
-
config: resolvedPlan.config,
|
|
194
|
-
...(primaryStashDir ? { stashDir: primaryStashDir } : {}),
|
|
195
|
-
...(options.sourceName && primaryStashDir
|
|
196
|
-
? { target: { source: options.sourceName, root: primaryStashDir } }
|
|
197
|
-
: {}),
|
|
198
|
-
...(reflectErrors.length > 0 ? { avoidPatterns: [...reflectErrors] } : {}),
|
|
199
|
-
eventSource: "improve",
|
|
200
|
-
// #639 — resolve the low-value filter from the ACTIVE improve profile
|
|
201
|
-
// (default off when unset), so the running strategy decides.
|
|
202
|
-
lowValueFilter: improveProfile.processes?.reflect?.lowValueFilter?.enabled === true,
|
|
203
|
-
...(reflectBudgetMs > 0 ? { timeoutMs: reflectBudgetMs } : {}),
|
|
204
|
-
signal: budgetSignal,
|
|
205
|
-
// R25: reflect's event emits reuse the run's long-lived state.db handle.
|
|
206
|
-
eventsCtx: env.eventsCtx,
|
|
207
|
-
// Attribution: carry the eligibility lane so reflect stamps it on
|
|
208
|
-
// the reflect_invoked event and the persisted proposal.
|
|
209
|
-
...(planned.eligibilitySource ? { eligibilitySource: planned.eligibilitySource } : {}),
|
|
210
|
-
};
|
|
211
|
-
const reflectResult = await withLlmStage("reflect", () => reflectFn(reflectCallArgs), {
|
|
212
|
-
engine: resolvedPlan.processes.reflect.runner?.engine,
|
|
213
|
-
process: "reflect",
|
|
214
|
-
});
|
|
215
|
-
const isCooldown = !reflectResult.ok && reflectResult.reason === "cooldown";
|
|
216
|
-
// Content-policy guard hits (reflect size-rail rejections) are NOT
|
|
217
|
-
// LLM faults — the agent responded fine, the downstream guard
|
|
218
|
-
// blocked the output. Route them to a distinct `reflect-guard-rejected`
|
|
219
|
-
// mode so health metrics can split deterministic guard hits out of
|
|
220
|
-
// true LLM failures. See
|
|
221
|
-
// `/tmp/akm-health-investigations/metrics-taxonomy-review.md` §1a.
|
|
222
|
-
const isGuardReject = !reflectResult.ok && reflectResult.reason === "content_policy_reject";
|
|
223
|
-
// Type-guard rejection (reflect refused a script/env/task ref) is
|
|
224
|
-
// also NOT an LLM failure — the LLM is never invoked. Route to the
|
|
225
|
-
// existing `reflect-skipped` bucket so it does not inflate the
|
|
226
|
-
// failure-rate numerator. ~9% of `reflect-failed` events in the
|
|
227
|
-
// user's stack were this case; see review §1a row "Reflect refused
|
|
228
|
-
// asset type".
|
|
229
|
-
const isTypeRefused = !reflectResult.ok && reflectResult.reason === "unsupported_type";
|
|
230
|
-
// Noise-gate suppression (#580): the candidate edit was an empty
|
|
231
|
-
// diff or a cosmetic-only reformat of the current asset. Like
|
|
232
|
-
// `unsupported_type`, this is a deterministic skip — not an LLM
|
|
233
|
-
// fault — so it routes to the `reflect-skipped` bucket and stays
|
|
234
|
-
// out of recentErrors/avoidPatterns.
|
|
235
|
-
const isNoChange = !reflectResult.ok && reflectResult.reason === "no_change";
|
|
236
|
-
tally.actions.push({
|
|
237
|
-
ref: planned.ref,
|
|
238
|
-
mode: reflectResult.ok
|
|
239
|
-
? "reflect"
|
|
240
|
-
: isCooldown
|
|
241
|
-
? "reflect-cooldown"
|
|
242
|
-
: isGuardReject
|
|
243
|
-
? "reflect-guard-rejected"
|
|
244
|
-
: isTypeRefused || isNoChange
|
|
245
|
-
? "reflect-skipped"
|
|
246
|
-
: "reflect-failed",
|
|
247
|
-
result: reflectResult,
|
|
248
|
-
});
|
|
249
|
-
// Cooldown skips, guard rejects, type-refused skips, and noise-gate
|
|
250
|
-
// skips are not failures — do not pollute recentErrors with them
|
|
251
|
-
// (those get injected as `avoidPatterns` into the next reflect
|
|
252
|
-
// prompt). Guard rejects ARE worth showing the LLM as a learn-signal
|
|
253
|
-
// so the next iteration sees "your last expansion was too large";
|
|
254
|
-
// type-refused and no-change are deterministic and add no learning
|
|
255
|
-
// signal.
|
|
256
|
-
if (!reflectResult.ok && !isCooldown && !isTypeRefused && !isNoChange) {
|
|
257
|
-
const errMsg = reflectResult.error ?? reflectResult.reason ?? "unknown reflect error";
|
|
258
|
-
tally.recentErrorPushes.push({ originator: "reflect", message: errMsg });
|
|
259
|
-
}
|
|
260
|
-
// improve_reflect_outcome — per-asset metric for tuning the reflect path.
|
|
261
|
-
appendEvent({
|
|
262
|
-
eventType: "improve_reflect_outcome",
|
|
263
|
-
ref: planned.ref,
|
|
264
|
-
metadata: {
|
|
265
|
-
ok: reflectResult.ok,
|
|
266
|
-
durationMs: reflectResult.ok ? reflectResult.durationMs : undefined,
|
|
267
|
-
engine: reflectResult.engine,
|
|
268
|
-
reason: reflectResult.ok ? undefined : reflectResult.reason,
|
|
269
|
-
},
|
|
270
|
-
}, eventsCtx);
|
|
271
|
-
// Plasticity counter (plan §WS-1 step 8): record no-ops so the
|
|
272
|
-
// WS-1 selection comparator (effectiveScore, ~line 3073) can dampen
|
|
273
|
-
// repeatedly-silent assets during consolidation-selection.
|
|
274
|
-
// A no_change reflect means the LLM was invoked but found nothing to
|
|
275
|
-
// improve — the asset is stable. Track it. A successful reflect means
|
|
276
|
-
// the asset changed; reset the counter so the dampener lifts.
|
|
277
|
-
// Use the same item_ref-or-conceptId salience key as preparation/distill.
|
|
278
|
-
const plasticityKey = planned.itemRef ?? durableImproveRef(planned.ref);
|
|
279
|
-
if (isNoChange && eventsCtx?.db) {
|
|
280
|
-
try {
|
|
281
|
-
recordNoOp(eventsCtx.db, plasticityKey);
|
|
282
|
-
}
|
|
283
|
-
catch {
|
|
284
|
-
// best-effort: plasticity counter failure never blocks the run
|
|
285
|
-
}
|
|
286
|
-
}
|
|
287
|
-
else if (reflectResult.ok && eventsCtx?.db) {
|
|
288
|
-
try {
|
|
289
|
-
resetConsecutiveNoOps(eventsCtx.db, plasticityKey);
|
|
290
|
-
}
|
|
291
|
-
catch {
|
|
292
|
-
// best-effort
|
|
293
|
-
}
|
|
294
|
-
}
|
|
295
|
-
} // end else (reflect type/profile check)
|
|
296
|
-
}
|
|
297
|
-
else if (!isDistillOnly && planned.ref.endsWith(".derived")) {
|
|
298
|
-
// B6: .derived refs skip reflect; record synthetic skip action.
|
|
299
|
-
tally.actions.push({
|
|
300
|
-
ref: planned.ref,
|
|
301
|
-
mode: "distill-skipped",
|
|
302
|
-
result: { ok: true, reason: "derived-memory-reflect-skipped" },
|
|
303
|
-
});
|
|
304
|
-
appendEvent({
|
|
305
|
-
eventType: "improve_skipped",
|
|
306
|
-
ref: planned.ref,
|
|
307
|
-
metadata: { reason: "derived_memory_reflect_skipped" },
|
|
308
|
-
}, eventsCtx);
|
|
309
|
-
}
|
|
310
|
-
}
|
|
311
|
-
/**
|
|
312
|
-
* Distill half of one loop iteration: the profile / requirePlannedRefs /
|
|
313
|
-
* candidate-type / weak-signal / cooldown gates, then the pending-proposal and
|
|
314
|
-
* reject-grace dedup checks, then {@link invokeDistillAndRecord}. Each gate
|
|
315
|
-
* that was a `continue` in the old inline loop body is an early `return` here.
|
|
316
|
-
*/
|
|
317
|
-
async function runLoopDistillPass(planned, parsedPlannedRef, isDistillOnly, env, tally) {
|
|
318
|
-
const { options, primaryStashDir, eventsCtx, improveProfile } = env;
|
|
319
|
-
const hasRecentFeedbackSignal = env.signalBearingSet.has(planned.ref);
|
|
320
|
-
const explicitRefScope = env.scope.mode === "ref";
|
|
321
|
-
// Profile gate: apply the full type-filter / raw-wiki / disabled rules to
|
|
322
|
-
// distill so callers who configure `profile.processes.distill.allowedTypes`
|
|
323
|
-
// or land on raw-wiki refs get a recorded skip action instead of silently
|
|
324
|
-
// proceeding.
|
|
325
|
-
const distillSkip = shouldSkipRef(planned.ref, "distill", improveProfile);
|
|
326
|
-
if (distillSkip.skip) {
|
|
327
|
-
tally.actions.push({
|
|
328
|
-
ref: planned.ref,
|
|
329
|
-
mode: "distill-skipped",
|
|
330
|
-
result: { ok: true, reason: distillSkip.reason },
|
|
331
|
-
});
|
|
122
|
+
async function runLoopReflectPass(planned, env, tally) {
|
|
123
|
+
const { options, primaryStashDir, improveProfile, resolvedPlan } = env;
|
|
124
|
+
// Derived memories are machine-generated: never reflected.
|
|
125
|
+
if (planned.ref.endsWith(".derived")) {
|
|
126
|
+
recordSkip(tally, planned.ref, "derived-memory-reflect-skipped", { env, reason: "derived_memory_reflect_skipped" });
|
|
332
127
|
return;
|
|
333
128
|
}
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
tally.actions.push({
|
|
338
|
-
ref: planned.ref,
|
|
339
|
-
mode: "distill-skipped",
|
|
340
|
-
result: { ok: true, reason: "require_planned_refs" },
|
|
341
|
-
});
|
|
129
|
+
const reflectSkip = shouldSkipRef(planned.ref, "reflect", improveProfile);
|
|
130
|
+
if (reflectSkip.skip) {
|
|
131
|
+
tally.actions.push({ ref: planned.ref, mode: "reflect-skipped", result: { ok: true, reason: reflectSkip.reason } });
|
|
342
132
|
return;
|
|
343
133
|
}
|
|
344
|
-
//
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
const
|
|
349
|
-
|
|
350
|
-
// refs; non-candidate refs in the set are blocked here.
|
|
351
|
-
// O-2 (#365): bypass the distill cooldown when the user explicitly targeted
|
|
352
|
-
// this ref via --scope — their intent overrides unattended-run policies.
|
|
353
|
-
if (shouldAttemptDistill &&
|
|
354
|
-
!skipMemoryDistillForWeakSignal &&
|
|
355
|
-
(!env.distillCooledRefs.has(planned.ref) || explicitRefScope)) {
|
|
356
|
-
// TODO(refactor): single call site needs both lesson+knowledge refs for proposal dedup. If a third target ref type is added, extract deriveAllTargetRefs(inputRef): string[].
|
|
357
|
-
const lessonRef = deriveLessonRef(planned.ref);
|
|
358
|
-
const knowledgeRef = deriveKnowledgeRef(planned.ref);
|
|
359
|
-
const dedupeStashDir = primaryStashDir ?? options.stashDir;
|
|
360
|
-
if (dedupeStashDir) {
|
|
361
|
-
// B2: check both lesson ref and knowledge ref since auto-promoted memories
|
|
362
|
-
// create knowledge: proposals, not lesson: proposals.
|
|
363
|
-
const hasExistingPending = env.pendingProposalRefSet.has(lessonRef) || env.pendingProposalRefSet.has(knowledgeRef);
|
|
364
|
-
if (hasExistingPending) {
|
|
365
|
-
tally.actions.push({
|
|
366
|
-
ref: planned.ref,
|
|
367
|
-
mode: "distill-skipped",
|
|
368
|
-
result: { ok: true, reason: "pending proposal exists" },
|
|
369
|
-
});
|
|
370
|
-
appendEvent({
|
|
371
|
-
eventType: "improve_skipped",
|
|
372
|
-
ref: planned.ref,
|
|
373
|
-
metadata: { reason: "pending_proposal_exists" },
|
|
374
|
-
}, eventsCtx);
|
|
375
|
-
return;
|
|
376
|
-
}
|
|
377
|
-
// D-2 (#370): reject-aware cooldown for distill. When the reviewer
|
|
378
|
-
// recently rejected a distilled lesson or knowledge proposal for this
|
|
379
|
-
// asset, skip re-distillation for a 1-day grace window. Prevents the
|
|
380
|
-
// same rejected proposal from being regenerated immediately. The
|
|
381
|
-
// window is fixed (the 0.8.0 redesign moved per-ref cooldowns to
|
|
382
|
-
// signal-delta gates and dropped --distill-cooldown-days; a short
|
|
383
|
-
// reject grace is preserved here so a fresh rejection isn't
|
|
384
|
-
// overridden by the same run).
|
|
385
|
-
// References: ExpeL arXiv:2308.10144, STaR arXiv:2203.14465.
|
|
386
|
-
const DISTILL_REJECT_COOLDOWN_MS = daysToMs(1);
|
|
387
|
-
const recentlyRejectedLesson = !explicitRefScope && // O-2: bypass when --scope <ref> is explicit
|
|
388
|
-
(env.rejectedProposalsByRef.has(lessonRef) || env.rejectedProposalsByRef.has(knowledgeRef));
|
|
389
|
-
if (recentlyRejectedLesson) {
|
|
390
|
-
const rejectedEntry = env.rejectedProposalsByRef.get(lessonRef) ?? env.rejectedProposalsByRef.get(knowledgeRef);
|
|
391
|
-
const rejectedAgeMs = rejectedEntry ? Date.now() - new Date(rejectedEntry.ts).getTime() : 0;
|
|
392
|
-
if (rejectedAgeMs < DISTILL_REJECT_COOLDOWN_MS) {
|
|
393
|
-
tally.actions.push({
|
|
394
|
-
ref: planned.ref,
|
|
395
|
-
mode: "distill-skipped",
|
|
396
|
-
result: { ok: true, reason: "distill reject grace window" },
|
|
397
|
-
});
|
|
398
|
-
appendEvent({
|
|
399
|
-
eventType: "improve_skipped",
|
|
400
|
-
ref: planned.ref,
|
|
401
|
-
metadata: {
|
|
402
|
-
reason: "distill_reject_grace_window",
|
|
403
|
-
},
|
|
404
|
-
}, eventsCtx);
|
|
405
|
-
return;
|
|
406
|
-
}
|
|
407
|
-
}
|
|
408
|
-
}
|
|
409
|
-
await invokeDistillAndRecord(planned, parsedPlannedRef, env, tally);
|
|
410
|
-
}
|
|
411
|
-
else if (skipMemoryDistillForWeakSignal) {
|
|
412
|
-
tally.actions.push({
|
|
413
|
-
ref: planned.ref,
|
|
414
|
-
mode: "distill-skipped",
|
|
415
|
-
result: { ok: true, reason: "memory requires recent feedback signal" },
|
|
416
|
-
});
|
|
417
|
-
appendEvent({
|
|
418
|
-
eventType: "improve_skipped",
|
|
419
|
-
ref: planned.ref,
|
|
420
|
-
metadata: { reason: "memory_distill_requires_feedback" },
|
|
421
|
-
}, eventsCtx);
|
|
422
|
-
}
|
|
423
|
-
}
|
|
424
|
-
/**
|
|
425
|
-
* The distill invocation for one ref that passed every gate: the `distillFn`
|
|
426
|
-
* call, memory-inference queueing, plasticity counters, and the
|
|
427
|
-
* quality-rejected / proposal-rejected eval-case writes.
|
|
428
|
-
*/
|
|
429
|
-
async function invokeDistillAndRecord(planned, parsedPlannedRef, env, tally) {
|
|
430
|
-
const { options, primaryStashDir, distillFn, eventsCtx, improveProfile, resolvedPlan, budgetSignal } = env;
|
|
431
|
-
const distillResult = await withLlmStage("distill", () => distillFn({
|
|
134
|
+
// Only reflect's own recent errors reach its prompt.
|
|
135
|
+
const reflectErrors = env.recentErrors.reflect ?? [];
|
|
136
|
+
if (reflectErrors.length > 0)
|
|
137
|
+
tally.reflectsWithErrorContext++;
|
|
138
|
+
const budgetMs = env.remainingBudgetMs();
|
|
139
|
+
const reflectArgs = {
|
|
432
140
|
ref: planned.ref,
|
|
433
|
-
// Carry the resolved item_ref so distill matches preparation's state key.
|
|
434
141
|
...(planned.itemRef ? { itemRef: planned.itemRef } : {}),
|
|
435
|
-
|
|
436
|
-
...(primaryStashDir ? { stashDir: primaryStashDir } : {}),
|
|
437
|
-
// Active profile so distill's per-process reads honor `--profile`.
|
|
142
|
+
task: options.task,
|
|
438
143
|
...(improveProfile ? { improveProfile } : {}),
|
|
439
|
-
config:
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
144
|
+
config: resolvedPlan.config,
|
|
145
|
+
...(primaryStashDir ? { stashDir: primaryStashDir } : {}),
|
|
146
|
+
...(options.sourceName && primaryStashDir ? { target: { source: options.sourceName, root: primaryStashDir } } : {}),
|
|
147
|
+
...(reflectErrors.length > 0 ? { avoidPatterns: [...reflectErrors] } : {}),
|
|
148
|
+
eventSource: "improve",
|
|
149
|
+
lowValueFilter: improveProfile.processes?.reflect?.lowValueFilter?.enabled === true,
|
|
150
|
+
...(budgetMs > 0 ? { timeoutMs: budgetMs } : {}),
|
|
151
|
+
signal: env.budgetSignal,
|
|
152
|
+
eventsCtx: env.eventsCtx,
|
|
446
153
|
...(planned.eligibilitySource ? { eligibilitySource: planned.eligibilitySource } : {}),
|
|
447
|
-
}
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
154
|
+
};
|
|
155
|
+
const result = await attributeStage(resolvedPlan, "reflect", () => env.reflectFn(reflectArgs));
|
|
156
|
+
const reason = result.ok ? undefined : result.reason;
|
|
157
|
+
// A refused type or an unchanged asset is a deterministic skip, not an LLM
|
|
158
|
+
// fault; a guard rejection (size rail) gets its own bucket for health.
|
|
159
|
+
const skipped = reason === "unsupported_type" || reason === "no_change";
|
|
160
|
+
tally.actions.push({
|
|
161
|
+
ref: planned.ref,
|
|
162
|
+
mode: result.ok
|
|
163
|
+
? "reflect"
|
|
164
|
+
: reason === "content_policy_reject"
|
|
165
|
+
? "reflect-guard-rejected"
|
|
166
|
+
: skipped
|
|
167
|
+
? "reflect-skipped"
|
|
168
|
+
: "reflect-failed",
|
|
169
|
+
result,
|
|
170
|
+
});
|
|
171
|
+
// A quality rejection recorded itself, and the judge's text is no lesson for
|
|
172
|
+
// the next prompt; skips revisit on the `unchanged` cadence.
|
|
173
|
+
if (!result.ok && reason !== "quality_rejected") {
|
|
174
|
+
recordLoopAttempt(planned, env, "reflect", skipped ? "unchanged" : "failed", reason);
|
|
175
|
+
if (!skipped) {
|
|
176
|
+
tally.recentErrorPushes.push({
|
|
177
|
+
originator: "reflect",
|
|
178
|
+
message: result.error ?? reason ?? "unknown reflect error",
|
|
179
|
+
});
|
|
471
180
|
}
|
|
472
181
|
}
|
|
473
|
-
if (distillResult.outcome === "quality_rejected" && primaryStashDir) {
|
|
474
|
-
const slug = refSlug(planned.ref);
|
|
475
|
-
writeEvalCase(primaryStashDir, {
|
|
476
|
-
ref: planned.ref,
|
|
477
|
-
failureReason: distillResult.reason ?? "quality gate rejected",
|
|
478
|
-
assetType: parseRefInput(planned.ref).type ?? "unknown",
|
|
479
|
-
rejectedAt: Date.now(),
|
|
480
|
-
source: "distill_quality_rejected",
|
|
481
|
-
slug: `${slug}-${Date.now()}`,
|
|
482
|
-
});
|
|
483
|
-
}
|
|
484
|
-
// D6: use pre-loaded map instead of per-iteration DB query
|
|
485
|
-
const rejectedProposalEvent = env.rejectedProposalsByRef.get(planned.ref);
|
|
486
|
-
if (rejectedProposalEvent && primaryStashDir) {
|
|
487
|
-
const slug = refSlug(planned.ref);
|
|
488
|
-
writeEvalCase(primaryStashDir, {
|
|
489
|
-
ref: planned.ref,
|
|
490
|
-
failureReason: rejectedProposalEvent.metadata?.reason ?? "proposal rejected",
|
|
491
|
-
assetType: parseRefInput(planned.ref).type ?? "unknown",
|
|
492
|
-
rejectedAt: new Date(rejectedProposalEvent.ts).getTime(),
|
|
493
|
-
source: "proposal_rejected",
|
|
494
|
-
slug: `${slug}-rejected`,
|
|
495
|
-
});
|
|
496
|
-
}
|
|
497
|
-
}
|
|
498
|
-
/**
|
|
499
|
-
* Wall-clock budget exhausted mid-loop (O-1 / #364): emit the improve_skipped
|
|
500
|
-
* events for the current and remaining refs (B11) and return the terminal
|
|
501
|
-
* error action for the orchestrator to record before breaking out of the loop.
|
|
502
|
-
*/
|
|
503
|
-
function recordBudgetExhausted(args) {
|
|
504
|
-
const { planned, loopRefs, completedCount, startMs, eventsCtx } = args;
|
|
505
|
-
const remaining = loopRefs.length - completedCount;
|
|
506
|
-
info(`[improve] budget exhausted after ${Math.round((Date.now() - startMs) / 60000)}min — ${remaining} assets skipped`);
|
|
507
182
|
appendEvent({
|
|
508
|
-
eventType: "
|
|
183
|
+
eventType: "improve_reflect_outcome",
|
|
509
184
|
ref: planned.ref,
|
|
510
185
|
metadata: {
|
|
511
|
-
|
|
512
|
-
|
|
186
|
+
ok: result.ok,
|
|
187
|
+
durationMs: result.ok ? result.durationMs : undefined,
|
|
188
|
+
engine: result.engine,
|
|
189
|
+
reason,
|
|
513
190
|
},
|
|
514
|
-
}, eventsCtx);
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
191
|
+
}, env.eventsCtx);
|
|
192
|
+
recordPlasticity(env, planned, reason === "no_change" ? "noop" : result.ok ? "changed" : undefined);
|
|
193
|
+
}
|
|
194
|
+
async function runLoopDistillPass(planned, refType, isDistillOnly, env, tally) {
|
|
195
|
+
const { options, primaryStashDir, improveProfile, resolvedPlan } = env;
|
|
196
|
+
const distillSkip = shouldSkipRef(planned.ref, "distill", improveProfile);
|
|
197
|
+
if (distillSkip.skip)
|
|
198
|
+
return recordSkip(tally, planned.ref, distillSkip.reason);
|
|
199
|
+
if (env.skipDistillDueToRequirePlannedRefs && isDistillOnly) {
|
|
200
|
+
return recordSkip(tally, planned.ref, "require_planned_refs");
|
|
522
201
|
}
|
|
523
|
-
|
|
202
|
+
const explicitRefScope = env.scope.mode === "ref";
|
|
203
|
+
const weakMemorySignal = !isDistillOnly && refType === "memory" && !env.signalBearingSet.has(planned.ref) && !explicitRefScope;
|
|
204
|
+
if (weakMemorySignal) {
|
|
205
|
+
return recordSkip(tally, planned.ref, "memory requires recent feedback signal", {
|
|
206
|
+
env,
|
|
207
|
+
reason: "memory_distill_requires_feedback",
|
|
208
|
+
});
|
|
209
|
+
}
|
|
210
|
+
// The ledger holds cooled refs; an explicit `--scope` ref overrides it.
|
|
211
|
+
if (!isDistillCandidateRef(planned.ref, options.stashDir))
|
|
212
|
+
return;
|
|
213
|
+
if (env.distillCooledRefs.has(planned.ref) && !explicitRefScope)
|
|
214
|
+
return;
|
|
215
|
+
const result = await attributeStage(resolvedPlan, "distill", () => env.distillFn({
|
|
524
216
|
ref: planned.ref,
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
217
|
+
...(planned.itemRef ? { itemRef: planned.itemRef } : {}),
|
|
218
|
+
...(refType === "memory" ? { proposalKind: "auto" } : {}),
|
|
219
|
+
...(primaryStashDir ? { stashDir: primaryStashDir } : {}),
|
|
220
|
+
...(improveProfile ? { improveProfile } : {}),
|
|
221
|
+
config: options.config,
|
|
222
|
+
llmRunner: resolvedPlan.processes.distill.runner,
|
|
223
|
+
signal: env.budgetSignal,
|
|
224
|
+
eventsCtx: env.eventsCtx,
|
|
225
|
+
...(planned.eligibilitySource ? { eligibilitySource: planned.eligibilitySource } : {}),
|
|
226
|
+
}));
|
|
227
|
+
tally.actions.push({ ref: planned.ref, mode: "distill", result });
|
|
228
|
+
// `queued` and the quality outcomes recorded themselves; a transport failure
|
|
229
|
+
// or a disabled process is not an attempt, so the ref stays eligible.
|
|
230
|
+
if (result.outcome === "skipped") {
|
|
231
|
+
recordLoopAttempt(planned, env, "distill", "unchanged", result.skipReason ?? result.message);
|
|
232
|
+
}
|
|
233
|
+
if (refType === "memory" && !(result.outcome === "queued" && result.proposalKind === "knowledge")) {
|
|
234
|
+
tally.memoryRefsForInference.push(planned.ref);
|
|
235
|
+
}
|
|
236
|
+
recordPlasticity(env, planned, result.outcome === "quality_rejected" || result.outcome === "skipped"
|
|
237
|
+
? "noop"
|
|
238
|
+
: result.outcome === "queued"
|
|
239
|
+
? "changed"
|
|
240
|
+
: undefined);
|
|
528
241
|
}
|
|
529
242
|
export async function runImproveLoopStage(args) {
|
|
530
|
-
const {
|
|
531
|
-
const eventsCtx = ctx.eventsCtx;
|
|
243
|
+
const { loopRefs, actions, recentErrors, startMs, budgetMs, eventsCtx } = args;
|
|
532
244
|
const env = prepareImproveLoopEnv(args);
|
|
533
|
-
let completedCount = 0;
|
|
534
245
|
let reflectsWithErrorContext = 0;
|
|
535
246
|
const memoryRefsForInference = new Set();
|
|
536
|
-
for (const planned of loopRefs) {
|
|
247
|
+
for (const [index, planned] of loopRefs.entries()) {
|
|
537
248
|
if (Date.now() - startMs >= budgetMs) {
|
|
538
|
-
|
|
249
|
+
const remaining = loopRefs.length - index;
|
|
250
|
+
info(`[improve] budget exhausted after ${Math.round((Date.now() - startMs) / 60000)}min — ${remaining} assets skipped`);
|
|
251
|
+
appendEvent({ eventType: "improve_skipped", ref: planned.ref, metadata: { reason: "budget_exhausted", remaining } }, eventsCtx);
|
|
252
|
+
for (const rest of loopRefs.slice(index + 1)) {
|
|
253
|
+
appendEvent({
|
|
254
|
+
eventType: "improve_skipped",
|
|
255
|
+
ref: rest.ref,
|
|
256
|
+
metadata: { reason: "budget_exhausted_batch", remaining: remaining - 1 },
|
|
257
|
+
}, eventsCtx);
|
|
258
|
+
}
|
|
259
|
+
actions.push({
|
|
260
|
+
ref: planned.ref,
|
|
261
|
+
mode: "error",
|
|
262
|
+
result: { ok: false, error: "timeout: improve wall-clock budget exhausted" },
|
|
263
|
+
});
|
|
539
264
|
break;
|
|
540
265
|
}
|
|
541
266
|
const tally = await processImproveLoopRef(planned, env);
|
|
542
|
-
// Fold the per-ref tally into run-level state — the passes never touch it.
|
|
543
267
|
actions.push(...tally.actions);
|
|
544
268
|
for (const push of tally.recentErrorPushes)
|
|
545
269
|
pushRecentError(recentErrors, push.originator, push.message);
|
|
546
270
|
reflectsWithErrorContext += tally.reflectsWithErrorContext;
|
|
547
271
|
for (const ref of tally.memoryRefsForInference)
|
|
548
272
|
memoryRefsForInference.add(ref);
|
|
549
|
-
|
|
550
|
-
info(`[improve] ${completedCount}/${loopRefs.length} ${planned.ref}`);
|
|
273
|
+
info(`[improve] ${index + 1}/${loopRefs.length} ${planned.ref}`);
|
|
551
274
|
}
|
|
552
275
|
return { reflectsWithErrorContext, memoryRefsForInference };
|
|
553
276
|
}
|
|
554
277
|
export async function runImprovePostLoopStage(args) {
|
|
555
|
-
const { scope,
|
|
556
|
-
const allWarnings = [...cleanupWarnings, ...(appliedCleanup?.warnings ?? [])];
|
|
278
|
+
const { scope, primaryStashDir, actionableRefs } = args;
|
|
279
|
+
const allWarnings = [...args.cleanupWarnings, ...(args.appliedCleanup?.warnings ?? [])];
|
|
557
280
|
info("[improve] post-loop maintenance starting");
|
|
558
|
-
const
|
|
559
|
-
options,
|
|
560
|
-
primaryStashDir,
|
|
561
|
-
actionableRefs,
|
|
562
|
-
memoryRefsForInference,
|
|
563
|
-
allWarnings,
|
|
564
|
-
reindexFn,
|
|
565
|
-
consolidationRan,
|
|
566
|
-
// O-1 (#364): forward the budget signal to memory inference + graph extraction.
|
|
567
|
-
budgetSignal,
|
|
568
|
-
eventsCtx,
|
|
569
|
-
improveProfile,
|
|
570
|
-
resolvedPlan,
|
|
571
|
-
});
|
|
281
|
+
const maintenance = await runImproveMaintenancePasses({ ...args, allWarnings });
|
|
572
282
|
let deadUrls;
|
|
573
283
|
let deadUrlCoverage;
|
|
574
284
|
if (scope.mode === "all" && primaryStashDir && actionableRefs.length > 0) {
|
|
575
285
|
try {
|
|
576
|
-
// Every actionable knowledge ref is scanned
|
|
577
|
-
//
|
|
578
|
-
// `deadUrlCoverage.total` counted only those, so a real bundle reported
|
|
579
|
-
// checked === total while most refs were never looked at (#892). URL
|
|
580
|
-
// extraction (a regex over already-loaded text) is cheap; it is the
|
|
581
|
-
// network requests that are expensive, and those are bounded by
|
|
582
|
-
// `checkDeadUrls`'s concurrency limit, not by trimming what gets scanned.
|
|
286
|
+
// Every actionable knowledge ref is scanned; checkDeadUrls bounds the
|
|
287
|
+
// network concurrency (#892).
|
|
583
288
|
const knowledgeEntries = actionableRefs
|
|
584
289
|
.filter((r) => {
|
|
585
290
|
try {
|
|
@@ -590,9 +295,6 @@ export async function runImprovePostLoopStage(args) {
|
|
|
590
295
|
}
|
|
591
296
|
})
|
|
592
297
|
.map((r) => {
|
|
593
|
-
// The URL scan needs the document body; filePath is pre-resolved on
|
|
594
|
-
// eligible refs at planning time (#591). Best-effort — an unreadable
|
|
595
|
-
// or unresolved file contributes no URLs, same as before.
|
|
596
298
|
let body = "";
|
|
597
299
|
if (r.filePath) {
|
|
598
300
|
try {
|
|
@@ -616,154 +318,77 @@ export async function runImprovePostLoopStage(args) {
|
|
|
616
318
|
// best-effort
|
|
617
319
|
}
|
|
618
320
|
}
|
|
619
|
-
// ── R5: collapse/churn detector ────────────────────────────────────────────
|
|
620
|
-
// One snapshot per QUALIFYING cycle: consolidate processed work. Runs AFTER
|
|
621
|
-
// the maintenance reindex so FTS sees the post-merge index. Deterministic,
|
|
622
|
-
// observe-only, fail-open (the orchestrator catches everything) — and inert
|
|
623
|
-
// on the ~9-in-10 default-profile runs that touch no merges.
|
|
624
|
-
let cycleMetrics;
|
|
625
|
-
if (!options.dryRun && consolidationRan) {
|
|
626
|
-
cycleMetrics = runCollapseDetector({
|
|
627
|
-
runId: options.runId ?? "improve-adhoc",
|
|
628
|
-
...(improveProfile ? { improveProfile } : {}),
|
|
629
|
-
pass: "consolidate",
|
|
630
|
-
mergeFloorViolations: args.consolidationMergeFloorViolations ?? 0,
|
|
631
|
-
config: options.config ?? loadConfig(),
|
|
632
|
-
...(eventsCtx ? { eventsCtx } : {}),
|
|
633
|
-
});
|
|
634
|
-
}
|
|
635
321
|
return {
|
|
636
322
|
allWarnings,
|
|
637
323
|
deadUrls,
|
|
638
324
|
...(deadUrlCoverage ? { deadUrlCoverage } : {}),
|
|
639
|
-
...(
|
|
640
|
-
...(
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
: {}),
|
|
645
|
-
memoryInferenceDurationMs: maintenanceResult.memoryInferenceDurationMs,
|
|
646
|
-
graphExtractionDurationMs: maintenanceResult.graphExtractionDurationMs,
|
|
647
|
-
orphansPurged: maintenanceResult.orphansPurged,
|
|
648
|
-
proposalsExpired: maintenanceResult.proposalsExpired,
|
|
325
|
+
...(maintenance.memoryInference ? { memoryInference: maintenance.memoryInference } : {}),
|
|
326
|
+
...(maintenance.actions && maintenance.actions.length > 0 ? { maintenanceActions: maintenance.actions } : {}),
|
|
327
|
+
memoryInferenceDurationMs: maintenance.memoryInferenceDurationMs,
|
|
328
|
+
orphansPurged: maintenance.orphansPurged,
|
|
329
|
+
proposalsExpired: maintenance.proposalsExpired,
|
|
649
330
|
};
|
|
650
331
|
}
|
|
651
|
-
|
|
652
|
-
|
|
332
|
+
/**
|
|
333
|
+
* Memory inference → index what it wrote → proposal hygiene (orphan purge,
|
|
334
|
+
* expiration) → orphan-state GC → retention purges. Warnings go to
|
|
335
|
+
* `allWarnings`.
|
|
336
|
+
*/
|
|
653
337
|
export async function runImproveMaintenancePasses(args) {
|
|
654
|
-
const { options, primaryStashDir,
|
|
655
|
-
if (!primaryStashDir)
|
|
656
|
-
return { memoryInferenceDurationMs: 0
|
|
657
|
-
if (budgetSignal?.aborted)
|
|
658
|
-
return { memoryInferenceDurationMs: 0, graphExtractionDurationMs: 0 };
|
|
338
|
+
const { options, primaryStashDir, allWarnings, budgetSignal, eventsCtx } = args;
|
|
339
|
+
if (!primaryStashDir || budgetSignal?.aborted)
|
|
340
|
+
return { memoryInferenceDurationMs: 0 };
|
|
659
341
|
const config = options.config ?? loadConfig();
|
|
660
|
-
const sources = resolveSourceEntries(options.stashDir, config);
|
|
661
|
-
const memoryInferenceFn = options.memoryInferenceFn ?? runMemoryInferencePass;
|
|
662
|
-
const graphExtractionFn = options.graphExtractionFn ?? runGraphExtractionPass;
|
|
663
|
-
const openIndexDb = () => openIndexDatabase(getDbPath(), config.embedding?.dimension ? { embeddingDim: config.embedding.dimension } : undefined);
|
|
664
|
-
const dbCell = {};
|
|
665
|
-
// #584: see the MaintenanceCtx.reindexWithIndexDbReleased doc — close before
|
|
666
|
-
// every reindex, reopen in `finally` so a failed reindex still leaves a
|
|
667
|
-
// usable handle in the cell.
|
|
668
|
-
const reindexWithIndexDbReleased = async (stashDir) => {
|
|
669
|
-
if (dbCell.current) {
|
|
670
|
-
closeDatabase(dbCell.current);
|
|
671
|
-
dbCell.current = undefined;
|
|
672
|
-
}
|
|
673
|
-
try {
|
|
674
|
-
await reindexFn({ stashDir, signal: budgetSignal });
|
|
675
|
-
}
|
|
676
|
-
finally {
|
|
677
|
-
dbCell.current = openIndexDb();
|
|
678
|
-
}
|
|
679
|
-
};
|
|
680
342
|
const ctx = {
|
|
681
343
|
config,
|
|
682
|
-
sources,
|
|
344
|
+
sources: resolveSourceEntries(options.stashDir, config),
|
|
683
345
|
primaryStashDir,
|
|
684
346
|
eventsCtx,
|
|
685
347
|
budgetSignal,
|
|
686
348
|
improveProfile: args.improveProfile,
|
|
687
349
|
resolvedPlan: args.resolvedPlan,
|
|
688
|
-
memoryInferenceFn,
|
|
689
|
-
graphExtractionFn,
|
|
690
|
-
reindexWithIndexDbReleased,
|
|
350
|
+
memoryInferenceFn: options.memoryInferenceFn ?? runMemoryInferencePass,
|
|
691
351
|
};
|
|
692
|
-
const
|
|
693
|
-
|
|
694
|
-
memoryRefsForInference,
|
|
695
|
-
consolidationRan: args.consolidationRan,
|
|
696
|
-
allWarnings,
|
|
697
|
-
openIndexDb,
|
|
698
|
-
});
|
|
699
|
-
return {
|
|
700
|
-
...(collected.memoryInference ? { memoryInference: collected.memoryInference } : {}),
|
|
701
|
-
...(collected.graphExtraction ? { graphExtraction: collected.graphExtraction } : {}),
|
|
702
|
-
...(collected.actions.length > 0 ? { actions: collected.actions } : {}),
|
|
703
|
-
memoryInferenceDurationMs: collected.memoryInferenceDurationMs,
|
|
704
|
-
graphExtractionDurationMs: collected.graphExtractionDurationMs,
|
|
705
|
-
orphansPurged: collected.orphansPurged,
|
|
706
|
-
proposalsExpired: collected.proposalsExpired,
|
|
707
|
-
};
|
|
708
|
-
}
|
|
709
|
-
/**
|
|
710
|
-
* The maintenance sequence (formerly the ~389-line anonymous
|
|
711
|
-
* `withIndexWriterLease` callback, before #872 removed the index-rebuild
|
|
712
|
-
* lease): memory inference → reindex-after-inference → graph extraction →
|
|
713
|
-
* proposal hygiene (orphan purge, expiration) → retention purges. Each pass
|
|
714
|
-
* returns its results and warnings; this orchestrator folds warnings into the
|
|
715
|
-
* caller's `allWarnings` sink at the same points the inline code pushed them.
|
|
716
|
-
*/
|
|
717
|
-
async function runMaintenancePassesUnderLease(ctx, dbCell, args) {
|
|
718
|
-
const { allWarnings } = args;
|
|
352
|
+
const openIndexDb = () => openIndexDatabase(getDbPath());
|
|
353
|
+
const dbCell = {};
|
|
719
354
|
const actions = [];
|
|
720
|
-
let reindexedAfterInference = false;
|
|
721
355
|
try {
|
|
722
|
-
dbCell.current =
|
|
356
|
+
dbCell.current = openIndexDb();
|
|
723
357
|
const inference = await runMemoryInferenceMaintenancePass(ctx, dbCell, args.memoryRefsForInference);
|
|
724
358
|
if (inference.action)
|
|
725
359
|
actions.push(inference.action);
|
|
726
360
|
allWarnings.push(...inference.warnings);
|
|
727
|
-
const
|
|
728
|
-
if (
|
|
729
|
-
|
|
361
|
+
const written = inference.memoryInference?.writtenPaths ?? [];
|
|
362
|
+
if (written.length > 0) {
|
|
363
|
+
// Index exactly the files inference wrote. indexWrittenAssets opens its
|
|
364
|
+
// own write handle, so ours closes first and reopens after.
|
|
365
|
+
info(`[improve] indexing ${written.length} file(s) written by memory inference`);
|
|
730
366
|
try {
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
367
|
+
if (dbCell.current)
|
|
368
|
+
closeDatabase(dbCell.current);
|
|
369
|
+
dbCell.current = undefined;
|
|
370
|
+
try {
|
|
371
|
+
await indexWrittenAssets(primaryStashDir, written);
|
|
372
|
+
}
|
|
373
|
+
finally {
|
|
374
|
+
dbCell.current = openIndexDb();
|
|
375
|
+
}
|
|
376
|
+
info("[improve] indexing after memory inference complete");
|
|
734
377
|
}
|
|
735
378
|
catch (err) {
|
|
736
|
-
allWarnings.push(`
|
|
379
|
+
allWarnings.push(`indexing after memory inference failed: ${errMessage(err)}`);
|
|
737
380
|
}
|
|
738
381
|
}
|
|
739
|
-
const
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
consolidationRan: args.consolidationRan,
|
|
743
|
-
reindexedAfterInference,
|
|
744
|
-
});
|
|
745
|
-
if (graph.action)
|
|
746
|
-
actions.push(graph.action);
|
|
747
|
-
allWarnings.push(...graph.warnings);
|
|
748
|
-
const orphan = runOrphanProposalPurgePass(ctx);
|
|
749
|
-
allWarnings.push(...orphan.warnings);
|
|
750
|
-
// #733: orphan-state GC — stamps/clears/(optionally) collects
|
|
751
|
-
// asset_salience/asset_outcome rows whose ref no longer resolves in
|
|
752
|
-
// index.db. Needs the SAME already-open index.db handle (dbCell.current)
|
|
753
|
-
// the passes above share.
|
|
754
|
-
const stateGc = runOrphanStateGcPass(ctx, dbCell);
|
|
755
|
-
allWarnings.push(...stateGc.warnings);
|
|
756
|
-
const expiration = runProposalExpirationPass(ctx);
|
|
757
|
-
allWarnings.push(...expiration.warnings);
|
|
382
|
+
const hygiene = runProposalHygienePass(ctx);
|
|
383
|
+
allWarnings.push(...hygiene.warnings);
|
|
384
|
+
allWarnings.push(...runOrphanStateGcPass(ctx, dbCell).warnings);
|
|
758
385
|
allWarnings.push(...runRetentionPurgePass(ctx).warnings);
|
|
759
386
|
return {
|
|
760
|
-
memoryInference,
|
|
761
|
-
|
|
762
|
-
actions,
|
|
387
|
+
...(inference.memoryInference ? { memoryInference: inference.memoryInference } : {}),
|
|
388
|
+
...(actions.length > 0 ? { actions } : {}),
|
|
763
389
|
memoryInferenceDurationMs: inference.durationMs,
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
proposalsExpired: expiration.proposalsExpired,
|
|
390
|
+
orphansPurged: hygiene.orphansPurged,
|
|
391
|
+
proposalsExpired: hygiene.proposalsExpired,
|
|
767
392
|
};
|
|
768
393
|
}
|
|
769
394
|
finally {
|
|
@@ -771,540 +396,257 @@ async function runMaintenancePassesUnderLease(ctx, dbCell, args) {
|
|
|
771
396
|
closeDatabase(dbCell.current);
|
|
772
397
|
}
|
|
773
398
|
}
|
|
774
|
-
/**
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
* already split (`inferenceProcessed: true`) and `isPendingMemory` excludes
|
|
781
|
-
* them. The genuinely-pending parents in the stash never entered the
|
|
782
|
-
* filter. Result: 0/0/0 for 25 consecutive runs.
|
|
783
|
-
*
|
|
784
|
-
* Fix: always run the pass when the feature is enabled; let the pass's
|
|
785
|
-
* own `collectPendingMemories` + `isPendingMemory` predicate find
|
|
786
|
-
* candidates from the filesystem-of-truth. The this-run set is still
|
|
787
|
-
* logged as a hint but no longer used as a filter.
|
|
788
|
-
*/
|
|
789
|
-
export async function runMemoryInferenceMaintenancePass(ctx, dbCell, memoryRefsForInference) {
|
|
790
|
-
const { config, sources, primaryStashDir, budgetSignal, improveProfile, resolvedPlan, memoryInferenceFn } = ctx;
|
|
791
|
-
const warnings = [];
|
|
792
|
-
let memoryInference;
|
|
793
|
-
let durationMs = 0;
|
|
794
|
-
let action;
|
|
795
|
-
const memoryInferenceDisabledByProfile = improveProfile?.processes?.memoryInference?.enabled === false;
|
|
796
|
-
const minPendingCount = improveProfile?.processes?.memoryInference?.minPendingCount;
|
|
797
|
-
const pendingBelowMinCount = (() => {
|
|
798
|
-
if (!primaryStashDir || minPendingCount === undefined || minPendingCount <= 0)
|
|
799
|
-
return false;
|
|
800
|
-
const pending = collectPendingMemories(primaryStashDir).length;
|
|
801
|
-
if (pending < minPendingCount) {
|
|
802
|
-
info(`[improve] memory inference skipped (${pending} pending < minPendingCount ${minPendingCount})`);
|
|
803
|
-
return true;
|
|
804
|
-
}
|
|
805
|
-
return false;
|
|
806
|
-
})();
|
|
807
|
-
if (memoryInferenceDisabledByProfile) {
|
|
808
|
-
info("[improve] memory inference skipped (disabled by improve profile)");
|
|
809
|
-
}
|
|
810
|
-
else if (pendingBelowMinCount) {
|
|
811
|
-
// skipped — message already emitted above
|
|
399
|
+
/** Time one LLM maintenance pass; a throw becomes a `<label> failed: …` warning. */
|
|
400
|
+
async function timedLlmPass(label, run) {
|
|
401
|
+
const start = Date.now();
|
|
402
|
+
try {
|
|
403
|
+
const result = await run();
|
|
404
|
+
return { result, durationMs: Date.now() - start, warnings: [] };
|
|
812
405
|
}
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
info(hintRefs > 0
|
|
816
|
-
? `[improve] memory inference starting (${hintRefs} hint refs touched this run; pass discovers all pending)`
|
|
817
|
-
: "[improve] memory inference starting (discovering pending parents)");
|
|
818
|
-
const inferenceStart = Date.now();
|
|
819
|
-
try {
|
|
820
|
-
// O-1 (#364): pass budget signal so a hung inference call is cancelled.
|
|
821
|
-
memoryInference = await withLlmStage("memory-inference", () => memoryInferenceFn({
|
|
822
|
-
config,
|
|
823
|
-
...(resolvedPlan
|
|
824
|
-
? {
|
|
825
|
-
llmRunner: resolvedPlan.processes.memoryInference.runner,
|
|
826
|
-
}
|
|
827
|
-
: {}),
|
|
828
|
-
sources,
|
|
829
|
-
signal: budgetSignal,
|
|
830
|
-
db: dbCell.current,
|
|
831
|
-
reEnrich: false,
|
|
832
|
-
onProgress: (event) => {
|
|
833
|
-
const current = event.currentRef ? ` ${event.currentRef}` : "";
|
|
834
|
-
info(`[improve] memory inference ${event.processed}/${event.total}${current} (written ${event.writtenFacts}, skipped ${event.skippedNoFacts})`);
|
|
835
|
-
},
|
|
836
|
-
}), { engine: resolvedPlan?.processes.memoryInference.runner?.engine, process: "memoryInference" });
|
|
837
|
-
durationMs = Date.now() - inferenceStart;
|
|
838
|
-
// Synthetic sentinel ref (ref-grammar decision D-R3): a colon-free
|
|
839
|
-
// `<domain>/_<marker>` label on the event row, never parsed as an asset
|
|
840
|
-
// ref. The domain is the asset stash-subdir for asset-scoped sentinels
|
|
841
|
-
// (`memories/…`) and the subsystem name for maintenance/artifact sentinels
|
|
842
|
-
// (`graph/…`, `events/…`, `proposals/…`, `health/…`, …). Readers match the
|
|
843
|
-
// event by `eventType`, never by this string.
|
|
844
|
-
action = { ref: "memories/_inference", mode: "memory-inference", result: memoryInference };
|
|
845
|
-
info(`[improve] memory inference complete (${memoryInference.writtenFacts} facts written from ${memoryInference.splitParents} parents)`);
|
|
846
|
-
}
|
|
847
|
-
catch (err) {
|
|
848
|
-
durationMs = Date.now() - inferenceStart;
|
|
849
|
-
warnings.push(`memory inference failed: ${errMessage(err)}`);
|
|
850
|
-
}
|
|
406
|
+
catch (err) {
|
|
407
|
+
return { durationMs: Date.now() - start, warnings: [`${label} failed: ${errMessage(err)}`] };
|
|
851
408
|
}
|
|
852
|
-
return { memoryInference, durationMs, action, warnings };
|
|
853
409
|
}
|
|
854
410
|
/**
|
|
855
|
-
*
|
|
856
|
-
*
|
|
857
|
-
* INVARIANT: graph extraction normally runs only on files touched by
|
|
858
|
-
* actionable refs (candidatePaths). Full-corpus scans are opt-in via
|
|
859
|
-
* profile.processes.graphExtraction.fullScan = true (used by the
|
|
860
|
-
* `graph-refresh` built-in profile and its weekly scheduled task).
|
|
861
|
-
* The empty-Set fallback is intentional when no refs were touched —
|
|
862
|
-
* the extractor's filter rejects every file and returns empty, keeping
|
|
863
|
-
* the pass invoked so the action is recorded and tests stay exercised.
|
|
411
|
+
* Memory inference over every pending parent in the stash. The pass discovers
|
|
412
|
+
* its own candidates; the refs distilled this run are only logged as a hint.
|
|
864
413
|
*/
|
|
865
|
-
export async function
|
|
866
|
-
const { config, sources, primaryStashDir,
|
|
867
|
-
const
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
let reindexedAfterInference = args.reindexedAfterInference;
|
|
872
|
-
const graphEnabled = resolvedPlan ? true : isProcessEnabled("index", "graph_extraction", config);
|
|
873
|
-
const graphExtractionDisabledByProfile = improveProfile?.processes?.graphExtraction?.enabled === false;
|
|
874
|
-
const graphExtractionFullScan = improveProfile?.processes?.graphExtraction?.fullScan === true;
|
|
875
|
-
// #624 P2: optional incremental high-signal-first cap. Unset = process all
|
|
876
|
-
// eligible (byte-identical to today; no ranking/slice).
|
|
877
|
-
const graphExtractionTopN = improveProfile?.processes?.graphExtraction?.topN;
|
|
878
|
-
const graphExtractionIncludeTypes = improveProfile?.processes?.graphExtraction?.includeTypes ?? [
|
|
879
|
-
...DEFAULT_GRAPH_EXTRACTION_INCLUDE_TYPES,
|
|
880
|
-
];
|
|
881
|
-
const graphExtractionBatchSize = improveProfile?.processes?.graphExtraction?.batchSize ?? DEFAULT_GRAPH_EXTRACTION_BATCH_SIZE;
|
|
882
|
-
// Build the set of refs actually touched this run.
|
|
883
|
-
const touchedRefs = new Set();
|
|
884
|
-
for (const r of args.actionableRefs)
|
|
885
|
-
touchedRefs.add(r.ref);
|
|
886
|
-
for (const r of args.memoryRefsForInference)
|
|
887
|
-
touchedRefs.add(r);
|
|
888
|
-
if (graphExtractionDisabledByProfile) {
|
|
889
|
-
info("[improve] graph extraction skipped (disabled by improve profile)");
|
|
414
|
+
export async function runMemoryInferenceMaintenancePass(ctx, dbCell, memoryRefsForInference) {
|
|
415
|
+
const { config, sources, primaryStashDir, resolvedPlan } = ctx;
|
|
416
|
+
const settings = ctx.improveProfile?.processes?.memoryInference;
|
|
417
|
+
if (settings?.enabled === false) {
|
|
418
|
+
info("[improve] memory inference skipped (disabled by improve profile)");
|
|
419
|
+
return { durationMs: 0, warnings: [] };
|
|
890
420
|
}
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
const
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
if (args.consolidationRan && !reindexedAfterInference) {
|
|
898
|
-
info("[improve] reindexing after consolidation (graph extraction needs current state)");
|
|
899
|
-
try {
|
|
900
|
-
await ctx.reindexWithIndexDbReleased(primaryStashDir);
|
|
901
|
-
reindexedAfterInference = true;
|
|
902
|
-
info("[improve] reindex after consolidation complete");
|
|
903
|
-
}
|
|
904
|
-
catch (err) {
|
|
905
|
-
warnings.push(`reindex after consolidation failed: ${errMessage(err)}`);
|
|
906
|
-
}
|
|
907
|
-
}
|
|
908
|
-
// #584: no close/reopen needed here — reindexWithIndexDbReleased
|
|
909
|
-
// already swapped in a fresh post-reindex handle.
|
|
910
|
-
// Resolve touched refs to absolute file paths. Skipped for fullScan
|
|
911
|
-
// (candidatePaths stays undefined → extractor processes all files).
|
|
912
|
-
let candidatePaths;
|
|
913
|
-
if (!graphExtractionFullScan) {
|
|
914
|
-
candidatePaths = new Set();
|
|
915
|
-
if (primaryStashDir && touchedRefs.size > 0) {
|
|
916
|
-
const writableBundleIds = deriveWritableBundleIds(resolveSourceEntries(primaryStashDir));
|
|
917
|
-
const resolved = await Promise.all([...touchedRefs].map((ref) => findAssetFilePath(ref, primaryStashDir, writableBundleIds).catch(() => null)));
|
|
918
|
-
for (const p of resolved) {
|
|
919
|
-
if (typeof p === "string" && p.length > 0)
|
|
920
|
-
candidatePaths.add(p);
|
|
921
|
-
}
|
|
922
|
-
}
|
|
923
|
-
}
|
|
924
|
-
const progressHandler = (event) => {
|
|
925
|
-
const current = event.currentPath ? ` ${path.basename(event.currentPath)}` : "";
|
|
926
|
-
info(`[improve] graph extraction ${event.processed}/${event.total}${current} (extracted ${event.extracted}, entities ${event.totalEntities}, relations ${event.totalRelations})`);
|
|
927
|
-
};
|
|
928
|
-
// O-1 (#364): pass budget signal so a hung graph extraction call is cancelled.
|
|
929
|
-
graphExtraction = await withLlmStage("graph-extraction", () => graphExtractionFn({
|
|
930
|
-
config,
|
|
931
|
-
...(resolvedPlan
|
|
932
|
-
? {
|
|
933
|
-
llmRunner: resolvedPlan.processes.graphExtraction.runner,
|
|
934
|
-
}
|
|
935
|
-
: {}),
|
|
936
|
-
sources,
|
|
937
|
-
signal: budgetSignal,
|
|
938
|
-
db: dbCell.current,
|
|
939
|
-
reEnrich: false,
|
|
940
|
-
onProgress: progressHandler,
|
|
941
|
-
options: {
|
|
942
|
-
candidatePaths,
|
|
943
|
-
includeTypes: graphExtractionIncludeTypes,
|
|
944
|
-
batchSize: graphExtractionBatchSize,
|
|
945
|
-
...(graphExtractionTopN != null ? { topN: graphExtractionTopN } : {}),
|
|
946
|
-
},
|
|
947
|
-
}), { engine: resolvedPlan?.processes.graphExtraction.runner?.engine, process: "graphExtraction" });
|
|
948
|
-
durationMs = Date.now() - extractionStart;
|
|
949
|
-
// Synthetic sentinel ref (D-R3): `graph` has no asset stash-subdir, so the
|
|
950
|
-
// colon-free `graph/_artifact` names the subsystem, per the sentinel
|
|
951
|
-
// convention documented at the memory-inference writer above.
|
|
952
|
-
action = { ref: "graph/_artifact", mode: "graph-extraction", result: graphExtraction };
|
|
953
|
-
info(`[improve] graph extraction complete (${graphExtraction.quality.extractedFiles} files, ${graphExtraction.quality.entityCount} entities, ${graphExtraction.quality.relationCount} relations)`);
|
|
954
|
-
}
|
|
955
|
-
catch (err) {
|
|
956
|
-
durationMs = Date.now() - extractionStart;
|
|
957
|
-
warnings.push(`graph extraction failed: ${errMessage(err)}`);
|
|
421
|
+
const minPendingCount = settings?.minPendingCount;
|
|
422
|
+
if (primaryStashDir && minPendingCount !== undefined && minPendingCount > 0) {
|
|
423
|
+
const pending = collectPendingMemories(primaryStashDir).length;
|
|
424
|
+
if (pending < minPendingCount) {
|
|
425
|
+
info(`[improve] memory inference skipped (${pending} pending < minPendingCount ${minPendingCount})`);
|
|
426
|
+
return { durationMs: 0, warnings: [] };
|
|
958
427
|
}
|
|
959
428
|
}
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
429
|
+
const hintRefs = memoryRefsForInference.size;
|
|
430
|
+
info(hintRefs > 0
|
|
431
|
+
? `[improve] memory inference starting (${hintRefs} hint refs touched this run; pass discovers all pending)`
|
|
432
|
+
: "[improve] memory inference starting (discovering pending parents)");
|
|
433
|
+
const pass = await timedLlmPass("memory inference", () => attributeStage(resolvedPlan, "memoryInference", () => ctx.memoryInferenceFn({
|
|
434
|
+
config,
|
|
435
|
+
...(resolvedPlan ? { llmRunner: resolvedPlan.processes.memoryInference.runner } : {}),
|
|
436
|
+
sources,
|
|
437
|
+
signal: ctx.budgetSignal,
|
|
438
|
+
db: dbCell.current,
|
|
439
|
+
reEnrich: false,
|
|
440
|
+
onProgress: (event) => {
|
|
441
|
+
const current = event.currentRef ? ` ${event.currentRef}` : "";
|
|
442
|
+
info(`[improve] memory inference ${event.processed}/${event.total}${current} (written ${event.writtenFacts}, skipped ${event.skippedNoFacts})`);
|
|
443
|
+
},
|
|
444
|
+
})));
|
|
445
|
+
const memoryInference = pass.result;
|
|
446
|
+
if (!memoryInference)
|
|
447
|
+
return { durationMs: pass.durationMs, warnings: pass.warnings };
|
|
448
|
+
info(`[improve] memory inference complete (${memoryInference.writtenFacts} facts written from ${memoryInference.splitParents} parents)`);
|
|
449
|
+
return {
|
|
450
|
+
memoryInference,
|
|
451
|
+
durationMs: pass.durationMs,
|
|
452
|
+
// Sentinel refs (`<domain>/_<marker>`) label maintenance events; never parsed as assets.
|
|
453
|
+
action: { ref: "memories/_inference", mode: "memory-inference", result: memoryInference },
|
|
454
|
+
warnings: pass.warnings,
|
|
455
|
+
};
|
|
964
456
|
}
|
|
965
457
|
/**
|
|
966
|
-
*
|
|
967
|
-
*
|
|
968
|
-
* promoted assets from accept flows during this run are already present.
|
|
458
|
+
* Reject pending proposals whose target no longer exists, then expire pending
|
|
459
|
+
* proposals past the retention window; each emits a roll-up event.
|
|
969
460
|
*/
|
|
970
|
-
function
|
|
971
|
-
const { primaryStashDir,
|
|
461
|
+
function runProposalHygienePass(ctx) {
|
|
462
|
+
const { primaryStashDir, eventsCtx } = ctx;
|
|
972
463
|
const warnings = [];
|
|
973
464
|
let orphansPurged = 0;
|
|
465
|
+
let proposalsExpired = 0;
|
|
974
466
|
try {
|
|
975
|
-
const
|
|
976
|
-
orphansPurged =
|
|
977
|
-
if (
|
|
978
|
-
info(`[improve] orphan purge: ${
|
|
467
|
+
const purge = purgeOrphanProposals(primaryStashDir, ctx.sources.map((s) => s.path));
|
|
468
|
+
orphansPurged = purge.rejected;
|
|
469
|
+
if (purge.rejected > 0) {
|
|
470
|
+
info(`[improve] orphan purge: ${purge.rejected}/${purge.checked} orphaned proposals rejected (${purge.durationMs}ms)`);
|
|
979
471
|
}
|
|
980
472
|
appendEvent({
|
|
981
473
|
eventType: "proposal_orphan_purge",
|
|
982
474
|
ref: "proposals/_orphan-purge",
|
|
983
475
|
metadata: {
|
|
984
|
-
checked:
|
|
985
|
-
rejected:
|
|
986
|
-
durationMs:
|
|
987
|
-
byType:
|
|
988
|
-
orphans:
|
|
476
|
+
checked: purge.checked,
|
|
477
|
+
rejected: purge.rejected,
|
|
478
|
+
durationMs: purge.durationMs,
|
|
479
|
+
byType: purge.byType,
|
|
480
|
+
orphans: purge.orphans.map((o) => o.ref),
|
|
989
481
|
},
|
|
990
482
|
}, eventsCtx);
|
|
991
483
|
}
|
|
992
484
|
catch (err) {
|
|
993
485
|
warnings.push(`orphan purge failed: ${errMessage(err)}`);
|
|
994
486
|
}
|
|
995
|
-
return { orphansPurged, warnings };
|
|
996
|
-
}
|
|
997
|
-
/**
|
|
998
|
-
* Phase 6B (Advantage D6b): expire pending proposals that have aged past
|
|
999
|
-
* the retention window. Runs AFTER orphan purge so we never double-archive
|
|
1000
|
-
* a proposal that orphan-purge already moved. `expireStaleProposals` emits
|
|
1001
|
-
* its own per-proposal `proposal_expired` events; we additionally emit a
|
|
1002
|
-
* single roll-up event here for parity with the orphan-purge surface.
|
|
1003
|
-
*/
|
|
1004
|
-
function runProposalExpirationPass(ctx) {
|
|
1005
|
-
const { primaryStashDir, config, eventsCtx } = ctx;
|
|
1006
|
-
const warnings = [];
|
|
1007
|
-
let proposalsExpired = 0;
|
|
1008
487
|
try {
|
|
1009
|
-
const
|
|
1010
|
-
proposalsExpired =
|
|
1011
|
-
if (
|
|
1012
|
-
info(`[improve] expiration: ${
|
|
1013
|
-
`(retention=${
|
|
488
|
+
const expiry = expireStaleProposals(primaryStashDir, ctx.config);
|
|
489
|
+
proposalsExpired = expiry.expired;
|
|
490
|
+
if (expiry.expired > 0) {
|
|
491
|
+
info(`[improve] expiration: ${expiry.expired}/${expiry.checked} pending proposals expired ` +
|
|
492
|
+
`(retention=${expiry.retentionDays}d, ${expiry.durationMs}ms)`);
|
|
1014
493
|
}
|
|
1015
494
|
appendEvent({
|
|
1016
495
|
eventType: "proposal_expiration_pass",
|
|
1017
496
|
ref: "proposals/_expiration",
|
|
1018
497
|
metadata: {
|
|
1019
|
-
checked:
|
|
1020
|
-
expired:
|
|
1021
|
-
durationMs:
|
|
1022
|
-
retentionDays:
|
|
1023
|
-
expiredProposals:
|
|
498
|
+
checked: expiry.checked,
|
|
499
|
+
expired: expiry.expired,
|
|
500
|
+
durationMs: expiry.durationMs,
|
|
501
|
+
retentionDays: expiry.retentionDays,
|
|
502
|
+
expiredProposals: expiry.expiredProposals,
|
|
1024
503
|
},
|
|
1025
504
|
}, eventsCtx);
|
|
1026
505
|
}
|
|
1027
506
|
catch (err) {
|
|
1028
507
|
warnings.push(`proposal expiration failed: ${errMessage(err)}`);
|
|
1029
508
|
}
|
|
1030
|
-
return { proposalsExpired, warnings };
|
|
509
|
+
return { orphansPurged, proposalsExpired, warnings };
|
|
1031
510
|
}
|
|
1032
511
|
/**
|
|
1033
|
-
*
|
|
1034
|
-
*
|
|
1035
|
-
*
|
|
1036
|
-
*
|
|
1037
|
-
*
|
|
1038
|
-
*
|
|
1039
|
-
* the index handle the other passes use).
|
|
512
|
+
* Trim the observability data that grows append-only — state.db events and
|
|
513
|
+
* improve_runs, logs.db task_logs, and the per-run task log files — to
|
|
514
|
+
* `improve.eventRetentionDays` (default 90; 0 disables), then VACUUM state.db
|
|
515
|
+
* when enough pages are free. Each store fails on its own. state.db work
|
|
516
|
+
* borrows the run's long-lived handle: a second writer on the same WAL file
|
|
517
|
+
* locks (#585).
|
|
1040
518
|
*/
|
|
1041
519
|
export function runRetentionPurgePass(ctx) {
|
|
1042
520
|
const { config, eventsCtx } = ctx;
|
|
1043
521
|
const warnings = [];
|
|
1044
522
|
const retentionDays = typeof config.improve?.eventRetentionDays === "number" ? config.improve.eventRetentionDays : 90;
|
|
1045
|
-
if (retentionDays
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
}
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
|
|
1073
|
-
eventType: "improve_runs_purged",
|
|
1074
|
-
ref: "improve_runs/_purge",
|
|
1075
|
-
metadata: { purgedCount: improveRunsPurged, retentionDays },
|
|
1076
|
-
}, eventsCtx);
|
|
1077
|
-
// R5: improve_cycle_metrics has its OWN retention window
|
|
1078
|
-
// (default 365d — a slow collapse needs a longer trend than
|
|
1079
|
-
// the 90d events window). canary_queries rows are never purged.
|
|
1080
|
-
const cycleRetention = config.improve?.collapseDetector?.retentionDays ?? CYCLE_METRICS_RETENTION_DAYS;
|
|
1081
|
-
const cycleMetricsPurged = purgeOldCycleMetrics(stateDb, cycleRetention);
|
|
1082
|
-
if (cycleMetricsPurged > 0) {
|
|
1083
|
-
info(`[improve] cycle-metrics purge: ${cycleMetricsPurged} row(s) older than ${cycleRetention}d removed from state.db`);
|
|
1084
|
-
appendEvent({
|
|
1085
|
-
// Dedicated type (mirrors improve_runs_purged) so consumers
|
|
1086
|
-
// never have to disambiguate purge targets via the ref string.
|
|
1087
|
-
eventType: "improve_cycle_metrics_purged",
|
|
1088
|
-
ref: "improve_cycle_metrics/_purge",
|
|
1089
|
-
metadata: { purgedCount: cycleMetricsPurged, retentionDays: cycleRetention },
|
|
1090
|
-
}, eventsCtx);
|
|
1091
|
-
}
|
|
1092
|
-
}, { path: eventsCtx?.dbPath, borrowed: eventsCtx?.db });
|
|
1093
|
-
}
|
|
1094
|
-
catch (err) {
|
|
1095
|
-
warnings.push(`events purge failed: ${errMessage(err)}`);
|
|
1096
|
-
}
|
|
1097
|
-
// task_logs in logs.db (#579) shares the same retention window as
|
|
1098
|
-
// events/improve_runs — all three are observability data governed by
|
|
1099
|
-
// the single improve.eventRetentionDays knob. Separate try/finally
|
|
1100
|
-
// because logs.db is a different file: a locked/missing logs.db must
|
|
1101
|
-
// not block the state.db purges above.
|
|
1102
|
-
let logsDb;
|
|
1103
|
-
try {
|
|
1104
|
-
logsDb = openLogsDatabase();
|
|
1105
|
-
const taskLogsPurged = purgeOldTaskLogs(logsDb, retentionDays);
|
|
1106
|
-
if (taskLogsPurged > 0) {
|
|
1107
|
-
info(`[improve] task_logs purge: ${taskLogsPurged} log line(s) older than ${retentionDays}d removed from logs.db`);
|
|
1108
|
-
}
|
|
1109
|
-
appendEvent({
|
|
1110
|
-
eventType: "task_logs_purged",
|
|
1111
|
-
ref: "task_logs/_purge",
|
|
1112
|
-
metadata: { purgedCount: taskLogsPurged, retentionDays },
|
|
1113
|
-
}, eventsCtx);
|
|
1114
|
-
}
|
|
1115
|
-
catch (err) {
|
|
1116
|
-
warnings.push(`task_logs purge failed: ${errMessage(err)}`);
|
|
1117
|
-
}
|
|
1118
|
-
finally {
|
|
1119
|
-
if (logsDb) {
|
|
1120
|
-
try {
|
|
1121
|
-
logsDb.close();
|
|
1122
|
-
}
|
|
1123
|
-
catch {
|
|
1124
|
-
// best-effort
|
|
1125
|
-
}
|
|
1126
|
-
}
|
|
1127
|
-
}
|
|
1128
|
-
// Per-run flat log files under getTaskLogDir() (#951): logs.db above is
|
|
1129
|
-
// the durable record and already retention-purged, so the transitional
|
|
1130
|
-
// `<taskId>/<timestamp>.log` tail files can be deleted on the same
|
|
1131
|
-
// window without losing anything. A separate try/catch — a filesystem
|
|
1132
|
-
// problem here must not block the DB purges above.
|
|
523
|
+
if (retentionDays <= 0)
|
|
524
|
+
return { warnings };
|
|
525
|
+
const report = (eventType, ref, purgedCount, what) => {
|
|
526
|
+
if (purgedCount > 0)
|
|
527
|
+
info(`[improve] ${eventType}: ${purgedCount} ${what} older than ${retentionDays}d removed`);
|
|
528
|
+
appendEvent({ eventType, ref, metadata: { purgedCount, retentionDays } }, eventsCtx);
|
|
529
|
+
};
|
|
530
|
+
try {
|
|
531
|
+
withStateDb((stateDb) => {
|
|
532
|
+
report("events_purged", "events/_purge", purgeOldEvents(stateDb, retentionDays), "event(s)");
|
|
533
|
+
report("improve_runs_purged", "improve_runs/_purge", purgeOldImproveRuns(stateDb, retentionDays), "run(s)");
|
|
534
|
+
const vacuum = vacuumIfReclaimable(stateDb, readFreelistInfo(stateDb), { eventType: STATE_DB_VACUUMED_EVENT }, eventsCtx);
|
|
535
|
+
if (vacuum.ran)
|
|
536
|
+
info(`[improve] state.db vacuum: ${vacuum.pagesBefore} -> ${vacuum.pagesAfter} pages`);
|
|
537
|
+
}, { path: eventsCtx?.dbPath, borrowed: eventsCtx?.db });
|
|
538
|
+
}
|
|
539
|
+
catch (err) {
|
|
540
|
+
warnings.push(`events purge failed: ${errMessage(err)}`);
|
|
541
|
+
}
|
|
542
|
+
let logsDb;
|
|
543
|
+
try {
|
|
544
|
+
logsDb = openLogsDatabase();
|
|
545
|
+
report("task_logs_purged", "task_logs/_purge", purgeOldTaskLogs(logsDb, retentionDays), "log line(s)");
|
|
546
|
+
}
|
|
547
|
+
catch (err) {
|
|
548
|
+
warnings.push(`task_logs purge failed: ${errMessage(err)}`);
|
|
549
|
+
}
|
|
550
|
+
finally {
|
|
1133
551
|
try {
|
|
1134
|
-
|
|
1135
|
-
if (taskLogFilesPurged > 0) {
|
|
1136
|
-
info(`[improve] task log files purge: ${taskLogFilesPurged} file(s) older than ${retentionDays}d removed from ${getTaskLogDir()}`);
|
|
1137
|
-
}
|
|
1138
|
-
appendEvent({
|
|
1139
|
-
eventType: "task_log_files_purged",
|
|
1140
|
-
ref: "task_log_files/_purge",
|
|
1141
|
-
metadata: { purgedCount: taskLogFilesPurged, retentionDays },
|
|
1142
|
-
}, eventsCtx);
|
|
552
|
+
logsDb?.close();
|
|
1143
553
|
}
|
|
1144
|
-
catch
|
|
1145
|
-
|
|
554
|
+
catch {
|
|
555
|
+
// best-effort
|
|
1146
556
|
}
|
|
1147
557
|
}
|
|
558
|
+
try {
|
|
559
|
+
report("task_log_files_purged", "task_log_files/_purge", purgeOldTaskLogFiles(undefined, retentionDays), `file(s) under ${getTaskLogDir()}`);
|
|
560
|
+
}
|
|
561
|
+
catch (err) {
|
|
562
|
+
warnings.push(`task log files purge failed: ${errMessage(err)}`);
|
|
563
|
+
}
|
|
1148
564
|
return { warnings };
|
|
1149
565
|
}
|
|
1150
|
-
// ── #733 — orphan-state GC pass (Workstream C) ──────────────────────────────
|
|
1151
|
-
//
|
|
1152
|
-
// Deliberately lean: one maintenance pass, one additive migration (021), one
|
|
1153
|
-
// event type (asset_state_gc), one config gate (improve.stateGc.collect,
|
|
1154
|
-
// default false). See docs/architecture/specs/0.9.0-close-out-plan.md
|
|
1155
|
-
// Workstream C for the full design rationale. No quarantine archive, no
|
|
1156
|
-
// circuit breaker, no health-advisory plumbing, no new tables.
|
|
1157
566
|
/**
|
|
1158
|
-
* Grace window
|
|
1159
|
-
*
|
|
1160
|
-
* A named constant, not a config knob (owner ruling — see the close-out
|
|
1161
|
-
* plan's Workstream C). Mirrors `TXN_SWEEP_GRACE_MS` (src/core/fs-txn.ts:298).
|
|
567
|
+
* Grace window before an unresolved salience/outcome row may be deleted (only
|
|
568
|
+
* when `improve.stateGc.collect` is true).
|
|
1162
569
|
*/
|
|
1163
570
|
export const STATE_GC_GRACE_MS = daysToMs(7);
|
|
1164
571
|
/**
|
|
1165
|
-
*
|
|
1166
|
-
*
|
|
1167
|
-
*
|
|
1168
|
-
* predicate (see the pass doc comment below), so this is a thin wrapper
|
|
1169
|
-
* around the same single-ref probe the rest of improve uses
|
|
1170
|
-
* (`getEntryByRef`, index-entries-repository.ts) — which already resolves
|
|
1171
|
-
* both storage spellings a write can produce (`salienceWriteKey`/
|
|
1172
|
-
* `outcomeWriteKey` = `itemRef ?? ref`): an exact bundle-qualified item_ref,
|
|
1173
|
-
* or a bare conceptId matched by suffix across all bundles.
|
|
1174
|
-
*
|
|
1175
|
-
* On top of that, falls back to the BARE conceptId form (`bareImproveRef` —
|
|
1176
|
-
* the same primitive `preparation.ts`'s `normalizeStoredKey` map is built
|
|
1177
|
-
* from via `improveStateReadRefs`) when the stored ref carries a bundle
|
|
1178
|
-
* prefix that no longer matches exactly. This is the legacy-spelling
|
|
1179
|
-
* normalization trap: a naive `asset_ref NOT IN (SELECT item_ref FROM
|
|
1180
|
-
* entries)` would treat a live asset whose row predates bundle-qualification
|
|
1181
|
-
* (or whose bundle prefix is stale) as an orphan and delete it. Preferring
|
|
1182
|
-
* "never delete a live row" over "never miss a genuinely dead one" mirrors
|
|
1183
|
-
* `getEntryByRef`'s own bare-conceptId suffix-match trade-off.
|
|
572
|
+
* A stored state ref is live when it, or its bare conceptId, resolves in the
|
|
573
|
+
* index. The bare fallback keeps a live asset whose row predates
|
|
574
|
+
* bundle-qualification from being collected.
|
|
1184
575
|
*/
|
|
1185
|
-
function isStateRefLive(
|
|
1186
|
-
if (
|
|
576
|
+
function isStateRefLive(snapshot, storedRef) {
|
|
577
|
+
if (isRefLiveInSnapshot(snapshot, storedRef))
|
|
1187
578
|
return true;
|
|
1188
|
-
const bare =
|
|
1189
|
-
return bare !== storedRef &&
|
|
1190
|
-
}
|
|
1191
|
-
/**
|
|
1192
|
-
* Sweep ONE state table: stamp refs that just went unresolved, clear refs
|
|
1193
|
-
* that resolved again, and — only when `collect` is true — delete rows whose
|
|
1194
|
-
* `missing_since` is older than {@link STATE_GC_GRACE_MS}. `pending` is a
|
|
1195
|
-
* point-in-time snapshot taken AFTER stamp/clear/delete (re-queried, not
|
|
1196
|
-
* accumulated), so it reflects the current backlog rather than this run's
|
|
1197
|
-
* delta — "every run emits the counts either way, so live data accumulates
|
|
1198
|
-
* proof" (close-out plan, Workstream C).
|
|
1199
|
-
*/
|
|
1200
|
-
function gcOneStateTable(args) {
|
|
1201
|
-
const { refRows, indexDb, now, collect, stamp, clear, deleteOlderThan, countPending } = args;
|
|
1202
|
-
const toStamp = [];
|
|
1203
|
-
const toClear = [];
|
|
1204
|
-
for (const row of refRows) {
|
|
1205
|
-
const live = isStateRefLive(indexDb, row.asset_ref);
|
|
1206
|
-
if (!live && row.missing_since == null)
|
|
1207
|
-
toStamp.push(row.asset_ref);
|
|
1208
|
-
else if (live && row.missing_since != null)
|
|
1209
|
-
toClear.push(row.asset_ref);
|
|
1210
|
-
}
|
|
1211
|
-
if (toStamp.length > 0)
|
|
1212
|
-
stamp(toStamp, now);
|
|
1213
|
-
if (toClear.length > 0)
|
|
1214
|
-
clear(toClear);
|
|
1215
|
-
const collected = collect ? deleteOlderThan(now - STATE_GC_GRACE_MS) : 0;
|
|
1216
|
-
const pending = countPending();
|
|
1217
|
-
return { pending, collected };
|
|
579
|
+
const bare = stripBundle(storedRef);
|
|
580
|
+
return bare !== storedRef && isRefLiveInSnapshot(snapshot, bare);
|
|
1218
581
|
}
|
|
1219
582
|
/**
|
|
1220
|
-
* Orphan-state GC
|
|
1221
|
-
*
|
|
1222
|
-
* that
|
|
1223
|
-
*
|
|
1224
|
-
*
|
|
1225
|
-
*
|
|
1226
|
-
*
|
|
1227
|
-
* predicate: "absent ≠ deleted" is inherited from the indexer, not
|
|
1228
|
-
* re-implemented here — an incomplete or failed source scan preserves that
|
|
1229
|
-
* source's last-known-good `entries` rows (indexer.ts ~1195-1199), and
|
|
1230
|
-
* mass-wipe is already gated upstream (`preserveExistingIndex` +
|
|
1231
|
-
* `fullDelete && scanComplete`). A temporarily unreachable source therefore
|
|
1232
|
-
* never surfaces candidates; no separate scan-status tracking is needed.
|
|
1233
|
-
*
|
|
1234
|
-
* Runs under the SAME index-writer lease / borrowed-state.db-connection
|
|
1235
|
-
* discipline as the neighboring maintenance passes: `dbCell.current` supplies
|
|
1236
|
-
* the already-open index.db handle (#584), and state.db access goes through
|
|
1237
|
-
* `withStateDb(..., { borrowed: eventsCtx?.db })` so this never opens a
|
|
1238
|
-
* second live writer alongside a long-lived `eventsCtx.db` connection — see
|
|
1239
|
-
* the #585 comment on {@link runRetentionPurgePass}'s events-purge call for
|
|
1240
|
-
* why that matters ("database is locked").
|
|
1241
|
-
*
|
|
1242
|
-
* Exported for direct test coverage (tests/integration/commands/improve/
|
|
1243
|
-
* state-gc.test.ts), mirroring the `runMemoryInferenceMaintenancePass` /
|
|
1244
|
-
* `runGraphExtractionMaintenancePass` / `runRetentionPurgePass` precedent;
|
|
1245
|
-
* production callers reach it only through `runMaintenancePassesUnderLease`.
|
|
583
|
+
* Orphan-state GC (#733) over `asset_salience` and `asset_outcome`: stamp
|
|
584
|
+
* `missing_since` on refs that no longer resolve in index.db, clear it on refs
|
|
585
|
+
* that resolve again, and — only with `improve.stateGc.collect` — delete rows
|
|
586
|
+
* stamped longer ago than {@link STATE_GC_GRACE_MS}. An unreachable source keeps
|
|
587
|
+
* its last-known index rows, so it never surfaces candidates. `pending` is the
|
|
588
|
+
* backlog after the sweep; the event is emitted only when there is something to
|
|
589
|
+
* report.
|
|
1246
590
|
*/
|
|
1247
591
|
export function runOrphanStateGcPass(ctx, dbCell) {
|
|
1248
|
-
const { eventsCtx
|
|
1249
|
-
const warnings = [];
|
|
592
|
+
const { eventsCtx } = ctx;
|
|
1250
593
|
const indexDb = dbCell.current;
|
|
1251
|
-
if (!indexDb)
|
|
1252
|
-
warnings
|
|
1253
|
-
|
|
1254
|
-
}
|
|
1255
|
-
const collect = config.improve?.stateGc?.collect === true;
|
|
594
|
+
if (!indexDb)
|
|
595
|
+
return { pending: 0, collected: 0, warnings: ["orphan state GC skipped: no index.db handle available"] };
|
|
596
|
+
const collect = ctx.config.improve?.stateGc?.collect === true;
|
|
1256
597
|
const now = Date.now();
|
|
1257
598
|
let pending = 0;
|
|
1258
599
|
let collected = 0;
|
|
1259
600
|
try {
|
|
601
|
+
const liveRefs = getLiveRefSnapshot(indexDb);
|
|
602
|
+
const sweep = (table) => {
|
|
603
|
+
const toStamp = [];
|
|
604
|
+
const toClear = [];
|
|
605
|
+
for (const row of table.rows) {
|
|
606
|
+
const live = isStateRefLive(liveRefs, row.asset_ref);
|
|
607
|
+
if (!live && row.missing_since == null)
|
|
608
|
+
toStamp.push(row.asset_ref);
|
|
609
|
+
else if (live && row.missing_since != null)
|
|
610
|
+
toClear.push(row.asset_ref);
|
|
611
|
+
}
|
|
612
|
+
if (toStamp.length > 0)
|
|
613
|
+
table.stamp(toStamp, now);
|
|
614
|
+
if (toClear.length > 0)
|
|
615
|
+
table.clear(toClear);
|
|
616
|
+
const removed = collect ? table.deleteOlderThan(now - STATE_GC_GRACE_MS) : 0;
|
|
617
|
+
return { pending: table.countPending(), collected: removed };
|
|
618
|
+
};
|
|
1260
619
|
withStateDb((stateDb) => {
|
|
1261
|
-
|
|
1262
|
-
|
|
1263
|
-
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
|
|
1268
|
-
|
|
1269
|
-
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
|
|
1276
|
-
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
// Table-name-shaped keys ("salience"/"outcome", not the guarded
|
|
1282
|
-
// "asset_salience"/"asset_outcome" table names) — this file sits
|
|
1283
|
-
// outside src/storage/repositories/**, where the state-table-sql
|
|
1284
|
-
// lint rule (#672) forbids naming those tables even in a log string
|
|
1285
|
-
// or an object key, not just in raw SQL.
|
|
1286
|
-
const byTable = { salience: salienceResult, outcome: outcomeResult };
|
|
1287
|
-
pending = salienceResult.pending + outcomeResult.pending;
|
|
1288
|
-
collected = salienceResult.collected + outcomeResult.collected;
|
|
620
|
+
// Keys avoid the state table names: the state-table-sql lint rule (#672)
|
|
621
|
+
// forbids them outside the repositories.
|
|
622
|
+
const byTable = {
|
|
623
|
+
salience: sweep({
|
|
624
|
+
rows: listAssetSalienceMissingState(stateDb),
|
|
625
|
+
stamp: (refs, ts) => stampAssetSalienceMissing(stateDb, refs, ts),
|
|
626
|
+
clear: (refs) => clearAssetSalienceMissing(stateDb, refs),
|
|
627
|
+
deleteOlderThan: (cutoff) => deleteAssetSalienceMissingBefore(stateDb, cutoff),
|
|
628
|
+
countPending: () => countAssetSalienceMissing(stateDb),
|
|
629
|
+
}),
|
|
630
|
+
outcome: sweep({
|
|
631
|
+
rows: listAssetOutcomeMissingState(stateDb),
|
|
632
|
+
stamp: (refs, ts) => stampAssetOutcomeMissing(stateDb, refs, ts),
|
|
633
|
+
clear: (refs) => clearAssetOutcomeMissing(stateDb, refs),
|
|
634
|
+
deleteOlderThan: (cutoff) => deleteAssetOutcomeMissingBefore(stateDb, cutoff),
|
|
635
|
+
countPending: () => countAssetOutcomeMissing(stateDb),
|
|
636
|
+
}),
|
|
637
|
+
};
|
|
638
|
+
pending = byTable.salience.pending + byTable.outcome.pending;
|
|
639
|
+
collected = byTable.salience.collected + byTable.outcome.collected;
|
|
1289
640
|
if (pending > 0 || collected > 0) {
|
|
1290
641
|
info(`[improve] orphan state GC: ${pending} pending, ${collected} collected ` +
|
|
1291
|
-
`(salience ${
|
|
1292
|
-
`outcome ${
|
|
1293
|
-
|
|
1294
|
-
// snapshot (`pending`) plus this run's deletions (`collected`);
|
|
1295
|
-
// emitted only when there is something to report (mirrors the
|
|
1296
|
-
// rekey script's no-op-stays-silent precedent) so a perpetually
|
|
1297
|
-
// clean stash never accumulates events.
|
|
1298
|
-
appendEvent({
|
|
1299
|
-
eventType: "asset_state_gc",
|
|
1300
|
-
ref: "asset_state/_gc",
|
|
1301
|
-
metadata: { pending, collected, byTable },
|
|
1302
|
-
}, eventsCtx);
|
|
642
|
+
`(salience ${byTable.salience.pending}/${byTable.salience.collected}, ` +
|
|
643
|
+
`outcome ${byTable.outcome.pending}/${byTable.outcome.collected})`);
|
|
644
|
+
appendEvent({ eventType: "asset_state_gc", ref: "asset_state/_gc", metadata: { pending, collected, byTable } }, eventsCtx);
|
|
1303
645
|
}
|
|
1304
646
|
}, { path: eventsCtx?.dbPath, borrowed: eventsCtx?.db });
|
|
1305
647
|
}
|
|
1306
648
|
catch (err) {
|
|
1307
|
-
warnings
|
|
649
|
+
return { pending, collected, warnings: [`orphan state GC failed: ${errMessage(err)}`] };
|
|
1308
650
|
}
|
|
1309
|
-
return { pending, collected, warnings };
|
|
651
|
+
return { pending, collected, warnings: [] };
|
|
1310
652
|
}
|