akm-cli 0.9.1 → 0.9.2-alpha.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +103 -28
- package/README.md +3 -1
- package/SECURITY.md +1 -1
- package/STABILITY.md +1 -1
- package/dist/akm +2 -2
- package/dist/akm-migrate +2 -2
- package/dist/assets/hints/cli-hints-full.md +14 -9
- package/dist/assets/improve-strategies/proactive-maintenance.json +1 -1
- package/dist/assets/improve-strategies/reflect-distill.json +1 -1
- package/dist/assets/models.json +35 -0
- package/dist/assets/stash-skeleton/facts/conventions/backlinks.md +3 -4
- package/dist/assets/stash-skeleton/facts/conventions/organization.md +1 -3
- package/dist/assets/tasks/core/extract.yml +6 -5
- package/dist/assets/tasks/core/improve.yml +6 -5
- package/dist/assets/tasks/core/index-refresh.yml +6 -5
- package/dist/assets/tasks/core/sync.yml +6 -5
- package/dist/assets/tasks/core/version-check.yml +6 -5
- package/dist/assets/tasks/improve/akm-graph-refresh-weekly.yml +6 -5
- package/dist/assets/tasks/improve/akm-improve-catchup.yml +6 -5
- package/dist/assets/tasks/improve/akm-improve-consolidate.yml +6 -5
- package/dist/assets/tasks/improve/akm-improve-frequent.yml +6 -5
- package/dist/assets/tasks/improve/akm-improve-nightly.yml +6 -5
- package/dist/cli/confirm.js +2 -2
- package/dist/cli/parse-args.js +3 -24
- package/dist/cli/retired-commands.js +1 -1
- package/dist/cli/shared.js +2 -2
- package/dist/cli.js +11 -9
- package/dist/commands/agent/agent-dispatch.js +55 -89
- package/dist/commands/agent/contribute-cli.js +12 -45
- package/dist/commands/command/builtin-action.js +32 -0
- package/dist/commands/command/command-cli.js +99 -0
- package/dist/commands/command/command-execution.js +308 -0
- package/dist/commands/command/execution-source-loader.js +176 -0
- package/dist/commands/command/portable-template.js +60 -0
- package/dist/commands/config-cli.js +10 -4
- package/dist/commands/env/env.js +4 -2
- package/dist/commands/feedback-cli.js +1 -1
- package/dist/commands/health/checks.js +241 -29
- package/dist/commands/health/html-report.js +0 -14
- package/dist/commands/health/report-view-model.js +0 -1
- package/dist/commands/health/surfaces.js +6 -7
- package/dist/commands/health/types.js +0 -2
- package/dist/commands/health.js +63 -18
- package/dist/commands/improve/collapse-detector.js +5 -6
- package/dist/commands/improve/consolidate.js +251 -214
- package/dist/commands/improve/distill/promote-memory.js +71 -34
- package/dist/commands/improve/distill/quality-gate.js +17 -5
- package/dist/commands/improve/distill.js +232 -155
- package/dist/commands/improve/eligibility.js +112 -79
- package/dist/commands/improve/execution.js +57 -0
- package/dist/commands/improve/extract-cli.js +5 -5
- package/dist/commands/improve/extract-prompt.js +64 -22
- package/dist/commands/improve/extract.js +608 -360
- package/dist/commands/improve/improve-strategies.js +43 -14
- package/dist/commands/improve/improve.js +249 -29
- package/dist/commands/improve/loop-stages.js +11 -17
- package/dist/commands/improve/memory/memory-contradiction-detect.js +90 -66
- package/dist/commands/improve/outcome-loop.js +22 -38
- package/dist/commands/improve/planner.js +134 -0
- package/dist/commands/improve/preparation.js +730 -409
- package/dist/commands/improve/reflect.js +386 -223
- package/dist/commands/improve/run-context.js +3 -4
- package/dist/commands/improve/salience.js +6 -58
- package/dist/commands/improve/session-asset.js +12 -12
- package/dist/commands/lint/index.js +101 -29
- package/dist/commands/migrate-cli.js +11 -69
- package/dist/commands/migration-tool.js +6 -9
- package/dist/commands/models-cli.js +27 -0
- package/dist/commands/proposal/drain.js +258 -186
- package/dist/commands/proposal/proposal-cli.js +32 -10
- package/dist/commands/proposal/proposal.js +2 -5
- package/dist/commands/proposal/propose.js +192 -172
- package/dist/commands/proposal/repository.js +54 -91
- package/dist/commands/proposal/validators/proposal-validators.js +9 -7
- package/dist/commands/read/curate.js +53 -22
- package/dist/commands/read/registry-search.js +25 -9
- package/dist/commands/read/remember-cli.js +14 -2
- package/dist/commands/read/search.js +10 -4
- package/dist/commands/read/show.js +139 -153
- package/dist/commands/registry-cli.js +16 -7
- package/dist/commands/remember.js +33 -18
- package/dist/commands/sources/add-cli.js +19 -178
- package/dist/commands/sources/bundle-cli.js +15 -3
- package/dist/commands/sources/dangerous-env-audit.js +135 -0
- package/dist/commands/sources/info.js +2 -1
- package/dist/commands/sources/installed-stashes.js +901 -177
- package/dist/commands/sources/schema-repair.js +174 -95
- package/dist/commands/sources/self-update.js +30 -74
- package/dist/commands/sources/source-add.js +3 -5
- package/dist/commands/sources/sources-cli.js +2 -15
- package/dist/commands/sources/update-transaction.js +220 -0
- package/dist/commands/tasks/tasks-cli.js +3 -3
- package/dist/commands/tasks/tasks.js +736 -317
- package/dist/commands/workflow-cli.js +2 -2
- package/dist/core/adapter/adapters/agent-skills-adapter.js +3 -0
- package/dist/core/adapter/adapters/akm-adapter.js +85 -35
- package/dist/core/adapter/adapters/akm-lint.js +54 -39
- package/dist/core/adapter/adapters/akm-metadata.js +45 -45
- package/dist/core/adapter/adapters/akm-task-adapter.js +32 -49
- package/dist/core/adapter/adapters/akm-workflow-adapter.js +38 -23
- package/dist/core/adapter/adapters/dotenv-adapter.js +30 -1
- package/dist/core/adapter/adapters/generic-files-adapter.js +11 -0
- package/dist/core/adapter/adapters/index.js +0 -9
- package/dist/core/adapter/adapters/llm-wiki-adapter.js +4 -0
- package/dist/core/adapter/adapters/okf-adapter.js +4 -0
- package/dist/core/adapter/adapters/opencode-adapter.js +5 -8
- package/dist/core/adapter/adapters/tool-dir-shared.js +63 -6
- package/dist/core/adapter/adapters/website-snapshot-adapter.js +4 -0
- package/dist/core/adapter/execution-source.js +308 -0
- package/dist/core/adapter/recognize-match.js +36 -13
- package/dist/core/adapter/registry.js +0 -9
- package/dist/core/asset/stash-meta.js +94 -4
- package/dist/core/common.js +6 -11
- package/dist/core/config/config-io.js +3 -3
- package/dist/core/config/config-schema.js +18 -40
- package/dist/core/config/config-sources.js +11 -21
- package/dist/core/config/config-walker.js +31 -13
- package/dist/core/config/config.js +23 -26
- package/dist/core/config/schema/engines.js +8 -7
- package/dist/core/config/schema/improve-processes.js +29 -5
- package/dist/core/config/schema/index-config.js +0 -27
- package/dist/core/config/schema/primitives.js +1 -23
- package/dist/core/config/schema/sources-bundles.js +13 -16
- package/dist/core/errors.js +2 -0
- package/dist/core/events.js +68 -32
- package/dist/core/extra-params.js +1 -0
- package/dist/core/improve-result.js +315 -0
- package/dist/core/lesson-lint.js +0 -6
- package/dist/core/maintenance-barrier.js +4 -4
- package/dist/core/network-policy.js +152 -0
- package/dist/core/paths.js +1 -1
- package/dist/core/recognition-util.js +4 -4
- package/dist/core/registry-url.js +456 -0
- package/dist/core/state/migrations.js +161 -47
- package/dist/core/state-db.js +453 -80
- package/dist/core/system-error.js +32 -0
- package/dist/core/time.js +2 -12
- package/dist/core/write-source.js +0 -18
- package/dist/execution/directory-identity.js +52 -0
- package/dist/execution/executable-identity.js +107 -0
- package/dist/execution/guarded-source.js +398 -0
- package/dist/execution/json.js +95 -0
- package/dist/{commands/health/types-session-log.js → execution/limits.js} +2 -1
- package/dist/execution/record.js +55 -0
- package/dist/execution/resolved-request.js +730 -0
- package/dist/execution/source.js +320 -0
- package/dist/indexer/bundle-identity-guard.js +5 -4
- package/dist/indexer/db/graph-db.js +33 -0
- package/dist/indexer/graph/graph-boost.js +3 -4
- package/dist/indexer/graph/graph-extraction.js +562 -373
- package/dist/indexer/index-written-assets.js +78 -39
- package/dist/indexer/indexer.js +471 -432
- package/dist/indexer/installations.js +6 -0
- package/dist/indexer/lookup/adapter-concept-owner.js +283 -0
- package/dist/indexer/materialize-embeddings.js +155 -0
- package/dist/indexer/passes/memory-inference.js +227 -174
- package/dist/indexer/passes/metadata.js +263 -118
- package/dist/indexer/scan/doc-to-entry.js +7 -10
- package/dist/indexer/scan/drain-dir.js +51 -23
- package/dist/indexer/search/db-search.js +156 -50
- package/dist/indexer/search/fts-query.js +40 -40
- package/dist/indexer/search/ranking.js +36 -1
- package/dist/indexer/search/search-attribution.js +3 -1
- package/dist/indexer/search/search-fields.js +23 -14
- package/dist/indexer/search/search-hit-enrichers.js +1 -1
- package/dist/indexer/search/search-source.js +7 -16
- package/dist/indexer/search/semantic-status.js +10 -1
- package/dist/indexer/usage/show-usage.js +105 -0
- package/dist/indexer/usage/usage-events.js +7 -2
- package/dist/indexer/walk/matchers.js +40 -10
- package/dist/indexer/walk/path-resolver.js +5 -2
- package/dist/indexer/walk/walker.js +20 -2
- package/dist/integrations/agent/builder-shared.js +3 -6
- package/dist/integrations/agent/conversation-fallback.js +16 -0
- package/dist/integrations/agent/engine-resolution.js +87 -87
- package/dist/integrations/agent/execution-cascade.js +566 -0
- package/dist/integrations/agent/execution-definitions.js +211 -0
- package/dist/integrations/agent/execution-lowering.js +811 -0
- package/dist/integrations/agent/execution-preparation.js +67 -0
- package/dist/integrations/agent/index.js +0 -2
- package/dist/integrations/agent/inline-execution.js +74 -0
- package/dist/integrations/agent/model-map.js +515 -0
- package/dist/integrations/agent/persona-fallback.js +30 -0
- package/dist/integrations/agent/request-lowering.js +186 -0
- package/dist/integrations/agent/runner-dispatch.js +230 -37
- package/dist/integrations/agent/runner.js +12 -83
- package/dist/integrations/harnesses/aider/agent-builder.js +8 -0
- package/dist/integrations/harnesses/aider/index.js +0 -1
- package/dist/integrations/harnesses/amazonq/agent-builder.js +8 -0
- package/dist/integrations/harnesses/amazonq/index.js +0 -1
- package/dist/integrations/harnesses/claude/agent-builder.js +14 -1
- package/dist/integrations/harnesses/claude/index.js +1 -5
- package/dist/integrations/harnesses/claude/session-log.js +3 -33
- package/dist/integrations/harnesses/codex/agent-builder.js +8 -0
- package/dist/integrations/harnesses/codex/index.js +0 -1
- package/dist/integrations/harnesses/copilot/agent-builder.js +8 -0
- package/dist/integrations/harnesses/copilot/index.js +0 -1
- package/dist/integrations/harnesses/gemini/agent-builder.js +8 -0
- package/dist/integrations/harnesses/gemini/index.js +0 -1
- package/dist/integrations/harnesses/index.js +4 -44
- package/dist/integrations/harnesses/opencode/agent-builder.js +16 -9
- package/dist/integrations/harnesses/opencode/index.js +0 -2
- package/dist/integrations/harnesses/opencode/session-log.js +14 -204
- package/dist/integrations/harnesses/opencode-sdk/harness.js +12 -1
- package/dist/integrations/harnesses/opencode-sdk/sdk-runner.js +40 -42
- package/dist/integrations/harnesses/openhands/agent-builder.js +8 -0
- package/dist/integrations/harnesses/openhands/index.js +0 -1
- package/dist/integrations/harnesses/pi/agent-builder.js +8 -0
- package/dist/integrations/harnesses/pi/index.js +0 -1
- package/dist/integrations/harnesses/shared.js +0 -1
- package/dist/integrations/harnesses/types.js +1 -3
- package/dist/integrations/lockfile.js +82 -79
- package/dist/integrations/session-logs/index.js +6 -17
- package/dist/integrations/session-logs/provider-base.js +1 -29
- package/dist/llm/client.js +10 -5
- package/dist/llm/embedder.js +6 -7
- package/dist/llm/embedders/local.js +37 -88
- package/dist/llm/embedders/types.js +1 -1
- package/dist/llm/graph-extract.js +75 -50
- package/dist/llm/index-passes.js +43 -5
- package/dist/llm/memory-infer.js +8 -6
- package/dist/llm/metadata-enhance.js +5 -3
- package/dist/llm/structured-call.js +122 -25
- package/dist/output/format-exempt.js +1 -1
- package/dist/output/render-registry.js +0 -16
- package/dist/output/renderers.js +12 -7
- package/dist/output/shapes/curate.js +1 -0
- package/dist/output/shapes/helpers.js +10 -2
- package/dist/output/shapes/passthrough.js +2 -0
- package/dist/output/text/command-format.js +31 -33
- package/dist/output/text/health-format.js +1 -29
- package/dist/output/text/migrate.js +6 -56
- package/dist/output/text/proposal-format.js +16 -1
- package/dist/output/text/workflow-format.js +16 -0
- package/dist/registry/network.js +279 -0
- package/dist/registry/pinned-request-helper.js +247 -0
- package/dist/registry/pinned-transport.js +717 -0
- package/dist/registry/providers/skills-sh.js +18 -6
- package/dist/registry/providers/static-index.js +20 -7
- package/dist/registry/resolve.js +53 -28
- package/dist/scripts/akm-migrate-node.js +19334 -52269
- package/dist/scripts/akm-migrate.js +19270 -51612
- package/dist/setup/registry-stash-loader.js +64 -20
- package/dist/setup/semantic-assets.js +9 -34
- package/dist/setup/setup.js +12 -30
- package/dist/setup/source-identity.js +17 -0
- package/dist/setup/steps/sources.js +36 -15
- package/dist/setup/steps/tasks.js +39 -11
- package/dist/sources/providers/git-provider.js +3 -3
- package/dist/sources/providers/npm.js +2 -2
- package/dist/sources/providers/provider-utils.js +4 -3
- package/dist/sources/providers/website.js +11 -7
- package/dist/sources/snapshot-fetchers/host-guard.js +9 -136
- package/dist/sources/snapshot-fetchers/website-ingest.js +25 -109
- package/dist/sources/website-url.js +73 -0
- package/dist/storage/engines/sqlite-migrations.js +81 -26
- package/dist/storage/managed-db.js +27 -24
- package/dist/storage/repositories/events-repository.js +3 -0
- package/dist/storage/repositories/index-connection.js +42 -10
- package/dist/storage/repositories/index-entries-repository.js +203 -229
- package/dist/storage/repositories/index-entry-mapper.js +8 -12
- package/dist/storage/repositories/index-entry-schema.js +255 -0
- package/dist/storage/repositories/index-fts-repository.js +64 -71
- package/dist/storage/repositories/index-llm-cache-repository.js +8 -13
- package/dist/storage/repositories/index-meta-repository.js +0 -11
- package/dist/storage/repositories/index-schema.js +74 -350
- package/dist/storage/repositories/index-utility-repository.js +12 -17
- package/dist/storage/repositories/index-vec-repository.js +56 -7
- package/dist/storage/repositories/proposals-repository.js +4 -127
- package/dist/storage/repositories/registry-cache.js +2 -1
- package/dist/storage/repositories/task-history-repository.js +20 -40
- package/dist/storage/repositories/workflow-runs-repository.js +228 -129
- package/dist/storage/sqlite-read-snapshot.js +148 -0
- package/dist/tasks/backends/cron.js +170 -42
- package/dist/tasks/backends/index.js +1 -1
- package/dist/tasks/backends/launchd.js +787 -202
- package/dist/tasks/backends/schtasks.js +282 -83
- package/dist/tasks/embedded.js +7 -7
- package/dist/tasks/frozen-script.js +50 -0
- package/dist/tasks/resolve-akm-bin.js +5 -1
- package/dist/tasks/runner.js +239 -251
- package/dist/tasks/runtime-v3.js +281 -0
- package/dist/tasks/scheduler-binding.js +272 -0
- package/dist/tasks/scheduler-invocation.js +57 -43
- package/dist/tasks/scheduler-sync.js +654 -0
- package/dist/tasks/source-v3.js +752 -0
- package/dist/tasks/standalone-script-entry.js +5 -0
- package/dist/tasks/task-id.js +29 -0
- package/dist/workflows/authoring/authoring.js +15 -32
- package/dist/workflows/exec/dispatch-redaction.js +14 -8
- package/dist/workflows/exec/exec-unit.js +7 -28
- package/dist/workflows/exec/frozen-judge.js +57 -89
- package/dist/workflows/exec/lowering-notices.js +23 -0
- package/dist/workflows/exec/native-executor.js +301 -458
- package/dist/workflows/exec/param-secrets.js +4 -3
- package/dist/workflows/exec/run-workflow.js +26 -32
- package/dist/workflows/exec/step-work.js +105 -109
- package/dist/workflows/exec/unit-dispatch.js +103 -27
- package/dist/workflows/exec/unit-writer.js +3 -3
- package/dist/workflows/exec/worktree.js +2 -2
- package/dist/workflows/ir/compile.js +86 -72
- package/dist/workflows/ir/environment-v4.js +328 -0
- package/dist/workflows/ir/freeze-v4.js +122 -0
- package/dist/workflows/ir/plan-hash.js +13 -7
- package/dist/workflows/ir/schema-v4.js +525 -0
- package/dist/workflows/ir/schema.js +25 -284
- package/dist/workflows/ir/source-freeze-v4.js +506 -0
- package/dist/workflows/parser.js +27 -24
- package/dist/workflows/program/schema.js +1 -2
- package/dist/workflows/renderer.js +42 -29
- package/dist/workflows/resource-limits.js +4 -5
- package/dist/workflows/runtime/agent-identity.js +11 -13
- package/dist/workflows/runtime/plan-classifier.js +8 -8
- package/dist/workflows/runtime/runs.js +27 -43
- package/dist/workflows/runtime/workflow-asset-loader.js +45 -205
- package/dist/workflows/source-files.js +373 -0
- package/dist/workflows/source-ir/compile.js +196 -0
- package/dist/workflows/source-ir/github-yaml.js +577 -0
- package/dist/workflows/source-ir/ordering.js +38 -0
- package/dist/workflows/source-ir/program.js +50 -0
- package/dist/workflows/source-ir/result.js +26 -0
- package/dist/workflows/source-ir/schema.js +772 -0
- package/dist/workflows/source-ir/semantics.js +242 -0
- package/dist/workflows/source-ir/uses.js +14 -0
- package/docs/README.md +2 -0
- package/docs/migration/README.md +3 -1
- package/docs/migration/release-notes/0.9.2.md +55 -0
- package/docs/migration/release-notes/README.md +5 -0
- package/docs/migration/v0.8-to-v0.9.md +76 -1077
- package/docs/migration/v0.9.0-troubleshooting.md +104 -516
- package/docs/migration/v0.9.1-to-v0.9.2.md +150 -0
- package/docs/reference/README.md +1 -0
- package/docs/reference/cli.md +230 -98
- package/docs/reference/configuration.md +159 -36
- package/docs/reference/data-and-telemetry.md +19 -1
- package/docs/reference/supported-formats.md +23 -3
- package/docs/reference/tasks.md +182 -0
- package/docs/reference/workflow-schema.md +91 -40
- package/docs/reference/workflows.md +33 -6
- package/package.json +10 -6
- package/schemas/akm-config.json +372 -224
- package/schemas/akm-task.json +324 -80
- package/schemas/akm-workflow.json +6 -9
- package/dist/core/migration-operation.js +0 -75
- package/dist/integrations/agent/model-aliases.js +0 -74
- package/dist/tasks/parser.js +0 -380
- package/dist/tasks/schema.js +0 -123
- package/dist/tasks/validator.js +0 -80
- package/dist/workflows/ir/freeze.js +0 -320
- package/dist/workflows/runtime/document-cache.js +0 -13
|
@@ -13,23 +13,24 @@ import { appendEvent, readEvents } from "../../core/events.js";
|
|
|
13
13
|
import { openStateDatabase, withStateDb } from "../../core/state-db.js";
|
|
14
14
|
import { info, warn } from "../../core/warn.js";
|
|
15
15
|
import { countUsageEventsByType } from "../../indexer/usage/usage-events.js";
|
|
16
|
-
import { materializeLlmRunnerConnection } from "../../integrations/agent/runner.js";
|
|
17
16
|
import { getAvailableHarnesses } from "../../integrations/session-logs/index.js";
|
|
18
17
|
import { withLlmStage } from "../../llm/usage-telemetry.js";
|
|
19
|
-
import { closeDatabase, openExistingDatabase } from "../../storage/repositories/index-connection.js";
|
|
18
|
+
import { closeDatabase, openExistingDatabase, openReadonlyExistingDatabase, } from "../../storage/repositories/index-connection.js";
|
|
20
19
|
import { getZeroResultSearches } from "../../storage/repositories/index-entries-repository.js";
|
|
21
20
|
import { getRetrievalCounts } from "../../storage/repositories/index-utility-repository.js";
|
|
22
21
|
import { listStateProposals } from "../../storage/repositories/proposals-repository.js";
|
|
23
22
|
import { akmLint } from "../lint/index.js";
|
|
24
23
|
import { runSchemaRepairPass } from "../sources/schema-repair.js";
|
|
25
24
|
import { isAutonomyLaneAllowed } from "./autonomy-gate.js";
|
|
26
|
-
import { akmConsolidate } from "./consolidate.js";
|
|
25
|
+
import { akmConsolidate, inspectConsolidationPool } from "./consolidate.js";
|
|
26
|
+
import { computeSafeChunkSize, DEFAULT_CONTEXT_LENGTH_TOKENS } from "./consolidate/chunking.js";
|
|
27
27
|
// Eligibility / candidate-selection predicates live in ./eligibility.
|
|
28
28
|
import { buildLatestFeedbackTsMap, buildLatestProposalTsMap, buildUtilityMap, dedupeRefs, findAssetFilePath, isDistillCandidateRef, isLessonCandidate, isSignalDeltaEligible, } from "./eligibility.js";
|
|
29
29
|
import { akmExtract, countNewExtractCandidates } from "./extract.js";
|
|
30
30
|
import { computeValenceScore, FEEDBACK_WEIGHT, UTILITY_WEIGHT } from "./feedback-valence.js";
|
|
31
31
|
import { applyMemoryCleanup } from "./memory/memory-improve.js";
|
|
32
|
-
import { computeProxyAdequacy, getAllAssetOutcomes, getOutcomeScoresByRef, OUTCOME_SCORE_MAX, outcomeScoreToSalience, updateAssetOutcome, } from "./outcome-loop.js";
|
|
32
|
+
import { computeProxyAdequacy, getAllAssetOutcomes, getAssetOutcome, getOutcomeScoresByRef, OUTCOME_SCORE_MAX, outcomeScoreToSalience, projectAssetOutcome, updateAssetOutcome, } from "./outcome-loop.js";
|
|
33
|
+
import { projectMemoryCleanup, selectEffectiveImproveRefs } from "./planner.js";
|
|
33
34
|
import { DEFAULT_DUE_DAYS, DEFAULT_MAX_PER_RUN, selectProactiveMaintenanceRefs } from "./proactive-maintenance.js";
|
|
34
35
|
import { buildRankChangeReport, computeSalience, getAllRankScores, getAssetSalience, getLastUseMsByRef, isContentEncodingRow, SALIENCE_NO_OP_DAMPEN_FACTOR, SALIENCE_NO_OP_DAMPEN_THRESHOLD, upsertAssetSalience, } from "./salience.js";
|
|
35
36
|
import { bareImproveRef, improveStateReadRefs } from "./source-identity.js";
|
|
@@ -82,7 +83,7 @@ function assetTypeOf(ref) {
|
|
|
82
83
|
* Extracted verbatim from `runConsolidationPass` — logic is byte-identical.
|
|
83
84
|
*/
|
|
84
85
|
function evaluateConsolidationEligibility(args) {
|
|
85
|
-
const { options, primaryStashDir, memorySummary, improveProfile, resolvedPlan } = args;
|
|
86
|
+
const { options, primaryStashDir, memorySummary, improveProfile, resolvedPlan, eventsCtx } = args;
|
|
86
87
|
const MEMORY_VOLUME_THRESHOLD = options.memoryVolumeConsolidationThreshold ?? 100;
|
|
87
88
|
const hasLlm = resolvedPlan.processes.consolidate.runner !== null;
|
|
88
89
|
const volumeTriggered = typeof memorySummary.eligible === "number" && memorySummary.eligible > MEMORY_VOLUME_THRESHOLD && hasLlm;
|
|
@@ -92,7 +93,7 @@ function evaluateConsolidationEligibility(args) {
|
|
|
92
93
|
// synchronised-wave failure mode the reflect/distill cooldowns did; the
|
|
93
94
|
// pool-delta gate ties consolidation to actual work-to-do.
|
|
94
95
|
const sourceName = options.sourceName ?? options.writeTarget?.source.name ?? options.config?.defaultBundle ?? "stash";
|
|
95
|
-
const recentConsolidations = readEvents({ type: "consolidate_completed" });
|
|
96
|
+
const recentConsolidations = readEvents({ type: "consolidate_completed" }, eventsCtx);
|
|
96
97
|
const lastConsolidation = recentConsolidations.events
|
|
97
98
|
.filter((e) => e.metadata?.source === sourceName && Number(e.metadata?.processed) > 0)
|
|
98
99
|
.sort((a, b) => new Date(b.ts ?? 0).getTime() - new Date(a.ts ?? 0).getTime())[0];
|
|
@@ -111,7 +112,7 @@ function evaluateConsolidationEligibility(args) {
|
|
|
111
112
|
const promoted = readEvents({
|
|
112
113
|
type: "promoted",
|
|
113
114
|
...(lastConsolidateTs ? { since: lastConsolidateTs } : {}),
|
|
114
|
-
}).events;
|
|
115
|
+
}, eventsCtx).events;
|
|
115
116
|
for (const e of promoted) {
|
|
116
117
|
const ap = e.metadata?.assetPath;
|
|
117
118
|
if (typeof ap === "string" && ap.length > 0)
|
|
@@ -198,11 +199,102 @@ function evaluateConsolidationEligibility(args) {
|
|
|
198
199
|
...(lastConsolidateTs ? { lastConsolidationTs: lastConsolidateTs } : {}),
|
|
199
200
|
};
|
|
200
201
|
}
|
|
202
|
+
/** Build the no-dispatch consolidation projection consumed by dry and live. */
|
|
203
|
+
function planConsolidationPass(args) {
|
|
204
|
+
const { options, primaryStashDir, memorySummary, improveProfile, resolvedPlan, eventsCtx } = args;
|
|
205
|
+
const processConfig = improveProfile?.processes?.consolidate;
|
|
206
|
+
const eligibility = evaluateConsolidationEligibility({
|
|
207
|
+
options,
|
|
208
|
+
primaryStashDir,
|
|
209
|
+
memorySummary,
|
|
210
|
+
improveProfile,
|
|
211
|
+
resolvedPlan,
|
|
212
|
+
eventsCtx,
|
|
213
|
+
});
|
|
214
|
+
const effectiveOptions = {
|
|
215
|
+
...options.consolidateOptions,
|
|
216
|
+
config: options.config,
|
|
217
|
+
stashDir: options.stashDir,
|
|
218
|
+
writeTarget: options.writeTarget,
|
|
219
|
+
limit: processConfig?.limit,
|
|
220
|
+
incrementalSince: processConfig?.incrementalSince,
|
|
221
|
+
neighborsPerChanged: processConfig?.neighborsPerChanged,
|
|
222
|
+
maxChunkSize: processConfig?.maxChunkSize,
|
|
223
|
+
};
|
|
224
|
+
const poolWarnings = [];
|
|
225
|
+
const pool = primaryStashDir
|
|
226
|
+
? inspectConsolidationPool(effectiveOptions, primaryStashDir, poolWarnings, {
|
|
227
|
+
readOnly: eventsCtx?.readOnly === true,
|
|
228
|
+
})
|
|
229
|
+
: { poolSize: 0, candidatePoolSize: 0, dedupPoolSize: 0, memories: [] };
|
|
230
|
+
const chunkSize = computeSafeChunkSize(resolvedPlan.processes.consolidate.runner?.connection.contextLength ?? DEFAULT_CONTEXT_LENGTH_TOKENS, 500, processConfig?.maxChunkSize);
|
|
231
|
+
const profilePassed = !eligibility.consolidateDisabledByProfile;
|
|
232
|
+
const minimumPoolPassed = !eligibility.poolBelowMinSize;
|
|
233
|
+
const deltaPassed = !eligibility.consolidationOnCooldown;
|
|
234
|
+
const nonEmptyPool = pool.candidatePoolSize > 0;
|
|
235
|
+
const wouldRun = profilePassed && minimumPoolPassed && deltaPassed && nonEmptyPool;
|
|
236
|
+
const reason = !profilePassed
|
|
237
|
+
? "disabled by improve profile"
|
|
238
|
+
: !minimumPoolPassed
|
|
239
|
+
? `pool ${eligibility.eligiblePoolSize} is below minPoolSize ${eligibility.minPoolSize}`
|
|
240
|
+
: !deltaPassed
|
|
241
|
+
? "no memory updates since the last completed consolidation"
|
|
242
|
+
: !nonEmptyPool
|
|
243
|
+
? "candidate pool is empty after narrowing"
|
|
244
|
+
: "all consolidation gates pass";
|
|
245
|
+
return {
|
|
246
|
+
eligibility,
|
|
247
|
+
plan: {
|
|
248
|
+
configured: {
|
|
249
|
+
...(processConfig?.enabled !== undefined ? { enabled: processConfig.enabled } : {}),
|
|
250
|
+
...(processConfig?.minPoolSize !== undefined ? { minPoolSize: processConfig.minPoolSize } : {}),
|
|
251
|
+
...(processConfig?.limit !== undefined ? { limit: processConfig.limit } : {}),
|
|
252
|
+
...(processConfig?.maxChunkSize !== undefined ? { maxChunkSize: processConfig.maxChunkSize } : {}),
|
|
253
|
+
...(processConfig?.incrementalSince !== undefined ? { incrementalSince: processConfig.incrementalSince } : {}),
|
|
254
|
+
},
|
|
255
|
+
effective: {
|
|
256
|
+
enabled: profilePassed,
|
|
257
|
+
minPoolSize: eligibility.minPoolSize,
|
|
258
|
+
...(processConfig?.limit !== undefined ? { limit: processConfig.limit } : {}),
|
|
259
|
+
chunkSize,
|
|
260
|
+
},
|
|
261
|
+
poolSize: pool.poolSize,
|
|
262
|
+
candidatePoolSize: pool.candidatePoolSize,
|
|
263
|
+
gates: {
|
|
264
|
+
profile: {
|
|
265
|
+
passed: profilePassed,
|
|
266
|
+
reason: profilePassed ? "consolidation enabled" : "disabled by improve profile",
|
|
267
|
+
},
|
|
268
|
+
minimumPool: {
|
|
269
|
+
passed: minimumPoolPassed,
|
|
270
|
+
reason: minimumPoolPassed
|
|
271
|
+
? `pool satisfies minPoolSize ${eligibility.minPoolSize}`
|
|
272
|
+
: `pool ${eligibility.eligiblePoolSize} is below minPoolSize ${eligibility.minPoolSize}`,
|
|
273
|
+
},
|
|
274
|
+
delta: {
|
|
275
|
+
passed: deltaPassed,
|
|
276
|
+
reason: deltaPassed ? "memory pool has work" : "no updates since the last completed consolidation",
|
|
277
|
+
},
|
|
278
|
+
},
|
|
279
|
+
wouldRun,
|
|
280
|
+
reason,
|
|
281
|
+
estimatedChunks: wouldRun ? Math.ceil(pool.candidatePoolSize / chunkSize) : 0,
|
|
282
|
+
},
|
|
283
|
+
};
|
|
284
|
+
}
|
|
201
285
|
export async function runConsolidationPass(args) {
|
|
202
286
|
const { options, primaryStashDir, memorySummary, improveProfile, resolvedPlan, eventsCtx, budgetSignal, runBudgetMs, } = args;
|
|
203
287
|
const baseConfig = options.config ?? loadConfig();
|
|
204
288
|
const consolidationConfig = baseConfig;
|
|
205
|
-
const
|
|
289
|
+
const planned = planConsolidationPass({
|
|
290
|
+
options,
|
|
291
|
+
primaryStashDir,
|
|
292
|
+
memorySummary,
|
|
293
|
+
improveProfile,
|
|
294
|
+
resolvedPlan,
|
|
295
|
+
eventsCtx,
|
|
296
|
+
});
|
|
297
|
+
const { volumeTriggered, consolidationOnCooldown, consolidateDisabledByProfile, poolBelowMinSize, eligiblePoolSize, minPoolSize, lastConsolidationTs, } = planned.eligibility;
|
|
206
298
|
let consolidation = {
|
|
207
299
|
schemaVersion: 1,
|
|
208
300
|
ok: true,
|
|
@@ -246,9 +338,7 @@ export async function runConsolidationPass(args) {
|
|
|
246
338
|
// Active profile for this improve run — lets consolidate's secondary
|
|
247
339
|
// process-config reads honor `--profile <name>` instead of `default`.
|
|
248
340
|
improveProfile,
|
|
249
|
-
|
|
250
|
-
? materializeLlmRunnerConnection(resolvedPlan.processes.consolidate.runner)
|
|
251
|
-
: null,
|
|
341
|
+
llmRunner: resolvedPlan.processes.consolidate.runner,
|
|
252
342
|
autoTriggered: volumeTriggered,
|
|
253
343
|
// Tie consolidate proposals back to this improve invocation so
|
|
254
344
|
// accept-rate-per-run aggregation works. Mirrors reflect/propose/extract.
|
|
@@ -258,6 +348,7 @@ export async function runConsolidationPass(args) {
|
|
|
258
348
|
// passes (quick-shredder). Leave absent in the nightly default profile for
|
|
259
349
|
// a full-pool sweep that catches stale-but-unmerged duplicates.
|
|
260
350
|
limit: improveProfile?.processes?.consolidate?.limit,
|
|
351
|
+
incrementalSince: improveProfile?.processes?.consolidate?.incrementalSince,
|
|
261
352
|
neighborsPerChanged: improveProfile?.processes?.consolidate?.neighborsPerChanged,
|
|
262
353
|
maxChunkSize: improveProfile?.processes?.consolidate?.maxChunkSize,
|
|
263
354
|
// WS-3a: forward budget signal for graceful abort on timeout, and pass
|
|
@@ -312,7 +403,55 @@ export async function runConsolidationPass(args) {
|
|
|
312
403
|
!consolidationOnCooldown &&
|
|
313
404
|
!consolidation.previewOnly &&
|
|
314
405
|
consolidation.processed > 0;
|
|
315
|
-
return { consolidation, consolidationRan };
|
|
406
|
+
return { consolidation, consolidationRan, plan: planned.plan };
|
|
407
|
+
}
|
|
408
|
+
/**
|
|
409
|
+
* Evaluate the exact pre-dispatch extract gates. Live execution consumes this
|
|
410
|
+
* snapshot and dry-run only reports it, so neither path reconstructs the
|
|
411
|
+
* selector independently.
|
|
412
|
+
*/
|
|
413
|
+
function inspectExtractPass(args) {
|
|
414
|
+
const { options, improveProfile, resolvedPlan, eventsCtx, readOnly } = args;
|
|
415
|
+
const enabled = resolvedPlan.processes.extract.enabled;
|
|
416
|
+
const hasRunner = resolvedPlan.processes.extract.runner?.engine !== undefined;
|
|
417
|
+
const availableHarnesses = (options.extractHarnesses ?? getAvailableHarnesses()).filter((harness) => harness.isAvailable());
|
|
418
|
+
const configuredMinNewSessions = improveProfile.processes?.extract?.minNewSessions;
|
|
419
|
+
const minNewSessions = typeof configuredMinNewSessions === "number" ? configuredMinNewSessions : 0;
|
|
420
|
+
let newCandidateCount;
|
|
421
|
+
if (enabled && hasRunner && availableHarnesses.length > 0 && minNewSessions > 0) {
|
|
422
|
+
const countFn = options.extractCandidateCountFn ?? countNewExtractCandidates;
|
|
423
|
+
newCandidateCount = countFn(options.config ?? loadConfig(), {
|
|
424
|
+
harnesses: availableHarnesses,
|
|
425
|
+
improveProfile,
|
|
426
|
+
...(improveProfile.processes?.extract?.defaultSince
|
|
427
|
+
? { since: improveProfile.processes.extract.defaultSince }
|
|
428
|
+
: {}),
|
|
429
|
+
...(eventsCtx?.db ? { stateDb: eventsCtx.db } : {}),
|
|
430
|
+
...(!readOnly && eventsCtx?.dbPath ? { stateDbPath: eventsCtx.dbPath } : {}),
|
|
431
|
+
...(readOnly ? { readOnly: true } : {}),
|
|
432
|
+
});
|
|
433
|
+
}
|
|
434
|
+
const belowMinNewSessions = minNewSessions > 0 && newCandidateCount !== undefined && newCandidateCount < minNewSessions;
|
|
435
|
+
const wouldRun = enabled && hasRunner && availableHarnesses.length > 0 && !belowMinNewSessions;
|
|
436
|
+
const reason = !enabled
|
|
437
|
+
? "disabled"
|
|
438
|
+
: !hasRunner
|
|
439
|
+
? "enabled but no runner is resolved"
|
|
440
|
+
: availableHarnesses.length === 0
|
|
441
|
+
? "enabled but no session-log harness is available"
|
|
442
|
+
: belowMinNewSessions
|
|
443
|
+
? `${newCandidateCount ?? 0} new sessions is below minNewSessions ${minNewSessions}`
|
|
444
|
+
: minNewSessions > 0
|
|
445
|
+
? `${newCandidateCount ?? 0} new sessions satisfies minNewSessions ${minNewSessions}`
|
|
446
|
+
: `enabled with ${availableHarnesses.length} available session-log harness(es); minNewSessions is disabled`;
|
|
447
|
+
return {
|
|
448
|
+
availableHarnesses,
|
|
449
|
+
minNewSessions,
|
|
450
|
+
...(newCandidateCount !== undefined ? { newCandidateCount } : {}),
|
|
451
|
+
belowMinNewSessions,
|
|
452
|
+
wouldRun,
|
|
453
|
+
reason,
|
|
454
|
+
};
|
|
316
455
|
}
|
|
317
456
|
/**
|
|
318
457
|
* Phase 0.4 — session-extract pass. Reads native session files through the
|
|
@@ -325,7 +464,7 @@ async function runSessionExtractPass(args) {
|
|
|
325
464
|
const warnings = [];
|
|
326
465
|
// Phase 0.4 — session-extract pass.
|
|
327
466
|
//
|
|
328
|
-
// Reads native session files (claude
|
|
467
|
+
// Reads native session files (claude JSONL, opencode storage tree)
|
|
329
468
|
// through the SessionLogHarness registry, pre-filters noise, and asks a
|
|
330
469
|
// bounded in-tree LLM to produce candidate memory/lesson/knowledge
|
|
331
470
|
// proposals for content the agent did NOT preserve via inline `akm remember`
|
|
@@ -354,12 +493,7 @@ async function runSessionExtractPass(args) {
|
|
|
354
493
|
// call so a skip costs zero LLM work AND writes nothing. A skipped extract
|
|
355
494
|
// never flags work for the NEXT run's consolidation mtime-gate (the
|
|
356
495
|
// downstream trigger #554 asks us to suppress).
|
|
357
|
-
const
|
|
358
|
-
// Read from the ACTIVE resolved profile (not always `default`), matching how
|
|
359
|
-
// `extract.enabled` resolves — otherwise a non-default profile (e.g.
|
|
360
|
-
// `frequent`) setting `minNewSessions` was silently ignored.
|
|
361
|
-
const configuredMinNewSessions = improveProfile.processes?.extract?.minNewSessions;
|
|
362
|
-
const minNewSessions = typeof configuredMinNewSessions === "number" ? configuredMinNewSessions : EXTRACT_DEFAULT_MIN_NEW_SESSIONS;
|
|
496
|
+
const plan = args.plan ?? inspectExtractPass({ options, improveProfile, resolvedPlan, eventsCtx, readOnly: false });
|
|
363
497
|
// #593/#594: the ACTIVE resolved improve profile is the single source of
|
|
364
498
|
// truth for whether extract runs. (Previously this also ANDed in the legacy
|
|
365
499
|
// `session_extraction` feature flag, which only reads
|
|
@@ -378,40 +512,24 @@ async function runSessionExtractPass(args) {
|
|
|
378
512
|
runner: extractRunner,
|
|
379
513
|
timeoutMs: extractRunner.timeoutMs === undefined ? 600_000 : extractRunner.timeoutMs,
|
|
380
514
|
embeddingConfig: Object.freeze(structuredClone(extractConfig.embedding)),
|
|
515
|
+
...(resolvedPlan.processes.extract.notices?.length ? { notices: resolvedPlan.processes.extract.notices } : {}),
|
|
381
516
|
});
|
|
382
|
-
const availableHarnesses =
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
...(eventsCtx?.dbPath ? { stateDbPath: eventsCtx.dbPath } : {}),
|
|
397
|
-
});
|
|
398
|
-
if (newCandidateCount < minNewSessions) {
|
|
399
|
-
belowMinNewSessions = true;
|
|
400
|
-
// Reuse the #551/#553 `improve_skipped` emission path so health's dynamic
|
|
401
|
-
// skipReasons aggregation surfaces this under `below_min_new_sessions`.
|
|
402
|
-
appendEvent({
|
|
403
|
-
eventType: "improve_skipped",
|
|
404
|
-
ref: "memories/_extract",
|
|
405
|
-
metadata: {
|
|
406
|
-
reason: "below_min_new_sessions",
|
|
407
|
-
newSessions: newCandidateCount,
|
|
408
|
-
minNewSessions,
|
|
409
|
-
},
|
|
410
|
-
}, eventsCtx);
|
|
411
|
-
info(`[improve] extract skipped (new sessions ${newCandidateCount} < minNewSessions ${minNewSessions})`);
|
|
412
|
-
}
|
|
517
|
+
const availableHarnesses = plan.availableHarnesses;
|
|
518
|
+
if (plan.belowMinNewSessions) {
|
|
519
|
+
// Reuse the #551/#553 `improve_skipped` emission path so health's dynamic
|
|
520
|
+
// skipReasons aggregation surfaces this under `below_min_new_sessions`.
|
|
521
|
+
appendEvent({
|
|
522
|
+
eventType: "improve_skipped",
|
|
523
|
+
ref: "memories/_extract",
|
|
524
|
+
metadata: {
|
|
525
|
+
reason: "below_min_new_sessions",
|
|
526
|
+
newSessions: plan.newCandidateCount ?? 0,
|
|
527
|
+
minNewSessions: plan.minNewSessions,
|
|
528
|
+
},
|
|
529
|
+
}, eventsCtx);
|
|
530
|
+
info(`[improve] extract skipped (new sessions ${plan.newCandidateCount ?? 0} < minNewSessions ${plan.minNewSessions})`);
|
|
413
531
|
}
|
|
414
|
-
if (
|
|
532
|
+
if (plan.wouldRun) {
|
|
415
533
|
extractResults = [];
|
|
416
534
|
for (const h of availableHarnesses) {
|
|
417
535
|
try {
|
|
@@ -489,14 +607,13 @@ export async function runValidationAndRepairPass(args) {
|
|
|
489
607
|
let schemaRepairs = [];
|
|
490
608
|
const repairedRefs = new Set();
|
|
491
609
|
// Schema repair pass: attempt to fix validation failures via LLM before skipping.
|
|
492
|
-
if (validationFailures.length > 0) {
|
|
610
|
+
if (repairValidationFailures && validationFailures.length > 0) {
|
|
493
611
|
const validationRunner = resolvedPlan.processes.validation.runner;
|
|
494
|
-
|
|
495
|
-
if (llmCfg) {
|
|
612
|
+
if (validationRunner) {
|
|
496
613
|
const result = await withLlmStage("validation", () => schemaRepairFn(validationFailures, {
|
|
497
614
|
startMs,
|
|
498
615
|
budgetMs,
|
|
499
|
-
|
|
616
|
+
llmRunner: validationRunner,
|
|
500
617
|
// #591/#379 regression: options.stashDir is the raw, unresolved CLI
|
|
501
618
|
// flag (only set when --stash-dir is passed explicitly — never true
|
|
502
619
|
// for the scheduled tasks). primaryStashDir is the already-resolved
|
|
@@ -526,78 +643,127 @@ export async function runValidationAndRepairPass(args) {
|
|
|
526
643
|
}
|
|
527
644
|
return { validationFailures, validationFailureRefs, schemaRepairs };
|
|
528
645
|
}
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
646
|
+
/**
|
|
647
|
+
* Resolve the preparation stages that precede candidate ranking. Keeping these
|
|
648
|
+
* lifecycle decisions in one named pass preserves the 220-line orchestrator
|
|
649
|
+
* ratchet while giving dry and live execution one implementation.
|
|
650
|
+
*/
|
|
651
|
+
async function runPreparationPrelude(args) {
|
|
652
|
+
const { scope, options, plannedRefs, memoryCleanupPlan, primaryStashDir, memorySummary, reindexFn, startMs, budgetMs, eventsCtx, improveProfile, resolvedPlan, strategyName, budgetSignal, planOnly, actions, cleanupWarnings, } = args;
|
|
533
653
|
const memoryBudget = assessMemoryIndexBudget(primaryStashDir);
|
|
534
|
-
const memoryIndexHealth = memoryBudget.memoryIndexHealth;
|
|
535
654
|
if (memoryBudget.warning)
|
|
536
655
|
cleanupWarnings.push(memoryBudget.warning);
|
|
537
|
-
//
|
|
538
|
-
//
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
656
|
+
// Consolidation intentionally precedes extract so current-run promotions
|
|
657
|
+
// cannot force the pool-delta gate open (#551).
|
|
658
|
+
const consolidationPass = planOnly
|
|
659
|
+
? (() => {
|
|
660
|
+
const planned = planConsolidationPass({
|
|
661
|
+
options,
|
|
662
|
+
primaryStashDir,
|
|
663
|
+
memorySummary,
|
|
664
|
+
improveProfile,
|
|
665
|
+
resolvedPlan,
|
|
666
|
+
eventsCtx,
|
|
667
|
+
});
|
|
668
|
+
return {
|
|
669
|
+
consolidation: {
|
|
670
|
+
schemaVersion: 1,
|
|
671
|
+
ok: true,
|
|
672
|
+
shape: "consolidate-result",
|
|
673
|
+
dryRun: true,
|
|
674
|
+
previewOnly: true,
|
|
675
|
+
target: options.target ?? options.stashDir ?? "",
|
|
676
|
+
processed: 0,
|
|
677
|
+
merged: 0,
|
|
678
|
+
deleted: 0,
|
|
679
|
+
promoted: [],
|
|
680
|
+
contradicted: 0,
|
|
681
|
+
warnings: [],
|
|
682
|
+
durationMs: 0,
|
|
683
|
+
},
|
|
684
|
+
consolidationRan: false,
|
|
685
|
+
plan: planned.plan,
|
|
686
|
+
};
|
|
687
|
+
})()
|
|
688
|
+
: await runConsolidationPass({
|
|
689
|
+
options,
|
|
690
|
+
primaryStashDir,
|
|
691
|
+
memorySummary,
|
|
692
|
+
improveProfile,
|
|
693
|
+
resolvedPlan,
|
|
694
|
+
eventsCtx,
|
|
695
|
+
budgetSignal,
|
|
696
|
+
runBudgetMs: budgetMs,
|
|
697
|
+
});
|
|
698
|
+
const extractPlan = inspectExtractPass({ options, improveProfile, resolvedPlan, eventsCtx, readOnly: planOnly });
|
|
699
|
+
const extractPass = planOnly
|
|
700
|
+
? { extractResults: undefined, warnings: [] }
|
|
701
|
+
: await runSessionExtractPass({
|
|
702
|
+
options,
|
|
703
|
+
primaryStashDir,
|
|
704
|
+
improveProfile,
|
|
705
|
+
resolvedPlan,
|
|
706
|
+
eventsCtx,
|
|
707
|
+
budgetSignal,
|
|
708
|
+
plan: extractPlan,
|
|
709
|
+
});
|
|
567
710
|
if (extractPass.warnings.length > 0)
|
|
568
711
|
cleanupWarnings.push(...extractPass.warnings);
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
}
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
712
|
+
if (!planOnly) {
|
|
713
|
+
appendEvent({
|
|
714
|
+
eventType: "improve_invoked",
|
|
715
|
+
ref: scope.mode === "ref" ? scope.value : `improve:${scope.mode}:${scope.value ?? "all"}`,
|
|
716
|
+
metadata: { strategy: strategyName, scope, dryRun: options.dryRun ?? false, eligibleCount: plannedRefs.length },
|
|
717
|
+
}, eventsCtx);
|
|
718
|
+
}
|
|
719
|
+
const allowCleanupApply = isAutonomyLaneAllowed("memoryCleanup", options.config ?? loadConfig());
|
|
720
|
+
const cleanup = planOnly
|
|
721
|
+
? {
|
|
722
|
+
...projectMemoryCleanup({
|
|
723
|
+
mode: "estimate",
|
|
724
|
+
plannedRefs,
|
|
725
|
+
candidateRefs: memoryCleanupPlan?.pruneCandidates.map((candidate) => candidate.ref) ?? [],
|
|
726
|
+
allowApply: allowCleanupApply,
|
|
727
|
+
}),
|
|
728
|
+
pruneActions: [],
|
|
729
|
+
warnings: [],
|
|
730
|
+
appliedCleanup: undefined,
|
|
731
|
+
}
|
|
732
|
+
: await applyCleanupPass({
|
|
733
|
+
primaryStashDir,
|
|
734
|
+
memoryCleanupPlan,
|
|
735
|
+
plannedRefs,
|
|
736
|
+
reindexFn,
|
|
737
|
+
budgetSignal,
|
|
738
|
+
allowApply: allowCleanupApply,
|
|
739
|
+
});
|
|
590
740
|
actions.push(...cleanup.pruneActions);
|
|
591
741
|
cleanupWarnings.push(...cleanup.warnings);
|
|
592
|
-
const
|
|
593
|
-
postCleanupRefs,
|
|
742
|
+
const validation = await runValidationAndRepairPass({
|
|
743
|
+
postCleanupRefs: cleanup.postCleanupRefs,
|
|
594
744
|
options,
|
|
595
745
|
startMs,
|
|
596
746
|
budgetMs,
|
|
597
747
|
primaryStashDir,
|
|
598
748
|
resolvedPlan,
|
|
599
|
-
repairValidationFailures: resolvedPlan.processes.validation.enabled && options.repairValidationFailures !== false,
|
|
749
|
+
repairValidationFailures: !planOnly && resolvedPlan.processes.validation.enabled && options.repairValidationFailures !== false,
|
|
600
750
|
});
|
|
751
|
+
return {
|
|
752
|
+
memoryIndexHealth: memoryBudget.memoryIndexHealth,
|
|
753
|
+
consolidationPass,
|
|
754
|
+
extractPlan,
|
|
755
|
+
extractResults: extractPass.extractResults,
|
|
756
|
+
appliedCleanup: cleanup.appliedCleanup,
|
|
757
|
+
postCleanupRefs: cleanup.postCleanupRefs,
|
|
758
|
+
cleanupGate: cleanup.gate,
|
|
759
|
+
...validation,
|
|
760
|
+
};
|
|
761
|
+
}
|
|
762
|
+
export async function runImprovePreparationStage(args) {
|
|
763
|
+
const { scope, options, plannedRefs, primaryStashDir, eventsCtx, initialCleanupWarnings, improveProfile, resolvedPlan, planOnly = options.dryRun === true, } = args;
|
|
764
|
+
const actions = [];
|
|
765
|
+
const cleanupWarnings = initialCleanupWarnings ? [...initialCleanupWarnings] : [];
|
|
766
|
+
const { memoryIndexHealth, consolidationPass, extractPlan, extractResults, appliedCleanup, postCleanupRefs, cleanupGate, validationFailures, validationFailureRefs, schemaRepairs, } = await runPreparationPrelude({ ...args, planOnly, actions, cleanupWarnings });
|
|
601
767
|
// Phase 0.5 — structural hygiene pass
|
|
602
768
|
let lintSummary;
|
|
603
769
|
if (primaryStashDir) {
|
|
@@ -610,7 +776,7 @@ export async function runImprovePreparationStage(args) {
|
|
|
610
776
|
}
|
|
611
777
|
}
|
|
612
778
|
const recentErrors = seedRecentErrorWindows(schemaRepairs);
|
|
613
|
-
const snapshot = buildSnapshotManifest({ postCleanupRefs, validationFailureRefs });
|
|
779
|
+
const snapshot = buildSnapshotManifest({ postCleanupRefs, validationFailureRefs, eventsCtx });
|
|
614
780
|
const gathered = gatherCandidates({
|
|
615
781
|
scope,
|
|
616
782
|
options,
|
|
@@ -621,8 +787,13 @@ export async function runImprovePreparationStage(args) {
|
|
|
621
787
|
postCleanupRefs,
|
|
622
788
|
validationFailureRefs,
|
|
623
789
|
snapshot,
|
|
790
|
+
persist: !planOnly,
|
|
624
791
|
});
|
|
625
|
-
|
|
792
|
+
// Shared admission boundary for every synthetic fallback lane. Cleanup and
|
|
793
|
+
// structural validation are exclusive selectors: no later rank/replay state
|
|
794
|
+
// may re-create a candidate they removed. Keep the exact surviving objects
|
|
795
|
+
// so any admitted fallback preserves its index-resolved file/item provenance.
|
|
796
|
+
const fallbackEligibleRefs = postCleanupRefs.filter((candidate) => !validationFailureRefs.has(candidate.ref));
|
|
626
797
|
const scored = scoreSalience({
|
|
627
798
|
scope,
|
|
628
799
|
options,
|
|
@@ -635,11 +806,17 @@ export async function runImprovePreparationStage(args) {
|
|
|
635
806
|
signalFiltered: gathered.signalFiltered,
|
|
636
807
|
proactiveRefs: gathered.proactiveRefs,
|
|
637
808
|
highSalienceRefs: gathered.highSalienceRefs,
|
|
809
|
+
forgettingEligibleRefs: fallbackEligibleRefs,
|
|
810
|
+
persist: !planOnly,
|
|
638
811
|
});
|
|
812
|
+
// Replay is additive to the signal lanes, but it must not bypass selectors
|
|
813
|
+
// that have already removed a ref. Use the exact surviving objects so a
|
|
814
|
+
// replay admission preserves the index-resolved file/item provenance while
|
|
815
|
+
// excluding cleanup-pruned and structurally-invalid candidates.
|
|
639
816
|
const filtered = await filterEligibility({
|
|
640
817
|
scope,
|
|
641
818
|
options,
|
|
642
|
-
|
|
819
|
+
replayEligibleRefs: fallbackEligibleRefs,
|
|
643
820
|
eventsCtx,
|
|
644
821
|
mergedRefs: scored.mergedRefs,
|
|
645
822
|
salienceMap: scored.salienceMap,
|
|
@@ -647,12 +824,56 @@ export async function runImprovePreparationStage(args) {
|
|
|
647
824
|
distillOnlyRefs: gathered.distillOnlyRefs,
|
|
648
825
|
validationFailureRefs,
|
|
649
826
|
summary: {
|
|
650
|
-
fullySkippedCount: gathered.fullySkippedCount,
|
|
651
|
-
preCooldownCount: gathered.preCooldownCount,
|
|
652
827
|
signalAndRetrievalRefs: gathered.signalAndRetrievalRefs,
|
|
653
828
|
signalFiltered: gathered.signalFiltered,
|
|
654
829
|
},
|
|
830
|
+
persist: !planOnly,
|
|
831
|
+
});
|
|
832
|
+
const preDiskRefSet = new Set(filtered.preDiskRefs.map((candidate) => candidate.ref));
|
|
833
|
+
const terminalSignalSkippedRefs = fallbackEligibleRefs.filter((candidate) => !preDiskRefSet.has(candidate.ref));
|
|
834
|
+
recordSignalSkipObservability({
|
|
835
|
+
actions,
|
|
836
|
+
terminalSignalSkippedRefs,
|
|
837
|
+
distillCooledRefs: gathered.distillCooledRefs,
|
|
838
|
+
eventsCtx,
|
|
839
|
+
persist: !planOnly,
|
|
655
840
|
});
|
|
841
|
+
// Gate counts are an exclusive, sequential accounting of the raw pool.
|
|
842
|
+
// Replay, proactive maintenance, high-salience, and forgetting-safety are
|
|
843
|
+
// legitimate signal-gate fallback lanes, so a ref admitted by any of them
|
|
844
|
+
// was not removed by the signal gate. Derive this count from the actual
|
|
845
|
+
// pre-disk survivor set instead of the earlier lane-rescue snapshot; the
|
|
846
|
+
// latter is intentionally assembled before replay and is also broader than
|
|
847
|
+
// the effective pool when --require-feedback-signal suppresses fallbacks.
|
|
848
|
+
const signalRemoved = terminalSignalSkippedRefs.length;
|
|
849
|
+
const totalReflectBlocked = terminalSignalSkippedRefs.length + gathered.distillOnlyRefs.length;
|
|
850
|
+
if (totalReflectBlocked > 0) {
|
|
851
|
+
info(`[improve] ${totalReflectBlocked} of ${gathered.preCooldownCount} indexed refs blocked by reflect signal-delta ` +
|
|
852
|
+
`(${terminalSignalSkippedRefs.length} fully skipped, ${gathered.distillOnlyRefs.length} routed to distill-only)`);
|
|
853
|
+
}
|
|
854
|
+
const planningGates = [
|
|
855
|
+
cleanupGate,
|
|
856
|
+
{
|
|
857
|
+
name: "validation",
|
|
858
|
+
removed: validationFailureRefs.size,
|
|
859
|
+
reason: "structural validation failures",
|
|
860
|
+
},
|
|
861
|
+
{
|
|
862
|
+
name: "signal",
|
|
863
|
+
removed: signalRemoved,
|
|
864
|
+
reason: "no fresh signal and no fallback lane selected the ref",
|
|
865
|
+
},
|
|
866
|
+
{
|
|
867
|
+
name: "disk",
|
|
868
|
+
removed: filtered.missingDiskCount,
|
|
869
|
+
reason: "backing asset is absent on disk",
|
|
870
|
+
},
|
|
871
|
+
{
|
|
872
|
+
name: "limit",
|
|
873
|
+
removed: filtered.limitRemoved,
|
|
874
|
+
reason: "deferred by the effective run limit",
|
|
875
|
+
},
|
|
876
|
+
];
|
|
656
877
|
return {
|
|
657
878
|
actions,
|
|
658
879
|
cleanupWarnings,
|
|
@@ -673,6 +894,13 @@ export async function runImprovePreparationStage(args) {
|
|
|
673
894
|
consolidation: consolidationPass.consolidation,
|
|
674
895
|
consolidationRan: consolidationPass.consolidationRan,
|
|
675
896
|
...(gathered.proactiveMaintenanceSummary ? { proactiveMaintenance: gathered.proactiveMaintenanceSummary } : {}),
|
|
897
|
+
planning: {
|
|
898
|
+
gates: planningGates,
|
|
899
|
+
replayBudget: filtered.replayBudget,
|
|
900
|
+
...(gathered.proactivePlan ? { proactive: gathered.proactivePlan } : {}),
|
|
901
|
+
consolidation: consolidationPass.plan,
|
|
902
|
+
extract: { wouldRun: extractPlan.wouldRun, reason: extractPlan.reason },
|
|
903
|
+
},
|
|
676
904
|
};
|
|
677
905
|
}
|
|
678
906
|
// ── preparation-stage passes (WI-7.6 decomposition, R31) ────────────────────
|
|
@@ -733,9 +961,12 @@ async function applyCleanupPass(args) {
|
|
|
733
961
|
catch (err) {
|
|
734
962
|
warnings.push(`applyMemoryCleanup failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
735
963
|
}
|
|
736
|
-
const
|
|
737
|
-
|
|
738
|
-
|
|
964
|
+
const projection = projectMemoryCleanup({
|
|
965
|
+
mode: "execution",
|
|
966
|
+
plannedRefs,
|
|
967
|
+
archivedRefs: appliedCleanup?.archived.map((record) => record.ref) ?? [],
|
|
968
|
+
allowApply,
|
|
969
|
+
});
|
|
739
970
|
// ── Phase 1: validation pass + schema repair (run on full postCleanupRefs) ──
|
|
740
971
|
// Identifies refs whose on-disk asset has structural problems. Validation
|
|
741
972
|
// failures are excluded from every downstream bucket. Run early so the
|
|
@@ -760,7 +991,7 @@ async function applyCleanupPass(args) {
|
|
|
760
991
|
}
|
|
761
992
|
}
|
|
762
993
|
}
|
|
763
|
-
return { appliedCleanup,
|
|
994
|
+
return { appliedCleanup, ...projection, pruneActions, warnings };
|
|
764
995
|
}
|
|
765
996
|
/** Seed the per-originator rolling error windows from schema-repair errors. */
|
|
766
997
|
function seedRecentErrorWindows(schemaRepairs) {
|
|
@@ -790,7 +1021,7 @@ function seedRecentErrorWindows(schemaRepairs) {
|
|
|
790
1021
|
}
|
|
791
1022
|
/** Pass: snapshot-manifest — the three timestamp maps + the 30-day signal window. */
|
|
792
1023
|
export function buildSnapshotManifest(args) {
|
|
793
|
-
const { postCleanupRefs, validationFailureRefs } = args;
|
|
1024
|
+
const { postCleanupRefs, validationFailureRefs, eventsCtx } = args;
|
|
794
1025
|
// ── Phase 2: signal-delta eligibility sets built EARLY ────────────────────
|
|
795
1026
|
// 0.8.0 replaces the flat time-based cooldowns (which produced synchronised
|
|
796
1027
|
// waves whenever many refs cooled at the same instant — see the 2026-05-26
|
|
@@ -815,9 +1046,9 @@ export function buildSnapshotManifest(args) {
|
|
|
815
1046
|
const candidateRefs = postCleanupRefs.filter((r) => !validationFailureRefs.has(r.ref)).map((r) => r.ref);
|
|
816
1047
|
// Carry each candidate's item_ref into the feedback/proposal timestamp reads.
|
|
817
1048
|
const itemRefByRef = buildItemRefByRef(postCleanupRefs);
|
|
818
|
-
const latestFeedbackTs = buildLatestFeedbackTsMap(candidateRefs, feedbackSinceCutoff, itemRefByRef);
|
|
819
|
-
const lastReflectProposalTs = buildLatestProposalTsMap(candidateRefs, "reflect", itemRefByRef);
|
|
820
|
-
const lastDistillProposalTs = buildLatestProposalTsMap(candidateRefs, "distill", itemRefByRef);
|
|
1049
|
+
const latestFeedbackTs = buildLatestFeedbackTsMap(candidateRefs, feedbackSinceCutoff, itemRefByRef, eventsCtx);
|
|
1050
|
+
const lastReflectProposalTs = buildLatestProposalTsMap(candidateRefs, "reflect", itemRefByRef, eventsCtx);
|
|
1051
|
+
const lastDistillProposalTs = buildLatestProposalTsMap(candidateRefs, "distill", itemRefByRef, eventsCtx);
|
|
821
1052
|
return { feedbackSinceCutoff, latestFeedbackTs, lastReflectProposalTs, lastDistillProposalTs };
|
|
822
1053
|
}
|
|
823
1054
|
/**
|
|
@@ -826,25 +1057,24 @@ export function buildSnapshotManifest(args) {
|
|
|
826
1057
|
* rescue lanes, the merged candidate set, and lane attribution stamping.
|
|
827
1058
|
*/
|
|
828
1059
|
function gatherCandidates(args) {
|
|
829
|
-
const { scope, options, primaryStashDir, eventsCtx, improveProfile, resolvedPlan, postCleanupRefs } = args;
|
|
1060
|
+
const { scope, options, primaryStashDir, eventsCtx, improveProfile, resolvedPlan, postCleanupRefs, persist } = args;
|
|
830
1061
|
const { feedbackSinceCutoff, lastReflectProposalTs, lastDistillProposalTs } = args.snapshot;
|
|
831
1062
|
const partition = partitionBySignalDelta({
|
|
832
1063
|
scope,
|
|
833
1064
|
options,
|
|
834
|
-
eventsCtx,
|
|
835
1065
|
postCleanupRefs,
|
|
836
1066
|
validationFailureRefs: args.validationFailureRefs,
|
|
837
1067
|
snapshot: args.snapshot,
|
|
838
1068
|
});
|
|
839
|
-
const
|
|
840
|
-
const { distillCooledRefs, preCooldownCount, eligibleRefs, distillOnlyRefs, noFeedbackPool, fullySkippedCount } = partition;
|
|
1069
|
+
const { distillCooledRefs, preCooldownCount, eligibleRefs, distillOnlyRefs, noFeedbackPool } = partition;
|
|
841
1070
|
// ── Phase 4: signal/feedback/utility/sort on the reduced set ──────────────
|
|
842
1071
|
// Everything from here works on (eligibleRefs ∪ distillOnlyRefs) plus the
|
|
843
1072
|
// deferred noFeedbackPool that may be rescued by the proactive-maintenance
|
|
844
1073
|
// (Layer 2) or high-salience (Layer 3) fallbacks below. The fully-skipped
|
|
845
|
-
// bucket
|
|
846
|
-
//
|
|
847
|
-
// rejected with feedback
|
|
1074
|
+
// bucket is retained as partition metadata only; terminal skip observability
|
|
1075
|
+
// is delayed until every fallback lane has finalized. We deliberately avoid
|
|
1076
|
+
// spending DB/CPU on refs that the signal-delta gate rejected with feedback
|
|
1077
|
+
// already on record.
|
|
848
1078
|
const processableRefs = [...eligibleRefs, ...distillOnlyRefs];
|
|
849
1079
|
const feedbackSummary = buildFeedbackSummaryMap({
|
|
850
1080
|
processableRefs,
|
|
@@ -871,46 +1101,40 @@ function gatherCandidates(args) {
|
|
|
871
1101
|
primaryStashDir,
|
|
872
1102
|
signalFiltered,
|
|
873
1103
|
noFeedbackCandidates,
|
|
874
|
-
});
|
|
875
|
-
const proactive = selectProactiveMaintenanceLane({
|
|
876
|
-
scope,
|
|
877
|
-
improveProfile,
|
|
878
|
-
resolvedPlan,
|
|
879
1104
|
eventsCtx,
|
|
880
|
-
|
|
881
|
-
lastReflectProposalTs,
|
|
882
|
-
lastDistillProposalTs,
|
|
883
|
-
retrievalCounts,
|
|
884
|
-
lastUseMsForProactive,
|
|
1105
|
+
persist,
|
|
885
1106
|
});
|
|
1107
|
+
// `--require-feedback-signal` is a hard policy boundary, not merely a final
|
|
1108
|
+
// list filter. Do not run or report fallback selectors that the invocation
|
|
1109
|
+
// explicitly disabled (and do not emit their live selection events).
|
|
1110
|
+
const allowFallbacks = options.requireFeedbackSignal !== true;
|
|
1111
|
+
const proactive = allowFallbacks
|
|
1112
|
+
? selectProactiveMaintenanceLane({
|
|
1113
|
+
scope,
|
|
1114
|
+
improveProfile,
|
|
1115
|
+
resolvedPlan,
|
|
1116
|
+
eventsCtx,
|
|
1117
|
+
noFeedbackCandidates,
|
|
1118
|
+
lastReflectProposalTs,
|
|
1119
|
+
lastDistillProposalTs,
|
|
1120
|
+
retrievalCounts,
|
|
1121
|
+
lastUseMsForProactive,
|
|
1122
|
+
persist,
|
|
1123
|
+
})
|
|
1124
|
+
: { proactiveRefs: [] };
|
|
886
1125
|
const proactiveRefs = proactive.proactiveRefs;
|
|
887
1126
|
const proactiveMaintenanceSummary = proactive.proactiveMaintenanceSummary;
|
|
888
|
-
const highSalienceRefs =
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
// already on record). These never make it into mergedRefs, so without this
|
|
900
|
-
// they would silently vanish from the run summary. No DB event is written
|
|
901
|
-
// here — these refs carry no signal at all, so there is nothing for the skip
|
|
902
|
-
// histogram to aggregate; the action log alone preserves the per-ref audit
|
|
903
|
-
// trail (mirrors the fully-skipped action above).
|
|
904
|
-
const rescuedSet = new Set([...proactiveRefs, ...highSalienceRefs].map((r) => r.ref));
|
|
905
|
-
for (const r of noFeedbackPool) {
|
|
906
|
-
if (rescuedSet.has(r.ref))
|
|
907
|
-
continue;
|
|
908
|
-
actions.push({
|
|
909
|
-
ref: r.ref,
|
|
910
|
-
mode: "distill-skipped",
|
|
911
|
-
result: { ok: true, reason: "no new signal since last proposal" },
|
|
912
|
-
});
|
|
913
|
-
}
|
|
1127
|
+
const highSalienceRefs = allowFallbacks
|
|
1128
|
+
? selectHighSalienceLane({
|
|
1129
|
+
options,
|
|
1130
|
+
improveProfile,
|
|
1131
|
+
eventsCtx,
|
|
1132
|
+
noFeedbackCandidates,
|
|
1133
|
+
proactiveRefs,
|
|
1134
|
+
lastReflectProposalTs,
|
|
1135
|
+
persist,
|
|
1136
|
+
})
|
|
1137
|
+
: [];
|
|
914
1138
|
// If the user explicitly scoped to a single ref, always act on it —
|
|
915
1139
|
// skip the signal/retrieval filter entirely. The filter exists to avoid
|
|
916
1140
|
// noisy "improve everything" runs; it should not gate an intentional
|
|
@@ -965,17 +1189,16 @@ function gatherCandidates(args) {
|
|
|
965
1189
|
r.eligibilitySource = eligibilitySourceByRef.get(r.ref) ?? "unknown";
|
|
966
1190
|
}
|
|
967
1191
|
return {
|
|
968
|
-
actions,
|
|
969
1192
|
distillCooledRefs,
|
|
970
1193
|
preCooldownCount,
|
|
971
1194
|
distillOnlyRefs,
|
|
972
|
-
fullySkippedCount,
|
|
973
1195
|
feedbackSummary,
|
|
974
1196
|
signalFiltered,
|
|
975
1197
|
signalBearingSet,
|
|
976
1198
|
retrievalCounts,
|
|
977
1199
|
proactiveRefs,
|
|
978
1200
|
proactiveMaintenanceSummary,
|
|
1201
|
+
proactivePlan: proactive.proactivePlan,
|
|
979
1202
|
highSalienceRefs,
|
|
980
1203
|
signalAndRetrievalRefs,
|
|
981
1204
|
mergedRefs,
|
|
@@ -988,9 +1211,8 @@ function gatherCandidates(args) {
|
|
|
988
1211
|
* VERBATIM — see the phase-2/3 comments inside.
|
|
989
1212
|
*/
|
|
990
1213
|
export function partitionBySignalDelta(args) {
|
|
991
|
-
const { scope, options,
|
|
1214
|
+
const { scope, options, postCleanupRefs, validationFailureRefs } = args;
|
|
992
1215
|
const { latestFeedbackTs, lastReflectProposalTs, lastDistillProposalTs } = args.snapshot;
|
|
993
|
-
const actions = [];
|
|
994
1216
|
// Refs the distill signal-delta gate rejected at planning time. The main
|
|
995
1217
|
// loop reads this to skip distill for these refs without re-checking
|
|
996
1218
|
// eligibility per iteration.
|
|
@@ -1010,8 +1232,8 @@ export function partitionBySignalDelta(args) {
|
|
|
1010
1232
|
// below so never-rated assets can still be improved.
|
|
1011
1233
|
// Only refs those lanes decline are fully skipped.
|
|
1012
1234
|
// fullySkippedCount — has stale feedback but no signal delta → genuine
|
|
1013
|
-
// skip
|
|
1014
|
-
//
|
|
1235
|
+
// skip candidate, excluded from sort. Final skip
|
|
1236
|
+
// observability is emitted only after fallbacks.
|
|
1015
1237
|
const eligibleRefs = [];
|
|
1016
1238
|
const distillOnlyRefs = [];
|
|
1017
1239
|
// Zero-(recent-)feedback refs deferred to the proactive/high-salience fallbacks.
|
|
@@ -1031,16 +1253,10 @@ export function partitionBySignalDelta(args) {
|
|
|
1031
1253
|
const isDistillCandidate = isDistillCandidateRef(r.ref, options.stashDir);
|
|
1032
1254
|
if (reflectOk) {
|
|
1033
1255
|
if (!distillOk && isDistillCandidate) {
|
|
1034
|
-
// Reflect passes the gate, distill does not
|
|
1035
|
-
//
|
|
1036
|
-
//
|
|
1256
|
+
// Reflect passes the gate, distill does not. Record only partition
|
|
1257
|
+
// metadata here; observability is emitted after every fallback selector
|
|
1258
|
+
// has finalized the invocation's terminal skipped set.
|
|
1037
1259
|
distillCooledRefs.add(r.ref);
|
|
1038
|
-
actions.push({ ref: r.ref, mode: "distill-skipped", result: { ok: true, reason: "distill signal-delta" } });
|
|
1039
|
-
appendEvent({
|
|
1040
|
-
eventType: "improve_skipped",
|
|
1041
|
-
ref: r.ref,
|
|
1042
|
-
metadata: { reason: "distill_no_new_signal" },
|
|
1043
|
-
}, eventsCtx);
|
|
1044
1260
|
}
|
|
1045
1261
|
else if (!distillOk) {
|
|
1046
1262
|
// Not a distill candidate AND distill gate doesn't pass — just mark
|
|
@@ -1062,36 +1278,13 @@ export function partitionBySignalDelta(args) {
|
|
|
1062
1278
|
}
|
|
1063
1279
|
else {
|
|
1064
1280
|
// Has feedback on record but no signal delta since the last proposal —
|
|
1065
|
-
// genuinely fully
|
|
1066
|
-
//
|
|
1067
|
-
//
|
|
1281
|
+
// genuinely a fully-skipped candidate. Count it as partition metadata;
|
|
1282
|
+
// final observability waits until replay and every other fallback lane
|
|
1283
|
+
// has had a chance to rescue it.
|
|
1068
1284
|
fullySkippedCount++;
|
|
1069
|
-
actions.push({
|
|
1070
|
-
ref: r.ref,
|
|
1071
|
-
mode: "distill-skipped",
|
|
1072
|
-
result: { ok: true, reason: "no new signal since last proposal" },
|
|
1073
|
-
});
|
|
1074
1285
|
}
|
|
1075
1286
|
}
|
|
1076
|
-
// Emit ONE aggregated skip event for the fully-skipped bucket rather than one
|
|
1077
|
-
// improve_skipped event per ref (#592 pattern, mirrors
|
|
1078
|
-
// strategy_filtered_all_passes above). The per-ref loop previously produced
|
|
1079
|
-
// ~11K state.db writes per run on a large stash, the dominant contributor to
|
|
1080
|
-
// 900 s timeouts. The in-memory `actions` log keeps the per-ref detail for the
|
|
1081
|
-
// run summary; no downstream consumer needs a per-ref DB audit trail (health's
|
|
1082
|
-
// skip histogram reads the `no_new_signal` counter from the count field).
|
|
1083
|
-
if (fullySkippedCount > 0) {
|
|
1084
|
-
appendEvent({
|
|
1085
|
-
eventType: "improve_skipped",
|
|
1086
|
-
ref: undefined,
|
|
1087
|
-
metadata: {
|
|
1088
|
-
reason: "no_new_signal",
|
|
1089
|
-
count: fullySkippedCount,
|
|
1090
|
-
},
|
|
1091
|
-
}, eventsCtx);
|
|
1092
|
-
}
|
|
1093
1287
|
return {
|
|
1094
|
-
actions,
|
|
1095
1288
|
distillCooledRefs,
|
|
1096
1289
|
preCooldownCount,
|
|
1097
1290
|
eligibleRefs,
|
|
@@ -1100,6 +1293,44 @@ export function partitionBySignalDelta(args) {
|
|
|
1100
1293
|
fullySkippedCount,
|
|
1101
1294
|
};
|
|
1102
1295
|
}
|
|
1296
|
+
/**
|
|
1297
|
+
* Emit signal-delta skip observability only after every fallback lane has
|
|
1298
|
+
* finalized the pre-disk survivor set. This prevents replay, proactive,
|
|
1299
|
+
* high-salience, or forgetting-safety winners from also being recorded as
|
|
1300
|
+
* terminally skipped work.
|
|
1301
|
+
*/
|
|
1302
|
+
function recordSignalSkipObservability(args) {
|
|
1303
|
+
const { actions, terminalSignalSkippedRefs, distillCooledRefs, eventsCtx, persist } = args;
|
|
1304
|
+
for (const ref of distillCooledRefs) {
|
|
1305
|
+
actions.push({ ref, mode: "distill-skipped", result: { ok: true, reason: "distill signal-delta" } });
|
|
1306
|
+
if (persist) {
|
|
1307
|
+
appendEvent({
|
|
1308
|
+
eventType: "improve_skipped",
|
|
1309
|
+
ref,
|
|
1310
|
+
metadata: { reason: "distill_no_new_signal" },
|
|
1311
|
+
}, eventsCtx);
|
|
1312
|
+
}
|
|
1313
|
+
}
|
|
1314
|
+
for (const candidate of terminalSignalSkippedRefs) {
|
|
1315
|
+
actions.push({
|
|
1316
|
+
ref: candidate.ref,
|
|
1317
|
+
mode: "distill-skipped",
|
|
1318
|
+
result: { ok: true, reason: "no new signal since last proposal" },
|
|
1319
|
+
});
|
|
1320
|
+
}
|
|
1321
|
+
// One aggregate row preserves health accounting without restoring the old
|
|
1322
|
+
// O(n) event-write path. The count now exactly matches the signal gate.
|
|
1323
|
+
if (persist && terminalSignalSkippedRefs.length > 0) {
|
|
1324
|
+
appendEvent({
|
|
1325
|
+
eventType: "improve_skipped",
|
|
1326
|
+
ref: undefined,
|
|
1327
|
+
metadata: {
|
|
1328
|
+
reason: "no_new_signal",
|
|
1329
|
+
count: terminalSignalSkippedRefs.length,
|
|
1330
|
+
},
|
|
1331
|
+
}, eventsCtx);
|
|
1332
|
+
}
|
|
1333
|
+
}
|
|
1103
1334
|
/** Bulk per-ref feedback summary in a SINGLE readEvents pass (candidate-gather). */
|
|
1104
1335
|
function buildFeedbackSummaryMap(args) {
|
|
1105
1336
|
const { processableRefs, noFeedbackPool, eventsCtx, feedbackSinceCutoff } = args;
|
|
@@ -1163,7 +1394,7 @@ function buildFeedbackSummaryMap(args) {
|
|
|
1163
1394
|
}
|
|
1164
1395
|
/** Retrieval counts + last-use timestamps for the candidate pools (candidate-gather). */
|
|
1165
1396
|
function fetchRetrievalSignals(args) {
|
|
1166
|
-
const { options,
|
|
1397
|
+
const { options, signalFiltered, noFeedbackCandidates, eventsCtx, persist } = args;
|
|
1167
1398
|
// Retrieval counts for the zero-feedback pool, hoisted so the Layer-2
|
|
1168
1399
|
// proactive-maintenance selector below can reuse them without a second DB pass.
|
|
1169
1400
|
// Also fetch lastUseMs here for the proactive-maintenance recency term (plan §WS-1
|
|
@@ -1172,28 +1403,33 @@ function fetchRetrievalSignals(args) {
|
|
|
1172
1403
|
let lastUseMsForProactive = new Map();
|
|
1173
1404
|
let dbForRetrieval;
|
|
1174
1405
|
try {
|
|
1175
|
-
dbForRetrieval =
|
|
1406
|
+
dbForRetrieval = persist
|
|
1407
|
+
? openExistingDatabase()
|
|
1408
|
+
: openReadonlyExistingDatabase(undefined, { isolatedSnapshot: true });
|
|
1409
|
+
if (!dbForRetrieval)
|
|
1410
|
+
return { retrievalCounts, lastUseMsForProactive };
|
|
1176
1411
|
// usage_events lives in state.db (Chunk-8 WI-8.3); entries stay in index.db,
|
|
1177
1412
|
// so the retrieval-count reads take both handles.
|
|
1178
1413
|
const dbForRetrievalIndex = dbForRetrieval;
|
|
1179
|
-
|
|
1180
|
-
|
|
1181
|
-
|
|
1182
|
-
|
|
1183
|
-
|
|
1184
|
-
|
|
1185
|
-
|
|
1186
|
-
|
|
1187
|
-
|
|
1188
|
-
|
|
1189
|
-
|
|
1190
|
-
|
|
1191
|
-
|
|
1192
|
-
|
|
1193
|
-
|
|
1194
|
-
|
|
1195
|
-
|
|
1196
|
-
|
|
1414
|
+
if (persist || eventsCtx?.db) {
|
|
1415
|
+
withStateDb((stateDb) => {
|
|
1416
|
+
const showEventCount = countUsageEventsByType(stateDb, "show");
|
|
1417
|
+
if (showEventCount === 0) {
|
|
1418
|
+
warn("Warning: show events not yet in usage_events — zero-feedback fallback will match only search-retrieved assets.");
|
|
1419
|
+
}
|
|
1420
|
+
// Fetch retrieval counts for ALL candidates — not only the zero-feedback pool.
|
|
1421
|
+
// Previously only noFeedbackCandidates were looked up, so feedback-bearing refs
|
|
1422
|
+
// had retrievalFreq=0 in computeSalience(), collapsing their retrievalSalience
|
|
1423
|
+
// to 0 regardless of actual use. Two assets of the same type — one
|
|
1424
|
+
// heavily-retrieved, one never-touched — would receive identical rankScores.
|
|
1425
|
+
// Fix (WS-1 blocker 3): union the feedback pool into the lookup.
|
|
1426
|
+
const allCandidateRefs = [...new Set([...signalFiltered, ...noFeedbackCandidates].map((r) => r.ref))];
|
|
1427
|
+
retrievalCounts = getRetrievalCounts(dbForRetrievalIndex, stateDb, allCandidateRefs, {
|
|
1428
|
+
sourceName: options.sourceName,
|
|
1429
|
+
});
|
|
1430
|
+
}, { path: eventsCtx?.dbPath, borrowed: eventsCtx?.db });
|
|
1431
|
+
}
|
|
1432
|
+
lastUseMsForProactive = getLastUseMsByRef(dbForRetrieval, noFeedbackCandidates);
|
|
1197
1433
|
}
|
|
1198
1434
|
catch (err) {
|
|
1199
1435
|
rethrowIfTestIsolationError(err);
|
|
@@ -1207,7 +1443,7 @@ function fetchRetrievalSignals(args) {
|
|
|
1207
1443
|
}
|
|
1208
1444
|
/** Layer 2 — the proactive-maintenance selector lane (candidate-gather). */
|
|
1209
1445
|
function selectProactiveMaintenanceLane(args) {
|
|
1210
|
-
const { scope, improveProfile, resolvedPlan, eventsCtx, noFeedbackCandidates, lastReflectProposalTs, lastDistillProposalTs, retrievalCounts, lastUseMsForProactive, } = args;
|
|
1446
|
+
const { scope, improveProfile, resolvedPlan, eventsCtx, noFeedbackCandidates, lastReflectProposalTs, lastDistillProposalTs, retrievalCounts, lastUseMsForProactive, persist, } = args;
|
|
1211
1447
|
// ── Layer 2: PROACTIVE MAINTENANCE SELECTOR (second eligibility source) ────
|
|
1212
1448
|
// The signal-delta gate only surfaces assets with fresh feedback. It never
|
|
1213
1449
|
// revisits a stable, high-value asset on a schedule, so on a quiet stash
|
|
@@ -1223,6 +1459,7 @@ function selectProactiveMaintenanceLane(args) {
|
|
|
1223
1459
|
// runs rotate through the due pool rather than re-selecting the same heads.
|
|
1224
1460
|
let proactiveRefs = [];
|
|
1225
1461
|
let proactiveMaintenanceSummary;
|
|
1462
|
+
let proactivePlan;
|
|
1226
1463
|
const proactiveEnabled = scope.mode !== "ref" && resolvedPlan.processes.proactiveMaintenance.enabled;
|
|
1227
1464
|
if (proactiveEnabled) {
|
|
1228
1465
|
const pmCfg = improveProfile.processes?.proactiveMaintenance;
|
|
@@ -1257,28 +1494,44 @@ function selectProactiveMaintenanceLane(args) {
|
|
|
1257
1494
|
selected: selection.selected.length,
|
|
1258
1495
|
dueTotal: selection.dueTotal,
|
|
1259
1496
|
neverReflected: selection.neverReflected,
|
|
1497
|
+
selectedRefs: selection.selected.map((entry) => entry.ref),
|
|
1498
|
+
};
|
|
1499
|
+
proactivePlan = {
|
|
1500
|
+
configured: {
|
|
1501
|
+
...(pmCfg?.dueDays !== undefined ? { dueDays: pmCfg.dueDays } : {}),
|
|
1502
|
+
...(pmCfg?.maxPerRun !== undefined ? { maxPerRun: pmCfg.maxPerRun } : {}),
|
|
1503
|
+
...(pmCfg?.limit !== undefined ? { limit: pmCfg.limit } : {}),
|
|
1504
|
+
},
|
|
1505
|
+
effective: { dueDays, maxPerRun },
|
|
1506
|
+
candidatePool: pmCandidates.length,
|
|
1507
|
+
dueTotal: selection.dueTotal,
|
|
1508
|
+
neverReflected: selection.neverReflected,
|
|
1509
|
+
selected: selection.selected.length,
|
|
1510
|
+
selectedRefs: selection.selected.map((entry) => entry.ref),
|
|
1260
1511
|
};
|
|
1261
1512
|
// Aggregated observability event (never per-ref — avoids the event flood the
|
|
1262
1513
|
// Layer-1 work eliminated). Mirrors the `no_new_signal` aggregation pattern.
|
|
1263
|
-
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
|
|
1268
|
-
|
|
1269
|
-
|
|
1270
|
-
|
|
1271
|
-
|
|
1514
|
+
if (persist) {
|
|
1515
|
+
appendEvent({
|
|
1516
|
+
eventType: "proactive_selected",
|
|
1517
|
+
ref: undefined,
|
|
1518
|
+
metadata: {
|
|
1519
|
+
count: selection.selected.length,
|
|
1520
|
+
dueTotal: selection.dueTotal,
|
|
1521
|
+
neverReflected: selection.neverReflected,
|
|
1522
|
+
},
|
|
1523
|
+
}, eventsCtx);
|
|
1524
|
+
}
|
|
1272
1525
|
if (selection.selected.length > 0) {
|
|
1273
1526
|
info(`[improve] proactive maintenance selected ${selection.selected.length}/${selection.dueTotal} due refs ` +
|
|
1274
1527
|
`(${selection.neverReflected} never reflected, dueDays=${dueDays}, maxPerRun=${maxPerRun})`);
|
|
1275
1528
|
}
|
|
1276
1529
|
}
|
|
1277
|
-
return { proactiveRefs, proactiveMaintenanceSummary };
|
|
1530
|
+
return { proactiveRefs, proactiveMaintenanceSummary, proactivePlan };
|
|
1278
1531
|
}
|
|
1279
1532
|
/** Layer 3 — the high-salience admission gate (#608/#644; candidate-gather). */
|
|
1280
1533
|
function selectHighSalienceLane(args) {
|
|
1281
|
-
const { options, improveProfile, eventsCtx, noFeedbackCandidates, proactiveRefs, lastReflectProposalTs } = args;
|
|
1534
|
+
const { options, improveProfile, eventsCtx, noFeedbackCandidates, proactiveRefs, lastReflectProposalTs, persist } = args;
|
|
1282
1535
|
// ── Layer 3: HIGH-SALIENCE ADMISSION GATE (#608) ──────────────────────────
|
|
1283
1536
|
// Zero-feedback refs whose encoding_salience (set at distill time by
|
|
1284
1537
|
// scoreEncodingSalience) exceeds the configured salienceThreshold are admitted
|
|
@@ -1313,6 +1566,8 @@ function selectHighSalienceLane(args) {
|
|
|
1313
1566
|
const salienceThreshold = salienceCfg?.salienceThreshold ?? 0.75;
|
|
1314
1567
|
const proactiveSelectedSet = new Set(proactiveRefs.map((r) => r.ref));
|
|
1315
1568
|
try {
|
|
1569
|
+
if (!persist && !eventsCtx?.db)
|
|
1570
|
+
return highSalienceRefs;
|
|
1316
1571
|
withStateDb((dbForHighSalience) => {
|
|
1317
1572
|
// Derive the cap from the resolved reflect limit (mirrors improve.ts's
|
|
1318
1573
|
// options.limit resolution) so an unbounded whole-stash run does not
|
|
@@ -1337,7 +1592,7 @@ function selectHighSalienceLane(args) {
|
|
|
1337
1592
|
for (const q of qualifying.slice(0, highSalienceCap)) {
|
|
1338
1593
|
highSalienceRefs.push(q.ref);
|
|
1339
1594
|
}
|
|
1340
|
-
}, { path: eventsCtx?.dbPath });
|
|
1595
|
+
}, { path: eventsCtx?.dbPath, borrowed: eventsCtx?.db });
|
|
1341
1596
|
}
|
|
1342
1597
|
catch (err) {
|
|
1343
1598
|
rethrowIfTestIsolationError(err);
|
|
@@ -1358,7 +1613,7 @@ function selectHighSalienceLane(args) {
|
|
|
1358
1613
|
* attribution identity is load-bearing (see the candidate-gather comments).
|
|
1359
1614
|
*/
|
|
1360
1615
|
function scoreSalience(args) {
|
|
1361
|
-
const { scope, options, primaryStashDir, eventsCtx, eligibilitySourceByRef, feedbackSummary, retrievalCounts, signalFiltered, proactiveRefs, highSalienceRefs, } = args;
|
|
1616
|
+
const { scope, options, primaryStashDir, eventsCtx, eligibilitySourceByRef, feedbackSummary, retrievalCounts, signalFiltered, proactiveRefs, highSalienceRefs, forgettingEligibleRefs, persist, } = args;
|
|
1362
1617
|
const mergedRefs = args.mergedRefs;
|
|
1363
1618
|
// Chunk-5 flip F5e — resolve each candidate's durable item_ref ONCE for this
|
|
1364
1619
|
// pass (the write/read key source for the outcome + salience state writers).
|
|
@@ -1375,11 +1630,15 @@ function scoreSalience(args) {
|
|
|
1375
1630
|
// lightweight open is used here to avoid holding the connection longer than needed.
|
|
1376
1631
|
let lastUseMsByRef = new Map();
|
|
1377
1632
|
// Health and outcome reporting consume the utility projection.
|
|
1378
|
-
const utilityMap = buildUtilityMap(mergedRefs);
|
|
1633
|
+
const utilityMap = buildUtilityMap(mergedRefs, !persist);
|
|
1379
1634
|
let dbForSalience;
|
|
1380
1635
|
try {
|
|
1381
|
-
dbForSalience =
|
|
1382
|
-
|
|
1636
|
+
dbForSalience = persist
|
|
1637
|
+
? openExistingDatabase()
|
|
1638
|
+
: openReadonlyExistingDatabase(undefined, { isolatedSnapshot: true });
|
|
1639
|
+
if (dbForSalience) {
|
|
1640
|
+
lastUseMsByRef = getLastUseMsByRef(dbForSalience, mergedRefs);
|
|
1641
|
+
}
|
|
1383
1642
|
}
|
|
1384
1643
|
catch (err) {
|
|
1385
1644
|
rethrowIfTestIsolationError(err);
|
|
@@ -1398,6 +1657,7 @@ function scoreSalience(args) {
|
|
|
1398
1657
|
utilityMap,
|
|
1399
1658
|
primaryStashDir,
|
|
1400
1659
|
eventsCtx,
|
|
1660
|
+
persist,
|
|
1401
1661
|
});
|
|
1402
1662
|
const { salienceMap, nowForSalience } = computeSalienceVectors({
|
|
1403
1663
|
mergedRefs,
|
|
@@ -1408,6 +1668,7 @@ function scoreSalience(args) {
|
|
|
1408
1668
|
lastUseMsByRef,
|
|
1409
1669
|
utilityMap,
|
|
1410
1670
|
outcomeSalienceByRef,
|
|
1671
|
+
persist,
|
|
1411
1672
|
});
|
|
1412
1673
|
const pendingForgettingRefs = persistSalienceAndReportRanks({
|
|
1413
1674
|
salienceMap,
|
|
@@ -1417,11 +1678,14 @@ function scoreSalience(args) {
|
|
|
1417
1678
|
options,
|
|
1418
1679
|
eventsCtx,
|
|
1419
1680
|
nowForSalience,
|
|
1681
|
+
persist,
|
|
1420
1682
|
});
|
|
1421
1683
|
const finalMergedRefs = applyForgettingSafety({
|
|
1422
1684
|
pendingForgettingRefs,
|
|
1423
1685
|
scope,
|
|
1424
1686
|
mergedRefs,
|
|
1687
|
+
eligibleRefs: forgettingEligibleRefs,
|
|
1688
|
+
allowFallbacks: options.requireFeedbackSignal !== true,
|
|
1425
1689
|
eligibilitySourceByRef,
|
|
1426
1690
|
highSalienceRefs,
|
|
1427
1691
|
proactiveRefs,
|
|
@@ -1431,7 +1695,7 @@ function scoreSalience(args) {
|
|
|
1431
1695
|
}
|
|
1432
1696
|
/** WS-2 — update asset_outcome for the merged set; returns outcomeSalience by ref. */
|
|
1433
1697
|
function updateOutcomeScores(args) {
|
|
1434
|
-
const { mergedRefs, itemRefByRef, feedbackSummary, retrievalCounts, lastUseMsByRef, utilityMap, primaryStashDir, eventsCtx, } = args;
|
|
1698
|
+
const { mergedRefs, itemRefByRef, feedbackSummary, retrievalCounts, lastUseMsByRef, utilityMap, primaryStashDir, eventsCtx, persist, } = args;
|
|
1435
1699
|
// ── WS-2 Outcome loop ─────────────────────────────────────────────────────
|
|
1436
1700
|
//
|
|
1437
1701
|
// Update asset_outcome for every ref in the merged set BEFORE computing the
|
|
@@ -1447,6 +1711,32 @@ function updateOutcomeScores(args) {
|
|
|
1447
1711
|
//
|
|
1448
1712
|
// Best-effort: outcome failures never block the salience or ranking pass.
|
|
1449
1713
|
const outcomeSalienceByRef = new Map();
|
|
1714
|
+
// Missing state.db is itself a complete snapshot: no prior outcome rows and
|
|
1715
|
+
// no accepted proposals. Project the same warm-start values a live run would
|
|
1716
|
+
// insert, without creating the database merely to represent empty tables.
|
|
1717
|
+
if (!persist && !eventsCtx?.db) {
|
|
1718
|
+
const projectedScores = new Map();
|
|
1719
|
+
const nowForOutcome = Date.now();
|
|
1720
|
+
for (const ref of mergedRefs) {
|
|
1721
|
+
const feedback = feedbackSummary.get(ref.ref) ?? { positive: 0, negative: 0 };
|
|
1722
|
+
const result = projectAssetOutcome(undefined, {
|
|
1723
|
+
ref: outcomeWriteKey(ref.ref, itemRefByRef),
|
|
1724
|
+
currentRetrievalCount: retrievalCounts.get(ref.ref) ?? 0,
|
|
1725
|
+
lastRetrievedAt: lastUseMsByRef.get(ref.ref) ?? 0,
|
|
1726
|
+
acceptedChangeCount: 0,
|
|
1727
|
+
negativeFeedbackCount: feedback.negative,
|
|
1728
|
+
valence: computeValenceScore(feedback).valence,
|
|
1729
|
+
utilityScore: utilityMap.get(ref.ref),
|
|
1730
|
+
now: nowForOutcome,
|
|
1731
|
+
});
|
|
1732
|
+
projectedScores.set(ref.ref, result.outcomeScore);
|
|
1733
|
+
}
|
|
1734
|
+
const maxOutcomeScore = Math.min(OUTCOME_SCORE_MAX, Math.max(0, ...projectedScores.values()));
|
|
1735
|
+
for (const [ref, score] of projectedScores) {
|
|
1736
|
+
outcomeSalienceByRef.set(ref, outcomeScoreToSalience(score, maxOutcomeScore));
|
|
1737
|
+
}
|
|
1738
|
+
return outcomeSalienceByRef;
|
|
1739
|
+
}
|
|
1450
1740
|
try {
|
|
1451
1741
|
withStateDb((outcomeDb) => {
|
|
1452
1742
|
// Count accepted proposals per ref in one pass (avoid N separate queries).
|
|
@@ -1467,15 +1757,17 @@ function updateOutcomeScores(args) {
|
|
|
1467
1757
|
}
|
|
1468
1758
|
// Update each ref's outcome row and collect the resulting outcome scores.
|
|
1469
1759
|
const rawOutcomeScores = new Map();
|
|
1760
|
+
const projectedByWriteKey = new Map();
|
|
1470
1761
|
const nowForOutcome = Date.now();
|
|
1471
1762
|
for (const r of mergedRefs) {
|
|
1472
1763
|
const fb = feedbackSummary.get(r.ref) ?? { positive: 0, negative: 0 };
|
|
1473
1764
|
const valenceResult = computeValenceScore(fb);
|
|
1474
1765
|
try {
|
|
1475
|
-
const
|
|
1766
|
+
const writeKey = outcomeWriteKey(r.ref, itemRefByRef);
|
|
1767
|
+
const inputs = {
|
|
1476
1768
|
// Key by item_ref when resolved, else by the conceptId. Keep
|
|
1477
1769
|
// rawOutcomeScores keyed by r.ref, its in-memory identity.
|
|
1478
|
-
ref:
|
|
1770
|
+
ref: writeKey,
|
|
1479
1771
|
currentRetrievalCount: retrievalCounts.get(r.ref) ?? 0,
|
|
1480
1772
|
lastRetrievedAt: lastUseMsByRef.get(r.ref) ?? 0,
|
|
1481
1773
|
acceptedChangeCount: acceptedCountByRef.get(r.ref) ?? 0,
|
|
@@ -1483,8 +1775,12 @@ function updateOutcomeScores(args) {
|
|
|
1483
1775
|
valence: valenceResult.valence,
|
|
1484
1776
|
utilityScore: utilityMap.get(r.ref),
|
|
1485
1777
|
now: nowForOutcome,
|
|
1486
|
-
}
|
|
1778
|
+
};
|
|
1779
|
+
const result = persist
|
|
1780
|
+
? updateAssetOutcome(outcomeDb, inputs)
|
|
1781
|
+
: projectAssetOutcome(getAssetOutcome(outcomeDb, writeKey), inputs);
|
|
1487
1782
|
rawOutcomeScores.set(r.ref, result.outcomeScore);
|
|
1783
|
+
projectedByWriteKey.set(writeKey, result.outcomeScore);
|
|
1488
1784
|
}
|
|
1489
1785
|
catch {
|
|
1490
1786
|
// best-effort per-ref: skip this ref's outcome update on failure
|
|
@@ -1496,16 +1792,19 @@ function updateOutcomeScores(args) {
|
|
|
1496
1792
|
let maxOutcomeScore = 0;
|
|
1497
1793
|
try {
|
|
1498
1794
|
const allOutcomes = getAllAssetOutcomes(outcomeDb);
|
|
1499
|
-
|
|
1500
|
-
|
|
1501
|
-
|
|
1795
|
+
const scoreByRef = new Map(allOutcomes.map((row) => [row.asset_ref, row.outcome_score]));
|
|
1796
|
+
for (const [ref, score] of projectedByWriteKey)
|
|
1797
|
+
scoreByRef.set(ref, score);
|
|
1798
|
+
for (const score of scoreByRef.values()) {
|
|
1799
|
+
if (score > maxOutcomeScore)
|
|
1800
|
+
maxOutcomeScore = score;
|
|
1502
1801
|
}
|
|
1503
1802
|
// Keep the normalization denominator within the writer's score bounds.
|
|
1504
1803
|
maxOutcomeScore = Math.min(maxOutcomeScore, OUTCOME_SCORE_MAX);
|
|
1505
1804
|
// Proxy-adequacy tripwire (two-tailed): inverted (corr < −0.3) and
|
|
1506
1805
|
// dead (|corr| < 0.1 at n ≥ 500) both emit health events.
|
|
1507
|
-
const adequacy = computeProxyAdequacy(allOutcomes);
|
|
1508
|
-
if (adequacy
|
|
1806
|
+
const adequacy = persist ? computeProxyAdequacy(allOutcomes) : undefined;
|
|
1807
|
+
if (adequacy?.isInverted) {
|
|
1509
1808
|
appendEvent({
|
|
1510
1809
|
eventType: "outcome_proxy_inverted",
|
|
1511
1810
|
ref: undefined,
|
|
@@ -1516,7 +1815,7 @@ function updateOutcomeScores(args) {
|
|
|
1516
1815
|
},
|
|
1517
1816
|
}, eventsCtx);
|
|
1518
1817
|
}
|
|
1519
|
-
if (adequacy
|
|
1818
|
+
if (adequacy?.isDead) {
|
|
1520
1819
|
appendEvent({
|
|
1521
1820
|
eventType: "outcome_proxy_dead",
|
|
1522
1821
|
ref: undefined,
|
|
@@ -1562,7 +1861,7 @@ function updateOutcomeScores(args) {
|
|
|
1562
1861
|
}
|
|
1563
1862
|
/** WS-1 — compute the salience vector per ref (#644 provenance preserved). */
|
|
1564
1863
|
function computeSalienceVectors(args) {
|
|
1565
|
-
const { mergedRefs, itemRefByRef, options, eventsCtx, retrievalCounts, lastUseMsByRef, utilityMap, outcomeSalienceByRef, } = args;
|
|
1864
|
+
const { mergedRefs, itemRefByRef, options, eventsCtx, retrievalCounts, lastUseMsByRef, utilityMap, outcomeSalienceByRef, persist, } = args;
|
|
1566
1865
|
// Compute the salience vector for every ref in the merged set.
|
|
1567
1866
|
// retrievalCounts now covers the full candidate set (feedback-bearing + zero-feedback)
|
|
1568
1867
|
// so feedback refs get their genuine retrieval frequency, not a 0-floor fallback.
|
|
@@ -1586,14 +1885,16 @@ function computeSalienceVectors(args) {
|
|
|
1586
1885
|
// Refs that have never been content-scored keep the type-weight stub fallback.
|
|
1587
1886
|
const storedEncodingByRef = new Map();
|
|
1588
1887
|
try {
|
|
1589
|
-
|
|
1590
|
-
|
|
1591
|
-
|
|
1592
|
-
|
|
1593
|
-
|
|
1888
|
+
if (persist || eventsCtx?.db) {
|
|
1889
|
+
withStateDb((dbForStoredEncoding) => {
|
|
1890
|
+
for (const r of mergedRefs) {
|
|
1891
|
+
const row = readAssetSalienceForImproveRef(dbForStoredEncoding, r.ref, itemRefByRef.get(r.ref));
|
|
1892
|
+
if (row && isContentEncodingRow(row)) {
|
|
1893
|
+
storedEncodingByRef.set(r.ref, row.encoding_salience);
|
|
1894
|
+
}
|
|
1594
1895
|
}
|
|
1595
|
-
}
|
|
1596
|
-
}
|
|
1896
|
+
}, { path: eventsCtx?.dbPath, borrowed: eventsCtx?.db });
|
|
1897
|
+
}
|
|
1597
1898
|
}
|
|
1598
1899
|
catch (err) {
|
|
1599
1900
|
rethrowIfTestIsolationError(err);
|
|
@@ -1658,7 +1959,7 @@ function buildSalienceWriteKeyMaps(itemRefByRef) {
|
|
|
1658
1959
|
}
|
|
1659
1960
|
/** Persist salience vectors + the WS-1 step-7 rank-change/forgetting report. */
|
|
1660
1961
|
function persistSalienceAndReportRanks(args) {
|
|
1661
|
-
const { salienceMap, itemRefByRef, utilityMap, feedbackSummary, options, eventsCtx, nowForSalience } = args;
|
|
1962
|
+
const { salienceMap, itemRefByRef, utilityMap, feedbackSummary, options, eventsCtx, nowForSalience, persist } = args;
|
|
1662
1963
|
// Chunk-5 flip F5e — the WRITE-key space. salienceMap stays keyed by each
|
|
1663
1964
|
// candidate's own short `r.ref`; the state.db boundary keys by item_ref when
|
|
1664
1965
|
// available and otherwise by conceptId.
|
|
@@ -1711,6 +2012,8 @@ function persistSalienceAndReportRanks(args) {
|
|
|
1711
2012
|
// empty on scenario A or when no candidates dropped below the threshold.
|
|
1712
2013
|
let pendingForgettingRefs = [];
|
|
1713
2014
|
try {
|
|
2015
|
+
if (!persist && !eventsCtx?.db)
|
|
2016
|
+
return pendingForgettingRefs;
|
|
1714
2017
|
withStateDb((stateDb) => {
|
|
1715
2018
|
// Step 7: stash-wide rank-change report BEFORE overwriting the table.
|
|
1716
2019
|
//
|
|
@@ -1761,20 +2064,22 @@ function persistSalienceAndReportRanks(args) {
|
|
|
1761
2064
|
.join(", ")}`);
|
|
1762
2065
|
pendingForgettingRefs = firstRunReport.forgettingCandidates.map((e) => e.ref);
|
|
1763
2066
|
}
|
|
1764
|
-
|
|
1765
|
-
|
|
1766
|
-
|
|
1767
|
-
|
|
1768
|
-
|
|
1769
|
-
|
|
1770
|
-
|
|
1771
|
-
|
|
1772
|
-
|
|
1773
|
-
|
|
1774
|
-
|
|
1775
|
-
|
|
1776
|
-
|
|
1777
|
-
|
|
2067
|
+
if (persist) {
|
|
2068
|
+
appendEvent({
|
|
2069
|
+
eventType: "improve_salience_first_run",
|
|
2070
|
+
ref: undefined,
|
|
2071
|
+
metadata: {
|
|
2072
|
+
candidateCount: salienceMap.size,
|
|
2073
|
+
note: "first WS-1 salience run — partial reconstruction of old combinedEligibilityScore ordering for candidate pool (stash-wide ordering not available); WS-1 step 7",
|
|
2074
|
+
forgettingCandidates: firstRunReport.forgettingCandidates.length,
|
|
2075
|
+
topDrops: firstRunReport.forgettingCandidates.slice(0, 10).map((e) => ({
|
|
2076
|
+
ref: e.ref,
|
|
2077
|
+
oldRank: e.oldRank,
|
|
2078
|
+
newRank: e.newRank,
|
|
2079
|
+
})),
|
|
2080
|
+
},
|
|
2081
|
+
}, eventsCtx);
|
|
2082
|
+
}
|
|
1778
2083
|
}
|
|
1779
2084
|
else {
|
|
1780
2085
|
// Scenario B: subsequent run — compare stash-wide old vs. new ranks.
|
|
@@ -1804,30 +2109,35 @@ function persistSalienceAndReportRanks(args) {
|
|
|
1804
2109
|
// this try block, bypassing cooldown/signal-delta gating.
|
|
1805
2110
|
// Chunk-5 flip F5e — map an in-pool candidate's write-key spelling
|
|
1806
2111
|
// back to its bare `r.ref` so applyForgettingSafety re-stamps the
|
|
1807
|
-
// existing pool ref
|
|
1808
|
-
//
|
|
2112
|
+
// existing pool ref. Other stored spellings stay qualified here;
|
|
2113
|
+
// the downstream admission boundary resolves them only when they
|
|
2114
|
+
// match an exact current-plan item_ref.
|
|
1809
2115
|
pendingForgettingRefs = report.forgettingCandidates.map((e) => refByWriteKey.get(e.ref) ?? e.ref);
|
|
1810
2116
|
}
|
|
1811
|
-
|
|
1812
|
-
|
|
1813
|
-
|
|
1814
|
-
|
|
1815
|
-
|
|
1816
|
-
|
|
1817
|
-
|
|
1818
|
-
|
|
1819
|
-
|
|
1820
|
-
|
|
1821
|
-
|
|
1822
|
-
|
|
1823
|
-
|
|
1824
|
-
|
|
2117
|
+
if (persist) {
|
|
2118
|
+
appendEvent({
|
|
2119
|
+
eventType: "improve_salience_rank_change",
|
|
2120
|
+
ref: undefined,
|
|
2121
|
+
metadata: {
|
|
2122
|
+
stashSize: existingAllScores.size,
|
|
2123
|
+
totalChanged: report.allChanges.length,
|
|
2124
|
+
forgettingCandidates: report.forgettingCandidates.length,
|
|
2125
|
+
topDrops: report.forgettingCandidates.slice(0, 10).map((e) => ({
|
|
2126
|
+
ref: e.ref,
|
|
2127
|
+
oldRank: e.oldRank,
|
|
2128
|
+
newRank: e.newRank,
|
|
2129
|
+
})),
|
|
2130
|
+
},
|
|
2131
|
+
}, eventsCtx);
|
|
2132
|
+
}
|
|
1825
2133
|
}
|
|
1826
|
-
|
|
1827
|
-
|
|
1828
|
-
|
|
2134
|
+
if (persist) {
|
|
2135
|
+
for (const [ref, vector] of salienceMap) {
|
|
2136
|
+
// Persist salience under item_ref when resolved, else the conceptId.
|
|
2137
|
+
upsertAssetSalience(stateDb, wk(ref), vector, nowForSalience);
|
|
2138
|
+
}
|
|
1829
2139
|
}
|
|
1830
|
-
}, { path: eventsCtx?.dbPath });
|
|
2140
|
+
}, { path: eventsCtx?.dbPath, borrowed: eventsCtx?.db });
|
|
1831
2141
|
}
|
|
1832
2142
|
catch (err) {
|
|
1833
2143
|
rethrowIfTestIsolationError(err);
|
|
@@ -1841,29 +2151,38 @@ function persistSalienceAndReportRanks(args) {
|
|
|
1841
2151
|
* order on the SHARED ref objects and eligibilitySourceByRef map.
|
|
1842
2152
|
*/
|
|
1843
2153
|
export function applyForgettingSafety(args) {
|
|
1844
|
-
const { pendingForgettingRefs, scope, eligibilitySourceByRef, highSalienceRefs, proactiveRefs, signalFiltered } = args;
|
|
2154
|
+
const { pendingForgettingRefs, scope, eligibleRefs, allowFallbacks, eligibilitySourceByRef, highSalienceRefs, proactiveRefs, signalFiltered, } = args;
|
|
1845
2155
|
let mergedRefs = args.mergedRefs;
|
|
1846
2156
|
// ── Protective consolidation pass (plan §WS-1 step 7) ─────────────────────
|
|
1847
2157
|
// Forgetting candidates detected in scenario B are force-injected into
|
|
1848
2158
|
// mergedRefs here, BEFORE the effectiveScore sort, bypassing cooldown and
|
|
1849
|
-
// signal-delta gating.
|
|
1850
|
-
//
|
|
1851
|
-
//
|
|
1852
|
-
|
|
1853
|
-
// ensures no ref can enter the loop twice.
|
|
1854
|
-
if (pendingForgettingRefs.length > 0 && scope.mode !== "ref") {
|
|
2159
|
+
// signal-delta gating. The lane may only reuse exact objects from this
|
|
2160
|
+
// invocation's post-cleanup/post-validation plan. Stale, out-of-scope, and
|
|
2161
|
+
// differently-qualified durable state must never synthesize executable work.
|
|
2162
|
+
if (pendingForgettingRefs.length > 0 && scope.mode !== "ref" && allowFallbacks) {
|
|
1855
2163
|
const existingRefSet = new Set(mergedRefs.map((r) => r.ref));
|
|
2164
|
+
const eligibleByRef = new Map(eligibleRefs.map((candidate) => [candidate.ref, candidate]));
|
|
2165
|
+
const eligibleByItemRef = new Map();
|
|
2166
|
+
for (const candidate of eligibleRefs) {
|
|
2167
|
+
if (candidate.itemRef)
|
|
2168
|
+
eligibleByItemRef.set(candidate.itemRef, candidate);
|
|
2169
|
+
}
|
|
1856
2170
|
const newForgettingRefs = [];
|
|
1857
|
-
|
|
1858
|
-
|
|
1859
|
-
|
|
1860
|
-
|
|
1861
|
-
|
|
2171
|
+
const forgettingRefSet = new Set();
|
|
2172
|
+
for (const stateRef of pendingForgettingRefs) {
|
|
2173
|
+
const boundary = stateRef.indexOf("//");
|
|
2174
|
+
const candidate = eligibleByItemRef.get(stateRef) ?? (boundary < 0 ? eligibleByRef.get(bareImproveRef(stateRef)) : undefined);
|
|
2175
|
+
if (!candidate || forgettingRefSet.has(candidate.ref))
|
|
2176
|
+
continue;
|
|
2177
|
+
forgettingRefSet.add(candidate.ref);
|
|
2178
|
+
if (!existingRefSet.has(candidate.ref)) {
|
|
2179
|
+
newForgettingRefs.push(candidate);
|
|
2180
|
+
existingRefSet.add(candidate.ref);
|
|
1862
2181
|
}
|
|
1863
2182
|
// Always stamp the lane in the attribution map (overwrites weaker lanes;
|
|
1864
2183
|
// stronger reactive signals — scope/signal-delta/proactive — are written
|
|
1865
2184
|
// after this block so they take precedence).
|
|
1866
|
-
eligibilitySourceByRef.set(ref, "forgetting-safety");
|
|
2185
|
+
eligibilitySourceByRef.set(candidate.ref, "forgetting-safety");
|
|
1867
2186
|
}
|
|
1868
2187
|
if (newForgettingRefs.length > 0) {
|
|
1869
2188
|
mergedRefs = dedupeRefs([...mergedRefs, ...newForgettingRefs]);
|
|
@@ -1887,7 +2206,7 @@ export function applyForgettingSafety(args) {
|
|
|
1887
2206
|
// set() calls above for proactive/high-salience overwrite the earlier
|
|
1888
2207
|
// forgetting-safety stamp — so we re-apply forgetting-safety now for those
|
|
1889
2208
|
// refs that are both forgetting candidates AND in another fallback lane.
|
|
1890
|
-
for (const ref of
|
|
2209
|
+
for (const ref of forgettingRefSet) {
|
|
1891
2210
|
eligibilitySourceByRef.set(ref, "forgetting-safety");
|
|
1892
2211
|
}
|
|
1893
2212
|
// signal-delta is the strongest reactive signal and overrides forgetting-safety.
|
|
@@ -1907,17 +2226,18 @@ export function applyForgettingSafety(args) {
|
|
|
1907
2226
|
* info emits.
|
|
1908
2227
|
*/
|
|
1909
2228
|
async function filterEligibility(args) {
|
|
1910
|
-
const { scope, options,
|
|
1911
|
-
const {
|
|
2229
|
+
const { scope, options, replayEligibleRefs, eventsCtx, salienceMap, eligibilitySourceByRef, distillOnlyRefs, persist, } = args;
|
|
2230
|
+
const { signalAndRetrievalRefs, signalFiltered } = args.summary;
|
|
1912
2231
|
const validationFailureRefs = args.validationFailureRefs;
|
|
1913
2232
|
const replay = applyReplaySelection({
|
|
1914
2233
|
scope,
|
|
1915
2234
|
options,
|
|
1916
|
-
plannedRefs,
|
|
2235
|
+
plannedRefs: replayEligibleRefs,
|
|
1917
2236
|
eventsCtx,
|
|
1918
2237
|
mergedRefs: args.mergedRefs,
|
|
1919
2238
|
salienceMap,
|
|
1920
2239
|
eligibilitySourceByRef,
|
|
2240
|
+
persist,
|
|
1921
2241
|
});
|
|
1922
2242
|
const mergedRefs = replay.mergedRefs;
|
|
1923
2243
|
const { replayRefSet, replayBudget } = replay;
|
|
@@ -1927,7 +2247,7 @@ async function filterEligibility(args) {
|
|
|
1927
2247
|
// order — the persisted rank_score in asset_salience is never mutated here.
|
|
1928
2248
|
const noOpMap = new Map();
|
|
1929
2249
|
try {
|
|
1930
|
-
const noOpDb = eventsCtx?.db ?? (eventsCtx?.dbPath ? openStateDatabase(eventsCtx.dbPath) : null);
|
|
2250
|
+
const noOpDb = eventsCtx?.db ?? (persist && eventsCtx?.dbPath ? openStateDatabase(eventsCtx.dbPath) : null);
|
|
1931
2251
|
if (noOpDb) {
|
|
1932
2252
|
const ownsNoOpDb = !eventsCtx?.db;
|
|
1933
2253
|
try {
|
|
@@ -1978,35 +2298,28 @@ async function filterEligibility(args) {
|
|
|
1978
2298
|
// Phase 0: surface coverage gaps from zero-result search queries
|
|
1979
2299
|
let coverageGaps = [];
|
|
1980
2300
|
try {
|
|
1981
|
-
const dbForGaps =
|
|
1982
|
-
|
|
1983
|
-
|
|
1984
|
-
|
|
1985
|
-
|
|
1986
|
-
|
|
2301
|
+
const dbForGaps = persist
|
|
2302
|
+
? openExistingDatabase()
|
|
2303
|
+
: openReadonlyExistingDatabase(undefined, { isolatedSnapshot: true });
|
|
2304
|
+
if (dbForGaps) {
|
|
2305
|
+
try {
|
|
2306
|
+
coverageGaps = getZeroResultSearches(dbForGaps);
|
|
2307
|
+
}
|
|
2308
|
+
finally {
|
|
2309
|
+
closeDatabase(dbForGaps);
|
|
2310
|
+
}
|
|
1987
2311
|
}
|
|
1988
2312
|
}
|
|
1989
2313
|
catch (err) {
|
|
1990
2314
|
rethrowIfTestIsolationError(err);
|
|
1991
2315
|
// best-effort
|
|
1992
2316
|
}
|
|
1993
|
-
const diskCheck = await dropRefsMissingOnDisk({ sorted, options, eventsCtx });
|
|
2317
|
+
const diskCheck = await dropRefsMissingOnDisk({ sorted, options, eventsCtx, persist });
|
|
1994
2318
|
const assetMissingOnDisk = diskCheck.assetMissingOnDisk;
|
|
1995
2319
|
const actionableRefs = diskCheck.actionableRefs;
|
|
1996
2320
|
// Re-split actionableRefs (sorted) into reflect-path vs distill-only-path while
|
|
1997
2321
|
// preserving sort order. distillOnlyRefs participate in the sort so --limit
|
|
1998
2322
|
// picks them by score, not by arbitrary position.
|
|
1999
|
-
const distillOnlyRefSetForSort = new Set(distillOnlyRefs.map((r) => r.ref));
|
|
2000
|
-
const reflectAndDistillRefsAfterSort = [];
|
|
2001
|
-
const distillOnlyRefsAfterSort = [];
|
|
2002
|
-
for (const r of actionableRefs) {
|
|
2003
|
-
if (distillOnlyRefSetForSort.has(r.ref)) {
|
|
2004
|
-
distillOnlyRefsAfterSort.push(r);
|
|
2005
|
-
}
|
|
2006
|
-
else {
|
|
2007
|
-
reflectAndDistillRefsAfterSort.push(r);
|
|
2008
|
-
}
|
|
2009
|
-
}
|
|
2010
2323
|
// ── Phase 5: --limit applies to the post-cooldown actionable set ──────────
|
|
2011
2324
|
//
|
|
2012
2325
|
// #610 ADDITIVITY: replay-lane refs are budgeted SEPARATELY from the --limit
|
|
@@ -2019,37 +2332,40 @@ async function filterEligibility(args) {
|
|
|
2019
2332
|
// Default replayBudget=0 reduces this to the exact pre-#610 expression: with no
|
|
2020
2333
|
// replay refs, `nonReplayLoop === allLoopRefs`, so `baseLoop === old slice` and
|
|
2021
2334
|
// `replayLoop.slice(0, 0) === []` — byte-identical.
|
|
2022
|
-
const
|
|
2023
|
-
|
|
2024
|
-
|
|
2025
|
-
|
|
2026
|
-
|
|
2027
|
-
|
|
2028
|
-
|
|
2029
|
-
|
|
2030
|
-
const distillOnlyRefsResult = distillOnlyRefsAfterSort;
|
|
2031
|
-
const totalReflectBlocked = fullySkippedCount + distillOnlyRefs.length;
|
|
2032
|
-
if (totalReflectBlocked > 0) {
|
|
2033
|
-
info(`[improve] ${totalReflectBlocked} of ${preCooldownCount} indexed refs blocked by reflect signal-delta ` +
|
|
2034
|
-
`(${fullySkippedCount} fully skipped, ${distillOnlyRefs.length} routed to distill-only)`);
|
|
2035
|
-
}
|
|
2335
|
+
const selection = selectEffectiveImproveRefs({
|
|
2336
|
+
rankedRefs: actionableRefs,
|
|
2337
|
+
distillOnlyRefs,
|
|
2338
|
+
limit: options.limit,
|
|
2339
|
+
replayBudget,
|
|
2340
|
+
});
|
|
2341
|
+
const loopRefs = selection.loopRefs;
|
|
2342
|
+
const distillOnlyRefsResult = selection.distillOnlyRefs;
|
|
2036
2343
|
if (signalAndRetrievalRefs.length > 0) {
|
|
2037
2344
|
info(`[improve] ${signalAndRetrievalRefs.length} refs with usage signals (${signalFiltered.length} feedback${replayRefSet.size > 0 ? `, ${replayRefSet.size} replay` : ""})`);
|
|
2038
2345
|
}
|
|
2039
2346
|
if (validationFailureRefs.size > 0) {
|
|
2040
2347
|
info(`[improve] ${validationFailureRefs.size} with validation failures excluded`);
|
|
2041
2348
|
}
|
|
2042
|
-
if (assetMissingOnDisk.length > 0) {
|
|
2349
|
+
if (persist && assetMissingOnDisk.length > 0) {
|
|
2043
2350
|
info(`[improve] ${assetMissingOnDisk.length} candidates dropped — file not on disk`);
|
|
2044
2351
|
}
|
|
2045
2352
|
const deferredCount = actionableRefs.length - loopRefs.length;
|
|
2046
2353
|
info(`[improve] ${actionableRefs.length} actionable; ${loopRefs.length} will be processed` +
|
|
2047
2354
|
(options.limit && deferredCount > 0 ? ` (--limit ${options.limit} applied; ${deferredCount} deferred)` : ""));
|
|
2048
|
-
return {
|
|
2355
|
+
return {
|
|
2356
|
+
loopRefs,
|
|
2357
|
+
actionableRefs,
|
|
2358
|
+
distillOnlyRefs: distillOnlyRefsResult,
|
|
2359
|
+
coverageGaps,
|
|
2360
|
+
limitRemoved: selection.limitRemoved,
|
|
2361
|
+
missingDiskCount: assetMissingOnDisk.length,
|
|
2362
|
+
replayBudget,
|
|
2363
|
+
preDiskRefs: sorted,
|
|
2364
|
+
};
|
|
2049
2365
|
}
|
|
2050
2366
|
/** The #610 bounded, additive replay-selection lane (eligibility-filter). */
|
|
2051
2367
|
function applyReplaySelection(args) {
|
|
2052
|
-
const { scope, options, plannedRefs, eventsCtx, salienceMap, eligibilitySourceByRef } = args;
|
|
2368
|
+
const { scope, options, plannedRefs, eventsCtx, salienceMap, eligibilitySourceByRef, persist } = args;
|
|
2053
2369
|
let mergedRefs = args.mergedRefs;
|
|
2054
2370
|
// ── REPLAY SELECTION layer (#610) ─────────────────────────────────────────
|
|
2055
2371
|
// Bounded, ADDITIVE replay budget: up to `replayBudget` top-salience refs are
|
|
@@ -2069,67 +2385,70 @@ function applyReplaySelection(args) {
|
|
|
2069
2385
|
const replayRefSet = new Set();
|
|
2070
2386
|
if (replayBudget > 0 && scope.mode !== "ref" && !options.requireFeedbackSignal) {
|
|
2071
2387
|
try {
|
|
2388
|
+
if (!persist && !eventsCtx?.db)
|
|
2389
|
+
return { mergedRefs, replayRefSet, replayBudget };
|
|
2072
2390
|
withStateDb((replayDb) => {
|
|
2073
2391
|
const alreadyInPool = new Set(mergedRefs.map((r) => r.ref));
|
|
2074
2392
|
const storedRankScores = getAllRankScores(replayDb);
|
|
2075
|
-
|
|
2076
|
-
|
|
2077
|
-
|
|
2078
|
-
|
|
2079
|
-
|
|
2080
|
-
|
|
2081
|
-
|
|
2082
|
-
|
|
2083
|
-
|
|
2084
|
-
|
|
2085
|
-
|
|
2086
|
-
|
|
2087
|
-
|
|
2088
|
-
|
|
2089
|
-
|
|
2090
|
-
|
|
2091
|
-
|
|
2092
|
-
|
|
2093
|
-
|
|
2094
|
-
|
|
2095
|
-
|
|
2096
|
-
|
|
2097
|
-
|
|
2098
|
-
|
|
2099
|
-
//
|
|
2100
|
-
//
|
|
2101
|
-
//
|
|
2102
|
-
//
|
|
2393
|
+
const plannedByRef = new Map(plannedRefs.map((planned) => [planned.ref, planned]));
|
|
2394
|
+
const plannedByItemRef = new Map();
|
|
2395
|
+
for (const planned of plannedRefs) {
|
|
2396
|
+
if (planned.itemRef)
|
|
2397
|
+
plannedByItemRef.set(planned.itemRef, planned);
|
|
2398
|
+
}
|
|
2399
|
+
// Replay can only revisit an entry selected into THIS invocation's
|
|
2400
|
+
// source/type plan. Match durable item_ref rows by exact provenance;
|
|
2401
|
+
// legacy bare rows may match the current plan's concept ref. Folding
|
|
2402
|
+
// both spellings onto the planned ref also prevents duplicate budget
|
|
2403
|
+
// spend when old and current state rows coexist.
|
|
2404
|
+
const allRankScores = new Map();
|
|
2405
|
+
for (const [stateRef, score] of storedRankScores) {
|
|
2406
|
+
const boundary = stateRef.indexOf("//");
|
|
2407
|
+
if (options.sourceName && (boundary < 0 || stateRef.slice(0, boundary) !== options.sourceName))
|
|
2408
|
+
continue;
|
|
2409
|
+
const planned = plannedByItemRef.get(stateRef) ?? (boundary < 0 ? plannedByRef.get(bareImproveRef(stateRef)) : undefined);
|
|
2410
|
+
if (!planned)
|
|
2411
|
+
continue;
|
|
2412
|
+
const previous = allRankScores.get(planned.ref);
|
|
2413
|
+
if (previous === undefined || score > previous)
|
|
2414
|
+
allRankScores.set(planned.ref, score);
|
|
2415
|
+
}
|
|
2416
|
+
// Candidate universe = every current-plan salience match NOT already in the
|
|
2417
|
+
// pool, ordered by rank_score desc with a deterministic ref-string tie-break
|
|
2418
|
+
// (mirrors the main sort). Converged refs (consecutive_no_ops >= dampener
|
|
2419
|
+
// threshold) are fully EXCLUDED — a stronger skip than the dampener (which
|
|
2420
|
+
// only halves order).
|
|
2103
2421
|
let convergedSkipped = 0;
|
|
2104
2422
|
const candidates = [];
|
|
2105
2423
|
for (const [ref, rankScore] of allRankScores) {
|
|
2106
2424
|
if (alreadyInPool.has(ref))
|
|
2107
2425
|
continue;
|
|
2108
|
-
const
|
|
2426
|
+
const planned = plannedByRef.get(ref);
|
|
2427
|
+
if (!planned)
|
|
2428
|
+
continue;
|
|
2429
|
+
const noOps = readConsecutiveNoOpsForImproveRef(replayDb, ref, planned.itemRef);
|
|
2109
2430
|
if (noOps >= SALIENCE_NO_OP_DAMPEN_THRESHOLD) {
|
|
2110
2431
|
convergedSkipped++;
|
|
2111
2432
|
continue;
|
|
2112
2433
|
}
|
|
2113
|
-
candidates.push({
|
|
2434
|
+
candidates.push({ planned, rankScore });
|
|
2114
2435
|
}
|
|
2115
|
-
candidates.sort((a, b) => b.rankScore !== a.rankScore
|
|
2436
|
+
candidates.sort((a, b) => b.rankScore !== a.rankScore
|
|
2437
|
+
? b.rankScore - a.rankScore
|
|
2438
|
+
: a.planned.ref < b.planned.ref
|
|
2439
|
+
? -1
|
|
2440
|
+
: a.planned.ref > b.planned.ref
|
|
2441
|
+
? 1
|
|
2442
|
+
: 0);
|
|
2116
2443
|
const candidatePool = candidates.length;
|
|
2117
2444
|
const selected = candidates.slice(0, replayBudget);
|
|
2118
2445
|
const newReplayRefs = [];
|
|
2119
|
-
for (const {
|
|
2446
|
+
for (const { planned } of selected) {
|
|
2447
|
+
const ref = planned.ref;
|
|
2120
2448
|
replayRefSet.add(ref);
|
|
2121
|
-
// Synthesise a stub (mirror the forgetting-safety stub). Resolve the
|
|
2122
|
-
// backing file from the planned-ref pool so the downstream existsSync
|
|
2123
|
-
// guard keeps the ref (a replay candidate from asset_salience whose file
|
|
2124
|
-
// is gone correctly drops out). Only refs present in the indexed pool can
|
|
2125
|
-
// be revisited — refs without a planned entry get no filePath and are
|
|
2126
|
-
// dropped by the disk check, which is the desired behavior.
|
|
2127
|
-
const planned = plannedRefs.find((p) => p.ref === ref);
|
|
2128
2449
|
newReplayRefs.push({
|
|
2129
|
-
|
|
2130
|
-
reason: "scope-type",
|
|
2450
|
+
...planned,
|
|
2131
2451
|
eligibilitySource: "replay",
|
|
2132
|
-
...(planned?.filePath ? { filePath: planned.filePath } : {}),
|
|
2133
2452
|
});
|
|
2134
2453
|
// Seed the salienceMap so the sort/effectiveScore can rank the replay ref.
|
|
2135
2454
|
if (!salienceMap.has(ref)) {
|
|
@@ -2154,17 +2473,19 @@ function applyReplaySelection(args) {
|
|
|
2154
2473
|
}
|
|
2155
2474
|
}
|
|
2156
2475
|
// Aggregated observability event (never per-ref).
|
|
2157
|
-
|
|
2158
|
-
|
|
2159
|
-
|
|
2160
|
-
|
|
2161
|
-
|
|
2162
|
-
|
|
2163
|
-
|
|
2164
|
-
|
|
2165
|
-
|
|
2166
|
-
|
|
2167
|
-
|
|
2476
|
+
if (persist) {
|
|
2477
|
+
appendEvent({
|
|
2478
|
+
eventType: "improve_replay_selected",
|
|
2479
|
+
ref: undefined,
|
|
2480
|
+
metadata: {
|
|
2481
|
+
count: newReplayRefs.length,
|
|
2482
|
+
budget: replayBudget,
|
|
2483
|
+
convergedSkipped,
|
|
2484
|
+
candidatePool,
|
|
2485
|
+
},
|
|
2486
|
+
}, eventsCtx);
|
|
2487
|
+
}
|
|
2488
|
+
}, { path: eventsCtx?.dbPath, borrowed: eventsCtx?.db });
|
|
2168
2489
|
}
|
|
2169
2490
|
catch (err) {
|
|
2170
2491
|
rethrowIfTestIsolationError(err);
|
|
@@ -2175,7 +2496,7 @@ function applyReplaySelection(args) {
|
|
|
2175
2496
|
}
|
|
2176
2497
|
/** The final disk-existence guard + its aggregated audit event (eligibility-filter). */
|
|
2177
2498
|
async function dropRefsMissingOnDisk(args) {
|
|
2178
|
-
const { sorted, options, eventsCtx } = args;
|
|
2499
|
+
const { sorted, options, eventsCtx, persist } = args;
|
|
2179
2500
|
// actionableRefs is the post-cooldown, post-validation, post-signal, post-sort
|
|
2180
2501
|
// set — i.e. the genuinely processable refs in priority order. Note: this is
|
|
2181
2502
|
// a semantic shift from earlier code where actionableRefs was the pre-cooldown
|
|
@@ -2208,7 +2529,7 @@ async function dropRefsMissingOnDisk(args) {
|
|
|
2208
2529
|
// #592 audit: one summary event instead of one per missing ref. Normally
|
|
2209
2530
|
// tiny, but a stash deletion racing the run could make this O(n) sequential
|
|
2210
2531
|
// state.db writes. `refs` is capped so the metadata row stays bounded.
|
|
2211
|
-
if (assetMissingOnDisk.length > 0) {
|
|
2532
|
+
if (persist && assetMissingOnDisk.length > 0) {
|
|
2212
2533
|
appendEvent({
|
|
2213
2534
|
eventType: "improve_skipped",
|
|
2214
2535
|
ref: undefined,
|