akm-cli 0.8.7 → 0.8.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +428 -0
- package/dist/assets/help/help-proposals.md +1 -2
- package/dist/assets/hints/cli-hints-full.md +34 -19
- package/dist/assets/hints/cli-hints-short.md +1 -1
- package/dist/assets/profiles/catchup.json +13 -0
- package/dist/assets/profiles/consolidate.json +13 -0
- package/dist/assets/profiles/frequent.json +13 -0
- package/dist/assets/tasks/core/backup.yml +4 -0
- package/dist/assets/tasks/core/extract.yml +4 -0
- package/dist/assets/tasks/core/improve.yml +4 -0
- package/dist/assets/tasks/core/index-refresh.yml +4 -0
- package/dist/assets/tasks/core/sync.yml +4 -0
- package/dist/assets/tasks/core/update-stashes.yml +4 -0
- package/dist/assets/tasks/core/version-check.yml +4 -0
- package/dist/assets/templates/html/default.html +78 -0
- package/dist/assets/templates/html/health.html +560 -0
- package/dist/assets/templates/html/vendor/echarts.min.js +45 -0
- package/dist/cli/config-migrate.js +6 -6
- package/dist/cli/config-validate.js +4 -4
- package/dist/cli/confirm.js +3 -3
- package/dist/cli/parse-args.js +1 -1
- package/dist/cli/shared.js +72 -19
- package/dist/cli-node.mjs +26 -0
- package/dist/cli.js +206 -3866
- package/dist/commands/{agent-dispatch.js → agent/agent-dispatch.js} +6 -6
- package/dist/commands/{agent-support.js → agent/agent-support.js} +2 -2
- package/dist/commands/agent/contribute-cli.js +200 -0
- package/dist/commands/completions.js +1 -1
- package/dist/commands/config-cli.js +230 -3
- package/dist/commands/db-cli.js +2 -2
- package/dist/commands/env/env-cli.js +529 -0
- package/dist/commands/env/env.js +410 -0
- package/dist/commands/env/secret-cli.js +259 -0
- package/dist/commands/{secret.js → env/secret.js} +6 -47
- package/dist/commands/events.js +4 -4
- package/dist/commands/feedback-cli.js +18 -34
- package/dist/commands/graph/graph-cli.js +132 -0
- package/dist/commands/{graph.js → graph/graph.js} +22 -16
- package/dist/commands/health/checks.js +279 -0
- package/dist/commands/health/html-report.js +448 -0
- package/dist/commands/health.js +189 -266
- package/dist/commands/{consolidate.js → improve/consolidate.js} +48 -36
- package/dist/commands/{distill-promotion-policy.js → improve/distill-promotion-policy.js} +3 -3
- package/dist/commands/{distill.js → improve/distill.js} +39 -18
- package/dist/commands/{eval-cases.js → improve/eval-cases.js} +1 -1
- package/dist/commands/{extract-cli.js → improve/extract-cli.js} +4 -4
- package/dist/commands/{extract-prompt.js → improve/extract-prompt.js} +2 -2
- package/dist/commands/{extract.js → improve/extract.js} +221 -26
- package/dist/commands/{improve-auto-accept.js → improve/improve-auto-accept.js} +30 -4
- package/dist/commands/{improve-cli.js → improve/improve-cli.js} +44 -22
- package/dist/commands/{improve-profiles.js → improve/improve-profiles.js} +13 -7
- package/dist/commands/{improve-result-file.js → improve/improve-result-file.js} +1 -1
- package/dist/commands/{improve.js → improve/improve.js} +672 -292
- package/dist/{core → commands/improve/memory}/memory-belief.js +2 -2
- package/dist/{core → commands/improve/memory}/memory-contradiction-detect.js +5 -5
- package/dist/{core → commands/improve/memory}/memory-improve.js +4 -4
- package/dist/commands/improve/reflect-noise.js +0 -0
- package/dist/commands/{reflect.js → improve/reflect.js} +58 -28
- package/dist/commands/improve/session-asset.js +248 -0
- package/dist/commands/lint/agent-linter.js +1 -1
- package/dist/commands/lint/base-linter.js +55 -37
- package/dist/commands/lint/command-linter.js +1 -1
- package/dist/commands/lint/default-linter.js +1 -1
- package/dist/commands/lint/env-key-rules.js +1 -1
- package/dist/commands/lint/index.js +19 -25
- package/dist/commands/lint/knowledge-linter.js +1 -1
- package/dist/commands/lint/memory-linter.js +1 -1
- package/dist/commands/lint/registry.js +8 -8
- package/dist/commands/lint/skill-linter.js +1 -1
- package/dist/commands/lint/task-linter.js +1 -1
- package/dist/commands/lint/workflow-linter.js +1 -1
- package/dist/commands/lint.js +1 -1
- package/dist/commands/observability-cli.js +244 -0
- package/dist/commands/proposal/drain-policies.js +3 -3
- package/dist/commands/proposal/drain.js +87 -15
- package/dist/commands/proposal/proposal-cli.js +490 -0
- package/dist/commands/{proposal.js → proposal/proposal.js} +17 -6
- package/dist/commands/{propose.js → proposal/propose.js} +11 -11
- package/dist/{core → commands/proposal/validators}/proposal-quality-validators.js +8 -3
- package/dist/{core → commands/proposal/validators}/proposal-validators.js +5 -5
- package/dist/{core → commands/proposal/validators}/proposals.js +374 -345
- package/dist/commands/{curate.js → read/curate.js} +7 -7
- package/dist/commands/{knowledge.js → read/knowledge.js} +22 -9
- package/dist/commands/{registry-search.js → read/registry-search.js} +5 -5
- package/dist/commands/{remember-cli.js → read/remember-cli.js} +15 -7
- package/dist/commands/read/search-cli.js +207 -0
- package/dist/commands/{search.js → read/search.js} +22 -27
- package/dist/commands/{show.js → read/show.js} +31 -45
- package/dist/commands/registry-cli.js +8 -8
- package/dist/commands/remember.js +14 -10
- package/dist/commands/sources/add-cli.js +293 -0
- package/dist/commands/{history.js → sources/history.js} +27 -25
- package/dist/commands/{info.js → sources/info.js} +6 -6
- package/dist/commands/{init.js → sources/init.js} +6 -6
- package/dist/commands/{installed-stashes.js → sources/installed-stashes.js} +12 -12
- package/dist/commands/{migration-help.js → sources/migration-help.js} +3 -2
- package/dist/commands/{schema-repair.js → sources/schema-repair.js} +8 -8
- package/dist/commands/{self-update.js → sources/self-update.js} +10 -9
- package/dist/commands/{source-add.js → sources/source-add.js} +10 -10
- package/dist/commands/{source-clone.js → sources/source-clone.js} +7 -7
- package/dist/commands/{source-manage.js → sources/source-manage.js} +4 -4
- package/dist/commands/sources/sources-cli.js +305 -0
- package/dist/commands/sources/stash-cli.js +219 -0
- package/dist/commands/{stash-skeleton.js → sources/stash-skeleton.js} +2 -1
- package/dist/commands/tasks/default-tasks.js +173 -0
- package/dist/commands/tasks/tasks-cli.js +210 -0
- package/dist/commands/{tasks.js → tasks/tasks.js} +14 -14
- package/dist/commands/wiki-cli.js +307 -0
- package/dist/commands/workflow-cli.js +329 -0
- package/dist/core/action-contributors.js +1 -1
- package/dist/core/assert.js +40 -0
- package/dist/core/asset/asset-create.js +54 -0
- package/dist/core/{asset-ref.js → asset/asset-ref.js} +21 -4
- package/dist/core/{asset-registry.js → asset/asset-registry.js} +3 -3
- package/dist/core/{asset-spec.js → asset/asset-spec.js} +17 -31
- package/dist/core/{markdown.js → asset/markdown.js} +1 -1
- package/dist/core/{stash-meta.js → asset/stash-meta.js} +1 -1
- package/dist/core/best-effort.js +64 -0
- package/dist/core/common.js +32 -18
- package/dist/core/{config-io.js → config/config-io.js} +29 -19
- package/dist/core/{config-migration.js → config/config-migration.js} +11 -9
- package/dist/core/{config-schema.js → config/config-schema.js} +50 -7
- package/dist/core/config/config-types.js +16 -0
- package/dist/core/{config-walker.js → config/config-walker.js} +2 -2
- package/dist/core/{config.js → config/config.js} +10 -8
- package/dist/core/env-secret-ref.js +90 -0
- package/dist/core/errors.js +13 -3
- package/dist/core/events.js +27 -4
- package/dist/core/file-lock.js +1 -1
- package/dist/core/improve-types.js +48 -0
- package/dist/core/lesson-lint.js +2 -2
- package/dist/core/logs-db.js +304 -0
- package/dist/core/paths.js +2 -2
- package/dist/core/ripgrep/install.js +2 -2
- package/dist/core/ripgrep/resolve.js +2 -2
- package/dist/core/state-db.js +195 -60
- package/dist/core/text-truncation.js +148 -0
- package/dist/core/time.js +1 -1
- package/dist/core/write-source.js +98 -85
- package/dist/indexer/{db-backup.js → db/db-backup.js} +9 -24
- package/dist/indexer/{db.js → db/db.js} +128 -118
- package/dist/indexer/{graph-db.js → db/graph-db.js} +9 -4
- package/dist/indexer/{llm-cache.js → db/llm-cache.js} +15 -12
- package/dist/indexer/ensure-index.js +4 -4
- package/dist/indexer/{graph-boost.js → graph/graph-boost.js} +1 -1
- package/dist/indexer/{graph-extraction.js → graph/graph-extraction.js} +55 -13
- package/dist/indexer/indexer.js +37 -30
- package/dist/indexer/init.js +54 -0
- package/dist/indexer/manifest.js +10 -10
- package/dist/indexer/{memory-inference.js → passes/memory-inference.js} +141 -33
- package/dist/indexer/{metadata-contributors.js → passes/metadata-contributors.js} +10 -8
- package/dist/indexer/{metadata.js → passes/metadata.js} +15 -19
- package/dist/indexer/{staleness-detect.js → passes/staleness-detect.js} +53 -12
- package/dist/indexer/{db-search.js → search/db-search.js} +28 -16
- package/dist/indexer/{ranking-contributors.js → search/ranking-contributors.js} +1 -1
- package/dist/indexer/{ranking.js → search/ranking.js} +2 -2
- package/dist/indexer/{search-hit-enrichers.js → search/search-hit-enrichers.js} +3 -3
- package/dist/indexer/{search-source.js → search/search-source.js} +8 -8
- package/dist/indexer/{semantic-status.js → search/semantic-status.js} +3 -3
- package/dist/indexer/usage/unmigrated-vaults-guard.js +94 -0
- package/dist/indexer/{usage-events.js → usage/usage-events.js} +32 -0
- package/dist/indexer/{file-context.js → walk/file-context.js} +10 -15
- package/dist/indexer/{matchers.js → walk/matchers.js} +13 -9
- package/dist/indexer/{path-resolver.js → walk/path-resolver.js} +6 -6
- package/dist/indexer/{project-context.js → walk/project-context.js} +1 -1
- package/dist/indexer/{walker.js → walk/walker.js} +4 -3
- package/dist/integrations/agent/builder-shared.js +39 -0
- package/dist/integrations/agent/builders.js +14 -81
- package/dist/integrations/agent/config.js +6 -4
- package/dist/integrations/agent/detect.js +1 -1
- package/dist/integrations/agent/index.js +23 -8
- package/dist/integrations/agent/prompts.js +2 -3
- package/dist/integrations/agent/runner.js +22 -3
- package/dist/integrations/agent/spawn.js +9 -10
- package/dist/integrations/harnesses/claude/agent-builder.js +48 -0
- package/dist/integrations/harnesses/claude/config-import.js +70 -0
- package/dist/integrations/harnesses/claude/index.js +64 -0
- package/dist/integrations/{session-logs/providers/claude-code.js → harnesses/claude/session-log.js} +32 -5
- package/dist/integrations/harnesses/index.js +144 -0
- package/dist/integrations/harnesses/opencode/agent-builder.js +43 -0
- package/dist/integrations/harnesses/opencode/config-import.js +82 -0
- package/dist/integrations/harnesses/opencode/index.js +59 -0
- package/dist/integrations/{session-logs/providers/opencode.js → harnesses/opencode/session-log.js} +1 -1
- package/dist/integrations/harnesses/opencode-sdk/index.js +49 -0
- package/dist/integrations/harnesses/opencode-sdk/sdk-runner.js +234 -0
- package/dist/integrations/harnesses/types.js +43 -0
- package/dist/integrations/lockfile.js +7 -16
- package/dist/integrations/session-logs/index.js +82 -9
- package/dist/llm/call-ai.js +4 -4
- package/dist/llm/client.js +146 -6
- package/dist/llm/embedder.js +6 -6
- package/dist/llm/embedders/local.js +9 -22
- package/dist/llm/embedders/remote.js +2 -2
- package/dist/llm/embedders/types.js +1 -1
- package/dist/llm/graph-extract.js +31 -12
- package/dist/llm/index-passes.js +1 -1
- package/dist/llm/memory-infer.js +12 -5
- package/dist/llm/metadata-enhance.js +2 -2
- package/dist/llm/usage-persist.js +77 -0
- package/dist/llm/usage-telemetry.js +103 -0
- package/dist/output/context.js +9 -46
- package/dist/output/html-render.js +73 -0
- package/dist/output/renderers.js +88 -58
- package/dist/output/shapes/curate.js +7 -3
- package/dist/output/shapes/distill.js +7 -3
- package/dist/output/shapes/env-list.js +18 -16
- package/dist/output/shapes/events.js +5 -4
- package/dist/output/shapes/helpers.js +19 -5
- package/dist/output/shapes/history.js +7 -3
- package/dist/output/shapes/passthrough.js +8 -11
- package/dist/output/shapes/{proposal-accept.js → proposal/accept.js} +7 -3
- package/dist/output/shapes/{proposal-diff.js → proposal/diff.js} +7 -3
- package/dist/output/shapes/{proposal-list.js → proposal/list.js} +7 -3
- package/dist/output/shapes/{proposal-producer.js → proposal/producer.js} +5 -4
- package/dist/output/shapes/{proposal-reject.js → proposal/reject.js} +7 -3
- package/dist/output/shapes/{proposal-show.js → proposal/show.js} +7 -3
- package/dist/output/shapes/registry-search.js +7 -3
- package/dist/output/shapes/registry.js +12 -0
- package/dist/output/shapes/search.js +7 -3
- package/dist/output/shapes/secret-list.js +18 -16
- package/dist/output/shapes/show.js +7 -3
- package/dist/output/shapes.js +55 -30
- package/dist/output/text/add.js +2 -3
- package/dist/output/text/clone.js +2 -3
- package/dist/output/text/config.js +2 -3
- package/dist/output/text/curate.js +4 -3
- package/dist/output/text/distill.js +2 -3
- package/dist/output/text/enable-disable.js +5 -4
- package/dist/output/text/env.js +13 -0
- package/dist/output/text/events.js +5 -4
- package/dist/output/text/feedback.js +4 -3
- package/dist/output/text/helpers.js +123 -40
- package/dist/output/text/history.js +2 -3
- package/dist/output/text/import.js +2 -3
- package/dist/output/text/index.js +2 -3
- package/dist/output/text/info.js +2 -3
- package/dist/output/text/init.js +2 -3
- package/dist/output/text/list.js +2 -3
- package/dist/output/text/proposal/producer.js +9 -0
- package/dist/output/text/proposal/proposal.js +13 -0
- package/dist/output/text/registry-commands.js +8 -7
- package/dist/output/text/registry.js +12 -0
- package/dist/output/text/remember.js +4 -3
- package/dist/output/text/remove.js +2 -3
- package/dist/output/text/save.js +2 -3
- package/dist/output/text/search.js +4 -3
- package/dist/output/text/show.js +4 -3
- package/dist/output/text/update.js +2 -3
- package/dist/output/text/upgrade.js +2 -3
- package/dist/output/text/wiki.js +12 -11
- package/dist/output/text/workflow.js +12 -10
- package/dist/output/text.js +66 -32
- package/dist/registry/build-index.js +11 -10
- package/dist/registry/factory.js +1 -1
- package/dist/registry/origin-resolve.js +1 -1
- package/dist/registry/providers/index.js +2 -2
- package/dist/registry/providers/skills-sh.js +91 -72
- package/dist/registry/providers/static-index.js +75 -52
- package/dist/registry/resolve.js +3 -3
- package/dist/runtime.js +242 -0
- package/dist/scripts/migrate-storage.js +1654 -683
- package/dist/scripts/migrations/import-fs-improve-runs-to-db.js +254 -168
- package/dist/setup/detect.js +311 -9
- package/dist/setup/harness-config-import.js +6 -120
- package/dist/setup/setup.js +454 -43
- package/dist/sources/include.js +1 -1
- package/dist/sources/provider-factory.js +2 -2
- package/dist/sources/providers/filesystem.js +3 -3
- package/dist/sources/providers/git.js +9 -9
- package/dist/sources/providers/index.js +4 -4
- package/dist/sources/providers/npm.js +6 -6
- package/dist/sources/providers/provider-utils.js +13 -20
- package/dist/sources/providers/sync-from-ref.js +5 -5
- package/dist/sources/providers/tar-utils.js +2 -2
- package/dist/sources/providers/website.js +2 -2
- package/dist/sources/resolve.js +5 -5
- package/dist/sources/website-ingest.js +5 -5
- package/dist/storage/database.js +102 -0
- package/dist/storage/engines/sqlite-migrations.js +42 -0
- package/dist/storage/locations.js +25 -0
- package/dist/storage/repositories/index-db.js +43 -0
- package/dist/storage/repositories/workflow-runs-repository.js +141 -0
- package/dist/tasks/backends/cron.js +4 -4
- package/dist/tasks/backends/exec-utils.js +32 -0
- package/dist/tasks/backends/index.js +3 -3
- package/dist/tasks/backends/launchd.js +7 -14
- package/dist/tasks/backends/schtasks.js +7 -16
- package/dist/tasks/embedded.js +71 -0
- package/dist/tasks/parser.js +2 -2
- package/dist/tasks/resolveAkmBin.js +1 -1
- package/dist/tasks/runner.js +127 -31
- package/dist/tasks/schedule.js +1 -1
- package/dist/tasks/validator.js +7 -7
- package/dist/text-import-hook.mjs +51 -0
- package/dist/version.js +2 -1
- package/dist/wiki/wiki.js +7 -7
- package/dist/workflows/{authoring.js → authoring/authoring.js} +6 -6
- package/dist/workflows/{scope-key.js → authoring/scope-key.js} +1 -1
- package/dist/workflows/cli.js +1 -1
- package/dist/workflows/db.js +54 -32
- package/dist/workflows/parser.js +4 -4
- package/dist/workflows/renderer.js +5 -5
- package/dist/workflows/runtime/agent-identity.js +56 -0
- package/dist/workflows/runtime/checkin.js +57 -0
- package/dist/workflows/{runs.js → runtime/runs.js} +197 -101
- package/dist/workflows/validate-summary.js +82 -0
- package/docs/README.md +1 -1
- package/docs/data-and-telemetry.md +6 -6
- package/package.json +17 -8
- package/dist/commands/add-cli.js +0 -279
- package/dist/commands/env.js +0 -213
- package/dist/integrations/agent/sdk-runner.js +0 -126
- package/dist/output/shapes/vault-list.js +0 -19
- package/dist/output/text/proposal-producer.js +0 -8
- package/dist/output/text/proposal.js +0 -12
- package/dist/output/text/vault.js +0 -16
- /package/dist/core/{asset-serialize.js → asset/asset-serialize.js} +0 -0
- /package/dist/core/{frontmatter.js → asset/frontmatter.js} +0 -0
- /package/dist/core/{config-sources.js → config/config-sources.js} +0 -0
- /package/dist/indexer/{graph-dedup.js → graph/graph-dedup.js} +0 -0
- /package/dist/{core/config-types.js → indexer/passes/pass-context.js} +0 -0
- /package/dist/indexer/{search-fields.js → search/search-fields.js} +0 -0
- /package/dist/indexer/{index-context.js → walk/index-context.js} +0 -0
- /package/dist/workflows/{document-cache.js → runtime/document-cache.js} +0 -0
|
@@ -3,44 +3,50 @@
|
|
|
3
3
|
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
4
4
|
import fs from "node:fs";
|
|
5
5
|
import path from "node:path";
|
|
6
|
-
import {
|
|
7
|
-
import {
|
|
8
|
-
import {
|
|
9
|
-
import {
|
|
10
|
-
import {
|
|
11
|
-
import {
|
|
12
|
-
import {
|
|
13
|
-
import {
|
|
14
|
-
import {
|
|
15
|
-
import {
|
|
16
|
-
import {
|
|
17
|
-
import { openStateDatabase, purgeOldEvents, purgeOldImproveRuns } from "
|
|
18
|
-
import { info, warn } from "
|
|
19
|
-
import { closeDatabase, getAllEntries, getEntryCount, getRetrievalCounts, getUtilityScoresByIds, getZeroResultSearches, openDatabase, openExistingDatabase, } from "
|
|
20
|
-
import { ensureIndex } from "
|
|
21
|
-
import { runGraphExtractionPass } from "
|
|
22
|
-
import { akmIndex } from "
|
|
23
|
-
import { runMemoryInferencePass
|
|
24
|
-
import {
|
|
25
|
-
import { getWritableStashDirs, resolveSourceEntries } from "
|
|
26
|
-
import {
|
|
27
|
-
import {
|
|
28
|
-
import {
|
|
29
|
-
import {
|
|
30
|
-
import {
|
|
31
|
-
import {
|
|
32
|
-
import {
|
|
33
|
-
import {
|
|
34
|
-
import {
|
|
35
|
-
import {
|
|
36
|
-
import {
|
|
37
|
-
import {
|
|
38
|
-
import {
|
|
39
|
-
import {
|
|
40
|
-
import {
|
|
41
|
-
import {
|
|
42
|
-
import {
|
|
43
|
-
import {
|
|
6
|
+
import { assertNever } from "../../core/assert.js";
|
|
7
|
+
import { makeAssetRef, parseAssetRef } from "../../core/asset/asset-ref.js";
|
|
8
|
+
import { parseFrontmatter } from "../../core/asset/frontmatter.js";
|
|
9
|
+
import { daysToMs, isAssetType } from "../../core/common.js";
|
|
10
|
+
import { getDefaultLlmConfig, loadConfig } from "../../core/config/config.js";
|
|
11
|
+
import { ConfigError, NotFoundError, rethrowIfTestIsolationError, UsageError } from "../../core/errors.js";
|
|
12
|
+
import { appendEvent, readEvents } from "../../core/events.js";
|
|
13
|
+
import { probeLock, releaseLock, releaseLockIfOwned, tryAcquireLockSync } from "../../core/file-lock.js";
|
|
14
|
+
import { classifyImproveAction } from "../../core/improve-types.js";
|
|
15
|
+
import { openLogsDatabase, purgeOldTaskLogs } from "../../core/logs-db.js";
|
|
16
|
+
import { getDbPath, getStateDbPathInDataDir } from "../../core/paths.js";
|
|
17
|
+
import { openStateDatabase, purgeOldEvents, purgeOldImproveRuns } from "../../core/state-db.js";
|
|
18
|
+
import { info, warn } from "../../core/warn.js";
|
|
19
|
+
import { closeDatabase, getAllEntries, getEntryCount, getRetrievalCounts, getUtilityScoresByIds, getZeroResultSearches, openDatabase, openExistingDatabase, } from "../../indexer/db/db.js";
|
|
20
|
+
import { ensureIndex } from "../../indexer/ensure-index.js";
|
|
21
|
+
import { runGraphExtractionPass } from "../../indexer/graph/graph-extraction.js";
|
|
22
|
+
import { akmIndex } from "../../indexer/indexer.js";
|
|
23
|
+
import { runMemoryInferencePass } from "../../indexer/passes/memory-inference.js";
|
|
24
|
+
import { runStalenessDetectionPass } from "../../indexer/passes/staleness-detect.js";
|
|
25
|
+
import { getWritableStashDirs, resolveSourceEntries } from "../../indexer/search/search-source.js";
|
|
26
|
+
import { countUsageEventsByType } from "../../indexer/usage/usage-events.js";
|
|
27
|
+
import { resolveAssetPath } from "../../indexer/walk/path-resolver.js";
|
|
28
|
+
import { resolveImproveProcessRunnerFromProfile, resolveTriageJudgmentRunner } from "../../integrations/agent/runner.js";
|
|
29
|
+
import { getAvailableHarnesses } from "../../integrations/session-logs/index.js";
|
|
30
|
+
import { isLlmFeatureEnabled, isProcessEnabled } from "../../llm/feature-gate.js";
|
|
31
|
+
import { installLlmUsagePersistence } from "../../llm/usage-persist.js";
|
|
32
|
+
import { withLlmStage } from "../../llm/usage-telemetry.js";
|
|
33
|
+
import { isGitBackedStash, resolveWritableOverride, saveGitStash } from "../../sources/providers/git.js";
|
|
34
|
+
import { akmLint } from "../lint/index.js";
|
|
35
|
+
import { drainProposals } from "../proposal/drain.js";
|
|
36
|
+
import { resolveDrainPolicy } from "../proposal/drain-policies.js";
|
|
37
|
+
import { createProposal, expireStaleProposals, getProposal, isProposalSkipped, listProposals, purgeOrphanProposals, } from "../proposal/validators/proposals.js";
|
|
38
|
+
import { runSchemaRepairPass } from "../sources/schema-repair.js";
|
|
39
|
+
import { checkDeadUrls } from "../url-checker.js";
|
|
40
|
+
import { akmConsolidate } from "./consolidate.js";
|
|
41
|
+
import { akmDistill, deriveLessonRef, isDistillRefusedInputType } from "./distill.js";
|
|
42
|
+
import { deriveKnowledgeRef } from "./distill-promotion-policy.js";
|
|
43
|
+
import { countEvalCases, writeEvalCase } from "./eval-cases.js";
|
|
44
|
+
import { akmExtract, countNewExtractCandidates } from "./extract.js";
|
|
45
|
+
import { makeGateConfig, resolveExtractConfidence, runAutoAcceptGate } from "./improve-auto-accept.js";
|
|
46
|
+
import { isProfileFilteredForAllPasses, resolveImproveProfile, resolveProcessEnabled, shouldSkipRef, } from "./improve-profiles.js";
|
|
47
|
+
import { detectAndWriteContradictions } from "./memory/memory-contradiction-detect.js";
|
|
48
|
+
import { analyzeMemoryCleanup, applyMemoryCleanup } from "./memory/memory-improve.js";
|
|
49
|
+
import { akmReflect } from "./reflect.js";
|
|
44
50
|
function resolveImproveScope(scope) {
|
|
45
51
|
const trimmed = scope?.trim();
|
|
46
52
|
if (!trimmed)
|
|
@@ -51,7 +57,7 @@ function resolveImproveScope(scope) {
|
|
|
51
57
|
}
|
|
52
58
|
catch {
|
|
53
59
|
if (!isAssetType(trimmed)) {
|
|
54
|
-
throw new UsageError(`Unknown asset type: "${trimmed}". Valid types: memory, knowledge, skill, lesson, workflow, agent, command, script, wiki, env,
|
|
60
|
+
throw new UsageError(`Unknown asset type: "${trimmed}". Valid types: memory, knowledge, skill, lesson, workflow, agent, command, script, wiki, env, secret, task.\n` +
|
|
55
61
|
`If you passed --format to akm improve, that flag is not supported — use it with akm search or akm show instead.`, "INVALID_FLAG_VALUE");
|
|
56
62
|
}
|
|
57
63
|
return { mode: "type", value: trimmed };
|
|
@@ -70,6 +76,9 @@ function resolveImproveScope(scope) {
|
|
|
70
76
|
* {scope} scope value (e.g. a ref/type) or the scope mode (`all`)
|
|
71
77
|
* {refs} number of planned refs this run processed
|
|
72
78
|
* {accepted} number of proposals auto-accepted by the confidence gate
|
|
79
|
+
* {triage_promoted} proposals promoted by the triage pre-pass (0 if triage did not run)
|
|
80
|
+
* {triage_rejected} proposals rejected by the triage pre-pass (0 if triage did not run)
|
|
81
|
+
* {runId} this run's id (empty string when absent)
|
|
73
82
|
*
|
|
74
83
|
* The result is still passed through `sanitizeCommitMessage` downstream in
|
|
75
84
|
* `saveGitStash`, so token values never widen the commit-message attack surface
|
|
@@ -87,6 +96,9 @@ export function renderSyncCommitMessage(template, result, nowMs) {
|
|
|
87
96
|
scope: result.scope.value ?? result.scope.mode,
|
|
88
97
|
refs: String(result.plannedRefs.length),
|
|
89
98
|
accepted: String(result.gateAutoAcceptedCount ?? 0),
|
|
99
|
+
triage_promoted: String(result.triage?.promoted ?? 0),
|
|
100
|
+
triage_rejected: String(result.triage?.rejected ?? 0),
|
|
101
|
+
runId: result.runId ?? "",
|
|
90
102
|
};
|
|
91
103
|
return template.replace(/\{(\w+)\}/g, (match, key) => (Object.hasOwn(tokens, key) ? tokens[key] : match));
|
|
92
104
|
}
|
|
@@ -103,7 +115,7 @@ async function collectEligibleRefs(scope, stashDir, improveProfile) {
|
|
|
103
115
|
};
|
|
104
116
|
}
|
|
105
117
|
return {
|
|
106
|
-
plannedRefs: [{ ref: scope.value, reason: "scope-ref" }],
|
|
118
|
+
plannedRefs: [{ ref: scope.value, reason: "scope-ref", filePath }],
|
|
107
119
|
memorySummary: {
|
|
108
120
|
eligible: parsed.type === "memory" ? 1 : 0,
|
|
109
121
|
derived: parsed.type === "memory" && parsed.name.endsWith(".derived") ? 1 : 0,
|
|
@@ -167,12 +179,14 @@ async function collectEligibleRefs(scope, stashDir, improveProfile) {
|
|
|
167
179
|
profileFiltered.set(ref, {
|
|
168
180
|
ref,
|
|
169
181
|
reason: "profile_filtered_all_passes",
|
|
182
|
+
filePath: indexed.filePath,
|
|
170
183
|
});
|
|
171
184
|
}
|
|
172
185
|
else {
|
|
173
186
|
planned.set(ref, {
|
|
174
187
|
ref,
|
|
175
188
|
reason: scope.mode === "type" ? "scope-type" : indexed.entry.type === "memory" ? "memory-cleanup" : "scope-type",
|
|
189
|
+
filePath: indexed.filePath,
|
|
176
190
|
});
|
|
177
191
|
}
|
|
178
192
|
}
|
|
@@ -395,6 +409,51 @@ function isSignalDeltaEligible(ref, latestFeedback, lastProposal) {
|
|
|
395
409
|
return true;
|
|
396
410
|
return fb > lp;
|
|
397
411
|
}
|
|
412
|
+
/**
|
|
413
|
+
* H7 (#566): cooperative budget watchdog with a captured, RAII-cleared hard-kill.
|
|
414
|
+
*
|
|
415
|
+
* When the wall-clock budget expires, `onExhausted` (normally an
|
|
416
|
+
* `AbortController.abort`) signals cooperative cancellation so the run can drain
|
|
417
|
+
* its in-flight log/`state.db` flush and unwind naturally. A second hard-kill
|
|
418
|
+
* timer is then armed as a watchdog: it only `exit(0)`s if the drain itself
|
|
419
|
+
* overruns `hardKillGraceMs`, preventing the process from outliving the task
|
|
420
|
+
* timeout window (lock-cascade fix).
|
|
421
|
+
*
|
|
422
|
+
* Both timers are captured; the returned dispose() clears whichever is still
|
|
423
|
+
* pending. Callers invoke it from a `finally`, so a *clean* drain reaches the
|
|
424
|
+
* `finally` and cancels the pending hard-kill before it can fire — the previous
|
|
425
|
+
* detached `setTimeout(() => process.exit(0), 5000)` always fired, truncating a
|
|
426
|
+
* clean flush. The hard-kill timer is `unref()`-ed so it never keeps the event
|
|
427
|
+
* loop alive on its own: once the run drains it exits with its own code, not the
|
|
428
|
+
* forced 0.
|
|
429
|
+
*
|
|
430
|
+
* Dependencies are injectable purely so the concurrency-sensitive timing
|
|
431
|
+
* contract can be exercised deterministically in unit tests.
|
|
432
|
+
*/
|
|
433
|
+
export function armBudgetWatchdog(budgetMs, controller, deps) {
|
|
434
|
+
const setTimeoutFn = deps?.setTimeoutFn ?? setTimeout;
|
|
435
|
+
const clearTimeoutFn = deps?.clearTimeoutFn ?? clearTimeout;
|
|
436
|
+
const exitFn = deps?.exitFn ?? ((code) => process.exit(code));
|
|
437
|
+
const hardKillGraceMs = deps?.hardKillGraceMs ?? 5_000;
|
|
438
|
+
let hardKillTimer;
|
|
439
|
+
const budgetTimer = setTimeoutFn(() => {
|
|
440
|
+
// Cooperative cancellation first: let the run drain.
|
|
441
|
+
controller.abort("improve budget exhausted");
|
|
442
|
+
// Watchdog: only force-exit if the drain itself overruns the grace period.
|
|
443
|
+
// Exit 0: budget exhaustion is a normal scheduled-task condition, not an error.
|
|
444
|
+
hardKillTimer = setTimeoutFn(() => exitFn(0), hardKillGraceMs);
|
|
445
|
+
// Never keep the event loop alive solely for the watchdog.
|
|
446
|
+
hardKillTimer.unref?.();
|
|
447
|
+
}, budgetMs);
|
|
448
|
+
// RAII dispose: clears whichever timer is still pending. Idempotent.
|
|
449
|
+
return () => {
|
|
450
|
+
clearTimeoutFn(budgetTimer);
|
|
451
|
+
if (hardKillTimer !== undefined) {
|
|
452
|
+
clearTimeoutFn(hardKillTimer);
|
|
453
|
+
hardKillTimer = undefined;
|
|
454
|
+
}
|
|
455
|
+
};
|
|
456
|
+
}
|
|
398
457
|
export async function akmImprove(options = {}) {
|
|
399
458
|
const scope = resolveImproveScope(options.scope);
|
|
400
459
|
const reflectFn = options.reflectFn ?? akmReflect;
|
|
@@ -421,6 +480,15 @@ export async function akmImprove(options = {}) {
|
|
|
421
480
|
catch {
|
|
422
481
|
primaryStashDir = undefined;
|
|
423
482
|
}
|
|
483
|
+
// C2 (#553/#554/#499): resolve the state.db path ONCE, synchronously, at the
|
|
484
|
+
// command boundary — before the first `await` below. Every state.db open in
|
|
485
|
+
// this run (`openStateDatabase`, every default-path `appendEvent`) is pinned
|
|
486
|
+
// to this snapshot via `eventsCtx.dbPath`, so a parallel test file mutating
|
|
487
|
+
// `process.env.XDG_DATA_HOME` across an await boundary can never redirect this
|
|
488
|
+
// run's DB opens to a wrong/just-deleted tmpdir mid-flight (the parallel-load
|
|
489
|
+
// timeout root cause). Because beforeEach runs synchronously, env is still the
|
|
490
|
+
// calling test's own at this point; we capture it before yielding the loop.
|
|
491
|
+
const resolvedStateDbPath = getStateDbPathInDataDir();
|
|
424
492
|
// Phase 4 lock hoist (§7): the `improve.lock` setup is hoisted ABOVE
|
|
425
493
|
// ensureIndex/collectEligibleRefs so the triage pre-pass (and improve's own
|
|
426
494
|
// queue writes) run fully serialized under the lock. The dry-run early-return
|
|
@@ -507,14 +575,14 @@ export async function akmImprove(options = {}) {
|
|
|
507
575
|
}
|
|
508
576
|
lockAcquired = false;
|
|
509
577
|
};
|
|
510
|
-
// Signal-safe lock release
|
|
511
|
-
//
|
|
512
|
-
//
|
|
513
|
-
// `
|
|
514
|
-
//
|
|
515
|
-
//
|
|
516
|
-
//
|
|
517
|
-
//
|
|
578
|
+
// Signal-safe lock release. The SIGTERM/SIGINT/SIGHUP handler in improve-cli.ts
|
|
579
|
+
// calls `process.exit()`, which does NOT run the `finally` below that owns lock
|
|
580
|
+
// release — so a cron-timeout SIGTERM leaked `improve.lock` every run.
|
|
581
|
+
// `process.exit()` DOES fire `'exit'` listeners, so we release the lock from
|
|
582
|
+
// one. `releaseLockIfOwned` only unlinks a lock still owned by this PID, so it
|
|
583
|
+
// is safe even if a later run re-acquired it. The listener is removed in the
|
|
584
|
+
// `finally` so the normal path stays single-release and repeated in-process
|
|
585
|
+
// `akmImprove` calls (tests) do not accumulate listeners.
|
|
518
586
|
const releaseLockOnExit = () => {
|
|
519
587
|
releaseLockIfOwned(resolvedLockPath, process.pid);
|
|
520
588
|
};
|
|
@@ -524,6 +592,7 @@ export async function akmImprove(options = {}) {
|
|
|
524
592
|
let profileFilteredRefs;
|
|
525
593
|
let memoryCleanupPlan;
|
|
526
594
|
let guidance;
|
|
595
|
+
let triageDrain;
|
|
527
596
|
try {
|
|
528
597
|
// Acquire the lock and run the triage pre-pass for non-dry-run executions.
|
|
529
598
|
// The dry-run branch below produces plannedRefs/memorySummary WITHOUT the lock
|
|
@@ -569,7 +638,7 @@ export async function akmImprove(options = {}) {
|
|
|
569
638
|
const judgment = triageConfig?.judgment
|
|
570
639
|
? resolveTriageJudgmentRunner(triageConfig.judgment, _earlyConfig)
|
|
571
640
|
: null;
|
|
572
|
-
await drainProposalsFn({
|
|
641
|
+
triageDrain = await drainProposalsFn({
|
|
573
642
|
stashDir: primaryStashDir,
|
|
574
643
|
policy,
|
|
575
644
|
applyMode,
|
|
@@ -653,7 +722,7 @@ export async function akmImprove(options = {}) {
|
|
|
653
722
|
if (primaryStashDir && shouldAnalyzeMemoryCleanup(scope, memorySummary.eligible, primaryStashDir)) {
|
|
654
723
|
try {
|
|
655
724
|
// Reuse the config resolved at the top of the run instead of a second load.
|
|
656
|
-
await detectAndWriteContradictions(primaryStashDir, _earlyConfig);
|
|
725
|
+
await withLlmStage("memory-contradiction", () => detectAndWriteContradictions(primaryStashDir, _earlyConfig));
|
|
657
726
|
}
|
|
658
727
|
catch (err) {
|
|
659
728
|
// Non-fatal: contradiction detection is a best-effort pass.
|
|
@@ -710,37 +779,51 @@ export async function akmImprove(options = {}) {
|
|
|
710
779
|
let eventsDb;
|
|
711
780
|
// `eventsCtx` is read by the main catch (improve_failed) and finally, so it
|
|
712
781
|
// lives in the outer scope. It is always assigned at the top of the try.
|
|
713
|
-
|
|
782
|
+
// Pinned to the boundary snapshot so the fallback per-call `appendEvent`
|
|
783
|
+
// opens (when the long-lived handle below fails to open) never re-read env.
|
|
784
|
+
let eventsCtx = { dbPath: resolvedStateDbPath };
|
|
785
|
+
// #576: clears the per-run LLM usage sink. Defaults to a no-op until the sink
|
|
786
|
+
// is installed inside the try; the `finally` always calls it.
|
|
787
|
+
let disposeLlmUsageSink = () => { };
|
|
714
788
|
try {
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
clearBudgetTimer = () => clearTimeout(budgetTimer);
|
|
789
|
+
// H7 (#566): arm the budget watchdog. `armBudgetWatchdog` captures both the
|
|
790
|
+
// budget timer and the hard-kill timer it schedules on exhaustion, returning
|
|
791
|
+
// a single dispose() that clears whichever are still pending. The `finally`
|
|
792
|
+
// calls dispose() via `clearBudgetTimer` (RAII), so a clean cooperative
|
|
793
|
+
// drain cancels the pending hard-kill before it can fire — the process then
|
|
794
|
+
// exits naturally instead of being force-`exit(0)`-ed mid-flush, which could
|
|
795
|
+
// truncate an in-flight log or `state.db` transaction.
|
|
796
|
+
clearBudgetTimer = armBudgetWatchdog(budgetMs, budgetAbortController);
|
|
724
797
|
try {
|
|
725
|
-
eventsDb = openStateDatabase();
|
|
798
|
+
eventsDb = openStateDatabase(resolvedStateDbPath);
|
|
726
799
|
eventsCtx = { db: eventsDb };
|
|
727
800
|
}
|
|
728
801
|
catch (err) {
|
|
729
802
|
rethrowIfTestIsolationError(err);
|
|
730
|
-
// If we cannot open state.db up-front, fall back to per-call opens
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
803
|
+
// If we cannot open state.db up-front, fall back to per-call opens — but
|
|
804
|
+
// still pinned to the boundary-resolved path, never a live env re-read.
|
|
805
|
+
eventsCtx = { dbPath: resolvedStateDbPath };
|
|
806
|
+
}
|
|
807
|
+
// #576: persist per-call LLM usage telemetry for this run as `llm_usage`
|
|
808
|
+
// events, reusing the same boundary-pinned events context (and long-lived
|
|
809
|
+
// handle when available). Disposed in `finally` so the sink never leaks
|
|
810
|
+
// across runs. Wrapping is best-effort end to end — see usage-telemetry.ts.
|
|
811
|
+
disposeLlmUsageSink = installLlmUsagePersistence(eventsCtx);
|
|
812
|
+
// 2026-05-27: emit an `improve_skipped` audit event for refs the planner
|
|
734
813
|
// pre-filtered (reflect AND distill both refuse them under the active
|
|
735
|
-
// profile).
|
|
736
|
-
//
|
|
737
|
-
//
|
|
738
|
-
//
|
|
739
|
-
|
|
814
|
+
// profile). Emitted as a single summary event (count only) rather than one
|
|
815
|
+
// event per ref (#592) — the per-ref loop caused O(n) sequential state.db
|
|
816
|
+
// writes that consumed ~500 s on a 9 000-ref stash. No downstream consumer
|
|
817
|
+
// needs the per-ref audit trail: health's skip histogram reads the
|
|
818
|
+
// `profile_filtered_all_passes` counters from `improve_completed` metadata.
|
|
819
|
+
if (profileFilteredRefs.length > 0) {
|
|
740
820
|
appendEvent({
|
|
741
821
|
eventType: "improve_skipped",
|
|
742
|
-
ref:
|
|
743
|
-
metadata: {
|
|
822
|
+
ref: undefined,
|
|
823
|
+
metadata: {
|
|
824
|
+
reason: "profile_filtered_all_passes",
|
|
825
|
+
count: profileFilteredRefs.length,
|
|
826
|
+
},
|
|
744
827
|
}, eventsCtx);
|
|
745
828
|
}
|
|
746
829
|
const preparation = await runImprovePreparationStage({
|
|
@@ -787,20 +870,23 @@ export async function akmImprove(options = {}) {
|
|
|
787
870
|
eventsCtx,
|
|
788
871
|
improveProfile,
|
|
789
872
|
});
|
|
790
|
-
|
|
873
|
+
// #551: consolidation now runs in the preparation stage (before extract);
|
|
874
|
+
// its result and run-flag are read from `preparation`, not the post-loop.
|
|
875
|
+
const consolidation = preparation.consolidation;
|
|
876
|
+
const { allWarnings, deadUrls, memoryInference, graphExtraction, stalenessDetection, maintenanceActions, memoryInferenceDurationMs, graphExtractionDurationMs, orphansPurged, proposalsExpired, gateAutoAcceptedCount: postLoopGateCount, gateAutoAcceptFailedCount: postLoopGateFailedCount, } = await runImprovePostLoopStage({
|
|
791
877
|
scope,
|
|
792
878
|
options,
|
|
793
879
|
primaryStashDir,
|
|
794
880
|
actionableRefs: preparation.actionableRefs,
|
|
795
881
|
appliedCleanup: preparation.appliedCleanup,
|
|
796
882
|
cleanupWarnings: preparation.cleanupWarnings,
|
|
797
|
-
memorySummary,
|
|
798
883
|
memoryRefsForInference,
|
|
799
884
|
reindexFn,
|
|
800
885
|
eventsCtx,
|
|
801
886
|
// O-1 (#364): propagate wall-clock budget signal to post-loop maintenance.
|
|
802
887
|
budgetSignal: budgetAbortController.signal,
|
|
803
888
|
improveProfile,
|
|
889
|
+
consolidationRan: preparation.consolidationRan,
|
|
804
890
|
});
|
|
805
891
|
const finalActions = maintenanceActions && maintenanceActions.length > 0
|
|
806
892
|
? [...preparation.actions, ...maintenanceActions]
|
|
@@ -876,6 +962,17 @@ export async function akmImprove(options = {}) {
|
|
|
876
962
|
const f = preparation.gateAutoAcceptFailedCount + loopGateFailedCount + postLoopGateFailedCount;
|
|
877
963
|
return f > 0 ? { gateAutoAcceptFailedCount: f } : {};
|
|
878
964
|
})(),
|
|
965
|
+
...(triageDrain
|
|
966
|
+
? {
|
|
967
|
+
triage: {
|
|
968
|
+
promoted: triageDrain.promoted.length,
|
|
969
|
+
rejected: triageDrain.rejected.length,
|
|
970
|
+
deferred: triageDrain.deferred.length,
|
|
971
|
+
skippedByCap: triageDrain.skippedByCap.length,
|
|
972
|
+
},
|
|
973
|
+
}
|
|
974
|
+
: {}),
|
|
975
|
+
...(options.runId !== undefined ? { runId: options.runId } : {}),
|
|
879
976
|
};
|
|
880
977
|
if (!result.dryRun)
|
|
881
978
|
emitImproveCompletedEvent(result, {
|
|
@@ -951,6 +1048,9 @@ export async function akmImprove(options = {}) {
|
|
|
951
1048
|
throw err;
|
|
952
1049
|
}
|
|
953
1050
|
finally {
|
|
1051
|
+
// #576: clear the per-run LLM usage sink BEFORE closing `eventsDb` below, so
|
|
1052
|
+
// no late sink invocation can write through a closed handle.
|
|
1053
|
+
disposeLlmUsageSink();
|
|
954
1054
|
// O-1 (#364): Clear the budget abort timer so it does not keep the event
|
|
955
1055
|
// loop alive after the run completes.
|
|
956
1056
|
clearBudgetTimer();
|
|
@@ -978,6 +1078,7 @@ function emitImproveCompletedEvent(result, durations, eventsCtx) {
|
|
|
978
1078
|
reflectFailed: 0,
|
|
979
1079
|
reflectCooldown: 0,
|
|
980
1080
|
reflectSkipped: 0,
|
|
1081
|
+
reflectGuardRejected: 0,
|
|
981
1082
|
distill: 0,
|
|
982
1083
|
distillSkipped: 0,
|
|
983
1084
|
memoryPrune: 0,
|
|
@@ -985,7 +1086,16 @@ function emitImproveCompletedEvent(result, durations, eventsCtx) {
|
|
|
985
1086
|
graphExtraction: 0,
|
|
986
1087
|
error: 0,
|
|
987
1088
|
};
|
|
1089
|
+
// Coarse audit buckets, derived from the SAME classifyImproveAction the
|
|
1090
|
+
// persisted metrics_json uses (state-db.ts#computeImproveRunMetrics) so the
|
|
1091
|
+
// emitted event and the stored row can never disagree.
|
|
1092
|
+
const classCounts = { accepted: 0, rejected: 0, error: 0, noop: 0 };
|
|
988
1093
|
for (const action of result.actions ?? []) {
|
|
1094
|
+
classCounts[classifyImproveAction(action.mode)] += 1;
|
|
1095
|
+
// Per-variant counters for the event metadata. The default arm makes any
|
|
1096
|
+
// new ImproveActionMode variant a compile error so a future variant cannot
|
|
1097
|
+
// be silently dropped from the improve_completed event (the `reflect-guard-
|
|
1098
|
+
// rejected` case below was previously missing here entirely).
|
|
989
1099
|
switch (action.mode) {
|
|
990
1100
|
case "reflect":
|
|
991
1101
|
actionCounts.reflect += 1;
|
|
@@ -999,6 +1109,9 @@ function emitImproveCompletedEvent(result, durations, eventsCtx) {
|
|
|
999
1109
|
case "reflect-skipped":
|
|
1000
1110
|
actionCounts.reflectSkipped += 1;
|
|
1001
1111
|
break;
|
|
1112
|
+
case "reflect-guard-rejected":
|
|
1113
|
+
actionCounts.reflectGuardRejected += 1;
|
|
1114
|
+
break;
|
|
1002
1115
|
case "distill":
|
|
1003
1116
|
actionCounts.distill += 1;
|
|
1004
1117
|
break;
|
|
@@ -1017,6 +1130,8 @@ function emitImproveCompletedEvent(result, durations, eventsCtx) {
|
|
|
1017
1130
|
case "error":
|
|
1018
1131
|
actionCounts.error += 1;
|
|
1019
1132
|
break;
|
|
1133
|
+
default:
|
|
1134
|
+
assertNever(action.mode);
|
|
1020
1135
|
}
|
|
1021
1136
|
}
|
|
1022
1137
|
appendEvent({
|
|
@@ -1036,6 +1151,12 @@ function emitImproveCompletedEvent(result, durations, eventsCtx) {
|
|
|
1036
1151
|
reflectFailedActions: actionCounts.reflectFailed,
|
|
1037
1152
|
reflectCooldownActions: actionCounts.reflectCooldown,
|
|
1038
1153
|
reflectSkippedActions: actionCounts.reflectSkipped,
|
|
1154
|
+
// Previously dropped from the event entirely; now emitted so the guard
|
|
1155
|
+
// rejections are visible in improve_completed telemetry.
|
|
1156
|
+
reflectGuardRejectedActions: actionCounts.reflectGuardRejected,
|
|
1157
|
+
acceptedActions: classCounts.accepted,
|
|
1158
|
+
rejectedActions: classCounts.rejected,
|
|
1159
|
+
noopActions: classCounts.noop,
|
|
1039
1160
|
reflectsWithErrorContext: result.reflectsWithErrorContext ?? 0,
|
|
1040
1161
|
coverageGapCount: result.coverageGaps?.length ?? 0,
|
|
1041
1162
|
evalCasesWritten: result.evalCasesWritten ?? 0,
|
|
@@ -1068,12 +1189,258 @@ function emitImproveCompletedEvent(result, durations, eventsCtx) {
|
|
|
1068
1189
|
},
|
|
1069
1190
|
}, eventsCtx);
|
|
1070
1191
|
}
|
|
1192
|
+
/**
|
|
1193
|
+
* Run (or gate-skip) the memory consolidation pass.
|
|
1194
|
+
*
|
|
1195
|
+
* #551 — two coordinated changes live here:
|
|
1196
|
+
*
|
|
1197
|
+
* 1. STRUCTURAL: this runs before extract in the improve pipeline (see
|
|
1198
|
+
* `runImprovePreparationStage`). Consolidation therefore only ever judges
|
|
1199
|
+
* PRIOR-run memories; current-run extract promotions are invisible to it.
|
|
1200
|
+
*
|
|
1201
|
+
* 2. SMARTER POOL-DELTA GATE: even among on-disk files, a memory whose only
|
|
1202
|
+
* post-`lastConsolidateTs` mtime bump came from its OWN auto-accept
|
|
1203
|
+
* promotion (i.e. it was just promoted by extract in the immediately
|
|
1204
|
+
* preceding run and has not had a full improve cycle to settle) does NOT
|
|
1205
|
+
* count as "work to do". We exclude those paths from the pool-delta check
|
|
1206
|
+
* using the `promoted` events already emitted with each promotion's
|
|
1207
|
+
* `assetPath`. A genuinely-settled prior memory — one edited by feedback,
|
|
1208
|
+
* reflect, manual edit, or simply older than the last consolidate — still
|
|
1209
|
+
* triggers the run. This is gate-option (a) from the issue (same-run /
|
|
1210
|
+
* adjacent-run promotion exclusion), chosen over option (b) because there
|
|
1211
|
+
* is no `extract_completed` event in the data model to gate against;
|
|
1212
|
+
* `promoted` events with `assetPath` already carry exactly the signal we
|
|
1213
|
+
* need, so the fix is non-invasive and provably correct.
|
|
1214
|
+
*/
|
|
1215
|
+
async function runConsolidationPass(args) {
|
|
1216
|
+
const { options, primaryStashDir, memorySummary, improveProfile, eventsCtx } = args;
|
|
1217
|
+
const baseConfig = options.config ?? loadConfig();
|
|
1218
|
+
const MEMORY_VOLUME_THRESHOLD = options.memoryVolumeConsolidationThreshold ?? 100;
|
|
1219
|
+
const hasLlm = !!(baseConfig.defaults?.llm || baseConfig.defaults?.agent);
|
|
1220
|
+
const volumeTriggered = typeof memorySummary.eligible === "number" && memorySummary.eligible > MEMORY_VOLUME_THRESHOLD && hasLlm;
|
|
1221
|
+
// When volume triggers a consolidation pass, force-enable the consolidate
|
|
1222
|
+
// process on the default improve profile so the gate accepts the run even
|
|
1223
|
+
// if the user's config disabled it. We synthesise a new profile override
|
|
1224
|
+
// rather than mutating connection settings.
|
|
1225
|
+
const consolidationConfig = volumeTriggered
|
|
1226
|
+
? {
|
|
1227
|
+
...baseConfig,
|
|
1228
|
+
profiles: {
|
|
1229
|
+
...(baseConfig.profiles ?? {}),
|
|
1230
|
+
improve: {
|
|
1231
|
+
...(baseConfig.profiles?.improve ?? {}),
|
|
1232
|
+
default: {
|
|
1233
|
+
...(baseConfig.profiles?.improve?.default ?? {}),
|
|
1234
|
+
processes: {
|
|
1235
|
+
...(baseConfig.profiles?.improve?.default?.processes ?? {}),
|
|
1236
|
+
consolidate: {
|
|
1237
|
+
...(baseConfig.profiles?.improve?.default?.processes?.consolidate ?? {}),
|
|
1238
|
+
enabled: true,
|
|
1239
|
+
},
|
|
1240
|
+
},
|
|
1241
|
+
},
|
|
1242
|
+
},
|
|
1243
|
+
},
|
|
1244
|
+
}
|
|
1245
|
+
: baseConfig;
|
|
1246
|
+
// 0.8.0 pool-delta gate for consolidate: re-eligible iff at least one
|
|
1247
|
+
// memory file has been updated since the most recent successful
|
|
1248
|
+
// consolidate_completed event. Time-based cooldowns produced the same
|
|
1249
|
+
// synchronised-wave failure mode the reflect/distill cooldowns did; the
|
|
1250
|
+
// pool-delta gate ties consolidation to actual work-to-do.
|
|
1251
|
+
const recentConsolidations = readEvents({ type: "consolidate_completed" });
|
|
1252
|
+
const lastConsolidation = recentConsolidations.events
|
|
1253
|
+
.filter((e) => e.metadata?.processed && Number(e.metadata.processed) > 0)
|
|
1254
|
+
.sort((a, b) => new Date(b.ts ?? 0).getTime() - new Date(a.ts ?? 0).getTime())[0];
|
|
1255
|
+
const lastConsolidateTs = lastConsolidation?.ts;
|
|
1256
|
+
// #551 smarter gate: build the set of memory asset paths whose only delta
|
|
1257
|
+
// since the last consolidate is their OWN auto-accept promotion. Those files
|
|
1258
|
+
// have not had a full improve cycle to settle, so they offer no merge /
|
|
1259
|
+
// contradiction candidates yet — excluding them stops the gate firing on
|
|
1260
|
+
// freshly-promoted single-source memories. We read `promoted` events emitted
|
|
1261
|
+
// after the last consolidate; each carries the written `assetPath`.
|
|
1262
|
+
const promotedSinceConsolidate = (() => {
|
|
1263
|
+
const paths = new Set();
|
|
1264
|
+
try {
|
|
1265
|
+
const promoted = readEvents({
|
|
1266
|
+
type: "promoted",
|
|
1267
|
+
...(lastConsolidateTs ? { since: lastConsolidateTs } : {}),
|
|
1268
|
+
}).events;
|
|
1269
|
+
for (const e of promoted) {
|
|
1270
|
+
const ap = e.metadata?.assetPath;
|
|
1271
|
+
if (typeof ap === "string" && ap.length > 0)
|
|
1272
|
+
paths.add(path.resolve(ap));
|
|
1273
|
+
}
|
|
1274
|
+
}
|
|
1275
|
+
catch {
|
|
1276
|
+
// best-effort: if the events query fails, fall back to no exclusions
|
|
1277
|
+
// (preserves pre-#551 behaviour rather than over-skipping).
|
|
1278
|
+
}
|
|
1279
|
+
return paths;
|
|
1280
|
+
})();
|
|
1281
|
+
// Pool-delta: any memory file with mtime > lastConsolidateTs flags work to do,
|
|
1282
|
+
// EXCEPT files whose only post-consolidate change was their own promotion.
|
|
1283
|
+
// Using file mtime keeps this query DB-free and matches what the indexer
|
|
1284
|
+
// already uses as the canonical `memory.updated_at` proxy.
|
|
1285
|
+
//
|
|
1286
|
+
// Bootstrap: when no successful consolidate_completed event has ever been
|
|
1287
|
+
// recorded, we cannot evaluate the pool-delta — treat as eligible so a
|
|
1288
|
+
// fresh stash runs consolidate once before the steady-state gate kicks in.
|
|
1289
|
+
const memoryUpdatedAfterLastConsolidate = (() => {
|
|
1290
|
+
if (volumeTriggered)
|
|
1291
|
+
return true; // volume override forces the run regardless.
|
|
1292
|
+
if (!lastConsolidateTs)
|
|
1293
|
+
return true; // bootstrap path: never consolidated.
|
|
1294
|
+
if (!primaryStashDir)
|
|
1295
|
+
return false;
|
|
1296
|
+
const memoriesDir = path.join(primaryStashDir, "memories");
|
|
1297
|
+
if (!fs.existsSync(memoriesDir))
|
|
1298
|
+
return false;
|
|
1299
|
+
try {
|
|
1300
|
+
return fs.readdirSync(memoriesDir).some((f) => {
|
|
1301
|
+
if (!f.endsWith(".md"))
|
|
1302
|
+
return false;
|
|
1303
|
+
const filePath = path.join(memoriesDir, f);
|
|
1304
|
+
// #551: skip files that were only touched by their own promotion this
|
|
1305
|
+
// cohort — they have no settled merge/contradiction candidates yet.
|
|
1306
|
+
if (promotedSinceConsolidate.has(path.resolve(filePath)))
|
|
1307
|
+
return false;
|
|
1308
|
+
try {
|
|
1309
|
+
return fs.statSync(filePath).mtime.toISOString() > lastConsolidateTs;
|
|
1310
|
+
}
|
|
1311
|
+
catch {
|
|
1312
|
+
return false;
|
|
1313
|
+
}
|
|
1314
|
+
});
|
|
1315
|
+
}
|
|
1316
|
+
catch {
|
|
1317
|
+
return false;
|
|
1318
|
+
}
|
|
1319
|
+
})();
|
|
1320
|
+
const consolidationOnCooldown = !volumeTriggered && !memoryUpdatedAfterLastConsolidate;
|
|
1321
|
+
// Profile gate: if profile explicitly disables consolidate, skip the entire pass.
|
|
1322
|
+
const consolidateDisabledByProfile = improveProfile?.processes?.consolidate?.enabled === false;
|
|
1323
|
+
// #553 minPoolSize guard: skip consolidation when the eligible memory pool is
|
|
1324
|
+
// below a minimum size, rather than spending an LLM pass on a handful of
|
|
1325
|
+
// memories. This is an INDEPENDENT skip condition from #551's mtime pool-delta
|
|
1326
|
+
// gate — either can skip. Default 500; `minPoolSize: 0` disables the guard.
|
|
1327
|
+
// Evaluated against the eligible-pool count BEFORE entering the LLM loop so a
|
|
1328
|
+
// skip costs ZERO LLM calls.
|
|
1329
|
+
const CONSOLIDATE_DEFAULT_MIN_POOL_SIZE = 500;
|
|
1330
|
+
const configuredMinPoolSize = improveProfile?.processes?.consolidate?.minPoolSize;
|
|
1331
|
+
const minPoolSize = typeof configuredMinPoolSize === "number" ? configuredMinPoolSize : CONSOLIDATE_DEFAULT_MIN_POOL_SIZE;
|
|
1332
|
+
const eligiblePoolSize = typeof memorySummary.eligible === "number" ? memorySummary.eligible : 0;
|
|
1333
|
+
// volumeTriggered means the pool already exceeds the volume threshold (100),
|
|
1334
|
+
// so a force-triggered run never trips the pool-size guard. The guard only
|
|
1335
|
+
// engages when minPoolSize > 0 and the eligible pool is strictly below it.
|
|
1336
|
+
const poolBelowMinSize = !volumeTriggered && minPoolSize > 0 && eligiblePoolSize < minPoolSize;
|
|
1337
|
+
let consolidation = {
|
|
1338
|
+
schemaVersion: 1,
|
|
1339
|
+
ok: true,
|
|
1340
|
+
shape: "consolidate-result",
|
|
1341
|
+
dryRun: false,
|
|
1342
|
+
previewOnly: false,
|
|
1343
|
+
target: "",
|
|
1344
|
+
processed: 0,
|
|
1345
|
+
merged: 0,
|
|
1346
|
+
deleted: 0,
|
|
1347
|
+
promoted: [],
|
|
1348
|
+
contradicted: 0,
|
|
1349
|
+
warnings: [],
|
|
1350
|
+
durationMs: 0,
|
|
1351
|
+
};
|
|
1352
|
+
let gateAutoAcceptedCount = 0;
|
|
1353
|
+
let gateAutoAcceptFailedCount = 0;
|
|
1354
|
+
const consolidateGateCfg = makeGateConfig("consolidate", {
|
|
1355
|
+
globalThreshold: options.autoAccept,
|
|
1356
|
+
dryRun: options.dryRun ?? false,
|
|
1357
|
+
stashDir: primaryStashDir,
|
|
1358
|
+
config: consolidationConfig,
|
|
1359
|
+
eventsCtx,
|
|
1360
|
+
}, { minimumThreshold: 95 });
|
|
1361
|
+
if (consolidateDisabledByProfile) {
|
|
1362
|
+
info("[improve] consolidation skipped (disabled by improve profile)");
|
|
1363
|
+
}
|
|
1364
|
+
else if (poolBelowMinSize) {
|
|
1365
|
+
// #553: eligible pool below the configured minimum — skip with zero LLM
|
|
1366
|
+
// calls. Reuse the #551 `improve_skipped` emission path so health surfaces
|
|
1367
|
+
// it via the dynamic skipReasons aggregation under `pool_below_min_size`.
|
|
1368
|
+
appendEvent({
|
|
1369
|
+
eventType: "improve_skipped",
|
|
1370
|
+
ref: "memory:_consolidation",
|
|
1371
|
+
metadata: {
|
|
1372
|
+
reason: "pool_below_min_size",
|
|
1373
|
+
poolSize: eligiblePoolSize,
|
|
1374
|
+
minPoolSize,
|
|
1375
|
+
},
|
|
1376
|
+
}, eventsCtx);
|
|
1377
|
+
info(`[improve] consolidation skipped (pool ${eligiblePoolSize} < minPoolSize ${minPoolSize})`);
|
|
1378
|
+
}
|
|
1379
|
+
else if (!consolidationOnCooldown) {
|
|
1380
|
+
consolidation = await withLlmStage("consolidate", () => akmConsolidate({
|
|
1381
|
+
...options.consolidateOptions,
|
|
1382
|
+
config: consolidationConfig,
|
|
1383
|
+
stashDir: options.stashDir,
|
|
1384
|
+
autoTriggered: volumeTriggered,
|
|
1385
|
+
// Tie consolidate proposals back to this improve invocation so
|
|
1386
|
+
// accept-rate-per-run aggregation works. Mirrors reflect/propose/extract.
|
|
1387
|
+
sourceRun: `consolidate-${Date.now()}`,
|
|
1388
|
+
// Full-pool sweep: consolidation only runs on the nightly default-profile
|
|
1389
|
+
// pass (quick/frequent disable it), so a complete re-cluster is correct and
|
|
1390
|
+
// affordable here. Do NOT pass incrementalSince — the time-window narrowing
|
|
1391
|
+
// it triggers permanently excludes stale-but-unmerged duplicate clusters,
|
|
1392
|
+
// starving merge recall and letting the pool grow unbounded. (The narrowing
|
|
1393
|
+
// was a band-aid for an every-30-min consolidation cadence that the profile
|
|
1394
|
+
// split has since eliminated.) lastConsolidateTs still gates whether we run.
|
|
1395
|
+
maxChunkSize: improveProfile?.processes?.consolidate?.maxChunkSize,
|
|
1396
|
+
// Honor profile.autoAccept (already merged into options.autoAccept at the
|
|
1397
|
+
// top of akmImprove). The CLI parser always supplies 90 when --auto-accept
|
|
1398
|
+
// is absent, so ?? 90 is not needed here and would prevent --auto-accept=false
|
|
1399
|
+
// (which maps to undefined) from disabling consolidation auto-accept.
|
|
1400
|
+
// options.consolidateOptions.autoAccept (if explicitly provided by caller)
|
|
1401
|
+
// still wins because the spread above runs first.
|
|
1402
|
+
autoAccept: options.consolidateOptions?.autoAccept ?? options.autoAccept,
|
|
1403
|
+
}));
|
|
1404
|
+
{
|
|
1405
|
+
const consolidateGr = await runAutoAcceptGate(consolidation.promoted.map((proposalId) => {
|
|
1406
|
+
try {
|
|
1407
|
+
if (!primaryStashDir)
|
|
1408
|
+
return { proposalId, confidence: undefined };
|
|
1409
|
+
const proposal = getProposal(primaryStashDir, proposalId);
|
|
1410
|
+
return { proposalId, confidence: proposal.confidence };
|
|
1411
|
+
}
|
|
1412
|
+
catch {
|
|
1413
|
+
return { proposalId, confidence: undefined };
|
|
1414
|
+
}
|
|
1415
|
+
}), consolidateGateCfg);
|
|
1416
|
+
gateAutoAcceptedCount += consolidateGr.promoted.length;
|
|
1417
|
+
gateAutoAcceptFailedCount += consolidateGr.failed.length;
|
|
1418
|
+
}
|
|
1419
|
+
if (consolidation.processed > 0) {
|
|
1420
|
+
appendEvent({
|
|
1421
|
+
eventType: "consolidate_completed",
|
|
1422
|
+
ref: "memory:_consolidation",
|
|
1423
|
+
metadata: { processed: consolidation.processed, merged: consolidation.merged },
|
|
1424
|
+
}, eventsCtx);
|
|
1425
|
+
}
|
|
1426
|
+
}
|
|
1427
|
+
else {
|
|
1428
|
+
appendEvent({
|
|
1429
|
+
eventType: "improve_skipped",
|
|
1430
|
+
ref: "memory:_consolidation",
|
|
1431
|
+
metadata: {
|
|
1432
|
+
reason: "consolidation_no_memory_updates",
|
|
1433
|
+
lastEventTs: lastConsolidation?.ts ?? null,
|
|
1434
|
+
},
|
|
1435
|
+
}, eventsCtx);
|
|
1436
|
+
info("[improve] consolidation skipped (no memory updates since last run)");
|
|
1437
|
+
}
|
|
1438
|
+
// D9: track whether consolidation wrote any data so graph extraction can reindex if needed
|
|
1439
|
+
const consolidationRan = !consolidateDisabledByProfile && !poolBelowMinSize && !consolidationOnCooldown && consolidation.processed > 0;
|
|
1440
|
+
return { consolidation, consolidationRan, gateAutoAcceptedCount, gateAutoAcceptFailedCount };
|
|
1441
|
+
}
|
|
1071
1442
|
async function runImprovePreparationStage(args) {
|
|
1072
|
-
const { scope, options, plannedRefs, memoryCleanupPlan, primaryStashDir, reindexFn, startMs, budgetMs, eventsCtx, initialCleanupWarnings,
|
|
1073
|
-
// improveProfile is part of the preparation-stage signature for future use
|
|
1074
|
-
// (per-process gating moved into the in-loop stage). Kept here so the
|
|
1075
|
-
// signature does not drift away from the rest of the planner stack.
|
|
1076
|
-
improveProfile: _improveProfile, } = args;
|
|
1443
|
+
const { scope, options, plannedRefs, memoryCleanupPlan, primaryStashDir, memorySummary, reindexFn, startMs, budgetMs, eventsCtx, initialCleanupWarnings, improveProfile, } = args;
|
|
1077
1444
|
const actions = [];
|
|
1078
1445
|
const cleanupWarnings = initialCleanupWarnings ? [...initialCleanupWarnings] : [];
|
|
1079
1446
|
// Phase 0 — MEMORY.md budget check (200-line cap; warn at 180)
|
|
@@ -1094,6 +1461,23 @@ async function runImprovePreparationStage(args) {
|
|
|
1094
1461
|
}
|
|
1095
1462
|
}
|
|
1096
1463
|
}
|
|
1464
|
+
// Phase 0.3 — memory consolidation pass (#551).
|
|
1465
|
+
//
|
|
1466
|
+
// Consolidation runs BEFORE the session-extract pass. This is the structural
|
|
1467
|
+
// half of the #551 fix: extract auto-accept writes brand-new memory .md files
|
|
1468
|
+
// on every run, which previously made the consolidation pool-delta gate fire
|
|
1469
|
+
// unconditionally (any new file => "memory updated since last consolidate").
|
|
1470
|
+
// By running consolidation first, the gate and akmConsolidate only ever see
|
|
1471
|
+
// memories that existed at the start of the run — current-run extract
|
|
1472
|
+
// promotions are not on disk yet. The complementary smarter-gate logic
|
|
1473
|
+
// (excluding adjacent-run promotions) lives in `runConsolidationPass`.
|
|
1474
|
+
const consolidationPass = await runConsolidationPass({
|
|
1475
|
+
options,
|
|
1476
|
+
primaryStashDir,
|
|
1477
|
+
memorySummary,
|
|
1478
|
+
improveProfile,
|
|
1479
|
+
eventsCtx,
|
|
1480
|
+
});
|
|
1097
1481
|
// Phase 0.4 — session-extract pass.
|
|
1098
1482
|
//
|
|
1099
1483
|
// Reads native session files (claude-code JSONL, opencode storage tree)
|
|
@@ -1103,7 +1487,9 @@ async function runImprovePreparationStage(args) {
|
|
|
1103
1487
|
// / `akm feedback` invocations. Replaces the akm-plugin session-checkpoint
|
|
1104
1488
|
// hook with an on-demand pull pipeline.
|
|
1105
1489
|
//
|
|
1106
|
-
// Default-on; opt out via `
|
|
1490
|
+
// Default-on; opt out via the ACTIVE profile's `processes.extract.enabled: false`
|
|
1491
|
+
// (#593: the gate respects the resolved improve profile, not just the
|
|
1492
|
+
// hardcoded `default` profile path the legacy feature flag reads).
|
|
1107
1493
|
// Each available harness gets one call with the default --since window;
|
|
1108
1494
|
// already-seen sessions (tracked in state.db.extract_sessions_seen) are
|
|
1109
1495
|
// skipped automatically so re-runs don't burn LLM calls on unchanged data.
|
|
@@ -1111,8 +1497,11 @@ async function runImprovePreparationStage(args) {
|
|
|
1111
1497
|
// Failures are non-fatal — one harness throwing doesn't abort improve.
|
|
1112
1498
|
// The extract envelope's own `warnings` field surfaces what went wrong.
|
|
1113
1499
|
let extractResults;
|
|
1114
|
-
|
|
1115
|
-
|
|
1500
|
+
// Seed the preparation-stage gate counters with consolidation's auto-accept
|
|
1501
|
+
// gate results (#551: consolidation now runs in this stage), then accumulate
|
|
1502
|
+
// extract's gate results on top.
|
|
1503
|
+
let gateAutoAcceptedCount = consolidationPass.gateAutoAcceptedCount;
|
|
1504
|
+
let gateAutoAcceptFailedCount = consolidationPass.gateAutoAcceptFailedCount;
|
|
1116
1505
|
const extractConfig = options.config ?? loadConfig();
|
|
1117
1506
|
const extractGateCfg = makeGateConfig("extract", {
|
|
1118
1507
|
globalThreshold: options.autoAccept,
|
|
@@ -1121,18 +1510,65 @@ async function runImprovePreparationStage(args) {
|
|
|
1121
1510
|
config: extractConfig,
|
|
1122
1511
|
eventsCtx,
|
|
1123
1512
|
});
|
|
1124
|
-
|
|
1125
|
-
|
|
1126
|
-
|
|
1513
|
+
// #554 minNewSessions gate: skip the entire extract pass (ensureIndex was
|
|
1514
|
+
// already done upstream; here we elide every akmExtract/processSession call)
|
|
1515
|
+
// when the NEW (unseen, in-window) candidate-session pool is below a minimum.
|
|
1516
|
+
// 22% of improve runs produce zero memory-inference writes because extract
|
|
1517
|
+
// finds no new sessions, yet still burns the full extract pipeline. Default 0
|
|
1518
|
+
// (disabled) preserves existing always-run behaviour; only opted-in profiles
|
|
1519
|
+
// (e.g. `frequent`) set it. Evaluated BEFORE any LLM call so a skip costs zero
|
|
1520
|
+
// LLM work AND writes nothing — which also means no extract auto-accept bumps
|
|
1521
|
+
// memory mtimes, so a skipped extract never flags work for the NEXT run's
|
|
1522
|
+
// consolidation mtime-gate (the downstream trigger #554 asks us to suppress).
|
|
1523
|
+
const EXTRACT_DEFAULT_MIN_NEW_SESSIONS = 0;
|
|
1524
|
+
const configuredMinNewSessions = extractConfig.profiles?.improve?.default?.processes?.extract?.minNewSessions;
|
|
1525
|
+
const minNewSessions = typeof configuredMinNewSessions === "number" ? configuredMinNewSessions : EXTRACT_DEFAULT_MIN_NEW_SESSIONS;
|
|
1526
|
+
// #593: gate on BOTH the legacy feature flag (which only reads
|
|
1527
|
+
// `profiles.improve.default.processes.extract.enabled` — kept for back-compat
|
|
1528
|
+
// with users who disable extract via the default-profile path) AND the active
|
|
1529
|
+
// resolved profile. Without the second check a non-default profile setting
|
|
1530
|
+
// `extract.enabled: false` (e.g. the built-in `quick`) was silently ignored
|
|
1531
|
+
// and extract ran on every improve call regardless.
|
|
1532
|
+
if (isLlmFeatureEnabled(extractConfig, "session_extraction") && resolveProcessEnabled("extract", improveProfile)) {
|
|
1533
|
+
const availableHarnesses = options.extractHarnesses ?? getAvailableHarnesses();
|
|
1534
|
+
// The guard engages only when minNewSessions > 0; 0 disables it entirely.
|
|
1535
|
+
let belowMinNewSessions = false;
|
|
1536
|
+
if (minNewSessions > 0 && availableHarnesses.length > 0) {
|
|
1537
|
+
const countFn = options.extractCandidateCountFn ?? countNewExtractCandidates;
|
|
1538
|
+
const newCandidateCount = countFn(extractConfig, {
|
|
1539
|
+
...(options.extractHarnesses ? { harnesses: options.extractHarnesses } : {}),
|
|
1540
|
+
// C2: pin the candidate-count state.db open to the boundary-resolved path.
|
|
1541
|
+
...(eventsCtx?.dbPath ? { stateDbPath: eventsCtx.dbPath } : {}),
|
|
1542
|
+
});
|
|
1543
|
+
if (newCandidateCount < minNewSessions) {
|
|
1544
|
+
belowMinNewSessions = true;
|
|
1545
|
+
// Reuse the #551/#553 `improve_skipped` emission path so health's dynamic
|
|
1546
|
+
// skipReasons aggregation surfaces this under `below_min_new_sessions`.
|
|
1547
|
+
appendEvent({
|
|
1548
|
+
eventType: "improve_skipped",
|
|
1549
|
+
ref: "memory:_extract",
|
|
1550
|
+
metadata: {
|
|
1551
|
+
reason: "below_min_new_sessions",
|
|
1552
|
+
newSessions: newCandidateCount,
|
|
1553
|
+
minNewSessions,
|
|
1554
|
+
},
|
|
1555
|
+
}, eventsCtx);
|
|
1556
|
+
info(`[improve] extract skipped (new sessions ${newCandidateCount} < minNewSessions ${minNewSessions})`);
|
|
1557
|
+
}
|
|
1558
|
+
}
|
|
1559
|
+
if (!belowMinNewSessions && availableHarnesses.length > 0) {
|
|
1127
1560
|
extractResults = [];
|
|
1128
1561
|
for (const h of availableHarnesses) {
|
|
1129
1562
|
try {
|
|
1130
|
-
const result = await akmExtract({
|
|
1563
|
+
const result = await withLlmStage("session-extraction", () => akmExtract({
|
|
1131
1564
|
type: h.name,
|
|
1132
1565
|
...(primaryStashDir !== undefined ? { stashDir: primaryStashDir } : {}),
|
|
1133
1566
|
config: extractConfig,
|
|
1134
1567
|
dryRun: options.dryRun ?? false,
|
|
1135
|
-
|
|
1568
|
+
...(options.extractHarnesses ? { harnesses: options.extractHarnesses } : {}),
|
|
1569
|
+
// C2: pin extract's skip-tracking state.db open to the boundary path.
|
|
1570
|
+
...(eventsCtx?.dbPath ? { stateDbPath: eventsCtx.dbPath } : {}),
|
|
1571
|
+
}));
|
|
1136
1572
|
extractResults.push(result);
|
|
1137
1573
|
{
|
|
1138
1574
|
const gr = await runAutoAcceptGate(primaryStashDir
|
|
@@ -1223,7 +1659,13 @@ async function runImprovePreparationStage(args) {
|
|
|
1223
1659
|
const validationFailures = [];
|
|
1224
1660
|
for (const candidate of postCleanupRefs) {
|
|
1225
1661
|
try {
|
|
1226
|
-
|
|
1662
|
+
// #591: use the path pre-resolved at planning time when it is still on
|
|
1663
|
+
// disk — a serial async DB lookup per ref cost ~500 s on a 9 000-ref
|
|
1664
|
+
// stash. Fall back to findAssetFilePath only for refs that bypassed
|
|
1665
|
+
// collectEligibleRefs' index scan or whose file moved since planning.
|
|
1666
|
+
const filePath = candidate.filePath && fs.existsSync(candidate.filePath)
|
|
1667
|
+
? candidate.filePath
|
|
1668
|
+
: await findAssetFilePath(candidate.ref, options.stashDir);
|
|
1227
1669
|
if (!filePath) {
|
|
1228
1670
|
validationFailures.push({ ref: candidate.ref, reason: "file not found on disk" });
|
|
1229
1671
|
continue;
|
|
@@ -1444,7 +1886,7 @@ async function runImprovePreparationStage(args) {
|
|
|
1444
1886
|
let dbForRetrieval;
|
|
1445
1887
|
try {
|
|
1446
1888
|
dbForRetrieval = openExistingDatabase();
|
|
1447
|
-
const showEventCount = dbForRetrieval
|
|
1889
|
+
const showEventCount = countUsageEventsByType(dbForRetrieval, "show");
|
|
1448
1890
|
if (showEventCount === 0) {
|
|
1449
1891
|
warn("Warning: show events not yet in usage_events — zero-feedback fallback will match only search-retrieved assets.");
|
|
1450
1892
|
}
|
|
@@ -1531,15 +1973,32 @@ async function runImprovePreparationStage(args) {
|
|
|
1531
1973
|
const assetMissingOnDisk = [];
|
|
1532
1974
|
const existsCheckedActionable = [];
|
|
1533
1975
|
for (const candidate of sorted) {
|
|
1534
|
-
|
|
1976
|
+
// #591: prefer the path pre-resolved at planning time (synchronous
|
|
1977
|
+
// existsSync) over a serial async DB lookup per ref.
|
|
1978
|
+
const filePath = candidate.filePath && fs.existsSync(candidate.filePath)
|
|
1979
|
+
? candidate.filePath
|
|
1980
|
+
: await findAssetFilePath(candidate.ref, options.stashDir);
|
|
1535
1981
|
if (filePath && fs.existsSync(filePath)) {
|
|
1536
1982
|
existsCheckedActionable.push(candidate);
|
|
1537
1983
|
}
|
|
1538
1984
|
else {
|
|
1539
1985
|
assetMissingOnDisk.push(candidate.ref);
|
|
1540
|
-
appendEvent({ eventType: "improve_skipped", ref: candidate.ref, metadata: { reason: "asset_missing_on_disk" } }, eventsCtx);
|
|
1541
1986
|
}
|
|
1542
1987
|
}
|
|
1988
|
+
// #592 audit: one summary event instead of one per missing ref. Normally
|
|
1989
|
+
// tiny, but a stash deletion racing the run could make this O(n) sequential
|
|
1990
|
+
// state.db writes. `refs` is capped so the metadata row stays bounded.
|
|
1991
|
+
if (assetMissingOnDisk.length > 0) {
|
|
1992
|
+
appendEvent({
|
|
1993
|
+
eventType: "improve_skipped",
|
|
1994
|
+
ref: undefined,
|
|
1995
|
+
metadata: {
|
|
1996
|
+
reason: "asset_missing_on_disk",
|
|
1997
|
+
count: assetMissingOnDisk.length,
|
|
1998
|
+
refs: assetMissingOnDisk.slice(0, 50),
|
|
1999
|
+
},
|
|
2000
|
+
}, eventsCtx);
|
|
2001
|
+
}
|
|
1543
2002
|
const actionableRefs = existsCheckedActionable;
|
|
1544
2003
|
// Re-split actionableRefs (sorted) into reflect-path vs distill-only-path while
|
|
1545
2004
|
// preserving sort order. distillOnlyRefs participate in the sort so --limit
|
|
@@ -1598,9 +2057,10 @@ async function runImprovePreparationStage(args) {
|
|
|
1598
2057
|
utilityMap,
|
|
1599
2058
|
gateAutoAcceptedCount,
|
|
1600
2059
|
gateAutoAcceptFailedCount,
|
|
2060
|
+
consolidation: consolidationPass.consolidation,
|
|
2061
|
+
consolidationRan: consolidationPass.consolidationRan,
|
|
1601
2062
|
};
|
|
1602
2063
|
}
|
|
1603
|
-
// TODO(refactor): 13 args including `actions`/`recentErrors` mutation channels. Restructure into immutable plan + mutable context objects — deferred to dedicated refactor with isolated testing.
|
|
1604
2064
|
async function runImproveLoopStage(args) {
|
|
1605
2065
|
const { scope, options, primaryStashDir, reflectFn, distillFn, loopRefs, actions, signalBearingSet, distillCooledRefs, distillOnlyRefs, recentErrors, rejectedProposalsByRef, utilityMap, startMs, budgetMs, eventsCtx, improveProfile, } = args;
|
|
1606
2066
|
// O-1 (#364): compute remaining budget at call time so each sub-call
|
|
@@ -1734,7 +2194,7 @@ async function runImproveLoopStage(args) {
|
|
|
1734
2194
|
// path is also a no-op for them — we just avoid unnecessary agent spawns.
|
|
1735
2195
|
// D2: distillOnlyRefs also skip the reflect call (reflect-cooled, distill path only).
|
|
1736
2196
|
if (!isDistillOnly && !planned.ref.endsWith(".derived")) {
|
|
1737
|
-
// Type guard: skip reflect for unsupported types (script,
|
|
2197
|
+
// Type guard: skip reflect for unsupported types (script, env, task, etc.)
|
|
1738
2198
|
// and raw wiki directories, driven by the active improve profile.
|
|
1739
2199
|
const reflectSkip = shouldSkipRef(planned.ref, "reflect", improveProfile);
|
|
1740
2200
|
if (reflectSkip.skip) {
|
|
@@ -1779,9 +2239,11 @@ async function runImproveLoopStage(args) {
|
|
|
1779
2239
|
if (remainingBudgetMs() <= 0)
|
|
1780
2240
|
break;
|
|
1781
2241
|
// draftMode: skip DB write so each sample doesn't create a proposal.
|
|
1782
|
-
samples.push(await reflectFn({ ...reflectCallArgs, draftMode: true }));
|
|
2242
|
+
samples.push(await withLlmStage("reflect", () => reflectFn({ ...reflectCallArgs, draftMode: true })));
|
|
1783
2243
|
}
|
|
1784
|
-
const winner = pickMajorityVote(samples.length > 0
|
|
2244
|
+
const winner = pickMajorityVote(samples.length > 0
|
|
2245
|
+
? samples
|
|
2246
|
+
: [await withLlmStage("reflect", () => reflectFn({ ...reflectCallArgs, draftMode: true }))]);
|
|
1785
2247
|
// Persist only the majority-vote winner as a single real proposal.
|
|
1786
2248
|
if (winner.ok && primaryStashDir) {
|
|
1787
2249
|
const persistResult = createProposal(primaryStashDir, {
|
|
@@ -1806,7 +2268,7 @@ async function runImproveLoopStage(args) {
|
|
|
1806
2268
|
}
|
|
1807
2269
|
}
|
|
1808
2270
|
else {
|
|
1809
|
-
reflectResult = await reflectFn(reflectCallArgs);
|
|
2271
|
+
reflectResult = await withLlmStage("reflect", () => reflectFn(reflectCallArgs));
|
|
1810
2272
|
}
|
|
1811
2273
|
const isCooldown = !reflectResult.ok && reflectResult.reason === "cooldown";
|
|
1812
2274
|
// Content-policy guard hits (reflect size-rail rejections) are NOT
|
|
@@ -1816,13 +2278,19 @@ async function runImproveLoopStage(args) {
|
|
|
1816
2278
|
// true LLM failures. See
|
|
1817
2279
|
// `/tmp/akm-health-investigations/metrics-taxonomy-review.md` §1a.
|
|
1818
2280
|
const isGuardReject = !reflectResult.ok && reflectResult.reason === "content_policy_reject";
|
|
1819
|
-
// Type-guard rejection (reflect refused a script/
|
|
2281
|
+
// Type-guard rejection (reflect refused a script/env/task ref) is
|
|
1820
2282
|
// also NOT an LLM failure — the LLM is never invoked. Route to the
|
|
1821
2283
|
// existing `reflect-skipped` bucket so it does not inflate the
|
|
1822
2284
|
// failure-rate numerator. ~9% of `reflect-failed` events in the
|
|
1823
2285
|
// user's stack were this case; see review §1a row "Reflect refused
|
|
1824
2286
|
// asset type".
|
|
1825
2287
|
const isTypeRefused = !reflectResult.ok && reflectResult.reason === "unsupported_type";
|
|
2288
|
+
// Noise-gate suppression (#580): the candidate edit was an empty
|
|
2289
|
+
// diff or a cosmetic-only reformat of the current asset. Like
|
|
2290
|
+
// `unsupported_type`, this is a deterministic skip — not an LLM
|
|
2291
|
+
// fault — so it routes to the `reflect-skipped` bucket and stays
|
|
2292
|
+
// out of recentErrors/avoidPatterns.
|
|
2293
|
+
const isNoChange = !reflectResult.ok && reflectResult.reason === "no_change";
|
|
1826
2294
|
actions.push({
|
|
1827
2295
|
ref: planned.ref,
|
|
1828
2296
|
mode: reflectResult.ok
|
|
@@ -1831,18 +2299,19 @@ async function runImproveLoopStage(args) {
|
|
|
1831
2299
|
? "reflect-cooldown"
|
|
1832
2300
|
: isGuardReject
|
|
1833
2301
|
? "reflect-guard-rejected"
|
|
1834
|
-
: isTypeRefused
|
|
2302
|
+
: isTypeRefused || isNoChange
|
|
1835
2303
|
? "reflect-skipped"
|
|
1836
2304
|
: "reflect-failed",
|
|
1837
2305
|
result: reflectResult,
|
|
1838
2306
|
});
|
|
1839
|
-
// Cooldown skips, guard rejects,
|
|
1840
|
-
// failures — do not pollute recentErrors with them
|
|
1841
|
-
// injected as `avoidPatterns` into the next reflect
|
|
1842
|
-
// rejects ARE worth showing the LLM as a learn-signal
|
|
1843
|
-
// iteration sees "your last expansion was too large";
|
|
1844
|
-
//
|
|
1845
|
-
|
|
2307
|
+
// Cooldown skips, guard rejects, type-refused skips, and noise-gate
|
|
2308
|
+
// skips are not failures — do not pollute recentErrors with them
|
|
2309
|
+
// (those get injected as `avoidPatterns` into the next reflect
|
|
2310
|
+
// prompt). Guard rejects ARE worth showing the LLM as a learn-signal
|
|
2311
|
+
// so the next iteration sees "your last expansion was too large";
|
|
2312
|
+
// type-refused and no-change are deterministic and add no learning
|
|
2313
|
+
// signal.
|
|
2314
|
+
if (!reflectResult.ok && !isCooldown && !isTypeRefused && !isNoChange) {
|
|
1846
2315
|
const errMsg = reflectResult.error ?? reflectResult.reason ?? "unknown reflect error";
|
|
1847
2316
|
pushRecentError("reflect", errMsg);
|
|
1848
2317
|
}
|
|
@@ -1964,11 +2433,11 @@ async function runImproveLoopStage(args) {
|
|
|
1964
2433
|
}
|
|
1965
2434
|
}
|
|
1966
2435
|
}
|
|
1967
|
-
const distillResult = await distillFn({
|
|
2436
|
+
const distillResult = await withLlmStage("distill", () => distillFn({
|
|
1968
2437
|
ref: planned.ref,
|
|
1969
2438
|
...(parsedPlannedRef.type === "memory" ? { proposalKind: "auto" } : {}),
|
|
1970
2439
|
...(options.stashDir ? { stashDir: options.stashDir } : {}),
|
|
1971
|
-
});
|
|
2440
|
+
}));
|
|
1972
2441
|
actions.push({ ref: planned.ref, mode: "distill", result: distillResult });
|
|
1973
2442
|
if (distillResult.outcome === "queued" && distillResult.proposal) {
|
|
1974
2443
|
const distillGr = await runAutoAcceptGate([{ proposalId: distillResult.proposal.id, confidence: distillResult.proposal.confidence }], distillGateCfg);
|
|
@@ -2049,169 +2518,8 @@ async function runImproveLoopStage(args) {
|
|
|
2049
2518
|
return { reflectsWithErrorContext, memoryRefsForInference, gateAutoAcceptedCount, gateAutoAcceptFailedCount };
|
|
2050
2519
|
}
|
|
2051
2520
|
async function runImprovePostLoopStage(args) {
|
|
2052
|
-
const { scope, options, primaryStashDir, actionableRefs, appliedCleanup, cleanupWarnings,
|
|
2521
|
+
const { scope, options, primaryStashDir, actionableRefs, appliedCleanup, cleanupWarnings, memoryRefsForInference, reindexFn, eventsCtx, budgetSignal, improveProfile, consolidationRan, } = args;
|
|
2053
2522
|
const allWarnings = [...cleanupWarnings, ...(appliedCleanup?.warnings ?? [])];
|
|
2054
|
-
const baseConfig = options.config ?? loadConfig();
|
|
2055
|
-
const MEMORY_VOLUME_THRESHOLD = options.memoryVolumeConsolidationThreshold ?? 100;
|
|
2056
|
-
const hasLlm = !!(baseConfig.defaults?.llm || baseConfig.defaults?.agent);
|
|
2057
|
-
const volumeTriggered = typeof memorySummary.eligible === "number" && memorySummary.eligible > MEMORY_VOLUME_THRESHOLD && hasLlm;
|
|
2058
|
-
// When volume triggers a consolidation pass, force-enable the consolidate
|
|
2059
|
-
// process on the default improve profile so the gate accepts the run even
|
|
2060
|
-
// if the user's config disabled it. We synthesise a new profile override
|
|
2061
|
-
// rather than mutating connection settings.
|
|
2062
|
-
const consolidationConfig = volumeTriggered
|
|
2063
|
-
? {
|
|
2064
|
-
...baseConfig,
|
|
2065
|
-
profiles: {
|
|
2066
|
-
...(baseConfig.profiles ?? {}),
|
|
2067
|
-
improve: {
|
|
2068
|
-
...(baseConfig.profiles?.improve ?? {}),
|
|
2069
|
-
default: {
|
|
2070
|
-
...(baseConfig.profiles?.improve?.default ?? {}),
|
|
2071
|
-
processes: {
|
|
2072
|
-
...(baseConfig.profiles?.improve?.default?.processes ?? {}),
|
|
2073
|
-
consolidate: {
|
|
2074
|
-
...(baseConfig.profiles?.improve?.default?.processes?.consolidate ?? {}),
|
|
2075
|
-
enabled: true,
|
|
2076
|
-
},
|
|
2077
|
-
},
|
|
2078
|
-
},
|
|
2079
|
-
},
|
|
2080
|
-
},
|
|
2081
|
-
}
|
|
2082
|
-
: baseConfig;
|
|
2083
|
-
// 0.8.0 pool-delta gate for consolidate: re-eligible iff at least one
|
|
2084
|
-
// memory file has been updated since the most recent successful
|
|
2085
|
-
// consolidate_completed event. Time-based cooldowns produced the same
|
|
2086
|
-
// synchronised-wave failure mode the reflect/distill cooldowns did; the
|
|
2087
|
-
// pool-delta gate ties consolidation to actual work-to-do.
|
|
2088
|
-
const recentConsolidations = readEvents({ type: "consolidate_completed" });
|
|
2089
|
-
const lastConsolidation = recentConsolidations.events
|
|
2090
|
-
.filter((e) => e.metadata?.processed && Number(e.metadata.processed) > 0)
|
|
2091
|
-
.sort((a, b) => new Date(b.ts ?? 0).getTime() - new Date(a.ts ?? 0).getTime())[0];
|
|
2092
|
-
const lastConsolidateTs = lastConsolidation?.ts;
|
|
2093
|
-
// Pool-delta: any memory file with mtime > lastConsolidateTs flags work to do.
|
|
2094
|
-
// Using file mtime keeps this query DB-free and matches what the indexer
|
|
2095
|
-
// already uses as the canonical `memory.updated_at` proxy.
|
|
2096
|
-
//
|
|
2097
|
-
// Bootstrap: when no successful consolidate_completed event has ever been
|
|
2098
|
-
// recorded, we cannot evaluate the pool-delta — treat as eligible so a
|
|
2099
|
-
// fresh stash runs consolidate once before the steady-state gate kicks in.
|
|
2100
|
-
const memoryUpdatedAfterLastConsolidate = (() => {
|
|
2101
|
-
if (volumeTriggered)
|
|
2102
|
-
return true; // volume override forces the run regardless.
|
|
2103
|
-
if (!lastConsolidateTs)
|
|
2104
|
-
return true; // bootstrap path: never consolidated.
|
|
2105
|
-
if (!primaryStashDir)
|
|
2106
|
-
return false;
|
|
2107
|
-
const memoriesDir = path.join(primaryStashDir, "memories");
|
|
2108
|
-
if (!fs.existsSync(memoriesDir))
|
|
2109
|
-
return false;
|
|
2110
|
-
try {
|
|
2111
|
-
return fs.readdirSync(memoriesDir).some((f) => {
|
|
2112
|
-
if (!f.endsWith(".md"))
|
|
2113
|
-
return false;
|
|
2114
|
-
try {
|
|
2115
|
-
return fs.statSync(path.join(memoriesDir, f)).mtime.toISOString() > lastConsolidateTs;
|
|
2116
|
-
}
|
|
2117
|
-
catch {
|
|
2118
|
-
return false;
|
|
2119
|
-
}
|
|
2120
|
-
});
|
|
2121
|
-
}
|
|
2122
|
-
catch {
|
|
2123
|
-
return false;
|
|
2124
|
-
}
|
|
2125
|
-
})();
|
|
2126
|
-
const consolidationOnCooldown = !volumeTriggered && !memoryUpdatedAfterLastConsolidate;
|
|
2127
|
-
// Profile gate: if profile explicitly disables consolidate, skip the entire pass.
|
|
2128
|
-
const consolidateDisabledByProfile = improveProfile?.processes?.consolidate?.enabled === false;
|
|
2129
|
-
let consolidation = {
|
|
2130
|
-
schemaVersion: 1,
|
|
2131
|
-
ok: true,
|
|
2132
|
-
shape: "consolidate-result",
|
|
2133
|
-
dryRun: false,
|
|
2134
|
-
previewOnly: false,
|
|
2135
|
-
target: "",
|
|
2136
|
-
processed: 0,
|
|
2137
|
-
merged: 0,
|
|
2138
|
-
deleted: 0,
|
|
2139
|
-
promoted: [],
|
|
2140
|
-
contradicted: 0,
|
|
2141
|
-
warnings: [],
|
|
2142
|
-
durationMs: 0,
|
|
2143
|
-
};
|
|
2144
|
-
let gateAutoAcceptedCount = 0;
|
|
2145
|
-
let gateAutoAcceptFailedCount = 0;
|
|
2146
|
-
const consolidateGateCfg = makeGateConfig("consolidate", {
|
|
2147
|
-
globalThreshold: options.autoAccept,
|
|
2148
|
-
dryRun: options.dryRun ?? false,
|
|
2149
|
-
stashDir: primaryStashDir,
|
|
2150
|
-
config: consolidationConfig,
|
|
2151
|
-
eventsCtx,
|
|
2152
|
-
}, { minimumThreshold: 95 });
|
|
2153
|
-
if (consolidateDisabledByProfile) {
|
|
2154
|
-
info("[improve] consolidation skipped (disabled by improve profile)");
|
|
2155
|
-
}
|
|
2156
|
-
else if (!consolidationOnCooldown) {
|
|
2157
|
-
consolidation = await akmConsolidate({
|
|
2158
|
-
...options.consolidateOptions,
|
|
2159
|
-
config: consolidationConfig,
|
|
2160
|
-
stashDir: options.stashDir,
|
|
2161
|
-
autoTriggered: volumeTriggered,
|
|
2162
|
-
// Tie consolidate proposals back to this improve invocation so
|
|
2163
|
-
// accept-rate-per-run aggregation works. Mirrors reflect/propose/extract.
|
|
2164
|
-
sourceRun: `consolidate-${Date.now()}`,
|
|
2165
|
-
// incrementalSince: when set in the profile config, narrows the candidate
|
|
2166
|
-
// pool to memories modified within that window + their graph neighbours,
|
|
2167
|
-
// keeping each pass focused on recent changes. Omit for a full-pool sweep
|
|
2168
|
-
// (default for nightly passes). See config-schema.ts for guidance.
|
|
2169
|
-
incrementalSince: improveProfile?.processes?.consolidate?.incrementalSince,
|
|
2170
|
-
maxChunkSize: improveProfile?.processes?.consolidate?.maxChunkSize,
|
|
2171
|
-
// Honor profile.autoAccept (already merged into options.autoAccept at the
|
|
2172
|
-
// top of akmImprove). The CLI parser always supplies 90 when --auto-accept
|
|
2173
|
-
// is absent, so ?? 90 is not needed here and would prevent --auto-accept=false
|
|
2174
|
-
// (which maps to undefined) from disabling consolidation auto-accept.
|
|
2175
|
-
// options.consolidateOptions.autoAccept (if explicitly provided by caller)
|
|
2176
|
-
// still wins because the spread above runs first.
|
|
2177
|
-
autoAccept: options.consolidateOptions?.autoAccept ?? options.autoAccept,
|
|
2178
|
-
});
|
|
2179
|
-
{
|
|
2180
|
-
const consolidateGr = await runAutoAcceptGate(consolidation.promoted.map((proposalId) => {
|
|
2181
|
-
try {
|
|
2182
|
-
if (!primaryStashDir)
|
|
2183
|
-
return { proposalId, confidence: undefined };
|
|
2184
|
-
const proposal = getProposal(primaryStashDir, proposalId);
|
|
2185
|
-
return { proposalId, confidence: proposal.confidence };
|
|
2186
|
-
}
|
|
2187
|
-
catch {
|
|
2188
|
-
return { proposalId, confidence: undefined };
|
|
2189
|
-
}
|
|
2190
|
-
}), consolidateGateCfg);
|
|
2191
|
-
gateAutoAcceptedCount += consolidateGr.promoted.length;
|
|
2192
|
-
gateAutoAcceptFailedCount += consolidateGr.failed.length;
|
|
2193
|
-
}
|
|
2194
|
-
if (consolidation.processed > 0) {
|
|
2195
|
-
appendEvent({
|
|
2196
|
-
eventType: "consolidate_completed",
|
|
2197
|
-
ref: "memory:_consolidation",
|
|
2198
|
-
metadata: { processed: consolidation.processed, merged: consolidation.merged },
|
|
2199
|
-
}, eventsCtx);
|
|
2200
|
-
}
|
|
2201
|
-
}
|
|
2202
|
-
else {
|
|
2203
|
-
appendEvent({
|
|
2204
|
-
eventType: "improve_skipped",
|
|
2205
|
-
ref: "memory:_consolidation",
|
|
2206
|
-
metadata: {
|
|
2207
|
-
reason: "consolidation_no_memory_updates",
|
|
2208
|
-
lastEventTs: lastConsolidation?.ts ?? null,
|
|
2209
|
-
},
|
|
2210
|
-
}, eventsCtx);
|
|
2211
|
-
info("[improve] consolidation skipped (no memory updates since last run)");
|
|
2212
|
-
}
|
|
2213
|
-
// D9: track whether consolidation wrote any data so graph extraction can reindex if needed
|
|
2214
|
-
const consolidationRan = !consolidateDisabledByProfile && !consolidationOnCooldown && consolidation.processed > 0;
|
|
2215
2523
|
info("[improve] post-loop maintenance starting");
|
|
2216
2524
|
const maintenanceResult = await runImproveMaintenancePasses({
|
|
2217
2525
|
options,
|
|
@@ -2252,7 +2560,6 @@ async function runImprovePostLoopStage(args) {
|
|
|
2252
2560
|
}
|
|
2253
2561
|
return {
|
|
2254
2562
|
allWarnings,
|
|
2255
|
-
consolidation,
|
|
2256
2563
|
deadUrls,
|
|
2257
2564
|
...(maintenanceResult.memoryInference ? { memoryInference: maintenanceResult.memoryInference } : {}),
|
|
2258
2565
|
...(maintenanceResult.graphExtraction ? { graphExtraction: maintenanceResult.graphExtraction } : {}),
|
|
@@ -2264,12 +2571,16 @@ async function runImprovePostLoopStage(args) {
|
|
|
2264
2571
|
graphExtractionDurationMs: maintenanceResult.graphExtractionDurationMs,
|
|
2265
2572
|
orphansPurged: maintenanceResult.orphansPurged,
|
|
2266
2573
|
proposalsExpired: maintenanceResult.proposalsExpired,
|
|
2267
|
-
|
|
2268
|
-
|
|
2574
|
+
// Consolidation's auto-accept gate counts now accrue in the preparation
|
|
2575
|
+
// stage (#551); post-loop no longer runs an auto-accept gate of its own.
|
|
2576
|
+
gateAutoAcceptedCount: 0,
|
|
2577
|
+
gateAutoAcceptFailedCount: 0,
|
|
2269
2578
|
};
|
|
2270
2579
|
}
|
|
2271
2580
|
// TODO(refactor): mutates the passed-in `allWarnings` array as a hidden side channel. Return warnings in ImproveMaintenanceResult and merge in caller — invasive signature change deferred to next refactor pass.
|
|
2272
|
-
|
|
2581
|
+
// Exported for tests (#584/#585 DB-locking regression coverage); production
|
|
2582
|
+
// callers reach it only through akmImprove → runImprovePostLoopStage.
|
|
2583
|
+
export async function runImproveMaintenancePasses(args) {
|
|
2273
2584
|
const { options, primaryStashDir, memoryRefsForInference, allWarnings, reindexFn, consolidationRan, budgetSignal, eventsCtx, improveProfile, } = args;
|
|
2274
2585
|
if (!primaryStashDir)
|
|
2275
2586
|
return { memoryInferenceDurationMs: 0, graphExtractionDurationMs: 0 };
|
|
@@ -2288,8 +2599,27 @@ async function runImproveMaintenancePasses(args) {
|
|
|
2288
2599
|
let graphExtractionDurationMs = 0;
|
|
2289
2600
|
let orphansPurged = 0;
|
|
2290
2601
|
let proposalsExpired = 0;
|
|
2602
|
+
const openIndexDb = () => openDatabase(getDbPath(), config.embedding?.dimension ? { embeddingDim: config.embedding.dimension } : undefined);
|
|
2603
|
+
// #584: reindexFn opens its own write handle on the same index.db WAL file.
|
|
2604
|
+
// Holding our handle across that call produced SQLITE_BUSY / "database is
|
|
2605
|
+
// locked" failures in production, so the handle is closed BEFORE every
|
|
2606
|
+
// reindex and reopened after — the fresh handle also sees the post-reindex
|
|
2607
|
+
// state that graph extraction and staleness detection below rely on. The
|
|
2608
|
+
// reopen runs in `finally` so a failed reindex still leaves a usable handle.
|
|
2609
|
+
const reindexWithIndexDbReleased = async (stashDir) => {
|
|
2610
|
+
if (db) {
|
|
2611
|
+
closeDatabase(db);
|
|
2612
|
+
db = undefined;
|
|
2613
|
+
}
|
|
2614
|
+
try {
|
|
2615
|
+
await reindexFn({ stashDir });
|
|
2616
|
+
}
|
|
2617
|
+
finally {
|
|
2618
|
+
db = openIndexDb();
|
|
2619
|
+
}
|
|
2620
|
+
};
|
|
2291
2621
|
try {
|
|
2292
|
-
db =
|
|
2622
|
+
db = openIndexDb();
|
|
2293
2623
|
// Memory inference candidate-discovery (post-Item 9 fix from
|
|
2294
2624
|
// memory:akm-improve-critical-review-2026-05-20). Previously this pass
|
|
2295
2625
|
// was gated on memoryRefsForInference.size > 0 AND passed those refs as a
|
|
@@ -2315,10 +2645,17 @@ async function runImproveMaintenancePasses(args) {
|
|
|
2315
2645
|
const inferenceStart = Date.now();
|
|
2316
2646
|
try {
|
|
2317
2647
|
// O-1 (#364): pass budget signal so a hung inference call is cancelled.
|
|
2318
|
-
memoryInference = await
|
|
2319
|
-
|
|
2320
|
-
|
|
2321
|
-
|
|
2648
|
+
memoryInference = await withLlmStage("memory-inference", () => memoryInferenceFn({
|
|
2649
|
+
config,
|
|
2650
|
+
sources,
|
|
2651
|
+
signal: budgetSignal,
|
|
2652
|
+
db,
|
|
2653
|
+
reEnrich: false,
|
|
2654
|
+
onProgress: (event) => {
|
|
2655
|
+
const current = event.currentRef ? ` ${event.currentRef}` : "";
|
|
2656
|
+
info(`[improve] memory inference ${event.processed}/${event.total}${current} (written ${event.writtenFacts}, skipped ${event.skippedNoFacts})`);
|
|
2657
|
+
},
|
|
2658
|
+
}));
|
|
2322
2659
|
memoryInferenceDurationMs = Date.now() - inferenceStart;
|
|
2323
2660
|
actions.push({ ref: "memory:_inference", mode: "memory-inference", result: memoryInference });
|
|
2324
2661
|
info(`[improve] memory inference complete (${memoryInference.writtenFacts} facts written from ${memoryInference.splitParents} parents)`);
|
|
@@ -2331,7 +2668,7 @@ async function runImproveMaintenancePasses(args) {
|
|
|
2331
2668
|
if (memoryInference && (memoryInference.splitParents > 0 || memoryInference.writtenFacts > 0)) {
|
|
2332
2669
|
info("[improve] reindexing after memory inference writes");
|
|
2333
2670
|
try {
|
|
2334
|
-
await
|
|
2671
|
+
await reindexWithIndexDbReleased(primaryStashDir);
|
|
2335
2672
|
reindexedAfterInference = true;
|
|
2336
2673
|
info("[improve] reindex after memory inference complete");
|
|
2337
2674
|
}
|
|
@@ -2367,7 +2704,7 @@ async function runImproveMaintenancePasses(args) {
|
|
|
2367
2704
|
if (consolidationRan && !reindexedAfterInference) {
|
|
2368
2705
|
info("[improve] reindexing after consolidation (graph extraction needs current state)");
|
|
2369
2706
|
try {
|
|
2370
|
-
await
|
|
2707
|
+
await reindexWithIndexDbReleased(primaryStashDir);
|
|
2371
2708
|
reindexedAfterInference = true;
|
|
2372
2709
|
info("[improve] reindex after consolidation complete");
|
|
2373
2710
|
}
|
|
@@ -2375,10 +2712,8 @@ async function runImproveMaintenancePasses(args) {
|
|
|
2375
2712
|
allWarnings.push(`reindex after consolidation failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
2376
2713
|
}
|
|
2377
2714
|
}
|
|
2378
|
-
|
|
2379
|
-
|
|
2380
|
-
db = openDatabase(getDbPath(), config.embedding?.dimension ? { embeddingDim: config.embedding.dimension } : undefined);
|
|
2381
|
-
}
|
|
2715
|
+
// #584: no close/reopen needed here — reindexWithIndexDbReleased
|
|
2716
|
+
// already swapped in a fresh post-reindex handle.
|
|
2382
2717
|
// Resolve touched refs to absolute file paths. Skipped for fullScan
|
|
2383
2718
|
// (candidatePaths stays undefined → extractor processes all files).
|
|
2384
2719
|
let candidatePaths;
|
|
@@ -2398,9 +2733,15 @@ async function runImproveMaintenancePasses(args) {
|
|
|
2398
2733
|
info(`[improve] graph extraction ${event.processed}/${event.total}${current} (extracted ${event.extracted}, entities ${event.totalEntities}, relations ${event.totalRelations})`);
|
|
2399
2734
|
};
|
|
2400
2735
|
// O-1 (#364): pass budget signal so a hung graph extraction call is cancelled.
|
|
2401
|
-
graphExtraction = await
|
|
2402
|
-
|
|
2403
|
-
|
|
2736
|
+
graphExtraction = await withLlmStage("graph-extraction", () => graphExtractionFn({
|
|
2737
|
+
config,
|
|
2738
|
+
sources,
|
|
2739
|
+
signal: budgetSignal,
|
|
2740
|
+
db,
|
|
2741
|
+
reEnrich: false,
|
|
2742
|
+
onProgress: progressHandler,
|
|
2743
|
+
options: { candidatePaths },
|
|
2744
|
+
}));
|
|
2404
2745
|
graphExtractionDurationMs = Date.now() - extractionStart;
|
|
2405
2746
|
actions.push({ ref: "graph:_artifact", mode: "graph-extraction", result: graphExtraction });
|
|
2406
2747
|
info(`[improve] graph extraction complete (${graphExtraction.quality.extractedFiles} files, ${graphExtraction.quality.entityCount} entities, ${graphExtraction.quality.relationCount} relations)`);
|
|
@@ -2471,14 +2812,22 @@ async function runImproveMaintenancePasses(args) {
|
|
|
2471
2812
|
// invocation, and every command surface emits at least one event besides —
|
|
2472
2813
|
// without this trim, state.db is a permanent append-only log. Config key
|
|
2473
2814
|
// `improve.eventRetentionDays` (default 90, set 0 to disable) controls the
|
|
2474
|
-
// window.
|
|
2475
|
-
// the index `db` above
|
|
2815
|
+
// window. The purge runs against state.db (a different SQLite file from
|
|
2816
|
+
// the index `db` above).
|
|
2476
2817
|
{
|
|
2477
2818
|
const retentionDays = typeof config.improve?.eventRetentionDays === "number" ? config.improve.eventRetentionDays : 90;
|
|
2478
2819
|
if (retentionDays > 0) {
|
|
2820
|
+
// #585: reuse the long-lived eventsCtx.db connection when akmImprove
|
|
2821
|
+
// opened one — opening a second state.db write connection while
|
|
2822
|
+
// eventsDb is still live made two simultaneous writers contend on the
|
|
2823
|
+
// same WAL file ("database is locked"). Only the eventsCtx.dbPath
|
|
2824
|
+
// fallback path (state.db failed to open up-front) opens — and then
|
|
2825
|
+
// owns and closes — its own handle. C2 still holds: the fallback uses
|
|
2826
|
+
// the boundary-pinned path, never a live `process.env` re-read.
|
|
2827
|
+
const ownsStateDb = !eventsCtx?.db;
|
|
2479
2828
|
let stateDb;
|
|
2480
2829
|
try {
|
|
2481
|
-
stateDb = openStateDatabase();
|
|
2830
|
+
stateDb = eventsCtx?.db ?? openStateDatabase(eventsCtx?.dbPath);
|
|
2482
2831
|
const purgedCount = purgeOldEvents(stateDb, retentionDays);
|
|
2483
2832
|
if (purgedCount > 0) {
|
|
2484
2833
|
info(`[improve] events purge: ${purgedCount} event(s) older than ${retentionDays}d removed from state.db`);
|
|
@@ -2506,7 +2855,7 @@ async function runImproveMaintenancePasses(args) {
|
|
|
2506
2855
|
allWarnings.push(`events purge failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
2507
2856
|
}
|
|
2508
2857
|
finally {
|
|
2509
|
-
if (stateDb) {
|
|
2858
|
+
if (ownsStateDb && stateDb) {
|
|
2510
2859
|
try {
|
|
2511
2860
|
stateDb.close();
|
|
2512
2861
|
}
|
|
@@ -2515,6 +2864,37 @@ async function runImproveMaintenancePasses(args) {
|
|
|
2515
2864
|
}
|
|
2516
2865
|
}
|
|
2517
2866
|
}
|
|
2867
|
+
// task_logs in logs.db (#579) shares the same retention window as
|
|
2868
|
+
// events/improve_runs — all three are observability data governed by
|
|
2869
|
+
// the single improve.eventRetentionDays knob. Separate try/finally
|
|
2870
|
+
// because logs.db is a different file: a locked/missing logs.db must
|
|
2871
|
+
// not block the state.db purges above.
|
|
2872
|
+
let logsDb;
|
|
2873
|
+
try {
|
|
2874
|
+
logsDb = openLogsDatabase();
|
|
2875
|
+
const taskLogsPurged = purgeOldTaskLogs(logsDb, retentionDays);
|
|
2876
|
+
if (taskLogsPurged > 0) {
|
|
2877
|
+
info(`[improve] task_logs purge: ${taskLogsPurged} log line(s) older than ${retentionDays}d removed from logs.db`);
|
|
2878
|
+
}
|
|
2879
|
+
appendEvent({
|
|
2880
|
+
eventType: "task_logs_purged",
|
|
2881
|
+
ref: "task_logs:_purge",
|
|
2882
|
+
metadata: { purgedCount: taskLogsPurged, retentionDays },
|
|
2883
|
+
}, eventsCtx);
|
|
2884
|
+
}
|
|
2885
|
+
catch (err) {
|
|
2886
|
+
allWarnings.push(`task_logs purge failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
2887
|
+
}
|
|
2888
|
+
finally {
|
|
2889
|
+
if (logsDb) {
|
|
2890
|
+
try {
|
|
2891
|
+
logsDb.close();
|
|
2892
|
+
}
|
|
2893
|
+
catch {
|
|
2894
|
+
// best-effort
|
|
2895
|
+
}
|
|
2896
|
+
}
|
|
2897
|
+
}
|
|
2518
2898
|
}
|
|
2519
2899
|
}
|
|
2520
2900
|
// Phase 4A (staleness detection). Activates the `deprecated` belief-state
|
|
@@ -2523,7 +2903,7 @@ async function runImproveMaintenancePasses(args) {
|
|
|
2523
2903
|
// and before the URL check (which lives in the outer caller).
|
|
2524
2904
|
if (sources.length > 0) {
|
|
2525
2905
|
try {
|
|
2526
|
-
stalenessDetection = await stalenessDetectionFn(config, sources, budgetSignal, db);
|
|
2906
|
+
stalenessDetection = await withLlmStage("staleness-detection", () => stalenessDetectionFn({ config, sources, signal: budgetSignal, db }));
|
|
2527
2907
|
if (stalenessDetection.considered > 0) {
|
|
2528
2908
|
info(`[improve] staleness detection complete (considered ${stalenessDetection.considered}, ` +
|
|
2529
2909
|
`deprecated ${stalenessDetection.deprecated}, confirmed ${stalenessDetection.confirmed}, ` +
|