akm-cli 0.8.2 → 0.9.0-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +187 -0
- package/dist/assets/help/help-proposals.md +1 -2
- package/dist/assets/hints/cli-hints-full.md +34 -19
- package/dist/assets/hints/cli-hints-short.md +1 -1
- package/dist/assets/profiles/catchup.json +13 -0
- package/dist/assets/profiles/consolidate.json +13 -0
- package/dist/assets/profiles/frequent.json +13 -0
- package/dist/assets/tasks/core/backup.yml +4 -0
- package/dist/assets/tasks/core/extract.yml +4 -0
- package/dist/assets/tasks/core/improve.yml +4 -0
- package/dist/assets/tasks/core/index-refresh.yml +4 -0
- package/dist/assets/tasks/core/sync.yml +4 -0
- package/dist/assets/tasks/core/update-stashes.yml +4 -0
- package/dist/assets/tasks/core/version-check.yml +4 -0
- package/dist/cli/config-migrate.js +6 -6
- package/dist/cli/config-validate.js +4 -4
- package/dist/cli/confirm.js +3 -3
- package/dist/cli/parse-args.js +1 -1
- package/dist/cli/shared.js +51 -14
- package/dist/cli-node.mjs +26 -0
- package/dist/cli.js +171 -3862
- package/dist/commands/{agent-dispatch.js → agent/agent-dispatch.js} +6 -6
- package/dist/commands/{agent-support.js → agent/agent-support.js} +2 -2
- package/dist/commands/agent/contribute-cli.js +200 -0
- package/dist/commands/completions.js +1 -1
- package/dist/commands/config-cli.js +240 -3
- package/dist/commands/config-edit.js +344 -0
- package/dist/commands/db-cli.js +2 -2
- package/dist/commands/env/env-cli.js +529 -0
- package/dist/commands/env/env.js +410 -0
- package/dist/commands/env/secret-cli.js +259 -0
- package/dist/commands/{secret.js → env/secret.js} +6 -47
- package/dist/commands/events.js +4 -4
- package/dist/commands/feedback-cli.js +18 -34
- package/dist/commands/graph/graph-cli.js +132 -0
- package/dist/commands/{graph.js → graph/graph.js} +22 -16
- package/dist/commands/health/checks.js +279 -0
- package/dist/commands/health.js +94 -262
- package/dist/commands/{consolidate.js → improve/consolidate.js} +48 -36
- package/dist/commands/{distill-promotion-policy.js → improve/distill-promotion-policy.js} +3 -3
- package/dist/commands/{distill.js → improve/distill.js} +39 -18
- package/dist/commands/{eval-cases.js → improve/eval-cases.js} +1 -1
- package/dist/commands/{extract-cli.js → improve/extract-cli.js} +4 -4
- package/dist/commands/{extract-prompt.js → improve/extract-prompt.js} +2 -2
- package/dist/commands/{extract.js → improve/extract.js} +185 -26
- package/dist/commands/{improve-auto-accept.js → improve/improve-auto-accept.js} +4 -4
- package/dist/commands/{improve-cli.js → improve/improve-cli.js} +44 -22
- package/dist/commands/{improve-profiles.js → improve/improve-profiles.js} +13 -7
- package/dist/commands/{improve-result-file.js → improve/improve-result-file.js} +1 -1
- package/dist/commands/{improve.js → improve/improve.js} +509 -245
- package/dist/{core → commands/improve/memory}/memory-belief.js +2 -2
- package/dist/{core → commands/improve/memory}/memory-contradiction-detect.js +5 -5
- package/dist/{core → commands/improve/memory}/memory-improve.js +4 -4
- package/dist/commands/{reflect.js → improve/reflect.js} +33 -28
- package/dist/commands/improve/session-asset.js +248 -0
- package/dist/commands/lint/agent-linter.js +1 -1
- package/dist/commands/lint/base-linter.js +55 -37
- package/dist/commands/lint/command-linter.js +1 -1
- package/dist/commands/lint/default-linter.js +1 -1
- package/dist/commands/lint/env-key-rules.js +1 -1
- package/dist/commands/lint/index.js +19 -25
- package/dist/commands/lint/knowledge-linter.js +1 -1
- package/dist/commands/lint/memory-linter.js +1 -1
- package/dist/commands/lint/registry.js +8 -8
- package/dist/commands/lint/skill-linter.js +1 -1
- package/dist/commands/lint/task-linter.js +1 -1
- package/dist/commands/lint/workflow-linter.js +1 -1
- package/dist/commands/lint.js +1 -1
- package/dist/commands/observability-cli.js +244 -0
- package/dist/commands/proposal/drain-policies.js +3 -3
- package/dist/commands/proposal/drain.js +15 -10
- package/dist/commands/proposal/proposal-cli.js +478 -0
- package/dist/commands/{proposal.js → proposal/proposal.js} +5 -5
- package/dist/commands/{propose.js → proposal/propose.js} +11 -11
- package/dist/{core → commands/proposal/validators}/proposal-quality-validators.js +8 -3
- package/dist/{core → commands/proposal/validators}/proposal-validators.js +5 -5
- package/dist/{core → commands/proposal/validators}/proposals.js +13 -7
- package/dist/commands/{curate.js → read/curate.js} +7 -7
- package/dist/commands/{knowledge.js → read/knowledge.js} +22 -9
- package/dist/commands/{registry-search.js → read/registry-search.js} +5 -5
- package/dist/commands/{remember-cli.js → read/remember-cli.js} +15 -7
- package/dist/commands/read/search-cli.js +207 -0
- package/dist/commands/{search.js → read/search.js} +22 -27
- package/dist/commands/{show.js → read/show.js} +31 -45
- package/dist/commands/registry-cli.js +8 -8
- package/dist/commands/remember.js +8 -8
- package/dist/commands/sources/add-cli.js +293 -0
- package/dist/commands/{history.js → sources/history.js} +27 -25
- package/dist/commands/{info.js → sources/info.js} +6 -6
- package/dist/commands/{init.js → sources/init.js} +6 -6
- package/dist/commands/{installed-stashes.js → sources/installed-stashes.js} +12 -12
- package/dist/commands/{migration-help.js → sources/migration-help.js} +3 -2
- package/dist/commands/{schema-repair.js → sources/schema-repair.js} +8 -8
- package/dist/commands/{self-update.js → sources/self-update.js} +10 -9
- package/dist/commands/{source-add.js → sources/source-add.js} +10 -10
- package/dist/commands/{source-clone.js → sources/source-clone.js} +7 -7
- package/dist/commands/{source-manage.js → sources/source-manage.js} +4 -4
- package/dist/commands/sources/sources-cli.js +305 -0
- package/dist/commands/sources/stash-cli.js +219 -0
- package/dist/commands/{stash-skeleton.js → sources/stash-skeleton.js} +2 -1
- package/dist/commands/tasks/default-tasks.js +173 -0
- package/dist/commands/tasks/tasks-cli.js +210 -0
- package/dist/commands/{tasks.js → tasks/tasks.js} +14 -14
- package/dist/commands/wiki-cli.js +307 -0
- package/dist/commands/workflow-cli.js +329 -0
- package/dist/core/action-contributors.js +1 -1
- package/dist/core/assert.js +40 -0
- package/dist/core/asset/asset-create.js +54 -0
- package/dist/core/{asset-ref.js → asset/asset-ref.js} +21 -4
- package/dist/core/{asset-registry.js → asset/asset-registry.js} +3 -3
- package/dist/core/{asset-spec.js → asset/asset-spec.js} +17 -31
- package/dist/core/{markdown.js → asset/markdown.js} +1 -1
- package/dist/core/{stash-meta.js → asset/stash-meta.js} +1 -1
- package/dist/core/best-effort.js +64 -0
- package/dist/core/common.js +32 -18
- package/dist/core/{config-io.js → config/config-io.js} +29 -19
- package/dist/core/{config-migration.js → config/config-migration.js} +11 -9
- package/dist/core/{config-schema.js → config/config-schema.js} +45 -1
- package/dist/core/config/config-types.js +16 -0
- package/dist/core/{config-walker.js → config/config-walker.js} +2 -2
- package/dist/core/{config.js → config/config.js} +10 -8
- package/dist/core/env-secret-ref.js +90 -0
- package/dist/core/errors.js +13 -3
- package/dist/core/events.js +27 -4
- package/dist/core/file-lock.js +1 -1
- package/dist/core/improve-types.js +48 -0
- package/dist/core/lesson-lint.js +2 -2
- package/dist/core/paths.js +2 -2
- package/dist/core/ripgrep/install.js +2 -2
- package/dist/core/ripgrep/resolve.js +2 -2
- package/dist/core/state-db.js +88 -46
- package/dist/core/text-truncation.js +148 -0
- package/dist/core/time.js +1 -1
- package/dist/core/write-source.js +98 -85
- package/dist/indexer/{db-backup.js → db/db-backup.js} +9 -24
- package/dist/indexer/{db.js → db/db.js} +126 -116
- package/dist/indexer/{graph-db.js → db/graph-db.js} +9 -4
- package/dist/indexer/{llm-cache.js → db/llm-cache.js} +15 -12
- package/dist/indexer/ensure-index.js +4 -4
- package/dist/indexer/{graph-boost.js → graph/graph-boost.js} +1 -1
- package/dist/indexer/{graph-extraction.js → graph/graph-extraction.js} +55 -13
- package/dist/indexer/indexer.js +37 -30
- package/dist/indexer/init.js +54 -0
- package/dist/indexer/manifest.js +10 -10
- package/dist/indexer/{memory-inference.js → passes/memory-inference.js} +92 -23
- package/dist/indexer/{metadata-contributors.js → passes/metadata-contributors.js} +10 -8
- package/dist/indexer/{metadata.js → passes/metadata.js} +15 -19
- package/dist/indexer/{staleness-detect.js → passes/staleness-detect.js} +53 -12
- package/dist/indexer/{db-search.js → search/db-search.js} +28 -16
- package/dist/indexer/{ranking-contributors.js → search/ranking-contributors.js} +1 -1
- package/dist/indexer/{ranking.js → search/ranking.js} +2 -2
- package/dist/indexer/{search-hit-enrichers.js → search/search-hit-enrichers.js} +3 -3
- package/dist/indexer/{search-source.js → search/search-source.js} +8 -8
- package/dist/indexer/{semantic-status.js → search/semantic-status.js} +3 -3
- package/dist/indexer/usage/unmigrated-vaults-guard.js +94 -0
- package/dist/indexer/{usage-events.js → usage/usage-events.js} +32 -0
- package/dist/indexer/{file-context.js → walk/file-context.js} +10 -15
- package/dist/indexer/{matchers.js → walk/matchers.js} +13 -9
- package/dist/indexer/{path-resolver.js → walk/path-resolver.js} +6 -6
- package/dist/indexer/{project-context.js → walk/project-context.js} +1 -1
- package/dist/indexer/{walker.js → walk/walker.js} +4 -3
- package/dist/integrations/agent/builder-shared.js +39 -0
- package/dist/integrations/agent/builders.js +14 -81
- package/dist/integrations/agent/config.js +6 -4
- package/dist/integrations/agent/detect.js +1 -1
- package/dist/integrations/agent/index.js +23 -8
- package/dist/integrations/agent/prompts.js +2 -3
- package/dist/integrations/agent/runner.js +22 -3
- package/dist/integrations/agent/spawn.js +9 -10
- package/dist/integrations/harnesses/claude/agent-builder.js +48 -0
- package/dist/integrations/harnesses/claude/config-import.js +70 -0
- package/dist/integrations/harnesses/claude/index.js +64 -0
- package/dist/integrations/{session-logs/providers/claude-code.js → harnesses/claude/session-log.js} +16 -1
- package/dist/integrations/harnesses/index.js +144 -0
- package/dist/integrations/harnesses/opencode/agent-builder.js +43 -0
- package/dist/integrations/harnesses/opencode/config-import.js +82 -0
- package/dist/integrations/harnesses/opencode/index.js +59 -0
- package/dist/integrations/{session-logs/providers/opencode.js → harnesses/opencode/session-log.js} +1 -1
- package/dist/integrations/harnesses/opencode-sdk/index.js +49 -0
- package/dist/integrations/harnesses/opencode-sdk/sdk-runner.js +234 -0
- package/dist/integrations/harnesses/types.js +43 -0
- package/dist/integrations/lockfile.js +7 -16
- package/dist/integrations/session-logs/index.js +82 -9
- package/dist/llm/call-ai.js +4 -4
- package/dist/llm/client.js +131 -6
- package/dist/llm/embedder.js +6 -6
- package/dist/llm/embedders/local.js +9 -22
- package/dist/llm/embedders/remote.js +2 -2
- package/dist/llm/embedders/types.js +1 -1
- package/dist/llm/graph-extract.js +31 -12
- package/dist/llm/index-passes.js +1 -1
- package/dist/llm/memory-infer.js +12 -5
- package/dist/llm/metadata-enhance.js +2 -2
- package/dist/output/context.js +6 -44
- package/dist/output/renderers.js +88 -58
- package/dist/output/shapes/curate.js +7 -3
- package/dist/output/shapes/distill.js +7 -3
- package/dist/output/shapes/env-list.js +18 -16
- package/dist/output/shapes/events.js +5 -4
- package/dist/output/shapes/helpers.js +2 -4
- package/dist/output/shapes/history.js +7 -3
- package/dist/output/shapes/passthrough.js +8 -11
- package/dist/output/shapes/{proposal-accept.js → proposal/accept.js} +7 -3
- package/dist/output/shapes/{proposal-diff.js → proposal/diff.js} +7 -3
- package/dist/output/shapes/{proposal-list.js → proposal/list.js} +7 -3
- package/dist/output/shapes/{proposal-producer.js → proposal/producer.js} +5 -4
- package/dist/output/shapes/{proposal-reject.js → proposal/reject.js} +7 -3
- package/dist/output/shapes/{proposal-show.js → proposal/show.js} +7 -3
- package/dist/output/shapes/registry-search.js +7 -3
- package/dist/output/shapes/registry.js +12 -0
- package/dist/output/shapes/search.js +7 -3
- package/dist/output/shapes/secret-list.js +18 -16
- package/dist/output/shapes/show.js +7 -3
- package/dist/output/shapes.js +55 -30
- package/dist/output/text/add.js +2 -3
- package/dist/output/text/clone.js +2 -3
- package/dist/output/text/config.js +2 -3
- package/dist/output/text/curate.js +4 -3
- package/dist/output/text/distill.js +2 -3
- package/dist/output/text/enable-disable.js +5 -4
- package/dist/output/text/env.js +13 -0
- package/dist/output/text/events.js +5 -4
- package/dist/output/text/feedback.js +4 -3
- package/dist/output/text/helpers.js +54 -39
- package/dist/output/text/history.js +2 -3
- package/dist/output/text/import.js +2 -3
- package/dist/output/text/index.js +2 -3
- package/dist/output/text/info.js +2 -3
- package/dist/output/text/init.js +2 -3
- package/dist/output/text/list.js +2 -3
- package/dist/output/text/proposal/producer.js +9 -0
- package/dist/output/text/proposal/proposal.js +13 -0
- package/dist/output/text/registry-commands.js +8 -7
- package/dist/output/text/registry.js +12 -0
- package/dist/output/text/remember.js +4 -3
- package/dist/output/text/remove.js +2 -3
- package/dist/output/text/save.js +2 -3
- package/dist/output/text/search.js +4 -3
- package/dist/output/text/show.js +4 -3
- package/dist/output/text/update.js +2 -3
- package/dist/output/text/upgrade.js +2 -3
- package/dist/output/text/wiki.js +12 -11
- package/dist/output/text/workflow.js +12 -10
- package/dist/output/text.js +66 -32
- package/dist/registry/build-index.js +11 -10
- package/dist/registry/factory.js +1 -1
- package/dist/registry/origin-resolve.js +1 -1
- package/dist/registry/providers/index.js +2 -2
- package/dist/registry/providers/skills-sh.js +91 -72
- package/dist/registry/providers/static-index.js +75 -52
- package/dist/registry/resolve.js +3 -3
- package/dist/runtime.js +242 -0
- package/dist/scripts/migrate-storage.js +1594 -673
- package/dist/scripts/migrations/import-fs-improve-runs-to-db.js +240 -166
- package/dist/setup/detect.js +311 -9
- package/dist/setup/harness-config-import.js +6 -120
- package/dist/setup/setup.js +454 -43
- package/dist/sources/include.js +1 -1
- package/dist/sources/provider-factory.js +2 -2
- package/dist/sources/providers/filesystem.js +3 -3
- package/dist/sources/providers/git.js +9 -9
- package/dist/sources/providers/index.js +4 -4
- package/dist/sources/providers/npm.js +6 -6
- package/dist/sources/providers/provider-utils.js +13 -20
- package/dist/sources/providers/sync-from-ref.js +5 -5
- package/dist/sources/providers/tar-utils.js +2 -2
- package/dist/sources/providers/website.js +2 -2
- package/dist/sources/resolve.js +5 -5
- package/dist/sources/website-ingest.js +5 -5
- package/dist/storage/database.js +102 -0
- package/dist/storage/engines/sqlite-migrations.js +42 -0
- package/dist/storage/locations.js +25 -0
- package/dist/storage/repositories/index-db.js +43 -0
- package/dist/storage/repositories/workflow-runs-repository.js +141 -0
- package/dist/tasks/backends/cron.js +4 -4
- package/dist/tasks/backends/exec-utils.js +32 -0
- package/dist/tasks/backends/index.js +3 -3
- package/dist/tasks/backends/launchd.js +7 -14
- package/dist/tasks/backends/schtasks.js +7 -16
- package/dist/tasks/embedded.js +71 -0
- package/dist/tasks/parser.js +2 -2
- package/dist/tasks/resolveAkmBin.js +1 -1
- package/dist/tasks/runner.js +28 -15
- package/dist/tasks/schedule.js +1 -1
- package/dist/tasks/validator.js +7 -7
- package/dist/text-import-hook.mjs +51 -0
- package/dist/version.js +2 -1
- package/dist/wiki/wiki.js +7 -7
- package/dist/workflows/{authoring.js → authoring/authoring.js} +6 -6
- package/dist/workflows/{scope-key.js → authoring/scope-key.js} +1 -1
- package/dist/workflows/cli.js +1 -1
- package/dist/workflows/db.js +50 -32
- package/dist/workflows/parser.js +4 -4
- package/dist/workflows/renderer.js +5 -5
- package/dist/workflows/runtime/agent-identity.js +56 -0
- package/dist/workflows/runtime/checkin.js +57 -0
- package/dist/workflows/{runs.js → runtime/runs.js} +197 -101
- package/dist/workflows/validate-summary.js +82 -0
- package/docs/README.md +1 -1
- package/docs/data-and-telemetry.md +6 -6
- package/package.json +16 -8
- package/dist/commands/add-cli.js +0 -279
- package/dist/commands/env.js +0 -213
- package/dist/integrations/agent/sdk-runner.js +0 -126
- package/dist/output/shapes/vault-list.js +0 -19
- package/dist/output/text/proposal-producer.js +0 -8
- package/dist/output/text/proposal.js +0 -12
- package/dist/output/text/vault.js +0 -16
- /package/dist/core/{asset-serialize.js → asset/asset-serialize.js} +0 -0
- /package/dist/core/{frontmatter.js → asset/frontmatter.js} +0 -0
- /package/dist/core/{config-sources.js → config/config-sources.js} +0 -0
- /package/dist/indexer/{graph-dedup.js → graph/graph-dedup.js} +0 -0
- /package/dist/{core/config-types.js → indexer/passes/pass-context.js} +0 -0
- /package/dist/indexer/{search-fields.js → search/search-fields.js} +0 -0
- /package/dist/indexer/{index-context.js → walk/index-context.js} +0 -0
- /package/dist/workflows/{document-cache.js → runtime/document-cache.js} +0 -0
|
@@ -3,44 +3,47 @@
|
|
|
3
3
|
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
4
4
|
import fs from "node:fs";
|
|
5
5
|
import path from "node:path";
|
|
6
|
-
import {
|
|
7
|
-
import {
|
|
8
|
-
import {
|
|
9
|
-
import {
|
|
10
|
-
import {
|
|
11
|
-
import {
|
|
12
|
-
import {
|
|
13
|
-
import {
|
|
14
|
-
import {
|
|
15
|
-
import { getDbPath } from "
|
|
16
|
-
import {
|
|
17
|
-
import {
|
|
18
|
-
import {
|
|
19
|
-
import {
|
|
20
|
-
import {
|
|
21
|
-
import {
|
|
22
|
-
import {
|
|
23
|
-
import {
|
|
24
|
-
import {
|
|
25
|
-
import {
|
|
26
|
-
import {
|
|
27
|
-
import { resolveImproveProcessRunnerFromProfile, resolveTriageJudgmentRunner } from "
|
|
28
|
-
import { getAvailableHarnesses } from "
|
|
29
|
-
import { isLlmFeatureEnabled, isProcessEnabled } from "
|
|
30
|
-
import { isGitBackedStash, resolveWritableOverride, saveGitStash } from "
|
|
31
|
-
import {
|
|
32
|
-
import {
|
|
33
|
-
import {
|
|
34
|
-
import {
|
|
35
|
-
import {
|
|
36
|
-
import {
|
|
37
|
-
import {
|
|
38
|
-
import {
|
|
39
|
-
import {
|
|
40
|
-
import {
|
|
41
|
-
import {
|
|
42
|
-
import {
|
|
43
|
-
import {
|
|
6
|
+
import { assertNever } from "../../core/assert.js";
|
|
7
|
+
import { makeAssetRef, parseAssetRef } from "../../core/asset/asset-ref.js";
|
|
8
|
+
import { parseFrontmatter } from "../../core/asset/frontmatter.js";
|
|
9
|
+
import { daysToMs, isAssetType } from "../../core/common.js";
|
|
10
|
+
import { getDefaultLlmConfig, loadConfig } from "../../core/config/config.js";
|
|
11
|
+
import { ConfigError, NotFoundError, rethrowIfTestIsolationError, UsageError } from "../../core/errors.js";
|
|
12
|
+
import { appendEvent, readEvents } from "../../core/events.js";
|
|
13
|
+
import { probeLock, releaseLock, tryAcquireLockSync } from "../../core/file-lock.js";
|
|
14
|
+
import { classifyImproveAction } from "../../core/improve-types.js";
|
|
15
|
+
import { getDbPath, getStateDbPathInDataDir } from "../../core/paths.js";
|
|
16
|
+
import { openStateDatabase, purgeOldEvents, purgeOldImproveRuns } from "../../core/state-db.js";
|
|
17
|
+
import { info, warn } from "../../core/warn.js";
|
|
18
|
+
import { closeDatabase, getAllEntries, getEntryCount, getRetrievalCounts, getUtilityScoresByIds, getZeroResultSearches, openDatabase, openExistingDatabase, } from "../../indexer/db/db.js";
|
|
19
|
+
import { ensureIndex } from "../../indexer/ensure-index.js";
|
|
20
|
+
import { runGraphExtractionPass } from "../../indexer/graph/graph-extraction.js";
|
|
21
|
+
import { akmIndex } from "../../indexer/indexer.js";
|
|
22
|
+
import { runMemoryInferencePass } from "../../indexer/passes/memory-inference.js";
|
|
23
|
+
import { runStalenessDetectionPass } from "../../indexer/passes/staleness-detect.js";
|
|
24
|
+
import { getWritableStashDirs, resolveSourceEntries } from "../../indexer/search/search-source.js";
|
|
25
|
+
import { countUsageEventsByType } from "../../indexer/usage/usage-events.js";
|
|
26
|
+
import { resolveAssetPath } from "../../indexer/walk/path-resolver.js";
|
|
27
|
+
import { resolveImproveProcessRunnerFromProfile, resolveTriageJudgmentRunner } from "../../integrations/agent/runner.js";
|
|
28
|
+
import { getAvailableHarnesses } from "../../integrations/session-logs/index.js";
|
|
29
|
+
import { isLlmFeatureEnabled, isProcessEnabled } from "../../llm/feature-gate.js";
|
|
30
|
+
import { isGitBackedStash, resolveWritableOverride, saveGitStash } from "../../sources/providers/git.js";
|
|
31
|
+
import { akmLint } from "../lint/index.js";
|
|
32
|
+
import { drainProposals } from "../proposal/drain.js";
|
|
33
|
+
import { resolveDrainPolicy } from "../proposal/drain-policies.js";
|
|
34
|
+
import { createProposal, expireStaleProposals, getProposal, isProposalSkipped, listProposals, purgeOrphanProposals, } from "../proposal/validators/proposals.js";
|
|
35
|
+
import { runSchemaRepairPass } from "../sources/schema-repair.js";
|
|
36
|
+
import { checkDeadUrls } from "../url-checker.js";
|
|
37
|
+
import { akmConsolidate } from "./consolidate.js";
|
|
38
|
+
import { akmDistill, deriveLessonRef, isDistillRefusedInputType } from "./distill.js";
|
|
39
|
+
import { deriveKnowledgeRef } from "./distill-promotion-policy.js";
|
|
40
|
+
import { countEvalCases, writeEvalCase } from "./eval-cases.js";
|
|
41
|
+
import { akmExtract, countNewExtractCandidates } from "./extract.js";
|
|
42
|
+
import { makeGateConfig, resolveExtractConfidence, runAutoAcceptGate } from "./improve-auto-accept.js";
|
|
43
|
+
import { isProfileFilteredForAllPasses, resolveImproveProfile, resolveProcessEnabled, shouldSkipRef, } from "./improve-profiles.js";
|
|
44
|
+
import { detectAndWriteContradictions } from "./memory/memory-contradiction-detect.js";
|
|
45
|
+
import { analyzeMemoryCleanup, applyMemoryCleanup } from "./memory/memory-improve.js";
|
|
46
|
+
import { akmReflect } from "./reflect.js";
|
|
44
47
|
function resolveImproveScope(scope) {
|
|
45
48
|
const trimmed = scope?.trim();
|
|
46
49
|
if (!trimmed)
|
|
@@ -51,7 +54,7 @@ function resolveImproveScope(scope) {
|
|
|
51
54
|
}
|
|
52
55
|
catch {
|
|
53
56
|
if (!isAssetType(trimmed)) {
|
|
54
|
-
throw new UsageError(`Unknown asset type: "${trimmed}". Valid types: memory, knowledge, skill, lesson, workflow, agent, command, script, wiki, env,
|
|
57
|
+
throw new UsageError(`Unknown asset type: "${trimmed}". Valid types: memory, knowledge, skill, lesson, workflow, agent, command, script, wiki, env, secret, task.\n` +
|
|
55
58
|
`If you passed --format to akm improve, that flag is not supported — use it with akm search or akm show instead.`, "INVALID_FLAG_VALUE");
|
|
56
59
|
}
|
|
57
60
|
return { mode: "type", value: trimmed };
|
|
@@ -70,6 +73,9 @@ function resolveImproveScope(scope) {
|
|
|
70
73
|
* {scope} scope value (e.g. a ref/type) or the scope mode (`all`)
|
|
71
74
|
* {refs} number of planned refs this run processed
|
|
72
75
|
* {accepted} number of proposals auto-accepted by the confidence gate
|
|
76
|
+
* {triage_promoted} proposals promoted by the triage pre-pass (0 if triage did not run)
|
|
77
|
+
* {triage_rejected} proposals rejected by the triage pre-pass (0 if triage did not run)
|
|
78
|
+
* {runId} this run's id (empty string when absent)
|
|
73
79
|
*
|
|
74
80
|
* The result is still passed through `sanitizeCommitMessage` downstream in
|
|
75
81
|
* `saveGitStash`, so token values never widen the commit-message attack surface
|
|
@@ -87,6 +93,9 @@ export function renderSyncCommitMessage(template, result, nowMs) {
|
|
|
87
93
|
scope: result.scope.value ?? result.scope.mode,
|
|
88
94
|
refs: String(result.plannedRefs.length),
|
|
89
95
|
accepted: String(result.gateAutoAcceptedCount ?? 0),
|
|
96
|
+
triage_promoted: String(result.triage?.promoted ?? 0),
|
|
97
|
+
triage_rejected: String(result.triage?.rejected ?? 0),
|
|
98
|
+
runId: result.runId ?? "",
|
|
90
99
|
};
|
|
91
100
|
return template.replace(/\{(\w+)\}/g, (match, key) => (Object.hasOwn(tokens, key) ? tokens[key] : match));
|
|
92
101
|
}
|
|
@@ -395,6 +404,51 @@ function isSignalDeltaEligible(ref, latestFeedback, lastProposal) {
|
|
|
395
404
|
return true;
|
|
396
405
|
return fb > lp;
|
|
397
406
|
}
|
|
407
|
+
/**
|
|
408
|
+
* H7 (#566): cooperative budget watchdog with a captured, RAII-cleared hard-kill.
|
|
409
|
+
*
|
|
410
|
+
* When the wall-clock budget expires, `onExhausted` (normally an
|
|
411
|
+
* `AbortController.abort`) signals cooperative cancellation so the run can drain
|
|
412
|
+
* its in-flight log/`state.db` flush and unwind naturally. A second hard-kill
|
|
413
|
+
* timer is then armed as a watchdog: it only `exit(0)`s if the drain itself
|
|
414
|
+
* overruns `hardKillGraceMs`, preventing the process from outliving the task
|
|
415
|
+
* timeout window (lock-cascade fix).
|
|
416
|
+
*
|
|
417
|
+
* Both timers are captured; the returned dispose() clears whichever is still
|
|
418
|
+
* pending. Callers invoke it from a `finally`, so a *clean* drain reaches the
|
|
419
|
+
* `finally` and cancels the pending hard-kill before it can fire — the previous
|
|
420
|
+
* detached `setTimeout(() => process.exit(0), 5000)` always fired, truncating a
|
|
421
|
+
* clean flush. The hard-kill timer is `unref()`-ed so it never keeps the event
|
|
422
|
+
* loop alive on its own: once the run drains it exits with its own code, not the
|
|
423
|
+
* forced 0.
|
|
424
|
+
*
|
|
425
|
+
* Dependencies are injectable purely so the concurrency-sensitive timing
|
|
426
|
+
* contract can be exercised deterministically in unit tests.
|
|
427
|
+
*/
|
|
428
|
+
export function armBudgetWatchdog(budgetMs, controller, deps) {
|
|
429
|
+
const setTimeoutFn = deps?.setTimeoutFn ?? setTimeout;
|
|
430
|
+
const clearTimeoutFn = deps?.clearTimeoutFn ?? clearTimeout;
|
|
431
|
+
const exitFn = deps?.exitFn ?? ((code) => process.exit(code));
|
|
432
|
+
const hardKillGraceMs = deps?.hardKillGraceMs ?? 5_000;
|
|
433
|
+
let hardKillTimer;
|
|
434
|
+
const budgetTimer = setTimeoutFn(() => {
|
|
435
|
+
// Cooperative cancellation first: let the run drain.
|
|
436
|
+
controller.abort("improve budget exhausted");
|
|
437
|
+
// Watchdog: only force-exit if the drain itself overruns the grace period.
|
|
438
|
+
// Exit 0: budget exhaustion is a normal scheduled-task condition, not an error.
|
|
439
|
+
hardKillTimer = setTimeoutFn(() => exitFn(0), hardKillGraceMs);
|
|
440
|
+
// Never keep the event loop alive solely for the watchdog.
|
|
441
|
+
hardKillTimer.unref?.();
|
|
442
|
+
}, budgetMs);
|
|
443
|
+
// RAII dispose: clears whichever timer is still pending. Idempotent.
|
|
444
|
+
return () => {
|
|
445
|
+
clearTimeoutFn(budgetTimer);
|
|
446
|
+
if (hardKillTimer !== undefined) {
|
|
447
|
+
clearTimeoutFn(hardKillTimer);
|
|
448
|
+
hardKillTimer = undefined;
|
|
449
|
+
}
|
|
450
|
+
};
|
|
451
|
+
}
|
|
398
452
|
export async function akmImprove(options = {}) {
|
|
399
453
|
const scope = resolveImproveScope(options.scope);
|
|
400
454
|
const reflectFn = options.reflectFn ?? akmReflect;
|
|
@@ -421,6 +475,15 @@ export async function akmImprove(options = {}) {
|
|
|
421
475
|
catch {
|
|
422
476
|
primaryStashDir = undefined;
|
|
423
477
|
}
|
|
478
|
+
// C2 (#553/#554/#499): resolve the state.db path ONCE, synchronously, at the
|
|
479
|
+
// command boundary — before the first `await` below. Every state.db open in
|
|
480
|
+
// this run (`openStateDatabase`, every default-path `appendEvent`) is pinned
|
|
481
|
+
// to this snapshot via `eventsCtx.dbPath`, so a parallel test file mutating
|
|
482
|
+
// `process.env.XDG_DATA_HOME` across an await boundary can never redirect this
|
|
483
|
+
// run's DB opens to a wrong/just-deleted tmpdir mid-flight (the parallel-load
|
|
484
|
+
// timeout root cause). Because beforeEach runs synchronously, env is still the
|
|
485
|
+
// calling test's own at this point; we capture it before yielding the loop.
|
|
486
|
+
const resolvedStateDbPath = getStateDbPathInDataDir();
|
|
424
487
|
// Phase 4 lock hoist (§7): the `improve.lock` setup is hoisted ABOVE
|
|
425
488
|
// ensureIndex/collectEligibleRefs so the triage pre-pass (and improve's own
|
|
426
489
|
// queue writes) run fully serialized under the lock. The dry-run early-return
|
|
@@ -503,6 +566,7 @@ export async function akmImprove(options = {}) {
|
|
|
503
566
|
let profileFilteredRefs;
|
|
504
567
|
let memoryCleanupPlan;
|
|
505
568
|
let guidance;
|
|
569
|
+
let triageDrain;
|
|
506
570
|
try {
|
|
507
571
|
// Acquire the lock and run the triage pre-pass for non-dry-run executions.
|
|
508
572
|
// The dry-run branch below produces plannedRefs/memorySummary WITHOUT the lock
|
|
@@ -531,7 +595,7 @@ export async function akmImprove(options = {}) {
|
|
|
531
595
|
const judgment = triageConfig?.judgment
|
|
532
596
|
? resolveTriageJudgmentRunner(triageConfig.judgment, _earlyConfig)
|
|
533
597
|
: null;
|
|
534
|
-
await drainProposalsFn({
|
|
598
|
+
triageDrain = await drainProposalsFn({
|
|
535
599
|
stashDir: primaryStashDir,
|
|
536
600
|
policy,
|
|
537
601
|
applyMode,
|
|
@@ -672,25 +736,27 @@ export async function akmImprove(options = {}) {
|
|
|
672
736
|
let eventsDb;
|
|
673
737
|
// `eventsCtx` is read by the main catch (improve_failed) and finally, so it
|
|
674
738
|
// lives in the outer scope. It is always assigned at the top of the try.
|
|
675
|
-
|
|
739
|
+
// Pinned to the boundary snapshot so the fallback per-call `appendEvent`
|
|
740
|
+
// opens (when the long-lived handle below fails to open) never re-read env.
|
|
741
|
+
let eventsCtx = { dbPath: resolvedStateDbPath };
|
|
676
742
|
try {
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
clearBudgetTimer = () => clearTimeout(budgetTimer);
|
|
743
|
+
// H7 (#566): arm the budget watchdog. `armBudgetWatchdog` captures both the
|
|
744
|
+
// budget timer and the hard-kill timer it schedules on exhaustion, returning
|
|
745
|
+
// a single dispose() that clears whichever are still pending. The `finally`
|
|
746
|
+
// calls dispose() via `clearBudgetTimer` (RAII), so a clean cooperative
|
|
747
|
+
// drain cancels the pending hard-kill before it can fire — the process then
|
|
748
|
+
// exits naturally instead of being force-`exit(0)`-ed mid-flush, which could
|
|
749
|
+
// truncate an in-flight log or `state.db` transaction.
|
|
750
|
+
clearBudgetTimer = armBudgetWatchdog(budgetMs, budgetAbortController);
|
|
686
751
|
try {
|
|
687
|
-
eventsDb = openStateDatabase();
|
|
752
|
+
eventsDb = openStateDatabase(resolvedStateDbPath);
|
|
688
753
|
eventsCtx = { db: eventsDb };
|
|
689
754
|
}
|
|
690
755
|
catch (err) {
|
|
691
756
|
rethrowIfTestIsolationError(err);
|
|
692
|
-
// If we cannot open state.db up-front, fall back to per-call opens
|
|
693
|
-
|
|
757
|
+
// If we cannot open state.db up-front, fall back to per-call opens — but
|
|
758
|
+
// still pinned to the boundary-resolved path, never a live env re-read.
|
|
759
|
+
eventsCtx = { dbPath: resolvedStateDbPath };
|
|
694
760
|
}
|
|
695
761
|
// 2026-05-27: emit `improve_skipped` audit events for refs the planner
|
|
696
762
|
// pre-filtered (reflect AND distill both refuse them under the active
|
|
@@ -749,20 +815,23 @@ export async function akmImprove(options = {}) {
|
|
|
749
815
|
eventsCtx,
|
|
750
816
|
improveProfile,
|
|
751
817
|
});
|
|
752
|
-
|
|
818
|
+
// #551: consolidation now runs in the preparation stage (before extract);
|
|
819
|
+
// its result and run-flag are read from `preparation`, not the post-loop.
|
|
820
|
+
const consolidation = preparation.consolidation;
|
|
821
|
+
const { allWarnings, deadUrls, memoryInference, graphExtraction, stalenessDetection, maintenanceActions, memoryInferenceDurationMs, graphExtractionDurationMs, orphansPurged, proposalsExpired, gateAutoAcceptedCount: postLoopGateCount, gateAutoAcceptFailedCount: postLoopGateFailedCount, } = await runImprovePostLoopStage({
|
|
753
822
|
scope,
|
|
754
823
|
options,
|
|
755
824
|
primaryStashDir,
|
|
756
825
|
actionableRefs: preparation.actionableRefs,
|
|
757
826
|
appliedCleanup: preparation.appliedCleanup,
|
|
758
827
|
cleanupWarnings: preparation.cleanupWarnings,
|
|
759
|
-
memorySummary,
|
|
760
828
|
memoryRefsForInference,
|
|
761
829
|
reindexFn,
|
|
762
830
|
eventsCtx,
|
|
763
831
|
// O-1 (#364): propagate wall-clock budget signal to post-loop maintenance.
|
|
764
832
|
budgetSignal: budgetAbortController.signal,
|
|
765
833
|
improveProfile,
|
|
834
|
+
consolidationRan: preparation.consolidationRan,
|
|
766
835
|
});
|
|
767
836
|
const finalActions = maintenanceActions && maintenanceActions.length > 0
|
|
768
837
|
? [...preparation.actions, ...maintenanceActions]
|
|
@@ -838,6 +907,17 @@ export async function akmImprove(options = {}) {
|
|
|
838
907
|
const f = preparation.gateAutoAcceptFailedCount + loopGateFailedCount + postLoopGateFailedCount;
|
|
839
908
|
return f > 0 ? { gateAutoAcceptFailedCount: f } : {};
|
|
840
909
|
})(),
|
|
910
|
+
...(triageDrain
|
|
911
|
+
? {
|
|
912
|
+
triage: {
|
|
913
|
+
promoted: triageDrain.promoted.length,
|
|
914
|
+
rejected: triageDrain.rejected.length,
|
|
915
|
+
deferred: triageDrain.deferred.length,
|
|
916
|
+
skippedByCap: triageDrain.skippedByCap.length,
|
|
917
|
+
},
|
|
918
|
+
}
|
|
919
|
+
: {}),
|
|
920
|
+
...(options.runId !== undefined ? { runId: options.runId } : {}),
|
|
841
921
|
};
|
|
842
922
|
if (!result.dryRun)
|
|
843
923
|
emitImproveCompletedEvent(result, {
|
|
@@ -937,6 +1017,7 @@ function emitImproveCompletedEvent(result, durations, eventsCtx) {
|
|
|
937
1017
|
reflectFailed: 0,
|
|
938
1018
|
reflectCooldown: 0,
|
|
939
1019
|
reflectSkipped: 0,
|
|
1020
|
+
reflectGuardRejected: 0,
|
|
940
1021
|
distill: 0,
|
|
941
1022
|
distillSkipped: 0,
|
|
942
1023
|
memoryPrune: 0,
|
|
@@ -944,7 +1025,16 @@ function emitImproveCompletedEvent(result, durations, eventsCtx) {
|
|
|
944
1025
|
graphExtraction: 0,
|
|
945
1026
|
error: 0,
|
|
946
1027
|
};
|
|
1028
|
+
// Coarse audit buckets, derived from the SAME classifyImproveAction the
|
|
1029
|
+
// persisted metrics_json uses (state-db.ts#computeImproveRunMetrics) so the
|
|
1030
|
+
// emitted event and the stored row can never disagree.
|
|
1031
|
+
const classCounts = { accepted: 0, rejected: 0, error: 0, noop: 0 };
|
|
947
1032
|
for (const action of result.actions ?? []) {
|
|
1033
|
+
classCounts[classifyImproveAction(action.mode)] += 1;
|
|
1034
|
+
// Per-variant counters for the event metadata. The default arm makes any
|
|
1035
|
+
// new ImproveActionMode variant a compile error so a future variant cannot
|
|
1036
|
+
// be silently dropped from the improve_completed event (the `reflect-guard-
|
|
1037
|
+
// rejected` case below was previously missing here entirely).
|
|
948
1038
|
switch (action.mode) {
|
|
949
1039
|
case "reflect":
|
|
950
1040
|
actionCounts.reflect += 1;
|
|
@@ -958,6 +1048,9 @@ function emitImproveCompletedEvent(result, durations, eventsCtx) {
|
|
|
958
1048
|
case "reflect-skipped":
|
|
959
1049
|
actionCounts.reflectSkipped += 1;
|
|
960
1050
|
break;
|
|
1051
|
+
case "reflect-guard-rejected":
|
|
1052
|
+
actionCounts.reflectGuardRejected += 1;
|
|
1053
|
+
break;
|
|
961
1054
|
case "distill":
|
|
962
1055
|
actionCounts.distill += 1;
|
|
963
1056
|
break;
|
|
@@ -976,6 +1069,8 @@ function emitImproveCompletedEvent(result, durations, eventsCtx) {
|
|
|
976
1069
|
case "error":
|
|
977
1070
|
actionCounts.error += 1;
|
|
978
1071
|
break;
|
|
1072
|
+
default:
|
|
1073
|
+
assertNever(action.mode);
|
|
979
1074
|
}
|
|
980
1075
|
}
|
|
981
1076
|
appendEvent({
|
|
@@ -995,6 +1090,12 @@ function emitImproveCompletedEvent(result, durations, eventsCtx) {
|
|
|
995
1090
|
reflectFailedActions: actionCounts.reflectFailed,
|
|
996
1091
|
reflectCooldownActions: actionCounts.reflectCooldown,
|
|
997
1092
|
reflectSkippedActions: actionCounts.reflectSkipped,
|
|
1093
|
+
// Previously dropped from the event entirely; now emitted so the guard
|
|
1094
|
+
// rejections are visible in improve_completed telemetry.
|
|
1095
|
+
reflectGuardRejectedActions: actionCounts.reflectGuardRejected,
|
|
1096
|
+
acceptedActions: classCounts.accepted,
|
|
1097
|
+
rejectedActions: classCounts.rejected,
|
|
1098
|
+
noopActions: classCounts.noop,
|
|
998
1099
|
reflectsWithErrorContext: result.reflectsWithErrorContext ?? 0,
|
|
999
1100
|
coverageGapCount: result.coverageGaps?.length ?? 0,
|
|
1000
1101
|
evalCasesWritten: result.evalCasesWritten ?? 0,
|
|
@@ -1027,12 +1128,261 @@ function emitImproveCompletedEvent(result, durations, eventsCtx) {
|
|
|
1027
1128
|
},
|
|
1028
1129
|
}, eventsCtx);
|
|
1029
1130
|
}
|
|
1131
|
+
/**
|
|
1132
|
+
* Run (or gate-skip) the memory consolidation pass.
|
|
1133
|
+
*
|
|
1134
|
+
* #551 — two coordinated changes live here:
|
|
1135
|
+
*
|
|
1136
|
+
* 1. STRUCTURAL: this runs before extract in the improve pipeline (see
|
|
1137
|
+
* `runImprovePreparationStage`). Consolidation therefore only ever judges
|
|
1138
|
+
* PRIOR-run memories; current-run extract promotions are invisible to it.
|
|
1139
|
+
*
|
|
1140
|
+
* 2. SMARTER POOL-DELTA GATE: even among on-disk files, a memory whose only
|
|
1141
|
+
* post-`lastConsolidateTs` mtime bump came from its OWN auto-accept
|
|
1142
|
+
* promotion (i.e. it was just promoted by extract in the immediately
|
|
1143
|
+
* preceding run and has not had a full improve cycle to settle) does NOT
|
|
1144
|
+
* count as "work to do". We exclude those paths from the pool-delta check
|
|
1145
|
+
* using the `promoted` events already emitted with each promotion's
|
|
1146
|
+
* `assetPath`. A genuinely-settled prior memory — one edited by feedback,
|
|
1147
|
+
* reflect, manual edit, or simply older than the last consolidate — still
|
|
1148
|
+
* triggers the run. This is gate-option (a) from the issue (same-run /
|
|
1149
|
+
* adjacent-run promotion exclusion), chosen over option (b) because there
|
|
1150
|
+
* is no `extract_completed` event in the data model to gate against;
|
|
1151
|
+
* `promoted` events with `assetPath` already carry exactly the signal we
|
|
1152
|
+
* need, so the fix is non-invasive and provably correct.
|
|
1153
|
+
*/
|
|
1154
|
+
async function runConsolidationPass(args) {
|
|
1155
|
+
const { options, primaryStashDir, memorySummary, improveProfile, eventsCtx } = args;
|
|
1156
|
+
const baseConfig = options.config ?? loadConfig();
|
|
1157
|
+
const MEMORY_VOLUME_THRESHOLD = options.memoryVolumeConsolidationThreshold ?? 100;
|
|
1158
|
+
const hasLlm = !!(baseConfig.defaults?.llm || baseConfig.defaults?.agent);
|
|
1159
|
+
const volumeTriggered = typeof memorySummary.eligible === "number" && memorySummary.eligible > MEMORY_VOLUME_THRESHOLD && hasLlm;
|
|
1160
|
+
// When volume triggers a consolidation pass, force-enable the consolidate
|
|
1161
|
+
// process on the default improve profile so the gate accepts the run even
|
|
1162
|
+
// if the user's config disabled it. We synthesise a new profile override
|
|
1163
|
+
// rather than mutating connection settings.
|
|
1164
|
+
const consolidationConfig = volumeTriggered
|
|
1165
|
+
? {
|
|
1166
|
+
...baseConfig,
|
|
1167
|
+
profiles: {
|
|
1168
|
+
...(baseConfig.profiles ?? {}),
|
|
1169
|
+
improve: {
|
|
1170
|
+
...(baseConfig.profiles?.improve ?? {}),
|
|
1171
|
+
default: {
|
|
1172
|
+
...(baseConfig.profiles?.improve?.default ?? {}),
|
|
1173
|
+
processes: {
|
|
1174
|
+
...(baseConfig.profiles?.improve?.default?.processes ?? {}),
|
|
1175
|
+
consolidate: {
|
|
1176
|
+
...(baseConfig.profiles?.improve?.default?.processes?.consolidate ?? {}),
|
|
1177
|
+
enabled: true,
|
|
1178
|
+
},
|
|
1179
|
+
},
|
|
1180
|
+
},
|
|
1181
|
+
},
|
|
1182
|
+
},
|
|
1183
|
+
}
|
|
1184
|
+
: baseConfig;
|
|
1185
|
+
// 0.8.0 pool-delta gate for consolidate: re-eligible iff at least one
|
|
1186
|
+
// memory file has been updated since the most recent successful
|
|
1187
|
+
// consolidate_completed event. Time-based cooldowns produced the same
|
|
1188
|
+
// synchronised-wave failure mode the reflect/distill cooldowns did; the
|
|
1189
|
+
// pool-delta gate ties consolidation to actual work-to-do.
|
|
1190
|
+
const recentConsolidations = readEvents({ type: "consolidate_completed" });
|
|
1191
|
+
const lastConsolidation = recentConsolidations.events
|
|
1192
|
+
.filter((e) => e.metadata?.processed && Number(e.metadata.processed) > 0)
|
|
1193
|
+
.sort((a, b) => new Date(b.ts ?? 0).getTime() - new Date(a.ts ?? 0).getTime())[0];
|
|
1194
|
+
const lastConsolidateTs = lastConsolidation?.ts;
|
|
1195
|
+
// #551 smarter gate: build the set of memory asset paths whose only delta
|
|
1196
|
+
// since the last consolidate is their OWN auto-accept promotion. Those files
|
|
1197
|
+
// have not had a full improve cycle to settle, so they offer no merge /
|
|
1198
|
+
// contradiction candidates yet — excluding them stops the gate firing on
|
|
1199
|
+
// freshly-promoted single-source memories. We read `promoted` events emitted
|
|
1200
|
+
// after the last consolidate; each carries the written `assetPath`.
|
|
1201
|
+
const promotedSinceConsolidate = (() => {
|
|
1202
|
+
const paths = new Set();
|
|
1203
|
+
try {
|
|
1204
|
+
const promoted = readEvents({
|
|
1205
|
+
type: "promoted",
|
|
1206
|
+
...(lastConsolidateTs ? { since: lastConsolidateTs } : {}),
|
|
1207
|
+
}).events;
|
|
1208
|
+
for (const e of promoted) {
|
|
1209
|
+
const ap = e.metadata?.assetPath;
|
|
1210
|
+
if (typeof ap === "string" && ap.length > 0)
|
|
1211
|
+
paths.add(path.resolve(ap));
|
|
1212
|
+
}
|
|
1213
|
+
}
|
|
1214
|
+
catch {
|
|
1215
|
+
// best-effort: if the events query fails, fall back to no exclusions
|
|
1216
|
+
// (preserves pre-#551 behaviour rather than over-skipping).
|
|
1217
|
+
}
|
|
1218
|
+
return paths;
|
|
1219
|
+
})();
|
|
1220
|
+
// Pool-delta: any memory file with mtime > lastConsolidateTs flags work to do,
|
|
1221
|
+
// EXCEPT files whose only post-consolidate change was their own promotion.
|
|
1222
|
+
// Using file mtime keeps this query DB-free and matches what the indexer
|
|
1223
|
+
// already uses as the canonical `memory.updated_at` proxy.
|
|
1224
|
+
//
|
|
1225
|
+
// Bootstrap: when no successful consolidate_completed event has ever been
|
|
1226
|
+
// recorded, we cannot evaluate the pool-delta — treat as eligible so a
|
|
1227
|
+
// fresh stash runs consolidate once before the steady-state gate kicks in.
|
|
1228
|
+
const memoryUpdatedAfterLastConsolidate = (() => {
|
|
1229
|
+
if (volumeTriggered)
|
|
1230
|
+
return true; // volume override forces the run regardless.
|
|
1231
|
+
if (!lastConsolidateTs)
|
|
1232
|
+
return true; // bootstrap path: never consolidated.
|
|
1233
|
+
if (!primaryStashDir)
|
|
1234
|
+
return false;
|
|
1235
|
+
const memoriesDir = path.join(primaryStashDir, "memories");
|
|
1236
|
+
if (!fs.existsSync(memoriesDir))
|
|
1237
|
+
return false;
|
|
1238
|
+
try {
|
|
1239
|
+
return fs.readdirSync(memoriesDir).some((f) => {
|
|
1240
|
+
if (!f.endsWith(".md"))
|
|
1241
|
+
return false;
|
|
1242
|
+
const filePath = path.join(memoriesDir, f);
|
|
1243
|
+
// #551: skip files that were only touched by their own promotion this
|
|
1244
|
+
// cohort — they have no settled merge/contradiction candidates yet.
|
|
1245
|
+
if (promotedSinceConsolidate.has(path.resolve(filePath)))
|
|
1246
|
+
return false;
|
|
1247
|
+
try {
|
|
1248
|
+
return fs.statSync(filePath).mtime.toISOString() > lastConsolidateTs;
|
|
1249
|
+
}
|
|
1250
|
+
catch {
|
|
1251
|
+
return false;
|
|
1252
|
+
}
|
|
1253
|
+
});
|
|
1254
|
+
}
|
|
1255
|
+
catch {
|
|
1256
|
+
return false;
|
|
1257
|
+
}
|
|
1258
|
+
})();
|
|
1259
|
+
const consolidationOnCooldown = !volumeTriggered && !memoryUpdatedAfterLastConsolidate;
|
|
1260
|
+
// Profile gate: if profile explicitly disables consolidate, skip the entire pass.
|
|
1261
|
+
const consolidateDisabledByProfile = improveProfile?.processes?.consolidate?.enabled === false;
|
|
1262
|
+
// #553 minPoolSize guard: skip consolidation when the eligible memory pool is
|
|
1263
|
+
// below a minimum size, rather than spending an LLM pass on a handful of
|
|
1264
|
+
// memories. This is an INDEPENDENT skip condition from #551's mtime pool-delta
|
|
1265
|
+
// gate — either can skip. Default 500; `minPoolSize: 0` disables the guard.
|
|
1266
|
+
// Evaluated against the eligible-pool count BEFORE entering the LLM loop so a
|
|
1267
|
+
// skip costs ZERO LLM calls.
|
|
1268
|
+
const CONSOLIDATE_DEFAULT_MIN_POOL_SIZE = 500;
|
|
1269
|
+
const configuredMinPoolSize = improveProfile?.processes?.consolidate?.minPoolSize;
|
|
1270
|
+
const minPoolSize = typeof configuredMinPoolSize === "number" ? configuredMinPoolSize : CONSOLIDATE_DEFAULT_MIN_POOL_SIZE;
|
|
1271
|
+
const eligiblePoolSize = typeof memorySummary.eligible === "number" ? memorySummary.eligible : 0;
|
|
1272
|
+
// volumeTriggered means the pool already exceeds the volume threshold (100),
|
|
1273
|
+
// so a force-triggered run never trips the pool-size guard. The guard only
|
|
1274
|
+
// engages when minPoolSize > 0 and the eligible pool is strictly below it.
|
|
1275
|
+
const poolBelowMinSize = !volumeTriggered && minPoolSize > 0 && eligiblePoolSize < minPoolSize;
|
|
1276
|
+
let consolidation = {
|
|
1277
|
+
schemaVersion: 1,
|
|
1278
|
+
ok: true,
|
|
1279
|
+
shape: "consolidate-result",
|
|
1280
|
+
dryRun: false,
|
|
1281
|
+
previewOnly: false,
|
|
1282
|
+
target: "",
|
|
1283
|
+
processed: 0,
|
|
1284
|
+
merged: 0,
|
|
1285
|
+
deleted: 0,
|
|
1286
|
+
promoted: [],
|
|
1287
|
+
contradicted: 0,
|
|
1288
|
+
warnings: [],
|
|
1289
|
+
durationMs: 0,
|
|
1290
|
+
};
|
|
1291
|
+
let gateAutoAcceptedCount = 0;
|
|
1292
|
+
let gateAutoAcceptFailedCount = 0;
|
|
1293
|
+
const consolidateGateCfg = makeGateConfig("consolidate", {
|
|
1294
|
+
globalThreshold: options.autoAccept,
|
|
1295
|
+
dryRun: options.dryRun ?? false,
|
|
1296
|
+
stashDir: primaryStashDir,
|
|
1297
|
+
config: consolidationConfig,
|
|
1298
|
+
eventsCtx,
|
|
1299
|
+
}, { minimumThreshold: 95 });
|
|
1300
|
+
if (consolidateDisabledByProfile) {
|
|
1301
|
+
info("[improve] consolidation skipped (disabled by improve profile)");
|
|
1302
|
+
}
|
|
1303
|
+
else if (poolBelowMinSize) {
|
|
1304
|
+
// #553: eligible pool below the configured minimum — skip with zero LLM
|
|
1305
|
+
// calls. Reuse the #551 `improve_skipped` emission path so health surfaces
|
|
1306
|
+
// it via the dynamic skipReasons aggregation under `pool_below_min_size`.
|
|
1307
|
+
appendEvent({
|
|
1308
|
+
eventType: "improve_skipped",
|
|
1309
|
+
ref: "memory:_consolidation",
|
|
1310
|
+
metadata: {
|
|
1311
|
+
reason: "pool_below_min_size",
|
|
1312
|
+
poolSize: eligiblePoolSize,
|
|
1313
|
+
minPoolSize,
|
|
1314
|
+
},
|
|
1315
|
+
}, eventsCtx);
|
|
1316
|
+
info(`[improve] consolidation skipped (pool ${eligiblePoolSize} < minPoolSize ${minPoolSize})`);
|
|
1317
|
+
}
|
|
1318
|
+
else if (!consolidationOnCooldown) {
|
|
1319
|
+
consolidation = await akmConsolidate({
|
|
1320
|
+
...options.consolidateOptions,
|
|
1321
|
+
config: consolidationConfig,
|
|
1322
|
+
stashDir: options.stashDir,
|
|
1323
|
+
autoTriggered: volumeTriggered,
|
|
1324
|
+
// Tie consolidate proposals back to this improve invocation so
|
|
1325
|
+
// accept-rate-per-run aggregation works. Mirrors reflect/propose/extract.
|
|
1326
|
+
sourceRun: `consolidate-${Date.now()}`,
|
|
1327
|
+
// Incremental consolidation: pass the last-consolidation timestamp so
|
|
1328
|
+
// akmConsolidate skips chunks with no memory changed since then. Converts
|
|
1329
|
+
// consolidation cost from O(pool) to O(changed clusters) — the fix for
|
|
1330
|
+
// the rising p95 tail where full-pool re-judging produced 5–10 min runs
|
|
1331
|
+
// that promoted ~0. undefined → full pass on first-ever run (bootstrap).
|
|
1332
|
+
// volumeTriggered correctly forces the run past cooldown but must NOT
|
|
1333
|
+
// override incrementalSince — the stash has ~1400 eligible memories so
|
|
1334
|
+
// volumeTriggered=true on every run, permanently forcing full 12-chunk
|
|
1335
|
+
// scans (~264s) instead of the intended 1-2 chunk incremental path (~44s).
|
|
1336
|
+
incrementalSince: lastConsolidateTs,
|
|
1337
|
+
maxChunkSize: improveProfile?.processes?.consolidate?.maxChunkSize,
|
|
1338
|
+
// Honor profile.autoAccept (already merged into options.autoAccept at the
|
|
1339
|
+
// top of akmImprove). The CLI parser always supplies 90 when --auto-accept
|
|
1340
|
+
// is absent, so ?? 90 is not needed here and would prevent --auto-accept=false
|
|
1341
|
+
// (which maps to undefined) from disabling consolidation auto-accept.
|
|
1342
|
+
// options.consolidateOptions.autoAccept (if explicitly provided by caller)
|
|
1343
|
+
// still wins because the spread above runs first.
|
|
1344
|
+
autoAccept: options.consolidateOptions?.autoAccept ?? options.autoAccept,
|
|
1345
|
+
});
|
|
1346
|
+
{
|
|
1347
|
+
const consolidateGr = await runAutoAcceptGate(consolidation.promoted.map((proposalId) => {
|
|
1348
|
+
try {
|
|
1349
|
+
if (!primaryStashDir)
|
|
1350
|
+
return { proposalId, confidence: undefined };
|
|
1351
|
+
const proposal = getProposal(primaryStashDir, proposalId);
|
|
1352
|
+
return { proposalId, confidence: proposal.confidence };
|
|
1353
|
+
}
|
|
1354
|
+
catch {
|
|
1355
|
+
return { proposalId, confidence: undefined };
|
|
1356
|
+
}
|
|
1357
|
+
}), consolidateGateCfg);
|
|
1358
|
+
gateAutoAcceptedCount += consolidateGr.promoted.length;
|
|
1359
|
+
gateAutoAcceptFailedCount += consolidateGr.failed.length;
|
|
1360
|
+
}
|
|
1361
|
+
if (consolidation.processed > 0) {
|
|
1362
|
+
appendEvent({
|
|
1363
|
+
eventType: "consolidate_completed",
|
|
1364
|
+
ref: "memory:_consolidation",
|
|
1365
|
+
metadata: { processed: consolidation.processed, merged: consolidation.merged },
|
|
1366
|
+
}, eventsCtx);
|
|
1367
|
+
}
|
|
1368
|
+
}
|
|
1369
|
+
else {
|
|
1370
|
+
appendEvent({
|
|
1371
|
+
eventType: "improve_skipped",
|
|
1372
|
+
ref: "memory:_consolidation",
|
|
1373
|
+
metadata: {
|
|
1374
|
+
reason: "consolidation_no_memory_updates",
|
|
1375
|
+
lastEventTs: lastConsolidation?.ts ?? null,
|
|
1376
|
+
},
|
|
1377
|
+
}, eventsCtx);
|
|
1378
|
+
info("[improve] consolidation skipped (no memory updates since last run)");
|
|
1379
|
+
}
|
|
1380
|
+
// D9: track whether consolidation wrote any data so graph extraction can reindex if needed
|
|
1381
|
+
const consolidationRan = !consolidateDisabledByProfile && !poolBelowMinSize && !consolidationOnCooldown && consolidation.processed > 0;
|
|
1382
|
+
return { consolidation, consolidationRan, gateAutoAcceptedCount, gateAutoAcceptFailedCount };
|
|
1383
|
+
}
|
|
1030
1384
|
async function runImprovePreparationStage(args) {
|
|
1031
|
-
const { scope, options, plannedRefs, memoryCleanupPlan, primaryStashDir, reindexFn, startMs, budgetMs, eventsCtx, initialCleanupWarnings,
|
|
1032
|
-
// improveProfile is part of the preparation-stage signature for future use
|
|
1033
|
-
// (per-process gating moved into the in-loop stage). Kept here so the
|
|
1034
|
-
// signature does not drift away from the rest of the planner stack.
|
|
1035
|
-
improveProfile: _improveProfile, } = args;
|
|
1385
|
+
const { scope, options, plannedRefs, memoryCleanupPlan, primaryStashDir, memorySummary, reindexFn, startMs, budgetMs, eventsCtx, initialCleanupWarnings, improveProfile, } = args;
|
|
1036
1386
|
const actions = [];
|
|
1037
1387
|
const cleanupWarnings = initialCleanupWarnings ? [...initialCleanupWarnings] : [];
|
|
1038
1388
|
// Phase 0 — MEMORY.md budget check (200-line cap; warn at 180)
|
|
@@ -1053,6 +1403,23 @@ async function runImprovePreparationStage(args) {
|
|
|
1053
1403
|
}
|
|
1054
1404
|
}
|
|
1055
1405
|
}
|
|
1406
|
+
// Phase 0.3 — memory consolidation pass (#551).
|
|
1407
|
+
//
|
|
1408
|
+
// Consolidation runs BEFORE the session-extract pass. This is the structural
|
|
1409
|
+
// half of the #551 fix: extract auto-accept writes brand-new memory .md files
|
|
1410
|
+
// on every run, which previously made the consolidation pool-delta gate fire
|
|
1411
|
+
// unconditionally (any new file => "memory updated since last consolidate").
|
|
1412
|
+
// By running consolidation first, the gate and akmConsolidate only ever see
|
|
1413
|
+
// memories that existed at the start of the run — current-run extract
|
|
1414
|
+
// promotions are not on disk yet. The complementary smarter-gate logic
|
|
1415
|
+
// (excluding adjacent-run promotions) lives in `runConsolidationPass`.
|
|
1416
|
+
const consolidationPass = await runConsolidationPass({
|
|
1417
|
+
options,
|
|
1418
|
+
primaryStashDir,
|
|
1419
|
+
memorySummary,
|
|
1420
|
+
improveProfile,
|
|
1421
|
+
eventsCtx,
|
|
1422
|
+
});
|
|
1056
1423
|
// Phase 0.4 — session-extract pass.
|
|
1057
1424
|
//
|
|
1058
1425
|
// Reads native session files (claude-code JSONL, opencode storage tree)
|
|
@@ -1070,8 +1437,11 @@ async function runImprovePreparationStage(args) {
|
|
|
1070
1437
|
// Failures are non-fatal — one harness throwing doesn't abort improve.
|
|
1071
1438
|
// The extract envelope's own `warnings` field surfaces what went wrong.
|
|
1072
1439
|
let extractResults;
|
|
1073
|
-
|
|
1074
|
-
|
|
1440
|
+
// Seed the preparation-stage gate counters with consolidation's auto-accept
|
|
1441
|
+
// gate results (#551: consolidation now runs in this stage), then accumulate
|
|
1442
|
+
// extract's gate results on top.
|
|
1443
|
+
let gateAutoAcceptedCount = consolidationPass.gateAutoAcceptedCount;
|
|
1444
|
+
let gateAutoAcceptFailedCount = consolidationPass.gateAutoAcceptFailedCount;
|
|
1075
1445
|
const extractConfig = options.config ?? loadConfig();
|
|
1076
1446
|
const extractGateCfg = makeGateConfig("extract", {
|
|
1077
1447
|
globalThreshold: options.autoAccept,
|
|
@@ -1080,9 +1450,47 @@ async function runImprovePreparationStage(args) {
|
|
|
1080
1450
|
config: extractConfig,
|
|
1081
1451
|
eventsCtx,
|
|
1082
1452
|
});
|
|
1453
|
+
// #554 minNewSessions gate: skip the entire extract pass (ensureIndex was
|
|
1454
|
+
// already done upstream; here we elide every akmExtract/processSession call)
|
|
1455
|
+
// when the NEW (unseen, in-window) candidate-session pool is below a minimum.
|
|
1456
|
+
// 22% of improve runs produce zero memory-inference writes because extract
|
|
1457
|
+
// finds no new sessions, yet still burns the full extract pipeline. Default 0
|
|
1458
|
+
// (disabled) preserves existing always-run behaviour; only opted-in profiles
|
|
1459
|
+
// (e.g. `frequent`) set it. Evaluated BEFORE any LLM call so a skip costs zero
|
|
1460
|
+
// LLM work AND writes nothing — which also means no extract auto-accept bumps
|
|
1461
|
+
// memory mtimes, so a skipped extract never flags work for the NEXT run's
|
|
1462
|
+
// consolidation mtime-gate (the downstream trigger #554 asks us to suppress).
|
|
1463
|
+
const EXTRACT_DEFAULT_MIN_NEW_SESSIONS = 0;
|
|
1464
|
+
const configuredMinNewSessions = extractConfig.profiles?.improve?.default?.processes?.extract?.minNewSessions;
|
|
1465
|
+
const minNewSessions = typeof configuredMinNewSessions === "number" ? configuredMinNewSessions : EXTRACT_DEFAULT_MIN_NEW_SESSIONS;
|
|
1083
1466
|
if (isLlmFeatureEnabled(extractConfig, "session_extraction")) {
|
|
1084
|
-
const availableHarnesses = getAvailableHarnesses();
|
|
1085
|
-
|
|
1467
|
+
const availableHarnesses = options.extractHarnesses ?? getAvailableHarnesses();
|
|
1468
|
+
// The guard engages only when minNewSessions > 0; 0 disables it entirely.
|
|
1469
|
+
let belowMinNewSessions = false;
|
|
1470
|
+
if (minNewSessions > 0 && availableHarnesses.length > 0) {
|
|
1471
|
+
const countFn = options.extractCandidateCountFn ?? countNewExtractCandidates;
|
|
1472
|
+
const newCandidateCount = countFn(extractConfig, {
|
|
1473
|
+
...(options.extractHarnesses ? { harnesses: options.extractHarnesses } : {}),
|
|
1474
|
+
// C2: pin the candidate-count state.db open to the boundary-resolved path.
|
|
1475
|
+
...(eventsCtx?.dbPath ? { stateDbPath: eventsCtx.dbPath } : {}),
|
|
1476
|
+
});
|
|
1477
|
+
if (newCandidateCount < minNewSessions) {
|
|
1478
|
+
belowMinNewSessions = true;
|
|
1479
|
+
// Reuse the #551/#553 `improve_skipped` emission path so health's dynamic
|
|
1480
|
+
// skipReasons aggregation surfaces this under `below_min_new_sessions`.
|
|
1481
|
+
appendEvent({
|
|
1482
|
+
eventType: "improve_skipped",
|
|
1483
|
+
ref: "memory:_extract",
|
|
1484
|
+
metadata: {
|
|
1485
|
+
reason: "below_min_new_sessions",
|
|
1486
|
+
newSessions: newCandidateCount,
|
|
1487
|
+
minNewSessions,
|
|
1488
|
+
},
|
|
1489
|
+
}, eventsCtx);
|
|
1490
|
+
info(`[improve] extract skipped (new sessions ${newCandidateCount} < minNewSessions ${minNewSessions})`);
|
|
1491
|
+
}
|
|
1492
|
+
}
|
|
1493
|
+
if (!belowMinNewSessions && availableHarnesses.length > 0) {
|
|
1086
1494
|
extractResults = [];
|
|
1087
1495
|
for (const h of availableHarnesses) {
|
|
1088
1496
|
try {
|
|
@@ -1091,6 +1499,9 @@ async function runImprovePreparationStage(args) {
|
|
|
1091
1499
|
...(primaryStashDir !== undefined ? { stashDir: primaryStashDir } : {}),
|
|
1092
1500
|
config: extractConfig,
|
|
1093
1501
|
dryRun: options.dryRun ?? false,
|
|
1502
|
+
...(options.extractHarnesses ? { harnesses: options.extractHarnesses } : {}),
|
|
1503
|
+
// C2: pin extract's skip-tracking state.db open to the boundary path.
|
|
1504
|
+
...(eventsCtx?.dbPath ? { stateDbPath: eventsCtx.dbPath } : {}),
|
|
1094
1505
|
});
|
|
1095
1506
|
extractResults.push(result);
|
|
1096
1507
|
{
|
|
@@ -1403,7 +1814,7 @@ async function runImprovePreparationStage(args) {
|
|
|
1403
1814
|
let dbForRetrieval;
|
|
1404
1815
|
try {
|
|
1405
1816
|
dbForRetrieval = openExistingDatabase();
|
|
1406
|
-
const showEventCount = dbForRetrieval
|
|
1817
|
+
const showEventCount = countUsageEventsByType(dbForRetrieval, "show");
|
|
1407
1818
|
if (showEventCount === 0) {
|
|
1408
1819
|
warn("Warning: show events not yet in usage_events — zero-feedback fallback will match only search-retrieved assets.");
|
|
1409
1820
|
}
|
|
@@ -1557,9 +1968,10 @@ async function runImprovePreparationStage(args) {
|
|
|
1557
1968
|
utilityMap,
|
|
1558
1969
|
gateAutoAcceptedCount,
|
|
1559
1970
|
gateAutoAcceptFailedCount,
|
|
1971
|
+
consolidation: consolidationPass.consolidation,
|
|
1972
|
+
consolidationRan: consolidationPass.consolidationRan,
|
|
1560
1973
|
};
|
|
1561
1974
|
}
|
|
1562
|
-
// TODO(refactor): 13 args including `actions`/`recentErrors` mutation channels. Restructure into immutable plan + mutable context objects — deferred to dedicated refactor with isolated testing.
|
|
1563
1975
|
async function runImproveLoopStage(args) {
|
|
1564
1976
|
const { scope, options, primaryStashDir, reflectFn, distillFn, loopRefs, actions, signalBearingSet, distillCooledRefs, distillOnlyRefs, recentErrors, rejectedProposalsByRef, utilityMap, startMs, budgetMs, eventsCtx, improveProfile, } = args;
|
|
1565
1977
|
// O-1 (#364): compute remaining budget at call time so each sub-call
|
|
@@ -1693,7 +2105,7 @@ async function runImproveLoopStage(args) {
|
|
|
1693
2105
|
// path is also a no-op for them — we just avoid unnecessary agent spawns.
|
|
1694
2106
|
// D2: distillOnlyRefs also skip the reflect call (reflect-cooled, distill path only).
|
|
1695
2107
|
if (!isDistillOnly && !planned.ref.endsWith(".derived")) {
|
|
1696
|
-
// Type guard: skip reflect for unsupported types (script,
|
|
2108
|
+
// Type guard: skip reflect for unsupported types (script, env, task, etc.)
|
|
1697
2109
|
// and raw wiki directories, driven by the active improve profile.
|
|
1698
2110
|
const reflectSkip = shouldSkipRef(planned.ref, "reflect", improveProfile);
|
|
1699
2111
|
if (reflectSkip.skip) {
|
|
@@ -1775,7 +2187,7 @@ async function runImproveLoopStage(args) {
|
|
|
1775
2187
|
// true LLM failures. See
|
|
1776
2188
|
// `/tmp/akm-health-investigations/metrics-taxonomy-review.md` §1a.
|
|
1777
2189
|
const isGuardReject = !reflectResult.ok && reflectResult.reason === "content_policy_reject";
|
|
1778
|
-
// Type-guard rejection (reflect refused a script/
|
|
2190
|
+
// Type-guard rejection (reflect refused a script/env/task ref) is
|
|
1779
2191
|
// also NOT an LLM failure — the LLM is never invoked. Route to the
|
|
1780
2192
|
// existing `reflect-skipped` bucket so it does not inflate the
|
|
1781
2193
|
// failure-rate numerator. ~9% of `reflect-failed` events in the
|
|
@@ -2008,174 +2420,8 @@ async function runImproveLoopStage(args) {
|
|
|
2008
2420
|
return { reflectsWithErrorContext, memoryRefsForInference, gateAutoAcceptedCount, gateAutoAcceptFailedCount };
|
|
2009
2421
|
}
|
|
2010
2422
|
async function runImprovePostLoopStage(args) {
|
|
2011
|
-
const { scope, options, primaryStashDir, actionableRefs, appliedCleanup, cleanupWarnings,
|
|
2423
|
+
const { scope, options, primaryStashDir, actionableRefs, appliedCleanup, cleanupWarnings, memoryRefsForInference, reindexFn, eventsCtx, budgetSignal, improveProfile, consolidationRan, } = args;
|
|
2012
2424
|
const allWarnings = [...cleanupWarnings, ...(appliedCleanup?.warnings ?? [])];
|
|
2013
|
-
const baseConfig = options.config ?? loadConfig();
|
|
2014
|
-
const MEMORY_VOLUME_THRESHOLD = options.memoryVolumeConsolidationThreshold ?? 100;
|
|
2015
|
-
const hasLlm = !!(baseConfig.defaults?.llm || baseConfig.defaults?.agent);
|
|
2016
|
-
const volumeTriggered = typeof memorySummary.eligible === "number" && memorySummary.eligible > MEMORY_VOLUME_THRESHOLD && hasLlm;
|
|
2017
|
-
// When volume triggers a consolidation pass, force-enable the consolidate
|
|
2018
|
-
// process on the default improve profile so the gate accepts the run even
|
|
2019
|
-
// if the user's config disabled it. We synthesise a new profile override
|
|
2020
|
-
// rather than mutating connection settings.
|
|
2021
|
-
const consolidationConfig = volumeTriggered
|
|
2022
|
-
? {
|
|
2023
|
-
...baseConfig,
|
|
2024
|
-
profiles: {
|
|
2025
|
-
...(baseConfig.profiles ?? {}),
|
|
2026
|
-
improve: {
|
|
2027
|
-
...(baseConfig.profiles?.improve ?? {}),
|
|
2028
|
-
default: {
|
|
2029
|
-
...(baseConfig.profiles?.improve?.default ?? {}),
|
|
2030
|
-
processes: {
|
|
2031
|
-
...(baseConfig.profiles?.improve?.default?.processes ?? {}),
|
|
2032
|
-
consolidate: {
|
|
2033
|
-
...(baseConfig.profiles?.improve?.default?.processes?.consolidate ?? {}),
|
|
2034
|
-
enabled: true,
|
|
2035
|
-
},
|
|
2036
|
-
},
|
|
2037
|
-
},
|
|
2038
|
-
},
|
|
2039
|
-
},
|
|
2040
|
-
}
|
|
2041
|
-
: baseConfig;
|
|
2042
|
-
// 0.8.0 pool-delta gate for consolidate: re-eligible iff at least one
|
|
2043
|
-
// memory file has been updated since the most recent successful
|
|
2044
|
-
// consolidate_completed event. Time-based cooldowns produced the same
|
|
2045
|
-
// synchronised-wave failure mode the reflect/distill cooldowns did; the
|
|
2046
|
-
// pool-delta gate ties consolidation to actual work-to-do.
|
|
2047
|
-
const recentConsolidations = readEvents({ type: "consolidate_completed" });
|
|
2048
|
-
const lastConsolidation = recentConsolidations.events
|
|
2049
|
-
.filter((e) => e.metadata?.processed && Number(e.metadata.processed) > 0)
|
|
2050
|
-
.sort((a, b) => new Date(b.ts ?? 0).getTime() - new Date(a.ts ?? 0).getTime())[0];
|
|
2051
|
-
const lastConsolidateTs = lastConsolidation?.ts;
|
|
2052
|
-
// Pool-delta: any memory file with mtime > lastConsolidateTs flags work to do.
|
|
2053
|
-
// Using file mtime keeps this query DB-free and matches what the indexer
|
|
2054
|
-
// already uses as the canonical `memory.updated_at` proxy.
|
|
2055
|
-
//
|
|
2056
|
-
// Bootstrap: when no successful consolidate_completed event has ever been
|
|
2057
|
-
// recorded, we cannot evaluate the pool-delta — treat as eligible so a
|
|
2058
|
-
// fresh stash runs consolidate once before the steady-state gate kicks in.
|
|
2059
|
-
const memoryUpdatedAfterLastConsolidate = (() => {
|
|
2060
|
-
if (volumeTriggered)
|
|
2061
|
-
return true; // volume override forces the run regardless.
|
|
2062
|
-
if (!lastConsolidateTs)
|
|
2063
|
-
return true; // bootstrap path: never consolidated.
|
|
2064
|
-
if (!primaryStashDir)
|
|
2065
|
-
return false;
|
|
2066
|
-
const memoriesDir = path.join(primaryStashDir, "memories");
|
|
2067
|
-
if (!fs.existsSync(memoriesDir))
|
|
2068
|
-
return false;
|
|
2069
|
-
try {
|
|
2070
|
-
return fs.readdirSync(memoriesDir).some((f) => {
|
|
2071
|
-
if (!f.endsWith(".md"))
|
|
2072
|
-
return false;
|
|
2073
|
-
try {
|
|
2074
|
-
return fs.statSync(path.join(memoriesDir, f)).mtime.toISOString() > lastConsolidateTs;
|
|
2075
|
-
}
|
|
2076
|
-
catch {
|
|
2077
|
-
return false;
|
|
2078
|
-
}
|
|
2079
|
-
});
|
|
2080
|
-
}
|
|
2081
|
-
catch {
|
|
2082
|
-
return false;
|
|
2083
|
-
}
|
|
2084
|
-
})();
|
|
2085
|
-
const consolidationOnCooldown = !volumeTriggered && !memoryUpdatedAfterLastConsolidate;
|
|
2086
|
-
// Profile gate: if profile explicitly disables consolidate, skip the entire pass.
|
|
2087
|
-
const consolidateDisabledByProfile = improveProfile?.processes?.consolidate?.enabled === false;
|
|
2088
|
-
let consolidation = {
|
|
2089
|
-
schemaVersion: 1,
|
|
2090
|
-
ok: true,
|
|
2091
|
-
shape: "consolidate-result",
|
|
2092
|
-
dryRun: false,
|
|
2093
|
-
previewOnly: false,
|
|
2094
|
-
target: "",
|
|
2095
|
-
processed: 0,
|
|
2096
|
-
merged: 0,
|
|
2097
|
-
deleted: 0,
|
|
2098
|
-
promoted: [],
|
|
2099
|
-
contradicted: 0,
|
|
2100
|
-
warnings: [],
|
|
2101
|
-
durationMs: 0,
|
|
2102
|
-
};
|
|
2103
|
-
let gateAutoAcceptedCount = 0;
|
|
2104
|
-
let gateAutoAcceptFailedCount = 0;
|
|
2105
|
-
const consolidateGateCfg = makeGateConfig("consolidate", {
|
|
2106
|
-
globalThreshold: options.autoAccept,
|
|
2107
|
-
dryRun: options.dryRun ?? false,
|
|
2108
|
-
stashDir: primaryStashDir,
|
|
2109
|
-
config: consolidationConfig,
|
|
2110
|
-
eventsCtx,
|
|
2111
|
-
}, { minimumThreshold: 95 });
|
|
2112
|
-
if (consolidateDisabledByProfile) {
|
|
2113
|
-
info("[improve] consolidation skipped (disabled by improve profile)");
|
|
2114
|
-
}
|
|
2115
|
-
else if (!consolidationOnCooldown) {
|
|
2116
|
-
consolidation = await akmConsolidate({
|
|
2117
|
-
...options.consolidateOptions,
|
|
2118
|
-
config: consolidationConfig,
|
|
2119
|
-
stashDir: options.stashDir,
|
|
2120
|
-
autoTriggered: volumeTriggered,
|
|
2121
|
-
// Tie consolidate proposals back to this improve invocation so
|
|
2122
|
-
// accept-rate-per-run aggregation works. Mirrors reflect/propose/extract.
|
|
2123
|
-
sourceRun: `consolidate-${Date.now()}`,
|
|
2124
|
-
// Incremental consolidation: pass the last-consolidation timestamp so
|
|
2125
|
-
// akmConsolidate skips chunks with no memory changed since then. Converts
|
|
2126
|
-
// consolidation cost from O(pool) to O(changed clusters) — the fix for
|
|
2127
|
-
// the rising p95 tail where full-pool re-judging produced 5–10 min runs
|
|
2128
|
-
// that promoted ~0. undefined → full pass on first-ever run (bootstrap).
|
|
2129
|
-
// volumeTriggered correctly forces the run past cooldown but must NOT
|
|
2130
|
-
// override incrementalSince — the stash has ~1400 eligible memories so
|
|
2131
|
-
// volumeTriggered=true on every run, permanently forcing full 12-chunk
|
|
2132
|
-
// scans (~264s) instead of the intended 1-2 chunk incremental path (~44s).
|
|
2133
|
-
incrementalSince: lastConsolidateTs,
|
|
2134
|
-
maxChunkSize: improveProfile?.processes?.consolidate?.maxChunkSize,
|
|
2135
|
-
// Honor profile.autoAccept (already merged into options.autoAccept at the
|
|
2136
|
-
// top of akmImprove). The CLI parser always supplies 90 when --auto-accept
|
|
2137
|
-
// is absent, so ?? 90 is not needed here and would prevent --auto-accept=false
|
|
2138
|
-
// (which maps to undefined) from disabling consolidation auto-accept.
|
|
2139
|
-
// options.consolidateOptions.autoAccept (if explicitly provided by caller)
|
|
2140
|
-
// still wins because the spread above runs first.
|
|
2141
|
-
autoAccept: options.consolidateOptions?.autoAccept ?? options.autoAccept,
|
|
2142
|
-
});
|
|
2143
|
-
{
|
|
2144
|
-
const consolidateGr = await runAutoAcceptGate(consolidation.promoted.map((proposalId) => {
|
|
2145
|
-
try {
|
|
2146
|
-
if (!primaryStashDir)
|
|
2147
|
-
return { proposalId, confidence: undefined };
|
|
2148
|
-
const proposal = getProposal(primaryStashDir, proposalId);
|
|
2149
|
-
return { proposalId, confidence: proposal.confidence };
|
|
2150
|
-
}
|
|
2151
|
-
catch {
|
|
2152
|
-
return { proposalId, confidence: undefined };
|
|
2153
|
-
}
|
|
2154
|
-
}), consolidateGateCfg);
|
|
2155
|
-
gateAutoAcceptedCount += consolidateGr.promoted.length;
|
|
2156
|
-
gateAutoAcceptFailedCount += consolidateGr.failed.length;
|
|
2157
|
-
}
|
|
2158
|
-
if (consolidation.processed > 0) {
|
|
2159
|
-
appendEvent({
|
|
2160
|
-
eventType: "consolidate_completed",
|
|
2161
|
-
ref: "memory:_consolidation",
|
|
2162
|
-
metadata: { processed: consolidation.processed, merged: consolidation.merged },
|
|
2163
|
-
}, eventsCtx);
|
|
2164
|
-
}
|
|
2165
|
-
}
|
|
2166
|
-
else {
|
|
2167
|
-
appendEvent({
|
|
2168
|
-
eventType: "improve_skipped",
|
|
2169
|
-
ref: "memory:_consolidation",
|
|
2170
|
-
metadata: {
|
|
2171
|
-
reason: "consolidation_no_memory_updates",
|
|
2172
|
-
lastEventTs: lastConsolidation?.ts ?? null,
|
|
2173
|
-
},
|
|
2174
|
-
}, eventsCtx);
|
|
2175
|
-
info("[improve] consolidation skipped (no memory updates since last run)");
|
|
2176
|
-
}
|
|
2177
|
-
// D9: track whether consolidation wrote any data so graph extraction can reindex if needed
|
|
2178
|
-
const consolidationRan = !consolidateDisabledByProfile && !consolidationOnCooldown && consolidation.processed > 0;
|
|
2179
2425
|
info("[improve] post-loop maintenance starting");
|
|
2180
2426
|
const maintenanceResult = await runImproveMaintenancePasses({
|
|
2181
2427
|
options,
|
|
@@ -2216,7 +2462,6 @@ async function runImprovePostLoopStage(args) {
|
|
|
2216
2462
|
}
|
|
2217
2463
|
return {
|
|
2218
2464
|
allWarnings,
|
|
2219
|
-
consolidation,
|
|
2220
2465
|
deadUrls,
|
|
2221
2466
|
...(maintenanceResult.memoryInference ? { memoryInference: maintenanceResult.memoryInference } : {}),
|
|
2222
2467
|
...(maintenanceResult.graphExtraction ? { graphExtraction: maintenanceResult.graphExtraction } : {}),
|
|
@@ -2228,8 +2473,10 @@ async function runImprovePostLoopStage(args) {
|
|
|
2228
2473
|
graphExtractionDurationMs: maintenanceResult.graphExtractionDurationMs,
|
|
2229
2474
|
orphansPurged: maintenanceResult.orphansPurged,
|
|
2230
2475
|
proposalsExpired: maintenanceResult.proposalsExpired,
|
|
2231
|
-
|
|
2232
|
-
|
|
2476
|
+
// Consolidation's auto-accept gate counts now accrue in the preparation
|
|
2477
|
+
// stage (#551); post-loop no longer runs an auto-accept gate of its own.
|
|
2478
|
+
gateAutoAcceptedCount: 0,
|
|
2479
|
+
gateAutoAcceptFailedCount: 0,
|
|
2233
2480
|
};
|
|
2234
2481
|
}
|
|
2235
2482
|
// TODO(refactor): mutates the passed-in `allWarnings` array as a hidden side channel. Return warnings in ImproveMaintenanceResult and merge in caller — invasive signature change deferred to next refactor pass.
|
|
@@ -2279,9 +2526,16 @@ async function runImproveMaintenancePasses(args) {
|
|
|
2279
2526
|
const inferenceStart = Date.now();
|
|
2280
2527
|
try {
|
|
2281
2528
|
// O-1 (#364): pass budget signal so a hung inference call is cancelled.
|
|
2282
|
-
memoryInference = await memoryInferenceFn(
|
|
2283
|
-
|
|
2284
|
-
|
|
2529
|
+
memoryInference = await memoryInferenceFn({
|
|
2530
|
+
config,
|
|
2531
|
+
sources,
|
|
2532
|
+
signal: budgetSignal,
|
|
2533
|
+
db,
|
|
2534
|
+
reEnrich: false,
|
|
2535
|
+
onProgress: (event) => {
|
|
2536
|
+
const current = event.currentRef ? ` ${event.currentRef}` : "";
|
|
2537
|
+
info(`[improve] memory inference ${event.processed}/${event.total}${current} (written ${event.writtenFacts}, skipped ${event.skippedNoFacts})`);
|
|
2538
|
+
},
|
|
2285
2539
|
});
|
|
2286
2540
|
memoryInferenceDurationMs = Date.now() - inferenceStart;
|
|
2287
2541
|
actions.push({ ref: "memory:_inference", mode: "memory-inference", result: memoryInference });
|
|
@@ -2362,8 +2616,14 @@ async function runImproveMaintenancePasses(args) {
|
|
|
2362
2616
|
info(`[improve] graph extraction ${event.processed}/${event.total}${current} (extracted ${event.extracted}, entities ${event.totalEntities}, relations ${event.totalRelations})`);
|
|
2363
2617
|
};
|
|
2364
2618
|
// O-1 (#364): pass budget signal so a hung graph extraction call is cancelled.
|
|
2365
|
-
graphExtraction = await graphExtractionFn(
|
|
2366
|
-
|
|
2619
|
+
graphExtraction = await graphExtractionFn({
|
|
2620
|
+
config,
|
|
2621
|
+
sources,
|
|
2622
|
+
signal: budgetSignal,
|
|
2623
|
+
db,
|
|
2624
|
+
reEnrich: false,
|
|
2625
|
+
onProgress: progressHandler,
|
|
2626
|
+
options: { candidatePaths },
|
|
2367
2627
|
});
|
|
2368
2628
|
graphExtractionDurationMs = Date.now() - extractionStart;
|
|
2369
2629
|
actions.push({ ref: "graph:_artifact", mode: "graph-extraction", result: graphExtraction });
|
|
@@ -2442,7 +2702,11 @@ async function runImproveMaintenancePasses(args) {
|
|
|
2442
2702
|
if (retentionDays > 0) {
|
|
2443
2703
|
let stateDb;
|
|
2444
2704
|
try {
|
|
2445
|
-
|
|
2705
|
+
// C2: reuse the boundary-pinned state.db path carried on eventsCtx so
|
|
2706
|
+
// this purge open never re-reads `process.env` live mid-run. The path
|
|
2707
|
+
// is always set by akmImprove; openStateDatabase() falls back to the
|
|
2708
|
+
// env-derived default only if a caller omitted it entirely.
|
|
2709
|
+
stateDb = openStateDatabase(eventsCtx?.dbPath);
|
|
2446
2710
|
const purgedCount = purgeOldEvents(stateDb, retentionDays);
|
|
2447
2711
|
if (purgedCount > 0) {
|
|
2448
2712
|
info(`[improve] events purge: ${purgedCount} event(s) older than ${retentionDays}d removed from state.db`);
|
|
@@ -2487,7 +2751,7 @@ async function runImproveMaintenancePasses(args) {
|
|
|
2487
2751
|
// and before the URL check (which lives in the outer caller).
|
|
2488
2752
|
if (sources.length > 0) {
|
|
2489
2753
|
try {
|
|
2490
|
-
stalenessDetection = await stalenessDetectionFn(config, sources, budgetSignal, db);
|
|
2754
|
+
stalenessDetection = await stalenessDetectionFn({ config, sources, signal: budgetSignal, db });
|
|
2491
2755
|
if (stalenessDetection.considered > 0) {
|
|
2492
2756
|
info(`[improve] staleness detection complete (considered ${stalenessDetection.considered}, ` +
|
|
2493
2757
|
`deprecated ${stalenessDetection.deprecated}, confirmed ${stalenessDetection.confirmed}, ` +
|