akm-cli 0.9.14 → 0.9.15-beta.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (120) hide show
  1. package/CHANGELOG.md +559 -0
  2. package/STABILITY.md +6 -3
  3. package/dist/akm +54 -1
  4. package/dist/akm-migrate +34 -1
  5. package/dist/assets/prompts/reflect-feedback-framing.md +1 -0
  6. package/dist/assets/prompts/reflect-llm-framed-contract.md +2 -0
  7. package/dist/assets/prompts/reflect-llm-schema-contract.md +2 -0
  8. package/dist/assets/tasks/core/improve.yml +1 -1
  9. package/dist/assets/tasks/core/index-refresh.yml +1 -1
  10. package/dist/assets/tasks/improve/akm-graph-refresh-weekly.yml +1 -1
  11. package/dist/assets/tasks/improve/akm-improve-catchup.yml +1 -1
  12. package/dist/assets/tasks/improve/akm-improve-consolidate.yml +1 -1
  13. package/dist/assets/tasks/improve/akm-improve-frequent.yml +1 -1
  14. package/dist/assets/tasks/improve/akm-improve-nightly.yml +1 -1
  15. package/dist/cli/retired-commands.js +0 -1
  16. package/dist/cli/shared.js +9 -0
  17. package/dist/cli/unknown-flags.js +1 -0
  18. package/dist/cli.js +40 -3
  19. package/dist/commands/config-cli.js +85 -3
  20. package/dist/commands/env/env-cli.js +1 -42
  21. package/dist/commands/env/env.js +1 -1
  22. package/dist/commands/env/secret-cli.js +1 -2
  23. package/dist/commands/health/checks.js +357 -63
  24. package/dist/commands/health/engine-usage.js +45 -0
  25. package/dist/commands/health/improve-metrics.js +18 -0
  26. package/dist/commands/health/llm-usage.js +41 -1
  27. package/dist/commands/health/plugin-staleness.js +7 -3
  28. package/dist/commands/health/version-drift.js +93 -0
  29. package/dist/commands/health/windows.js +3 -1
  30. package/dist/commands/health.js +44 -9
  31. package/dist/commands/improve/consolidate/chunking.js +4 -2
  32. package/dist/commands/improve/improve-cli.js +99 -5
  33. package/dist/commands/improve/improve-report.js +154 -0
  34. package/dist/commands/improve/improve-result-file.js +45 -33
  35. package/dist/commands/improve/improve-strategies.js +133 -3
  36. package/dist/commands/improve/improve-usage-report.js +182 -0
  37. package/dist/commands/improve/improve.js +40 -3
  38. package/dist/commands/improve/locks.js +28 -78
  39. package/dist/commands/improve/planner.js +1 -0
  40. package/dist/commands/improve/preparation.js +9 -1
  41. package/dist/commands/improve/reflect.js +44 -4
  42. package/dist/commands/models-cli.js +50 -1
  43. package/dist/commands/proposal/repository.js +8 -3
  44. package/dist/commands/proposal/validators/proposal-quality-validators.js +41 -6
  45. package/dist/commands/proposal/validators/proposal-validators.js +24 -0
  46. package/dist/commands/read/search-cli.js +38 -2
  47. package/dist/commands/read/show.js +103 -4
  48. package/dist/commands/sources/info.js +5 -1
  49. package/dist/commands/sources/installed-stashes.js +58 -16
  50. package/dist/commands/sources/self-update.js +2 -2
  51. package/dist/commands/sources/stash-cli.js +48 -0
  52. package/dist/commands/tasks/tasks-cli.js +49 -2
  53. package/dist/commands/workflow-cli.js +86 -12
  54. package/dist/core/asset/markdown-fragments.js +35 -0
  55. package/dist/core/config/config-schema.js +14 -0
  56. package/dist/core/config/config.js +302 -24
  57. package/dist/core/config/schema/embedding.js +41 -0
  58. package/dist/core/env-secret-ref.js +58 -5
  59. package/dist/core/errors.js +30 -0
  60. package/dist/core/file-lock.js +49 -15
  61. package/dist/core/improve-result.js +51 -0
  62. package/dist/core/loopback.js +17 -0
  63. package/dist/core/parent-watchdog.js +64 -0
  64. package/dist/core/paths.js +11 -0
  65. package/dist/core/run-lock.js +107 -0
  66. package/dist/core/sensitive-marker-path.js +19 -0
  67. package/dist/core/state-db.js +74 -14
  68. package/dist/indexer/index-rebuild-lock.js +73 -0
  69. package/dist/indexer/index-writer-lock.js +40 -1
  70. package/dist/indexer/index-written-assets.js +29 -1
  71. package/dist/indexer/indexer.js +93 -29
  72. package/dist/indexer/materialize-embeddings.js +564 -48
  73. package/dist/indexer/search/db-search.js +49 -2
  74. package/dist/indexer/search/search-source.js +23 -1
  75. package/dist/integrations/agent/engine-resolution.js +96 -6
  76. package/dist/integrations/agent/execution-definitions.js +6 -15
  77. package/dist/integrations/agent/execution-lowering.js +6 -1
  78. package/dist/integrations/agent/execution-preparation.js +1 -1
  79. package/dist/integrations/agent/model-map.js +123 -20
  80. package/dist/integrations/agent/prompts.js +40 -8
  81. package/dist/integrations/agent/runner-dispatch.js +9 -3
  82. package/dist/integrations/agent/runner.js +2 -0
  83. package/dist/llm/client.js +8 -3
  84. package/dist/llm/embedder.js +20 -8
  85. package/dist/llm/embedders/local.js +10 -2
  86. package/dist/llm/embedders/remote.js +497 -32
  87. package/dist/output/shapes/helpers.js +38 -2
  88. package/dist/output/shapes/models-list.js +16 -0
  89. package/dist/output/shapes/passthrough.js +2 -0
  90. package/dist/output/shapes.js +4 -0
  91. package/dist/output/text/command-format.js +29 -0
  92. package/dist/output/text/helpers.js +1 -1
  93. package/dist/output/text/improve-report.js +27 -0
  94. package/dist/{commands/env/marker-path.js → output/text/models.js} +4 -3
  95. package/dist/output/text/show-format.js +4 -0
  96. package/dist/output/text.js +4 -0
  97. package/dist/scripts/akm-migrate-node.js +25146 -21759
  98. package/dist/scripts/akm-migrate.js +24271 -20885
  99. package/dist/storage/repositories/embedding-salvage-repository.js +184 -0
  100. package/dist/storage/repositories/improve-runs-repository.js +34 -0
  101. package/dist/storage/repositories/index-fts-repository.js +49 -6
  102. package/dist/storage/repositories/index-schema.js +16 -0
  103. package/dist/storage/repositories/index-vec-repository.js +30 -0
  104. package/dist/storage/repositories/workflow-runs-repository.js +55 -18
  105. package/dist/tasks/backends/cron.js +14 -7
  106. package/dist/tasks/run/run-native-task.js +23 -1
  107. package/dist/tasks/run/run-workflow-task.js +16 -0
  108. package/dist/workflows/exec/child-workflow.js +2 -2
  109. package/dist/workflows/exec/dispatch-redaction.js +21 -9
  110. package/dist/workflows/exec/run-workflow.js +6 -5
  111. package/dist/workflows/runtime/runs.js +33 -5
  112. package/docs/migration/release-notes/0.9.15.md +133 -0
  113. package/docs/migration/release-notes/README.md +5 -0
  114. package/docs/reference/cli.md +271 -30
  115. package/docs/reference/configuration.md +234 -21
  116. package/docs/reference/data-and-telemetry.md +8 -0
  117. package/docs/reference/tasks.md +16 -1
  118. package/docs/reference/workflow-schema.md +5 -1
  119. package/package.json +1 -1
  120. package/schemas/akm-config.json +47 -0
@@ -22,11 +22,12 @@ import { akmIndex } from "../../indexer/indexer.js";
22
22
  import { collectPendingMemories } from "../../indexer/passes/memory-inference.js";
23
23
  import { resolveEntryContentDir, resolveSourceEntries } from "../../indexer/search/search-source.js";
24
24
  import { collectEngineCredentialValues } from "../../integrations/agent/engine-resolution.js";
25
- import { installLlmUsagePersistence } from "../../llm/usage-persist.js";
25
+ import { installLlmUsagePersistence, LLM_USAGE_EVENT } from "../../llm/usage-persist.js";
26
26
  import { isGitBackedStash, listGitChangedPaths, resolveWritableOverride, saveGitStash, } from "../../sources/providers/git.js";
27
27
  import { closeDatabase, openExistingDatabase } from "../../storage/repositories/index-connection.js";
28
28
  import { getEntryCount } from "../../storage/repositories/index-entries-repository.js";
29
29
  import { openSqliteReadSnapshot, SqliteReadSnapshotUnavailableError } from "../../storage/sqlite-read-snapshot.js";
30
+ import { summarizeLlmUsageCrossTab } from "../health/llm-usage.js";
30
31
  import { drainProposals } from "../proposal/drain.js";
31
32
  import { resolveDrainPolicy } from "../proposal/drain-policies.js";
32
33
  import { describeGatedLanes, isAutonomyLaneAllowed } from "./autonomy-gate.js";
@@ -34,7 +35,8 @@ import { akmDistill } from "./distill.js";
34
35
  // Eligibility / candidate-selection predicates live in ./eligibility.
35
36
  import { buildLatestProposalTsMap, collectEligibleRefs, collectEligibleRefsReadOnly, memoryCleanupParentRef, resolveImproveScope, shouldAnalyzeMemoryCleanup, } from "./eligibility.js";
36
37
  import { countEvalCases } from "./eval-cases.js";
37
- import { resolveImprovePlan, resolveImproveStrategy } from "./improve-strategies.js";
38
+ import { projectResolvedProcessRouting, resolveImprovePlan, resolveImproveStrategy, shouldSkipRef, } from "./improve-strategies.js";
39
+ import { buildImproveUsageReport } from "./improve-usage-report.js";
38
40
  import { improveLockPath, releaseImproveLock, tryAcquireImproveLock } from "./locks.js";
39
41
  // The cycle loop / post-loop / maintenance stages live in ./loop-stages.
40
42
  import { runImproveLoopStage, runImprovePostLoopStage } from "./loop-stages.js";
@@ -402,8 +404,12 @@ function resolveImproveRunSetup(options) {
402
404
  const _earlyConfig = options.config ?? loadConfig();
403
405
  const configuredImproveProfile = resolveImproveStrategy(options.strategy, _earlyConfig).config;
404
406
  const resolvedPlan = options.resolvedPlan ??
407
+ // #800/#957 round 3 — same dry-run exemption as improve-cli.ts's own
408
+ // resolveImprovePlan call: a dry run never dispatches, so a strategy left
409
+ // fully disabled by an unreachable credential must not abort here either.
405
410
  resolveImprovePlan(options.strategy, _earlyConfig, {
406
411
  repairValidationFailures: options.repairValidationFailures,
412
+ allowAllDisabled: options.dryRun,
407
413
  });
408
414
  const selectedStrategy = resolvedPlan.strategy;
409
415
  const improveSensitiveValues = collectEngineCredentialValues(_earlyConfig);
@@ -759,6 +765,7 @@ export function buildDryRunResult(run, collected, preparation) {
759
765
  }
760
766
  : {}),
761
767
  ...(strategyFilteredRefs.length > 0 ? { strategyFilteredRefs } : {}),
768
+ ...(run.resolvedPlan.engineUnavailable.length > 0 ? { skippedProcesses: run.resolvedPlan.engineUnavailable } : {}),
762
769
  ...(preparation?.proactiveMaintenance ? { proactiveMaintenance: preparation.proactiveMaintenance } : {}),
763
770
  };
764
771
  }
@@ -783,6 +790,17 @@ function buildResultExecutionPlan(run, preparation, rawProfileEligibleRefs, stra
783
790
  removed: strategyFilteredRefs.length,
784
791
  reason: "all enabled per-ref processes refuse the asset type",
785
792
  };
793
+ // #947 — per-process resolved engine/model/notices, plus how many of this
794
+ // run's effective refs each ref-scoped process (reflect/distill/consolidate)
795
+ // would act on. Counts only (not a per-ref matrix) to bound result_json size.
796
+ const REF_SCOPED_PROCESSES = new Set(["reflect", "distill", "consolidate"]);
797
+ const processes = projectResolvedProcessRouting(resolvedPlan).map((row) => {
798
+ if (!REF_SCOPED_PROCESSES.has(row.process))
799
+ return row;
800
+ const eligibleRefs = preparation.loopRefs.filter((entry) => !shouldSkipRef(entry.ref, row.process, resolvedPlan.strategy.config)
801
+ .skip).length;
802
+ return { ...row, eligibleRefs };
803
+ });
786
804
  const proactive = preparation.planning.proactive
787
805
  ? {
788
806
  ...preparation.planning.proactive,
@@ -823,6 +841,7 @@ function buildResultExecutionPlan(run, preparation, rawProfileEligibleRefs, stra
823
841
  effectiveLimit,
824
842
  replayBudget: preparation.planning.replayBudget,
825
843
  gates: [profileGate, ...preparation.planning.gates],
844
+ processes,
826
845
  ...(proactive ? { proactive } : {}),
827
846
  consolidation,
828
847
  stageConfig: {
@@ -1281,7 +1300,7 @@ async function runImproveStageSequence(args) {
1281
1300
  */
1282
1301
  function finalizeImproveResult(args) {
1283
1302
  const { guidance, memorySummary, memoryCleanupPlan, strategyFilteredRefs, rawPlannedRefs, indexSnapshot, triageDrain, eventsCtx, } = args;
1284
- const { selectedStrategy, scope, options, primaryStashDir, startMs } = args.run;
1303
+ const { selectedStrategy, scope, options, primaryStashDir, startMs, resolvedPlan } = args.run;
1285
1304
  const { preparation, consolidation, memoryInference, graphExtraction, cycleMetrics, reflectsWithErrorContext, memoryInferenceDurationMs, graphExtractionDurationMs, orphansPurged, proposalsExpired, allWarnings, deadUrls, deadUrlCoverage, finalActions, } = args.seq;
1286
1305
  // C1 (13-bus-factor): fold the per-ref `distill-skipped` rows (~13k/run,
1287
1306
  // ~91% of result_json bytes) into a bounded aggregate BEFORE persistence.
@@ -1289,6 +1308,22 @@ function finalizeImproveResult(args) {
1289
1308
  // the unbounded row list never reaches result_json. Reflect skip counters
1290
1309
  // below still read `finalActions` (reflect skips are not folded).
1291
1310
  const { actions: persistedActions, aggregate: distillSkippedAggregate } = foldDistillSkipped(finalActions);
1311
+ // #944 — this run's LLM call/token accounting, split by process x engine x
1312
+ // model, plus which enabled processes made zero calls and why. `llm_usage`
1313
+ // events carry no runId column, so bound the read to this run's own wall
1314
+ // clock — the same per-run event-scoping technique `health/windows.ts`
1315
+ // already uses for wall time. `until` is "now" (assembly happens at
1316
+ // teardown, after every LLM call this run will make has already emitted
1317
+ // its event).
1318
+ const usageEvents = readEvents({ since: new Date(startMs).toISOString(), type: LLM_USAGE_EVENT }, eventsCtx).events;
1319
+ const usageReport = buildImproveUsageReport({
1320
+ resolvedPlan,
1321
+ byProcessEngineModel: summarizeLlmUsageCrossTab(usageEvents),
1322
+ strategyFilteredRefsCount: strategyFilteredRefs.length,
1323
+ loopRefs: preparation.loopRefs,
1324
+ persistedActions,
1325
+ distillSkippedAggregate,
1326
+ });
1292
1327
  const notices = collectImproveNotices({
1293
1328
  resolvedPlan: args.run.resolvedPlan,
1294
1329
  actions: finalActions,
@@ -1336,6 +1371,8 @@ function finalizeImproveResult(args) {
1336
1371
  }
1337
1372
  : {}),
1338
1373
  ...(strategyFilteredRefs.length > 0 ? { strategyFilteredRefs } : {}),
1374
+ ...(resolvedPlan.engineUnavailable.length > 0 ? { skippedProcesses: resolvedPlan.engineUnavailable } : {}),
1375
+ ...(usageReport ? { usageReport } : {}),
1339
1376
  actions: persistedActions,
1340
1377
  ...(distillSkippedAggregate ? { distillSkipped: distillSkippedAggregate } : {}),
1341
1378
  ...(preparation.validationFailures.length > 0 ? { validationFailures: preparation.validationFailures } : {}),
@@ -1,13 +1,12 @@
1
1
  // This Source Code Form is subject to the terms of the Mozilla Public
2
2
  // License, v. 2.0. If a copy of the MPL was not distributed with this
3
3
  // file, You can obtain one at https://mozilla.org/MPL/2.0/.
4
- import fs from "node:fs";
5
4
  import path from "node:path";
6
5
  import { ConfigError } from "../../core/errors.js";
7
6
  import { appendEvent } from "../../core/events.js";
8
- import { createLockPayload, probeLock, reclaimStaleLock, releaseLock, tryAcquireLockSync, } from "../../core/file-lock.js";
7
+ import { releaseLock } from "../../core/file-lock.js";
9
8
  import { tryWithMaintenanceStartBarrier, withMaintenanceStartBarrier } from "../../core/maintenance-barrier.js";
10
- import { describeInaccessiblePath } from "../../core/path-access.js";
9
+ import { formatLockHolderPid, tryAcquireRunLock } from "../../core/run-lock.js";
11
10
  import { warn } from "../../core/warn.js";
12
11
  export function improveLockPath(lockBaseDir) {
13
12
  return path.join(lockBaseDir, "improve.lock");
@@ -37,85 +36,36 @@ export function tryAcquireImproveLock(lockPath, skipIfLocked, eventsCtx) {
37
36
  return result;
38
37
  }
39
38
  function tryAcquireImproveLockUnlocked(lockPath, skipIfLocked, onRecovered) {
40
- fs.mkdirSync(path.dirname(lockPath), { recursive: true });
41
- const lockPayload = () => createLockPayload({ startedAt: new Date().toISOString() });
42
- let ownership = tryAcquireLockSync(lockPath, lockPayload());
43
- if (ownership) {
44
- return { state: "acquired", ownership };
45
- }
46
- // No `staleAfterMs`: only a verifiably dead holder is ever reclaimed. A
47
- // wedged-but-alive `akm improve` (SQLite WAL + busy_timeout + BEGIN
48
- // IMMEDIATE already serialize concurrent writes to state.db at the
49
- // correctness layer, so this lock only avoids duplicate LOGICAL work) must
50
- // not have its lease silently taken away purely because a clock elapsed —
51
- // that was the #872-shaped hazard here: a live holder passed the
52
- // PID-liveness check forever, so only a multi-hour age window could ever
53
- // free it, stranding every `akm improve --skip-if-locked` invocation for
54
- // up to that long while reporting success.
55
- const probe = probeLock(lockPath);
56
- // Race: the holder released the lock between our failed `tryAcquireLockSync`
57
- // and this probe, so the probe sees no file (`absent`). Retry acquisition once
58
- // rather than falling through to the contended skip/throw below — otherwise we
59
- // would warn/throw with a null PID for a lock that nobody actually holds.
60
- // (Mirrors the absent/stale reclaim-and-retry in `acquireExtractSessionLock`.)
61
- if (probe.state === "absent") {
62
- ownership = tryAcquireLockSync(lockPath, lockPayload());
63
- if (ownership) {
64
- return { state: "acquired", ownership };
65
- }
66
- // Re-grabbed by another racer in the window — fall through and treat as held.
67
- }
68
- // A lock we cannot READ is a hard stop, never a reclaim candidate (#791): the
69
- // holder may be alive and working, and we simply lack permission to see it.
70
- // Stealing the lease here would let two improve runs mutate the same bundle.
71
- if (probe.state === "inaccessible") {
72
- throw new ConfigError(`improve lock exists but is not readable: ${describeInaccessiblePath(lockPath, probe.code)}.`, "DATA_DIR_UNREADABLE");
73
- }
74
- const rawContent = probe.state === "absent" ? undefined : probe.rawContent;
75
- const lock = rawContent
76
- ? (() => {
77
- try {
78
- return JSON.parse(rawContent);
79
- }
80
- catch {
81
- return null;
82
- }
83
- })()
84
- : null;
85
- if (probe.state === "stale") {
86
- if (!reclaimStaleLock(lockPath, probe)) {
87
- if (skipIfLocked) {
88
- warn("[improve] lock changed ownership during stale recovery; skipping (--skip-if-locked)");
89
- return { state: "skipped" };
90
- }
91
- throw new ConfigError(`akm improve is already running. Delete ${lockPath} to force.`, "INVALID_CONFIG_FILE");
92
- }
93
- onRecovered({
94
- eventType: "improve_lock_recovered",
95
- metadata: {
96
- lockName: "improve",
97
- stalePid: lock?.pid ?? null,
98
- lockedAt: lock?.startedAt ?? null,
99
- recoveredAt: new Date().toISOString(),
100
- lockAgeMs: probe.ageMs ?? null,
101
- reason: probe.reason === "pid_dead" ? "pid_not_alive" : probe.reason,
102
- },
103
- });
104
- ownership = tryAcquireLockSync(lockPath, lockPayload());
105
- if (ownership) {
106
- return { state: "acquired", ownership };
107
- }
108
- if (skipIfLocked) {
109
- warn("[improve] lock acquired by another run during stale recovery; skipping (--skip-if-locked)");
110
- return { state: "skipped" };
111
- }
112
- throw new ConfigError(`akm improve is already running. Delete ${lockPath} to force.`, "INVALID_CONFIG_FILE");
39
+ // Mechanics (PID-liveness-only acquire/probe/absent-race-retry/stale-reclaim)
40
+ // live in the shared `core/run-lock.ts` module (#956) — this wrapper only
41
+ // owns improve's own policy: what "held" means (skip vs throw) and the
42
+ // improve_lock_recovered audit event.
43
+ const result = tryAcquireRunLock(lockPath, {
44
+ label: "improve",
45
+ onReclaimed: (info) => {
46
+ onRecovered({
47
+ eventType: "improve_lock_recovered",
48
+ metadata: {
49
+ lockName: "improve",
50
+ stalePid: info.holderPid,
51
+ lockedAt: info.lockedAt,
52
+ recoveredAt: new Date().toISOString(),
53
+ lockAgeMs: info.ageMs,
54
+ reason: info.reason,
55
+ },
56
+ });
57
+ },
58
+ });
59
+ if (result.state === "acquired") {
60
+ return { state: "acquired", ownership: result.ownership };
113
61
  }
62
+ const { startedAt } = result.holder;
63
+ const pid = formatLockHolderPid(result.holder);
114
64
  if (skipIfLocked) {
115
- warn(`[improve] another improve run holds the lock (PID ${lock?.pid}, started ${lock?.startedAt}); skipping (--skip-if-locked)`);
65
+ warn(`[improve] another improve run holds the lock (PID ${pid}, started ${startedAt}); skipping (--skip-if-locked)`);
116
66
  return { state: "skipped" };
117
67
  }
118
- throw new ConfigError(`akm improve is already running (PID ${lock?.pid}, started ${lock?.startedAt}). Delete ${lockPath} to force.`, "INVALID_CONFIG_FILE");
68
+ throw new ConfigError(`akm improve is already running (PID ${pid}, started ${startedAt}). Delete ${lockPath} to force.`, "INVALID_CONFIG_FILE");
119
69
  }
120
70
  export function releaseImproveLock(ownership) {
121
71
  releaseLock(ownership);
@@ -87,6 +87,7 @@ export function buildImproveExecutionPlan(input) {
87
87
  },
88
88
  gates: input.gates.map((gate) => ({ ...gate })),
89
89
  effectiveRefs,
90
+ processes: input.processes.map((row) => ({ ...row })),
90
91
  ...(input.proactive
91
92
  ? {
92
93
  proactive: {
@@ -230,7 +230,15 @@ function planConsolidationPass(args) {
230
230
  readOnly: eventsCtx?.readOnly === true,
231
231
  })
232
232
  : { poolSize: 0, candidatePoolSize: 0, dedupPoolSize: 0, memories: [] };
233
- const chunkSize = computeSafeChunkSize(resolvedPlan.processes.consolidate.runner?.connection.contextLength ?? DEFAULT_CONTEXT_LENGTH_TOKENS, 500, processConfig?.maxChunkSize);
233
+ // #800/#957 round 3 a credential-unavailable consolidate engine still
234
+ // resolved a context length structurally; read it off the `engineUnavailable`
235
+ // entry instead of falling back to the generic default, so a dry-run
236
+ // preview reflects the real engine even when its credential isn't
237
+ // materialized here.
238
+ const consolidateUnavailable = resolvedPlan.engineUnavailable.find((item) => item.process === "consolidate");
239
+ const chunkSize = computeSafeChunkSize(resolvedPlan.processes.consolidate.runner?.connection.contextLength ??
240
+ consolidateUnavailable?.contextLength ??
241
+ DEFAULT_CONTEXT_LENGTH_TOKENS, 500, processConfig?.maxChunkSize);
234
242
  const profilePassed = !eligibility.consolidateDisabledByProfile;
235
243
  const minimumPoolPassed = !eligibility.poolBelowMinSize;
236
244
  const deltaPassed = !eligibility.consolidationOnCooldown;
@@ -41,7 +41,7 @@ import { DEFAULT_LLM_TIMEOUT_MS } from "../../integrations/agent/config.js";
41
41
  import { fallbackAnnouncement, NO_ENGINE_MESSAGE_SUFFIX, NO_ENGINE_REMEDY, withEngineFallback, } from "../../integrations/agent/engine-fallback.js";
42
42
  import { acquireLoweredExecutionDispatchLease, dispatchLoweredExecutionRequest, disposeLoweredExecutionDispatchLease, lowerResolvedExecutionRequest, lowerResolvedExecutionRequestWithRunner, } from "../../integrations/agent/execution-lowering.js";
43
43
  import { prepareInlineExecution, prepareInlineExecutionWithRunner } from "../../integrations/agent/inline-execution.js";
44
- import { buildReflectOutputRepairPrompt, buildReflectPrompt, extractDraftConfidence, parseAgentProposalPayload, } from "../../integrations/agent/prompts.js";
44
+ import { buildReflectOutputRepairPrompt, buildReflectPrompt, extractDraftConfidence, parseAgentProposalPayload, REFLECT_CONTENT_CAP, REFLECT_TRUNCATION_MARKER, } from "../../integrations/agent/prompts.js";
45
45
  import { runnerIsLlm, runnerSupportsFileWrite } from "../../integrations/agent/runner.js";
46
46
  import { collectDispatchSensitiveValues } from "../../integrations/agent/runner-dispatch.js";
47
47
  import { isJsonSchemaKnownUnsupported, LlmCallError } from "../../llm/client.js";
@@ -49,6 +49,7 @@ import { callStructured } from "../../llm/structured-call.js";
49
49
  import { baseFailureFields, enoentHintMessage, isEnoentFailure } from "../agent/agent-support.js";
50
50
  import { isProposalSkipped, listProposalsReadOnly, proposalContent, recordGateDecision, } from "../proposal/repository.js";
51
51
  import { checkReflectSize, isValidDescription } from "../proposal/validators/proposal-quality-validators.js";
52
+ import { CHARS_PER_TOKEN, DEFAULT_CONTEXT_LENGTH_TOKENS } from "./consolidate/chunking.js";
52
53
  import { deriveLessonRef } from "./distill.js";
53
54
  import { runReflectQualityJudge } from "./distill/quality-gate.js";
54
55
  import { findAssetFilePath } from "./eligibility.js";
@@ -553,6 +554,17 @@ export function sanitizeReflectPayload(payload, sourceContent, targetRef) {
553
554
  warnings.push(`${sizeOutcome.code} — proposed body is ${pct}% of source (${limit}) for ref ${targetRef}. ${cause} Flagged for review.`);
554
555
  sizeGuardRatio = { code: sizeOutcome.code, ratio: sizeOutcome.ratio };
555
556
  }
557
+ // Truncation-marker leak (#952) — a model that saw a capped/truncated
558
+ // asset sometimes echoes the "[truncated ...]" notice verbatim into its
559
+ // rewrite instead of proposing real content for the missing tail. The
560
+ // body-length ratio check above does not reliably catch this (a leaked
561
+ // marker can still fall inside the 50%-250% band). Flag and defer to
562
+ // human review — same "degrade with a warning" rung as the size guard,
563
+ // not a new hard reject.
564
+ const truncationMarkerLeaked = cleanedBody.includes(REFLECT_TRUNCATION_MARKER);
565
+ if (truncationMarkerLeaked) {
566
+ warnings.push(`Proposed body for ref ${targetRef} contains the truncation-notice text the model was shown for a capped source asset ("${REFLECT_TRUNCATION_MARKER}"). The model likely echoed the notice instead of writing real content. Flagged for review.`);
567
+ }
556
568
  // Reassemble final content: merged frontmatter + cleaned body.
557
569
  // When there is no frontmatter at all (no source fm and no LLM fm), emit body
558
570
  // only so we don't add a stray `---` to e.g. a script asset that bypassed the
@@ -566,6 +578,7 @@ export function sanitizeReflectPayload(payload, sourceContent, targetRef) {
566
578
  ...(hasFrontmatter ? { frontmatter: mergedFm } : {}),
567
579
  warnings,
568
580
  ...(sizeGuardRatio ? { sizeGuardRatio } : {}),
581
+ ...(truncationMarkerLeaked ? { truncationMarkerLeaked } : {}),
569
582
  };
570
583
  }
571
584
  /**
@@ -962,7 +975,9 @@ async function finalizeReflectProposal(args) {
962
975
  }
963
976
  // 7c. Judge the exact sanitized content that can be persisted. Fail closed
964
977
  // on cancellation, transport failure, malformed output, or an invalid score.
965
- if (qualityGateEnabled && !sanitizeOutcome.sizeGuardRatio) {
978
+ // Skipped when the size guard or the truncation-marker leak already fired —
979
+ // that content is deferred to human review regardless of what the judge says.
980
+ if (qualityGateEnabled && !sanitizeOutcome.sizeGuardRatio && !sanitizeOutcome.truncationMarkerLeaked) {
966
981
  const judgeResult = await runReflectQualityJudge(config, payload.content, assetContent ?? "", feedback, options.chat, {
967
982
  runnerSelectionFrozen: true,
968
983
  ...(qualityJudgeRunner ? { llmRunner: qualityJudgeRunner } : {}),
@@ -1004,6 +1019,7 @@ async function finalizeReflectProposal(args) {
1004
1019
  outputTelemetry,
1005
1020
  qualityGateSkippedNoJudge,
1006
1021
  sizeGuardRatio: sanitizeOutcome.sizeGuardRatio,
1022
+ truncationMarkerLeaked: sanitizeOutcome.truncationMarkerLeaked,
1007
1023
  });
1008
1024
  }
1009
1025
  /**
@@ -1013,7 +1029,7 @@ async function finalizeReflectProposal(args) {
1013
1029
  * `akmReflect`'s finalize tail.
1014
1030
  */
1015
1031
  function createReflectProposal(args) {
1016
- const { payload, options, stash, engineName, durationMs, emitReflectFailed, outputTelemetry, qualityGateSkippedNoJudge, sizeGuardRatio, } = args;
1032
+ const { payload, options, stash, engineName, durationMs, emitReflectFailed, outputTelemetry, qualityGateSkippedNoJudge, sizeGuardRatio, truncationMarkerLeaked, } = args;
1017
1033
  // 8. Create the proposal. The proposal queue is the ONLY thing reflect
1018
1034
  // writes — promotion to a real asset is gated by `akm proposal accept`.
1019
1035
  //
@@ -1082,6 +1098,8 @@ function createReflectProposal(args) {
1082
1098
  reviewReasons.push("no-judge-configured");
1083
1099
  if (sizeGuardRatio)
1084
1100
  reviewReasons.push("reflect-size-ratio");
1101
+ if (truncationMarkerLeaked)
1102
+ reviewReasons.push("reflect-truncation-leak");
1085
1103
  if (reviewReasons.length > 0) {
1086
1104
  proposal =
1087
1105
  recordGateDecision(stash, proposal.id, {
@@ -1100,6 +1118,7 @@ function createReflectProposal(args) {
1100
1118
  engine: engineName,
1101
1119
  ...(qualityGateSkippedNoJudge ? { qualityGateSkippedNoJudge: true } : {}),
1102
1120
  ...(sizeGuardRatio ? { sizeGuardRatio: sizeGuardRatio.code, sizeGuardRatioValue: sizeGuardRatio.ratio } : {}),
1121
+ ...(truncationMarkerLeaked ? { truncationMarkerLeaked: true } : {}),
1103
1122
  ...(outputTelemetry ?? {}),
1104
1123
  },
1105
1124
  }, options.eventsCtx);
@@ -1385,7 +1404,7 @@ async function runReflectRefineIterations(args) {
1385
1404
  draftPathsToCleanup.push(iterDraftPath);
1386
1405
  lastDraftPath = iterDraftPath;
1387
1406
  }
1388
- const { prompt } = buildReflectPrompt({
1407
+ const promptInput = {
1389
1408
  ...(options.ref ? { ref: options.ref } : {}),
1390
1409
  ...(parsedRef?.type ? { type: parsedRef.type } : {}),
1391
1410
  ...(parsedRef?.name ? { name: parsedRef.name } : {}),
@@ -1405,6 +1424,27 @@ async function runReflectRefineIterations(args) {
1405
1424
  // on long bodies (e.g. knowledge/systems/KOKORO_USAGE_GUIDE 8.4KB).
1406
1425
  ...(iterDraftPath ? { draftFilePath: iterDraftPath } : {}),
1407
1426
  ...(outputMode ? { outputMode } : {}),
1427
+ };
1428
+ // #952 — the flat REFLECT_CONTENT_CAP (12 000 chars) exists only to avoid
1429
+ // E2BIG when the prompt travels through CLI argv (agent/SDK runners). The
1430
+ // direct-LLM HTTP path never touches argv, so it can use the resolved
1431
+ // engine's own context window instead. The reserve for "the rest of the
1432
+ // prompt" is measured directly (not guessed): build the same prompt with
1433
+ // the content cap forced to zero and use its length as the overhead, so
1434
+ // feedback/standards/schema-hints/prior-draft size is accounted for
1435
+ // exactly, per this call. A reflect rewrite returns a body roughly the
1436
+ // size of the input, so the budget only spends HALF of the usable window
1437
+ // on input content and reserves the other half for the model's own
1438
+ // output — otherwise a full-context request leaves no room for a
1439
+ // response. Never drops below the flat floor.
1440
+ const contentBudgetChars = runnerIsLlm(runnerSpec) && assetContent?.trim()
1441
+ ? Math.max(REFLECT_CONTENT_CAP, Math.floor(((runnerSpec.connection.contextLength ?? DEFAULT_CONTEXT_LENGTH_TOKENS) * CHARS_PER_TOKEN -
1442
+ buildReflectPrompt({ ...promptInput, contentBudgetChars: 0 }).prompt.length) /
1443
+ 2))
1444
+ : undefined;
1445
+ const { prompt } = buildReflectPrompt({
1446
+ ...promptInput,
1447
+ ...(contentBudgetChars !== undefined ? { contentBudgetChars } : {}),
1408
1448
  });
1409
1449
  let iterResult;
1410
1450
  if (runnerIsLlm(runnerSpec)) {
@@ -1,8 +1,48 @@
1
1
  // This Source Code Form is subject to the terms of the Mozilla Public
2
2
  // License, v. 2.0. If a copy of the MPL was not distributed with this
3
3
  // file, You can obtain one at https://mozilla.org/MPL/2.0/.
4
+ import { isDeepStrictEqual } from "node:util";
4
5
  import { defineGroupCommand, defineJsonCommand, output } from "../cli/shared.js";
5
- import { copyDefaultModelMap } from "../integrations/agent/model-map.js";
6
+ import { loadConfig } from "../core/config/config.js";
7
+ import { copyDefaultModelMap, loadModelMapLayers, mergedModelMapProfiles, mergeModelMapLayers, } from "../integrations/agent/model-map.js";
8
+ /**
9
+ * Compute the effective alias table (#946): every (alias, column) pair from
10
+ * the fully resolved map, labeled with where its value came from.
11
+ *
12
+ * `source` compares the installed-only merge against the full (installed +
13
+ * user) merge for the same pair — identical means the user file never
14
+ * touched it. `via`/`engine` come from the raw overlaid-but-unresolved
15
+ * profile (before `engine` indirection is expanded), so a column that
16
+ * resolves through `{ engine: "local-fast" }` reports which engine it
17
+ * borrowed its model from.
18
+ */
19
+ function modelsListRows() {
20
+ const config = loadConfig();
21
+ const layers = loadModelMapLayers();
22
+ const rawProfiles = mergedModelMapProfiles(layers.installed, layers.user);
23
+ const defaultsOnly = mergeModelMapLayers(layers.installed, undefined, config.engines);
24
+ const resolved = mergeModelMapLayers(layers.installed, layers.user, config.engines);
25
+ const rows = [];
26
+ for (const [alias, columns] of Object.entries(resolved.aliases)) {
27
+ for (const [column, profile] of Object.entries(columns)) {
28
+ const raw = rawProfiles[alias]?.[column];
29
+ const via = raw?.engine !== undefined ? "engine" : "literal";
30
+ const defaultProfile = defaultsOnly.aliases[alias]?.[column];
31
+ const source = defaultProfile !== undefined && isDeepStrictEqual(defaultProfile, profile) ? "default" : "user";
32
+ rows.push({
33
+ alias,
34
+ column,
35
+ model: profile.model,
36
+ ...(Object.hasOwn(profile, "inference") ? { inference: profile.inference } : {}),
37
+ source,
38
+ via,
39
+ ...(via === "engine" && raw?.engine !== undefined ? { engine: raw.engine } : {}),
40
+ });
41
+ }
42
+ }
43
+ rows.sort((left, right) => left.alias === right.alias ? left.column.localeCompare(right.column) : left.alias.localeCompare(right.alias));
44
+ return rows;
45
+ }
6
46
  /** Operator-owned model-map lifecycle commands. Alias expansion is consumed through the runtime API. */
7
47
  export const modelsCommand = defineGroupCommand({
8
48
  meta: { name: "models", description: "Inspect and customize model intent alias defaults" },
@@ -23,5 +63,14 @@ export const modelsCommand = defineGroupCommand({
23
63
  output("models", copyDefaultModelMap({ overwrite: args.overwrite === true }));
24
64
  },
25
65
  }),
66
+ list: defineJsonCommand({
67
+ meta: {
68
+ name: "list",
69
+ description: "Show the effective model intent alias table and where each mapping resolved from",
70
+ },
71
+ run() {
72
+ output("models-list", { rows: modelsListRows() });
73
+ },
74
+ }),
26
75
  },
27
76
  });
@@ -67,7 +67,7 @@ import { pkgVersion } from "../../version.js";
67
67
  import { runBaseChecks } from "../lint/base-linter.js";
68
68
  import { formatNewAssetDiff, formatUnifiedDiff } from "./diff-format.js";
69
69
  import { isAutomatedProposalSource, isValidProposalSource, PROPOSAL_SOURCES, } from "./proposal-types.js";
70
- import { hasCanonicalProposalValidator } from "./validators/proposal-validators.js";
70
+ import { canonicalOnlyProposalValidators, hasCanonicalProposalValidator, runProposalValidators, } from "./validators/proposal-validators.js";
71
71
  import { repairProposalContent, validateProposal } from "./validators/proposals.js";
72
72
  const PROMOTION_LINT_ISSUE_TYPES = new Set(["unquoted-colon", "missing-ref", "stale-path"]);
73
73
  // ── Proposal domain types (moved to ./proposal-types.ts, WI-9.8 KILL 1) ─────
@@ -343,7 +343,12 @@ export function createProposal(stashDir, input, ctx) {
343
343
  ];
344
344
  const mintedBeforeHash = mintBeforeContent !== undefined ? contentHash(mintBeforeContent) : undefined;
345
345
  if (hasCanonicalProposalValidator(parsedRef.type)) {
346
- const report = validateProposal({
346
+ // Mint-time gate: structural shape only (generic + canonical-per-type),
347
+ // NOT the full quality-validator list — see canonicalOnlyProposalValidators'
348
+ // doc comment (#952 review round 2). Quality validators (including the
349
+ // blocking reflect-truncation-marker guard) run at `proposal accept` /
350
+ // drain-promotion time via validateProposal instead.
351
+ const report = runProposalValidators({
347
352
  id: "pending",
348
353
  ref: normalizedRef,
349
354
  status: "pending",
@@ -353,7 +358,7 @@ export function createProposal(stashDir, input, ctx) {
353
358
  payload: { ...input.payload, content: proposalContent },
354
359
  changes: mintedChanges,
355
360
  proposedTarget: { source: proposalTarget.source, root: targetRoot },
356
- });
361
+ }, canonicalOnlyProposalValidators);
357
362
  if (!report.ok) {
358
363
  return rejectProposal("invalid_canonical_structure", `Proposal for "${input.ref}" has invalid ${parsedRef.type} structure:\n${report.findings
359
364
  .map((finding) => `[${finding.kind}] ${finding.message}`)
@@ -71,6 +71,7 @@ function refNameTail(inputRef) {
71
71
  return parseRefInput(inputRef).name.toLowerCase();
72
72
  }
73
73
  import { detectTruncatedDescription, TRUNCATION_TRAILING_WORDS } from "../../../core/text-truncation.js";
74
+ import { REFLECT_TRUNCATION_MARKER } from "../../../integrations/agent/prompts.js";
74
75
  // ── Description / when_to_use shape ─────────────────────────────────────────
75
76
  export const HEADING_FRAGMENT_PATTERNS = [
76
77
  /^for example\b/i,
@@ -370,6 +371,34 @@ const reflectSizeGuardValidator = {
370
371
  ];
371
372
  },
372
373
  };
374
+ /**
375
+ * Accept-time data-loss guard (#952 Addendum). `sanitizeReflectPayload`
376
+ * already defers a proposal whose body echoes {@link REFLECT_TRUNCATION_MARKER}
377
+ * (the notice appended when the source asset was too large to send in full)
378
+ * with `reflect-truncation-leak` — but that is a creation-time check, and a
379
+ * proposal can reach `proposal accept` / drain promotion without ever going
380
+ * through it (e.g. a defer that a human then accepts anyway, or a future
381
+ * reflect code path that mints proposals directly). A leaked marker replacing
382
+ * real asset content on disk is data loss, not a quality nit, so unlike the
383
+ * rest of this file's validators this one is NOT wrapped by {@link advisory}
384
+ * — it blocks acceptance the same way the generic/canonical validators do.
385
+ */
386
+ const reflectTruncationMarkerValidator = {
387
+ name: "reflect-truncation-marker",
388
+ appliesTo(proposal) {
389
+ return proposal.source === "reflect" && typeof proposal.payload?.content === "string";
390
+ },
391
+ validate(proposal) {
392
+ if (!proposalContent(proposal).includes(REFLECT_TRUNCATION_MARKER))
393
+ return [];
394
+ return [
395
+ {
396
+ kind: "reflect-truncation-marker-leak",
397
+ message: `Proposal ${proposal.id} (${proposal.ref}) body still contains the reflect truncation marker "${REFLECT_TRUNCATION_MARKER}" — the source asset was too large to send in full and this body would overwrite it with an incomplete rewrite. Reflect this ref again (or raise its content budget) and re-propose.`,
398
+ },
399
+ ];
400
+ },
401
+ };
373
402
  /**
374
403
  * Report a validator's findings as advisory.
375
404
  *
@@ -394,11 +423,17 @@ function advisory(validator) {
394
423
  /**
395
424
  * Full set of quality validators in registration order. Appended onto
396
425
  * {@link defaultProposalValidators} so they run inside `validateProposal` on
397
- * `proposal accept` automatically, and report without blocking.
426
+ * `proposal accept` automatically. All prose-quality checks report without
427
+ * blocking (see {@link advisory}); {@link reflectTruncationMarkerValidator} is
428
+ * the one exception and blocks, since it guards against data loss rather than
429
+ * prose quality.
398
430
  */
399
431
  export const defaultProposalQualityValidators = [
400
- descriptionQualityValidator,
401
- lessonContentQualityValidator,
402
- sourceNotSupersededValidator,
403
- reflectSizeGuardValidator,
404
- ].map(advisory);
432
+ ...[
433
+ descriptionQualityValidator,
434
+ lessonContentQualityValidator,
435
+ sourceNotSupersededValidator,
436
+ reflectSizeGuardValidator,
437
+ ].map(advisory),
438
+ reflectTruncationMarkerValidator,
439
+ ];
@@ -123,6 +123,30 @@ export const defaultProposalValidators = [
123
123
  canonicalProposalValidator,
124
124
  ...defaultProposalQualityValidators,
125
125
  ];
126
+ /**
127
+ * Structural-only subset used by {@link createProposal}'s mint-time
128
+ * canonical-structure gate (repository.ts, `hasCanonicalProposalValidator`).
129
+ * That gate exists to reject a lesson/task/workflow proposal whose body is
130
+ * not parseable as its type — it predates {@link defaultProposalQualityValidators}
131
+ * and was previously safe to run in full there because every quality
132
+ * validator was advisory (`advisory()` downgrades findings to `severity:
133
+ * "warn"`, which {@link runProposalValidators}'s `ok` never treats as
134
+ * failing). #952's `reflect-truncation-marker` validator is deliberately
135
+ * NOT advisory (it guards against data loss), so running the full
136
+ * {@link defaultProposalValidators} list at mint time would throw
137
+ * `invalid_canonical_structure` for any lesson/task/workflow reflect
138
+ * proposal whose body leaks the truncation marker — instead of letting
139
+ * `sanitizeReflectPayload` mint the proposal and defer it with
140
+ * `reflect-truncation-leak`, per the #952 design. Quality validators (prose
141
+ * shape, reflect size ratio, the truncation-marker guard) belong at
142
+ * `proposal accept` / drain-promotion time, which already calls
143
+ * {@link validateProposal} (the full list) via `preflightProposalPromotion`
144
+ * / `promoteProposalWithLease`.
145
+ */
146
+ export const canonicalOnlyProposalValidators = [
147
+ genericProposalValidator,
148
+ canonicalProposalValidator,
149
+ ];
126
150
  export function runProposalValidators(proposal, validators = defaultProposalValidators, initialContext = {}) {
127
151
  const findings = [];
128
152
  const ctx = { ...initialContext };