akm-cli 0.8.0-rc2 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/{.github/CHANGELOG.md → CHANGELOG.md} +191 -3
- package/README.md +22 -6
- package/SECURITY.md +93 -0
- package/dist/cli/config-migrate.js +144 -0
- package/dist/cli/config-validate.js +39 -0
- package/dist/cli/confirm.js +73 -0
- package/dist/cli/parse-args.js +93 -3
- package/dist/cli/shared.js +129 -0
- package/dist/cli.js +2141 -1268
- package/dist/commands/add-cli.js +279 -0
- package/dist/commands/agent-dispatch.js +20 -12
- package/dist/commands/agent-support.js +11 -5
- package/dist/commands/completions.js +3 -0
- package/dist/commands/config-cli.js +129 -517
- package/dist/commands/consolidate.js +1533 -144
- package/dist/commands/curate.js +44 -3
- package/dist/commands/db-cli.js +23 -0
- package/dist/commands/distill-promotion-policy.js +5 -3
- package/dist/commands/distill.js +906 -100
- package/dist/commands/env.js +213 -0
- package/dist/commands/eval-cases.js +3 -0
- package/dist/commands/events.js +3 -0
- package/dist/commands/extract-cli.js +127 -0
- package/dist/commands/extract-prompt.js +204 -0
- package/dist/commands/extract.js +477 -0
- package/dist/commands/feedback-cli.js +331 -0
- package/dist/commands/graph.js +260 -5
- package/dist/commands/health.js +977 -51
- package/dist/commands/help/help-accept.md +6 -3
- package/dist/commands/help/help-improve.md +36 -8
- package/dist/commands/help/help-proposals.md +7 -4
- package/dist/commands/help/help-reject.md +5 -2
- package/dist/commands/history.js +51 -16
- package/dist/commands/improve-auto-accept.js +97 -0
- package/dist/commands/improve-cli.js +236 -0
- package/dist/commands/improve-profiles.js +184 -0
- package/dist/commands/improve-result-file.js +167 -0
- package/dist/commands/improve.js +1725 -332
- package/dist/commands/info.js +3 -0
- package/dist/commands/init.js +49 -1
- package/dist/commands/installed-stashes.js +6 -23
- package/dist/commands/knowledge.js +3 -0
- package/dist/commands/lint/agent-linter.js +3 -0
- package/dist/commands/lint/base-linter.js +199 -5
- package/dist/commands/lint/command-linter.js +3 -0
- package/dist/commands/lint/default-linter.js +3 -0
- package/dist/commands/lint/env-key-rules.js +154 -0
- package/dist/commands/lint/index.js +92 -3
- package/dist/commands/lint/knowledge-linter.js +3 -0
- package/dist/commands/lint/markdown-insertion.js +343 -0
- package/dist/commands/lint/memory-linter.js +3 -0
- package/dist/commands/lint/registry.js +3 -0
- package/dist/commands/lint/skill-linter.js +3 -0
- package/dist/commands/lint/task-linter.js +15 -12
- package/dist/commands/lint/types.js +3 -0
- package/dist/commands/lint/workflow-linter.js +3 -0
- package/dist/commands/lint.js +3 -0
- package/dist/commands/migration-help.js +5 -2
- package/dist/commands/proposal-drain-policies.js +128 -0
- package/dist/commands/proposal-drain.js +477 -0
- package/dist/commands/proposal.js +60 -6
- package/dist/commands/propose.js +24 -19
- package/dist/commands/reflect.js +1004 -94
- package/dist/commands/registry-cli.js +150 -0
- package/dist/commands/registry-search.js +3 -0
- package/dist/commands/remember-cli.js +257 -0
- package/dist/commands/remember.js +15 -6
- package/dist/commands/schema-repair.js +88 -15
- package/dist/commands/search.js +99 -14
- package/dist/commands/secret.js +173 -0
- package/dist/commands/self-update.js +3 -0
- package/dist/commands/show.js +32 -13
- package/dist/commands/source-add.js +7 -35
- package/dist/commands/source-clone.js +3 -0
- package/dist/commands/source-manage.js +3 -0
- package/dist/commands/tasks.js +161 -95
- package/dist/commands/url-checker.js +3 -0
- package/dist/core/action-contributors.js +3 -0
- package/dist/core/asset-ref.js +13 -2
- package/dist/core/asset-registry.js +9 -2
- package/dist/core/asset-serialize.js +88 -0
- package/dist/core/asset-spec.js +61 -5
- package/dist/core/common.js +93 -5
- package/dist/core/concurrent.js +3 -0
- package/dist/core/config-io.js +347 -0
- package/dist/core/config-migration.js +622 -0
- package/dist/core/config-schema.js +558 -0
- package/dist/core/config-sources.js +108 -0
- package/dist/core/config-types.js +4 -0
- package/dist/core/config-walker.js +337 -0
- package/dist/core/config.js +366 -1077
- package/dist/core/errors.js +42 -20
- package/dist/core/events.js +31 -25
- package/dist/core/file-lock.js +104 -0
- package/dist/core/frontmatter.js +75 -10
- package/dist/core/lesson-lint.js +3 -0
- package/dist/core/markdown.js +3 -0
- package/dist/core/memory-belief.js +62 -0
- package/dist/core/memory-contradiction-detect.js +274 -0
- package/dist/core/memory-improve.js +142 -14
- package/dist/core/parse.js +3 -0
- package/dist/core/paths.js +218 -50
- package/dist/core/proposal-quality-validators.js +380 -0
- package/dist/core/proposal-validators.js +11 -3
- package/dist/core/proposals.js +464 -5
- package/dist/core/state-db.js +349 -56
- package/dist/core/text-truncation.js +107 -0
- package/dist/core/time.js +3 -0
- package/dist/core/tty.js +59 -0
- package/dist/core/warn.js +7 -2
- package/dist/core/write-source.js +12 -0
- package/dist/indexer/db-backup.js +391 -0
- package/dist/indexer/db-search.js +136 -28
- package/dist/indexer/db.js +661 -166
- package/dist/indexer/ensure-index.js +3 -0
- package/dist/indexer/file-context.js +3 -0
- package/dist/indexer/graph-boost.js +162 -40
- package/dist/indexer/graph-db.js +241 -51
- package/dist/indexer/graph-dedup.js +3 -7
- package/dist/indexer/graph-extraction.js +242 -149
- package/dist/indexer/index-context.js +3 -9
- package/dist/indexer/indexer.js +84 -14
- package/dist/indexer/llm-cache.js +24 -19
- package/dist/indexer/manifest.js +3 -0
- package/dist/indexer/matchers.js +184 -11
- package/dist/indexer/memory-inference.js +94 -50
- package/dist/indexer/metadata-contributors.js +3 -0
- package/dist/indexer/metadata.js +110 -50
- package/dist/indexer/path-resolver.js +3 -0
- package/dist/indexer/project-context.js +192 -0
- package/dist/indexer/ranking-contributors.js +134 -7
- package/dist/indexer/ranking.js +8 -1
- package/dist/indexer/search-fields.js +5 -9
- package/dist/indexer/search-hit-enrichers.js +91 -2
- package/dist/indexer/search-source.js +20 -1
- package/dist/indexer/semantic-status.js +4 -1
- package/dist/indexer/staleness-detect.js +447 -0
- package/dist/indexer/usage-events.js +12 -9
- package/dist/indexer/walker.js +3 -0
- package/dist/integrations/agent/builders.js +135 -0
- package/dist/integrations/agent/config.js +121 -401
- package/dist/integrations/agent/detect.js +3 -0
- package/dist/integrations/agent/index.js +6 -14
- package/dist/integrations/agent/model-aliases.js +55 -0
- package/dist/integrations/agent/profiles.js +3 -0
- package/dist/integrations/agent/prompts.js +137 -8
- package/dist/integrations/agent/runner.js +208 -0
- package/dist/integrations/agent/sdk-runner.js +8 -2
- package/dist/integrations/agent/spawn.js +54 -14
- package/dist/integrations/github.js +3 -0
- package/dist/integrations/lockfile.js +22 -51
- package/dist/integrations/session-logs/index.js +4 -0
- package/dist/integrations/session-logs/inline-refs.js +35 -0
- package/dist/integrations/session-logs/pre-filter.js +152 -0
- package/dist/integrations/session-logs/providers/claude-code.js +226 -0
- package/dist/integrations/session-logs/providers/opencode.js +231 -25
- package/dist/integrations/session-logs/types.js +3 -0
- package/dist/llm/call-ai.js +14 -26
- package/dist/llm/client.js +16 -2
- package/dist/llm/embedder.js +20 -29
- package/dist/llm/embedders/cache.js +3 -7
- package/dist/llm/embedders/local.js +42 -1
- package/dist/llm/embedders/remote.js +20 -8
- package/dist/llm/embedders/types.js +3 -7
- package/dist/llm/feature-gate.js +92 -56
- package/dist/llm/graph-extract.js +401 -30
- package/dist/llm/index-passes.js +44 -29
- package/dist/llm/memory-infer.js +30 -2
- package/dist/llm/metadata-enhance.js +3 -7
- package/dist/llm/prompts/extract-session.md +80 -0
- package/dist/llm/prompts/graph-extract-user-prompt.md +24 -1
- package/dist/output/cli-hints-full.md +60 -32
- package/dist/output/cli-hints-short.md +10 -7
- package/dist/output/cli-hints.js +5 -2
- package/dist/output/context.js +60 -8
- package/dist/output/renderers.js +170 -194
- package/dist/output/shapes/curate.js +56 -0
- package/dist/output/shapes/distill.js +10 -0
- package/dist/output/shapes/env-list.js +19 -0
- package/dist/output/shapes/events.js +11 -0
- package/dist/output/shapes/helpers.js +424 -0
- package/dist/output/shapes/history.js +7 -0
- package/dist/output/shapes/passthrough.js +105 -0
- package/dist/output/shapes/proposal-accept.js +7 -0
- package/dist/output/shapes/proposal-diff.js +7 -0
- package/dist/output/shapes/proposal-list.js +7 -0
- package/dist/output/shapes/proposal-producer.js +11 -0
- package/dist/output/shapes/proposal-reject.js +7 -0
- package/dist/output/shapes/proposal-show.js +7 -0
- package/dist/output/shapes/registry-search.js +6 -0
- package/dist/output/shapes/registry.js +30 -0
- package/dist/output/shapes/search.js +6 -0
- package/dist/output/shapes/secret-list.js +19 -0
- package/dist/output/shapes/show.js +6 -0
- package/dist/output/shapes/vault-list.js +19 -0
- package/dist/output/shapes.js +51 -549
- package/dist/output/text/add.js +6 -0
- package/dist/output/text/clone.js +6 -0
- package/dist/output/text/config.js +6 -0
- package/dist/output/text/curate.js +6 -0
- package/dist/output/text/distill.js +7 -0
- package/dist/output/text/enable-disable.js +7 -0
- package/dist/output/text/events.js +10 -0
- package/dist/output/text/feedback.js +6 -0
- package/dist/output/text/helpers.js +1059 -0
- package/dist/output/text/history.js +7 -0
- package/dist/output/text/import.js +6 -0
- package/dist/output/text/index.js +6 -0
- package/dist/output/text/info.js +6 -0
- package/dist/output/text/init.js +6 -0
- package/dist/output/text/list.js +6 -0
- package/dist/output/text/proposal-producer.js +8 -0
- package/dist/output/text/proposal.js +12 -0
- package/dist/output/text/registry-commands.js +11 -0
- package/dist/output/text/registry.js +30 -0
- package/dist/output/text/remember.js +6 -0
- package/dist/output/text/remove.js +6 -0
- package/dist/output/text/save.js +6 -0
- package/dist/output/text/search.js +6 -0
- package/dist/output/text/show.js +6 -0
- package/dist/output/text/update.js +6 -0
- package/dist/output/text/upgrade.js +6 -0
- package/dist/output/text/vault.js +16 -0
- package/dist/output/text/wiki.js +15 -0
- package/dist/output/text/workflow.js +14 -0
- package/dist/output/text.js +44 -1329
- package/dist/registry/build-index.js +3 -0
- package/dist/registry/create-provider-registry.js +3 -0
- package/dist/registry/factory.js +4 -1
- package/dist/registry/origin-resolve.js +3 -0
- package/dist/registry/providers/index.js +3 -0
- package/dist/registry/providers/skills-sh.js +11 -2
- package/dist/registry/providers/static-index.js +10 -1
- package/dist/registry/providers/types.js +3 -24
- package/dist/registry/resolve.js +11 -16
- package/dist/registry/types.js +3 -0
- package/dist/scripts/migrate-storage.js +17767 -0
- package/dist/scripts/migrations/import-fs-improve-runs-to-db.js +9031 -0
- package/dist/scripts/migrations/v16-to-v17.js +141 -0
- package/dist/setup/detect.js +3 -0
- package/dist/setup/ripgrep-install.js +3 -0
- package/dist/setup/ripgrep-resolve.js +3 -0
- package/dist/setup/setup.js +306 -67
- package/dist/setup/steps.js +3 -15
- package/dist/sources/include.js +3 -0
- package/dist/sources/provider-factory.js +3 -11
- package/dist/sources/provider.js +3 -20
- package/dist/sources/providers/filesystem.js +19 -23
- package/dist/sources/providers/git.js +171 -21
- package/dist/sources/providers/index.js +3 -0
- package/dist/sources/providers/install-types.js +3 -13
- package/dist/sources/providers/npm.js +3 -4
- package/dist/sources/providers/provider-utils.js +3 -0
- package/dist/sources/providers/sync-from-ref.js +3 -11
- package/dist/sources/providers/tar-utils.js +3 -0
- package/dist/sources/providers/website.js +18 -22
- package/dist/sources/resolve.js +3 -0
- package/dist/sources/types.js +3 -0
- package/dist/sources/website-ingest.js +3 -0
- package/dist/tasks/backends/cron.js +3 -0
- package/dist/tasks/backends/exec-utils.js +3 -0
- package/dist/tasks/backends/index.js +3 -11
- package/dist/tasks/backends/launchd.js +3 -0
- package/dist/tasks/backends/schtasks.js +3 -0
- package/dist/tasks/parser.js +51 -38
- package/dist/tasks/resolveAkmBin.js +3 -0
- package/dist/tasks/runner.js +35 -9
- package/dist/tasks/schedule.js +20 -1
- package/dist/tasks/schema.js +5 -3
- package/dist/tasks/validator.js +6 -3
- package/dist/version.js +3 -0
- package/dist/wiki/wiki-templates.js +3 -0
- package/dist/wiki/wiki.js +3 -0
- package/dist/workflows/authoring.js +3 -0
- package/dist/workflows/cli.js +3 -0
- package/dist/workflows/db.js +140 -10
- package/dist/workflows/document-cache.js +3 -10
- package/dist/workflows/parser.js +3 -0
- package/dist/workflows/renderer.js +3 -0
- package/dist/workflows/runs.js +18 -1
- package/dist/workflows/schema.js +3 -0
- package/dist/workflows/scope-key.js +3 -0
- package/dist/workflows/validator.js +5 -9
- package/docs/README.md +7 -2
- package/docs/data-and-telemetry.md +225 -0
- package/docs/migration/release-notes/0.7.5.md +2 -2
- package/docs/migration/release-notes/0.8.0.md +57 -5
- package/docs/migration/v0.7-to-v0.8.md +1378 -0
- package/package.json +28 -11
- package/.github/LICENSE +0 -374
- package/dist/commands/install-audit.js +0 -385
- package/dist/commands/vault.js +0 -310
- package/dist/indexer/match-contributors.js +0 -141
- package/dist/integrations/agent/pipeline.js +0 -39
- package/dist/integrations/agent/runners.js +0 -31
package/dist/commands/health.js
CHANGED
|
@@ -1,3 +1,6 @@
|
|
|
1
|
+
// This Source Code Form is subject to the terms of the Mozilla Public
|
|
2
|
+
// License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
3
|
+
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
1
4
|
import { spawnSync } from "node:child_process";
|
|
2
5
|
import fs from "node:fs";
|
|
3
6
|
import { loadConfig } from "../core/config";
|
|
@@ -48,17 +51,27 @@ function createUnknownImproveMetrics() {
|
|
|
48
51
|
skipped: 0,
|
|
49
52
|
skipReasons: {},
|
|
50
53
|
plannedRefs: 0,
|
|
54
|
+
profileFilteredRefs: 0,
|
|
51
55
|
actions: {
|
|
52
|
-
reflect: 0,
|
|
53
|
-
distill:
|
|
54
|
-
|
|
56
|
+
reflect: { ok: 0, failed: 0, cooldown: 0, skipped: 0, guardRejected: 0, skippedByReason: {} },
|
|
57
|
+
distill: {
|
|
58
|
+
queued: 0,
|
|
59
|
+
llmFailed: 0,
|
|
60
|
+
qualityRejected: 0,
|
|
61
|
+
judgeRejected: 0,
|
|
62
|
+
validatorRejected: 0,
|
|
63
|
+
configDisabled: 0,
|
|
64
|
+
skipped: 0,
|
|
65
|
+
skippedByReason: {},
|
|
66
|
+
deferred: 0,
|
|
67
|
+
deferredByReason: {},
|
|
68
|
+
},
|
|
55
69
|
memoryPrune: 0,
|
|
56
70
|
memoryInference: 0,
|
|
57
71
|
graphExtraction: 0,
|
|
58
72
|
error: 0,
|
|
59
73
|
},
|
|
60
|
-
|
|
61
|
-
feedbackRatioUsed: false,
|
|
74
|
+
reflectsWithErrorContext: 0,
|
|
62
75
|
coverageGapCount: 0,
|
|
63
76
|
executionLogCandidateCount: 0,
|
|
64
77
|
evalCasesWritten: 0,
|
|
@@ -72,9 +85,63 @@ function createUnknownImproveMetrics() {
|
|
|
72
85
|
archived: 0,
|
|
73
86
|
warnings: 0,
|
|
74
87
|
},
|
|
75
|
-
consolidation: {
|
|
76
|
-
|
|
77
|
-
|
|
88
|
+
consolidation: {
|
|
89
|
+
ran: false,
|
|
90
|
+
processed: 0,
|
|
91
|
+
promoted: 0,
|
|
92
|
+
merged: 0,
|
|
93
|
+
deleted: 0,
|
|
94
|
+
contradicted: 0,
|
|
95
|
+
judgedNoAction: 0,
|
|
96
|
+
mergedSecondaries: 0,
|
|
97
|
+
failedChunkMemories: 0,
|
|
98
|
+
skipReasons: {},
|
|
99
|
+
failedChunks: 0,
|
|
100
|
+
totalChunks: 0,
|
|
101
|
+
durationMs: 0,
|
|
102
|
+
},
|
|
103
|
+
memoryInference: {
|
|
104
|
+
ran: false,
|
|
105
|
+
considered: 0,
|
|
106
|
+
cacheHits: 0,
|
|
107
|
+
freshAttempts: 0,
|
|
108
|
+
splitParents: 0,
|
|
109
|
+
written: 0,
|
|
110
|
+
skippedNoFacts: 0,
|
|
111
|
+
skippedChildExists: 0,
|
|
112
|
+
skippedAborted: 0,
|
|
113
|
+
unaccounted: 0,
|
|
114
|
+
yieldEligibleRuns: 0,
|
|
115
|
+
yieldEligibleConsidered: 0,
|
|
116
|
+
yieldEligibleWritten: 0,
|
|
117
|
+
yieldRate: 0,
|
|
118
|
+
durationMs: 0,
|
|
119
|
+
writes: 0,
|
|
120
|
+
},
|
|
121
|
+
graphExtraction: {
|
|
122
|
+
ran: false,
|
|
123
|
+
extractedFiles: 0,
|
|
124
|
+
entities: 0,
|
|
125
|
+
relations: 0,
|
|
126
|
+
cacheHits: 0,
|
|
127
|
+
cacheMisses: 0,
|
|
128
|
+
cacheHitRate: 0,
|
|
129
|
+
truncations: 0,
|
|
130
|
+
failures: 0,
|
|
131
|
+
durationMs: 0,
|
|
132
|
+
},
|
|
133
|
+
wallTime: {
|
|
134
|
+
count: 0,
|
|
135
|
+
medianMs: 0,
|
|
136
|
+
p95Ms: 0,
|
|
137
|
+
minMs: 0,
|
|
138
|
+
maxMs: 0,
|
|
139
|
+
byPhase: {
|
|
140
|
+
consolidation: { count: 0, totalMs: 0, medianMs: 0, p95Ms: 0 },
|
|
141
|
+
memoryInference: { count: 0, totalMs: 0, medianMs: 0, p95Ms: 0 },
|
|
142
|
+
graphExtraction: { count: 0, totalMs: 0, medianMs: 0, p95Ms: 0 },
|
|
143
|
+
},
|
|
144
|
+
},
|
|
78
145
|
};
|
|
79
146
|
}
|
|
80
147
|
function toFiniteNumber(value) {
|
|
@@ -87,46 +154,558 @@ function toFiniteNumber(value) {
|
|
|
87
154
|
}
|
|
88
155
|
return 0;
|
|
89
156
|
}
|
|
157
|
+
/**
|
|
158
|
+
* Event-derived metrics. Only `completed` and skipReasons/invoked are sourced
|
|
159
|
+
* from events in v2 — the richer fields come from {@link summarizeImproveRuns}.
|
|
160
|
+
* The function still receives `improve_completed` events so that the completed
|
|
161
|
+
* count reflects the canonical event stream (it lines up 1:1 with improve_runs
|
|
162
|
+
* rows in practice, but the events table remains the system-of-record for the
|
|
163
|
+
* existence of a run).
|
|
164
|
+
*/
|
|
90
165
|
function summarizeImproveCompleted(events) {
|
|
91
166
|
const metrics = createUnknownImproveMetrics();
|
|
92
167
|
metrics.completed = events.length;
|
|
93
|
-
for (const event of events) {
|
|
94
|
-
const meta = event.metadata ?? {};
|
|
95
|
-
metrics.plannedRefs += toFiniteNumber(meta.plannedRefs);
|
|
96
|
-
metrics.actions.reflect += toFiniteNumber(meta.reflectActions);
|
|
97
|
-
metrics.actions.distill += toFiniteNumber(meta.distillActions);
|
|
98
|
-
metrics.actions.distillSkipped += toFiniteNumber(meta.distillSkippedActions);
|
|
99
|
-
metrics.actions.memoryPrune += toFiniteNumber(meta.memoryPruneActions);
|
|
100
|
-
metrics.actions.memoryInference += toFiniteNumber(meta.memoryInferenceActions);
|
|
101
|
-
metrics.actions.graphExtraction += toFiniteNumber(meta.graphExtractionActions);
|
|
102
|
-
metrics.actions.error += toFiniteNumber(meta.errorActions);
|
|
103
|
-
metrics.crossStepErrorsInjected += toFiniteNumber(meta.crossStepErrorsInjected);
|
|
104
|
-
metrics.coverageGapCount += toFiniteNumber(meta.coverageGapCount);
|
|
105
|
-
metrics.executionLogCandidateCount += toFiniteNumber(meta.executionLogCandidateCount);
|
|
106
|
-
metrics.evalCasesWritten += toFiniteNumber(meta.evalCasesWritten);
|
|
107
|
-
metrics.deadUrlCount += toFiniteNumber(meta.deadUrlCount);
|
|
108
|
-
metrics.memorySummary.eligible += toFiniteNumber(meta.memoryEligible);
|
|
109
|
-
metrics.memorySummary.derived += toFiniteNumber(meta.memoryDerived);
|
|
110
|
-
metrics.memoryCleanup.pruneCandidates += toFiniteNumber(meta.memoryCleanupPruneCandidates);
|
|
111
|
-
metrics.memoryCleanup.contradictionCandidates += toFiniteNumber(meta.memoryCleanupContradictionCandidates);
|
|
112
|
-
metrics.memoryCleanup.beliefStateTransitions += toFiniteNumber(meta.memoryCleanupBeliefStateTransitions);
|
|
113
|
-
metrics.memoryCleanup.consolidationCandidates += toFiniteNumber(meta.memoryCleanupConsolidationCandidates);
|
|
114
|
-
metrics.memoryCleanup.archived += toFiniteNumber(meta.memoryCleanupArchived);
|
|
115
|
-
metrics.memoryCleanup.warnings += toFiniteNumber(meta.memoryCleanupWarnings);
|
|
116
|
-
metrics.consolidation.processed += toFiniteNumber(meta.consolidationProcessed);
|
|
117
|
-
metrics.consolidation.durationMs += toFiniteNumber(meta.consolidationDurationMs);
|
|
118
|
-
metrics.memoryInference.writes += toFiniteNumber(meta.memoryInferenceWrites);
|
|
119
|
-
metrics.memoryInference.durationMs += toFiniteNumber(meta.memoryInferenceDurationMs);
|
|
120
|
-
metrics.graphExtraction.extractedFiles += toFiniteNumber(meta.graphExtractionExtractedFiles);
|
|
121
|
-
metrics.graphExtraction.durationMs += toFiniteNumber(meta.graphExtractionDurationMs);
|
|
122
|
-
if (meta.feedbackRatioUsed === true)
|
|
123
|
-
metrics.feedbackRatioUsed = true;
|
|
124
|
-
}
|
|
125
|
-
metrics.consolidation.ran = metrics.consolidation.processed > 0 || metrics.consolidation.durationMs > 0;
|
|
126
|
-
metrics.memoryInference.ran = metrics.memoryInference.writes > 0 || metrics.memoryInference.durationMs > 0;
|
|
127
|
-
metrics.graphExtraction.ran = metrics.graphExtraction.extractedFiles > 0 || metrics.graphExtraction.durationMs > 0;
|
|
128
168
|
return metrics;
|
|
129
169
|
}
|
|
170
|
+
/**
|
|
171
|
+
* Project a single `improve_runs.result_json` envelope into an accumulator-shaped
|
|
172
|
+
* ImproveHealthMetrics. The aggregator merges these per-row metrics into one
|
|
173
|
+
* window-level metric.
|
|
174
|
+
*/
|
|
175
|
+
function projectRunMetrics(result) {
|
|
176
|
+
const metrics = createUnknownImproveMetrics();
|
|
177
|
+
// plannedRefs (array of {ref, reason})
|
|
178
|
+
const plannedRefs = result.plannedRefs;
|
|
179
|
+
if (Array.isArray(plannedRefs))
|
|
180
|
+
metrics.plannedRefs += plannedRefs.length;
|
|
181
|
+
// profileFilteredRefs (array of {ref, reason}) — 2026-05-27: pre-filter
|
|
182
|
+
// bucket from `collectEligibleRefs` so the metric reflects work the
|
|
183
|
+
// planner dropped before signal-delta / per-pass dispatch.
|
|
184
|
+
const profileFilteredRefs = result.profileFilteredRefs;
|
|
185
|
+
if (Array.isArray(profileFilteredRefs))
|
|
186
|
+
metrics.profileFilteredRefs += profileFilteredRefs.length;
|
|
187
|
+
// actions: split reflect / distill by outcome, count others.
|
|
188
|
+
const actions = result.actions;
|
|
189
|
+
if (Array.isArray(actions)) {
|
|
190
|
+
for (const action of actions) {
|
|
191
|
+
const mode = typeof action.mode === "string" ? action.mode : "";
|
|
192
|
+
switch (mode) {
|
|
193
|
+
case "reflect":
|
|
194
|
+
metrics.actions.reflect.ok += 1;
|
|
195
|
+
break;
|
|
196
|
+
case "reflect-failed":
|
|
197
|
+
metrics.actions.reflect.failed += 1;
|
|
198
|
+
break;
|
|
199
|
+
case "reflect-cooldown":
|
|
200
|
+
metrics.actions.reflect.cooldown += 1;
|
|
201
|
+
break;
|
|
202
|
+
case "reflect-skipped": {
|
|
203
|
+
metrics.actions.reflect.skipped += 1;
|
|
204
|
+
const r = action.result;
|
|
205
|
+
const reason = typeof r?.reason === "string" && r.reason.trim() ? r.reason : "unknown";
|
|
206
|
+
metrics.actions.reflect.skippedByReason[reason] = (metrics.actions.reflect.skippedByReason[reason] ?? 0) + 1;
|
|
207
|
+
break;
|
|
208
|
+
}
|
|
209
|
+
case "reflect-guard-rejected":
|
|
210
|
+
metrics.actions.reflect.guardRejected += 1;
|
|
211
|
+
break;
|
|
212
|
+
case "distill": {
|
|
213
|
+
const r = action.result;
|
|
214
|
+
const outcome = typeof r?.outcome === "string" ? r.outcome : "";
|
|
215
|
+
switch (outcome) {
|
|
216
|
+
case "queued":
|
|
217
|
+
metrics.actions.distill.queued += 1;
|
|
218
|
+
break;
|
|
219
|
+
case "llm_failed":
|
|
220
|
+
metrics.actions.distill.llmFailed += 1;
|
|
221
|
+
break;
|
|
222
|
+
case "quality_rejected":
|
|
223
|
+
case "review_needed":
|
|
224
|
+
metrics.actions.distill.qualityRejected += 1;
|
|
225
|
+
metrics.actions.distill.judgeRejected += 1;
|
|
226
|
+
break;
|
|
227
|
+
case "validation_failed":
|
|
228
|
+
metrics.actions.distill.qualityRejected += 1;
|
|
229
|
+
metrics.actions.distill.validatorRejected += 1;
|
|
230
|
+
break;
|
|
231
|
+
case "config_disabled":
|
|
232
|
+
metrics.actions.distill.configDisabled += 1;
|
|
233
|
+
break;
|
|
234
|
+
case "skipped": {
|
|
235
|
+
// Previously dropped on the floor. The four sub-paths that emit
|
|
236
|
+
// `outcome: "skipped"` (see distill.ts:893, 1024, 1120, 1576):
|
|
237
|
+
// - recursive_lesson_input (type guard refused a lesson input)
|
|
238
|
+
// - conflict_noop (LLM resolved destination conflict as NOOP)
|
|
239
|
+
// - proposal-skipped cooldown / dedup at persistence
|
|
240
|
+
// 465 events/7d in the user's live stack. The result message
|
|
241
|
+
// typically encodes the reason; we also accept an explicit
|
|
242
|
+
// `skipReason` field when downstream code sets it.
|
|
243
|
+
metrics.actions.distill.deferred += 1;
|
|
244
|
+
const explicitReason = typeof r?.skipReason === "string" ? r.skipReason : undefined;
|
|
245
|
+
const msg = typeof r?.message === "string" ? r.message : "";
|
|
246
|
+
let reason = explicitReason ?? "unknown";
|
|
247
|
+
if (!explicitReason) {
|
|
248
|
+
if (/lesson inputs/i.test(msg))
|
|
249
|
+
reason = "recursive_lesson_input";
|
|
250
|
+
else if (/NOOP/.test(msg))
|
|
251
|
+
reason = "conflict_noop";
|
|
252
|
+
else if (/cooldown/i.test(msg))
|
|
253
|
+
reason = "proposal_cooldown";
|
|
254
|
+
else if (/content[_ ]?hash/i.test(msg))
|
|
255
|
+
reason = "content_hash_match";
|
|
256
|
+
}
|
|
257
|
+
metrics.actions.distill.deferredByReason[reason] =
|
|
258
|
+
(metrics.actions.distill.deferredByReason[reason] ?? 0) + 1;
|
|
259
|
+
break;
|
|
260
|
+
}
|
|
261
|
+
default:
|
|
262
|
+
break;
|
|
263
|
+
}
|
|
264
|
+
break;
|
|
265
|
+
}
|
|
266
|
+
case "distill-skipped": {
|
|
267
|
+
metrics.actions.distill.skipped += 1;
|
|
268
|
+
const r = action.result;
|
|
269
|
+
const reason = typeof r?.reason === "string" && r.reason.trim() ? r.reason : "unknown";
|
|
270
|
+
metrics.actions.distill.skippedByReason[reason] = (metrics.actions.distill.skippedByReason[reason] ?? 0) + 1;
|
|
271
|
+
break;
|
|
272
|
+
}
|
|
273
|
+
case "memory-prune":
|
|
274
|
+
metrics.actions.memoryPrune += 1;
|
|
275
|
+
break;
|
|
276
|
+
case "memory-inference":
|
|
277
|
+
metrics.actions.memoryInference += 1;
|
|
278
|
+
break;
|
|
279
|
+
case "graph-extraction":
|
|
280
|
+
metrics.actions.graphExtraction += 1;
|
|
281
|
+
break;
|
|
282
|
+
case "error":
|
|
283
|
+
metrics.actions.error += 1;
|
|
284
|
+
break;
|
|
285
|
+
}
|
|
286
|
+
}
|
|
287
|
+
}
|
|
288
|
+
metrics.reflectsWithErrorContext += toFiniteNumber(result.reflectsWithErrorContext);
|
|
289
|
+
if (Array.isArray(result.coverageGaps))
|
|
290
|
+
metrics.coverageGapCount += result.coverageGaps.length;
|
|
291
|
+
if (Array.isArray(result.executionLogCandidates))
|
|
292
|
+
metrics.executionLogCandidateCount += result.executionLogCandidates.length;
|
|
293
|
+
metrics.evalCasesWritten += toFiniteNumber(result.evalCasesWritten);
|
|
294
|
+
if (Array.isArray(result.deadUrls))
|
|
295
|
+
metrics.deadUrlCount += result.deadUrls.length;
|
|
296
|
+
const memorySummary = result.memorySummary;
|
|
297
|
+
if (memorySummary) {
|
|
298
|
+
metrics.memorySummary.eligible += toFiniteNumber(memorySummary.eligible);
|
|
299
|
+
metrics.memorySummary.derived += toFiniteNumber(memorySummary.derived);
|
|
300
|
+
}
|
|
301
|
+
const memoryCleanup = result.memoryCleanup;
|
|
302
|
+
if (memoryCleanup) {
|
|
303
|
+
if (Array.isArray(memoryCleanup.pruneCandidates))
|
|
304
|
+
metrics.memoryCleanup.pruneCandidates += memoryCleanup.pruneCandidates.length;
|
|
305
|
+
if (Array.isArray(memoryCleanup.contradictionCandidates))
|
|
306
|
+
metrics.memoryCleanup.contradictionCandidates += memoryCleanup.contradictionCandidates.length;
|
|
307
|
+
if (Array.isArray(memoryCleanup.beliefStateTransitions))
|
|
308
|
+
metrics.memoryCleanup.beliefStateTransitions += memoryCleanup.beliefStateTransitions.length;
|
|
309
|
+
if (Array.isArray(memoryCleanup.consolidationCandidates))
|
|
310
|
+
metrics.memoryCleanup.consolidationCandidates += memoryCleanup.consolidationCandidates.length;
|
|
311
|
+
if (Array.isArray(memoryCleanup.archived))
|
|
312
|
+
metrics.memoryCleanup.archived += memoryCleanup.archived.length;
|
|
313
|
+
if (Array.isArray(memoryCleanup.warnings))
|
|
314
|
+
metrics.memoryCleanup.warnings += memoryCleanup.warnings.length;
|
|
315
|
+
}
|
|
316
|
+
const consolidation = result.consolidation;
|
|
317
|
+
if (consolidation) {
|
|
318
|
+
metrics.consolidation.processed += toFiniteNumber(consolidation.processed);
|
|
319
|
+
metrics.consolidation.merged += toFiniteNumber(consolidation.merged);
|
|
320
|
+
metrics.consolidation.deleted += toFiniteNumber(consolidation.deleted);
|
|
321
|
+
metrics.consolidation.contradicted += toFiniteNumber(consolidation.contradicted);
|
|
322
|
+
if (Array.isArray(consolidation.promoted))
|
|
323
|
+
metrics.consolidation.promoted += consolidation.promoted.length;
|
|
324
|
+
metrics.consolidation.failedChunks += toFiniteNumber(consolidation.failedChunks);
|
|
325
|
+
metrics.consolidation.totalChunks += toFiniteNumber(consolidation.totalChunks);
|
|
326
|
+
metrics.consolidation.durationMs += toFiniteNumber(consolidation.durationMs);
|
|
327
|
+
metrics.consolidation.judgedNoAction += toFiniteNumber(consolidation.judgedNoAction);
|
|
328
|
+
metrics.consolidation.mergedSecondaries += toFiniteNumber(consolidation.mergedSecondaries);
|
|
329
|
+
metrics.consolidation.failedChunkMemories += toFiniteNumber(consolidation.failedChunkMemories);
|
|
330
|
+
// Structured emitter (new on this branch): consolidate.ts now pushes
|
|
331
|
+
// `{op, ref, reason}` entries to `skipReasons` for every deterministic
|
|
332
|
+
// post-LLM rejection. Pre-fix envelopes have neither field, so be
|
|
333
|
+
// defensive.
|
|
334
|
+
const skipReasons = consolidation.skipReasons;
|
|
335
|
+
if (Array.isArray(skipReasons)) {
|
|
336
|
+
for (const entry of skipReasons) {
|
|
337
|
+
if (!entry || typeof entry !== "object")
|
|
338
|
+
continue;
|
|
339
|
+
const reason = entry.reason;
|
|
340
|
+
if (typeof reason !== "string" || !reason.trim())
|
|
341
|
+
continue;
|
|
342
|
+
metrics.consolidation.skipReasons[reason] = (metrics.consolidation.skipReasons[reason] ?? 0) + 1;
|
|
343
|
+
}
|
|
344
|
+
}
|
|
345
|
+
}
|
|
346
|
+
const memoryInference = result.memoryInference;
|
|
347
|
+
if (memoryInference) {
|
|
348
|
+
const considered = toFiniteNumber(memoryInference.considered);
|
|
349
|
+
const writtenFacts = toFiniteNumber(memoryInference.writtenFacts);
|
|
350
|
+
metrics.memoryInference.considered += considered;
|
|
351
|
+
metrics.memoryInference.cacheHits += toFiniteNumber(memoryInference.cacheHits);
|
|
352
|
+
metrics.memoryInference.splitParents += toFiniteNumber(memoryInference.splitParents);
|
|
353
|
+
metrics.memoryInference.written += writtenFacts;
|
|
354
|
+
metrics.memoryInference.skippedNoFacts += toFiniteNumber(memoryInference.skippedNoFacts);
|
|
355
|
+
metrics.memoryInference.skippedChildExists += toFiniteNumber(memoryInference.skippedChildExists);
|
|
356
|
+
metrics.memoryInference.skippedAborted += toFiniteNumber(memoryInference.skippedAborted);
|
|
357
|
+
metrics.memoryInference.unaccounted += toFiniteNumber(memoryInference.unaccounted);
|
|
358
|
+
// Yield-rate gating: pre-cache-feature envelopes lack the `cacheHits`
|
|
359
|
+
// field entirely. Treating their `considered` as freshAttempts (since
|
|
360
|
+
// cacheHits=0) is mathematically tempting but operationally wrong —
|
|
361
|
+
// historical runs with the legacy schema have no cache instrumentation
|
|
362
|
+
// and the SUM dragged the reported rate to ~14% in local data. Only
|
|
363
|
+
// contribute to the yield aggregate when the envelope actually carries
|
|
364
|
+
// the field. See investigation 2026-05-26.
|
|
365
|
+
if (Object.hasOwn(memoryInference, "cacheHits")) {
|
|
366
|
+
metrics.memoryInference.yieldEligibleRuns += 1;
|
|
367
|
+
metrics.memoryInference.yieldEligibleConsidered += considered;
|
|
368
|
+
metrics.memoryInference.yieldEligibleWritten += writtenFacts;
|
|
369
|
+
}
|
|
370
|
+
}
|
|
371
|
+
metrics.memoryInference.durationMs += toFiniteNumber(result.memoryInferenceDurationMs);
|
|
372
|
+
const graphExtraction = result.graphExtraction;
|
|
373
|
+
if (graphExtraction) {
|
|
374
|
+
const quality = graphExtraction.quality;
|
|
375
|
+
if (quality)
|
|
376
|
+
metrics.graphExtraction.extractedFiles += toFiniteNumber(quality.extractedFiles);
|
|
377
|
+
metrics.graphExtraction.entities += toFiniteNumber(graphExtraction.totalEntities);
|
|
378
|
+
metrics.graphExtraction.relations += toFiniteNumber(graphExtraction.totalRelations);
|
|
379
|
+
const telemetry = graphExtraction.telemetry;
|
|
380
|
+
if (telemetry) {
|
|
381
|
+
metrics.graphExtraction.cacheHits += toFiniteNumber(telemetry.cacheHits);
|
|
382
|
+
metrics.graphExtraction.cacheMisses += toFiniteNumber(telemetry.cacheMisses);
|
|
383
|
+
metrics.graphExtraction.truncations += toFiniteNumber(telemetry.truncationCount);
|
|
384
|
+
metrics.graphExtraction.failures += toFiniteNumber(telemetry.failureCount);
|
|
385
|
+
}
|
|
386
|
+
}
|
|
387
|
+
metrics.graphExtraction.durationMs += toFiniteNumber(result.graphExtractionDurationMs);
|
|
388
|
+
return metrics;
|
|
389
|
+
}
|
|
390
|
+
/**
|
|
391
|
+
* Finalize derived flags and rates on an accumulator. Used both for the
|
|
392
|
+
* window-level aggregate and for each per-run row in --detail per-run mode
|
|
393
|
+
* so the single-row metrics still expose `ran` / `yieldRate` / `cacheHitRate`.
|
|
394
|
+
*/
|
|
395
|
+
function finalizeImproveMetrics(metrics) {
|
|
396
|
+
metrics.consolidation.ran =
|
|
397
|
+
metrics.consolidation.processed > 0 ||
|
|
398
|
+
metrics.consolidation.durationMs > 0 ||
|
|
399
|
+
metrics.consolidation.promoted > 0 ||
|
|
400
|
+
metrics.consolidation.merged > 0 ||
|
|
401
|
+
metrics.consolidation.deleted > 0 ||
|
|
402
|
+
metrics.consolidation.contradicted > 0 ||
|
|
403
|
+
metrics.consolidation.totalChunks > 0;
|
|
404
|
+
metrics.memoryInference.ran =
|
|
405
|
+
metrics.memoryInference.considered > 0 ||
|
|
406
|
+
metrics.memoryInference.written > 0 ||
|
|
407
|
+
metrics.memoryInference.durationMs > 0;
|
|
408
|
+
metrics.memoryInference.writes = metrics.memoryInference.written;
|
|
409
|
+
// Yield denominator excludes cache hits AND legacy (pre-cacheHits-field)
|
|
410
|
+
// envelopes. Only runs whose envelope carries a `cacheHits` field
|
|
411
|
+
// contribute to freshAttempts/yieldRate; legacy rows remain in
|
|
412
|
+
// `considered`/`written` for totals but are excluded from the rate so
|
|
413
|
+
// they cannot drag it down. See ImproveHealthMetrics.memoryInference
|
|
414
|
+
// jsdoc for the rationale.
|
|
415
|
+
metrics.memoryInference.freshAttempts = Math.max(0, metrics.memoryInference.yieldEligibleConsidered - metrics.memoryInference.cacheHits);
|
|
416
|
+
metrics.memoryInference.yieldRate =
|
|
417
|
+
metrics.memoryInference.freshAttempts > 0
|
|
418
|
+
? roundRate(metrics.memoryInference.yieldEligibleWritten / metrics.memoryInference.freshAttempts)
|
|
419
|
+
: 0;
|
|
420
|
+
metrics.graphExtraction.ran =
|
|
421
|
+
metrics.graphExtraction.extractedFiles > 0 ||
|
|
422
|
+
metrics.graphExtraction.entities > 0 ||
|
|
423
|
+
metrics.graphExtraction.durationMs > 0;
|
|
424
|
+
const cacheTotal = metrics.graphExtraction.cacheHits + metrics.graphExtraction.cacheMisses;
|
|
425
|
+
metrics.graphExtraction.cacheHitRate = cacheTotal > 0 ? roundRate(metrics.graphExtraction.cacheHits / cacheTotal) : 0;
|
|
426
|
+
}
|
|
427
|
+
/**
|
|
428
|
+
* Merge per-row metrics from `src` into accumulator `dst`. All numeric fields
|
|
429
|
+
* are additive; cumulative rates are recomputed by finalizeImproveMetrics.
|
|
430
|
+
*/
|
|
431
|
+
function mergeImproveMetrics(dst, src) {
|
|
432
|
+
dst.plannedRefs += src.plannedRefs;
|
|
433
|
+
dst.profileFilteredRefs += src.profileFilteredRefs;
|
|
434
|
+
dst.actions.reflect.ok += src.actions.reflect.ok;
|
|
435
|
+
dst.actions.reflect.failed += src.actions.reflect.failed;
|
|
436
|
+
dst.actions.reflect.cooldown += src.actions.reflect.cooldown;
|
|
437
|
+
dst.actions.reflect.skipped += src.actions.reflect.skipped;
|
|
438
|
+
dst.actions.reflect.guardRejected += src.actions.reflect.guardRejected;
|
|
439
|
+
for (const [reason, count] of Object.entries(src.actions.reflect.skippedByReason)) {
|
|
440
|
+
dst.actions.reflect.skippedByReason[reason] = (dst.actions.reflect.skippedByReason[reason] ?? 0) + count;
|
|
441
|
+
}
|
|
442
|
+
dst.actions.distill.queued += src.actions.distill.queued;
|
|
443
|
+
dst.actions.distill.llmFailed += src.actions.distill.llmFailed;
|
|
444
|
+
dst.actions.distill.qualityRejected += src.actions.distill.qualityRejected;
|
|
445
|
+
dst.actions.distill.judgeRejected += src.actions.distill.judgeRejected;
|
|
446
|
+
dst.actions.distill.validatorRejected += src.actions.distill.validatorRejected;
|
|
447
|
+
dst.actions.distill.configDisabled += src.actions.distill.configDisabled;
|
|
448
|
+
dst.actions.distill.skipped += src.actions.distill.skipped;
|
|
449
|
+
for (const [reason, count] of Object.entries(src.actions.distill.skippedByReason)) {
|
|
450
|
+
dst.actions.distill.skippedByReason[reason] = (dst.actions.distill.skippedByReason[reason] ?? 0) + count;
|
|
451
|
+
}
|
|
452
|
+
dst.actions.distill.deferred += src.actions.distill.deferred;
|
|
453
|
+
for (const [reason, count] of Object.entries(src.actions.distill.deferredByReason)) {
|
|
454
|
+
dst.actions.distill.deferredByReason[reason] = (dst.actions.distill.deferredByReason[reason] ?? 0) + count;
|
|
455
|
+
}
|
|
456
|
+
dst.actions.memoryPrune += src.actions.memoryPrune;
|
|
457
|
+
dst.actions.memoryInference += src.actions.memoryInference;
|
|
458
|
+
dst.actions.graphExtraction += src.actions.graphExtraction;
|
|
459
|
+
dst.actions.error += src.actions.error;
|
|
460
|
+
dst.reflectsWithErrorContext += src.reflectsWithErrorContext;
|
|
461
|
+
dst.coverageGapCount += src.coverageGapCount;
|
|
462
|
+
dst.executionLogCandidateCount += src.executionLogCandidateCount;
|
|
463
|
+
dst.evalCasesWritten += src.evalCasesWritten;
|
|
464
|
+
dst.deadUrlCount += src.deadUrlCount;
|
|
465
|
+
dst.memorySummary.eligible += src.memorySummary.eligible;
|
|
466
|
+
dst.memorySummary.derived += src.memorySummary.derived;
|
|
467
|
+
dst.memoryCleanup.pruneCandidates += src.memoryCleanup.pruneCandidates;
|
|
468
|
+
dst.memoryCleanup.contradictionCandidates += src.memoryCleanup.contradictionCandidates;
|
|
469
|
+
dst.memoryCleanup.beliefStateTransitions += src.memoryCleanup.beliefStateTransitions;
|
|
470
|
+
dst.memoryCleanup.consolidationCandidates += src.memoryCleanup.consolidationCandidates;
|
|
471
|
+
dst.memoryCleanup.archived += src.memoryCleanup.archived;
|
|
472
|
+
dst.memoryCleanup.warnings += src.memoryCleanup.warnings;
|
|
473
|
+
dst.consolidation.processed += src.consolidation.processed;
|
|
474
|
+
dst.consolidation.promoted += src.consolidation.promoted;
|
|
475
|
+
dst.consolidation.merged += src.consolidation.merged;
|
|
476
|
+
dst.consolidation.deleted += src.consolidation.deleted;
|
|
477
|
+
dst.consolidation.contradicted += src.consolidation.contradicted;
|
|
478
|
+
dst.consolidation.failedChunks += src.consolidation.failedChunks;
|
|
479
|
+
dst.consolidation.totalChunks += src.consolidation.totalChunks;
|
|
480
|
+
dst.consolidation.durationMs += src.consolidation.durationMs;
|
|
481
|
+
dst.consolidation.judgedNoAction += src.consolidation.judgedNoAction;
|
|
482
|
+
dst.consolidation.mergedSecondaries += src.consolidation.mergedSecondaries;
|
|
483
|
+
dst.consolidation.failedChunkMemories += src.consolidation.failedChunkMemories;
|
|
484
|
+
for (const [reason, count] of Object.entries(src.consolidation.skipReasons)) {
|
|
485
|
+
dst.consolidation.skipReasons[reason] = (dst.consolidation.skipReasons[reason] ?? 0) + count;
|
|
486
|
+
}
|
|
487
|
+
dst.memoryInference.considered += src.memoryInference.considered;
|
|
488
|
+
dst.memoryInference.cacheHits += src.memoryInference.cacheHits;
|
|
489
|
+
dst.memoryInference.splitParents += src.memoryInference.splitParents;
|
|
490
|
+
dst.memoryInference.written += src.memoryInference.written;
|
|
491
|
+
dst.memoryInference.skippedNoFacts += src.memoryInference.skippedNoFacts;
|
|
492
|
+
dst.memoryInference.skippedChildExists += src.memoryInference.skippedChildExists;
|
|
493
|
+
dst.memoryInference.skippedAborted += src.memoryInference.skippedAborted;
|
|
494
|
+
dst.memoryInference.unaccounted += src.memoryInference.unaccounted;
|
|
495
|
+
dst.memoryInference.yieldEligibleRuns += src.memoryInference.yieldEligibleRuns;
|
|
496
|
+
dst.memoryInference.yieldEligibleConsidered += src.memoryInference.yieldEligibleConsidered;
|
|
497
|
+
dst.memoryInference.yieldEligibleWritten += src.memoryInference.yieldEligibleWritten;
|
|
498
|
+
dst.memoryInference.durationMs += src.memoryInference.durationMs;
|
|
499
|
+
dst.graphExtraction.extractedFiles += src.graphExtraction.extractedFiles;
|
|
500
|
+
dst.graphExtraction.entities += src.graphExtraction.entities;
|
|
501
|
+
dst.graphExtraction.relations += src.graphExtraction.relations;
|
|
502
|
+
dst.graphExtraction.cacheHits += src.graphExtraction.cacheHits;
|
|
503
|
+
dst.graphExtraction.cacheMisses += src.graphExtraction.cacheMisses;
|
|
504
|
+
dst.graphExtraction.truncations += src.graphExtraction.truncations;
|
|
505
|
+
dst.graphExtraction.failures += src.graphExtraction.failures;
|
|
506
|
+
dst.graphExtraction.durationMs += src.graphExtraction.durationMs;
|
|
507
|
+
}
|
|
508
|
+
function loadImproveRunRows(db, since, until) {
|
|
509
|
+
const sql = until
|
|
510
|
+
? "SELECT id, started_at, completed_at, ok, scope_mode, scope_value, result_json FROM improve_runs WHERE started_at >= ? AND started_at < ? AND dry_run = 0 ORDER BY started_at DESC"
|
|
511
|
+
: "SELECT id, started_at, completed_at, ok, scope_mode, scope_value, result_json FROM improve_runs WHERE started_at >= ? AND dry_run = 0 ORDER BY started_at DESC";
|
|
512
|
+
return (until ? db.prepare(sql).all(since, until) : db.prepare(sql).all(since));
|
|
513
|
+
}
|
|
514
|
+
function summarizeImproveRuns(db, since, until) {
|
|
515
|
+
const accum = createUnknownImproveMetrics();
|
|
516
|
+
const rows = loadImproveRunRows(db, since, until);
|
|
517
|
+
// Per-phase wall-time samples. Each entry is one envelope's durationMs for
|
|
518
|
+
// that phase. Phases that did not run on a given envelope are simply
|
|
519
|
+
// omitted (NOT counted as 0) so the median/p95 reflect actual phase work.
|
|
520
|
+
const phaseDurations = {
|
|
521
|
+
consolidation: [],
|
|
522
|
+
memoryInference: [],
|
|
523
|
+
graphExtraction: [],
|
|
524
|
+
};
|
|
525
|
+
for (const row of rows) {
|
|
526
|
+
let result;
|
|
527
|
+
try {
|
|
528
|
+
result = JSON.parse(row.result_json);
|
|
529
|
+
}
|
|
530
|
+
catch {
|
|
531
|
+
continue;
|
|
532
|
+
}
|
|
533
|
+
const perRow = projectRunMetrics(result);
|
|
534
|
+
mergeImproveMetrics(accum, perRow);
|
|
535
|
+
// Collect per-phase durations directly off the envelope. consolidation's
|
|
536
|
+
// duration lives inside the sub-object; memoryInference and graphExtraction
|
|
537
|
+
// expose top-level *DurationMs keys (`memoryInferenceDurationMs`,
|
|
538
|
+
// `graphExtractionDurationMs`) when they actually ran on that envelope.
|
|
539
|
+
const consol = result.consolidation;
|
|
540
|
+
const consolMs = toFiniteNumber(consol?.durationMs);
|
|
541
|
+
if (consolMs > 0)
|
|
542
|
+
phaseDurations.consolidation.push(consolMs);
|
|
543
|
+
const memMs = toFiniteNumber(result.memoryInferenceDurationMs);
|
|
544
|
+
if (memMs > 0)
|
|
545
|
+
phaseDurations.memoryInference.push(memMs);
|
|
546
|
+
const graphMs = toFiniteNumber(result.graphExtractionDurationMs);
|
|
547
|
+
if (graphMs > 0)
|
|
548
|
+
phaseDurations.graphExtraction.push(graphMs);
|
|
549
|
+
}
|
|
550
|
+
finalizeImproveMetrics(accum);
|
|
551
|
+
accum.wallTime.byPhase = {
|
|
552
|
+
consolidation: summarizePhaseDurations(phaseDurations.consolidation),
|
|
553
|
+
memoryInference: summarizePhaseDurations(phaseDurations.memoryInference),
|
|
554
|
+
graphExtraction: summarizePhaseDurations(phaseDurations.graphExtraction),
|
|
555
|
+
};
|
|
556
|
+
return { metrics: accum, runCount: rows.length };
|
|
557
|
+
}
|
|
558
|
+
/**
|
|
559
|
+
* Aggregate a list of per-envelope phase durations into the
|
|
560
|
+
* `wallTime.byPhase.*` shape: count, total, median, p95. Median/p95 use the
|
|
561
|
+
* same nearest-rank picker as the top-level wallTime stats so the two are
|
|
562
|
+
* comparable.
|
|
563
|
+
*/
|
|
564
|
+
function summarizePhaseDurations(samples) {
|
|
565
|
+
if (samples.length === 0)
|
|
566
|
+
return { count: 0, totalMs: 0, medianMs: 0, p95Ms: 0 };
|
|
567
|
+
const sorted = [...samples].sort((a, b) => a - b);
|
|
568
|
+
const pick = (q) => sorted[Math.min(sorted.length - 1, Math.floor(q * sorted.length))] ?? 0;
|
|
569
|
+
const totalMs = sorted.reduce((acc, n) => acc + n, 0);
|
|
570
|
+
return {
|
|
571
|
+
count: sorted.length,
|
|
572
|
+
totalMs,
|
|
573
|
+
medianMs: pick(0.5),
|
|
574
|
+
p95Ms: pick(0.95),
|
|
575
|
+
};
|
|
576
|
+
}
|
|
577
|
+
/**
|
|
578
|
+
* Project an improve_runs row + wall-time lookup into a single ImproveRunSummary.
|
|
579
|
+
* Used by `akm health --detail per-run`.
|
|
580
|
+
*/
|
|
581
|
+
function projectImproveRunSummary(row, wallTimeMs) {
|
|
582
|
+
let result = {};
|
|
583
|
+
try {
|
|
584
|
+
result = JSON.parse(row.result_json);
|
|
585
|
+
}
|
|
586
|
+
catch {
|
|
587
|
+
// fall through with empty result so per-stage rollups are zeros
|
|
588
|
+
}
|
|
589
|
+
const perRow = projectRunMetrics(result);
|
|
590
|
+
finalizeImproveMetrics(perRow);
|
|
591
|
+
const orphansPurged = toFiniteNumber(result.orphansPurged);
|
|
592
|
+
const lintSummary = result.lintSummary;
|
|
593
|
+
const lintFixed = lintSummary ? toFiniteNumber(lintSummary.fixed) : 0;
|
|
594
|
+
const lintFlagged = lintSummary ? toFiniteNumber(lintSummary.flagged) : 0;
|
|
595
|
+
return {
|
|
596
|
+
id: row.id,
|
|
597
|
+
startedAt: row.started_at,
|
|
598
|
+
completedAt: row.completed_at,
|
|
599
|
+
wallTimeMs,
|
|
600
|
+
ok: row.ok === 1,
|
|
601
|
+
scope: {
|
|
602
|
+
mode: row.scope_mode,
|
|
603
|
+
...(row.scope_value ? { value: row.scope_value } : {}),
|
|
604
|
+
},
|
|
605
|
+
actions: perRow.actions,
|
|
606
|
+
memorySummary: perRow.memorySummary,
|
|
607
|
+
memoryCleanup: perRow.memoryCleanup,
|
|
608
|
+
consolidation: perRow.consolidation,
|
|
609
|
+
memoryInference: perRow.memoryInference,
|
|
610
|
+
graphExtraction: perRow.graphExtraction,
|
|
611
|
+
reflectsWithErrorContext: perRow.reflectsWithErrorContext,
|
|
612
|
+
evalCasesWritten: perRow.evalCasesWritten,
|
|
613
|
+
orphansPurged,
|
|
614
|
+
lintFixed,
|
|
615
|
+
lintFlagged,
|
|
616
|
+
};
|
|
617
|
+
}
|
|
618
|
+
/**
|
|
619
|
+
* Load task_history intervals for `task_id='akm-improve'` in the window.
|
|
620
|
+
* Returned sorted by startMs ascending so containment lookups can use a
|
|
621
|
+
* linear scan (typical N is ~24/day; not worth a tree).
|
|
622
|
+
*
|
|
623
|
+
* The window filter is widened by 5 minutes on each side because the cron
|
|
624
|
+
* task wraps `akm improve` — the task `started_at` fires at e.g. :07:01
|
|
625
|
+
* while `recordImproveRun` writes the matching `improve_runs.started_at`
|
|
626
|
+
* later (after config load, planning, etc.), so the improve_runs row can
|
|
627
|
+
* be inside the window even when its enclosing task_history row started
|
|
628
|
+
* just before the window opened.
|
|
629
|
+
*/
|
|
630
|
+
function loadTaskIntervals(db, since, until) {
|
|
631
|
+
const sinceMs = new Date(since).getTime();
|
|
632
|
+
const untilMs = until ? new Date(until).getTime() : Number.POSITIVE_INFINITY;
|
|
633
|
+
const widenedSince = new Date(sinceMs - 5 * 60 * 1000).toISOString();
|
|
634
|
+
const widenedUntil = Number.isFinite(untilMs) ? new Date(untilMs + 5 * 60 * 1000).toISOString() : undefined;
|
|
635
|
+
const sql = widenedUntil
|
|
636
|
+
? "SELECT started_at, completed_at FROM task_history WHERE task_id = 'akm-improve' AND started_at >= ? AND started_at < ? AND completed_at IS NOT NULL ORDER BY started_at"
|
|
637
|
+
: "SELECT started_at, completed_at FROM task_history WHERE task_id = 'akm-improve' AND started_at >= ? AND completed_at IS NOT NULL ORDER BY started_at";
|
|
638
|
+
const rows = (widenedUntil ? db.prepare(sql).all(widenedSince, widenedUntil) : db.prepare(sql).all(widenedSince));
|
|
639
|
+
const intervals = [];
|
|
640
|
+
for (const row of rows) {
|
|
641
|
+
const startMs = new Date(row.started_at).getTime();
|
|
642
|
+
const endMs = new Date(row.completed_at).getTime();
|
|
643
|
+
if (!Number.isFinite(startMs) || !Number.isFinite(endMs) || endMs < startMs)
|
|
644
|
+
continue;
|
|
645
|
+
intervals.push({ startMs, endMs, durationMs: endMs - startMs });
|
|
646
|
+
}
|
|
647
|
+
return intervals;
|
|
648
|
+
}
|
|
649
|
+
/**
|
|
650
|
+
* Find the task_history interval that contains the given timestamp. The
|
|
651
|
+
* task wraps `akm improve`, so `improve_runs.started_at` (when
|
|
652
|
+
* `recordImproveRun` writes) always falls inside the enclosing task's
|
|
653
|
+
* [started_at, completed_at]. Returns undefined when no interval
|
|
654
|
+
* contains the timestamp (which happens for manually-invoked improve
|
|
655
|
+
* runs not driven by the `akm-improve` task).
|
|
656
|
+
*
|
|
657
|
+
* Linear scan because N is small. We tolerate a 1s slop on the upper
|
|
658
|
+
* bound to handle clock skew between the wrapper's `completed_at` write
|
|
659
|
+
* and recordImproveRun's `started_at` write.
|
|
660
|
+
*/
|
|
661
|
+
function findContainingTaskInterval(timestampMs, intervals) {
|
|
662
|
+
const SLOP_MS = 1000;
|
|
663
|
+
for (const interval of intervals) {
|
|
664
|
+
if (timestampMs >= interval.startMs && timestampMs <= interval.endMs + SLOP_MS) {
|
|
665
|
+
return interval;
|
|
666
|
+
}
|
|
667
|
+
}
|
|
668
|
+
return undefined;
|
|
669
|
+
}
|
|
670
|
+
function buildPerRunSummaries(db, since, until) {
|
|
671
|
+
const rows = loadImproveRunRows(db, since, until);
|
|
672
|
+
const taskIntervals = loadTaskIntervals(db, since, until);
|
|
673
|
+
const summaries = [];
|
|
674
|
+
for (const row of rows) {
|
|
675
|
+
const startMs = new Date(row.started_at).getTime();
|
|
676
|
+
const endMs = new Date(row.completed_at).getTime();
|
|
677
|
+
// Prefer the task_history interval (which has distinct start/end timestamps).
|
|
678
|
+
// Fall back to the improve_runs row's own delta (usually 0 because
|
|
679
|
+
// recordImproveRun writes started_at == completed_at == end-of-run timestamp).
|
|
680
|
+
const fallbackWallMs = Number.isFinite(startMs) && Number.isFinite(endMs) && endMs >= startMs ? endMs - startMs : 0;
|
|
681
|
+
const interval = Number.isFinite(startMs) ? findContainingTaskInterval(startMs, taskIntervals) : undefined;
|
|
682
|
+
const wallTimeMs = interval?.durationMs ?? fallbackWallMs;
|
|
683
|
+
summaries.push(projectImproveRunSummary(row, wallTimeMs));
|
|
684
|
+
}
|
|
685
|
+
return summaries;
|
|
686
|
+
}
|
|
687
|
+
function emptyPhaseStats() {
|
|
688
|
+
return {
|
|
689
|
+
consolidation: { count: 0, totalMs: 0, medianMs: 0, p95Ms: 0 },
|
|
690
|
+
memoryInference: { count: 0, totalMs: 0, medianMs: 0, p95Ms: 0 },
|
|
691
|
+
graphExtraction: { count: 0, totalMs: 0, medianMs: 0, p95Ms: 0 },
|
|
692
|
+
};
|
|
693
|
+
}
|
|
694
|
+
function computeWallTimeStats(durationsMs, byPhase) {
|
|
695
|
+
const phase = byPhase ?? emptyPhaseStats();
|
|
696
|
+
if (durationsMs.length === 0)
|
|
697
|
+
return { count: 0, medianMs: 0, p95Ms: 0, minMs: 0, maxMs: 0, byPhase: phase };
|
|
698
|
+
const sorted = [...durationsMs].sort((a, b) => a - b);
|
|
699
|
+
const pick = (q) => sorted[Math.min(sorted.length - 1, Math.floor(q * sorted.length))] ?? 0;
|
|
700
|
+
return {
|
|
701
|
+
count: sorted.length,
|
|
702
|
+
medianMs: pick(0.5),
|
|
703
|
+
p95Ms: pick(0.95),
|
|
704
|
+
minMs: sorted[0] ?? 0,
|
|
705
|
+
maxMs: sorted[sorted.length - 1] ?? 0,
|
|
706
|
+
byPhase: phase,
|
|
707
|
+
};
|
|
708
|
+
}
|
|
130
709
|
function buildImproveSkipSummary(events) {
|
|
131
710
|
const skipReasons = {};
|
|
132
711
|
for (const event of events) {
|
|
@@ -148,7 +727,31 @@ function probeStateDbRoundTrip(stateDbPath) {
|
|
|
148
727
|
}
|
|
149
728
|
function runAgentProbe() {
|
|
150
729
|
const config = loadConfig();
|
|
151
|
-
|
|
730
|
+
// v2: check profiles.agent first
|
|
731
|
+
if (config.profiles?.agent) {
|
|
732
|
+
const defaultName = config.defaults?.agent;
|
|
733
|
+
const profileCount = Object.keys(config.profiles.agent).length;
|
|
734
|
+
if (profileCount === 0) {
|
|
735
|
+
return {
|
|
736
|
+
name: "agent-profile",
|
|
737
|
+
kind: "deterministic",
|
|
738
|
+
status: "unknown",
|
|
739
|
+
confidence: "high",
|
|
740
|
+
message: "No agent profiles configured in profiles.agent.",
|
|
741
|
+
};
|
|
742
|
+
}
|
|
743
|
+
const profileName = defaultName ?? Object.keys(config.profiles.agent)[0];
|
|
744
|
+
const profile = config.profiles.agent[profileName];
|
|
745
|
+
return {
|
|
746
|
+
name: "agent-profile",
|
|
747
|
+
kind: "deterministic",
|
|
748
|
+
status: "pass",
|
|
749
|
+
confidence: "high",
|
|
750
|
+
message: `v2 agent profile "${profileName}" configured (platform: ${profile?.platform ?? "unknown"}).`,
|
|
751
|
+
evidence: { profile: profileName, platform: profile?.platform, profileCount },
|
|
752
|
+
};
|
|
753
|
+
}
|
|
754
|
+
if (!config.profiles?.agent && !config.defaults?.agent) {
|
|
152
755
|
return {
|
|
153
756
|
name: "agent-profile",
|
|
154
757
|
kind: "deterministic",
|
|
@@ -159,7 +762,7 @@ function runAgentProbe() {
|
|
|
159
762
|
}
|
|
160
763
|
let profile;
|
|
161
764
|
try {
|
|
162
|
-
profile = requireAgentProfile(config
|
|
765
|
+
profile = requireAgentProfile(config);
|
|
163
766
|
}
|
|
164
767
|
catch (error) {
|
|
165
768
|
return {
|
|
@@ -182,7 +785,7 @@ function runAgentProbe() {
|
|
|
182
785
|
evidence: { profile: profile.name, sdkMode: true, model: profile.model ?? null },
|
|
183
786
|
};
|
|
184
787
|
}
|
|
185
|
-
const detections = detectAgentCliProfiles(config
|
|
788
|
+
const detections = detectAgentCliProfiles(config);
|
|
186
789
|
const detection = detections.find((entry) => entry.name === profile.name);
|
|
187
790
|
if (!detection?.available) {
|
|
188
791
|
return {
|
|
@@ -219,7 +822,173 @@ function runAgentProbe() {
|
|
|
219
822
|
evidence: { profile: profile.name, bin: profile.bin, version: (version.stdout ?? "").trim() },
|
|
220
823
|
};
|
|
221
824
|
}
|
|
825
|
+
/**
|
|
826
|
+
* Parse a `--window-compare <duration>` shorthand into two adjacent windows
|
|
827
|
+
* (current, prior). Duration syntax matches {@link parseHealthSince}.
|
|
828
|
+
*/
|
|
829
|
+
function resolveWindowCompare(duration) {
|
|
830
|
+
const trimmed = duration.trim();
|
|
831
|
+
const durationMatch = trimmed.match(/^(\d+)([dhm])$/i);
|
|
832
|
+
if (!durationMatch) {
|
|
833
|
+
throw new UsageError("--window-compare must be a duration like '24h', '7d', or '30m'.", "INVALID_FLAG_VALUE");
|
|
834
|
+
}
|
|
835
|
+
const amount = Number.parseInt(durationMatch[1] ?? "0", 10);
|
|
836
|
+
const unit = (durationMatch[2] ?? "h").toLowerCase();
|
|
837
|
+
if (!Number.isFinite(amount) || amount <= 0) {
|
|
838
|
+
throw new UsageError("--window-compare must be a positive duration.", "INVALID_FLAG_VALUE");
|
|
839
|
+
}
|
|
840
|
+
const multiplier = unit === "h" ? 60 * 60 * 1000 : unit === "m" ? 60 * 1000 : 24 * 60 * 60 * 1000;
|
|
841
|
+
const ms = amount * multiplier;
|
|
842
|
+
const now = Date.now();
|
|
843
|
+
const currentSince = new Date(now - ms).toISOString();
|
|
844
|
+
const currentUntil = new Date(now).toISOString();
|
|
845
|
+
const priorSince = new Date(now - 2 * ms).toISOString();
|
|
846
|
+
const priorUntil = currentSince;
|
|
847
|
+
return [
|
|
848
|
+
{ name: "current", since: currentSince, until: currentUntil },
|
|
849
|
+
{ name: "prior", since: priorSince, until: priorUntil },
|
|
850
|
+
];
|
|
851
|
+
}
|
|
852
|
+
/**
|
|
853
|
+
* Parse a single repeatable `--windows` value of the form
|
|
854
|
+
* `name=...,since=...,until=...`. All keys are optional EXCEPT name and since.
|
|
855
|
+
*/
|
|
856
|
+
export function parseWindowSpec(raw) {
|
|
857
|
+
const fields = {};
|
|
858
|
+
for (const part of raw.split(",")) {
|
|
859
|
+
const trimmed = part.trim();
|
|
860
|
+
if (!trimmed)
|
|
861
|
+
continue;
|
|
862
|
+
const eq = trimmed.indexOf("=");
|
|
863
|
+
if (eq < 0) {
|
|
864
|
+
throw new UsageError(`--windows entry must be a comma-separated list of key=value pairs: ${raw}`, "INVALID_FLAG_VALUE");
|
|
865
|
+
}
|
|
866
|
+
const key = trimmed.slice(0, eq).trim();
|
|
867
|
+
const value = trimmed.slice(eq + 1).trim();
|
|
868
|
+
fields[key] = value;
|
|
869
|
+
}
|
|
870
|
+
if (!fields.name) {
|
|
871
|
+
throw new UsageError(`--windows entry is missing required 'name': ${raw}`, "INVALID_FLAG_VALUE");
|
|
872
|
+
}
|
|
873
|
+
if (!fields.since) {
|
|
874
|
+
throw new UsageError(`--windows entry is missing required 'since': ${raw}`, "INVALID_FLAG_VALUE");
|
|
875
|
+
}
|
|
876
|
+
return {
|
|
877
|
+
name: fields.name,
|
|
878
|
+
since: fields.since,
|
|
879
|
+
...(fields.until ? { until: fields.until } : {}),
|
|
880
|
+
};
|
|
881
|
+
}
|
|
882
|
+
/** Hard-coded list of "interesting" metric paths for window-compare deltas. */
|
|
883
|
+
const INTERESTING_DELTA_PATHS = [
|
|
884
|
+
"improve.actions.reflect.failed",
|
|
885
|
+
"improve.actions.reflect.guardRejected",
|
|
886
|
+
"improve.actions.distill.llmFailed",
|
|
887
|
+
"improve.actions.distill.queued",
|
|
888
|
+
"improve.actions.distill.deferred",
|
|
889
|
+
"improve.consolidation.promoted",
|
|
890
|
+
"improve.memoryInference.written",
|
|
891
|
+
"improve.memoryInference.yieldRate",
|
|
892
|
+
"improve.memoryInference.skippedNoFacts",
|
|
893
|
+
"improve.graphExtraction.cacheHitRate",
|
|
894
|
+
"improve.graphExtraction.failures",
|
|
895
|
+
"improve.wallTime.medianMs",
|
|
896
|
+
"improve.wallTime.p95Ms",
|
|
897
|
+
];
|
|
898
|
+
function readNumericPath(obj, path) {
|
|
899
|
+
const parts = path.split(".");
|
|
900
|
+
let cursor = obj;
|
|
901
|
+
for (const part of parts) {
|
|
902
|
+
if (typeof cursor !== "object" || cursor === null)
|
|
903
|
+
return 0;
|
|
904
|
+
cursor = cursor[part];
|
|
905
|
+
}
|
|
906
|
+
return typeof cursor === "number" && Number.isFinite(cursor) ? cursor : 0;
|
|
907
|
+
}
|
|
908
|
+
function computeDeltas(first, last) {
|
|
909
|
+
const out = {};
|
|
910
|
+
for (const path of INTERESTING_DELTA_PATHS) {
|
|
911
|
+
const from = readNumericPath(first, path);
|
|
912
|
+
const to = readNumericPath(last, path);
|
|
913
|
+
if (from === 0 && to === 0)
|
|
914
|
+
continue;
|
|
915
|
+
let pctChange;
|
|
916
|
+
if (from === 0) {
|
|
917
|
+
pctChange = to === 0 ? 0 : "+inf";
|
|
918
|
+
}
|
|
919
|
+
else {
|
|
920
|
+
pctChange = Number((((to - from) / from) * 100).toFixed(2));
|
|
921
|
+
}
|
|
922
|
+
out[path] = { from, to, pctChange };
|
|
923
|
+
}
|
|
924
|
+
return out;
|
|
925
|
+
}
|
|
926
|
+
function buildWindowMetrics(db, stateDbPath, since, until) {
|
|
927
|
+
const taskRows = queryTaskHistory(db, { since }).filter((row) => {
|
|
928
|
+
const startMs = new Date(row.started_at).getTime();
|
|
929
|
+
const untilMs = new Date(until).getTime();
|
|
930
|
+
return !Number.isFinite(untilMs) || startMs < untilMs;
|
|
931
|
+
});
|
|
932
|
+
const taskRowsWithLogs = taskRows.filter((row) => row.log_path !== null);
|
|
933
|
+
const existingLogRows = taskRowsWithLogs.filter((row) => row.log_path && fs.existsSync(row.log_path));
|
|
934
|
+
const failedTaskRows = taskRows.filter((row) => row.status === "failed");
|
|
935
|
+
const activeRows = taskRows.filter((row) => row.status === "active");
|
|
936
|
+
const stuckActiveRuns = activeRows.filter((row) => Date.now() - new Date(row.started_at).getTime() > ACTIVE_RUN_WARN_MS).length;
|
|
937
|
+
const promptRows = taskRows.filter((row) => row.target_kind === "prompt");
|
|
938
|
+
const promptFailures = promptRows.filter((row) => {
|
|
939
|
+
const detail = parseTaskMetadata(row).detail;
|
|
940
|
+
return typeof detail?.reason === "string" && detail.reason.length > 0;
|
|
941
|
+
});
|
|
942
|
+
const logBackingRate = taskRowsWithLogs.length === 0 ? 1 : existingLogRows.length / taskRowsWithLogs.length;
|
|
943
|
+
const taskFailRate = taskRows.length === 0 ? 0 : failedTaskRows.length / taskRows.length;
|
|
944
|
+
const agentFailureRate = promptRows.length === 0 ? 0 : promptFailures.length / promptRows.length;
|
|
945
|
+
const improveInvoked = readEvents({ since, type: "improve_invoked" }, { dbPath: stateDbPath }).events.filter((event) => new Date(event.ts ?? since).getTime() < new Date(until).getTime()).length;
|
|
946
|
+
const improveCompletedEvents = readEvents({ since, type: IMPROVE_COMPLETED_EVENT }, { dbPath: stateDbPath }).events.filter((event) => new Date(event.ts ?? since).getTime() < new Date(until).getTime());
|
|
947
|
+
const improveSkippedEvents = readEvents({ since, type: "improve_skipped" }, { dbPath: stateDbPath }).events.filter((event) => new Date(event.ts ?? since).getTime() < new Date(until).getTime());
|
|
948
|
+
const eventsMetrics = summarizeImproveCompleted(improveCompletedEvents);
|
|
949
|
+
const { metrics: improveSummary, runCount } = summarizeImproveRuns(db, since, until);
|
|
950
|
+
improveSummary.invoked = improveInvoked;
|
|
951
|
+
improveSummary.completed = eventsMetrics.completed;
|
|
952
|
+
const skipSummary = buildImproveSkipSummary(improveSkippedEvents);
|
|
953
|
+
improveSummary.skipped = skipSummary.skipped;
|
|
954
|
+
improveSummary.skipReasons = skipSummary.skipReasons;
|
|
955
|
+
// Preserve the per-phase aggregation computed by summarizeImproveRuns and
|
|
956
|
+
// derive top-level wall times from the same improve-runs window so counts
|
|
957
|
+
// and percentiles stay aligned with per-run reporting.
|
|
958
|
+
const perRunSummaries = buildPerRunSummaries(db, since, until);
|
|
959
|
+
const wallTimes = perRunSummaries.map((run) => run.wallTimeMs).filter((ms) => Number.isFinite(ms) && ms > 0);
|
|
960
|
+
improveSummary.wallTime = computeWallTimeStats(wallTimes, improveSummary.wallTime.byPhase);
|
|
961
|
+
const metrics = {
|
|
962
|
+
taskFailRate: roundRate(taskFailRate),
|
|
963
|
+
agentFailureRate: roundRate(agentFailureRate),
|
|
964
|
+
stuckActiveRuns,
|
|
965
|
+
logBackingRate: roundRate(logBackingRate),
|
|
966
|
+
probeRoundTripMs: null,
|
|
967
|
+
};
|
|
968
|
+
return { improve: improveSummary, metrics, runs: runCount };
|
|
969
|
+
}
|
|
970
|
+
function validateAkmHealthOptions(options) {
|
|
971
|
+
if (options.groupBy !== undefined && options.groupBy !== "run") {
|
|
972
|
+
throw new UsageError(`Invalid value for --group-by: ${options.groupBy}. Expected: run`, "INVALID_FLAG_VALUE");
|
|
973
|
+
}
|
|
974
|
+
if (options.windowCompare !== undefined && options.windows !== undefined && options.windows.length > 0) {
|
|
975
|
+
throw new UsageError("--window-compare and --windows are mutually exclusive.", "INVALID_FLAG_VALUE");
|
|
976
|
+
}
|
|
977
|
+
if (options.windows) {
|
|
978
|
+
if (options.windows.length > 4) {
|
|
979
|
+
throw new UsageError("--windows accepts at most 4 entries.", "INVALID_FLAG_VALUE");
|
|
980
|
+
}
|
|
981
|
+
const seen = new Set();
|
|
982
|
+
for (const spec of options.windows) {
|
|
983
|
+
if (seen.has(spec.name)) {
|
|
984
|
+
throw new UsageError(`--windows has duplicate name: ${spec.name}`, "INVALID_FLAG_VALUE");
|
|
985
|
+
}
|
|
986
|
+
seen.add(spec.name);
|
|
987
|
+
}
|
|
988
|
+
}
|
|
989
|
+
}
|
|
222
990
|
export function akmHealth(options = {}) {
|
|
991
|
+
validateAkmHealthOptions(options);
|
|
223
992
|
const since = parseHealthSince(options.since);
|
|
224
993
|
const stateDbPath = getStateDbPathInDataDir();
|
|
225
994
|
const hardChecks = [];
|
|
@@ -320,11 +1089,16 @@ export function akmHealth(options = {}) {
|
|
|
320
1089
|
const improveInvoked = readEvents({ since, type: "improve_invoked" }, { dbPath: stateDbPath }).events.length;
|
|
321
1090
|
const improveCompletedEvents = readEvents({ since, type: IMPROVE_COMPLETED_EVENT }, { dbPath: stateDbPath }).events;
|
|
322
1091
|
const improveSkippedEvents = readEvents({ since, type: "improve_skipped" }, { dbPath: stateDbPath }).events;
|
|
323
|
-
const
|
|
1092
|
+
const eventsMetrics = summarizeImproveCompleted(improveCompletedEvents);
|
|
1093
|
+
const { metrics: improveSummary } = summarizeImproveRuns(db, since);
|
|
324
1094
|
improveSummary.invoked = improveInvoked;
|
|
1095
|
+
improveSummary.completed = eventsMetrics.completed;
|
|
325
1096
|
const skipSummary = buildImproveSkipSummary(improveSkippedEvents);
|
|
326
1097
|
improveSummary.skipped = skipSummary.skipped;
|
|
327
1098
|
improveSummary.skipReasons = skipSummary.skipReasons;
|
|
1099
|
+
const perRunSummaries = buildPerRunSummaries(db, since);
|
|
1100
|
+
const wallTimes = perRunSummaries.map((run) => run.wallTimeMs).filter((ms) => Number.isFinite(ms) && ms > 0);
|
|
1101
|
+
improveSummary.wallTime = computeWallTimeStats(wallTimes, improveSummary.wallTime.byPhase);
|
|
328
1102
|
let sessionLogEntries = [];
|
|
329
1103
|
try {
|
|
330
1104
|
const sinceDays = Math.max(0, Math.ceil((Date.now() - new Date(since).getTime()) / (24 * 60 * 60 * 1000)));
|
|
@@ -358,19 +1132,171 @@ export function akmHealth(options = {}) {
|
|
|
358
1132
|
const hardFailure = hardChecks.some((check) => check.status === "fail");
|
|
359
1133
|
const deterministicWarnings = [...hardChecks, ...advisories].some((check) => check.status === "warn" && check.kind === "deterministic");
|
|
360
1134
|
const status = hardFailure ? "fail" : deterministicWarnings ? "warn" : "pass";
|
|
1135
|
+
// ── Window-compare mode (Phase 3) ─────────────────────────────────────
|
|
1136
|
+
let windowSpecs;
|
|
1137
|
+
if (options.windowCompare) {
|
|
1138
|
+
windowSpecs = resolveWindowCompare(options.windowCompare);
|
|
1139
|
+
}
|
|
1140
|
+
else if (options.windows && options.windows.length > 0) {
|
|
1141
|
+
windowSpecs = options.windows;
|
|
1142
|
+
}
|
|
1143
|
+
let windowResults;
|
|
1144
|
+
let deltas;
|
|
1145
|
+
let topLevelImprove = improveSummary;
|
|
1146
|
+
let topLevelMetrics = metrics;
|
|
1147
|
+
let topLevelSince = since;
|
|
1148
|
+
if (windowSpecs && db) {
|
|
1149
|
+
windowResults = windowSpecs.map((spec) => {
|
|
1150
|
+
const winSince = parseHealthSince(spec.since);
|
|
1151
|
+
const winUntil = spec.until ? parseHealthSince(spec.until) : new Date().toISOString();
|
|
1152
|
+
const bundle = buildWindowMetrics(db, stateDbPath, winSince, winUntil);
|
|
1153
|
+
return {
|
|
1154
|
+
name: spec.name,
|
|
1155
|
+
since: winSince,
|
|
1156
|
+
until: winUntil,
|
|
1157
|
+
runs: bundle.runs,
|
|
1158
|
+
improve: bundle.improve,
|
|
1159
|
+
metrics: bundle.metrics,
|
|
1160
|
+
};
|
|
1161
|
+
});
|
|
1162
|
+
// Preserve backward compat: top-level improve/metrics reflect window 0.
|
|
1163
|
+
if (windowResults.length > 0) {
|
|
1164
|
+
topLevelImprove = windowResults[0].improve;
|
|
1165
|
+
topLevelMetrics = { ...windowResults[0].metrics, probeRoundTripMs: probe.durationMs };
|
|
1166
|
+
topLevelSince = windowResults[0].since;
|
|
1167
|
+
}
|
|
1168
|
+
if (windowResults.length >= 2) {
|
|
1169
|
+
// Deltas always read chronologically: `from` = earliest window,
|
|
1170
|
+
// `to` = latest. Positive pctChange on a failure metric (e.g.
|
|
1171
|
+
// distill.llmFailed) means things got WORSE going forward in
|
|
1172
|
+
// time; negative means improvement. Window 0 in the output
|
|
1173
|
+
// array is whatever the user specified first (typically
|
|
1174
|
+
// `current` for --window-compare), but the delta direction is
|
|
1175
|
+
// independent of that array order.
|
|
1176
|
+
const sorted = [...windowResults].sort((a, b) => new Date(a.since).getTime() - new Date(b.since).getTime());
|
|
1177
|
+
deltas = computeDeltas(sorted[0], sorted[sorted.length - 1]);
|
|
1178
|
+
}
|
|
1179
|
+
}
|
|
1180
|
+
// ── Per-run mode (Phase 2) ────────────────────────────────────────────
|
|
1181
|
+
let runs;
|
|
1182
|
+
if (options.groupBy === "run") {
|
|
1183
|
+
runs = buildPerRunSummaries(db, since);
|
|
1184
|
+
}
|
|
361
1185
|
return {
|
|
362
|
-
schemaVersion:
|
|
1186
|
+
schemaVersion: 2,
|
|
363
1187
|
ok: !hardFailure,
|
|
364
1188
|
status,
|
|
365
|
-
since,
|
|
1189
|
+
since: topLevelSince,
|
|
366
1190
|
hardChecks,
|
|
367
1191
|
advisories,
|
|
368
|
-
metrics,
|
|
369
|
-
improve:
|
|
1192
|
+
metrics: topLevelMetrics,
|
|
1193
|
+
improve: topLevelImprove,
|
|
370
1194
|
sessionLogAdvisories: sessionLogEntries,
|
|
1195
|
+
...(runs ? { runs } : {}),
|
|
1196
|
+
...(windowResults ? { windows: windowResults } : {}),
|
|
1197
|
+
...(deltas ? { deltas } : {}),
|
|
371
1198
|
};
|
|
372
1199
|
}
|
|
373
1200
|
finally {
|
|
374
1201
|
db.close();
|
|
375
1202
|
}
|
|
376
1203
|
}
|
|
1204
|
+
// ── Markdown renderers ───────────────────────────────────────────────────────
|
|
1205
|
+
function padRight(s, width) {
|
|
1206
|
+
return s.length >= width ? s : s + " ".repeat(width - s.length);
|
|
1207
|
+
}
|
|
1208
|
+
function renderTable(headers, rows) {
|
|
1209
|
+
const widths = headers.map((h, i) => Math.max(h.length, ...rows.map((r) => (r[i] ?? "").length)));
|
|
1210
|
+
const lines = [];
|
|
1211
|
+
lines.push(headers.map((h, i) => padRight(h, widths[i] ?? 0)).join(" "));
|
|
1212
|
+
for (const row of rows) {
|
|
1213
|
+
lines.push(row.map((cell, i) => padRight(cell ?? "", widths[i] ?? 0)).join(" "));
|
|
1214
|
+
}
|
|
1215
|
+
return lines.join("\n");
|
|
1216
|
+
}
|
|
1217
|
+
/**
|
|
1218
|
+
* Render `--detail per-run` rows as a TSV-ish aligned table. The column
|
|
1219
|
+
* shape was originally inherited from the retired
|
|
1220
|
+
* `scripts/improve-stats/runs-detail` bash helper; keep the same shape
|
|
1221
|
+
* so operator muscle memory carries over.
|
|
1222
|
+
*
|
|
1223
|
+
* Columns: ts | ok | actions | refl_ok/fail/cd/skip |
|
|
1224
|
+
* distill_q/llm-fail/qrej/cfg/skip | cons_proc/promo/merge/del |
|
|
1225
|
+
* mem_cons/written/skip | graph_f/e/r | orphans | lint_f/fl
|
|
1226
|
+
*/
|
|
1227
|
+
export function renderRunsDetailMd(runs) {
|
|
1228
|
+
const headers = [
|
|
1229
|
+
"ts",
|
|
1230
|
+
"ok",
|
|
1231
|
+
"actions",
|
|
1232
|
+
"refl_ok/fail/cd/skip",
|
|
1233
|
+
"distill_q/llm-fail/qrej/cfg/skip",
|
|
1234
|
+
"cons_proc/promo/merge/del",
|
|
1235
|
+
"mem_cons/written/skip",
|
|
1236
|
+
"graph_f/e/r",
|
|
1237
|
+
"orphans",
|
|
1238
|
+
"lint_f/fl",
|
|
1239
|
+
];
|
|
1240
|
+
const rows = runs.map((r) => {
|
|
1241
|
+
const totalActions = r.actions.reflect.ok +
|
|
1242
|
+
r.actions.reflect.failed +
|
|
1243
|
+
r.actions.reflect.cooldown +
|
|
1244
|
+
r.actions.reflect.skipped +
|
|
1245
|
+
r.actions.distill.queued +
|
|
1246
|
+
r.actions.distill.llmFailed +
|
|
1247
|
+
r.actions.distill.qualityRejected +
|
|
1248
|
+
r.actions.distill.configDisabled +
|
|
1249
|
+
r.actions.distill.skipped +
|
|
1250
|
+
r.actions.memoryPrune +
|
|
1251
|
+
r.actions.memoryInference +
|
|
1252
|
+
r.actions.graphExtraction +
|
|
1253
|
+
r.actions.error;
|
|
1254
|
+
return [
|
|
1255
|
+
r.startedAt,
|
|
1256
|
+
String(r.ok),
|
|
1257
|
+
String(totalActions),
|
|
1258
|
+
`${r.actions.reflect.ok}/${r.actions.reflect.failed}/${r.actions.reflect.cooldown}/${r.actions.reflect.skipped}`,
|
|
1259
|
+
`${r.actions.distill.queued}/${r.actions.distill.llmFailed}/${r.actions.distill.qualityRejected}/${r.actions.distill.configDisabled}/${r.actions.distill.skipped}`,
|
|
1260
|
+
`${r.consolidation.processed}/${r.consolidation.promoted}/${r.consolidation.merged}/${r.consolidation.deleted}`,
|
|
1261
|
+
`${r.memoryInference.considered}/${r.memoryInference.written}/${r.memoryInference.skippedNoFacts}`,
|
|
1262
|
+
`${r.graphExtraction.extractedFiles}/${r.graphExtraction.entities}/${r.graphExtraction.relations}`,
|
|
1263
|
+
String(r.orphansPurged),
|
|
1264
|
+
`${r.lintFixed}/${r.lintFlagged}`,
|
|
1265
|
+
];
|
|
1266
|
+
});
|
|
1267
|
+
return renderTable(headers, rows);
|
|
1268
|
+
}
|
|
1269
|
+
/**
|
|
1270
|
+
* Render a window-compare comparison as a side-by-side metric table with a
|
|
1271
|
+
* delta column. Bad-direction deltas (e.g. +pct on failed counts) get a `!`
|
|
1272
|
+
* marker prefix.
|
|
1273
|
+
*/
|
|
1274
|
+
export function renderWindowCompareMd(windows, deltas) {
|
|
1275
|
+
if (windows.length === 0)
|
|
1276
|
+
return "";
|
|
1277
|
+
const headers = ["metric", ...windows.map((w) => w.name), "delta"];
|
|
1278
|
+
const badIfPositive = new Set([
|
|
1279
|
+
"improve.actions.reflect.failed",
|
|
1280
|
+
"improve.actions.distill.llmFailed",
|
|
1281
|
+
"improve.graphExtraction.failures",
|
|
1282
|
+
"improve.wallTime.medianMs",
|
|
1283
|
+
"improve.wallTime.p95Ms",
|
|
1284
|
+
"improve.memoryInference.skippedNoFacts",
|
|
1285
|
+
]);
|
|
1286
|
+
const rows = [];
|
|
1287
|
+
for (const path of INTERESTING_DELTA_PATHS) {
|
|
1288
|
+
const values = windows.map((w) => String(readNumericPath(w, path)));
|
|
1289
|
+
const delta = deltas?.[path];
|
|
1290
|
+
let deltaStr = "—";
|
|
1291
|
+
if (delta) {
|
|
1292
|
+
const pct = delta.pctChange;
|
|
1293
|
+
const num = typeof pct === "number" ? pct : pct;
|
|
1294
|
+
const sign = typeof num === "number" && num > 0 ? "+" : "";
|
|
1295
|
+
const formatted = typeof num === "number" ? `${sign}${num}%` : String(num);
|
|
1296
|
+
const marker = badIfPositive.has(path) && typeof num === "number" && num > 0 ? "!" : "";
|
|
1297
|
+
deltaStr = marker + formatted;
|
|
1298
|
+
}
|
|
1299
|
+
rows.push([path, ...values, deltaStr]);
|
|
1300
|
+
}
|
|
1301
|
+
return renderTable(headers, rows);
|
|
1302
|
+
}
|