akm-cli 0.9.0-beta.11 → 0.9.0-beta.26
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +163 -0
- package/dist/assets/prompts/consolidate-system.md +23 -0
- package/dist/assets/prompts/contradiction-judge.md +33 -0
- package/dist/assets/prompts/distill-knowledge-system.md +22 -0
- package/dist/assets/prompts/distill-lesson-system.md +36 -0
- package/dist/assets/prompts/extract-session.md +5 -1
- package/dist/assets/prompts/graph-extract-system.md +1 -0
- package/dist/assets/prompts/memory-infer-system.md +1 -0
- package/dist/assets/prompts/memory-infer-user.md +5 -0
- package/dist/assets/prompts/metadata-enhance-system.md +1 -0
- package/dist/assets/prompts/procedural-system.md +44 -0
- package/dist/assets/prompts/recombine-system.md +40 -0
- package/dist/assets/prompts/staleness-detect-system.md +6 -0
- package/dist/assets/prompts/validate-summary-judge.md +1 -0
- package/dist/assets/templates/html/health.html +25 -27
- package/dist/cli.js +2 -2
- package/dist/commands/agent/contribute-cli.js +16 -3
- package/dist/commands/feedback-cli.js +48 -44
- package/dist/commands/health/html-report.js +140 -16
- package/dist/commands/health.js +277 -1
- package/dist/commands/improve/calibration.js +161 -0
- package/dist/commands/improve/consolidate.js +595 -105
- package/dist/commands/improve/dedup.js +482 -0
- package/dist/commands/improve/distill.js +119 -64
- package/dist/commands/improve/encoding-salience.js +205 -0
- package/dist/commands/improve/extract-cli.js +115 -1
- package/dist/commands/improve/extract-prompt.js +32 -1
- package/dist/commands/improve/extract-watch.js +140 -0
- package/dist/commands/improve/extract.js +210 -30
- package/dist/commands/improve/feedback-valence.js +54 -0
- package/dist/commands/improve/homeostatic.js +467 -0
- package/dist/commands/improve/improve-auto-accept.js +80 -7
- package/dist/commands/improve/improve-profiles.js +8 -0
- package/dist/commands/improve/improve.js +991 -61
- package/dist/commands/improve/memory/memory-contradiction-detect.js +23 -28
- package/dist/commands/improve/outcome-loop.js +256 -0
- package/dist/commands/improve/proactive-maintenance.js +9 -35
- package/dist/commands/improve/procedural.js +409 -0
- package/dist/commands/improve/recombine.js +488 -0
- package/dist/commands/improve/reflect.js +20 -1
- package/dist/commands/improve/related-sessions.js +120 -0
- package/dist/commands/improve/salience.js +386 -0
- package/dist/commands/improve/triage.js +95 -0
- package/dist/commands/lint/agent-linter.js +19 -24
- package/dist/commands/lint/base-linter.js +173 -60
- package/dist/commands/lint/command-linter.js +19 -24
- package/dist/commands/lint/env-key-rules.js +34 -1
- package/dist/commands/lint/index.js +30 -13
- package/dist/commands/lint/memory-linter.js +1 -1
- package/dist/commands/lint/registry.js +5 -2
- package/dist/commands/lint/task-linter.js +3 -3
- package/dist/commands/lint/workflow-linter.js +26 -1
- package/dist/commands/proposal/validators/proposals.js +4 -0
- package/dist/commands/read/curate.js +284 -86
- package/dist/commands/read/search-cli.js +7 -0
- package/dist/commands/read/search.js +1 -0
- package/dist/commands/sources/installed-stashes.js +5 -1
- package/dist/core/asset/frontmatter.js +166 -167
- package/dist/core/asset/markdown.js +8 -0
- package/dist/core/config/config-schema.js +211 -3
- package/dist/core/config/config.js +2 -2
- package/dist/core/logs-db.js +4 -3
- package/dist/core/state-db.js +555 -29
- package/dist/indexer/db/db.js +250 -27
- package/dist/indexer/db/graph-db.js +81 -86
- package/dist/indexer/graph/graph-boost.js +51 -41
- package/dist/indexer/passes/memory-inference.js +10 -3
- package/dist/indexer/passes/staleness-detect.js +2 -5
- package/dist/indexer/search/db-search.js +15 -4
- package/dist/indexer/search/ranking.js +4 -0
- package/dist/integrations/harnesses/claude/session-log.js +10 -0
- package/dist/integrations/harnesses/opencode/session-log.js +9 -0
- package/dist/integrations/session-logs/index.js +16 -0
- package/dist/llm/embedder.js +27 -3
- package/dist/llm/embedders/local.js +66 -2
- package/dist/llm/graph-extract.js +2 -1
- package/dist/llm/memory-infer.js +4 -8
- package/dist/llm/metadata-enhance.js +9 -1
- package/dist/output/shapes/curate.js +14 -2
- package/dist/output/text/helpers.js +9 -0
- package/dist/runtime.js +25 -1
- package/dist/scripts/migrate-storage.js +1025 -567
- package/dist/scripts/migrations/import-fs-improve-runs-to-db.js +435 -269
- package/dist/storage/sqlite-pragmas.js +146 -0
- package/dist/workflows/db.js +3 -4
- package/dist/workflows/validate-summary.js +2 -7
- package/docs/data-and-telemetry.md +1 -0
- package/package.json +5 -4
package/dist/commands/health.js
CHANGED
|
@@ -6,12 +6,13 @@ import { ConfigError, UsageError } from "../core/errors.js";
|
|
|
6
6
|
import { appendEvent, readEvents } from "../core/events.js";
|
|
7
7
|
import { buildTaskRunId, getLoggedRunIds, openLogsDatabase } from "../core/logs-db.js";
|
|
8
8
|
import { getStateDbPathInDataDir } from "../core/paths.js";
|
|
9
|
-
import { listExistingTableNames, openStateDatabase, queryCompletedTaskIntervals, queryImproveRuns, queryTaskHistory, } from "../core/state-db.js";
|
|
9
|
+
import { listExistingTableNames, listProposalGateDecisions, listStateProposals, openStateDatabase, queryCompletedTaskIntervals, queryImproveRuns, queryTaskHistory, } from "../core/state-db.js";
|
|
10
10
|
import { parseSinceToIso } from "../core/time.js";
|
|
11
11
|
import { readSemanticStatus } from "../indexer/search/semantic-status.js";
|
|
12
12
|
import { getExecutionLogCandidates } from "../integrations/session-logs/index.js";
|
|
13
13
|
import { LLM_USAGE_EVENT } from "../llm/usage-persist.js";
|
|
14
14
|
import { HEALTH_CHECKS } from "./health/checks.js";
|
|
15
|
+
import { gateDecisionsToSamples, summarizeCalibration } from "./improve/calibration.js";
|
|
15
16
|
const DEFAULT_SINCE_MS = 24 * 60 * 60 * 1000;
|
|
16
17
|
const IMPROVE_COMPLETED_EVENT = "improve_completed";
|
|
17
18
|
const HEALTH_PROBE_EVENT = "health_probe";
|
|
@@ -72,6 +73,7 @@ function createUnknownImproveMetrics() {
|
|
|
72
73
|
error: 0,
|
|
73
74
|
},
|
|
74
75
|
autoAccept: { promoted: 0, validationFailed: 0 },
|
|
76
|
+
calibration: summarizeCalibration([]),
|
|
75
77
|
reflectsWithErrorContext: 0,
|
|
76
78
|
coverageGapCount: 0,
|
|
77
79
|
evalCasesWritten: 0,
|
|
@@ -155,6 +157,22 @@ function createUnknownImproveMetrics() {
|
|
|
155
157
|
graphExtraction: { count: 0, totalMs: 0, medianMs: 0, p95Ms: 0 },
|
|
156
158
|
},
|
|
157
159
|
},
|
|
160
|
+
perfTelemetry: {
|
|
161
|
+
dedupPoolSize: 0,
|
|
162
|
+
llmPoolSize: 0,
|
|
163
|
+
judgedCacheSkipped: 0,
|
|
164
|
+
embedMs: 0,
|
|
165
|
+
embedCacheHits: 0,
|
|
166
|
+
embedCacheMisses: 0,
|
|
167
|
+
overBudgetRuns: 0,
|
|
168
|
+
runsWithTelemetry: 0,
|
|
169
|
+
},
|
|
170
|
+
coverage: {
|
|
171
|
+
rate: Number.NaN,
|
|
172
|
+
eligibleFraction: Number.NaN,
|
|
173
|
+
acceptedProposals: 0,
|
|
174
|
+
totalAssets: 0,
|
|
175
|
+
},
|
|
158
176
|
};
|
|
159
177
|
}
|
|
160
178
|
function toFiniteNumber(value) {
|
|
@@ -363,6 +381,21 @@ function projectRunMetrics(result) {
|
|
|
363
381
|
}
|
|
364
382
|
}
|
|
365
383
|
}
|
|
384
|
+
// WS-5: extract perf telemetry from the consolidation envelope.
|
|
385
|
+
// Pre-WS-5 envelopes lack `perfTelemetry`; be defensive.
|
|
386
|
+
const perf = consolidation.perfTelemetry;
|
|
387
|
+
if (perf) {
|
|
388
|
+
metrics.perfTelemetry.runsWithTelemetry += 1;
|
|
389
|
+
metrics.perfTelemetry.dedupPoolSize += toFiniteNumber(perf.dedupPoolSize);
|
|
390
|
+
metrics.perfTelemetry.llmPoolSize += toFiniteNumber(perf.llmPoolSize);
|
|
391
|
+
metrics.perfTelemetry.judgedCacheSkipped += toFiniteNumber(perf.judgedCacheSkipped);
|
|
392
|
+
metrics.perfTelemetry.embedMs += toFiniteNumber(perf.embedMs);
|
|
393
|
+
metrics.perfTelemetry.embedCacheHits += toFiniteNumber(perf.embedCacheHits);
|
|
394
|
+
metrics.perfTelemetry.embedCacheMisses += toFiniteNumber(perf.embedCacheMisses);
|
|
395
|
+
const budgetFrac = toFiniteNumber(perf.estimatedBudgetFractionUsed);
|
|
396
|
+
if (budgetFrac > 1.0)
|
|
397
|
+
metrics.perfTelemetry.overBudgetRuns += 1;
|
|
398
|
+
}
|
|
366
399
|
}
|
|
367
400
|
const memoryInference = result.memoryInference;
|
|
368
401
|
if (memoryInference) {
|
|
@@ -561,6 +594,18 @@ function mergeImproveMetrics(dst, src) {
|
|
|
561
594
|
dst.sessionExtraction.proposalsCreated += src.sessionExtraction.proposalsCreated;
|
|
562
595
|
dst.sessionExtraction.warnings += src.sessionExtraction.warnings;
|
|
563
596
|
dst.sessionExtraction.durationMs += src.sessionExtraction.durationMs;
|
|
597
|
+
// WS-5: merge perf telemetry (additive sums).
|
|
598
|
+
dst.perfTelemetry.dedupPoolSize += src.perfTelemetry.dedupPoolSize;
|
|
599
|
+
dst.perfTelemetry.llmPoolSize += src.perfTelemetry.llmPoolSize;
|
|
600
|
+
dst.perfTelemetry.judgedCacheSkipped += src.perfTelemetry.judgedCacheSkipped;
|
|
601
|
+
dst.perfTelemetry.embedMs += src.perfTelemetry.embedMs;
|
|
602
|
+
dst.perfTelemetry.embedCacheHits += src.perfTelemetry.embedCacheHits;
|
|
603
|
+
dst.perfTelemetry.embedCacheMisses += src.perfTelemetry.embedCacheMisses;
|
|
604
|
+
dst.perfTelemetry.overBudgetRuns += src.perfTelemetry.overBudgetRuns;
|
|
605
|
+
dst.perfTelemetry.runsWithTelemetry += src.perfTelemetry.runsWithTelemetry;
|
|
606
|
+
// coverage: acceptedProposals is additive; totalAssets is a snapshot (like memorySummary).
|
|
607
|
+
// totalAssets is intentionally NOT merged here — set from the most recent run in summarizeImproveRuns.
|
|
608
|
+
dst.coverage.acceptedProposals += src.coverage.acceptedProposals;
|
|
564
609
|
}
|
|
565
610
|
function summarizeImproveRuns(db, since, until) {
|
|
566
611
|
const accum = createUnknownImproveMetrics();
|
|
@@ -1051,6 +1096,197 @@ function readLlmUsageAggregate(stateDbPath, since, until) {
|
|
|
1051
1096
|
});
|
|
1052
1097
|
return summarizeLlmUsage(events);
|
|
1053
1098
|
}
|
|
1099
|
+
/**
|
|
1100
|
+
* Read the auto-accept gate calibration summary (#612) over `[since, until)`.
|
|
1101
|
+
* Reads every proposal's `gateDecision` from the open state.db, projects the
|
|
1102
|
+
* acted-on (auto-accepted / auto-rejected) decisions into calibration samples
|
|
1103
|
+
* within the window, and aggregates them deterministically.
|
|
1104
|
+
*/
|
|
1105
|
+
function readCalibration(db, since, until) {
|
|
1106
|
+
const decisions = listProposalGateDecisions(db);
|
|
1107
|
+
const samples = gateDecisionsToSamples(decisions, { since, ...(until !== undefined ? { until } : {}) });
|
|
1108
|
+
return summarizeCalibration(samples);
|
|
1109
|
+
}
|
|
1110
|
+
// ── WS-5 Observability helpers ───────────────────────────────────────────────
|
|
1111
|
+
/**
|
|
1112
|
+
* Compute WS-5 denominator-fixed coverage metrics.
|
|
1113
|
+
*
|
|
1114
|
+
* `coverage = accepted_proposals / total_assets` (Part V §3).
|
|
1115
|
+
* The denominator is the TOTAL stash size (not the moving eligible set) so
|
|
1116
|
+
* more-inclusive WS-1 ranking cannot spuriously inflate coverage.
|
|
1117
|
+
* `eligibleFraction = eligible_assets / total_assets` is reported separately.
|
|
1118
|
+
*
|
|
1119
|
+
* Proposals are counted only when their `updatedAt` falls within `[since, until)`
|
|
1120
|
+
* so the rate is genuinely window-scoped (matching the JSDoc on the type).
|
|
1121
|
+
*
|
|
1122
|
+
* @param db - Open state.db connection.
|
|
1123
|
+
* @param totalAssets - Total stash asset count (eligible + derived) from the
|
|
1124
|
+
* most recent run's memorySummary. 0 = denominator unknown, returns NaN rates.
|
|
1125
|
+
* @param eligibleAssets - Eligible (non-derived) asset count from the most recent run.
|
|
1126
|
+
* @param since - Window start (ISO-8601). Proposals accepted before this are excluded.
|
|
1127
|
+
* @param until - Window end (ISO-8601, exclusive). Absent = open-ended (up to now).
|
|
1128
|
+
* @param stashDir - Optional: scope accepted proposals to one stash. Absent = all stashes.
|
|
1129
|
+
*/
|
|
1130
|
+
function computeDenominatorFixedCoverage(db, totalAssets, eligibleAssets, since, until, stashDir) {
|
|
1131
|
+
let acceptedProposals = 0;
|
|
1132
|
+
try {
|
|
1133
|
+
const proposals = listStateProposals(db, {
|
|
1134
|
+
status: "accepted",
|
|
1135
|
+
...(stashDir ? { stashDir } : {}),
|
|
1136
|
+
}).filter((p) => {
|
|
1137
|
+
const updatedAt = p.updatedAt ?? "";
|
|
1138
|
+
if (updatedAt < since)
|
|
1139
|
+
return false;
|
|
1140
|
+
if (until !== undefined && updatedAt >= until)
|
|
1141
|
+
return false;
|
|
1142
|
+
return true;
|
|
1143
|
+
});
|
|
1144
|
+
acceptedProposals = proposals.length;
|
|
1145
|
+
}
|
|
1146
|
+
catch {
|
|
1147
|
+
// Fail open: table may not exist on older installs.
|
|
1148
|
+
}
|
|
1149
|
+
if (totalAssets === 0) {
|
|
1150
|
+
return {
|
|
1151
|
+
rate: Number.NaN,
|
|
1152
|
+
eligibleFraction: Number.NaN,
|
|
1153
|
+
acceptedProposals,
|
|
1154
|
+
totalAssets: 0,
|
|
1155
|
+
};
|
|
1156
|
+
}
|
|
1157
|
+
return {
|
|
1158
|
+
rate: roundRate(acceptedProposals / totalAssets),
|
|
1159
|
+
eligibleFraction: roundRate(eligibleAssets / totalAssets),
|
|
1160
|
+
acceptedProposals,
|
|
1161
|
+
totalAssets,
|
|
1162
|
+
};
|
|
1163
|
+
}
|
|
1164
|
+
/**
|
|
1165
|
+
* Compute WS-5 per-run degradation metrics (Part V §4).
|
|
1166
|
+
*
|
|
1167
|
+
* Health VIEWS only — reads from state.db tables populated by prior improve
|
|
1168
|
+
* runs. Gracefully returns partial data when tables are absent (pre-WS-1/2).
|
|
1169
|
+
*
|
|
1170
|
+
* @param db - Open state.db connection.
|
|
1171
|
+
* @param since - Window start (ISO-8601).
|
|
1172
|
+
* @param until - Window end (ISO-8601).
|
|
1173
|
+
*/
|
|
1174
|
+
function computeDegradationMetrics(db, since, until) {
|
|
1175
|
+
// (a) Corpus diversity — salience rank distribution of the top-100 assets.
|
|
1176
|
+
// We use the Gini coefficient of retrieval_salience scores as an intra-corpus
|
|
1177
|
+
// diversity proxy. A Gini close to 1 = highly concentrated (entrenched top
|
|
1178
|
+
// assets), Gini near 0 = flat/diverse. This is a single-snapshot metric;
|
|
1179
|
+
// consecutive-run centroid distance requires cross-run history not yet stored.
|
|
1180
|
+
let corpusCentroidDistance = Number.NaN;
|
|
1181
|
+
let entrenchmentFlagged;
|
|
1182
|
+
try {
|
|
1183
|
+
const rows = db
|
|
1184
|
+
.prepare(`SELECT retrieval_salience FROM asset_salience
|
|
1185
|
+
ORDER BY rank_score DESC LIMIT 100`)
|
|
1186
|
+
.all();
|
|
1187
|
+
if (rows.length >= 5) {
|
|
1188
|
+
const vals = rows.map((r) => r.retrieval_salience).sort((a, b) => a - b);
|
|
1189
|
+
const n = vals.length;
|
|
1190
|
+
const sumAbsDiff = vals.reduce((acc, xi, i) => {
|
|
1191
|
+
return acc + vals.slice(i + 1).reduce((a, xj) => a + Math.abs(xi - xj), 0);
|
|
1192
|
+
}, 0);
|
|
1193
|
+
const mean = vals.reduce((a, b) => a + b, 0) / n;
|
|
1194
|
+
// Gini = (sum |xi - xj|) / (2 n^2 mean); 0 = perfect equality, 1 = perfect inequality.
|
|
1195
|
+
const gini = mean > 0 ? sumAbsDiff / (2 * n * n * mean) : 0;
|
|
1196
|
+
// Re-express as a diversity proxy in [0,1]: high gini = low diversity.
|
|
1197
|
+
// corpusCentroidDistance approximation: gini is "distance from uniform".
|
|
1198
|
+
// Note: retrieval_salience values are in [0,1], so the max achievable Gini
|
|
1199
|
+
// with this formula is ~0.5 (when one asset dominates and others are near 0).
|
|
1200
|
+
// Threshold: >0.35 flags entrenchment (robustly above the ~0.1 uniform baseline).
|
|
1201
|
+
corpusCentroidDistance = roundRate(gini);
|
|
1202
|
+
entrenchmentFlagged = gini > 0.35;
|
|
1203
|
+
}
|
|
1204
|
+
}
|
|
1205
|
+
catch {
|
|
1206
|
+
// Table not present (pre-WS-1 install) — leave NaN.
|
|
1207
|
+
}
|
|
1208
|
+
// (b) Merge fidelity — fraction of consolidate accepted proposals in the window
|
|
1209
|
+
// whose ref also has a consolidate skip-reason of "contradict_target_missing"
|
|
1210
|
+
// or an event indicating contradiction. Uses the improve_runs result_json
|
|
1211
|
+
// consolidation.contradicted count as a proxy.
|
|
1212
|
+
// Simple implementation: contradictionRate = total_contradicted / max(1, total_processed)
|
|
1213
|
+
// sourced from the window's consolidation envelope.
|
|
1214
|
+
// (The full "merge proposal → later contradiction" correlation requires cross-run
|
|
1215
|
+
// history; this is the available proxy.)
|
|
1216
|
+
let mergeFidelityContradictionRate = 0;
|
|
1217
|
+
try {
|
|
1218
|
+
const runs = queryImproveRuns(db, since, until);
|
|
1219
|
+
let totalContradicted = 0;
|
|
1220
|
+
let totalProcessed = 0;
|
|
1221
|
+
for (const row of runs) {
|
|
1222
|
+
try {
|
|
1223
|
+
const result = JSON.parse(row.result_json);
|
|
1224
|
+
const cons = result.consolidation;
|
|
1225
|
+
if (cons) {
|
|
1226
|
+
totalContradicted += toFiniteNumber(cons.contradicted);
|
|
1227
|
+
totalProcessed += toFiniteNumber(cons.processed);
|
|
1228
|
+
}
|
|
1229
|
+
}
|
|
1230
|
+
catch {
|
|
1231
|
+
// Skip malformed rows.
|
|
1232
|
+
}
|
|
1233
|
+
}
|
|
1234
|
+
if (totalProcessed > 0) {
|
|
1235
|
+
mergeFidelityContradictionRate = roundRate(totalContradicted / totalProcessed);
|
|
1236
|
+
}
|
|
1237
|
+
}
|
|
1238
|
+
catch {
|
|
1239
|
+
// Fail open.
|
|
1240
|
+
}
|
|
1241
|
+
// (c) Generation distribution — fraction of asset_salience rows with
|
|
1242
|
+
// generation >= 2. Generation is NOT currently stored in asset_salience
|
|
1243
|
+
// (it's in frontmatter). We approximate using consecutive_no_ops as a
|
|
1244
|
+
// maturity proxy: assets that have never been no-op'd are "fresh".
|
|
1245
|
+
// TODO(0.10+): store generation in asset_salience for proper tracking.
|
|
1246
|
+
let highGenerationFraction = Number.NaN;
|
|
1247
|
+
try {
|
|
1248
|
+
const genRows = db.prepare("SELECT consecutive_no_ops FROM asset_salience").all();
|
|
1249
|
+
if (genRows.length > 0) {
|
|
1250
|
+
// Use consecutive_no_ops >= 2 as a proxy for "has been through merge cycles".
|
|
1251
|
+
const highGen = genRows.filter((r) => r.consecutive_no_ops >= 2).length;
|
|
1252
|
+
highGenerationFraction = roundRate(highGen / genRows.length);
|
|
1253
|
+
}
|
|
1254
|
+
}
|
|
1255
|
+
catch {
|
|
1256
|
+
// Table not present.
|
|
1257
|
+
}
|
|
1258
|
+
// (d) Oracle spot-check — up to 5 recently accepted proposals in the window.
|
|
1259
|
+
const oracleSpotCheck = [];
|
|
1260
|
+
try {
|
|
1261
|
+
const accepted = listStateProposals(db, { status: "accepted" }).filter((p) => {
|
|
1262
|
+
const updatedAt = p.updatedAt ?? "";
|
|
1263
|
+
return updatedAt >= since && updatedAt < until;
|
|
1264
|
+
});
|
|
1265
|
+
// Sample up to 5: pick evenly spaced (not just the first 5).
|
|
1266
|
+
const step = Math.max(1, Math.floor(accepted.length / 5));
|
|
1267
|
+
for (let i = 0; i < accepted.length && oracleSpotCheck.length < 5; i += step) {
|
|
1268
|
+
const p = accepted[i];
|
|
1269
|
+
if (p) {
|
|
1270
|
+
oracleSpotCheck.push({
|
|
1271
|
+
proposalId: p.id,
|
|
1272
|
+
ref: p.ref,
|
|
1273
|
+
source: p.source ?? "unknown",
|
|
1274
|
+
acceptedAt: p.updatedAt ?? p.createdAt ?? "",
|
|
1275
|
+
});
|
|
1276
|
+
}
|
|
1277
|
+
}
|
|
1278
|
+
}
|
|
1279
|
+
catch {
|
|
1280
|
+
// Fail open.
|
|
1281
|
+
}
|
|
1282
|
+
return {
|
|
1283
|
+
corpusCentroidDistance,
|
|
1284
|
+
entrenchmentFlagged,
|
|
1285
|
+
mergeFidelityContradictionRate,
|
|
1286
|
+
highGenerationFraction,
|
|
1287
|
+
oracleSpotCheck,
|
|
1288
|
+
};
|
|
1289
|
+
}
|
|
1054
1290
|
function buildWindowMetrics(db, stateDbPath, since, until, now = () => Date.now(), logsDb) {
|
|
1055
1291
|
const taskRows = queryTaskHistory(db, { since }).filter((row) => {
|
|
1056
1292
|
const startMs = new Date(row.started_at).getTime();
|
|
@@ -1085,6 +1321,17 @@ function buildWindowMetrics(db, stateDbPath, since, until, now = () => Date.now(
|
|
|
1085
1321
|
const perRunSummaries = buildPerRunSummaries(db, since, until);
|
|
1086
1322
|
const wallTimes = perRunSummaries.map((run) => run.wallTimeMs).filter((ms) => Number.isFinite(ms) && ms > 0);
|
|
1087
1323
|
improveSummary.wallTime = computeWallTimeStats(wallTimes, improveSummary.wallTime.byPhase);
|
|
1324
|
+
improveSummary.calibration = readCalibration(db, since, until);
|
|
1325
|
+
// WS-5: Compute denominator-fixed coverage from the most recent run's
|
|
1326
|
+
// memorySummary (totalAssets = eligible + derived — the fixed denominator).
|
|
1327
|
+
const totalAssets = improveSummary.memorySummary.eligible + improveSummary.memorySummary.derived;
|
|
1328
|
+
improveSummary.coverage = computeDenominatorFixedCoverage(db, totalAssets, improveSummary.memorySummary.eligible, since, until);
|
|
1329
|
+
// WS-5: Compute per-run degradation metrics (corpus diversity, merge fidelity,
|
|
1330
|
+
// generation distribution, oracle spot-check). Health VIEWS only.
|
|
1331
|
+
const degradation = computeDegradationMetrics(db, since, until);
|
|
1332
|
+
if (degradation) {
|
|
1333
|
+
improveSummary.degradation = degradation;
|
|
1334
|
+
}
|
|
1088
1335
|
const metrics = {
|
|
1089
1336
|
taskFailRate: roundRate(taskFailRate),
|
|
1090
1337
|
agentFailureRate: roundRate(agentFailureRate),
|
|
@@ -1173,6 +1420,35 @@ export function akmHealth(options = {}) {
|
|
|
1173
1420
|
const perRunSummaries = buildPerRunSummaries(db, since);
|
|
1174
1421
|
const wallTimes = perRunSummaries.map((run) => run.wallTimeMs).filter((ms) => Number.isFinite(ms) && ms > 0);
|
|
1175
1422
|
improveSummary.wallTime = computeWallTimeStats(wallTimes, improveSummary.wallTime.byPhase);
|
|
1423
|
+
improveSummary.calibration = readCalibration(db, since);
|
|
1424
|
+
// WS-5: Compute denominator-fixed coverage and per-run degradation metrics
|
|
1425
|
+
// for the main health path (not just window-compare mode).
|
|
1426
|
+
const until = new Date(now()).toISOString();
|
|
1427
|
+
const totalAssetsMain = improveSummary.memorySummary.eligible + improveSummary.memorySummary.derived;
|
|
1428
|
+
improveSummary.coverage = computeDenominatorFixedCoverage(db, totalAssetsMain, improveSummary.memorySummary.eligible, since, until);
|
|
1429
|
+
const degradationMain = computeDegradationMetrics(db, since, until);
|
|
1430
|
+
if (degradationMain) {
|
|
1431
|
+
improveSummary.degradation = degradationMain;
|
|
1432
|
+
}
|
|
1433
|
+
// WS-2 proxy-adequacy tripwire: surface any outcome_proxy_inverted events
|
|
1434
|
+
// in the health window as an advisory so operators know when the 0.10+
|
|
1435
|
+
// rich in-session signal is no longer deferrable.
|
|
1436
|
+
const proxyInvertedEvents = readEvents({ since, type: "outcome_proxy_inverted" }, { dbPath: stateDbPath }).events;
|
|
1437
|
+
if (proxyInvertedEvents.length > 0) {
|
|
1438
|
+
const lastEvent = proxyInvertedEvents[proxyInvertedEvents.length - 1];
|
|
1439
|
+
const correlation = typeof lastEvent.metadata?.correlation === "number" ? lastEvent.metadata.correlation.toFixed(3) : "unknown";
|
|
1440
|
+
advisories.push({
|
|
1441
|
+
name: "outcome-proxy-adequacy",
|
|
1442
|
+
status: "warn",
|
|
1443
|
+
kind: "deterministic",
|
|
1444
|
+
confidence: "high",
|
|
1445
|
+
message: `WS-2 outcome proxy inverted (${proxyInvertedEvents.length} event(s) in window). ` +
|
|
1446
|
+
`corr(outcome_score, accepted_change_rate) = ${correlation} < −0.3. ` +
|
|
1447
|
+
"Popular assets are also the most-needing-improvement assets — " +
|
|
1448
|
+
"the retrieval-based proxy is inverted. " +
|
|
1449
|
+
"The 0.10+ rich in-session outcome signal is no longer deferrable. See plan §WS-2.",
|
|
1450
|
+
});
|
|
1451
|
+
}
|
|
1176
1452
|
let sessionLogEntries = [];
|
|
1177
1453
|
try {
|
|
1178
1454
|
const sinceDays = Math.max(0, Math.ceil((now() - new Date(since).getTime()) / (24 * 60 * 60 * 1000)));
|
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
// This Source Code Form is subject to the terms of the Mozilla Public
|
|
2
|
+
// License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
3
|
+
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
4
|
+
// ---------------------------------------------------------------------------
|
|
5
|
+
// Reliability computation
|
|
6
|
+
// ---------------------------------------------------------------------------
|
|
7
|
+
/** Number of fixed-width reliability buckets over [0, 1]. */
|
|
8
|
+
export const CALIBRATION_BUCKET_COUNT = 10;
|
|
9
|
+
function roundRate(value) {
|
|
10
|
+
return Number(value.toFixed(4));
|
|
11
|
+
}
|
|
12
|
+
/**
|
|
13
|
+
* Assign a confidence in [0, 1] to one of {@link CALIBRATION_BUCKET_COUNT}
|
|
14
|
+
* fixed-width buckets. The final bucket is closed on the right so confidence
|
|
15
|
+
* === 1 lands in the top bucket rather than overflowing.
|
|
16
|
+
*/
|
|
17
|
+
function bucketIndex(confidence) {
|
|
18
|
+
const clamped = Math.min(1, Math.max(0, confidence));
|
|
19
|
+
const idx = Math.floor(clamped * CALIBRATION_BUCKET_COUNT);
|
|
20
|
+
return Math.min(CALIBRATION_BUCKET_COUNT - 1, idx);
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* Compute a deterministic calibration summary from a list of acted-on gate
|
|
24
|
+
* decisions. Pure: identical input always yields identical output, with no
|
|
25
|
+
* dependency on wall-clock time or randomness.
|
|
26
|
+
*/
|
|
27
|
+
export function summarizeCalibration(samples) {
|
|
28
|
+
const buckets = [];
|
|
29
|
+
const bucketCounts = new Array(CALIBRATION_BUCKET_COUNT).fill(0);
|
|
30
|
+
const bucketAccepted = new Array(CALIBRATION_BUCKET_COUNT).fill(0);
|
|
31
|
+
const bucketConfSum = new Array(CALIBRATION_BUCKET_COUNT).fill(0);
|
|
32
|
+
let accepted = 0;
|
|
33
|
+
let confSum = 0;
|
|
34
|
+
for (const sample of samples) {
|
|
35
|
+
const idx = bucketIndex(sample.confidence);
|
|
36
|
+
bucketCounts[idx] = (bucketCounts[idx] ?? 0) + 1;
|
|
37
|
+
bucketConfSum[idx] = (bucketConfSum[idx] ?? 0) + sample.confidence;
|
|
38
|
+
confSum += sample.confidence;
|
|
39
|
+
if (sample.outcome === "auto-accepted") {
|
|
40
|
+
accepted += 1;
|
|
41
|
+
bucketAccepted[idx] = (bucketAccepted[idx] ?? 0) + 1;
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
const total = samples.length;
|
|
45
|
+
const width = 1 / CALIBRATION_BUCKET_COUNT;
|
|
46
|
+
for (let i = 0; i < CALIBRATION_BUCKET_COUNT; i += 1) {
|
|
47
|
+
const count = bucketCounts[i] ?? 0;
|
|
48
|
+
const acc = bucketAccepted[i] ?? 0;
|
|
49
|
+
buckets.push({
|
|
50
|
+
lower: roundRate(i * width),
|
|
51
|
+
upper: roundRate((i + 1) * width),
|
|
52
|
+
count,
|
|
53
|
+
accepted: acc,
|
|
54
|
+
acceptRate: count > 0 ? roundRate(acc / count) : 0,
|
|
55
|
+
meanConfidence: count > 0 ? roundRate((bucketConfSum[i] ?? 0) / count) : 0,
|
|
56
|
+
});
|
|
57
|
+
}
|
|
58
|
+
const overallAcceptRate = total > 0 ? roundRate(accepted / total) : 0;
|
|
59
|
+
const meanConfidence = total > 0 ? roundRate(confSum / total) : 0;
|
|
60
|
+
return {
|
|
61
|
+
samples: total,
|
|
62
|
+
accepted,
|
|
63
|
+
rejected: total - accepted,
|
|
64
|
+
overallAcceptRate,
|
|
65
|
+
meanConfidence,
|
|
66
|
+
calibrationGap: total > 0 ? roundRate(meanConfidence - overallAcceptRate) : 0,
|
|
67
|
+
buckets,
|
|
68
|
+
};
|
|
69
|
+
}
|
|
70
|
+
/**
|
|
71
|
+
* Project a list of `gateDecision` records (read from the proposal store) into
|
|
72
|
+
* the acted-on calibration samples within an optional `[since, until)` window.
|
|
73
|
+
*
|
|
74
|
+
* Only `auto-accepted` / `auto-rejected` decisions with a finite confidence in
|
|
75
|
+
* [0, 1] contribute. `deferred` decisions and decisions missing a confidence
|
|
76
|
+
* are excluded (no realized accept/reject signal). The window filter uses each
|
|
77
|
+
* decision's `decidedAt` timestamp; decisions with an unparseable timestamp are
|
|
78
|
+
* kept only when no window is supplied.
|
|
79
|
+
*
|
|
80
|
+
* Exploration-budget promotions (`reason === "exploration-budget"`) are EXCLUDED:
|
|
81
|
+
* they are accepted regardless of confidence, so they carry no reliability signal
|
|
82
|
+
* about the gate threshold. Counting them would inflate the apparent accept-rate
|
|
83
|
+
* and bias the auto-tuner downward — they are exempt from auto-tune by design
|
|
84
|
+
* (WS-4 exploration budget).
|
|
85
|
+
*/
|
|
86
|
+
export function gateDecisionsToSamples(decisions, window) {
|
|
87
|
+
const sinceMs = window?.since ? new Date(window.since).getTime() : undefined;
|
|
88
|
+
const untilMs = window?.until ? new Date(window.until).getTime() : undefined;
|
|
89
|
+
const samples = [];
|
|
90
|
+
for (const decision of decisions) {
|
|
91
|
+
if (!decision)
|
|
92
|
+
continue;
|
|
93
|
+
if (decision.outcome !== "auto-accepted" && decision.outcome !== "auto-rejected")
|
|
94
|
+
continue;
|
|
95
|
+
if (decision.reason === "exploration-budget")
|
|
96
|
+
continue;
|
|
97
|
+
const confidence = decision.confidence;
|
|
98
|
+
if (typeof confidence !== "number" || !Number.isFinite(confidence) || confidence < 0 || confidence > 1)
|
|
99
|
+
continue;
|
|
100
|
+
if (sinceMs !== undefined || untilMs !== undefined) {
|
|
101
|
+
const ts = new Date(decision.decidedAt).getTime();
|
|
102
|
+
if (!Number.isFinite(ts))
|
|
103
|
+
continue;
|
|
104
|
+
if (sinceMs !== undefined && ts < sinceMs)
|
|
105
|
+
continue;
|
|
106
|
+
if (untilMs !== undefined && ts >= untilMs)
|
|
107
|
+
continue;
|
|
108
|
+
}
|
|
109
|
+
samples.push({ confidence, outcome: decision.outcome });
|
|
110
|
+
}
|
|
111
|
+
return samples;
|
|
112
|
+
}
|
|
113
|
+
/**
|
|
114
|
+
* Compute a bounded, opt-in threshold adjustment from a calibration summary.
|
|
115
|
+
* PURE and deterministic — does not mutate config or read the clock. The
|
|
116
|
+
* caller is responsible for persisting `newThreshold` and logging the result.
|
|
117
|
+
*
|
|
118
|
+
* Algorithm (deliberately simple and bounded):
|
|
119
|
+
* - When `autoTune` is false → no-op (`disabled`).
|
|
120
|
+
* - When samples < `minSamples` → no-op (`insufficient-samples`).
|
|
121
|
+
* - Otherwise nudge by at most `maxStep` toward `targetAcceptRate`, then
|
|
122
|
+
* clamp into `[minThreshold, maxThreshold]`. The step size scales with the
|
|
123
|
+
* gap from target but is capped, so a single run can never make a large
|
|
124
|
+
* swing.
|
|
125
|
+
*/
|
|
126
|
+
export function computeThresholdAutoTune(currentThreshold, summary, config) {
|
|
127
|
+
const previousThreshold = Math.round(currentThreshold);
|
|
128
|
+
const noop = (reason) => ({
|
|
129
|
+
adjusted: false,
|
|
130
|
+
previousThreshold,
|
|
131
|
+
newThreshold: previousThreshold,
|
|
132
|
+
delta: 0,
|
|
133
|
+
reason,
|
|
134
|
+
});
|
|
135
|
+
if (!config.autoTune)
|
|
136
|
+
return noop("disabled");
|
|
137
|
+
if (summary.samples < config.minSamples)
|
|
138
|
+
return noop("insufficient-samples");
|
|
139
|
+
const gap = config.targetAcceptRate - summary.overallAcceptRate;
|
|
140
|
+
// A small dead-band so tiny noise doesn't churn the threshold every run.
|
|
141
|
+
const DEAD_BAND = 0.01;
|
|
142
|
+
if (Math.abs(gap) <= DEAD_BAND)
|
|
143
|
+
return noop("within-target");
|
|
144
|
+
// gap > 0 ⇒ realized below target ⇒ raise threshold (be stricter).
|
|
145
|
+
// gap < 0 ⇒ realized above target ⇒ lower threshold (be more permissive).
|
|
146
|
+
const direction = gap > 0 ? 1 : -1;
|
|
147
|
+
// Scale the step with the gap magnitude (in points) but cap at maxStep.
|
|
148
|
+
const desiredMagnitude = Math.min(config.maxStep, Math.max(1, Math.round(Math.abs(gap) * 100)));
|
|
149
|
+
const proposed = previousThreshold + direction * desiredMagnitude;
|
|
150
|
+
const clamped = Math.min(config.maxThreshold, Math.max(config.minThreshold, proposed));
|
|
151
|
+
const delta = clamped - previousThreshold;
|
|
152
|
+
if (delta === 0)
|
|
153
|
+
return noop("clamped-at-bound");
|
|
154
|
+
return {
|
|
155
|
+
adjusted: true,
|
|
156
|
+
previousThreshold,
|
|
157
|
+
newThreshold: clamped,
|
|
158
|
+
delta,
|
|
159
|
+
reason: direction > 0 ? "below-target-raise" : "above-target-lower",
|
|
160
|
+
};
|
|
161
|
+
}
|