akm-cli 0.9.0-beta.11 → 0.9.0-beta.26

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (88) hide show
  1. package/CHANGELOG.md +163 -0
  2. package/dist/assets/prompts/consolidate-system.md +23 -0
  3. package/dist/assets/prompts/contradiction-judge.md +33 -0
  4. package/dist/assets/prompts/distill-knowledge-system.md +22 -0
  5. package/dist/assets/prompts/distill-lesson-system.md +36 -0
  6. package/dist/assets/prompts/extract-session.md +5 -1
  7. package/dist/assets/prompts/graph-extract-system.md +1 -0
  8. package/dist/assets/prompts/memory-infer-system.md +1 -0
  9. package/dist/assets/prompts/memory-infer-user.md +5 -0
  10. package/dist/assets/prompts/metadata-enhance-system.md +1 -0
  11. package/dist/assets/prompts/procedural-system.md +44 -0
  12. package/dist/assets/prompts/recombine-system.md +40 -0
  13. package/dist/assets/prompts/staleness-detect-system.md +6 -0
  14. package/dist/assets/prompts/validate-summary-judge.md +1 -0
  15. package/dist/assets/templates/html/health.html +25 -27
  16. package/dist/cli.js +2 -2
  17. package/dist/commands/agent/contribute-cli.js +16 -3
  18. package/dist/commands/feedback-cli.js +48 -44
  19. package/dist/commands/health/html-report.js +140 -16
  20. package/dist/commands/health.js +277 -1
  21. package/dist/commands/improve/calibration.js +161 -0
  22. package/dist/commands/improve/consolidate.js +595 -105
  23. package/dist/commands/improve/dedup.js +482 -0
  24. package/dist/commands/improve/distill.js +119 -64
  25. package/dist/commands/improve/encoding-salience.js +205 -0
  26. package/dist/commands/improve/extract-cli.js +115 -1
  27. package/dist/commands/improve/extract-prompt.js +32 -1
  28. package/dist/commands/improve/extract-watch.js +140 -0
  29. package/dist/commands/improve/extract.js +210 -30
  30. package/dist/commands/improve/feedback-valence.js +54 -0
  31. package/dist/commands/improve/homeostatic.js +467 -0
  32. package/dist/commands/improve/improve-auto-accept.js +80 -7
  33. package/dist/commands/improve/improve-profiles.js +8 -0
  34. package/dist/commands/improve/improve.js +991 -61
  35. package/dist/commands/improve/memory/memory-contradiction-detect.js +23 -28
  36. package/dist/commands/improve/outcome-loop.js +256 -0
  37. package/dist/commands/improve/proactive-maintenance.js +9 -35
  38. package/dist/commands/improve/procedural.js +409 -0
  39. package/dist/commands/improve/recombine.js +488 -0
  40. package/dist/commands/improve/reflect.js +20 -1
  41. package/dist/commands/improve/related-sessions.js +120 -0
  42. package/dist/commands/improve/salience.js +386 -0
  43. package/dist/commands/improve/triage.js +95 -0
  44. package/dist/commands/lint/agent-linter.js +19 -24
  45. package/dist/commands/lint/base-linter.js +173 -60
  46. package/dist/commands/lint/command-linter.js +19 -24
  47. package/dist/commands/lint/env-key-rules.js +34 -1
  48. package/dist/commands/lint/index.js +30 -13
  49. package/dist/commands/lint/memory-linter.js +1 -1
  50. package/dist/commands/lint/registry.js +5 -2
  51. package/dist/commands/lint/task-linter.js +3 -3
  52. package/dist/commands/lint/workflow-linter.js +26 -1
  53. package/dist/commands/proposal/validators/proposals.js +4 -0
  54. package/dist/commands/read/curate.js +284 -86
  55. package/dist/commands/read/search-cli.js +7 -0
  56. package/dist/commands/read/search.js +1 -0
  57. package/dist/commands/sources/installed-stashes.js +5 -1
  58. package/dist/core/asset/frontmatter.js +166 -167
  59. package/dist/core/asset/markdown.js +8 -0
  60. package/dist/core/config/config-schema.js +211 -3
  61. package/dist/core/config/config.js +2 -2
  62. package/dist/core/logs-db.js +4 -3
  63. package/dist/core/state-db.js +555 -29
  64. package/dist/indexer/db/db.js +250 -27
  65. package/dist/indexer/db/graph-db.js +81 -86
  66. package/dist/indexer/graph/graph-boost.js +51 -41
  67. package/dist/indexer/passes/memory-inference.js +10 -3
  68. package/dist/indexer/passes/staleness-detect.js +2 -5
  69. package/dist/indexer/search/db-search.js +15 -4
  70. package/dist/indexer/search/ranking.js +4 -0
  71. package/dist/integrations/harnesses/claude/session-log.js +10 -0
  72. package/dist/integrations/harnesses/opencode/session-log.js +9 -0
  73. package/dist/integrations/session-logs/index.js +16 -0
  74. package/dist/llm/embedder.js +27 -3
  75. package/dist/llm/embedders/local.js +66 -2
  76. package/dist/llm/graph-extract.js +2 -1
  77. package/dist/llm/memory-infer.js +4 -8
  78. package/dist/llm/metadata-enhance.js +9 -1
  79. package/dist/output/shapes/curate.js +14 -2
  80. package/dist/output/text/helpers.js +9 -0
  81. package/dist/runtime.js +25 -1
  82. package/dist/scripts/migrate-storage.js +1025 -567
  83. package/dist/scripts/migrations/import-fs-improve-runs-to-db.js +435 -269
  84. package/dist/storage/sqlite-pragmas.js +146 -0
  85. package/dist/workflows/db.js +3 -4
  86. package/dist/workflows/validate-summary.js +2 -7
  87. package/docs/data-and-telemetry.md +1 -0
  88. package/package.json +5 -4
@@ -6,12 +6,13 @@ import { ConfigError, UsageError } from "../core/errors.js";
6
6
  import { appendEvent, readEvents } from "../core/events.js";
7
7
  import { buildTaskRunId, getLoggedRunIds, openLogsDatabase } from "../core/logs-db.js";
8
8
  import { getStateDbPathInDataDir } from "../core/paths.js";
9
- import { listExistingTableNames, openStateDatabase, queryCompletedTaskIntervals, queryImproveRuns, queryTaskHistory, } from "../core/state-db.js";
9
+ import { listExistingTableNames, listProposalGateDecisions, listStateProposals, openStateDatabase, queryCompletedTaskIntervals, queryImproveRuns, queryTaskHistory, } from "../core/state-db.js";
10
10
  import { parseSinceToIso } from "../core/time.js";
11
11
  import { readSemanticStatus } from "../indexer/search/semantic-status.js";
12
12
  import { getExecutionLogCandidates } from "../integrations/session-logs/index.js";
13
13
  import { LLM_USAGE_EVENT } from "../llm/usage-persist.js";
14
14
  import { HEALTH_CHECKS } from "./health/checks.js";
15
+ import { gateDecisionsToSamples, summarizeCalibration } from "./improve/calibration.js";
15
16
  const DEFAULT_SINCE_MS = 24 * 60 * 60 * 1000;
16
17
  const IMPROVE_COMPLETED_EVENT = "improve_completed";
17
18
  const HEALTH_PROBE_EVENT = "health_probe";
@@ -72,6 +73,7 @@ function createUnknownImproveMetrics() {
72
73
  error: 0,
73
74
  },
74
75
  autoAccept: { promoted: 0, validationFailed: 0 },
76
+ calibration: summarizeCalibration([]),
75
77
  reflectsWithErrorContext: 0,
76
78
  coverageGapCount: 0,
77
79
  evalCasesWritten: 0,
@@ -155,6 +157,22 @@ function createUnknownImproveMetrics() {
155
157
  graphExtraction: { count: 0, totalMs: 0, medianMs: 0, p95Ms: 0 },
156
158
  },
157
159
  },
160
+ perfTelemetry: {
161
+ dedupPoolSize: 0,
162
+ llmPoolSize: 0,
163
+ judgedCacheSkipped: 0,
164
+ embedMs: 0,
165
+ embedCacheHits: 0,
166
+ embedCacheMisses: 0,
167
+ overBudgetRuns: 0,
168
+ runsWithTelemetry: 0,
169
+ },
170
+ coverage: {
171
+ rate: Number.NaN,
172
+ eligibleFraction: Number.NaN,
173
+ acceptedProposals: 0,
174
+ totalAssets: 0,
175
+ },
158
176
  };
159
177
  }
160
178
  function toFiniteNumber(value) {
@@ -363,6 +381,21 @@ function projectRunMetrics(result) {
363
381
  }
364
382
  }
365
383
  }
384
+ // WS-5: extract perf telemetry from the consolidation envelope.
385
+ // Pre-WS-5 envelopes lack `perfTelemetry`; be defensive.
386
+ const perf = consolidation.perfTelemetry;
387
+ if (perf) {
388
+ metrics.perfTelemetry.runsWithTelemetry += 1;
389
+ metrics.perfTelemetry.dedupPoolSize += toFiniteNumber(perf.dedupPoolSize);
390
+ metrics.perfTelemetry.llmPoolSize += toFiniteNumber(perf.llmPoolSize);
391
+ metrics.perfTelemetry.judgedCacheSkipped += toFiniteNumber(perf.judgedCacheSkipped);
392
+ metrics.perfTelemetry.embedMs += toFiniteNumber(perf.embedMs);
393
+ metrics.perfTelemetry.embedCacheHits += toFiniteNumber(perf.embedCacheHits);
394
+ metrics.perfTelemetry.embedCacheMisses += toFiniteNumber(perf.embedCacheMisses);
395
+ const budgetFrac = toFiniteNumber(perf.estimatedBudgetFractionUsed);
396
+ if (budgetFrac > 1.0)
397
+ metrics.perfTelemetry.overBudgetRuns += 1;
398
+ }
366
399
  }
367
400
  const memoryInference = result.memoryInference;
368
401
  if (memoryInference) {
@@ -561,6 +594,18 @@ function mergeImproveMetrics(dst, src) {
561
594
  dst.sessionExtraction.proposalsCreated += src.sessionExtraction.proposalsCreated;
562
595
  dst.sessionExtraction.warnings += src.sessionExtraction.warnings;
563
596
  dst.sessionExtraction.durationMs += src.sessionExtraction.durationMs;
597
+ // WS-5: merge perf telemetry (additive sums).
598
+ dst.perfTelemetry.dedupPoolSize += src.perfTelemetry.dedupPoolSize;
599
+ dst.perfTelemetry.llmPoolSize += src.perfTelemetry.llmPoolSize;
600
+ dst.perfTelemetry.judgedCacheSkipped += src.perfTelemetry.judgedCacheSkipped;
601
+ dst.perfTelemetry.embedMs += src.perfTelemetry.embedMs;
602
+ dst.perfTelemetry.embedCacheHits += src.perfTelemetry.embedCacheHits;
603
+ dst.perfTelemetry.embedCacheMisses += src.perfTelemetry.embedCacheMisses;
604
+ dst.perfTelemetry.overBudgetRuns += src.perfTelemetry.overBudgetRuns;
605
+ dst.perfTelemetry.runsWithTelemetry += src.perfTelemetry.runsWithTelemetry;
606
+ // coverage: acceptedProposals is additive; totalAssets is a snapshot (like memorySummary).
607
+ // totalAssets is intentionally NOT merged here — set from the most recent run in summarizeImproveRuns.
608
+ dst.coverage.acceptedProposals += src.coverage.acceptedProposals;
564
609
  }
565
610
  function summarizeImproveRuns(db, since, until) {
566
611
  const accum = createUnknownImproveMetrics();
@@ -1051,6 +1096,197 @@ function readLlmUsageAggregate(stateDbPath, since, until) {
1051
1096
  });
1052
1097
  return summarizeLlmUsage(events);
1053
1098
  }
1099
+ /**
1100
+ * Read the auto-accept gate calibration summary (#612) over `[since, until)`.
1101
+ * Reads every proposal's `gateDecision` from the open state.db, projects the
1102
+ * acted-on (auto-accepted / auto-rejected) decisions into calibration samples
1103
+ * within the window, and aggregates them deterministically.
1104
+ */
1105
+ function readCalibration(db, since, until) {
1106
+ const decisions = listProposalGateDecisions(db);
1107
+ const samples = gateDecisionsToSamples(decisions, { since, ...(until !== undefined ? { until } : {}) });
1108
+ return summarizeCalibration(samples);
1109
+ }
1110
+ // ── WS-5 Observability helpers ───────────────────────────────────────────────
1111
+ /**
1112
+ * Compute WS-5 denominator-fixed coverage metrics.
1113
+ *
1114
+ * `coverage = accepted_proposals / total_assets` (Part V §3).
1115
+ * The denominator is the TOTAL stash size (not the moving eligible set) so
1116
+ * more-inclusive WS-1 ranking cannot spuriously inflate coverage.
1117
+ * `eligibleFraction = eligible_assets / total_assets` is reported separately.
1118
+ *
1119
+ * Proposals are counted only when their `updatedAt` falls within `[since, until)`
1120
+ * so the rate is genuinely window-scoped (matching the JSDoc on the type).
1121
+ *
1122
+ * @param db - Open state.db connection.
1123
+ * @param totalAssets - Total stash asset count (eligible + derived) from the
1124
+ * most recent run's memorySummary. 0 = denominator unknown, returns NaN rates.
1125
+ * @param eligibleAssets - Eligible (non-derived) asset count from the most recent run.
1126
+ * @param since - Window start (ISO-8601). Proposals accepted before this are excluded.
1127
+ * @param until - Window end (ISO-8601, exclusive). Absent = open-ended (up to now).
1128
+ * @param stashDir - Optional: scope accepted proposals to one stash. Absent = all stashes.
1129
+ */
1130
+ function computeDenominatorFixedCoverage(db, totalAssets, eligibleAssets, since, until, stashDir) {
1131
+ let acceptedProposals = 0;
1132
+ try {
1133
+ const proposals = listStateProposals(db, {
1134
+ status: "accepted",
1135
+ ...(stashDir ? { stashDir } : {}),
1136
+ }).filter((p) => {
1137
+ const updatedAt = p.updatedAt ?? "";
1138
+ if (updatedAt < since)
1139
+ return false;
1140
+ if (until !== undefined && updatedAt >= until)
1141
+ return false;
1142
+ return true;
1143
+ });
1144
+ acceptedProposals = proposals.length;
1145
+ }
1146
+ catch {
1147
+ // Fail open: table may not exist on older installs.
1148
+ }
1149
+ if (totalAssets === 0) {
1150
+ return {
1151
+ rate: Number.NaN,
1152
+ eligibleFraction: Number.NaN,
1153
+ acceptedProposals,
1154
+ totalAssets: 0,
1155
+ };
1156
+ }
1157
+ return {
1158
+ rate: roundRate(acceptedProposals / totalAssets),
1159
+ eligibleFraction: roundRate(eligibleAssets / totalAssets),
1160
+ acceptedProposals,
1161
+ totalAssets,
1162
+ };
1163
+ }
1164
+ /**
1165
+ * Compute WS-5 per-run degradation metrics (Part V §4).
1166
+ *
1167
+ * Health VIEWS only — reads from state.db tables populated by prior improve
1168
+ * runs. Gracefully returns partial data when tables are absent (pre-WS-1/2).
1169
+ *
1170
+ * @param db - Open state.db connection.
1171
+ * @param since - Window start (ISO-8601).
1172
+ * @param until - Window end (ISO-8601).
1173
+ */
1174
+ function computeDegradationMetrics(db, since, until) {
1175
+ // (a) Corpus diversity — salience rank distribution of the top-100 assets.
1176
+ // We use the Gini coefficient of retrieval_salience scores as an intra-corpus
1177
+ // diversity proxy. A Gini close to 1 = highly concentrated (entrenched top
1178
+ // assets), Gini near 0 = flat/diverse. This is a single-snapshot metric;
1179
+ // consecutive-run centroid distance requires cross-run history not yet stored.
1180
+ let corpusCentroidDistance = Number.NaN;
1181
+ let entrenchmentFlagged;
1182
+ try {
1183
+ const rows = db
1184
+ .prepare(`SELECT retrieval_salience FROM asset_salience
1185
+ ORDER BY rank_score DESC LIMIT 100`)
1186
+ .all();
1187
+ if (rows.length >= 5) {
1188
+ const vals = rows.map((r) => r.retrieval_salience).sort((a, b) => a - b);
1189
+ const n = vals.length;
1190
+ const sumAbsDiff = vals.reduce((acc, xi, i) => {
1191
+ return acc + vals.slice(i + 1).reduce((a, xj) => a + Math.abs(xi - xj), 0);
1192
+ }, 0);
1193
+ const mean = vals.reduce((a, b) => a + b, 0) / n;
1194
+ // Gini = (sum |xi - xj|) / (2 n^2 mean); 0 = perfect equality, 1 = perfect inequality.
1195
+ const gini = mean > 0 ? sumAbsDiff / (2 * n * n * mean) : 0;
1196
+ // Re-express as a diversity proxy in [0,1]: high gini = low diversity.
1197
+ // corpusCentroidDistance approximation: gini is "distance from uniform".
1198
+ // Note: retrieval_salience values are in [0,1], so the max achievable Gini
1199
+ // with this formula is ~0.5 (when one asset dominates and others are near 0).
1200
+ // Threshold: >0.35 flags entrenchment (robustly above the ~0.1 uniform baseline).
1201
+ corpusCentroidDistance = roundRate(gini);
1202
+ entrenchmentFlagged = gini > 0.35;
1203
+ }
1204
+ }
1205
+ catch {
1206
+ // Table not present (pre-WS-1 install) — leave NaN.
1207
+ }
1208
+ // (b) Merge fidelity — fraction of consolidate accepted proposals in the window
1209
+ // whose ref also has a consolidate skip-reason of "contradict_target_missing"
1210
+ // or an event indicating contradiction. Uses the improve_runs result_json
1211
+ // consolidation.contradicted count as a proxy.
1212
+ // Simple implementation: contradictionRate = total_contradicted / max(1, total_processed)
1213
+ // sourced from the window's consolidation envelope.
1214
+ // (The full "merge proposal → later contradiction" correlation requires cross-run
1215
+ // history; this is the available proxy.)
1216
+ let mergeFidelityContradictionRate = 0;
1217
+ try {
1218
+ const runs = queryImproveRuns(db, since, until);
1219
+ let totalContradicted = 0;
1220
+ let totalProcessed = 0;
1221
+ for (const row of runs) {
1222
+ try {
1223
+ const result = JSON.parse(row.result_json);
1224
+ const cons = result.consolidation;
1225
+ if (cons) {
1226
+ totalContradicted += toFiniteNumber(cons.contradicted);
1227
+ totalProcessed += toFiniteNumber(cons.processed);
1228
+ }
1229
+ }
1230
+ catch {
1231
+ // Skip malformed rows.
1232
+ }
1233
+ }
1234
+ if (totalProcessed > 0) {
1235
+ mergeFidelityContradictionRate = roundRate(totalContradicted / totalProcessed);
1236
+ }
1237
+ }
1238
+ catch {
1239
+ // Fail open.
1240
+ }
1241
+ // (c) Generation distribution — fraction of asset_salience rows with
1242
+ // generation >= 2. Generation is NOT currently stored in asset_salience
1243
+ // (it's in frontmatter). We approximate using consecutive_no_ops as a
1244
+ // maturity proxy: assets that have never been no-op'd are "fresh".
1245
+ // TODO(0.10+): store generation in asset_salience for proper tracking.
1246
+ let highGenerationFraction = Number.NaN;
1247
+ try {
1248
+ const genRows = db.prepare("SELECT consecutive_no_ops FROM asset_salience").all();
1249
+ if (genRows.length > 0) {
1250
+ // Use consecutive_no_ops >= 2 as a proxy for "has been through merge cycles".
1251
+ const highGen = genRows.filter((r) => r.consecutive_no_ops >= 2).length;
1252
+ highGenerationFraction = roundRate(highGen / genRows.length);
1253
+ }
1254
+ }
1255
+ catch {
1256
+ // Table not present.
1257
+ }
1258
+ // (d) Oracle spot-check — up to 5 recently accepted proposals in the window.
1259
+ const oracleSpotCheck = [];
1260
+ try {
1261
+ const accepted = listStateProposals(db, { status: "accepted" }).filter((p) => {
1262
+ const updatedAt = p.updatedAt ?? "";
1263
+ return updatedAt >= since && updatedAt < until;
1264
+ });
1265
+ // Sample up to 5: pick evenly spaced (not just the first 5).
1266
+ const step = Math.max(1, Math.floor(accepted.length / 5));
1267
+ for (let i = 0; i < accepted.length && oracleSpotCheck.length < 5; i += step) {
1268
+ const p = accepted[i];
1269
+ if (p) {
1270
+ oracleSpotCheck.push({
1271
+ proposalId: p.id,
1272
+ ref: p.ref,
1273
+ source: p.source ?? "unknown",
1274
+ acceptedAt: p.updatedAt ?? p.createdAt ?? "",
1275
+ });
1276
+ }
1277
+ }
1278
+ }
1279
+ catch {
1280
+ // Fail open.
1281
+ }
1282
+ return {
1283
+ corpusCentroidDistance,
1284
+ entrenchmentFlagged,
1285
+ mergeFidelityContradictionRate,
1286
+ highGenerationFraction,
1287
+ oracleSpotCheck,
1288
+ };
1289
+ }
1054
1290
  function buildWindowMetrics(db, stateDbPath, since, until, now = () => Date.now(), logsDb) {
1055
1291
  const taskRows = queryTaskHistory(db, { since }).filter((row) => {
1056
1292
  const startMs = new Date(row.started_at).getTime();
@@ -1085,6 +1321,17 @@ function buildWindowMetrics(db, stateDbPath, since, until, now = () => Date.now(
1085
1321
  const perRunSummaries = buildPerRunSummaries(db, since, until);
1086
1322
  const wallTimes = perRunSummaries.map((run) => run.wallTimeMs).filter((ms) => Number.isFinite(ms) && ms > 0);
1087
1323
  improveSummary.wallTime = computeWallTimeStats(wallTimes, improveSummary.wallTime.byPhase);
1324
+ improveSummary.calibration = readCalibration(db, since, until);
1325
+ // WS-5: Compute denominator-fixed coverage from the most recent run's
1326
+ // memorySummary (totalAssets = eligible + derived — the fixed denominator).
1327
+ const totalAssets = improveSummary.memorySummary.eligible + improveSummary.memorySummary.derived;
1328
+ improveSummary.coverage = computeDenominatorFixedCoverage(db, totalAssets, improveSummary.memorySummary.eligible, since, until);
1329
+ // WS-5: Compute per-run degradation metrics (corpus diversity, merge fidelity,
1330
+ // generation distribution, oracle spot-check). Health VIEWS only.
1331
+ const degradation = computeDegradationMetrics(db, since, until);
1332
+ if (degradation) {
1333
+ improveSummary.degradation = degradation;
1334
+ }
1088
1335
  const metrics = {
1089
1336
  taskFailRate: roundRate(taskFailRate),
1090
1337
  agentFailureRate: roundRate(agentFailureRate),
@@ -1173,6 +1420,35 @@ export function akmHealth(options = {}) {
1173
1420
  const perRunSummaries = buildPerRunSummaries(db, since);
1174
1421
  const wallTimes = perRunSummaries.map((run) => run.wallTimeMs).filter((ms) => Number.isFinite(ms) && ms > 0);
1175
1422
  improveSummary.wallTime = computeWallTimeStats(wallTimes, improveSummary.wallTime.byPhase);
1423
+ improveSummary.calibration = readCalibration(db, since);
1424
+ // WS-5: Compute denominator-fixed coverage and per-run degradation metrics
1425
+ // for the main health path (not just window-compare mode).
1426
+ const until = new Date(now()).toISOString();
1427
+ const totalAssetsMain = improveSummary.memorySummary.eligible + improveSummary.memorySummary.derived;
1428
+ improveSummary.coverage = computeDenominatorFixedCoverage(db, totalAssetsMain, improveSummary.memorySummary.eligible, since, until);
1429
+ const degradationMain = computeDegradationMetrics(db, since, until);
1430
+ if (degradationMain) {
1431
+ improveSummary.degradation = degradationMain;
1432
+ }
1433
+ // WS-2 proxy-adequacy tripwire: surface any outcome_proxy_inverted events
1434
+ // in the health window as an advisory so operators know when the 0.10+
1435
+ // rich in-session signal is no longer deferrable.
1436
+ const proxyInvertedEvents = readEvents({ since, type: "outcome_proxy_inverted" }, { dbPath: stateDbPath }).events;
1437
+ if (proxyInvertedEvents.length > 0) {
1438
+ const lastEvent = proxyInvertedEvents[proxyInvertedEvents.length - 1];
1439
+ const correlation = typeof lastEvent.metadata?.correlation === "number" ? lastEvent.metadata.correlation.toFixed(3) : "unknown";
1440
+ advisories.push({
1441
+ name: "outcome-proxy-adequacy",
1442
+ status: "warn",
1443
+ kind: "deterministic",
1444
+ confidence: "high",
1445
+ message: `WS-2 outcome proxy inverted (${proxyInvertedEvents.length} event(s) in window). ` +
1446
+ `corr(outcome_score, accepted_change_rate) = ${correlation} < −0.3. ` +
1447
+ "Popular assets are also the most-needing-improvement assets — " +
1448
+ "the retrieval-based proxy is inverted. " +
1449
+ "The 0.10+ rich in-session outcome signal is no longer deferrable. See plan §WS-2.",
1450
+ });
1451
+ }
1176
1452
  let sessionLogEntries = [];
1177
1453
  try {
1178
1454
  const sinceDays = Math.max(0, Math.ceil((now() - new Date(since).getTime()) / (24 * 60 * 60 * 1000)));
@@ -0,0 +1,161 @@
1
+ // This Source Code Form is subject to the terms of the Mozilla Public
2
+ // License, v. 2.0. If a copy of the MPL was not distributed with this
3
+ // file, You can obtain one at https://mozilla.org/MPL/2.0/.
4
+ // ---------------------------------------------------------------------------
5
+ // Reliability computation
6
+ // ---------------------------------------------------------------------------
7
+ /** Number of fixed-width reliability buckets over [0, 1]. */
8
+ export const CALIBRATION_BUCKET_COUNT = 10;
9
+ function roundRate(value) {
10
+ return Number(value.toFixed(4));
11
+ }
12
+ /**
13
+ * Assign a confidence in [0, 1] to one of {@link CALIBRATION_BUCKET_COUNT}
14
+ * fixed-width buckets. The final bucket is closed on the right so confidence
15
+ * === 1 lands in the top bucket rather than overflowing.
16
+ */
17
+ function bucketIndex(confidence) {
18
+ const clamped = Math.min(1, Math.max(0, confidence));
19
+ const idx = Math.floor(clamped * CALIBRATION_BUCKET_COUNT);
20
+ return Math.min(CALIBRATION_BUCKET_COUNT - 1, idx);
21
+ }
22
+ /**
23
+ * Compute a deterministic calibration summary from a list of acted-on gate
24
+ * decisions. Pure: identical input always yields identical output, with no
25
+ * dependency on wall-clock time or randomness.
26
+ */
27
+ export function summarizeCalibration(samples) {
28
+ const buckets = [];
29
+ const bucketCounts = new Array(CALIBRATION_BUCKET_COUNT).fill(0);
30
+ const bucketAccepted = new Array(CALIBRATION_BUCKET_COUNT).fill(0);
31
+ const bucketConfSum = new Array(CALIBRATION_BUCKET_COUNT).fill(0);
32
+ let accepted = 0;
33
+ let confSum = 0;
34
+ for (const sample of samples) {
35
+ const idx = bucketIndex(sample.confidence);
36
+ bucketCounts[idx] = (bucketCounts[idx] ?? 0) + 1;
37
+ bucketConfSum[idx] = (bucketConfSum[idx] ?? 0) + sample.confidence;
38
+ confSum += sample.confidence;
39
+ if (sample.outcome === "auto-accepted") {
40
+ accepted += 1;
41
+ bucketAccepted[idx] = (bucketAccepted[idx] ?? 0) + 1;
42
+ }
43
+ }
44
+ const total = samples.length;
45
+ const width = 1 / CALIBRATION_BUCKET_COUNT;
46
+ for (let i = 0; i < CALIBRATION_BUCKET_COUNT; i += 1) {
47
+ const count = bucketCounts[i] ?? 0;
48
+ const acc = bucketAccepted[i] ?? 0;
49
+ buckets.push({
50
+ lower: roundRate(i * width),
51
+ upper: roundRate((i + 1) * width),
52
+ count,
53
+ accepted: acc,
54
+ acceptRate: count > 0 ? roundRate(acc / count) : 0,
55
+ meanConfidence: count > 0 ? roundRate((bucketConfSum[i] ?? 0) / count) : 0,
56
+ });
57
+ }
58
+ const overallAcceptRate = total > 0 ? roundRate(accepted / total) : 0;
59
+ const meanConfidence = total > 0 ? roundRate(confSum / total) : 0;
60
+ return {
61
+ samples: total,
62
+ accepted,
63
+ rejected: total - accepted,
64
+ overallAcceptRate,
65
+ meanConfidence,
66
+ calibrationGap: total > 0 ? roundRate(meanConfidence - overallAcceptRate) : 0,
67
+ buckets,
68
+ };
69
+ }
70
+ /**
71
+ * Project a list of `gateDecision` records (read from the proposal store) into
72
+ * the acted-on calibration samples within an optional `[since, until)` window.
73
+ *
74
+ * Only `auto-accepted` / `auto-rejected` decisions with a finite confidence in
75
+ * [0, 1] contribute. `deferred` decisions and decisions missing a confidence
76
+ * are excluded (no realized accept/reject signal). The window filter uses each
77
+ * decision's `decidedAt` timestamp; decisions with an unparseable timestamp are
78
+ * kept only when no window is supplied.
79
+ *
80
+ * Exploration-budget promotions (`reason === "exploration-budget"`) are EXCLUDED:
81
+ * they are accepted regardless of confidence, so they carry no reliability signal
82
+ * about the gate threshold. Counting them would inflate the apparent accept-rate
83
+ * and bias the auto-tuner downward — they are exempt from auto-tune by design
84
+ * (WS-4 exploration budget).
85
+ */
86
+ export function gateDecisionsToSamples(decisions, window) {
87
+ const sinceMs = window?.since ? new Date(window.since).getTime() : undefined;
88
+ const untilMs = window?.until ? new Date(window.until).getTime() : undefined;
89
+ const samples = [];
90
+ for (const decision of decisions) {
91
+ if (!decision)
92
+ continue;
93
+ if (decision.outcome !== "auto-accepted" && decision.outcome !== "auto-rejected")
94
+ continue;
95
+ if (decision.reason === "exploration-budget")
96
+ continue;
97
+ const confidence = decision.confidence;
98
+ if (typeof confidence !== "number" || !Number.isFinite(confidence) || confidence < 0 || confidence > 1)
99
+ continue;
100
+ if (sinceMs !== undefined || untilMs !== undefined) {
101
+ const ts = new Date(decision.decidedAt).getTime();
102
+ if (!Number.isFinite(ts))
103
+ continue;
104
+ if (sinceMs !== undefined && ts < sinceMs)
105
+ continue;
106
+ if (untilMs !== undefined && ts >= untilMs)
107
+ continue;
108
+ }
109
+ samples.push({ confidence, outcome: decision.outcome });
110
+ }
111
+ return samples;
112
+ }
113
+ /**
114
+ * Compute a bounded, opt-in threshold adjustment from a calibration summary.
115
+ * PURE and deterministic — does not mutate config or read the clock. The
116
+ * caller is responsible for persisting `newThreshold` and logging the result.
117
+ *
118
+ * Algorithm (deliberately simple and bounded):
119
+ * - When `autoTune` is false → no-op (`disabled`).
120
+ * - When samples < `minSamples` → no-op (`insufficient-samples`).
121
+ * - Otherwise nudge by at most `maxStep` toward `targetAcceptRate`, then
122
+ * clamp into `[minThreshold, maxThreshold]`. The step size scales with the
123
+ * gap from target but is capped, so a single run can never make a large
124
+ * swing.
125
+ */
126
+ export function computeThresholdAutoTune(currentThreshold, summary, config) {
127
+ const previousThreshold = Math.round(currentThreshold);
128
+ const noop = (reason) => ({
129
+ adjusted: false,
130
+ previousThreshold,
131
+ newThreshold: previousThreshold,
132
+ delta: 0,
133
+ reason,
134
+ });
135
+ if (!config.autoTune)
136
+ return noop("disabled");
137
+ if (summary.samples < config.minSamples)
138
+ return noop("insufficient-samples");
139
+ const gap = config.targetAcceptRate - summary.overallAcceptRate;
140
+ // A small dead-band so tiny noise doesn't churn the threshold every run.
141
+ const DEAD_BAND = 0.01;
142
+ if (Math.abs(gap) <= DEAD_BAND)
143
+ return noop("within-target");
144
+ // gap > 0 ⇒ realized below target ⇒ raise threshold (be stricter).
145
+ // gap < 0 ⇒ realized above target ⇒ lower threshold (be more permissive).
146
+ const direction = gap > 0 ? 1 : -1;
147
+ // Scale the step with the gap magnitude (in points) but cap at maxStep.
148
+ const desiredMagnitude = Math.min(config.maxStep, Math.max(1, Math.round(Math.abs(gap) * 100)));
149
+ const proposed = previousThreshold + direction * desiredMagnitude;
150
+ const clamped = Math.min(config.maxThreshold, Math.max(config.minThreshold, proposed));
151
+ const delta = clamped - previousThreshold;
152
+ if (delta === 0)
153
+ return noop("clamped-at-bound");
154
+ return {
155
+ adjusted: true,
156
+ previousThreshold,
157
+ newThreshold: clamped,
158
+ delta,
159
+ reason: direction > 0 ? "below-target-raise" : "above-target-lower",
160
+ };
161
+ }