@ngockhoale/ukit 3.3.2 → 3.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (89) hide show
  1. package/CHANGELOG.md +44 -0
  2. package/manifests/engineConformance.yaml +17 -1
  3. package/manifests/hostCapabilities.yaml +68 -1
  4. package/manifests/platform.full.yaml +138 -0
  5. package/manifests/platform.user.yaml +255 -3
  6. package/package.json +1 -1
  7. package/scripts/bench/subagent-orchestrator-corpus.mjs +275 -0
  8. package/scripts/bench/subagent-orchestrator-eval.mjs +565 -0
  9. package/scripts/probe/codex-capability-probe.mjs +169 -0
  10. package/src/cli/commands/doctor.js +168 -0
  11. package/src/cli/commands/indexTools.js +7 -0
  12. package/src/cli/commands/metrics.js +66 -2
  13. package/src/cli/commands/playbook.js +4 -4
  14. package/src/cli/commands/vm.js +49 -8
  15. package/src/core/agentRuntime/adapters.js +328 -27
  16. package/src/core/agentRuntime/artifacts.js +89 -0
  17. package/src/core/agentRuntime/context.js +345 -1
  18. package/src/core/agentRuntime/contract.js +296 -0
  19. package/src/core/agentRuntime/eventStore.js +176 -0
  20. package/src/core/agentRuntime/shadowRun.js +481 -5
  21. package/src/core/agentRuntime/telemetry.js +121 -0
  22. package/src/core/observability/emit/lifecycle.js +68 -1
  23. package/src/core/observability/emit/sessionBoot.js +393 -0
  24. package/src/core/observability/privacy/allowlist.js +10 -1
  25. package/src/core/observability/schema/registry.js +10 -0
  26. package/src/core/runtimeConfig.js +133 -0
  27. package/src/core/userPlaybooks.js +18 -3
  28. package/src/decision/registry.js +19 -0
  29. package/src/diagnostics/feedbackEvents.js +7 -4
  30. package/src/diagnostics/routeOutcomes.js +51 -6
  31. package/src/diagnostics/skillAccuracy.js +43 -3
  32. package/src/index/crossCheckMatrix.js +412 -0
  33. package/src/index/fixLoopEscalation.js +453 -0
  34. package/src/index/playbookRegistry.js +691 -0
  35. package/src/index/reviewPolicy.js +368 -0
  36. package/src/index/routeResolver.js +915 -0
  37. package/src/index/sessionHistoryExtractor.js +359 -0
  38. package/src/index/taskRouting.js +764 -581
  39. package/src/index/tierSelection.js +308 -0
  40. package/src/index/verificationMap.js +404 -0
  41. package/template_project/.claude/hooks/observability-emit.mjs +14 -0
  42. package/template_project/.claude/hooks/record-execution.mjs +19 -1
  43. package/template_project/.claude/hooks/skill-router.sh +691 -25
  44. package/template_project/.claude/hooks/verification-guard.sh +230 -1
  45. package/template_project/.claude/settings.json +2 -2
  46. package/template_project/.claude/ukit/index/cross-check-matrix.mjs +415 -0
  47. package/template_project/.claude/ukit/index/fix-loop-escalation.mjs +456 -0
  48. package/template_project/.claude/ukit/index/playbook-registry.mjs +690 -0
  49. package/template_project/.claude/ukit/index/review-panel-aggregate.mjs +20 -2
  50. package/template_project/.claude/ukit/index/review-policy.mjs +376 -0
  51. package/template_project/.claude/ukit/index/route-resolver.mjs +1059 -0
  52. package/template_project/.claude/ukit/index/route-task.mjs +1253 -846
  53. package/template_project/.claude/ukit/index/session-history-extractor.mjs +362 -0
  54. package/template_project/.claude/ukit/index/tier-selection.mjs +309 -0
  55. package/template_project/.claude/ukit/index/verification-map.mjs +403 -0
  56. package/template_project/.claude/ukit/index/worktree-sweep.mjs +195 -0
  57. package/template_project/.claude/ukit/runtime/execution-ledger.mjs +789 -11
  58. package/template_project/.claude/ukit/runtime/observability-emit.mjs +1102 -0
  59. package/template_project/.claude/ukit/runtime/reinject-context.mjs +9 -1
  60. package/template_project/.claude/ukit/runtime/resumable-run.mjs +149 -5
  61. package/template_project/.claude/ukit/runtime/stop-coordinator.mjs +323 -6
  62. package/template_project/.codex/README.md +8 -0
  63. package/template_project/.omp/hooks/pre/ukit-bridge.js +8 -1
  64. package/template_project/ukit/README.md +1 -1
  65. package/template_project/ukit/storage/config.json +20 -0
  66. package/template_user/playbooks/architecture-decision.md +28 -0
  67. package/template_user/playbooks/autonomous-run.md +43 -0
  68. package/template_user/playbooks/autopilot-full.md +59 -0
  69. package/template_user/playbooks/autopilot-stack.md +54 -0
  70. package/template_user/playbooks/babysit.md +39 -0
  71. package/template_user/playbooks/bug-fix.md +3 -1
  72. package/template_user/playbooks/{issue-implementation.md → feature-implementation.md} +4 -2
  73. package/template_user/playbooks/hillclimb.md +44 -0
  74. package/template_user/playbooks/investigation.md +21 -0
  75. package/template_user/playbooks/migration.md +21 -0
  76. package/template_user/playbooks/open-pr.md +48 -0
  77. package/template_user/playbooks/orchestrate.md +45 -0
  78. package/template_user/playbooks/performance.md +33 -0
  79. package/template_user/playbooks/prototype.md +28 -0
  80. package/template_user/playbooks/refactor.md +19 -0
  81. package/template_user/playbooks/release.md +28 -0
  82. package/template_user/playbooks/runtime-forensics.md +23 -0
  83. package/template_user/playbooks/session-pickup.md +31 -0
  84. package/template_user/playbooks/shipping.md +53 -0
  85. package/template_user/playbooks/skill-evaluation.md +48 -0
  86. package/template_user/playbooks/small-feature.md +20 -0
  87. package/template_user/playbooks/verification-map.json +153 -0
  88. package/template_user/playbooks/verification.md +22 -0
  89. package/template_user/playbooks/worktree-cleanup.md +37 -0
@@ -0,0 +1,362 @@
1
+ // session-history-extractor.mjs — parity-locked mirror of
2
+ // src/index/sessionHistoryExtractor.js (BL-013). Mirrors never import src/;
3
+ // tests/consistency/sessionHistoryParity.test.js locks the export surface and
4
+ // asserts identical extraction on shared fixtures. Keep logic identical.
5
+ //
6
+ // Reads the transcript tail (<=16KiB by default) carried on hook/bridge payloads
7
+ // (`transcriptPath`, ukit-bridge.js:217,228) and emits bounded counters/enums:
8
+ //
9
+ // { priorAttemptCount, sameSymptomReask, correctionEvents, fixLoopCount,
10
+ // degraded, reason? }
11
+ //
12
+ // Detection is deliberately heuristic + bounded (SPEC §14):
13
+ // * Route markers — any tail entry carrying `requestKey`/`routeFingerprint`
14
+ // (top-level or nested under route/ukit/metadata/message) is a prior route
15
+ // record; markers matching the current requestKey OR routeFingerprint count
16
+ // as prior attempts, and a failure-valued outcome counts a fix-loop round.
17
+ // * Re-ask — a user entry after the last matched marker (or >=2 matched
18
+ // markers) means the same route was asked again.
19
+ // * Corrections — explicit correction markers (`ukit-correction`/`correction`
20
+ // kinds) plus bounded correction phrasing in user text.
21
+ //
22
+ // Contracts (SPEC FR-001 + §10):
23
+ // * NEVER THROWS. Missing/unreadable transcript → zeroed struct with
24
+ // degraded:true + reason; malformed lines are skipped, never fatal.
25
+ // * BOUNDED: tail-only read (<=maxBytes, hard ceiling 4MiB like
26
+ // transcript-tail.mjs), counters clamp at 99, ≤50ms wall budget.
27
+ // * PRIVACY: output is counters/enums only — no transcript text may ever
28
+ // reach route state, decisions.tsv, or any receipt through this module.
29
+ //
30
+ // Two entry points share one classifier: extractHistorySignals (sync) for
31
+ // callers outside deadline-guarded hook blocks, extractHistorySignalsAsync for
32
+ // the hook path where every fs call must be async (BUG-C21-04 posture — a
33
+ // stalled mount must not park the event loop under an armed deadline).
34
+
35
+ import fs from 'node:fs';
36
+ import fsp from 'node:fs/promises';
37
+
38
+ // SPEC §14: the history extraction bound is a 16KiB tail. The ceiling mirrors
39
+ // transcript-tail.mjs's hard cap so one number describes a UKit tail scan.
40
+ export const DEFAULT_HISTORY_MAX_BYTES = 16 * 1024;
41
+ const HISTORY_MAX_BYTES_CEILING = 4 * 1024 * 1024;
42
+
43
+ const COUNTER_MAX = 99;
44
+
45
+ // Outcome fields probed on markers, in priority order. A marker without an
46
+ // outcome field counts as an attempt but never as a failed round.
47
+ const OUTCOME_FIELDS = ['outcome', 'verdict', 'status', 'result'];
48
+ const FAILURE_PATTERN = /fail|error|timeout|abort|crash|block/i;
49
+
50
+ // Object containers probed for marker keys — top-level plus the bounded nests
51
+ // writers may use. `message` is included so stamped user-prompt entries
52
+ // ({type:'user', message:{content, ukit:{requestKey}}}) still match.
53
+ const MARKER_PROBE_KEYS = ['ukit', 'route', 'routeRecord', 'routeMeta', 'metadata', 'meta', 'message'];
54
+
55
+ const USER_ENTRY_TYPES = new Set(['user', 'human']);
56
+ const CORRECTION_TYPES = new Set(['ukit-correction', 'correction', 'user-correction']);
57
+
58
+ // Bounded correction phrasing — explicit pushback patterns only; frustration
59
+ // noise stays out of the counter by design (GAP M12 failure-mode note).
60
+ const CORRECTION_PATTERN = /\b(?:no[,.!]?\s+that'?s|that'?s not|not what i|wrong|you (?:said|did|were|meant)|still (?:fail|failing|broken|wrong|crash|not working)|didn'?t work|incorrect|try again)\b/i;
61
+
62
+ // User message text is scanned only this far — long pasted diffs/logs in a
63
+ // user entry never extend the classification surface.
64
+ const USER_TEXT_SCAN_CHARS = 4096;
65
+
66
+ function isPlainObject(value) {
67
+ return Boolean(value) && typeof value === 'object' && !Array.isArray(value);
68
+ }
69
+
70
+ function nonEmptyString(value) {
71
+ return typeof value === 'string' && value.trim() ? value.trim() : null;
72
+ }
73
+
74
+ function boundedCap(maxBytes) {
75
+ const requested = Number.isFinite(maxBytes) && maxBytes > 0
76
+ ? Math.floor(maxBytes)
77
+ : DEFAULT_HISTORY_MAX_BYTES;
78
+ return Math.min(requested, HISTORY_MAX_BYTES_CEILING);
79
+ }
80
+
81
+ function degradedResult(reason) {
82
+ return {
83
+ priorAttemptCount: 0,
84
+ sameSymptomReask: false,
85
+ correctionEvents: 0,
86
+ fixLoopCount: 0,
87
+ degraded: true,
88
+ reason,
89
+ };
90
+ }
91
+
92
+ function zeroedResult() {
93
+ return {
94
+ priorAttemptCount: 0,
95
+ sameSymptomReask: false,
96
+ correctionEvents: 0,
97
+ fixLoopCount: 0,
98
+ degraded: false,
99
+ };
100
+ }
101
+
102
+ function clampCounter(value) {
103
+ const count = Number.isFinite(value) ? Math.max(0, Math.floor(value)) : 0;
104
+ return Math.min(count, COUNTER_MAX);
105
+ }
106
+
107
+ // --- bounded tail read -------------------------------------------------------
108
+
109
+ // A tail read almost certainly starts mid-record: drop everything before the
110
+ // first complete line. Malformed or non-object lines are skipped — consumers
111
+ // count only well-formed bounded events.
112
+ function tailTextToEntries(text, truncated) {
113
+ let payload = text;
114
+ if (truncated) {
115
+ const firstBreak = payload.indexOf('\n');
116
+ payload = firstBreak >= 0 ? payload.slice(firstBreak + 1) : '';
117
+ }
118
+ const entries = [];
119
+ for (const line of payload.split('\n')) {
120
+ const trimmed = line.trim();
121
+ if (!trimmed) continue;
122
+ try {
123
+ const parsed = JSON.parse(trimmed);
124
+ if (isPlainObject(parsed)) entries.push(parsed);
125
+ } catch {
126
+ // partial/corrupt line — skipped, never fatal
127
+ }
128
+ }
129
+ return entries;
130
+ }
131
+
132
+ async function readTailEntriesAsync(filePath, cap) {
133
+ const stat = await fsp.stat(filePath);
134
+ if (!stat.isFile()) {
135
+ throw new Error('not-a-file');
136
+ }
137
+ if (stat.size <= 0) {
138
+ return { entries: [], bytesRead: 0 };
139
+ }
140
+ const start = Math.max(0, stat.size - cap);
141
+ const length = Math.min(cap, stat.size);
142
+ const buffer = Buffer.allocUnsafe(length);
143
+ const handle = await fsp.open(filePath, 'r');
144
+ let bytesRead = 0;
145
+ try {
146
+ const result = await handle.read(buffer, 0, length, start);
147
+ bytesRead = result.bytesRead;
148
+ } finally {
149
+ try { await handle.close(); } catch {}
150
+ }
151
+ return {
152
+ entries: tailTextToEntries(buffer.toString('utf8', 0, bytesRead), start > 0),
153
+ bytesRead,
154
+ };
155
+ }
156
+
157
+ function readTailEntriesSync(filePath, cap) {
158
+ const stat = fs.statSync(filePath);
159
+ if (!stat.isFile()) {
160
+ throw new Error('not-a-file');
161
+ }
162
+ if (stat.size <= 0) {
163
+ return { entries: [], bytesRead: 0 };
164
+ }
165
+ const start = Math.max(0, stat.size - cap);
166
+ const length = Math.min(cap, stat.size);
167
+ const buffer = Buffer.allocUnsafe(length);
168
+ const fd = fs.openSync(filePath, 'r');
169
+ let bytesRead = 0;
170
+ try {
171
+ bytesRead = fs.readSync(fd, buffer, 0, length, start);
172
+ } finally {
173
+ try { fs.closeSync(fd); } catch {}
174
+ }
175
+ return {
176
+ entries: tailTextToEntries(buffer.toString('utf8', 0, bytesRead), start > 0),
177
+ bytesRead,
178
+ };
179
+ }
180
+
181
+ // --- bounded classification --------------------------------------------------
182
+
183
+ // Probe order for marker keys + outcomes: the entry itself first, then the
184
+ // bounded nests. Returns the first non-empty string found.
185
+ function probeField(entry, names) {
186
+ if (!isPlainObject(entry)) return null;
187
+ for (const name of names) {
188
+ const direct = nonEmptyString(entry[name]);
189
+ if (direct) return direct;
190
+ }
191
+ for (const nestKey of MARKER_PROBE_KEYS) {
192
+ const nested = entry[nestKey];
193
+ if (!isPlainObject(nested)) continue;
194
+ for (const name of names) {
195
+ const value = nonEmptyString(nested[name]);
196
+ if (value) return value;
197
+ }
198
+ }
199
+ return null;
200
+ }
201
+
202
+ function markerKeysOf(entry) {
203
+ const requestKey = probeField(entry, ['requestKey']);
204
+ const routeFingerprint = probeField(entry, ['routeFingerprint', 'fingerprint']);
205
+ return requestKey || routeFingerprint ? { requestKey, routeFingerprint } : null;
206
+ }
207
+
208
+ function markerOutcome(entry) {
209
+ return probeField(entry, OUTCOME_FIELDS);
210
+ }
211
+
212
+ function isFailedOutcome(outcome) {
213
+ return typeof outcome === 'string' && FAILURE_PATTERN.test(outcome);
214
+ }
215
+
216
+ function isCorrectionMarker(entry) {
217
+ if (!isPlainObject(entry)) return false;
218
+ if (CORRECTION_TYPES.has(entry.type)) return true;
219
+ if (entry.event === 'correction' || entry.kind === 'correction') return true;
220
+ if (isPlainObject(entry.event) && entry.event.kind === 'correction') return true;
221
+ if (isPlainObject(entry.ukit) && entry.ukit.kind === 'correction') return true;
222
+ return false;
223
+ }
224
+
225
+ function isUserEntry(entry) {
226
+ if (!isPlainObject(entry)) return false;
227
+ if (USER_ENTRY_TYPES.has(entry.type)) return true;
228
+ if (isPlainObject(entry.message) && entry.message.role === 'user') return true;
229
+ return false;
230
+ }
231
+
232
+ // Bounded user text: string content, or the text blocks of a content array.
233
+ // Tool-result/thinking blocks are ignored — only user-authored text feeds the
234
+ // correction heuristic.
235
+ function userTextOf(entry) {
236
+ if (!isUserEntry(entry)) return null;
237
+ const content = entry.message?.content ?? entry.text ?? entry.content;
238
+ let text = '';
239
+ if (typeof content === 'string') {
240
+ text = content;
241
+ } else if (Array.isArray(content)) {
242
+ text = content
243
+ .map((block) => (isPlainObject(block) && block.type === 'text' ? String(block.text ?? '') : ''))
244
+ .filter(Boolean)
245
+ .join('\n');
246
+ }
247
+ return text ? text.slice(0, USER_TEXT_SCAN_CHARS) : null;
248
+ }
249
+
250
+ function computeHistorySignals(entries, { routeFingerprint = null, requestKey = null } = {}) {
251
+ const result = zeroedResult();
252
+ let attempts = 0;
253
+ let failedAttempts = 0;
254
+ let corrections = 0;
255
+ let lastMatchedIndex = -1;
256
+ let userAfterLastMatch = false;
257
+
258
+ entries.forEach((entry, index) => {
259
+ const marker = markerKeysOf(entry);
260
+ if (marker) {
261
+ const matches = Boolean(
262
+ (marker.requestKey && requestKey && marker.requestKey === requestKey)
263
+ || (marker.routeFingerprint && routeFingerprint && marker.routeFingerprint === routeFingerprint),
264
+ );
265
+ if (matches) {
266
+ attempts += 1;
267
+ lastMatchedIndex = index;
268
+ if (isFailedOutcome(markerOutcome(entry))) {
269
+ failedAttempts += 1;
270
+ }
271
+ }
272
+ }
273
+
274
+ const userText = userTextOf(entry);
275
+ if (isCorrectionMarker(entry) || (userText !== null && CORRECTION_PATTERN.test(userText))) {
276
+ corrections += 1;
277
+ }
278
+
279
+ if (lastMatchedIndex >= 0 && index > lastMatchedIndex && userText !== null) {
280
+ userAfterLastMatch = true;
281
+ }
282
+ });
283
+
284
+ result.priorAttemptCount = clampCounter(attempts);
285
+ result.fixLoopCount = clampCounter(failedAttempts);
286
+ result.correctionEvents = clampCounter(corrections);
287
+ // A re-ask on the same route shape: either the route resolved more than once
288
+ // in the tail, or a user prompt followed the last matched attempt (the
289
+ // current prompt is appended before the hook runs).
290
+ result.sameSymptomReask = attempts > 0 && (attempts >= 2 || userAfterLastMatch);
291
+ return result;
292
+ }
293
+
294
+ // --- public contract ---------------------------------------------------------
295
+
296
+ // Coerce arbitrary input into the historySignals route-field shape. Non-object
297
+ // input yields null so callers can emit the additive nullable field honestly.
298
+ export function normalizeHistorySignals(value) {
299
+ if (!isPlainObject(value)) return null;
300
+ const normalized = {
301
+ priorAttemptCount: clampCounter(value.priorAttemptCount),
302
+ sameSymptomReask: Boolean(value.sameSymptomReask),
303
+ correctionEvents: clampCounter(value.correctionEvents),
304
+ fixLoopCount: clampCounter(value.fixLoopCount),
305
+ degraded: Boolean(value.degraded),
306
+ };
307
+ const reason = nonEmptyString(value.reason);
308
+ if (reason) normalized.reason = reason.slice(0, 120);
309
+ return normalized;
310
+ }
311
+
312
+ // SPEC §8: extractHistorySignals({transcriptPath, routeFingerprint, requestKey,
313
+ // maxBytes?}) → bounded counters, never throws. Sync reader — the ≤50ms wall
314
+ // budget assumes a healthy filesystem; deadline-guarded hook code must use the
315
+ // async twin so a stalled mount cannot park the event loop.
316
+ export function extractHistorySignals({
317
+ transcriptPath,
318
+ routeFingerprint = null,
319
+ requestKey = null,
320
+ maxBytes,
321
+ } = {}) {
322
+ try {
323
+ const normalizedPath = nonEmptyString(transcriptPath);
324
+ if (!normalizedPath) {
325
+ return degradedResult('missing-transcript-path');
326
+ }
327
+ let tail;
328
+ try {
329
+ tail = readTailEntriesSync(normalizedPath, boundedCap(maxBytes));
330
+ } catch {
331
+ return degradedResult('transcript-unreadable');
332
+ }
333
+ return computeHistorySignals(tail.entries, { routeFingerprint, requestKey });
334
+ } catch {
335
+ return degradedResult('extract-failed');
336
+ }
337
+ }
338
+
339
+ // Async twin of extractHistorySignals — identical classification, fsp-based
340
+ // bounded read for deadline-guarded hook code (skill-router.sh).
341
+ export async function extractHistorySignalsAsync({
342
+ transcriptPath,
343
+ routeFingerprint = null,
344
+ requestKey = null,
345
+ maxBytes,
346
+ } = {}) {
347
+ try {
348
+ const normalizedPath = nonEmptyString(transcriptPath);
349
+ if (!normalizedPath) {
350
+ return degradedResult('missing-transcript-path');
351
+ }
352
+ let tail;
353
+ try {
354
+ tail = await readTailEntriesAsync(normalizedPath, boundedCap(maxBytes));
355
+ } catch {
356
+ return degradedResult('transcript-unreadable');
357
+ }
358
+ return computeHistorySignals(tail.entries, { routeFingerprint, requestKey });
359
+ } catch {
360
+ return degradedResult('extract-failed');
361
+ }
362
+ }
@@ -0,0 +1,309 @@
1
+ // tier-selection.mjs — installed mirror of src/index/tierSelection.js.
2
+ // Measured reliability → tier selection (C86 BL-019, SPEC FR-001 §5/§7/§8/§14).
3
+ //
4
+ // modelTier stops being purely static: the route-audit ↔ exec-ledger join
5
+ // (BL-003/004/007) produces measured per-(execution-mode, tier) reliability,
6
+ // every route logs the measured pick vs the static pick as a `tier-selection`
7
+ // shadow receipt, and the measured map becomes authoritative ONLY when the
8
+ // promotion gates pass (≥50 joined routes, ≥60% agreement, disagreements skew
9
+ // cheaper, no verdict regression). Thin data → the static map stays.
10
+ //
11
+ // Boundary rules (SPEC §14, ARCH §Safety Boundary):
12
+ // * Tier names only — 'lite'|'code'|'smart'. A model ID in the artifact is
13
+ // dropped at read time so it can never reach a route field or receipt.
14
+ // * `modelRoles` config is never written — the measured map is derived state
15
+ // under .ukit/storage/cache/, not user configuration.
16
+ // * handoff-model-guard.sh enforcement is unchanged (not touched here).
17
+ //
18
+ // Mirror parity: template_project/.claude/ukit/index/tier-selection.mjs
19
+ // carries the identical logic (the mirror cannot import src/). Both twins are
20
+ // locked by tests/consistency/tierSelectionParity.test.js.
21
+
22
+ import fs from 'node:fs/promises';
23
+ import path from 'node:path';
24
+ import crypto from 'node:crypto';
25
+
26
+ // Provider-neutral cost-tier vocabulary — literal mirror of the
27
+ // lite→code→smart ladder in src/core/executionContracts.js (MODEL_TIER_ORDER).
28
+ export const MEASURED_TIER_ORDER = Object.freeze(['lite', 'code', 'smart']);
29
+
30
+ // Derived cache artifact (SPEC §7): metrics/diagnostics writes, router reads.
31
+ export const TIER_MAP_PATH_REL = path.join('.ukit', 'storage', 'cache', 'tier-map.json');
32
+
33
+ // Executor-decided defaults (task Discussion): a conservative reliability
34
+ // floor so a cheap-but-flaky tier never displaces a reliable one, and a 24h
35
+ // artifact TTL so a stale promote cannot silently own routing forever.
36
+ const DEFAULT_RELIABILITY_FLOOR = 0.8;
37
+ const TIER_MAP_MAX_AGE_MS = 24 * 60 * 60 * 1000;
38
+
39
+ const PROMOTION_REASONS = Object.freeze({
40
+ thin: 'joined<50',
41
+ agreement: 'agreement<0.6',
42
+ skew: 'disagreements-not-cheaper',
43
+ });
44
+
45
+ function tierName(value) {
46
+ return MEASURED_TIER_ORDER.includes(value) ? value : null;
47
+ }
48
+
49
+ // One joined row is either the {audit, ledger} pair collectRouteOutcomes
50
+ // emits or a flat {executionMode, modelTier, writeSucceeded} row — both are
51
+ // normalized here so callers never have to care which shape arrived.
52
+ function normalizeJoinedRow(row) {
53
+ if (!row || typeof row !== 'object' || Array.isArray(row)) return null;
54
+ const audit = row.audit && typeof row.audit === 'object' ? row.audit : row;
55
+ const ledger = row.ledger && typeof row.ledger === 'object' ? row.ledger : row;
56
+ // An empty object is not a joined row — nothing identifiable to count.
57
+ if (Object.keys(audit).length === 0) return null;
58
+ return { audit, ledger };
59
+ }
60
+
61
+ // SPEC §8: deriveTierStats(joinedRows) → { byModeTier, joinedRoutes }.
62
+ // byModeTier is "<mode>|<tier>"-keyed; `routes` counts joined rows where the
63
+ // route ran on that tier, `successes` counts writeSucceeded === true (the
64
+ // ledger outcome the existing byMode buckets already count), `reliability`
65
+ // is successes/routes. Rows without a provider-neutral tier name are counted
66
+ // in joinedRoutes (coverage) but never land in a tier bucket — a model ID
67
+ // must not create a pseudo-tier.
68
+ export function deriveTierStats(joinedRows = []) {
69
+ const byModeTier = {};
70
+ const rows = Array.isArray(joinedRows) ? joinedRows : [];
71
+ let joinedRoutes = 0;
72
+ for (const raw of rows) {
73
+ const normalized = normalizeJoinedRow(raw);
74
+ if (!normalized) continue;
75
+ joinedRoutes += 1;
76
+ const mode = typeof normalized.audit.executionMode === 'string'
77
+ ? normalized.audit.executionMode
78
+ : null;
79
+ const tier = tierName(normalized.audit.modelTier);
80
+ if (!mode || !tier) continue;
81
+ const key = `${mode}|${tier}`;
82
+ const bucket = byModeTier[key] ??= { routes: 0, successes: 0, reliability: 0 };
83
+ bucket.routes += 1;
84
+ if (normalized.ledger.writeSucceeded === true) bucket.successes += 1;
85
+ bucket.reliability = bucket.successes / bucket.routes;
86
+ }
87
+ return { byModeTier, joinedRoutes };
88
+ }
89
+
90
+ // SPEC §8: resolveMeasuredTier({executionMode, staticTier, stats,
91
+ // reliabilityFloor?}) → tier name | null. Cheapest tier whose measured
92
+ // reliability for that mode meets the floor; null when the data is thin or
93
+ // absent. `staticTier` rides the signature per SPEC but does not bias the
94
+ // pick — the measured table decides on its own.
95
+ export function resolveMeasuredTier({
96
+ executionMode = null,
97
+ staticTier = null,
98
+ stats = null,
99
+ reliabilityFloor = DEFAULT_RELIABILITY_FLOOR,
100
+ } = {}) {
101
+ if (!executionMode || !stats || typeof stats !== 'object') return null;
102
+ const floor = Number.isFinite(reliabilityFloor)
103
+ ? reliabilityFloor
104
+ : DEFAULT_RELIABILITY_FLOOR;
105
+ const byModeTier = stats.byModeTier ?? {};
106
+ for (const tier of MEASURED_TIER_ORDER) {
107
+ const bucket = byModeTier[`${executionMode}|${tier}`];
108
+ if (bucket && bucket.routes >= 1 && bucket.reliability >= floor) {
109
+ return tier;
110
+ }
111
+ }
112
+ return null;
113
+ }
114
+
115
+ // SPEC §8/§14: evaluateTierPromotion({stats, staticMap, minJoined?=50,
116
+ // minAgreement?=0.60}) → {decision, agreement, cheaperShare, regressionFree,
117
+ // reasons[], byMode, joinedRoutes}.
118
+ //
119
+ // Promote only when ALL gates pass:
120
+ // * joined ≥ minJoined (default 50 joined route↔outcome rows);
121
+ // * agreement ≥ minAgreement — the measured pick equals the static pick for
122
+ // ≥60% of modes where a pick exists;
123
+ // * disagreements skew cheaper — measured picks that differ must be a
124
+ // lower rung (cheaperShare = cheaper/disagreements > 0.5; zero
125
+ // disagreements passes as perfectly agreeing);
126
+ // * regressionFree — no mode where the measured pick's measured
127
+ // reliability is lower than the static pick's measured reliability
128
+ // (a tier-verdict regression, verdict = writeSucceeded reliability).
129
+ // `byMode` carries the measured pick per mode for the artifact; modes with
130
+ // no pick fall back to static at read time.
131
+ export function evaluateTierPromotion({
132
+ stats = null,
133
+ staticMap = {},
134
+ minJoined = 50,
135
+ minAgreement = 0.60,
136
+ reliabilityFloor = DEFAULT_RELIABILITY_FLOOR,
137
+ } = {}) {
138
+ const byModeTier = stats?.byModeTier ?? {};
139
+ const joinedRoutes = Number.isFinite(stats?.joinedRoutes) ? stats.joinedRoutes : 0;
140
+ const modes = Object.keys(staticMap && typeof staticMap === 'object' ? staticMap : {});
141
+ const byMode = {};
142
+
143
+ let compared = 0;
144
+ let agreed = 0;
145
+ let disagreements = 0;
146
+ let cheaper = 0;
147
+ const regressedModes = [];
148
+
149
+ for (const mode of modes) {
150
+ const staticTier = staticMap[mode];
151
+ const measured = resolveMeasuredTier({
152
+ executionMode: mode,
153
+ staticTier,
154
+ stats,
155
+ reliabilityFloor,
156
+ });
157
+ if (measured !== null) byMode[mode] = measured;
158
+ if (!tierName(staticTier) || measured === null) continue;
159
+ compared += 1;
160
+ if (measured === staticTier) {
161
+ agreed += 1;
162
+ continue;
163
+ }
164
+ disagreements += 1;
165
+ const measuredIdx = MEASURED_TIER_ORDER.indexOf(measured);
166
+ const staticIdx = MEASURED_TIER_ORDER.indexOf(staticTier);
167
+ if (measuredIdx !== -1 && staticIdx !== -1 && measuredIdx < staticIdx) {
168
+ cheaper += 1;
169
+ }
170
+ const measuredRel = byModeTier[`${mode}|${measured}`]?.reliability;
171
+ const staticRel = byModeTier[`${mode}|${staticTier}`]?.reliability;
172
+ if (Number.isFinite(measuredRel) && Number.isFinite(staticRel) && measuredRel < staticRel) {
173
+ regressedModes.push(mode);
174
+ }
175
+ }
176
+
177
+ const agreement = compared > 0 ? agreed / compared : 0;
178
+ const cheaperShare = disagreements > 0 ? cheaper / disagreements : 1;
179
+ const regressionFree = regressedModes.length === 0;
180
+
181
+ const reasons = [];
182
+ if (!(joinedRoutes >= minJoined)) reasons.push(PROMOTION_REASONS.thin);
183
+ if (!(agreement >= minAgreement)) reasons.push(PROMOTION_REASONS.agreement);
184
+ if (!(cheaperShare > 0.5)) reasons.push(PROMOTION_REASONS.skew);
185
+ for (const mode of regressedModes) reasons.push(`regression:${mode}`);
186
+
187
+ return {
188
+ decision: reasons.length === 0 ? 'promote' : 'hold',
189
+ agreement,
190
+ cheaperShare,
191
+ regressionFree,
192
+ reasons,
193
+ byMode,
194
+ joinedRoutes,
195
+ };
196
+ }
197
+
198
+ // SPEC §7: { version: 1, derivedAt, joinedRoutes, agreement, decision,
199
+ // byMode: {<mode>: <tier>} } — plus cheaperShare/reasons for the metrics
200
+ // rollup (additive, bounded enums/numbers only).
201
+ export function buildTierMapArtifact({
202
+ stats = null,
203
+ staticMap = {},
204
+ minJoined = 50,
205
+ minAgreement = 0.60,
206
+ now = () => new Date(),
207
+ } = {}) {
208
+ const evaluation = evaluateTierPromotion({ stats, staticMap, minJoined, minAgreement });
209
+ return {
210
+ version: 1,
211
+ derivedAt: now().toISOString(),
212
+ joinedRoutes: evaluation.joinedRoutes,
213
+ agreement: evaluation.agreement,
214
+ decision: evaluation.decision,
215
+ byMode: evaluation.byMode,
216
+ cheaperShare: evaluation.cheaperShare,
217
+ regressionFree: evaluation.regressionFree,
218
+ reasons: evaluation.reasons,
219
+ };
220
+ }
221
+
222
+ // Atomic tmp+rename write — same discipline as route-audit/state writes so a
223
+ // reader never sees a torn artifact. Advisory: a write failure returns
224
+ // {written:false}, never throws (metrics must stay read-mostly-safe).
225
+ export async function writeTierMap(projectRoot, artifact) {
226
+ try {
227
+ const filePath = path.join(projectRoot, TIER_MAP_PATH_REL);
228
+ await fs.mkdir(path.dirname(filePath), { recursive: true });
229
+ const tempPath = `${filePath}.tmp-${process.pid}-${crypto.randomBytes(6).toString('hex')}`;
230
+ try {
231
+ await fs.writeFile(tempPath, JSON.stringify(artifact, null, 2), 'utf8');
232
+ await fs.rename(tempPath, filePath);
233
+ } catch (error) {
234
+ try { await fs.rm(tempPath, { force: true }); } catch { /* best effort */ }
235
+ throw error;
236
+ }
237
+ return { written: true, path: filePath };
238
+ } catch (error) {
239
+ return { written: false, error: error?.message ?? String(error) };
240
+ }
241
+ }
242
+
243
+ // SPEC §8: loadTierMap(projectRoot) → TierMap|null. Absent, corrupt, wrong
244
+ // version, invalid decision, or stale (derivedAt older than the 24h TTL) →
245
+ // null so the static map stays authoritative. byMode values are filtered to
246
+ // the tier vocabulary — anything else (e.g. a model ID) is dropped so it can
247
+ // never reach a route field or receipt.
248
+ export async function loadTierMap(projectRoot, { now = () => Date.now() } = {}) {
249
+ try {
250
+ const filePath = path.join(projectRoot, TIER_MAP_PATH_REL);
251
+ const doc = JSON.parse(await fs.readFile(filePath, 'utf8'));
252
+ if (!doc || typeof doc !== 'object' || doc.version !== 1) return null;
253
+ if (doc.decision !== 'promote' && doc.decision !== 'hold') return null;
254
+ const derivedAtMs = Date.parse(doc.derivedAt ?? '');
255
+ if (!Number.isFinite(derivedAtMs)) return null;
256
+ if (now() - derivedAtMs > TIER_MAP_MAX_AGE_MS) return null;
257
+ const byMode = {};
258
+ if (doc.byMode && typeof doc.byMode === 'object') {
259
+ for (const [mode, tier] of Object.entries(doc.byMode)) {
260
+ const name = tierName(tier);
261
+ if (name) byMode[mode] = name;
262
+ }
263
+ }
264
+ return {
265
+ version: 1,
266
+ derivedAt: doc.derivedAt,
267
+ joinedRoutes: Number.isFinite(doc.joinedRoutes) ? doc.joinedRoutes : 0,
268
+ agreement: Number.isFinite(doc.agreement) ? doc.agreement : 0,
269
+ decision: doc.decision,
270
+ byMode,
271
+ ...(Number.isFinite(doc.cheaperShare) ? { cheaperShare: doc.cheaperShare } : {}),
272
+ ...(Array.isArray(doc.reasons) ? { reasons: doc.reasons.filter((r) => typeof r === 'string') } : {}),
273
+ };
274
+ } catch {
275
+ return null;
276
+ }
277
+ }
278
+
279
+ // Route-side emission (SPEC §5 FR-001): folds the persisted measured map into
280
+ // a route summary. Sets the additive `measuredTier` field (nullable — null
281
+ // when data is thin or promotion is off) and, ONLY when the persisted map
282
+ // says 'promote' AND a pick exists for this mode, overrides the authoritative
283
+ // executionContract.modelTier with the measured tier. Never throws; absent or
284
+ // invalid input leaves the static map authoritative.
285
+ //
286
+ // Returns { map, staticTier, measuredPick, promoted } — `measuredPick` is the
287
+ // raw map pick (logged on the A/B receipt even on hold), `promoted` whether
288
+ // the measured pick took over modelTier.
289
+ export async function applyMeasuredTier({
290
+ projectRoot = process.cwd(),
291
+ routeSummary = null,
292
+ tierMap = undefined,
293
+ } = {}) {
294
+ const map = tierMap !== undefined ? tierMap : await loadTierMap(projectRoot);
295
+ if (!routeSummary || typeof routeSummary !== 'object') {
296
+ return { map, staticTier: null, measuredPick: null, promoted: false };
297
+ }
298
+ const executionMode = routeSummary.executionMode ?? null;
299
+ const staticTier = routeSummary.executionContract?.modelTier ?? null;
300
+ const pick = map?.byMode?.[executionMode] ?? null;
301
+ const measuredPick = tierName(pick);
302
+ const promoted = map?.decision === 'promote' && measuredPick !== null;
303
+
304
+ routeSummary.measuredTier = promoted ? measuredPick : null;
305
+ if (promoted && routeSummary.executionContract && typeof routeSummary.executionContract === 'object') {
306
+ routeSummary.executionContract.modelTier = measuredPick;
307
+ }
308
+ return { map, staticTier, measuredPick, promoted };
309
+ }