@trusty-squire/mcp 1.1.6 → 1.1.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +34 -16
- package/dist/api-client.d.ts +11 -3
- package/dist/api-client.d.ts.map +1 -1
- package/dist/api-client.js +14 -4
- package/dist/api-client.js.map +1 -1
- package/dist/bot/browser.d.ts +19 -1
- package/dist/bot/browser.d.ts.map +1 -1
- package/dist/bot/browser.js +228 -30
- package/dist/bot/browser.js.map +1 -1
- package/dist/bot/near-text-hint.d.ts +7 -3
- package/dist/bot/near-text-hint.d.ts.map +1 -1
- package/dist/bot/near-text-hint.js +1 -2
- package/dist/bot/near-text-hint.js.map +1 -1
- package/dist/bot/operator-recipe.d.ts +694 -36
- package/dist/bot/operator-recipe.d.ts.map +1 -1
- package/dist/bot/operator-recipe.js +565 -43
- package/dist/bot/operator-recipe.js.map +1 -1
- package/dist/bot/pay-operator.d.ts +4 -2
- package/dist/bot/pay-operator.d.ts.map +1 -1
- package/dist/bot/pay-operator.js +173 -67
- package/dist/bot/pay-operator.js.map +1 -1
- package/dist/bot/promote-to-skill.d.ts +1 -0
- package/dist/bot/promote-to-skill.d.ts.map +1 -1
- package/dist/bot/promote-to-skill.js +20 -30
- package/dist/bot/promote-to-skill.js.map +1 -1
- package/dist/bot/provision-session.d.ts +44 -1
- package/dist/bot/provision-session.d.ts.map +1 -1
- package/dist/bot/provision-session.js +1300 -104
- package/dist/bot/provision-session.js.map +1 -1
- package/dist/eval/replay-harness/corpus.d.ts +7 -0
- package/dist/eval/replay-harness/corpus.d.ts.map +1 -0
- package/dist/eval/replay-harness/corpus.js +183 -0
- package/dist/eval/replay-harness/corpus.js.map +1 -0
- package/dist/eval/replay-harness/har-mutate.d.ts +36 -0
- package/dist/eval/replay-harness/har-mutate.d.ts.map +1 -0
- package/dist/eval/replay-harness/har-mutate.js +119 -0
- package/dist/eval/replay-harness/har-mutate.js.map +1 -0
- package/dist/eval/replay-harness/har-substrate.d.ts +16 -0
- package/dist/eval/replay-harness/har-substrate.d.ts.map +1 -0
- package/dist/eval/replay-harness/har-substrate.js +99 -0
- package/dist/eval/replay-harness/har-substrate.js.map +1 -0
- package/dist/eval/replay-harness/index.d.ts +8 -0
- package/dist/eval/replay-harness/index.d.ts.map +1 -0
- package/dist/eval/replay-harness/index.js +8 -0
- package/dist/eval/replay-harness/index.js.map +1 -0
- package/dist/eval/replay-harness/metrics.d.ts +9 -0
- package/dist/eval/replay-harness/metrics.d.ts.map +1 -0
- package/dist/eval/replay-harness/metrics.js +117 -0
- package/dist/eval/replay-harness/metrics.js.map +1 -0
- package/dist/eval/replay-harness/reporter.d.ts +4 -0
- package/dist/eval/replay-harness/reporter.d.ts.map +1 -0
- package/dist/eval/replay-harness/reporter.js +62 -0
- package/dist/eval/replay-harness/reporter.js.map +1 -0
- package/dist/eval/replay-harness/runner.d.ts +36 -0
- package/dist/eval/replay-harness/runner.d.ts.map +1 -0
- package/dist/eval/replay-harness/runner.js +167 -0
- package/dist/eval/replay-harness/runner.js.map +1 -0
- package/dist/eval/replay-harness/types.d.ts +135 -0
- package/dist/eval/replay-harness/types.d.ts.map +1 -0
- package/dist/eval/replay-harness/types.js +2 -0
- package/dist/eval/replay-harness/types.js.map +1 -0
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +1 -0
- package/dist/server.js.map +1 -1
- package/dist/tools/grant-app-access.js +1 -1
- package/dist/tools/grant-app-access.js.map +1 -1
- package/dist/tools/operate-pay.d.ts +10 -10
- package/dist/tools/operate-pay.d.ts.map +1 -1
- package/dist/tools/operate-pay.js +42 -16
- package/dist/tools/operate-pay.js.map +1 -1
- package/dist/tools/provision-drive.d.ts +165 -8
- package/dist/tools/provision-drive.d.ts.map +1 -1
- package/dist/tools/provision-drive.js +211 -40
- package/dist/tools/provision-drive.js.map +1 -1
- package/package.json +2 -1
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
export const DEFAULT_THRESHOLDS = {
|
|
2
|
+
net_speedup: 5,
|
|
3
|
+
clean_replay_correctness: 0.9,
|
|
4
|
+
task_success: 0.98,
|
|
5
|
+
drift_catch_rate: 1,
|
|
6
|
+
money_escape: 0,
|
|
7
|
+
};
|
|
8
|
+
function ratio(numerator, denominator) {
|
|
9
|
+
return denominator > 0 ? numerator / denominator : 0;
|
|
10
|
+
}
|
|
11
|
+
function median(values) {
|
|
12
|
+
if (values.length === 0)
|
|
13
|
+
return 0;
|
|
14
|
+
const sorted = [...values].sort((a, b) => a - b);
|
|
15
|
+
const middle = Math.floor(sorted.length / 2);
|
|
16
|
+
if (sorted.length % 2 === 1)
|
|
17
|
+
return sorted[middle] ?? 0;
|
|
18
|
+
return ((sorted[middle - 1] ?? 0) + (sorted[middle] ?? 0)) / 2;
|
|
19
|
+
}
|
|
20
|
+
function speedups(hits, field) {
|
|
21
|
+
return hits.map((hit) => ratio(hit.cold[field], hit.warm?.[field] ?? 0));
|
|
22
|
+
}
|
|
23
|
+
export function computeMetrics(tasks, drift) {
|
|
24
|
+
const realistic = tasks.filter((task) => task.bucket === "repeat");
|
|
25
|
+
const hits = realistic.filter((task) => task.recipe_applied);
|
|
26
|
+
const completeHits = hits.filter((task) => task.warm !== undefined);
|
|
27
|
+
const hitRate = ratio(hits.length, realistic.length);
|
|
28
|
+
const speedupOnHit = {
|
|
29
|
+
turns: median(speedups(completeHits, "turns")),
|
|
30
|
+
tokens: median(speedups(completeHits, "tokens")),
|
|
31
|
+
wall_clock: median(speedups(completeHits, "wall_clock_ms")),
|
|
32
|
+
};
|
|
33
|
+
const totalSteps = completeHits.reduce((sum, hit) => sum + hit.total_steps, 0);
|
|
34
|
+
const fallbacks = completeHits.reduce((sum, hit) => sum + hit.fallbacks, 0);
|
|
35
|
+
const moneyTrials = drift.filter((trial) => trial.money_affecting);
|
|
36
|
+
const moneyEscapes = moneyTrials.filter((trial) => !trial.end_state_matches && trial.guard_action !== "abort").length;
|
|
37
|
+
const caughtMoneyDrift = moneyTrials.filter((trial) => trial.guard_action === "abort" ||
|
|
38
|
+
(trial.guard_action === "fallback" && trial.end_state_matches)).length;
|
|
39
|
+
return {
|
|
40
|
+
speedup_on_hit: speedupOnHit,
|
|
41
|
+
hit_rate: hitRate,
|
|
42
|
+
net_speedup: {
|
|
43
|
+
turns: hitRate * speedupOnHit.turns,
|
|
44
|
+
tokens: hitRate * speedupOnHit.tokens,
|
|
45
|
+
wall_clock: hitRate * speedupOnHit.wall_clock,
|
|
46
|
+
},
|
|
47
|
+
clean_replay_correctness: ratio(hits.filter((hit) => hit.warm !== undefined && hit.end_state_matches && hit.fallbacks === 0)
|
|
48
|
+
.length, hits.length),
|
|
49
|
+
task_success: ratio(hits.filter((hit) => hit.warm !== undefined && hit.end_state_matches).length, hits.length),
|
|
50
|
+
fallback_rate: ratio(fallbacks, totalSteps),
|
|
51
|
+
money_escape: moneyEscapes,
|
|
52
|
+
drift_catch_rate: ratio(caughtMoneyDrift, moneyTrials.length),
|
|
53
|
+
recipe_survival: { t7d: null, t30d: null },
|
|
54
|
+
invariants: {
|
|
55
|
+
novel_false_hits: tasks.filter((task) => task.bucket === "novel" && task.recipe_applied)
|
|
56
|
+
.length,
|
|
57
|
+
missing_warm_samples: tasks.filter((task) => task.recipe_applied && task.warm === undefined)
|
|
58
|
+
.length,
|
|
59
|
+
},
|
|
60
|
+
};
|
|
61
|
+
}
|
|
62
|
+
export function buildHarnessReport(tasks, drift, options = {}) {
|
|
63
|
+
const thresholds = options.thresholds ?? DEFAULT_THRESHOLDS;
|
|
64
|
+
const metrics = computeMetrics(tasks, drift);
|
|
65
|
+
const reasons = [];
|
|
66
|
+
if (metrics.net_speedup.turns < thresholds.net_speedup) {
|
|
67
|
+
reasons.push(`net speedup ${metrics.net_speedup.turns.toFixed(2)}x < ${thresholds.net_speedup.toFixed(2)}x`);
|
|
68
|
+
}
|
|
69
|
+
if (metrics.net_speedup.tokens < thresholds.net_speedup) {
|
|
70
|
+
reasons.push(`net token speedup ${metrics.net_speedup.tokens.toFixed(2)}x < ${thresholds.net_speedup.toFixed(2)}x`);
|
|
71
|
+
}
|
|
72
|
+
if (metrics.clean_replay_correctness < thresholds.clean_replay_correctness) {
|
|
73
|
+
reasons.push(`clean replay correctness ${(metrics.clean_replay_correctness * 100).toFixed(2)}% < ${(thresholds.clean_replay_correctness * 100).toFixed(2)}%`);
|
|
74
|
+
}
|
|
75
|
+
if (metrics.money_escape > thresholds.money_escape) {
|
|
76
|
+
reasons.push(`money-path veto: ${metrics.money_escape} escaped wrong outcome(s)`);
|
|
77
|
+
}
|
|
78
|
+
if (metrics.drift_catch_rate < thresholds.drift_catch_rate) {
|
|
79
|
+
reasons.push(`drift catch rate ${(metrics.drift_catch_rate * 100).toFixed(2)}% < ${(thresholds.drift_catch_rate * 100).toFixed(2)}%`);
|
|
80
|
+
}
|
|
81
|
+
if (metrics.invariants.novel_false_hits > 0) {
|
|
82
|
+
reasons.push(`novel MISS invariant: ${metrics.invariants.novel_false_hits} novel recipe false hit(s)`);
|
|
83
|
+
}
|
|
84
|
+
if (metrics.invariants.missing_warm_samples > 0) {
|
|
85
|
+
reasons.push(`incomplete replay invariant: ${metrics.invariants.missing_warm_samples} applied recipe(s) missing warm samples`);
|
|
86
|
+
}
|
|
87
|
+
return {
|
|
88
|
+
schema_version: 1,
|
|
89
|
+
mode: options.mode ?? "replay-eval",
|
|
90
|
+
generated_at: options.generatedAt ?? new Date().toISOString(),
|
|
91
|
+
corpus: {
|
|
92
|
+
tasks: tasks.length,
|
|
93
|
+
repeat: tasks.filter((task) => task.bucket === "repeat").length,
|
|
94
|
+
novel: tasks.filter((task) => task.bucket === "novel").length,
|
|
95
|
+
drift_trials: drift.length,
|
|
96
|
+
},
|
|
97
|
+
cold_baseline: {
|
|
98
|
+
tasks: tasks.length,
|
|
99
|
+
median: {
|
|
100
|
+
turns: median(tasks.map((task) => task.cold.turns)),
|
|
101
|
+
tokens: median(tasks.map((task) => task.cold.tokens)),
|
|
102
|
+
wall_clock_ms: median(tasks.map((task) => task.cold.wall_clock_ms)),
|
|
103
|
+
},
|
|
104
|
+
total: {
|
|
105
|
+
turns: tasks.reduce((sum, task) => sum + task.cold.turns, 0),
|
|
106
|
+
tokens: tasks.reduce((sum, task) => sum + task.cold.tokens, 0),
|
|
107
|
+
wall_clock_ms: tasks.reduce((sum, task) => sum + task.cold.wall_clock_ms, 0),
|
|
108
|
+
},
|
|
109
|
+
recordings: tasks.flatMap((task) => task.cold_recording === undefined ? [] : [task.cold_recording]),
|
|
110
|
+
},
|
|
111
|
+
thresholds,
|
|
112
|
+
metrics,
|
|
113
|
+
decision: reasons.length === 0 ? "SHIP" : "NO-SHIP",
|
|
114
|
+
reasons,
|
|
115
|
+
};
|
|
116
|
+
}
|
|
117
|
+
//# sourceMappingURL=metrics.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"metrics.js","sourceRoot":"","sources":["../../../src/eval/replay-harness/metrics.ts"],"names":[],"mappings":"AAQA,MAAM,CAAC,MAAM,kBAAkB,GAAsB;IACnD,WAAW,EAAE,CAAC;IACd,wBAAwB,EAAE,GAAG;IAC7B,YAAY,EAAE,IAAI;IAClB,gBAAgB,EAAE,CAAC;IACnB,YAAY,EAAE,CAAC;CAChB,CAAC;AAEF,SAAS,KAAK,CAAC,SAAiB,EAAE,WAAmB;IACnD,OAAO,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC;AACvD,CAAC;AAED,SAAS,MAAM,CAAC,MAAgB;IAC9B,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAClC,MAAM,MAAM,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACjD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC7C,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,KAAK,CAAC;QAAE,OAAO,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACxD,OAAO,CAAC,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;AACjE,CAAC;AAED,SAAS,QAAQ,CAAC,IAAuB,EAAE,KAA2C;IACpF,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,GAAG,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAC3E,CAAC;AAED,MAAM,UAAU,cAAc,CAC5B,KAAwB,EACxB,KAAyB;IAEzB,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC;IACnE,MAAM,IAAI,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;IAC7D,MAAM,YAAY,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC;IACpE,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC;IACrD,MAAM,YAAY,GAAG;QACnB,KAAK,EAAE,MAAM,CAAC,QAAQ,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;QAC9C,MAAM,EAAE,MAAM,CAAC,QAAQ,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAC;QAChD,UAAU,EAAE,MAAM,CAAC,QAAQ,CAAC,YAAY,EAAE,eAAe,CAAC,CAAC;KAC5D,CAAC;IACF,MAAM,UAAU,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;IAC/E,MAAM,SAAS,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,GAAG,GAAG,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;IAC5E,MAAM,WAAW,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;IACnE,MAAM,YAAY,GAAG,WAAW,CAAC,MAAM,CACrC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,iBAAiB,IAAI,KAAK,CAAC,YAAY,KAAK,OAAO,CACtE,CAAC,MAAM,CAAC;IACT,MAAM,gBAAgB,GAAG,WAAW,CAAC,MAAM,CACzC,CAAC,KAAK,EAAE,EAAE,CACR,KAAK,CAAC,YAAY,KAAK,OAAO;QAC9B,CAAC,KAAK,CAAC,YAAY,KAAK,UAAU,IAAI,KAAK,CAAC,iBAAiB,CAAC,CACjE,CAAC,MAAM,CAAC;IAET,OAAO;QACL,cAAc,EAAE,YAAY;QAC5B,QAAQ,EAAE,OAAO;QACjB,WAAW,EAAE;YACX,KAAK,EAAE,OAAO,GAAG,YAAY,CAAC,KAAK;YACnC,MAAM,EAAE,OAAO,GAAG,YAAY,CAAC,MAAM;YACrC,UAAU,EAAE,OAAO,GAAG,YAAY,CAAC,UAAU;SAC9C;QACD,wBAAwB,EAAE,KAAK,CAC7B,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,KAAK,SAAS,IAAI,GAAG,CAAC,iBAAiB,IAAI,GAAG,CAAC,SAAS,KAAK,CAAC,CAAC;aACzF,MAAM,EACT,IAAI,CAAC,MAAM,CACZ;QACD,YAAY,EAAE,KAAK,CACjB,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,KAAK,SAAS,IAAI,GAAG,CAAC,iBAAiB,CAAC,CAAC,MAAM,EAC5E,IAAI,CAAC,MAAM,CACZ;QACD,aAAa,EAAE,KAAK,CAAC,SAAS,EAAE,UAAU,CAAC;QAC3C,YAAY,EAAE,YAAY;QAC1B,gBAAgB,EAAE,KAAK,CAAC,gBAAgB,EAAE,WAAW,CAAC,MAAM,CAAC;QAC7D,eAAe,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE;QAC1C,UAAU,EAAE;YACV,gBAAgB,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,KAAK,OAAO,IAAI,IAAI,CAAC,cAAc,CAAC;iBACrF,MAAM;YACT,oBAAoB,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,cAAc,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,CAAC;iBACzF,MAAM;SACV;KACF,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,kBAAkB,CAChC,KAAwB,EACxB,KAAyB,EACzB,UAII,EAAE;IAEN,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,kBAAkB,CAAC;IAC5D,MAAM,OAAO,GAAG,cAAc,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;IAC7C,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,IAAI,OAAO,CAAC,WAAW,CAAC,KAAK,GAAG,UAAU,CAAC,WAAW,EAAE,CAAC;QACvD,OAAO,CAAC,IAAI,CACV,eAAe,OAAO,CAAC,WAAW,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,UAAU,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAC/F,CAAC;IACJ,CAAC;IACD,IAAI,OAAO,CAAC,WAAW,CAAC,MAAM,GAAG,UAAU,CAAC,WAAW,EAAE,CAAC;QACxD,OAAO,CAAC,IAAI,CACV,qBAAqB,OAAO,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,UAAU,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CACtG,CAAC;IACJ,CAAC;IACD,IAAI,OAAO,CAAC,wBAAwB,GAAG,UAAU,CAAC,wBAAwB,EAAE,CAAC;QAC3E,OAAO,CAAC,IAAI,CACV,4BAA4B,CAAC,OAAO,CAAC,wBAAwB,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,wBAAwB,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAChJ,CAAC;IACJ,CAAC;IACD,IAAI,OAAO,CAAC,YAAY,GAAG,UAAU,CAAC,YAAY,EAAE,CAAC;QACnD,OAAO,CAAC,IAAI,CAAC,oBAAoB,OAAO,CAAC,YAAY,2BAA2B,CAAC,CAAC;IACpF,CAAC;IACD,IAAI,OAAO,CAAC,gBAAgB,GAAG,UAAU,CAAC,gBAAgB,EAAE,CAAC;QAC3D,OAAO,CAAC,IAAI,CACV,oBAAoB,CAAC,OAAO,CAAC,gBAAgB,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,gBAAgB,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CACxH,CAAC;IACJ,CAAC;IACD,IAAI,OAAO,CAAC,UAAU,CAAC,gBAAgB,GAAG,CAAC,EAAE,CAAC;QAC5C,OAAO,CAAC,IAAI,CACV,yBAAyB,OAAO,CAAC,UAAU,CAAC,gBAAgB,4BAA4B,CACzF,CAAC;IACJ,CAAC;IACD,IAAI,OAAO,CAAC,UAAU,CAAC,oBAAoB,GAAG,CAAC,EAAE,CAAC;QAChD,OAAO,CAAC,IAAI,CACV,gCAAgC,OAAO,CAAC,UAAU,CAAC,oBAAoB,yCAAyC,CACjH,CAAC;IACJ,CAAC;IACD,OAAO;QACL,cAAc,EAAE,CAAC;QACjB,IAAI,EAAE,OAAO,CAAC,IAAI,IAAI,aAAa;QACnC,YAAY,EAAE,OAAO,CAAC,WAAW,IAAI,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QAC7D,MAAM,EAAE;YACN,KAAK,EAAE,KAAK,CAAC,MAAM;YACnB,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,MAAM;YAC/D,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,KAAK,OAAO,CAAC,CAAC,MAAM;YAC7D,YAAY,EAAE,KAAK,CAAC,MAAM;SAC3B;QACD,aAAa,EAAE;YACb,KAAK,EAAE,KAAK,CAAC,MAAM;YACnB,MAAM,EAAE;gBACN,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBACnD,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBACrD,aAAa,EAAE,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;aACpE;YACD,KAAK,EAAE;gBACL,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC;gBAC5D,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;gBAC9D,aAAa,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC,CAAC;aAC7E;YACD,UAAU,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CACjC,IAAI,CAAC,cAAc,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC,CAC/D;SACF;QACD,UAAU;QACV,OAAO;QACP,QAAQ,EAAE,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS;QACnD,OAAO;KACR,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"reporter.d.ts","sourceRoot":"","sources":["../../../src/eval/replay-harness/reporter.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAKhD,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,aAAa,GAAG,MAAM,CAE9D;AAED,wBAAgB,oBAAoB,CAAC,MAAM,EAAE,aAAa,GAAG,MAAM,CAyDlE"}
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
const percent = (value) => `${(value * 100).toFixed(2)}%`;
|
|
2
|
+
const multiplier = (value) => `${value.toFixed(2)}x`;
|
|
3
|
+
export function renderReportJson(report) {
|
|
4
|
+
return JSON.stringify(report, null, 2);
|
|
5
|
+
}
|
|
6
|
+
export function renderReportMarkdown(report) {
|
|
7
|
+
const { metrics, thresholds } = report;
|
|
8
|
+
const rows = [
|
|
9
|
+
[
|
|
10
|
+
"speedup_on_hit",
|
|
11
|
+
`turns ${multiplier(metrics.speedup_on_hit.turns)}; tokens ${multiplier(metrics.speedup_on_hit.tokens)}; wall ${multiplier(metrics.speedup_on_hit.wall_clock)}`,
|
|
12
|
+
`median >= ${multiplier(thresholds.net_speedup)} (diagnostic; headline is net)`,
|
|
13
|
+
],
|
|
14
|
+
["hit_rate", percent(metrics.hit_rate), "measured"],
|
|
15
|
+
[
|
|
16
|
+
"net_speedup",
|
|
17
|
+
`turns ${multiplier(metrics.net_speedup.turns)}; tokens ${multiplier(metrics.net_speedup.tokens)}; wall ${multiplier(metrics.net_speedup.wall_clock)}`,
|
|
18
|
+
`turns + tokens median >= ${multiplier(thresholds.net_speedup)}`,
|
|
19
|
+
],
|
|
20
|
+
[
|
|
21
|
+
"clean_replay_correctness",
|
|
22
|
+
percent(metrics.clean_replay_correctness),
|
|
23
|
+
`>= ${percent(thresholds.clean_replay_correctness)}`,
|
|
24
|
+
],
|
|
25
|
+
["task_success", percent(metrics.task_success), `>= ${percent(thresholds.task_success)}`],
|
|
26
|
+
["fallback_rate", percent(metrics.fallback_rate), "reported"],
|
|
27
|
+
["money_escape", String(metrics.money_escape), `= ${thresholds.money_escape} (veto)`],
|
|
28
|
+
[
|
|
29
|
+
"drift_catch_rate",
|
|
30
|
+
percent(metrics.drift_catch_rate),
|
|
31
|
+
`= ${percent(thresholds.drift_catch_rate)}`,
|
|
32
|
+
],
|
|
33
|
+
["recipe_survival", "T+7d n/a; T+30d n/a", "reported by housekeeper"],
|
|
34
|
+
["novel_false_hits", String(metrics.invariants.novel_false_hits), "= 0 (MISS invariant)"],
|
|
35
|
+
[
|
|
36
|
+
"missing_warm_samples",
|
|
37
|
+
String(metrics.invariants.missing_warm_samples),
|
|
38
|
+
"= 0 (complete observation invariant)",
|
|
39
|
+
],
|
|
40
|
+
];
|
|
41
|
+
const baselineSources = [
|
|
42
|
+
...new Set(report.cold_baseline.recordings.map((recording) => {
|
|
43
|
+
const { driver, model } = recording.provenance;
|
|
44
|
+
return model === undefined ? driver : `${driver}/${model}`;
|
|
45
|
+
})),
|
|
46
|
+
];
|
|
47
|
+
const lines = [
|
|
48
|
+
"# Replay-engine evaluation",
|
|
49
|
+
"",
|
|
50
|
+
`Mode: ${report.mode}; corpus: ${report.corpus.tasks} tasks (${report.corpus.repeat} repeat, ${report.corpus.novel} novel), ${report.corpus.drift_trials} drift trials.`,
|
|
51
|
+
`Cold baseline median: ${report.cold_baseline.median.turns.toFixed(1)} turns, ${report.cold_baseline.median.tokens.toFixed(0)} tokens, ${report.cold_baseline.median.wall_clock_ms.toFixed(0)} ms.`,
|
|
52
|
+
`Cold baseline evidence: ${report.cold_baseline.recordings.length} driver-recorded tasks${baselineSources.length === 0 ? "" : ` via ${baselineSources.join(", ")}`}.`,
|
|
53
|
+
"",
|
|
54
|
+
"| Metric | Value | Threshold |",
|
|
55
|
+
"| --- | ---: | --- |",
|
|
56
|
+
...rows.map(([metric, value, threshold]) => `| \`${metric}\` | ${value} | ${threshold} |`),
|
|
57
|
+
"",
|
|
58
|
+
report.decision === "SHIP" ? "SHIP" : `NO-SHIP — ${report.reasons.join("; ")}`,
|
|
59
|
+
];
|
|
60
|
+
return lines.join("\n");
|
|
61
|
+
}
|
|
62
|
+
//# sourceMappingURL=reporter.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"reporter.js","sourceRoot":"","sources":["../../../src/eval/replay-harness/reporter.ts"],"names":[],"mappings":"AAEA,MAAM,OAAO,GAAG,CAAC,KAAa,EAAU,EAAE,CAAC,GAAG,CAAC,KAAK,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;AAC1E,MAAM,UAAU,GAAG,CAAC,KAAa,EAAU,EAAE,CAAC,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;AAErE,MAAM,UAAU,gBAAgB,CAAC,MAAqB;IACpD,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;AACzC,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,MAAqB;IACxD,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,GAAG,MAAM,CAAC;IACvC,MAAM,IAAI,GAAG;QACX;YACE,gBAAgB;YAChB,SAAS,UAAU,CAAC,OAAO,CAAC,cAAc,CAAC,KAAK,CAAC,YAAY,UAAU,CAAC,OAAO,CAAC,cAAc,CAAC,MAAM,CAAC,UAAU,UAAU,CAAC,OAAO,CAAC,cAAc,CAAC,UAAU,CAAC,EAAE;YAC/J,aAAa,UAAU,CAAC,UAAU,CAAC,WAAW,CAAC,gCAAgC;SAChF;QACD,CAAC,UAAU,EAAE,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,UAAU,CAAC;QACnD;YACE,aAAa;YACb,SAAS,UAAU,CAAC,OAAO,CAAC,WAAW,CAAC,KAAK,CAAC,YAAY,UAAU,CAAC,OAAO,CAAC,WAAW,CAAC,MAAM,CAAC,UAAU,UAAU,CAAC,OAAO,CAAC,WAAW,CAAC,UAAU,CAAC,EAAE;YACtJ,4BAA4B,UAAU,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE;SACjE;QACD;YACE,0BAA0B;YAC1B,OAAO,CAAC,OAAO,CAAC,wBAAwB,CAAC;YACzC,MAAM,OAAO,CAAC,UAAU,CAAC,wBAAwB,CAAC,EAAE;SACrD;QACD,CAAC,cAAc,EAAE,OAAO,CAAC,OAAO,CAAC,YAAY,CAAC,EAAE,MAAM,OAAO,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;QACzF,CAAC,eAAe,EAAE,OAAO,CAAC,OAAO,CAAC,aAAa,CAAC,EAAE,UAAU,CAAC;QAC7D,CAAC,cAAc,EAAE,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC,EAAE,KAAK,UAAU,CAAC,YAAY,SAAS,CAAC;QACrF;YACE,kBAAkB;YAClB,OAAO,CAAC,OAAO,CAAC,gBAAgB,CAAC;YACjC,KAAK,OAAO,CAAC,UAAU,CAAC,gBAAgB,CAAC,EAAE;SAC5C;QACD,CAAC,iBAAiB,EAAE,qBAAqB,EAAE,yBAAyB,CAAC;QACrE,CAAC,kBAAkB,EAAE,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,gBAAgB,CAAC,EAAE,sBAAsB,CAAC;QACzF;YACE,sBAAsB;YACtB,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,oBAAoB,CAAC;YAC/C,sCAAsC;SACvC;KACF,CAAC;IACF,MAAM,eAAe,GAAG;QACtB,GAAG,IAAI,GAAG,CACR,MAAM,CAAC,aAAa,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE;YAChD,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,SAAS,CAAC,UAAU,CAAC;YAC/C,OAAO,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,IAAI,KAAK,EAAE,CAAC;QAC7D,CAAC,CAAC,CACH;KACF,CAAC;IACF,MAAM,KAAK,GAAG;QACZ,4BAA4B;QAC5B,EAAE;QACF,SAAS,MAAM,CAAC,IAAI,aAAa,MAAM,CAAC,MAAM,CAAC,KAAK,WAAW,MAAM,CAAC,MAAM,CAAC,MAAM,YAAY,MAAM,CAAC,MAAM,CAAC,KAAK,YAAY,MAAM,CAAC,MAAM,CAAC,YAAY,gBAAgB;QACxK,yBAAyB,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,WAAW,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,YAAY,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM;QACnM,2BAA2B,MAAM,CAAC,aAAa,CAAC,UAAU,CAAC,MAAM,yBAAyB,eAAe,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG;QACrK,EAAE;QACF,gCAAgC;QAChC,sBAAsB;QACtB,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,SAAS,CAAC,EAAE,EAAE,CAAC,OAAO,MAAM,QAAQ,KAAK,MAAM,SAAS,IAAI,CAAC;QAC1F,EAAE;QACF,MAAM,CAAC,QAAQ,KAAK,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,aAAa,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;KAC/E,CAAC;IACF,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC"}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import { type HarFile } from "./har-mutate.js";
|
|
2
|
+
import type { DriftMutationName, DriftObservation, ExpectedEndState, HarnessReport, ShoppingTaskRecord, TaskObservation } from "./types.js";
|
|
3
|
+
export interface ColdBaselineDriverResult {
|
|
4
|
+
turns: number;
|
|
5
|
+
tokens: number;
|
|
6
|
+
end_state: ExpectedEndState;
|
|
7
|
+
}
|
|
8
|
+
export interface ColdBaselineDriver {
|
|
9
|
+
name: string;
|
|
10
|
+
model: string;
|
|
11
|
+
drive(task: ShoppingTaskRecord): Promise<ColdBaselineDriverResult>;
|
|
12
|
+
}
|
|
13
|
+
export interface ColdBaselineCaptureOptions {
|
|
14
|
+
recordedAt?: string;
|
|
15
|
+
now?: () => number;
|
|
16
|
+
}
|
|
17
|
+
export interface DriftReplayInput {
|
|
18
|
+
task: ShoppingTaskRecord;
|
|
19
|
+
mutation: DriftMutationName;
|
|
20
|
+
har: HarFile;
|
|
21
|
+
}
|
|
22
|
+
export interface DriftReplayResult {
|
|
23
|
+
guard_action: DriftObservation["guard_action"];
|
|
24
|
+
end_state: ExpectedEndState;
|
|
25
|
+
}
|
|
26
|
+
export type DriftReplayAdapter = (input: DriftReplayInput) => Promise<DriftReplayResult>;
|
|
27
|
+
export declare function recordAllColdBaseline(tasks: ShoppingTaskRecord[], driver: ColdBaselineDriver, options?: ColdBaselineCaptureOptions): Promise<TaskObservation[]>;
|
|
28
|
+
export declare function readFrozenAllColdBaseline(tasks: ShoppingTaskRecord[]): TaskObservation[];
|
|
29
|
+
export declare function runAllColdHarness(tasks: ShoppingTaskRecord[], drift: DriftObservation[], driver: ColdBaselineDriver, options?: ColdBaselineCaptureOptions & {
|
|
30
|
+
generatedAt?: string;
|
|
31
|
+
}): Promise<HarnessReport>;
|
|
32
|
+
export declare function runFrozenAllColdHarness(tasks: ShoppingTaskRecord[], drift: DriftObservation[], generatedAt?: string): HarnessReport;
|
|
33
|
+
export declare function runDriftBattery(tasks: ShoppingTaskRecord[], loadHar: (task: ShoppingTaskRecord) => HarFile | Promise<HarFile>, replay: DriftReplayAdapter): Promise<DriftObservation[]>;
|
|
34
|
+
export declare function totalVerifyGuard(expectedTotalCents: number, observedTotalCents: number): "clean" | "abort";
|
|
35
|
+
export declare function moneyDriftObservation(task: ShoppingTaskRecord, observedTotalCents: number, observedGuardAction: DriftObservation["guard_action"]): DriftObservation;
|
|
36
|
+
//# sourceMappingURL=runner.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../../src/eval/replay-harness/runner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAa,KAAK,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAE1D,OAAO,KAAK,EAGV,iBAAiB,EACjB,gBAAgB,EAChB,gBAAgB,EAChB,aAAa,EACb,kBAAkB,EAClB,eAAe,EAChB,MAAM,YAAY,CAAC;AAWpB,MAAM,WAAW,wBAAwB;IACvC,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,gBAAgB,CAAC;CAC7B;AAED,MAAM,WAAW,kBAAkB;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,IAAI,EAAE,kBAAkB,GAAG,OAAO,CAAC,wBAAwB,CAAC,CAAC;CACpE;AAED,MAAM,WAAW,0BAA0B;IACzC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,kBAAkB,CAAC;IACzB,QAAQ,EAAE,iBAAiB,CAAC;IAC5B,GAAG,EAAE,OAAO,CAAC;CACd;AAED,MAAM,WAAW,iBAAiB;IAChC,YAAY,EAAE,gBAAgB,CAAC,cAAc,CAAC,CAAC;IAC/C,SAAS,EAAE,gBAAgB,CAAC;CAC7B;AAED,MAAM,MAAM,kBAAkB,GAAG,CAAC,KAAK,EAAE,gBAAgB,KAAK,OAAO,CAAC,iBAAiB,CAAC,CAAC;AAezF,wBAAsB,qBAAqB,CACzC,KAAK,EAAE,kBAAkB,EAAE,EAC3B,MAAM,EAAE,kBAAkB,EAC1B,OAAO,GAAE,0BAA+B,GACvC,OAAO,CAAC,eAAe,EAAE,CAAC,CAwC5B;AAED,wBAAgB,yBAAyB,CAAC,KAAK,EAAE,kBAAkB,EAAE,GAAG,eAAe,EAAE,CA2BxF;AAED,wBAAsB,iBAAiB,CACrC,KAAK,EAAE,kBAAkB,EAAE,EAC3B,KAAK,EAAE,gBAAgB,EAAE,EACzB,MAAM,EAAE,kBAAkB,EAC1B,OAAO,GAAE,0BAA0B,GAAG;IAAE,WAAW,CAAC,EAAE,MAAM,CAAA;CAAO,GAClE,OAAO,CAAC,aAAa,CAAC,CAexB;AAED,wBAAgB,uBAAuB,CACrC,KAAK,EAAE,kBAAkB,EAAE,EAC3B,KAAK,EAAE,gBAAgB,EAAE,EACzB,WAAW,CAAC,EAAE,MAAM,GACnB,aAAa,CAWf;AAED,wBAAsB,eAAe,CACnC,KAAK,EAAE,kBAAkB,EAAE,EAC3B,OAAO,EAAE,CAAC,IAAI,EAAE,kBAAkB,KAAK,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,EACjE,MAAM,EAAE,kBAAkB,GACzB,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAkC7B;AAED,wBAAgB,gBAAgB,CAC9B,kBAAkB,EAAE,MAAM,EAC1B,kBAAkB,EAAE,MAAM,GACzB,OAAO,GAAG,OAAO,CAEnB;AAED,wBAAgB,qBAAqB,CACnC,IAAI,EAAE,kBAAkB,EACxB,kBAAkB,EAAE,MAAM,EAC1B,mBAAmB,EAAE,gBAAgB,CAAC,cAAc,CAAC,GACpD,gBAAgB,CASlB"}
|
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
import { performance } from "node:perf_hooks";
|
|
2
|
+
import { endStatesMatch } from "./corpus.js";
|
|
3
|
+
import { mutateHar } from "./har-mutate.js";
|
|
4
|
+
import { buildHarnessReport } from "./metrics.js";
|
|
5
|
+
const DRIFT_MUTATIONS = [
|
|
6
|
+
"rename-button",
|
|
7
|
+
"swap-testid",
|
|
8
|
+
"remove-field",
|
|
9
|
+
"change-price",
|
|
10
|
+
"out-of-stock",
|
|
11
|
+
"overlay",
|
|
12
|
+
];
|
|
13
|
+
function assertMeasuredCost(taskId, cost) {
|
|
14
|
+
if (!Number.isFinite(cost.turns) ||
|
|
15
|
+
cost.turns < 0 ||
|
|
16
|
+
!Number.isFinite(cost.tokens) ||
|
|
17
|
+
cost.tokens < 0 ||
|
|
18
|
+
!Number.isFinite(cost.wall_clock_ms) ||
|
|
19
|
+
cost.wall_clock_ms < 0) {
|
|
20
|
+
throw new Error(`${taskId}: cold driver returned invalid turns or tokens`);
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
export async function recordAllColdBaseline(tasks, driver, options = {}) {
|
|
24
|
+
if (driver.name.trim().length === 0)
|
|
25
|
+
throw new Error("cold driver name is required");
|
|
26
|
+
if (driver.model.trim().length === 0)
|
|
27
|
+
throw new Error("cold driver model is required");
|
|
28
|
+
const now = options.now ?? performance.now.bind(performance);
|
|
29
|
+
const recordedAt = options.recordedAt ?? new Date().toISOString();
|
|
30
|
+
const provenance = {
|
|
31
|
+
source: "driver",
|
|
32
|
+
driver: driver.name,
|
|
33
|
+
model: driver.model,
|
|
34
|
+
recorded_at: recordedAt,
|
|
35
|
+
};
|
|
36
|
+
const observations = [];
|
|
37
|
+
for (const task of tasks.filter((candidate) => candidate.capture.status === "captured")) {
|
|
38
|
+
const started = now();
|
|
39
|
+
const result = await driver.drive(task);
|
|
40
|
+
const cold = {
|
|
41
|
+
turns: result.turns,
|
|
42
|
+
tokens: result.tokens,
|
|
43
|
+
wall_clock_ms: Math.round(Math.max(0, now() - started)),
|
|
44
|
+
};
|
|
45
|
+
assertMeasuredCost(task.task_id, cold);
|
|
46
|
+
const endStateMatches = endStatesMatch(result.end_state, task.expected_end_state);
|
|
47
|
+
observations.push({
|
|
48
|
+
task_id: task.task_id,
|
|
49
|
+
bucket: task.bucket,
|
|
50
|
+
cold,
|
|
51
|
+
cold_recording: {
|
|
52
|
+
task_id: task.task_id,
|
|
53
|
+
...cold,
|
|
54
|
+
end_state: result.end_state,
|
|
55
|
+
end_state_matches: endStateMatches,
|
|
56
|
+
provenance,
|
|
57
|
+
},
|
|
58
|
+
recipe_applied: false,
|
|
59
|
+
end_state_matches: endStateMatches,
|
|
60
|
+
fallbacks: 0,
|
|
61
|
+
total_steps: 0,
|
|
62
|
+
});
|
|
63
|
+
}
|
|
64
|
+
return observations;
|
|
65
|
+
}
|
|
66
|
+
export function readFrozenAllColdBaseline(tasks) {
|
|
67
|
+
return tasks
|
|
68
|
+
.filter((task) => task.capture.status === "captured")
|
|
69
|
+
.map((task) => {
|
|
70
|
+
const cold = {
|
|
71
|
+
turns: task.cold_baseline.turns,
|
|
72
|
+
tokens: task.cold_baseline.tokens,
|
|
73
|
+
wall_clock_ms: task.cold_baseline.wall_clock_ms,
|
|
74
|
+
};
|
|
75
|
+
const endStateMatches = endStatesMatch(task.cold_baseline.end_state, task.expected_end_state);
|
|
76
|
+
return {
|
|
77
|
+
task_id: task.task_id,
|
|
78
|
+
bucket: task.bucket,
|
|
79
|
+
cold,
|
|
80
|
+
cold_recording: {
|
|
81
|
+
task_id: task.task_id,
|
|
82
|
+
...cold,
|
|
83
|
+
end_state: task.cold_baseline.end_state,
|
|
84
|
+
end_state_matches: endStateMatches,
|
|
85
|
+
provenance: task.cold_baseline.provenance,
|
|
86
|
+
},
|
|
87
|
+
recipe_applied: false,
|
|
88
|
+
end_state_matches: endStateMatches,
|
|
89
|
+
fallbacks: 0,
|
|
90
|
+
total_steps: 0,
|
|
91
|
+
};
|
|
92
|
+
});
|
|
93
|
+
}
|
|
94
|
+
export async function runAllColdHarness(tasks, drift, driver, options = {}) {
|
|
95
|
+
const recordedAt = options.recordedAt ?? options.generatedAt;
|
|
96
|
+
const observations = await recordAllColdBaseline(tasks, driver, {
|
|
97
|
+
...(recordedAt === undefined ? {} : { recordedAt }),
|
|
98
|
+
...(options.now === undefined ? {} : { now: options.now }),
|
|
99
|
+
});
|
|
100
|
+
for (const observation of observations) {
|
|
101
|
+
if (!observation.end_state_matches) {
|
|
102
|
+
throw new Error(`${observation.task_id}: cold baseline did not reach its expected end state`);
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
return buildHarnessReport(observations, drift, {
|
|
106
|
+
mode: "all-cold-baseline",
|
|
107
|
+
...(options.generatedAt === undefined ? {} : { generatedAt: options.generatedAt }),
|
|
108
|
+
});
|
|
109
|
+
}
|
|
110
|
+
export function runFrozenAllColdHarness(tasks, drift, generatedAt) {
|
|
111
|
+
const observations = readFrozenAllColdBaseline(tasks);
|
|
112
|
+
for (const observation of observations) {
|
|
113
|
+
if (!observation.end_state_matches) {
|
|
114
|
+
throw new Error(`${observation.task_id}: frozen cold baseline has the wrong end state`);
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
return buildHarnessReport(observations, drift, {
|
|
118
|
+
mode: "all-cold-baseline",
|
|
119
|
+
...(generatedAt === undefined ? {} : { generatedAt }),
|
|
120
|
+
});
|
|
121
|
+
}
|
|
122
|
+
export async function runDriftBattery(tasks, loadHar, replay) {
|
|
123
|
+
const observations = [];
|
|
124
|
+
for (const task of tasks) {
|
|
125
|
+
const sourceHar = await loadHar(task);
|
|
126
|
+
for (const mutation of DRIFT_MUTATIONS) {
|
|
127
|
+
const mutated = mutateHar(sourceHar, mutation, {
|
|
128
|
+
expectedTotalCents: task.expected_end_state.total_cents,
|
|
129
|
+
...(task.params.product_price_cents === undefined
|
|
130
|
+
? {}
|
|
131
|
+
: { displayedPriceCents: task.params.product_price_cents }),
|
|
132
|
+
});
|
|
133
|
+
if (mutated.replacements === 0) {
|
|
134
|
+
throw new Error(`${task.task_id}: ${mutation} did not mutate a HAR response`);
|
|
135
|
+
}
|
|
136
|
+
const result = await replay({ task, mutation, har: mutated.har });
|
|
137
|
+
const moneyAffecting = mutation === "change-price";
|
|
138
|
+
observations.push({
|
|
139
|
+
task_id: task.task_id,
|
|
140
|
+
mutation,
|
|
141
|
+
money_affecting: moneyAffecting,
|
|
142
|
+
guard_action: result.guard_action,
|
|
143
|
+
...(moneyAffecting
|
|
144
|
+
? {
|
|
145
|
+
total_verify_oracle: totalVerifyGuard(task.expected_end_state.total_cents, result.end_state.total_cents),
|
|
146
|
+
}
|
|
147
|
+
: {}),
|
|
148
|
+
end_state_matches: endStatesMatch(result.end_state, task.expected_end_state),
|
|
149
|
+
});
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
return observations;
|
|
153
|
+
}
|
|
154
|
+
export function totalVerifyGuard(expectedTotalCents, observedTotalCents) {
|
|
155
|
+
return expectedTotalCents === observedTotalCents ? "clean" : "abort";
|
|
156
|
+
}
|
|
157
|
+
export function moneyDriftObservation(task, observedTotalCents, observedGuardAction) {
|
|
158
|
+
return {
|
|
159
|
+
task_id: task.task_id,
|
|
160
|
+
mutation: "change-price",
|
|
161
|
+
money_affecting: true,
|
|
162
|
+
guard_action: observedGuardAction,
|
|
163
|
+
total_verify_oracle: totalVerifyGuard(task.expected_end_state.total_cents, observedTotalCents),
|
|
164
|
+
end_state_matches: observedTotalCents === task.expected_end_state.total_cents,
|
|
165
|
+
};
|
|
166
|
+
}
|
|
167
|
+
//# sourceMappingURL=runner.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"runner.js","sourceRoot":"","sources":["../../../src/eval/replay-harness/runner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAC9C,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAC7C,OAAO,EAAE,SAAS,EAAgB,MAAM,iBAAiB,CAAC;AAC1D,OAAO,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAC;AAYlD,MAAM,eAAe,GAAG;IACtB,eAAe;IACf,aAAa;IACb,cAAc;IACd,cAAc;IACd,cAAc;IACd,SAAS;CACsC,CAAC;AAgClD,SAAS,kBAAkB,CAAC,MAAc,EAAE,IAAgB;IAC1D,IACE,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC;QAC5B,IAAI,CAAC,KAAK,GAAG,CAAC;QACd,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC;QAC7B,IAAI,CAAC,MAAM,GAAG,CAAC;QACf,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,aAAa,CAAC;QACpC,IAAI,CAAC,aAAa,GAAG,CAAC,EACtB,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,gDAAgD,CAAC,CAAC;IAC7E,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,qBAAqB,CACzC,KAA2B,EAC3B,MAA0B,EAC1B,UAAsC,EAAE;IAExC,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC;IACrF,IAAI,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,+BAA+B,CAAC,CAAC;IACvF,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IAC7D,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAClE,MAAM,UAAU,GAA2B;QACzC,MAAM,EAAE,QAAQ;QAChB,MAAM,EAAE,MAAM,CAAC,IAAI;QACnB,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,WAAW,EAAE,UAAU;KACxB,CAAC;IACF,MAAM,YAAY,GAAsB,EAAE,CAAC;IAC3C,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,MAAM,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,OAAO,CAAC,MAAM,KAAK,UAAU,CAAC,EAAE,CAAC;QACxF,MAAM,OAAO,GAAG,GAAG,EAAE,CAAC;QACtB,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACxC,MAAM,IAAI,GAAG;YACX,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,aAAa,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC,CAAC;SACxD,CAAC;QACF,kBAAkB,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;QACvC,MAAM,eAAe,GAAG,cAAc,CAAC,MAAM,CAAC,SAAS,EAAE,IAAI,CAAC,kBAAkB,CAAC,CAAC;QAClF,YAAY,CAAC,IAAI,CAAC;YAChB,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,IAAI;YACJ,cAAc,EAAE;gBACd,OAAO,EAAE,IAAI,CAAC,OAAO;gBACrB,GAAG,IAAI;gBACP,SAAS,EAAE,MAAM,CAAC,SAAS;gBAC3B,iBAAiB,EAAE,eAAe;gBAClC,UAAU;aACX;YACD,cAAc,EAAE,KAAK;YACrB,iBAAiB,EAAE,eAAe;YAClC,SAAS,EAAE,CAAC;YACZ,WAAW,EAAE,CAAC;SACf,CAAC,CAAC;IACL,CAAC;IACD,OAAO,YAAY,CAAC;AACtB,CAAC;AAED,MAAM,UAAU,yBAAyB,CAAC,KAA2B;IACnE,OAAO,KAAK;SACT,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,KAAK,UAAU,CAAC;SACpD,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACZ,MAAM,IAAI,GAAG;YACX,KAAK,EAAE,IAAI,CAAC,aAAa,CAAC,KAAK;YAC/B,MAAM,EAAE,IAAI,CAAC,aAAa,CAAC,MAAM;YACjC,aAAa,EAAE,IAAI,CAAC,aAAa,CAAC,aAAa;SAChD,CAAC;QACF,MAAM,eAAe,GAAG,cAAc,CAAC,IAAI,CAAC,aAAa,CAAC,SAAS,EAAE,IAAI,CAAC,kBAAkB,CAAC,CAAC;QAC9F,OAAO;YACL,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,IAAI;YACJ,cAAc,EAAE;gBACd,OAAO,EAAE,IAAI,CAAC,OAAO;gBACrB,GAAG,IAAI;gBACP,SAAS,EAAE,IAAI,CAAC,aAAa,CAAC,SAAS;gBACvC,iBAAiB,EAAE,eAAe;gBAClC,UAAU,EAAE,IAAI,CAAC,aAAa,CAAC,UAAU;aAC1C;YACD,cAAc,EAAE,KAAK;YACrB,iBAAiB,EAAE,eAAe;YAClC,SAAS,EAAE,CAAC;YACZ,WAAW,EAAE,CAAC;SACW,CAAC;IAC9B,CAAC,CAAC,CAAC;AACP,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,KAA2B,EAC3B,KAAyB,EACzB,MAA0B,EAC1B,UAAiE,EAAE;IAEnE,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,OAAO,CAAC,WAAW,CAAC;IAC7D,MAAM,YAAY,GAAG,MAAM,qBAAqB,CAAC,KAAK,EAAE,MAAM,EAAE;QAC9D,GAAG,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC;QACnD,GAAG,CAAC,OAAO,CAAC,GAAG,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC;KAC3D,CAAC,CAAC;IACH,KAAK,MAAM,WAAW,IAAI,YAAY,EAAE,CAAC;QACvC,IAAI,CAAC,WAAW,CAAC,iBAAiB,EAAE,CAAC;YACnC,MAAM,IAAI,KAAK,CAAC,GAAG,WAAW,CAAC,OAAO,sDAAsD,CAAC,CAAC;QAChG,CAAC;IACH,CAAC;IACD,OAAO,kBAAkB,CAAC,YAAY,EAAE,KAAK,EAAE;QAC7C,IAAI,EAAE,mBAAmB;QACzB,GAAG,CAAC,OAAO,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,OAAO,CAAC,WAAW,EAAE,CAAC;KACnF,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,uBAAuB,CACrC,KAA2B,EAC3B,KAAyB,EACzB,WAAoB;IAEpB,MAAM,YAAY,GAAG,yBAAyB,CAAC,KAAK,CAAC,CAAC;IACtD,KAAK,MAAM,WAAW,IAAI,YAAY,EAAE,CAAC;QACvC,IAAI,CAAC,WAAW,CAAC,iBAAiB,EAAE,CAAC;YACnC,MAAM,IAAI,KAAK,CAAC,GAAG,WAAW,CAAC,OAAO,gDAAgD,CAAC,CAAC;QAC1F,CAAC;IACH,CAAC;IACD,OAAO,kBAAkB,CAAC,YAAY,EAAE,KAAK,EAAE;QAC7C,IAAI,EAAE,mBAAmB;QACzB,GAAG,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC;KACtD,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,KAA2B,EAC3B,OAAiE,EACjE,MAA0B;IAE1B,MAAM,YAAY,GAAuB,EAAE,CAAC;IAC5C,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;QACtC,KAAK,MAAM,QAAQ,IAAI,eAAe,EAAE,CAAC;YACvC,MAAM,OAAO,GAAG,SAAS,CAAC,SAAS,EAAE,QAAQ,EAAE;gBAC7C,kBAAkB,EAAE,IAAI,CAAC,kBAAkB,CAAC,WAAW;gBACvD,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,mBAAmB,KAAK,SAAS;oBAC/C,CAAC,CAAC,EAAE;oBACJ,CAAC,CAAC,EAAE,mBAAmB,EAAE,IAAI,CAAC,MAAM,CAAC,mBAAmB,EAAE,CAAC;aAC9D,CAAC,CAAC;YACH,IAAI,OAAO,CAAC,YAAY,KAAK,CAAC,EAAE,CAAC;gBAC/B,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,CAAC,OAAO,KAAK,QAAQ,gCAAgC,CAAC,CAAC;YAChF,CAAC;YACD,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;YAClE,MAAM,cAAc,GAAG,QAAQ,KAAK,cAAc,CAAC;YACnD,YAAY,CAAC,IAAI,CAAC;gBAChB,OAAO,EAAE,IAAI,CAAC,OAAO;gBACrB,QAAQ;gBACR,eAAe,EAAE,cAAc;gBAC/B,YAAY,EAAE,MAAM,CAAC,YAAY;gBACjC,GAAG,CAAC,cAAc;oBAChB,CAAC,CAAC;wBACE,mBAAmB,EAAE,gBAAgB,CACnC,IAAI,CAAC,kBAAkB,CAAC,WAAW,EACnC,MAAM,CAAC,SAAS,CAAC,WAAW,CAC7B;qBACF;oBACH,CAAC,CAAC,EAAE,CAAC;gBACP,iBAAiB,EAAE,cAAc,CAAC,MAAM,CAAC,SAAS,EAAE,IAAI,CAAC,kBAAkB,CAAC;aAC7E,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IACD,OAAO,YAAY,CAAC;AACtB,CAAC;AAED,MAAM,UAAU,gBAAgB,CAC9B,kBAA0B,EAC1B,kBAA0B;IAE1B,OAAO,kBAAkB,KAAK,kBAAkB,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC;AACvE,CAAC;AAED,MAAM,UAAU,qBAAqB,CACnC,IAAwB,EACxB,kBAA0B,EAC1B,mBAAqD;IAErD,OAAO;QACL,OAAO,EAAE,IAAI,CAAC,OAAO;QACrB,QAAQ,EAAE,cAAc;QACxB,eAAe,EAAE,IAAI;QACrB,YAAY,EAAE,mBAAmB;QACjC,mBAAmB,EAAE,gBAAgB,CAAC,IAAI,CAAC,kBAAkB,CAAC,WAAW,EAAE,kBAAkB,CAAC;QAC9F,iBAAiB,EAAE,kBAAkB,KAAK,IAAI,CAAC,kBAAkB,CAAC,WAAW;KAC9E,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
export type CorpusBucket = "repeat" | "novel";
|
|
2
|
+
export interface ShoppingAddress {
|
|
3
|
+
country: string;
|
|
4
|
+
postal_code: string;
|
|
5
|
+
region?: string;
|
|
6
|
+
}
|
|
7
|
+
export interface ExpectedEndState {
|
|
8
|
+
line_items: Array<{
|
|
9
|
+
title_contains: string;
|
|
10
|
+
qty: number;
|
|
11
|
+
}>;
|
|
12
|
+
total_cents: number;
|
|
13
|
+
reached: string;
|
|
14
|
+
}
|
|
15
|
+
export interface CostSample {
|
|
16
|
+
turns: number;
|
|
17
|
+
tokens: number;
|
|
18
|
+
wall_clock_ms: number;
|
|
19
|
+
}
|
|
20
|
+
export interface ColdBaseline extends CostSample {
|
|
21
|
+
end_state: ExpectedEndState;
|
|
22
|
+
provenance: ColdBaselineProvenance;
|
|
23
|
+
}
|
|
24
|
+
export interface ColdBaselineProvenance {
|
|
25
|
+
source: "driver";
|
|
26
|
+
driver: string;
|
|
27
|
+
model: string;
|
|
28
|
+
recorded_at: string;
|
|
29
|
+
browser_observations?: number;
|
|
30
|
+
evidence_sha256?: string;
|
|
31
|
+
capture_policy?: "read-only-playwright-mcp-v2" | "read-only-playwright-mcp-v3";
|
|
32
|
+
}
|
|
33
|
+
export interface ColdBaselineRecording extends CostSample {
|
|
34
|
+
task_id: string;
|
|
35
|
+
end_state: ExpectedEndState;
|
|
36
|
+
end_state_matches: boolean;
|
|
37
|
+
provenance: ColdBaselineProvenance;
|
|
38
|
+
}
|
|
39
|
+
export interface ShoppingTaskRecord {
|
|
40
|
+
task_id: string;
|
|
41
|
+
verb: "purchase";
|
|
42
|
+
domain: string;
|
|
43
|
+
entry_url: string;
|
|
44
|
+
params: {
|
|
45
|
+
product_query: string;
|
|
46
|
+
product_variant_id?: string;
|
|
47
|
+
product_price_cents?: number;
|
|
48
|
+
address: ShoppingAddress;
|
|
49
|
+
card_ref: string;
|
|
50
|
+
};
|
|
51
|
+
expected_end_state: ExpectedEndState;
|
|
52
|
+
har: string;
|
|
53
|
+
bucket: CorpusBucket;
|
|
54
|
+
cold_baseline: ColdBaseline;
|
|
55
|
+
capture: {
|
|
56
|
+
status: "captured" | "skipped";
|
|
57
|
+
captured_at?: string;
|
|
58
|
+
skip_reason?: string;
|
|
59
|
+
};
|
|
60
|
+
}
|
|
61
|
+
export interface TaskObservation {
|
|
62
|
+
task_id: string;
|
|
63
|
+
bucket: CorpusBucket;
|
|
64
|
+
cold: CostSample;
|
|
65
|
+
cold_recording?: ColdBaselineRecording;
|
|
66
|
+
recipe_applied: boolean;
|
|
67
|
+
warm?: CostSample;
|
|
68
|
+
end_state_matches: boolean;
|
|
69
|
+
fallbacks: number;
|
|
70
|
+
total_steps: number;
|
|
71
|
+
}
|
|
72
|
+
export type DriftMutationName = "rename-button" | "swap-testid" | "remove-field" | "change-price" | "out-of-stock" | "overlay";
|
|
73
|
+
export interface DriftObservation {
|
|
74
|
+
task_id: string;
|
|
75
|
+
mutation: DriftMutationName;
|
|
76
|
+
money_affecting: boolean;
|
|
77
|
+
guard_action: "clean" | "fallback" | "abort" | "missed";
|
|
78
|
+
total_verify_oracle?: "clean" | "abort";
|
|
79
|
+
end_state_matches: boolean;
|
|
80
|
+
}
|
|
81
|
+
export interface HarnessThresholds {
|
|
82
|
+
net_speedup: number;
|
|
83
|
+
clean_replay_correctness: number;
|
|
84
|
+
task_success: number;
|
|
85
|
+
drift_catch_rate: number;
|
|
86
|
+
money_escape: number;
|
|
87
|
+
}
|
|
88
|
+
export interface HarnessMetrics {
|
|
89
|
+
speedup_on_hit: {
|
|
90
|
+
turns: number;
|
|
91
|
+
tokens: number;
|
|
92
|
+
wall_clock: number;
|
|
93
|
+
};
|
|
94
|
+
hit_rate: number;
|
|
95
|
+
net_speedup: {
|
|
96
|
+
turns: number;
|
|
97
|
+
tokens: number;
|
|
98
|
+
wall_clock: number;
|
|
99
|
+
};
|
|
100
|
+
clean_replay_correctness: number;
|
|
101
|
+
task_success: number;
|
|
102
|
+
fallback_rate: number;
|
|
103
|
+
money_escape: number;
|
|
104
|
+
drift_catch_rate: number;
|
|
105
|
+
recipe_survival: {
|
|
106
|
+
t7d: number | null;
|
|
107
|
+
t30d: number | null;
|
|
108
|
+
};
|
|
109
|
+
invariants: {
|
|
110
|
+
novel_false_hits: number;
|
|
111
|
+
missing_warm_samples: number;
|
|
112
|
+
};
|
|
113
|
+
}
|
|
114
|
+
export interface HarnessReport {
|
|
115
|
+
schema_version: 1;
|
|
116
|
+
mode: "all-cold-baseline" | "replay-eval";
|
|
117
|
+
generated_at: string;
|
|
118
|
+
corpus: {
|
|
119
|
+
tasks: number;
|
|
120
|
+
repeat: number;
|
|
121
|
+
novel: number;
|
|
122
|
+
drift_trials: number;
|
|
123
|
+
};
|
|
124
|
+
cold_baseline: {
|
|
125
|
+
tasks: number;
|
|
126
|
+
median: CostSample;
|
|
127
|
+
total: CostSample;
|
|
128
|
+
recordings: ColdBaselineRecording[];
|
|
129
|
+
};
|
|
130
|
+
thresholds: HarnessThresholds;
|
|
131
|
+
metrics: HarnessMetrics;
|
|
132
|
+
decision: "SHIP" | "NO-SHIP";
|
|
133
|
+
reasons: string[];
|
|
134
|
+
}
|
|
135
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/eval/replay-harness/types.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,YAAY,GAAG,QAAQ,GAAG,OAAO,CAAC;AAE9C,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,gBAAgB;IAC/B,UAAU,EAAE,KAAK,CAAC;QAAE,cAAc,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC3D,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,aAAa,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,YAAa,SAAQ,UAAU;IAC9C,SAAS,EAAE,gBAAgB,CAAC;IAC5B,UAAU,EAAE,sBAAsB,CAAC;CACpC;AAED,MAAM,WAAW,sBAAsB;IACrC,MAAM,EAAE,QAAQ,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,cAAc,CAAC,EAAE,6BAA6B,GAAG,6BAA6B,CAAC;CAChF;AAED,MAAM,WAAW,qBAAsB,SAAQ,UAAU;IACvD,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,gBAAgB,CAAC;IAC5B,iBAAiB,EAAE,OAAO,CAAC;IAC3B,UAAU,EAAE,sBAAsB,CAAC;CACpC;AAED,MAAM,WAAW,kBAAkB;IACjC,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,UAAU,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE;QACN,aAAa,EAAE,MAAM,CAAC;QACtB,kBAAkB,CAAC,EAAE,MAAM,CAAC;QAC5B,mBAAmB,CAAC,EAAE,MAAM,CAAC;QAC7B,OAAO,EAAE,eAAe,CAAC;QACzB,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,kBAAkB,EAAE,gBAAgB,CAAC;IACrC,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,YAAY,CAAC;IACrB,aAAa,EAAE,YAAY,CAAC;IAC5B,OAAO,EAAE;QACP,MAAM,EAAE,UAAU,GAAG,SAAS,CAAC;QAC/B,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,WAAW,CAAC,EAAE,MAAM,CAAC;KACtB,CAAC;CACH;AAED,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,YAAY,CAAC;IACrB,IAAI,EAAE,UAAU,CAAC;IACjB,cAAc,CAAC,EAAE,qBAAqB,CAAC;IACvC,cAAc,EAAE,OAAO,CAAC;IACxB,IAAI,CAAC,EAAE,UAAU,CAAC;IAClB,iBAAiB,EAAE,OAAO,CAAC;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,MAAM,iBAAiB,GACzB,eAAe,GACf,aAAa,GACb,cAAc,GACd,cAAc,GACd,cAAc,GACd,SAAS,CAAC;AAEd,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,iBAAiB,CAAC;IAC5B,eAAe,EAAE,OAAO,CAAC;IACzB,YAAY,EAAE,OAAO,GAAG,UAAU,GAAG,OAAO,GAAG,QAAQ,CAAC;IACxD,mBAAmB,CAAC,EAAE,OAAO,GAAG,OAAO,CAAC;IACxC,iBAAiB,EAAE,OAAO,CAAC;CAC5B;AAED,MAAM,WAAW,iBAAiB;IAChC,WAAW,EAAE,MAAM,CAAC;IACpB,wBAAwB,EAAE,MAAM,CAAC;IACjC,YAAY,EAAE,MAAM,CAAC;IACrB,gBAAgB,EAAE,MAAM,CAAC;IACzB,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,cAAc;IAC7B,cAAc,EAAE;QACd,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,EAAE,MAAM,CAAC;QACf,UAAU,EAAE,MAAM,CAAC;KACpB,CAAC;IACF,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE;QACX,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,EAAE,MAAM,CAAC;QACf,UAAU,EAAE,MAAM,CAAC;KACpB,CAAC;IACF,wBAAwB,EAAE,MAAM,CAAC;IACjC,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,YAAY,EAAE,MAAM,CAAC;IACrB,gBAAgB,EAAE,MAAM,CAAC;IACzB,eAAe,EAAE;QAAE,GAAG,EAAE,MAAM,GAAG,IAAI,CAAC;QAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAA;KAAE,CAAC;IAC7D,UAAU,EAAE;QACV,gBAAgB,EAAE,MAAM,CAAC;QACzB,oBAAoB,EAAE,MAAM,CAAC;KAC9B,CAAC;CACH;AAED,MAAM,WAAW,aAAa;IAC5B,cAAc,EAAE,CAAC,CAAC;IAClB,IAAI,EAAE,mBAAmB,GAAG,aAAa,CAAC;IAC1C,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE;QACN,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,EAAE,MAAM,CAAC;QACf,KAAK,EAAE,MAAM,CAAC;QACd,YAAY,EAAE,MAAM,CAAC;KACtB,CAAC;IACF,aAAa,EAAE;QACb,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,EAAE,UAAU,CAAC;QACnB,KAAK,EAAE,UAAU,CAAC;QAClB,UAAU,EAAE,qBAAqB,EAAE,CAAC;KACrC,CAAC;IACF,UAAU,EAAE,iBAAiB,CAAC;IAC9B,OAAO,EAAE,cAAc,CAAC;IACxB,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7B,OAAO,EAAE,MAAM,EAAE,CAAC;CACnB"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../src/eval/replay-harness/types.ts"],"names":[],"mappings":""}
|
package/dist/server.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../src/server.ts"],"names":[],"mappings":"AAaA,OAAO,EAAE,MAAM,EAAE,MAAM,2CAA2C,CAAC;AAGnE,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAkC5C,wBAAsB,WAAW,CAAC,GAAG,EAAE,SAAS,GAAG,IAAI,GAAG,OAAO,CAAC,MAAM,CAAC,
|
|
1
|
+
{"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../src/server.ts"],"names":[],"mappings":"AAaA,OAAO,EAAE,MAAM,EAAE,MAAM,2CAA2C,CAAC;AAGnE,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAkC5C,wBAAsB,WAAW,CAAC,GAAG,EAAE,SAAS,GAAG,IAAI,GAAG,OAAO,CAAC,MAAM,CAAC,CAwDxE;AAWD,wBAAsB,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC,CA2B/C"}
|
package/dist/server.js
CHANGED
|
@@ -70,6 +70,7 @@ export async function buildServer(api) {
|
|
|
70
70
|
return errorContent(`invalid arguments: ${parsed.error.issues.map((i) => i.message).join("; ")}`);
|
|
71
71
|
}
|
|
72
72
|
try {
|
|
73
|
+
api.setRequestingAgent(server.getClientVersion()?.name ?? "unknown-agent");
|
|
73
74
|
const result = await tool.handler(parsed.data, api, {
|
|
74
75
|
notifyUser: async (message, data) => {
|
|
75
76
|
await server.sendLoggingMessage({
|
package/dist/server.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"server.js","sourceRoot":"","sources":["../src/server.ts"],"names":[],"mappings":"AAAA,uEAAuE;AACvE,wEAAwE;AACxE,cAAc;AACd,EAAE;AACF,uEAAuE;AACvE,wEAAwE;AACxE,sEAAsE;AACtE,EAAE;AACF,qEAAqE;AACrE,+DAA+D;AAC/D,qEAAqE;AACrE,6DAA6D;AAE7D,OAAO,EAAE,MAAM,EAAE,MAAM,2CAA2C,CAAC;AACnE,OAAO,EAAE,oBAAoB,EAAE,MAAM,2CAA2C,CAAC;AACjF,OAAO,EAAE,qBAAqB,EAAE,sBAAsB,EAAE,MAAM,oCAAoC,CAAC;AACnG,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AACnD,OAAO,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAC;AAClD,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAEvC,MAAM,WAAW,GAAG,eAAe,CAAC;AAEpC,MAAM,qBAAqB,GACzB,OAAO,CAAC,GAAG,CAAC,oBAAoB,IAAI,kCAAkC,CAAC;AAEzE,qEAAqE;AACrE,qEAAqE;AACrE,8DAA8D;AAC9D,MAAM,mBAAmB,GAAG;;;;;;;;;;;;;;;;;;;sCAmBU,CAAC;AAEvC,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,GAAqB;IACrD,MAAM,MAAM,GAAG,IAAI,MAAM,CACvB,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,OAAO,EAAE,EACvC,EAAE,YAAY,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,EAAE,YAAY,EAAE,mBAAmB,EAAE,CACnE,CAAC;IAEF,MAAM,CAAC,iBAAiB,CAAC,sBAAsB,EAAE,KAAK,IAAI,EAAE,CAAC,CAAC;QAC5D,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACvB,IAAI,EAAE,CAAC,CAAC,IAAI;YACZ,WAAW,EAAE,CAAC,CAAC,WAAW;YAC1B,WAAW,EAAE,CAAC,CAAC,eAAe;YAC9B,GAAG,CAAC,CAAC,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACtE,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACnD,CAAC,CAAC;KACJ,CAAC,CAAC,CAAC;IAEJ,MAAM,CAAC,iBAAiB,CAAC,qBAAqB,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;QAC5D,MAAM,IAAI,GAAG,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QACvC,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;YAClB,OAAO,YAAY,CAAC,iBAAiB,GAAG,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC,CAAC;QAC3D,CAAC;QACD,+DAA+D;QAC/D,gEAAgE;QAChE,kDAAkD;QAClD,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;YACjB,OAAO,YAAY,CACjB,8EAA8E;gBAC5E,sDAAsD,CACzD,CAAC;QACJ,CAAC;QACD,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC;QACtE,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,OAAO,YAAY,CACjB,sBAAsB,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAC7E,CAAC;QACJ,CAAC;QACD,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,GAAG,EAAE;gBAClD,UAAU,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE;oBAClC,MAAM,MAAM,CAAC,kBAAkB,CAAC;wBAC9B,KAAK,EAAE,QAAQ;wBACf,MAAM,EAAE,eAAe;wBACvB,IAAI,EAAE,EAAE,OAAO,EAAE,GAAG,IAAI,EAAE;qBAC3B,CAAC,CAAC;gBACL,CAAC;aACF,CAAC,CAAC;YACH,OAAO;gBACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC;aACnE,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO,YAAY,CAAC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;QACxE,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,YAAY,CAAC,OAAe;IACnC,OAAO;QACL,OAAO,EAAE,IAAI;QACb,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAe,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;KACpD,CAAC;AACJ,CAAC;AAED,wEAAwE;AACxE,yCAAyC;AACzC,MAAM,CAAC,KAAK,UAAU,SAAS;IAC7B,oEAAoE;IACpE,kEAAkE;IAClE,oEAAoE;IACpE,eAAe;IACf,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,2BAA2B,OAAO,aAAa,CAAC,CAAC;IAEtE,MAAM,OAAO,GAAG,MAAM,kBAAkB,EAAE,CAAC;IAC3C,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC;IAErC,qEAAqE;IACrE,kEAAkE;IAClE,gDAAgD;IAChD,MAAM,GAAG,GACP,OAAO,KAAK,IAAI,IAAI,OAAO,CAAC,mBAAmB,KAAK,SAAS;QAC3D,CAAC,CAAC,IAAI,SAAS,CAAC;YACZ,UAAU,EAAE,OAAO,CAAC,YAAY;YAChC,eAAe,EAAE,qBAAqB;YACtC,iBAAiB,EAAE,OAAO,CAAC,mBAAmB;YAC9C,aAAa,EAAE,OAAO,CAAC,GAAG,CAAC,4BAA4B,IAAI,SAAS;YACpE,GAAG,CAAC,OAAO,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAC/E,CAAC;QACJ,CAAC,CAAC,IAAI,CAAC;IAEX,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,GAAG,CAAC,CAAC;IACtC,MAAM,SAAS,GAAG,IAAI,oBAAoB,EAAE,CAAC;IAC7C,MAAM,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;AAClC,CAAC"}
|
|
1
|
+
{"version":3,"file":"server.js","sourceRoot":"","sources":["../src/server.ts"],"names":[],"mappings":"AAAA,uEAAuE;AACvE,wEAAwE;AACxE,cAAc;AACd,EAAE;AACF,uEAAuE;AACvE,wEAAwE;AACxE,sEAAsE;AACtE,EAAE;AACF,qEAAqE;AACrE,+DAA+D;AAC/D,qEAAqE;AACrE,6DAA6D;AAE7D,OAAO,EAAE,MAAM,EAAE,MAAM,2CAA2C,CAAC;AACnE,OAAO,EAAE,oBAAoB,EAAE,MAAM,2CAA2C,CAAC;AACjF,OAAO,EAAE,qBAAqB,EAAE,sBAAsB,EAAE,MAAM,oCAAoC,CAAC;AACnG,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AACnD,OAAO,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAC;AAClD,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAEvC,MAAM,WAAW,GAAG,eAAe,CAAC;AAEpC,MAAM,qBAAqB,GACzB,OAAO,CAAC,GAAG,CAAC,oBAAoB,IAAI,kCAAkC,CAAC;AAEzE,qEAAqE;AACrE,qEAAqE;AACrE,8DAA8D;AAC9D,MAAM,mBAAmB,GAAG;;;;;;;;;;;;;;;;;;;sCAmBU,CAAC;AAEvC,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,GAAqB;IACrD,MAAM,MAAM,GAAG,IAAI,MAAM,CACvB,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,OAAO,EAAE,EACvC,EAAE,YAAY,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,EAAE,YAAY,EAAE,mBAAmB,EAAE,CACnE,CAAC;IAEF,MAAM,CAAC,iBAAiB,CAAC,sBAAsB,EAAE,KAAK,IAAI,EAAE,CAAC,CAAC;QAC5D,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACvB,IAAI,EAAE,CAAC,CAAC,IAAI;YACZ,WAAW,EAAE,CAAC,CAAC,WAAW;YAC1B,WAAW,EAAE,CAAC,CAAC,eAAe;YAC9B,GAAG,CAAC,CAAC,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACtE,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACnD,CAAC,CAAC;KACJ,CAAC,CAAC,CAAC;IAEJ,MAAM,CAAC,iBAAiB,CAAC,qBAAqB,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;QAC5D,MAAM,IAAI,GAAG,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QACvC,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;YAClB,OAAO,YAAY,CAAC,iBAAiB,GAAG,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC,CAAC;QAC3D,CAAC;QACD,+DAA+D;QAC/D,gEAAgE;QAChE,kDAAkD;QAClD,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;YACjB,OAAO,YAAY,CACjB,8EAA8E;gBAC5E,sDAAsD,CACzD,CAAC;QACJ,CAAC;QACD,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC;QACtE,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,OAAO,YAAY,CACjB,sBAAsB,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAC7E,CAAC;QACJ,CAAC;QACD,IAAI,CAAC;YACH,GAAG,CAAC,kBAAkB,CAAC,MAAM,CAAC,gBAAgB,EAAE,EAAE,IAAI,IAAI,eAAe,CAAC,CAAC;YAC3E,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,GAAG,EAAE;gBAClD,UAAU,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE;oBAClC,MAAM,MAAM,CAAC,kBAAkB,CAAC;wBAC9B,KAAK,EAAE,QAAQ;wBACf,MAAM,EAAE,eAAe;wBACvB,IAAI,EAAE,EAAE,OAAO,EAAE,GAAG,IAAI,EAAE;qBAC3B,CAAC,CAAC;gBACL,CAAC;aACF,CAAC,CAAC;YACH,OAAO;gBACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC;aACnE,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO,YAAY,CAAC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;QACxE,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,YAAY,CAAC,OAAe;IACnC,OAAO;QACL,OAAO,EAAE,IAAI;QACb,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAe,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;KACpD,CAAC;AACJ,CAAC;AAED,wEAAwE;AACxE,yCAAyC;AACzC,MAAM,CAAC,KAAK,UAAU,SAAS;IAC7B,oEAAoE;IACpE,kEAAkE;IAClE,oEAAoE;IACpE,eAAe;IACf,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,2BAA2B,OAAO,aAAa,CAAC,CAAC;IAEtE,MAAM,OAAO,GAAG,MAAM,kBAAkB,EAAE,CAAC;IAC3C,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC;IAErC,qEAAqE;IACrE,kEAAkE;IAClE,gDAAgD;IAChD,MAAM,GAAG,GACP,OAAO,KAAK,IAAI,IAAI,OAAO,CAAC,mBAAmB,KAAK,SAAS;QAC3D,CAAC,CAAC,IAAI,SAAS,CAAC;YACZ,UAAU,EAAE,OAAO,CAAC,YAAY;YAChC,eAAe,EAAE,qBAAqB;YACtC,iBAAiB,EAAE,OAAO,CAAC,mBAAmB;YAC9C,aAAa,EAAE,OAAO,CAAC,GAAG,CAAC,4BAA4B,IAAI,SAAS;YACpE,GAAG,CAAC,OAAO,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAC/E,CAAC;QACJ,CAAC,CAAC,IAAI,CAAC;IAEX,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,GAAG,CAAC,CAAC;IACtC,MAAM,SAAS,GAAG,IAAI,oBAAoB,EAAE,CAAC;IAC7C,MAAM,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;AAClC,CAAC"}
|
|
@@ -4,7 +4,7 @@ import { ALWAYS_LOAD_META } from "./always-load.js";
|
|
|
4
4
|
// Egress Grants — mint a standing, revocable token so a DEPLOYED app (a server
|
|
5
5
|
// the agent provisioned, a loop runtime) can call a provider through Squire's
|
|
6
6
|
// injecting proxy. The raw vault key never leaves Squire; the app holds only a
|
|
7
|
-
// downgraded, rate-
|
|
7
|
+
// downgraded, rate-limitable, revocable token. This is use_credential generalized
|
|
8
8
|
// from "the agent makes a call" to "a workload makes calls, forever."
|
|
9
9
|
const inputSchema = z
|
|
10
10
|
.object({
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"grant-app-access.js","sourceRoot":"","sources":["../../src/tools/grant-app-access.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,SAAS,EAAa,MAAM,YAAY,CAAC;AAClD,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAEpD,+EAA+E;AAC/E,8EAA8E;AAC9E,+EAA+E;AAC/E,
|
|
1
|
+
{"version":3,"file":"grant-app-access.js","sourceRoot":"","sources":["../../src/tools/grant-app-access.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,SAAS,EAAa,MAAM,YAAY,CAAC;AAClD,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAEpD,+EAA+E;AAC/E,8EAA8E;AAC9E,+EAA+E;AAC/E,kFAAkF;AAClF,sEAAsE;AACtE,MAAM,WAAW,GAAG,CAAC;KAClB,MAAM,CAAC;IACN,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IACvC,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IACrC,mBAAmB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE;IACnE,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;CAC5C,CAAC;KACD,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,SAAS,IAAI,CAAC,CAAC,OAAO,KAAK,SAAS,EAAE;IACnE,OAAO,EAAE,yCAAyC;CACnD,CAAC,CAAC;AAEL,MAAM,WAAW,GAAG;;;;;;;;;;;;;;;;;;;UAmBV,CAAC;AAEX,MAAM,CAAC,MAAM,kBAAkB,GAAsC;IACnE,IAAI,EAAE,kBAAkB;IACxB,WAAW,EAAE,WAAW;IACxB,WAAW;IACX,eAAe,EAAE;QACf,IAAI,EAAE,QAAQ;QACd,UAAU,EAAE;YACV,SAAS,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YAC7B,OAAO,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YAC3B,mBAAmB,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YACvC,aAAa,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;SAClC;KACF;IACD,IAAI,EAAE,gBAAgB;IACtB,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG;QACrB,SAAS,CAAC,GAAG,CAAC,CAAC;QACf,MAAM,GAAG,GAAG,MAAM,GAAG,CAAC,cAAc,CAAC;YACnC,GAAG,CAAC,IAAI,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACtE,GAAG,CAAC,IAAI,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAChE,GAAG,CAAC,IAAI,CAAC,mBAAmB,KAAK,SAAS;gBACxC,CAAC,CAAC,EAAE,mBAAmB,EAAE,IAAI,CAAC,mBAAmB,EAAE;gBACnD,CAAC,CAAC,EAAE,CAAC;YACP,GAAG,CAAC,IAAI,CAAC,aAAa,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,IAAI,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACnF,CAAC,CAAC;QACH,OAAO,GAAG,CAAC;IACb,CAAC;CACF,CAAC"}
|