@tangle-network/agent-eval 0.120.1 → 0.120.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (171) hide show
  1. package/CHANGELOG.md +6 -0
  2. package/package.json +1 -1
  3. package/dist/analyst/index.d.ts +0 -3111
  4. package/dist/analyst/index.js +0 -403
  5. package/dist/analyst/index.js.map +0 -1
  6. package/dist/authenticity/index.d.ts +0 -161
  7. package/dist/authenticity/index.js +0 -215
  8. package/dist/authenticity/index.js.map +0 -1
  9. package/dist/belief-state/index.d.ts +0 -1301
  10. package/dist/belief-state/index.js +0 -2152
  11. package/dist/belief-state/index.js.map +0 -1
  12. package/dist/benchmarks/index.d.ts +0 -974
  13. package/dist/benchmarks/index.js +0 -60
  14. package/dist/benchmarks/index.js.map +0 -1
  15. package/dist/builder-eval/index.d.ts +0 -695
  16. package/dist/builder-eval/index.js +0 -366
  17. package/dist/builder-eval/index.js.map +0 -1
  18. package/dist/campaign/index.d.ts +0 -7454
  19. package/dist/campaign/index.js +0 -272
  20. package/dist/campaign/index.js.map +0 -1
  21. package/dist/chunk-32BZXMSO.js +0 -3878
  22. package/dist/chunk-32BZXMSO.js.map +0 -1
  23. package/dist/chunk-3A246TSA.js +0 -998
  24. package/dist/chunk-3A246TSA.js.map +0 -1
  25. package/dist/chunk-3RF76KTD.js +0 -84
  26. package/dist/chunk-3RF76KTD.js.map +0 -1
  27. package/dist/chunk-3YYRZDON.js +0 -45
  28. package/dist/chunk-3YYRZDON.js.map +0 -1
  29. package/dist/chunk-4I2E3LLO.js +0 -1030
  30. package/dist/chunk-4I2E3LLO.js.map +0 -1
  31. package/dist/chunk-ARU2PZFM.js +0 -312
  32. package/dist/chunk-ARU2PZFM.js.map +0 -1
  33. package/dist/chunk-BOD4O7OF.js +0 -40
  34. package/dist/chunk-BOD4O7OF.js.map +0 -1
  35. package/dist/chunk-DPZAEKA6.js +0 -880
  36. package/dist/chunk-DPZAEKA6.js.map +0 -1
  37. package/dist/chunk-DTJ6QUQB.js +0 -131
  38. package/dist/chunk-DTJ6QUQB.js.map +0 -1
  39. package/dist/chunk-GGE4NNQT.js +0 -65
  40. package/dist/chunk-GGE4NNQT.js.map +0 -1
  41. package/dist/chunk-H5UD2323.js +0 -286
  42. package/dist/chunk-H5UD2323.js.map +0 -1
  43. package/dist/chunk-HHWE3POT.js +0 -94
  44. package/dist/chunk-HHWE3POT.js.map +0 -1
  45. package/dist/chunk-HKUCJ437.js +0 -787
  46. package/dist/chunk-HKUCJ437.js.map +0 -1
  47. package/dist/chunk-JHCHEVET.js +0 -274
  48. package/dist/chunk-JHCHEVET.js.map +0 -1
  49. package/dist/chunk-JHOJHHU7.js +0 -867
  50. package/dist/chunk-JHOJHHU7.js.map +0 -1
  51. package/dist/chunk-JM2SKQMS.js +0 -750
  52. package/dist/chunk-JM2SKQMS.js.map +0 -1
  53. package/dist/chunk-JN2FCO5W.js +0 -7958
  54. package/dist/chunk-JN2FCO5W.js.map +0 -1
  55. package/dist/chunk-K4DBDHLK.js +0 -158
  56. package/dist/chunk-K4DBDHLK.js.map +0 -1
  57. package/dist/chunk-K6N6XJJX.js +0 -306
  58. package/dist/chunk-K6N6XJJX.js.map +0 -1
  59. package/dist/chunk-MA6HLL3S.js +0 -65
  60. package/dist/chunk-MA6HLL3S.js.map +0 -1
  61. package/dist/chunk-MAZ26DC7.js +0 -99
  62. package/dist/chunk-MAZ26DC7.js.map +0 -1
  63. package/dist/chunk-MOXWMGPC.js +0 -577
  64. package/dist/chunk-MOXWMGPC.js.map +0 -1
  65. package/dist/chunk-NJC7U437.js +0 -626
  66. package/dist/chunk-NJC7U437.js.map +0 -1
  67. package/dist/chunk-NPCTHQIO.js +0 -91
  68. package/dist/chunk-NPCTHQIO.js.map +0 -1
  69. package/dist/chunk-ONWEPEDO.js +0 -57
  70. package/dist/chunk-ONWEPEDO.js.map +0 -1
  71. package/dist/chunk-OYZAPX5G.js +0 -1526
  72. package/dist/chunk-OYZAPX5G.js.map +0 -1
  73. package/dist/chunk-PC4UYEBM.js +0 -166
  74. package/dist/chunk-PC4UYEBM.js.map +0 -1
  75. package/dist/chunk-PICTDURQ.js +0 -766
  76. package/dist/chunk-PICTDURQ.js.map +0 -1
  77. package/dist/chunk-PJQFMIOX.js +0 -1182
  78. package/dist/chunk-PJQFMIOX.js.map +0 -1
  79. package/dist/chunk-PXD6ZFNY.js +0 -1107
  80. package/dist/chunk-PXD6ZFNY.js.map +0 -1
  81. package/dist/chunk-PXE2VKMX.js +0 -140
  82. package/dist/chunk-PXE2VKMX.js.map +0 -1
  83. package/dist/chunk-PZ5AY32C.js +0 -10
  84. package/dist/chunk-PZ5AY32C.js.map +0 -1
  85. package/dist/chunk-QBRSJK47.js +0 -622
  86. package/dist/chunk-QBRSJK47.js.map +0 -1
  87. package/dist/chunk-QWMPPZ3X.js +0 -550
  88. package/dist/chunk-QWMPPZ3X.js.map +0 -1
  89. package/dist/chunk-S3UZOQ5Y.js +0 -328
  90. package/dist/chunk-S3UZOQ5Y.js.map +0 -1
  91. package/dist/chunk-S5TT5R3L.js +0 -2668
  92. package/dist/chunk-S5TT5R3L.js.map +0 -1
  93. package/dist/chunk-T4SQEITX.js +0 -95
  94. package/dist/chunk-T4SQEITX.js.map +0 -1
  95. package/dist/chunk-TT4KNT67.js +0 -124
  96. package/dist/chunk-TT4KNT67.js.map +0 -1
  97. package/dist/chunk-U5CHZ5M3.js +0 -357
  98. package/dist/chunk-U5CHZ5M3.js.map +0 -1
  99. package/dist/chunk-ULOKLHIQ.js +0 -1937
  100. package/dist/chunk-ULOKLHIQ.js.map +0 -1
  101. package/dist/chunk-VI2UW6B6.js +0 -162
  102. package/dist/chunk-VI2UW6B6.js.map +0 -1
  103. package/dist/chunk-VQMK5FMP.js +0 -247
  104. package/dist/chunk-VQMK5FMP.js.map +0 -1
  105. package/dist/chunk-VSMTAMNK.js +0 -53
  106. package/dist/chunk-VSMTAMNK.js.map +0 -1
  107. package/dist/chunk-VZSRQ272.js +0 -149
  108. package/dist/chunk-VZSRQ272.js.map +0 -1
  109. package/dist/chunk-WW2A73HW.js +0 -159
  110. package/dist/chunk-WW2A73HW.js.map +0 -1
  111. package/dist/chunk-X4UCIOTZ.js +0 -136
  112. package/dist/chunk-X4UCIOTZ.js.map +0 -1
  113. package/dist/chunk-XDIRG3TO.js +0 -1266
  114. package/dist/chunk-XDIRG3TO.js.map +0 -1
  115. package/dist/chunk-XJYR7XFV.js +0 -317
  116. package/dist/chunk-XJYR7XFV.js.map +0 -1
  117. package/dist/chunk-ZET2UAYW.js +0 -89
  118. package/dist/chunk-ZET2UAYW.js.map +0 -1
  119. package/dist/chunk-ZZUXHH3R.js +0 -99
  120. package/dist/chunk-ZZUXHH3R.js.map +0 -1
  121. package/dist/cli.d.ts +0 -1
  122. package/dist/cli.js +0 -112
  123. package/dist/cli.js.map +0 -1
  124. package/dist/contract/index.d.ts +0 -4972
  125. package/dist/contract/index.js +0 -1654
  126. package/dist/contract/index.js.map +0 -1
  127. package/dist/control.d.ts +0 -1013
  128. package/dist/control.js +0 -34
  129. package/dist/control.js.map +0 -1
  130. package/dist/fuzz.d.ts +0 -759
  131. package/dist/fuzz.js +0 -714
  132. package/dist/fuzz.js.map +0 -1
  133. package/dist/hosted/index.d.ts +0 -730
  134. package/dist/hosted/index.js +0 -14
  135. package/dist/hosted/index.js.map +0 -1
  136. package/dist/index.d.ts +0 -16780
  137. package/dist/index.js +0 -12168
  138. package/dist/index.js.map +0 -1
  139. package/dist/matrix/index.d.ts +0 -155
  140. package/dist/matrix/index.js +0 -8
  141. package/dist/matrix/index.js.map +0 -1
  142. package/dist/meta-eval/index.d.ts +0 -1030
  143. package/dist/meta-eval/index.js +0 -417
  144. package/dist/meta-eval/index.js.map +0 -1
  145. package/dist/multishot/index.d.ts +0 -579
  146. package/dist/multishot/index.js +0 -589
  147. package/dist/multishot/index.js.map +0 -1
  148. package/dist/openapi.json +0 -992
  149. package/dist/pipelines/index.d.ts +0 -567
  150. package/dist/pipelines/index.js +0 -515
  151. package/dist/pipelines/index.js.map +0 -1
  152. package/dist/reporting.d.ts +0 -1277
  153. package/dist/reporting.js +0 -48
  154. package/dist/reporting.js.map +0 -1
  155. package/dist/rl.d.ts +0 -4092
  156. package/dist/rl.js +0 -1724
  157. package/dist/rl.js.map +0 -1
  158. package/dist/run-campaign-HNFPJET4.js +0 -14
  159. package/dist/run-campaign-HNFPJET4.js.map +0 -1
  160. package/dist/storyboard/index.d.ts +0 -279
  161. package/dist/storyboard/index.js +0 -767
  162. package/dist/storyboard/index.js.map +0 -1
  163. package/dist/trace-attributes.d.ts +0 -52
  164. package/dist/trace-attributes.js +0 -62
  165. package/dist/trace-attributes.js.map +0 -1
  166. package/dist/traces.d.ts +0 -2343
  167. package/dist/traces.js +0 -249
  168. package/dist/traces.js.map +0 -1
  169. package/dist/wire/index.d.ts +0 -1252
  170. package/dist/wire/index.js +0 -81
  171. package/dist/wire/index.js.map +0 -1
package/dist/fuzz.js DELETED
@@ -1,714 +0,0 @@
1
- import {
2
- CostLedger,
3
- CostReceiptCaptureError
4
- } from "./chunk-JHOJHHU7.js";
5
- import "./chunk-VI2UW6B6.js";
6
- import {
7
- varianceBasedCurriculum
8
- } from "./chunk-VZSRQ272.js";
9
- import {
10
- ValidationError
11
- } from "./chunk-ONWEPEDO.js";
12
- import "./chunk-PZ5AY32C.js";
13
-
14
- // src/fuzz/cube.ts
15
- function enumerateCells(space) {
16
- if (space.axes.length === 0) return [];
17
- let partials = [{}];
18
- for (const axis of space.axes) {
19
- const next = [];
20
- for (const partial of partials) {
21
- for (const value of axis.values) next.push({ ...partial, [axis.name]: value });
22
- }
23
- partials = next;
24
- }
25
- return partials.map((coords) => ({ id: cellId(space, coords), coords }));
26
- }
27
- function cellId(space, coords) {
28
- return space.axes.map((a) => `${a.name}=${coords[a.name]}`).join("|");
29
- }
30
- function percentile(sorted, p) {
31
- const idx = Math.min(sorted.length - 1, Math.max(0, Math.ceil(p * sorted.length) - 1));
32
- return sorted[idx];
33
- }
34
- function distribution(values) {
35
- if (values.length === 0)
36
- throw new Error("distribution: empty sample \u2014 represent missing data as null, not zeros");
37
- const sorted = [...values].sort((a, b) => a - b);
38
- const mean = sorted.reduce((a, b) => a + b, 0) / sorted.length;
39
- return {
40
- mean,
41
- median: percentile(sorted, 0.5),
42
- p90: percentile(sorted, 0.9),
43
- min: sorted[0],
44
- max: sorted[sorted.length - 1],
45
- n: sorted.length
46
- };
47
- }
48
- function buildCoverage(cells, log, threshold) {
49
- const byCell = /* @__PURE__ */ new Map();
50
- for (const r of log) {
51
- const arr = byCell.get(r.cell.id) ?? [];
52
- arr.push(r);
53
- byCell.set(r.cell.id, arr);
54
- }
55
- return cells.map((cell) => {
56
- const recs = byCell.get(cell.id) ?? [];
57
- const runs = recs.length;
58
- if (runs === 0)
59
- return { cell, runs: 0, score: null, findingRate: 0, dimensions: {}, latencyMs: null };
60
- const score = distribution(recs.map((r) => r.ev.score));
61
- const latencyMs = distribution(recs.map((r) => r.latencyMs));
62
- const findingRate = recs.filter((r) => r.interest >= threshold).length / runs;
63
- const dimSamples = {};
64
- for (const r of recs) {
65
- for (const [k, v] of Object.entries(r.ev.scores ?? {})) {
66
- ;
67
- (dimSamples[k] ??= []).push(v);
68
- }
69
- }
70
- const dimensions = {};
71
- for (const [k, xs] of Object.entries(dimSamples)) dimensions[k] = distribution(xs);
72
- const tracked = recs.filter((r) => r.costUsd !== void 0);
73
- const known = tracked.filter((r) => r.costUsd !== null);
74
- const cost = tracked.length > 0 ? {
75
- costUsd: known.reduce((a, r) => a + r.costUsd, 0),
76
- ...tracked.length > known.length ? { costUnknownRuns: tracked.length - known.length } : {}
77
- } : {};
78
- return { cell, runs, score, findingRate, dimensions, latencyMs, ...cost };
79
- });
80
- }
81
-
82
- // src/fuzz/capsule.ts
83
- function buildCapsule(input) {
84
- const coverage = buildCoverage(input.cells, input.log, input.threshold);
85
- const covered = coverage.filter((c) => c.runs > 0);
86
- const robustness = covered.length === 0 ? null : distribution(covered.map((c) => c.score.mean));
87
- const latencyMs = input.log.length === 0 ? null : distribution(input.log.map((r) => r.latencyMs));
88
- const behaviorBinsObserved = input.archive.filter((e) => e.binId !== e.cell.id).length;
89
- return {
90
- target: input.target,
91
- objective: input.objective,
92
- coverage,
93
- findings: [...input.findings].sort((a, b) => b.interest - a.interest),
94
- archive: [...input.archive].sort((a, b) => b.interest - a.interest),
95
- stats: {
96
- totalRuns: input.runsUsed,
97
- cellsTotal: input.cells.length,
98
- cellsCovered: covered.length,
99
- behaviorBinsObserved,
100
- candidateFindings: input.candidateFindings,
101
- verifiedFindings: input.findings.length,
102
- robustness,
103
- latencyMs,
104
- ...input.cost ? { costUsd: input.cost.costUsd, costUnknownRuns: input.cost.costUnknownRuns } : {},
105
- evalErrors: input.evalErrors,
106
- ...input.stoppedEarly ? { stoppedEarly: input.stoppedEarly } : {}
107
- }
108
- };
109
- }
110
- function esc(s) {
111
- return s.replace(
112
- /[&<>"]/g,
113
- (c) => ({ "&": "&amp;", "<": "&lt;", ">": "&gt;", '"': "&quot;" })[c]
114
- );
115
- }
116
- function robustnessColor(r) {
117
- if (r == null) return "#2a2a2e";
118
- return `hsl(${Math.round(r * 120)} 70% 42%)`;
119
- }
120
- function pct(x) {
121
- return `${Math.round(x * 100)}%`;
122
- }
123
- function deriveAxes(coverage) {
124
- const order = [];
125
- const seen = /* @__PURE__ */ new Map();
126
- for (const c of coverage) {
127
- for (const [k, v] of Object.entries(c.cell.coords)) {
128
- if (!seen.has(k)) {
129
- seen.set(k, /* @__PURE__ */ new Set());
130
- order.push(k);
131
- }
132
- seen.get(k)?.add(v);
133
- }
134
- }
135
- return order.map((name) => ({ name, values: [...seen.get(name) ?? []] }));
136
- }
137
- function weakestDim(c) {
138
- const entries = Object.entries(c.dimensions).map(([k, d]) => [k, d.mean]);
139
- if (entries.length === 0) return "";
140
- const sorted = entries.sort((a, b) => a[1] - b[1]);
141
- const w = sorted[0];
142
- if (!w) return "";
143
- return `<span class="dim">${esc(w[0])} ${pct(w[1])}</span>`;
144
- }
145
- function heatmapHtml(coverage) {
146
- const axes = deriveAxes(coverage);
147
- const byId = new Map(coverage.map((c) => [c.cell.id, c]));
148
- const tile = (c, label) => {
149
- const r = c?.score?.mean ?? null;
150
- const title = c?.score ? `${pct(c.score.mean)} robust (median ${pct(c.score.median)}, min ${pct(c.score.min)}) \xB7 ${c.runs} runs \xB7 ${pct(c.findingRate)} flagged${c.latencyMs ? ` \xB7 ${(c.latencyMs.median / 1e3).toFixed(1)}s median` : ""}${c.costUsd !== void 0 ? ` \xB7 $${c.costUsd.toFixed(2)}` : ""}` : "not covered";
151
- return `<div class="tile" style="background:${robustnessColor(r)}" title="${esc(title)}">${label ? `<span class="tl">${esc(label)}</span>` : ""}<span class="tv">${c && r != null ? pct(r) : "\u2014"}</span>${c ? weakestDim(c) : ""}</div>`;
152
- };
153
- const rowAxis = axes[0];
154
- const colAxis = axes[1];
155
- if (axes.length === 2 && rowAxis && colAxis) {
156
- const head = `<tr><th></th>${colAxis.values.map((v) => `<th>${esc(v)}</th>`).join("")}</tr>`;
157
- const rows = rowAxis.values.map((rv) => {
158
- const cells = colAxis.values.map((cv) => {
159
- const id = `${rowAxis.name}=${rv}|${colAxis.name}=${cv}`;
160
- return `<td>${tile(byId.get(id), "")}</td>`;
161
- }).join("");
162
- return `<tr><th class="rh">${esc(rv)}</th>${cells}</tr>`;
163
- }).join("");
164
- return `<div class="axis-label">rows: <b>${esc(rowAxis.name)}</b> \xB7 cols: <b>${esc(colAxis.name)}</b></div><table class="heat">${head}${rows}</table>`;
165
- }
166
- const sorted = [...coverage].sort((a, b) => (a.score?.mean ?? 2) - (b.score?.mean ?? 2));
167
- return `<div class="grid">${sorted.map((c) => tile(c, Object.values(c.cell.coords).join(" \xB7 "))).join("")}</div>`;
168
- }
169
- function findingsHtml(findings, limit) {
170
- if (findings.length === 0)
171
- return `<p class="none">No verified findings \u2014 the target held across every covered cell.</p>`;
172
- return findings.slice(0, limit).map((f) => {
173
- const coords = Object.entries(f.cell.coords).map(([k, v]) => `${k}:${v}`).join(" \xB7 ");
174
- const labels = (f.evaluation.labels ?? []).map((l) => `<span class="fclass">${esc(l)}</span>`).join("");
175
- const dims = Object.entries(f.evaluation.scores ?? {}).sort((a, b) => a[1] - b[1]).slice(0, 3).map(([k, v]) => `<span class="dim">${esc(k)} ${pct(v)}</span>`).join("");
176
- return `<div class="fail"><div class="fmeta"><span class="fcell">${esc(coords)}</span>${labels}<span class="fsev">interest ${pct(f.interest)}</span></div><div class="sevbar"><div class="sevfill" style="width:${pct(f.interest)}"></div></div><div class="ftext">${esc(f.text ?? "(scenario text not captured)")}</div>${dims ? `<div class="fdims">${dims}</div>` : ""}</div>`;
177
- }).join("");
178
- }
179
- function renderCapsuleHtml(capsule, opts = {}) {
180
- const s = capsule.stats;
181
- const kpi = (label, value, accent = "#e6e6e6") => `<div class="kpi"><div class="kv" style="color:${accent}">${esc(value)}</div><div class="kl">${esc(label)}</div></div>`;
182
- const lift = capsule.lift ? kpi(
183
- "hardening lift",
184
- `${capsule.lift.before.toFixed(2)} \u2192 ${capsule.lift.after.toFixed(2)}`,
185
- "#5ad17a"
186
- ) : "";
187
- const cost = s.costUsd !== void 0 ? kpi(
188
- s.costUnknownRuns ? `cost \xB7 ${s.costUnknownRuns} runs unpriced` : "cost",
189
- `$${s.costUsd.toFixed(2)}`,
190
- s.costUnknownRuns ? "#e5c07b" : "#e6e6e6"
191
- ) : "";
192
- const stamp = opts.generatedAt ?? capsule.generatedAt ?? "";
193
- return `<!doctype html><html><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
194
- <title>${esc(capsule.objective)} capsule \u2014 ${esc(capsule.target)}</title>
195
- <style>
196
- :root{color-scheme:dark}
197
- *{box-sizing:border-box}
198
- body{margin:0;background:#0c0c0f;color:#e6e6e6;font:14px/1.5 ui-sans-serif,system-ui,-apple-system,Segoe UI,Roboto,sans-serif}
199
- .wrap{max-width:980px;margin:0 auto;padding:40px 24px}
200
- h1{font-size:22px;margin:0 0 2px;letter-spacing:-.01em}
201
- .sub{color:#8a8a93;font-size:13px;margin-bottom:28px}
202
- .kpis{display:flex;gap:14px;flex-wrap:wrap;margin-bottom:32px}
203
- .kpi{background:#16161b;border:1px solid #24242b;border-radius:12px;padding:14px 18px;min-width:120px}
204
- .kv{font-size:24px;font-weight:650;letter-spacing:-.02em}
205
- .kl{color:#8a8a93;font-size:12px;margin-top:2px}
206
- h2{font-size:13px;text-transform:uppercase;letter-spacing:.08em;color:#8a8a93;margin:34px 0 14px}
207
- .axis-label{color:#8a8a93;font-size:12px;margin-bottom:10px}
208
- table.heat{border-collapse:separate;border-spacing:6px}
209
- table.heat th{font-weight:500;color:#a8a8b0;font-size:12px;padding:4px 8px;text-align:center}
210
- table.heat th.rh{text-align:right}
211
- .tile{position:relative;width:104px;height:66px;border-radius:10px;display:flex;flex-direction:column;justify-content:center;align-items:center;gap:1px;color:#06120a;font-weight:600}
212
- .tile .tv{font-size:17px}
213
- .tile .tl{font-size:10px;font-weight:600;opacity:.9;padding:0 6px;text-align:center}
214
- .tile .dim{font-size:9px;font-weight:600;opacity:.85;background:rgba(0,0,0,.22);padding:0 6px;border-radius:999px}
215
- .grid{display:flex;flex-wrap:wrap;gap:8px}
216
- .grid .tile{width:138px;height:74px}
217
- .fail{background:#16161b;border:1px solid #24242b;border-left:3px solid #d1495b;border-radius:10px;padding:12px 14px;margin-bottom:10px}
218
- .fmeta{display:flex;gap:10px;align-items:center;font-size:12px;color:#a8a8b0;margin-bottom:8px}
219
- .fcell{color:#cfcfd6}
220
- .fclass{background:#2a1a1d;color:#e58a96;padding:1px 8px;border-radius:999px;font-size:11px}
221
- .fsev{margin-left:auto;color:#e58a96}
222
- .sevbar{height:4px;background:#24242b;border-radius:2px;overflow:hidden;margin-bottom:10px}
223
- .sevfill{height:100%;background:#d1495b}
224
- .ftext{font-size:13px;color:#d8d8df;white-space:pre-wrap}
225
- .fdims{display:flex;gap:6px;margin-top:8px}
226
- .fdims .dim{background:#1d2530;color:#9fc1e8;padding:1px 8px;border-radius:999px;font-size:11px}
227
- .none{color:#5ad17a}
228
- .foot{color:#5a5a63;font-size:11px;margin-top:36px}
229
- </style></head><body><div class="wrap">
230
- <h1>${esc(capsule.objective)} exploration \xB7 ${esc(capsule.target)}</h1>
231
- <div class="sub">${s.totalRuns} scenarios across ${s.cellsCovered}/${s.cellsTotal} planned cells${s.behaviorBinsObserved > 0 ? ` \xB7 ${s.behaviorBinsObserved} measured behavior bins` : ""}${stamp ? ` \xB7 ${esc(stamp)}` : ""}</div>
232
- <div class="kpis">
233
- ${s.robustness ? kpi("robustness", `${pct(s.robustness.mean)}`, s.robustness.mean < 0.6 ? "#e58a96" : "#5ad17a") : ""}
234
- ${s.robustness ? kpi("cell spread", `${pct(s.robustness.min)}\u2013${pct(s.robustness.max)}`) : ""}
235
- ${s.latencyMs ? kpi("median latency", `${(s.latencyMs.median / 1e3).toFixed(1)}s`, s.latencyMs.p90 > 4 * s.latencyMs.median ? "#e5b566" : "#e6e6e6") : ""}
236
- ${kpi("verified findings", String(s.verifiedFindings), s.verifiedFindings > 0 ? "#e58a96" : "#5ad17a")}
237
- ${kpi("cells covered", `${s.cellsCovered}/${s.cellsTotal}`)}
238
- ${kpi("scenarios run", String(s.totalRuns))}
239
- ${cost}
240
- ${s.evalErrors > 0 ? kpi("eval errors", String(s.evalErrors), "#e5b566") : ""}
241
- ${lift}
242
- </div>
243
- ${s.stoppedEarly ? `<div class="sub" style="color:#e5b566">stopped early: ${esc(s.stoppedEarly.detail)} \u2014 coverage below reflects the completed portion only</div>` : ""}
244
- <h2>Coverage map</h2>
245
- ${heatmapHtml(capsule.coverage)}
246
- <h2>Verified findings${s.candidateFindings > s.verifiedFindings ? ` \xB7 ${s.verifiedFindings} of ${s.candidateFindings} candidates passed the validity gates` : ""}</h2>
247
- ${findingsHtml(capsule.findings, opts.maxFindings ?? 8)}
248
- <div class="foot">Findings are gate-verified: each is a fair, answerable task that reproduces under a meaning-preserving rephrase, minimized to the smallest trigger.</div>
249
- </div></body></html>`;
250
- }
251
-
252
- // src/fuzz/policies.ts
253
- var clamp01 = (x) => Math.max(0, Math.min(1, x));
254
- function mutationProposer(opts) {
255
- return async (ctx) => {
256
- const mutations = opts.mutationsFor(ctx.cell);
257
- const parents = [...ctx.elites, ...ctx.seeds];
258
- const seen = new Set(parents.map(opts.scenarioId));
259
- const out = [];
260
- for (const parent of parents) {
261
- if (out.length >= ctx.count) break;
262
- for (const m of mutations) {
263
- const children = await m.mutate(parent, ctx.rng);
264
- for (const child of children) {
265
- const id = opts.scenarioId(child);
266
- if (seen.has(id)) continue;
267
- seen.add(id);
268
- out.push(child);
269
- if (out.length >= ctx.count) break;
270
- }
271
- if (out.length >= ctx.count) break;
272
- }
273
- }
274
- return out;
275
- };
276
- }
277
- function adversarialObjective(threshold = 0.5) {
278
- return { kind: "adversarial", threshold, interest: (ev) => clamp01(1 - ev.score) };
279
- }
280
- function hamming(a, b) {
281
- if (!a || !b) return 1;
282
- const keys = /* @__PURE__ */ new Set([...Object.keys(a), ...Object.keys(b)]);
283
- if (keys.size === 0) return 0;
284
- let diff = 0;
285
- for (const k of keys) if (a[k] !== b[k]) diff++;
286
- return diff / keys.size;
287
- }
288
- function noveltyObjective(threshold = 0.3) {
289
- return {
290
- kind: "novelty",
291
- threshold,
292
- interest: (ev, ctx) => {
293
- const scoreNovelty = ctx.archiveScores.length === 0 ? 1 : Math.min(...ctx.archiveScores.map((s) => Math.abs(s - ev.score)));
294
- const descNovelty = ctx.archiveDescriptors.length === 0 ? 1 : Math.min(...ctx.archiveDescriptors.map((d) => hamming(d, ev.descriptor)));
295
- return clamp01(0.5 * scoreNovelty + 0.5 * descNovelty);
296
- }
297
- };
298
- }
299
-
300
- // src/fuzz/explorer.ts
301
- async function pMap(items, fn, concurrency, signal) {
302
- let i = 0;
303
- const workers = Array.from(
304
- { length: Math.max(1, Math.min(concurrency, items.length)) },
305
- async () => {
306
- while (i < items.length) {
307
- if (signal?.aborted) return;
308
- const item = items[i++];
309
- if (item === void 0) return;
310
- await fn(item);
311
- }
312
- }
313
- );
314
- await Promise.all(workers);
315
- }
316
- var BehaviorExplorer = class {
317
- constructor(opts) {
318
- this.opts = opts;
319
- this.cells = enumerateCells(opts.space);
320
- if (this.cells.length === 0)
321
- throw new Error("BehaviorExplorer: space has no cells \u2014 every axis needs \u22651 value");
322
- if (opts.costBudgetUsd !== void 0) {
323
- if (typeof opts.costBudgetUsd !== "number" || !Number.isFinite(opts.costBudgetUsd) || opts.costBudgetUsd < 0) {
324
- throw new RangeError(
325
- `BehaviorExplorer: costBudgetUsd must be a nonnegative finite number, got ${String(opts.costBudgetUsd)}`
326
- );
327
- }
328
- }
329
- if (!opts.costOf && (opts.costBudgetUsd !== void 0 || opts.ledger || opts.onCost)) {
330
- throw new ValidationError(
331
- "BehaviorExplorer: costBudgetUsd/ledger/onCost require costOf \u2014 the explorer cannot know run cost without it; supply costOf or drop the cost options"
332
- );
333
- }
334
- if ((opts.costBudgetUsd !== void 0 || opts.ledger?.costCeilingUsd !== void 0) && !opts.maximumChargeOf) {
335
- throw new ValidationError(
336
- "BehaviorExplorer: capped cost tracking requires maximumChargeOf before evaluation"
337
- );
338
- }
339
- if (opts.ledger && opts.costBudgetUsd !== void 0 && opts.ledger.costCeilingUsd !== opts.costBudgetUsd) {
340
- throw new ValidationError(
341
- "BehaviorExplorer: costBudgetUsd must match the supplied CostLedger ceiling"
342
- );
343
- }
344
- if (opts.costOf) this.costLedger = opts.ledger ?? new CostLedger(opts.costBudgetUsd);
345
- this.cellById = new Map(this.cells.map((c) => [c.id, c]));
346
- this.objective = opts.objective ?? adversarialObjective(0.5);
347
- this.threshold = this.objective.threshold ?? 0.5;
348
- this.floorPerCell = opts.floorPerCell ?? 2;
349
- this.perRoundBudget = Math.max(
350
- this.cells.length * this.floorPerCell,
351
- Math.ceil(opts.budget / 4)
352
- );
353
- this.rngState = (opts.seed ?? 1) >>> 0;
354
- }
355
- opts;
356
- cells;
357
- cellById;
358
- objective;
359
- threshold;
360
- floorPerCell;
361
- perRoundBudget;
362
- /** The single evaluation log — coverage + allocation are projections of it. */
363
- log = [];
364
- /** binId (input × measured coords) → the most interesting entry seen. */
365
- archiveByBin = /* @__PURE__ */ new Map();
366
- _findings = [];
367
- runsUsed = 0;
368
- candidateFindings = 0;
369
- evalErrors = 0;
370
- consecutiveEvalErrors = 0;
371
- stoppedEarly;
372
- rngState;
373
- costLedger;
374
- costPhase = "fuzz.explore";
375
- // mulberry32 — deterministic per session.
376
- rng = () => {
377
- this.rngState = this.rngState + 1831565813 | 0;
378
- let t = this.rngState;
379
- t = Math.imul(t ^ t >>> 15, t | 1);
380
- t ^= t + Math.imul(t ^ t >>> 7, t | 61);
381
- return ((t ^ t >>> 14) >>> 0) / 4294967296;
382
- };
383
- binId(cell, descriptor) {
384
- if (!descriptor || Object.keys(descriptor).length === 0) return cell.id;
385
- const measured = Object.keys(descriptor).sort().map((k) => `${k}=${descriptor[k]}`).join("|");
386
- return `${cell.id}|${measured}`;
387
- }
388
- allocate(budget) {
389
- if ((this.opts.allocation ?? "variance") === "uniform") {
390
- const per = Math.max(this.floorPerCell, Math.floor(budget / this.cells.length));
391
- return this.cells.map((c) => ({ cellId: c.id, count: per }));
392
- }
393
- return varianceBasedCurriculum(
394
- this.log.map((r) => ({
395
- variantId: r.cell.id,
396
- scenarioId: r.scenarioId,
397
- score: r.ev.score,
398
- pass: r.ev.valid && r.ev.score >= 0.5
399
- })),
400
- this.cells.map((c) => ({ variantId: c.id, scenarioId: "*" })),
401
- { budget, floorPerCell: this.floorPerCell }
402
- ).map((a) => ({ cellId: a.variantId, count: a.count }));
403
- }
404
- objectiveContext() {
405
- const entries = [...this.archiveByBin.values()];
406
- return {
407
- archiveScores: entries.map((e) => e.evaluation.score),
408
- archiveDescriptors: entries.map((e) => e.evaluation.descriptor)
409
- };
410
- }
411
- /** Elites whose INPUT cell matches — what the proposer mutates/deepens from. */
412
- elitesFor(cellId2) {
413
- const out = [];
414
- for (const e of this.archiveByBin.values()) if (e.cell.id === cellId2) out.push(e.scenario);
415
- return out;
416
- }
417
- /** One allocate → propose → evaluate → gate → archive round. */
418
- async step() {
419
- const remaining = this.opts.budget - this.runsUsed;
420
- if (remaining <= 0 || this.opts.signal?.aborted) return { runs: 0, findings: [] };
421
- const allocations = this.allocate(Math.min(this.perRoundBudget, remaining));
422
- const newFindings = [];
423
- let runsThisStep = 0;
424
- for (const alloc of allocations) {
425
- if (this.runsUsed >= this.opts.budget || this.stoppedEarly !== void 0 || this.opts.signal?.aborted)
426
- break;
427
- const cell = this.cellById.get(alloc.cellId);
428
- if (!cell) continue;
429
- const cap = Math.min(alloc.count, this.opts.budget - this.runsUsed);
430
- if (cap <= 0) continue;
431
- this.opts.onProgress?.({ type: "cell-allocated", cell, count: cap });
432
- const seeds = await this.opts.seedsFor(cell);
433
- const elites = this.elitesFor(cell.id);
434
- const proposed = await this.opts.proposer({
435
- cell,
436
- seeds,
437
- elites,
438
- findings: this._findings,
439
- count: cap,
440
- rng: this.rng
441
- });
442
- const cold = this.log.every((r) => r.cell.id !== cell.id);
443
- const toEval = [...cold ? seeds : [], ...proposed].slice(0, cap);
444
- await pMap(
445
- toEval,
446
- async (scenario) => {
447
- if (this.runsUsed >= this.opts.budget || this.stoppedEarly !== void 0 || this.opts.signal?.aborted)
448
- return;
449
- try {
450
- const startedAt = performance.now();
451
- const paid = this.costLedger ? await this.costLedger.runPaidCall({
452
- channel: "agent",
453
- phase: this.costPhase,
454
- actor: "fuzz.evaluate",
455
- tags: { target: this.opts.target, cell: cell.id },
456
- signal: this.opts.signal,
457
- maximumCharge: this.opts.maximumChargeOf?.(scenario, cell),
458
- execute: () => this.opts.evaluate(scenario, cell),
459
- receipt: (evaluation) => {
460
- const cost = this.opts.costOf(scenario, cell, evaluation);
461
- if (cost === null) {
462
- return {
463
- model: "unattributed",
464
- inputTokens: 0,
465
- outputTokens: 0,
466
- costUnknown: true
467
- };
468
- }
469
- if (!Number.isFinite(cost.usd) || cost.usd < 0) {
470
- throw new RangeError(
471
- `BehaviorExplorer: costOf returned an invalid usd (${String(cost.usd)}) \u2014 return null when cost is unknown, never a fabricated number`
472
- );
473
- }
474
- return {
475
- model: cost.model ?? "unattributed",
476
- inputTokens: 0,
477
- outputTokens: 0,
478
- actualCostUsd: cost.usd
479
- };
480
- }
481
- }) : void 0;
482
- if (paid && !paid.succeeded) {
483
- if (paid.error instanceof CostReceiptCaptureError && paid.error.receiptError instanceof RangeError) {
484
- throw paid.error.receiptError;
485
- }
486
- throw paid.error;
487
- }
488
- const ev = paid ? paid.value : await this.opts.evaluate(scenario, cell);
489
- const latencyMs = ev.latencyMs ?? performance.now() - startedAt;
490
- this.runsUsed++;
491
- runsThisStep++;
492
- this.consecutiveEvalErrors = 0;
493
- const costUsd = paid ? paid.receipt.costUnknown ? null : paid.receipt.costUsd : void 0;
494
- if (paid?.receipt.actualCostUsd !== void 0) {
495
- this.opts.onCost?.({ usd: paid.receipt.actualCostUsd, channel: "agent" });
496
- }
497
- const interest = this.objective.interest(ev, this.objectiveContext());
498
- this.log.push({
499
- cell,
500
- ev,
501
- interest,
502
- latencyMs,
503
- ...costUsd !== void 0 ? { costUsd } : {},
504
- scenarioId: this.opts.scenarioId(scenario)
505
- });
506
- this.opts.onProgress?.({ type: "evaluated", cell, scenario, evaluation: ev });
507
- const bin = this.binId(cell, ev.descriptor);
508
- const cur = this.archiveByBin.get(bin);
509
- if (!cur || interest > cur.interest)
510
- this.archiveByBin.set(bin, { binId: bin, cell, scenario, evaluation: ev, interest });
511
- if (interest < this.threshold) return;
512
- this.candidateFindings++;
513
- if (this.opts.gates?.isValid && !await this.opts.gates.isValid(scenario, ev, cell))
514
- return;
515
- if (this.opts.gates?.isUncontaminated && !await this.opts.gates.isUncontaminated(scenario, ev, cell))
516
- return;
517
- const minimized = this.opts.minimize ? await this.opts.minimize(scenario, this.opts.evaluate, cell) : scenario;
518
- const finding = {
519
- id: this.opts.scenarioId(scenario),
520
- cell,
521
- scenario,
522
- minimized,
523
- text: this.opts.scenarioText?.(minimized),
524
- evaluation: ev,
525
- interest,
526
- objective: this.objective.kind
527
- };
528
- this._findings.push(finding);
529
- newFindings.push(finding);
530
- this.opts.onProgress?.({ type: "finding", finding });
531
- } catch (err) {
532
- if (err instanceof RangeError) throw err;
533
- this.evalErrors++;
534
- this.consecutiveEvalErrors++;
535
- const message = err instanceof Error ? err.message : String(err);
536
- this.opts.onProgress?.({
537
- type: "eval-error",
538
- cell,
539
- scenarioId: this.opts.scenarioId(scenario),
540
- message
541
- });
542
- const limit = this.opts.maxConsecutiveEvalErrors ?? 5;
543
- if (this.consecutiveEvalErrors >= limit) {
544
- this.stoppedEarly = {
545
- reason: "eval-errors",
546
- detail: `${this.consecutiveEvalErrors} consecutive eval errors (last: ${message})`
547
- };
548
- }
549
- }
550
- },
551
- this.opts.concurrency ?? 1,
552
- this.opts.signal
553
- );
554
- }
555
- this.opts.onProgress?.({ type: "round", runsUsed: this.runsUsed, budget: this.opts.budget });
556
- return { runs: runsThisStep, findings: newFindings };
557
- }
558
- /** Loop `step()` until the run or dollar budget is spent, the signal aborts,
559
- * or no progress is made. */
560
- async run() {
561
- while (this.runsUsed < this.opts.budget && this.stoppedEarly === void 0 && !this.opts.signal?.aborted) {
562
- const { runs } = await this.step();
563
- if (runs === 0 && this.stoppedEarly === void 0) break;
564
- }
565
- return this.capsule();
566
- }
567
- coverage() {
568
- return this.capsule().coverage;
569
- }
570
- findings() {
571
- return [...this._findings].sort((a, b) => b.interest - a.interest);
572
- }
573
- capsule() {
574
- const cost = this.costLedger?.summary({
575
- phase: this.costPhase,
576
- tags: { target: this.opts.target }
577
- });
578
- return buildCapsule({
579
- target: this.opts.target,
580
- objective: this.objective.kind,
581
- cells: this.cells,
582
- log: this.log,
583
- threshold: this.threshold,
584
- archive: [...this.archiveByBin.values()],
585
- findings: this._findings,
586
- candidateFindings: this.candidateFindings,
587
- runsUsed: this.runsUsed,
588
- cost: cost ? {
589
- costUsd: cost.totalCostUsd,
590
- costUnknownRuns: cost.byChannel.reduce((sum, row) => sum + row.unpricedCalls, 0)
591
- } : void 0,
592
- evalErrors: this.evalErrors,
593
- stoppedEarly: this.stoppedEarly
594
- });
595
- }
596
- };
597
-
598
- // src/fuzz/fuzz-agent.ts
599
- async function fuzzAgent(opts) {
600
- const { failureThreshold, ...rest } = opts;
601
- const explorer = new BehaviorExplorer({
602
- ...rest,
603
- objective: adversarialObjective(failureThreshold ?? 0.5)
604
- });
605
- return { capsule: await explorer.run() };
606
- }
607
-
608
- // src/fuzz/gates.ts
609
- function composeGates(...sets) {
610
- const present = sets.filter((s) => s != null);
611
- return {
612
- isValid: async (scenario, ev, cell) => {
613
- for (const g of present) {
614
- if (g.isValid && !await g.isValid(scenario, ev, cell)) return false;
615
- }
616
- return true;
617
- },
618
- isUncontaminated: async (scenario, ev, cell) => {
619
- for (const g of present) {
620
- if (g.isUncontaminated && !await g.isUncontaminated(scenario, ev, cell)) return false;
621
- }
622
- return true;
623
- }
624
- };
625
- }
626
- function perturbationStabilityGate(opts) {
627
- const threshold = opts.failureThreshold ?? 0.5;
628
- return {
629
- isUncontaminated: async (scenario, _ev, cell) => {
630
- const rephrased = opts.perturb(scenario);
631
- if (rephrased == null) return true;
632
- const re = await opts.evaluate(rephrased, cell);
633
- return re.score < threshold;
634
- }
635
- };
636
- }
637
- function severityFloorGate(opts) {
638
- const threshold = opts.failureThreshold ?? 0.5;
639
- const margin = opts.margin ?? 0.1;
640
- return {
641
- isValid: (_scenario, ev) => ev.score <= threshold - margin
642
- };
643
- }
644
-
645
- // src/fuzz/tools.ts
646
- var NO_ARGS = { type: "object", properties: {}, additionalProperties: false };
647
- function makeExploreTools(explorer) {
648
- return [
649
- {
650
- name: "explore_step",
651
- description: "Run one exploration round: allocate budget across cells, propose + evaluate scenarios, archive elites, admit gate-verified findings. Returns runs spent and new findings.",
652
- parameters: NO_ARGS,
653
- handler: async () => {
654
- const { runs, findings } = await explorer.step();
655
- return { runs, newFindings: findings.length, findings };
656
- }
657
- },
658
- {
659
- name: "explore_coverage",
660
- description: "Read the live coverage map: per planned cell \u2014 runs, robustness (null = uncovered), finding rate, per-dimension means.",
661
- parameters: NO_ARGS,
662
- handler: async () => explorer.coverage()
663
- },
664
- {
665
- name: "explore_findings",
666
- description: "List gate-verified findings so far, sorted by descending interest.",
667
- parameters: {
668
- type: "object",
669
- properties: { limit: { type: "number", description: "max findings to return" } },
670
- additionalProperties: false
671
- },
672
- handler: async (args) => {
673
- const limit = args?.limit;
674
- const all = explorer.findings();
675
- return typeof limit === "number" ? all.slice(0, limit) : all;
676
- }
677
- },
678
- {
679
- name: "explore_capsule",
680
- description: 'Build the capsule artifact from the current session state. format "data" returns the structured CapsuleData; "html" returns the standalone page.',
681
- parameters: {
682
- type: "object",
683
- properties: {
684
- format: { type: "string", enum: ["data", "html"] },
685
- generatedAt: { type: "string", description: "ISO timestamp to stamp into the page" }
686
- },
687
- additionalProperties: false
688
- },
689
- handler: async (args) => {
690
- const a = args ?? {};
691
- const capsule = explorer.capsule();
692
- if (a.format === "html") return renderCapsuleHtml(capsule, { generatedAt: a.generatedAt });
693
- return capsule;
694
- }
695
- }
696
- ];
697
- }
698
- export {
699
- BehaviorExplorer,
700
- adversarialObjective,
701
- buildCapsule,
702
- buildCoverage,
703
- cellId,
704
- composeGates,
705
- enumerateCells,
706
- fuzzAgent,
707
- makeExploreTools,
708
- mutationProposer,
709
- noveltyObjective,
710
- perturbationStabilityGate,
711
- renderCapsuleHtml,
712
- severityFloorGate
713
- };
714
- //# sourceMappingURL=fuzz.js.map