@tangle-network/agent-eval 0.90.1 → 0.92.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/campaign/index.d.ts +1 -1
- package/dist/campaign/index.js +1 -1
- package/dist/{chunk-FIUKOSWI.js → chunk-S6OZEZQK.js} +39 -9
- package/dist/chunk-S6OZEZQK.js.map +1 -0
- package/dist/fuzz.d.ts +43 -9
- package/dist/fuzz.js +59 -18
- package/dist/fuzz.js.map +1 -1
- package/dist/index.d.ts +2 -2
- package/dist/index.js +1 -1
- package/dist/openapi.json +1 -1
- package/dist/{pre-registration-DELOEJ8v.d.ts → pre-registration-mAnCugl9.d.ts} +10 -3
- package/docs/concepts.md +1 -0
- package/docs/eval-surface-map.md +59 -0
- package/package.json +1 -1
- package/dist/chunk-FIUKOSWI.js.map +0 -1
package/dist/campaign/index.d.ts
CHANGED
|
@@ -7,7 +7,7 @@ export { A as AxisEvidence, a as AxisVerdict, B as BuildEvidenceVectorOptions, k
|
|
|
7
7
|
import { L as LlmClientOptions } from '../llm-client-BeEcAokY.js';
|
|
8
8
|
import { T as TraceAnalystKindSpec } from '../kind-factory-5b7xXXOr.js';
|
|
9
9
|
import { c as AnalystFinding } from '../types-2VVIL04s.js';
|
|
10
|
-
import { S as SignedManifest, A as AgentProfile, B as BackendIntegrityReport, C as CompletionRequirement, R as RuntimeEventLike, a as CompletionVerdict, P as ProducedState, b as CorrectnessChecker } from '../pre-registration-
|
|
10
|
+
import { S as SignedManifest, A as AgentProfile, B as BackendIntegrityReport, C as CompletionRequirement, R as RuntimeEventLike, a as CompletionVerdict, P as ProducedState, b as CorrectnessChecker } from '../pre-registration-mAnCugl9.js';
|
|
11
11
|
import { E as EProcessState, a as PairedBootstrapResult } from '../statistics-C7PozGrZ.js';
|
|
12
12
|
import { A as AgentEvalError } from '../errors-CzMUYo7b.js';
|
|
13
13
|
import { a as RunSplitTag, R as RunRecord } from '../run-record-e7vj1uZQ.js';
|
package/dist/campaign/index.js
CHANGED
|
@@ -38,15 +38,36 @@ var STOPWORDS = /* @__PURE__ */ new Set([
|
|
|
38
38
|
"with",
|
|
39
39
|
"by"
|
|
40
40
|
]);
|
|
41
|
+
var REQUIREMENT_FORM_STOPWORDS = /* @__PURE__ */ new Set([
|
|
42
|
+
"generated",
|
|
43
|
+
"generate",
|
|
44
|
+
"view",
|
|
45
|
+
"render",
|
|
46
|
+
"rendered",
|
|
47
|
+
"persisted",
|
|
48
|
+
"persist",
|
|
49
|
+
"artifact",
|
|
50
|
+
"file",
|
|
51
|
+
"document",
|
|
52
|
+
"note",
|
|
53
|
+
"proposal",
|
|
54
|
+
"deliverable",
|
|
55
|
+
"output",
|
|
56
|
+
"created",
|
|
57
|
+
"create",
|
|
58
|
+
"produce",
|
|
59
|
+
"produced",
|
|
60
|
+
"flag"
|
|
61
|
+
]);
|
|
41
62
|
var MATCH_THRESHOLD = 0.5;
|
|
42
63
|
var MIN_CONTENT_CHARS = 50;
|
|
43
|
-
function tokens(s) {
|
|
64
|
+
function tokens(s, extraStop) {
|
|
44
65
|
return new Set(
|
|
45
|
-
s.toLowerCase().split(/[^a-z0-9]+/).filter((t) => t.length > 1 && !STOPWORDS.has(t))
|
|
66
|
+
s.toLowerCase().split(/[^a-z0-9]+/).filter((t) => t.length > 1 && !STOPWORDS.has(t) && !extraStop?.has(t))
|
|
46
67
|
);
|
|
47
68
|
}
|
|
48
69
|
function tokenRecall(requirementText, candidateText) {
|
|
49
|
-
const req = tokens(requirementText);
|
|
70
|
+
const req = tokens(requirementText, REQUIREMENT_FORM_STOPWORDS);
|
|
50
71
|
if (req.size === 0) return 0;
|
|
51
72
|
const cand = tokens(candidateText);
|
|
52
73
|
let hit = 0;
|
|
@@ -58,7 +79,10 @@ function artifactCandidates(req, reqIndex, artifacts) {
|
|
|
58
79
|
const out = [];
|
|
59
80
|
artifacts.forEach((a, i) => {
|
|
60
81
|
if ((a.content ?? "").trim().length < MIN_CONTENT_CHARS) return;
|
|
61
|
-
let score = tokenRecall(
|
|
82
|
+
let score = tokenRecall(
|
|
83
|
+
reqText,
|
|
84
|
+
`${a.path ?? ""} ${a.kind} ${(a.content ?? "").slice(0, 4e3)}`
|
|
85
|
+
);
|
|
62
86
|
if (req.category && a.kind && req.category.toLowerCase() === a.kind.toLowerCase()) {
|
|
63
87
|
score = Math.max(score, 1);
|
|
64
88
|
}
|
|
@@ -78,15 +102,16 @@ function proposalCandidates(req, reqIndex, proposals) {
|
|
|
78
102
|
const out = [];
|
|
79
103
|
for (const p of proposals) {
|
|
80
104
|
if (p.status !== "approved") continue;
|
|
81
|
-
const
|
|
105
|
+
const body = (p.content ?? "").trim();
|
|
106
|
+
if (body.length < MIN_CONTENT_CHARS) continue;
|
|
107
|
+
const score = tokenRecall(reqText, `${p.title} ${body}`);
|
|
82
108
|
if (score < MATCH_THRESHOLD) continue;
|
|
83
|
-
const body = p.content ?? "";
|
|
84
109
|
out.push({
|
|
85
110
|
reqIndex,
|
|
86
111
|
itemKey: `proposal:${p.id}`,
|
|
87
112
|
score,
|
|
88
113
|
evidence: `approved proposal '${p.title}' matched (token recall ${score.toFixed(2)})`,
|
|
89
|
-
content: body
|
|
114
|
+
content: body
|
|
90
115
|
});
|
|
91
116
|
}
|
|
92
117
|
return out;
|
|
@@ -278,7 +303,12 @@ function extractProducedState(events) {
|
|
|
278
303
|
});
|
|
279
304
|
} else if (ev.type === "proposal_created") {
|
|
280
305
|
const p = ev;
|
|
281
|
-
proposals.push({
|
|
306
|
+
proposals.push({
|
|
307
|
+
id: p.proposalId,
|
|
308
|
+
title: p.title,
|
|
309
|
+
status: p.status ?? "pending",
|
|
310
|
+
...p.content !== void 0 ? { content: p.content } : {}
|
|
311
|
+
});
|
|
282
312
|
}
|
|
283
313
|
}
|
|
284
314
|
return { artifacts, proposals, toolCalls };
|
|
@@ -309,4 +339,4 @@ export {
|
|
|
309
339
|
extractProducedState,
|
|
310
340
|
agentProfileHash
|
|
311
341
|
};
|
|
312
|
-
//# sourceMappingURL=chunk-
|
|
342
|
+
//# sourceMappingURL=chunk-S6OZEZQK.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/completion-verifier.ts","../src/produced-state.ts","../src/agent-profile.ts"],"sourcesContent":["/**\n * Completion verifier — the task-completion oracle.\n *\n * Answers the only eval question that is not a proxy: did the agent actually\n * COMPLETE the task — produce every required deliverable, persisted and\n * correct — rather than describe what should be done. A fluent transcript\n * that never produces the artifact scores zero here.\n *\n * Per requirement, a two-stage check:\n * 1. Structural — a produced item (vault artifact / approved proposal /\n * tool call) of the right kind is matched against the requirement and\n * carries non-empty content. Deterministic; no LLM.\n * 2. Correctness — only if structurally present AND the matched item\n * carries content, one targeted check decides whether that item\n * actually fulfils the requirement. A hallucinated artifact fails here;\n * an absent one already failed stage 1.\n *\n * `completionRate` is satisfied / total. Quality dimensions are meaningless\n * on an incomplete task — callers gate on `fullyComplete` / `completionRate`\n * before scoring quality.\n */\n\nimport type { TCloud } from '@tangle-network/tcloud'\nimport type { Artifact } from './artifact-validator'\nimport type { DefaultVerdict } from './verdict'\n\n/** What kind of produced state can satisfy a requirement structurally. */\nexport type SatisfiedBy = 'artifact' | 'proposal' | 'tool-call' | 'any'\n\nexport interface CompletionRequirement {\n /** Stable id from the task gold (e.g. a persona's `expected_requirements[].req_id`). */\n reqId: string\n /** Human-readable description of the required deliverable. */\n title: string\n /** Optional kind/category hint, matched against a produced item's kind. */\n category?: string\n /** What produced state satisfies this requirement. Defaults to 'any'. */\n satisfiedBy?: SatisfiedBy\n}\n\nexport interface TaskGold {\n taskId: string\n requirements: CompletionRequirement[]\n}\n\nexport interface ProducedProposal {\n id: string\n title: string\n status: 'pending' | 'approved' | 'rejected'\n /** Optional persisted body — when present, enables a correctness check. */\n content?: string\n}\n\n/** Everything observable about what a run actually produced. */\nexport interface ProducedState {\n /** Persisted vault artifacts. Reuses the shared `Artifact` shape. */\n artifacts: Artifact[]\n /** Proposals / filings the agent created. */\n proposals: ProducedProposal[]\n /** Names of tools the agent invoked. */\n toolCalls: string[]\n}\n\nexport interface RequirementCheck {\n reqId: string\n title: string\n /** A produced item of the right kind matched the requirement, non-empty. */\n structurallyPresent: boolean\n /**\n * Whether the matched item actually fulfils the requirement. `null` when\n * not structurally present, or when the matched item carries no content\n * to assess.\n */\n correct: boolean | null\n /** structurallyPresent && correct !== false. */\n satisfied: boolean\n /** Human-readable evidence for the verdict. */\n evidence: string[]\n}\n\n/** Extends the substrate verdict spine: `valid` = `fullyComplete` and\n * `score` = `completionRate` — derived in `completionVerdict()`, the one\n * place those equalities hold by construction. */\nexport interface CompletionVerdict extends DefaultVerdict {\n taskId: string\n requirements: RequirementCheck[]\n /** satisfied / total requirements. */\n completionRate: number\n /** Every requirement satisfied. */\n fullyComplete: boolean\n}\n\n/**\n * Construct a `CompletionVerdict` from the per-requirement checks, deriving\n * `completionRate` / `fullyComplete` and the spine fields (`valid` =\n * `fullyComplete`, `score` = `completionRate`) in one place. Throws on zero\n * requirements — a verdict over nothing is a misconfiguration, mirroring\n * `verifyCompletion`'s gold-spec guard.\n */\nexport function completionVerdict(input: {\n taskId: string\n requirements: RequirementCheck[]\n}): CompletionVerdict {\n if (input.requirements.length === 0) {\n throw new Error(\n `completionVerdict: task '${input.taskId}' has no requirement checks — nothing to derive a verdict from`,\n )\n }\n const satisfiedCount = input.requirements.filter((r) => r.satisfied).length\n const completionRate = satisfiedCount / input.requirements.length\n const fullyComplete = satisfiedCount === input.requirements.length\n return {\n taskId: input.taskId,\n requirements: input.requirements,\n completionRate,\n fullyComplete,\n valid: fullyComplete,\n score: completionRate,\n }\n}\n\n/**\n * Decides whether a produced item's content actually fulfils a requirement.\n * Injected so the structural verifier stays pure and unit-testable; the\n * production implementation is `createLlmCorrectnessChecker`.\n */\nexport type CorrectnessChecker = (\n requirement: CompletionRequirement,\n content: string,\n) => Promise<{ correct: boolean; reason: string }>\n\nconst STOPWORDS = new Set([\n 'the',\n 'a',\n 'an',\n 'of',\n 'for',\n 'and',\n 'or',\n 'to',\n 'in',\n 'on',\n 'with',\n 'by',\n])\n\n// Deliverable-FORM vocabulary — words that name the SHAPE of an output, not its\n// domain content. A correct \"swap-comparison view persisted as a ui/** artifact\"\n// is an OpenUI JSON whose body says nothing about \"artifact\" / \"persisted\" /\n// \"view\"; the discriminative tokens are the domain nouns (swap, comparison).\n// Stripped from the REQUIREMENT side of structural recall so a deliverable is\n// matched on what it IS about, not on the boilerplate describing its form. The\n// correctness checker strips the same class via TITLE_STOPWORDS. Anti-game holds:\n// the distinctive domain tokens remain, so an off-topic item still fails.\nconst REQUIREMENT_FORM_STOPWORDS = new Set([\n 'generated',\n 'generate',\n 'view',\n 'render',\n 'rendered',\n 'persisted',\n 'persist',\n 'artifact',\n 'file',\n 'document',\n 'note',\n 'proposal',\n 'deliverable',\n 'output',\n 'created',\n 'create',\n 'produce',\n 'produced',\n 'flag',\n])\n\nconst MATCH_THRESHOLD = 0.5\nconst MIN_CONTENT_CHARS = 50\n\nfunction tokens(s: string, extraStop?: Set<string>): Set<string> {\n return new Set(\n s\n .toLowerCase()\n .split(/[^a-z0-9]+/)\n .filter((t) => t.length > 1 && !STOPWORDS.has(t) && !extraStop?.has(t)),\n )\n}\n\n/**\n * Recall of the requirement's tokens within a candidate's identifying text.\n * Recall, not Jaccard — a candidate's path/id legitimately carries extra\n * tokens the requirement does not name. The requirement side drops\n * deliverable-FORM vocabulary so recall keys on the distinctive domain tokens.\n */\nfunction tokenRecall(requirementText: string, candidateText: string): number {\n const req = tokens(requirementText, REQUIREMENT_FORM_STOPWORDS)\n if (req.size === 0) return 0\n const cand = tokens(candidateText)\n let hit = 0\n for (const t of req) if (cand.has(t)) hit++\n return hit / req.size\n}\n\ninterface Candidate {\n reqIndex: number\n /** Unique key for a produced item — each item satisfies at most one requirement. */\n itemKey: string\n score: number\n evidence: string\n /** Content to correctness-check, or null when the matched item has none. */\n content: string | null\n}\n\nfunction artifactCandidates(\n req: CompletionRequirement,\n reqIndex: number,\n artifacts: Artifact[],\n): Candidate[] {\n const reqText = `${req.title} ${req.category ?? ''}`\n const out: Candidate[] = []\n artifacts.forEach((a, i) => {\n if ((a.content ?? '').trim().length < MIN_CONTENT_CHARS) return\n // Match against the artifact CONTENT too, not just its path + kind — a\n // generated view / note whose path is generic still satisfies a requirement\n // when its body covers it (e.g. an OpenUI comparison grounded in the on-file\n // figures). Bounded slice keeps the recall text cheap; MATCH_THRESHOLD holds.\n let score = tokenRecall(\n reqText,\n `${a.path ?? ''} ${a.kind} ${(a.content ?? '').slice(0, 4000)}`,\n )\n if (req.category && a.kind && req.category.toLowerCase() === a.kind.toLowerCase()) {\n score = Math.max(score, 1)\n }\n if (score < MATCH_THRESHOLD) return\n out.push({\n reqIndex,\n itemKey: `artifact:${i}`,\n score,\n evidence: `artifact '${a.path ?? a.kind}' matched (token recall ${score.toFixed(2)})`,\n content: a.content ?? null,\n })\n })\n return out\n}\n\nfunction proposalCandidates(\n req: CompletionRequirement,\n reqIndex: number,\n proposals: ProducedProposal[],\n): Candidate[] {\n const reqText = `${req.title} ${req.category ?? ''}`\n const out: Candidate[] = []\n for (const p of proposals) {\n // Pending or rejected work is not a completed deliverable.\n if (p.status !== 'approved') continue\n // A proposal needs an assessable BODY to be a deliverable. A bare title is\n // not completion: correctness cannot be judged on it, so a title-only match\n // would auto-pass the oracle (structurallyPresent && correct===null →\n // satisfied) with no verifiable content. Tool calls are the only\n // legitimately content-less deliverable (`toolCallCandidates`).\n const body = (p.content ?? '').trim()\n if (body.length < MIN_CONTENT_CHARS) continue\n // Match against the body as well as the (often short) title — a refusal /\n // flag / analysis proposal whose title is a label still satisfies a\n // descriptively-worded requirement when its content covers it. MATCH_THRESHOLD\n // + the requirement's distinctive tokens keep an off-topic proposal out;\n // correctness (a SEMANTIC checker, NOT this lexical pass) then judges\n // polarity/fulfilment, so a negation that merely contains the tokens fails.\n // Structural and correctness must use different evidence or the two-stage\n // check collapses to one lexical gate.\n const score = tokenRecall(reqText, `${p.title} ${body}`)\n if (score < MATCH_THRESHOLD) continue\n out.push({\n reqIndex,\n itemKey: `proposal:${p.id}`,\n score,\n evidence: `approved proposal '${p.title}' matched (token recall ${score.toFixed(2)})`,\n content: body,\n })\n }\n return out\n}\n\nfunction toolCallCandidates(\n req: CompletionRequirement,\n reqIndex: number,\n toolCalls: string[],\n): Candidate[] {\n const out: Candidate[] = []\n toolCalls.forEach((name, i) => {\n const score = tokenRecall(req.title, name)\n if (score < MATCH_THRESHOLD) return\n out.push({\n reqIndex,\n itemKey: `tool:${i}`,\n score,\n evidence: `tool call '${name}' matched (token recall ${score.toFixed(2)})`,\n content: null,\n })\n })\n return out\n}\n\n/**\n * Verify whether a run completed the task. `checkCorrectness` is injected —\n * `createLlmCorrectnessChecker` for production, a deterministic stub in tests.\n *\n * Throws on a gold spec with no requirements: an eval task that requires\n * nothing is a misconfiguration, not a vacuously-complete task.\n */\nexport async function verifyCompletion(\n gold: TaskGold,\n state: ProducedState,\n checkCorrectness: CorrectnessChecker,\n): Promise<CompletionVerdict> {\n if (gold.requirements.length === 0) {\n throw new Error(\n `verifyCompletion: task '${gold.taskId}' has no requirements — malformed gold spec`,\n )\n }\n\n // Collect every above-threshold (requirement, produced-item) candidate, then\n // assign greedily by descending score: each requirement and each produced\n // item is used at most once. One deliverable fulfils one requirement.\n const candidates: Candidate[] = []\n gold.requirements.forEach((req, i) => {\n const by = req.satisfiedBy ?? 'any'\n if (by === 'artifact' || by === 'any') {\n candidates.push(...artifactCandidates(req, i, state.artifacts))\n }\n if (by === 'proposal' || by === 'any') {\n candidates.push(...proposalCandidates(req, i, state.proposals))\n }\n if (by === 'tool-call' || by === 'any') {\n candidates.push(...toolCallCandidates(req, i, state.toolCalls))\n }\n })\n candidates.sort((a, b) => b.score - a.score)\n\n const assigned = new Map<number, Candidate>()\n const itemTaken = new Set<string>()\n for (const c of candidates) {\n if (assigned.has(c.reqIndex) || itemTaken.has(c.itemKey)) continue\n assigned.set(c.reqIndex, c)\n itemTaken.add(c.itemKey)\n }\n\n const requirements: RequirementCheck[] = []\n for (let i = 0; i < gold.requirements.length; i++) {\n const req = gold.requirements[i]!\n const match = assigned.get(i)\n const evidence: string[] = []\n let correct: boolean | null = null\n\n if (match) {\n evidence.push(match.evidence)\n if (match.content !== null) {\n const r = await checkCorrectness(req, match.content)\n correct = r.correct\n evidence.push(`correctness: ${r.correct ? 'pass' : 'fail'} — ${r.reason}`)\n } else {\n evidence.push('correctness: not assessed — matched item carries no content')\n }\n } else {\n const by = req.satisfiedBy ?? 'any'\n const kind = by === 'any' ? 'artifact/proposal/tool-call' : by\n evidence.push(`no produced ${kind} matched this requirement`)\n }\n\n const structurallyPresent = match !== undefined\n const satisfied = structurallyPresent && correct !== false\n requirements.push({\n reqId: req.reqId,\n title: req.title,\n structurallyPresent,\n correct,\n satisfied,\n evidence,\n })\n }\n\n return completionVerdict({ taskId: gold.taskId, requirements })\n}\n\nexport interface LlmCorrectnessCheckerOpts {\n model?: string\n /** Max chars of artifact content sent to the checker. */\n maxContentChars?: number\n}\n\n/** Parse the correctness checker's model response. Fails loud on a bad shape. */\nexport function parseCorrectnessResponse(raw: string): { correct: boolean; reason: string } {\n const match = raw.match(/\\{[\\s\\S]*\\}/)\n if (!match) {\n throw new Error(`correctness checker: no JSON object in model response: ${raw.slice(0, 200)}`)\n }\n const parsed = JSON.parse(match[0]) as { correct?: unknown; reason?: unknown }\n if (typeof parsed.correct !== 'boolean') {\n throw new Error(`correctness checker: 'correct' is not a boolean in: ${match[0].slice(0, 200)}`)\n }\n return { correct: parsed.correct, reason: typeof parsed.reason === 'string' ? parsed.reason : '' }\n}\n\n/**\n * Production `CorrectnessChecker` — one LLM call per matched artifact,\n * deterministic (temperature 0), structured JSON out. Judges fulfilment\n * only: a plan, a gesture, or a description of what should be done does not\n * fulfil a requirement — the artifact must BE the deliverable.\n */\nexport function createLlmCorrectnessChecker(\n tc: TCloud,\n opts: LlmCorrectnessCheckerOpts = {},\n): CorrectnessChecker {\n const model = opts.model ?? 'claude-sonnet-4-6'\n const maxContentChars = opts.maxContentChars ?? 8000\n return async (requirement, content) => {\n const resp = await tc.chat({\n model,\n messages: [\n {\n role: 'system',\n content:\n 'You verify whether a produced work artifact actually fulfils a stated requirement. Judge fulfilment only — is the deliverable substantively present and on-point — not polish. A plan to do it later, a vague gesture, or a description of what should be done does NOT fulfil a requirement; the artifact must BE the deliverable. Respond with a single JSON object: {\"correct\": boolean, \"reason\": string (<= 30 words)}.',\n },\n {\n role: 'user',\n content: `Requirement: ${requirement.title}\\n${\n requirement.category ? `Category: ${requirement.category}\\n` : ''\n }\\nProduced artifact:\\n${content.slice(0, maxContentChars)}`,\n },\n ],\n temperature: 0,\n maxTokens: 200,\n })\n const raw =\n (resp as { choices?: { message?: { content?: string } }[] }).choices?.[0]?.message?.content ??\n ''\n return parseCorrectnessResponse(raw)\n }\n}\n\n/** Stopwords for requirement-title tokenization — drops the imperative verbs\n * ('review', 'update', …) common to deliverable titles so recall keys on the\n * substantive nouns, not the boilerplate ask. */\nconst TITLE_STOPWORDS = new Set([\n 'the',\n 'a',\n 'an',\n 'and',\n 'or',\n 'for',\n 'to',\n 'of',\n 'in',\n 'on',\n 'with',\n 'review',\n 'update',\n 'new',\n 'proposed',\n])\n\n/**\n * Deterministic `CorrectnessChecker` — the no-LLM counterpart to\n * `createLlmCorrectnessChecker`. A produced item fulfils a requirement when its\n * content is substantive (≥ `minContentLength` chars) AND recalls ≥ `minRecall`\n * of the requirement title's significant tokens. No network.\n *\n * Polarity-blind: token recall credits a negation that contains the\n * requirement's tokens (\"I will NOT produce the comparison\" recalls every token\n * of \"produce the comparison\"). The structural match stage is ALSO lexical, so\n * pairing the two collapses to a single gameable gate. Use this only as an\n * opt-in structural pre-filter or for tasks whose requirements have no polarity\n * to invert; for produced-state grading the correctness checker MUST be semantic\n * (`createLlmCorrectnessChecker`). See the anti-game fixtures in the test suite.\n */\nexport function createTokenRecallChecker(\n opts: { minRecall?: number; minContentLength?: number } = {},\n): CorrectnessChecker {\n const minRecall = opts.minRecall ?? 0.5\n const minLen = opts.minContentLength ?? 120\n return async (requirement, content) => {\n const body = content.trim()\n if (body.length < minLen)\n return {\n correct: false,\n reason: `content too thin (${body.length} chars) to be the deliverable`,\n }\n const titleTokens = requirement.title\n .toLowerCase()\n .split(/[^a-z0-9]+/)\n .filter((t) => t.length > 2 && !TITLE_STOPWORDS.has(t))\n if (titleTokens.length === 0)\n return {\n correct: true,\n reason: 'requirement title has no significant tokens — structural match accepted',\n }\n const lower = body.toLowerCase()\n const hits = titleTokens.filter((t) => lower.includes(t)).length\n const recall = hits / titleTokens.length\n return recall >= minRecall\n ? {\n correct: true,\n reason: `content recalls ${hits}/${titleTokens.length} requirement tokens`,\n }\n : {\n correct: false,\n reason: `content recalls only ${hits}/${titleTokens.length} requirement tokens`,\n }\n }\n}\n","/**\n * Produced-state extraction — normalize a run's runtime event stream into the\n * typed `ProducedState` the completion oracle consumes.\n *\n * `ProducedState` answers \"what did the agent actually produce\" — vault\n * artifacts, proposals, tool calls. The runtime emits these as a stream of\n * events; this module is the single normalization point from that stream to\n * the shape `verifyCompletion` expects.\n *\n * Input is structurally typed (`RuntimeEventLike`) so this module does not\n * depend on agent-runtime — agent-runtime's `RuntimeStreamEvent` satisfies it\n * structurally. The `content` on `ArtifactEventLike` and the whole\n * `proposal_created` variant are the runtime-side enrichments this contract\n * requires; the runtime emits them, this module consumes them.\n */\n\nimport type { Artifact } from './artifact-validator'\nimport type { ProducedProposal, ProducedState } from './completion-verifier'\n\n/** A tool the agent invoked. */\nexport interface ToolCallEventLike {\n type: 'tool_call'\n toolName: string\n}\n\n/**\n * An artifact the agent produced. `content` is the enriched field — the\n * runtime's base `artifact` event carries only metadata; the completion\n * oracle needs the body to verify the deliverable, so the runtime emits it.\n */\nexport interface ArtifactEventLike {\n type: 'artifact'\n artifactId: string\n name?: string\n mimeType?: string\n uri?: string\n content?: string\n}\n\n/** A proposal / filing the agent created. */\nexport interface ProposalEventLike {\n type: 'proposal_created'\n proposalId: string\n title: string\n status?: 'pending' | 'approved' | 'rejected'\n // body of the proposal (e.g. a submit_proposal `description`). When present,\n // the completion oracle correctness-checks it like artifact content; absent,\n // the proposal is graded presence-only.\n content?: string\n}\n\n/**\n * The subset of runtime stream events `extractProducedState` consumes.\n * agent-runtime's full `RuntimeStreamEvent` union satisfies this structurally;\n * the `{ type: string }` catch-all keeps the input permissive so callers can\n * pass the whole unfiltered telemetry stream — unrecognized events are skipped.\n */\nexport type RuntimeEventLike =\n | ToolCallEventLike\n | ArtifactEventLike\n | ProposalEventLike\n | { type: string }\n\nfunction artifactKind(mimeType: string | undefined): string {\n if (!mimeType) return 'file'\n if (mimeType.includes('json')) return 'json'\n if (mimeType.startsWith('text/')) return 'text'\n return 'file'\n}\n\n/**\n * Normalize a run's runtime event stream into `ProducedState`.\n *\n * Pure and total — unrecognized event types are skipped. `toolCalls` is\n * deduplicated by name in first-seen order (completion cares about a tool's\n * presence, not its call count). An artifact with neither a name nor a uri\n * still yields an entry keyed by its `artifactId` so it is never silently\n * dropped; an artifact with no `content` yields empty content, which the\n * completion oracle's structural check then rejects on its own.\n */\nexport function extractProducedState(events: readonly RuntimeEventLike[]): ProducedState {\n const artifacts: Artifact[] = []\n const proposals: ProducedProposal[] = []\n const toolCalls: string[] = []\n const seenTools = new Set<string>()\n\n for (const ev of events) {\n if (ev.type === 'tool_call') {\n const name = (ev as ToolCallEventLike).toolName\n if (name && !seenTools.has(name)) {\n seenTools.add(name)\n toolCalls.push(name)\n }\n } else if (ev.type === 'artifact') {\n const a = ev as ArtifactEventLike\n artifacts.push({\n kind: artifactKind(a.mimeType),\n path: a.name ?? a.uri ?? a.artifactId,\n content: a.content ?? '',\n })\n } else if (ev.type === 'proposal_created') {\n const p = ev as ProposalEventLike\n proposals.push({\n id: p.proposalId,\n title: p.title,\n status: p.status ?? 'pending',\n ...(p.content !== undefined ? { content: p.content } : {}),\n })\n }\n }\n\n return { artifacts, proposals, toolCalls }\n}\n","/**\n * @stable\n *\n * AgentProfile — the eval harness's unit of variation.\n *\n * A profile pins everything that changes agent behaviour for a benchmark\n * cell: the model, the active skills, the prompt version, the available\n * tools. Vary the profile — swap a model, add a skill — and re-run the suite\n * to benchmark the change. The scorecard keys a cell on\n * `(scenarioId, profileHash)`, so the model is not a separate axis: it lives\n * inside the profile, and two profiles with the same model but different\n * skills are different cells.\n *\n * `agentProfileHash` is the profile's behaviour identity. Two profiles that\n * produce the same agent behaviour share a hash (and a scorecard cell);\n * reordering `skills` or `tools` does not change it; the human-facing `id`\n * label does not affect it.\n */\n\nimport { createHash } from 'node:crypto'\nimport { ValidationError } from './errors'\nimport { canonicalize } from './pre-registration'\n\nexport interface AgentProfile {\n /** Human-facing label, e.g. `sonnet-legal-skills-v3`. Not part of the hash. */\n id: string\n /** Model snapshot id this profile pins, e.g. `claude-sonnet-4-6@2025-04-15`. */\n model: string\n /** Skill ids/versions active in this profile — the primary behaviour lever. */\n skills?: string[]\n /** Prompt version identifier. */\n promptVersion?: string\n /** Tool ids available to the agent. */\n tools?: string[]\n /** Any other behaviour-bearing knobs that should fingerprint into the hash. */\n metadata?: Record<string, string | number | boolean>\n}\n\n/**\n * Deterministic behaviour identity of a profile — a sha256 over the\n * behaviour-bearing fields. `skills` and `tools` are order-insensitive; the\n * `id` label is excluded. Throws on a profile with no `model` — an unkeyable\n * profile must fail loud rather than collapse into a blank-model cell.\n */\nexport function agentProfileHash(profile: AgentProfile): string {\n if (typeof profile.model !== 'string' || profile.model.trim().length === 0) {\n throw new ValidationError(`AgentProfile \"${profile.id}\" has no model — cannot hash`)\n }\n const behaviour = {\n model: profile.model.trim(),\n skills: [...(profile.skills ?? [])].sort(),\n promptVersion: profile.promptVersion ?? null,\n tools: [...(profile.tools ?? [])].sort(),\n metadata: profile.metadata ?? {},\n }\n return createHash('sha256')\n .update(JSON.stringify(canonicalize(behaviour)))\n .digest('hex')\n}\n"],"mappings":";;;;;;;;AAmGO,SAAS,kBAAkB,OAGZ;AACpB,MAAI,MAAM,aAAa,WAAW,GAAG;AACnC,UAAM,IAAI;AAAA,MACR,4BAA4B,MAAM,MAAM;AAAA,IAC1C;AAAA,EACF;AACA,QAAM,iBAAiB,MAAM,aAAa,OAAO,CAAC,MAAM,EAAE,SAAS,EAAE;AACrE,QAAM,iBAAiB,iBAAiB,MAAM,aAAa;AAC3D,QAAM,gBAAgB,mBAAmB,MAAM,aAAa;AAC5D,SAAO;AAAA,IACL,QAAQ,MAAM;AAAA,IACd,cAAc,MAAM;AAAA,IACpB;AAAA,IACA;AAAA,IACA,OAAO;AAAA,IACP,OAAO;AAAA,EACT;AACF;AAYA,IAAM,YAAY,oBAAI,IAAI;AAAA,EACxB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,CAAC;AAUD,IAAM,6BAA6B,oBAAI,IAAI;AAAA,EACzC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,CAAC;AAED,IAAM,kBAAkB;AACxB,IAAM,oBAAoB;AAE1B,SAAS,OAAO,GAAW,WAAsC;AAC/D,SAAO,IAAI;AAAA,IACT,EACG,YAAY,EACZ,MAAM,YAAY,EAClB,OAAO,CAAC,MAAM,EAAE,SAAS,KAAK,CAAC,UAAU,IAAI,CAAC,KAAK,CAAC,WAAW,IAAI,CAAC,CAAC;AAAA,EAC1E;AACF;AAQA,SAAS,YAAY,iBAAyB,eAA+B;AAC3E,QAAM,MAAM,OAAO,iBAAiB,0BAA0B;AAC9D,MAAI,IAAI,SAAS,EAAG,QAAO;AAC3B,QAAM,OAAO,OAAO,aAAa;AACjC,MAAI,MAAM;AACV,aAAW,KAAK,IAAK,KAAI,KAAK,IAAI,CAAC,EAAG;AACtC,SAAO,MAAM,IAAI;AACnB;AAYA,SAAS,mBACP,KACA,UACA,WACa;AACb,QAAM,UAAU,GAAG,IAAI,KAAK,IAAI,IAAI,YAAY,EAAE;AAClD,QAAM,MAAmB,CAAC;AAC1B,YAAU,QAAQ,CAAC,GAAG,MAAM;AAC1B,SAAK,EAAE,WAAW,IAAI,KAAK,EAAE,SAAS,kBAAmB;AAKzD,QAAI,QAAQ;AAAA,MACV;AAAA,MACA,GAAG,EAAE,QAAQ,EAAE,IAAI,EAAE,IAAI,KAAK,EAAE,WAAW,IAAI,MAAM,GAAG,GAAI,CAAC;AAAA,IAC/D;AACA,QAAI,IAAI,YAAY,EAAE,QAAQ,IAAI,SAAS,YAAY,MAAM,EAAE,KAAK,YAAY,GAAG;AACjF,cAAQ,KAAK,IAAI,OAAO,CAAC;AAAA,IAC3B;AACA,QAAI,QAAQ,gBAAiB;AAC7B,QAAI,KAAK;AAAA,MACP;AAAA,MACA,SAAS,YAAY,CAAC;AAAA,MACtB;AAAA,MACA,UAAU,aAAa,EAAE,QAAQ,EAAE,IAAI,2BAA2B,MAAM,QAAQ,CAAC,CAAC;AAAA,MAClF,SAAS,EAAE,WAAW;AAAA,IACxB,CAAC;AAAA,EACH,CAAC;AACD,SAAO;AACT;AAEA,SAAS,mBACP,KACA,UACA,WACa;AACb,QAAM,UAAU,GAAG,IAAI,KAAK,IAAI,IAAI,YAAY,EAAE;AAClD,QAAM,MAAmB,CAAC;AAC1B,aAAW,KAAK,WAAW;AAEzB,QAAI,EAAE,WAAW,WAAY;AAM7B,UAAM,QAAQ,EAAE,WAAW,IAAI,KAAK;AACpC,QAAI,KAAK,SAAS,kBAAmB;AASrC,UAAM,QAAQ,YAAY,SAAS,GAAG,EAAE,KAAK,IAAI,IAAI,EAAE;AACvD,QAAI,QAAQ,gBAAiB;AAC7B,QAAI,KAAK;AAAA,MACP;AAAA,MACA,SAAS,YAAY,EAAE,EAAE;AAAA,MACzB;AAAA,MACA,UAAU,sBAAsB,EAAE,KAAK,2BAA2B,MAAM,QAAQ,CAAC,CAAC;AAAA,MAClF,SAAS;AAAA,IACX,CAAC;AAAA,EACH;AACA,SAAO;AACT;AAEA,SAAS,mBACP,KACA,UACA,WACa;AACb,QAAM,MAAmB,CAAC;AAC1B,YAAU,QAAQ,CAAC,MAAM,MAAM;AAC7B,UAAM,QAAQ,YAAY,IAAI,OAAO,IAAI;AACzC,QAAI,QAAQ,gBAAiB;AAC7B,QAAI,KAAK;AAAA,MACP;AAAA,MACA,SAAS,QAAQ,CAAC;AAAA,MAClB;AAAA,MACA,UAAU,cAAc,IAAI,2BAA2B,MAAM,QAAQ,CAAC,CAAC;AAAA,MACvE,SAAS;AAAA,IACX,CAAC;AAAA,EACH,CAAC;AACD,SAAO;AACT;AASA,eAAsB,iBACpB,MACA,OACA,kBAC4B;AAC5B,MAAI,KAAK,aAAa,WAAW,GAAG;AAClC,UAAM,IAAI;AAAA,MACR,2BAA2B,KAAK,MAAM;AAAA,IACxC;AAAA,EACF;AAKA,QAAM,aAA0B,CAAC;AACjC,OAAK,aAAa,QAAQ,CAAC,KAAK,MAAM;AACpC,UAAM,KAAK,IAAI,eAAe;AAC9B,QAAI,OAAO,cAAc,OAAO,OAAO;AACrC,iBAAW,KAAK,GAAG,mBAAmB,KAAK,GAAG,MAAM,SAAS,CAAC;AAAA,IAChE;AACA,QAAI,OAAO,cAAc,OAAO,OAAO;AACrC,iBAAW,KAAK,GAAG,mBAAmB,KAAK,GAAG,MAAM,SAAS,CAAC;AAAA,IAChE;AACA,QAAI,OAAO,eAAe,OAAO,OAAO;AACtC,iBAAW,KAAK,GAAG,mBAAmB,KAAK,GAAG,MAAM,SAAS,CAAC;AAAA,IAChE;AAAA,EACF,CAAC;AACD,aAAW,KAAK,CAAC,GAAG,MAAM,EAAE,QAAQ,EAAE,KAAK;AAE3C,QAAM,WAAW,oBAAI,IAAuB;AAC5C,QAAM,YAAY,oBAAI,IAAY;AAClC,aAAW,KAAK,YAAY;AAC1B,QAAI,SAAS,IAAI,EAAE,QAAQ,KAAK,UAAU,IAAI,EAAE,OAAO,EAAG;AAC1D,aAAS,IAAI,EAAE,UAAU,CAAC;AAC1B,cAAU,IAAI,EAAE,OAAO;AAAA,EACzB;AAEA,QAAM,eAAmC,CAAC;AAC1C,WAAS,IAAI,GAAG,IAAI,KAAK,aAAa,QAAQ,KAAK;AACjD,UAAM,MAAM,KAAK,aAAa,CAAC;AAC/B,UAAM,QAAQ,SAAS,IAAI,CAAC;AAC5B,UAAM,WAAqB,CAAC;AAC5B,QAAI,UAA0B;AAE9B,QAAI,OAAO;AACT,eAAS,KAAK,MAAM,QAAQ;AAC5B,UAAI,MAAM,YAAY,MAAM;AAC1B,cAAM,IAAI,MAAM,iBAAiB,KAAK,MAAM,OAAO;AACnD,kBAAU,EAAE;AACZ,iBAAS,KAAK,gBAAgB,EAAE,UAAU,SAAS,MAAM,WAAM,EAAE,MAAM,EAAE;AAAA,MAC3E,OAAO;AACL,iBAAS,KAAK,kEAA6D;AAAA,MAC7E;AAAA,IACF,OAAO;AACL,YAAM,KAAK,IAAI,eAAe;AAC9B,YAAM,OAAO,OAAO,QAAQ,gCAAgC;AAC5D,eAAS,KAAK,eAAe,IAAI,2BAA2B;AAAA,IAC9D;AAEA,UAAM,sBAAsB,UAAU;AACtC,UAAM,YAAY,uBAAuB,YAAY;AACrD,iBAAa,KAAK;AAAA,MAChB,OAAO,IAAI;AAAA,MACX,OAAO,IAAI;AAAA,MACX;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF,CAAC;AAAA,EACH;AAEA,SAAO,kBAAkB,EAAE,QAAQ,KAAK,QAAQ,aAAa,CAAC;AAChE;AASO,SAAS,yBAAyB,KAAmD;AAC1F,QAAM,QAAQ,IAAI,MAAM,aAAa;AACrC,MAAI,CAAC,OAAO;AACV,UAAM,IAAI,MAAM,0DAA0D,IAAI,MAAM,GAAG,GAAG,CAAC,EAAE;AAAA,EAC/F;AACA,QAAM,SAAS,KAAK,MAAM,MAAM,CAAC,CAAC;AAClC,MAAI,OAAO,OAAO,YAAY,WAAW;AACvC,UAAM,IAAI,MAAM,uDAAuD,MAAM,CAAC,EAAE,MAAM,GAAG,GAAG,CAAC,EAAE;AAAA,EACjG;AACA,SAAO,EAAE,SAAS,OAAO,SAAS,QAAQ,OAAO,OAAO,WAAW,WAAW,OAAO,SAAS,GAAG;AACnG;AAQO,SAAS,4BACd,IACA,OAAkC,CAAC,GACf;AACpB,QAAM,QAAQ,KAAK,SAAS;AAC5B,QAAM,kBAAkB,KAAK,mBAAmB;AAChD,SAAO,OAAO,aAAa,YAAY;AACrC,UAAM,OAAO,MAAM,GAAG,KAAK;AAAA,MACzB;AAAA,MACA,UAAU;AAAA,QACR;AAAA,UACE,MAAM;AAAA,UACN,SACE;AAAA,QACJ;AAAA,QACA;AAAA,UACE,MAAM;AAAA,UACN,SAAS,gBAAgB,YAAY,KAAK;AAAA,EACxC,YAAY,WAAW,aAAa,YAAY,QAAQ;AAAA,IAAO,EACjE;AAAA;AAAA,EAAyB,QAAQ,MAAM,GAAG,eAAe,CAAC;AAAA,QAC5D;AAAA,MACF;AAAA,MACA,aAAa;AAAA,MACb,WAAW;AAAA,IACb,CAAC;AACD,UAAM,MACH,KAA4D,UAAU,CAAC,GAAG,SAAS,WACpF;AACF,WAAO,yBAAyB,GAAG;AAAA,EACrC;AACF;AAKA,IAAM,kBAAkB,oBAAI,IAAI;AAAA,EAC9B;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,CAAC;AAgBM,SAAS,yBACd,OAA0D,CAAC,GACvC;AACpB,QAAM,YAAY,KAAK,aAAa;AACpC,QAAM,SAAS,KAAK,oBAAoB;AACxC,SAAO,OAAO,aAAa,YAAY;AACrC,UAAM,OAAO,QAAQ,KAAK;AAC1B,QAAI,KAAK,SAAS;AAChB,aAAO;AAAA,QACL,SAAS;AAAA,QACT,QAAQ,qBAAqB,KAAK,MAAM;AAAA,MAC1C;AACF,UAAM,cAAc,YAAY,MAC7B,YAAY,EACZ,MAAM,YAAY,EAClB,OAAO,CAAC,MAAM,EAAE,SAAS,KAAK,CAAC,gBAAgB,IAAI,CAAC,CAAC;AACxD,QAAI,YAAY,WAAW;AACzB,aAAO;AAAA,QACL,SAAS;AAAA,QACT,QAAQ;AAAA,MACV;AACF,UAAM,QAAQ,KAAK,YAAY;AAC/B,UAAM,OAAO,YAAY,OAAO,CAAC,MAAM,MAAM,SAAS,CAAC,CAAC,EAAE;AAC1D,UAAM,SAAS,OAAO,YAAY;AAClC,WAAO,UAAU,YACb;AAAA,MACE,SAAS;AAAA,MACT,QAAQ,mBAAmB,IAAI,IAAI,YAAY,MAAM;AAAA,IACvD,IACA;AAAA,MACE,SAAS;AAAA,MACT,QAAQ,wBAAwB,IAAI,IAAI,YAAY,MAAM;AAAA,IAC5D;AAAA,EACN;AACF;;;AC/bA,SAAS,aAAa,UAAsC;AAC1D,MAAI,CAAC,SAAU,QAAO;AACtB,MAAI,SAAS,SAAS,MAAM,EAAG,QAAO;AACtC,MAAI,SAAS,WAAW,OAAO,EAAG,QAAO;AACzC,SAAO;AACT;AAYO,SAAS,qBAAqB,QAAoD;AACvF,QAAM,YAAwB,CAAC;AAC/B,QAAM,YAAgC,CAAC;AACvC,QAAM,YAAsB,CAAC;AAC7B,QAAM,YAAY,oBAAI,IAAY;AAElC,aAAW,MAAM,QAAQ;AACvB,QAAI,GAAG,SAAS,aAAa;AAC3B,YAAM,OAAQ,GAAyB;AACvC,UAAI,QAAQ,CAAC,UAAU,IAAI,IAAI,GAAG;AAChC,kBAAU,IAAI,IAAI;AAClB,kBAAU,KAAK,IAAI;AAAA,MACrB;AAAA,IACF,WAAW,GAAG,SAAS,YAAY;AACjC,YAAM,IAAI;AACV,gBAAU,KAAK;AAAA,QACb,MAAM,aAAa,EAAE,QAAQ;AAAA,QAC7B,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE;AAAA,QAC3B,SAAS,EAAE,WAAW;AAAA,MACxB,CAAC;AAAA,IACH,WAAW,GAAG,SAAS,oBAAoB;AACzC,YAAM,IAAI;AACV,gBAAU,KAAK;AAAA,QACb,IAAI,EAAE;AAAA,QACN,OAAO,EAAE;AAAA,QACT,QAAQ,EAAE,UAAU;AAAA,QACpB,GAAI,EAAE,YAAY,SAAY,EAAE,SAAS,EAAE,QAAQ,IAAI,CAAC;AAAA,MAC1D,CAAC;AAAA,IACH;AAAA,EACF;AAEA,SAAO,EAAE,WAAW,WAAW,UAAU;AAC3C;;;AC7FA,SAAS,kBAAkB;AAyBpB,SAAS,iBAAiB,SAA+B;AAC9D,MAAI,OAAO,QAAQ,UAAU,YAAY,QAAQ,MAAM,KAAK,EAAE,WAAW,GAAG;AAC1E,UAAM,IAAI,gBAAgB,iBAAiB,QAAQ,EAAE,mCAA8B;AAAA,EACrF;AACA,QAAM,YAAY;AAAA,IAChB,OAAO,QAAQ,MAAM,KAAK;AAAA,IAC1B,QAAQ,CAAC,GAAI,QAAQ,UAAU,CAAC,CAAE,EAAE,KAAK;AAAA,IACzC,eAAe,QAAQ,iBAAiB;AAAA,IACxC,OAAO,CAAC,GAAI,QAAQ,SAAS,CAAC,CAAE,EAAE,KAAK;AAAA,IACvC,UAAU,QAAQ,YAAY,CAAC;AAAA,EACjC;AACA,SAAO,WAAW,QAAQ,EACvB,OAAO,KAAK,UAAU,aAAa,SAAS,CAAC,CAAC,EAC9C,OAAO,KAAK;AACjB;","names":[]}
|
package/dist/fuzz.d.ts
CHANGED
|
@@ -52,6 +52,9 @@ interface Evaluation extends DefaultVerdict {
|
|
|
52
52
|
runId?: string;
|
|
53
53
|
/** Structured labels, e.g. failure classes (`hallucination`, `refusal`). */
|
|
54
54
|
labels?: string[];
|
|
55
|
+
/** Wall-clock for the evaluation, when the consumer measures it more precisely
|
|
56
|
+
* than the engine can (e.g. excluding judge time). Engine-measured otherwise. */
|
|
57
|
+
latencyMs?: number;
|
|
55
58
|
}
|
|
56
59
|
/** Run the target against one scenario in a cell. */
|
|
57
60
|
type Evaluator<S> = (scenario: S, cell: Cell) => Promise<Evaluation>;
|
|
@@ -123,16 +126,34 @@ interface ArchiveEntry<S> {
|
|
|
123
126
|
evaluation: Evaluation;
|
|
124
127
|
interest: number;
|
|
125
128
|
}
|
|
129
|
+
/** Summary of a sample — every aggregate carries its spread, never a bare mean. */
|
|
130
|
+
interface Distribution {
|
|
131
|
+
mean: number;
|
|
132
|
+
median: number;
|
|
133
|
+
p90: number;
|
|
134
|
+
min: number;
|
|
135
|
+
max: number;
|
|
136
|
+
n: number;
|
|
137
|
+
}
|
|
126
138
|
/** Per-INPUT-cell coverage — the planned-vs-covered map. */
|
|
127
139
|
interface CoverageCell {
|
|
128
140
|
cell: Cell;
|
|
129
141
|
runs: number;
|
|
130
|
-
/**
|
|
131
|
-
|
|
142
|
+
/** Headline score distribution in [0,1]; `null` when the cell was never run
|
|
143
|
+
* (honestly uncovered — never a fabricated zero). */
|
|
144
|
+
score: Distribution | null;
|
|
132
145
|
/** Fraction of runs the objective flagged as notable. */
|
|
133
146
|
findingRate: number;
|
|
134
|
-
/**
|
|
135
|
-
|
|
147
|
+
/** Per-dimension score distributions — surfaces WHICH dimension is weak and
|
|
148
|
+
* how consistently. */
|
|
149
|
+
dimensions: Record<string, Distribution>;
|
|
150
|
+
/** Evaluation wall-clock per run; engine-measured unless the evaluation
|
|
151
|
+
* carried its own `latencyMs`. `null` when the cell was never run. */
|
|
152
|
+
latencyMs: Distribution | null;
|
|
153
|
+
/** Known dollars spent in this cell — present only when cost tracking was
|
|
154
|
+
* wired; runs with unknown cost are counted apart, never folded in as $0. */
|
|
155
|
+
costUsd?: number;
|
|
156
|
+
costUnknownRuns?: number;
|
|
136
157
|
}
|
|
137
158
|
/** The artifact every exploration produces. */
|
|
138
159
|
interface CapsuleData<S> {
|
|
@@ -160,7 +181,12 @@ interface CapsuleData<S> {
|
|
|
160
181
|
behaviorBinsObserved: number;
|
|
161
182
|
candidateFindings: number;
|
|
162
183
|
verifiedFindings: number;
|
|
163
|
-
|
|
184
|
+
/** Distribution of per-cell mean scores across covered cells (cells weigh
|
|
185
|
+
* equally — variance steering sends more runs to weak cells, so a
|
|
186
|
+
* run-weighted average would bias low). `null` when nothing ran. */
|
|
187
|
+
robustness: Distribution | null;
|
|
188
|
+
/** Evaluation wall-clock across all runs. `null` when nothing ran. */
|
|
189
|
+
latencyMs: Distribution | null;
|
|
164
190
|
/** Known dollars spent on this exploration's runs. Present only when cost
|
|
165
191
|
* tracking was wired (`costOf`) — absent means "not tracked", never $0. */
|
|
166
192
|
costUsd?: number;
|
|
@@ -285,9 +311,10 @@ interface ExploreOptions<S> {
|
|
|
285
311
|
*
|
|
286
312
|
* Cells are the cartesian product of the input axes — the stratification plan,
|
|
287
313
|
* enumerable up front so the planned-vs-covered denominator is honest. Coverage
|
|
288
|
-
* is projected from the evaluation log: per cell,
|
|
289
|
-
*
|
|
290
|
-
*
|
|
314
|
+
* is projected from the evaluation log: per cell, the full DISTRIBUTION of the
|
|
315
|
+
* headline score, of each scored dimension, and of evaluation latency — a bare
|
|
316
|
+
* mean hides outliers, so every aggregate carries its spread. Per-cell cost is
|
|
317
|
+
* split known-dollars vs unknown-runs, never folded into a fabricated $0.
|
|
291
318
|
*/
|
|
292
319
|
|
|
293
320
|
/** One recorded evaluation — the unit coverage and the capsule are built from. */
|
|
@@ -296,6 +323,11 @@ interface EvalRecord {
|
|
|
296
323
|
ev: Evaluation;
|
|
297
324
|
/** The objective's interest score for this evaluation. */
|
|
298
325
|
interest: number;
|
|
326
|
+
/** Evaluation wall-clock — engine-measured unless `ev.latencyMs` overrode it. */
|
|
327
|
+
latencyMs: number;
|
|
328
|
+
/** Known dollars for this run. `null` = cost tracking was wired but this
|
|
329
|
+
* run's cost was unknowable (counted apart). Absent = not tracked at all. */
|
|
330
|
+
costUsd?: number | null;
|
|
299
331
|
}
|
|
300
332
|
/** Enumerate every input cell (cartesian product of the axes), in stable order. */
|
|
301
333
|
declare function enumerateCells(space: BehaviorSpace): Cell[];
|
|
@@ -303,7 +335,7 @@ declare function enumerateCells(space: BehaviorSpace): Cell[];
|
|
|
303
335
|
declare function cellId(space: BehaviorSpace, coords: Record<string, string>): string;
|
|
304
336
|
/**
|
|
305
337
|
* Project the evaluation log into the per-input-cell coverage map. A cell with
|
|
306
|
-
* no evaluations reports `
|
|
338
|
+
* no evaluations reports `score: null` (honestly uncovered), never zeros.
|
|
307
339
|
*/
|
|
308
340
|
declare function buildCoverage(cells: Cell[], log: EvalRecord[], threshold: number): CoverageCell[];
|
|
309
341
|
|
|
@@ -398,6 +430,8 @@ declare class BehaviorExplorer<S> {
|
|
|
398
430
|
private costExhausted;
|
|
399
431
|
/** Fold one run's cost in: null counts as unknown (never $0); a known cost
|
|
400
432
|
* accrues toward the budget, lands in the ledger, and fires `onCost`. */
|
|
433
|
+
/** Returns the run's known cost, `null` when tracked-but-unknown, `undefined`
|
|
434
|
+
* when cost tracking is not wired — the log row mirrors this exactly. */
|
|
401
435
|
private recordRunCost;
|
|
402
436
|
/** Elites whose INPUT cell matches — what the proposer mutates/deepens from. */
|
|
403
437
|
private elitesFor;
|
package/dist/fuzz.js
CHANGED
|
@@ -22,7 +22,24 @@ function enumerateCells(space) {
|
|
|
22
22
|
function cellId(space, coords) {
|
|
23
23
|
return space.axes.map((a) => `${a.name}=${coords[a.name]}`).join("|");
|
|
24
24
|
}
|
|
25
|
-
|
|
25
|
+
function percentile(sorted, p) {
|
|
26
|
+
const idx = Math.min(sorted.length - 1, Math.max(0, Math.ceil(p * sorted.length) - 1));
|
|
27
|
+
return sorted[idx];
|
|
28
|
+
}
|
|
29
|
+
function distribution(values) {
|
|
30
|
+
if (values.length === 0)
|
|
31
|
+
throw new Error("distribution: empty sample \u2014 represent missing data as null, not zeros");
|
|
32
|
+
const sorted = [...values].sort((a, b) => a - b);
|
|
33
|
+
const mean = sorted.reduce((a, b) => a + b, 0) / sorted.length;
|
|
34
|
+
return {
|
|
35
|
+
mean,
|
|
36
|
+
median: percentile(sorted, 0.5),
|
|
37
|
+
p90: percentile(sorted, 0.9),
|
|
38
|
+
min: sorted[0],
|
|
39
|
+
max: sorted[sorted.length - 1],
|
|
40
|
+
n: sorted.length
|
|
41
|
+
};
|
|
42
|
+
}
|
|
26
43
|
function buildCoverage(cells, log, threshold) {
|
|
27
44
|
const byCell = /* @__PURE__ */ new Map();
|
|
28
45
|
for (const r of log) {
|
|
@@ -33,19 +50,27 @@ function buildCoverage(cells, log, threshold) {
|
|
|
33
50
|
return cells.map((cell) => {
|
|
34
51
|
const recs = byCell.get(cell.id) ?? [];
|
|
35
52
|
const runs = recs.length;
|
|
36
|
-
if (runs === 0)
|
|
37
|
-
|
|
53
|
+
if (runs === 0)
|
|
54
|
+
return { cell, runs: 0, score: null, findingRate: 0, dimensions: {}, latencyMs: null };
|
|
55
|
+
const score = distribution(recs.map((r) => r.ev.score));
|
|
56
|
+
const latencyMs = distribution(recs.map((r) => r.latencyMs));
|
|
38
57
|
const findingRate = recs.filter((r) => r.interest >= threshold).length / runs;
|
|
39
|
-
const
|
|
58
|
+
const dimSamples = {};
|
|
40
59
|
for (const r of recs) {
|
|
41
60
|
for (const [k, v] of Object.entries(r.ev.scores ?? {})) {
|
|
42
61
|
;
|
|
43
|
-
(
|
|
62
|
+
(dimSamples[k] ??= []).push(v);
|
|
44
63
|
}
|
|
45
64
|
}
|
|
46
65
|
const dimensions = {};
|
|
47
|
-
for (const [k, xs] of Object.entries(
|
|
48
|
-
|
|
66
|
+
for (const [k, xs] of Object.entries(dimSamples)) dimensions[k] = distribution(xs);
|
|
67
|
+
const tracked = recs.filter((r) => r.costUsd !== void 0);
|
|
68
|
+
const known = tracked.filter((r) => r.costUsd !== null);
|
|
69
|
+
const cost = tracked.length > 0 ? {
|
|
70
|
+
costUsd: known.reduce((a, r) => a + r.costUsd, 0),
|
|
71
|
+
...tracked.length > known.length ? { costUnknownRuns: tracked.length - known.length } : {}
|
|
72
|
+
} : {};
|
|
73
|
+
return { cell, runs, score, findingRate, dimensions, latencyMs, ...cost };
|
|
49
74
|
});
|
|
50
75
|
}
|
|
51
76
|
|
|
@@ -53,7 +78,8 @@ function buildCoverage(cells, log, threshold) {
|
|
|
53
78
|
function buildCapsule(input) {
|
|
54
79
|
const coverage = buildCoverage(input.cells, input.log, input.threshold);
|
|
55
80
|
const covered = coverage.filter((c) => c.runs > 0);
|
|
56
|
-
const
|
|
81
|
+
const robustness = covered.length === 0 ? null : distribution(covered.map((c) => c.score.mean));
|
|
82
|
+
const latencyMs = input.log.length === 0 ? null : distribution(input.log.map((r) => r.latencyMs));
|
|
57
83
|
const behaviorBinsObserved = input.archive.filter((e) => e.binId !== e.cell.id).length;
|
|
58
84
|
return {
|
|
59
85
|
target: input.target,
|
|
@@ -68,7 +94,8 @@ function buildCapsule(input) {
|
|
|
68
94
|
behaviorBinsObserved,
|
|
69
95
|
candidateFindings: input.candidateFindings,
|
|
70
96
|
verifiedFindings: input.findings.length,
|
|
71
|
-
|
|
97
|
+
robustness,
|
|
98
|
+
latencyMs,
|
|
72
99
|
...input.cost ? { costUsd: input.cost.costUsd, costUnknownRuns: input.cost.costUnknownRuns } : {},
|
|
73
100
|
evalErrors: input.evalErrors,
|
|
74
101
|
...input.stoppedEarly ? { stoppedEarly: input.stoppedEarly } : {}
|
|
@@ -103,7 +130,7 @@ function deriveAxes(coverage) {
|
|
|
103
130
|
return order.map((name) => ({ name, values: [...seen.get(name) ?? []] }));
|
|
104
131
|
}
|
|
105
132
|
function weakestDim(c) {
|
|
106
|
-
const entries = Object.entries(c.dimensions);
|
|
133
|
+
const entries = Object.entries(c.dimensions).map(([k, d]) => [k, d.mean]);
|
|
107
134
|
if (entries.length === 0) return "";
|
|
108
135
|
const sorted = entries.sort((a, b) => a[1] - b[1]);
|
|
109
136
|
const w = sorted[0];
|
|
@@ -114,8 +141,8 @@ function heatmapHtml(coverage) {
|
|
|
114
141
|
const axes = deriveAxes(coverage);
|
|
115
142
|
const byId = new Map(coverage.map((c) => [c.cell.id, c]));
|
|
116
143
|
const tile = (c, label) => {
|
|
117
|
-
const r = c?.
|
|
118
|
-
const title = c ? `${pct(
|
|
144
|
+
const r = c?.score?.mean ?? null;
|
|
145
|
+
const title = c?.score ? `${pct(c.score.mean)} robust (median ${pct(c.score.median)}, min ${pct(c.score.min)}) \xB7 ${c.runs} runs \xB7 ${pct(c.findingRate)} flagged${c.latencyMs ? ` \xB7 ${(c.latencyMs.median / 1e3).toFixed(1)}s median` : ""}${c.costUsd !== void 0 ? ` \xB7 $${c.costUsd.toFixed(2)}` : ""}` : "not covered";
|
|
119
146
|
return `<div class="tile" style="background:${robustnessColor(r)}" title="${esc(title)}">${label ? `<span class="tl">${esc(label)}</span>` : ""}<span class="tv">${c && r != null ? pct(r) : "\u2014"}</span>${c ? weakestDim(c) : ""}</div>`;
|
|
120
147
|
};
|
|
121
148
|
const rowAxis = axes[0];
|
|
@@ -131,7 +158,7 @@ function heatmapHtml(coverage) {
|
|
|
131
158
|
}).join("");
|
|
132
159
|
return `<div class="axis-label">rows: <b>${esc(rowAxis.name)}</b> \xB7 cols: <b>${esc(colAxis.name)}</b></div><table class="heat">${head}${rows}</table>`;
|
|
133
160
|
}
|
|
134
|
-
const sorted = [...coverage].sort((a, b) => (a.
|
|
161
|
+
const sorted = [...coverage].sort((a, b) => (a.score?.mean ?? 2) - (b.score?.mean ?? 2));
|
|
135
162
|
return `<div class="grid">${sorted.map((c) => tile(c, Object.values(c.cell.coords).join(" \xB7 "))).join("")}</div>`;
|
|
136
163
|
}
|
|
137
164
|
function findingsHtml(findings, limit) {
|
|
@@ -198,7 +225,9 @@ table.heat th.rh{text-align:right}
|
|
|
198
225
|
<h1>${esc(capsule.objective)} exploration \xB7 ${esc(capsule.target)}</h1>
|
|
199
226
|
<div class="sub">${s.totalRuns} scenarios across ${s.cellsCovered}/${s.cellsTotal} planned cells${s.behaviorBinsObserved > 0 ? ` \xB7 ${s.behaviorBinsObserved} measured behavior bins` : ""}${stamp ? ` \xB7 ${esc(stamp)}` : ""}</div>
|
|
200
227
|
<div class="kpis">
|
|
201
|
-
${kpi("
|
|
228
|
+
${s.robustness ? kpi("robustness", `${pct(s.robustness.mean)}`, s.robustness.mean < 0.6 ? "#e58a96" : "#5ad17a") : ""}
|
|
229
|
+
${s.robustness ? kpi("cell spread", `${pct(s.robustness.min)}\u2013${pct(s.robustness.max)}`) : ""}
|
|
230
|
+
${s.latencyMs ? kpi("median latency", `${(s.latencyMs.median / 1e3).toFixed(1)}s`, s.latencyMs.p90 > 4 * s.latencyMs.median ? "#e5b566" : "#e6e6e6") : ""}
|
|
202
231
|
${kpi("verified findings", String(s.verifiedFindings), s.verifiedFindings > 0 ? "#e58a96" : "#5ad17a")}
|
|
203
232
|
${kpi("cells covered", `${s.cellsCovered}/${s.cellsTotal}`)}
|
|
204
233
|
${kpi("scenarios run", String(s.totalRuns))}
|
|
@@ -370,12 +399,14 @@ var BehaviorExplorer = class {
|
|
|
370
399
|
}
|
|
371
400
|
/** Fold one run's cost in: null counts as unknown (never $0); a known cost
|
|
372
401
|
* accrues toward the budget, lands in the ledger, and fires `onCost`. */
|
|
402
|
+
/** Returns the run's known cost, `null` when tracked-but-unknown, `undefined`
|
|
403
|
+
* when cost tracking is not wired — the log row mirrors this exactly. */
|
|
373
404
|
recordRunCost(scenario, cell, ev) {
|
|
374
|
-
if (!this.opts.costOf) return;
|
|
405
|
+
if (!this.opts.costOf) return void 0;
|
|
375
406
|
const cost = this.opts.costOf(scenario, cell, ev);
|
|
376
407
|
if (cost === null) {
|
|
377
408
|
this.costUnknownRuns++;
|
|
378
|
-
return;
|
|
409
|
+
return null;
|
|
379
410
|
}
|
|
380
411
|
if (typeof cost.usd !== "number" || !Number.isFinite(cost.usd) || cost.usd < 0) {
|
|
381
412
|
throw new RangeError(
|
|
@@ -391,6 +422,7 @@ var BehaviorExplorer = class {
|
|
|
391
422
|
tags: { target: this.opts.target, cell: cell.id }
|
|
392
423
|
});
|
|
393
424
|
this.opts.onCost?.({ usd: cost.usd, channel: "agent" });
|
|
425
|
+
return cost.usd;
|
|
394
426
|
}
|
|
395
427
|
/** Elites whose INPUT cell matches — what the proposer mutates/deepens from. */
|
|
396
428
|
elitesFor(cellId2) {
|
|
@@ -432,13 +464,22 @@ var BehaviorExplorer = class {
|
|
|
432
464
|
if (this.runsUsed >= this.opts.budget || this.costExhausted() || this.stoppedEarly !== void 0 || this.opts.signal?.aborted)
|
|
433
465
|
return;
|
|
434
466
|
try {
|
|
467
|
+
const startedAt = performance.now();
|
|
435
468
|
const ev = await this.opts.evaluate(scenario, cell);
|
|
469
|
+
const latencyMs = ev.latencyMs ?? performance.now() - startedAt;
|
|
436
470
|
this.runsUsed++;
|
|
437
471
|
runsThisStep++;
|
|
438
472
|
this.consecutiveEvalErrors = 0;
|
|
439
|
-
this.recordRunCost(scenario, cell, ev);
|
|
473
|
+
const costUsd = this.recordRunCost(scenario, cell, ev);
|
|
440
474
|
const interest = this.objective.interest(ev, this.objectiveContext());
|
|
441
|
-
this.log.push({
|
|
475
|
+
this.log.push({
|
|
476
|
+
cell,
|
|
477
|
+
ev,
|
|
478
|
+
interest,
|
|
479
|
+
latencyMs,
|
|
480
|
+
...costUsd !== void 0 ? { costUsd } : {},
|
|
481
|
+
scenarioId: this.opts.scenarioId(scenario)
|
|
482
|
+
});
|
|
442
483
|
this.opts.onProgress?.({ type: "evaluated", cell, scenario, evaluation: ev });
|
|
443
484
|
const bin = this.binId(cell, ev.descriptor);
|
|
444
485
|
const cur = this.archiveByBin.get(bin);
|
package/dist/fuzz.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../src/fuzz/cube.ts","../src/fuzz/capsule.ts","../src/fuzz/policies.ts","../src/fuzz/explorer.ts","../src/fuzz/fuzz-agent.ts","../src/fuzz/gates.ts","../src/fuzz/tools.ts"],"sourcesContent":["/**\n * Input-space tiling + coverage projection.\n *\n * Cells are the cartesian product of the input axes — the stratification plan,\n * enumerable up front so the planned-vs-covered denominator is honest. Coverage\n * is projected from the evaluation log: per cell, mean headline robustness, the\n * mean of each scored dimension (so the map shows WHICH dimension is weak), and\n * the rate at which the active objective flagged a candidate.\n */\n\nimport type { BehaviorSpace, Cell, CoverageCell, Evaluation } from './types'\n\n/** One recorded evaluation — the unit coverage and the capsule are built from. */\nexport interface EvalRecord {\n cell: Cell\n ev: Evaluation\n /** The objective's interest score for this evaluation. */\n interest: number\n}\n\n/** Enumerate every input cell (cartesian product of the axes), in stable order. */\nexport function enumerateCells(space: BehaviorSpace): Cell[] {\n if (space.axes.length === 0) return []\n let partials: Array<Record<string, string>> = [{}]\n for (const axis of space.axes) {\n const next: Array<Record<string, string>> = []\n for (const partial of partials) {\n for (const value of axis.values) next.push({ ...partial, [axis.name]: value })\n }\n partials = next\n }\n return partials.map((coords) => ({ id: cellId(space, coords), coords }))\n}\n\n/** Deterministic id for a coordinate map, e.g. `matterType=nda|difficulty=hard`. */\nexport function cellId(space: BehaviorSpace, coords: Record<string, string>): string {\n return space.axes.map((a) => `${a.name}=${coords[a.name]}`).join('|')\n}\n\nconst mean = (xs: number[]): number =>\n xs.length === 0 ? 0 : xs.reduce((a, b) => a + b, 0) / xs.length\n\n/**\n * Project the evaluation log into the per-input-cell coverage map. A cell with\n * no evaluations reports `robustness: null` (honestly uncovered), never 0.\n */\nexport function buildCoverage(cells: Cell[], log: EvalRecord[], threshold: number): CoverageCell[] {\n const byCell = new Map<string, EvalRecord[]>()\n for (const r of log) {\n const arr = byCell.get(r.cell.id) ?? []\n arr.push(r)\n byCell.set(r.cell.id, arr)\n }\n return cells.map((cell) => {\n const recs = byCell.get(cell.id) ?? []\n const runs = recs.length\n if (runs === 0) return { cell, runs: 0, robustness: null, findingRate: 0, dimensions: {} }\n const robustness = mean(recs.map((r) => r.ev.score))\n const findingRate = recs.filter((r) => r.interest >= threshold).length / runs\n const dims: Record<string, number[]> = {}\n for (const r of recs) {\n for (const [k, v] of Object.entries(r.ev.scores ?? {})) {\n ;(dims[k] ??= []).push(v)\n }\n }\n const dimensions: Record<string, number> = {}\n for (const [k, xs] of Object.entries(dims)) dimensions[k] = mean(xs)\n return { cell, runs, robustness, findingRate, dimensions }\n })\n}\n","/**\n * The capsule — the artifact every exploration produces.\n *\n * `buildCapsule` assembles coverage + verified findings + the QD archive into a\n * pure `CapsuleData` (no clock, no I/O — deterministic and snapshot-testable).\n * `renderCapsuleHtml` turns it into a standalone page: the input-cell heat-map\n * (planned vs covered), per-dimension weakness chips, and the minimized finding\n * exemplars. One artifact — the hardening map and the shareable proof object.\n */\n\nimport type { EvalRecord } from './cube'\nimport { buildCoverage } from './cube'\nimport type { ArchiveEntry, CapsuleData, Cell, CoverageCell, Finding } from './types'\n\nexport interface BuildCapsuleInput<S> {\n target: string\n objective: string\n cells: Cell[]\n log: EvalRecord[]\n /** The objective's notable threshold — drives findingRate. */\n threshold: number\n archive: ArchiveEntry<S>[]\n findings: Finding<S>[]\n candidateFindings: number\n runsUsed: number\n /** Known-dollar / unknown-run split — present only when cost tracking was\n * wired; the capsule never fabricates a $0 total. */\n cost?: { costUsd: number; costUnknownRuns: number }\n /** Evaluations that threw — infra outcomes, never folded into robustness. */\n evalErrors: number\n /** Set when the consecutive-error circuit breaker stopped the run early. */\n stoppedEarly?: { reason: 'eval-errors'; detail: string }\n}\n\nexport function buildCapsule<S>(input: BuildCapsuleInput<S>): CapsuleData<S> {\n const coverage = buildCoverage(input.cells, input.log, input.threshold)\n const covered = coverage.filter((c) => c.runs > 0)\n const meanRobustness =\n covered.length === 0 ? 0 : covered.reduce((a, c) => a + (c.robustness ?? 0), 0) / covered.length\n // Measured-descriptor bins beyond the bare input cell — observed, never planned.\n const behaviorBinsObserved = input.archive.filter((e) => e.binId !== e.cell.id).length\n\n return {\n target: input.target,\n objective: input.objective,\n coverage,\n findings: [...input.findings].sort((a, b) => b.interest - a.interest),\n archive: [...input.archive].sort((a, b) => b.interest - a.interest),\n stats: {\n totalRuns: input.runsUsed,\n cellsTotal: input.cells.length,\n cellsCovered: covered.length,\n behaviorBinsObserved,\n candidateFindings: input.candidateFindings,\n verifiedFindings: input.findings.length,\n meanRobustness,\n ...(input.cost\n ? { costUsd: input.cost.costUsd, costUnknownRuns: input.cost.costUnknownRuns }\n : {}),\n evalErrors: input.evalErrors,\n ...(input.stoppedEarly ? { stoppedEarly: input.stoppedEarly } : {}),\n },\n }\n}\n\n// ── HTML capsule ──────────────────────────────────────────────────────────────\n\nfunction esc(s: string): string {\n return s.replace(\n /[&<>\"]/g,\n (c) => ({ '&': '&', '<': '<', '>': '>', '\"': '"' })[c] as string,\n )\n}\n\n/** red (0) → amber (.5) → green (1); uncovered cells render gray. */\nfunction robustnessColor(r: number | null): string {\n if (r == null) return '#2a2a2e'\n return `hsl(${Math.round(r * 120)} 70% 42%)`\n}\n\nfunction pct(x: number): string {\n return `${Math.round(x * 100)}%`\n}\n\nfunction deriveAxes(coverage: CoverageCell[]): Array<{ name: string; values: string[] }> {\n const order: string[] = []\n const seen = new Map<string, Set<string>>()\n for (const c of coverage) {\n for (const [k, v] of Object.entries(c.cell.coords)) {\n if (!seen.has(k)) {\n seen.set(k, new Set())\n order.push(k)\n }\n seen.get(k)?.add(v)\n }\n }\n return order.map((name) => ({ name, values: [...(seen.get(name) ?? [])] }))\n}\n\n/** The weakest dimension chip for a cell, e.g. `safety 32%` — shown when scores exist. */\nfunction weakestDim(c: CoverageCell): string {\n const entries = Object.entries(c.dimensions)\n if (entries.length === 0) return ''\n const sorted = entries.sort((a, b) => a[1] - b[1])\n const w = sorted[0]\n if (!w) return ''\n return `<span class=\"dim\">${esc(w[0])} ${pct(w[1])}</span>`\n}\n\nfunction heatmapHtml(coverage: CoverageCell[]): string {\n const axes = deriveAxes(coverage)\n const byId = new Map(coverage.map((c) => [c.cell.id, c]))\n const tile = (c: CoverageCell | undefined, label: string): string => {\n const r = c?.robustness ?? null\n const title = c\n ? `${pct(r ?? 0)} robust · ${c.runs} runs · ${pct(c.findingRate)} flagged`\n : 'not covered'\n return `<div class=\"tile\" style=\"background:${robustnessColor(r)}\" title=\"${esc(title)}\">${label ? `<span class=\"tl\">${esc(label)}</span>` : ''}<span class=\"tv\">${c && r != null ? pct(r) : '—'}</span>${c ? weakestDim(c) : ''}</div>`\n }\n\n const rowAxis = axes[0]\n const colAxis = axes[1]\n if (axes.length === 2 && rowAxis && colAxis) {\n const head = `<tr><th></th>${colAxis.values.map((v) => `<th>${esc(v)}</th>`).join('')}</tr>`\n const rows = rowAxis.values\n .map((rv) => {\n const cells = colAxis.values\n .map((cv) => {\n const id = `${rowAxis.name}=${rv}|${colAxis.name}=${cv}`\n return `<td>${tile(byId.get(id), '')}</td>`\n })\n .join('')\n return `<tr><th class=\"rh\">${esc(rv)}</th>${cells}</tr>`\n })\n .join('')\n return `<div class=\"axis-label\">rows: <b>${esc(rowAxis.name)}</b> · cols: <b>${esc(colAxis.name)}</b></div><table class=\"heat\">${head}${rows}</table>`\n }\n\n const sorted = [...coverage].sort((a, b) => (a.robustness ?? 2) - (b.robustness ?? 2))\n return `<div class=\"grid\">${sorted.map((c) => tile(c, Object.values(c.cell.coords).join(' · '))).join('')}</div>`\n}\n\nfunction findingsHtml<S>(findings: Finding<S>[], limit: number): string {\n if (findings.length === 0)\n return `<p class=\"none\">No verified findings — the target held across every covered cell.</p>`\n return findings\n .slice(0, limit)\n .map((f) => {\n const coords = Object.entries(f.cell.coords)\n .map(([k, v]) => `${k}:${v}`)\n .join(' · ')\n const labels = (f.evaluation.labels ?? [])\n .map((l) => `<span class=\"fclass\">${esc(l)}</span>`)\n .join('')\n const dims = Object.entries(f.evaluation.scores ?? {})\n .sort((a, b) => a[1] - b[1])\n .slice(0, 3)\n .map(([k, v]) => `<span class=\"dim\">${esc(k)} ${pct(v)}</span>`)\n .join('')\n return `<div class=\"fail\"><div class=\"fmeta\"><span class=\"fcell\">${esc(coords)}</span>${labels}<span class=\"fsev\">interest ${pct(f.interest)}</span></div><div class=\"sevbar\"><div class=\"sevfill\" style=\"width:${pct(f.interest)}\"></div></div><div class=\"ftext\">${esc(f.text ?? '(scenario text not captured)')}</div>${dims ? `<div class=\"fdims\">${dims}</div>` : ''}</div>`\n })\n .join('')\n}\n\nexport interface RenderCapsuleOptions {\n /** Max finding exemplars to show. Default 8. */\n maxFindings?: number\n /** ISO timestamp to stamp into the page (keeps the pure capsule clock-free). */\n generatedAt?: string\n}\n\n/** Render a self-contained HTML capsule — heat-map + per-dimension chips + verified findings. */\nexport function renderCapsuleHtml<S>(\n capsule: CapsuleData<S>,\n opts: RenderCapsuleOptions = {},\n): string {\n const s = capsule.stats\n const kpi = (label: string, value: string, accent = '#e6e6e6'): string =>\n `<div class=\"kpi\"><div class=\"kv\" style=\"color:${accent}\">${esc(value)}</div><div class=\"kl\">${esc(label)}</div></div>`\n const lift = capsule.lift\n ? kpi(\n 'hardening lift',\n `${capsule.lift.before.toFixed(2)} → ${capsule.lift.after.toFixed(2)}`,\n '#5ad17a',\n )\n : ''\n // Cost KPI only when tracking was wired — an untracked run never shows $0.\n // Unpriced runs are named in the label (amber): the total is a lower bound.\n const cost =\n s.costUsd !== undefined\n ? kpi(\n s.costUnknownRuns ? `cost · ${s.costUnknownRuns} runs unpriced` : 'cost',\n `$${s.costUsd.toFixed(2)}`,\n s.costUnknownRuns ? '#e5c07b' : '#e6e6e6',\n )\n : ''\n const stamp = opts.generatedAt ?? capsule.generatedAt ?? ''\n\n return `<!doctype html><html><head><meta charset=\"utf-8\"><meta name=\"viewport\" content=\"width=device-width,initial-scale=1\">\n<title>${esc(capsule.objective)} capsule — ${esc(capsule.target)}</title>\n<style>\n:root{color-scheme:dark}\n*{box-sizing:border-box}\nbody{margin:0;background:#0c0c0f;color:#e6e6e6;font:14px/1.5 ui-sans-serif,system-ui,-apple-system,Segoe UI,Roboto,sans-serif}\n.wrap{max-width:980px;margin:0 auto;padding:40px 24px}\nh1{font-size:22px;margin:0 0 2px;letter-spacing:-.01em}\n.sub{color:#8a8a93;font-size:13px;margin-bottom:28px}\n.kpis{display:flex;gap:14px;flex-wrap:wrap;margin-bottom:32px}\n.kpi{background:#16161b;border:1px solid #24242b;border-radius:12px;padding:14px 18px;min-width:120px}\n.kv{font-size:24px;font-weight:650;letter-spacing:-.02em}\n.kl{color:#8a8a93;font-size:12px;margin-top:2px}\nh2{font-size:13px;text-transform:uppercase;letter-spacing:.08em;color:#8a8a93;margin:34px 0 14px}\n.axis-label{color:#8a8a93;font-size:12px;margin-bottom:10px}\ntable.heat{border-collapse:separate;border-spacing:6px}\ntable.heat th{font-weight:500;color:#a8a8b0;font-size:12px;padding:4px 8px;text-align:center}\ntable.heat th.rh{text-align:right}\n.tile{position:relative;width:104px;height:66px;border-radius:10px;display:flex;flex-direction:column;justify-content:center;align-items:center;gap:1px;color:#06120a;font-weight:600}\n.tile .tv{font-size:17px}\n.tile .tl{font-size:10px;font-weight:600;opacity:.9;padding:0 6px;text-align:center}\n.tile .dim{font-size:9px;font-weight:600;opacity:.85;background:rgba(0,0,0,.22);padding:0 6px;border-radius:999px}\n.grid{display:flex;flex-wrap:wrap;gap:8px}\n.grid .tile{width:138px;height:74px}\n.fail{background:#16161b;border:1px solid #24242b;border-left:3px solid #d1495b;border-radius:10px;padding:12px 14px;margin-bottom:10px}\n.fmeta{display:flex;gap:10px;align-items:center;font-size:12px;color:#a8a8b0;margin-bottom:8px}\n.fcell{color:#cfcfd6}\n.fclass{background:#2a1a1d;color:#e58a96;padding:1px 8px;border-radius:999px;font-size:11px}\n.fsev{margin-left:auto;color:#e58a96}\n.sevbar{height:4px;background:#24242b;border-radius:2px;overflow:hidden;margin-bottom:10px}\n.sevfill{height:100%;background:#d1495b}\n.ftext{font-size:13px;color:#d8d8df;white-space:pre-wrap}\n.fdims{display:flex;gap:6px;margin-top:8px}\n.fdims .dim{background:#1d2530;color:#9fc1e8;padding:1px 8px;border-radius:999px;font-size:11px}\n.none{color:#5ad17a}\n.foot{color:#5a5a63;font-size:11px;margin-top:36px}\n</style></head><body><div class=\"wrap\">\n<h1>${esc(capsule.objective)} exploration · ${esc(capsule.target)}</h1>\n<div class=\"sub\">${s.totalRuns} scenarios across ${s.cellsCovered}/${s.cellsTotal} planned cells${s.behaviorBinsObserved > 0 ? ` · ${s.behaviorBinsObserved} measured behavior bins` : ''}${stamp ? ` · ${esc(stamp)}` : ''}</div>\n<div class=\"kpis\">\n${kpi('mean robustness', pct(s.meanRobustness), s.meanRobustness < 0.6 ? '#e58a96' : '#5ad17a')}\n${kpi('verified findings', String(s.verifiedFindings), s.verifiedFindings > 0 ? '#e58a96' : '#5ad17a')}\n${kpi('cells covered', `${s.cellsCovered}/${s.cellsTotal}`)}\n${kpi('scenarios run', String(s.totalRuns))}\n${cost}\n${s.evalErrors > 0 ? kpi('eval errors', String(s.evalErrors), '#e5b566') : ''}\n${lift}\n</div>\n${s.stoppedEarly ? `<div class=\"sub\" style=\"color:#e5b566\">stopped early: ${esc(s.stoppedEarly.detail)} — coverage below reflects the completed portion only</div>` : ''}\n<h2>Coverage map</h2>\n${heatmapHtml(capsule.coverage)}\n<h2>Verified findings${s.candidateFindings > s.verifiedFindings ? ` · ${s.verifiedFindings} of ${s.candidateFindings} candidates passed the validity gates` : ''}</h2>\n${findingsHtml(capsule.findings, opts.maxFindings ?? 8)}\n<div class=\"foot\">Findings are gate-verified: each is a fair, answerable task that reproduces under a meaning-preserving rephrase, minimized to the smallest trigger.</div>\n</div></body></html>`\n}\n","/**\n * Shipped policies for the exploration engine.\n *\n * `Proposer` is a plain function type — an agent running a generator skill IS a\n * proposer (`(ctx) => dispatchToSkill(ctx)`), no wrapper needed. `mutationProposer`\n * builds the deterministic, LLM-free one from mutation operators. Objectives are\n * interfaces because the engine reads `kind` + `threshold` off them.\n */\n\nimport type { AdversarialMutation } from '../rl/adversarial'\nimport type {\n Cell,\n Evaluation,\n Objective,\n ObjectiveContext,\n ProposeContext,\n Proposer,\n} from './types'\n\nconst clamp01 = (x: number): number => Math.max(0, Math.min(1, x))\n\n// ── proposers ─────────────────────────────────────────────────────────────────\n\n/**\n * Perturbation-based search: apply the cell's mutation operators to the current\n * elites + seeds, deduping by id. Elites first — mutating the most interesting\n * scenario found so far is what makes the search deepen across rounds.\n */\nexport function mutationProposer<S>(opts: {\n mutationsFor: (cell: Cell) => AdversarialMutation<S>[]\n scenarioId: (s: S) => string\n}): Proposer<S> {\n return async (ctx: ProposeContext<S>): Promise<S[]> => {\n const mutations = opts.mutationsFor(ctx.cell)\n const parents = [...ctx.elites, ...ctx.seeds]\n const seen = new Set(parents.map(opts.scenarioId))\n const out: S[] = []\n for (const parent of parents) {\n if (out.length >= ctx.count) break\n for (const m of mutations) {\n const children = await m.mutate(parent, ctx.rng)\n for (const child of children) {\n const id = opts.scenarioId(child)\n if (seen.has(id)) continue\n seen.add(id)\n out.push(child)\n if (out.length >= ctx.count) break\n }\n if (out.length >= ctx.count) break\n }\n }\n return out\n }\n}\n\n// ── objectives ────────────────────────────────────────────────────────────────\n\n/** Adversarial: a low headline score is interesting — find where the agent fails. */\nexport function adversarialObjective(threshold = 0.5): Objective {\n return { kind: 'adversarial', threshold, interest: (ev) => clamp01(1 - ev.score) }\n}\n\nfunction hamming(\n a: Record<string, string> | undefined,\n b: Record<string, string> | undefined,\n): number {\n if (!a || !b) return 1\n const keys = new Set([...Object.keys(a), ...Object.keys(b)])\n if (keys.size === 0) return 0\n let diff = 0\n for (const k of keys) if (a[k] !== b[k]) diff++\n return diff / keys.size\n}\n\n/**\n * Novelty: interesting when far from the archive in score AND measured behavior\n * descriptor — quality-diversity's diversity pressure; drives corpus growth\n * rather than re-finding the same hole.\n */\nexport function noveltyObjective(threshold = 0.3): Objective {\n return {\n kind: 'novelty',\n threshold,\n interest: (ev: Evaluation, ctx: ObjectiveContext) => {\n const scoreNovelty =\n ctx.archiveScores.length === 0\n ? 1\n : Math.min(...ctx.archiveScores.map((s) => Math.abs(s - ev.score)))\n const descNovelty =\n ctx.archiveDescriptors.length === 0\n ? 1\n : Math.min(...ctx.archiveDescriptors.map((d) => hamming(d, ev.descriptor)))\n return clamp01(0.5 * scoreNovelty + 0.5 * descNovelty)\n },\n }\n}\n","/**\n * The exploration engine — a stateful session over a behavior space.\n *\n * Each `step()`: allocate budget across INPUT cells (floor first, then variance\n * steering toward the least-certain cells), propose candidates (the proposer\n * reads current elites + findings, so the search deepens generationally),\n * evaluate with bounded concurrency, archive the most interesting scenario per\n * input×measured bin, and admit notable candidates that pass the validity gates.\n * `run()` loops to budget. `coverage()`/`findings()`/`capsule()` read live state —\n * the surface `makeExploreTools` exposes so an agent can drive the session.\n *\n * One evaluation log (`EvalRecord[]`) is the source of truth; allocation\n * observations and coverage are projections of it.\n */\n\nimport { ValidationError } from '../errors'\nimport { varianceBasedCurriculum } from '../rl/active-curriculum'\nimport { buildCapsule } from './capsule'\nimport type { EvalRecord } from './cube'\nimport { enumerateCells } from './cube'\nimport { adversarialObjective } from './policies'\nimport type {\n ArchiveEntry,\n CapsuleData,\n Cell,\n CoverageCell,\n Evaluation,\n ExploreOptions,\n Finding,\n Objective,\n} from './types'\n\nasync function pMap<T>(\n items: T[],\n fn: (item: T) => Promise<void>,\n concurrency: number,\n signal?: AbortSignal,\n): Promise<void> {\n let i = 0\n const workers = Array.from(\n { length: Math.max(1, Math.min(concurrency, items.length)) },\n async () => {\n while (i < items.length) {\n if (signal?.aborted) return\n const item = items[i++]\n if (item === undefined) return\n await fn(item)\n }\n },\n )\n await Promise.all(workers)\n}\n\nexport class BehaviorExplorer<S> {\n private readonly cells: Cell[]\n private readonly cellById: Map<string, Cell>\n private readonly objective: Objective\n private readonly threshold: number\n private readonly floorPerCell: number\n private readonly perRoundBudget: number\n\n /** The single evaluation log — coverage + allocation are projections of it. */\n private readonly log: Array<EvalRecord & { scenarioId: string }> = []\n /** binId (input × measured coords) → the most interesting entry seen. */\n private readonly archiveByBin = new Map<string, ArchiveEntry<S>>()\n private readonly _findings: Finding<S>[] = []\n private runsUsed = 0\n private candidateFindings = 0\n private evalErrors = 0\n private consecutiveEvalErrors = 0\n private stoppedEarly: { reason: 'eval-errors'; detail: string } | undefined\n private rngState: number\n /** Accumulated KNOWN dollars — unknown-cost runs never inflate it. */\n private spentKnownUsd = 0\n private costUnknownRuns = 0\n\n constructor(private readonly opts: ExploreOptions<S>) {\n this.cells = enumerateCells(opts.space)\n if (this.cells.length === 0)\n throw new Error('BehaviorExplorer: space has no cells — every axis needs ≥1 value')\n if (opts.costBudgetUsd !== undefined) {\n if (\n typeof opts.costBudgetUsd !== 'number' ||\n !Number.isFinite(opts.costBudgetUsd) ||\n opts.costBudgetUsd < 0\n ) {\n throw new RangeError(\n `BehaviorExplorer: costBudgetUsd must be a nonnegative finite number, got ${String(opts.costBudgetUsd)}`,\n )\n }\n }\n if (!opts.costOf && (opts.costBudgetUsd !== undefined || opts.ledger || opts.onCost)) {\n throw new ValidationError(\n 'BehaviorExplorer: costBudgetUsd/ledger/onCost require costOf — the explorer ' +\n 'cannot know run cost without it; supply costOf or drop the cost options',\n )\n }\n this.cellById = new Map(this.cells.map((c) => [c.id, c]))\n this.objective = opts.objective ?? adversarialObjective(0.5)\n this.threshold = this.objective.threshold ?? 0.5\n this.floorPerCell = opts.floorPerCell ?? 2\n this.perRoundBudget = Math.max(\n this.cells.length * this.floorPerCell,\n Math.ceil(opts.budget / 4),\n )\n this.rngState = (opts.seed ?? 1) >>> 0\n }\n\n // mulberry32 — deterministic per session.\n private rng = (): number => {\n this.rngState = (this.rngState + 0x6d2b79f5) | 0\n let t = this.rngState\n t = Math.imul(t ^ (t >>> 15), t | 1)\n t ^= t + Math.imul(t ^ (t >>> 7), t | 61)\n return ((t ^ (t >>> 14)) >>> 0) / 4294967296\n }\n\n private binId(cell: Cell, descriptor: Record<string, string> | undefined): string {\n if (!descriptor || Object.keys(descriptor).length === 0) return cell.id\n const measured = Object.keys(descriptor)\n .sort()\n .map((k) => `${k}=${descriptor[k]}`)\n .join('|')\n return `${cell.id}|${measured}`\n }\n\n private allocate(budget: number): Array<{ cellId: string; count: number }> {\n if ((this.opts.allocation ?? 'variance') === 'uniform') {\n const per = Math.max(this.floorPerCell, Math.floor(budget / this.cells.length))\n return this.cells.map((c) => ({ cellId: c.id, count: per }))\n }\n return varianceBasedCurriculum(\n this.log.map((r) => ({\n variantId: r.cell.id,\n scenarioId: r.scenarioId,\n score: r.ev.score,\n pass: r.ev.valid && r.ev.score >= 0.5,\n })),\n this.cells.map((c) => ({ variantId: c.id, scenarioId: '*' })),\n { budget, floorPerCell: this.floorPerCell },\n ).map((a) => ({ cellId: a.variantId, count: a.count }))\n }\n\n private objectiveContext() {\n const entries = [...this.archiveByBin.values()]\n return {\n archiveScores: entries.map((e) => e.evaluation.score),\n archiveDescriptors: entries.map((e) => e.evaluation.descriptor),\n }\n }\n\n /** Mirrors control-runtime: stop once accumulated KNOWN cost ≥ the ceiling. */\n private costExhausted(): boolean {\n return this.opts.costBudgetUsd !== undefined && this.spentKnownUsd >= this.opts.costBudgetUsd\n }\n\n /** Fold one run's cost in: null counts as unknown (never $0); a known cost\n * accrues toward the budget, lands in the ledger, and fires `onCost`. */\n private recordRunCost(scenario: S, cell: Cell, ev: Evaluation): void {\n if (!this.opts.costOf) return\n const cost = this.opts.costOf(scenario, cell, ev)\n if (cost === null) {\n this.costUnknownRuns++\n return\n }\n if (typeof cost.usd !== 'number' || !Number.isFinite(cost.usd) || cost.usd < 0) {\n throw new RangeError(\n `BehaviorExplorer: costOf returned an invalid usd (${String(cost?.usd)}) — ` +\n 'return null when cost is unknown, never a fabricated number',\n )\n }\n this.spentKnownUsd += cost.usd\n this.opts.ledger?.record({\n model: cost.model ?? 'unattributed',\n channel: 'agent',\n usage: { inputTokens: 0, outputTokens: 0 },\n actualCostUsd: cost.usd,\n tags: { target: this.opts.target, cell: cell.id },\n })\n this.opts.onCost?.({ usd: cost.usd, channel: 'agent' })\n }\n\n /** Elites whose INPUT cell matches — what the proposer mutates/deepens from. */\n private elitesFor(cellId: string): S[] {\n const out: S[] = []\n for (const e of this.archiveByBin.values()) if (e.cell.id === cellId) out.push(e.scenario)\n return out\n }\n\n /** One allocate → propose → evaluate → gate → archive round. */\n async step(): Promise<{ runs: number; findings: Finding<S>[] }> {\n const remaining = this.opts.budget - this.runsUsed\n if (remaining <= 0 || this.costExhausted() || this.opts.signal?.aborted)\n return { runs: 0, findings: [] }\n\n const allocations = this.allocate(Math.min(this.perRoundBudget, remaining))\n const newFindings: Finding<S>[] = []\n let runsThisStep = 0\n\n for (const alloc of allocations) {\n if (\n this.runsUsed >= this.opts.budget ||\n this.costExhausted() ||\n this.stoppedEarly !== undefined ||\n this.opts.signal?.aborted\n )\n break\n const cell = this.cellById.get(alloc.cellId)\n if (!cell) continue\n const cap = Math.min(alloc.count, this.opts.budget - this.runsUsed)\n if (cap <= 0) continue\n this.opts.onProgress?.({ type: 'cell-allocated', cell, count: cap })\n\n const seeds = await this.opts.seedsFor(cell)\n const elites = this.elitesFor(cell.id)\n const proposed = await this.opts.proposer({\n cell,\n seeds,\n elites,\n findings: this._findings,\n count: cap,\n rng: this.rng,\n })\n // Cold cells evaluate their seeds first (the coverage floor); warm cells\n // trust the proposer, which already saw the elites.\n const cold = this.log.every((r) => r.cell.id !== cell.id)\n const toEval = [...(cold ? seeds : []), ...proposed].slice(0, cap)\n\n await pMap(\n toEval,\n async (scenario) => {\n if (\n this.runsUsed >= this.opts.budget ||\n this.costExhausted() ||\n this.stoppedEarly !== undefined ||\n this.opts.signal?.aborted\n )\n return\n // evaluate/gates/minimize cross an external boundary (router, backend,\n // judge). A throw there is an infra outcome: record it as a typed\n // eval-error and keep exploring — one 5xx must not kill a campaign.\n // Consecutive failures trip the circuit breaker instead, so a dead\n // backend stops the run rather than burning the remaining budget.\n try {\n const ev = await this.opts.evaluate(scenario, cell)\n this.runsUsed++\n runsThisStep++\n this.consecutiveEvalErrors = 0\n this.recordRunCost(scenario, cell, ev)\n const interest = this.objective.interest(ev, this.objectiveContext())\n this.log.push({ cell, ev, interest, scenarioId: this.opts.scenarioId(scenario) })\n this.opts.onProgress?.({ type: 'evaluated', cell, scenario, evaluation: ev })\n\n const bin = this.binId(cell, ev.descriptor)\n const cur = this.archiveByBin.get(bin)\n if (!cur || interest > cur.interest)\n this.archiveByBin.set(bin, { binId: bin, cell, scenario, evaluation: ev, interest })\n\n if (interest < this.threshold) return\n this.candidateFindings++\n if (this.opts.gates?.isValid && !(await this.opts.gates.isValid(scenario, ev, cell)))\n return\n if (\n this.opts.gates?.isUncontaminated &&\n !(await this.opts.gates.isUncontaminated(scenario, ev, cell))\n )\n return\n\n const minimized = this.opts.minimize\n ? await this.opts.minimize(scenario, this.opts.evaluate, cell)\n : scenario\n const finding: Finding<S> = {\n id: this.opts.scenarioId(scenario),\n cell,\n scenario,\n minimized,\n text: this.opts.scenarioText?.(minimized),\n evaluation: ev,\n interest,\n objective: this.objective.kind,\n }\n this._findings.push(finding)\n newFindings.push(finding)\n this.opts.onProgress?.({ type: 'finding', finding })\n } catch (err) {\n // Internal validation errors (e.g. a fabricated costOf number) are\n // programming mistakes, not backend outcomes — they stay loud.\n if (err instanceof RangeError) throw err\n this.evalErrors++\n this.consecutiveEvalErrors++\n const message = err instanceof Error ? err.message : String(err)\n this.opts.onProgress?.({\n type: 'eval-error',\n cell,\n scenarioId: this.opts.scenarioId(scenario),\n message,\n })\n const limit = this.opts.maxConsecutiveEvalErrors ?? 5\n if (this.consecutiveEvalErrors >= limit) {\n this.stoppedEarly = {\n reason: 'eval-errors',\n detail: `${this.consecutiveEvalErrors} consecutive eval errors (last: ${message})`,\n }\n }\n }\n },\n this.opts.concurrency ?? 1,\n this.opts.signal,\n )\n }\n\n this.opts.onProgress?.({ type: 'round', runsUsed: this.runsUsed, budget: this.opts.budget })\n return { runs: runsThisStep, findings: newFindings }\n }\n\n /** Loop `step()` until the run or dollar budget is spent, the signal aborts,\n * or no progress is made. */\n async run(): Promise<CapsuleData<S>> {\n while (\n this.runsUsed < this.opts.budget &&\n !this.costExhausted() &&\n this.stoppedEarly === undefined &&\n !this.opts.signal?.aborted\n ) {\n const { runs } = await this.step()\n if (runs === 0 && this.stoppedEarly === undefined) break\n }\n return this.capsule()\n }\n\n coverage(): CoverageCell[] {\n return this.capsule().coverage\n }\n\n findings(): Finding<S>[] {\n return [...this._findings].sort((a, b) => b.interest - a.interest)\n }\n\n capsule(): CapsuleData<S> {\n return buildCapsule({\n target: this.opts.target,\n objective: this.objective.kind,\n cells: this.cells,\n log: this.log,\n threshold: this.threshold,\n archive: [...this.archiveByBin.values()],\n findings: this._findings,\n candidateFindings: this.candidateFindings,\n runsUsed: this.runsUsed,\n cost: this.opts.costOf\n ? { costUsd: this.spentKnownUsd, costUnknownRuns: this.costUnknownRuns }\n : undefined,\n evalErrors: this.evalErrors,\n stoppedEarly: this.stoppedEarly,\n })\n }\n}\n","/**\n * `fuzzAgent` — the adversarial batch preset over `BehaviorExplorer`.\n *\n * One call: explore the space to budget with the adversarial objective and\n * return the capsule. For agent-driven, incremental, or multi-objective use,\n * construct a `BehaviorExplorer` and drive it via `makeExploreTools`.\n */\n\nimport { BehaviorExplorer } from './explorer'\nimport { adversarialObjective } from './policies'\nimport type { CapsuleData, ExploreOptions } from './types'\n\nexport type FuzzAgentOptions<S> = Omit<ExploreOptions<S>, 'objective'> & {\n /** Score strictly below this is a candidate failure. Default 0.5. */\n failureThreshold?: number\n}\n\nexport async function fuzzAgent<S>(\n opts: FuzzAgentOptions<S>,\n): Promise<{ capsule: CapsuleData<S> }> {\n const { failureThreshold, ...rest } = opts\n const explorer = new BehaviorExplorer<S>({\n ...rest,\n objective: adversarialObjective(failureThreshold ?? 0.5),\n })\n return { capsule: await explorer.run() }\n}\n","/**\n * Validity gates — what separates a fuzzer from a slop generator.\n *\n * A notable candidate is admitted only when it is fair and reproducible. None of\n * these are on by default: the live wiring opts in, so reported findings carry\n * their proof.\n */\n\nimport type { Cell, Evaluation, Evaluator, ValidityGates } from './types'\n\n/** Combine gate sets; a candidate must pass every gate in every set. */\nexport function composeGates<S>(...sets: Array<ValidityGates<S> | undefined>): ValidityGates<S> {\n const present = sets.filter((s): s is ValidityGates<S> => s != null)\n return {\n isValid: async (scenario, ev, cell) => {\n for (const g of present) {\n if (g.isValid && !(await g.isValid(scenario, ev, cell))) return false\n }\n return true\n },\n isUncontaminated: async (scenario, ev, cell) => {\n for (const g of present) {\n if (g.isUncontaminated && !(await g.isUncontaminated(scenario, ev, cell))) return false\n }\n return true\n },\n }\n}\n\n/**\n * Reproducibility gate. Re-run the target on a meaning-preserving rephrase of the\n * flagged scenario; keep the finding only when the rephrase ALSO scores below the\n * threshold. A finding that flips under a cosmetic rewrite was keyed to surface\n * form, not the task — a false signal we must not report. Costs one extra\n * evaluation per candidate (candidates are rare, so cheap).\n */\nexport function perturbationStabilityGate<S>(opts: {\n evaluate: Evaluator<S>\n /** Produce a semantic-preserving rephrase. Return null to skip (treated as pass). */\n perturb: (scenario: S) => S | null\n /** Score strictly below this still counts as failing. Default 0.5. */\n failureThreshold?: number\n}): ValidityGates<S> {\n const threshold = opts.failureThreshold ?? 0.5\n return {\n isUncontaminated: async (scenario: S, _ev: Evaluation, cell: Cell) => {\n const rephrased = opts.perturb(scenario)\n if (rephrased == null) return true\n const re = await opts.evaluate(rephrased, cell)\n return re.score < threshold\n },\n }\n}\n\n/**\n * Severity-floor gate. Reject borderline candidates whose score sits in a band\n * just under the threshold — judge noise, not a real defect.\n */\nexport function severityFloorGate<S>(opts: {\n failureThreshold?: number\n margin?: number\n}): ValidityGates<S> {\n const threshold = opts.failureThreshold ?? 0.5\n const margin = opts.margin ?? 0.1\n return {\n isValid: (_scenario, ev: Evaluation) => ev.score <= threshold - margin,\n }\n}\n","/**\n * Agent-drivable surface over a live exploration session.\n *\n * Framework-neutral tool defs ({name, description, parameters: JSON Schema,\n * handler}) so the on-demand agent — not a batch script — drives the search:\n * step it, read coverage, inspect findings, render the capsule. Transport\n * encodings (OpenAI function shape, MCP) are one-line mappings the host owns.\n */\n\nimport { renderCapsuleHtml } from './capsule'\nimport type { BehaviorExplorer } from './explorer'\n\nexport interface ExploreToolDef {\n name: string\n description: string\n /** JSON Schema (draft-07+) for the arguments. */\n parameters: Record<string, unknown>\n handler: (args: unknown, ctx?: { signal?: AbortSignal }) => Promise<unknown>\n}\n\nconst NO_ARGS = { type: 'object', properties: {}, additionalProperties: false }\n\nexport function makeExploreTools<S>(explorer: BehaviorExplorer<S>): ExploreToolDef[] {\n return [\n {\n name: 'explore_step',\n description:\n 'Run one exploration round: allocate budget across cells, propose + evaluate scenarios, archive elites, admit gate-verified findings. Returns runs spent and new findings.',\n parameters: NO_ARGS,\n handler: async () => {\n const { runs, findings } = await explorer.step()\n return { runs, newFindings: findings.length, findings }\n },\n },\n {\n name: 'explore_coverage',\n description:\n 'Read the live coverage map: per planned cell — runs, robustness (null = uncovered), finding rate, per-dimension means.',\n parameters: NO_ARGS,\n handler: async () => explorer.coverage(),\n },\n {\n name: 'explore_findings',\n description: 'List gate-verified findings so far, sorted by descending interest.',\n parameters: {\n type: 'object',\n properties: { limit: { type: 'number', description: 'max findings to return' } },\n additionalProperties: false,\n },\n handler: async (args) => {\n const limit = (args as { limit?: number } | undefined)?.limit\n const all = explorer.findings()\n return typeof limit === 'number' ? all.slice(0, limit) : all\n },\n },\n {\n name: 'explore_capsule',\n description:\n 'Build the capsule artifact from the current session state. format \"data\" returns the structured CapsuleData; \"html\" returns the standalone page.',\n parameters: {\n type: 'object',\n properties: {\n format: { type: 'string', enum: ['data', 'html'] },\n generatedAt: { type: 'string', description: 'ISO timestamp to stamp into the page' },\n },\n additionalProperties: false,\n },\n handler: async (args) => {\n const a = (args ?? {}) as { format?: string; generatedAt?: string }\n const capsule = explorer.capsule()\n if (a.format === 'html') return renderCapsuleHtml(capsule, { generatedAt: a.generatedAt })\n return capsule\n },\n },\n ]\n}\n"],"mappings":";;;;;;;;;AAqBO,SAAS,eAAe,OAA8B;AAC3D,MAAI,MAAM,KAAK,WAAW,EAAG,QAAO,CAAC;AACrC,MAAI,WAA0C,CAAC,CAAC,CAAC;AACjD,aAAW,QAAQ,MAAM,MAAM;AAC7B,UAAM,OAAsC,CAAC;AAC7C,eAAW,WAAW,UAAU;AAC9B,iBAAW,SAAS,KAAK,OAAQ,MAAK,KAAK,EAAE,GAAG,SAAS,CAAC,KAAK,IAAI,GAAG,MAAM,CAAC;AAAA,IAC/E;AACA,eAAW;AAAA,EACb;AACA,SAAO,SAAS,IAAI,CAAC,YAAY,EAAE,IAAI,OAAO,OAAO,MAAM,GAAG,OAAO,EAAE;AACzE;AAGO,SAAS,OAAO,OAAsB,QAAwC;AACnF,SAAO,MAAM,KAAK,IAAI,CAAC,MAAM,GAAG,EAAE,IAAI,IAAI,OAAO,EAAE,IAAI,CAAC,EAAE,EAAE,KAAK,GAAG;AACtE;AAEA,IAAM,OAAO,CAAC,OACZ,GAAG,WAAW,IAAI,IAAI,GAAG,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,GAAG;AAMpD,SAAS,cAAc,OAAe,KAAmB,WAAmC;AACjG,QAAM,SAAS,oBAAI,IAA0B;AAC7C,aAAW,KAAK,KAAK;AACnB,UAAM,MAAM,OAAO,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC;AACtC,QAAI,KAAK,CAAC;AACV,WAAO,IAAI,EAAE,KAAK,IAAI,GAAG;AAAA,EAC3B;AACA,SAAO,MAAM,IAAI,CAAC,SAAS;AACzB,UAAM,OAAO,OAAO,IAAI,KAAK,EAAE,KAAK,CAAC;AACrC,UAAM,OAAO,KAAK;AAClB,QAAI,SAAS,EAAG,QAAO,EAAE,MAAM,MAAM,GAAG,YAAY,MAAM,aAAa,GAAG,YAAY,CAAC,EAAE;AACzF,UAAM,aAAa,KAAK,KAAK,IAAI,CAAC,MAAM,EAAE,GAAG,KAAK,CAAC;AACnD,UAAM,cAAc,KAAK,OAAO,CAAC,MAAM,EAAE,YAAY,SAAS,EAAE,SAAS;AACzE,UAAM,OAAiC,CAAC;AACxC,eAAW,KAAK,MAAM;AACpB,iBAAW,CAAC,GAAG,CAAC,KAAK,OAAO,QAAQ,EAAE,GAAG,UAAU,CAAC,CAAC,GAAG;AACtD;AAAC,SAAC,KAAK,CAAC,MAAM,CAAC,GAAG,KAAK,CAAC;AAAA,MAC1B;AAAA,IACF;AACA,UAAM,aAAqC,CAAC;AAC5C,eAAW,CAAC,GAAG,EAAE,KAAK,OAAO,QAAQ,IAAI,EAAG,YAAW,CAAC,IAAI,KAAK,EAAE;AACnE,WAAO,EAAE,MAAM,MAAM,YAAY,aAAa,WAAW;AAAA,EAC3D,CAAC;AACH;;;ACnCO,SAAS,aAAgB,OAA6C;AAC3E,QAAM,WAAW,cAAc,MAAM,OAAO,MAAM,KAAK,MAAM,SAAS;AACtE,QAAM,UAAU,SAAS,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC;AACjD,QAAM,iBACJ,QAAQ,WAAW,IAAI,IAAI,QAAQ,OAAO,CAAC,GAAG,MAAM,KAAK,EAAE,cAAc,IAAI,CAAC,IAAI,QAAQ;AAE5F,QAAM,uBAAuB,MAAM,QAAQ,OAAO,CAAC,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,EAAE;AAEhF,SAAO;AAAA,IACL,QAAQ,MAAM;AAAA,IACd,WAAW,MAAM;AAAA,IACjB;AAAA,IACA,UAAU,CAAC,GAAG,MAAM,QAAQ,EAAE,KAAK,CAAC,GAAG,MAAM,EAAE,WAAW,EAAE,QAAQ;AAAA,IACpE,SAAS,CAAC,GAAG,MAAM,OAAO,EAAE,KAAK,CAAC,GAAG,MAAM,EAAE,WAAW,EAAE,QAAQ;AAAA,IAClE,OAAO;AAAA,MACL,WAAW,MAAM;AAAA,MACjB,YAAY,MAAM,MAAM;AAAA,MACxB,cAAc,QAAQ;AAAA,MACtB;AAAA,MACA,mBAAmB,MAAM;AAAA,MACzB,kBAAkB,MAAM,SAAS;AAAA,MACjC;AAAA,MACA,GAAI,MAAM,OACN,EAAE,SAAS,MAAM,KAAK,SAAS,iBAAiB,MAAM,KAAK,gBAAgB,IAC3E,CAAC;AAAA,MACL,YAAY,MAAM;AAAA,MAClB,GAAI,MAAM,eAAe,EAAE,cAAc,MAAM,aAAa,IAAI,CAAC;AAAA,IACnE;AAAA,EACF;AACF;AAIA,SAAS,IAAI,GAAmB;AAC9B,SAAO,EAAE;AAAA,IACP;AAAA,IACA,CAAC,OAAO,EAAE,KAAK,SAAS,KAAK,QAAQ,KAAK,QAAQ,KAAK,SAAS,GAAG,CAAC;AAAA,EACtE;AACF;AAGA,SAAS,gBAAgB,GAA0B;AACjD,MAAI,KAAK,KAAM,QAAO;AACtB,SAAO,OAAO,KAAK,MAAM,IAAI,GAAG,CAAC;AACnC;AAEA,SAAS,IAAI,GAAmB;AAC9B,SAAO,GAAG,KAAK,MAAM,IAAI,GAAG,CAAC;AAC/B;AAEA,SAAS,WAAW,UAAqE;AACvF,QAAM,QAAkB,CAAC;AACzB,QAAM,OAAO,oBAAI,IAAyB;AAC1C,aAAW,KAAK,UAAU;AACxB,eAAW,CAAC,GAAG,CAAC,KAAK,OAAO,QAAQ,EAAE,KAAK,MAAM,GAAG;AAClD,UAAI,CAAC,KAAK,IAAI,CAAC,GAAG;AAChB,aAAK,IAAI,GAAG,oBAAI,IAAI,CAAC;AACrB,cAAM,KAAK,CAAC;AAAA,MACd;AACA,WAAK,IAAI,CAAC,GAAG,IAAI,CAAC;AAAA,IACpB;AAAA,EACF;AACA,SAAO,MAAM,IAAI,CAAC,UAAU,EAAE,MAAM,QAAQ,CAAC,GAAI,KAAK,IAAI,IAAI,KAAK,CAAC,CAAE,EAAE,EAAE;AAC5E;AAGA,SAAS,WAAW,GAAyB;AAC3C,QAAM,UAAU,OAAO,QAAQ,EAAE,UAAU;AAC3C,MAAI,QAAQ,WAAW,EAAG,QAAO;AACjC,QAAM,SAAS,QAAQ,KAAK,CAAC,GAAG,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC;AACjD,QAAM,IAAI,OAAO,CAAC;AAClB,MAAI,CAAC,EAAG,QAAO;AACf,SAAO,qBAAqB,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,CAAC;AACpD;AAEA,SAAS,YAAY,UAAkC;AACrD,QAAM,OAAO,WAAW,QAAQ;AAChC,QAAM,OAAO,IAAI,IAAI,SAAS,IAAI,CAAC,MAAM,CAAC,EAAE,KAAK,IAAI,CAAC,CAAC,CAAC;AACxD,QAAM,OAAO,CAAC,GAA6B,UAA0B;AACnE,UAAM,IAAI,GAAG,cAAc;AAC3B,UAAM,QAAQ,IACV,GAAG,IAAI,KAAK,CAAC,CAAC,gBAAa,EAAE,IAAI,cAAW,IAAI,EAAE,WAAW,CAAC,aAC9D;AACJ,WAAO,uCAAuC,gBAAgB,CAAC,CAAC,YAAY,IAAI,KAAK,CAAC,KAAK,QAAQ,oBAAoB,IAAI,KAAK,CAAC,YAAY,EAAE,oBAAoB,KAAK,KAAK,OAAO,IAAI,CAAC,IAAI,QAAG,UAAU,IAAI,WAAW,CAAC,IAAI,EAAE;AAAA,EAClO;AAEA,QAAM,UAAU,KAAK,CAAC;AACtB,QAAM,UAAU,KAAK,CAAC;AACtB,MAAI,KAAK,WAAW,KAAK,WAAW,SAAS;AAC3C,UAAM,OAAO,gBAAgB,QAAQ,OAAO,IAAI,CAAC,MAAM,OAAO,IAAI,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,CAAC;AACrF,UAAM,OAAO,QAAQ,OAClB,IAAI,CAAC,OAAO;AACX,YAAM,QAAQ,QAAQ,OACnB,IAAI,CAAC,OAAO;AACX,cAAM,KAAK,GAAG,QAAQ,IAAI,IAAI,EAAE,IAAI,QAAQ,IAAI,IAAI,EAAE;AACtD,eAAO,OAAO,KAAK,KAAK,IAAI,EAAE,GAAG,EAAE,CAAC;AAAA,MACtC,CAAC,EACA,KAAK,EAAE;AACV,aAAO,sBAAsB,IAAI,EAAE,CAAC,QAAQ,KAAK;AAAA,IACnD,CAAC,EACA,KAAK,EAAE;AACV,WAAO,oCAAoC,IAAI,QAAQ,IAAI,CAAC,sBAAmB,IAAI,QAAQ,IAAI,CAAC,iCAAiC,IAAI,GAAG,IAAI;AAAA,EAC9I;AAEA,QAAM,SAAS,CAAC,GAAG,QAAQ,EAAE,KAAK,CAAC,GAAG,OAAO,EAAE,cAAc,MAAM,EAAE,cAAc,EAAE;AACrF,SAAO,qBAAqB,OAAO,IAAI,CAAC,MAAM,KAAK,GAAG,OAAO,OAAO,EAAE,KAAK,MAAM,EAAE,KAAK,QAAK,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC;AAC3G;AAEA,SAAS,aAAgB,UAAwB,OAAuB;AACtE,MAAI,SAAS,WAAW;AACtB,WAAO;AACT,SAAO,SACJ,MAAM,GAAG,KAAK,EACd,IAAI,CAAC,MAAM;AACV,UAAM,SAAS,OAAO,QAAQ,EAAE,KAAK,MAAM,EACxC,IAAI,CAAC,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,EAAE,EAC3B,KAAK,QAAK;AACb,UAAM,UAAU,EAAE,WAAW,UAAU,CAAC,GACrC,IAAI,CAAC,MAAM,wBAAwB,IAAI,CAAC,CAAC,SAAS,EAClD,KAAK,EAAE;AACV,UAAM,OAAO,OAAO,QAAQ,EAAE,WAAW,UAAU,CAAC,CAAC,EAClD,KAAK,CAAC,GAAG,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC,EAC1B,MAAM,GAAG,CAAC,EACV,IAAI,CAAC,CAAC,GAAG,CAAC,MAAM,qBAAqB,IAAI,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,SAAS,EAC9D,KAAK,EAAE;AACV,WAAO,4DAA4D,IAAI,MAAM,CAAC,UAAU,MAAM,+BAA+B,IAAI,EAAE,QAAQ,CAAC,sEAAsE,IAAI,EAAE,QAAQ,CAAC,oCAAoC,IAAI,EAAE,QAAQ,8BAA8B,CAAC,SAAS,OAAO,sBAAsB,IAAI,WAAW,EAAE;AAAA,EAC3W,CAAC,EACA,KAAK,EAAE;AACZ;AAUO,SAAS,kBACd,SACA,OAA6B,CAAC,GACtB;AACR,QAAM,IAAI,QAAQ;AAClB,QAAM,MAAM,CAAC,OAAe,OAAe,SAAS,cAClD,iDAAiD,MAAM,KAAK,IAAI,KAAK,CAAC,yBAAyB,IAAI,KAAK,CAAC;AAC3G,QAAM,OAAO,QAAQ,OACjB;AAAA,IACE;AAAA,IACA,GAAG,QAAQ,KAAK,OAAO,QAAQ,CAAC,CAAC,WAAM,QAAQ,KAAK,MAAM,QAAQ,CAAC,CAAC;AAAA,IACpE;AAAA,EACF,IACA;AAGJ,QAAM,OACJ,EAAE,YAAY,SACV;AAAA,IACE,EAAE,kBAAkB,aAAU,EAAE,eAAe,mBAAmB;AAAA,IAClE,IAAI,EAAE,QAAQ,QAAQ,CAAC,CAAC;AAAA,IACxB,EAAE,kBAAkB,YAAY;AAAA,EAClC,IACA;AACN,QAAM,QAAQ,KAAK,eAAe,QAAQ,eAAe;AAEzD,SAAO;AAAA,SACA,IAAI,QAAQ,SAAS,CAAC,mBAAc,IAAI,QAAQ,MAAM,CAAC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,MAoC1D,IAAI,QAAQ,SAAS,CAAC,qBAAkB,IAAI,QAAQ,MAAM,CAAC;AAAA,mBAC9C,EAAE,SAAS,qBAAqB,EAAE,YAAY,IAAI,EAAE,UAAU,iBAAiB,EAAE,uBAAuB,IAAI,SAAM,EAAE,oBAAoB,4BAA4B,EAAE,GAAG,QAAQ,SAAM,IAAI,KAAK,CAAC,KAAK,EAAE;AAAA;AAAA,EAEzN,IAAI,mBAAmB,IAAI,EAAE,cAAc,GAAG,EAAE,iBAAiB,MAAM,YAAY,SAAS,CAAC;AAAA,EAC7F,IAAI,qBAAqB,OAAO,EAAE,gBAAgB,GAAG,EAAE,mBAAmB,IAAI,YAAY,SAAS,CAAC;AAAA,EACpG,IAAI,iBAAiB,GAAG,EAAE,YAAY,IAAI,EAAE,UAAU,EAAE,CAAC;AAAA,EACzD,IAAI,iBAAiB,OAAO,EAAE,SAAS,CAAC,CAAC;AAAA,EACzC,IAAI;AAAA,EACJ,EAAE,aAAa,IAAI,IAAI,eAAe,OAAO,EAAE,UAAU,GAAG,SAAS,IAAI,EAAE;AAAA,EAC3E,IAAI;AAAA;AAAA,EAEJ,EAAE,eAAe,yDAAyD,IAAI,EAAE,aAAa,MAAM,CAAC,qEAAgE,EAAE;AAAA;AAAA,EAEtK,YAAY,QAAQ,QAAQ,CAAC;AAAA,uBACR,EAAE,oBAAoB,EAAE,mBAAmB,SAAM,EAAE,gBAAgB,OAAO,EAAE,iBAAiB,0CAA0C,EAAE;AAAA,EAC9J,aAAa,QAAQ,UAAU,KAAK,eAAe,CAAC,CAAC;AAAA;AAAA;AAGvD;;;AC1OA,IAAM,UAAU,CAAC,MAAsB,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,CAAC,CAAC;AAS1D,SAAS,iBAAoB,MAGpB;AACd,SAAO,OAAO,QAAyC;AACrD,UAAM,YAAY,KAAK,aAAa,IAAI,IAAI;AAC5C,UAAM,UAAU,CAAC,GAAG,IAAI,QAAQ,GAAG,IAAI,KAAK;AAC5C,UAAM,OAAO,IAAI,IAAI,QAAQ,IAAI,KAAK,UAAU,CAAC;AACjD,UAAM,MAAW,CAAC;AAClB,eAAW,UAAU,SAAS;AAC5B,UAAI,IAAI,UAAU,IAAI,MAAO;AAC7B,iBAAW,KAAK,WAAW;AACzB,cAAM,WAAW,MAAM,EAAE,OAAO,QAAQ,IAAI,GAAG;AAC/C,mBAAW,SAAS,UAAU;AAC5B,gBAAM,KAAK,KAAK,WAAW,KAAK;AAChC,cAAI,KAAK,IAAI,EAAE,EAAG;AAClB,eAAK,IAAI,EAAE;AACX,cAAI,KAAK,KAAK;AACd,cAAI,IAAI,UAAU,IAAI,MAAO;AAAA,QAC/B;AACA,YAAI,IAAI,UAAU,IAAI,MAAO;AAAA,MAC/B;AAAA,IACF;AACA,WAAO;AAAA,EACT;AACF;AAKO,SAAS,qBAAqB,YAAY,KAAgB;AAC/D,SAAO,EAAE,MAAM,eAAe,WAAW,UAAU,CAAC,OAAO,QAAQ,IAAI,GAAG,KAAK,EAAE;AACnF;AAEA,SAAS,QACP,GACA,GACQ;AACR,MAAI,CAAC,KAAK,CAAC,EAAG,QAAO;AACrB,QAAM,OAAO,oBAAI,IAAI,CAAC,GAAG,OAAO,KAAK,CAAC,GAAG,GAAG,OAAO,KAAK,CAAC,CAAC,CAAC;AAC3D,MAAI,KAAK,SAAS,EAAG,QAAO;AAC5B,MAAI,OAAO;AACX,aAAW,KAAK,KAAM,KAAI,EAAE,CAAC,MAAM,EAAE,CAAC,EAAG;AACzC,SAAO,OAAO,KAAK;AACrB;AAOO,SAAS,iBAAiB,YAAY,KAAgB;AAC3D,SAAO;AAAA,IACL,MAAM;AAAA,IACN;AAAA,IACA,UAAU,CAAC,IAAgB,QAA0B;AACnD,YAAM,eACJ,IAAI,cAAc,WAAW,IACzB,IACA,KAAK,IAAI,GAAG,IAAI,cAAc,IAAI,CAAC,MAAM,KAAK,IAAI,IAAI,GAAG,KAAK,CAAC,CAAC;AACtE,YAAM,cACJ,IAAI,mBAAmB,WAAW,IAC9B,IACA,KAAK,IAAI,GAAG,IAAI,mBAAmB,IAAI,CAAC,MAAM,QAAQ,GAAG,GAAG,UAAU,CAAC,CAAC;AAC9E,aAAO,QAAQ,MAAM,eAAe,MAAM,WAAW;AAAA,IACvD;AAAA,EACF;AACF;;;AC/DA,eAAe,KACb,OACA,IACA,aACA,QACe;AACf,MAAI,IAAI;AACR,QAAM,UAAU,MAAM;AAAA,IACpB,EAAE,QAAQ,KAAK,IAAI,GAAG,KAAK,IAAI,aAAa,MAAM,MAAM,CAAC,EAAE;AAAA,IAC3D,YAAY;AACV,aAAO,IAAI,MAAM,QAAQ;AACvB,YAAI,QAAQ,QAAS;AACrB,cAAM,OAAO,MAAM,GAAG;AACtB,YAAI,SAAS,OAAW;AACxB,cAAM,GAAG,IAAI;AAAA,MACf;AAAA,IACF;AAAA,EACF;AACA,QAAM,QAAQ,IAAI,OAAO;AAC3B;AAEO,IAAM,mBAAN,MAA0B;AAAA,EAuB/B,YAA6B,MAAyB;AAAzB;AAC3B,SAAK,QAAQ,eAAe,KAAK,KAAK;AACtC,QAAI,KAAK,MAAM,WAAW;AACxB,YAAM,IAAI,MAAM,4EAAkE;AACpF,QAAI,KAAK,kBAAkB,QAAW;AACpC,UACE,OAAO,KAAK,kBAAkB,YAC9B,CAAC,OAAO,SAAS,KAAK,aAAa,KACnC,KAAK,gBAAgB,GACrB;AACA,cAAM,IAAI;AAAA,UACR,4EAA4E,OAAO,KAAK,aAAa,CAAC;AAAA,QACxG;AAAA,MACF;AAAA,IACF;AACA,QAAI,CAAC,KAAK,WAAW,KAAK,kBAAkB,UAAa,KAAK,UAAU,KAAK,SAAS;AACpF,YAAM,IAAI;AAAA,QACR;AAAA,MAEF;AAAA,IACF;AACA,SAAK,WAAW,IAAI,IAAI,KAAK,MAAM,IAAI,CAAC,MAAM,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC;AACxD,SAAK,YAAY,KAAK,aAAa,qBAAqB,GAAG;AAC3D,SAAK,YAAY,KAAK,UAAU,aAAa;AAC7C,SAAK,eAAe,KAAK,gBAAgB;AACzC,SAAK,iBAAiB,KAAK;AAAA,MACzB,KAAK,MAAM,SAAS,KAAK;AAAA,MACzB,KAAK,KAAK,KAAK,SAAS,CAAC;AAAA,IAC3B;AACA,SAAK,YAAY,KAAK,QAAQ,OAAO;AAAA,EACvC;AAAA,EA9B6B;AAAA,EAtBZ;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA;AAAA,EAGA,MAAkD,CAAC;AAAA;AAAA,EAEnD,eAAe,oBAAI,IAA6B;AAAA,EAChD,YAA0B,CAAC;AAAA,EACpC,WAAW;AAAA,EACX,oBAAoB;AAAA,EACpB,aAAa;AAAA,EACb,wBAAwB;AAAA,EACxB;AAAA,EACA;AAAA;AAAA,EAEA,gBAAgB;AAAA,EAChB,kBAAkB;AAAA;AAAA,EAmClB,MAAM,MAAc;AAC1B,SAAK,WAAY,KAAK,WAAW,aAAc;AAC/C,QAAI,IAAI,KAAK;AACb,QAAI,KAAK,KAAK,IAAK,MAAM,IAAK,IAAI,CAAC;AACnC,SAAK,IAAI,KAAK,KAAK,IAAK,MAAM,GAAI,IAAI,EAAE;AACxC,aAAS,IAAK,MAAM,QAAS,KAAK;AAAA,EACpC;AAAA,EAEQ,MAAM,MAAY,YAAwD;AAChF,QAAI,CAAC,cAAc,OAAO,KAAK,UAAU,EAAE,WAAW,EAAG,QAAO,KAAK;AACrE,UAAM,WAAW,OAAO,KAAK,UAAU,EACpC,KAAK,EACL,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,WAAW,CAAC,CAAC,EAAE,EAClC,KAAK,GAAG;AACX,WAAO,GAAG,KAAK,EAAE,IAAI,QAAQ;AAAA,EAC/B;AAAA,EAEQ,SAAS,QAA0D;AACzE,SAAK,KAAK,KAAK,cAAc,gBAAgB,WAAW;AACtD,YAAM,MAAM,KAAK,IAAI,KAAK,cAAc,KAAK,MAAM,SAAS,KAAK,MAAM,MAAM,CAAC;AAC9E,aAAO,KAAK,MAAM,IAAI,CAAC,OAAO,EAAE,QAAQ,EAAE,IAAI,OAAO,IAAI,EAAE;AAAA,IAC7D;AACA,WAAO;AAAA,MACL,KAAK,IAAI,IAAI,CAAC,OAAO;AAAA,QACnB,WAAW,EAAE,KAAK;AAAA,QAClB,YAAY,EAAE;AAAA,QACd,OAAO,EAAE,GAAG;AAAA,QACZ,MAAM,EAAE,GAAG,SAAS,EAAE,GAAG,SAAS;AAAA,MACpC,EAAE;AAAA,MACF,KAAK,MAAM,IAAI,CAAC,OAAO,EAAE,WAAW,EAAE,IAAI,YAAY,IAAI,EAAE;AAAA,MAC5D,EAAE,QAAQ,cAAc,KAAK,aAAa;AAAA,IAC5C,EAAE,IAAI,CAAC,OAAO,EAAE,QAAQ,EAAE,WAAW,OAAO,EAAE,MAAM,EAAE;AAAA,EACxD;AAAA,EAEQ,mBAAmB;AACzB,UAAM,UAAU,CAAC,GAAG,KAAK,aAAa,OAAO,CAAC;AAC9C,WAAO;AAAA,MACL,eAAe,QAAQ,IAAI,CAAC,MAAM,EAAE,WAAW,KAAK;AAAA,MACpD,oBAAoB,QAAQ,IAAI,CAAC,MAAM,EAAE,WAAW,UAAU;AAAA,IAChE;AAAA,EACF;AAAA;AAAA,EAGQ,gBAAyB;AAC/B,WAAO,KAAK,KAAK,kBAAkB,UAAa,KAAK,iBAAiB,KAAK,KAAK;AAAA,EAClF;AAAA;AAAA;AAAA,EAIQ,cAAc,UAAa,MAAY,IAAsB;AACnE,QAAI,CAAC,KAAK,KAAK,OAAQ;AACvB,UAAM,OAAO,KAAK,KAAK,OAAO,UAAU,MAAM,EAAE;AAChD,QAAI,SAAS,MAAM;AACjB,WAAK;AACL;AAAA,IACF;AACA,QAAI,OAAO,KAAK,QAAQ,YAAY,CAAC,OAAO,SAAS,KAAK,GAAG,KAAK,KAAK,MAAM,GAAG;AAC9E,YAAM,IAAI;AAAA,QACR,qDAAqD,OAAO,MAAM,GAAG,CAAC;AAAA,MAExE;AAAA,IACF;AACA,SAAK,iBAAiB,KAAK;AAC3B,SAAK,KAAK,QAAQ,OAAO;AAAA,MACvB,OAAO,KAAK,SAAS;AAAA,MACrB,SAAS;AAAA,MACT,OAAO,EAAE,aAAa,GAAG,cAAc,EAAE;AAAA,MACzC,eAAe,KAAK;AAAA,MACpB,MAAM,EAAE,QAAQ,KAAK,KAAK,QAAQ,MAAM,KAAK,GAAG;AAAA,IAClD,CAAC;AACD,SAAK,KAAK,SAAS,EAAE,KAAK,KAAK,KAAK,SAAS,QAAQ,CAAC;AAAA,EACxD;AAAA;AAAA,EAGQ,UAAUA,SAAqB;AACrC,UAAM,MAAW,CAAC;AAClB,eAAW,KAAK,KAAK,aAAa,OAAO,EAAG,KAAI,EAAE,KAAK,OAAOA,QAAQ,KAAI,KAAK,EAAE,QAAQ;AACzF,WAAO;AAAA,EACT;AAAA;AAAA,EAGA,MAAM,OAA0D;AAC9D,UAAM,YAAY,KAAK,KAAK,SAAS,KAAK;AAC1C,QAAI,aAAa,KAAK,KAAK,cAAc,KAAK,KAAK,KAAK,QAAQ;AAC9D,aAAO,EAAE,MAAM,GAAG,UAAU,CAAC,EAAE;AAEjC,UAAM,cAAc,KAAK,SAAS,KAAK,IAAI,KAAK,gBAAgB,SAAS,CAAC;AAC1E,UAAM,cAA4B,CAAC;AACnC,QAAI,eAAe;AAEnB,eAAW,SAAS,aAAa;AAC/B,UACE,KAAK,YAAY,KAAK,KAAK,UAC3B,KAAK,cAAc,KACnB,KAAK,iBAAiB,UACtB,KAAK,KAAK,QAAQ;AAElB;AACF,YAAM,OAAO,KAAK,SAAS,IAAI,MAAM,MAAM;AAC3C,UAAI,CAAC,KAAM;AACX,YAAM,MAAM,KAAK,IAAI,MAAM,OAAO,KAAK,KAAK,SAAS,KAAK,QAAQ;AAClE,UAAI,OAAO,EAAG;AACd,WAAK,KAAK,aAAa,EAAE,MAAM,kBAAkB,MAAM,OAAO,IAAI,CAAC;AAEnE,YAAM,QAAQ,MAAM,KAAK,KAAK,SAAS,IAAI;AAC3C,YAAM,SAAS,KAAK,UAAU,KAAK,EAAE;AACrC,YAAM,WAAW,MAAM,KAAK,KAAK,SAAS;AAAA,QACxC;AAAA,QACA;AAAA,QACA;AAAA,QACA,UAAU,KAAK;AAAA,QACf,OAAO;AAAA,QACP,KAAK,KAAK;AAAA,MACZ,CAAC;AAGD,YAAM,OAAO,KAAK,IAAI,MAAM,CAAC,MAAM,EAAE,KAAK,OAAO,KAAK,EAAE;AACxD,YAAM,SAAS,CAAC,GAAI,OAAO,QAAQ,CAAC,GAAI,GAAG,QAAQ,EAAE,MAAM,GAAG,GAAG;AAEjE,YAAM;AAAA,QACJ;AAAA,QACA,OAAO,aAAa;AAClB,cACE,KAAK,YAAY,KAAK,KAAK,UAC3B,KAAK,cAAc,KACnB,KAAK,iBAAiB,UACtB,KAAK,KAAK,QAAQ;AAElB;AAMF,cAAI;AACF,kBAAM,KAAK,MAAM,KAAK,KAAK,SAAS,UAAU,IAAI;AAClD,iBAAK;AACL;AACA,iBAAK,wBAAwB;AAC7B,iBAAK,cAAc,UAAU,MAAM,EAAE;AACrC,kBAAM,WAAW,KAAK,UAAU,SAAS,IAAI,KAAK,iBAAiB,CAAC;AACpE,iBAAK,IAAI,KAAK,EAAE,MAAM,IAAI,UAAU,YAAY,KAAK,KAAK,WAAW,QAAQ,EAAE,CAAC;AAChF,iBAAK,KAAK,aAAa,EAAE,MAAM,aAAa,MAAM,UAAU,YAAY,GAAG,CAAC;AAE5E,kBAAM,MAAM,KAAK,MAAM,MAAM,GAAG,UAAU;AAC1C,kBAAM,MAAM,KAAK,aAAa,IAAI,GAAG;AACrC,gBAAI,CAAC,OAAO,WAAW,IAAI;AACzB,mBAAK,aAAa,IAAI,KAAK,EAAE,OAAO,KAAK,MAAM,UAAU,YAAY,IAAI,SAAS,CAAC;AAErF,gBAAI,WAAW,KAAK,UAAW;AAC/B,iBAAK;AACL,gBAAI,KAAK,KAAK,OAAO,WAAW,CAAE,MAAM,KAAK,KAAK,MAAM,QAAQ,UAAU,IAAI,IAAI;AAChF;AACF,gBACE,KAAK,KAAK,OAAO,oBACjB,CAAE,MAAM,KAAK,KAAK,MAAM,iBAAiB,UAAU,IAAI,IAAI;AAE3D;AAEF,kBAAM,YAAY,KAAK,KAAK,WACxB,MAAM,KAAK,KAAK,SAAS,UAAU,KAAK,KAAK,UAAU,IAAI,IAC3D;AACJ,kBAAM,UAAsB;AAAA,cAC1B,IAAI,KAAK,KAAK,WAAW,QAAQ;AAAA,cACjC;AAAA,cACA;AAAA,cACA;AAAA,cACA,MAAM,KAAK,KAAK,eAAe,SAAS;AAAA,cACxC,YAAY;AAAA,cACZ;AAAA,cACA,WAAW,KAAK,UAAU;AAAA,YAC5B;AACA,iBAAK,UAAU,KAAK,OAAO;AAC3B,wBAAY,KAAK,OAAO;AACxB,iBAAK,KAAK,aAAa,EAAE,MAAM,WAAW,QAAQ,CAAC;AAAA,UACrD,SAAS,KAAK;AAGZ,gBAAI,eAAe,WAAY,OAAM;AACrC,iBAAK;AACL,iBAAK;AACL,kBAAM,UAAU,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAC/D,iBAAK,KAAK,aAAa;AAAA,cACrB,MAAM;AAAA,cACN;AAAA,cACA,YAAY,KAAK,KAAK,WAAW,QAAQ;AAAA,cACzC;AAAA,YACF,CAAC;AACD,kBAAM,QAAQ,KAAK,KAAK,4BAA4B;AACpD,gBAAI,KAAK,yBAAyB,OAAO;AACvC,mBAAK,eAAe;AAAA,gBAClB,QAAQ;AAAA,gBACR,QAAQ,GAAG,KAAK,qBAAqB,mCAAmC,OAAO;AAAA,cACjF;AAAA,YACF;AAAA,UACF;AAAA,QACF;AAAA,QACA,KAAK,KAAK,eAAe;AAAA,QACzB,KAAK,KAAK;AAAA,MACZ;AAAA,IACF;AAEA,SAAK,KAAK,aAAa,EAAE,MAAM,SAAS,UAAU,KAAK,UAAU,QAAQ,KAAK,KAAK,OAAO,CAAC;AAC3F,WAAO,EAAE,MAAM,cAAc,UAAU,YAAY;AAAA,EACrD;AAAA;AAAA;AAAA,EAIA,MAAM,MAA+B;AACnC,WACE,KAAK,WAAW,KAAK,KAAK,UAC1B,CAAC,KAAK,cAAc,KACpB,KAAK,iBAAiB,UACtB,CAAC,KAAK,KAAK,QAAQ,SACnB;AACA,YAAM,EAAE,KAAK,IAAI,MAAM,KAAK,KAAK;AACjC,UAAI,SAAS,KAAK,KAAK,iBAAiB,OAAW;AAAA,IACrD;AACA,WAAO,KAAK,QAAQ;AAAA,EACtB;AAAA,EAEA,WAA2B;AACzB,WAAO,KAAK,QAAQ,EAAE;AAAA,EACxB;AAAA,EAEA,WAAyB;AACvB,WAAO,CAAC,GAAG,KAAK,SAAS,EAAE,KAAK,CAAC,GAAG,MAAM,EAAE,WAAW,EAAE,QAAQ;AAAA,EACnE;AAAA,EAEA,UAA0B;AACxB,WAAO,aAAa;AAAA,MAClB,QAAQ,KAAK,KAAK;AAAA,MAClB,WAAW,KAAK,UAAU;AAAA,MAC1B,OAAO,KAAK;AAAA,MACZ,KAAK,KAAK;AAAA,MACV,WAAW,KAAK;AAAA,MAChB,SAAS,CAAC,GAAG,KAAK,aAAa,OAAO,CAAC;AAAA,MACvC,UAAU,KAAK;AAAA,MACf,mBAAmB,KAAK;AAAA,MACxB,UAAU,KAAK;AAAA,MACf,MAAM,KAAK,KAAK,SACZ,EAAE,SAAS,KAAK,eAAe,iBAAiB,KAAK,gBAAgB,IACrE;AAAA,MACJ,YAAY,KAAK;AAAA,MACjB,cAAc,KAAK;AAAA,IACrB,CAAC;AAAA,EACH;AACF;;;ACnVA,eAAsB,UACpB,MACsC;AACtC,QAAM,EAAE,kBAAkB,GAAG,KAAK,IAAI;AACtC,QAAM,WAAW,IAAI,iBAAoB;AAAA,IACvC,GAAG;AAAA,IACH,WAAW,qBAAqB,oBAAoB,GAAG;AAAA,EACzD,CAAC;AACD,SAAO,EAAE,SAAS,MAAM,SAAS,IAAI,EAAE;AACzC;;;ACfO,SAAS,gBAAmB,MAA6D;AAC9F,QAAM,UAAU,KAAK,OAAO,CAAC,MAA6B,KAAK,IAAI;AACnE,SAAO;AAAA,IACL,SAAS,OAAO,UAAU,IAAI,SAAS;AACrC,iBAAW,KAAK,SAAS;AACvB,YAAI,EAAE,WAAW,CAAE,MAAM,EAAE,QAAQ,UAAU,IAAI,IAAI,EAAI,QAAO;AAAA,MAClE;AACA,aAAO;AAAA,IACT;AAAA,IACA,kBAAkB,OAAO,UAAU,IAAI,SAAS;AAC9C,iBAAW,KAAK,SAAS;AACvB,YAAI,EAAE,oBAAoB,CAAE,MAAM,EAAE,iBAAiB,UAAU,IAAI,IAAI,EAAI,QAAO;AAAA,MACpF;AACA,aAAO;AAAA,IACT;AAAA,EACF;AACF;AASO,SAAS,0BAA6B,MAMxB;AACnB,QAAM,YAAY,KAAK,oBAAoB;AAC3C,SAAO;AAAA,IACL,kBAAkB,OAAO,UAAa,KAAiB,SAAe;AACpE,YAAM,YAAY,KAAK,QAAQ,QAAQ;AACvC,UAAI,aAAa,KAAM,QAAO;AAC9B,YAAM,KAAK,MAAM,KAAK,SAAS,WAAW,IAAI;AAC9C,aAAO,GAAG,QAAQ;AAAA,IACpB;AAAA,EACF;AACF;AAMO,SAAS,kBAAqB,MAGhB;AACnB,QAAM,YAAY,KAAK,oBAAoB;AAC3C,QAAM,SAAS,KAAK,UAAU;AAC9B,SAAO;AAAA,IACL,SAAS,CAAC,WAAW,OAAmB,GAAG,SAAS,YAAY;AAAA,EAClE;AACF;;;AC/CA,IAAM,UAAU,EAAE,MAAM,UAAU,YAAY,CAAC,GAAG,sBAAsB,MAAM;AAEvE,SAAS,iBAAoB,UAAiD;AACnF,SAAO;AAAA,IACL;AAAA,MACE,MAAM;AAAA,MACN,aACE;AAAA,MACF,YAAY;AAAA,MACZ,SAAS,YAAY;AACnB,cAAM,EAAE,MAAM,SAAS,IAAI,MAAM,SAAS,KAAK;AAC/C,eAAO,EAAE,MAAM,aAAa,SAAS,QAAQ,SAAS;AAAA,MACxD;AAAA,IACF;AAAA,IACA;AAAA,MACE,MAAM;AAAA,MACN,aACE;AAAA,MACF,YAAY;AAAA,MACZ,SAAS,YAAY,SAAS,SAAS;AAAA,IACzC;AAAA,IACA;AAAA,MACE,MAAM;AAAA,MACN,aAAa;AAAA,MACb,YAAY;AAAA,QACV,MAAM;AAAA,QACN,YAAY,EAAE,OAAO,EAAE,MAAM,UAAU,aAAa,yBAAyB,EAAE;AAAA,QAC/E,sBAAsB;AAAA,MACxB;AAAA,MACA,SAAS,OAAO,SAAS;AACvB,cAAM,QAAS,MAAyC;AACxD,cAAM,MAAM,SAAS,SAAS;AAC9B,eAAO,OAAO,UAAU,WAAW,IAAI,MAAM,GAAG,KAAK,IAAI;AAAA,MAC3D;AAAA,IACF;AAAA,IACA;AAAA,MACE,MAAM;AAAA,MACN,aACE;AAAA,MACF,YAAY;AAAA,QACV,MAAM;AAAA,QACN,YAAY;AAAA,UACV,QAAQ,EAAE,MAAM,UAAU,MAAM,CAAC,QAAQ,MAAM,EAAE;AAAA,UACjD,aAAa,EAAE,MAAM,UAAU,aAAa,uCAAuC;AAAA,QACrF;AAAA,QACA,sBAAsB;AAAA,MACxB;AAAA,MACA,SAAS,OAAO,SAAS;AACvB,cAAM,IAAK,QAAQ,CAAC;AACpB,cAAM,UAAU,SAAS,QAAQ;AACjC,YAAI,EAAE,WAAW,OAAQ,QAAO,kBAAkB,SAAS,EAAE,aAAa,EAAE,YAAY,CAAC;AACzF,eAAO;AAAA,MACT;AAAA,IACF;AAAA,EACF;AACF;","names":["cellId"]}
|
|
1
|
+
{"version":3,"sources":["../src/fuzz/cube.ts","../src/fuzz/capsule.ts","../src/fuzz/policies.ts","../src/fuzz/explorer.ts","../src/fuzz/fuzz-agent.ts","../src/fuzz/gates.ts","../src/fuzz/tools.ts"],"sourcesContent":["/**\n * Input-space tiling + coverage projection.\n *\n * Cells are the cartesian product of the input axes — the stratification plan,\n * enumerable up front so the planned-vs-covered denominator is honest. Coverage\n * is projected from the evaluation log: per cell, the full DISTRIBUTION of the\n * headline score, of each scored dimension, and of evaluation latency — a bare\n * mean hides outliers, so every aggregate carries its spread. Per-cell cost is\n * split known-dollars vs unknown-runs, never folded into a fabricated $0.\n */\n\nimport type { BehaviorSpace, Cell, CoverageCell, Distribution, Evaluation } from './types'\n\n/** One recorded evaluation — the unit coverage and the capsule are built from. */\nexport interface EvalRecord {\n cell: Cell\n ev: Evaluation\n /** The objective's interest score for this evaluation. */\n interest: number\n /** Evaluation wall-clock — engine-measured unless `ev.latencyMs` overrode it. */\n latencyMs: number\n /** Known dollars for this run. `null` = cost tracking was wired but this\n * run's cost was unknowable (counted apart). Absent = not tracked at all. */\n costUsd?: number | null\n}\n\n/** Enumerate every input cell (cartesian product of the axes), in stable order. */\nexport function enumerateCells(space: BehaviorSpace): Cell[] {\n if (space.axes.length === 0) return []\n let partials: Array<Record<string, string>> = [{}]\n for (const axis of space.axes) {\n const next: Array<Record<string, string>> = []\n for (const partial of partials) {\n for (const value of axis.values) next.push({ ...partial, [axis.name]: value })\n }\n partials = next\n }\n return partials.map((coords) => ({ id: cellId(space, coords), coords }))\n}\n\n/** Deterministic id for a coordinate map, e.g. `matterType=nda|difficulty=hard`. */\nexport function cellId(space: BehaviorSpace, coords: Record<string, string>): string {\n return space.axes.map((a) => `${a.name}=${coords[a.name]}`).join('|')\n}\n\n/** Nearest-rank percentile on a pre-sorted ascending sample. */\nfunction percentile(sorted: number[], p: number): number {\n const idx = Math.min(sorted.length - 1, Math.max(0, Math.ceil(p * sorted.length) - 1))\n return sorted[idx] as number\n}\n\n/** Summarize a sample. Throws on an empty sample — callers represent \"no data\"\n * as `null`, never as a zeroed distribution. */\nexport function distribution(values: number[]): Distribution {\n if (values.length === 0)\n throw new Error('distribution: empty sample — represent missing data as null, not zeros')\n const sorted = [...values].sort((a, b) => a - b)\n const mean = sorted.reduce((a, b) => a + b, 0) / sorted.length\n return {\n mean,\n median: percentile(sorted, 0.5),\n p90: percentile(sorted, 0.9),\n min: sorted[0] as number,\n max: sorted[sorted.length - 1] as number,\n n: sorted.length,\n }\n}\n\n/**\n * Project the evaluation log into the per-input-cell coverage map. A cell with\n * no evaluations reports `score: null` (honestly uncovered), never zeros.\n */\nexport function buildCoverage(cells: Cell[], log: EvalRecord[], threshold: number): CoverageCell[] {\n const byCell = new Map<string, EvalRecord[]>()\n for (const r of log) {\n const arr = byCell.get(r.cell.id) ?? []\n arr.push(r)\n byCell.set(r.cell.id, arr)\n }\n return cells.map((cell) => {\n const recs = byCell.get(cell.id) ?? []\n const runs = recs.length\n if (runs === 0)\n return { cell, runs: 0, score: null, findingRate: 0, dimensions: {}, latencyMs: null }\n\n const score = distribution(recs.map((r) => r.ev.score))\n const latencyMs = distribution(recs.map((r) => r.latencyMs))\n const findingRate = recs.filter((r) => r.interest >= threshold).length / runs\n\n const dimSamples: Record<string, number[]> = {}\n for (const r of recs) {\n for (const [k, v] of Object.entries(r.ev.scores ?? {})) {\n ;(dimSamples[k] ??= []).push(v)\n }\n }\n const dimensions: Record<string, Distribution> = {}\n for (const [k, xs] of Object.entries(dimSamples)) dimensions[k] = distribution(xs)\n\n // Cost fields appear only when tracking was wired: known dollars sum, and\n // tracked-but-unknown runs counted apart — never folded in as $0.\n const tracked = recs.filter((r) => r.costUsd !== undefined)\n const known = tracked.filter((r) => r.costUsd !== null)\n const cost =\n tracked.length > 0\n ? {\n costUsd: known.reduce((a, r) => a + (r.costUsd as number), 0),\n ...(tracked.length > known.length\n ? { costUnknownRuns: tracked.length - known.length }\n : {}),\n }\n : {}\n\n return { cell, runs, score, findingRate, dimensions, latencyMs, ...cost }\n })\n}\n","/**\n * The capsule — the artifact every exploration produces.\n *\n * `buildCapsule` assembles coverage + verified findings + the QD archive into a\n * pure `CapsuleData` (no clock, no I/O — deterministic and snapshot-testable).\n * `renderCapsuleHtml` turns it into a standalone page: the input-cell heat-map\n * (planned vs covered), per-dimension weakness chips, and the minimized finding\n * exemplars. One artifact — the hardening map and the shareable proof object.\n */\n\nimport type { EvalRecord } from './cube'\nimport { buildCoverage, distribution } from './cube'\nimport type { ArchiveEntry, CapsuleData, Cell, CoverageCell, Distribution, Finding } from './types'\n\nexport interface BuildCapsuleInput<S> {\n target: string\n objective: string\n cells: Cell[]\n log: EvalRecord[]\n /** The objective's notable threshold — drives findingRate. */\n threshold: number\n archive: ArchiveEntry<S>[]\n findings: Finding<S>[]\n candidateFindings: number\n runsUsed: number\n /** Known-dollar / unknown-run split — present only when cost tracking was\n * wired; the capsule never fabricates a $0 total. */\n cost?: { costUsd: number; costUnknownRuns: number }\n /** Evaluations that threw — infra outcomes, never folded into robustness. */\n evalErrors: number\n /** Set when the consecutive-error circuit breaker stopped the run early. */\n stoppedEarly?: { reason: 'eval-errors'; detail: string }\n}\n\nexport function buildCapsule<S>(input: BuildCapsuleInput<S>): CapsuleData<S> {\n const coverage = buildCoverage(input.cells, input.log, input.threshold)\n const covered = coverage.filter((c) => c.runs > 0)\n // Cells weigh equally: variance steering sends more runs to weak cells, so a\n // run-weighted average would bias the headline low.\n const robustness =\n covered.length === 0 ? null : distribution(covered.map((c) => (c.score as Distribution).mean))\n const latencyMs = input.log.length === 0 ? null : distribution(input.log.map((r) => r.latencyMs))\n // Measured-descriptor bins beyond the bare input cell — observed, never planned.\n const behaviorBinsObserved = input.archive.filter((e) => e.binId !== e.cell.id).length\n\n return {\n target: input.target,\n objective: input.objective,\n coverage,\n findings: [...input.findings].sort((a, b) => b.interest - a.interest),\n archive: [...input.archive].sort((a, b) => b.interest - a.interest),\n stats: {\n totalRuns: input.runsUsed,\n cellsTotal: input.cells.length,\n cellsCovered: covered.length,\n behaviorBinsObserved,\n candidateFindings: input.candidateFindings,\n verifiedFindings: input.findings.length,\n robustness,\n latencyMs,\n ...(input.cost\n ? { costUsd: input.cost.costUsd, costUnknownRuns: input.cost.costUnknownRuns }\n : {}),\n evalErrors: input.evalErrors,\n ...(input.stoppedEarly ? { stoppedEarly: input.stoppedEarly } : {}),\n },\n }\n}\n\n// ── HTML capsule ──────────────────────────────────────────────────────────────\n\nfunction esc(s: string): string {\n return s.replace(\n /[&<>\"]/g,\n (c) => ({ '&': '&', '<': '<', '>': '>', '\"': '"' })[c] as string,\n )\n}\n\n/** red (0) → amber (.5) → green (1); uncovered cells render gray. */\nfunction robustnessColor(r: number | null): string {\n if (r == null) return '#2a2a2e'\n return `hsl(${Math.round(r * 120)} 70% 42%)`\n}\n\nfunction pct(x: number): string {\n return `${Math.round(x * 100)}%`\n}\n\nfunction deriveAxes(coverage: CoverageCell[]): Array<{ name: string; values: string[] }> {\n const order: string[] = []\n const seen = new Map<string, Set<string>>()\n for (const c of coverage) {\n for (const [k, v] of Object.entries(c.cell.coords)) {\n if (!seen.has(k)) {\n seen.set(k, new Set())\n order.push(k)\n }\n seen.get(k)?.add(v)\n }\n }\n return order.map((name) => ({ name, values: [...(seen.get(name) ?? [])] }))\n}\n\n/** The weakest dimension chip for a cell, e.g. `safety 32%` — shown when scores exist. */\nfunction weakestDim(c: CoverageCell): string {\n const entries = Object.entries(c.dimensions).map(([k, d]) => [k, d.mean] as [string, number])\n if (entries.length === 0) return ''\n const sorted = entries.sort((a, b) => a[1] - b[1])\n const w = sorted[0]\n if (!w) return ''\n return `<span class=\"dim\">${esc(w[0])} ${pct(w[1])}</span>`\n}\n\nfunction heatmapHtml(coverage: CoverageCell[]): string {\n const axes = deriveAxes(coverage)\n const byId = new Map(coverage.map((c) => [c.cell.id, c]))\n const tile = (c: CoverageCell | undefined, label: string): string => {\n const r = c?.score?.mean ?? null\n const title = c?.score\n ? `${pct(c.score.mean)} robust (median ${pct(c.score.median)}, min ${pct(c.score.min)}) · ${c.runs} runs · ${pct(c.findingRate)} flagged${c.latencyMs ? ` · ${(c.latencyMs.median / 1000).toFixed(1)}s median` : ''}${c.costUsd !== undefined ? ` · $${c.costUsd.toFixed(2)}` : ''}`\n : 'not covered'\n return `<div class=\"tile\" style=\"background:${robustnessColor(r)}\" title=\"${esc(title)}\">${label ? `<span class=\"tl\">${esc(label)}</span>` : ''}<span class=\"tv\">${c && r != null ? pct(r) : '—'}</span>${c ? weakestDim(c) : ''}</div>`\n }\n\n const rowAxis = axes[0]\n const colAxis = axes[1]\n if (axes.length === 2 && rowAxis && colAxis) {\n const head = `<tr><th></th>${colAxis.values.map((v) => `<th>${esc(v)}</th>`).join('')}</tr>`\n const rows = rowAxis.values\n .map((rv) => {\n const cells = colAxis.values\n .map((cv) => {\n const id = `${rowAxis.name}=${rv}|${colAxis.name}=${cv}`\n return `<td>${tile(byId.get(id), '')}</td>`\n })\n .join('')\n return `<tr><th class=\"rh\">${esc(rv)}</th>${cells}</tr>`\n })\n .join('')\n return `<div class=\"axis-label\">rows: <b>${esc(rowAxis.name)}</b> · cols: <b>${esc(colAxis.name)}</b></div><table class=\"heat\">${head}${rows}</table>`\n }\n\n const sorted = [...coverage].sort((a, b) => (a.score?.mean ?? 2) - (b.score?.mean ?? 2))\n return `<div class=\"grid\">${sorted.map((c) => tile(c, Object.values(c.cell.coords).join(' · '))).join('')}</div>`\n}\n\nfunction findingsHtml<S>(findings: Finding<S>[], limit: number): string {\n if (findings.length === 0)\n return `<p class=\"none\">No verified findings — the target held across every covered cell.</p>`\n return findings\n .slice(0, limit)\n .map((f) => {\n const coords = Object.entries(f.cell.coords)\n .map(([k, v]) => `${k}:${v}`)\n .join(' · ')\n const labels = (f.evaluation.labels ?? [])\n .map((l) => `<span class=\"fclass\">${esc(l)}</span>`)\n .join('')\n const dims = Object.entries(f.evaluation.scores ?? {})\n .sort((a, b) => a[1] - b[1])\n .slice(0, 3)\n .map(([k, v]) => `<span class=\"dim\">${esc(k)} ${pct(v)}</span>`)\n .join('')\n return `<div class=\"fail\"><div class=\"fmeta\"><span class=\"fcell\">${esc(coords)}</span>${labels}<span class=\"fsev\">interest ${pct(f.interest)}</span></div><div class=\"sevbar\"><div class=\"sevfill\" style=\"width:${pct(f.interest)}\"></div></div><div class=\"ftext\">${esc(f.text ?? '(scenario text not captured)')}</div>${dims ? `<div class=\"fdims\">${dims}</div>` : ''}</div>`\n })\n .join('')\n}\n\nexport interface RenderCapsuleOptions {\n /** Max finding exemplars to show. Default 8. */\n maxFindings?: number\n /** ISO timestamp to stamp into the page (keeps the pure capsule clock-free). */\n generatedAt?: string\n}\n\n/** Render a self-contained HTML capsule — heat-map + per-dimension chips + verified findings. */\nexport function renderCapsuleHtml<S>(\n capsule: CapsuleData<S>,\n opts: RenderCapsuleOptions = {},\n): string {\n const s = capsule.stats\n const kpi = (label: string, value: string, accent = '#e6e6e6'): string =>\n `<div class=\"kpi\"><div class=\"kv\" style=\"color:${accent}\">${esc(value)}</div><div class=\"kl\">${esc(label)}</div></div>`\n const lift = capsule.lift\n ? kpi(\n 'hardening lift',\n `${capsule.lift.before.toFixed(2)} → ${capsule.lift.after.toFixed(2)}`,\n '#5ad17a',\n )\n : ''\n // Cost KPI only when tracking was wired — an untracked run never shows $0.\n // Unpriced runs are named in the label (amber): the total is a lower bound.\n const cost =\n s.costUsd !== undefined\n ? kpi(\n s.costUnknownRuns ? `cost · ${s.costUnknownRuns} runs unpriced` : 'cost',\n `$${s.costUsd.toFixed(2)}`,\n s.costUnknownRuns ? '#e5c07b' : '#e6e6e6',\n )\n : ''\n const stamp = opts.generatedAt ?? capsule.generatedAt ?? ''\n\n return `<!doctype html><html><head><meta charset=\"utf-8\"><meta name=\"viewport\" content=\"width=device-width,initial-scale=1\">\n<title>${esc(capsule.objective)} capsule — ${esc(capsule.target)}</title>\n<style>\n:root{color-scheme:dark}\n*{box-sizing:border-box}\nbody{margin:0;background:#0c0c0f;color:#e6e6e6;font:14px/1.5 ui-sans-serif,system-ui,-apple-system,Segoe UI,Roboto,sans-serif}\n.wrap{max-width:980px;margin:0 auto;padding:40px 24px}\nh1{font-size:22px;margin:0 0 2px;letter-spacing:-.01em}\n.sub{color:#8a8a93;font-size:13px;margin-bottom:28px}\n.kpis{display:flex;gap:14px;flex-wrap:wrap;margin-bottom:32px}\n.kpi{background:#16161b;border:1px solid #24242b;border-radius:12px;padding:14px 18px;min-width:120px}\n.kv{font-size:24px;font-weight:650;letter-spacing:-.02em}\n.kl{color:#8a8a93;font-size:12px;margin-top:2px}\nh2{font-size:13px;text-transform:uppercase;letter-spacing:.08em;color:#8a8a93;margin:34px 0 14px}\n.axis-label{color:#8a8a93;font-size:12px;margin-bottom:10px}\ntable.heat{border-collapse:separate;border-spacing:6px}\ntable.heat th{font-weight:500;color:#a8a8b0;font-size:12px;padding:4px 8px;text-align:center}\ntable.heat th.rh{text-align:right}\n.tile{position:relative;width:104px;height:66px;border-radius:10px;display:flex;flex-direction:column;justify-content:center;align-items:center;gap:1px;color:#06120a;font-weight:600}\n.tile .tv{font-size:17px}\n.tile .tl{font-size:10px;font-weight:600;opacity:.9;padding:0 6px;text-align:center}\n.tile .dim{font-size:9px;font-weight:600;opacity:.85;background:rgba(0,0,0,.22);padding:0 6px;border-radius:999px}\n.grid{display:flex;flex-wrap:wrap;gap:8px}\n.grid .tile{width:138px;height:74px}\n.fail{background:#16161b;border:1px solid #24242b;border-left:3px solid #d1495b;border-radius:10px;padding:12px 14px;margin-bottom:10px}\n.fmeta{display:flex;gap:10px;align-items:center;font-size:12px;color:#a8a8b0;margin-bottom:8px}\n.fcell{color:#cfcfd6}\n.fclass{background:#2a1a1d;color:#e58a96;padding:1px 8px;border-radius:999px;font-size:11px}\n.fsev{margin-left:auto;color:#e58a96}\n.sevbar{height:4px;background:#24242b;border-radius:2px;overflow:hidden;margin-bottom:10px}\n.sevfill{height:100%;background:#d1495b}\n.ftext{font-size:13px;color:#d8d8df;white-space:pre-wrap}\n.fdims{display:flex;gap:6px;margin-top:8px}\n.fdims .dim{background:#1d2530;color:#9fc1e8;padding:1px 8px;border-radius:999px;font-size:11px}\n.none{color:#5ad17a}\n.foot{color:#5a5a63;font-size:11px;margin-top:36px}\n</style></head><body><div class=\"wrap\">\n<h1>${esc(capsule.objective)} exploration · ${esc(capsule.target)}</h1>\n<div class=\"sub\">${s.totalRuns} scenarios across ${s.cellsCovered}/${s.cellsTotal} planned cells${s.behaviorBinsObserved > 0 ? ` · ${s.behaviorBinsObserved} measured behavior bins` : ''}${stamp ? ` · ${esc(stamp)}` : ''}</div>\n<div class=\"kpis\">\n${s.robustness ? kpi('robustness', `${pct(s.robustness.mean)}`, s.robustness.mean < 0.6 ? '#e58a96' : '#5ad17a') : ''}\n${s.robustness ? kpi('cell spread', `${pct(s.robustness.min)}–${pct(s.robustness.max)}`) : ''}\n${s.latencyMs ? kpi('median latency', `${(s.latencyMs.median / 1000).toFixed(1)}s`, s.latencyMs.p90 > 4 * s.latencyMs.median ? '#e5b566' : '#e6e6e6') : ''}\n${kpi('verified findings', String(s.verifiedFindings), s.verifiedFindings > 0 ? '#e58a96' : '#5ad17a')}\n${kpi('cells covered', `${s.cellsCovered}/${s.cellsTotal}`)}\n${kpi('scenarios run', String(s.totalRuns))}\n${cost}\n${s.evalErrors > 0 ? kpi('eval errors', String(s.evalErrors), '#e5b566') : ''}\n${lift}\n</div>\n${s.stoppedEarly ? `<div class=\"sub\" style=\"color:#e5b566\">stopped early: ${esc(s.stoppedEarly.detail)} — coverage below reflects the completed portion only</div>` : ''}\n<h2>Coverage map</h2>\n${heatmapHtml(capsule.coverage)}\n<h2>Verified findings${s.candidateFindings > s.verifiedFindings ? ` · ${s.verifiedFindings} of ${s.candidateFindings} candidates passed the validity gates` : ''}</h2>\n${findingsHtml(capsule.findings, opts.maxFindings ?? 8)}\n<div class=\"foot\">Findings are gate-verified: each is a fair, answerable task that reproduces under a meaning-preserving rephrase, minimized to the smallest trigger.</div>\n</div></body></html>`\n}\n","/**\n * Shipped policies for the exploration engine.\n *\n * `Proposer` is a plain function type — an agent running a generator skill IS a\n * proposer (`(ctx) => dispatchToSkill(ctx)`), no wrapper needed. `mutationProposer`\n * builds the deterministic, LLM-free one from mutation operators. Objectives are\n * interfaces because the engine reads `kind` + `threshold` off them.\n */\n\nimport type { AdversarialMutation } from '../rl/adversarial'\nimport type {\n Cell,\n Evaluation,\n Objective,\n ObjectiveContext,\n ProposeContext,\n Proposer,\n} from './types'\n\nconst clamp01 = (x: number): number => Math.max(0, Math.min(1, x))\n\n// ── proposers ─────────────────────────────────────────────────────────────────\n\n/**\n * Perturbation-based search: apply the cell's mutation operators to the current\n * elites + seeds, deduping by id. Elites first — mutating the most interesting\n * scenario found so far is what makes the search deepen across rounds.\n */\nexport function mutationProposer<S>(opts: {\n mutationsFor: (cell: Cell) => AdversarialMutation<S>[]\n scenarioId: (s: S) => string\n}): Proposer<S> {\n return async (ctx: ProposeContext<S>): Promise<S[]> => {\n const mutations = opts.mutationsFor(ctx.cell)\n const parents = [...ctx.elites, ...ctx.seeds]\n const seen = new Set(parents.map(opts.scenarioId))\n const out: S[] = []\n for (const parent of parents) {\n if (out.length >= ctx.count) break\n for (const m of mutations) {\n const children = await m.mutate(parent, ctx.rng)\n for (const child of children) {\n const id = opts.scenarioId(child)\n if (seen.has(id)) continue\n seen.add(id)\n out.push(child)\n if (out.length >= ctx.count) break\n }\n if (out.length >= ctx.count) break\n }\n }\n return out\n }\n}\n\n// ── objectives ────────────────────────────────────────────────────────────────\n\n/** Adversarial: a low headline score is interesting — find where the agent fails. */\nexport function adversarialObjective(threshold = 0.5): Objective {\n return { kind: 'adversarial', threshold, interest: (ev) => clamp01(1 - ev.score) }\n}\n\nfunction hamming(\n a: Record<string, string> | undefined,\n b: Record<string, string> | undefined,\n): number {\n if (!a || !b) return 1\n const keys = new Set([...Object.keys(a), ...Object.keys(b)])\n if (keys.size === 0) return 0\n let diff = 0\n for (const k of keys) if (a[k] !== b[k]) diff++\n return diff / keys.size\n}\n\n/**\n * Novelty: interesting when far from the archive in score AND measured behavior\n * descriptor — quality-diversity's diversity pressure; drives corpus growth\n * rather than re-finding the same hole.\n */\nexport function noveltyObjective(threshold = 0.3): Objective {\n return {\n kind: 'novelty',\n threshold,\n interest: (ev: Evaluation, ctx: ObjectiveContext) => {\n const scoreNovelty =\n ctx.archiveScores.length === 0\n ? 1\n : Math.min(...ctx.archiveScores.map((s) => Math.abs(s - ev.score)))\n const descNovelty =\n ctx.archiveDescriptors.length === 0\n ? 1\n : Math.min(...ctx.archiveDescriptors.map((d) => hamming(d, ev.descriptor)))\n return clamp01(0.5 * scoreNovelty + 0.5 * descNovelty)\n },\n }\n}\n","/**\n * The exploration engine — a stateful session over a behavior space.\n *\n * Each `step()`: allocate budget across INPUT cells (floor first, then variance\n * steering toward the least-certain cells), propose candidates (the proposer\n * reads current elites + findings, so the search deepens generationally),\n * evaluate with bounded concurrency, archive the most interesting scenario per\n * input×measured bin, and admit notable candidates that pass the validity gates.\n * `run()` loops to budget. `coverage()`/`findings()`/`capsule()` read live state —\n * the surface `makeExploreTools` exposes so an agent can drive the session.\n *\n * One evaluation log (`EvalRecord[]`) is the source of truth; allocation\n * observations and coverage are projections of it.\n */\n\nimport { ValidationError } from '../errors'\nimport { varianceBasedCurriculum } from '../rl/active-curriculum'\nimport { buildCapsule } from './capsule'\nimport type { EvalRecord } from './cube'\nimport { enumerateCells } from './cube'\nimport { adversarialObjective } from './policies'\nimport type {\n ArchiveEntry,\n CapsuleData,\n Cell,\n CoverageCell,\n Evaluation,\n ExploreOptions,\n Finding,\n Objective,\n} from './types'\n\nasync function pMap<T>(\n items: T[],\n fn: (item: T) => Promise<void>,\n concurrency: number,\n signal?: AbortSignal,\n): Promise<void> {\n let i = 0\n const workers = Array.from(\n { length: Math.max(1, Math.min(concurrency, items.length)) },\n async () => {\n while (i < items.length) {\n if (signal?.aborted) return\n const item = items[i++]\n if (item === undefined) return\n await fn(item)\n }\n },\n )\n await Promise.all(workers)\n}\n\nexport class BehaviorExplorer<S> {\n private readonly cells: Cell[]\n private readonly cellById: Map<string, Cell>\n private readonly objective: Objective\n private readonly threshold: number\n private readonly floorPerCell: number\n private readonly perRoundBudget: number\n\n /** The single evaluation log — coverage + allocation are projections of it. */\n private readonly log: Array<EvalRecord & { scenarioId: string }> = []\n /** binId (input × measured coords) → the most interesting entry seen. */\n private readonly archiveByBin = new Map<string, ArchiveEntry<S>>()\n private readonly _findings: Finding<S>[] = []\n private runsUsed = 0\n private candidateFindings = 0\n private evalErrors = 0\n private consecutiveEvalErrors = 0\n private stoppedEarly: { reason: 'eval-errors'; detail: string } | undefined\n private rngState: number\n /** Accumulated KNOWN dollars — unknown-cost runs never inflate it. */\n private spentKnownUsd = 0\n private costUnknownRuns = 0\n\n constructor(private readonly opts: ExploreOptions<S>) {\n this.cells = enumerateCells(opts.space)\n if (this.cells.length === 0)\n throw new Error('BehaviorExplorer: space has no cells — every axis needs ≥1 value')\n if (opts.costBudgetUsd !== undefined) {\n if (\n typeof opts.costBudgetUsd !== 'number' ||\n !Number.isFinite(opts.costBudgetUsd) ||\n opts.costBudgetUsd < 0\n ) {\n throw new RangeError(\n `BehaviorExplorer: costBudgetUsd must be a nonnegative finite number, got ${String(opts.costBudgetUsd)}`,\n )\n }\n }\n if (!opts.costOf && (opts.costBudgetUsd !== undefined || opts.ledger || opts.onCost)) {\n throw new ValidationError(\n 'BehaviorExplorer: costBudgetUsd/ledger/onCost require costOf — the explorer ' +\n 'cannot know run cost without it; supply costOf or drop the cost options',\n )\n }\n this.cellById = new Map(this.cells.map((c) => [c.id, c]))\n this.objective = opts.objective ?? adversarialObjective(0.5)\n this.threshold = this.objective.threshold ?? 0.5\n this.floorPerCell = opts.floorPerCell ?? 2\n this.perRoundBudget = Math.max(\n this.cells.length * this.floorPerCell,\n Math.ceil(opts.budget / 4),\n )\n this.rngState = (opts.seed ?? 1) >>> 0\n }\n\n // mulberry32 — deterministic per session.\n private rng = (): number => {\n this.rngState = (this.rngState + 0x6d2b79f5) | 0\n let t = this.rngState\n t = Math.imul(t ^ (t >>> 15), t | 1)\n t ^= t + Math.imul(t ^ (t >>> 7), t | 61)\n return ((t ^ (t >>> 14)) >>> 0) / 4294967296\n }\n\n private binId(cell: Cell, descriptor: Record<string, string> | undefined): string {\n if (!descriptor || Object.keys(descriptor).length === 0) return cell.id\n const measured = Object.keys(descriptor)\n .sort()\n .map((k) => `${k}=${descriptor[k]}`)\n .join('|')\n return `${cell.id}|${measured}`\n }\n\n private allocate(budget: number): Array<{ cellId: string; count: number }> {\n if ((this.opts.allocation ?? 'variance') === 'uniform') {\n const per = Math.max(this.floorPerCell, Math.floor(budget / this.cells.length))\n return this.cells.map((c) => ({ cellId: c.id, count: per }))\n }\n return varianceBasedCurriculum(\n this.log.map((r) => ({\n variantId: r.cell.id,\n scenarioId: r.scenarioId,\n score: r.ev.score,\n pass: r.ev.valid && r.ev.score >= 0.5,\n })),\n this.cells.map((c) => ({ variantId: c.id, scenarioId: '*' })),\n { budget, floorPerCell: this.floorPerCell },\n ).map((a) => ({ cellId: a.variantId, count: a.count }))\n }\n\n private objectiveContext() {\n const entries = [...this.archiveByBin.values()]\n return {\n archiveScores: entries.map((e) => e.evaluation.score),\n archiveDescriptors: entries.map((e) => e.evaluation.descriptor),\n }\n }\n\n /** Mirrors control-runtime: stop once accumulated KNOWN cost ≥ the ceiling. */\n private costExhausted(): boolean {\n return this.opts.costBudgetUsd !== undefined && this.spentKnownUsd >= this.opts.costBudgetUsd\n }\n\n /** Fold one run's cost in: null counts as unknown (never $0); a known cost\n * accrues toward the budget, lands in the ledger, and fires `onCost`. */\n /** Returns the run's known cost, `null` when tracked-but-unknown, `undefined`\n * when cost tracking is not wired — the log row mirrors this exactly. */\n private recordRunCost(scenario: S, cell: Cell, ev: Evaluation): number | null | undefined {\n if (!this.opts.costOf) return undefined\n const cost = this.opts.costOf(scenario, cell, ev)\n if (cost === null) {\n this.costUnknownRuns++\n return null\n }\n if (typeof cost.usd !== 'number' || !Number.isFinite(cost.usd) || cost.usd < 0) {\n throw new RangeError(\n `BehaviorExplorer: costOf returned an invalid usd (${String(cost?.usd)}) — ` +\n 'return null when cost is unknown, never a fabricated number',\n )\n }\n this.spentKnownUsd += cost.usd\n this.opts.ledger?.record({\n model: cost.model ?? 'unattributed',\n channel: 'agent',\n usage: { inputTokens: 0, outputTokens: 0 },\n actualCostUsd: cost.usd,\n tags: { target: this.opts.target, cell: cell.id },\n })\n this.opts.onCost?.({ usd: cost.usd, channel: 'agent' })\n return cost.usd\n }\n\n /** Elites whose INPUT cell matches — what the proposer mutates/deepens from. */\n private elitesFor(cellId: string): S[] {\n const out: S[] = []\n for (const e of this.archiveByBin.values()) if (e.cell.id === cellId) out.push(e.scenario)\n return out\n }\n\n /** One allocate → propose → evaluate → gate → archive round. */\n async step(): Promise<{ runs: number; findings: Finding<S>[] }> {\n const remaining = this.opts.budget - this.runsUsed\n if (remaining <= 0 || this.costExhausted() || this.opts.signal?.aborted)\n return { runs: 0, findings: [] }\n\n const allocations = this.allocate(Math.min(this.perRoundBudget, remaining))\n const newFindings: Finding<S>[] = []\n let runsThisStep = 0\n\n for (const alloc of allocations) {\n if (\n this.runsUsed >= this.opts.budget ||\n this.costExhausted() ||\n this.stoppedEarly !== undefined ||\n this.opts.signal?.aborted\n )\n break\n const cell = this.cellById.get(alloc.cellId)\n if (!cell) continue\n const cap = Math.min(alloc.count, this.opts.budget - this.runsUsed)\n if (cap <= 0) continue\n this.opts.onProgress?.({ type: 'cell-allocated', cell, count: cap })\n\n const seeds = await this.opts.seedsFor(cell)\n const elites = this.elitesFor(cell.id)\n const proposed = await this.opts.proposer({\n cell,\n seeds,\n elites,\n findings: this._findings,\n count: cap,\n rng: this.rng,\n })\n // Cold cells evaluate their seeds first (the coverage floor); warm cells\n // trust the proposer, which already saw the elites.\n const cold = this.log.every((r) => r.cell.id !== cell.id)\n const toEval = [...(cold ? seeds : []), ...proposed].slice(0, cap)\n\n await pMap(\n toEval,\n async (scenario) => {\n if (\n this.runsUsed >= this.opts.budget ||\n this.costExhausted() ||\n this.stoppedEarly !== undefined ||\n this.opts.signal?.aborted\n )\n return\n // evaluate/gates/minimize cross an external boundary (router, backend,\n // judge). A throw there is an infra outcome: record it as a typed\n // eval-error and keep exploring — one 5xx must not kill a campaign.\n // Consecutive failures trip the circuit breaker instead, so a dead\n // backend stops the run rather than burning the remaining budget.\n try {\n const startedAt = performance.now()\n const ev = await this.opts.evaluate(scenario, cell)\n // Consumer-measured latency wins (it can exclude judge time); the\n // engine's wall-clock is the default so latency is never missing.\n const latencyMs = ev.latencyMs ?? performance.now() - startedAt\n this.runsUsed++\n runsThisStep++\n this.consecutiveEvalErrors = 0\n const costUsd = this.recordRunCost(scenario, cell, ev)\n const interest = this.objective.interest(ev, this.objectiveContext())\n this.log.push({\n cell,\n ev,\n interest,\n latencyMs,\n ...(costUsd !== undefined ? { costUsd } : {}),\n scenarioId: this.opts.scenarioId(scenario),\n })\n this.opts.onProgress?.({ type: 'evaluated', cell, scenario, evaluation: ev })\n\n const bin = this.binId(cell, ev.descriptor)\n const cur = this.archiveByBin.get(bin)\n if (!cur || interest > cur.interest)\n this.archiveByBin.set(bin, { binId: bin, cell, scenario, evaluation: ev, interest })\n\n if (interest < this.threshold) return\n this.candidateFindings++\n if (this.opts.gates?.isValid && !(await this.opts.gates.isValid(scenario, ev, cell)))\n return\n if (\n this.opts.gates?.isUncontaminated &&\n !(await this.opts.gates.isUncontaminated(scenario, ev, cell))\n )\n return\n\n const minimized = this.opts.minimize\n ? await this.opts.minimize(scenario, this.opts.evaluate, cell)\n : scenario\n const finding: Finding<S> = {\n id: this.opts.scenarioId(scenario),\n cell,\n scenario,\n minimized,\n text: this.opts.scenarioText?.(minimized),\n evaluation: ev,\n interest,\n objective: this.objective.kind,\n }\n this._findings.push(finding)\n newFindings.push(finding)\n this.opts.onProgress?.({ type: 'finding', finding })\n } catch (err) {\n // Internal validation errors (e.g. a fabricated costOf number) are\n // programming mistakes, not backend outcomes — they stay loud.\n if (err instanceof RangeError) throw err\n this.evalErrors++\n this.consecutiveEvalErrors++\n const message = err instanceof Error ? err.message : String(err)\n this.opts.onProgress?.({\n type: 'eval-error',\n cell,\n scenarioId: this.opts.scenarioId(scenario),\n message,\n })\n const limit = this.opts.maxConsecutiveEvalErrors ?? 5\n if (this.consecutiveEvalErrors >= limit) {\n this.stoppedEarly = {\n reason: 'eval-errors',\n detail: `${this.consecutiveEvalErrors} consecutive eval errors (last: ${message})`,\n }\n }\n }\n },\n this.opts.concurrency ?? 1,\n this.opts.signal,\n )\n }\n\n this.opts.onProgress?.({ type: 'round', runsUsed: this.runsUsed, budget: this.opts.budget })\n return { runs: runsThisStep, findings: newFindings }\n }\n\n /** Loop `step()` until the run or dollar budget is spent, the signal aborts,\n * or no progress is made. */\n async run(): Promise<CapsuleData<S>> {\n while (\n this.runsUsed < this.opts.budget &&\n !this.costExhausted() &&\n this.stoppedEarly === undefined &&\n !this.opts.signal?.aborted\n ) {\n const { runs } = await this.step()\n if (runs === 0 && this.stoppedEarly === undefined) break\n }\n return this.capsule()\n }\n\n coverage(): CoverageCell[] {\n return this.capsule().coverage\n }\n\n findings(): Finding<S>[] {\n return [...this._findings].sort((a, b) => b.interest - a.interest)\n }\n\n capsule(): CapsuleData<S> {\n return buildCapsule({\n target: this.opts.target,\n objective: this.objective.kind,\n cells: this.cells,\n log: this.log,\n threshold: this.threshold,\n archive: [...this.archiveByBin.values()],\n findings: this._findings,\n candidateFindings: this.candidateFindings,\n runsUsed: this.runsUsed,\n cost: this.opts.costOf\n ? { costUsd: this.spentKnownUsd, costUnknownRuns: this.costUnknownRuns }\n : undefined,\n evalErrors: this.evalErrors,\n stoppedEarly: this.stoppedEarly,\n })\n }\n}\n","/**\n * `fuzzAgent` — the adversarial batch preset over `BehaviorExplorer`.\n *\n * One call: explore the space to budget with the adversarial objective and\n * return the capsule. For agent-driven, incremental, or multi-objective use,\n * construct a `BehaviorExplorer` and drive it via `makeExploreTools`.\n */\n\nimport { BehaviorExplorer } from './explorer'\nimport { adversarialObjective } from './policies'\nimport type { CapsuleData, ExploreOptions } from './types'\n\nexport type FuzzAgentOptions<S> = Omit<ExploreOptions<S>, 'objective'> & {\n /** Score strictly below this is a candidate failure. Default 0.5. */\n failureThreshold?: number\n}\n\nexport async function fuzzAgent<S>(\n opts: FuzzAgentOptions<S>,\n): Promise<{ capsule: CapsuleData<S> }> {\n const { failureThreshold, ...rest } = opts\n const explorer = new BehaviorExplorer<S>({\n ...rest,\n objective: adversarialObjective(failureThreshold ?? 0.5),\n })\n return { capsule: await explorer.run() }\n}\n","/**\n * Validity gates — what separates a fuzzer from a slop generator.\n *\n * A notable candidate is admitted only when it is fair and reproducible. None of\n * these are on by default: the live wiring opts in, so reported findings carry\n * their proof.\n */\n\nimport type { Cell, Evaluation, Evaluator, ValidityGates } from './types'\n\n/** Combine gate sets; a candidate must pass every gate in every set. */\nexport function composeGates<S>(...sets: Array<ValidityGates<S> | undefined>): ValidityGates<S> {\n const present = sets.filter((s): s is ValidityGates<S> => s != null)\n return {\n isValid: async (scenario, ev, cell) => {\n for (const g of present) {\n if (g.isValid && !(await g.isValid(scenario, ev, cell))) return false\n }\n return true\n },\n isUncontaminated: async (scenario, ev, cell) => {\n for (const g of present) {\n if (g.isUncontaminated && !(await g.isUncontaminated(scenario, ev, cell))) return false\n }\n return true\n },\n }\n}\n\n/**\n * Reproducibility gate. Re-run the target on a meaning-preserving rephrase of the\n * flagged scenario; keep the finding only when the rephrase ALSO scores below the\n * threshold. A finding that flips under a cosmetic rewrite was keyed to surface\n * form, not the task — a false signal we must not report. Costs one extra\n * evaluation per candidate (candidates are rare, so cheap).\n */\nexport function perturbationStabilityGate<S>(opts: {\n evaluate: Evaluator<S>\n /** Produce a semantic-preserving rephrase. Return null to skip (treated as pass). */\n perturb: (scenario: S) => S | null\n /** Score strictly below this still counts as failing. Default 0.5. */\n failureThreshold?: number\n}): ValidityGates<S> {\n const threshold = opts.failureThreshold ?? 0.5\n return {\n isUncontaminated: async (scenario: S, _ev: Evaluation, cell: Cell) => {\n const rephrased = opts.perturb(scenario)\n if (rephrased == null) return true\n const re = await opts.evaluate(rephrased, cell)\n return re.score < threshold\n },\n }\n}\n\n/**\n * Severity-floor gate. Reject borderline candidates whose score sits in a band\n * just under the threshold — judge noise, not a real defect.\n */\nexport function severityFloorGate<S>(opts: {\n failureThreshold?: number\n margin?: number\n}): ValidityGates<S> {\n const threshold = opts.failureThreshold ?? 0.5\n const margin = opts.margin ?? 0.1\n return {\n isValid: (_scenario, ev: Evaluation) => ev.score <= threshold - margin,\n }\n}\n","/**\n * Agent-drivable surface over a live exploration session.\n *\n * Framework-neutral tool defs ({name, description, parameters: JSON Schema,\n * handler}) so the on-demand agent — not a batch script — drives the search:\n * step it, read coverage, inspect findings, render the capsule. Transport\n * encodings (OpenAI function shape, MCP) are one-line mappings the host owns.\n */\n\nimport { renderCapsuleHtml } from './capsule'\nimport type { BehaviorExplorer } from './explorer'\n\nexport interface ExploreToolDef {\n name: string\n description: string\n /** JSON Schema (draft-07+) for the arguments. */\n parameters: Record<string, unknown>\n handler: (args: unknown, ctx?: { signal?: AbortSignal }) => Promise<unknown>\n}\n\nconst NO_ARGS = { type: 'object', properties: {}, additionalProperties: false }\n\nexport function makeExploreTools<S>(explorer: BehaviorExplorer<S>): ExploreToolDef[] {\n return [\n {\n name: 'explore_step',\n description:\n 'Run one exploration round: allocate budget across cells, propose + evaluate scenarios, archive elites, admit gate-verified findings. Returns runs spent and new findings.',\n parameters: NO_ARGS,\n handler: async () => {\n const { runs, findings } = await explorer.step()\n return { runs, newFindings: findings.length, findings }\n },\n },\n {\n name: 'explore_coverage',\n description:\n 'Read the live coverage map: per planned cell — runs, robustness (null = uncovered), finding rate, per-dimension means.',\n parameters: NO_ARGS,\n handler: async () => explorer.coverage(),\n },\n {\n name: 'explore_findings',\n description: 'List gate-verified findings so far, sorted by descending interest.',\n parameters: {\n type: 'object',\n properties: { limit: { type: 'number', description: 'max findings to return' } },\n additionalProperties: false,\n },\n handler: async (args) => {\n const limit = (args as { limit?: number } | undefined)?.limit\n const all = explorer.findings()\n return typeof limit === 'number' ? all.slice(0, limit) : all\n },\n },\n {\n name: 'explore_capsule',\n description:\n 'Build the capsule artifact from the current session state. format \"data\" returns the structured CapsuleData; \"html\" returns the standalone page.',\n parameters: {\n type: 'object',\n properties: {\n format: { type: 'string', enum: ['data', 'html'] },\n generatedAt: { type: 'string', description: 'ISO timestamp to stamp into the page' },\n },\n additionalProperties: false,\n },\n handler: async (args) => {\n const a = (args ?? {}) as { format?: string; generatedAt?: string }\n const capsule = explorer.capsule()\n if (a.format === 'html') return renderCapsuleHtml(capsule, { generatedAt: a.generatedAt })\n return capsule\n },\n },\n ]\n}\n"],"mappings":";;;;;;;;;AA2BO,SAAS,eAAe,OAA8B;AAC3D,MAAI,MAAM,KAAK,WAAW,EAAG,QAAO,CAAC;AACrC,MAAI,WAA0C,CAAC,CAAC,CAAC;AACjD,aAAW,QAAQ,MAAM,MAAM;AAC7B,UAAM,OAAsC,CAAC;AAC7C,eAAW,WAAW,UAAU;AAC9B,iBAAW,SAAS,KAAK,OAAQ,MAAK,KAAK,EAAE,GAAG,SAAS,CAAC,KAAK,IAAI,GAAG,MAAM,CAAC;AAAA,IAC/E;AACA,eAAW;AAAA,EACb;AACA,SAAO,SAAS,IAAI,CAAC,YAAY,EAAE,IAAI,OAAO,OAAO,MAAM,GAAG,OAAO,EAAE;AACzE;AAGO,SAAS,OAAO,OAAsB,QAAwC;AACnF,SAAO,MAAM,KAAK,IAAI,CAAC,MAAM,GAAG,EAAE,IAAI,IAAI,OAAO,EAAE,IAAI,CAAC,EAAE,EAAE,KAAK,GAAG;AACtE;AAGA,SAAS,WAAW,QAAkB,GAAmB;AACvD,QAAM,MAAM,KAAK,IAAI,OAAO,SAAS,GAAG,KAAK,IAAI,GAAG,KAAK,KAAK,IAAI,OAAO,MAAM,IAAI,CAAC,CAAC;AACrF,SAAO,OAAO,GAAG;AACnB;AAIO,SAAS,aAAa,QAAgC;AAC3D,MAAI,OAAO,WAAW;AACpB,UAAM,IAAI,MAAM,6EAAwE;AAC1F,QAAM,SAAS,CAAC,GAAG,MAAM,EAAE,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC;AAC/C,QAAM,OAAO,OAAO,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,OAAO;AACxD,SAAO;AAAA,IACL;AAAA,IACA,QAAQ,WAAW,QAAQ,GAAG;AAAA,IAC9B,KAAK,WAAW,QAAQ,GAAG;AAAA,IAC3B,KAAK,OAAO,CAAC;AAAA,IACb,KAAK,OAAO,OAAO,SAAS,CAAC;AAAA,IAC7B,GAAG,OAAO;AAAA,EACZ;AACF;AAMO,SAAS,cAAc,OAAe,KAAmB,WAAmC;AACjG,QAAM,SAAS,oBAAI,IAA0B;AAC7C,aAAW,KAAK,KAAK;AACnB,UAAM,MAAM,OAAO,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC;AACtC,QAAI,KAAK,CAAC;AACV,WAAO,IAAI,EAAE,KAAK,IAAI,GAAG;AAAA,EAC3B;AACA,SAAO,MAAM,IAAI,CAAC,SAAS;AACzB,UAAM,OAAO,OAAO,IAAI,KAAK,EAAE,KAAK,CAAC;AACrC,UAAM,OAAO,KAAK;AAClB,QAAI,SAAS;AACX,aAAO,EAAE,MAAM,MAAM,GAAG,OAAO,MAAM,aAAa,GAAG,YAAY,CAAC,GAAG,WAAW,KAAK;AAEvF,UAAM,QAAQ,aAAa,KAAK,IAAI,CAAC,MAAM,EAAE,GAAG,KAAK,CAAC;AACtD,UAAM,YAAY,aAAa,KAAK,IAAI,CAAC,MAAM,EAAE,SAAS,CAAC;AAC3D,UAAM,cAAc,KAAK,OAAO,CAAC,MAAM,EAAE,YAAY,SAAS,EAAE,SAAS;AAEzE,UAAM,aAAuC,CAAC;AAC9C,eAAW,KAAK,MAAM;AACpB,iBAAW,CAAC,GAAG,CAAC,KAAK,OAAO,QAAQ,EAAE,GAAG,UAAU,CAAC,CAAC,GAAG;AACtD;AAAC,SAAC,WAAW,CAAC,MAAM,CAAC,GAAG,KAAK,CAAC;AAAA,MAChC;AAAA,IACF;AACA,UAAM,aAA2C,CAAC;AAClD,eAAW,CAAC,GAAG,EAAE,KAAK,OAAO,QAAQ,UAAU,EAAG,YAAW,CAAC,IAAI,aAAa,EAAE;AAIjF,UAAM,UAAU,KAAK,OAAO,CAAC,MAAM,EAAE,YAAY,MAAS;AAC1D,UAAM,QAAQ,QAAQ,OAAO,CAAC,MAAM,EAAE,YAAY,IAAI;AACtD,UAAM,OACJ,QAAQ,SAAS,IACb;AAAA,MACE,SAAS,MAAM,OAAO,CAAC,GAAG,MAAM,IAAK,EAAE,SAAoB,CAAC;AAAA,MAC5D,GAAI,QAAQ,SAAS,MAAM,SACvB,EAAE,iBAAiB,QAAQ,SAAS,MAAM,OAAO,IACjD,CAAC;AAAA,IACP,IACA,CAAC;AAEP,WAAO,EAAE,MAAM,MAAM,OAAO,aAAa,YAAY,WAAW,GAAG,KAAK;AAAA,EAC1E,CAAC;AACH;;;AChFO,SAAS,aAAgB,OAA6C;AAC3E,QAAM,WAAW,cAAc,MAAM,OAAO,MAAM,KAAK,MAAM,SAAS;AACtE,QAAM,UAAU,SAAS,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC;AAGjD,QAAM,aACJ,QAAQ,WAAW,IAAI,OAAO,aAAa,QAAQ,IAAI,CAAC,MAAO,EAAE,MAAuB,IAAI,CAAC;AAC/F,QAAM,YAAY,MAAM,IAAI,WAAW,IAAI,OAAO,aAAa,MAAM,IAAI,IAAI,CAAC,MAAM,EAAE,SAAS,CAAC;AAEhG,QAAM,uBAAuB,MAAM,QAAQ,OAAO,CAAC,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,EAAE;AAEhF,SAAO;AAAA,IACL,QAAQ,MAAM;AAAA,IACd,WAAW,MAAM;AAAA,IACjB;AAAA,IACA,UAAU,CAAC,GAAG,MAAM,QAAQ,EAAE,KAAK,CAAC,GAAG,MAAM,EAAE,WAAW,EAAE,QAAQ;AAAA,IACpE,SAAS,CAAC,GAAG,MAAM,OAAO,EAAE,KAAK,CAAC,GAAG,MAAM,EAAE,WAAW,EAAE,QAAQ;AAAA,IAClE,OAAO;AAAA,MACL,WAAW,MAAM;AAAA,MACjB,YAAY,MAAM,MAAM;AAAA,MACxB,cAAc,QAAQ;AAAA,MACtB;AAAA,MACA,mBAAmB,MAAM;AAAA,MACzB,kBAAkB,MAAM,SAAS;AAAA,MACjC;AAAA,MACA;AAAA,MACA,GAAI,MAAM,OACN,EAAE,SAAS,MAAM,KAAK,SAAS,iBAAiB,MAAM,KAAK,gBAAgB,IAC3E,CAAC;AAAA,MACL,YAAY,MAAM;AAAA,MAClB,GAAI,MAAM,eAAe,EAAE,cAAc,MAAM,aAAa,IAAI,CAAC;AAAA,IACnE;AAAA,EACF;AACF;AAIA,SAAS,IAAI,GAAmB;AAC9B,SAAO,EAAE;AAAA,IACP;AAAA,IACA,CAAC,OAAO,EAAE,KAAK,SAAS,KAAK,QAAQ,KAAK,QAAQ,KAAK,SAAS,GAAG,CAAC;AAAA,EACtE;AACF;AAGA,SAAS,gBAAgB,GAA0B;AACjD,MAAI,KAAK,KAAM,QAAO;AACtB,SAAO,OAAO,KAAK,MAAM,IAAI,GAAG,CAAC;AACnC;AAEA,SAAS,IAAI,GAAmB;AAC9B,SAAO,GAAG,KAAK,MAAM,IAAI,GAAG,CAAC;AAC/B;AAEA,SAAS,WAAW,UAAqE;AACvF,QAAM,QAAkB,CAAC;AACzB,QAAM,OAAO,oBAAI,IAAyB;AAC1C,aAAW,KAAK,UAAU;AACxB,eAAW,CAAC,GAAG,CAAC,KAAK,OAAO,QAAQ,EAAE,KAAK,MAAM,GAAG;AAClD,UAAI,CAAC,KAAK,IAAI,CAAC,GAAG;AAChB,aAAK,IAAI,GAAG,oBAAI,IAAI,CAAC;AACrB,cAAM,KAAK,CAAC;AAAA,MACd;AACA,WAAK,IAAI,CAAC,GAAG,IAAI,CAAC;AAAA,IACpB;AAAA,EACF;AACA,SAAO,MAAM,IAAI,CAAC,UAAU,EAAE,MAAM,QAAQ,CAAC,GAAI,KAAK,IAAI,IAAI,KAAK,CAAC,CAAE,EAAE,EAAE;AAC5E;AAGA,SAAS,WAAW,GAAyB;AAC3C,QAAM,UAAU,OAAO,QAAQ,EAAE,UAAU,EAAE,IAAI,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,EAAE,IAAI,CAAqB;AAC5F,MAAI,QAAQ,WAAW,EAAG,QAAO;AACjC,QAAM,SAAS,QAAQ,KAAK,CAAC,GAAG,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC;AACjD,QAAM,IAAI,OAAO,CAAC;AAClB,MAAI,CAAC,EAAG,QAAO;AACf,SAAO,qBAAqB,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,CAAC;AACpD;AAEA,SAAS,YAAY,UAAkC;AACrD,QAAM,OAAO,WAAW,QAAQ;AAChC,QAAM,OAAO,IAAI,IAAI,SAAS,IAAI,CAAC,MAAM,CAAC,EAAE,KAAK,IAAI,CAAC,CAAC,CAAC;AACxD,QAAM,OAAO,CAAC,GAA6B,UAA0B;AACnE,UAAM,IAAI,GAAG,OAAO,QAAQ;AAC5B,UAAM,QAAQ,GAAG,QACb,GAAG,IAAI,EAAE,MAAM,IAAI,CAAC,mBAAmB,IAAI,EAAE,MAAM,MAAM,CAAC,SAAS,IAAI,EAAE,MAAM,GAAG,CAAC,UAAO,EAAE,IAAI,cAAW,IAAI,EAAE,WAAW,CAAC,WAAW,EAAE,YAAY,UAAO,EAAE,UAAU,SAAS,KAAM,QAAQ,CAAC,CAAC,aAAa,EAAE,GAAG,EAAE,YAAY,SAAY,UAAO,EAAE,QAAQ,QAAQ,CAAC,CAAC,KAAK,EAAE,KAChR;AACJ,WAAO,uCAAuC,gBAAgB,CAAC,CAAC,YAAY,IAAI,KAAK,CAAC,KAAK,QAAQ,oBAAoB,IAAI,KAAK,CAAC,YAAY,EAAE,oBAAoB,KAAK,KAAK,OAAO,IAAI,CAAC,IAAI,QAAG,UAAU,IAAI,WAAW,CAAC,IAAI,EAAE;AAAA,EAClO;AAEA,QAAM,UAAU,KAAK,CAAC;AACtB,QAAM,UAAU,KAAK,CAAC;AACtB,MAAI,KAAK,WAAW,KAAK,WAAW,SAAS;AAC3C,UAAM,OAAO,gBAAgB,QAAQ,OAAO,IAAI,CAAC,MAAM,OAAO,IAAI,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,CAAC;AACrF,UAAM,OAAO,QAAQ,OAClB,IAAI,CAAC,OAAO;AACX,YAAM,QAAQ,QAAQ,OACnB,IAAI,CAAC,OAAO;AACX,cAAM,KAAK,GAAG,QAAQ,IAAI,IAAI,EAAE,IAAI,QAAQ,IAAI,IAAI,EAAE;AACtD,eAAO,OAAO,KAAK,KAAK,IAAI,EAAE,GAAG,EAAE,CAAC;AAAA,MACtC,CAAC,EACA,KAAK,EAAE;AACV,aAAO,sBAAsB,IAAI,EAAE,CAAC,QAAQ,KAAK;AAAA,IACnD,CAAC,EACA,KAAK,EAAE;AACV,WAAO,oCAAoC,IAAI,QAAQ,IAAI,CAAC,sBAAmB,IAAI,QAAQ,IAAI,CAAC,iCAAiC,IAAI,GAAG,IAAI;AAAA,EAC9I;AAEA,QAAM,SAAS,CAAC,GAAG,QAAQ,EAAE,KAAK,CAAC,GAAG,OAAO,EAAE,OAAO,QAAQ,MAAM,EAAE,OAAO,QAAQ,EAAE;AACvF,SAAO,qBAAqB,OAAO,IAAI,CAAC,MAAM,KAAK,GAAG,OAAO,OAAO,EAAE,KAAK,MAAM,EAAE,KAAK,QAAK,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC;AAC3G;AAEA,SAAS,aAAgB,UAAwB,OAAuB;AACtE,MAAI,SAAS,WAAW;AACtB,WAAO;AACT,SAAO,SACJ,MAAM,GAAG,KAAK,EACd,IAAI,CAAC,MAAM;AACV,UAAM,SAAS,OAAO,QAAQ,EAAE,KAAK,MAAM,EACxC,IAAI,CAAC,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,EAAE,EAC3B,KAAK,QAAK;AACb,UAAM,UAAU,EAAE,WAAW,UAAU,CAAC,GACrC,IAAI,CAAC,MAAM,wBAAwB,IAAI,CAAC,CAAC,SAAS,EAClD,KAAK,EAAE;AACV,UAAM,OAAO,OAAO,QAAQ,EAAE,WAAW,UAAU,CAAC,CAAC,EAClD,KAAK,CAAC,GAAG,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC,EAC1B,MAAM,GAAG,CAAC,EACV,IAAI,CAAC,CAAC,GAAG,CAAC,MAAM,qBAAqB,IAAI,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,SAAS,EAC9D,KAAK,EAAE;AACV,WAAO,4DAA4D,IAAI,MAAM,CAAC,UAAU,MAAM,+BAA+B,IAAI,EAAE,QAAQ,CAAC,sEAAsE,IAAI,EAAE,QAAQ,CAAC,oCAAoC,IAAI,EAAE,QAAQ,8BAA8B,CAAC,SAAS,OAAO,sBAAsB,IAAI,WAAW,EAAE;AAAA,EAC3W,CAAC,EACA,KAAK,EAAE;AACZ;AAUO,SAAS,kBACd,SACA,OAA6B,CAAC,GACtB;AACR,QAAM,IAAI,QAAQ;AAClB,QAAM,MAAM,CAAC,OAAe,OAAe,SAAS,cAClD,iDAAiD,MAAM,KAAK,IAAI,KAAK,CAAC,yBAAyB,IAAI,KAAK,CAAC;AAC3G,QAAM,OAAO,QAAQ,OACjB;AAAA,IACE;AAAA,IACA,GAAG,QAAQ,KAAK,OAAO,QAAQ,CAAC,CAAC,WAAM,QAAQ,KAAK,MAAM,QAAQ,CAAC,CAAC;AAAA,IACpE;AAAA,EACF,IACA;AAGJ,QAAM,OACJ,EAAE,YAAY,SACV;AAAA,IACE,EAAE,kBAAkB,aAAU,EAAE,eAAe,mBAAmB;AAAA,IAClE,IAAI,EAAE,QAAQ,QAAQ,CAAC,CAAC;AAAA,IACxB,EAAE,kBAAkB,YAAY;AAAA,EAClC,IACA;AACN,QAAM,QAAQ,KAAK,eAAe,QAAQ,eAAe;AAEzD,SAAO;AAAA,SACA,IAAI,QAAQ,SAAS,CAAC,mBAAc,IAAI,QAAQ,MAAM,CAAC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,MAoC1D,IAAI,QAAQ,SAAS,CAAC,qBAAkB,IAAI,QAAQ,MAAM,CAAC;AAAA,mBAC9C,EAAE,SAAS,qBAAqB,EAAE,YAAY,IAAI,EAAE,UAAU,iBAAiB,EAAE,uBAAuB,IAAI,SAAM,EAAE,oBAAoB,4BAA4B,EAAE,GAAG,QAAQ,SAAM,IAAI,KAAK,CAAC,KAAK,EAAE;AAAA;AAAA,EAEzN,EAAE,aAAa,IAAI,cAAc,GAAG,IAAI,EAAE,WAAW,IAAI,CAAC,IAAI,EAAE,WAAW,OAAO,MAAM,YAAY,SAAS,IAAI,EAAE;AAAA,EACnH,EAAE,aAAa,IAAI,eAAe,GAAG,IAAI,EAAE,WAAW,GAAG,CAAC,SAAI,IAAI,EAAE,WAAW,GAAG,CAAC,EAAE,IAAI,EAAE;AAAA,EAC3F,EAAE,YAAY,IAAI,kBAAkB,IAAI,EAAE,UAAU,SAAS,KAAM,QAAQ,CAAC,CAAC,KAAK,EAAE,UAAU,MAAM,IAAI,EAAE,UAAU,SAAS,YAAY,SAAS,IAAI,EAAE;AAAA,EACxJ,IAAI,qBAAqB,OAAO,EAAE,gBAAgB,GAAG,EAAE,mBAAmB,IAAI,YAAY,SAAS,CAAC;AAAA,EACpG,IAAI,iBAAiB,GAAG,EAAE,YAAY,IAAI,EAAE,UAAU,EAAE,CAAC;AAAA,EACzD,IAAI,iBAAiB,OAAO,EAAE,SAAS,CAAC,CAAC;AAAA,EACzC,IAAI;AAAA,EACJ,EAAE,aAAa,IAAI,IAAI,eAAe,OAAO,EAAE,UAAU,GAAG,SAAS,IAAI,EAAE;AAAA,EAC3E,IAAI;AAAA;AAAA,EAEJ,EAAE,eAAe,yDAAyD,IAAI,EAAE,aAAa,MAAM,CAAC,qEAAgE,EAAE;AAAA;AAAA,EAEtK,YAAY,QAAQ,QAAQ,CAAC;AAAA,uBACR,EAAE,oBAAoB,EAAE,mBAAmB,SAAM,EAAE,gBAAgB,OAAO,EAAE,iBAAiB,0CAA0C,EAAE;AAAA,EAC9J,aAAa,QAAQ,UAAU,KAAK,eAAe,CAAC,CAAC;AAAA;AAAA;AAGvD;;;AChPA,IAAM,UAAU,CAAC,MAAsB,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,CAAC,CAAC;AAS1D,SAAS,iBAAoB,MAGpB;AACd,SAAO,OAAO,QAAyC;AACrD,UAAM,YAAY,KAAK,aAAa,IAAI,IAAI;AAC5C,UAAM,UAAU,CAAC,GAAG,IAAI,QAAQ,GAAG,IAAI,KAAK;AAC5C,UAAM,OAAO,IAAI,IAAI,QAAQ,IAAI,KAAK,UAAU,CAAC;AACjD,UAAM,MAAW,CAAC;AAClB,eAAW,UAAU,SAAS;AAC5B,UAAI,IAAI,UAAU,IAAI,MAAO;AAC7B,iBAAW,KAAK,WAAW;AACzB,cAAM,WAAW,MAAM,EAAE,OAAO,QAAQ,IAAI,GAAG;AAC/C,mBAAW,SAAS,UAAU;AAC5B,gBAAM,KAAK,KAAK,WAAW,KAAK;AAChC,cAAI,KAAK,IAAI,EAAE,EAAG;AAClB,eAAK,IAAI,EAAE;AACX,cAAI,KAAK,KAAK;AACd,cAAI,IAAI,UAAU,IAAI,MAAO;AAAA,QAC/B;AACA,YAAI,IAAI,UAAU,IAAI,MAAO;AAAA,MAC/B;AAAA,IACF;AACA,WAAO;AAAA,EACT;AACF;AAKO,SAAS,qBAAqB,YAAY,KAAgB;AAC/D,SAAO,EAAE,MAAM,eAAe,WAAW,UAAU,CAAC,OAAO,QAAQ,IAAI,GAAG,KAAK,EAAE;AACnF;AAEA,SAAS,QACP,GACA,GACQ;AACR,MAAI,CAAC,KAAK,CAAC,EAAG,QAAO;AACrB,QAAM,OAAO,oBAAI,IAAI,CAAC,GAAG,OAAO,KAAK,CAAC,GAAG,GAAG,OAAO,KAAK,CAAC,CAAC,CAAC;AAC3D,MAAI,KAAK,SAAS,EAAG,QAAO;AAC5B,MAAI,OAAO;AACX,aAAW,KAAK,KAAM,KAAI,EAAE,CAAC,MAAM,EAAE,CAAC,EAAG;AACzC,SAAO,OAAO,KAAK;AACrB;AAOO,SAAS,iBAAiB,YAAY,KAAgB;AAC3D,SAAO;AAAA,IACL,MAAM;AAAA,IACN;AAAA,IACA,UAAU,CAAC,IAAgB,QAA0B;AACnD,YAAM,eACJ,IAAI,cAAc,WAAW,IACzB,IACA,KAAK,IAAI,GAAG,IAAI,cAAc,IAAI,CAAC,MAAM,KAAK,IAAI,IAAI,GAAG,KAAK,CAAC,CAAC;AACtE,YAAM,cACJ,IAAI,mBAAmB,WAAW,IAC9B,IACA,KAAK,IAAI,GAAG,IAAI,mBAAmB,IAAI,CAAC,MAAM,QAAQ,GAAG,GAAG,UAAU,CAAC,CAAC;AAC9E,aAAO,QAAQ,MAAM,eAAe,MAAM,WAAW;AAAA,IACvD;AAAA,EACF;AACF;;;AC/DA,eAAe,KACb,OACA,IACA,aACA,QACe;AACf,MAAI,IAAI;AACR,QAAM,UAAU,MAAM;AAAA,IACpB,EAAE,QAAQ,KAAK,IAAI,GAAG,KAAK,IAAI,aAAa,MAAM,MAAM,CAAC,EAAE;AAAA,IAC3D,YAAY;AACV,aAAO,IAAI,MAAM,QAAQ;AACvB,YAAI,QAAQ,QAAS;AACrB,cAAM,OAAO,MAAM,GAAG;AACtB,YAAI,SAAS,OAAW;AACxB,cAAM,GAAG,IAAI;AAAA,MACf;AAAA,IACF;AAAA,EACF;AACA,QAAM,QAAQ,IAAI,OAAO;AAC3B;AAEO,IAAM,mBAAN,MAA0B;AAAA,EAuB/B,YAA6B,MAAyB;AAAzB;AAC3B,SAAK,QAAQ,eAAe,KAAK,KAAK;AACtC,QAAI,KAAK,MAAM,WAAW;AACxB,YAAM,IAAI,MAAM,4EAAkE;AACpF,QAAI,KAAK,kBAAkB,QAAW;AACpC,UACE,OAAO,KAAK,kBAAkB,YAC9B,CAAC,OAAO,SAAS,KAAK,aAAa,KACnC,KAAK,gBAAgB,GACrB;AACA,cAAM,IAAI;AAAA,UACR,4EAA4E,OAAO,KAAK,aAAa,CAAC;AAAA,QACxG;AAAA,MACF;AAAA,IACF;AACA,QAAI,CAAC,KAAK,WAAW,KAAK,kBAAkB,UAAa,KAAK,UAAU,KAAK,SAAS;AACpF,YAAM,IAAI;AAAA,QACR;AAAA,MAEF;AAAA,IACF;AACA,SAAK,WAAW,IAAI,IAAI,KAAK,MAAM,IAAI,CAAC,MAAM,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC;AACxD,SAAK,YAAY,KAAK,aAAa,qBAAqB,GAAG;AAC3D,SAAK,YAAY,KAAK,UAAU,aAAa;AAC7C,SAAK,eAAe,KAAK,gBAAgB;AACzC,SAAK,iBAAiB,KAAK;AAAA,MACzB,KAAK,MAAM,SAAS,KAAK;AAAA,MACzB,KAAK,KAAK,KAAK,SAAS,CAAC;AAAA,IAC3B;AACA,SAAK,YAAY,KAAK,QAAQ,OAAO;AAAA,EACvC;AAAA,EA9B6B;AAAA,EAtBZ;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA;AAAA,EAGA,MAAkD,CAAC;AAAA;AAAA,EAEnD,eAAe,oBAAI,IAA6B;AAAA,EAChD,YAA0B,CAAC;AAAA,EACpC,WAAW;AAAA,EACX,oBAAoB;AAAA,EACpB,aAAa;AAAA,EACb,wBAAwB;AAAA,EACxB;AAAA,EACA;AAAA;AAAA,EAEA,gBAAgB;AAAA,EAChB,kBAAkB;AAAA;AAAA,EAmClB,MAAM,MAAc;AAC1B,SAAK,WAAY,KAAK,WAAW,aAAc;AAC/C,QAAI,IAAI,KAAK;AACb,QAAI,KAAK,KAAK,IAAK,MAAM,IAAK,IAAI,CAAC;AACnC,SAAK,IAAI,KAAK,KAAK,IAAK,MAAM,GAAI,IAAI,EAAE;AACxC,aAAS,IAAK,MAAM,QAAS,KAAK;AAAA,EACpC;AAAA,EAEQ,MAAM,MAAY,YAAwD;AAChF,QAAI,CAAC,cAAc,OAAO,KAAK,UAAU,EAAE,WAAW,EAAG,QAAO,KAAK;AACrE,UAAM,WAAW,OAAO,KAAK,UAAU,EACpC,KAAK,EACL,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,WAAW,CAAC,CAAC,EAAE,EAClC,KAAK,GAAG;AACX,WAAO,GAAG,KAAK,EAAE,IAAI,QAAQ;AAAA,EAC/B;AAAA,EAEQ,SAAS,QAA0D;AACzE,SAAK,KAAK,KAAK,cAAc,gBAAgB,WAAW;AACtD,YAAM,MAAM,KAAK,IAAI,KAAK,cAAc,KAAK,MAAM,SAAS,KAAK,MAAM,MAAM,CAAC;AAC9E,aAAO,KAAK,MAAM,IAAI,CAAC,OAAO,EAAE,QAAQ,EAAE,IAAI,OAAO,IAAI,EAAE;AAAA,IAC7D;AACA,WAAO;AAAA,MACL,KAAK,IAAI,IAAI,CAAC,OAAO;AAAA,QACnB,WAAW,EAAE,KAAK;AAAA,QAClB,YAAY,EAAE;AAAA,QACd,OAAO,EAAE,GAAG;AAAA,QACZ,MAAM,EAAE,GAAG,SAAS,EAAE,GAAG,SAAS;AAAA,MACpC,EAAE;AAAA,MACF,KAAK,MAAM,IAAI,CAAC,OAAO,EAAE,WAAW,EAAE,IAAI,YAAY,IAAI,EAAE;AAAA,MAC5D,EAAE,QAAQ,cAAc,KAAK,aAAa;AAAA,IAC5C,EAAE,IAAI,CAAC,OAAO,EAAE,QAAQ,EAAE,WAAW,OAAO,EAAE,MAAM,EAAE;AAAA,EACxD;AAAA,EAEQ,mBAAmB;AACzB,UAAM,UAAU,CAAC,GAAG,KAAK,aAAa,OAAO,CAAC;AAC9C,WAAO;AAAA,MACL,eAAe,QAAQ,IAAI,CAAC,MAAM,EAAE,WAAW,KAAK;AAAA,MACpD,oBAAoB,QAAQ,IAAI,CAAC,MAAM,EAAE,WAAW,UAAU;AAAA,IAChE;AAAA,EACF;AAAA;AAAA,EAGQ,gBAAyB;AAC/B,WAAO,KAAK,KAAK,kBAAkB,UAAa,KAAK,iBAAiB,KAAK,KAAK;AAAA,EAClF;AAAA;AAAA;AAAA;AAAA;AAAA,EAMQ,cAAc,UAAa,MAAY,IAA2C;AACxF,QAAI,CAAC,KAAK,KAAK,OAAQ,QAAO;AAC9B,UAAM,OAAO,KAAK,KAAK,OAAO,UAAU,MAAM,EAAE;AAChD,QAAI,SAAS,MAAM;AACjB,WAAK;AACL,aAAO;AAAA,IACT;AACA,QAAI,OAAO,KAAK,QAAQ,YAAY,CAAC,OAAO,SAAS,KAAK,GAAG,KAAK,KAAK,MAAM,GAAG;AAC9E,YAAM,IAAI;AAAA,QACR,qDAAqD,OAAO,MAAM,GAAG,CAAC;AAAA,MAExE;AAAA,IACF;AACA,SAAK,iBAAiB,KAAK;AAC3B,SAAK,KAAK,QAAQ,OAAO;AAAA,MACvB,OAAO,KAAK,SAAS;AAAA,MACrB,SAAS;AAAA,MACT,OAAO,EAAE,aAAa,GAAG,cAAc,EAAE;AAAA,MACzC,eAAe,KAAK;AAAA,MACpB,MAAM,EAAE,QAAQ,KAAK,KAAK,QAAQ,MAAM,KAAK,GAAG;AAAA,IAClD,CAAC;AACD,SAAK,KAAK,SAAS,EAAE,KAAK,KAAK,KAAK,SAAS,QAAQ,CAAC;AACtD,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGQ,UAAUA,SAAqB;AACrC,UAAM,MAAW,CAAC;AAClB,eAAW,KAAK,KAAK,aAAa,OAAO,EAAG,KAAI,EAAE,KAAK,OAAOA,QAAQ,KAAI,KAAK,EAAE,QAAQ;AACzF,WAAO;AAAA,EACT;AAAA;AAAA,EAGA,MAAM,OAA0D;AAC9D,UAAM,YAAY,KAAK,KAAK,SAAS,KAAK;AAC1C,QAAI,aAAa,KAAK,KAAK,cAAc,KAAK,KAAK,KAAK,QAAQ;AAC9D,aAAO,EAAE,MAAM,GAAG,UAAU,CAAC,EAAE;AAEjC,UAAM,cAAc,KAAK,SAAS,KAAK,IAAI,KAAK,gBAAgB,SAAS,CAAC;AAC1E,UAAM,cAA4B,CAAC;AACnC,QAAI,eAAe;AAEnB,eAAW,SAAS,aAAa;AAC/B,UACE,KAAK,YAAY,KAAK,KAAK,UAC3B,KAAK,cAAc,KACnB,KAAK,iBAAiB,UACtB,KAAK,KAAK,QAAQ;AAElB;AACF,YAAM,OAAO,KAAK,SAAS,IAAI,MAAM,MAAM;AAC3C,UAAI,CAAC,KAAM;AACX,YAAM,MAAM,KAAK,IAAI,MAAM,OAAO,KAAK,KAAK,SAAS,KAAK,QAAQ;AAClE,UAAI,OAAO,EAAG;AACd,WAAK,KAAK,aAAa,EAAE,MAAM,kBAAkB,MAAM,OAAO,IAAI,CAAC;AAEnE,YAAM,QAAQ,MAAM,KAAK,KAAK,SAAS,IAAI;AAC3C,YAAM,SAAS,KAAK,UAAU,KAAK,EAAE;AACrC,YAAM,WAAW,MAAM,KAAK,KAAK,SAAS;AAAA,QACxC;AAAA,QACA;AAAA,QACA;AAAA,QACA,UAAU,KAAK;AAAA,QACf,OAAO;AAAA,QACP,KAAK,KAAK;AAAA,MACZ,CAAC;AAGD,YAAM,OAAO,KAAK,IAAI,MAAM,CAAC,MAAM,EAAE,KAAK,OAAO,KAAK,EAAE;AACxD,YAAM,SAAS,CAAC,GAAI,OAAO,QAAQ,CAAC,GAAI,GAAG,QAAQ,EAAE,MAAM,GAAG,GAAG;AAEjE,YAAM;AAAA,QACJ;AAAA,QACA,OAAO,aAAa;AAClB,cACE,KAAK,YAAY,KAAK,KAAK,UAC3B,KAAK,cAAc,KACnB,KAAK,iBAAiB,UACtB,KAAK,KAAK,QAAQ;AAElB;AAMF,cAAI;AACF,kBAAM,YAAY,YAAY,IAAI;AAClC,kBAAM,KAAK,MAAM,KAAK,KAAK,SAAS,UAAU,IAAI;AAGlD,kBAAM,YAAY,GAAG,aAAa,YAAY,IAAI,IAAI;AACtD,iBAAK;AACL;AACA,iBAAK,wBAAwB;AAC7B,kBAAM,UAAU,KAAK,cAAc,UAAU,MAAM,EAAE;AACrD,kBAAM,WAAW,KAAK,UAAU,SAAS,IAAI,KAAK,iBAAiB,CAAC;AACpE,iBAAK,IAAI,KAAK;AAAA,cACZ;AAAA,cACA;AAAA,cACA;AAAA,cACA;AAAA,cACA,GAAI,YAAY,SAAY,EAAE,QAAQ,IAAI,CAAC;AAAA,cAC3C,YAAY,KAAK,KAAK,WAAW,QAAQ;AAAA,YAC3C,CAAC;AACD,iBAAK,KAAK,aAAa,EAAE,MAAM,aAAa,MAAM,UAAU,YAAY,GAAG,CAAC;AAE5E,kBAAM,MAAM,KAAK,MAAM,MAAM,GAAG,UAAU;AAC1C,kBAAM,MAAM,KAAK,aAAa,IAAI,GAAG;AACrC,gBAAI,CAAC,OAAO,WAAW,IAAI;AACzB,mBAAK,aAAa,IAAI,KAAK,EAAE,OAAO,KAAK,MAAM,UAAU,YAAY,IAAI,SAAS,CAAC;AAErF,gBAAI,WAAW,KAAK,UAAW;AAC/B,iBAAK;AACL,gBAAI,KAAK,KAAK,OAAO,WAAW,CAAE,MAAM,KAAK,KAAK,MAAM,QAAQ,UAAU,IAAI,IAAI;AAChF;AACF,gBACE,KAAK,KAAK,OAAO,oBACjB,CAAE,MAAM,KAAK,KAAK,MAAM,iBAAiB,UAAU,IAAI,IAAI;AAE3D;AAEF,kBAAM,YAAY,KAAK,KAAK,WACxB,MAAM,KAAK,KAAK,SAAS,UAAU,KAAK,KAAK,UAAU,IAAI,IAC3D;AACJ,kBAAM,UAAsB;AAAA,cAC1B,IAAI,KAAK,KAAK,WAAW,QAAQ;AAAA,cACjC;AAAA,cACA;AAAA,cACA;AAAA,cACA,MAAM,KAAK,KAAK,eAAe,SAAS;AAAA,cACxC,YAAY;AAAA,cACZ;AAAA,cACA,WAAW,KAAK,UAAU;AAAA,YAC5B;AACA,iBAAK,UAAU,KAAK,OAAO;AAC3B,wBAAY,KAAK,OAAO;AACxB,iBAAK,KAAK,aAAa,EAAE,MAAM,WAAW,QAAQ,CAAC;AAAA,UACrD,SAAS,KAAK;AAGZ,gBAAI,eAAe,WAAY,OAAM;AACrC,iBAAK;AACL,iBAAK;AACL,kBAAM,UAAU,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAC/D,iBAAK,KAAK,aAAa;AAAA,cACrB,MAAM;AAAA,cACN;AAAA,cACA,YAAY,KAAK,KAAK,WAAW,QAAQ;AAAA,cACzC;AAAA,YACF,CAAC;AACD,kBAAM,QAAQ,KAAK,KAAK,4BAA4B;AACpD,gBAAI,KAAK,yBAAyB,OAAO;AACvC,mBAAK,eAAe;AAAA,gBAClB,QAAQ;AAAA,gBACR,QAAQ,GAAG,KAAK,qBAAqB,mCAAmC,OAAO;AAAA,cACjF;AAAA,YACF;AAAA,UACF;AAAA,QACF;AAAA,QACA,KAAK,KAAK,eAAe;AAAA,QACzB,KAAK,KAAK;AAAA,MACZ;AAAA,IACF;AAEA,SAAK,KAAK,aAAa,EAAE,MAAM,SAAS,UAAU,KAAK,UAAU,QAAQ,KAAK,KAAK,OAAO,CAAC;AAC3F,WAAO,EAAE,MAAM,cAAc,UAAU,YAAY;AAAA,EACrD;AAAA;AAAA;AAAA,EAIA,MAAM,MAA+B;AACnC,WACE,KAAK,WAAW,KAAK,KAAK,UAC1B,CAAC,KAAK,cAAc,KACpB,KAAK,iBAAiB,UACtB,CAAC,KAAK,KAAK,QAAQ,SACnB;AACA,YAAM,EAAE,KAAK,IAAI,MAAM,KAAK,KAAK;AACjC,UAAI,SAAS,KAAK,KAAK,iBAAiB,OAAW;AAAA,IACrD;AACA,WAAO,KAAK,QAAQ;AAAA,EACtB;AAAA,EAEA,WAA2B;AACzB,WAAO,KAAK,QAAQ,EAAE;AAAA,EACxB;AAAA,EAEA,WAAyB;AACvB,WAAO,CAAC,GAAG,KAAK,SAAS,EAAE,KAAK,CAAC,GAAG,MAAM,EAAE,WAAW,EAAE,QAAQ;AAAA,EACnE;AAAA,EAEA,UAA0B;AACxB,WAAO,aAAa;AAAA,MAClB,QAAQ,KAAK,KAAK;AAAA,MAClB,WAAW,KAAK,UAAU;AAAA,MAC1B,OAAO,KAAK;AAAA,MACZ,KAAK,KAAK;AAAA,MACV,WAAW,KAAK;AAAA,MAChB,SAAS,CAAC,GAAG,KAAK,aAAa,OAAO,CAAC;AAAA,MACvC,UAAU,KAAK;AAAA,MACf,mBAAmB,KAAK;AAAA,MACxB,UAAU,KAAK;AAAA,MACf,MAAM,KAAK,KAAK,SACZ,EAAE,SAAS,KAAK,eAAe,iBAAiB,KAAK,gBAAgB,IACrE;AAAA,MACJ,YAAY,KAAK;AAAA,MACjB,cAAc,KAAK;AAAA,IACrB,CAAC;AAAA,EACH;AACF;;;ACjWA,eAAsB,UACpB,MACsC;AACtC,QAAM,EAAE,kBAAkB,GAAG,KAAK,IAAI;AACtC,QAAM,WAAW,IAAI,iBAAoB;AAAA,IACvC,GAAG;AAAA,IACH,WAAW,qBAAqB,oBAAoB,GAAG;AAAA,EACzD,CAAC;AACD,SAAO,EAAE,SAAS,MAAM,SAAS,IAAI,EAAE;AACzC;;;ACfO,SAAS,gBAAmB,MAA6D;AAC9F,QAAM,UAAU,KAAK,OAAO,CAAC,MAA6B,KAAK,IAAI;AACnE,SAAO;AAAA,IACL,SAAS,OAAO,UAAU,IAAI,SAAS;AACrC,iBAAW,KAAK,SAAS;AACvB,YAAI,EAAE,WAAW,CAAE,MAAM,EAAE,QAAQ,UAAU,IAAI,IAAI,EAAI,QAAO;AAAA,MAClE;AACA,aAAO;AAAA,IACT;AAAA,IACA,kBAAkB,OAAO,UAAU,IAAI,SAAS;AAC9C,iBAAW,KAAK,SAAS;AACvB,YAAI,EAAE,oBAAoB,CAAE,MAAM,EAAE,iBAAiB,UAAU,IAAI,IAAI,EAAI,QAAO;AAAA,MACpF;AACA,aAAO;AAAA,IACT;AAAA,EACF;AACF;AASO,SAAS,0BAA6B,MAMxB;AACnB,QAAM,YAAY,KAAK,oBAAoB;AAC3C,SAAO;AAAA,IACL,kBAAkB,OAAO,UAAa,KAAiB,SAAe;AACpE,YAAM,YAAY,KAAK,QAAQ,QAAQ;AACvC,UAAI,aAAa,KAAM,QAAO;AAC9B,YAAM,KAAK,MAAM,KAAK,SAAS,WAAW,IAAI;AAC9C,aAAO,GAAG,QAAQ;AAAA,IACpB;AAAA,EACF;AACF;AAMO,SAAS,kBAAqB,MAGhB;AACnB,QAAM,YAAY,KAAK,oBAAoB;AAC3C,QAAM,SAAS,KAAK,UAAU;AAC9B,SAAO;AAAA,IACL,SAAS,CAAC,WAAW,OAAmB,GAAG,SAAS,YAAY;AAAA,EAClE;AACF;;;AC/CA,IAAM,UAAU,EAAE,MAAM,UAAU,YAAY,CAAC,GAAG,sBAAsB,MAAM;AAEvE,SAAS,iBAAoB,UAAiD;AACnF,SAAO;AAAA,IACL;AAAA,MACE,MAAM;AAAA,MACN,aACE;AAAA,MACF,YAAY;AAAA,MACZ,SAAS,YAAY;AACnB,cAAM,EAAE,MAAM,SAAS,IAAI,MAAM,SAAS,KAAK;AAC/C,eAAO,EAAE,MAAM,aAAa,SAAS,QAAQ,SAAS;AAAA,MACxD;AAAA,IACF;AAAA,IACA;AAAA,MACE,MAAM;AAAA,MACN,aACE;AAAA,MACF,YAAY;AAAA,MACZ,SAAS,YAAY,SAAS,SAAS;AAAA,IACzC;AAAA,IACA;AAAA,MACE,MAAM;AAAA,MACN,aAAa;AAAA,MACb,YAAY;AAAA,QACV,MAAM;AAAA,QACN,YAAY,EAAE,OAAO,EAAE,MAAM,UAAU,aAAa,yBAAyB,EAAE;AAAA,QAC/E,sBAAsB;AAAA,MACxB;AAAA,MACA,SAAS,OAAO,SAAS;AACvB,cAAM,QAAS,MAAyC;AACxD,cAAM,MAAM,SAAS,SAAS;AAC9B,eAAO,OAAO,UAAU,WAAW,IAAI,MAAM,GAAG,KAAK,IAAI;AAAA,MAC3D;AAAA,IACF;AAAA,IACA;AAAA,MACE,MAAM;AAAA,MACN,aACE;AAAA,MACF,YAAY;AAAA,QACV,MAAM;AAAA,QACN,YAAY;AAAA,UACV,QAAQ,EAAE,MAAM,UAAU,MAAM,CAAC,QAAQ,MAAM,EAAE;AAAA,UACjD,aAAa,EAAE,MAAM,UAAU,aAAa,uCAAuC;AAAA,QACrF;AAAA,QACA,sBAAsB;AAAA,MACxB;AAAA,MACA,SAAS,OAAO,SAAS;AACvB,cAAM,IAAK,QAAQ,CAAC;AACpB,cAAM,UAAU,SAAS,QAAQ;AACjC,YAAI,EAAE,WAAW,OAAQ,QAAO,kBAAkB,SAAS,EAAE,aAAa,EAAE,YAAY,CAAC;AACzF,eAAO;AAAA,MACT;AAAA,IACF;AAAA,EACF;AACF;","names":["cellId"]}
|
package/dist/index.d.ts
CHANGED
|
@@ -14,8 +14,8 @@ import { A as AgentEvalError, J as JudgeError, a as ConfigError } from './errors
|
|
|
14
14
|
export { b as AgentEvalErrorCode, C as CaptureIntegrityError, N as NotFoundError, R as ReplayError, V as ValidationError, c as VerificationError } from './errors-CzMUYo7b.js';
|
|
15
15
|
import { b as FeedbackLabel, F as FeedbackTrajectoryStore, a as FeedbackTrajectory } from './feedback-trajectory-D9OVLrg9.js';
|
|
16
16
|
export { c as FeedbackArtifactType, d as FeedbackAttempt, e as FeedbackLabelKind, f as FeedbackLabelSource, g as FeedbackOptimizerRow, h as FeedbackOutcome, i as FeedbackReplayAdapter, j as FeedbackReplayResult, k as FeedbackSeverity, l as FeedbackSplitPolicy, m as FeedbackTask, n as FeedbackTrajectoryFilter, o as FileSystemFeedbackTrajectoryStore, I as InMemoryFeedbackTrajectoryStore, P as PreferenceMemoryEntry, p as ProposedSideEffect, q as assignFeedbackSplit, r as controlRunToFeedbackTrajectory, s as createFeedbackTrajectory, t as feedbackTrajectoriesToDatasetScenarios, u as feedbackTrajectoriesToOptimizerRows, v as feedbackTrajectoryToDatasetScenario, w as feedbackTrajectoryToOptimizerRow, x as parseFeedbackTrajectoriesJsonl, y as renderPreferenceMemoryMarkdown, z as replayFeedbackTrajectories, A as replayFeedbackTrajectory, B as serializeFeedbackTrajectoriesJsonl, C as summarizePreferenceMemory, D as withAssignedFeedbackSplit } from './feedback-trajectory-D9OVLrg9.js';
|
|
17
|
-
import { b as CorrectnessChecker, A as AgentProfile$1 } from './pre-registration-
|
|
18
|
-
export { c as ArtifactCheckArtifact, d as ArtifactEventLike, e as ArtifactValidator, f as BackendIntegrityError, B as BackendIntegrityReport, C as CompletionRequirement, a as CompletionVerdict, H as HypothesisManifest, g as HypothesisResult, L as LlmCorrectnessCheckerOpts, h as ProducedProposal, P as ProducedState, i as ProposalEventLike, j as RequirementCheck, R as RuntimeEventLike, k as SatisfiedBy, S as SignedManifest, l as SignedManifestAlgo, T as TaskGold, m as ToolCallEventLike, V as ValidationContext, n as ValidationIssue, o as ValidationResult, p as agentProfileHash, q as assertRealBackend, r as byteLengthRange, s as canonicalize, t as completionVerdict, u as composeValidators, v as containsAll, w as createLlmCorrectnessChecker, x as createTokenRecallChecker, y as evaluateHypothesis, z as extractProducedState, D as hashJson, E as jsonHasKeys, F as parseCorrectnessResponse, G as regexMatch, I as signManifest, J as summarizeBackendIntegrity, K as verifyCompletion, M as verifyManifest } from './pre-registration-
|
|
17
|
+
import { b as CorrectnessChecker, A as AgentProfile$1 } from './pre-registration-mAnCugl9.js';
|
|
18
|
+
export { c as ArtifactCheckArtifact, d as ArtifactEventLike, e as ArtifactValidator, f as BackendIntegrityError, B as BackendIntegrityReport, C as CompletionRequirement, a as CompletionVerdict, H as HypothesisManifest, g as HypothesisResult, L as LlmCorrectnessCheckerOpts, h as ProducedProposal, P as ProducedState, i as ProposalEventLike, j as RequirementCheck, R as RuntimeEventLike, k as SatisfiedBy, S as SignedManifest, l as SignedManifestAlgo, T as TaskGold, m as ToolCallEventLike, V as ValidationContext, n as ValidationIssue, o as ValidationResult, p as agentProfileHash, q as assertRealBackend, r as byteLengthRange, s as canonicalize, t as completionVerdict, u as composeValidators, v as containsAll, w as createLlmCorrectnessChecker, x as createTokenRecallChecker, y as evaluateHypothesis, z as extractProducedState, D as hashJson, E as jsonHasKeys, F as parseCorrectnessResponse, G as regexMatch, I as signManifest, J as summarizeBackendIntegrity, K as verifyCompletion, M as verifyManifest } from './pre-registration-mAnCugl9.js';
|
|
19
19
|
export { DataAcquisitionPlan, KnowledgeAcquisitionMode, KnowledgeBundle, KnowledgeFallbackPolicy, KnowledgeFreshness, KnowledgeImportance, KnowledgeReadinessReport, KnowledgeRecommendedAction, KnowledgeRequirement, KnowledgeRequirementCategory, KnowledgeResponsibleSurface, KnowledgeSensitivity, ScoreKnowledgeReadinessOptions, UserQuestion, acquisitionPlansForKnowledgeGaps, blockingKnowledgeEval, knowledgeReadinessTracePayload, scoreKnowledgeReadiness, userQuestionsForKnowledgeGaps } from './knowledge/index.js';
|
|
20
20
|
import { h as ReleaseConfidenceThresholds, f as ReleaseConfidenceScorecard } from './release-report-euXIV_Sk.js';
|
|
21
21
|
export { A as ActionableSideInfo, o as AsiSeverity, B as BootstrapOptions, a as BootstrapResult, J as JudgeReplayGateArgs, R as ReleaseConfidenceAxis, b as ReleaseConfidenceAxisName, c as ReleaseConfidenceInput, d as ReleaseConfidenceIssue, e as ReleaseConfidenceMetrics, g as ReleaseConfidenceStatus, i as ReleaseTraceEvidence, j as RenderReleaseReportOptions, V as Verdict, k as assertReleaseConfidence, l as bootstrapCi, m as evaluateReleaseConfidence, n as judgeReplayGate, r as renderReleaseReport } from './release-report-euXIV_Sk.js';
|
package/dist/index.js
CHANGED
package/dist/openapi.json
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
"openapi": "3.1.0",
|
|
3
3
|
"info": {
|
|
4
4
|
"title": "@tangle-network/agent-eval — wire protocol",
|
|
5
|
-
"version": "0.
|
|
5
|
+
"version": "0.92.0",
|
|
6
6
|
"description": "HTTP and stdio RPC interface to agent-eval. The TypeScript runtime is the source of truth; this spec is the contract that cross-language clients (Python, Rust, Go) generate from.\n\nWire-protocol version: 1.0.0. Bumps on breaking changes to request/response schemas.",
|
|
7
7
|
"contact": {
|
|
8
8
|
"name": "Tangle Network",
|
|
@@ -277,9 +277,15 @@ declare function createLlmCorrectnessChecker(tc: TCloud, opts?: LlmCorrectnessCh
|
|
|
277
277
|
* Deterministic `CorrectnessChecker` — the no-LLM counterpart to
|
|
278
278
|
* `createLlmCorrectnessChecker`. A produced item fulfils a requirement when its
|
|
279
279
|
* content is substantive (≥ `minContentLength` chars) AND recalls ≥ `minRecall`
|
|
280
|
-
* of the requirement title's significant tokens. No network
|
|
281
|
-
*
|
|
282
|
-
*
|
|
280
|
+
* of the requirement title's significant tokens. No network.
|
|
281
|
+
*
|
|
282
|
+
* Polarity-blind: token recall credits a negation that contains the
|
|
283
|
+
* requirement's tokens ("I will NOT produce the comparison" recalls every token
|
|
284
|
+
* of "produce the comparison"). The structural match stage is ALSO lexical, so
|
|
285
|
+
* pairing the two collapses to a single gameable gate. Use this only as an
|
|
286
|
+
* opt-in structural pre-filter or for tasks whose requirements have no polarity
|
|
287
|
+
* to invert; for produced-state grading the correctness checker MUST be semantic
|
|
288
|
+
* (`createLlmCorrectnessChecker`). See the anti-game fixtures in the test suite.
|
|
283
289
|
*/
|
|
284
290
|
declare function createTokenRecallChecker(opts?: {
|
|
285
291
|
minRecall?: number;
|
|
@@ -326,6 +332,7 @@ interface ProposalEventLike {
|
|
|
326
332
|
proposalId: string;
|
|
327
333
|
title: string;
|
|
328
334
|
status?: 'pending' | 'approved' | 'rejected';
|
|
335
|
+
content?: string;
|
|
329
336
|
}
|
|
330
337
|
/**
|
|
331
338
|
* The subset of runtime stream events `extractProducedState` consumes.
|
package/docs/concepts.md
CHANGED
|
@@ -183,6 +183,7 @@ release decision.
|
|
|
183
183
|
## Where to go next
|
|
184
184
|
|
|
185
185
|
- **Confused by "GEPA / HALO / trace analysis / drivers everywhere"?** → [self-improvement-map.md](./self-improvement-map.md) — one loop, four roles, the seven-driver catalog (production vs bench-only), and why `gepa-refine` is the same loop on a test bench.
|
|
186
|
+
- **Which `run*` primitive do I use, and how do I grade produced state?** → [eval-surface-map.md](./eval-surface-map.md) — the campaign/matrix/optimization/gate primitives as a pick-by-"use-when" table, plus the produced-state grading composition (verifyCompletion-as-judge — there is no persona-dispatch wrapper) and the in-band body contract.
|
|
186
187
|
- **Need the layman feature map?** → [feature-guide.md](./feature-guide.md) — what each primitive does, when to use it, integration patterns, and guardrails.
|
|
187
188
|
- **Just want to score a string against a rubric?** → [wire-protocol.md](./wire-protocol.md) — HTTP/RPC interface, pluggable from any language.
|
|
188
189
|
- **Need a reusable driver/worker/evaluator loop?** → [control-runtime.md](./control-runtime.md) — generic runtime plus coding, browser, computer-use, and research integration patterns.
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
# Eval surface map — which primitive, when
|
|
2
|
+
|
|
3
|
+
The eval surface is a small set of orthogonal primitives. They compose; they do
|
|
4
|
+
not overlap. If two seem interchangeable, read the "use when" — the distinction
|
|
5
|
+
is real and load-bearing. **Do not add a new wrapper to bridge two of these; the
|
|
6
|
+
composition point already exists (see Produced-state grading below).**
|
|
7
|
+
|
|
8
|
+
## The run\* primitives
|
|
9
|
+
|
|
10
|
+
| Primitive | Use when | Returns |
|
|
11
|
+
|---|---|---|
|
|
12
|
+
| `runCampaign` | The measurement primitive. Run a dispatch over scenarios × seeds × reps, score each with judges, aggregate. Caller owns the dispatch. | `CampaignResult` |
|
|
13
|
+
| `runEval` | The simplest preset over `runCampaign` — just score, no loop, no gate. The 80% "I want a scorecard" case. | `CampaignResult` |
|
|
14
|
+
| `runProfileMatrix` | Factor the SAME scenarios across N agent **profiles** (models / prompt variants), with RunRecord stamping + a real-backend integrity guard. | `RunRecord[]` |
|
|
15
|
+
| `runOptimization` | GENERATE: baseline → N generations of propose → measure → rank → promote. No release gate. | generations + winner |
|
|
16
|
+
| `runImprovementLoop` | The release-gate shell around `runOptimization`: adds a held-out re-score + a promotion gate (+ optional auto-PR). | gate decision + winner |
|
|
17
|
+
| `runEvalCampaign` | Inversion-of-control variant of `runCampaign` — the runner is handed a pre-wired trace/sink/emitter and integrity gating as a precondition. Use when you need full capture by construction. | `CampaignResult` + records |
|
|
18
|
+
| `runAgentMatrix` | The bare N-axis cartesian scheduler with concurrency control. The layer beneath the eval surface — reach for it only when you need raw scheduling, not eval semantics. | cell results |
|
|
19
|
+
|
|
20
|
+
Mental model: **measure** (`runCampaign`/`runEval`) → **factor** (`runProfileMatrix`) →
|
|
21
|
+
**generate** (`runOptimization`) → **gate** (`runImprovementLoop`). `runEvalCampaign`
|
|
22
|
+
is `runCampaign` with capture inverted; `runAgentMatrix` is the scheduler underneath.
|
|
23
|
+
|
|
24
|
+
Merging any two of these conflates distinct mental models (measure ≠ search ≠
|
|
25
|
+
release-gate). Keep them separate; pick by the table.
|
|
26
|
+
|
|
27
|
+
## Produced-state grading — there is NO persona-dispatch wrapper
|
|
28
|
+
|
|
29
|
+
To grade what an agent actually **produced** (filed the proposal, wrote the
|
|
30
|
+
artifact) rather than what it said, the composition point is a **judge that wraps
|
|
31
|
+
`verifyCompletion`** — not a dedicated runner. The pipeline:
|
|
32
|
+
|
|
33
|
+
```
|
|
34
|
+
runtime/app-tool events ──► extractProducedState(events) ──► ProducedState
|
|
35
|
+
│
|
|
36
|
+
verifyCompletion(taskGold, state, correctnessChecker)
|
|
37
|
+
│
|
|
38
|
+
inject as a JudgeConfig into runProfileMatrix / runCampaign
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
`extractProducedState` is a pure function over the produced-event stream; the
|
|
42
|
+
judge calls it inline. This is why **`runProducedStatePersonaDispatch` does not
|
|
43
|
+
exist and should not be built** — it would be a fourth layer over a composition
|
|
44
|
+
that is already one judge. (Archetype: `playback.ts` `scoreUserStory`.)
|
|
45
|
+
|
|
46
|
+
### The in-band body contract
|
|
47
|
+
|
|
48
|
+
Produced events carry their **body in-band** — the grader never reaches into a
|
|
49
|
+
product database to recover it:
|
|
50
|
+
|
|
51
|
+
- `artifact` events carry `content` (the persisted file body).
|
|
52
|
+
- `proposal_created` events carry `content` (the `submit_proposal` description) —
|
|
53
|
+
same role, same field name. A title-only filing omits it; a content-less
|
|
54
|
+
proposal is graded presence-only (and, by the completion oracle's rule, does
|
|
55
|
+
not count as a completed deliverable).
|
|
56
|
+
|
|
57
|
+
A consumer that finds itself re-fetching a deliverable's body from its own DB to
|
|
58
|
+
grade it is working around a thin event — fix the event (carry `content`), don't
|
|
59
|
+
add an enrichment band-aid.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@tangle-network/agent-eval",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.92.0",
|
|
4
4
|
"description": "Evaluate and improve AI agents from runs, traces, judges, and feedback. Compare candidates, cluster failures, measure lift, and gate releases.",
|
|
5
5
|
"homepage": "https://github.com/tangle-network/agent-eval#readme",
|
|
6
6
|
"repository": {
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"sources":["../src/completion-verifier.ts","../src/produced-state.ts","../src/agent-profile.ts"],"sourcesContent":["/**\n * Completion verifier — the task-completion oracle.\n *\n * Answers the only eval question that is not a proxy: did the agent actually\n * COMPLETE the task — produce every required deliverable, persisted and\n * correct — rather than describe what should be done. A fluent transcript\n * that never produces the artifact scores zero here.\n *\n * Per requirement, a two-stage check:\n * 1. Structural — a produced item (vault artifact / approved proposal /\n * tool call) of the right kind is matched against the requirement and\n * carries non-empty content. Deterministic; no LLM.\n * 2. Correctness — only if structurally present AND the matched item\n * carries content, one targeted check decides whether that item\n * actually fulfils the requirement. A hallucinated artifact fails here;\n * an absent one already failed stage 1.\n *\n * `completionRate` is satisfied / total. Quality dimensions are meaningless\n * on an incomplete task — callers gate on `fullyComplete` / `completionRate`\n * before scoring quality.\n */\n\nimport type { TCloud } from '@tangle-network/tcloud'\nimport type { Artifact } from './artifact-validator'\nimport type { DefaultVerdict } from './verdict'\n\n/** What kind of produced state can satisfy a requirement structurally. */\nexport type SatisfiedBy = 'artifact' | 'proposal' | 'tool-call' | 'any'\n\nexport interface CompletionRequirement {\n /** Stable id from the task gold (e.g. a persona's `expected_requirements[].req_id`). */\n reqId: string\n /** Human-readable description of the required deliverable. */\n title: string\n /** Optional kind/category hint, matched against a produced item's kind. */\n category?: string\n /** What produced state satisfies this requirement. Defaults to 'any'. */\n satisfiedBy?: SatisfiedBy\n}\n\nexport interface TaskGold {\n taskId: string\n requirements: CompletionRequirement[]\n}\n\nexport interface ProducedProposal {\n id: string\n title: string\n status: 'pending' | 'approved' | 'rejected'\n /** Optional persisted body — when present, enables a correctness check. */\n content?: string\n}\n\n/** Everything observable about what a run actually produced. */\nexport interface ProducedState {\n /** Persisted vault artifacts. Reuses the shared `Artifact` shape. */\n artifacts: Artifact[]\n /** Proposals / filings the agent created. */\n proposals: ProducedProposal[]\n /** Names of tools the agent invoked. */\n toolCalls: string[]\n}\n\nexport interface RequirementCheck {\n reqId: string\n title: string\n /** A produced item of the right kind matched the requirement, non-empty. */\n structurallyPresent: boolean\n /**\n * Whether the matched item actually fulfils the requirement. `null` when\n * not structurally present, or when the matched item carries no content\n * to assess.\n */\n correct: boolean | null\n /** structurallyPresent && correct !== false. */\n satisfied: boolean\n /** Human-readable evidence for the verdict. */\n evidence: string[]\n}\n\n/** Extends the substrate verdict spine: `valid` = `fullyComplete` and\n * `score` = `completionRate` — derived in `completionVerdict()`, the one\n * place those equalities hold by construction. */\nexport interface CompletionVerdict extends DefaultVerdict {\n taskId: string\n requirements: RequirementCheck[]\n /** satisfied / total requirements. */\n completionRate: number\n /** Every requirement satisfied. */\n fullyComplete: boolean\n}\n\n/**\n * Construct a `CompletionVerdict` from the per-requirement checks, deriving\n * `completionRate` / `fullyComplete` and the spine fields (`valid` =\n * `fullyComplete`, `score` = `completionRate`) in one place. Throws on zero\n * requirements — a verdict over nothing is a misconfiguration, mirroring\n * `verifyCompletion`'s gold-spec guard.\n */\nexport function completionVerdict(input: {\n taskId: string\n requirements: RequirementCheck[]\n}): CompletionVerdict {\n if (input.requirements.length === 0) {\n throw new Error(\n `completionVerdict: task '${input.taskId}' has no requirement checks — nothing to derive a verdict from`,\n )\n }\n const satisfiedCount = input.requirements.filter((r) => r.satisfied).length\n const completionRate = satisfiedCount / input.requirements.length\n const fullyComplete = satisfiedCount === input.requirements.length\n return {\n taskId: input.taskId,\n requirements: input.requirements,\n completionRate,\n fullyComplete,\n valid: fullyComplete,\n score: completionRate,\n }\n}\n\n/**\n * Decides whether a produced item's content actually fulfils a requirement.\n * Injected so the structural verifier stays pure and unit-testable; the\n * production implementation is `createLlmCorrectnessChecker`.\n */\nexport type CorrectnessChecker = (\n requirement: CompletionRequirement,\n content: string,\n) => Promise<{ correct: boolean; reason: string }>\n\nconst STOPWORDS = new Set([\n 'the',\n 'a',\n 'an',\n 'of',\n 'for',\n 'and',\n 'or',\n 'to',\n 'in',\n 'on',\n 'with',\n 'by',\n])\n\nconst MATCH_THRESHOLD = 0.5\nconst MIN_CONTENT_CHARS = 50\n\nfunction tokens(s: string): Set<string> {\n return new Set(\n s\n .toLowerCase()\n .split(/[^a-z0-9]+/)\n .filter((t) => t.length > 1 && !STOPWORDS.has(t)),\n )\n}\n\n/**\n * Recall of the requirement's tokens within a candidate's identifying text.\n * Recall, not Jaccard — a candidate's path/id legitimately carries extra\n * tokens the requirement does not name.\n */\nfunction tokenRecall(requirementText: string, candidateText: string): number {\n const req = tokens(requirementText)\n if (req.size === 0) return 0\n const cand = tokens(candidateText)\n let hit = 0\n for (const t of req) if (cand.has(t)) hit++\n return hit / req.size\n}\n\ninterface Candidate {\n reqIndex: number\n /** Unique key for a produced item — each item satisfies at most one requirement. */\n itemKey: string\n score: number\n evidence: string\n /** Content to correctness-check, or null when the matched item has none. */\n content: string | null\n}\n\nfunction artifactCandidates(\n req: CompletionRequirement,\n reqIndex: number,\n artifacts: Artifact[],\n): Candidate[] {\n const reqText = `${req.title} ${req.category ?? ''}`\n const out: Candidate[] = []\n artifacts.forEach((a, i) => {\n if ((a.content ?? '').trim().length < MIN_CONTENT_CHARS) return\n let score = tokenRecall(reqText, `${a.path ?? ''} ${a.kind}`)\n if (req.category && a.kind && req.category.toLowerCase() === a.kind.toLowerCase()) {\n score = Math.max(score, 1)\n }\n if (score < MATCH_THRESHOLD) return\n out.push({\n reqIndex,\n itemKey: `artifact:${i}`,\n score,\n evidence: `artifact '${a.path ?? a.kind}' matched (token recall ${score.toFixed(2)})`,\n content: a.content ?? null,\n })\n })\n return out\n}\n\nfunction proposalCandidates(\n req: CompletionRequirement,\n reqIndex: number,\n proposals: ProducedProposal[],\n): Candidate[] {\n const reqText = `${req.title} ${req.category ?? ''}`\n const out: Candidate[] = []\n for (const p of proposals) {\n // Pending or rejected work is not a completed deliverable.\n if (p.status !== 'approved') continue\n const score = tokenRecall(reqText, p.title)\n if (score < MATCH_THRESHOLD) continue\n const body = p.content ?? ''\n out.push({\n reqIndex,\n itemKey: `proposal:${p.id}`,\n score,\n evidence: `approved proposal '${p.title}' matched (token recall ${score.toFixed(2)})`,\n content: body.trim().length >= MIN_CONTENT_CHARS ? body : null,\n })\n }\n return out\n}\n\nfunction toolCallCandidates(\n req: CompletionRequirement,\n reqIndex: number,\n toolCalls: string[],\n): Candidate[] {\n const out: Candidate[] = []\n toolCalls.forEach((name, i) => {\n const score = tokenRecall(req.title, name)\n if (score < MATCH_THRESHOLD) return\n out.push({\n reqIndex,\n itemKey: `tool:${i}`,\n score,\n evidence: `tool call '${name}' matched (token recall ${score.toFixed(2)})`,\n content: null,\n })\n })\n return out\n}\n\n/**\n * Verify whether a run completed the task. `checkCorrectness` is injected —\n * `createLlmCorrectnessChecker` for production, a deterministic stub in tests.\n *\n * Throws on a gold spec with no requirements: an eval task that requires\n * nothing is a misconfiguration, not a vacuously-complete task.\n */\nexport async function verifyCompletion(\n gold: TaskGold,\n state: ProducedState,\n checkCorrectness: CorrectnessChecker,\n): Promise<CompletionVerdict> {\n if (gold.requirements.length === 0) {\n throw new Error(\n `verifyCompletion: task '${gold.taskId}' has no requirements — malformed gold spec`,\n )\n }\n\n // Collect every above-threshold (requirement, produced-item) candidate, then\n // assign greedily by descending score: each requirement and each produced\n // item is used at most once. One deliverable fulfils one requirement.\n const candidates: Candidate[] = []\n gold.requirements.forEach((req, i) => {\n const by = req.satisfiedBy ?? 'any'\n if (by === 'artifact' || by === 'any') {\n candidates.push(...artifactCandidates(req, i, state.artifacts))\n }\n if (by === 'proposal' || by === 'any') {\n candidates.push(...proposalCandidates(req, i, state.proposals))\n }\n if (by === 'tool-call' || by === 'any') {\n candidates.push(...toolCallCandidates(req, i, state.toolCalls))\n }\n })\n candidates.sort((a, b) => b.score - a.score)\n\n const assigned = new Map<number, Candidate>()\n const itemTaken = new Set<string>()\n for (const c of candidates) {\n if (assigned.has(c.reqIndex) || itemTaken.has(c.itemKey)) continue\n assigned.set(c.reqIndex, c)\n itemTaken.add(c.itemKey)\n }\n\n const requirements: RequirementCheck[] = []\n for (let i = 0; i < gold.requirements.length; i++) {\n const req = gold.requirements[i]!\n const match = assigned.get(i)\n const evidence: string[] = []\n let correct: boolean | null = null\n\n if (match) {\n evidence.push(match.evidence)\n if (match.content !== null) {\n const r = await checkCorrectness(req, match.content)\n correct = r.correct\n evidence.push(`correctness: ${r.correct ? 'pass' : 'fail'} — ${r.reason}`)\n } else {\n evidence.push('correctness: not assessed — matched item carries no content')\n }\n } else {\n const by = req.satisfiedBy ?? 'any'\n const kind = by === 'any' ? 'artifact/proposal/tool-call' : by\n evidence.push(`no produced ${kind} matched this requirement`)\n }\n\n const structurallyPresent = match !== undefined\n const satisfied = structurallyPresent && correct !== false\n requirements.push({\n reqId: req.reqId,\n title: req.title,\n structurallyPresent,\n correct,\n satisfied,\n evidence,\n })\n }\n\n return completionVerdict({ taskId: gold.taskId, requirements })\n}\n\nexport interface LlmCorrectnessCheckerOpts {\n model?: string\n /** Max chars of artifact content sent to the checker. */\n maxContentChars?: number\n}\n\n/** Parse the correctness checker's model response. Fails loud on a bad shape. */\nexport function parseCorrectnessResponse(raw: string): { correct: boolean; reason: string } {\n const match = raw.match(/\\{[\\s\\S]*\\}/)\n if (!match) {\n throw new Error(`correctness checker: no JSON object in model response: ${raw.slice(0, 200)}`)\n }\n const parsed = JSON.parse(match[0]) as { correct?: unknown; reason?: unknown }\n if (typeof parsed.correct !== 'boolean') {\n throw new Error(`correctness checker: 'correct' is not a boolean in: ${match[0].slice(0, 200)}`)\n }\n return { correct: parsed.correct, reason: typeof parsed.reason === 'string' ? parsed.reason : '' }\n}\n\n/**\n * Production `CorrectnessChecker` — one LLM call per matched artifact,\n * deterministic (temperature 0), structured JSON out. Judges fulfilment\n * only: a plan, a gesture, or a description of what should be done does not\n * fulfil a requirement — the artifact must BE the deliverable.\n */\nexport function createLlmCorrectnessChecker(\n tc: TCloud,\n opts: LlmCorrectnessCheckerOpts = {},\n): CorrectnessChecker {\n const model = opts.model ?? 'claude-sonnet-4-6'\n const maxContentChars = opts.maxContentChars ?? 8000\n return async (requirement, content) => {\n const resp = await tc.chat({\n model,\n messages: [\n {\n role: 'system',\n content:\n 'You verify whether a produced work artifact actually fulfils a stated requirement. Judge fulfilment only — is the deliverable substantively present and on-point — not polish. A plan to do it later, a vague gesture, or a description of what should be done does NOT fulfil a requirement; the artifact must BE the deliverable. Respond with a single JSON object: {\"correct\": boolean, \"reason\": string (<= 30 words)}.',\n },\n {\n role: 'user',\n content: `Requirement: ${requirement.title}\\n${\n requirement.category ? `Category: ${requirement.category}\\n` : ''\n }\\nProduced artifact:\\n${content.slice(0, maxContentChars)}`,\n },\n ],\n temperature: 0,\n maxTokens: 200,\n })\n const raw =\n (resp as { choices?: { message?: { content?: string } }[] }).choices?.[0]?.message?.content ??\n ''\n return parseCorrectnessResponse(raw)\n }\n}\n\n/** Stopwords for requirement-title tokenization — drops the imperative verbs\n * ('review', 'update', …) common to deliverable titles so recall keys on the\n * substantive nouns, not the boilerplate ask. */\nconst TITLE_STOPWORDS = new Set([\n 'the',\n 'a',\n 'an',\n 'and',\n 'or',\n 'for',\n 'to',\n 'of',\n 'in',\n 'on',\n 'with',\n 'review',\n 'update',\n 'new',\n 'proposed',\n])\n\n/**\n * Deterministic `CorrectnessChecker` — the no-LLM counterpart to\n * `createLlmCorrectnessChecker`. A produced item fulfils a requirement when its\n * content is substantive (≥ `minContentLength` chars) AND recalls ≥ `minRecall`\n * of the requirement title's significant tokens. No network — the default gate\n * for apps and tests without an LLM judge. Pass to `verifyCompletion` as the\n * checker.\n */\nexport function createTokenRecallChecker(\n opts: { minRecall?: number; minContentLength?: number } = {},\n): CorrectnessChecker {\n const minRecall = opts.minRecall ?? 0.5\n const minLen = opts.minContentLength ?? 120\n return async (requirement, content) => {\n const body = content.trim()\n if (body.length < minLen)\n return {\n correct: false,\n reason: `content too thin (${body.length} chars) to be the deliverable`,\n }\n const titleTokens = requirement.title\n .toLowerCase()\n .split(/[^a-z0-9]+/)\n .filter((t) => t.length > 2 && !TITLE_STOPWORDS.has(t))\n if (titleTokens.length === 0)\n return {\n correct: true,\n reason: 'requirement title has no significant tokens — structural match accepted',\n }\n const lower = body.toLowerCase()\n const hits = titleTokens.filter((t) => lower.includes(t)).length\n const recall = hits / titleTokens.length\n return recall >= minRecall\n ? {\n correct: true,\n reason: `content recalls ${hits}/${titleTokens.length} requirement tokens`,\n }\n : {\n correct: false,\n reason: `content recalls only ${hits}/${titleTokens.length} requirement tokens`,\n }\n }\n}\n","/**\n * Produced-state extraction — normalize a run's runtime event stream into the\n * typed `ProducedState` the completion oracle consumes.\n *\n * `ProducedState` answers \"what did the agent actually produce\" — vault\n * artifacts, proposals, tool calls. The runtime emits these as a stream of\n * events; this module is the single normalization point from that stream to\n * the shape `verifyCompletion` expects.\n *\n * Input is structurally typed (`RuntimeEventLike`) so this module does not\n * depend on agent-runtime — agent-runtime's `RuntimeStreamEvent` satisfies it\n * structurally. The `content` on `ArtifactEventLike` and the whole\n * `proposal_created` variant are the runtime-side enrichments this contract\n * requires; the runtime emits them, this module consumes them.\n */\n\nimport type { Artifact } from './artifact-validator'\nimport type { ProducedProposal, ProducedState } from './completion-verifier'\n\n/** A tool the agent invoked. */\nexport interface ToolCallEventLike {\n type: 'tool_call'\n toolName: string\n}\n\n/**\n * An artifact the agent produced. `content` is the enriched field — the\n * runtime's base `artifact` event carries only metadata; the completion\n * oracle needs the body to verify the deliverable, so the runtime emits it.\n */\nexport interface ArtifactEventLike {\n type: 'artifact'\n artifactId: string\n name?: string\n mimeType?: string\n uri?: string\n content?: string\n}\n\n/** A proposal / filing the agent created. */\nexport interface ProposalEventLike {\n type: 'proposal_created'\n proposalId: string\n title: string\n status?: 'pending' | 'approved' | 'rejected'\n}\n\n/**\n * The subset of runtime stream events `extractProducedState` consumes.\n * agent-runtime's full `RuntimeStreamEvent` union satisfies this structurally;\n * the `{ type: string }` catch-all keeps the input permissive so callers can\n * pass the whole unfiltered telemetry stream — unrecognized events are skipped.\n */\nexport type RuntimeEventLike =\n | ToolCallEventLike\n | ArtifactEventLike\n | ProposalEventLike\n | { type: string }\n\nfunction artifactKind(mimeType: string | undefined): string {\n if (!mimeType) return 'file'\n if (mimeType.includes('json')) return 'json'\n if (mimeType.startsWith('text/')) return 'text'\n return 'file'\n}\n\n/**\n * Normalize a run's runtime event stream into `ProducedState`.\n *\n * Pure and total — unrecognized event types are skipped. `toolCalls` is\n * deduplicated by name in first-seen order (completion cares about a tool's\n * presence, not its call count). An artifact with neither a name nor a uri\n * still yields an entry keyed by its `artifactId` so it is never silently\n * dropped; an artifact with no `content` yields empty content, which the\n * completion oracle's structural check then rejects on its own.\n */\nexport function extractProducedState(events: readonly RuntimeEventLike[]): ProducedState {\n const artifacts: Artifact[] = []\n const proposals: ProducedProposal[] = []\n const toolCalls: string[] = []\n const seenTools = new Set<string>()\n\n for (const ev of events) {\n if (ev.type === 'tool_call') {\n const name = (ev as ToolCallEventLike).toolName\n if (name && !seenTools.has(name)) {\n seenTools.add(name)\n toolCalls.push(name)\n }\n } else if (ev.type === 'artifact') {\n const a = ev as ArtifactEventLike\n artifacts.push({\n kind: artifactKind(a.mimeType),\n path: a.name ?? a.uri ?? a.artifactId,\n content: a.content ?? '',\n })\n } else if (ev.type === 'proposal_created') {\n const p = ev as ProposalEventLike\n proposals.push({ id: p.proposalId, title: p.title, status: p.status ?? 'pending' })\n }\n }\n\n return { artifacts, proposals, toolCalls }\n}\n","/**\n * @stable\n *\n * AgentProfile — the eval harness's unit of variation.\n *\n * A profile pins everything that changes agent behaviour for a benchmark\n * cell: the model, the active skills, the prompt version, the available\n * tools. Vary the profile — swap a model, add a skill — and re-run the suite\n * to benchmark the change. The scorecard keys a cell on\n * `(scenarioId, profileHash)`, so the model is not a separate axis: it lives\n * inside the profile, and two profiles with the same model but different\n * skills are different cells.\n *\n * `agentProfileHash` is the profile's behaviour identity. Two profiles that\n * produce the same agent behaviour share a hash (and a scorecard cell);\n * reordering `skills` or `tools` does not change it; the human-facing `id`\n * label does not affect it.\n */\n\nimport { createHash } from 'node:crypto'\nimport { ValidationError } from './errors'\nimport { canonicalize } from './pre-registration'\n\nexport interface AgentProfile {\n /** Human-facing label, e.g. `sonnet-legal-skills-v3`. Not part of the hash. */\n id: string\n /** Model snapshot id this profile pins, e.g. `claude-sonnet-4-6@2025-04-15`. */\n model: string\n /** Skill ids/versions active in this profile — the primary behaviour lever. */\n skills?: string[]\n /** Prompt version identifier. */\n promptVersion?: string\n /** Tool ids available to the agent. */\n tools?: string[]\n /** Any other behaviour-bearing knobs that should fingerprint into the hash. */\n metadata?: Record<string, string | number | boolean>\n}\n\n/**\n * Deterministic behaviour identity of a profile — a sha256 over the\n * behaviour-bearing fields. `skills` and `tools` are order-insensitive; the\n * `id` label is excluded. Throws on a profile with no `model` — an unkeyable\n * profile must fail loud rather than collapse into a blank-model cell.\n */\nexport function agentProfileHash(profile: AgentProfile): string {\n if (typeof profile.model !== 'string' || profile.model.trim().length === 0) {\n throw new ValidationError(`AgentProfile \"${profile.id}\" has no model — cannot hash`)\n }\n const behaviour = {\n model: profile.model.trim(),\n skills: [...(profile.skills ?? [])].sort(),\n promptVersion: profile.promptVersion ?? null,\n tools: [...(profile.tools ?? [])].sort(),\n metadata: profile.metadata ?? {},\n }\n return createHash('sha256')\n .update(JSON.stringify(canonicalize(behaviour)))\n .digest('hex')\n}\n"],"mappings":";;;;;;;;AAmGO,SAAS,kBAAkB,OAGZ;AACpB,MAAI,MAAM,aAAa,WAAW,GAAG;AACnC,UAAM,IAAI;AAAA,MACR,4BAA4B,MAAM,MAAM;AAAA,IAC1C;AAAA,EACF;AACA,QAAM,iBAAiB,MAAM,aAAa,OAAO,CAAC,MAAM,EAAE,SAAS,EAAE;AACrE,QAAM,iBAAiB,iBAAiB,MAAM,aAAa;AAC3D,QAAM,gBAAgB,mBAAmB,MAAM,aAAa;AAC5D,SAAO;AAAA,IACL,QAAQ,MAAM;AAAA,IACd,cAAc,MAAM;AAAA,IACpB;AAAA,IACA;AAAA,IACA,OAAO;AAAA,IACP,OAAO;AAAA,EACT;AACF;AAYA,IAAM,YAAY,oBAAI,IAAI;AAAA,EACxB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,CAAC;AAED,IAAM,kBAAkB;AACxB,IAAM,oBAAoB;AAE1B,SAAS,OAAO,GAAwB;AACtC,SAAO,IAAI;AAAA,IACT,EACG,YAAY,EACZ,MAAM,YAAY,EAClB,OAAO,CAAC,MAAM,EAAE,SAAS,KAAK,CAAC,UAAU,IAAI,CAAC,CAAC;AAAA,EACpD;AACF;AAOA,SAAS,YAAY,iBAAyB,eAA+B;AAC3E,QAAM,MAAM,OAAO,eAAe;AAClC,MAAI,IAAI,SAAS,EAAG,QAAO;AAC3B,QAAM,OAAO,OAAO,aAAa;AACjC,MAAI,MAAM;AACV,aAAW,KAAK,IAAK,KAAI,KAAK,IAAI,CAAC,EAAG;AACtC,SAAO,MAAM,IAAI;AACnB;AAYA,SAAS,mBACP,KACA,UACA,WACa;AACb,QAAM,UAAU,GAAG,IAAI,KAAK,IAAI,IAAI,YAAY,EAAE;AAClD,QAAM,MAAmB,CAAC;AAC1B,YAAU,QAAQ,CAAC,GAAG,MAAM;AAC1B,SAAK,EAAE,WAAW,IAAI,KAAK,EAAE,SAAS,kBAAmB;AACzD,QAAI,QAAQ,YAAY,SAAS,GAAG,EAAE,QAAQ,EAAE,IAAI,EAAE,IAAI,EAAE;AAC5D,QAAI,IAAI,YAAY,EAAE,QAAQ,IAAI,SAAS,YAAY,MAAM,EAAE,KAAK,YAAY,GAAG;AACjF,cAAQ,KAAK,IAAI,OAAO,CAAC;AAAA,IAC3B;AACA,QAAI,QAAQ,gBAAiB;AAC7B,QAAI,KAAK;AAAA,MACP;AAAA,MACA,SAAS,YAAY,CAAC;AAAA,MACtB;AAAA,MACA,UAAU,aAAa,EAAE,QAAQ,EAAE,IAAI,2BAA2B,MAAM,QAAQ,CAAC,CAAC;AAAA,MAClF,SAAS,EAAE,WAAW;AAAA,IACxB,CAAC;AAAA,EACH,CAAC;AACD,SAAO;AACT;AAEA,SAAS,mBACP,KACA,UACA,WACa;AACb,QAAM,UAAU,GAAG,IAAI,KAAK,IAAI,IAAI,YAAY,EAAE;AAClD,QAAM,MAAmB,CAAC;AAC1B,aAAW,KAAK,WAAW;AAEzB,QAAI,EAAE,WAAW,WAAY;AAC7B,UAAM,QAAQ,YAAY,SAAS,EAAE,KAAK;AAC1C,QAAI,QAAQ,gBAAiB;AAC7B,UAAM,OAAO,EAAE,WAAW;AAC1B,QAAI,KAAK;AAAA,MACP;AAAA,MACA,SAAS,YAAY,EAAE,EAAE;AAAA,MACzB;AAAA,MACA,UAAU,sBAAsB,EAAE,KAAK,2BAA2B,MAAM,QAAQ,CAAC,CAAC;AAAA,MAClF,SAAS,KAAK,KAAK,EAAE,UAAU,oBAAoB,OAAO;AAAA,IAC5D,CAAC;AAAA,EACH;AACA,SAAO;AACT;AAEA,SAAS,mBACP,KACA,UACA,WACa;AACb,QAAM,MAAmB,CAAC;AAC1B,YAAU,QAAQ,CAAC,MAAM,MAAM;AAC7B,UAAM,QAAQ,YAAY,IAAI,OAAO,IAAI;AACzC,QAAI,QAAQ,gBAAiB;AAC7B,QAAI,KAAK;AAAA,MACP;AAAA,MACA,SAAS,QAAQ,CAAC;AAAA,MAClB;AAAA,MACA,UAAU,cAAc,IAAI,2BAA2B,MAAM,QAAQ,CAAC,CAAC;AAAA,MACvE,SAAS;AAAA,IACX,CAAC;AAAA,EACH,CAAC;AACD,SAAO;AACT;AASA,eAAsB,iBACpB,MACA,OACA,kBAC4B;AAC5B,MAAI,KAAK,aAAa,WAAW,GAAG;AAClC,UAAM,IAAI;AAAA,MACR,2BAA2B,KAAK,MAAM;AAAA,IACxC;AAAA,EACF;AAKA,QAAM,aAA0B,CAAC;AACjC,OAAK,aAAa,QAAQ,CAAC,KAAK,MAAM;AACpC,UAAM,KAAK,IAAI,eAAe;AAC9B,QAAI,OAAO,cAAc,OAAO,OAAO;AACrC,iBAAW,KAAK,GAAG,mBAAmB,KAAK,GAAG,MAAM,SAAS,CAAC;AAAA,IAChE;AACA,QAAI,OAAO,cAAc,OAAO,OAAO;AACrC,iBAAW,KAAK,GAAG,mBAAmB,KAAK,GAAG,MAAM,SAAS,CAAC;AAAA,IAChE;AACA,QAAI,OAAO,eAAe,OAAO,OAAO;AACtC,iBAAW,KAAK,GAAG,mBAAmB,KAAK,GAAG,MAAM,SAAS,CAAC;AAAA,IAChE;AAAA,EACF,CAAC;AACD,aAAW,KAAK,CAAC,GAAG,MAAM,EAAE,QAAQ,EAAE,KAAK;AAE3C,QAAM,WAAW,oBAAI,IAAuB;AAC5C,QAAM,YAAY,oBAAI,IAAY;AAClC,aAAW,KAAK,YAAY;AAC1B,QAAI,SAAS,IAAI,EAAE,QAAQ,KAAK,UAAU,IAAI,EAAE,OAAO,EAAG;AAC1D,aAAS,IAAI,EAAE,UAAU,CAAC;AAC1B,cAAU,IAAI,EAAE,OAAO;AAAA,EACzB;AAEA,QAAM,eAAmC,CAAC;AAC1C,WAAS,IAAI,GAAG,IAAI,KAAK,aAAa,QAAQ,KAAK;AACjD,UAAM,MAAM,KAAK,aAAa,CAAC;AAC/B,UAAM,QAAQ,SAAS,IAAI,CAAC;AAC5B,UAAM,WAAqB,CAAC;AAC5B,QAAI,UAA0B;AAE9B,QAAI,OAAO;AACT,eAAS,KAAK,MAAM,QAAQ;AAC5B,UAAI,MAAM,YAAY,MAAM;AAC1B,cAAM,IAAI,MAAM,iBAAiB,KAAK,MAAM,OAAO;AACnD,kBAAU,EAAE;AACZ,iBAAS,KAAK,gBAAgB,EAAE,UAAU,SAAS,MAAM,WAAM,EAAE,MAAM,EAAE;AAAA,MAC3E,OAAO;AACL,iBAAS,KAAK,kEAA6D;AAAA,MAC7E;AAAA,IACF,OAAO;AACL,YAAM,KAAK,IAAI,eAAe;AAC9B,YAAM,OAAO,OAAO,QAAQ,gCAAgC;AAC5D,eAAS,KAAK,eAAe,IAAI,2BAA2B;AAAA,IAC9D;AAEA,UAAM,sBAAsB,UAAU;AACtC,UAAM,YAAY,uBAAuB,YAAY;AACrD,iBAAa,KAAK;AAAA,MAChB,OAAO,IAAI;AAAA,MACX,OAAO,IAAI;AAAA,MACX;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF,CAAC;AAAA,EACH;AAEA,SAAO,kBAAkB,EAAE,QAAQ,KAAK,QAAQ,aAAa,CAAC;AAChE;AASO,SAAS,yBAAyB,KAAmD;AAC1F,QAAM,QAAQ,IAAI,MAAM,aAAa;AACrC,MAAI,CAAC,OAAO;AACV,UAAM,IAAI,MAAM,0DAA0D,IAAI,MAAM,GAAG,GAAG,CAAC,EAAE;AAAA,EAC/F;AACA,QAAM,SAAS,KAAK,MAAM,MAAM,CAAC,CAAC;AAClC,MAAI,OAAO,OAAO,YAAY,WAAW;AACvC,UAAM,IAAI,MAAM,uDAAuD,MAAM,CAAC,EAAE,MAAM,GAAG,GAAG,CAAC,EAAE;AAAA,EACjG;AACA,SAAO,EAAE,SAAS,OAAO,SAAS,QAAQ,OAAO,OAAO,WAAW,WAAW,OAAO,SAAS,GAAG;AACnG;AAQO,SAAS,4BACd,IACA,OAAkC,CAAC,GACf;AACpB,QAAM,QAAQ,KAAK,SAAS;AAC5B,QAAM,kBAAkB,KAAK,mBAAmB;AAChD,SAAO,OAAO,aAAa,YAAY;AACrC,UAAM,OAAO,MAAM,GAAG,KAAK;AAAA,MACzB;AAAA,MACA,UAAU;AAAA,QACR;AAAA,UACE,MAAM;AAAA,UACN,SACE;AAAA,QACJ;AAAA,QACA;AAAA,UACE,MAAM;AAAA,UACN,SAAS,gBAAgB,YAAY,KAAK;AAAA,EACxC,YAAY,WAAW,aAAa,YAAY,QAAQ;AAAA,IAAO,EACjE;AAAA;AAAA,EAAyB,QAAQ,MAAM,GAAG,eAAe,CAAC;AAAA,QAC5D;AAAA,MACF;AAAA,MACA,aAAa;AAAA,MACb,WAAW;AAAA,IACb,CAAC;AACD,UAAM,MACH,KAA4D,UAAU,CAAC,GAAG,SAAS,WACpF;AACF,WAAO,yBAAyB,GAAG;AAAA,EACrC;AACF;AAKA,IAAM,kBAAkB,oBAAI,IAAI;AAAA,EAC9B;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,CAAC;AAUM,SAAS,yBACd,OAA0D,CAAC,GACvC;AACpB,QAAM,YAAY,KAAK,aAAa;AACpC,QAAM,SAAS,KAAK,oBAAoB;AACxC,SAAO,OAAO,aAAa,YAAY;AACrC,UAAM,OAAO,QAAQ,KAAK;AAC1B,QAAI,KAAK,SAAS;AAChB,aAAO;AAAA,QACL,SAAS;AAAA,QACT,QAAQ,qBAAqB,KAAK,MAAM;AAAA,MAC1C;AACF,UAAM,cAAc,YAAY,MAC7B,YAAY,EACZ,MAAM,YAAY,EAClB,OAAO,CAAC,MAAM,EAAE,SAAS,KAAK,CAAC,gBAAgB,IAAI,CAAC,CAAC;AACxD,QAAI,YAAY,WAAW;AACzB,aAAO;AAAA,QACL,SAAS;AAAA,QACT,QAAQ;AAAA,MACV;AACF,UAAM,QAAQ,KAAK,YAAY;AAC/B,UAAM,OAAO,YAAY,OAAO,CAAC,MAAM,MAAM,SAAS,CAAC,CAAC,EAAE;AAC1D,UAAM,SAAS,OAAO,YAAY;AAClC,WAAO,UAAU,YACb;AAAA,MACE,SAAS;AAAA,MACT,QAAQ,mBAAmB,IAAI,IAAI,YAAY,MAAM;AAAA,IACvD,IACA;AAAA,MACE,SAAS;AAAA,MACT,QAAQ,wBAAwB,IAAI,IAAI,YAAY,MAAM;AAAA,IAC5D;AAAA,EACN;AACF;;;ACzYA,SAAS,aAAa,UAAsC;AAC1D,MAAI,CAAC,SAAU,QAAO;AACtB,MAAI,SAAS,SAAS,MAAM,EAAG,QAAO;AACtC,MAAI,SAAS,WAAW,OAAO,EAAG,QAAO;AACzC,SAAO;AACT;AAYO,SAAS,qBAAqB,QAAoD;AACvF,QAAM,YAAwB,CAAC;AAC/B,QAAM,YAAgC,CAAC;AACvC,QAAM,YAAsB,CAAC;AAC7B,QAAM,YAAY,oBAAI,IAAY;AAElC,aAAW,MAAM,QAAQ;AACvB,QAAI,GAAG,SAAS,aAAa;AAC3B,YAAM,OAAQ,GAAyB;AACvC,UAAI,QAAQ,CAAC,UAAU,IAAI,IAAI,GAAG;AAChC,kBAAU,IAAI,IAAI;AAClB,kBAAU,KAAK,IAAI;AAAA,MACrB;AAAA,IACF,WAAW,GAAG,SAAS,YAAY;AACjC,YAAM,IAAI;AACV,gBAAU,KAAK;AAAA,QACb,MAAM,aAAa,EAAE,QAAQ;AAAA,QAC7B,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE;AAAA,QAC3B,SAAS,EAAE,WAAW;AAAA,MACxB,CAAC;AAAA,IACH,WAAW,GAAG,SAAS,oBAAoB;AACzC,YAAM,IAAI;AACV,gBAAU,KAAK,EAAE,IAAI,EAAE,YAAY,OAAO,EAAE,OAAO,QAAQ,EAAE,UAAU,UAAU,CAAC;AAAA,IACpF;AAAA,EACF;AAEA,SAAO,EAAE,WAAW,WAAW,UAAU;AAC3C;;;ACpFA,SAAS,kBAAkB;AAyBpB,SAAS,iBAAiB,SAA+B;AAC9D,MAAI,OAAO,QAAQ,UAAU,YAAY,QAAQ,MAAM,KAAK,EAAE,WAAW,GAAG;AAC1E,UAAM,IAAI,gBAAgB,iBAAiB,QAAQ,EAAE,mCAA8B;AAAA,EACrF;AACA,QAAM,YAAY;AAAA,IAChB,OAAO,QAAQ,MAAM,KAAK;AAAA,IAC1B,QAAQ,CAAC,GAAI,QAAQ,UAAU,CAAC,CAAE,EAAE,KAAK;AAAA,IACzC,eAAe,QAAQ,iBAAiB;AAAA,IACxC,OAAO,CAAC,GAAI,QAAQ,SAAS,CAAC,CAAE,EAAE,KAAK;AAAA,IACvC,UAAU,QAAQ,YAAY,CAAC;AAAA,EACjC;AACA,SAAO,WAAW,QAAQ,EACvB,OAAO,KAAK,UAAU,aAAa,SAAS,CAAC,CAAC,EAC9C,OAAO,KAAK;AACjB;","names":[]}
|