@tangle-network/agent-eval 0.144.11 → 0.144.13

package/dist/produced-state-DU79a81m.js

@@ -0,0 +1,586 @@
+import { s as ValidationError } from "./errors-Dngq5h35.js";
+import { t as canonicalize } from "./pre-registration-DakwTRXk.js";
+import { I as recoverTruncatedJson, n as JudgeParseError } from "./llm-judge-dZ8P6nGI.js";
+import { i as CostLedger } from "./cost-ledger-BSe92yAV.js";
+import { t as certificationEvidenceDigest } from "./verdict-BndeTAh_.js";
+import { f as maximumChargeForLlmRequest, g as assertServedModel, h as ModelSubstitutionError, l as costReceiptFromLlm, u as costReceiptFromLlmError } from "./llm-client-d0-2TT1g.js";
+import { createHash, randomUUID } from "node:crypto";
+import { harnessSupportsModel } from "@tangle-network/agent-interface";
+//#region src/agent-profile.ts
+/**
+* The agentic coding harnesses an eval sweeps by default — the ones we care about
+* ranking. This is the SINGLE source of that list; consumers import it instead of
+* re-declaring their own (a re-declared list is how the fleet drifts). Pass an
+* explicit `harnesses` (e.g. `harnessTypeSchema.options` for literally every known
+* harness) to widen beyond these.
+*/
+const CODING_HARNESSES = [
+	"opencode",
+	"claude-code",
+	"codex",
+	"kimi-code"
+];
+/** Model sentinel for a vendor-locked harness that supports none of the swept models:
+*  it carries no provider prefix, so `harnessSupportsModel` accepts it and the harness
+*  resolves it to its own native default model at runtime (e.g. kimi-code → its Kimi
+*  model). Lets `expandProfileAxes` snap-instead-of-drop without a per-harness flagship
+*  table that would rot as router catalogs change. */
+const HARNESS_NATIVE_MODEL = "default";
+/**
+* Expand a base profile across the harness × model matrix into the `AgentProfile[]`
+* that `runProfileMatrix` / `selfImprove` score — the ONE place "which harnesses ×
+* which models do we evaluate" lives, so no product hand-rolls its own harness list
+* or column→profile mapping (the pattern that let those copies drift and silently
+* break the harness pivot).
+*
+* Each cell clones `base`, sets the canonical top-level `harness` and `model.default`,
+* and stamps `metadata.harness` + `metadata.harnessModel` for matrix grouping. Both
+* metadata fields are hash-bearing, so every cell gets a distinct `agentProfileId` row
+* and results join back by harness/model via {@link harnessAxisOf} with no
+* hand-recomputed key. A vendor-locked harness snaps to its family's swept models — or
+* its native default ({@link HARNESS_NATIVE_MODEL}) when it supports none — so every
+* requested harness runs; `keepIncompatible` forces every pair verbatim.
+*
+* Omit `harnesses`/`models` to sweep the full default set — the "turn it on for
+* everything we care about" switch, identical in shape whether one harness or all.
+*/
+function expandProfileAxes(spec) {
+	const harnesses = spec.harnesses ?? CODING_HARNESSES;
+	if (harnesses.length === 0) throw new ValidationError("expandProfileAxes: no harnesses to sweep");
+	const baseModel = spec.base.model?.default;
+	const models = spec.models ?? (baseModel ? [baseModel] : []);
+	if (models.length === 0) throw new ValidationError("expandProfileAxes: no models to sweep — base profile has no model.default and none were supplied");
+	const out = [];
+	const seen = /* @__PURE__ */ new Set();
+	for (const harness of harnesses) {
+		const supported = spec.keepIncompatible ? models : models.filter((model) => harnessSupportsModel(harness, model));
+		const effective = supported.length > 0 ? supported : [HARNESS_NATIVE_MODEL];
+		for (const model of effective) {
+			const profile = {
+				...spec.base,
+				harness,
+				name: `${spec.base.name ?? "agent"}/${harness}/${model}`,
+				model: {
+					...spec.base.model,
+					default: model
+				},
+				metadata: {
+					...spec.base.metadata ?? {},
+					harness,
+					harnessModel: model
+				}
+			};
+			const id = agentProfileId(profile);
+			if (seen.has(id)) continue;
+			seen.add(id);
+			out.push(profile);
+		}
+	}
+	if (out.length === 0) throw new ValidationError(`expandProfileAxes: produced no profiles (harnesses=[${harnesses.join(", ")}], models=[${models.join(", ")}]).`);
+	return out;
+}
+/**
+* Read the (harness, model) a matrix cell ran under, off a profile or a result row's
+* profile — the join-back for a `byHarness` pivot. Returns undefined when the profile
+* wasn't produced by {@link expandProfileAxes}. Callers group `result.byProfile` by
+* this instead of recomputing an id (recomputing the wrong key is what broke the pivot
+* in the hand-rolled copies).
+*/
+function harnessAxisOf(profile) {
+	const m = profile.metadata;
+	const harness = m?.harness;
+	const model = m?.harnessModel;
+	if (typeof harness === "string" && typeof model === "string") return {
+		harness,
+		model
+	};
+}
+/**
+* Collision-resistant, path-safe, human-readable profile id for eval artifacts.
+* Scorecard joins still use `agentProfileHash`; this id is for run ids, matrix
+* keys, and directory names where two profiles must not collapse onto one row.
+* The suffix is the first 64 bits of the behaviour hash, enough for ordinary
+* eval matrices while keeping filenames readable.
+*/
+function agentProfileId(profile) {
+	return `${pathSafeProfileLabel(agentProfileDisplayLabel(profile)) ?? "profile"}-${agentProfileHash(profile).slice(0, 16)}`;
+}
+/**
+* Model snapshot used for `RunRecord.model`. Eval surfaces require a concrete
+* model id because run records reject bare/missing model aliases.
+*/
+function agentProfileModelId(profile) {
+	const model = profile.model?.default?.trim();
+	if (!model) throw new ValidationError(`AgentProfile "${agentProfileDisplayLabel(profile) ?? "unnamed profile"}" has no model.default — cannot record eval run`);
+	return model;
+}
+function agentProfileDisplayLabel(profile) {
+	return profile.name?.trim() || profile.version?.trim() || void 0;
+}
+function pathSafeProfileLabel(label) {
+	return label?.trim().replace(/[^A-Za-z0-9._-]+/g, "-").replace(/-+/g, "-").replace(/^-|-$/g, "") || void 0;
+}
+function compact(input) {
+	const out = {};
+	for (const [key, value] of Object.entries(input)) if (value !== void 0) out[key] = value;
+	return out;
+}
+/**
+* Deterministic behaviour identity for the canonical
+* `@tangle-network/agent-interface` AgentProfile.
+*
+* `name` and `description` are labels and do not affect the hash. Profile
+* `version`, prompt, model hints, tools, resources, hooks, modes, permissions,
+* and extensions do affect the hash. Resource array order is hash-bearing
+* because mount order can change agent behaviour. Undefined fields are treated
+* as absent; explicit `null` fields remain hash-bearing.
+*/
+function agentProfileHash(profile) {
+	const model = agentProfileModelId(profile);
+	const behaviour = {
+		...profile,
+		name: void 0,
+		description: void 0,
+		tags: profile.tags ? [...profile.tags].sort() : void 0,
+		model: compact({
+			...profile.model,
+			default: model
+		})
+	};
+	return createHash("sha256").update(JSON.stringify(canonicalize(behaviour))).digest("hex");
+}
+//#endregion
+//#region src/completion-verifier.ts
+/**
+* Completion verifier — the task-completion oracle.
+*
+* Answers the only eval question that is not a proxy: did the agent actually
+* COMPLETE the task — produce every required deliverable, persisted and
+* correct — rather than describe what should be done. A fluent transcript
+* that never produces the artifact scores zero here.
+*
+* Per requirement, a two-stage check:
+*   1. Structural — a produced item (vault artifact / approved proposal /
+*      tool call) of the right kind is matched against the requirement and
+*      carries non-empty content. Deterministic; no LLM.
+*   2. Correctness — only if structurally present AND the matched item
+*      carries content, one targeted check decides whether that item
+*      actually fulfils the requirement. A hallucinated artifact fails here;
+*      an absent one already failed stage 1.
+*
+* `completionRate` is satisfied / MEASURABLE requirements (unmeasured rows —
+* checker failures — are excluded from the denominator, never scored as
+* zeros). Quality dimensions are meaningless on an incomplete task — callers
+* gate on `fullyComplete` / `completionRate` before scoring quality.
+*/
+/**
+* Construct a `CompletionVerdict` from the per-requirement checks, deriving
+* `completionRate` / `fullyComplete` and the spine fields (`valid` =
+* `fullyComplete`, `score` = `completionRate`) in one place. Throws on zero
+* requirements — a verdict over nothing is a misconfiguration, mirroring
+* `verifyCompletion`'s gold-spec guard.
+*/
+function completionVerdict(input) {
+	if (input.requirements.length === 0) throw new Error(`completionVerdict: task '${input.taskId}' has no requirement checks — nothing to derive a verdict from`);
+	const measurable = input.requirements.filter((r) => !r.unmeasured);
+	const unmeasuredCount = input.requirements.length - measurable.length;
+	if (measurable.length === 0) throw new Error(`completionVerdict: task '${input.taskId}' has no measurable requirements — all ${input.requirements.length} correctness checks failed (${input.requirements[0]?.unmeasuredReason ?? "unknown reason"})`);
+	const satisfiedCount = measurable.filter((r) => r.satisfied).length;
+	const completionRate = satisfiedCount / measurable.length;
+	const fullyComplete = unmeasuredCount === 0 && satisfiedCount === measurable.length;
+	return {
+		taskId: input.taskId,
+		requirements: input.requirements,
+		completionRate,
+		fullyComplete,
+		unmeasuredCount,
+		valid: fullyComplete,
+		score: completionRate,
+		...input.certification === void 0 ? {} : { certification: input.certification }
+	};
+}
+const STOPWORDS = /* @__PURE__ */ new Set([
+	"the",
+	"a",
+	"an",
+	"of",
+	"for",
+	"and",
+	"or",
+	"to",
+	"in",
+	"on",
+	"with",
+	"by"
+]);
+const REQUIREMENT_FORM_STOPWORDS = /* @__PURE__ */ new Set([
+	"generated",
+	"generate",
+	"view",
+	"render",
+	"rendered",
+	"persisted",
+	"persist",
+	"artifact",
+	"file",
+	"document",
+	"note",
+	"proposal",
+	"deliverable",
+	"output",
+	"created",
+	"create",
+	"produce",
+	"produced",
+	"flag"
+]);
+const MATCH_THRESHOLD = .5;
+const MIN_CONTENT_CHARS = 50;
+function tokens(s, extraStop) {
+	return new Set(s.toLowerCase().split(/[^a-z0-9]+/).filter((t) => t.length > 1 && !STOPWORDS.has(t) && !extraStop?.has(t)));
+}
+/**
+* Recall of the requirement's tokens within a candidate's identifying text.
+* Recall, not Jaccard — a candidate's path/id legitimately carries extra
+* tokens the requirement does not name. The requirement side drops
+* deliverable-FORM vocabulary so recall keys on the distinctive domain tokens.
+*/
+function tokenRecall(requirementText, candidateText) {
+	const req = tokens(requirementText, REQUIREMENT_FORM_STOPWORDS);
+	if (req.size === 0) return 0;
+	const cand = tokens(candidateText);
+	let hit = 0;
+	for (const t of req) if (cand.has(t)) hit++;
+	return hit / req.size;
+}
+function artifactCandidates(req, reqIndex, artifacts) {
+	const reqText = `${req.title} ${req.category ?? ""}`;
+	const out = [];
+	artifacts.forEach((a, i) => {
+		if ((a.content ?? "").trim().length < MIN_CONTENT_CHARS) return;
+		let score = tokenRecall(reqText, `${a.path ?? ""} ${a.kind} ${(a.content ?? "").slice(0, 4e3)}`);
+		if (req.category && a.kind && req.category.toLowerCase() === a.kind.toLowerCase()) score = Math.max(score, 1);
+		if (score < MATCH_THRESHOLD) return;
+		out.push({
+			reqIndex,
+			itemKey: `artifact:${i}`,
+			score,
+			evidence: `artifact '${a.path ?? a.kind}' matched (token recall ${score.toFixed(2)})`,
+			content: a.content ?? null
+		});
+	});
+	return out;
+}
+function proposalCandidates(req, reqIndex, proposals) {
+	const reqText = `${req.title} ${req.category ?? ""}`;
+	const out = [];
+	for (const p of proposals) {
+		if (p.status !== "approved") continue;
+		const body = (p.content ?? "").trim();
+		if (body.length < MIN_CONTENT_CHARS) continue;
+		const score = tokenRecall(reqText, `${p.title} ${body}`);
+		if (score < MATCH_THRESHOLD) continue;
+		out.push({
+			reqIndex,
+			itemKey: `proposal:${p.id}`,
+			score,
+			evidence: `approved proposal '${p.title}' matched (token recall ${score.toFixed(2)})`,
+			content: body
+		});
+	}
+	return out;
+}
+function toolCallCandidates(req, reqIndex, toolCalls) {
+	const out = [];
+	toolCalls.forEach((name, i) => {
+		const score = tokenRecall(req.title, name);
+		if (score < MATCH_THRESHOLD) return;
+		out.push({
+			reqIndex,
+			itemKey: `tool:${i}`,
+			score,
+			evidence: `tool call '${name}' matched (token recall ${score.toFixed(2)})`,
+			content: null
+		});
+	});
+	return out;
+}
+/**
+* Verify whether a run completed the task. `checkCorrectness` is injected —
+* `createLlmCorrectnessChecker` for production, a deterministic stub in tests.
+*
+* Throws on a gold spec with no requirements: an eval task that requires
+* nothing is a misconfiguration, not a vacuously-complete task.
+*/
+async function verifyCompletion(gold, state, checkCorrectness) {
+	if (gold.requirements.length === 0) throw new Error(`verifyCompletion: task '${gold.taskId}' has no requirements — malformed gold spec`);
+	const candidates = [];
+	gold.requirements.forEach((req, i) => {
+		const by = req.satisfiedBy ?? "any";
+		if (by === "artifact" || by === "any") candidates.push(...artifactCandidates(req, i, state.artifacts));
+		if (by === "proposal" || by === "any") candidates.push(...proposalCandidates(req, i, state.proposals));
+		if (by === "tool-call" || by === "any") candidates.push(...toolCallCandidates(req, i, state.toolCalls));
+	});
+	candidates.sort((a, b) => b.score - a.score);
+	const assigned = /* @__PURE__ */ new Map();
+	const itemTaken = /* @__PURE__ */ new Set();
+	for (const c of candidates) {
+		if (assigned.has(c.reqIndex) || itemTaken.has(c.itemKey)) continue;
+		assigned.set(c.reqIndex, c);
+		itemTaken.add(c.itemKey);
+	}
+	const requirements = [];
+	for (let i = 0; i < gold.requirements.length; i++) {
+		const req = gold.requirements[i];
+		const match = assigned.get(i);
+		const evidence = [];
+		let correct = null;
+		let unmeasuredReason;
+		if (match) {
+			evidence.push(match.evidence);
+			if (match.content !== null) try {
+				const r = await checkCorrectness(req, match.content);
+				correct = r.correct;
+				evidence.push(`correctness: ${r.correct ? "pass" : "fail"} — ${r.reason}`);
+			} catch (err) {
+				unmeasuredReason = err instanceof JudgeParseError ? `checker response unparseable after retry: ${err.raw.slice(0, 200)}` : `checker call failed: ${err instanceof Error ? err.message : String(err)}`;
+				evidence.push(`correctness: UNMEASURED — ${unmeasuredReason}`);
+			}
+			else evidence.push("correctness: not assessed — matched item carries no content");
+		} else {
+			const by = req.satisfiedBy ?? "any";
+			const kind = by === "any" ? "artifact/proposal/tool-call" : by;
+			evidence.push(`no produced ${kind} matched this requirement`);
+		}
+		const structurallyPresent = match !== void 0;
+		const unmeasured = unmeasuredReason !== void 0;
+		const satisfied = structurallyPresent && !unmeasured && correct !== false;
+		requirements.push({
+			reqId: req.reqId,
+			title: req.title,
+			structurallyPresent,
+			correct,
+			satisfied,
+			...unmeasured ? {
+				unmeasured: true,
+				unmeasuredReason
+			} : {},
+			evidence
+		});
+	}
+	const attestation = checkCorrectness.attestation;
+	const certification = attestation ? {
+		strategy: attestation.strategy,
+		checker: attestation.checker,
+		assumptions: ["structural matching is lexical token recall over produced items — the correctness stage only sees items it matched", ...attestation.assumptions],
+		evidenceDigest: certificationEvidenceDigest({
+			taskId: gold.taskId,
+			requirements
+		})
+	} : void 0;
+	return completionVerdict({
+		taskId: gold.taskId,
+		requirements,
+		...certification === void 0 ? {} : { certification }
+	});
+}
+/**
+* Parse the correctness checker's model response. Tolerates a response
+* truncated mid-JSON (max_tokens cap) by auto-closing the prefix — the
+* verdict boolean usually lands in the first few tokens, so a recovered
+* prefix with a boolean `correct` is a real measurement, not a guess.
+* Fails loud (JudgeParseError) when no boolean verdict is recoverable.
+*/
+function parseCorrectnessResponse(raw) {
+	const readVerdict = (candidate) => {
+		if (candidate === null || typeof candidate !== "object") return null;
+		const { correct, reason } = candidate;
+		if (typeof correct !== "boolean") return null;
+		return {
+			correct,
+			reason: typeof reason === "string" ? reason : ""
+		};
+	};
+	const match = raw.match(/\{[\s\S]*\}/);
+	if (match) try {
+		const strict = readVerdict(JSON.parse(match[0]));
+		if (strict) return strict;
+	} catch {}
+	const start = raw.indexOf("{");
+	if (start !== -1) {
+		const recovered = readVerdict(recoverTruncatedJson(raw.slice(start)));
+		if (recovered) return recovered;
+	}
+	throw new JudgeParseError("correctness-checker", raw);
+}
+/**
+* Production `CorrectnessChecker` — one LLM call per matched artifact,
+* deterministic (temperature 0), structured JSON out. Judges fulfilment
+* only: a plan, a gesture, or a description of what should be done does not
+* fulfil a requirement — the artifact must BE the deliverable.
+*/
+function createLlmCorrectnessChecker(chat, opts = {}) {
+	const model = opts.model ?? "claude-sonnet-4-6";
+	const maxContentChars = opts.maxContentChars ?? 8e3;
+	const maxAttempts = opts.maxAttempts ?? 2;
+	const costLedger = opts.costLedger ?? new CostLedger();
+	const sink = opts.rawSink;
+	const record = async (event) => {
+		try {
+			await sink?.record(event);
+		} catch {}
+	};
+	const checker = async (requirement, content) => {
+		const request = {
+			model,
+			messages: [{
+				role: "system",
+				content: "You verify whether a produced work artifact actually fulfils a stated requirement. Judge fulfilment only — is the deliverable substantively present and on-point — not polish. A plan to do it later, a vague gesture, or a description of what should be done does NOT fulfil a requirement; the artifact must BE the deliverable. Respond with a single JSON object: {\"correct\": boolean, \"reason\": string (<= 30 words)}."
+			}, {
+				role: "user",
+				content: `Requirement: ${requirement.title}\n${requirement.category ? `Category: ${requirement.category}\n` : ""}\nProduced artifact:\n${content.slice(0, maxContentChars)}`
+			}],
+			temperature: 0,
+			maxTokens: 200
+		};
+		let lastErr;
+		for (let attempt = 0; attempt < maxAttempts; attempt++) {
+			const started = Date.now();
+			await record({
+				eventId: randomUUID(),
+				provider: chat.transport,
+				model,
+				endpoint: "/chat",
+				baseUrl: "",
+				attemptIndex: attempt,
+				direction: "request",
+				timestamp: started,
+				requestBody: request,
+				redactedFields: []
+			});
+			try {
+				const paid = await costLedger.runPaidCall({
+					channel: "verifier",
+					phase: opts.costPhase ?? "completion.correctness",
+					actor: "correctness-checker",
+					model,
+					maximumCharge: chat.maximumAttempts === void 0 ? void 0 : maximumChargeForLlmRequest(request, { maximumAttempts: chat.maximumAttempts }),
+					tags: {
+						...opts.costTags,
+						requirementId: requirement.reqId,
+						attempt: String(attempt)
+					},
+					signal: opts.signal,
+					execute: (signal, callId) => chat.chat(request, {
+						signal,
+						idempotencyKey: callId
+					}),
+					receipt: costReceiptFromLlm,
+					receiptFromError: costReceiptFromLlmError
+				});
+				if (!paid.succeeded) throw paid.error;
+				const resp = paid.value;
+				assertServedModel(model, resp.servedModel, {
+					allowUnreported: true,
+					context: `correctness checker for requirement ${requirement.reqId}`
+				});
+				const raw = resp.content;
+				await record({
+					eventId: randomUUID(),
+					provider: chat.transport,
+					model,
+					endpoint: "/chat",
+					baseUrl: "",
+					attemptIndex: attempt,
+					direction: "response",
+					timestamp: Date.now(),
+					durationMs: Date.now() - started,
+					responseBody: resp,
+					redactedFields: []
+				});
+				return parseCorrectnessResponse(raw);
+			} catch (err) {
+				lastErr = err;
+				await record({
+					eventId: randomUUID(),
+					provider: chat.transport,
+					model,
+					endpoint: "/chat",
+					baseUrl: "",
+					attemptIndex: attempt,
+					direction: "error",
+					timestamp: Date.now(),
+					durationMs: Date.now() - started,
+					errorMessage: err instanceof Error ? err.message : String(err),
+					redactedFields: []
+				});
+				if (err instanceof ModelSubstitutionError) throw err;
+			}
+		}
+		throw lastErr instanceof Error ? lastErr : new Error(String(lastErr));
+	};
+	checker.attestation = {
+		strategy: "judge",
+		checker: {
+			name: "llm-correctness-checker",
+			version: model
+		},
+		assumptions: ["served-model identity is accepted when the transport reports none (assertServedModel allowUnreported)"]
+	};
+	return checker;
+}
+//#endregion
+//#region src/produced-state.ts
+function artifactKind(mimeType) {
+	if (!mimeType) return "file";
+	if (mimeType.includes("json")) return "json";
+	if (mimeType.startsWith("text/")) return "text";
+	return "file";
+}
+/**
+* Normalize a run's runtime event stream into `ProducedState`.
+*
+* Pure and total — unrecognized event types are skipped. `toolCalls` is
+* deduplicated by name in first-seen order (completion cares about a tool's
+* presence, not its call count). An artifact with neither a name nor a uri
+* still yields an entry keyed by its `artifactId` so it is never silently
+* dropped; an artifact with no `content` yields empty content, which the
+* completion oracle's structural check then rejects on its own.
+*/
+function extractProducedState(events) {
+	const artifacts = [];
+	const proposals = [];
+	const toolCalls = [];
+	const seenTools = /* @__PURE__ */ new Set();
+	for (const ev of events) if (ev.type === "tool_call") {
+		const name = ev.toolName;
+		if (name && !seenTools.has(name)) {
+			seenTools.add(name);
+			toolCalls.push(name);
+		}
+	} else if (ev.type === "artifact") {
+		const a = ev;
+		artifacts.push({
+			kind: artifactKind(a.mimeType),
+			path: a.name ?? a.uri ?? a.artifactId,
+			content: a.content ?? ""
+		});
+	} else if (ev.type === "proposal_created") {
+		const p = ev;
+		proposals.push({
+			id: p.proposalId,
+			title: p.title,
+			status: p.status ?? "pending",
+			...p.content !== void 0 ? { content: p.content } : {}
+		});
+	}
+	return {
+		artifacts,
+		proposals,
+		toolCalls
+	};
+}
+//#endregion
+export { CODING_HARNESSES as a, agentProfileId as c, harnessAxisOf as d, verifyCompletion as i, agentProfileModelId as l, completionVerdict as n, HARNESS_NATIVE_MODEL as o, createLlmCorrectnessChecker as r, agentProfileHash as s, extractProducedState as t, expandProfileAxes as u };
+
+//# sourceMappingURL=produced-state-DU79a81m.js.map