@tangle-network/agent-eval 0.95.0 → 0.96.0

package/dist/causal-attribution.d.ts

@@ -1,45 +0,0 @@
-/**
- * Causal attribution via factorial experiments.
- *
- * Run every combination of {model × prompt × scenario × seed}, then
- * decompose observed score variance into main effects + interactions.
- * Moves from correlational "variant B is better" to causal "the model
- * swap accounts for 42% of the lead; the prompt change accounts for 28%;
- * interaction is 30%."
- *
- * Minimal implementation: 2-way factorial (two factors at a time) with
- * main-effect + interaction decomposition via variance of cell means.
- * Consumers run the factorial design themselves (we don't schedule
- * runs); this module consumes the (factorLevels, observedScores)
- * table and does the attribution math.
- */
-export interface FactorialCell {
-    /** Map factor name → level id. e.g. { model: 'claude', prompt: 'v2' } */
-    levels: Record<string, string>;
-    /** Observed score for this cell (mean over replications if n > 1). */
-    score: number;
-    /** Number of replications averaged to produce `score`. */
-    n: number;
-}
-export interface FactorContribution {
-    factor: string;
-    /** Variance attributed to this factor's main effect, as a fraction of total. */
-    shareOfVariance: number;
-    /** Range of cell means across levels of this factor. */
-    range: number;
-}
-export interface InteractionContribution {
-    factors: [string, string];
-    shareOfVariance: number;
-}
-export interface CausalAttributionReport {
-    totalVariance: number;
-    mainEffects: FactorContribution[];
-    interactions: InteractionContribution[];
-    /** Residual = variance unexplained by main effects + modeled interactions. */
-    residualShare: number;
-    /** Sanity: shares sum to 1 (within fp). */
-    sharesSum: number;
-}
-export declare function causalAttribution(cells: FactorialCell[]): CausalAttributionReport;
-//# sourceMappingURL=causal-attribution.d.ts.map

package/dist/causal-attribution.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"causal-attribution.d.ts","sourceRoot":"","sources":["../src/causal-attribution.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,MAAM,WAAW,aAAa;IAC5B,yEAAyE;IACzE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAC9B,sEAAsE;IACtE,KAAK,EAAE,MAAM,CAAA;IACb,0DAA0D;IAC1D,CAAC,EAAE,MAAM,CAAA;CACV;AAED,MAAM,WAAW,kBAAkB;IACjC,MAAM,EAAE,MAAM,CAAA;IACd,gFAAgF;IAChF,eAAe,EAAE,MAAM,CAAA;IACvB,wDAAwD;IACxD,KAAK,EAAE,MAAM,CAAA;CACd;AAED,MAAM,WAAW,uBAAuB;IACtC,OAAO,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IACzB,eAAe,EAAE,MAAM,CAAA;CACxB;AAED,MAAM,WAAW,uBAAuB;IACtC,aAAa,EAAE,MAAM,CAAA;IACrB,WAAW,EAAE,kBAAkB,EAAE,CAAA;IACjC,YAAY,EAAE,uBAAuB,EAAE,CAAA;IACvC,8EAA8E;IAC9E,aAAa,EAAE,MAAM,CAAA;IACrB,2CAA2C;IAC3C,SAAS,EAAE,MAAM,CAAA;CAClB;AAED,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,aAAa,EAAE,GAAG,uBAAuB,CA8DjF"}

package/dist/chunk-AQ5WQAIV.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/trace-analyst/behavioral-metrics.ts","../src/analyst/behavioral-analyst.ts","../src/analyst/default-registry.ts"],"sourcesContent":["/**\n * Deterministic behavioral metrics over OTLP spans — pure arithmetic, no LLM.\n *\n * These are the model-independent multiplier: the four trace-quality signals a\n * tolerant analyzer (e.g. HALO) re-derives per run inside the model — token\n * growth, output decay, tool monoculture, missing self-verification — computed\n * here once, in TypeScript, with zero model judgment. A finding that falls out\n * of arithmetic is trivially model-agnostic and cannot hallucinate the trend.\n *\n * General, not trace-specific: the detectors key off token trajectories and\n * tool usage present in any agentic OTLP trace, not any one benchmark.\n */\n\nimport {\n  LLM_INPUT_TOKEN_ATTR_KEYS,\n  LLM_OUTPUT_TOKEN_ATTR_KEYS,\n  TOOL_NAME_ATTR_KEYS,\n} from '../trace/otlp-attributes'\nimport type { TraceAnalystSpan } from './types'\n\nexport type SuboptimalCode =\n  | 'monotonic-input-growth'\n  | 'output-length-decay'\n  | 'single-tool-dependency'\n  | 'no-self-verification'\n\nexport interface SuboptimalSignal {\n  code: SuboptimalCode\n  severity: 'high' | 'medium' | 'low'\n  /** Human-readable claim, with the backing numbers inlined. */\n  detail: string\n  /** The exact figures the detector fired on — auditable, no model in the loop. */\n  evidence: Record<string, number | string | boolean>\n}\n\nexport interface BehavioralMetrics {\n  llmCallCount: number\n  inputTokenTrajectory: number[]\n  outputTokenTrajectory: number[]\n  toolHistogram: Record<string, number>\n  totalToolCalls: number\n  distinctTools: number\n  /** distinct/total tool calls; 1.0 when there are no tool calls. */\n  toolDiversityRatio: number\n  hasSelfVerification: boolean\n  signals: SuboptimalSignal[]\n}\n\n/** ≥ this input-token growth ratio across a run, with no compression, fires. */\nconst INPUT_GROWTH_FACTOR = 3\n/** Tool-usage signals need at least this many calls to be meaningful. */\nconst MIN_TOOL_CALLS = 3\n/** Tool names matching this are self-verification, not state mutation. */\nconst VERIFY_RE = /verif|eval|inspect|check|assert|validat|review|confirm/i\n\nfunction num(v: unknown): number | null {\n  return typeof v === 'number' && Number.isFinite(v) ? v : null\n}\nfunction numAttr(attrs: Record<string, unknown>, keys: readonly string[]): number | null {\n  for (const key of keys) {\n    const value = num(attrs[key])\n    if (value !== null) return value\n  }\n  return null\n}\nfunction inputTokensOf(s: TraceAnalystSpan): number | null {\n  return (\n    numAttr(s.attributes, LLM_INPUT_TOKEN_ATTR_KEYS) ?? num(s.attributes['llm.usage.input_tokens'])\n  )\n}\nfunction outputTokensOf(s: TraceAnalystSpan): number | null {\n  return (\n    numAttr(s.attributes, LLM_OUTPUT_TOKEN_ATTR_KEYS) ??\n    num(s.attributes['llm.usage.output_tokens'])\n  )\n}\nfunction stepOf(s: TraceAnalystSpan): number | null {\n  return num(s.attributes.step)\n}\nfunction toolNameOf(s: TraceAnalystSpan): string | null {\n  if (s.tool_name) return s.tool_name\n  for (const key of TOOL_NAME_ATTR_KEYS) {\n    const t = s.attributes[key]\n    if (typeof t === 'string' && t.length > 0) return t\n  }\n  return null\n}\n\n/**\n * Reduce a span list to behavioral metrics + fired suboptimality signals.\n * Pure + deterministic: same spans → same output, on any machine, no model.\n */\nexport function computeTraceMetrics(spans: readonly TraceAnalystSpan[]): BehavioralMetrics {\n  // Order by step (when present) then start_time so trajectories reflect run order.\n  const ordered = [...spans].sort((a, b) => {\n    const sa = stepOf(a)\n    const sb = stepOf(b)\n    if (sa !== null && sb !== null && sa !== sb) return sa - sb\n    return a.start_time.localeCompare(b.start_time)\n  })\n\n  const inputTokenTrajectory: number[] = []\n  const outputTokenTrajectory: number[] = []\n  const toolHistogram: Record<string, number> = {}\n  let hasSelfVerification = false\n\n  for (const s of ordered) {\n    const inT = inputTokensOf(s)\n    if (inT !== null) inputTokenTrajectory.push(inT)\n    const outT = outputTokensOf(s)\n    if (outT !== null) outputTokenTrajectory.push(outT)\n    const tool = toolNameOf(s)\n    if (tool) {\n      toolHistogram[tool] = (toolHistogram[tool] ?? 0) + 1\n      if (VERIFY_RE.test(tool)) hasSelfVerification = true\n    }\n  }\n\n  const totalToolCalls = Object.values(toolHistogram).reduce((a, b) => a + b, 0)\n  const distinctTools = Object.keys(toolHistogram).length\n  const toolDiversityRatio = totalToolCalls === 0 ? 1 : distinctTools / totalToolCalls\n\n  const signals: SuboptimalSignal[] = []\n\n  if (inputTokenTrajectory.length >= 3) {\n    const first = inputTokenTrajectory[0]!\n    const last = inputTokenTrajectory[inputTokenTrajectory.length - 1]!\n    // first === 0 with later growth is an unbounded ratio (0→huge context blowup);\n    // treat it as infinite so the signal fires, and report it as such instead of\n    // dividing by zero for the displayed factor.\n    const growthFromZero = first === 0 && last > 0\n    const growth = growthFromZero ? Infinity : first > 0 ? last / first : 0\n    if (last > first && growth >= INPUT_GROWTH_FACTOR) {\n      const growthLabel = growthFromZero ? '0→nonzero (unbounded)' : `${growth.toFixed(1)}x`\n      signals.push({\n        code: 'monotonic-input-growth',\n        severity: 'high',\n        detail: `LLM input tokens grew ${growthLabel} (${first}→${last}) across ${inputTokenTrajectory.length} calls — full history re-sent each step with no compression.`,\n        evidence: {\n          first,\n          last,\n          growth_x: growthFromZero ? 'unbounded' : Number(growth.toFixed(2)),\n          calls: inputTokenTrajectory.length,\n        },\n      })\n    }\n  }\n\n  if (outputTokenTrajectory.length >= 3) {\n    const first = outputTokenTrajectory[0]!\n    const last = outputTokenTrajectory[outputTokenTrajectory.length - 1]!\n    if (last < first) {\n      signals.push({\n        code: 'output-length-decay',\n        severity: 'medium',\n        detail: `LLM output tokens shrank ${first}→${last} over ${outputTokenTrajectory.length} calls — less planning/reasoning per step as context grows.`,\n        evidence: { first, last, calls: outputTokenTrajectory.length },\n      })\n    }\n  }\n\n  if (totalToolCalls >= MIN_TOOL_CALLS && distinctTools === 1) {\n    const only = Object.keys(toolHistogram)[0]!\n    signals.push({\n      code: 'single-tool-dependency',\n      severity: 'medium',\n      detail: `All ${totalToolCalls} tool calls are \\`${only}\\` — no tool diversity and no fallback path.`,\n      evidence: { tool: only, calls: totalToolCalls, distinct_tools: 1 },\n    })\n  }\n\n  if (totalToolCalls >= MIN_TOOL_CALLS && !hasSelfVerification) {\n    signals.push({\n      code: 'no-self-verification',\n      severity: 'medium',\n      detail: `${totalToolCalls} tool calls and none verify/inspect/check state — the agent never validates its own actions.`,\n      evidence: { tool_calls: totalToolCalls, verification_calls: 0 },\n    })\n  }\n\n  return {\n    llmCallCount: inputTokenTrajectory.length,\n    inputTokenTrajectory,\n    outputTokenTrajectory,\n    toolHistogram,\n    totalToolCalls,\n    distinctTools,\n    toolDiversityRatio,\n    hasSelfVerification,\n    signals,\n  }\n}\n","/**\n * `behavioralAnalyst` — a DETERMINISTIC analyst (cost.kind = 'deterministic',\n * never calls the LLM). It produces the efficiency/behavioral findings a\n * tolerant agentic analyzer (HALO) re-derives per run inside the model —\n * context bloat, output decay, tool monoculture, missing self-verification —\n * directly from arithmetic over spans (`computeTraceMetrics`).\n *\n * Why it matters: these findings are model-agnostic BY CONSTRUCTION (no model\n * in the loop), so they cannot return 0 on a weak model the way the Ax-RLM\n * does — and they are strictly more reliable than HALO, which spends tokens\n * re-deriving the same numbers and can hallucinate the trend. The agentic\n * RLM kinds remain for SEMANTIC findings that genuinely need a model; this\n * analyst owns the behavioral class.\n */\n\nimport {\n  type BehavioralMetrics,\n  computeTraceMetrics,\n  type SuboptimalCode,\n} from '../trace-analyst/behavioral-metrics'\nimport type { TraceAnalysisStore } from '../trace-analyst/store'\nimport type { TraceAnalystSpan } from '../trace-analyst/types'\nimport { type Analyst, type AnalystFinding, makeFinding } from './types'\n\nconst RECOMMENDED_ACTION: Record<SuboptimalCode, string> = {\n  'monotonic-input-growth':\n    'Add a context-budget instruction: once prior context exceeds a threshold, summarize earlier steps into a short status line instead of re-sending full history.',\n  'output-length-decay':\n    'Require a minimum planning/reasoning budget per step so late steps do not degrade into terse, error-prone commands.',\n  'single-tool-dependency':\n    'Direct the agent to use the full toolset (verify / inspect / alternate actions), not a single execute call, and to plan a fallback when a call returns an unexpected result.',\n  'no-self-verification':\n    'After every state-mutating action, verify the result (eval / inspect / assert) before proceeding.',\n}\n\nconst ANALYST_ID = 'efficiency-behavioral'\n\n/**\n * Map computed signals → structured AnalystFindings. Pure: no LLM, no clock\n * dependence beyond `produced_at` (overridable for deterministic tests).\n */\nexport function deriveEfficiencyFindings(\n  metrics: BehavioralMetrics,\n  opts: { analystId?: string; producedAt?: string } = {},\n): AnalystFinding[] {\n  const analystId = opts.analystId ?? ANALYST_ID\n  return metrics.signals.map((sig) =>\n    makeFinding({\n      analyst_id: analystId,\n      area: 'efficiency',\n      subject: sig.code, // kebab — passes the cluster grammar; stable key for diffFindings\n      claim: sig.detail,\n      severity: sig.severity,\n      // Deterministic arithmetic over spans, not a model judgment → certain.\n      confidence: 1,\n      evidence_refs: [\n        {\n          kind: 'metric',\n          uri: `metric://efficiency/${sig.code}`,\n          excerpt: JSON.stringify(sig.evidence),\n        },\n      ],\n      recommended_action: RECOMMENDED_ACTION[sig.code],\n      metadata: { deterministic: true, evidence: sig.evidence },\n      ...(opts.producedAt ? { produced_at: opts.producedAt } : {}),\n    }),\n  )\n}\n\n/** The deterministic behavioral/efficiency analyst (no LLM, any-model). */\nexport function behavioralAnalyst(): Analyst<TraceAnalysisStore> {\n  return {\n    id: ANALYST_ID,\n    description:\n      'Deterministic behavioral/efficiency findings over OTLP spans — token-growth, output-decay, tool-monoculture, missing self-verification. Zero LLM; model-agnostic by construction.',\n    inputKind: 'trace-store',\n    cost: { kind: 'deterministic' },\n    version: '1.0.0',\n    async analyze(store) {\n      const overview = await store.getOverview()\n      const spans: TraceAnalystSpan[] = []\n      for (const traceId of overview.sample_trace_ids) {\n        const viewed = await store.viewTrace({ trace_id: traceId })\n        if (viewed.spans) spans.push(...viewed.spans)\n      }\n      return deriveEfficiencyFindings(computeTraceMetrics(spans))\n    },\n  }\n}\n","/**\n * `buildDefaultAnalystRegistry` — the canonical analyst suite, so consumers\n * stop hand-wiring `new AnalystRegistry()` + per-kind `createTraceAnalystKind`.\n *\n * The deterministic `behavioralAnalyst` is ALWAYS registered (it needs no\n * model and is model-agnostic by construction). The agentic RLM kinds are\n * registered only when an `ai` service is supplied — so a caller with no LLM\n * still gets the full behavioral/efficiency diagnosis, and the substrate's\n * \"any model (including no model)\" guarantee holds at the suite level.\n */\n\nimport type { AxAIService } from '@ax-llm/ax'\nimport { behavioralAnalyst } from './behavioral-analyst'\nimport { createTraceAnalystKind, type TraceAnalystKindSpec } from './kind-factory'\nimport { DEFAULT_TRACE_ANALYST_KINDS } from './kinds'\nimport { AnalystRegistry, type AnalystRegistryOptions } from './registry'\n\nexport interface DefaultAnalystRegistryOptions {\n  /** Ax service for the agentic RLM kinds. Omit → only the deterministic analyst. */\n  ai?: AxAIService\n  /** Model for the agentic kinds (falls back to the ai service default). */\n  model?: string\n  /** Which agentic kinds to register when `ai` is present. Default = the shipped suite. */\n  kinds?: readonly TraceAnalystKindSpec[]\n  /** Set false to omit the deterministic behavioral analyst (default: include). */\n  includeBehavioral?: boolean\n  /** Forwarded to the AnalystRegistry constructor (signal, tags, priorFindings). */\n  registry?: AnalystRegistryOptions\n}\n\nexport function buildDefaultAnalystRegistry(\n  opts: DefaultAnalystRegistryOptions = {},\n): AnalystRegistry {\n  const registry = new AnalystRegistry(opts.registry)\n  if (opts.includeBehavioral !== false) {\n    registry.register(behavioralAnalyst())\n  }\n  if (opts.ai) {\n    const kinds = opts.kinds ?? DEFAULT_TRACE_ANALYST_KINDS\n    for (const spec of kinds) {\n      registry.register(createTraceAnalystKind(spec, { ai: opts.ai, model: opts.model }))\n    }\n  }\n  return registry\n}\n"],"mappings":";;;;;;;;;;;;;;;AAiDA,IAAM,sBAAsB;AAE5B,IAAM,iBAAiB;AAEvB,IAAM,YAAY;AAElB,SAAS,IAAI,GAA2B;AACtC,SAAO,OAAO,MAAM,YAAY,OAAO,SAAS,CAAC,IAAI,IAAI;AAC3D;AACA,SAAS,QAAQ,OAAgC,MAAwC;AACvF,aAAW,OAAO,MAAM;AACtB,UAAM,QAAQ,IAAI,MAAM,GAAG,CAAC;AAC5B,QAAI,UAAU,KAAM,QAAO;AAAA,EAC7B;AACA,SAAO;AACT;AACA,SAAS,cAAc,GAAoC;AACzD,SACE,QAAQ,EAAE,YAAY,yBAAyB,KAAK,IAAI,EAAE,WAAW,wBAAwB,CAAC;AAElG;AACA,SAAS,eAAe,GAAoC;AAC1D,SACE,QAAQ,EAAE,YAAY,0BAA0B,KAChD,IAAI,EAAE,WAAW,yBAAyB,CAAC;AAE/C;AACA,SAAS,OAAO,GAAoC;AAClD,SAAO,IAAI,EAAE,WAAW,IAAI;AAC9B;AACA,SAAS,WAAW,GAAoC;AACtD,MAAI,EAAE,UAAW,QAAO,EAAE;AAC1B,aAAW,OAAO,qBAAqB;AACrC,UAAM,IAAI,EAAE,WAAW,GAAG;AAC1B,QAAI,OAAO,MAAM,YAAY,EAAE,SAAS,EAAG,QAAO;AAAA,EACpD;AACA,SAAO;AACT;AAMO,SAAS,oBAAoB,OAAuD;AAEzF,QAAM,UAAU,CAAC,GAAG,KAAK,EAAE,KAAK,CAAC,GAAG,MAAM;AACxC,UAAM,KAAK,OAAO,CAAC;AACnB,UAAM,KAAK,OAAO,CAAC;AACnB,QAAI,OAAO,QAAQ,OAAO,QAAQ,OAAO,GAAI,QAAO,KAAK;AACzD,WAAO,EAAE,WAAW,cAAc,EAAE,UAAU;AAAA,EAChD,CAAC;AAED,QAAM,uBAAiC,CAAC;AACxC,QAAM,wBAAkC,CAAC;AACzC,QAAM,gBAAwC,CAAC;AAC/C,MAAI,sBAAsB;AAE1B,aAAW,KAAK,SAAS;AACvB,UAAM,MAAM,cAAc,CAAC;AAC3B,QAAI,QAAQ,KAAM,sBAAqB,KAAK,GAAG;AAC/C,UAAM,OAAO,eAAe,CAAC;AAC7B,QAAI,SAAS,KAAM,uBAAsB,KAAK,IAAI;AAClD,UAAM,OAAO,WAAW,CAAC;AACzB,QAAI,MAAM;AACR,oBAAc,IAAI,KAAK,cAAc,IAAI,KAAK,KAAK;AACnD,UAAI,UAAU,KAAK,IAAI,EAAG,uBAAsB;AAAA,IAClD;AAAA,EACF;AAEA,QAAM,iBAAiB,OAAO,OAAO,aAAa,EAAE,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC;AAC7E,QAAM,gBAAgB,OAAO,KAAK,aAAa,EAAE;AACjD,QAAM,qBAAqB,mBAAmB,IAAI,IAAI,gBAAgB;AAEtE,QAAM,UAA8B,CAAC;AAErC,MAAI,qBAAqB,UAAU,GAAG;AACpC,UAAM,QAAQ,qBAAqB,CAAC;AACpC,UAAM,OAAO,qBAAqB,qBAAqB,SAAS,CAAC;AAIjE,UAAM,iBAAiB,UAAU,KAAK,OAAO;AAC7C,UAAM,SAAS,iBAAiB,WAAW,QAAQ,IAAI,OAAO,QAAQ;AACtE,QAAI,OAAO,SAAS,UAAU,qBAAqB;AACjD,YAAM,cAAc,iBAAiB,+BAA0B,GAAG,OAAO,QAAQ,CAAC,CAAC;AACnF,cAAQ,KAAK;AAAA,QACX,MAAM;AAAA,QACN,UAAU;AAAA,QACV,QAAQ,yBAAyB,WAAW,KAAK,KAAK,SAAI,IAAI,YAAY,qBAAqB,MAAM;AAAA,QACrG,UAAU;AAAA,UACR;AAAA,UACA;AAAA,UACA,UAAU,iBAAiB,cAAc,OAAO,OAAO,QAAQ,CAAC,CAAC;AAAA,UACjE,OAAO,qBAAqB;AAAA,QAC9B;AAAA,MACF,CAAC;AAAA,IACH;AAAA,EACF;AAEA,MAAI,sBAAsB,UAAU,GAAG;AACrC,UAAM,QAAQ,sBAAsB,CAAC;AACrC,UAAM,OAAO,sBAAsB,sBAAsB,SAAS,CAAC;AACnE,QAAI,OAAO,OAAO;AAChB,cAAQ,KAAK;AAAA,QACX,MAAM;AAAA,QACN,UAAU;AAAA,QACV,QAAQ,4BAA4B,KAAK,SAAI,IAAI,SAAS,sBAAsB,MAAM;AAAA,QACtF,UAAU,EAAE,OAAO,MAAM,OAAO,sBAAsB,OAAO;AAAA,MAC/D,CAAC;AAAA,IACH;AAAA,EACF;AAEA,MAAI,kBAAkB,kBAAkB,kBAAkB,GAAG;AAC3D,UAAM,OAAO,OAAO,KAAK,aAAa,EAAE,CAAC;AACzC,YAAQ,KAAK;AAAA,MACX,MAAM;AAAA,MACN,UAAU;AAAA,MACV,QAAQ,OAAO,cAAc,qBAAqB,IAAI;AAAA,MACtD,UAAU,EAAE,MAAM,MAAM,OAAO,gBAAgB,gBAAgB,EAAE;AAAA,IACnE,CAAC;AAAA,EACH;AAEA,MAAI,kBAAkB,kBAAkB,CAAC,qBAAqB;AAC5D,YAAQ,KAAK;AAAA,MACX,MAAM;AAAA,MACN,UAAU;AAAA,MACV,QAAQ,GAAG,cAAc;AAAA,MACzB,UAAU,EAAE,YAAY,gBAAgB,oBAAoB,EAAE;AAAA,IAChE,CAAC;AAAA,EACH;AAEA,SAAO;AAAA,IACL,cAAc,qBAAqB;AAAA,IACnC;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACF;;;ACvKA,IAAM,qBAAqD;AAAA,EACzD,0BACE;AAAA,EACF,uBACE;AAAA,EACF,0BACE;AAAA,EACF,wBACE;AACJ;AAEA,IAAM,aAAa;AAMZ,SAAS,yBACd,SACA,OAAoD,CAAC,GACnC;AAClB,QAAM,YAAY,KAAK,aAAa;AACpC,SAAO,QAAQ,QAAQ;AAAA,IAAI,CAAC,QAC1B,YAAY;AAAA,MACV,YAAY;AAAA,MACZ,MAAM;AAAA,MACN,SAAS,IAAI;AAAA;AAAA,MACb,OAAO,IAAI;AAAA,MACX,UAAU,IAAI;AAAA;AAAA,MAEd,YAAY;AAAA,MACZ,eAAe;AAAA,QACb;AAAA,UACE,MAAM;AAAA,UACN,KAAK,uBAAuB,IAAI,IAAI;AAAA,UACpC,SAAS,KAAK,UAAU,IAAI,QAAQ;AAAA,QACtC;AAAA,MACF;AAAA,MACA,oBAAoB,mBAAmB,IAAI,IAAI;AAAA,MAC/C,UAAU,EAAE,eAAe,MAAM,UAAU,IAAI,SAAS;AAAA,MACxD,GAAI,KAAK,aAAa,EAAE,aAAa,KAAK,WAAW,IAAI,CAAC;AAAA,IAC5D,CAAC;AAAA,EACH;AACF;AAGO,SAAS,oBAAiD;AAC/D,SAAO;AAAA,IACL,IAAI;AAAA,IACJ,aACE;AAAA,IACF,WAAW;AAAA,IACX,MAAM,EAAE,MAAM,gBAAgB;AAAA,IAC9B,SAAS;AAAA,IACT,MAAM,QAAQ,OAAO;AACnB,YAAM,WAAW,MAAM,MAAM,YAAY;AACzC,YAAM,QAA4B,CAAC;AACnC,iBAAW,WAAW,SAAS,kBAAkB;AAC/C,cAAM,SAAS,MAAM,MAAM,UAAU,EAAE,UAAU,QAAQ,CAAC;AAC1D,YAAI,OAAO,MAAO,OAAM,KAAK,GAAG,OAAO,KAAK;AAAA,MAC9C;AACA,aAAO,yBAAyB,oBAAoB,KAAK,CAAC;AAAA,IAC5D;AAAA,EACF;AACF;;;AC1DO,SAAS,4BACd,OAAsC,CAAC,GACtB;AACjB,QAAM,WAAW,IAAI,gBAAgB,KAAK,QAAQ;AAClD,MAAI,KAAK,sBAAsB,OAAO;AACpC,aAAS,SAAS,kBAAkB,CAAC;AAAA,EACvC;AACA,MAAI,KAAK,IAAI;AACX,UAAM,QAAQ,KAAK,SAAS;AAC5B,eAAW,QAAQ,OAAO;AACxB,eAAS,SAAS,uBAAuB,MAAM,EAAE,IAAI,KAAK,IAAI,OAAO,KAAK,MAAM,CAAC,CAAC;AAAA,IACpF;AAAA,EACF;AACA,SAAO;AACT;","names":[]}

package/dist/ci-gate.d.ts

@@ -1,44 +0,0 @@
-/**
- * CI gate — evaluate a corpus against threshold contracts and generate
- * a human-readable PR/build comment.
- *
- * Three layers:
- *   1. `ThresholdContract` declarations (YAML-equivalent TS objects)
- *   2. `evaluateContract` runs the contracts against a TraceStore and
- *      returns a structured report + overall pass/fail.
- *   3. `renderMarkdownReport` formats the report for GitHub PR comments.
- *
- * Consumers wrap this in their own `gh pr comment` / CI integration —
- * we don't ship the GitHub Action binary, just the library call that
- * the action invokes.
- */
-import type { BaselineReport } from './baseline';
-import { type Slo, type SloReport } from './slo';
-import type { Run } from './trace/schema';
-import type { RunFilter, TraceStore } from './trace/store';
-export interface ContractMetric {
-    /** Metric id matching either a predefined key or a custom extractor. */
-    metric: string;
-    higherIsBetter: boolean;
-    /** Max tolerated regression (e.g. 0.02 = 2pp worse than baseline). */
-    maxRegression?: number;
-    /** Optional extractor if the metric isn't in the default set. */
-    extract?: (run: Run, store: TraceStore) => Promise<number | null>;
-}
-export interface ThresholdContract {
-    name: string;
-    baseline: RunFilter;
-    candidate: RunFilter;
-    metrics: ContractMetric[];
-    slos?: Slo[];
-}
-export interface ContractReport {
-    name: string;
-    baselineReport: BaselineReport;
-    sloReport?: SloReport;
-    breaches: string[];
-    pass: boolean;
-}
-export declare function evaluateContract(store: TraceStore, contract: ThresholdContract): Promise<ContractReport>;
-export declare function renderMarkdownReport(reports: ContractReport[]): string;
-//# sourceMappingURL=ci-gate.d.ts.map

package/dist/ci-gate.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"ci-gate.d.ts","sourceRoot":"","sources":["../src/ci-gate.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,YAAY,CAAA;AAEhD,OAAO,EAAa,KAAK,GAAG,EAAE,KAAK,SAAS,EAAE,MAAM,OAAO,CAAA;AAE3D,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,gBAAgB,CAAA;AACzC,OAAO,KAAK,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,eAAe,CAAA;AAE1D,MAAM,WAAW,cAAc;IAC7B,wEAAwE;IACxE,MAAM,EAAE,MAAM,CAAA;IACd,cAAc,EAAE,OAAO,CAAA;IACvB,sEAAsE;IACtE,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,iEAAiE;IACjE,OAAO,CAAC,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,KAAK,EAAE,UAAU,KAAK,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAA;CAClE;AAED,MAAM,WAAW,iBAAiB;IAChC,IAAI,EAAE,MAAM,CAAA;IACZ,QAAQ,EAAE,SAAS,CAAA;IACnB,SAAS,EAAE,SAAS,CAAA;IACpB,OAAO,EAAE,cAAc,EAAE,CAAA;IACzB,IAAI,CAAC,EAAE,GAAG,EAAE,CAAA;CACb;AAED,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAA;IACZ,cAAc,EAAE,cAAc,CAAA;IAC9B,SAAS,CAAC,EAAE,SAAS,CAAA;IACrB,QAAQ,EAAE,MAAM,EAAE,CAAA;IAClB,IAAI,EAAE,OAAO,CAAA;CACd;AAED,wBAAsB,gBAAgB,CACpC,KAAK,EAAE,UAAU,EACjB,QAAQ,EAAE,iBAAiB,GAC1B,OAAO,CAAC,cAAc,CAAC,CAqDzB;AAED,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,cAAc,EAAE,GAAG,MAAM,CAgCtE"}

package/dist/cli.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":""}

package/dist/client.d.ts

@@ -1,77 +0,0 @@
-import type { CheckResult, ProductClientConfig, TestResult } from './types';
-export declare class ProductClient {
-    private baseUrl;
-    private routes;
-    private cookies;
-    private timeoutMs;
-    constructor(config: ProductClientConfig);
-    private route;
-    /**
-     * Single HTTP boundary for every JSON request. Aborts after `timeoutMs`,
-     * checks `res.ok` BEFORE parsing, and throws `${method} ${path} failed:
-     * HTTP ${status} — ${body}` on a non-ok response so a 4xx/5xx error body
-     * can never be parsed as a success shape. Callers inspect the resolved
-     * value only after a successful return.
-     */
-    private request;
-    /**
-     * Read a required collection field. A missing/non-array field is a contract
-     * violation (wrong route, drift, partial body) — surface it loud instead of
-     * masking it as a healthy empty set. A genuinely empty `[]` passes through.
-     */
-    private requireArray;
-    signup(name: string, email: string, password: string): Promise<{
-        userId: string;
-    }>;
-    login(email: string, password: string): Promise<void>;
-    createWorkspace(name: string, type?: string): Promise<string>;
-    createThread(workspaceId: string): Promise<string>;
-    chat(workspaceId: string, threadId: string, content: string, _opts?: {
-        blockPatterns?: RegExp[];
-    }): Promise<{
-        text: string;
-        blocks: {
-            type: string;
-            title: string;
-        }[];
-    }>;
-    getTasks(workspaceId: string): Promise<{
-        id: string;
-        title: string;
-        status: string;
-        priority: string;
-    }[]>;
-    getEvents(workspaceId: string): Promise<{
-        id: string;
-        title: string;
-        type: string;
-    }[]>;
-    getApprovals(workspaceId: string): Promise<{
-        id: string;
-        title: string;
-        status: string;
-        type: string;
-    }[]>;
-    getVaultTree(workspaceId: string): Promise<string[]>;
-    approveAction(workspaceId: string, id: string): Promise<void>;
-    rejectAction(workspaceId: string, id: string, reason: string): Promise<void>;
-    getGenerations(workspaceId: string): Promise<{
-        id: string;
-        type: string;
-        prompt: string;
-    }[]>;
-    /** Generic GET for custom routes */
-    get(path: string): Promise<Record<string, unknown>>;
-    /** Generic POST for custom routes */
-    post(path: string, body: Record<string, unknown>): Promise<Record<string, unknown>>;
-    /** Generic PATCH for custom routes */
-    patch(path: string, body: Record<string, unknown>): Promise<Record<string, unknown>>;
-}
-/**
- * Run a full e2e workflow test against a live product.
- *
- * The `workflow` callback receives a ProductClient and returns CheckResults.
- * This is the generic harness — each agent defines its own workflow steps.
- */
-export declare function runE2EWorkflow(client: ProductClient, name: string, workflow: (client: ProductClient) => Promise<CheckResult[]>): Promise<TestResult>;
-//# sourceMappingURL=client.d.ts.map

package/dist/client.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../src/client.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,mBAAmB,EAAY,UAAU,EAAE,MAAM,SAAS,CAAA;AASrF,qBAAa,aAAa;IACxB,OAAO,CAAC,OAAO,CAAQ;IACvB,OAAO,CAAC,MAAM,CAAU;IACxB,OAAO,CAAC,OAAO,CAAa;IAC5B,OAAO,CAAC,SAAS,CAAQ;gBAEb,MAAM,EAAE,mBAAmB;IAMvC,OAAO,CAAC,KAAK;IAMb;;;;;;OAMG;YACW,OAAO;IA6BrB;;;;OAIG;IACH,OAAO,CAAC,YAAY;IAUd,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;QAAE,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IAOlF,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAerD,eAAe,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,SAAY,GAAG,OAAO,CAAC,MAAM,CAAC;IAOhE,YAAY,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAOlD,IAAI,CACR,WAAW,EAAE,MAAM,EACnB,QAAQ,EAAE,MAAM,EAChB,OAAO,EAAE,MAAM,EACf,KAAK,CAAC,EAAE;QAAE,aAAa,CAAC,EAAE,MAAM,EAAE,CAAA;KAAE,GACnC,OAAO,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,KAAK,EAAE,MAAM,CAAA;SAAE,EAAE,CAAA;KAAE,CAAC;IAuDjE,QAAQ,CACZ,WAAW,EAAE,MAAM,GAClB,OAAO,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IAQvE,SAAS,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IAKtF,YAAY,CAChB,WAAW,EAAE,MAAM,GAClB,OAAO,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IAQnE,YAAY,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAcpD,aAAa,CAAC,WAAW,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAI7D,YAAY,CAAC,WAAW,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAI5E,cAAc,CAClB,WAAW,EAAE,MAAM,GAClB,OAAO,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IAK1D,oCAAoC;IAC9B,GAAG,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAIzD,qCAAqC;IAC/B,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAIzF,sCAAsC;IAChC,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAG3F;AAED;;;;;GAKG;AACH,wBAAsB,cAAc,CAClC,MAAM,EAAE,aAAa,EACrB,IAAI,EAAE,MAAM,EACZ,QAAQ,EAAE,CAAC,MAAM,EAAE,aAAa,KAAK,OAAO,CAAC,WAAW,EAAE,CAAC,GAC1D,OAAO,CAAC,UAAU,CAAC,CAuBrB"}

package/dist/client.test.d.ts

@@ -1,2 +0,0 @@
-export {};
-//# sourceMappingURL=client.test.d.ts.map

package/dist/client.test.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"client.test.d.ts","sourceRoot":"","sources":["../src/client.test.ts"],"names":[],"mappings":""}

package/dist/command-runner.d.ts

@@ -1,74 +0,0 @@
-/**
- * CommandRunner — abstract subprocess execution surface.
- *
- * Layers in a {@link MultiLayerVerifier} that need to invoke external
- * tools (compilers, test runners, package managers) call out via this
- * interface rather than directly using `child_process`. Two reasons:
- *
- *   1. **Sandbox interchangeability.** A run that targets a sandbox box
- *      (via SDK-specific Box.exec) and a run that targets the host both
- *      satisfy this same contract. The harness doesn't care which.
- *   2. **Testability.** Tests inject a fake runner and assert on calls
- *      without spawning real subprocesses.
- *
- * agent-eval ships only the local implementation (host-process). Sandbox
- * implementations live with their consumer because they depend on
- * SDK-specific Box / Sandbox types that don't belong in this package.
- */
-export interface RunCommandInput {
-    /** Executable name, looked up via PATH unless absolute. */
-    cmd: string;
-    /** Argument vector, NOT shell-interpolated. Each element passed to argv. */
-    argv: string[];
-    /** Working directory. Defaults to runner's notion of cwd if omitted. */
-    cwd?: string;
-    /**
-     * Wall-clock cap in ms. The runner SHOULD return `timedOut: true` when
-     * exceeded; callers MAY treat status null + timedOut as "killed."
-     */
-    capMs?: number;
-    /** Env overrides merged on top of the runner's base environment. */
-    env?: Record<string, string>;
-    /** Optional stdin payload. */
-    stdin?: string;
-}
-export interface RunCommandResult {
-    /** Exit code, or null when the process couldn't start / was killed. */
-    status: number | null;
-    stdout: string;
-    stderr: string;
-    durationMs: number;
-    timedOut: boolean;
-    /** Non-fatal runner-side error (binary missing, signal, etc.). */
-    runnerError?: string;
-}
-export interface DirEntry {
-    name: string;
-    isDirectory: boolean;
-    isFile: boolean;
-    /** File size in bytes. `null` for directories (not stat'd). */
-    sizeBytes: number | null;
-}
-export interface CommandRunner {
-    /** Identifier for telemetry + logs. Open-ended literal-union for new runners. */
-    readonly name: string;
-    /** Execute a command in the runner's environment. */
-    run(input: RunCommandInput): Promise<RunCommandResult>;
-    /** True iff `<name>` resolves on the runner's PATH. */
-    hasBin(name: string): Promise<boolean>;
-    /** True iff the given path exists in the runner's filesystem. */
-    fileExists(path: string): Promise<boolean>;
-    /** Read a file. Returns `null` if missing or unreadable. */
-    readFile(path: string): Promise<string | null>;
-    /** List a directory. Returns `[]` if unreadable / missing. */
-    readDir(path: string): Promise<DirEntry[]>;
-}
-/**
- * Host-process runner. Uses node:child_process spawnSync (synchronous
- * under the hood — wrapped in a Promise to satisfy the interface). For
- * very long-running commands consider an async-spawn variant; this
- * shape matches VB's existing behavior and is fine for build/test/lint
- * subprocesses that finish in seconds-to-minutes.
- */
-export declare const localCommandRunner: CommandRunner;
-//# sourceMappingURL=command-runner.d.ts.map

package/dist/command-runner.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"command-runner.d.ts","sourceRoot":"","sources":["../src/command-runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAeH,MAAM,WAAW,eAAe;IAC9B,2DAA2D;IAC3D,GAAG,EAAE,MAAM,CAAA;IACX,4EAA4E;IAC5E,IAAI,EAAE,MAAM,EAAE,CAAA;IACd,wEAAwE;IACxE,GAAG,CAAC,EAAE,MAAM,CAAA;IACZ;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,oEAAoE;IACpE,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAC5B,8BAA8B;IAC9B,KAAK,CAAC,EAAE,MAAM,CAAA;CACf;AAED,MAAM,WAAW,gBAAgB;IAC/B,uEAAuE;IACvE,MAAM,EAAE,MAAM,GAAG,IAAI,CAAA;IACrB,MAAM,EAAE,MAAM,CAAA;IACd,MAAM,EAAE,MAAM,CAAA;IACd,UAAU,EAAE,MAAM,CAAA;IAClB,QAAQ,EAAE,OAAO,CAAA;IACjB,kEAAkE;IAClE,WAAW,CAAC,EAAE,MAAM,CAAA;CACrB;AAED,MAAM,WAAW,QAAQ;IACvB,IAAI,EAAE,MAAM,CAAA;IACZ,WAAW,EAAE,OAAO,CAAA;IACpB,MAAM,EAAE,OAAO,CAAA;IACf,+DAA+D;IAC/D,SAAS,EAAE,MAAM,GAAG,IAAI,CAAA;CACzB;AAED,MAAM,WAAW,aAAa;IAC5B,iFAAiF;IACjF,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,qDAAqD;IACrD,GAAG,CAAC,KAAK,EAAE,eAAe,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAAA;IACtD,uDAAuD;IACvD,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAA;IACtC,iEAAiE;IACjE,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAA;IAC1C,4DAA4D;IAC5D,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAA;IAC9C,8DAA8D;IAC9D,OAAO,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAA;CAC3C;AAID;;;;;;GAMG;AACH,eAAO,MAAM,kBAAkB,EAAE,aA0EhC,CAAA"}

package/dist/command-runner.test.d.ts

@@ -1,2 +0,0 @@
-export {};
-//# sourceMappingURL=command-runner.test.d.ts.map

package/dist/command-runner.test.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"command-runner.test.d.ts","sourceRoot":"","sources":["../src/command-runner.test.ts"],"names":[],"mappings":""}

package/dist/completion-verifier.d.ts

@@ -1,147 +0,0 @@
-/**
- * Completion verifier — the task-completion oracle.
- *
- * Answers the only eval question that is not a proxy: did the agent actually
- * COMPLETE the task — produce every required deliverable, persisted and
- * correct — rather than describe what should be done. A fluent transcript
- * that never produces the artifact scores zero here.
- *
- * Per requirement, a two-stage check:
- *   1. Structural — a produced item (vault artifact / approved proposal /
- *      tool call) of the right kind is matched against the requirement and
- *      carries non-empty content. Deterministic; no LLM.
- *   2. Correctness — only if structurally present AND the matched item
- *      carries content, one targeted check decides whether that item
- *      actually fulfils the requirement. A hallucinated artifact fails here;
- *      an absent one already failed stage 1.
- *
- * `completionRate` is satisfied / total. Quality dimensions are meaningless
- * on an incomplete task — callers gate on `fullyComplete` / `completionRate`
- * before scoring quality.
- */
-import type { TCloud } from '@tangle-network/tcloud';
-import type { Artifact } from './artifact-validator';
-import type { DefaultVerdict } from './verdict';
-/** What kind of produced state can satisfy a requirement structurally. */
-export type SatisfiedBy = 'artifact' | 'proposal' | 'tool-call' | 'any';
-export interface CompletionRequirement {
-    /** Stable id from the task gold (e.g. a persona's `expected_requirements[].req_id`). */
-    reqId: string;
-    /** Human-readable description of the required deliverable. */
-    title: string;
-    /** Optional kind/category hint, matched against a produced item's kind. */
-    category?: string;
-    /** What produced state satisfies this requirement. Defaults to 'any'. */
-    satisfiedBy?: SatisfiedBy;
-}
-export interface TaskGold {
-    taskId: string;
-    requirements: CompletionRequirement[];
-}
-export interface ProducedProposal {
-    id: string;
-    title: string;
-    status: 'pending' | 'approved' | 'rejected';
-    /** Optional persisted body — when present, enables a correctness check. */
-    content?: string;
-}
-/** Everything observable about what a run actually produced. */
-export interface ProducedState {
-    /** Persisted vault artifacts. Reuses the shared `Artifact` shape. */
-    artifacts: Artifact[];
-    /** Proposals / filings the agent created. */
-    proposals: ProducedProposal[];
-    /** Names of tools the agent invoked. */
-    toolCalls: string[];
-}
-export interface RequirementCheck {
-    reqId: string;
-    title: string;
-    /** A produced item of the right kind matched the requirement, non-empty. */
-    structurallyPresent: boolean;
-    /**
-     * Whether the matched item actually fulfils the requirement. `null` when
-     * not structurally present, or when the matched item carries no content
-     * to assess.
-     */
-    correct: boolean | null;
-    /** structurallyPresent && correct !== false. */
-    satisfied: boolean;
-    /** Human-readable evidence for the verdict. */
-    evidence: string[];
-}
-/** Extends the substrate verdict spine: `valid` = `fullyComplete` and
- *  `score` = `completionRate` — derived in `completionVerdict()`, the one
- *  place those equalities hold by construction. */
-export interface CompletionVerdict extends DefaultVerdict {
-    taskId: string;
-    requirements: RequirementCheck[];
-    /** satisfied / total requirements. */
-    completionRate: number;
-    /** Every requirement satisfied. */
-    fullyComplete: boolean;
-}
-/**
- * Construct a `CompletionVerdict` from the per-requirement checks, deriving
- * `completionRate` / `fullyComplete` and the spine fields (`valid` =
- * `fullyComplete`, `score` = `completionRate`) in one place. Throws on zero
- * requirements — a verdict over nothing is a misconfiguration, mirroring
- * `verifyCompletion`'s gold-spec guard.
- */
-export declare function completionVerdict(input: {
-    taskId: string;
-    requirements: RequirementCheck[];
-}): CompletionVerdict;
-/**
- * Decides whether a produced item's content actually fulfils a requirement.
- * Injected so the structural verifier stays pure and unit-testable; the
- * production implementation is `createLlmCorrectnessChecker`.
- */
-export type CorrectnessChecker = (requirement: CompletionRequirement, content: string) => Promise<{
-    correct: boolean;
-    reason: string;
-}>;
-/**
- * Verify whether a run completed the task. `checkCorrectness` is injected —
- * `createLlmCorrectnessChecker` for production, a deterministic stub in tests.
- *
- * Throws on a gold spec with no requirements: an eval task that requires
- * nothing is a misconfiguration, not a vacuously-complete task.
- */
-export declare function verifyCompletion(gold: TaskGold, state: ProducedState, checkCorrectness: CorrectnessChecker): Promise<CompletionVerdict>;
-export interface LlmCorrectnessCheckerOpts {
-    model?: string;
-    /** Max chars of artifact content sent to the checker. */
-    maxContentChars?: number;
-}
-/** Parse the correctness checker's model response. Fails loud on a bad shape. */
-export declare function parseCorrectnessResponse(raw: string): {
-    correct: boolean;
-    reason: string;
-};
-/**
- * Production `CorrectnessChecker` — one LLM call per matched artifact,
- * deterministic (temperature 0), structured JSON out. Judges fulfilment
- * only: a plan, a gesture, or a description of what should be done does not
- * fulfil a requirement — the artifact must BE the deliverable.
- */
-export declare function createLlmCorrectnessChecker(tc: TCloud, opts?: LlmCorrectnessCheckerOpts): CorrectnessChecker;
-/**
- * Deterministic `CorrectnessChecker` — the no-LLM counterpart to
- * `createLlmCorrectnessChecker`. A produced item fulfils a requirement when its
- * content is substantive (≥ `minContentLength` chars) AND recalls ≥ `minRecall`
- * of the requirement title's significant tokens. No network.
- *
- * Polarity-blind: token recall credits a negation that contains the
- * requirement's tokens ("I will NOT produce the comparison" recalls every token
- * of "produce the comparison"). The structural match stage is ALSO lexical, so
- * pairing the two collapses to a single gameable gate. Use this only as an
- * opt-in structural pre-filter or for tasks whose requirements have no polarity
- * to invert; for produced-state grading the correctness checker MUST be semantic
- * (`createLlmCorrectnessChecker`). See the anti-game fixtures in the test suite.
- */
-export declare function createTokenRecallChecker(opts?: {
-    minRecall?: number;
-    minContentLength?: number;
-}): CorrectnessChecker;
-//# sourceMappingURL=completion-verifier.d.ts.map

package/dist/completion-verifier.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"completion-verifier.d.ts","sourceRoot":"","sources":["../src/completion-verifier.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,wBAAwB,CAAA;AACpD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAA;AACpD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,WAAW,CAAA;AAE/C,0EAA0E;AAC1E,MAAM,MAAM,WAAW,GAAG,UAAU,GAAG,UAAU,GAAG,WAAW,GAAG,KAAK,CAAA;AAEvE,MAAM,WAAW,qBAAqB;IACpC,wFAAwF;IACxF,KAAK,EAAE,MAAM,CAAA;IACb,8DAA8D;IAC9D,KAAK,EAAE,MAAM,CAAA;IACb,2EAA2E;IAC3E,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,yEAAyE;IACzE,WAAW,CAAC,EAAE,WAAW,CAAA;CAC1B;AAED,MAAM,WAAW,QAAQ;IACvB,MAAM,EAAE,MAAM,CAAA;IACd,YAAY,EAAE,qBAAqB,EAAE,CAAA;CACtC;AAED,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE,MAAM,CAAA;IACV,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE,SAAS,GAAG,UAAU,GAAG,UAAU,CAAA;IAC3C,2EAA2E;IAC3E,OAAO,CAAC,EAAE,MAAM,CAAA;CACjB;AAED,gEAAgE;AAChE,MAAM,WAAW,aAAa;IAC5B,qEAAqE;IACrE,SAAS,EAAE,QAAQ,EAAE,CAAA;IACrB,6CAA6C;IAC7C,SAAS,EAAE,gBAAgB,EAAE,CAAA;IAC7B,wCAAwC;IACxC,SAAS,EAAE,MAAM,EAAE,CAAA;CACpB;AAED,MAAM,WAAW,gBAAgB;IAC/B,KAAK,EAAE,MAAM,CAAA;IACb,KAAK,EAAE,MAAM,CAAA;IACb,4EAA4E;IAC5E,mBAAmB,EAAE,OAAO,CAAA;IAC5B;;;;OAIG;IACH,OAAO,EAAE,OAAO,GAAG,IAAI,CAAA;IACvB,gDAAgD;IAChD,SAAS,EAAE,OAAO,CAAA;IAClB,+CAA+C;IAC/C,QAAQ,EAAE,MAAM,EAAE,CAAA;CACnB;AAED;;mDAEmD;AACnD,MAAM,WAAW,iBAAkB,SAAQ,cAAc;IACvD,MAAM,EAAE,MAAM,CAAA;IACd,YAAY,EAAE,gBAAgB,EAAE,CAAA;IAChC,sCAAsC;IACtC,cAAc,EAAE,MAAM,CAAA;IACtB,mCAAmC;IACnC,aAAa,EAAE,OAAO,CAAA;CACvB;AAED;;;;;;GAMG;AACH,wBAAgB,iBAAiB,CAAC,KAAK,EAAE;IACvC,MAAM,EAAE,MAAM,CAAA;IACd,YAAY,EAAE,gBAAgB,EAAE,CAAA;CACjC,GAAG,iBAAiB,CAiBpB;AAED;;;;GAIG;AACH,MAAM,MAAM,kBAAkB,GAAG,CAC/B,WAAW,EAAE,qBAAqB,EAClC,OAAO,EAAE,MAAM,KACZ,OAAO,CAAC;IAAE,OAAO,EAAE,OAAO,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC,CAAA;AA8KlD;;;;;;GAMG;AACH,wBAAsB,gBAAgB,CACpC,IAAI,EAAE,QAAQ,EACd,KAAK,EAAE,aAAa,EACpB,gBAAgB,EAAE,kBAAkB,GACnC,OAAO,CAAC,iBAAiB,CAAC,CAoE5B;AAED,MAAM,WAAW,yBAAyB;IACxC,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,yDAAyD;IACzD,eAAe,CAAC,EAAE,MAAM,CAAA;CACzB;AAED,iFAAiF;AACjF,wBAAgB,wBAAwB,CAAC,GAAG,EAAE,MAAM,GAAG;IAAE,OAAO,EAAE,OAAO,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAU1F;AAED;;;;;GAKG;AACH,wBAAgB,2BAA2B,CACzC,EAAE,EAAE,MAAM,EACV,IAAI,GAAE,yBAA8B,GACnC,kBAAkB,CA2BpB;AAuBD;;;;;;;;;;;;;GAaG;AACH,wBAAgB,wBAAwB,CACtC,IAAI,GAAE;IAAE,SAAS,CAAC,EAAE,MAAM,CAAC;IAAC,gBAAgB,CAAC,EAAE,MAAM,CAAA;CAAO,GAC3D,kBAAkB,CAgCpB"}

package/dist/completion-verifier.test.d.ts

@@ -1,9 +0,0 @@
-/**
- * The completion verifier is the eval's anti-proxy guarantee: a score only
- * means something if this oracle confirms the task was actually done. These
- * tests pin the regressions that would silently re-inflate scores —
- * structurally-absent deliverables passing, hallucinated artifacts passing,
- * pending work counting as complete.
- */
-export {};
-//# sourceMappingURL=completion-verifier.test.d.ts.map

package/dist/completion-verifier.test.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"completion-verifier.test.d.ts","sourceRoot":"","sources":["../src/completion-verifier.test.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG"}

package/dist/concurrency.d.ts

@@ -1,23 +0,0 @@
-/**
- * concurrency — small primitives the evolution loop needs.
- *
- * `Mutex` is a zero-dep async lock with FIFO fairness. The evolution loop
- * uses it to serialise checkout/build/commit sequences inside a single
- * pool slot, and to gate concurrent JSONL writers (see
- * `lockedJsonlReferenceReplayStore`).
- *
- * Deliberately minimal — no priority queue, no timeouts. If you need
- * those, swap to `async-mutex` at the call site.
- */
-export declare class Mutex {
-    private locked;
-    private readonly waiters;
-    acquire(): Promise<() => void>;
-    private release;
-    runExclusive<T>(fn: () => Promise<T> | T): Promise<T>;
-    /** True iff someone holds the lock right now. Diagnostics only. */
-    get isLocked(): boolean;
-    /** Pending waiter count. Diagnostics only. */
-    get pending(): number;
-}
-//# sourceMappingURL=concurrency.d.ts.map

package/dist/concurrency.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"concurrency.d.ts","sourceRoot":"","sources":["../src/concurrency.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,qBAAa,KAAK;IAChB,OAAO,CAAC,MAAM,CAAQ;IACtB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAwB;IAE1C,OAAO,IAAI,OAAO,CAAC,MAAM,IAAI,CAAC;IAYpC,OAAO,CAAC,OAAO;IAST,YAAY,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;IAS3D,mEAAmE;IACnE,IAAI,QAAQ,IAAI,OAAO,CAEtB;IAED,8CAA8C;IAC9C,IAAI,OAAO,IAAI,MAAM,CAEpB;CACF"}

package/dist/contamination-guard.d.ts

@@ -1,81 +0,0 @@
-/**
- * ContaminationGuard — ensures held-out scenarios don't leak into
- * training/prompt paths, and flags model memorization.
- *
- * Three probes:
- *   1. `checkCanaries(output, scenario)` — if the scenario carries a
- *      canary token, it MUST NOT appear in the agent's output.
- *      Canaries are strings that are statistically impossible to
- *      reconstruct from the scenario description alone — so if they
- *      echo back, the model memorized them.
- *   2. `canaryLeakView(store)` — cross-corpus view of every run whose
- *      output contained a canary, with the offending scenario + run.
- *   3. `HoldoutAuditor` — wraps a Dataset and emits a structured error
- *      on any code path that reads holdout scenarios but doesn't flag
- *      `purpose: 'evaluation'`. Keeps engineers honest.
- */
-import type { DatasetScenario } from './dataset';
-import type { TraceStore } from './trace/store';
-export interface CanaryLeak {
-    scenarioId: string;
-    canary: string;
-    runId?: string;
-    evidence: string;
-}
-export declare function checkCanaries(output: string, scenarios: DatasetScenario[]): CanaryLeak[];
-/**
- * Behavioral canary: tests the AGENT, not the eval grep.
- *
- *   - `forbiddenPattern` PRESENT in output ⇒ AGENT EMITTED BAD CONTENT ⇒ LEAK.
- *   - `forbiddenPattern` ABSENT from output ⇒ AGENT HELD THE LINE       ⇒ PASS.
- *
- * Use when running known-bad-prompt scenarios against the agent under
- * test and you want to know if the agent misbehaved. The classical
- * {@link checkCanaries} / {@link import('./canary').runCanaries | runCanaries}
- * test whether the eval check fires when the bad output is forced
- * into the eval flow — different question, different answer.
- *
- * Pattern resolution order (first match wins):
- *   1. `scenario.forbiddenPattern` — if it parses as `/body/flags`,
- *      treated as a regex; otherwise a literal substring.
- *   2. `scenario.canary` — literal substring fallback so the helper
- *      works on existing scenario fixtures.
- *
- * Returns `null` when nothing forbidden was found OR the scenario
- * declared no pattern.
- */
-export declare function checkBehavioralCanary(output: string, scenario: DatasetScenario): CanaryLeak | null;
-/**
- * Behavioral canary over many (scenario, output) pairs. Sibling to
- * {@link import('./canary').runCanaries | runCanaries} — same idea
- * (run-many → report) but the question being answered is "did the
- * AGENT misbehave?" rather than "did the EVAL grep fire?".
- *
- * Returns one `CanaryLeak` per pair where the agent's output
- * contained its scenario's `forbiddenPattern` (or `canary` fallback).
- */
-export declare function runBehavioralCanaries(cases: Array<{
-    scenario: DatasetScenario;
-    output: string;
-    runId?: string;
-}>): CanaryLeak[];
-/**
- * Scan the LLM-output history in a corpus; returns every case where a
- * canary from a known scenario appeared in agent output. Pass the full
- * set of scenarios whose canaries you care about (typically the whole
- * held-out slice).
- */
-export declare function canaryLeakView(store: TraceStore, scenarios: DatasetScenario[]): Promise<CanaryLeak[]>;
-export declare class HoldoutAuditor {
-    private scenarios;
-    private accessLog;
-    constructor(scenarios: DatasetScenario[]);
-    /** Retrieve a holdout scenario for a declared purpose. Non-'evaluation' throws. */
-    get(scenarioId: string, purpose: 'evaluation' | 'debugging'): DatasetScenario;
-    getAccessLog(): ReadonlyArray<{
-        scenarioId: string;
-        purpose: string;
-        at: number;
-    }>;
-}
-//# sourceMappingURL=contamination-guard.d.ts.map

package/dist/contamination-guard.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"contamination-guard.d.ts","sourceRoot":"","sources":["../src/contamination-guard.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,WAAW,CAAA;AAEhD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,eAAe,CAAA;AAE/C,MAAM,WAAW,UAAU;IACzB,UAAU,EAAE,MAAM,CAAA;IAClB,MAAM,EAAE,MAAM,CAAA;IACd,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,MAAM,CAAA;CACjB;AAED,wBAAgB,aAAa,CAAC,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,eAAe,EAAE,GAAG,UAAU,EAAE,CASxF;AAED;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,wBAAgB,qBAAqB,CACnC,MAAM,EAAE,MAAM,EACd,QAAQ,EAAE,eAAe,GACxB,UAAU,GAAG,IAAI,CAUnB;AAED;;;;;;;;GAQG;AACH,wBAAgB,qBAAqB,CACnC,KAAK,EAAE,KAAK,CAAC;IAAE,QAAQ,EAAE,eAAe,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC,GAC1E,UAAU,EAAE,CAOd;AA8BD;;;;;GAKG;AACH,wBAAsB,cAAc,CAClC,KAAK,EAAE,UAAU,EACjB,SAAS,EAAE,eAAe,EAAE,GAC3B,OAAO,CAAC,UAAU,EAAE,CAAC,CAmBvB;AAED,qBAAa,cAAc;IACzB,OAAO,CAAC,SAAS,CAAmB;IACpC,OAAO,CAAC,SAAS,CAAiE;gBAEtE,SAAS,EAAE,eAAe,EAAE;IAIxC,mFAAmF;IACnF,GAAG,CAAC,UAAU,EAAE,MAAM,EAAE,OAAO,EAAE,YAAY,GAAG,WAAW,GAAG,eAAe;IAY7E,YAAY,IAAI,aAAa,CAAC;QAAE,UAAU,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,EAAE,EAAE,MAAM,CAAA;KAAE,CAAC;CAGnF"}

package/dist/contract/analyze-runs.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"analyze-runs.d.ts","sourceRoot":"","sources":["../../src/contract/analyze-runs.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAA;AAG1D,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,YAAY,CAAA;AAEjD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,eAAe,CAAA;AAE9C,OAAO,EAAE,KAAK,gBAAgB,EAAe,MAAM,mBAAmB,CAAA;AAEtE,OAAO,KAAK,EAGV,aAAa,EASd,MAAM,kBAAkB,CAAA;AAIzB,MAAM,WAAW,kBAAkB;IACjC,2BAA2B;IAC3B,IAAI,EAAE,SAAS,EAAE,CAAA;IACjB;2EACuE;IACvE,KAAK,CAAC,EAAE,QAAQ,GAAG,SAAS,GAAG,MAAM,CAAA;IACrC;;oEAEgE;IAChE,mBAAmB,CAAC,EAAE,MAAM,CAAA;IAC5B,oBAAoB,CAAC,EAAE,MAAM,CAAA;IAC7B;iCAC6B;IAC7B,eAAe,CAAC,EAAE,eAAe,EAAE,CAAA;IACnC;uDACmD;IACnD,OAAO,CAAC,EAAE,eAAe,CAAA;IACzB;;mEAE+D;IAC/D,aAAa,CAAC,EAAE;QACd,MAAM,EAAE,MAAM,CAAA;QACd,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;KACrC,CAAA;IACD;;uBAEmB;IACnB,WAAW,CAAC,EAAE,KAAK,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAA;IACpE,yEAAyE;IACzE,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB;;wBAEoB;IACpB,iBAAiB,CAAC,EAAE,MAAM,CAAA;IAC1B;;;;;;;wCAOoC;IACpC,YAAY,CAAC,EAAE,SAAS,EAAE,CAAA;IAC1B;sEACkE;IAClE,aAAa,CAAC,EAAE,MAAM,CAAA;CACvB;AAED,wBAAsB,WAAW,CAAC,IAAI,EAAE,kBAAkB,GAAG,OAAO,CAAC,aAAa,CAAC,CA2FlF;AAu5BD,YAAY,EAAE,gBAAgB,EAAE,CAAA"}