@tangle-network/agent-eval 0.98.0 → 0.100.0

package/dist/adapters/http.d.ts

@@ -1,4 +1,4 @@
-import { S as Scenario, D as DispatchFn, b as DispatchContext } from '../types-D9tuReDv.js';
+import { S as Scenario, D as DispatchFn, b as DispatchContext } from '../types-BTI16iFl.js';
 import '../run-record-CP2ObebC.js';
 import '@tangle-network/agent-interface';
 import '../errors-CzMUYo7b.js';

package/dist/adapters/langchain.d.ts

@@ -1,4 +1,4 @@
-import { S as Scenario, J as JudgeScore, D as DispatchFn, a as JudgeConfig } from '../types-D9tuReDv.js';
+import { S as Scenario, J as JudgeScore, D as DispatchFn, a as JudgeConfig } from '../types-BTI16iFl.js';
 import '../run-record-CP2ObebC.js';
 import '@tangle-network/agent-interface';
 import '../errors-CzMUYo7b.js';

package/dist/adapters/otel.d.ts

@@ -1,5 +1,5 @@
 import { TraceSpanEvent, HostedClient } from '../hosted/index.js';
-import '../types-D9tuReDv.js';
+import '../types-BTI16iFl.js';
 import '../run-record-CP2ObebC.js';
 import '@tangle-network/agent-interface';
 import '../errors-CzMUYo7b.js';

package/dist/adversarial-B7loGVVX.d.ts

@@ -0,0 +1,19 @@
+/**
+ * Adversarial mutation contract.
+ *
+ * `AdversarialMutation<S>` is the scenario-mutation strategy the fuzz harness
+ * (`fuzzAgent`, src/fuzz) drives: paraphrase, edge-case substitution, or
+ * compositional combination of a scenario the policy currently passes, looking
+ * for the tail inputs that break it. The harness supplies the loop; consumers
+ * supply the mutations and the failure detector.
+ */
+interface AdversarialMutation<S> {
+    id: string;
+    /**
+     * Mutate one scenario. Return null to skip; return one or more new
+     * scenarios. The harness deduplicates by `mutateScenarioId(scenario)`.
+     */
+    mutate(parent: S, rng: () => number): Promise<S[]> | S[];
+}
+
+export type { AdversarialMutation as A };

package/dist/analyst/index.js

@@ -11,13 +11,14 @@ import {
   diffFindings,
   emitSkillUsageFindings,
   runSemanticConceptJudge
-} from "../chunk-PSWWQXHF.js";
+} from "../chunk-SJISCGWD.js";
+import "../chunk-DJWX3GVS.js";
 import {
   behavioralAnalyst,
   buildDefaultAnalystRegistry,
   deriveEfficiencyFindings
 } from "../chunk-3NHEO6ZC.js";
-import "../chunk-DJWX3GVS.js";
+import "../chunk-BUTW4RGG.js";
 import {
   ANALYST_SEVERITIES,
   AnalystRegistry,

package/dist/analyst/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/analyst/adapters.ts","../../src/analyst/steer-firewall.ts"],"sourcesContent":["/**\n * Adapter factories — lift each existing agent-eval primitive into the\n * Analyst contract without re-implementing it.\n *\n * Five primitives, five factories. Each one:\n *   - Builds an Analyst with a stable id (caller chooses; defaults\n *     given), a sensible default `inputKind`, a version derived from\n *     the wrapped primitive's version + an adapter revision, and an\n *     `analyze()` that calls the primitive and lifts its output to\n *     AnalystFinding[] using `makeFinding()`.\n *   - Maps severities: the existing `Severity` ('critical' | 'major' |\n *     'minor' | 'info') projects onto AnalystSeverity ('critical' |\n *     'high' | 'medium' | 'low' | 'info'); 'major' → 'high', 'minor' →\n *     'medium'. Domain analysts that want finer-grained mapping override.\n *\n * Adapters never own state. Calling the same factory twice with the\n * same primitive instance is safe.\n */\n\nimport type {\n  Finding as LayerFinding,\n  Severity as LayerSeverity,\n  MultiLayerVerifier,\n  VerifyOptions,\n} from '../multi-layer-verifier'\nimport { RunCritic, type RunTrace } from '../run-critic'\nimport {\n  runSemanticConceptJudge,\n  SEMANTIC_CONCEPT_JUDGE_VERSION,\n  type SemanticConceptJudgeInput,\n  type SemanticConceptJudgeOptions,\n} from '../semantic-concept-judge'\nimport type { JudgeFn, JudgeInput, JudgeScore, TCloud } from '../types'\nimport type { Analyst, AnalystFinding, AnalystSeverity } from './types'\nimport { makeFinding } from './types'\n\nconst ADAPTER_REV = '1'\n\n// ── Severity bridges ───────────────────────────────────────────────\n\nexport function liftSeverity(s: LayerSeverity): AnalystSeverity {\n  switch (s) {\n    case 'critical':\n      return 'critical'\n    case 'major':\n      return 'high'\n    case 'minor':\n      return 'medium'\n    case 'info':\n      return 'info'\n  }\n}\n\n// ── 1. MultiLayerVerifier → Analyst ─────────────────────────────────\n\nexport interface VerifierAdapterOpts<Env> {\n  id?: string\n  area?: string\n  verifier: MultiLayerVerifier<Env>\n  /**\n   * The verifier expects an `env` per run. Adapters take it from\n   * `AnalystRunInputs.custom[<id>]` via the registry's 'custom' routing.\n   */\n  options?: Omit<VerifyOptions<Env>, 'env'>\n}\n\nexport function createVerifierAdapter<Env>(opts: VerifierAdapterOpts<Env>): Analyst<Env> {\n  const id = opts.id ?? 'multi-layer-verifier'\n  const area = opts.area ?? 'verification'\n  return {\n    id,\n    description:\n      \"Runs a MultiLayerVerifier and lifts each layer's findings into the analyst envelope.\",\n    inputKind: 'custom',\n    cost: { kind: 'deterministic' },\n    version: `verifier-${ADAPTER_REV}`,\n    async analyze(env, ctx) {\n      const report = await opts.verifier.run({ env, ...opts.options })\n      const out: AnalystFinding[] = []\n      for (const layer of report.layers) {\n        for (const finding of layer.findings) {\n          out.push(liftLayerFinding(id, area, layer.layer, finding))\n        }\n        // Layer-level signal: a failed/error layer is itself a finding\n        // even if it didn't emit per-finding rows.\n        if (layer.status === 'fail' || layer.status === 'error' || layer.status === 'timeout') {\n          out.push(\n            makeFinding({\n              analyst_id: id,\n              area,\n              subject: layer.layer,\n              claim: `layer \"${layer.layer}\" ${layer.status}: ${layer.reason ?? 'no reason given'}`,\n              severity:\n                layer.status === 'error' ? 'high' : layer.status === 'timeout' ? 'medium' : 'high',\n              confidence: 1,\n              evidence_refs: [],\n              metadata: {\n                layer_status: layer.status,\n                duration_ms: layer.durationMs,\n                score: layer.score,\n                diagnostics: layer.diagnostics,\n              },\n            }),\n          )\n        }\n      }\n      ctx.log?.('verifier complete', {\n        layers: report.layers.length,\n        blended: report.blendedScore,\n        all_pass: report.allPass,\n      })\n      return out\n    },\n  }\n}\n\nfunction liftLayerFinding(\n  analyst_id: string,\n  area: string,\n  layer: string,\n  f: LayerFinding,\n): AnalystFinding {\n  return makeFinding({\n    analyst_id,\n    area,\n    subject: f.layer ?? layer,\n    claim: f.message,\n    severity: liftSeverity(f.severity),\n    confidence: 0.85,\n    evidence_refs: f.evidence\n      ? [{ kind: 'artifact', uri: 'inline:evidence', excerpt: f.evidence }]\n      : [],\n    metadata: f.detail,\n  })\n}\n\n// ── 2. RunCritic → Analyst ──────────────────────────────────────────\n\nexport interface RunCriticAdapterOpts {\n  id?: string\n  area?: string\n  critic?: RunCritic\n  /** Optional threshold below which a dimension is reported as a finding. Default 0.5. */\n  threshold?: number\n}\n\nexport function createRunCriticAdapter(opts: RunCriticAdapterOpts = {}): Analyst<RunTrace> {\n  const id = opts.id ?? 'run-critic'\n  const area = opts.area ?? 'run-quality'\n  const critic = opts.critic ?? new RunCritic()\n  const threshold = opts.threshold ?? 0.5\n  return {\n    id,\n    description:\n      'Scores a single run across success / grounding / drift / tool-quality and surfaces below-threshold dimensions.',\n    inputKind: 'custom',\n    cost: { kind: 'deterministic' },\n    version: `run-critic-${ADAPTER_REV}`,\n    async analyze(trace) {\n      const score = critic.scoreTrace(trace)\n      const out: AnalystFinding[] = []\n      const dims: Array<[keyof typeof score, AnalystSeverity, string]> = [\n        ['success', 'critical', 'run did not complete successfully'],\n        ['goalProgress', 'high', 'goal progress is low'],\n        ['repoGroundedness', 'high', 'output is poorly grounded in the repository'],\n        ['toolUseQuality', 'medium', 'tool use quality is low'],\n        ['patchQuality', 'medium', 'no real patch/edit evidence'],\n        ['testReality', 'high', 'no real test/build evidence'],\n        ['finalGate', 'critical', 'final gate is blocking'],\n      ]\n      for (const [dim, sev, msg] of dims) {\n        const value = score[dim] as number\n        if (typeof value === 'number' && value < threshold) {\n          out.push(\n            makeFinding({\n              analyst_id: id,\n              area,\n              subject: dim,\n              claim: msg,\n              rationale: `${dim}=${value.toFixed(2)} below threshold ${threshold}`,\n              severity: sev,\n              confidence: 1,\n              evidence_refs: [],\n              metadata: { dimension: dim, value, threshold, run_id: trace.run.runId },\n            }),\n          )\n        }\n      }\n      // Drift penalty is high → surface as a finding (inverse threshold).\n      if (score.driftPenalty > 1 - threshold) {\n        out.push(\n          makeFinding({\n            analyst_id: id,\n            area,\n            subject: 'drift',\n            claim: 'agent output drifted from repository signal',\n            rationale: `driftPenalty=${score.driftPenalty.toFixed(2)}`,\n            severity: 'medium',\n            confidence: 0.9,\n            evidence_refs: [],\n            metadata: { drift_penalty: score.driftPenalty, notes: score.notes },\n          }),\n        )\n      }\n      return out\n    },\n  }\n}\n\n// ── 3. JudgeFn → Analyst ────────────────────────────────────────────\n\nexport interface JudgeAdapterOpts {\n  id?: string\n  area?: string\n  judge: JudgeFn\n  /** TCloud handle the JudgeFn calls. */\n  tcloud: TCloud\n  /** Optional cost classification — most judges call an LLM. */\n  cost?: Analyst['cost']\n  /** Optional threshold below which a JudgeScore becomes a finding. Default 6 (on 0-10 scale). */\n  threshold?: number\n}\n\nexport function createJudgeAdapter(opts: JudgeAdapterOpts): Analyst<JudgeInput> {\n  const id = opts.id ?? 'judge'\n  const area = opts.area ?? 'judge'\n  const threshold = opts.threshold ?? 6\n  return {\n    id,\n    description:\n      'Wraps an agent-eval JudgeFn into an analyst; below-threshold dimensions surface as findings.',\n    inputKind: 'judge-input',\n    cost: opts.cost ?? { kind: 'llm' },\n    version: `judge-${ADAPTER_REV}`,\n    async analyze(input) {\n      const scores = await opts.judge(opts.tcloud, input)\n      return scores\n        .filter((s) => normalize10(s.score) < threshold)\n        .map((s) => liftJudgeScore(id, area, s))\n    },\n  }\n}\n\nfunction normalize10(s: number): number {\n  // JudgeScore convention is 0-10 but some judges emit 0-1. Coerce to 0-10.\n  return s <= 1 ? s * 10 : s\n}\n\nfunction liftJudgeScore(analyst_id: string, area: string, s: JudgeScore): AnalystFinding {\n  const score10 = normalize10(s.score)\n  const severity: AnalystSeverity =\n    score10 < 3 ? 'critical' : score10 < 5 ? 'high' : score10 < 7 ? 'medium' : 'low'\n  return makeFinding({\n    analyst_id,\n    area,\n    subject: s.dimension,\n    claim: `${s.judgeName}/${s.dimension} scored ${score10.toFixed(1)}/10`,\n    rationale: s.reasoning,\n    severity,\n    confidence: 0.8,\n    evidence_refs: s.evidence\n      ? [{ kind: 'artifact', uri: 'inline:evidence', excerpt: s.evidence }]\n      : [],\n    // Provenance: this finding IS a judge verdict (an acceptance score), not an\n    // observation of behavior. The steer firewall (assertNoJudgeVerdict) rejects\n    // it from steering — even when it cites an artifact above — because letting a\n    // verdict steer the next attempt is the held-out judge leaking into the loop.\n    derived_from_judge: true,\n    metadata: { judge_name: s.judgeName, dimension: s.dimension, score_10: score10 },\n  })\n}\n\n// ── 4. SemanticConceptJudge → Analyst ──────────────────────────────\n\nexport interface SemanticConceptJudgeAdapterOpts {\n  id?: string\n  area?: string\n  options?: SemanticConceptJudgeOptions\n}\n\nexport function createSemanticConceptJudgeAdapter(\n  opts: SemanticConceptJudgeAdapterOpts = {},\n): Analyst<SemanticConceptJudgeInput> {\n  const id = opts.id ?? 'semantic-concept-judge'\n  const area = opts.area ?? 'concept-coverage'\n  return {\n    id,\n    description:\n      'Runs the semantic-concept judge and surfaces missing / weak concepts as findings.',\n    inputKind: 'custom',\n    cost: { kind: 'llm', models: opts.options?.model ? [opts.options.model] : undefined },\n    version: `${SEMANTIC_CONCEPT_JUDGE_VERSION}-adapter-${ADAPTER_REV}`,\n    async analyze(input) {\n      const result = await runSemanticConceptJudge(input, opts.options)\n      if (!result.available) {\n        return [\n          makeFinding({\n            analyst_id: id,\n            area,\n            claim: 'semantic-concept judge unavailable',\n            rationale: result.error,\n            severity: 'info',\n            confidence: 1,\n            evidence_refs: [],\n            metadata: { reason: result.error },\n          }),\n        ]\n      }\n      const out: AnalystFinding[] = []\n      for (const f of result.findings) {\n        // Only surface gaps: missing concepts or low scores. Concepts at\n        // 7+/10 with present=true are not findings — they're successes.\n        if (f.present && f.score >= 7) continue\n        out.push(\n          makeFinding({\n            analyst_id: id,\n            area,\n            subject: f.concept,\n            claim: f.present\n              ? `concept \"${f.concept}\" is weak (${f.score}/10)`\n              : `concept \"${f.concept}\" is missing`,\n            rationale: f.evidence,\n            severity: liftSeverity(f.severity),\n            confidence: 0.85,\n            evidence_refs: [{ kind: 'artifact', uri: 'inline:evidence', excerpt: f.evidence }],\n            metadata: {\n              concept: f.concept,\n              present: f.present,\n              score_10: f.score,\n              cost_usd: result.costUsd ?? undefined,\n            },\n          }),\n        )\n      }\n      return out\n    },\n  }\n}\n","// The realness-oracle firewall (docs/learning-flywheel.md, \"The steer is f(trace)\").\n//\n// A realness/authenticity signal has TWO legitimate roles that must stay\n// separated by a firewall:\n//   (a) anchor judge J — write-only: scores the chosen output, gates promotion,\n//       NEVER seen by the worker/optimizer mid-run (else the loop games it).\n//   (b) steer f(trace) — an analyst observes the agent's OWN behavior in the\n//       trace (\"imported a stub\", \"used a non-crypto PRNG where encryption was\n//       required\") and steers the next attempt. Legitimate, because it is derived\n//       from OBSERVABLE BEHAVIOR, not from J's held-out verdict.\n//\n// The correct discriminator is PROVENANCE, not evidence presence. A judge verdict\n// lifted into a finding (createJudgeAdapter → liftJudgeScore) is a verdict even\n// when it cites an artifact; an evidence-less trace-analyst bullet is an\n// observation even though it cites nothing. So the firewall keys on\n// `AnalystFinding.derived_from_judge` (set at the judge lift site), NOT on whether\n// evidence_refs is populated. The instant a verdict steers the next attempt it is\n// a back-channel for J and the loop Goodharts realness exactly as it would\n// Goodhart pass-rate.\n\nimport type { AnalystFinding, EvidenceRef } from './types'\n\n/** Evidence grounded in the agent's OWN execution: OTLP trace elements\n *  (`span`/`event`) or the artifact it produced (`artifact`). */\nconst OBSERVABLE_KINDS: ReadonlySet<EvidenceRef['kind']> = new Set<EvidenceRef['kind']>([\n  'span',\n  'event',\n  'artifact',\n])\n\n/** DESCRIPTIVE predicate: does the finding cite at least one observable\n *  (span/event/artifact) evidence ref. Useful for ranking evidence quality or\n *  rendering — it is NOT the steer gate. Evidence presence is the WRONG\n *  discriminator for steering: a legitimate trace-analyst observation may cite\n *  nothing (it would be wrongly rejected), and a judge verdict may cite an\n *  artifact (it would be wrongly admitted). Use `assertNoJudgeVerdict` to gate\n *  steering; use this only where \"is this grounded in observable evidence\" is the\n *  literal question. */\nexport function isTraceObservable(finding: AnalystFinding): boolean {\n  return finding.evidence_refs.some((ref) => OBSERVABLE_KINDS.has(ref.kind))\n}\n\n/** True iff the finding is a JUDGE VERDICT (an acceptance score lifted into a\n *  finding), identified by provenance set at the lift site — independent of\n *  whatever evidence it cites. */\nexport function isJudgeVerdict(finding: AnalystFinding): boolean {\n  return finding.derived_from_judge === true\n}\n\n/**\n * THE steer firewall. Fail-loud guard for any path that admits analyst findings\n * as STEERING input (the `f(trace)` role): rejects — naming the offenders — any\n * finding whose provenance is a judge verdict, rather than let `J` leak into the\n * loop. Returns the findings unchanged for chaining.\n *\n * Call this at the chokepoint where a detector that ALSO scores/gates has its\n * findings turned into a steer (the judge-and-steer dual-role case). It keys on\n * provenance, so it correctly admits evidence-less trace-analyst observations and\n * correctly rejects an artifact-citing judge verdict — the cases an evidence\n * check gets backwards.\n *\n * It is necessary, not sufficient: it stops PROVENANCE-tagged verdicts. A judge\n * whose output is laundered through a hand-built finding with no provenance flag\n * is out of its reach — provenance must be honestly set at every judge→finding\n * lift (today: createJudgeAdapter). That is why the integrity rule lives at the\n * lift site, and why ProposeContext.judgeScores?: never is the complementary\n * compile-time tripwire on the obvious direct channel.\n */\nexport function assertNoJudgeVerdict(\n  findings: ReadonlyArray<AnalystFinding>,\n  context = 'steer',\n): ReadonlyArray<AnalystFinding> {\n  const leaks = findings.filter(isJudgeVerdict)\n  if (leaks.length > 0) {\n    throw new Error(\n      `${context}: a judge verdict cannot be admitted as steering input — that is the ` +\n        `held-out judge leaking into the loop. Offending judge-derived findings: [${leaks\n          .map((f) => f.finding_id)\n          .join(', ')}]. Steering consumes observations of behavior, never acceptance verdicts.`,\n    )\n  }\n  return findings\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAoCA,IAAM,cAAc;AAIb,SAAS,aAAa,GAAmC;AAC9D,UAAQ,GAAG;AAAA,IACT,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AACH,aAAO;AAAA,EACX;AACF;AAeO,SAAS,sBAA2B,MAA8C;AACvF,QAAM,KAAK,KAAK,MAAM;AACtB,QAAM,OAAO,KAAK,QAAQ;AAC1B,SAAO;AAAA,IACL;AAAA,IACA,aACE;AAAA,IACF,WAAW;AAAA,IACX,MAAM,EAAE,MAAM,gBAAgB;AAAA,IAC9B,SAAS,YAAY,WAAW;AAAA,IAChC,MAAM,QAAQ,KAAK,KAAK;AACtB,YAAM,SAAS,MAAM,KAAK,SAAS,IAAI,EAAE,KAAK,GAAG,KAAK,QAAQ,CAAC;AAC/D,YAAM,MAAwB,CAAC;AAC/B,iBAAW,SAAS,OAAO,QAAQ;AACjC,mBAAW,WAAW,MAAM,UAAU;AACpC,cAAI,KAAK,iBAAiB,IAAI,MAAM,MAAM,OAAO,OAAO,CAAC;AAAA,QAC3D;AAGA,YAAI,MAAM,WAAW,UAAU,MAAM,WAAW,WAAW,MAAM,WAAW,WAAW;AACrF,cAAI;AAAA,YACF,YAAY;AAAA,cACV,YAAY;AAAA,cACZ;AAAA,cACA,SAAS,MAAM;AAAA,cACf,OAAO,UAAU,MAAM,KAAK,KAAK,MAAM,MAAM,KAAK,MAAM,UAAU,iBAAiB;AAAA,cACnF,UACE,MAAM,WAAW,UAAU,SAAS,MAAM,WAAW,YAAY,WAAW;AAAA,cAC9E,YAAY;AAAA,cACZ,eAAe,CAAC;AAAA,cAChB,UAAU;AAAA,gBACR,cAAc,MAAM;AAAA,gBACpB,aAAa,MAAM;AAAA,gBACnB,OAAO,MAAM;AAAA,gBACb,aAAa,MAAM;AAAA,cACrB;AAAA,YACF,CAAC;AAAA,UACH;AAAA,QACF;AAAA,MACF;AACA,UAAI,MAAM,qBAAqB;AAAA,QAC7B,QAAQ,OAAO,OAAO;AAAA,QACtB,SAAS,OAAO;AAAA,QAChB,UAAU,OAAO;AAAA,MACnB,CAAC;AACD,aAAO;AAAA,IACT;AAAA,EACF;AACF;AAEA,SAAS,iBACP,YACA,MACA,OACA,GACgB;AAChB,SAAO,YAAY;AAAA,IACjB;AAAA,IACA;AAAA,IACA,SAAS,EAAE,SAAS;AAAA,IACpB,OAAO,EAAE;AAAA,IACT,UAAU,aAAa,EAAE,QAAQ;AAAA,IACjC,YAAY;AAAA,IACZ,eAAe,EAAE,WACb,CAAC,EAAE,MAAM,YAAY,KAAK,mBAAmB,SAAS,EAAE,SAAS,CAAC,IAClE,CAAC;AAAA,IACL,UAAU,EAAE;AAAA,EACd,CAAC;AACH;AAYO,SAAS,uBAAuB,OAA6B,CAAC,GAAsB;AACzF,QAAM,KAAK,KAAK,MAAM;AACtB,QAAM,OAAO,KAAK,QAAQ;AAC1B,QAAM,SAAS,KAAK,UAAU,IAAI,UAAU;AAC5C,QAAM,YAAY,KAAK,aAAa;AACpC,SAAO;AAAA,IACL;AAAA,IACA,aACE;AAAA,IACF,WAAW;AAAA,IACX,MAAM,EAAE,MAAM,gBAAgB;AAAA,IAC9B,SAAS,cAAc,WAAW;AAAA,IAClC,MAAM,QAAQ,OAAO;AACnB,YAAM,QAAQ,OAAO,WAAW,KAAK;AACrC,YAAM,MAAwB,CAAC;AAC/B,YAAM,OAA6D;AAAA,QACjE,CAAC,WAAW,YAAY,mCAAmC;AAAA,QAC3D,CAAC,gBAAgB,QAAQ,sBAAsB;AAAA,QAC/C,CAAC,oBAAoB,QAAQ,6CAA6C;AAAA,QAC1E,CAAC,kBAAkB,UAAU,yBAAyB;AAAA,QACtD,CAAC,gBAAgB,UAAU,6BAA6B;AAAA,QACxD,CAAC,eAAe,QAAQ,6BAA6B;AAAA,QACrD,CAAC,aAAa,YAAY,wBAAwB;AAAA,MACpD;AACA,iBAAW,CAAC,KAAK,KAAK,GAAG,KAAK,MAAM;AAClC,cAAM,QAAQ,MAAM,GAAG;AACvB,YAAI,OAAO,UAAU,YAAY,QAAQ,WAAW;AAClD,cAAI;AAAA,YACF,YAAY;AAAA,cACV,YAAY;AAAA,cACZ;AAAA,cACA,SAAS;AAAA,cACT,OAAO;AAAA,cACP,WAAW,GAAG,GAAG,IAAI,MAAM,QAAQ,CAAC,CAAC,oBAAoB,SAAS;AAAA,cAClE,UAAU;AAAA,cACV,YAAY;AAAA,cACZ,eAAe,CAAC;AAAA,cAChB,UAAU,EAAE,WAAW,KAAK,OAAO,WAAW,QAAQ,MAAM,IAAI,MAAM;AAAA,YACxE,CAAC;AAAA,UACH;AAAA,QACF;AAAA,MACF;AAEA,UAAI,MAAM,eAAe,IAAI,WAAW;AACtC,YAAI;AAAA,UACF,YAAY;AAAA,YACV,YAAY;AAAA,YACZ;AAAA,YACA,SAAS;AAAA,YACT,OAAO;AAAA,YACP,WAAW,gBAAgB,MAAM,aAAa,QAAQ,CAAC,CAAC;AAAA,YACxD,UAAU;AAAA,YACV,YAAY;AAAA,YACZ,eAAe,CAAC;AAAA,YAChB,UAAU,EAAE,eAAe,MAAM,cAAc,OAAO,MAAM,MAAM;AAAA,UACpE,CAAC;AAAA,QACH;AAAA,MACF;AACA,aAAO;AAAA,IACT;AAAA,EACF;AACF;AAgBO,SAAS,mBAAmB,MAA6C;AAC9E,QAAM,KAAK,KAAK,MAAM;AACtB,QAAM,OAAO,KAAK,QAAQ;AAC1B,QAAM,YAAY,KAAK,aAAa;AACpC,SAAO;AAAA,IACL;AAAA,IACA,aACE;AAAA,IACF,WAAW;AAAA,IACX,MAAM,KAAK,QAAQ,EAAE,MAAM,MAAM;AAAA,IACjC,SAAS,SAAS,WAAW;AAAA,IAC7B,MAAM,QAAQ,OAAO;AACnB,YAAM,SAAS,MAAM,KAAK,MAAM,KAAK,QAAQ,KAAK;AAClD,aAAO,OACJ,OAAO,CAAC,MAAM,YAAY,EAAE,KAAK,IAAI,SAAS,EAC9C,IAAI,CAAC,MAAM,eAAe,IAAI,MAAM,CAAC,CAAC;AAAA,IAC3C;AAAA,EACF;AACF;AAEA,SAAS,YAAY,GAAmB;AAEtC,SAAO,KAAK,IAAI,IAAI,KAAK;AAC3B;AAEA,SAAS,eAAe,YAAoB,MAAc,GAA+B;AACvF,QAAM,UAAU,YAAY,EAAE,KAAK;AACnC,QAAM,WACJ,UAAU,IAAI,aAAa,UAAU,IAAI,SAAS,UAAU,IAAI,WAAW;AAC7E,SAAO,YAAY;AAAA,IACjB;AAAA,IACA;AAAA,IACA,SAAS,EAAE;AAAA,IACX,OAAO,GAAG,EAAE,SAAS,IAAI,EAAE,SAAS,WAAW,QAAQ,QAAQ,CAAC,CAAC;AAAA,IACjE,WAAW,EAAE;AAAA,IACb;AAAA,IACA,YAAY;AAAA,IACZ,eAAe,EAAE,WACb,CAAC,EAAE,MAAM,YAAY,KAAK,mBAAmB,SAAS,EAAE,SAAS,CAAC,IAClE,CAAC;AAAA;AAAA;AAAA;AAAA;AAAA,IAKL,oBAAoB;AAAA,IACpB,UAAU,EAAE,YAAY,EAAE,WAAW,WAAW,EAAE,WAAW,UAAU,QAAQ;AAAA,EACjF,CAAC;AACH;AAUO,SAAS,kCACd,OAAwC,CAAC,GACL;AACpC,QAAM,KAAK,KAAK,MAAM;AACtB,QAAM,OAAO,KAAK,QAAQ;AAC1B,SAAO;AAAA,IACL;AAAA,IACA,aACE;AAAA,IACF,WAAW;AAAA,IACX,MAAM,EAAE,MAAM,OAAO,QAAQ,KAAK,SAAS,QAAQ,CAAC,KAAK,QAAQ,KAAK,IAAI,OAAU;AAAA,IACpF,SAAS,GAAG,8BAA8B,YAAY,WAAW;AAAA,IACjE,MAAM,QAAQ,OAAO;AACnB,YAAM,SAAS,MAAM,wBAAwB,OAAO,KAAK,OAAO;AAChE,UAAI,CAAC,OAAO,WAAW;AACrB,eAAO;AAAA,UACL,YAAY;AAAA,YACV,YAAY;AAAA,YACZ;AAAA,YACA,OAAO;AAAA,YACP,WAAW,OAAO;AAAA,YAClB,UAAU;AAAA,YACV,YAAY;AAAA,YACZ,eAAe,CAAC;AAAA,YAChB,UAAU,EAAE,QAAQ,OAAO,MAAM;AAAA,UACnC,CAAC;AAAA,QACH;AAAA,MACF;AACA,YAAM,MAAwB,CAAC;AAC/B,iBAAW,KAAK,OAAO,UAAU;AAG/B,YAAI,EAAE,WAAW,EAAE,SAAS,EAAG;AAC/B,YAAI;AAAA,UACF,YAAY;AAAA,YACV,YAAY;AAAA,YACZ;AAAA,YACA,SAAS,EAAE;AAAA,YACX,OAAO,EAAE,UACL,YAAY,EAAE,OAAO,cAAc,EAAE,KAAK,SAC1C,YAAY,EAAE,OAAO;AAAA,YACzB,WAAW,EAAE;AAAA,YACb,UAAU,aAAa,EAAE,QAAQ;AAAA,YACjC,YAAY;AAAA,YACZ,eAAe,CAAC,EAAE,MAAM,YAAY,KAAK,mBAAmB,SAAS,EAAE,SAAS,CAAC;AAAA,YACjF,UAAU;AAAA,cACR,SAAS,EAAE;AAAA,cACX,SAAS,EAAE;AAAA,cACX,UAAU,EAAE;AAAA,cACZ,UAAU,OAAO,WAAW;AAAA,YAC9B;AAAA,UACF,CAAC;AAAA,QACH;AAAA,MACF;AACA,aAAO;AAAA,IACT;AAAA,EACF;AACF;;;ACzTA,IAAM,mBAAqD,oBAAI,IAAyB;AAAA,EACtF;AAAA,EACA;AAAA,EACA;AACF,CAAC;AAUM,SAAS,kBAAkB,SAAkC;AAClE,SAAO,QAAQ,cAAc,KAAK,CAAC,QAAQ,iBAAiB,IAAI,IAAI,IAAI,CAAC;AAC3E;AAKO,SAAS,eAAe,SAAkC;AAC/D,SAAO,QAAQ,uBAAuB;AACxC;AAqBO,SAAS,qBACd,UACA,UAAU,SACqB;AAC/B,QAAM,QAAQ,SAAS,OAAO,cAAc;AAC5C,MAAI,MAAM,SAAS,GAAG;AACpB,UAAM,IAAI;AAAA,MACR,GAAG,OAAO,sJACoE,MACzE,IAAI,CAAC,MAAM,EAAE,UAAU,EACvB,KAAK,IAAI,CAAC;AAAA,IACjB;AAAA,EACF;AACA,SAAO;AACT;","names":[]}
+{"version":3,"sources":["../../src/analyst/adapters.ts","../../src/analyst/steer-firewall.ts"],"sourcesContent":["/**\n * Adapter factories — lift each existing agent-eval primitive into the\n * Analyst contract without re-implementing it.\n *\n * Five primitives, five factories. Each one:\n *   - Builds an Analyst with a stable id (caller chooses; defaults\n *     given), a sensible default `inputKind`, a version derived from\n *     the wrapped primitive's version + an adapter revision, and an\n *     `analyze()` that calls the primitive and lifts its output to\n *     AnalystFinding[] using `makeFinding()`.\n *   - Maps severities: the existing `Severity` ('critical' | 'major' |\n *     'minor' | 'info') projects onto AnalystSeverity ('critical' |\n *     'high' | 'medium' | 'low' | 'info'); 'major' → 'high', 'minor' →\n *     'medium'. Domain analysts that want finer-grained mapping override.\n *\n * Adapters never own state. Calling the same factory twice with the\n * same primitive instance is safe.\n */\n\nimport type {\n  Finding as LayerFinding,\n  Severity as LayerSeverity,\n  MultiLayerVerifier,\n  VerifyOptions,\n} from '../multi-layer-verifier'\nimport { RunCritic, type RunTrace } from '../run-critic'\nimport {\n  runSemanticConceptJudge,\n  SEMANTIC_CONCEPT_JUDGE_VERSION,\n  type SemanticConceptJudgeInput,\n  type SemanticConceptJudgeOptions,\n} from '../semantic-concept-judge'\nimport type { JudgeFn, JudgeInput, JudgeScore, TCloud } from '../types'\nimport type { Analyst, AnalystFinding, AnalystSeverity } from './types'\nimport { makeFinding } from './types'\n\nconst ADAPTER_REV = '1'\n\n// ── Severity bridges ───────────────────────────────────────────────\n\nexport function liftSeverity(s: LayerSeverity): AnalystSeverity {\n  switch (s) {\n    case 'critical':\n      return 'critical'\n    case 'major':\n      return 'high'\n    case 'minor':\n      return 'medium'\n    case 'info':\n      return 'info'\n  }\n}\n\n// ── 1. MultiLayerVerifier → Analyst ─────────────────────────────────\n\nexport interface VerifierAdapterOpts<Env> {\n  id?: string\n  area?: string\n  verifier: MultiLayerVerifier<Env>\n  /**\n   * The verifier expects an `env` per run. Adapters take it from\n   * `AnalystRunInputs.custom[<id>]` via the registry's 'custom' routing.\n   */\n  options?: Omit<VerifyOptions<Env>, 'env'>\n}\n\nexport function createVerifierAdapter<Env>(opts: VerifierAdapterOpts<Env>): Analyst<Env> {\n  const id = opts.id ?? 'multi-layer-verifier'\n  const area = opts.area ?? 'verification'\n  return {\n    id,\n    description:\n      \"Runs a MultiLayerVerifier and lifts each layer's findings into the analyst envelope.\",\n    inputKind: 'custom',\n    cost: { kind: 'deterministic' },\n    version: `verifier-${ADAPTER_REV}`,\n    async analyze(env, ctx) {\n      const report = await opts.verifier.run({ env, ...opts.options })\n      const out: AnalystFinding[] = []\n      for (const layer of report.layers) {\n        for (const finding of layer.findings) {\n          out.push(liftLayerFinding(id, area, layer.layer, finding))\n        }\n        // Layer-level signal: a failed/error layer is itself a finding\n        // even if it didn't emit per-finding rows.\n        if (layer.status === 'fail' || layer.status === 'error' || layer.status === 'timeout') {\n          out.push(\n            makeFinding({\n              analyst_id: id,\n              area,\n              subject: layer.layer,\n              claim: `layer \"${layer.layer}\" ${layer.status}: ${layer.reason ?? 'no reason given'}`,\n              severity:\n                layer.status === 'error' ? 'high' : layer.status === 'timeout' ? 'medium' : 'high',\n              confidence: 1,\n              evidence_refs: [],\n              metadata: {\n                layer_status: layer.status,\n                duration_ms: layer.durationMs,\n                score: layer.score,\n                diagnostics: layer.diagnostics,\n              },\n            }),\n          )\n        }\n      }\n      ctx.log?.('verifier complete', {\n        layers: report.layers.length,\n        blended: report.blendedScore,\n        all_pass: report.allPass,\n      })\n      return out\n    },\n  }\n}\n\nfunction liftLayerFinding(\n  analyst_id: string,\n  area: string,\n  layer: string,\n  f: LayerFinding,\n): AnalystFinding {\n  return makeFinding({\n    analyst_id,\n    area,\n    subject: f.layer ?? layer,\n    claim: f.message,\n    severity: liftSeverity(f.severity),\n    confidence: 0.85,\n    evidence_refs: f.evidence\n      ? [{ kind: 'artifact', uri: 'inline:evidence', excerpt: f.evidence }]\n      : [],\n    metadata: f.detail,\n  })\n}\n\n// ── 2. RunCritic → Analyst ──────────────────────────────────────────\n\nexport interface RunCriticAdapterOpts {\n  id?: string\n  area?: string\n  critic?: RunCritic\n  /** Optional threshold below which a dimension is reported as a finding. Default 0.5. */\n  threshold?: number\n}\n\nexport function createRunCriticAdapter(opts: RunCriticAdapterOpts = {}): Analyst<RunTrace> {\n  const id = opts.id ?? 'run-critic'\n  const area = opts.area ?? 'run-quality'\n  const critic = opts.critic ?? new RunCritic()\n  const threshold = opts.threshold ?? 0.5\n  return {\n    id,\n    description:\n      'Scores a single run across success / grounding / drift / tool-quality and surfaces below-threshold dimensions.',\n    inputKind: 'custom',\n    cost: { kind: 'deterministic' },\n    version: `run-critic-${ADAPTER_REV}`,\n    async analyze(trace) {\n      const score = critic.scoreTrace(trace)\n      const out: AnalystFinding[] = []\n      const dims: Array<[keyof typeof score, AnalystSeverity, string]> = [\n        ['success', 'critical', 'run did not complete successfully'],\n        ['goalProgress', 'high', 'goal progress is low'],\n        ['repoGroundedness', 'high', 'output is poorly grounded in the repository'],\n        ['toolUseQuality', 'medium', 'tool use quality is low'],\n        ['patchQuality', 'medium', 'no real patch/edit evidence'],\n        ['testReality', 'high', 'no real test/build evidence'],\n        ['finalGate', 'critical', 'final gate is blocking'],\n      ]\n      for (const [dim, sev, msg] of dims) {\n        const value = score[dim] as number\n        if (typeof value === 'number' && value < threshold) {\n          out.push(\n            makeFinding({\n              analyst_id: id,\n              area,\n              subject: dim,\n              claim: msg,\n              rationale: `${dim}=${value.toFixed(2)} below threshold ${threshold}`,\n              severity: sev,\n              confidence: 1,\n              evidence_refs: [],\n              metadata: { dimension: dim, value, threshold, run_id: trace.run.runId },\n            }),\n          )\n        }\n      }\n      // Drift penalty is high → surface as a finding (inverse threshold).\n      if (score.driftPenalty > 1 - threshold) {\n        out.push(\n          makeFinding({\n            analyst_id: id,\n            area,\n            subject: 'drift',\n            claim: 'agent output drifted from repository signal',\n            rationale: `driftPenalty=${score.driftPenalty.toFixed(2)}`,\n            severity: 'medium',\n            confidence: 0.9,\n            evidence_refs: [],\n            metadata: { drift_penalty: score.driftPenalty, notes: score.notes },\n          }),\n        )\n      }\n      return out\n    },\n  }\n}\n\n// ── 3. JudgeFn → Analyst ────────────────────────────────────────────\n\nexport interface JudgeAdapterOpts {\n  id?: string\n  area?: string\n  judge: JudgeFn\n  /** TCloud handle the JudgeFn calls. */\n  tcloud: TCloud\n  /** Optional cost classification — most judges call an LLM. */\n  cost?: Analyst['cost']\n  /** Optional threshold below which a JudgeScore becomes a finding. Default 6 (on 0-10 scale). */\n  threshold?: number\n}\n\nexport function createJudgeAdapter(opts: JudgeAdapterOpts): Analyst<JudgeInput> {\n  const id = opts.id ?? 'judge'\n  const area = opts.area ?? 'judge'\n  const threshold = opts.threshold ?? 6\n  return {\n    id,\n    description:\n      'Wraps an agent-eval JudgeFn into an analyst; below-threshold dimensions surface as findings.',\n    inputKind: 'judge-input',\n    cost: opts.cost ?? { kind: 'llm' },\n    version: `judge-${ADAPTER_REV}`,\n    async analyze(input) {\n      const scores = await opts.judge(opts.tcloud, input)\n      return scores\n        .filter((s) => normalize10(s.score) < threshold)\n        .map((s) => liftJudgeScore(id, area, s))\n    },\n  }\n}\n\nfunction normalize10(s: number): number {\n  // JudgeScore convention is 0-10 but some judges emit 0-1. Coerce to 0-10.\n  return s <= 1 ? s * 10 : s\n}\n\nfunction liftJudgeScore(analyst_id: string, area: string, s: JudgeScore): AnalystFinding {\n  const score10 = normalize10(s.score)\n  const severity: AnalystSeverity =\n    score10 < 3 ? 'critical' : score10 < 5 ? 'high' : score10 < 7 ? 'medium' : 'low'\n  return makeFinding({\n    analyst_id,\n    area,\n    subject: s.dimension,\n    claim: `${s.judgeName}/${s.dimension} scored ${score10.toFixed(1)}/10`,\n    rationale: s.reasoning,\n    severity,\n    confidence: 0.8,\n    evidence_refs: s.evidence\n      ? [{ kind: 'artifact', uri: 'inline:evidence', excerpt: s.evidence }]\n      : [],\n    // Provenance: this finding IS a judge verdict (an acceptance score), not an\n    // observation of behavior. The steer firewall (assertNoJudgeVerdict) rejects\n    // it from steering — even when it cites an artifact above — because letting a\n    // verdict steer the next attempt is the held-out judge leaking into the loop.\n    derived_from_judge: true,\n    metadata: { judge_name: s.judgeName, dimension: s.dimension, score_10: score10 },\n  })\n}\n\n// ── 4. SemanticConceptJudge → Analyst ──────────────────────────────\n\nexport interface SemanticConceptJudgeAdapterOpts {\n  id?: string\n  area?: string\n  options?: SemanticConceptJudgeOptions\n}\n\nexport function createSemanticConceptJudgeAdapter(\n  opts: SemanticConceptJudgeAdapterOpts = {},\n): Analyst<SemanticConceptJudgeInput> {\n  const id = opts.id ?? 'semantic-concept-judge'\n  const area = opts.area ?? 'concept-coverage'\n  return {\n    id,\n    description:\n      'Runs the semantic-concept judge and surfaces missing / weak concepts as findings.',\n    inputKind: 'custom',\n    cost: { kind: 'llm', models: opts.options?.model ? [opts.options.model] : undefined },\n    version: `${SEMANTIC_CONCEPT_JUDGE_VERSION}-adapter-${ADAPTER_REV}`,\n    async analyze(input) {\n      const result = await runSemanticConceptJudge(input, opts.options)\n      if (!result.available) {\n        return [\n          makeFinding({\n            analyst_id: id,\n            area,\n            claim: 'semantic-concept judge unavailable',\n            rationale: result.error,\n            severity: 'info',\n            confidence: 1,\n            evidence_refs: [],\n            metadata: { reason: result.error },\n          }),\n        ]\n      }\n      const out: AnalystFinding[] = []\n      for (const f of result.findings) {\n        // Only surface gaps: missing concepts or low scores. Concepts at\n        // 7+/10 with present=true are not findings — they're successes.\n        if (f.present && f.score >= 7) continue\n        out.push(\n          makeFinding({\n            analyst_id: id,\n            area,\n            subject: f.concept,\n            claim: f.present\n              ? `concept \"${f.concept}\" is weak (${f.score}/10)`\n              : `concept \"${f.concept}\" is missing`,\n            rationale: f.evidence,\n            severity: liftSeverity(f.severity),\n            confidence: 0.85,\n            evidence_refs: [{ kind: 'artifact', uri: 'inline:evidence', excerpt: f.evidence }],\n            metadata: {\n              concept: f.concept,\n              present: f.present,\n              score_10: f.score,\n              cost_usd: result.costUsd ?? undefined,\n            },\n          }),\n        )\n      }\n      return out\n    },\n  }\n}\n","// The realness-oracle firewall (docs/learning-flywheel.md, \"The steer is f(trace)\").\n//\n// A realness/authenticity signal has TWO legitimate roles that must stay\n// separated by a firewall:\n//   (a) anchor judge J — write-only: scores the chosen output, gates promotion,\n//       NEVER seen by the worker/optimizer mid-run (else the loop games it).\n//   (b) steer f(trace) — an analyst observes the agent's OWN behavior in the\n//       trace (\"imported a stub\", \"used a non-crypto PRNG where encryption was\n//       required\") and steers the next attempt. Legitimate, because it is derived\n//       from OBSERVABLE BEHAVIOR, not from J's held-out verdict.\n//\n// The correct discriminator is PROVENANCE, not evidence presence. A judge verdict\n// lifted into a finding (createJudgeAdapter → liftJudgeScore) is a verdict even\n// when it cites an artifact; an evidence-less trace-analyst bullet is an\n// observation even though it cites nothing. So the firewall keys on\n// `AnalystFinding.derived_from_judge` (set at the judge lift site), NOT on whether\n// evidence_refs is populated. The instant a verdict steers the next attempt it is\n// a back-channel for J and the loop Goodharts realness exactly as it would\n// Goodhart pass-rate.\n\nimport type { AnalystFinding, EvidenceRef } from './types'\n\n/** Evidence grounded in the agent's OWN execution: OTLP trace elements\n *  (`span`/`event`) or the artifact it produced (`artifact`). */\nconst OBSERVABLE_KINDS: ReadonlySet<EvidenceRef['kind']> = new Set<EvidenceRef['kind']>([\n  'span',\n  'event',\n  'artifact',\n])\n\n/** DESCRIPTIVE predicate: does the finding cite at least one observable\n *  (span/event/artifact) evidence ref. Useful for ranking evidence quality or\n *  rendering — it is NOT the steer gate. Evidence presence is the WRONG\n *  discriminator for steering: a legitimate trace-analyst observation may cite\n *  nothing (it would be wrongly rejected), and a judge verdict may cite an\n *  artifact (it would be wrongly admitted). Use `assertNoJudgeVerdict` to gate\n *  steering; use this only where \"is this grounded in observable evidence\" is the\n *  literal question. */\nexport function isTraceObservable(finding: AnalystFinding): boolean {\n  return finding.evidence_refs.some((ref) => OBSERVABLE_KINDS.has(ref.kind))\n}\n\n/** True iff the finding is a JUDGE VERDICT (an acceptance score lifted into a\n *  finding), identified by provenance set at the lift site — independent of\n *  whatever evidence it cites. */\nexport function isJudgeVerdict(finding: AnalystFinding): boolean {\n  return finding.derived_from_judge === true\n}\n\n/**\n * THE steer firewall. Fail-loud guard for any path that admits analyst findings\n * as STEERING input (the `f(trace)` role): rejects — naming the offenders — any\n * finding whose provenance is a judge verdict, rather than let `J` leak into the\n * loop. Returns the findings unchanged for chaining.\n *\n * Call this at the chokepoint where a detector that ALSO scores/gates has its\n * findings turned into a steer (the judge-and-steer dual-role case). It keys on\n * provenance, so it correctly admits evidence-less trace-analyst observations and\n * correctly rejects an artifact-citing judge verdict — the cases an evidence\n * check gets backwards.\n *\n * It is necessary, not sufficient: it stops PROVENANCE-tagged verdicts. A judge\n * whose output is laundered through a hand-built finding with no provenance flag\n * is out of its reach — provenance must be honestly set at every judge→finding\n * lift (today: createJudgeAdapter). That is why the integrity rule lives at the\n * lift site, and why ProposeContext.judgeScores?: never is the complementary\n * compile-time tripwire on the obvious direct channel.\n */\nexport function assertNoJudgeVerdict(\n  findings: ReadonlyArray<AnalystFinding>,\n  context = 'steer',\n): ReadonlyArray<AnalystFinding> {\n  const leaks = findings.filter(isJudgeVerdict)\n  if (leaks.length > 0) {\n    throw new Error(\n      `${context}: a judge verdict cannot be admitted as steering input — that is the ` +\n        `held-out judge leaking into the loop. Offending judge-derived findings: [${leaks\n          .map((f) => f.finding_id)\n          .join(', ')}]. Steering consumes observations of behavior, never acceptance verdicts.`,\n    )\n  }\n  return findings\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAoCA,IAAM,cAAc;AAIb,SAAS,aAAa,GAAmC;AAC9D,UAAQ,GAAG;AAAA,IACT,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AACH,aAAO;AAAA,EACX;AACF;AAeO,SAAS,sBAA2B,MAA8C;AACvF,QAAM,KAAK,KAAK,MAAM;AACtB,QAAM,OAAO,KAAK,QAAQ;AAC1B,SAAO;AAAA,IACL;AAAA,IACA,aACE;AAAA,IACF,WAAW;AAAA,IACX,MAAM,EAAE,MAAM,gBAAgB;AAAA,IAC9B,SAAS,YAAY,WAAW;AAAA,IAChC,MAAM,QAAQ,KAAK,KAAK;AACtB,YAAM,SAAS,MAAM,KAAK,SAAS,IAAI,EAAE,KAAK,GAAG,KAAK,QAAQ,CAAC;AAC/D,YAAM,MAAwB,CAAC;AAC/B,iBAAW,SAAS,OAAO,QAAQ;AACjC,mBAAW,WAAW,MAAM,UAAU;AACpC,cAAI,KAAK,iBAAiB,IAAI,MAAM,MAAM,OAAO,OAAO,CAAC;AAAA,QAC3D;AAGA,YAAI,MAAM,WAAW,UAAU,MAAM,WAAW,WAAW,MAAM,WAAW,WAAW;AACrF,cAAI;AAAA,YACF,YAAY;AAAA,cACV,YAAY;AAAA,cACZ;AAAA,cACA,SAAS,MAAM;AAAA,cACf,OAAO,UAAU,MAAM,KAAK,KAAK,MAAM,MAAM,KAAK,MAAM,UAAU,iBAAiB;AAAA,cACnF,UACE,MAAM,WAAW,UAAU,SAAS,MAAM,WAAW,YAAY,WAAW;AAAA,cAC9E,YAAY;AAAA,cACZ,eAAe,CAAC;AAAA,cAChB,UAAU;AAAA,gBACR,cAAc,MAAM;AAAA,gBACpB,aAAa,MAAM;AAAA,gBACnB,OAAO,MAAM;AAAA,gBACb,aAAa,MAAM;AAAA,cACrB;AAAA,YACF,CAAC;AAAA,UACH;AAAA,QACF;AAAA,MACF;AACA,UAAI,MAAM,qBAAqB;AAAA,QAC7B,QAAQ,OAAO,OAAO;AAAA,QACtB,SAAS,OAAO;AAAA,QAChB,UAAU,OAAO;AAAA,MACnB,CAAC;AACD,aAAO;AAAA,IACT;AAAA,EACF;AACF;AAEA,SAAS,iBACP,YACA,MACA,OACA,GACgB;AAChB,SAAO,YAAY;AAAA,IACjB;AAAA,IACA;AAAA,IACA,SAAS,EAAE,SAAS;AAAA,IACpB,OAAO,EAAE;AAAA,IACT,UAAU,aAAa,EAAE,QAAQ;AAAA,IACjC,YAAY;AAAA,IACZ,eAAe,EAAE,WACb,CAAC,EAAE,MAAM,YAAY,KAAK,mBAAmB,SAAS,EAAE,SAAS,CAAC,IAClE,CAAC;AAAA,IACL,UAAU,EAAE;AAAA,EACd,CAAC;AACH;AAYO,SAAS,uBAAuB,OAA6B,CAAC,GAAsB;AACzF,QAAM,KAAK,KAAK,MAAM;AACtB,QAAM,OAAO,KAAK,QAAQ;AAC1B,QAAM,SAAS,KAAK,UAAU,IAAI,UAAU;AAC5C,QAAM,YAAY,KAAK,aAAa;AACpC,SAAO;AAAA,IACL;AAAA,IACA,aACE;AAAA,IACF,WAAW;AAAA,IACX,MAAM,EAAE,MAAM,gBAAgB;AAAA,IAC9B,SAAS,cAAc,WAAW;AAAA,IAClC,MAAM,QAAQ,OAAO;AACnB,YAAM,QAAQ,OAAO,WAAW,KAAK;AACrC,YAAM,MAAwB,CAAC;AAC/B,YAAM,OAA6D;AAAA,QACjE,CAAC,WAAW,YAAY,mCAAmC;AAAA,QAC3D,CAAC,gBAAgB,QAAQ,sBAAsB;AAAA,QAC/C,CAAC,oBAAoB,QAAQ,6CAA6C;AAAA,QAC1E,CAAC,kBAAkB,UAAU,yBAAyB;AAAA,QACtD,CAAC,gBAAgB,UAAU,6BAA6B;AAAA,QACxD,CAAC,eAAe,QAAQ,6BAA6B;AAAA,QACrD,CAAC,aAAa,YAAY,wBAAwB;AAAA,MACpD;AACA,iBAAW,CAAC,KAAK,KAAK,GAAG,KAAK,MAAM;AAClC,cAAM,QAAQ,MAAM,GAAG;AACvB,YAAI,OAAO,UAAU,YAAY,QAAQ,WAAW;AAClD,cAAI;AAAA,YACF,YAAY;AAAA,cACV,YAAY;AAAA,cACZ;AAAA,cACA,SAAS;AAAA,cACT,OAAO;AAAA,cACP,WAAW,GAAG,GAAG,IAAI,MAAM,QAAQ,CAAC,CAAC,oBAAoB,SAAS;AAAA,cAClE,UAAU;AAAA,cACV,YAAY;AAAA,cACZ,eAAe,CAAC;AAAA,cAChB,UAAU,EAAE,WAAW,KAAK,OAAO,WAAW,QAAQ,MAAM,IAAI,MAAM;AAAA,YACxE,CAAC;AAAA,UACH;AAAA,QACF;AAAA,MACF;AAEA,UAAI,MAAM,eAAe,IAAI,WAAW;AACtC,YAAI;AAAA,UACF,YAAY;AAAA,YACV,YAAY;AAAA,YACZ;AAAA,YACA,SAAS;AAAA,YACT,OAAO;AAAA,YACP,WAAW,gBAAgB,MAAM,aAAa,QAAQ,CAAC,CAAC;AAAA,YACxD,UAAU;AAAA,YACV,YAAY;AAAA,YACZ,eAAe,CAAC;AAAA,YAChB,UAAU,EAAE,eAAe,MAAM,cAAc,OAAO,MAAM,MAAM;AAAA,UACpE,CAAC;AAAA,QACH;AAAA,MACF;AACA,aAAO;AAAA,IACT;AAAA,EACF;AACF;AAgBO,SAAS,mBAAmB,MAA6C;AAC9E,QAAM,KAAK,KAAK,MAAM;AACtB,QAAM,OAAO,KAAK,QAAQ;AAC1B,QAAM,YAAY,KAAK,aAAa;AACpC,SAAO;AAAA,IACL;AAAA,IACA,aACE;AAAA,IACF,WAAW;AAAA,IACX,MAAM,KAAK,QAAQ,EAAE,MAAM,MAAM;AAAA,IACjC,SAAS,SAAS,WAAW;AAAA,IAC7B,MAAM,QAAQ,OAAO;AACnB,YAAM,SAAS,MAAM,KAAK,MAAM,KAAK,QAAQ,KAAK;AAClD,aAAO,OACJ,OAAO,CAAC,MAAM,YAAY,EAAE,KAAK,IAAI,SAAS,EAC9C,IAAI,CAAC,MAAM,eAAe,IAAI,MAAM,CAAC,CAAC;AAAA,IAC3C;AAAA,EACF;AACF;AAEA,SAAS,YAAY,GAAmB;AAEtC,SAAO,KAAK,IAAI,IAAI,KAAK;AAC3B;AAEA,SAAS,eAAe,YAAoB,MAAc,GAA+B;AACvF,QAAM,UAAU,YAAY,EAAE,KAAK;AACnC,QAAM,WACJ,UAAU,IAAI,aAAa,UAAU,IAAI,SAAS,UAAU,IAAI,WAAW;AAC7E,SAAO,YAAY;AAAA,IACjB;AAAA,IACA;AAAA,IACA,SAAS,EAAE;AAAA,IACX,OAAO,GAAG,EAAE,SAAS,IAAI,EAAE,SAAS,WAAW,QAAQ,QAAQ,CAAC,CAAC;AAAA,IACjE,WAAW,EAAE;AAAA,IACb;AAAA,IACA,YAAY;AAAA,IACZ,eAAe,EAAE,WACb,CAAC,EAAE,MAAM,YAAY,KAAK,mBAAmB,SAAS,EAAE,SAAS,CAAC,IAClE,CAAC;AAAA;AAAA;AAAA;AAAA;AAAA,IAKL,oBAAoB;AAAA,IACpB,UAAU,EAAE,YAAY,EAAE,WAAW,WAAW,EAAE,WAAW,UAAU,QAAQ;AAAA,EACjF,CAAC;AACH;AAUO,SAAS,kCACd,OAAwC,CAAC,GACL;AACpC,QAAM,KAAK,KAAK,MAAM;AACtB,QAAM,OAAO,KAAK,QAAQ;AAC1B,SAAO;AAAA,IACL;AAAA,IACA,aACE;AAAA,IACF,WAAW;AAAA,IACX,MAAM,EAAE,MAAM,OAAO,QAAQ,KAAK,SAAS,QAAQ,CAAC,KAAK,QAAQ,KAAK,IAAI,OAAU;AAAA,IACpF,SAAS,GAAG,8BAA8B,YAAY,WAAW;AAAA,IACjE,MAAM,QAAQ,OAAO;AACnB,YAAM,SAAS,MAAM,wBAAwB,OAAO,KAAK,OAAO;AAChE,UAAI,CAAC,OAAO,WAAW;AACrB,eAAO;AAAA,UACL,YAAY;AAAA,YACV,YAAY;AAAA,YACZ;AAAA,YACA,OAAO;AAAA,YACP,WAAW,OAAO;AAAA,YAClB,UAAU;AAAA,YACV,YAAY;AAAA,YACZ,eAAe,CAAC;AAAA,YAChB,UAAU,EAAE,QAAQ,OAAO,MAAM;AAAA,UACnC,CAAC;AAAA,QACH;AAAA,MACF;AACA,YAAM,MAAwB,CAAC;AAC/B,iBAAW,KAAK,OAAO,UAAU;AAG/B,YAAI,EAAE,WAAW,EAAE,SAAS,EAAG;AAC/B,YAAI;AAAA,UACF,YAAY;AAAA,YACV,YAAY;AAAA,YACZ;AAAA,YACA,SAAS,EAAE;AAAA,YACX,OAAO,EAAE,UACL,YAAY,EAAE,OAAO,cAAc,EAAE,KAAK,SAC1C,YAAY,EAAE,OAAO;AAAA,YACzB,WAAW,EAAE;AAAA,YACb,UAAU,aAAa,EAAE,QAAQ;AAAA,YACjC,YAAY;AAAA,YACZ,eAAe,CAAC,EAAE,MAAM,YAAY,KAAK,mBAAmB,SAAS,EAAE,SAAS,CAAC;AAAA,YACjF,UAAU;AAAA,cACR,SAAS,EAAE;AAAA,cACX,SAAS,EAAE;AAAA,cACX,UAAU,EAAE;AAAA,cACZ,UAAU,OAAO,WAAW;AAAA,YAC9B;AAAA,UACF,CAAC;AAAA,QACH;AAAA,MACF;AACA,aAAO;AAAA,IACT;AAAA,EACF;AACF;;;ACzTA,IAAM,mBAAqD,oBAAI,IAAyB;AAAA,EACtF;AAAA,EACA;AAAA,EACA;AACF,CAAC;AAUM,SAAS,kBAAkB,SAAkC;AAClE,SAAO,QAAQ,cAAc,KAAK,CAAC,QAAQ,iBAAiB,IAAI,IAAI,IAAI,CAAC;AAC3E;AAKO,SAAS,eAAe,SAAkC;AAC/D,SAAO,QAAQ,uBAAuB;AACxC;AAqBO,SAAS,qBACd,UACA,UAAU,SACqB;AAC/B,QAAM,QAAQ,SAAS,OAAO,cAAc;AAC5C,MAAI,MAAM,SAAS,GAAG;AACpB,UAAM,IAAI;AAAA,MACR,GAAG,OAAO,sJACoE,MACzE,IAAI,CAAC,MAAM,EAAE,UAAU,EACvB,KAAK,IAAI,CAAC;AAAA,IACjB;AAAA,EACF;AACA,SAAO;AACT;","names":[]}

package/dist/campaign/index.d.ts

@@ -1,10 +1,11 @@
+import { S as SignedManifest, B as BackendIntegrityReport, C as CompletionRequirement, R as RuntimeEventLike, a as CompletionVerdict, P as ProducedState, b as CorrectnessChecker } from '../pre-registration-CMm8cvrh.js';
+export { L as LlmJudgeDimension, c as LlmJudgeOptions, l as llmJudge } from '../pre-registration-CMm8cvrh.js';
 import { A as AnalyzeTracesOptions, a as AnalyzeTracesInput, b as AnalyzeTracesResult } from '../analyst-C8HHvfJp.js';
-import { S as Scenario, M as MutableSurface, b as DispatchContext, a as JudgeConfig, d as GenerationRecord, f as Gate, J as JudgeScore, L as LabeledScenarioStore, s as LabeledScenarioWrite, t as LabeledScenarioSampleArgs, u as LabeledScenarioRecord, v as LabelTrust, e as SurfaceProposer, w as ProposedCandidate, x as ProposeContext, y as LabeledScenarioSource, C as CampaignResult, n as CodeSurface } from '../types-D9tuReDv.js';
-export { j as CampaignAggregates, k as CampaignArtifactWriter, l as CampaignCellResult, m as CampaignCostMeter, z as CampaignTokenUsage, c as CampaignTraceWriter, D as DispatchFn, g as GateContext, i as GateDecision, G as GateResult, o as GenerationCandidate, A as JudgeAggregate, p as JudgeDimension, h as Mutator, O as OptimizationProposer, q as OptimizerConfig, P as ParetoParent, R as RedactionStatus, B as ScenarioAggregate, r as SessionScript, T as TraceSpan, E as isProposedCandidate, F as labelTrustRank } from '../types-D9tuReDv.js';
-import { R as RunCampaignOptions, c as RunImprovementLoopOptions, C as CampaignStorage } from '../gepa-3TG16SYf.js';
-export { e as GepaProposerConstraints, G as GepaProposerOptions, O as OpenAutoPrOptions, h as OpenAutoPrResult, b as RunImprovementLoopResult, a as RunOptimizationOptions, j as RunOptimizationResult, k as countSentenceEdits, l as defaultRenderDiff, m as extractH2Sections, f as fsCampaignStorage, g as gepaProposer, i as inMemoryCampaignStorage, o as openAutoPr, r as runCampaign, d as runImprovementLoop, n as runOptimization, s as surfaceHash } from '../gepa-3TG16SYf.js';
-export { A as AxisEvidence, a as AxisVerdict, B as BuildEvidenceVectorOptions, k as BuildLoopProvenanceArgs, D as DefaultProductionGateOptions, l as EmitLoopProvenanceArgs, m as EmitLoopProvenanceResult, E as EvidenceVector, b as EvolutionaryProposerOptions, H as HeldOutGateOptions, n as LoopProvenanceBackend, o as LoopProvenanceCandidate, L as LoopProvenanceRecord, O as ObjectiveSource, P as ParetoSignificanceGateOptions, c as PromotionObjective, d as PromotionPolicy, R as RunEvalOptions, e as buildEvidenceVector, q as buildLoopProvenanceRecord, f as composeGate, g as defaultProductionGate, s as emitLoopProvenance, h as evolutionaryProposer, i as heldOutGate, t as loopProvenanceSpans, p as paretoPolicy, j as paretoSignificanceGate, u as provenanceRecordPath, v as provenanceSpansPath, r as runEval, w as surfaceContentHash } from '../provenance-C5KAhiom.js';
-import { S as SignedManifest, B as BackendIntegrityReport, C as CompletionRequirement, R as RuntimeEventLike, a as CompletionVerdict, P as ProducedState, b as CorrectnessChecker } from '../pre-registration-nfUdc9EQ.js';
+import { S as Scenario, M as MutableSurface, b as DispatchContext, a as JudgeConfig, e as GenerationRecord, g as Gate, J as JudgeScore, L as LabeledScenarioStore, s as LabeledScenarioWrite, t as LabeledScenarioSampleArgs, u as LabeledScenarioRecord, v as LabelTrust, f as SurfaceProposer, w as ProposedCandidate, x as ProposeContext, y as LabeledScenarioSource, C as CampaignResult, o as CodeSurface } from '../types-BTI16iFl.js';
+export { k as CampaignAggregates, l as CampaignArtifactWriter, m as CampaignCellResult, n as CampaignCostMeter, z as CampaignTokenUsage, d as CampaignTraceWriter, D as DispatchFn, h as GateContext, j as GateDecision, G as GateResult, p as GenerationCandidate, A as JudgeAggregate, c as JudgeDimension, i as Mutator, O as OptimizationProposer, q as OptimizerConfig, P as ParetoParent, R as RedactionStatus, B as ScenarioAggregate, r as SessionScript, T as TraceSpan, E as isProposedCandidate, F as labelTrustRank } from '../types-BTI16iFl.js';
+import { R as RunCampaignOptions, c as RunImprovementLoopOptions, C as CampaignStorage } from '../gepa-H6mlM0KN.js';
+export { e as GepaProposerConstraints, G as GepaProposerOptions, O as OpenAutoPrOptions, h as OpenAutoPrResult, b as RunImprovementLoopResult, a as RunOptimizationOptions, j as RunOptimizationResult, k as countSentenceEdits, l as defaultRenderDiff, m as extractH2Sections, f as fsCampaignStorage, g as gepaProposer, i as inMemoryCampaignStorage, o as openAutoPr, r as runCampaign, d as runImprovementLoop, n as runOptimization, s as surfaceHash } from '../gepa-H6mlM0KN.js';
+export { A as AxisEvidence, a as AxisVerdict, B as BuildEvidenceVectorOptions, k as BuildLoopProvenanceArgs, D as DefaultProductionGateOptions, l as EmitLoopProvenanceArgs, m as EmitLoopProvenanceResult, E as EvidenceVector, b as EvolutionaryProposerOptions, H as HeldOutGateOptions, n as LoopProvenanceBackend, o as LoopProvenanceCandidate, L as LoopProvenanceRecord, O as ObjectiveSource, P as ParetoSignificanceGateOptions, c as PromotionObjective, d as PromotionPolicy, R as RunEvalOptions, e as buildEvidenceVector, q as buildLoopProvenanceRecord, f as composeGate, g as defaultProductionGate, s as emitLoopProvenance, h as evolutionaryProposer, i as heldOutGate, t as loopProvenanceSpans, p as paretoPolicy, j as paretoSignificanceGate, u as provenanceRecordPath, v as provenanceSpansPath, r as runEval, w as surfaceContentHash } from '../provenance-Bg_RttR8.js';
 import { E as EProcessState, a as PairedBootstrapResult } from '../statistics-xP-cWc5k.js';
 import { L as LlmClientOptions } from '../llm-client-Bj7g0rqu.js';
 import { AgentProfile } from '@tangle-network/agent-interface';
@@ -12,6 +13,8 @@ import { A as AgentEvalError } from '../errors-CzMUYo7b.js';
 import { a as RunSplitTag, R as RunRecord } from '../run-record-CP2ObebC.js';
 import { T as TraceAnalystKindSpec } from '../kind-factory-X3eDYbKn.js';
 import { c as AnalystFinding } from '../types-B5x54y6n.js';
+import '@tangle-network/tcloud';
+import '../verdict-C9MlYujm.js';
 import '@ax-llm/ax';
 import '../store-C1YxJDEK.js';
 import '../red-team-BWdoyleI.js';
@@ -24,8 +27,6 @@ import '../insight-report-BnRjTibG.js';
 import '../summary-report-CInXwsza.js';
 import '../failure-cluster-DH9Flgcf.js';
 import '../judge-calibration-0p2QcWNE.js';
-import '@tangle-network/tcloud';
-import '../verdict-C9MlYujm.js';
 import '../types-C7DGg5ex.js';
 import '../raw-provider-sink-C46HDghv.js';
 import 'zod';

package/dist/campaign/index.js

@@ -16,8 +16,9 @@ import {
   agentProfileId,
   agentProfileModelId,
   extractProducedState,
+  llmJudge,
   verifyCompletion
-} from "../chunk-HMA63UEO.js";
+} from "../chunk-OYU4D7FY.js";
 import {
   buildLoopProvenanceRecord,
   campaignBreakdown,
@@ -51,6 +52,7 @@ import {
   estimateCost,
   isModelPriced
 } from "../chunk-VI2UW6B6.js";
+import "../chunk-BUTW4RGG.js";
 import {
   AnalystRegistry,
   DEFAULT_TRACE_ANALYST_KINDS,
@@ -2498,6 +2500,7 @@ export {
   inMemoryCampaignStorage,
   isProposedCandidate,
   labelTrustRank,
+  llmJudge,
   loopProvenanceSpans,
   makePlaybackDispatch,
   memoryCurationProposer,