@tangle-network/agent-eval 0.106.3 → 0.108.0

package/dist/chunk-6QDKWHLS.js

@@ -1,223 +0,0 @@
-import {
-  __export
-} from "./chunk-PZ5AY32C.js";
-
-// src/benchmarks/index.ts
-var benchmarks_exports = {};
-__export(benchmarks_exports, {
-  BENCHMARK_SPLIT_SEED: () => BENCHMARK_SPLIT_SEED,
-  deterministicSplit: () => deterministicSplit,
-  routing: () => routing_exports
-});
-
-// src/benchmarks/routing/index.ts
-var routing_exports = {};
-__export(routing_exports, {
-  ROUTING_DATASET: () => ROUTING_DATASET,
-  RoutingAdapter: () => RoutingAdapter,
-  assignSplit: () => assignSplit,
-  evaluate: () => evaluate,
-  extractRouteTokens: () => extractRouteTokens,
-  loadDataset: () => loadDataset
-});
-
-// src/benchmarks/types.ts
-function fnv1a32(input) {
-  let h = 2166136261;
-  for (let i = 0; i < input.length; i++) {
-    h ^= input.charCodeAt(i) & 255;
-    h = h + ((h << 1) + (h << 4) + (h << 7) + (h << 8) + (h << 24)) >>> 0;
-  }
-  return h >>> 0;
-}
-var BENCHMARK_SPLIT_SEED = "agent-eval-v1";
-function deterministicSplit(itemId, seed = BENCHMARK_SPLIT_SEED) {
-  const h = fnv1a32(`${seed}::${itemId}`);
-  const pos = h / 4294967296;
-  if (pos < 0.6) return "search";
-  if (pos < 0.8) return "dev";
-  return "holdout";
-}
-
-// src/benchmarks/routing/dataset.ts
-var ROUTING_DATASET = [
-  {
-    id: "file_001",
-    category: "file",
-    prompt: "Save the meeting notes to /tmp/notes-2025-04.md as markdown.",
-    route: "fs.write",
-    synonyms: ["filesystem.write", "write_file"],
-    hardNegatives: ["fs.read", "chat.reply"]
-  },
-  {
-    id: "file_002",
-    category: "file",
-    prompt: "Read the contents of /etc/hosts and summarize the entries.",
-    route: "fs.read",
-    synonyms: ["filesystem.read", "read_file"],
-    hardNegatives: ["fs.write", "search.web"]
-  },
-  {
-    id: "file_003",
-    category: "file",
-    prompt: "List every Python file under src/ recursively.",
-    route: "fs.list",
-    synonyms: ["filesystem.list", "list_files"],
-    hardNegatives: ["fs.read", "search.code"]
-  },
-  {
-    id: "file_004",
-    category: "file",
-    prompt: "Delete the cached build at .turbo/cache.",
-    route: "fs.delete",
-    synonyms: ["filesystem.delete", "remove_file"],
-    hardNegatives: ["fs.write", "fs.list"]
-  },
-  {
-    id: "math_001",
-    category: "math",
-    prompt: "What is the integral of 3x^2 + 2x from 0 to 5?",
-    route: "math.integral",
-    synonyms: ["calculator.integral", "math.solve"],
-    hardNegatives: ["math.derivative", "chat.reply"]
-  },
-  {
-    id: "math_002",
-    category: "math",
-    prompt: "Compute the derivative of sin(x) * cos(x).",
-    route: "math.derivative",
-    synonyms: ["calculator.derivative", "math.solve"],
-    hardNegatives: ["math.integral", "math.algebra"]
-  },
-  {
-    id: "math_003",
-    category: "math",
-    prompt: "Solve 2x + 7 = 19 for x.",
-    route: "math.algebra",
-    synonyms: ["calculator.algebra", "math.solve"],
-    hardNegatives: ["math.derivative", "math.integral"]
-  },
-  {
-    id: "math_004",
-    category: "math",
-    prompt: "What is the prime factorization of 360?",
-    route: "math.numbertheory",
-    synonyms: ["calculator.factor", "math.solve"],
-    hardNegatives: ["math.algebra", "search.web"]
-  },
-  {
-    id: "search_001",
-    category: "search",
-    prompt: "Find recent papers on agent prompt optimization with held-out promotion gates.",
-    route: "search.web",
-    synonyms: ["web.search", "search.papers"],
-    hardNegatives: ["search.code", "chat.reply"]
-  },
-  {
-    id: "search_002",
-    category: "search",
-    prompt: "Search the codebase for every call site of `runProposeReview`.",
-    route: "search.code",
-    synonyms: ["code.search", "grep"],
-    hardNegatives: ["search.web", "fs.read"]
-  },
-  {
-    id: "search_003",
-    category: "search",
-    prompt: "What is the latest release of the Tangle network on GitHub?",
-    route: "search.web",
-    synonyms: ["web.search", "github.releases"],
-    hardNegatives: ["search.code", "chat.reply"]
-  },
-  {
-    id: "search_004",
-    category: "search",
-    prompt: "Find all TODO comments in the agent-eval src tree.",
-    route: "search.code",
-    synonyms: ["code.search", "grep"],
-    hardNegatives: ["search.web", "fs.list"]
-  },
-  {
-    id: "chat_001",
-    category: "chat",
-    prompt: "Hi there, how are you doing today?",
-    route: "chat.reply",
-    synonyms: ["conversation.reply"],
-    hardNegatives: ["search.web", "fs.read"]
-  },
-  {
-    id: "chat_002",
-    category: "chat",
-    prompt: "Please explain the difference between an LLM and a foundation model.",
-    route: "chat.reply",
-    synonyms: ["conversation.reply", "qa.answer"],
-    hardNegatives: ["search.web", "math.algebra"]
-  },
-  {
-    id: "chat_003",
-    category: "chat",
-    prompt: "Tell me a short joke about distributed systems.",
-    route: "chat.reply",
-    synonyms: ["conversation.reply"],
-    hardNegatives: ["search.web", "fs.read"]
-  },
-  {
-    id: "chat_004",
-    category: "chat",
-    prompt: "Acknowledge my last message with a thumbs up.",
-    route: "chat.reply",
-    synonyms: ["conversation.reply", "react"],
-    hardNegatives: ["fs.write", "search.web"]
-  }
-];
-
-// src/benchmarks/routing/index.ts
-var RoutingAdapter = class {
-  async loadDataset(split) {
-    return ROUTING_DATASET.map((item) => ({ id: item.id, payload: item })).filter(
-      (it) => assignSplitImpl(it.id) === split
-    );
-  }
-  async evaluate(item, response) {
-    const tokens = extractRouteTokens(response);
-    const correct = new Set(
-      [item.payload.route, ...item.payload.synonyms].map((s) => s.toLowerCase())
-    );
-    const hardNeg = new Set(item.payload.hardNegatives.map((s) => s.toLowerCase()));
-    const firstMatch = tokens.find((t) => correct.has(t.toLowerCase())) ?? null;
-    const firstHardNeg = tokens.find((t) => hardNeg.has(t.toLowerCase())) ?? null;
-    const score = firstMatch ? 1 : 0;
-    return {
-      score,
-      raw: {
-        firstToken: tokens[0] ?? null,
-        matchedRoute: firstMatch,
-        hitHardNegative: Boolean(firstHardNeg),
-        hardNegativeRoute: firstHardNeg,
-        category: item.payload.category
-      }
-    };
-  }
-  assignSplit(itemId) {
-    return assignSplitImpl(itemId);
-  }
-};
-function assignSplitImpl(itemId) {
-  return deterministicSplit(`routing::${itemId}`);
-}
-function extractRouteTokens(response) {
-  const matches = response.match(/[a-z][a-z0-9_]*\.[a-z][a-z0-9_]*/gi);
-  return matches ?? [];
-}
-var adapter = new RoutingAdapter();
-var loadDataset = adapter.loadDataset.bind(adapter);
-var evaluate = adapter.evaluate.bind(adapter);
-var assignSplit = adapter.assignSplit.bind(adapter);
-
-export {
-  BENCHMARK_SPLIT_SEED,
-  deterministicSplit,
-  routing_exports,
-  benchmarks_exports
-};
-//# sourceMappingURL=chunk-6QDKWHLS.js.map

package/dist/chunk-6QDKWHLS.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/benchmarks/index.ts","../src/benchmarks/routing/index.ts","../src/benchmarks/types.ts","../src/benchmarks/routing/dataset.ts"],"sourcesContent":["/**\n * Reference benchmark wrappers — entry point.\n *\n * Core surface (exported here):\n *   - The `BenchmarkAdapter` contract.\n *   - `deterministicSplit` + `BENCHMARK_SPLIT_SEED` for split assignment.\n *   - `routing` — synthetic 16-task router benchmark. The only novel\n *     benchmark we built; ships in the package.\n *\n * Example wrappers (under `examples/benchmarks/`, NOT in the bundle):\n *   - `gsm8k`         — exact-match math reasoning (HF mirror, dataset\n *                       not bundled).\n *   - `swebench-lite` — 30-instance SWE-Bench subset via an external\n *                       grader command.\n *\n * The example wrappers are reference implementations of `BenchmarkAdapter`.\n * Read them, copy them, adapt them. They're intentionally not in the main\n * entry — every team will configure them differently.\n */\n\nexport * as routing from './routing/index'\nexport type {\n  BenchmarkAdapter,\n  BenchmarkDatasetItem,\n  BenchmarkEvaluation,\n} from './types'\nexport { BENCHMARK_SPLIT_SEED, deterministicSplit } from './types'\n","/**\n * Routing benchmark — synthetic, dependency-free, ships in the\n * package. 16 cross-category items in `dataset.ts`. See\n * `routing/README.md` for the format.\n *\n * `evaluate` does case-insensitive exact match against the canonical\n * route plus declared synonyms. The first valid route token in the\n * response wins; everything else is ignored. Wrong answers also\n * report whether they hit a hard negative — useful when triaging\n * \"always picks the popular route\" failure modes.\n */\n\nimport type { RunSplitTag } from '../../run-record'\nimport type { BenchmarkAdapter, BenchmarkDatasetItem, BenchmarkEvaluation } from '../types'\nimport { deterministicSplit } from '../types'\nimport { ROUTING_DATASET, type RoutingItem } from './dataset'\n\nexport type { RoutingItem }\nexport type RoutingPayload = RoutingItem\nexport type RoutingDatasetItem = BenchmarkDatasetItem<RoutingPayload>\n\nclass RoutingAdapter implements BenchmarkAdapter<RoutingDatasetItem, RoutingPayload> {\n  async loadDataset(split: RunSplitTag): Promise<RoutingDatasetItem[]> {\n    return ROUTING_DATASET.map((item) => ({ id: item.id, payload: item })).filter(\n      (it) => assignSplitImpl(it.id) === split,\n    )\n  }\n\n  async evaluate(item: RoutingDatasetItem, response: string): Promise<BenchmarkEvaluation> {\n    const tokens = extractRouteTokens(response)\n    const correct = new Set<string>(\n      [item.payload.route, ...item.payload.synonyms].map((s) => s.toLowerCase()),\n    )\n    const hardNeg = new Set<string>(item.payload.hardNegatives.map((s) => s.toLowerCase()))\n    const firstMatch = tokens.find((t) => correct.has(t.toLowerCase())) ?? null\n    const firstHardNeg = tokens.find((t) => hardNeg.has(t.toLowerCase())) ?? null\n    const score = firstMatch ? 1 : 0\n    return {\n      score,\n      raw: {\n        firstToken: tokens[0] ?? null,\n        matchedRoute: firstMatch,\n        hitHardNegative: Boolean(firstHardNeg),\n        hardNegativeRoute: firstHardNeg,\n        category: item.payload.category,\n      },\n    }\n  }\n\n  assignSplit(itemId: string): RunSplitTag {\n    return assignSplitImpl(itemId)\n  }\n}\n\nfunction assignSplitImpl(itemId: string): RunSplitTag {\n  return deterministicSplit(`routing::${itemId}`)\n}\n\n/**\n * Pull route-shaped tokens out of a model response. Routes look like\n * `category.action` (`fs.write`, `chat.reply`). Bare alphanumerics\n * are not routes, but `category.action` patterns are robust to most\n * model wrappers (JSON output, prose explanations, code fences).\n */\nexport function extractRouteTokens(response: string): string[] {\n  const matches = response.match(/[a-z][a-z0-9_]*\\.[a-z][a-z0-9_]*/gi)\n  return matches ?? []\n}\n\nconst adapter = new RoutingAdapter()\n\nexport const loadDataset = adapter.loadDataset.bind(adapter)\nexport const evaluate = adapter.evaluate.bind(adapter)\nexport const assignSplit = adapter.assignSplit.bind(adapter)\nexport { ROUTING_DATASET, RoutingAdapter }\n","/**\n * Shared types for the reference benchmark wrappers under\n * `src/benchmarks/`. Each wrapper exports the three functions in\n * `BenchmarkAdapter` plus its own typed `DatasetItem` shape.\n */\n\nimport type { RunSplitTag } from '../run-record'\n\nexport interface BenchmarkDatasetItem<TPayload = unknown> {\n  /** Stable dataset-local item id (used for split assignment + paper\n   *  references). Unique within a benchmark. */\n  id: string\n  /** Free-form payload. Each benchmark defines its own shape. */\n  payload: TPayload\n}\n\nexport interface BenchmarkEvaluation {\n  /** [0, 1] score for the response on this item. Exact-match\n   *  benchmarks use 0/1; partial-credit benchmarks may return\n   *  fractional values. */\n  score: number\n  /** Optional bag of raw scoring signals — e.g. parsed numeric\n   *  answer, regex match, judge sub-scores. */\n  raw: Record<string, unknown>\n}\n\n/** Common signature implemented by every adapter under `src/benchmarks/*`. */\n// `TPayload` is the per-item payload type; `_TItem` is preserved for\n// downstream type-narrowing extensions (a richer `BenchmarkDatasetItem`\n// subclass that adds e.g. provenance metadata) but is intentionally\n// unused here. `noUnusedLocals` requires the leading underscore.\nexport interface BenchmarkAdapter<_TItem = unknown, TPayload = unknown> {\n  /** Load the dataset for the given split. May hit the network on\n   *  first call but should be cache-friendly. Adapters that don't\n   *  ship the dataset itself MUST throw a clearly-marked error\n   *  pointing the caller at the loader script. */\n  loadDataset(split: RunSplitTag): Promise<BenchmarkDatasetItem<TPayload>[]>\n  /** Score a single response. Pure with respect to the inputs. */\n  evaluate(item: BenchmarkDatasetItem<TPayload>, response: string): Promise<BenchmarkEvaluation>\n  /** Deterministic split assignment via item id hashing. The\n   *  fraction of items in each split is implementation-defined but\n   *  MUST be stable across processes and platforms. */\n  assignSplit(itemId: string): RunSplitTag\n}\n\n// ── Deterministic split assignment ───────────────────────────────────\n\n/**\n * 32-bit FNV-1a hash. Stable, allocation-free, deterministic across\n * runtimes. We use it to assign items to splits rather than depending\n * on a polyfilled crypto.subtle path.\n */\nfunction fnv1a32(input: string): number {\n  let h = 0x811c9dc5\n  for (let i = 0; i < input.length; i++) {\n    h ^= input.charCodeAt(i) & 0xff\n    h = (h + ((h << 1) + (h << 4) + (h << 7) + (h << 8) + (h << 24))) >>> 0\n  }\n  return h >>> 0\n}\n\n/** Split-assignment seed shared across all benchmarks. Bumping this\n *  value reshuffles every split — do NOT do that lightly. */\nexport const BENCHMARK_SPLIT_SEED = 'agent-eval-v1'\n\n/**\n * Assign an item id to one of `'search' | 'dev' | 'holdout'` using a\n * stable 32-bit hash of `${seed}::${id}`. Default proportions:\n *\n *   search: 60%   (optimization-readable)\n *   dev:    20%   (held-out for tuning, leak-on-purpose during dev)\n *   holdout:20%   (paper-grade held-out, gated reads)\n */\nexport function deterministicSplit(\n  itemId: string,\n  seed: string = BENCHMARK_SPLIT_SEED,\n): RunSplitTag {\n  const h = fnv1a32(`${seed}::${itemId}`)\n  const pos = h / 0x100000000\n  if (pos < 0.6) return 'search'\n  if (pos < 0.8) return 'dev'\n  return 'holdout'\n}\n","/**\n * Synthetic routing dataset. 16 tasks across 4 categories. Used as a\n * deterministic, dependency-free benchmark for any router that maps a\n * natural-language request to one of a fixed set of route labels.\n *\n * Format (see `routing/README.md` for prose):\n *\n *   {\n *     id:           stable per-task ID (matches across processes).\n *     category:     one of the four route labels.\n *     prompt:       the user-facing request the router must classify.\n *     route:        the ground-truth route the router should pick.\n *     synonyms:     other strings that count as a correct answer.\n *     hardNegatives:close-but-wrong route labels — used to detect the\n *                   \"always picks the popular route\" failure mode.\n *   }\n *\n * The four categories are intentionally cross-domain (file ops,\n * math, search, conversation) so a router that collapses to one\n * category is easy to spot.\n */\n\nexport interface RoutingItem {\n  id: string\n  category: 'file' | 'math' | 'search' | 'chat'\n  prompt: string\n  /** Canonical correct route label. */\n  route: string\n  /** Alternate route labels that also count as correct. */\n  synonyms: string[]\n  /** Wrong-but-tempting route labels (for analysis, not grading). */\n  hardNegatives: string[]\n}\n\nexport const ROUTING_DATASET: RoutingItem[] = [\n  {\n    id: 'file_001',\n    category: 'file',\n    prompt: 'Save the meeting notes to /tmp/notes-2025-04.md as markdown.',\n    route: 'fs.write',\n    synonyms: ['filesystem.write', 'write_file'],\n    hardNegatives: ['fs.read', 'chat.reply'],\n  },\n  {\n    id: 'file_002',\n    category: 'file',\n    prompt: 'Read the contents of /etc/hosts and summarize the entries.',\n    route: 'fs.read',\n    synonyms: ['filesystem.read', 'read_file'],\n    hardNegatives: ['fs.write', 'search.web'],\n  },\n  {\n    id: 'file_003',\n    category: 'file',\n    prompt: 'List every Python file under src/ recursively.',\n    route: 'fs.list',\n    synonyms: ['filesystem.list', 'list_files'],\n    hardNegatives: ['fs.read', 'search.code'],\n  },\n  {\n    id: 'file_004',\n    category: 'file',\n    prompt: 'Delete the cached build at .turbo/cache.',\n    route: 'fs.delete',\n    synonyms: ['filesystem.delete', 'remove_file'],\n    hardNegatives: ['fs.write', 'fs.list'],\n  },\n  {\n    id: 'math_001',\n    category: 'math',\n    prompt: 'What is the integral of 3x^2 + 2x from 0 to 5?',\n    route: 'math.integral',\n    synonyms: ['calculator.integral', 'math.solve'],\n    hardNegatives: ['math.derivative', 'chat.reply'],\n  },\n  {\n    id: 'math_002',\n    category: 'math',\n    prompt: 'Compute the derivative of sin(x) * cos(x).',\n    route: 'math.derivative',\n    synonyms: ['calculator.derivative', 'math.solve'],\n    hardNegatives: ['math.integral', 'math.algebra'],\n  },\n  {\n    id: 'math_003',\n    category: 'math',\n    prompt: 'Solve 2x + 7 = 19 for x.',\n    route: 'math.algebra',\n    synonyms: ['calculator.algebra', 'math.solve'],\n    hardNegatives: ['math.derivative', 'math.integral'],\n  },\n  {\n    id: 'math_004',\n    category: 'math',\n    prompt: 'What is the prime factorization of 360?',\n    route: 'math.numbertheory',\n    synonyms: ['calculator.factor', 'math.solve'],\n    hardNegatives: ['math.algebra', 'search.web'],\n  },\n  {\n    id: 'search_001',\n    category: 'search',\n    prompt: 'Find recent papers on agent prompt optimization with held-out promotion gates.',\n    route: 'search.web',\n    synonyms: ['web.search', 'search.papers'],\n    hardNegatives: ['search.code', 'chat.reply'],\n  },\n  {\n    id: 'search_002',\n    category: 'search',\n    prompt: 'Search the codebase for every call site of `runProposeReview`.',\n    route: 'search.code',\n    synonyms: ['code.search', 'grep'],\n    hardNegatives: ['search.web', 'fs.read'],\n  },\n  {\n    id: 'search_003',\n    category: 'search',\n    prompt: 'What is the latest release of the Tangle network on GitHub?',\n    route: 'search.web',\n    synonyms: ['web.search', 'github.releases'],\n    hardNegatives: ['search.code', 'chat.reply'],\n  },\n  {\n    id: 'search_004',\n    category: 'search',\n    prompt: 'Find all TODO comments in the agent-eval src tree.',\n    route: 'search.code',\n    synonyms: ['code.search', 'grep'],\n    hardNegatives: ['search.web', 'fs.list'],\n  },\n  {\n    id: 'chat_001',\n    category: 'chat',\n    prompt: 'Hi there, how are you doing today?',\n    route: 'chat.reply',\n    synonyms: ['conversation.reply'],\n    hardNegatives: ['search.web', 'fs.read'],\n  },\n  {\n    id: 'chat_002',\n    category: 'chat',\n    prompt: 'Please explain the difference between an LLM and a foundation model.',\n    route: 'chat.reply',\n    synonyms: ['conversation.reply', 'qa.answer'],\n    hardNegatives: ['search.web', 'math.algebra'],\n  },\n  {\n    id: 'chat_003',\n    category: 'chat',\n    prompt: 'Tell me a short joke about distributed systems.',\n    route: 'chat.reply',\n    synonyms: ['conversation.reply'],\n    hardNegatives: ['search.web', 'fs.read'],\n  },\n  {\n    id: 'chat_004',\n    category: 'chat',\n    prompt: 'Acknowledge my last message with a thumbs up.',\n    route: 'chat.reply',\n    synonyms: ['conversation.reply', 'react'],\n    hardNegatives: ['fs.write', 'search.web'],\n  },\n]\n"],"mappings":";;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;;;ACAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;;;ACoDA,SAAS,QAAQ,OAAuB;AACtC,MAAI,IAAI;AACR,WAAS,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK;AACrC,SAAK,MAAM,WAAW,CAAC,IAAI;AAC3B,QAAK,MAAM,KAAK,MAAM,KAAK,MAAM,KAAK,MAAM,KAAK,MAAM,KAAK,SAAU;AAAA,EACxE;AACA,SAAO,MAAM;AACf;AAIO,IAAM,uBAAuB;AAU7B,SAAS,mBACd,QACA,OAAe,sBACF;AACb,QAAM,IAAI,QAAQ,GAAG,IAAI,KAAK,MAAM,EAAE;AACtC,QAAM,MAAM,IAAI;AAChB,MAAI,MAAM,IAAK,QAAO;AACtB,MAAI,MAAM,IAAK,QAAO;AACtB,SAAO;AACT;;;AChDO,IAAM,kBAAiC;AAAA,EAC5C;AAAA,IACE,IAAI;AAAA,IACJ,UAAU;AAAA,IACV,QAAQ;AAAA,IACR,OAAO;AAAA,IACP,UAAU,CAAC,oBAAoB,YAAY;AAAA,IAC3C,eAAe,CAAC,WAAW,YAAY;AAAA,EACzC;AAAA,EACA;AAAA,IACE,IAAI;AAAA,IACJ,UAAU;AAAA,IACV,QAAQ;AAAA,IACR,OAAO;AAAA,IACP,UAAU,CAAC,mBAAmB,WAAW;AAAA,IACzC,eAAe,CAAC,YAAY,YAAY;AAAA,EAC1C;AAAA,EACA;AAAA,IACE,IAAI;AAAA,IACJ,UAAU;AAAA,IACV,QAAQ;AAAA,IACR,OAAO;AAAA,IACP,UAAU,CAAC,mBAAmB,YAAY;AAAA,IAC1C,eAAe,CAAC,WAAW,aAAa;AAAA,EAC1C;AAAA,EACA;AAAA,IACE,IAAI;AAAA,IACJ,UAAU;AAAA,IACV,QAAQ;AAAA,IACR,OAAO;AAAA,IACP,UAAU,CAAC,qBAAqB,aAAa;AAAA,IAC7C,eAAe,CAAC,YAAY,SAAS;AAAA,EACvC;AAAA,EACA;AAAA,IACE,IAAI;AAAA,IACJ,UAAU;AAAA,IACV,QAAQ;AAAA,IACR,OAAO;AAAA,IACP,UAAU,CAAC,uBAAuB,YAAY;AAAA,IAC9C,eAAe,CAAC,mBAAmB,YAAY;AAAA,EACjD;AAAA,EACA;AAAA,IACE,IAAI;AAAA,IACJ,UAAU;AAAA,IACV,QAAQ;AAAA,IACR,OAAO;AAAA,IACP,UAAU,CAAC,yBAAyB,YAAY;AAAA,IAChD,eAAe,CAAC,iBAAiB,cAAc;AAAA,EACjD;AAAA,EACA;AAAA,IACE,IAAI;AAAA,IACJ,UAAU;AAAA,IACV,QAAQ;AAAA,IACR,OAAO;AAAA,IACP,UAAU,CAAC,sBAAsB,YAAY;AAAA,IAC7C,eAAe,CAAC,mBAAmB,eAAe;AAAA,EACpD;AAAA,EACA;AAAA,IACE,IAAI;AAAA,IACJ,UAAU;AAAA,IACV,QAAQ;AAAA,IACR,OAAO;AAAA,IACP,UAAU,CAAC,qBAAqB,YAAY;AAAA,IAC5C,eAAe,CAAC,gBAAgB,YAAY;AAAA,EAC9C;AAAA,EACA;AAAA,IACE,IAAI;AAAA,IACJ,UAAU;AAAA,IACV,QAAQ;AAAA,IACR,OAAO;AAAA,IACP,UAAU,CAAC,cAAc,eAAe;AAAA,IACxC,eAAe,CAAC,eAAe,YAAY;AAAA,EAC7C;AAAA,EACA;AAAA,IACE,IAAI;AAAA,IACJ,UAAU;AAAA,IACV,QAAQ;AAAA,IACR,OAAO;AAAA,IACP,UAAU,CAAC,eAAe,MAAM;AAAA,IAChC,eAAe,CAAC,cAAc,SAAS;AAAA,EACzC;AAAA,EACA;AAAA,IACE,IAAI;AAAA,IACJ,UAAU;AAAA,IACV,QAAQ;AAAA,IACR,OAAO;AAAA,IACP,UAAU,CAAC,cAAc,iBAAiB;AAAA,IAC1C,eAAe,CAAC,eAAe,YAAY;AAAA,EAC7C;AAAA,EACA;AAAA,IACE,IAAI;AAAA,IACJ,UAAU;AAAA,IACV,QAAQ;AAAA,IACR,OAAO;AAAA,IACP,UAAU,CAAC,eAAe,MAAM;AAAA,IAChC,eAAe,CAAC,cAAc,SAAS;AAAA,EACzC;AAAA,EACA;AAAA,IACE,IAAI;AAAA,IACJ,UAAU;AAAA,IACV,QAAQ;AAAA,IACR,OAAO;AAAA,IACP,UAAU,CAAC,oBAAoB;AAAA,IAC/B,eAAe,CAAC,cAAc,SAAS;AAAA,EACzC;AAAA,EACA;AAAA,IACE,IAAI;AAAA,IACJ,UAAU;AAAA,IACV,QAAQ;AAAA,IACR,OAAO;AAAA,IACP,UAAU,CAAC,sBAAsB,WAAW;AAAA,IAC5C,eAAe,CAAC,cAAc,cAAc;AAAA,EAC9C;AAAA,EACA;AAAA,IACE,IAAI;AAAA,IACJ,UAAU;AAAA,IACV,QAAQ;AAAA,IACR,OAAO;AAAA,IACP,UAAU,CAAC,oBAAoB;AAAA,IAC/B,eAAe,CAAC,cAAc,SAAS;AAAA,EACzC;AAAA,EACA;AAAA,IACE,IAAI;AAAA,IACJ,UAAU;AAAA,IACV,QAAQ;AAAA,IACR,OAAO;AAAA,IACP,UAAU,CAAC,sBAAsB,OAAO;AAAA,IACxC,eAAe,CAAC,YAAY,YAAY;AAAA,EAC1C;AACF;;;AF9IA,IAAM,iBAAN,MAAqF;AAAA,EACnF,MAAM,YAAY,OAAmD;AACnE,WAAO,gBAAgB,IAAI,CAAC,UAAU,EAAE,IAAI,KAAK,IAAI,SAAS,KAAK,EAAE,EAAE;AAAA,MACrE,CAAC,OAAO,gBAAgB,GAAG,EAAE,MAAM;AAAA,IACrC;AAAA,EACF;AAAA,EAEA,MAAM,SAAS,MAA0B,UAAgD;AACvF,UAAM,SAAS,mBAAmB,QAAQ;AAC1C,UAAM,UAAU,IAAI;AAAA,MAClB,CAAC,KAAK,QAAQ,OAAO,GAAG,KAAK,QAAQ,QAAQ,EAAE,IAAI,CAAC,MAAM,EAAE,YAAY,CAAC;AAAA,IAC3E;AACA,UAAM,UAAU,IAAI,IAAY,KAAK,QAAQ,cAAc,IAAI,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;AACtF,UAAM,aAAa,OAAO,KAAK,CAAC,MAAM,QAAQ,IAAI,EAAE,YAAY,CAAC,CAAC,KAAK;AACvE,UAAM,eAAe,OAAO,KAAK,CAAC,MAAM,QAAQ,IAAI,EAAE,YAAY,CAAC,CAAC,KAAK;AACzE,UAAM,QAAQ,aAAa,IAAI;AAC/B,WAAO;AAAA,MACL;AAAA,MACA,KAAK;AAAA,QACH,YAAY,OAAO,CAAC,KAAK;AAAA,QACzB,cAAc;AAAA,QACd,iBAAiB,QAAQ,YAAY;AAAA,QACrC,mBAAmB;AAAA,QACnB,UAAU,KAAK,QAAQ;AAAA,MACzB;AAAA,IACF;AAAA,EACF;AAAA,EAEA,YAAY,QAA6B;AACvC,WAAO,gBAAgB,MAAM;AAAA,EAC/B;AACF;AAEA,SAAS,gBAAgB,QAA6B;AACpD,SAAO,mBAAmB,YAAY,MAAM,EAAE;AAChD;AAQO,SAAS,mBAAmB,UAA4B;AAC7D,QAAM,UAAU,SAAS,MAAM,oCAAoC;AACnE,SAAO,WAAW,CAAC;AACrB;AAEA,IAAM,UAAU,IAAI,eAAe;AAE5B,IAAM,cAAc,QAAQ,YAAY,KAAK,OAAO;AACpD,IAAM,WAAW,QAAQ,SAAS,KAAK,OAAO;AAC9C,IAAM,cAAc,QAAQ,YAAY,KAAK,OAAO;","names":[]}

package/dist/chunk-J22CKVXN.js

@@ -1,421 +0,0 @@
-import {
-  TIE_WARN_FRACTION,
-  detectScale,
-  dimensionRegressions,
-  heldoutSignificance,
-  pairHoldout,
-  runCanaries,
-  scoreRedTeamOutput
-} from "./chunk-3E5KUXYZ.js";
-import {
-  runCampaign
-} from "./chunk-3PFZBGMR.js";
-import {
-  detectRewardHacking
-} from "./chunk-N22ZO7FV.js";
-import {
-  pairedBootstrap
-} from "./chunk-XMSYF4A7.js";
-
-// src/campaign/gates/compose.ts
-function composeGate(...gates) {
-  if (gates.length === 0) {
-    throw new Error("composeGate requires at least one gate");
-  }
-  return {
-    name: `composed(${gates.map((g) => g.name).join(",")})`,
-    async decide(ctx) {
-      const results = [];
-      for (const gate of gates) {
-        const res = await gate.decide(ctx);
-        results.push({ gate, res });
-      }
-      const decisions = results.map((r) => r.res.decision);
-      const overall = decisions.every((d) => d === "ship") ? "ship" : decisions.includes("arch_ceiling") ? "arch_ceiling" : decisions.includes("model_ceiling") ? "model_ceiling" : decisions.includes("hold") ? "hold" : "need_more_work";
-      const contributing = results.flatMap(
-        (r) => r.res.contributingGates.length > 0 ? r.res.contributingGates : [{ name: r.gate.name, passed: r.res.decision === "ship", detail: r.res }]
-      );
-      const reasons = results.flatMap(
-        (r) => r.res.reasons.map((reason) => `[${r.gate.name}] ${reason}`)
-      );
-      return {
-        decision: overall,
-        reasons,
-        contributingGates: contributing,
-        delta: results[0]?.res.delta
-      };
-    }
-  };
-}
-
-// src/campaign/gates/default-production-gate.ts
-function defaultProductionGate(options) {
-  const deltaThreshold = options.deltaThreshold ?? 0;
-  const confidence = options.confidence ?? 0.95;
-  const resamples = options.bootstrapResamples ?? 2e3;
-  const seed = options.bootstrapSeed ?? 1337;
-  const minProductiveRuns = options.minProductiveRuns ?? 3;
-  const heldoutStatistic = options.heldoutStatistic ?? "mean";
-  const blockOnGaming = options.blockOnRewardHackingGaming ?? true;
-  return {
-    name: "defaultProductionGate",
-    async decide(ctx) {
-      const reasons = [];
-      const contributing = [];
-      const scenarioIds = new Set(options.holdoutScenarios.map((s) => s.id));
-      const sig = heldoutSignificance(
-        pairHoldout(
-          ctx.judgeScores,
-          ctx.baselineJudgeScores ?? ctx.judgeScores,
-          scenarioIds,
-          (s) => s.composite
-        ),
-        {
-          deltaThreshold,
-          minProductiveRuns,
-          confidence,
-          resamples,
-          seed,
-          statistic: heldoutStatistic
-        }
-      );
-      const delta = heldoutStatistic === "median" ? sig.bootstrap.median : sig.bootstrap.mean;
-      const heldoutPass = sig.significant;
-      contributing.push({
-        name: "heldout-significance",
-        passed: heldoutPass,
-        detail: {
-          n: sig.n,
-          delta,
-          deltaMean: sig.bootstrap.mean,
-          deltaMedianDiagnostic: sig.medianBootstrap.median,
-          // Back-compat: prior consumers read `deltaMedian`. It now always carries
-          // the median diagnostic (the ship decision keys on `delta`/mean).
-          deltaMedian: sig.medianBootstrap.median,
-          tieFraction: sig.tieFraction,
-          ciLow: sig.bootstrap.low,
-          ciHigh: sig.bootstrap.high,
-          confidence: sig.bootstrap.confidence,
-          deltaThreshold,
-          fewRuns: sig.fewRuns
-        }
-      });
-      if (!heldoutPass) {
-        const tieNote = sig.tieFraction >= TIE_WARN_FRACTION ? `; ${(sig.tieFraction * 100).toFixed(0)}% tied scenarios` : "";
-        reasons.push(
-          sig.fewRuns ? `held-out: only ${sig.n} paired runs (< ${minProductiveRuns}) \u2014 too few to claim significance` : `held-out CI.low ${sig.bootstrap.low.toFixed(3)} \u2264 threshold ${deltaThreshold} (${heldoutStatistic} \u0394 ${delta.toFixed(3)}, ${(sig.bootstrap.confidence * 100).toFixed(0)}% CI [${sig.bootstrap.low.toFixed(3)}, ${sig.bootstrap.high.toFixed(3)}]${tieNote})`
-        );
-      }
-      const dimRegs = options.criticalDimensions?.length ? dimensionRegressions(
-        ctx.judgeScores,
-        ctx.baselineJudgeScores ?? ctx.judgeScores,
-        scenarioIds,
-        options.criticalDimensions,
-        { tolerance: options.regressionTolerance, confidence, resamples, seed }
-      ) : [];
-      const regressed = dimRegs.filter((d) => d.regressed);
-      const dimPass = regressed.length === 0;
-      contributing.push({
-        name: "dimension-regression",
-        passed: dimPass,
-        detail: {
-          guarded: options.criticalDimensions ?? [],
-          regressions: dimRegs.map((d) => ({
-            dimension: d.dimension,
-            ciLow: d.bootstrap.low,
-            median: d.bootstrap.median,
-            tolerance: d.tolerance,
-            n: d.n,
-            regressed: d.regressed
-          }))
-        }
-      });
-      if (!dimPass) {
-        reasons.push(
-          `critical dimension(s) regressed: ${regressed.map((d) => `${d.dimension} CI.low ${d.bootstrap.low.toFixed(3)} < -${d.tolerance}`).join("; ")}`
-        );
-      }
-      const budgetPass = options.budgetUsd === void 0 || ctx.cost.candidate + ctx.cost.baseline <= options.budgetUsd;
-      contributing.push({
-        name: "budget",
-        passed: budgetPass,
-        detail: {
-          candidateUsd: ctx.cost.candidate,
-          baselineUsd: ctx.cost.baseline,
-          budgetUsd: options.budgetUsd
-        }
-      });
-      if (!budgetPass) {
-        reasons.push(
-          `spend ${(ctx.cost.candidate + ctx.cost.baseline).toFixed(2)} > budget ${options.budgetUsd}`
-        );
-      }
-      const redTeamFindings = options.redTeamBattery ? probeRedTeam(ctx.candidateArtifacts, options.redTeamBattery) : { passed: true, findings: [] };
-      contributing.push({
-        name: "red-team",
-        passed: redTeamFindings.passed,
-        detail: {
-          failures: redTeamFindings.findings.length,
-          sample: redTeamFindings.findings.slice(0, 3)
-        }
-      });
-      if (!redTeamFindings.passed) {
-        reasons.push(`red-team probe failed (${redTeamFindings.findings.length} findings)`);
-      }
-      let rewardHackingReport = null;
-      if (options.recentRuns && options.recentRuns.length >= 10) {
-        rewardHackingReport = detectRewardHacking({ runs: options.recentRuns });
-      }
-      const gamingThreshold = 0.6;
-      const gamingFindings = (rewardHackingReport?.findings ?? []).filter(
-        (f) => f.severity >= gamingThreshold
-      );
-      const rewardHackingPass = !rewardHackingReport || !blockOnGaming || gamingFindings.length === 0 && rewardHackingReport.verdict !== "gaming";
-      contributing.push({
-        name: "reward-hacking",
-        passed: rewardHackingPass,
-        detail: { report: rewardHackingReport, gamingFindingCount: gamingFindings.length }
-      });
-      if (!rewardHackingPass) {
-        reasons.push(
-          `reward-hacking detector flagged ${gamingFindings.length} gaming-severity findings (verdict=${rewardHackingReport.verdict})`
-        );
-      }
-      let canaryReport = null;
-      if (options.recentRuns && options.recentRuns.length >= 10) {
-        canaryReport = runCanaries(options.recentRuns, {});
-      }
-      const errorAlerts = (canaryReport?.alerts ?? []).filter((a) => a.severity === "error");
-      const canaryPass = errorAlerts.length === 0;
-      contributing.push({
-        name: "canary",
-        passed: canaryPass,
-        detail: { totalAlerts: canaryReport?.alerts.length ?? 0, errorAlerts: errorAlerts.length }
-      });
-      if (!canaryPass) {
-        reasons.push(`canary error alerts: ${errorAlerts.length}`);
-      }
-      const allPassed = contributing.every((c) => c.passed);
-      const decision = allPassed ? "ship" : "hold";
-      return {
-        decision,
-        reasons: reasons.length > 0 ? reasons : ["all gates passed"],
-        contributingGates: contributing,
-        delta
-      };
-    }
-  };
-}
-function probeRedTeam(artifacts, battery) {
-  const findings = [];
-  for (const [_cellId, artifact] of artifacts) {
-    const text = extractText(artifact);
-    if (text === void 0) continue;
-    for (const rtCase of battery) {
-      const finding = scoreRedTeamOutput(text, [], rtCase);
-      if (!finding.passed) {
-        findings.push({ scenarioId: rtCase.id, reason: finding.reason ?? "red-team probe failed" });
-      }
-    }
-  }
-  return { passed: findings.length === 0, findings };
-}
-function extractText(artifact) {
-  if (typeof artifact === "string") return artifact;
-  if (artifact && typeof artifact === "object") {
-    const rec = artifact;
-    if (typeof rec.text === "string") return rec.text;
-    if (typeof rec.output === "string") return rec.output;
-    if (typeof rec.content === "string") return rec.content;
-  }
-  return void 0;
-}
-
-// src/campaign/gates/power-preflight.ts
-function zFor(confidence) {
-  if (confidence >= 0.99) return 2.576;
-  if (confidence >= 0.95) return 1.96;
-  if (confidence >= 0.9) return 1.645;
-  return 1.282;
-}
-function powerPreflight(opts) {
-  const composites = opts.baselineComposites.filter((v) => Number.isFinite(v));
-  if (composites.length < 3) {
-    throw new Error(
-      `powerPreflight: need >= 3 finite baseline composites to estimate variance, got ${composites.length}`
-    );
-  }
-  const deltaThreshold = opts.deltaThreshold ?? 0.05;
-  const confidence = opts.confidence ?? 0.95;
-  const n = opts.pairedN ?? composites.length;
-  if (n < 2) throw new Error(`powerPreflight: pairedN must be >= 2, got ${n}`);
-  const mean = composites.reduce((a, b) => a + b, 0) / composites.length;
-  const variance = composites.reduce((a, b) => a + (b - mean) * (b - mean), 0) / (composites.length - 1);
-  const sd = Math.sqrt(variance);
-  const z = zFor(confidence);
-  const mde = deltaThreshold + z * Math.SQRT2 * sd / Math.sqrt(n);
-  const scaleAssumed = composites.every((v) => v >= -1e-3 && v <= 1.5);
-  const headroom = Math.max(0, 1 - mean);
-  const underpowered = scaleAssumed && mde > headroom;
-  const sharedChannelCaveat = opts.sharedScorerChannel ? "Holdout and gate share one scoring channel: raising n/reps reduces only idiosyncratic noise \u2014 systematic judge bias remains and this MDE is a lower bound. Full debiasing needs an independent second scoring channel (different judge/benchmark family)." : void 0;
-  const recommendation = underpowered ? `UNDERPOWERED: minimum detectable lift ${mde.toFixed(3)} exceeds the ${headroom.toFixed(3)} headroom above the baseline (${mean.toFixed(3)}) \u2014 no achievable effect can ship at this budget. Raise paired n (scenarios x reps) to ~${Math.ceil((z * Math.SQRT2 * sd / Math.max(headroom - deltaThreshold, 0.01)) ** 2)} or reduce worker variance before searching.` : `Minimum detectable lift at n=${n}: ${mde.toFixed(3)} (baseline sd ${sd.toFixed(3)}). Effects smaller than this cannot clear the gate; budget the search for effects you believe exceed it.`;
-  return {
-    n,
-    sd,
-    mde,
-    baselineMean: mean,
-    headroom,
-    underpowered,
-    scaleAssumed,
-    deltaThreshold,
-    confidence,
-    ...sharedChannelCaveat ? { sharedChannelCaveat } : {},
-    recommendation: sharedChannelCaveat ? `${recommendation} ${sharedChannelCaveat}` : recommendation
-  };
-}
-
-// src/campaign/gates/promotion-policy.ts
-function buildEvidenceVector(ctx, objectives, opts = {}) {
-  if (objectives.length === 0) {
-    throw new Error("buildEvidenceVector: at least 1 objective required");
-  }
-  const minProductiveRuns = opts.minProductiveRuns ?? 3;
-  const confidence = opts.confidence ?? 0.95;
-  const resamples = opts.resamples ?? 2e3;
-  const seed = opts.seed ?? 1337;
-  const baseline = ctx.baselineJudgeScores ?? ctx.judgeScores;
-  const scenarioIds = new Set(ctx.scenarios.map((s) => s.id));
-  const axes = [];
-  for (const obj of objectives) {
-    let select;
-    if (obj.source.kind === "composite") {
-      select = (s) => s.composite;
-    } else {
-      const dim = obj.source.dimension;
-      select = (s) => s.dimensions[dim];
-    }
-    const paired = pairHoldout(ctx.judgeScores, baseline, scenarioIds, select);
-    const before = obj.direction === "maximize" ? paired.before : paired.after;
-    const after = obj.direction === "maximize" ? paired.after : paired.before;
-    const bootstrap = pairedBootstrap(before, after, {
-      confidence,
-      resamples,
-      statistic: "median",
-      seed
-    });
-    const n = paired.before.length;
-    const floorTolerance = obj.floorTolerance ?? 0.05 * detectScale([...paired.before, ...paired.after]);
-    const gainThreshold = obj.gainThreshold ?? 0;
-    const verdict = n < minProductiveRuns ? "few_runs" : bootstrap.low < -floorTolerance ? "regressed" : bootstrap.low > gainThreshold ? "improved" : "flat";
-    axes.push({
-      name: obj.name,
-      source: obj.source,
-      direction: obj.direction,
-      bootstrap,
-      n,
-      gainThreshold,
-      floorTolerance,
-      verdict
-    });
-  }
-  const ns = axes.map((a) => a.n).filter((n) => n > 0);
-  const minN = ns.length > 0 ? Math.min(...ns) : 0;
-  return { axes, minN, cost: { candidate: ctx.cost.candidate, baseline: ctx.cost.baseline } };
-}
-var paretoPolicy = (ev) => {
-  const contributingGates = ev.axes.map((ax) => ({
-    name: `objective:${ax.name}`,
-    passed: ax.verdict === "improved",
-    detail: {
-      direction: ax.direction,
-      source: ax.source,
-      verdict: ax.verdict,
-      n: ax.n,
-      deltaMedian: ax.bootstrap.median,
-      ciLow: ax.bootstrap.low,
-      ciHigh: ax.bootstrap.high,
-      confidence: ax.bootstrap.confidence,
-      gainThreshold: ax.gainThreshold,
-      floorTolerance: ax.floorTolerance
-    }
-  }));
-  const regressed = ev.axes.filter((a) => a.verdict === "regressed");
-  const fewRuns = ev.axes.filter((a) => a.verdict === "few_runs");
-  const improved = ev.axes.filter((a) => a.verdict === "improved");
-  let decision;
-  const reasons = [];
-  if (regressed.length > 0) {
-    decision = "hold";
-    for (const a of regressed) {
-      reasons.push(
-        `objective '${a.name}' regressed: good-direction CI.low ${a.bootstrap.low.toFixed(3)} < -${a.floorTolerance} (n=${a.n})`
-      );
-    }
-  } else if (fewRuns.length > 0) {
-    decision = "need_more_work";
-    for (const a of fewRuns) {
-      reasons.push(
-        `objective '${a.name}' has only n=${a.n} paired runs \u2014 insufficient evidence to claim significance`
-      );
-    }
-  } else if (improved.length > 0) {
-    decision = "ship";
-    reasons.push(
-      `Pareto improvement at the confidence level: ${improved.map(
-        (a) => `'${a.name}' +${a.bootstrap.median.toFixed(3)} (CI.low ${a.bootstrap.low.toFixed(3)})`
-      ).join(", ")}; no objective regressed`
-    );
-  } else {
-    decision = "hold";
-    reasons.push(
-      "no Pareto improvement: candidate statistically equivalent to baseline on every objective"
-    );
-  }
-  const composite = ev.axes.find((a) => a.source.kind === "composite") ?? ev.axes[0];
-  return { decision, reasons, contributingGates, delta: composite?.bootstrap.median };
-};
-function paretoSignificanceGate(options) {
-  if (options.objectives.length === 0) {
-    throw new Error("paretoSignificanceGate: at least 1 objective required");
-  }
-  const policy = options.policy ?? paretoPolicy;
-  return {
-    name: options.name ?? "paretoSignificanceGate",
-    async decide(ctx) {
-      const ev = buildEvidenceVector(ctx, options.objectives, options);
-      return policy(ev);
-    }
-  };
-}
-
-// src/campaign/presets/run-eval.ts
-async function runEval(opts) {
-  return runCampaign(opts);
-}
-
-// src/campaign/proposers/evolutionary.ts
-function evolutionaryProposer(opts) {
-  return {
-    kind: `evolutionary:${opts.mutator.kind}`,
-    async propose({ currentSurface, findings, populationSize, signal }) {
-      return opts.mutator.mutate({
-        findings: findings.length > 0 ? findings : opts.findings ?? [],
-        currentSurface,
-        populationSize,
-        signal
-      });
-    }
-  };
-}
-
-export {
-  composeGate,
-  defaultProductionGate,
-  powerPreflight,
-  buildEvidenceVector,
-  paretoPolicy,
-  paretoSignificanceGate,
-  runEval,
-  evolutionaryProposer
-};
-//# sourceMappingURL=chunk-J22CKVXN.js.map