@tangle-network/agent-bench 0.4.0 → 0.4.3

package/dist/chunk-IA2FBTWC.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/benchmarks/appworld.ts"],"sourcesContent":["/**\n * AppWorld adapter (StonyBrookNLP/appworld). Worker artifact = the agent's\n * Python solution that calls the simulated apps' APIs (the same `apis.<app>.<fn>`\n * surface AppWorld exposes inside `world.execute(...)`), ending in\n * `apis.supervisor.complete_task()`. Judge = AppWorld's OWN programmatic\n * evaluator: a driver runs the solution in a fresh `AppWorld(task_id=...)` world,\n * then `world.evaluate().to_dict()` reports `success` (binary TGC), `num_tests`\n * (per-requirement total) and the `passes`/`failures` lists. Score =\n * passes / num_tests — GRADED; resolved = success. Fully deterministic — no LLM judge.\n *\n * loadTasks enumerates the real task suite via `load_task_ids(split)`\n * (train|dev|test_normal|test_challenge); the prompt = `world.task.instruction`.\n * The OutputAdapter is stream-only, so the worker emits its solution as a fenced\n * ```python block which the driver executes.\n *\n * Requires for a live run: the bench `.venv` with `appworld` installed + the\n * unpacked engine + downloaded data (`appworld install` ; `appworld download\n * data`). preflight + loadTasks + judge all fail loud with the exact step when the\n * engine/data is absent — never a fabricated score.\n */\n\nimport { spawn } from 'node:child_process'\nimport { join } from 'node:path'\nimport { createInterface } from 'node:readline'\nimport { type OutputAdapter, routerToolLoop, type ToolSpec } from '@tangle-network/agent-runtime/loops'\nimport { benchRoot, preflightVenvImports, runVenvScriptStdin, venvPython } from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst DRIVER = join(benchRoot, 'scripts', 'appworld_driver.py')\n\n/** AppWorld splits; only the test splits ship evaluation-only (no setup/solution). */\nconst DEFAULT_SPLIT = 'test_normal'\n\ninterface AppWorldMeta {\n  taskId: string\n  split: string\n}\n\n/** Worker solution code = the last fenced ```python block, else the raw text. */\nexport const appworldSolutionOutput: OutputAdapter<string> = {\n  parse(events) {\n    let text = ''\n    for (const ev of events) {\n      const d = (ev as { data?: Record<string, unknown> })?.data\n      const t = d?.finalText ?? d?.text ?? d?.result\n      if (typeof t === 'string' && t.length > 0) text = t\n    }\n    const fences = [...text.matchAll(/```(?:python|py)?\\s*\\n([\\s\\S]*?)```/g)]\n    return (fences.at(-1)?.[1] ?? text).trim()\n  },\n}\n\nconst WORKER_CONTRACT = [\n  '',\n  'Solve this by writing Python that calls the available app APIs (the `apis.<app>.<function>(...)` surface). You may inspect API docs with `apis.api_docs.show_api_descriptions(app_name=...)` and `apis.api_docs.show_api_doc(app_name=..., api_name=...)`.',\n  'Authenticate where needed via the supervisor-provided credentials, perform every step the task requires, and FINISH with `apis.supervisor.complete_task()`.',\n  'Emit your COMPLETE solution as the LAST thing in your reply, in a single fenced ```python block. Nothing after the closing fence.',\n].join('\\n')\n\nfunction readMeta(task: BenchTask): AppWorldMeta {\n  const md = task.metadata\n  if (!md || typeof md.taskId !== 'string') {\n    throw new Error(`appworld task ${task.id} missing metadata.taskId — loadTasks did not populate it`)\n  }\n  return md as unknown as AppWorldMeta\n}\n\n/**\n * Run the appworld engine driver with a subcommand; JSON on the LAST stdout line.\n * The solution code (evaluate) is piped to stdin via the shared stdin-aware runner —\n * execFile's `input` option is not honored async and hangs the driver's\n * sys.stdin.read() forever. `load` ignores stdin, so an empty pipe is harmless.\n */\nasync function driver(args: string[], input = ''): Promise<unknown> {\n  let stdout: string\n  try {\n    stdout = await runVenvScriptStdin(DRIVER, args, input, { cwd: benchRoot })\n  } catch (err) {\n    const e = err as { message?: string }\n    throw new Error(`appworld driver failed (${args.join(' ')}): ${(e.message || String(err)).slice(0, 1500)}`)\n  }\n  const last = stdout.trim().split('\\n').at(-1) ?? '{}'\n  const parsed = JSON.parse(last) as { error?: string }\n  if (parsed.error) throw new Error(`appworld driver error: ${parsed.error}`)\n  return parsed\n}\n\nexport function createAppWorldAdapter(): BenchmarkAdapter {\n  return {\n    name: 'appworld',\n    output: appworldSolutionOutput,\n\n    async preflight() {\n      await preflightVenvImports({\n        modules: ['appworld'],\n        requireDocker: false,\n        fix:\n          'Fix: bench/.venv/bin/pip install appworld ; ' +\n          'bench/.venv/bin/appworld install ; bench/.venv/bin/appworld download data ' +\n          '(unpacks the engine + downloads the simulated-app data/tasks). ' +\n          'Set APPWORLD_ROOT to the data root if not the default.',\n      })\n    },\n\n    async loadTasks(opts: LoadOptions = {}): Promise<BenchTask[]> {\n      const split = opts.split ?? DEFAULT_SPLIT\n      const out = (await driver([\n        'load',\n        '--split', split,\n        ...(opts.limit !== undefined ? ['--limit', String(opts.limit)] : []),\n        ...(opts.ids ? ['--ids', opts.ids.join(',')] : []),\n      ])) as { tasks?: Array<{ task_id: string; instruction: string }> }\n      const tasks = out.tasks ?? []\n      if (tasks.length === 0) {\n        throw new Error(`appworld loadTasks returned no tasks for split=${split} ${JSON.stringify(opts)}`)\n      }\n      return tasks.map(\n        (t): BenchTask => ({\n          id: t.task_id,\n          split,\n          prompt: t.instruction + WORKER_CONTRACT,\n          metadata: { taskId: t.task_id, split } as unknown as Record<string, unknown>,\n        }),\n      )\n    },\n\n    async goldArtifact() {\n      // Reference solution code ships only for train/dev, and only inside the\n      // engine's decrypted `.bundle` (it is not a portable string this adapter can\n      // emit across splits). The test splits are evaluation-only. So verify-judge\n      // here requires a real solve on a train/dev task through the live engine\n      // rather than a synthetic gold — returning a fabricated artifact would be a\n      // fake. Returns undefined.\n      return undefined\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      const out = (await driver(['evaluate', '--task-id', meta.taskId, '--split', meta.split], artifact)) as {\n        success?: boolean\n        passes?: number\n        fails?: number\n        num_tests?: number\n        failure_names?: string[]\n      }\n      const passes = out.passes ?? 0\n      const fails = out.fails ?? 0\n      // num_tests is the evaluator's authoritative per-requirement count; prefer it\n      // over passes+fails (which can disagree if a requirement neither passed nor\n      // failed). Never default the total to a phantom denominator.\n      const total = out.num_tests ?? passes + fails\n      const score = total > 0 ? passes / total : 0\n      // failure_names = WHICH sub-tests failed — the evidence a trace analyst\n      // steers on. Carried in `detail` so it reaches the verdict's `notes`.\n      const failures = Array.isArray(out.failure_names) ? out.failure_names : []\n      return {\n        resolved: out.success === true,\n        score,\n        detail: JSON.stringify({\n          taskId: meta.taskId,\n          success: out.success,\n          passes,\n          fails,\n          total,\n          ...(failures.length ? { failures } : {}),\n        }),\n      }\n    },\n  }\n}\n\n/**\n * AppWorld in its NATIVE protocol, run by OUR runtime: the worker is\n * `routerToolLoop` (the runtime's off-box agentic tool loop) with one tool —\n * `execute_python` — bound to a persistent AppWorld world session. The driver's\n * `session` subcommand is a dumb world shim (stdin JSONL: execute → output,\n * evaluate → verdict); every inference turn, the metering, and the typed\n * toolTrace the analyst steers on belong to the runtime, so runtime\n * improvements are what this benchmark measures.\n *\n * The one-shot codegen adapter above plays a strictly harder game (no execution\n * feedback — the first wrong API call kills the whole program at judge time),\n * which flatlines the score against ANY steering; this mode is what the\n * benchmark's published baselines use, where behavior can move sub-tests.\n *\n * Protocol: the round task string is `@appworld-react <taskId> <split>` on\n * line 1; everything after line 1 is the steer (an analyst correction, a push\n * directive) appended to the system prompt — so the existing arms steer this\n * worker without modification. The artifact is the episode evaluation JSON\n * (AppWorld's evaluator ran in-world); judge() parses it, never re-executes.\n */\n\ninterface ReactResult {\n  success?: boolean\n  passes?: number\n  fails?: number\n  num_tests?: number\n  failure_names?: string[]\n  turns?: number\n  input_tokens?: number\n  output_tokens?: number\n  transcript?: string\n}\n\nconst REACT_HEADER = /^@appworld-react (\\S+) (\\S+)\\n?/\n\nconst SESSION_SYSTEM = [\n  'You are completing a task in AppWorld, a simulated multi-app environment.',\n  'Use the execute_python tool to run Python that calls the app APIs (the `apis.<app>.<function>(...)` surface).',\n  'Inspect API docs with `apis.api_docs.show_api_descriptions(app_name=...)` and `apis.api_docs.show_api_doc(app_name=..., api_name=...)`.',\n  'Authenticate where needed via the supervisor-provided credentials (`apis.supervisor.show_account_passwords()`).',\n  'Work incrementally: small snippets, read each output, correct course.',\n  'When every step of the task is done, run `apis.supervisor.complete_task()` and then reply WITHOUT calling the tool again.',\n].join('\\n')\n\nconst EXECUTE_TOOL: ToolSpec = {\n  type: 'function',\n  function: {\n    name: 'execute_python',\n    description:\n      'Execute a Python snippet in the persistent AppWorld world. State persists across calls. Returns the execution output (API results or errors).',\n    parameters: {\n      type: 'object',\n      properties: { code: { type: 'string', description: 'Python code calling apis.<app>.<fn>(...)' } },\n      required: ['code'],\n    },\n  },\n}\n\n/** One persistent world session: line-JSONL request/response over the driver. */\nasync function withWorldSession<T>(\n  taskId: string,\n  split: string,\n  fn: (call: (cmd: Record<string, unknown>) => Promise<Record<string, unknown>>, instruction: string) => Promise<T>,\n): Promise<T> {\n  const child = spawn(venvPython, [DRIVER, 'session', '--task-id', taskId, '--split', split], {\n    cwd: benchRoot,\n  })\n  const rl = createInterface({ input: child.stdout })\n  const pending: Array<(line: string) => void> = []\n  const backlog: string[] = []\n  rl.on('line', (l) => {\n    const next = pending.shift()\n    if (next) next(l)\n    else backlog.push(l)\n  })\n  let stderr = ''\n  child.stderr.on('data', (c: Buffer) => {\n    stderr += c.toString('utf8')\n  })\n  const nextLine = (timeoutMs: number): Promise<string> =>\n    new Promise((resolve, reject) => {\n      const fromBacklog = backlog.shift()\n      if (fromBacklog !== undefined) return resolve(fromBacklog)\n      const t = setTimeout(\n        () => reject(new Error(`appworld session: no response in ${timeoutMs}ms; stderr: ${stderr.slice(-400)}`)),\n        timeoutMs,\n      )\n      // One exit listener per await leaks (25-turn episodes blow the listener\n      // cap) — remove it on the resolve path.\n      const onExit = (code: number | null): void => {\n        clearTimeout(t)\n        reject(new Error(`appworld session exited (${code}); stderr: ${stderr.slice(-400)}`))\n      }\n      pending.push((l) => {\n        clearTimeout(t)\n        child.removeListener('exit', onExit)\n        resolve(l)\n      })\n      child.once('exit', onExit)\n    })\n  try {\n    const ready = JSON.parse(await nextLine(120_000)) as { ready?: boolean; instruction?: string; error?: string }\n    if (!ready.ready) throw new Error(`appworld session failed to start: ${ready.error ?? 'no ready line'}`)\n    const call = async (cmd: Record<string, unknown>): Promise<Record<string, unknown>> => {\n      child.stdin.write(`${JSON.stringify(cmd)}\\n`)\n      const res = JSON.parse(await nextLine(180_000)) as Record<string, unknown>\n      if (typeof res.error === 'string') throw new Error(`appworld session op failed: ${res.error}`)\n      return res\n    }\n    return await fn(call, ready.instruction ?? '')\n  } finally {\n    child.stdin.end()\n    child.kill('SIGTERM')\n  }\n}\n\n/** SandboxClient whose leaf is OUR routerToolLoop driving a persistent world session. */\nexport function appworldToolLoopClient(cfg: {\n  model: string\n  routerBaseUrl: string\n  routerKey: string\n  maxTurns?: number\n}): unknown {\n  const maxTurns = cfg.maxTurns ?? Number(process.env.REACT_MAX_TURNS ?? 40)\n  let seq = 0\n  return {\n    async create() {\n      const id = `appworld-toolloop-${seq++}`\n      return {\n        id,\n        async *streamPrompt(prompt: string) {\n          const m = prompt.match(REACT_HEADER)\n          if (!m) {\n            throw new Error(\n              `appworld-react leaf: prompt missing '@appworld-react <taskId> <split>' header — got: ${prompt.slice(0, 120)}`,\n            )\n          }\n          const [, taskId, split] = m\n          const directive = prompt.replace(REACT_HEADER, '').trim()\n          const out = await withWorldSession(taskId as string, split as string, async (call, instruction) => {\n            const system = directive ? `${SESSION_SYSTEM}\\n\\n${directive}` : SESSION_SYSTEM\n            const loop = await routerToolLoop(\n              { routerBaseUrl: cfg.routerBaseUrl, routerKey: cfg.routerKey, model: cfg.model },\n              system,\n              `Task: ${instruction}`,\n              [EXECUTE_TOOL],\n              async (name, args) => {\n                if (name !== 'execute_python') return `error: unknown tool ${name}`\n                const res = await call({ op: 'execute', code: String(args.code ?? '') })\n                const done = res.task_completed === true\n                return `${String(res.output ?? '')}${done ? '\\n\\n[TASK MARKED COMPLETE — reply with a final summary and do not call the tool again]' : ''}`\n              },\n              { maxTurns },\n            )\n            const verdict = (await call({ op: 'evaluate' })) as unknown as ReactResult\n            const transcript = loop.toolTrace\n              .slice(-3)\n              .map((t) => `CODE:\\n${t.args.slice(0, 600)}\\nOUTPUT:\\n${t.result.slice(0, 600)}`)\n              .join('\\n---\\n')\n              .slice(0, 1600)\n            return {\n              ...verdict,\n              turns: loop.turns,\n              input_tokens: loop.usage.input,\n              output_tokens: loop.usage.output,\n              transcript,\n            } satisfies ReactResult\n          })\n          // Real usage from the episode — flat llm_call so the kernel meters it.\n          if (out.input_tokens || out.output_tokens) {\n            yield {\n              type: 'llm_call',\n              data: { tokensIn: out.input_tokens ?? 0, tokensOut: out.output_tokens ?? 0, model: cfg.model },\n            }\n          }\n          yield { type: 'result', data: { finalText: JSON.stringify(out) } }\n        },\n        async delete() {},\n      }\n    },\n  }\n}\n\n/** Artifact = the episode's evaluation JSON, verbatim (no fence extraction). */\nconst reactEpisodeOutput: OutputAdapter<string> = {\n  parse(events) {\n    let text = ''\n    for (const ev of events) {\n      const d = (ev as { data?: Record<string, unknown> })?.data\n      const t = d?.finalText\n      if (typeof t === 'string' && t.length > 0) text = t\n    }\n    return text\n  },\n}\n\nexport function createAppWorldReactAdapter(): BenchmarkAdapter {\n  const base = createAppWorldAdapter()\n  return {\n    name: 'appworld-react',\n    output: reactEpisodeOutput,\n    preflight: () => base.preflight(),\n\n    async loadTasks(opts: LoadOptions = {}): Promise<BenchTask[]> {\n      const tasks = await base.loadTasks(opts)\n      return tasks.map((t) => {\n        const meta = readMeta(t)\n        return {\n          ...t,\n          // Header carries task identity to the leaf; the body (empty at round 0)\n          // is the directive slot the arms append their steer into.\n          prompt: `@appworld-react ${meta.taskId} ${meta.split}\\n`,\n        }\n      })\n    },\n\n    goldArtifact: () => Promise.resolve(undefined),\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      let out: ReactResult\n      try {\n        out = JSON.parse(artifact) as ReactResult\n      } catch {\n        throw new Error(\n          `appworld-react judge: artifact is not the episode's evaluation JSON (task ${meta.taskId}): ${artifact.slice(0, 200)}`,\n        )\n      }\n      if (typeof out.success !== 'boolean' || typeof out.num_tests !== 'number') {\n        throw new Error(\n          `appworld-react judge: episode JSON missing success/num_tests (task ${meta.taskId}): ${artifact.slice(0, 200)}`,\n        )\n      }\n      const passes = out.passes ?? 0\n      const total = out.num_tests\n      const failures = Array.isArray(out.failure_names) ? out.failure_names : []\n      return {\n        resolved: out.success === true,\n        score: total > 0 ? passes / total : 0,\n        detail: JSON.stringify({\n          taskId: meta.taskId,\n          success: out.success,\n          passes,\n          fails: out.fails ?? 0,\n          total,\n          turns: out.turns,\n          ...(failures.length ? { failures } : {}),\n          ...(out.transcript ? { transcriptTail: out.transcript.slice(-800) } : {}),\n        }),\n      }\n    },\n\n    leafClient: (c) => appworldToolLoopClient(c),\n  }\n}\n"],"mappings":";;;;;;;;AAqBA,SAAS,aAAa;AACtB,SAAS,YAAY;AACrB,SAAS,uBAAuB;AAChC,SAA6B,sBAAqC;AAIlE,IAAM,SAAS,KAAK,WAAW,WAAW,oBAAoB;AAG9D,IAAM,gBAAgB;AAQf,IAAM,yBAAgD;AAAA,EAC3D,MAAM,QAAQ;AACZ,QAAI,OAAO;AACX,eAAW,MAAM,QAAQ;AACvB,YAAM,IAAK,IAA2C;AACtD,YAAM,IAAI,GAAG,aAAa,GAAG,QAAQ,GAAG;AACxC,UAAI,OAAO,MAAM,YAAY,EAAE,SAAS,EAAG,QAAO;AAAA,IACpD;AACA,UAAM,SAAS,CAAC,GAAG,KAAK,SAAS,sCAAsC,CAAC;AACxE,YAAQ,OAAO,GAAG,EAAE,IAAI,CAAC,KAAK,MAAM,KAAK;AAAA,EAC3C;AACF;AAEA,IAAM,kBAAkB;AAAA,EACtB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,EAAE,KAAK,IAAI;AAEX,SAAS,SAAS,MAA+B;AAC/C,QAAM,KAAK,KAAK;AAChB,MAAI,CAAC,MAAM,OAAO,GAAG,WAAW,UAAU;AACxC,UAAM,IAAI,MAAM,iBAAiB,KAAK,EAAE,+DAA0D;AAAA,EACpG;AACA,SAAO;AACT;AAQA,eAAe,OAAO,MAAgB,QAAQ,IAAsB;AAClE,MAAI;AACJ,MAAI;AACF,aAAS,MAAM,mBAAmB,QAAQ,MAAM,OAAO,EAAE,KAAK,UAAU,CAAC;AAAA,EAC3E,SAAS,KAAK;AACZ,UAAM,IAAI;AACV,UAAM,IAAI,MAAM,2BAA2B,KAAK,KAAK,GAAG,CAAC,OAAO,EAAE,WAAW,OAAO,GAAG,GAAG,MAAM,GAAG,IAAI,CAAC,EAAE;AAAA,EAC5G;AACA,QAAM,OAAO,OAAO,KAAK,EAAE,MAAM,IAAI,EAAE,GAAG,EAAE,KAAK;AACjD,QAAM,SAAS,KAAK,MAAM,IAAI;AAC9B,MAAI,OAAO,MAAO,OAAM,IAAI,MAAM,0BAA0B,OAAO,KAAK,EAAE;AAC1E,SAAO;AACT;AAEO,SAAS,wBAA0C;AACxD,SAAO;AAAA,IACL,MAAM;AAAA,IACN,QAAQ;AAAA,IAER,MAAM,YAAY;AAChB,YAAM,qBAAqB;AAAA,QACzB,SAAS,CAAC,UAAU;AAAA,QACpB,eAAe;AAAA,QACf,KACE;AAAA,MAIJ,CAAC;AAAA,IACH;AAAA,IAEA,MAAM,UAAU,OAAoB,CAAC,GAAyB;AAC5D,YAAM,QAAQ,KAAK,SAAS;AAC5B,YAAM,MAAO,MAAM,OAAO;AAAA,QACxB;AAAA,QACA;AAAA,QAAW;AAAA,QACX,GAAI,KAAK,UAAU,SAAY,CAAC,WAAW,OAAO,KAAK,KAAK,CAAC,IAAI,CAAC;AAAA,QAClE,GAAI,KAAK,MAAM,CAAC,SAAS,KAAK,IAAI,KAAK,GAAG,CAAC,IAAI,CAAC;AAAA,MAClD,CAAC;AACD,YAAM,QAAQ,IAAI,SAAS,CAAC;AAC5B,UAAI,MAAM,WAAW,GAAG;AACtB,cAAM,IAAI,MAAM,kDAAkD,KAAK,IAAI,KAAK,UAAU,IAAI,CAAC,EAAE;AAAA,MACnG;AACA,aAAO,MAAM;AAAA,QACX,CAAC,OAAkB;AAAA,UACjB,IAAI,EAAE;AAAA,UACN;AAAA,UACA,QAAQ,EAAE,cAAc;AAAA,UACxB,UAAU,EAAE,QAAQ,EAAE,SAAS,MAAM;AAAA,QACvC;AAAA,MACF;AAAA,IACF;AAAA,IAEA,MAAM,eAAe;AAOnB,aAAO;AAAA,IACT;AAAA,IAEA,MAAM,MAAM,MAAiB,UAAuC;AAClE,YAAM,OAAO,SAAS,IAAI;AAC1B,YAAM,MAAO,MAAM,OAAO,CAAC,YAAY,aAAa,KAAK,QAAQ,WAAW,KAAK,KAAK,GAAG,QAAQ;AAOjG,YAAM,SAAS,IAAI,UAAU;AAC7B,YAAM,QAAQ,IAAI,SAAS;AAI3B,YAAM,QAAQ,IAAI,aAAa,SAAS;AACxC,YAAM,QAAQ,QAAQ,IAAI,SAAS,QAAQ;AAG3C,YAAM,WAAW,MAAM,QAAQ,IAAI,aAAa,IAAI,IAAI,gBAAgB,CAAC;AACzE,aAAO;AAAA,QACL,UAAU,IAAI,YAAY;AAAA,QAC1B;AAAA,QACA,QAAQ,KAAK,UAAU;AAAA,UACrB,QAAQ,KAAK;AAAA,UACb,SAAS,IAAI;AAAA,UACb;AAAA,UACA;AAAA,UACA;AAAA,UACA,GAAI,SAAS,SAAS,EAAE,SAAS,IAAI,CAAC;AAAA,QACxC,CAAC;AAAA,MACH;AAAA,IACF;AAAA,EACF;AACF;AAmCA,IAAM,eAAe;AAErB,IAAM,iBAAiB;AAAA,EACrB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,EAAE,KAAK,IAAI;AAEX,IAAM,eAAyB;AAAA,EAC7B,MAAM;AAAA,EACN,UAAU;AAAA,IACR,MAAM;AAAA,IACN,aACE;AAAA,IACF,YAAY;AAAA,MACV,MAAM;AAAA,MACN,YAAY,EAAE,MAAM,EAAE,MAAM,UAAU,aAAa,2CAA2C,EAAE;AAAA,MAChG,UAAU,CAAC,MAAM;AAAA,IACnB;AAAA,EACF;AACF;AAGA,eAAe,iBACb,QACA,OACA,IACY;AACZ,QAAM,QAAQ,MAAM,YAAY,CAAC,QAAQ,WAAW,aAAa,QAAQ,WAAW,KAAK,GAAG;AAAA,IAC1F,KAAK;AAAA,EACP,CAAC;AACD,QAAM,KAAK,gBAAgB,EAAE,OAAO,MAAM,OAAO,CAAC;AAClD,QAAM,UAAyC,CAAC;AAChD,QAAM,UAAoB,CAAC;AAC3B,KAAG,GAAG,QAAQ,CAAC,MAAM;AACnB,UAAM,OAAO,QAAQ,MAAM;AAC3B,QAAI,KAAM,MAAK,CAAC;AAAA,QACX,SAAQ,KAAK,CAAC;AAAA,EACrB,CAAC;AACD,MAAI,SAAS;AACb,QAAM,OAAO,GAAG,QAAQ,CAAC,MAAc;AACrC,cAAU,EAAE,SAAS,MAAM;AAAA,EAC7B,CAAC;AACD,QAAM,WAAW,CAAC,cAChB,IAAI,QAAQ,CAAC,SAAS,WAAW;AAC/B,UAAM,cAAc,QAAQ,MAAM;AAClC,QAAI,gBAAgB,OAAW,QAAO,QAAQ,WAAW;AACzD,UAAM,IAAI;AAAA,MACR,MAAM,OAAO,IAAI,MAAM,oCAAoC,SAAS,eAAe,OAAO,MAAM,IAAI,CAAC,EAAE,CAAC;AAAA,MACxG;AAAA,IACF;AAGA,UAAM,SAAS,CAAC,SAA8B;AAC5C,mBAAa,CAAC;AACd,aAAO,IAAI,MAAM,4BAA4B,IAAI,cAAc,OAAO,MAAM,IAAI,CAAC,EAAE,CAAC;AAAA,IACtF;AACA,YAAQ,KAAK,CAAC,MAAM;AAClB,mBAAa,CAAC;AACd,YAAM,eAAe,QAAQ,MAAM;AACnC,cAAQ,CAAC;AAAA,IACX,CAAC;AACD,UAAM,KAAK,QAAQ,MAAM;AAAA,EAC3B,CAAC;AACH,MAAI;AACF,UAAM,QAAQ,KAAK,MAAM,MAAM,SAAS,IAAO,CAAC;AAChD,QAAI,CAAC,MAAM,MAAO,OAAM,IAAI,MAAM,qCAAqC,MAAM,SAAS,eAAe,EAAE;AACvG,UAAM,OAAO,OAAO,QAAmE;AACrF,YAAM,MAAM,MAAM,GAAG,KAAK,UAAU,GAAG,CAAC;AAAA,CAAI;AAC5C,YAAM,MAAM,KAAK,MAAM,MAAM,SAAS,IAAO,CAAC;AAC9C,UAAI,OAAO,IAAI,UAAU,SAAU,OAAM,IAAI,MAAM,+BAA+B,IAAI,KAAK,EAAE;AAC7F,aAAO;AAAA,IACT;AACA,WAAO,MAAM,GAAG,MAAM,MAAM,eAAe,EAAE;AAAA,EAC/C,UAAE;AACA,UAAM,MAAM,IAAI;AAChB,UAAM,KAAK,SAAS;AAAA,EACtB;AACF;AAGO,SAAS,uBAAuB,KAK3B;AACV,QAAM,WAAW,IAAI,YAAY,OAAO,QAAQ,IAAI,mBAAmB,EAAE;AACzE,MAAI,MAAM;AACV,SAAO;AAAA,IACL,MAAM,SAAS;AACb,YAAM,KAAK,qBAAqB,KAAK;AACrC,aAAO;AAAA,QACL;AAAA,QACA,OAAO,aAAa,QAAgB;AAClC,gBAAM,IAAI,OAAO,MAAM,YAAY;AACnC,cAAI,CAAC,GAAG;AACN,kBAAM,IAAI;AAAA,cACR,6FAAwF,OAAO,MAAM,GAAG,GAAG,CAAC;AAAA,YAC9G;AAAA,UACF;AACA,gBAAM,CAAC,EAAE,QAAQ,KAAK,IAAI;AAC1B,gBAAM,YAAY,OAAO,QAAQ,cAAc,EAAE,EAAE,KAAK;AACxD,gBAAM,MAAM,MAAM,iBAAiB,QAAkB,OAAiB,OAAO,MAAM,gBAAgB;AACjG,kBAAM,SAAS,YAAY,GAAG,cAAc;AAAA;AAAA,EAAO,SAAS,KAAK;AACjE,kBAAM,OAAO,MAAM;AAAA,cACjB,EAAE,eAAe,IAAI,eAAe,WAAW,IAAI,WAAW,OAAO,IAAI,MAAM;AAAA,cAC/E;AAAA,cACA,SAAS,WAAW;AAAA,cACpB,CAAC,YAAY;AAAA,cACb,OAAO,MAAM,SAAS;AACpB,oBAAI,SAAS,iBAAkB,QAAO,uBAAuB,IAAI;AACjE,sBAAM,MAAM,MAAM,KAAK,EAAE,IAAI,WAAW,MAAM,OAAO,KAAK,QAAQ,EAAE,EAAE,CAAC;AACvE,sBAAM,OAAO,IAAI,mBAAmB;AACpC,uBAAO,GAAG,OAAO,IAAI,UAAU,EAAE,CAAC,GAAG,OAAO,gGAA2F,EAAE;AAAA,cAC3I;AAAA,cACA,EAAE,SAAS;AAAA,YACb;AACA,kBAAM,UAAW,MAAM,KAAK,EAAE,IAAI,WAAW,CAAC;AAC9C,kBAAM,aAAa,KAAK,UACrB,MAAM,EAAE,EACR,IAAI,CAAC,MAAM;AAAA,EAAU,EAAE,KAAK,MAAM,GAAG,GAAG,CAAC;AAAA;AAAA,EAAc,EAAE,OAAO,MAAM,GAAG,GAAG,CAAC,EAAE,EAC/E,KAAK,SAAS,EACd,MAAM,GAAG,IAAI;AAChB,mBAAO;AAAA,cACL,GAAG;AAAA,cACH,OAAO,KAAK;AAAA,cACZ,cAAc,KAAK,MAAM;AAAA,cACzB,eAAe,KAAK,MAAM;AAAA,cAC1B;AAAA,YACF;AAAA,UACF,CAAC;AAED,cAAI,IAAI,gBAAgB,IAAI,eAAe;AACzC,kBAAM;AAAA,cACJ,MAAM;AAAA,cACN,MAAM,EAAE,UAAU,IAAI,gBAAgB,GAAG,WAAW,IAAI,iBAAiB,GAAG,OAAO,IAAI,MAAM;AAAA,YAC/F;AAAA,UACF;AACA,gBAAM,EAAE,MAAM,UAAU,MAAM,EAAE,WAAW,KAAK,UAAU,GAAG,EAAE,EAAE;AAAA,QACnE;AAAA,QACA,MAAM,SAAS;AAAA,QAAC;AAAA,MAClB;AAAA,IACF;AAAA,EACF;AACF;AAGA,IAAM,qBAA4C;AAAA,EAChD,MAAM,QAAQ;AACZ,QAAI,OAAO;AACX,eAAW,MAAM,QAAQ;AACvB,YAAM,IAAK,IAA2C;AACtD,YAAM,IAAI,GAAG;AACb,UAAI,OAAO,MAAM,YAAY,EAAE,SAAS,EAAG,QAAO;AAAA,IACpD;AACA,WAAO;AAAA,EACT;AACF;AAEO,SAAS,6BAA+C;AAC7D,QAAM,OAAO,sBAAsB;AACnC,SAAO;AAAA,IACL,MAAM;AAAA,IACN,QAAQ;AAAA,IACR,WAAW,MAAM,KAAK,UAAU;AAAA,IAEhC,MAAM,UAAU,OAAoB,CAAC,GAAyB;AAC5D,YAAM,QAAQ,MAAM,KAAK,UAAU,IAAI;AACvC,aAAO,MAAM,IAAI,CAAC,MAAM;AACtB,cAAM,OAAO,SAAS,CAAC;AACvB,eAAO;AAAA,UACL,GAAG;AAAA;AAAA;AAAA,UAGH,QAAQ,mBAAmB,KAAK,MAAM,IAAI,KAAK,KAAK;AAAA;AAAA,QACtD;AAAA,MACF,CAAC;AAAA,IACH;AAAA,IAEA,cAAc,MAAM,QAAQ,QAAQ,MAAS;AAAA,IAE7C,MAAM,MAAM,MAAiB,UAAuC;AAClE,YAAM,OAAO,SAAS,IAAI;AAC1B,UAAI;AACJ,UAAI;AACF,cAAM,KAAK,MAAM,QAAQ;AAAA,MAC3B,QAAQ;AACN,cAAM,IAAI;AAAA,UACR,6EAA6E,KAAK,MAAM,MAAM,SAAS,MAAM,GAAG,GAAG,CAAC;AAAA,QACtH;AAAA,MACF;AACA,UAAI,OAAO,IAAI,YAAY,aAAa,OAAO,IAAI,cAAc,UAAU;AACzE,cAAM,IAAI;AAAA,UACR,sEAAsE,KAAK,MAAM,MAAM,SAAS,MAAM,GAAG,GAAG,CAAC;AAAA,QAC/G;AAAA,MACF;AACA,YAAM,SAAS,IAAI,UAAU;AAC7B,YAAM,QAAQ,IAAI;AAClB,YAAM,WAAW,MAAM,QAAQ,IAAI,aAAa,IAAI,IAAI,gBAAgB,CAAC;AACzE,aAAO;AAAA,QACL,UAAU,IAAI,YAAY;AAAA,QAC1B,OAAO,QAAQ,IAAI,SAAS,QAAQ;AAAA,QACpC,QAAQ,KAAK,UAAU;AAAA,UACrB,QAAQ,KAAK;AAAA,UACb,SAAS,IAAI;AAAA,UACb;AAAA,UACA,OAAO,IAAI,SAAS;AAAA,UACpB;AAAA,UACA,OAAO,IAAI;AAAA,UACX,GAAI,SAAS,SAAS,EAAE,SAAS,IAAI,CAAC;AAAA,UACtC,GAAI,IAAI,aAAa,EAAE,gBAAgB,IAAI,WAAW,MAAM,IAAI,EAAE,IAAI,CAAC;AAAA,QACzE,CAAC;AAAA,MACH;AAAA,IACF;AAAA,IAEA,YAAY,CAAC,MAAM,uBAAuB,CAAC;AAAA,EAC7C;AACF;","names":[]}

package/dist/chunk-IFVINJ4B.js

@@ -1,142 +0,0 @@
-import {
-  FINAL_ANSWER_SENTINEL,
-  allStrings,
-  answerScoreToBenchScore,
-  contextBlock,
-  contextsFrom,
-  firstString,
-  isObject,
-  ragAnswerOutput,
-  readJsonRows,
-  scoreAnswerArtifact,
-  selectTasks,
-  stringFrom
-} from "./chunk-X3BTXCJ4.js";
-import {
-  benchRoot
-} from "./chunk-WSKWVEQB.js";
-
-// src/benchmarks/ragbench.ts
-import { readFile } from "fs/promises";
-import { join } from "path";
-var FIXTURES = join(benchRoot, "fixtures", "ragbench.json");
-var dataFile = () => process.env.RAGBENCH_DATA_FILE;
-function rowToTask(raw, index) {
-  if (!isObject(raw)) throw new Error(`RAGBench row ${index} must be an object`);
-  const query = firstString(raw, ["question", "query", "user_input", "prompt"]);
-  const goldAnswers = allStrings(raw, [
-    "response",
-    "responses",
-    "model_response",
-    "generated_response",
-    "reference",
-    "references",
-    "reference_answer",
-    "reference_answers",
-    "answer",
-    "answers",
-    "gold",
-    "gold_answer",
-    "ground_truth",
-    "expected_answer"
-  ]);
-  if (!query) throw new Error(`RAGBench row ${index} missing question/query`);
-  if (goldAnswers.length === 0) throw new Error(`RAGBench row ${index} missing reference answer`);
-  const contexts = contextsFrom(raw.contexts).length > 0 ? contextsFrom(raw.contexts) : contextsFrom(raw.retrieved_contexts).length > 0 ? contextsFrom(raw.retrieved_contexts) : contextsFrom(raw.documents);
-  const traceLabels = {};
-  for (const key of [
-    "adherence",
-    "completeness",
-    "relevance",
-    "utilization",
-    "all_relevant_sentence_keys",
-    "all_utilized_sentence_keys"
-  ]) {
-    if (raw[key] !== void 0) traceLabels[key] = raw[key];
-  }
-  const dataset = stringFrom(raw.dataset) ?? stringFrom(raw.source_dataset);
-  const id = stringFrom(raw.id) ?? stringFrom(raw.example_id) ?? `ragbench-${index}`;
-  const meta = {
-    benchmark: "ragbench",
-    query,
-    goldAnswers,
-    contexts,
-    ...dataset ? { dataset } : {},
-    traceLabels
-  };
-  return {
-    id,
-    split: stringFrom(raw.split) ?? dataset ?? "ragbench",
-    prompt: [
-      "Answer this RAGBench question using the supplied retrieved context.",
-      "End with a single final line: `FINAL ANSWER: <answer>`.",
-      "",
-      `Question: ${query}`,
-      contexts.length > 0 ? `
-Retrieved context:
-${contextBlock(contexts)}` : void 0
-    ].filter(Boolean).join("\n"),
-    metadata: meta
-  };
-}
-function readMeta(task) {
-  const md = task.metadata;
-  if (!md || !Array.isArray(md.goldAnswers)) {
-    throw new Error(`RAGBench task ${task.id} missing metadata \u2014 loadTasks did not populate it`);
-  }
-  return md;
-}
-async function loadRows(path) {
-  const rows = await readJsonRows(path);
-  if (rows.length === 0) throw new Error(`RAGBench: no rows in ${path}`);
-  return rows;
-}
-async function loadFixtures(opts) {
-  const rows = JSON.parse(await readFile(FIXTURES, "utf8"));
-  console.warn(`[ragbench] RAGBENCH_FIXTURES=1 \u2014 loading ${rows.length} adapter fixtures`);
-  return selectTasks(rows.map(rowToTask), opts, "RAGBench");
-}
-function createRagBenchAdapter() {
-  const fixturesMode = process.env.RAGBENCH_FIXTURES === "1";
-  return {
-    name: "ragbench",
-    output: ragAnswerOutput,
-    async preflight() {
-      if (fixturesMode) {
-        await readFile(FIXTURES, "utf8");
-        return;
-      }
-      const path = dataFile();
-      if (!path) {
-        throw new Error(
-          "RAGBENCH_DATA_FILE is required. Fix: export rungalileo/ragbench rows to JSONL and set RAGBENCH_DATA_FILE=/path/to/ragbench.jsonl, or set RAGBENCH_FIXTURES=1 for adapter plumbing."
-        );
-      }
-      await loadRows(path);
-    },
-    async loadTasks(opts = {}) {
-      if (fixturesMode) return loadFixtures(opts);
-      const path = dataFile();
-      if (!path) throw new Error("RAGBENCH_DATA_FILE is required to load RAGBench tasks");
-      return selectTasks((await loadRows(path)).map(rowToTask), opts, "RAGBench");
-    },
-    async goldArtifact(task) {
-      return `${FINAL_ANSWER_SENTINEL} ${readMeta(task).goldAnswers[0] ?? ""}`;
-    },
-    async judge(task, artifact) {
-      const meta = readMeta(task);
-      const score = scoreAnswerArtifact(artifact, meta.goldAnswers);
-      return answerScoreToBenchScore(score, {
-        benchmark: meta.benchmark,
-        dataset: meta.dataset ?? null,
-        traceLabels: meta.traceLabels,
-        contextCount: meta.contexts.length
-      });
-    }
-  };
-}
-
-export {
-  createRagBenchAdapter
-};
-//# sourceMappingURL=chunk-IFVINJ4B.js.map

package/dist/chunk-IFVINJ4B.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/benchmarks/ragbench.ts"],"sourcesContent":["/**\n * RAGBench-compatible adapter.\n *\n * Live mode expects a local JSON/JSONL export from rungalileo/ragbench or a\n * compatible table. Rows must carry a query and at least one reference answer.\n * Contexts, TRACe labels, and source metadata are preserved in task metadata\n * for diagnostics; the deterministic judge scores the worker's final answer\n * against the reference answer(s).\n */\n\nimport { readFile } from 'node:fs/promises'\nimport { join } from 'node:path'\nimport { benchRoot } from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\nimport {\n  FINAL_ANSWER_SENTINEL,\n  allStrings,\n  answerScoreToBenchScore,\n  contextBlock,\n  contextsFrom,\n  firstString,\n  isObject,\n  ragAnswerOutput,\n  readJsonRows,\n  scoreAnswerArtifact,\n  selectTasks,\n  stringFrom,\n  type RagContext,\n} from './rag-shared'\n\nconst FIXTURES = join(benchRoot, 'fixtures', 'ragbench.json')\n\ninterface RagBenchMeta {\n  benchmark: 'ragbench'\n  query: string\n  goldAnswers: string[]\n  contexts: RagContext[]\n  dataset?: string\n  traceLabels: Record<string, unknown>\n}\n\nconst dataFile = (): string | undefined => process.env.RAGBENCH_DATA_FILE\n\nfunction rowToTask(raw: unknown, index: number): BenchTask {\n  if (!isObject(raw)) throw new Error(`RAGBench row ${index} must be an object`)\n  const query = firstString(raw, ['question', 'query', 'user_input', 'prompt'])\n  const goldAnswers = allStrings(raw, [\n    'response',\n    'responses',\n    'model_response',\n    'generated_response',\n    'reference',\n    'references',\n    'reference_answer',\n    'reference_answers',\n    'answer',\n    'answers',\n    'gold',\n    'gold_answer',\n    'ground_truth',\n    'expected_answer',\n  ])\n  if (!query) throw new Error(`RAGBench row ${index} missing question/query`)\n  if (goldAnswers.length === 0) throw new Error(`RAGBench row ${index} missing reference answer`)\n  const contexts =\n    contextsFrom(raw.contexts).length > 0\n      ? contextsFrom(raw.contexts)\n      : contextsFrom(raw.retrieved_contexts).length > 0\n        ? contextsFrom(raw.retrieved_contexts)\n        : contextsFrom(raw.documents)\n  const traceLabels: Record<string, unknown> = {}\n  for (const key of [\n    'adherence',\n    'completeness',\n    'relevance',\n    'utilization',\n    'all_relevant_sentence_keys',\n    'all_utilized_sentence_keys',\n  ]) {\n    if (raw[key] !== undefined) traceLabels[key] = raw[key]\n  }\n  const dataset = stringFrom(raw.dataset) ?? stringFrom(raw.source_dataset)\n  const id = stringFrom(raw.id) ?? stringFrom(raw.example_id) ?? `ragbench-${index}`\n  const meta: RagBenchMeta = {\n    benchmark: 'ragbench',\n    query,\n    goldAnswers,\n    contexts,\n    ...(dataset ? { dataset } : {}),\n    traceLabels,\n  }\n  return {\n    id,\n    split: stringFrom(raw.split) ?? dataset ?? 'ragbench',\n    prompt: [\n      'Answer this RAGBench question using the supplied retrieved context.',\n      'End with a single final line: `FINAL ANSWER: <answer>`.',\n      '',\n      `Question: ${query}`,\n      contexts.length > 0 ? `\\nRetrieved context:\\n${contextBlock(contexts)}` : undefined,\n    ]\n      .filter(Boolean)\n      .join('\\n'),\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask): RagBenchMeta {\n  const md = task.metadata\n  if (!md || !Array.isArray(md.goldAnswers)) {\n    throw new Error(`RAGBench task ${task.id} missing metadata — loadTasks did not populate it`)\n  }\n  return md as unknown as RagBenchMeta\n}\n\nasync function loadRows(path: string): Promise<unknown[]> {\n  const rows = await readJsonRows(path)\n  if (rows.length === 0) throw new Error(`RAGBench: no rows in ${path}`)\n  return rows\n}\n\nasync function loadFixtures(opts: LoadOptions): Promise<BenchTask[]> {\n  const rows = JSON.parse(await readFile(FIXTURES, 'utf8')) as unknown[]\n  console.warn(`[ragbench] RAGBENCH_FIXTURES=1 — loading ${rows.length} adapter fixtures`)\n  return selectTasks(rows.map(rowToTask), opts, 'RAGBench')\n}\n\nexport function createRagBenchAdapter(): BenchmarkAdapter {\n  const fixturesMode = process.env.RAGBENCH_FIXTURES === '1'\n\n  return {\n    name: 'ragbench',\n    output: ragAnswerOutput,\n\n    async preflight() {\n      if (fixturesMode) {\n        await readFile(FIXTURES, 'utf8')\n        return\n      }\n      const path = dataFile()\n      if (!path) {\n        throw new Error(\n          'RAGBENCH_DATA_FILE is required. Fix: export rungalileo/ragbench rows to JSONL and set RAGBENCH_DATA_FILE=/path/to/ragbench.jsonl, or set RAGBENCH_FIXTURES=1 for adapter plumbing.',\n        )\n      }\n      await loadRows(path)\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      if (fixturesMode) return loadFixtures(opts)\n      const path = dataFile()\n      if (!path) throw new Error('RAGBENCH_DATA_FILE is required to load RAGBench tasks')\n      return selectTasks((await loadRows(path)).map(rowToTask), opts, 'RAGBench')\n    },\n\n    async goldArtifact(task: BenchTask) {\n      return `${FINAL_ANSWER_SENTINEL} ${readMeta(task).goldAnswers[0] ?? ''}`\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      const score = scoreAnswerArtifact(artifact, meta.goldAnswers)\n      return answerScoreToBenchScore(score, {\n        benchmark: meta.benchmark,\n        dataset: meta.dataset ?? null,\n        traceLabels: meta.traceLabels,\n        contextCount: meta.contexts.length,\n      })\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;AAUA,SAAS,gBAAgB;AACzB,SAAS,YAAY;AAmBrB,IAAM,WAAW,KAAK,WAAW,YAAY,eAAe;AAW5D,IAAM,WAAW,MAA0B,QAAQ,IAAI;AAEvD,SAAS,UAAU,KAAc,OAA0B;AACzD,MAAI,CAAC,SAAS,GAAG,EAAG,OAAM,IAAI,MAAM,gBAAgB,KAAK,oBAAoB;AAC7E,QAAM,QAAQ,YAAY,KAAK,CAAC,YAAY,SAAS,cAAc,QAAQ,CAAC;AAC5E,QAAM,cAAc,WAAW,KAAK;AAAA,IAClC;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,CAAC;AACD,MAAI,CAAC,MAAO,OAAM,IAAI,MAAM,gBAAgB,KAAK,yBAAyB;AAC1E,MAAI,YAAY,WAAW,EAAG,OAAM,IAAI,MAAM,gBAAgB,KAAK,2BAA2B;AAC9F,QAAM,WACJ,aAAa,IAAI,QAAQ,EAAE,SAAS,IAChC,aAAa,IAAI,QAAQ,IACzB,aAAa,IAAI,kBAAkB,EAAE,SAAS,IAC5C,aAAa,IAAI,kBAAkB,IACnC,aAAa,IAAI,SAAS;AAClC,QAAM,cAAuC,CAAC;AAC9C,aAAW,OAAO;AAAA,IAChB;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,GAAG;AACD,QAAI,IAAI,GAAG,MAAM,OAAW,aAAY,GAAG,IAAI,IAAI,GAAG;AAAA,EACxD;AACA,QAAM,UAAU,WAAW,IAAI,OAAO,KAAK,WAAW,IAAI,cAAc;AACxE,QAAM,KAAK,WAAW,IAAI,EAAE,KAAK,WAAW,IAAI,UAAU,KAAK,YAAY,KAAK;AAChF,QAAM,OAAqB;AAAA,IACzB,WAAW;AAAA,IACX;AAAA,IACA;AAAA,IACA;AAAA,IACA,GAAI,UAAU,EAAE,QAAQ,IAAI,CAAC;AAAA,IAC7B;AAAA,EACF;AACA,SAAO;AAAA,IACL;AAAA,IACA,OAAO,WAAW,IAAI,KAAK,KAAK,WAAW;AAAA,IAC3C,QAAQ;AAAA,MACN;AAAA,MACA;AAAA,MACA;AAAA,MACA,aAAa,KAAK;AAAA,MAClB,SAAS,SAAS,IAAI;AAAA;AAAA,EAAyB,aAAa,QAAQ,CAAC,KAAK;AAAA,IAC5E,EACG,OAAO,OAAO,EACd,KAAK,IAAI;AAAA,IACZ,UAAU;AAAA,EACZ;AACF;AAEA,SAAS,SAAS,MAA+B;AAC/C,QAAM,KAAK,KAAK;AAChB,MAAI,CAAC,MAAM,CAAC,MAAM,QAAQ,GAAG,WAAW,GAAG;AACzC,UAAM,IAAI,MAAM,iBAAiB,KAAK,EAAE,wDAAmD;AAAA,EAC7F;AACA,SAAO;AACT;AAEA,eAAe,SAAS,MAAkC;AACxD,QAAM,OAAO,MAAM,aAAa,IAAI;AACpC,MAAI,KAAK,WAAW,EAAG,OAAM,IAAI,MAAM,wBAAwB,IAAI,EAAE;AACrE,SAAO;AACT;AAEA,eAAe,aAAa,MAAyC;AACnE,QAAM,OAAO,KAAK,MAAM,MAAM,SAAS,UAAU,MAAM,CAAC;AACxD,UAAQ,KAAK,iDAA4C,KAAK,MAAM,mBAAmB;AACvF,SAAO,YAAY,KAAK,IAAI,SAAS,GAAG,MAAM,UAAU;AAC1D;AAEO,SAAS,wBAA0C;AACxD,QAAM,eAAe,QAAQ,IAAI,sBAAsB;AAEvD,SAAO;AAAA,IACL,MAAM;AAAA,IACN,QAAQ;AAAA,IAER,MAAM,YAAY;AAChB,UAAI,cAAc;AAChB,cAAM,SAAS,UAAU,MAAM;AAC/B;AAAA,MACF;AACA,YAAM,OAAO,SAAS;AACtB,UAAI,CAAC,MAAM;AACT,cAAM,IAAI;AAAA,UACR;AAAA,QACF;AAAA,MACF;AACA,YAAM,SAAS,IAAI;AAAA,IACrB;AAAA,IAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;AACtC,UAAI,aAAc,QAAO,aAAa,IAAI;AAC1C,YAAM,OAAO,SAAS;AACtB,UAAI,CAAC,KAAM,OAAM,IAAI,MAAM,uDAAuD;AAClF,aAAO,aAAa,MAAM,SAAS,IAAI,GAAG,IAAI,SAAS,GAAG,MAAM,UAAU;AAAA,IAC5E;AAAA,IAEA,MAAM,aAAa,MAAiB;AAClC,aAAO,GAAG,qBAAqB,IAAI,SAAS,IAAI,EAAE,YAAY,CAAC,KAAK,EAAE;AAAA,IACxE;AAAA,IAEA,MAAM,MAAM,MAAiB,UAAuC;AAClE,YAAM,OAAO,SAAS,IAAI;AAC1B,YAAM,QAAQ,oBAAoB,UAAU,KAAK,WAAW;AAC5D,aAAO,wBAAwB,OAAO;AAAA,QACpC,WAAW,KAAK;AAAA,QAChB,SAAS,KAAK,WAAW;AAAA,QACzB,aAAa,KAAK;AAAA,QAClB,cAAc,KAAK,SAAS;AAAA,MAC9B,CAAC;AAAA,IACH;AAAA,EACF;AACF;","names":[]}

package/dist/chunk-INNOYXCP.js

@@ -1,387 +0,0 @@
-// src/benchmarks/frames.ts
-import { execFile } from "child_process";
-import { readFile } from "fs/promises";
-import { join } from "path";
-import { fileURLToPath } from "url";
-import { promisify } from "util";
-var execFileAsync = promisify(execFile);
-var BENCH_ROOT = fileURLToPath(new URL("../..", import.meta.url));
-var PY = join(BENCH_ROOT, ".venv", "bin", "python");
-var FIXTURES = join(BENCH_ROOT, "fixtures", "frames.json");
-var DATASET = "google/frames-benchmark";
-var DATASET_REVISION = "main";
-var FINAL_ANSWER_SENTINEL = "FINAL ANSWER:";
-var WORKER_CONTRACT = [
-  "",
-  "Research the question using live web sources and answer it.",
-  "Cite the sources you used as full URLs, one per line, in a `CITATIONS:` block.",
-  `End your response with a single final line: \`${FINAL_ANSWER_SENTINEL} <answer>\``,
-  "The answer after the sentinel must be the bare value only (no explanation on that line)."
-].join("\n");
-async function py(script, args = []) {
-  const { stdout } = await execFileAsync(PY, ["-c", script, ...args], {
-    maxBuffer: 1024 * 1024 * 256
-  });
-  return stdout;
-}
-var ARTICLES = /* @__PURE__ */ new Set(["a", "an", "the"]);
-var UNIT_WORDS = /* @__PURE__ */ new Set([
-  "years",
-  "year",
-  "months",
-  "month",
-  "days",
-  "day",
-  "people",
-  "percent",
-  "dollars",
-  "meters",
-  "metres",
-  "kilometers",
-  "kilometres",
-  "miles",
-  "km",
-  "m"
-]);
-var WRITTEN_NUMBERS = {
-  zero: 0,
-  one: 1,
-  two: 2,
-  three: 3,
-  four: 4,
-  five: 5,
-  six: 6,
-  seven: 7,
-  eight: 8,
-  nine: 9,
-  ten: 10,
-  eleven: 11,
-  twelve: 12,
-  thirteen: 13,
-  fourteen: 14,
-  fifteen: 15,
-  sixteen: 16,
-  seventeen: 17,
-  eighteen: 18,
-  nineteen: 19,
-  twenty: 20,
-  thirty: 30,
-  forty: 40,
-  fifty: 50,
-  sixty: 60,
-  seventy: 70,
-  eighty: 80,
-  ninety: 90,
-  hundred: 100,
-  thousand: 1e3,
-  million: 1e6,
-  billion: 1e9
-};
-var MONTHS = {
-  january: "01",
-  february: "02",
-  march: "03",
-  april: "04",
-  may: "05",
-  june: "06",
-  july: "07",
-  august: "08",
-  september: "09",
-  october: "10",
-  november: "11",
-  december: "12"
-};
-var SCALES = /* @__PURE__ */ new Set(["hundred", "thousand", "million", "billion"]);
-function composeWrittenRun(words) {
-  let total = 0;
-  let current = 0;
-  for (const w of words) {
-    const v = WRITTEN_NUMBERS[w];
-    if (v === void 0) continue;
-    if (w === "hundred") {
-      current = (current === 0 ? 1 : current) * 100;
-    } else if (SCALES.has(w)) {
-      current = (current === 0 ? 1 : current) * v;
-      total += current;
-      current = 0;
-    } else {
-      current += v;
-    }
-  }
-  return total + current;
-}
-function canonicalizeNumbers(s) {
-  let out = s.replace(/(\d),(?=\d{3}\b)/g, "$1");
-  out = out.replace(/\b(\d+(?:\.\d+)?)\s+(hundred|thousand|million|billion)\b/g, (_m, num, scaleWord) => {
-    const scale = WRITTEN_NUMBERS[scaleWord] ?? 1;
-    return String(Number(num) * scale);
-  });
-  out = out.replace(/\b[a-z]+(?:[\s-]+[a-z]+)*\b/g, (run) => {
-    const words = run.split(/[\s-]+/);
-    if (!words.every((w) => w in WRITTEN_NUMBERS)) return run;
-    return String(composeWrittenRun(words));
-  });
-  return out;
-}
-function canonicalizeDates(s) {
-  let out = s.replace(
-    /\b(january|february|march|april|may|june|july|august|september|october|november|december)\s+(\d{1,2}),?\s+(\d{4})\b/g,
-    (_m, mon, day, year) => `${year}-${MONTHS[mon]}-${day.padStart(2, "0")}`
-  );
-  out = out.replace(
-    /\b(\d{1,2})\s+(january|february|march|april|may|june|july|august|september|october|november|december)\s+(\d{4})\b/g,
-    (_m, day, mon, year) => `${year}-${MONTHS[mon]}-${day.padStart(2, "0")}`
-  );
-  return out;
-}
-function normalizeAnswer(input) {
-  let s = input.toLowerCase();
-  s = canonicalizeDates(s);
-  s = canonicalizeNumbers(s);
-  s = s.replace(/[^\w\s-]/g, " ");
-  const tokens = s.split(/\s+/).filter((t) => t.length > 0).filter((t) => !ARTICLES.has(t)).filter((t) => !UNIT_WORDS.has(t));
-  return tokens.join(" ").trim();
-}
-function tier1Match(candidate, gold) {
-  const nc = normalizeAnswer(candidate);
-  const ng = normalizeAnswer(gold);
-  if (ng.length === 0) return false;
-  if (nc === ng) return true;
-  const re = new RegExp(`(^|\\s)${ng.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}(\\s|$)`);
-  return re.test(nc);
-}
-function parseFinalAnswer(artifact) {
-  const lines = artifact.split(/\r?\n/);
-  for (let i = lines.length - 1; i >= 0; i -= 1) {
-    const line = lines[i] ?? "";
-    const idx = line.toUpperCase().indexOf(FINAL_ANSWER_SENTINEL);
-    if (idx !== -1) return line.slice(idx + FINAL_ANSWER_SENTINEL.length).trim();
-  }
-  for (let i = lines.length - 1; i >= 0; i -= 1) {
-    const t = (lines[i] ?? "").trim();
-    if (t.length > 0) return t;
-  }
-  return "";
-}
-function parseCitations(artifact) {
-  const urls = /* @__PURE__ */ new Set();
-  for (const m of artifact.matchAll(/https?:\/\/[^\s)<>"']+/g)) {
-    urls.add(m[0].replace(/[.,;]+$/, ""));
-  }
-  return [...urls];
-}
-function citationCoverage(citations, goldSources) {
-  if (goldSources.length === 0) return 1;
-  const slug = (u) => {
-    const m = u.match(/\/wiki\/([^#?]+)/);
-    return m ? decodeURIComponent(m[1] ?? "").toLowerCase() : u.toLowerCase();
-  };
-  const got = new Set(citations.map(slug));
-  let hit = 0;
-  for (const g of goldSources) if (got.has(slug(g))) hit += 1;
-  return hit / goldSources.length;
-}
-var JUDGE_PROMPT = (question, gold, candidate) => [
-  "You are a strict answer-equivalence checker for a factual question-answering benchmark.",
-  "Decide ONLY whether the candidate answer is semantically equivalent to the gold answer for this question.",
-  "It is CORRECT iff it contains the same factual value as the gold answer. Extra correct detail is fine.",
-  "A different value, a missing value, or a wrong value is INCORRECT.",
-  "",
-  `Question: ${question}`,
-  `Gold answer: ${gold}`,
-  `Candidate answer: ${candidate}`,
-  "",
-  "Respond with ONLY a fenced JSON block and nothing else:",
-  "```json",
-  '{"verdict": "correct" | "incorrect"}',
-  "```"
-].join("\n");
-function judgeRouter() {
-  const model = process.env.JUDGE_MODEL;
-  if (!model) throw new Error("JUDGE_MODEL is required for the FRAMES Tier-2 equivalence judge");
-  const key = process.env.TANGLE_API_KEY;
-  if (!key) throw new Error("TANGLE_API_KEY is required for the FRAMES Tier-2 judge");
-  const baseUrl = process.env.JUDGE_ROUTER_BASE ?? process.env.ROUTER_BASE ?? "https://router.tangle.tools/v1";
-  return { baseUrl, key, model };
-}
-async function tier2Judge(question, gold, candidate, router) {
-  const res = await fetch(`${router.baseUrl}/chat/completions`, {
-    method: "POST",
-    headers: { "content-type": "application/json", authorization: `Bearer ${router.key}` },
-    body: JSON.stringify({
-      model: router.model,
-      temperature: 0,
-      seed: 0,
-      messages: [{ role: "user", content: JUDGE_PROMPT(question, gold, candidate) }]
-    })
-  });
-  if (!res.ok) {
-    throw new Error(`FRAMES Tier-2 judge HTTP ${res.status}: ${(await res.text()).slice(0, 300)}`);
-  }
-  const body = await res.json();
-  const content = body.choices?.[0]?.message?.content;
-  if (typeof content !== "string") {
-    throw new Error(`FRAMES Tier-2 judge returned no message content: ${JSON.stringify(body).slice(0, 300)}`);
-  }
-  const fenced = content.match(/```(?:json)?\s*([\s\S]*?)```/);
-  const raw = (fenced ? fenced[1] : content)?.trim() ?? "";
-  let parsed;
-  try {
-    parsed = JSON.parse(raw);
-  } catch {
-    throw new Error(`FRAMES Tier-2 judge produced unparseable output (no JSON verdict): ${content.slice(0, 300)}`);
-  }
-  if (parsed.verdict === "correct") return true;
-  if (parsed.verdict === "incorrect") return false;
-  throw new Error(`FRAMES Tier-2 judge verdict not in {correct,incorrect}: ${JSON.stringify(parsed).slice(0, 200)}`);
-}
-function parseWikiLinks(raw) {
-  if (!raw) return [];
-  const out = [];
-  for (const m of raw.matchAll(/https?:\/\/[^\s'"\]]+/g)) out.push(m[0]);
-  return out;
-}
-function rowToTask(row, index) {
-  const goldSources = parseWikiLinks(row.wiki_links);
-  const meta = {
-    gold: row.Answer,
-    goldSources,
-    reasoningTypes: row.reasoning_types ?? "",
-    rawPrompt: row.Prompt
-  };
-  return {
-    id: `frames-${index}`,
-    split: "test",
-    prompt: row.Prompt + WORKER_CONTRACT,
-    metadata: meta
-  };
-}
-function readMeta(task) {
-  const md = task.metadata;
-  if (!md || typeof md.gold !== "string") {
-    throw new Error(`FRAMES task ${task.id} missing metadata.gold \u2014 loadTasks did not populate it`);
-  }
-  return md;
-}
-async function loadFixtures(opts) {
-  const rows = JSON.parse(await readFile(FIXTURES, "utf8"));
-  console.log(`[frames] FRAMES_FIXTURES=1 \u2014 loading ${rows.length} committed fixtures (no HF download)`);
-  let tasks = rows.map(rowToTask);
-  if (opts.ids) {
-    const want = new Set(opts.ids);
-    tasks = tasks.filter((t) => want.has(t.id));
-  } else if (opts.limit !== void 0) {
-    tasks = tasks.slice(0, opts.limit);
-  }
-  return tasks;
-}
-function createFramesAdapter() {
-  const fixturesMode = process.env.FRAMES_FIXTURES === "1";
-  return {
-    name: "frames",
-    async preflight() {
-      judgeRouter();
-      if (fixturesMode) {
-        await readFile(FIXTURES, "utf8").catch((err) => {
-          throw new Error(`FRAMES_FIXTURES=1 but ${FIXTURES} unreadable: ${err instanceof Error ? err.message : err}`);
-        });
-        return;
-      }
-      try {
-        await py(
-          `from datasets import load_dataset
-load_dataset(${JSON.stringify(DATASET)}, split='test', revision=${JSON.stringify(DATASET_REVISION)})
-print('ok')`
-        );
-      } catch (err) {
-        const msg = err instanceof Error ? err.message : String(err);
-        throw new Error(
-          `frames preflight failed: ${msg}
-Fix: (1) python3 -m venv bench/.venv && bench/.venv/bin/pip install datasets ; (2) ensure network access to Hugging Face for ${DATASET} ; or set FRAMES_FIXTURES=1 to run against the committed fixtures offline.`
-        );
-      }
-    },
-    async loadTasks(opts = {}) {
-      if (fixturesMode) return loadFixtures(opts);
-      const limit = opts.limit ?? 10;
-      const script = `
-import json, sys
-from datasets import load_dataset
-ds = load_dataset(${JSON.stringify(DATASET)}, split='test', revision=${JSON.stringify(DATASET_REVISION)})
-ids = set(json.loads(sys.argv[1])) if len(sys.argv) > 1 and sys.argv[1] else None
-out = []
-for i, r in enumerate(ds):
-    rid = f"frames-{i}"
-    if ids is not None and rid not in ids:
-        continue
-    out.append({
-        "_index": i,
-        "Prompt": r.get("Prompt", ""),
-        "Answer": r.get("Answer", ""),
-        "wiki_links": str(r.get("wiki_links", "")),
-        "reasoning_types": str(r.get("reasoning_types", "")),
-    })
-    if ids is None and len(out) >= ${limit}:
-        break
-print(json.dumps(out))
-`;
-      const stdout = await py(script, [opts.ids ? JSON.stringify(opts.ids) : ""]);
-      const rows = JSON.parse(stdout);
-      return rows.map((r) => rowToTask(r, r._index));
-    },
-    async goldArtifact(task) {
-      const meta = readMeta(task);
-      return `${FINAL_ANSWER_SENTINEL} ${meta.gold}`;
-    },
-    async judge(task, artifact) {
-      const meta = readMeta(task);
-      const finalAnswer = parseFinalAnswer(artifact);
-      const citations = parseCitations(artifact);
-      const coverage = citationCoverage(citations, meta.goldSources);
-      if (finalAnswer.length === 0) {
-        return {
-          resolved: false,
-          score: 0,
-          detail: JSON.stringify({
-            tier: "none",
-            reason: "no parseable answer",
-            normalizedGold: normalizeAnswer(meta.gold),
-            citationCoverage: coverage
-          })
-        };
-      }
-      if (tier1Match(finalAnswer, meta.gold)) {
-        return {
-          resolved: true,
-          score: 1,
-          detail: JSON.stringify({
-            tier: 1,
-            normalizedAnswer: normalizeAnswer(finalAnswer),
-            normalizedGold: normalizeAnswer(meta.gold),
-            citationCoverage: coverage
-          })
-        };
-      }
-      const verdict = await tier2Judge(meta.rawPrompt, meta.gold, finalAnswer, judgeRouter());
-      return {
-        resolved: verdict,
-        score: verdict ? 1 : 0,
-        detail: JSON.stringify({
-          tier: 2,
-          normalizedAnswer: normalizeAnswer(finalAnswer),
-          normalizedGold: normalizeAnswer(meta.gold),
-          judgeVerdict: verdict ? "correct" : "incorrect",
-          citationCoverage: coverage
-        })
-      };
-    }
-  };
-}
-
-export {
-  normalizeAnswer,
-  parseFinalAnswer,
-  parseCitations,
-  createFramesAdapter
-};
-//# sourceMappingURL=chunk-INNOYXCP.js.map