@tangle-network/agent-eval 0.118.3 → 0.119.1

package/dist/primeintellect/index.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/primeintellect/index.ts"],"sourcesContent":["import { mkdir, writeFile } from 'node:fs/promises'\nimport { dirname, join } from 'node:path'\nimport { type RunRecord, validateRunRecord } from '../run-record'\n\nexport type PrimeIntellectJson =\n  | null\n  | boolean\n  | number\n  | string\n  | PrimeIntellectJson[]\n  | { [key: string]: PrimeIntellectJson }\n\nexport interface PrimeIntellectMessage {\n  role: 'system' | 'user' | 'assistant' | 'tool'\n  content: string\n}\n\nexport interface PrimeIntellectScenario {\n  id: string\n  prompt: string | readonly PrimeIntellectMessage[]\n  answer?: string\n  requiredSubstrings?: readonly string[]\n  split?: string\n  info?: Record<string, PrimeIntellectJson>\n}\n\nexport interface PrimeIntellectDatasetRow {\n  id: string\n  prompt: PrimeIntellectMessage[]\n  answer?: string\n  required_substrings?: string[]\n  split?: string\n  info: Record<string, PrimeIntellectJson>\n}\n\nexport type PrimeIntellectScenarioMap =\n  | ReadonlyMap<string, PrimeIntellectScenario>\n  | Record<string, PrimeIntellectScenario>\n  | readonly PrimeIntellectScenario[]\n\nexport interface PrimeIntellectRowsFromRunRecordsOptions {\n  records: readonly RunRecord[]\n  scenarios: PrimeIntellectScenarioMap\n  requireScorable?: boolean\n  includeFailed?: boolean\n}\n\nexport interface PrimeIntellectEnvironmentPackageInput {\n  name: string\n  version?: string\n  description?: string\n  tags?: readonly string[]\n  moduleName?: string\n  rows: readonly PrimeIntellectDatasetRow[]\n  runRecords?: readonly RunRecord[]\n  systemPrompt?: string\n  verifiersVersion?: string\n  dependencies?: readonly string[]\n  readme?: string\n}\n\nexport interface PrimeIntellectPackageFile {\n  path: string\n  content: string\n}\n\nexport interface PrimeIntellectPackageManifest {\n  schemaVersion: 1\n  name: string\n  moduleName: string\n  version: string\n  rowCount: number\n  runRecordCount: number\n  artifactKinds: readonly ['environment', 'dataset', 'run_records']\n}\n\nexport interface PrimeIntellectEnvironmentPackage {\n  files: PrimeIntellectPackageFile[]\n  manifest: PrimeIntellectPackageManifest\n}\n\nexport class PrimeIntellectBridgeError extends Error {\n  constructor(message: string) {\n    super(message)\n    this.name = 'PrimeIntellectBridgeError'\n  }\n}\n\nexport function primeIntellectRowsFromRunRecords(\n  options: PrimeIntellectRowsFromRunRecordsOptions,\n): PrimeIntellectDatasetRow[] {\n  const scenarioById = normalizeScenarios(options.scenarios)\n  const requireScorable = options.requireScorable ?? true\n  const includeFailed = options.includeFailed ?? true\n  const rows: PrimeIntellectDatasetRow[] = []\n\n  for (const record of options.records) {\n    const validRecord = validateRunRecord(record)\n    if (!includeFailed && validRecord.failureMode) continue\n\n    const scenarioId = validRecord.scenarioId\n    if (!scenarioId) {\n      throw new PrimeIntellectBridgeError(\n        `RunRecord ${validRecord.runId} is missing scenarioId; PrimeIntellect rows need stable task ids`,\n      )\n    }\n\n    const scenario = scenarioById.get(scenarioId)\n    if (!scenario) {\n      throw new PrimeIntellectBridgeError(\n        `RunRecord ${validRecord.runId} references unknown scenarioId ${JSON.stringify(scenarioId)}`,\n      )\n    }\n\n    const hasAnswer = typeof scenario.answer === 'string' && scenario.answer.trim().length > 0\n    const hasRequiredSubstrings = (scenario.requiredSubstrings?.length ?? 0) > 0\n    if (requireScorable && !hasAnswer && !hasRequiredSubstrings) {\n      throw new PrimeIntellectBridgeError(\n        `Scenario ${scenario.id} has no answer or requiredSubstrings; generated environments would always score 0`,\n      )\n    }\n\n    const score = validRecord.outcome.holdoutScore ?? validRecord.outcome.searchScore ?? null\n    const split = scenario.split ?? validRecord.splitTag\n    const tangleInfo = {\n      run_id: validRecord.runId,\n      experiment_id: validRecord.experimentId,\n      candidate_id: validRecord.candidateId,\n      scenario_id: scenarioId,\n      split,\n      score,\n      cost_usd: validRecord.costUsd,\n      wall_ms: validRecord.wallMs,\n      failure_mode: validRecord.failureMode ?? null,\n      model: validRecord.model,\n      commit_sha: validRecord.commitSha,\n    } satisfies Record<string, PrimeIntellectJson>\n\n    rows.push({\n      id: `${scenarioId}:${validRecord.runId}`,\n      prompt: normalizePrompt(scenario.prompt),\n      ...(hasAnswer ? { answer: scenario.answer } : {}),\n      ...(hasRequiredSubstrings\n        ? { required_substrings: [...(scenario.requiredSubstrings ?? [])] }\n        : {}),\n      split,\n      info: {\n        ...(scenario.info ?? {}),\n        tangle: tangleInfo,\n      },\n    })\n  }\n\n  return rows\n}\n\nexport function buildPrimeIntellectEnvironmentPackage(\n  input: PrimeIntellectEnvironmentPackageInput,\n): PrimeIntellectEnvironmentPackage {\n  if (input.rows.length === 0) {\n    throw new PrimeIntellectBridgeError('cannot build a PrimeIntellect environment with zero rows')\n  }\n\n  for (const record of input.runRecords ?? []) {\n    validateRunRecord(record)\n  }\n\n  const moduleName = input.moduleName ?? toPythonModuleName(input.name)\n  if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(moduleName)) {\n    throw new PrimeIntellectBridgeError(\n      `moduleName ${JSON.stringify(moduleName)} is not a valid Python module name`,\n    )\n  }\n\n  const version = input.version ?? '0.1.0'\n  const description =\n    input.description ?? 'PrimeIntellect Verifiers environment exported from Tangle runs.'\n  const verifiersVersion = input.verifiersVersion ?? '>=0.2.0,<0.3.0'\n  const dependencies = unique([\n    'datasets',\n    `verifiers${verifiersVersion}`,\n    ...(input.dependencies ?? []),\n  ])\n  const tags = unique(['tangle', 'agent-eval', ...(input.tags ?? [])])\n  const manifest: PrimeIntellectPackageManifest = {\n    schemaVersion: 1,\n    name: input.name,\n    moduleName,\n    version,\n    rowCount: input.rows.length,\n    runRecordCount: input.runRecords?.length ?? 0,\n    artifactKinds: ['environment', 'dataset', 'run_records'],\n  }\n\n  const files: PrimeIntellectPackageFile[] = [\n    {\n      path: 'pyproject.toml',\n      content: renderPyproject({\n        name: input.name,\n        version,\n        description,\n        dependencies,\n        tags,\n      }),\n    },\n    {\n      path: 'README.md',\n      content: input.readme ?? renderReadme({ name: input.name, description }),\n    },\n    {\n      path: `${moduleName}.py`,\n      content: renderEnvironmentModule({\n        systemPrompt: input.systemPrompt,\n      }),\n    },\n    {\n      path: 'data/dataset.jsonl',\n      content: toJsonl(input.rows),\n    },\n    {\n      path: 'data/run_records.jsonl',\n      content: toJsonl(input.runRecords ?? []),\n    },\n    {\n      path: 'tangle-primeintellect-manifest.json',\n      content: `${stableJson(manifest)}\\n`,\n    },\n  ]\n\n  return { files, manifest }\n}\n\nexport async function writePrimeIntellectEnvironmentPackage(\n  root: string,\n  input: PrimeIntellectEnvironmentPackageInput,\n): Promise<PrimeIntellectEnvironmentPackage> {\n  const pkg = buildPrimeIntellectEnvironmentPackage(input)\n  await Promise.all(\n    pkg.files.map(async (file) => {\n      const target = join(root, file.path)\n      await mkdir(dirname(target), { recursive: true })\n      await writeFile(target, file.content, 'utf8')\n    }),\n  )\n  return pkg\n}\n\nfunction normalizeScenarios(\n  scenarios: PrimeIntellectScenarioMap,\n): Map<string, PrimeIntellectScenario> {\n  const map = new Map<string, PrimeIntellectScenario>()\n  if (scenarios instanceof Map) {\n    for (const [id, scenario] of scenarios.entries()) {\n      map.set(id, { ...scenario, id: scenario.id ?? id })\n    }\n    return map\n  }\n\n  const values = Array.isArray(scenarios) ? scenarios : Object.values(scenarios)\n  for (const scenario of values) {\n    if (!scenario.id) {\n      throw new PrimeIntellectBridgeError('every PrimeIntellect scenario needs an id')\n    }\n    if (map.has(scenario.id)) {\n      throw new PrimeIntellectBridgeError(`duplicate PrimeIntellect scenario id ${scenario.id}`)\n    }\n    map.set(scenario.id, scenario)\n  }\n  return map\n}\n\nfunction normalizePrompt(\n  prompt: string | readonly PrimeIntellectMessage[],\n): PrimeIntellectMessage[] {\n  if (typeof prompt === 'string') {\n    return [{ role: 'user', content: prompt }]\n  }\n  return prompt.map((message) => ({ role: message.role, content: message.content }))\n}\n\nfunction toPythonModuleName(name: string): string {\n  return name\n    .trim()\n    .toLowerCase()\n    .replace(/[^a-z0-9_]+/g, '_')\n    .replace(/^_+|_+$/g, '')\n    .replace(/^[0-9]/, '_$&')\n}\n\nfunction unique(values: readonly string[]): string[] {\n  return [...new Set(values)]\n}\n\nfunction toJsonl(values: readonly unknown[]): string {\n  if (values.length === 0) return ''\n  return `${values.map((value) => stableJson(value)).join('\\n')}\\n`\n}\n\nfunction stableJson(value: unknown): string {\n  return JSON.stringify(sortJson(value))\n}\n\nfunction sortJson(value: unknown): unknown {\n  if (Array.isArray(value)) return value.map(sortJson)\n  if (value === null || typeof value !== 'object') return value\n  return Object.fromEntries(\n    Object.entries(value as Record<string, unknown>)\n      .sort(([a], [b]) => a.localeCompare(b))\n      .map(([key, nested]) => [key, sortJson(nested)]),\n  )\n}\n\nfunction renderPyproject(input: {\n  name: string\n  version: string\n  description: string\n  dependencies: readonly string[]\n  tags: readonly string[]\n}): string {\n  return `[project]\nname = ${tomlString(input.name)}\nversion = ${tomlString(input.version)}\ndescription = ${tomlString(input.description)}\nreadme = \"README.md\"\nrequires-python = \">=3.11\"\nlicense = \"MIT\"\nkeywords = ${tomlArray(input.tags)}\ndependencies = ${tomlArray(input.dependencies)}\n\n[build-system]\nrequires = [\"hatchling\"]\nbuild-backend = \"hatchling.build\"\n\n[tool.hatch.build]\ninclude = [\"*.py\", \"README.md\", \"data/*.jsonl\", \"tangle-primeintellect-manifest.json\"]\n\n[tool.uv]\nprerelease = \"allow\"\n\n[tool.verifiers.eval]\nnum_examples = 5\nrollouts_per_example = 3\n`\n}\n\nfunction renderReadme(input: { name: string; description: string }): string {\n  return `# ${input.name}\n\n${input.description}\n\nGenerated by \\`@tangle-network/agent-eval/primeintellect\\` from validated Tangle \\`RunRecord\\` rows.\n\n## Files\n\n- \\`data/dataset.jsonl\\`: prompts and scoring references for PrimeIntellect Verifiers.\n- \\`data/run_records.jsonl\\`: original Tangle run rows for provenance and analysis.\n- \\`tangle-primeintellect-manifest.json\\`: export metadata.\n\n## Local check\n\n\\`\\`\\`sh\nuv pip install --prerelease=allow -e .\nuv run vf-eval ${input.name}\n\\`\\`\\`\n\n## Upload\n\n\\`\\`\\`sh\nprime login\nprime env push\n\\`\\`\\`\n`\n}\n\nfunction renderEnvironmentModule(input: { systemPrompt?: string }): string {\n  const systemPrompt = input.systemPrompt === undefined ? 'None' : pyString(input.systemPrompt)\n  return `import json\nimport re\nfrom pathlib import Path\n\nfrom datasets import Dataset\nimport verifiers as vf\n\n\nDATA_PATH = Path(__file__).parent / \"data\" / \"dataset.jsonl\"\nDEFAULT_SYSTEM_PROMPT = ${systemPrompt}\n\n\ndef _load_rows(dataset_path=None, dataset_split=None):\n    path = Path(dataset_path) if dataset_path is not None else DATA_PATH\n    rows = []\n    with path.open(\"r\", encoding=\"utf-8\") as handle:\n        for line in handle:\n            row = json.loads(line)\n            if dataset_split is None or row.get(\"split\") == dataset_split:\n                rows.append(row)\n    if not rows:\n        raise ValueError(f\"No dataset rows found in {path} for split={dataset_split!r}\")\n    return rows\n\n\ndef _message_text(message):\n    if isinstance(message, dict):\n        content = message.get(\"content\", \"\")\n    else:\n        content = getattr(message, \"content\", \"\")\n    if isinstance(content, list):\n        return \" \".join(\n            str(part.get(\"text\", part)) if isinstance(part, dict) else str(part)\n            for part in content\n        )\n    return str(content)\n\n\ndef _completion_text(completion):\n    if isinstance(completion, str):\n        return completion\n    if isinstance(completion, list):\n        return \"\\\\n\".join(_message_text(message) for message in completion)\n    return _message_text(completion)\n\n\ndef _normalize(text):\n    return re.sub(r\"\\\\s+\", \" \", str(text).strip().lower())\n\n\nasync def tangle_reward(completion, answer=None, required_substrings=None, **kwargs):\n    response = _normalize(_completion_text(completion))\n    scores = []\n    if answer:\n        expected = _normalize(answer)\n        scores.append(1.0 if expected == response or expected in response else 0.0)\n    if required_substrings:\n        required = [_normalize(item) for item in required_substrings]\n        hits = sum(1 for item in required if item and item in response)\n        scores.append(hits / len(required))\n    return max(scores) if scores else 0.0\n\n\ndef build_dataset(dataset_path=None, dataset_split=None):\n    return Dataset.from_list(_load_rows(dataset_path=dataset_path, dataset_split=dataset_split))\n\n\ndef load_environment(dataset_path=None, dataset_split=None, system_prompt=DEFAULT_SYSTEM_PROMPT):\n    rubric = vf.Rubric(funcs=[tangle_reward])\n    return vf.SingleTurnEnv(\n        dataset=lambda: build_dataset(dataset_path=dataset_path, dataset_split=dataset_split),\n        system_prompt=system_prompt,\n        rubric=rubric,\n    )\n`\n}\n\nfunction tomlString(value: string): string {\n  return JSON.stringify(value)\n}\n\nfunction tomlArray(values: readonly string[]): string {\n  return `[${values.map(tomlString).join(', ')}]`\n}\n\nfunction pyString(value: string): string {\n  return JSON.stringify(value)\n}\n"],"mappings":";;;;;;;;;;AAAA,SAAS,OAAO,iBAAiB;AACjC,SAAS,SAAS,YAAY;AAgFvB,IAAM,4BAAN,cAAwC,MAAM;AAAA,EACnD,YAAY,SAAiB;AAC3B,UAAM,OAAO;AACb,SAAK,OAAO;AAAA,EACd;AACF;AAEO,SAAS,iCACd,SAC4B;AAC5B,QAAM,eAAe,mBAAmB,QAAQ,SAAS;AACzD,QAAM,kBAAkB,QAAQ,mBAAmB;AACnD,QAAM,gBAAgB,QAAQ,iBAAiB;AAC/C,QAAM,OAAmC,CAAC;AAE1C,aAAW,UAAU,QAAQ,SAAS;AACpC,UAAM,cAAc,kBAAkB,MAAM;AAC5C,QAAI,CAAC,iBAAiB,YAAY,YAAa;AAE/C,UAAM,aAAa,YAAY;AAC/B,QAAI,CAAC,YAAY;AACf,YAAM,IAAI;AAAA,QACR,aAAa,YAAY,KAAK;AAAA,MAChC;AAAA,IACF;AAEA,UAAM,WAAW,aAAa,IAAI,UAAU;AAC5C,QAAI,CAAC,UAAU;AACb,YAAM,IAAI;AAAA,QACR,aAAa,YAAY,KAAK,kCAAkC,KAAK,UAAU,UAAU,CAAC;AAAA,MAC5F;AAAA,IACF;AAEA,UAAM,YAAY,OAAO,SAAS,WAAW,YAAY,SAAS,OAAO,KAAK,EAAE,SAAS;AACzF,UAAM,yBAAyB,SAAS,oBAAoB,UAAU,KAAK;AAC3E,QAAI,mBAAmB,CAAC,aAAa,CAAC,uBAAuB;AAC3D,YAAM,IAAI;AAAA,QACR,YAAY,SAAS,EAAE;AAAA,MACzB;AAAA,IACF;AAEA,UAAM,QAAQ,YAAY,QAAQ,gBAAgB,YAAY,QAAQ,eAAe;AACrF,UAAM,QAAQ,SAAS,SAAS,YAAY;AAC5C,UAAM,aAAa;AAAA,MACjB,QAAQ,YAAY;AAAA,MACpB,eAAe,YAAY;AAAA,MAC3B,cAAc,YAAY;AAAA,MAC1B,aAAa;AAAA,MACb;AAAA,MACA;AAAA,MACA,UAAU,YAAY;AAAA,MACtB,SAAS,YAAY;AAAA,MACrB,cAAc,YAAY,eAAe;AAAA,MACzC,OAAO,YAAY;AAAA,MACnB,YAAY,YAAY;AAAA,IAC1B;AAEA,SAAK,KAAK;AAAA,MACR,IAAI,GAAG,UAAU,IAAI,YAAY,KAAK;AAAA,MACtC,QAAQ,gBAAgB,SAAS,MAAM;AAAA,MACvC,GAAI,YAAY,EAAE,QAAQ,SAAS,OAAO,IAAI,CAAC;AAAA,MAC/C,GAAI,wBACA,EAAE,qBAAqB,CAAC,GAAI,SAAS,sBAAsB,CAAC,CAAE,EAAE,IAChE,CAAC;AAAA,MACL;AAAA,MACA,MAAM;AAAA,QACJ,GAAI,SAAS,QAAQ,CAAC;AAAA,QACtB,QAAQ;AAAA,MACV;AAAA,IACF,CAAC;AAAA,EACH;AAEA,SAAO;AACT;AAEO,SAAS,sCACd,OACkC;AAClC,MAAI,MAAM,KAAK,WAAW,GAAG;AAC3B,UAAM,IAAI,0BAA0B,0DAA0D;AAAA,EAChG;AAEA,aAAW,UAAU,MAAM,cAAc,CAAC,GAAG;AAC3C,sBAAkB,MAAM;AAAA,EAC1B;AAEA,QAAM,aAAa,MAAM,cAAc,mBAAmB,MAAM,IAAI;AACpE,MAAI,CAAC,2BAA2B,KAAK,UAAU,GAAG;AAChD,UAAM,IAAI;AAAA,MACR,cAAc,KAAK,UAAU,UAAU,CAAC;AAAA,IAC1C;AAAA,EACF;AAEA,QAAM,UAAU,MAAM,WAAW;AACjC,QAAM,cACJ,MAAM,eAAe;AACvB,QAAM,mBAAmB,MAAM,oBAAoB;AACnD,QAAM,eAAe,OAAO;AAAA,IAC1B;AAAA,IACA,YAAY,gBAAgB;AAAA,IAC5B,GAAI,MAAM,gBAAgB,CAAC;AAAA,EAC7B,CAAC;AACD,QAAM,OAAO,OAAO,CAAC,UAAU,cAAc,GAAI,MAAM,QAAQ,CAAC,CAAE,CAAC;AACnE,QAAM,WAA0C;AAAA,IAC9C,eAAe;AAAA,IACf,MAAM,MAAM;AAAA,IACZ;AAAA,IACA;AAAA,IACA,UAAU,MAAM,KAAK;AAAA,IACrB,gBAAgB,MAAM,YAAY,UAAU;AAAA,IAC5C,eAAe,CAAC,eAAe,WAAW,aAAa;AAAA,EACzD;AAEA,QAAM,QAAqC;AAAA,IACzC;AAAA,MACE,MAAM;AAAA,MACN,SAAS,gBAAgB;AAAA,QACvB,MAAM,MAAM;AAAA,QACZ;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,MACF,CAAC;AAAA,IACH;AAAA,IACA;AAAA,MACE,MAAM;AAAA,MACN,SAAS,MAAM,UAAU,aAAa,EAAE,MAAM,MAAM,MAAM,YAAY,CAAC;AAAA,IACzE;AAAA,IACA;AAAA,MACE,MAAM,GAAG,UAAU;AAAA,MACnB,SAAS,wBAAwB;AAAA,QAC/B,cAAc,MAAM;AAAA,MACtB,CAAC;AAAA,IACH;AAAA,IACA;AAAA,MACE,MAAM;AAAA,MACN,SAAS,QAAQ,MAAM,IAAI;AAAA,IAC7B;AAAA,IACA;AAAA,MACE,MAAM;AAAA,MACN,SAAS,QAAQ,MAAM,cAAc,CAAC,CAAC;AAAA,IACzC;AAAA,IACA;AAAA,MACE,MAAM;AAAA,MACN,SAAS,GAAG,WAAW,QAAQ,CAAC;AAAA;AAAA,IAClC;AAAA,EACF;AAEA,SAAO,EAAE,OAAO,SAAS;AAC3B;AAEA,eAAsB,sCACpB,MACA,OAC2C;AAC3C,QAAM,MAAM,sCAAsC,KAAK;AACvD,QAAM,QAAQ;AAAA,IACZ,IAAI,MAAM,IAAI,OAAO,SAAS;AAC5B,YAAM,SAAS,KAAK,MAAM,KAAK,IAAI;AACnC,YAAM,MAAM,QAAQ,MAAM,GAAG,EAAE,WAAW,KAAK,CAAC;AAChD,YAAM,UAAU,QAAQ,KAAK,SAAS,MAAM;AAAA,IAC9C,CAAC;AAAA,EACH;AACA,SAAO;AACT;AAEA,SAAS,mBACP,WACqC;AACrC,QAAM,MAAM,oBAAI,IAAoC;AACpD,MAAI,qBAAqB,KAAK;AAC5B,eAAW,CAAC,IAAI,QAAQ,KAAK,UAAU,QAAQ,GAAG;AAChD,UAAI,IAAI,IAAI,EAAE,GAAG,UAAU,IAAI,SAAS,MAAM,GAAG,CAAC;AAAA,IACpD;AACA,WAAO;AAAA,EACT;AAEA,QAAM,SAAS,MAAM,QAAQ,SAAS,IAAI,YAAY,OAAO,OAAO,SAAS;AAC7E,aAAW,YAAY,QAAQ;AAC7B,QAAI,CAAC,SAAS,IAAI;AAChB,YAAM,IAAI,0BAA0B,2CAA2C;AAAA,IACjF;AACA,QAAI,IAAI,IAAI,SAAS,EAAE,GAAG;AACxB,YAAM,IAAI,0BAA0B,wCAAwC,SAAS,EAAE,EAAE;AAAA,IAC3F;AACA,QAAI,IAAI,SAAS,IAAI,QAAQ;AAAA,EAC/B;AACA,SAAO;AACT;AAEA,SAAS,gBACP,QACyB;AACzB,MAAI,OAAO,WAAW,UAAU;AAC9B,WAAO,CAAC,EAAE,MAAM,QAAQ,SAAS,OAAO,CAAC;AAAA,EAC3C;AACA,SAAO,OAAO,IAAI,CAAC,aAAa,EAAE,MAAM,QAAQ,MAAM,SAAS,QAAQ,QAAQ,EAAE;AACnF;AAEA,SAAS,mBAAmB,MAAsB;AAChD,SAAO,KACJ,KAAK,EACL,YAAY,EACZ,QAAQ,gBAAgB,GAAG,EAC3B,QAAQ,YAAY,EAAE,EACtB,QAAQ,UAAU,KAAK;AAC5B;AAEA,SAAS,OAAO,QAAqC;AACnD,SAAO,CAAC,GAAG,IAAI,IAAI,MAAM,CAAC;AAC5B;AAEA,SAAS,QAAQ,QAAoC;AACnD,MAAI,OAAO,WAAW,EAAG,QAAO;AAChC,SAAO,GAAG,OAAO,IAAI,CAAC,UAAU,WAAW,KAAK,CAAC,EAAE,KAAK,IAAI,CAAC;AAAA;AAC/D;AAEA,SAAS,WAAW,OAAwB;AAC1C,SAAO,KAAK,UAAU,SAAS,KAAK,CAAC;AACvC;AAEA,SAAS,SAAS,OAAyB;AACzC,MAAI,MAAM,QAAQ,KAAK,EAAG,QAAO,MAAM,IAAI,QAAQ;AACnD,MAAI,UAAU,QAAQ,OAAO,UAAU,SAAU,QAAO;AACxD,SAAO,OAAO;AAAA,IACZ,OAAO,QAAQ,KAAgC,EAC5C,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC,EACrC,IAAI,CAAC,CAAC,KAAK,MAAM,MAAM,CAAC,KAAK,SAAS,MAAM,CAAC,CAAC;AAAA,EACnD;AACF;AAEA,SAAS,gBAAgB,OAMd;AACT,SAAO;AAAA,SACA,WAAW,MAAM,IAAI,CAAC;AAAA,YACnB,WAAW,MAAM,OAAO,CAAC;AAAA,gBACrB,WAAW,MAAM,WAAW,CAAC;AAAA;AAAA;AAAA;AAAA,aAIhC,UAAU,MAAM,IAAI,CAAC;AAAA,iBACjB,UAAU,MAAM,YAAY,CAAC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAgB9C;AAEA,SAAS,aAAa,OAAsD;AAC1E,SAAO,KAAK,MAAM,IAAI;AAAA;AAAA,EAEtB,MAAM,WAAW;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,iBAcF,MAAM,IAAI;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAU3B;AAEA,SAAS,wBAAwB,OAA0C;AACzE,QAAM,eAAe,MAAM,iBAAiB,SAAY,SAAS,SAAS,MAAM,YAAY;AAC5F,SAAO;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,0BASiB,YAAY;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAkEtC;AAEA,SAAS,WAAW,OAAuB;AACzC,SAAO,KAAK,UAAU,KAAK;AAC7B;AAEA,SAAS,UAAU,QAAmC;AACpD,SAAO,IAAI,OAAO,IAAI,UAAU,EAAE,KAAK,IAAI,CAAC;AAC9C;AAEA,SAAS,SAAS,OAAuB;AACvC,SAAO,KAAK,UAAU,KAAK;AAC7B;","names":[]}

package/dist/rl.d.ts

@@ -2384,7 +2384,9 @@ interface ChannelRollup {
     calls: number;
     inputTokens: number;
     outputTokens: number;
+    reasoningTokens?: number;
     cachedTokens: number;
+    cacheWriteTokens?: number;
     costUsd: number;
     unpricedCalls: number;
     unknownUsageCalls: number;
@@ -2396,7 +2398,9 @@ interface CostLedgerSummary {
     reservedCostUsd: number;
     inputTokens: number;
     outputTokens: number;
+    reasoningTokens?: number;
     cachedTokens: number;
+    cacheWriteTokens?: number;
     totalCostUsd: number;
     byChannel: ChannelRollup[];
     unpricedModels: string[];

package/dist/{run-campaign-DWC67KJP.js → run-campaign-OBXSN5WK.js}

@@ -1,9 +1,9 @@
 import {
   planCampaignRun,
   runCampaign
-} from "./chunk-EGNRE7VA.js";
+} from "./chunk-PAHNGS65.js";
 import "./chunk-PJQFMIOX.js";
-import "./chunk-MLHRFWQK.js";
+import "./chunk-JHOJHHU7.js";
 import "./chunk-VI2UW6B6.js";
 import "./chunk-ONWEPEDO.js";
 import "./chunk-PZ5AY32C.js";
@@ -11,4 +11,4 @@ export {
   planCampaignRun,
   runCampaign
 };
-//# sourceMappingURL=run-campaign-DWC67KJP.js.map
+//# sourceMappingURL=run-campaign-OBXSN5WK.js.map

package/dist/traces.d.ts

@@ -1,4 +1,4 @@
-import { AxAIService, AxFunction } from '@ax-llm/ax';
+import { AxAIService, AxAgentActorTurnCallbackArgs, AxFunction } from '@ax-llm/ax';
 
 /**
  * Error taxonomy for `@tangle-network/agent-eval`.
@@ -1726,11 +1726,11 @@ interface AnalyzeTracesInput {
     question: string;
 }
 interface AnalyzeTracesResult {
-    /** The responder's prose answer. */
+    /** The actor's submitted prose answer. */
     answer: string;
-    /** Bulleted findings extracted from the responder's structured output. */
+    /** Bulleted findings from the actor's structured completion. */
     findings: string[];
-    /** Per-actor-turn snapshots captured via `actorTurnCallback`. */
+    /** Per-turn snapshots captured via `actorTurnCallback`. */
     turns: AnalyzeTracesTurnSnapshot[];
     /** Total turns the actor took. */
     turnCount: number;
@@ -1756,13 +1756,14 @@ interface TraceAnalystChatMessage {
     [key: string]: unknown;
 }
 interface AnalyzeTracesTurnSnapshot {
+    stage: AxAgentActorTurnCallbackArgs['stage'];
     turn: number;
     isError: boolean;
     /** The JS code the actor produced for this turn. */
     code: string;
     /** The formatted action-log entry the actor sees on the next turn. */
     output: string;
-    /** Provider thought (when `actorOptions.showThoughts` is true and the
+    /** Provider thought (when `executorOptions.showThoughts` is true and the
      *  provider returns it). */
     thought?: string;
 }
@@ -1771,18 +1772,18 @@ interface AnalyzeTracesOptions {
     source: string | TraceAnalysisStore;
     /** Caller-provided AxAIService. */
     ai: AxAIService;
-    /** Model id forwarded to actor + responder. */
+    /** Model id forwarded to the actor. */
     model?: string;
-    /** Recursion depth. 0 = no sub-agent dispatch. Default 1. */
-    maxDepth?: number;
+    /** Maximum model subqueries. 0 disables model fan-out. Default 4. */
+    maxSubqueries?: number;
     /** Maximum actor turns. Default 12. */
     maxTurns?: number;
-    /** Maximum parallel sub-agent calls in batched llmQuery. Default 2. */
-    maxParallelSubagents?: number;
+    /** Maximum parallel model subqueries. Default 2. */
+    maxParallelSubqueries?: number;
+    /** Cancels in-flight model and tool work. */
+    signal?: AbortSignal;
     /** Override the actor description. */
     actorDescription?: string;
-    /** Override the subagent description. */
-    subagentDescription?: string;
     /** Per-turn observability hook. */
     onTurn?: (turn: AnalyzeTracesTurnSnapshot) => void | Promise<void>;
     /** Override max runtime characters per turn. Default 6000. */
@@ -1790,8 +1791,7 @@ interface AnalyzeTracesOptions {
     /** When set, every turn's snapshot is appended to this JSONL file
      *  immediately. If the analyst crashes mid-loop (provider 503,
      *  network error, validator reject) the partial reasoning is still
-     *  on disk. Replay the file with the responder afterward to recover
-     *  evidence. */
+     *  on disk for diagnosis and recovery. */
     progressLogPath?: string;
 }
 /**
@@ -2043,7 +2043,8 @@ declare function firstStringAttr(attrs: Record<string, unknown>, keys: readonly
  * `otlpToRunRecords` — fold an OTLP traces.jsonl (one OTLP span per line;
  * the form AppWorld / HALO emit via their OpenInference OTLP exporter, the
  * same shape `flattenOtlpExportToNdjson` produces) into validated
- * `RunRecord[]` — one record per `trace_id` (one trace == one task).
+ * `RunRecord[]` — one record per `trace_id` by default, or per caller-defined
+ * logical run when one task is fragmented across multiple OTLP traces.
  *
  * This is the offline ingestion primitive the AppWorld proposer bench and the
  * hosted Intelligence product both stand on: traces in, paper-grade rows
@@ -2106,17 +2107,28 @@ interface OtlpToRunRecordsOptions {
      */
     priceUsdPerToken?: number;
     /**
-     * Score for a trace's outcome (AppWorld `world.evaluate()` → TGC/SGC, or
-     * any [0,1] task-success signal). Keyed by `trace_id`; falls through to
+     * Map each OTLP `trace_id` to the logical run it belongs to. Use this when a
+     * provider emits several traces for one task attempt. All mapped traces are
+     * aggregated into one record; duplicate span ids remain isolated by their
+     * source trace. The callback must return a non-empty id for every trace.
+     *
+     * When omitted, every `trace_id` remains an independent run.
+     */
+    logicalRunIdForTrace?: (traceId: string) => string;
+    /**
+     * Score for a produced run's outcome (AppWorld `world.evaluate()` →
+     * TGC/SGC, or
+     * any [0,1] task-success signal). Keyed by the logical run id when
+     * `logicalRunIdForTrace` is supplied, otherwise by `trace_id`; falls through to
      * the error-derived default (1 = no error span, 0 = had one) when the map
      * has no entry or the function returns undefined.
      */
-    scoreForTrace?: (traceId: string, span: TraceAggregate) => number | undefined;
+    scoreForTrace?: (runId: string, span: TraceAggregate) => number | undefined;
     /**
      * Per-record judge metadata when an external judge produced the score.
-     * Keyed by `trace_id`.
+     * Keyed by the logical run id when supplied, otherwise by `trace_id`.
      */
-    judgeMetadataForTrace?: (traceId: string) => RunRecord['judgeMetadata'] | undefined;
+    judgeMetadataForTrace?: (runId: string) => RunRecord['judgeMetadata'] | undefined;
 }
 /** A `RunRecord` plus the verbatim prompt/completion text when the trace's
  *  LLM spans exposed it. The text is NOT on the validated `RunRecord`
@@ -2132,6 +2144,10 @@ interface OtlpTraceRunRecord {
 /** Per-trace rollup the score callback can inspect. */
 interface TraceAggregate {
     traceId: string;
+    /** Number of source OTLP traces folded into this run. */
+    sourceTraceCount: number;
+    /** Source OTLP trace ids folded into this run, sorted for deterministic audit output. */
+    sourceTraceIds: readonly string[];
     spanCount: number;
     llmSpanCount: number;
     toolSpanCount: number;
@@ -2163,10 +2179,8 @@ declare function otlpToTraceRunRecords(otlpJsonl: string, opts: OtlpToRunRecords
 declare function otlpRowsToTraceRunRecords(rows: Iterable<object>, opts: OtlpToRunRecordsOptions): OtlpTraceRunRecord[];
 
 /** Ax RLM prompt for bounded trace discovery and evidence-backed analysis. */
-declare const TRACE_ANALYST_ACTOR_DESCRIPTION = "You answer questions about an OTLP-shaped JSONL trace dataset using the trace tools provided in the `traces` namespace.\n\nDISCOVERY \u2192 NARROW \u2192 DEEP-READ protocol \u2014 follow exactly:\n\n1. ALWAYS call `traces.getDatasetOverview({})` FIRST without a regex_pattern. The result tells you total_traces, raw_jsonl_bytes, services, agents, models, and sample_trace_ids (real ids \u2014 never fabricate one).\n\n2. Use raw_jsonl_bytes to gauge how expensive raw scans will be. `filters.regex_pattern` is the one scan-heavy filter on getDatasetOverview / queryTraces / countTraces \u2014 narrow with indexed fields (has_errors, model_names, service_names, agent_names, time bounds) BEFORE adding a regex on a large dataset.\n\n3. To list more traces than the sample, call `traces.queryTraces({ filters?, limit, offset? })`. Each summary carries raw_jsonl_bytes \u2014 use it to choose between viewTrace and searchTrace BEFORE calling either.\n\n4. Per-trace inspection:\n   - SMALL trace (raw_jsonl_bytes well under 150_000): call `traces.viewTrace({ trace_id })`. Returns all spans. Per-attribute payloads are head-capped at ~4KB; large `input.value` / `output.value` / `llm.input_messages` will show a `[trace-analyst truncated: N bytes]` marker.\n   - LARGE trace (raw_jsonl_bytes near or above 150_000, or you saw an `oversized` response): use `traces.searchTrace({ trace_id, regex_pattern })` to get bounded SpanMatchRecords (span metadata + matched text + surrounding context). Then call `traces.viewSpans({ trace_id, span_ids: [...] })` for surgical reads (~16KB cap, 4\u00D7 higher than discovery), or `traces.searchSpan({ trace_id, span_id, regex_pattern })` for one large span. Stays bounded regardless of trace size.\n   - Useful regex patterns: `STATUS_CODE_ERROR` (failures), tool names like `grep` or `view_trace`, error strings like `MaxTurnsExceeded`, model names, attribute keys.\n\n5. ONLY call viewTrace / viewSpans / searchTrace / searchSpan with trace/span ids you have already seen in sample_trace_ids, a queryTraces page, or a previous search result. Never invent ids.\n\n5a. **Result-shape contract** \u2014 searchTrace and searchSpan return `{ trace_id, hits, total_matches, has_more }`. Iterate `result.hits` (NOT result.matches). Each hit has `{ span_id, span_name, span_kind, attribute_path, matched_text, context_before, context_after, match_offset }`. viewTrace returns `{ trace_id, spans }` (or `oversized`). viewSpans returns `{ trace_id, spans, missing_span_ids, truncated_attribute_count }`. Never assume a field name \u2014 log the result shape first if unsure.\n\n6. If viewTrace returns an `oversized` summary instead of `spans`, DO NOT retry the same call. Read the summary's top_span_names, span_count, span_response_bytes_max, error_span_count to plan a follow-up: switch to searchTrace (or searchSpan for one large span), then viewSpans on a smaller, surgical span_ids set.\n\n7. If searchTrace or searchSpan returns has_more=true, REFINE the regex to be more specific rather than blindly raising max_matches.\n\n8. If a tool errors (invalid regex, range error), STOP and reconsider \u2014 don't retry with a guessed id or argument. Use the discovery tools above to recover.\n\n9. If a ~4KB-truncated payload from viewTrace / searchTrace matters for your answer, first try viewSpans on that span id (~16KB cap). If a 16KB-truncated payload from viewSpans still matters, narrow further with searchSpan against a more specific regex rather than asking for the full payload again.\n\n10. If maxDepth > 0 and the question splits into independent semantic branches, delegate well-defined subtasks to subagents using `await llmQuery(...)`. Pass narrow context and a focused query. Examples:\n\n    const reviews = await llmQuery([\n      { query: 'Drill into trace abc123 \u2014 what tool calls preceded the failure?', context: { trace_id: 'abc123' } },\n      { query: 'Drill into trace def456 \u2014 same failure mode?', context: { trace_id: 'def456' } },\n    ]);\n\nOBSERVABILITY rules:\n- Each non-final actor turn must emit at least one `console.log(...)` for evidence. Up to 3 logs per turn is fine when correlating multiple data sources (e.g. one log for findings list, one for source-file content, one for derived analysis).\n- Do NOT combine `console.log` with `final(...)` or `askClarification(...)` in the same turn \u2014 finish gathering data first, then call final on its own turn.\n- Reuse runtime variables across turns; don't recompute.\n- When done, call `await final(answer)` with the fully-formed report. The responder rewrites the answer into output fields; if you only pass a vague summary string the responder has nothing concrete to format.\n\nCRITICAL \u2014 `final()` payload contract for evidence-grounded analysis tasks:\n- Pass a STRUCTURED object as the second arg with the actual data the responder needs to format the answer. Do NOT pass abstract instructions; pass evidence.\n- Example for per-item verdict tasks:\n  ```js\n  await final(\"Format the per-item verdict report from the evidence below.\", {\n    findings: [\n      { id: 'sub-1-finding-1', claim: '...', verdict: 'TRUE-POSITIVE', evidence: 'lines 42-45 of contracts/X.sol show ...' },\n      ...all items\n    ],\n    systemic_summary: '3 sentences I wrote based on the evidence above'\n  });\n  ```\n- Calling `final(\"answer\", {})` with no evidence is a failure mode \u2014 the responder will hallucinate or echo back the field names. Always include the gathered data.\n- Premature final after a single viewSpans call is INSUFFICIENT for per-finding analysis tasks. Read the requested attributes (e.g. `spans[i].attributes['redteam.finding.title']`), and for each one perform the requested cross-reference (e.g. read the source SPAN's `attributes['source.content']`).\n\nOUTPUT contract \u2014 your final answer must include:\n- A clear prose conclusion answering the user's question.\n- Trace ids and span ids cited as evidence for each claim.\n- Failure modes named in the user's domain language, with frequency and concrete examples.\n\nDo NOT invent trace ids, span ids, error messages, or model names. Every fact must be traceable to a tool result.";
-declare const TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION = "trace-analyst-actor-v5-2026-05-06";
-/** Subagent prompt for focused trace-inspection subtasks. */
-declare const TRACE_ANALYST_SUBAGENT_DESCRIPTION = "You are a trace-analyst subagent. Your parent has delegated a focused trace-inspection question. Use the same DISCOVERY \u2192 NARROW \u2192 DEEP-READ protocol but stay tightly scoped: do exactly what was asked, return a concise compact answer, do NOT spawn further subagents unless the parent's question is genuinely multi-branch.\n\nCite trace ids and span ids for every claim. Do NOT invent ids.";
+declare const TRACE_ANALYST_ACTOR_DESCRIPTION = "You answer questions about an OTLP-shaped JSONL trace dataset using the trace tools provided in the `traces` namespace.\n\nDISCOVERY \u2192 NARROW \u2192 DEEP-READ protocol \u2014 follow exactly:\n\n1. ALWAYS call `traces.getDatasetOverview({})` FIRST without a regex_pattern. The result tells you total_traces, raw_jsonl_bytes, services, agents, models, and sample_trace_ids (real ids \u2014 never fabricate one).\n\n2. Use raw_jsonl_bytes to gauge how expensive raw scans will be. `filters.regex_pattern` is the one scan-heavy filter on getDatasetOverview / queryTraces / countTraces \u2014 narrow with indexed fields (has_errors, model_names, service_names, agent_names, time bounds) BEFORE adding a regex on a large dataset.\n\n3. To list more traces than the sample, call `traces.queryTraces({ filters?, limit, offset? })`. Each summary carries raw_jsonl_bytes \u2014 use it to choose between viewTrace and searchTrace BEFORE calling either.\n\n4. Per-trace inspection:\n   - SMALL trace (raw_jsonl_bytes well under 150_000): call `traces.viewTrace({ trace_id })`. Returns all spans. Per-attribute payloads are head-capped at ~4KB; large `input.value` / `output.value` / `llm.input_messages` will show a `[trace-analyst truncated: N bytes]` marker.\n   - LARGE trace (raw_jsonl_bytes near or above 150_000, or you saw an `oversized` response): use `traces.searchTrace({ trace_id, regex_pattern })` to get bounded SpanMatchRecords (span metadata + matched text + surrounding context). Then call `traces.viewSpans({ trace_id, span_ids: [...] })` for surgical reads (~16KB cap, 4\u00D7 higher than discovery), or `traces.searchSpan({ trace_id, span_id, regex_pattern })` for one large span. Stays bounded regardless of trace size.\n   - Useful regex patterns: `STATUS_CODE_ERROR` (failures), tool names like `grep` or `view_trace`, error strings like `MaxTurnsExceeded`, model names, attribute keys.\n\n5. ONLY call viewTrace / viewSpans / searchTrace / searchSpan with trace/span ids you have already seen in sample_trace_ids, a queryTraces page, or a previous search result. Never invent ids.\n\n5a. **Result-shape contract** \u2014 searchTrace and searchSpan return `{ trace_id, hits, total_matches, has_more }`. Iterate `result.hits` (NOT result.matches). Each hit has `{ span_id, span_name, span_kind, attribute_path, matched_text, context_before, context_after, match_offset }`. viewTrace returns `{ trace_id, spans }` (or `oversized`). viewSpans returns `{ trace_id, spans, missing_span_ids, truncated_attribute_count }`. Never assume a field name \u2014 log the result shape first if unsure.\n\n6. If viewTrace returns an `oversized` summary instead of `spans`, DO NOT retry the same call. Read the summary's top_span_names, span_count, span_response_bytes_max, error_span_count to plan a follow-up: switch to searchTrace (or searchSpan for one large span), then viewSpans on a smaller, surgical span_ids set.\n\n7. If searchTrace or searchSpan returns has_more=true, REFINE the regex to be more specific rather than blindly raising max_matches.\n\n8. If a tool errors (invalid regex, range error), STOP and reconsider \u2014 don't retry with a guessed id or argument. Use the discovery tools above to recover.\n\n9. If a ~4KB-truncated payload from viewTrace / searchTrace matters for your answer, first try viewSpans on that span id (~16KB cap). If a 16KB-truncated payload from viewSpans still matters, narrow further with searchSpan against a more specific regex rather than asking for the full payload again.\n\n10. If the question splits into independent reasoning branches, use bounded `llmQuery(...)` calls over evidence you already loaded. Subqueries cannot inspect the trace store, so pass the exact trace excerpts they need. Example:\n\n    const reviews = await llmQuery([\n      { query: 'Classify the failure mechanism in this excerpt.', context: traceAbcExcerpt },\n      { query: 'Classify the failure mechanism in this excerpt.', context: traceDefExcerpt },\n    ]);\n\nOBSERVABILITY rules:\n- Each discovery turn must emit at least one concise `console.log(...)` showing what evidence was learned.\n- Finish gathering evidence before submitting the analysis.\n- Reuse runtime variables across turns; don't recompute.\n\nOUTPUT contract \u2014 your final answer must include:\n- A clear prose conclusion answering the user's question.\n- Trace ids and span ids cited as evidence for each claim.\n- Failure modes named in the user's domain language, with frequency and concrete examples.\n- A concise findings array containing only claims supported by inspected evidence.\n\nDo NOT invent trace ids, span ids, error messages, or model names. Every fact must be traceable to a tool result.";
+declare const TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION = "trace-analyst-actor-v6-2026-07-14";
 
 /**
  * `OtlpFileTraceStore` — read-only OTLP-JSONL trace store for the
@@ -2326,4 +2340,4 @@ declare function traceAnalystFunctionGroup(opts: BuildTraceAnalystToolsOpts): {
     functions: AxFunction[];
 };
 
-export { type AnalyzeTracesInput, type AnalyzeTracesOptions, type AnalyzeTracesResult, type AnalyzeTracesTurnSnapshot, type Artifact, type BudgetLedgerEntry, type BudgetSpec, type CaptureFetchContext, type CaptureFetchOptions, DEFAULT_REDACTION_RULES, DEFAULT_TRACE_ANALYST_BUDGETS, type DatasetOverview, type ErrorCluster, type EventFilter, type EventKind, type ExportableSpan, type ExtractUsageFromSseOptions, type ExtractedUsage, FAILURE_CLASSES, type FailureClass, FileSystemRawProviderSink, type FileSystemRawProviderSinkOptions, FileSystemTraceStore, type FileSystemTraceStoreOptions, type FlattenOtlpOptions, type GenericSpan, INPUT_VALUE, InMemoryRawProviderSink, type InMemoryRawProviderSinkOptions, InMemoryTraceStore, type JudgeSpan, LLM_CACHED_TOKENS, LLM_CACHED_TOKEN_ATTR_KEYS, LLM_CACHE_WRITE_TOKENS, LLM_CACHE_WRITE_TOKEN_ATTR_KEYS, LLM_CONTEXT_TOKENS, LLM_COST_ATTR_KEYS, LLM_COST_USD, LLM_INPUT_TOKENS, LLM_INPUT_TOKEN_ATTR_KEYS, LLM_MODEL_ATTR_KEYS, LLM_MODEL_NAME, LLM_OUTPUT_TOKENS, LLM_OUTPUT_TOKEN_ATTR_KEYS, LLM_REASONING_TOKENS, LLM_REASONING_TOKEN_ATTR_KEYS, type LlmSpan, type LlmSpanOtlpInput, type Message, NoopRawProviderSink, OPENINFERENCE_SPAN_KIND, OTEL_AGENT_EVAL_SCOPE, OUTPUT_VALUE, type OtelExportConfig, type OtelExporter, type OtlpExport, OtlpFileTraceStore, type OtlpFileTraceStoreOptions, type OtlpFlatLine, type OtlpResourceSpans, type OtlpSpan, type OtlpToRunRecordsOptions, type OtlpTraceRunRecord, type ProjectedOtlpSpan, type ProviderRedactor, type QueryTracesPage, REDACTION_VERSION, RUN_COST_ATTR_KEYS, type RawProviderDirection, type RawProviderEvent, type RawProviderSink, type RawProviderSinkFilter, type RedactionReport, type RedactionRule, ReplayCache, type ReplayCacheEntry, ReplayCacheMissError, type ReplayCacheStats, type ReplayFetchOptions, type RetrievalSpan, type Run, type RunCompleteHook, type RunCompleteHookContext, type RunFilter, RunIntegrityError, type RunIntegrityExpectations, type RunIntegrityIssue, type RunIntegrityIssueCode, type RunIntegrityReport, type RunLayer, type RunOutcome$1 as RunOutcome, type RunStatus, SPAN_KIND_ATTR_KEYS, type SandboxSpan, type SearchSpanResult, type SearchTraceResult, type Span, type SpanBase, type SpanFilter, type SpanHandle, type SpanKind, type SpanMatchRecord, SpanNotFoundError, type SpanStatus, type SseUsageMode, TOOL_ARGS_CAPTURED, TOOL_LATENCY_MS, TOOL_NAME, TOOL_NAME_ATTR_KEYS, TRACE_ANALYST_ACTOR_DESCRIPTION, TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION, TRACE_ANALYST_SUBAGENT_DESCRIPTION, TRACE_ANALYST_TRUNCATION_MARKER_PREFIX, TRACE_SCHEMA_VERSION, type ToolSpan, type ToolSpanOtlpInput, type TraceAggregate, type TraceAnalysisStore, type TraceAnalystByteBudgets, type TraceAnalystFilters, type TraceAnalystHookOptions, type TraceAnalystSpan, type TraceAnalystSpanKind, type TraceAnalystSpanStatus, type TraceAnalystTraceSummary, TraceEmitter, type TraceEmitterOptions, type TraceEvent, TraceFileMissingError, type TraceInsightContext, type TraceInsightFinding, type TraceInsightPanelRole, type TraceInsightPromptInput, type TraceInsightQualityGate, type TraceInsightQuestion, type TraceInsightReadiness, type TraceInsightSuite, type TraceInsightTask, TraceNotFoundError, type TraceStore, type TraceStoreSource, type TraceStoreToOtlpOptions, type TracesToOtlpResult, type ViewSpansResult, type ViewTraceOversized, type ViewTraceResult, aggregateLlm, analyzeTraces, applyLlmSpanOtlpAttributes, applyToolSpanOtlpAttributes, argHash, asNumber, asString, assertRunCaptured, buildTraceAnalystTools, buildTraceInsightContext, buildTraceInsightPrompt, captureFetchToRawSink, contextInputTokens, convertTraceStoresToOtlp, createOtelExporter, createOtelTracingStore, createReplayFetch, defaultProviderRedactor, defaultTraceInsightPanel, describeTraceInsightScope, domainEvidencePattern, exportRunAsOtlp, extractOtlpAttributes, extractUsage, extractUsageFromResponse, extractUsageFromSse, firstNumberAttr, firstStringAttr, flattenOtlpExportToNdjson, groupBy, hasCapturedToolArgs, inferDomainKeywords, inferOtlpKind, isJudgeSpan, isLlmSpan, isRetrievalSpan, isSandboxSpan, isToolSpan, iterateRawCalls, judgeSpans, llmSpanFromProvider, llmSpans, otelRunCompleteHook, otlpRowsToRunRecords, otlpRowsToTraceRunRecords, otlpToRunRecords, otlpToTraceRunRecords, planTraceInsightQuestions, projectOtlpFlatLine, providerFromBaseUrl, readOtlpStatus, redactString, redactValue, runFailureClass, runsForScenario, scoreTraceInsightReadiness, stringField, throwIfRunIncomplete, tokenizeDomainWords, toolSpans, traceAnalystFunctionGroup, traceAnalystOnRunComplete, traceSpanKindToOpenInferenceKind };
+export { type AnalyzeTracesInput, type AnalyzeTracesOptions, type AnalyzeTracesResult, type AnalyzeTracesTurnSnapshot, type Artifact, type BudgetLedgerEntry, type BudgetSpec, type CaptureFetchContext, type CaptureFetchOptions, DEFAULT_REDACTION_RULES, DEFAULT_TRACE_ANALYST_BUDGETS, type DatasetOverview, type ErrorCluster, type EventFilter, type EventKind, type ExportableSpan, type ExtractUsageFromSseOptions, type ExtractedUsage, FAILURE_CLASSES, type FailureClass, FileSystemRawProviderSink, type FileSystemRawProviderSinkOptions, FileSystemTraceStore, type FileSystemTraceStoreOptions, type FlattenOtlpOptions, type GenericSpan, INPUT_VALUE, InMemoryRawProviderSink, type InMemoryRawProviderSinkOptions, InMemoryTraceStore, type JudgeSpan, LLM_CACHED_TOKENS, LLM_CACHED_TOKEN_ATTR_KEYS, LLM_CACHE_WRITE_TOKENS, LLM_CACHE_WRITE_TOKEN_ATTR_KEYS, LLM_CONTEXT_TOKENS, LLM_COST_ATTR_KEYS, LLM_COST_USD, LLM_INPUT_TOKENS, LLM_INPUT_TOKEN_ATTR_KEYS, LLM_MODEL_ATTR_KEYS, LLM_MODEL_NAME, LLM_OUTPUT_TOKENS, LLM_OUTPUT_TOKEN_ATTR_KEYS, LLM_REASONING_TOKENS, LLM_REASONING_TOKEN_ATTR_KEYS, type LlmSpan, type LlmSpanOtlpInput, type Message, NoopRawProviderSink, OPENINFERENCE_SPAN_KIND, OTEL_AGENT_EVAL_SCOPE, OUTPUT_VALUE, type OtelExportConfig, type OtelExporter, type OtlpExport, OtlpFileTraceStore, type OtlpFileTraceStoreOptions, type OtlpFlatLine, type OtlpResourceSpans, type OtlpSpan, type OtlpToRunRecordsOptions, type OtlpTraceRunRecord, type ProjectedOtlpSpan, type ProviderRedactor, type QueryTracesPage, REDACTION_VERSION, RUN_COST_ATTR_KEYS, type RawProviderDirection, type RawProviderEvent, type RawProviderSink, type RawProviderSinkFilter, type RedactionReport, type RedactionRule, ReplayCache, type ReplayCacheEntry, ReplayCacheMissError, type ReplayCacheStats, type ReplayFetchOptions, type RetrievalSpan, type Run, type RunCompleteHook, type RunCompleteHookContext, type RunFilter, RunIntegrityError, type RunIntegrityExpectations, type RunIntegrityIssue, type RunIntegrityIssueCode, type RunIntegrityReport, type RunLayer, type RunOutcome$1 as RunOutcome, type RunStatus, SPAN_KIND_ATTR_KEYS, type SandboxSpan, type SearchSpanResult, type SearchTraceResult, type Span, type SpanBase, type SpanFilter, type SpanHandle, type SpanKind, type SpanMatchRecord, SpanNotFoundError, type SpanStatus, type SseUsageMode, TOOL_ARGS_CAPTURED, TOOL_LATENCY_MS, TOOL_NAME, TOOL_NAME_ATTR_KEYS, TRACE_ANALYST_ACTOR_DESCRIPTION, TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION, TRACE_ANALYST_TRUNCATION_MARKER_PREFIX, TRACE_SCHEMA_VERSION, type ToolSpan, type ToolSpanOtlpInput, type TraceAggregate, type TraceAnalysisStore, type TraceAnalystByteBudgets, type TraceAnalystFilters, type TraceAnalystHookOptions, type TraceAnalystSpan, type TraceAnalystSpanKind, type TraceAnalystSpanStatus, type TraceAnalystTraceSummary, TraceEmitter, type TraceEmitterOptions, type TraceEvent, TraceFileMissingError, type TraceInsightContext, type TraceInsightFinding, type TraceInsightPanelRole, type TraceInsightPromptInput, type TraceInsightQualityGate, type TraceInsightQuestion, type TraceInsightReadiness, type TraceInsightSuite, type TraceInsightTask, TraceNotFoundError, type TraceStore, type TraceStoreSource, type TraceStoreToOtlpOptions, type TracesToOtlpResult, type ViewSpansResult, type ViewTraceOversized, type ViewTraceResult, aggregateLlm, analyzeTraces, applyLlmSpanOtlpAttributes, applyToolSpanOtlpAttributes, argHash, asNumber, asString, assertRunCaptured, buildTraceAnalystTools, buildTraceInsightContext, buildTraceInsightPrompt, captureFetchToRawSink, contextInputTokens, convertTraceStoresToOtlp, createOtelExporter, createOtelTracingStore, createReplayFetch, defaultProviderRedactor, defaultTraceInsightPanel, describeTraceInsightScope, domainEvidencePattern, exportRunAsOtlp, extractOtlpAttributes, extractUsage, extractUsageFromResponse, extractUsageFromSse, firstNumberAttr, firstStringAttr, flattenOtlpExportToNdjson, groupBy, hasCapturedToolArgs, inferDomainKeywords, inferOtlpKind, isJudgeSpan, isLlmSpan, isRetrievalSpan, isSandboxSpan, isToolSpan, iterateRawCalls, judgeSpans, llmSpanFromProvider, llmSpans, otelRunCompleteHook, otlpRowsToRunRecords, otlpRowsToTraceRunRecords, otlpToRunRecords, otlpToTraceRunRecords, planTraceInsightQuestions, projectOtlpFlatLine, providerFromBaseUrl, readOtlpStatus, redactString, redactValue, runFailureClass, runsForScenario, scoreTraceInsightReadiness, stringField, throwIfRunIncomplete, tokenizeDomainWords, toolSpans, traceAnalystFunctionGroup, traceAnalystOnRunComplete, traceSpanKindToOpenInferenceKind };

package/dist/traces.js

@@ -27,7 +27,7 @@ import {
   scoreTraceInsightReadiness,
   tokenizeDomainWords,
   traceAnalystOnRunComplete
-} from "./chunk-ZYHJNKI3.js";
+} from "./chunk-ULOKLHIQ.js";
 import "./chunk-H5UD2323.js";
 import {
   extractUsage,
@@ -48,9 +48,8 @@ import {
 import {
   TRACE_ANALYST_ACTOR_DESCRIPTION,
   TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION,
-  TRACE_ANALYST_SUBAGENT_DESCRIPTION,
   analyzeTraces
-} from "./chunk-FIUFRXP3.js";
+} from "./chunk-WW2A73HW.js";
 import {
   DEFAULT_REDACTION_RULES,
   REDACTION_VERSION,
@@ -75,7 +74,7 @@ import {
   stringField,
   traceAnalystFunctionGroup,
   traceSpanKindToOpenInferenceKind
-} from "./chunk-6QIM2EAP.js";
+} from "./chunk-PXD6ZFNY.js";
 import {
   RunIntegrityError,
   assertRunCaptured,
@@ -178,7 +177,6 @@ export {
   TOOL_NAME_ATTR_KEYS,
   TRACE_ANALYST_ACTOR_DESCRIPTION,
   TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION,
-  TRACE_ANALYST_SUBAGENT_DESCRIPTION,
   TRACE_ANALYST_TRUNCATION_MARKER_PREFIX,
   TRACE_SCHEMA_VERSION,
   TraceEmitter,

package/dist/wire/index.d.ts

@@ -7,8 +7,14 @@ import { Hono } from 'hono';
 type CostChannel = 'agent' | 'judge' | 'verifier' | 'analyst' | 'driver' | (string & {});
 interface CostUsage {
     inputTokens: number;
+    /** Includes reasoning tokens when the provider bills them as output. */
     outputTokens: number;
+    /** Reasoning-token subset of outputTokens, when reported. */
+    reasoningTokens?: number;
+    /** Prompt tokens served from a provider cache. */
     cachedTokens?: number;
+    /** Prompt tokens written into a provider cache. */
+    cacheWriteTokens?: number;
 }
 interface CostCallBase {
     callId: string;
@@ -75,7 +81,9 @@ interface ChannelRollup {
     calls: number;
     inputTokens: number;
     outputTokens: number;
+    reasoningTokens?: number;
     cachedTokens: number;
+    cacheWriteTokens?: number;
     costUsd: number;
     unpricedCalls: number;
     unknownUsageCalls: number;
@@ -87,7 +95,9 @@ interface CostLedgerSummary {
     reservedCostUsd: number;
     inputTokens: number;
     outputTokens: number;
+    reasoningTokens?: number;
     cachedTokens: number;
+    cacheWriteTokens?: number;
     totalCostUsd: number;
     byChannel: ChannelRollup[];
     unpricedModels: string[];
@@ -101,6 +111,10 @@ interface CostLedgerFilter {
     phase?: string;
     tags?: Record<string, string>;
 }
+interface CostLedgerWaitOptions {
+    /** Maximum time to wait for active provider calls. Default 5 seconds. */
+    timeoutMs?: number;
+}
 /** Append-only storage. `append` must atomically reject stale revisions. */
 interface CostLedgerPersistence {
     read(): {
@@ -120,6 +134,7 @@ declare class CostLedger {
     private readonly records;
     private readonly activeCallIds;
     private readonly lateCallIds;
+    private readonly idleWaiters;
     private completedTasks;
     private revision;
     private costLimitPersisted;
@@ -127,6 +142,8 @@ declare class CostLedger {
     private readonly persistence?;
     constructor(input?: number | CostLedgerOptions);
     runPaidCall<T>(input: RunPaidCallInput<T>): Promise<PaidCallResult<T>>;
+    /** Wait until every call started by this ledger has produced a durable outcome. */
+    waitForIdle(options?: CostLedgerWaitOptions): Promise<boolean>;
     /** Settle a call left pending by a crashed process after reconciling with the provider. */
     reconcile(callId: string, observed: CostReceiptInput, options?: {
         error?: string;
@@ -137,6 +154,7 @@ declare class CostLedger {
     costPerCompletedTask(): number | null;
     private execute;
     private captureLateOutcome;
+    private releaseActiveCall;
     private commitOutcome;
     private captureFailure;
     private commitReceipt;
@@ -152,7 +170,7 @@ declare class CostLedger {
  * Keeping callback contracts structural lets those subpaths compose while the
  * concrete {@link CostLedger} retains its private durable state.
  */
-type CostLedgerHandle = Pick<CostLedger, keyof CostLedger>;
+type CostLedgerHandle = Pick<CostLedger, Exclude<keyof CostLedger, 'waitForIdle'>> & Partial<Pick<CostLedger, 'waitForIdle'>>;
 
 type RunStatus = 'running' | 'completed' | 'failed' | 'aborted';
 interface BudgetSpec {

package/dist/wire/index.js

@@ -34,9 +34,9 @@ import {
   runRpcOnce,
   startServer,
   startServerAsync
-} from "../chunk-JTMFT5QZ.js";
+} from "../chunk-4I2E3LLO.js";
 import "../chunk-NJC7U437.js";
-import "../chunk-MLHRFWQK.js";
+import "../chunk-JHOJHHU7.js";
 import "../chunk-VI2UW6B6.js";
 import "../chunk-PC4UYEBM.js";
 import "../chunk-ONWEPEDO.js";

package/docs/trace-analysis.md

@@ -30,19 +30,23 @@ import {
   analyzeTraces,
 } from '@tangle-network/agent-eval'
 
+const abortController = new AbortController()
 const result = await analyzeTraces({
   question: 'Why did app-runtime holdout runs fail this week?',
 }, {
   source: new OtlpFileTraceStore({ path: 'traces/otlp.jsonl' }),
   ai,
   model: 'gpt-4o-2024-11-20',
+  maxSubqueries: 4,
+  maxParallelSubqueries: 2,
+  signal: abortController.signal,
 })
 
 console.log(result.findings)
 ```
 
-Products can pass any `TraceAnalysisStore`; they do not need to use the file
-store in production.
+Products can pass any `TraceAnalysisStore`; they do not need to use the file store in production.
+The analyst runs one Ax executor loop and accepts only an explicit structured `final(task, { report, findings })` result; max-turn fallback text fails loud.
 
 ## Deterministic failure coverage (no LLM)
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tangle-network/agent-eval",
-  "version": "0.118.3",
+  "version": "0.119.1",
   "description": "Evaluate and improve AI agents from runs, traces, judges, and feedback. Compare candidates, cluster failures, measure lift, and gate releases.",
   "homepage": "https://github.com/tangle-network/agent-eval#readme",
   "repository": {
@@ -119,6 +119,11 @@
       "import": "./dist/hosted/index.js",
       "default": "./dist/hosted/index.js"
     },
+    "./primeintellect": {
+      "types": "./dist/primeintellect/index.d.ts",
+      "import": "./dist/primeintellect/index.js",
+      "default": "./dist/primeintellect/index.js"
+    },
     "./openapi.json": {
       "default": "./dist/openapi.json"
     }
@@ -149,7 +154,7 @@
   },
   "dependencies": {
     "@asteasolutions/zod-to-openapi": "^8.5.0",
-    "@ax-llm/ax": "^19.0.25",
+    "@ax-llm/ax": "^23.0.0",
     "@hono/node-server": "^2.0.0",
     "@tangle-network/agent-core": "^0.4.11",
     "@tangle-network/agent-interface": "^0.26.0",