@tangle-network/agent-runtime 0.94.10 → 0.94.12

package/dist/primeintellect/index.js

@@ -0,0 +1,1120 @@
+import {
+  createOpenAICompatibleBackend
+} from "../chunk-AEG3NGJ2.js";
+import "../chunk-YEJR7IXO.js";
+import "../chunk-DGUM43GV.js";
+
+// src/primeintellect/package.ts
+import { createHash, randomUUID } from "crypto";
+import { mkdir, readFile, rename, rm, stat, writeFile } from "fs/promises";
+import { basename, dirname, isAbsolute, join, normalize, relative, resolve, sep } from "path";
+import { canonicalJson } from "@tangle-network/agent-eval";
+
+// src/primeintellect/validation.ts
+function validatePrimeIntellectPrompt(value, path) {
+  if (typeof value === "string" && value.length > 0) return value;
+  if (!Array.isArray(value) || value.length === 0) {
+    throw new Error(`${path} must be a non-empty string or message array`);
+  }
+  for (const [index, message] of value.entries()) {
+    validateMessage(message, `${path}[${index}]`);
+  }
+  return value;
+}
+function validatePrimeIntellectJson(value, path) {
+  if (value === null || ["string", "boolean"].includes(typeof value)) return;
+  if (typeof value === "number") {
+    if (!Number.isFinite(value)) throw new Error(`${path} contains a non-finite number`);
+    return;
+  }
+  if (Array.isArray(value)) {
+    value.forEach((entry, index) => {
+      validatePrimeIntellectJson(entry, `${path}[${index}]`);
+    });
+    return;
+  }
+  if (typeof value === "object") {
+    for (const [key, entry] of Object.entries(value)) {
+      validatePrimeIntellectJson(entry, `${path}.${key}`);
+    }
+    return;
+  }
+  throw new Error(`${path} is not JSON serializable`);
+}
+function validatePrimeIntellectJsonObject(value, path) {
+  const output = record(value, path);
+  validatePrimeIntellectJson(output, path);
+  return output;
+}
+function validateMessage(value, path) {
+  const message = record(value, path);
+  const role = message.role;
+  if (!["system", "user", "assistant", "tool"].includes(String(role))) {
+    throw new Error(`${path}.role is invalid`);
+  }
+  if (role === "system" || role === "user") {
+    assertOnlyKeys(message, ["role", "content"], path);
+    validateContent(message.content, `${path}.content`);
+  } else if (role === "assistant") {
+    assertOnlyKeys(
+      message,
+      ["role", "content", "reasoning_content", "tool_calls", "provider_state"],
+      path
+    );
+    optionalNullableString(message.content, `${path}.content`);
+    optionalNullableString(message.reasoning_content, `${path}.reasoning_content`);
+    if (message.tool_calls !== void 0) {
+      validateToolCalls(message.tool_calls, `${path}.tool_calls`);
+    }
+    if (message.provider_state !== void 0) {
+      validateProviderState(message.provider_state, `${path}.provider_state`);
+    }
+  } else {
+    assertOnlyKeys(message, ["role", "tool_call_id", "content", "name"], path);
+    nonEmptyString(message.tool_call_id, `${path}.tool_call_id`);
+    if (message.name !== void 0 && typeof message.name !== "string") {
+      throw new Error(`${path}.name must be a string`);
+    }
+    validateContent(message.content, `${path}.content`);
+  }
+  validatePrimeIntellectJson(message, path);
+}
+function validateToolCalls(value, path) {
+  if (!Array.isArray(value)) throw new Error(`${path} must be an array`);
+  for (const [index, rawCall] of value.entries()) {
+    const call = record(rawCall, `${path}[${index}]`);
+    assertOnlyKeys(call, ["id", "name", "arguments"], `${path}[${index}]`);
+    for (const field of ["id", "name", "arguments"]) {
+      nonEmptyString(call[field], `${path}[${index}].${field}`);
+    }
+  }
+}
+function validateProviderState(value, path) {
+  if (!Array.isArray(value)) throw new Error(`${path} must be an array`);
+  for (const [index, rawState] of value.entries()) {
+    const state = record(rawState, `${path}[${index}]`);
+    validatePrimeIntellectJson(state, `${path}[${index}]`);
+  }
+}
+function validateContent(value, path) {
+  if (typeof value === "string") return;
+  if (!Array.isArray(value) || value.length === 0) {
+    throw new Error(`${path} must be a string or non-empty content array`);
+  }
+  for (const [index, rawPart] of value.entries()) {
+    const part = record(rawPart, `${path}[${index}]`);
+    if (part.type === "text" && typeof part.text === "string") {
+      assertOnlyKeys(part, ["type", "text"], `${path}[${index}]`);
+      continue;
+    }
+    const image = part.image_url;
+    if (part.type === "image_url" && image !== null && typeof image === "object" && !Array.isArray(image) && typeof image.url === "string") {
+      assertOnlyKeys(part, ["type", "image_url"], `${path}[${index}]`);
+      assertOnlyKeys(image, ["url"], `${path}[${index}].image_url`);
+      continue;
+    }
+    throw new Error(`${path}[${index}] is not a supported text or image_url content part`);
+  }
+}
+function optionalNullableString(value, path) {
+  if (value !== void 0 && value !== null && typeof value !== "string") {
+    throw new Error(`${path} must be a string or null`);
+  }
+}
+function nonEmptyString(value, path) {
+  if (typeof value !== "string" || value.length === 0) {
+    throw new Error(`${path} must be a non-empty string`);
+  }
+  return value;
+}
+function record(value, path) {
+  if (value === null || typeof value !== "object" || Array.isArray(value)) {
+    throw new Error(`${path} must be an object`);
+  }
+  return value;
+}
+function assertOnlyKeys(value, allowed, path) {
+  for (const key of Object.keys(value)) {
+    if (!allowed.includes(key)) throw new Error(`${path}.${key} is not supported`);
+  }
+}
+
+// src/primeintellect/package.ts
+var VERIFIERS_RANGE = ">=0.2.0,<0.3.0";
+var ENV_NAME = /^[A-Z_][A-Z0-9_]*$/;
+var PACKAGE_NAME = /^[a-z][a-z0-9-]{0,62}$/;
+var VERSION = /^\d+\.\d+\.\d+(?:[-+][a-zA-Z0-9.-]+)?$/;
+var DEFAULT_MAX_TURNS = 16;
+var DEFAULT_ROLLOUT_TIMEOUT = 3600;
+var DEFAULT_SCORING_TIMEOUT = 300;
+function createPrimeIntellectPackage(options) {
+  const validated = validateOptions(options);
+  const moduleName = validated.name.replaceAll("-", "_");
+  const rows = validated.tasks.map((task, idx) => taskRow(task, idx));
+  const runnerFiles = validated.runner.files ?? {};
+  const scoringFiles = validated.scoring.kind === "command" ? validated.scoring.files ?? {} : {};
+  const files = {
+    "pyproject.toml": renderPyproject(validated, moduleName),
+    "prime.eval.toml": renderPrimeConfig(validated, "eval"),
+    "prime.train.toml": renderPrimeConfig(validated, "train"),
+    "README.md": renderReadme(validated),
+    [`${moduleName}/__init__.py`]: renderInit(moduleName),
+    [`${moduleName}/taskset.py`]: renderTaskset(moduleName, validated.scoring),
+    [`${moduleName}/harness.py`]: renderHarness(moduleName),
+    [`${moduleName}/tasks.jsonl`]: `${rows.map((row) => JSON.stringify(row)).join("\n")}
+`,
+    [`${moduleName}/runner.json`]: `${JSON.stringify(
+      {
+        command: validated.runner.command,
+        files: runnerFiles,
+        setup: validated.runner.setup ?? [],
+        forwardEnv: validated.runner.forwardEnv ?? []
+      },
+      null,
+      2
+    )}
+`
+  };
+  for (const [path, contents] of Object.entries(scoringFiles)) {
+    files[`${moduleName}/scoring/${path}`] = contents;
+  }
+  const filesSha256 = Object.fromEntries(
+    Object.entries(files).sort(([left], [right]) => left.localeCompare(right)).map(([path, contents]) => [path, sha256(contents)])
+  );
+  const manifest = {
+    schema: "tangle.primeintellect.package/v1",
+    name: validated.name,
+    moduleName,
+    version: validated.version,
+    verifiers: VERIFIERS_RANGE,
+    taskCount: validated.tasks.length,
+    splits: {
+      train: validated.tasks.filter((task) => task.split === "train").length,
+      eval: validated.tasks.filter((task) => task.split === "eval").length
+    },
+    taskIdsSha256: sha256(
+      validated.tasks.map((task) => `${task.split}:${task.id}`).sort().join("\n")
+    ),
+    filesSha256
+  };
+  files["manifest.json"] = `${JSON.stringify(manifest, null, 2)}
+`;
+  return { manifest, files: Object.freeze(files) };
+}
+async function writePrimeIntellectPackage(bundle, outputDirectory, options = {}) {
+  const output = resolve(outputDirectory);
+  const parent = dirname(output);
+  await mkdir(parent, { recursive: true });
+  const replacing = await pathExists(output);
+  if (replacing) {
+    if (!options.replace) throw new Error(`PrimeIntellect output already exists: ${output}`);
+    await assertGeneratedPackage(output);
+  }
+  const temporary = join(parent, `.${basename(output)}.${randomUUID()}.tmp`);
+  const backup = replacing ? join(parent, `.${basename(output)}.${randomUUID()}.backup`) : void 0;
+  try {
+    await mkdir(temporary);
+    for (const [path, contents] of Object.entries(bundle.files)) {
+      assertRelativePath(path, "bundle file");
+      const target = resolve(temporary, path);
+      if (target !== temporary && !target.startsWith(`${temporary}${sep}`)) {
+        throw new Error(`bundle file escapes output directory: ${path}`);
+      }
+      await mkdir(dirname(target), { recursive: true });
+      await writeFile(target, contents, "utf8");
+    }
+    if (backup) await rename(output, backup);
+    try {
+      await rename(temporary, output);
+    } catch (error) {
+      if (backup) {
+        try {
+          await rename(backup, output);
+        } catch (restoreError) {
+          throw new AggregateError(
+            [error, restoreError],
+            `failed to install PrimeIntellect package and restore ${output}`
+          );
+        }
+      }
+      throw error;
+    }
+    if (backup) await rm(backup, { recursive: true });
+  } catch (error) {
+    await rm(temporary, { recursive: true, force: true });
+    throw error;
+  }
+  return output;
+}
+function validateOptions(options) {
+  if (!PACKAGE_NAME.test(options.name)) {
+    throw new Error("PrimeIntellect package name must match /^[a-z][a-z0-9-]{0,62}$/");
+  }
+  if (!VERSION.test(options.version)) {
+    throw new Error("PrimeIntellect package version must be a numeric semantic version");
+  }
+  if (!Array.isArray(options.tasks) || options.tasks.length === 0) {
+    throw new Error("PrimeIntellect package requires tasks");
+  }
+  const tasks = options.tasks;
+  const seen = /* @__PURE__ */ new Set();
+  const seenInputs = /* @__PURE__ */ new Map();
+  const splitCounts = { train: 0, eval: 0 };
+  for (const [index, task] of tasks.entries()) {
+    validateTask(task, index, options.scoring);
+    if (seen.has(task.id)) throw new Error(`duplicate PrimeIntellect task id: ${task.id}`);
+    seen.add(task.id);
+    const input = canonicalJson({
+      prompt: task.prompt,
+      systemPrompt: task.systemPrompt ?? null,
+      metadata: task.metadata ?? {}
+    });
+    const duplicate = seenInputs.get(input);
+    if (duplicate) {
+      throw new Error(
+        `PrimeIntellect tasks ${duplicate.id} (${duplicate.split}) and ${task.id} (${task.split}) expose the same public input`
+      );
+    }
+    seenInputs.set(input, { id: task.id, split: task.split });
+    splitCounts[task.split] += 1;
+  }
+  if (splitCounts.train === 0 || splitCounts.eval === 0) {
+    throw new Error("PrimeIntellect package requires non-empty, disjoint train and eval splits");
+  }
+  validateScoring(options.scoring);
+  validateCommand(options.runner.command, "runner.command");
+  validateFiles(options.runner.files ?? {}, "runner.files");
+  for (const [index, command] of (options.runner.setup ?? []).entries()) {
+    validateCommand(command, `runner.setup[${index}]`);
+  }
+  validateEnvNames(options.runner.forwardEnv ?? [], "runner.forwardEnv");
+  if (typeof options.runner.image !== "string" || options.runner.image.trim().length === 0) {
+    throw new Error("runner.image must be a non-empty container image");
+  }
+  if (/(^|:)latest$/i.test(options.runner.image)) {
+    throw new Error("runner.image must not use the mutable latest tag");
+  }
+  positiveInteger(options.maxTurns ?? DEFAULT_MAX_TURNS, "maxTurns");
+  for (const [name, value] of [
+    ["maxInputTokens", options.maxInputTokens],
+    ["maxOutputTokens", options.maxOutputTokens],
+    ["maxTotalTokens", options.maxTotalTokens]
+  ]) {
+    if (value !== void 0) positiveInteger(value, name);
+  }
+  positiveNumber(options.rolloutTimeoutSeconds ?? DEFAULT_ROLLOUT_TIMEOUT, "rolloutTimeoutSeconds");
+  positiveNumber(options.scoringTimeoutSeconds ?? DEFAULT_SCORING_TIMEOUT, "scoringTimeoutSeconds");
+  return options;
+}
+function validateTask(task, index, scoring) {
+  const path = `tasks[${index}]`;
+  if (typeof task.id !== "string" || task.id.trim().length === 0) {
+    throw new Error(`${path}.id must be a non-empty string`);
+  }
+  if (task.split !== "train" && task.split !== "eval") {
+    throw new Error(`${path}.split must be train or eval`);
+  }
+  const prompt = validatePrimeIntellectPrompt(task.prompt, `${path}.prompt`);
+  if (Array.isArray(prompt)) {
+    if (task.systemPrompt !== void 0 && prompt.some((message) => message.role === "system")) {
+      throw new Error(`${path} must not set systemPrompt and include a system message`);
+    }
+  }
+  if (task.systemPrompt !== void 0 && typeof task.systemPrompt !== "string") {
+    throw new Error(`${path}.systemPrompt must be a string`);
+  }
+  if (task.metadata !== void 0) {
+    validatePrimeIntellectJson(task.metadata, `${path}.metadata`);
+  }
+  if (scoring.kind !== "command") {
+    const answers = Array.isArray(task.answer) ? task.answer : [task.answer];
+    if (answers.length === 0 || answers.some((answer) => typeof answer !== "string" || answer.length === 0)) {
+      throw new Error(`${path}.answer is required for ${scoring.kind} scoring`);
+    }
+  }
+}
+function validateScoring(scoring) {
+  if (scoring.kind === "exact") return;
+  if (scoring.kind === "reference-judge") {
+    if (typeof scoring.model !== "string" || scoring.model.length === 0) {
+      throw new Error("reference-judge scoring requires a model");
+    }
+    return;
+  }
+  validateCommand(scoring.command, "scoring.command");
+  validateFiles(scoring.files ?? {}, "scoring.files");
+  validateEnvNames(scoring.forwardEnv ?? [], "scoring.forwardEnv");
+  positiveNumber(scoring.timeoutSeconds ?? DEFAULT_SCORING_TIMEOUT, "scoring.timeoutSeconds");
+}
+function validateCommand(command, path) {
+  if (!Array.isArray(command) || command.length === 0) {
+    throw new Error(`${path} must be a non-empty argv array`);
+  }
+  for (const [index, argument] of command.entries()) {
+    if (typeof argument !== "string" || argument.length === 0 || argument.includes("\0")) {
+      throw new Error(`${path}[${index}] must be a non-empty string without NUL bytes`);
+    }
+  }
+}
+function validateFiles(files, path) {
+  for (const [file, contents] of Object.entries(files)) {
+    assertRelativePath(file, path);
+    if (typeof contents !== "string") throw new Error(`${path}.${file} must be a string`);
+  }
+}
+function validateEnvNames(names, path) {
+  const seen = /* @__PURE__ */ new Set();
+  for (const [index, name] of names.entries()) {
+    if (!ENV_NAME.test(name)) throw new Error(`${path}[${index}] is not a valid environment name`);
+    if (seen.has(name)) throw new Error(`${path} contains duplicate name ${name}`);
+    seen.add(name);
+  }
+}
+function assertRelativePath(path, label) {
+  const normalized = normalize(path);
+  if (path.length === 0 || path.includes("\0") || isAbsolute(path) || normalized === ".." || normalized.startsWith(`..${sep}`) || relative(".", normalized).startsWith("..")) {
+    throw new Error(`${label} contains unsafe path: ${path}`);
+  }
+}
+function taskRow(task, idx) {
+  return {
+    idx,
+    name: task.id,
+    prompt: task.prompt,
+    system_prompt: task.systemPrompt ?? null,
+    split: task.split,
+    answer: task.answer ?? null,
+    metadata: task.metadata ?? {}
+  };
+}
+function renderPyproject(options, moduleName) {
+  const description = options.description ?? `PrimeIntellect tasks for ${options.name}`;
+  return `[project]
+name = ${toml(options.name)}
+version = ${toml(options.version)}
+description = ${toml(description)}
+requires-python = ">=3.11,<3.14"
+dependencies = ["verifiers${VERIFIERS_RANGE}"]
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = [${toml(moduleName)}]
+
+[tool.uv]
+prerelease = "allow"
+`;
+}
+function renderPrimeConfig(options, split) {
+  const limits = [
+    `max_turns = ${options.maxTurns ?? DEFAULT_MAX_TURNS}`,
+    options.maxInputTokens === void 0 ? void 0 : `max_input_tokens = ${options.maxInputTokens}`,
+    options.maxOutputTokens === void 0 ? void 0 : `max_output_tokens = ${options.maxOutputTokens}`,
+    options.maxTotalTokens === void 0 ? void 0 : `max_total_tokens = ${options.maxTotalTokens}`
+  ].filter((line) => line !== void 0);
+  return `${limits.join("\n")}
+push = false
+
+[timeout]
+rollout = ${options.rolloutTimeoutSeconds ?? DEFAULT_ROLLOUT_TIMEOUT}
+scoring = ${options.scoringTimeoutSeconds ?? DEFAULT_SCORING_TIMEOUT}
+
+[taskset]
+id = ${toml(options.name)}
+split = ${toml(split)}
+
+[harness]
+id = ${toml(options.name)}
+program = ${tomlArray(options.runner.command)}
+forward_env = ${tomlArray(options.runner.forwardEnv ?? [])}
+
+[harness.runtime]
+type = "docker"
+image = ${toml(options.runner.image)}
+`;
+}
+function renderInit(moduleName) {
+  return `from ${moduleName}.harness import TangleRuntimeHarness
+from ${moduleName}.taskset import TangleTaskset
+
+__all__ = ["TangleRuntimeHarness", "TangleTaskset"]
+`;
+}
+function renderTaskset(moduleName, scoring) {
+  const config = scoringConfig(scoring);
+  return `import asyncio
+import json
+import math
+import os
+from importlib.resources import files
+from pathlib import Path
+from typing import Any, Literal
+
+import verifiers.v1 as vf
+
+SCORING = json.loads(${pythonString(JSON.stringify(config))})
+PACKAGE_ROOT = Path(__file__).resolve().parent
+
+
+class TangleTaskData(vf.TaskData):
+    split: Literal["train", "eval"]
+    answer: str | list[str] | None = None
+    metadata: dict[str, Any] = {}
+
+
+class TangleTaskConfig(vf.TaskConfig):
+    scoring: Literal["exact", "reference-judge", "command"] = SCORING["kind"]
+    normalization: Literal["none", "trim", "trim-casefold"] = SCORING.get("normalization", "trim")
+    judge_model: str = SCORING.get("model", "openai/gpt-5.4-nano")
+    judge_prompt: str | None = SCORING.get("prompt")
+    judge_view: Literal["last_reply", "full_trace"] = SCORING.get("view", "last_reply")
+    score_program: list[str] = SCORING.get("command", [])
+    score_forward_env: list[str] = SCORING.get("forwardEnv", [])
+    score_timeout_seconds: float = SCORING.get("timeoutSeconds", 300)
+
+
+def _normalize(value: str, mode: str) -> str:
+    if mode == "none":
+        return value
+    value = value.strip()
+    return value.casefold() if mode == "trim-casefold" else value
+
+
+async def _run_score_command(config: TangleTaskConfig, data: TangleTaskData, trace: vf.Trace) -> float:
+    if not config.score_program:
+        raise ValueError("command scoring requires score_program")
+    safe_env = {
+        key: os.environ[key]
+        for key in ("PATH", "HOME", "TMPDIR", "LANG")
+        if key in os.environ
+    }
+    safe_env.update(
+        {key: os.environ[key] for key in config.score_forward_env if key in os.environ}
+    )
+    request = {
+        "protocol": "tangle.primeintellect.score/v1",
+        "task": data.model_dump(mode="json", exclude_none=True),
+        "trace": trace.model_dump(mode="json", exclude_none=True),
+    }
+    process = await asyncio.create_subprocess_exec(
+        *config.score_program,
+        cwd=PACKAGE_ROOT,
+        env=safe_env,
+        stdin=asyncio.subprocess.PIPE,
+        stdout=asyncio.subprocess.PIPE,
+        stderr=asyncio.subprocess.PIPE,
+    )
+    payload = json.dumps(request, separators=(",", ":")).encode()
+    try:
+        stdout, stderr = await asyncio.wait_for(
+            process.communicate(payload), timeout=config.score_timeout_seconds
+        )
+    except TimeoutError:
+        process.kill()
+        await process.communicate()
+        raise RuntimeError(
+            f"score command timed out after {config.score_timeout_seconds}s"
+        )
+    if process.returncode != 0:
+        detail = (stderr or stdout).decode(errors="replace").strip()[-2000:]
+        raise RuntimeError(f"score command exited {process.returncode}: {detail}")
+    try:
+        result = json.loads(stdout)
+    except json.JSONDecodeError as error:
+        raise ValueError(f"score command returned invalid JSON: {error}") from error
+    if not isinstance(result, dict):
+        raise ValueError("score command must return an object")
+    reward = result.get("reward")
+    if isinstance(reward, bool) or not isinstance(reward, (int, float)) or not math.isfinite(reward):
+        raise ValueError("score command reward must be a finite number")
+    metrics = result.get("metrics", {})
+    if not isinstance(metrics, dict):
+        raise ValueError("score command metrics must be an object")
+    for name, value in metrics.items():
+        if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value):
+            raise ValueError(f"score command metric {name!r} must be a finite number")
+    trace.record_metrics(metrics)
+    return float(reward)
+
+
+class TangleTask(vf.Task[TangleTaskData, vf.State, TangleTaskConfig]):
+    @vf.reward(weight=1.0)
+    async def task_reward(self, trace: vf.Trace) -> float:
+        if self.config.scoring == "command":
+            return await _run_score_command(self.config, self.data, trace)
+        if self.data.answer is None:
+            raise ValueError(f"task {self.data.name!r} has no reference answer")
+        if self.config.scoring == "reference-judge":
+            judge = vf.ReferenceJudge(
+                vf.ReferenceJudgeConfig(
+                    model=self.config.judge_model,
+                    prompt=self.config.judge_prompt,
+                    view=self.config.judge_view,
+                )
+            )
+            return await judge.score(self.data, trace)
+        expected = self.data.answer if isinstance(self.data.answer, list) else [self.data.answer]
+        actual = _normalize(trace.last_reply, self.config.normalization)
+        return float(any(actual == _normalize(answer, self.config.normalization) for answer in expected))
+
+
+class TangleTasksetConfig(vf.TasksetConfig):
+    split: Literal["train", "eval"] = "eval"
+    task: TangleTaskConfig = TangleTaskConfig()
+
+
+class TangleTaskset(vf.Taskset[TangleTask, TangleTasksetConfig]):
+    def load(self) -> list[TangleTask]:
+        resource = files(${pythonString(moduleName)}).joinpath("tasks.jsonl")
+        tasks: list[TangleTask] = []
+        with resource.open("r", encoding="utf-8") as handle:
+            for line in handle:
+                if not line.strip():
+                    continue
+                row = json.loads(line)
+                if row["split"] != self.config.split:
+                    continue
+                tasks.append(TangleTask(TangleTaskData.model_validate(row), self.config.task))
+        if not tasks:
+            raise ValueError(f"task split {self.config.split!r} is empty")
+        return tasks
+
+
+__all__ = ["TangleTaskset"]
+`;
+}
+function renderHarness(moduleName) {
+  return `import json
+from importlib.resources import files
+
+import verifiers.v1 as vf
+
+RUNNER = json.loads(files(${pythonString(moduleName)}).joinpath("runner.json").read_text(encoding="utf-8"))
+
+
+class TangleRuntimeHarnessConfig(vf.HarnessConfig):
+    program: list[str] = RUNNER["command"]
+    setup_commands: list[list[str]] = RUNNER["setup"]
+    forward_env: list[str] = RUNNER["forwardEnv"]
+
+
+class TangleRuntimeHarness(vf.Harness[TangleRuntimeHarnessConfig]):
+    APPENDS_SYSTEM_PROMPT = True
+    SUPPORTS_MCP = True
+    SUPPORTS_MESSAGE_PROMPT = True
+
+    async def setup(self, runtime: vf.Runtime) -> None:
+        for path, contents in RUNNER["files"].items():
+            await runtime.write(path, contents.encode())
+        for command in self.config.setup_commands:
+            result = await runtime.run(command, self.config.resolved_env)
+            if result.exit_code != 0:
+                detail = (result.stderr or result.stdout).strip()[-2000:]
+                raise RuntimeError(
+                    f"runner setup command {command[0]!r} exited {result.exit_code}: {detail}"
+                )
+
+    async def launch(
+        self,
+        ctx: vf.ModelContext,
+        trace: vf.Trace,
+        runtime: vf.Runtime,
+        endpoint: str,
+        secret: str,
+        mcp_urls: dict[str, str],
+    ) -> vf.ProgramResult:
+        data = trace.task.data
+        public_task = {
+            "id": data.name or str(data.idx),
+            "split": data.split,
+            "prompt": data.prompt,
+            "metadata": data.metadata,
+        }
+        if data.system_prompt is not None:
+            public_task["systemPrompt"] = data.system_prompt
+        env = {
+            **self.config.resolved_env,
+            "OPENAI_BASE_URL": endpoint,
+            "OPENAI_API_KEY": secret,
+            "OPENAI_MODEL": ctx.model,
+            "TANGLE_PRIME_TASK_JSON": json.dumps(public_task, separators=(",", ":")),
+            "TANGLE_PRIME_MCP_SERVERS_JSON": json.dumps(mcp_urls, separators=(",", ":")),
+        }
+        if not self.config.program:
+            raise ValueError("Tangle runtime harness requires a program argv")
+        return await runtime.run_program(self.config.program, env)
+
+
+__all__ = ["TangleRuntimeHarness"]
+`;
+}
+function scoringConfig(scoring) {
+  if (scoring.kind === "exact") {
+    return { kind: scoring.kind, normalization: scoring.normalization ?? "trim" };
+  }
+  if (scoring.kind === "reference-judge") {
+    return {
+      kind: scoring.kind,
+      model: scoring.model,
+      prompt: scoring.prompt ?? null,
+      view: scoring.view ?? "last_reply"
+    };
+  }
+  return {
+    kind: scoring.kind,
+    command: [...scoring.command],
+    forwardEnv: [...scoring.forwardEnv ?? []],
+    timeoutSeconds: scoring.timeoutSeconds ?? DEFAULT_SCORING_TIMEOUT
+  };
+}
+function renderReadme(options) {
+  return `# ${options.name}
+
+PrimeIntellect Verifiers v1 tasks that run the caller's Tangle agent program.
+
+## Evaluate
+
+\`\`\`bash
+uv run eval @ prime.eval.toml --model <provider/model-snapshot>
+\`\`\`
+
+## Train
+
+Use \`prime.train.toml\` as the environment config for Prime RL. The train and eval rows are disjoint and selected by \`taskset.split\`.
+
+The runner receives only the prompt, metadata, intercepted model endpoint, and MCP URLs. Reference answers remain in the task process and are never written into the runner workspace or environment.
+`;
+}
+function toml(value) {
+  return JSON.stringify(value);
+}
+function tomlArray(values) {
+  return `[${values.map(toml).join(", ")}]`;
+}
+function pythonString(value) {
+  return JSON.stringify(value);
+}
+function positiveInteger(value, path) {
+  if (!Number.isSafeInteger(value) || value <= 0)
+    throw new Error(`${path} must be a positive integer`);
+}
+function positiveNumber(value, path) {
+  if (!Number.isFinite(value) || value <= 0) throw new Error(`${path} must be positive`);
+}
+async function pathExists(path) {
+  try {
+    await stat(path);
+    return true;
+  } catch (error) {
+    if (error.code === "ENOENT") return false;
+    throw error;
+  }
+}
+async function assertGeneratedPackage(output) {
+  let manifest;
+  try {
+    manifest = JSON.parse(await readFile(join(output, "manifest.json"), "utf8"));
+  } catch (error) {
+    throw new Error(
+      `refusing to replace ${output}: it is not a generated PrimeIntellect package (${error instanceof Error ? error.message : String(error)})`
+    );
+  }
+  if (manifest === null || typeof manifest !== "object" || manifest.schema !== "tangle.primeintellect.package/v1") {
+    throw new Error(`refusing to replace ${output}: manifest schema does not match`);
+  }
+}
+function sha256(value) {
+  return createHash("sha256").update(value, "utf8").digest("hex");
+}
+
+// src/primeintellect/runner.ts
+var ENV = {
+  task: "TANGLE_PRIME_TASK_JSON",
+  model: "OPENAI_MODEL",
+  baseUrl: "OPENAI_BASE_URL",
+  apiKey: "OPENAI_API_KEY",
+  mcpServers: "TANGLE_PRIME_MCP_SERVERS_JSON"
+};
+function readPrimeIntellectEpisodeContext(env = process.env) {
+  const rawTask = requiredEnv(env, ENV.task);
+  const rawMcp = env[ENV.mcpServers] ?? "{}";
+  const parsedTask = parseJson(rawTask, ENV.task);
+  const parsedMcp = parseJson(rawMcp, ENV.mcpServers);
+  const task = validatePublicTask(parsedTask);
+  const mcpServers = validateStringMap(parsedMcp, ENV.mcpServers);
+  return {
+    protocol: "tangle.primeintellect.episode/v1",
+    task,
+    model: {
+      name: requiredEnv(env, ENV.model),
+      baseUrl: requiredEnv(env, ENV.baseUrl),
+      apiKey: requiredEnv(env, ENV.apiKey)
+    },
+    mcpServers
+  };
+}
+function createPrimeIntellectBackend(context, options = {}) {
+  return createOpenAICompatibleBackend({
+    ...options,
+    apiKey: context.model.apiKey,
+    baseUrl: context.model.baseUrl,
+    model: context.model.name,
+    kind: options.kind ?? "primeintellect"
+  });
+}
+async function runPrimeIntellectProgram(run, options = {}) {
+  return run(readPrimeIntellectEpisodeContext(options.env));
+}
+function requiredEnv(env, name) {
+  const value = env[name];
+  if (typeof value !== "string" || value.length === 0) {
+    throw new Error(`PrimeIntellect runner requires ${name}`);
+  }
+  return value;
+}
+function parseJson(value, name) {
+  try {
+    return JSON.parse(value);
+  } catch (error) {
+    throw new Error(
+      `${name} must contain valid JSON: ${error instanceof Error ? error.message : String(error)}`
+    );
+  }
+}
+function validatePublicTask(value) {
+  const task = record2(value, ENV.task);
+  for (const privateField of ["answer", "reference", "scoring", "score"]) {
+    if (privateField in task) {
+      throw new Error(`${ENV.task} exposed private field ${privateField}`);
+    }
+  }
+  for (const field of Object.keys(task)) {
+    if (!["id", "split", "prompt", "systemPrompt", "metadata"].includes(field)) {
+      throw new Error(`${ENV.task}.${field} is not supported`);
+    }
+  }
+  const id = nonEmptyString2(task.id, `${ENV.task}.id`);
+  const split = validateSplit(task.split, `${ENV.task}.split`);
+  const prompt = validatePrimeIntellectPrompt(task.prompt, `${ENV.task}.prompt`);
+  const systemPrompt = optionalString(task.systemPrompt, `${ENV.task}.systemPrompt`);
+  if (systemPrompt !== void 0 && Array.isArray(prompt) && prompt.some((message) => message.role === "system")) {
+    throw new Error(`${ENV.task} must not set systemPrompt and include a system message`);
+  }
+  const metadata = task.metadata === void 0 ? void 0 : validatePrimeIntellectJsonObject(task.metadata, `${ENV.task}.metadata`);
+  return {
+    id,
+    split,
+    prompt,
+    ...systemPrompt !== void 0 ? { systemPrompt } : {},
+    ...metadata !== void 0 ? { metadata } : {}
+  };
+}
+function validateSplit(value, path) {
+  if (value !== "train" && value !== "eval") {
+    throw new Error(`${path} must be train or eval`);
+  }
+  return value;
+}
+function validateStringMap(value, path) {
+  const input = record2(value, path);
+  const output = {};
+  for (const [key, entry] of Object.entries(input)) {
+    if (typeof entry !== "string" || entry.length === 0) {
+      throw new Error(`${path}.${key} must be a non-empty string`);
+    }
+    output[key] = entry;
+  }
+  return output;
+}
+function record2(value, path) {
+  if (value === null || typeof value !== "object" || Array.isArray(value)) {
+    throw new Error(`${path} must be an object`);
+  }
+  return value;
+}
+function nonEmptyString2(value, path) {
+  if (typeof value !== "string" || value.length === 0) {
+    throw new Error(`${path} must be a non-empty string`);
+  }
+  return value;
+}
+function optionalString(value, path) {
+  if (value === void 0) return void 0;
+  if (typeof value !== "string") throw new Error(`${path} must be a string`);
+  return value;
+}
+
+// src/primeintellect/traces.ts
+import { validateRunRecord } from "@tangle-network/agent-eval";
+function parsePrimeIntellectTraces(jsonl) {
+  const traces = [];
+  for (const [lineIndex, line] of jsonl.split("\n").entries()) {
+    if (!line.trim()) continue;
+    let parsed;
+    try {
+      parsed = JSON.parse(line);
+    } catch (error) {
+      throw new Error(
+        `PrimeIntellect trace line ${lineIndex + 1} is invalid JSON: ${error instanceof Error ? error.message : String(error)}`
+      );
+    }
+    traces.push(validateTrace(parsed, lineIndex + 1));
+  }
+  if (traces.length === 0) throw new Error("PrimeIntellect traces.jsonl contains no traces");
+  return traces;
+}
+function importPrimeIntellectTraces(jsonl, defaults) {
+  return parsePrimeIntellectTraces(jsonl).map(
+    (trace) => primeIntellectTraceToRunRecord(trace, defaults)
+  );
+}
+function primeIntellectTraceToRunRecord(trace, options) {
+  const split = trace.task.data.split;
+  if (split !== "train" && split !== "eval") {
+    throw new Error(`PrimeIntellect trace ${trace.id} has no train/eval split`);
+  }
+  const reward = sumFinite(Object.values(trace.rewards), `trace ${trace.id} rewards`);
+  const usage = aggregateUsage([
+    ...trace.nodes.map((node) => node.usage).filter((value) => value != null),
+    ...trace.extra_usage ?? []
+  ]);
+  const errors = trace.errors ?? [];
+  const raw = {
+    reward,
+    "prime.turns": trace.nodes.filter((node) => node.sampled === true).length,
+    "prime.branches": countBranches(trace.nodes),
+    "prime.errors": errors.length,
+    "prime.completed": trace.is_completed ? 1 : 0,
+    "prime.cost_complete": usage.costComplete ? 1 : 0,
+    "prime.reported_cost_usd": usage.reportedCostUsd
+  };
+  for (const [name, value] of Object.entries(trace.rewards)) {
+    raw[`reward.${name}`] = finite(value, `trace ${trace.id} reward ${name}`);
+  }
+  for (const [name, value] of Object.entries(trace.metrics)) {
+    raw[`metric.${name}`] = finite(value, `trace ${trace.id} metric ${name}`);
+  }
+  const record3 = {
+    runId: trace.id,
+    experimentId: options.experimentId,
+    candidateId: options.candidateId,
+    seed: options.seed,
+    model: options.model,
+    promptHash: options.promptHash,
+    configHash: options.configHash,
+    commitSha: options.commitSha,
+    wallMs: traceWallMs(trace),
+    costUsd: usage.costComplete ? usage.reportedCostUsd : 0,
+    costProvenance: usage.costComplete ? { kind: "observed", usd: usage.reportedCostUsd } : { kind: "uncaptured", usd: null },
+    tokenUsage: {
+      input: usage.input,
+      output: usage.output,
+      ...usage.reasoning !== void 0 ? { reasoning: usage.reasoning } : {},
+      ...usage.cached !== void 0 ? { cached: usage.cached } : {}
+    },
+    outcome: split === "eval" ? { holdoutScore: reward, raw } : { searchScore: reward, raw },
+    splitTag: split === "eval" ? "holdout" : "search",
+    scenarioId: trace.task.data.name ?? String(trace.task.data.idx),
+    ...errors[0] ? { failureMode: `primeintellect:${errors[0].type}:${errors[0].message}` } : {}
+  };
+  return validateRunRecord(record3);
+}
+function validateTrace(value, line) {
+  const trace = object(value, `PrimeIntellect trace line ${line}`);
+  nonEmptyString3(trace.id, `PrimeIntellect trace line ${line}.id`);
+  const task = object(trace.task, `PrimeIntellect trace line ${line}.task`);
+  nonEmptyString3(task.type, `PrimeIntellect trace line ${line}.task.type`);
+  const data = object(task.data, `PrimeIntellect trace line ${line}.task.data`);
+  if (!Number.isSafeInteger(data.idx) || data.idx < 0) {
+    throw new Error(
+      `PrimeIntellect trace line ${line}.task.data.idx must be a non-negative integer`
+    );
+  }
+  if (!Array.isArray(trace.nodes)) {
+    throw new Error(`PrimeIntellect trace line ${line}.nodes must be an array`);
+  }
+  for (const [index, rawNode] of trace.nodes.entries()) {
+    const node = object(rawNode, `PrimeIntellect trace line ${line}.nodes[${index}]`);
+    const parent = node.parent;
+    if (parent !== void 0 && parent !== null && (!Number.isSafeInteger(parent) || parent < 0 || parent >= index)) {
+      throw new Error(
+        `PrimeIntellect trace line ${line}.nodes[${index}].parent must reference an earlier node`
+      );
+    }
+    if (node.sampled !== void 0 && typeof node.sampled !== "boolean") {
+      throw new Error(`PrimeIntellect trace line ${line}.nodes[${index}].sampled must be boolean`);
+    }
+    if (node.usage !== void 0 && node.usage !== null) {
+      validateUsage(node.usage, `PrimeIntellect trace line ${line}.nodes[${index}].usage`);
+    }
+  }
+  validateNumberMap(trace.rewards, `PrimeIntellect trace line ${line}.rewards`);
+  validateNumberMap(trace.metrics, `PrimeIntellect trace line ${line}.metrics`);
+  if (trace.extra_usage !== void 0 && !Array.isArray(trace.extra_usage)) {
+    throw new Error(`PrimeIntellect trace line ${line}.extra_usage must be an array`);
+  }
+  for (const [index, usage] of (trace.extra_usage ?? []).entries()) {
+    validateUsage(usage, `PrimeIntellect trace line ${line}.extra_usage[${index}]`);
+  }
+  if (trace.errors !== void 0 && !Array.isArray(trace.errors)) {
+    throw new Error(`PrimeIntellect trace line ${line}.errors must be an array`);
+  }
+  for (const [index, rawError] of (trace.errors ?? []).entries()) {
+    const error = object(rawError, `PrimeIntellect trace line ${line}.errors[${index}]`);
+    nonEmptyString3(error.type, `PrimeIntellect trace line ${line}.errors[${index}].type`);
+    nonEmptyString3(error.message, `PrimeIntellect trace line ${line}.errors[${index}].message`);
+    if (error.traceback !== void 0 && error.traceback !== null && typeof error.traceback !== "string") {
+      throw new Error(
+        `PrimeIntellect trace line ${line}.errors[${index}].traceback must be a string or null`
+      );
+    }
+  }
+  if (trace.is_completed !== void 0 && typeof trace.is_completed !== "boolean") {
+    throw new Error(`PrimeIntellect trace line ${line}.is_completed must be boolean`);
+  }
+  if (trace.stop_condition !== void 0 && trace.stop_condition !== null && typeof trace.stop_condition !== "string") {
+    throw new Error(`PrimeIntellect trace line ${line}.stop_condition must be a string or null`);
+  }
+  if (trace.timing !== void 0) validateTiming(trace.timing, line);
+  return value;
+}
+function validateTiming(value, line) {
+  const timing = object(value, `PrimeIntellect trace line ${line}.timing`);
+  optionalFinite(timing.start, `PrimeIntellect trace line ${line}.timing.start`);
+  for (const phase of ["setup", "generation", "finalize", "scoring"]) {
+    if (timing[phase] === void 0) continue;
+    const span = object(timing[phase], `PrimeIntellect trace line ${line}.timing.${phase}`);
+    optionalFinite(span.start, `PrimeIntellect trace line ${line}.timing.${phase}.start`);
+    optionalFinite(span.end, `PrimeIntellect trace line ${line}.timing.${phase}.end`);
+  }
+}
+function aggregateUsage(usages) {
+  let input = 0;
+  let output = 0;
+  let reasoning = 0;
+  let cached = 0;
+  let costUsd = 0;
+  let sawReasoning = false;
+  let sawCached = false;
+  let costsReported = 0;
+  for (const [index, usage] of usages.entries()) {
+    const prompt = nonNegative(usage.prompt_tokens, `usage[${index}].prompt_tokens`);
+    const completion = nonNegative(usage.completion_tokens, `usage[${index}].completion_tokens`);
+    const cachedInput = optionalNonNegative(
+      usage.cached_input_tokens,
+      `usage[${index}].cached_input_tokens`
+    );
+    const reasoningTokens = optionalNonNegative(
+      usage.reasoning_tokens,
+      `usage[${index}].reasoning_tokens`
+    );
+    const cost = optionalNonNegative(usage.cost, `usage[${index}].cost`);
+    input += prompt + (cachedInput ?? 0);
+    output += completion;
+    if (cachedInput !== void 0) {
+      cached += cachedInput;
+      sawCached = true;
+    }
+    if (reasoningTokens !== void 0) {
+      reasoning += reasoningTokens;
+      sawReasoning = true;
+    }
+    if (cost !== void 0) {
+      costUsd += cost;
+      costsReported += 1;
+    }
+  }
+  if (reasoning > output) {
+    throw new Error(
+      `PrimeIntellect reasoning token total ${reasoning} exceeds output total ${output}`
+    );
+  }
+  return {
+    input,
+    output,
+    ...sawReasoning ? { reasoning } : {},
+    ...sawCached ? { cached } : {},
+    reportedCostUsd: costUsd,
+    costComplete: usages.length > 0 && costsReported === usages.length
+  };
+}
+function validateUsage(value, path) {
+  const usage = object(value, path);
+  nonNegative(usage.prompt_tokens, `${path}.prompt_tokens`);
+  nonNegative(usage.completion_tokens, `${path}.completion_tokens`);
+  optionalNonNegative(usage.cached_input_tokens, `${path}.cached_input_tokens`);
+  optionalNonNegative(usage.reasoning_tokens, `${path}.reasoning_tokens`);
+  optionalNonNegative(usage.cost, `${path}.cost`);
+}
+function traceWallMs(trace) {
+  const timing = trace.timing;
+  if (!timing || typeof timing.start !== "number" || !Number.isFinite(timing.start)) return 0;
+  const ends = [
+    timing.setup?.end,
+    timing.generation?.end,
+    timing.finalize?.end,
+    timing.scoring?.end
+  ].filter((value) => typeof value === "number" && Number.isFinite(value));
+  if (ends.length === 0) return 0;
+  return Math.max(0, (Math.max(...ends) - timing.start) * 1e3);
+}
+function countBranches(nodes) {
+  if (nodes.length === 0) return 0;
+  const parents = new Set(
+    nodes.map((node) => node.parent).filter(
+      (parent) => Number.isSafeInteger(parent) && parent >= 0
+    )
+  );
+  return nodes.reduce((count, _node, index) => count + (parents.has(index) ? 0 : 1), 0);
+}
+function validateNumberMap(value, path) {
+  const map = object(value, path);
+  for (const [key, entry] of Object.entries(map)) finite(entry, `${path}.${key}`);
+}
+function sumFinite(values, path) {
+  return values.reduce((sum, value, index) => sum + finite(value, `${path}[${index}]`), 0);
+}
+function finite(value, path) {
+  if (typeof value !== "number" || !Number.isFinite(value)) {
+    throw new Error(`${path} must be a finite number`);
+  }
+  return value;
+}
+function nonNegative(value, path) {
+  const parsed = finite(value, path);
+  if (parsed < 0) throw new Error(`${path} must be non-negative`);
+  return parsed;
+}
+function optionalNonNegative(value, path) {
+  if (value === void 0 || value === null) return void 0;
+  return nonNegative(value, path);
+}
+function optionalFinite(value, path) {
+  if (value === void 0 || value === null) return void 0;
+  return finite(value, path);
+}
+function object(value, path) {
+  if (value === null || typeof value !== "object" || Array.isArray(value)) {
+    throw new Error(`${path} must be an object`);
+  }
+  return value;
+}
+function nonEmptyString3(value, path) {
+  if (typeof value !== "string" || value.length === 0) {
+    throw new Error(`${path} must be a non-empty string`);
+  }
+  return value;
+}
+export {
+  createPrimeIntellectBackend,
+  createPrimeIntellectPackage,
+  importPrimeIntellectTraces,
+  parsePrimeIntellectTraces,
+  primeIntellectTraceToRunRecord,
+  readPrimeIntellectEpisodeContext,
+  runPrimeIntellectProgram,
+  writePrimeIntellectPackage
+};
+//# sourceMappingURL=index.js.map