@tangle-network/agent-eval 0.119.1 → 0.120.0

package/dist/primeintellect/index.d.ts

@@ -1,311 +0,0 @@
-type AgentProfileCellSchemaVersion = 'agent-profile-cell/v1';
-type AgentProfileDimensionValue = string | number | boolean | null;
-interface AgentProfileSource {
-    /** Runtime/profile contract being fingerprinted, e.g. `agent-interface-profile`. */
-    kind: string;
-    /** sha256 over the canonical source profile object. */
-    hash: string;
-}
-interface AgentProfileHarness {
-    id: string;
-    version?: string;
-    hash?: string;
-}
-interface AgentProfileCell {
-    schemaVersion: AgentProfileCellSchemaVersion;
-    cellId: string;
-    profileId: string;
-    sourceProfile: AgentProfileSource;
-    harness?: AgentProfileHarness;
-    model?: string;
-    promptHash?: string;
-    dimensions?: Record<string, AgentProfileDimensionValue>;
-}
-
-type FailureClass = 'success' | 'reasoning_error' | 'tool_selection_error' | 'tool_argument_error' | 'tool_recovery_failure' | 'hallucination' | 'instruction_following' | 'safety_refusal_miss' | 'policy_violation' | 'budget_exceeded' | 'format_drift' | 'permission_escalation' | 'pii_leak' | 'cost_overrun' | 'timeout' | 'sandbox_failure' | 'missing_user_data' | 'missing_domain_data' | 'missing_codebase_context' | 'missing_runtime_context' | 'missing_credentials' | 'missing_integration_connection' | 'missing_integration_scope' | 'integration_approval_required' | 'integration_auth_expired' | 'integration_provider_failure' | 'bad_integration_manifest' | 'unsafe_integration_write_denied' | 'stale_external_data' | 'bad_retrieval' | 'insufficient_evidence' | 'contradictory_evidence' | 'ambiguous_user_intent' | 'knowledge_readiness_blocked' | 'unknown';
-
-/**
- * Paper-grade RunRecord schema + runtime validator.
- *
- * Every run that participates in a promotion gate, paper table, or
- * researcher loop SHOULD be recorded as a `RunRecord`. The mandatory
- * fields are exactly those the paper "Two Loops, Three Roles" requires
- * for reproducibility: who/what/when/cost/seed/hash, plus the search vs
- * holdout split tag and either a `searchScore` or a `holdoutScore`.
- *
- * This is intentionally NOT a replacement for the rich `Run` /
- * `ProposeReviewReport` / `ScenarioResult` types already in the
- * package. Those are runtime structures with full provenance. A
- * `RunRecord` is the analysis-time projection — the JSON-friendly
- * row you'd put in a parquet file or paste into a notebook.
- *
- * Validate at the boundary:
- *
- *   const rec = validateRunRecord(rawJson)         // throws on missing
- *   const ok  = isRunRecord(rawJson)               // boolean check
- *   const rec = parseRunRecordSafe(rawJson)        // { ok, value | error }
- *
- * The validator runs in pure TS — zod is intentionally NOT a
- * dependency. Round-trip tested in `tests/run-record.test.ts`.
- */
-
-/** Search/dev/holdout split tag. 'search' is the paper-grade alias for the
- *  combined train+test pool that the optimizer is allowed to read. */
-type RunSplitTag = 'search' | 'dev' | 'holdout';
-interface RunTokenUsage {
-    input: number;
-    /** All generated tokens charged as output, including reasoning tokens. */
-    output: number;
-    /** Reasoning-token subset of `output`, when the provider reports it. */
-    reasoning?: number;
-    /** Prompt tokens served from a provider cache. */
-    cached?: number;
-    /** Prompt tokens written into a provider cache. */
-    cacheWrite?: number;
-}
-/**
- * How a run's USD amount was obtained.
- *
- * `costUsd` remains mandatory for wire compatibility. New producers should
- * always populate this discriminated union so a missing bill is never
- * mistaken for an observed zero-dollar run. For `uncaptured`, `costUsd` uses
- * the legacy `0` sentinel while this field carries the truthful null.
- */
-type RunCostProvenance = {
-    kind: 'observed';
-    usd: number;
-} | {
-    kind: 'estimated';
-    usd: number;
-} | {
-    kind: 'uncaptured';
-    usd: null;
-};
-interface RunJudgeMetadata {
-    model: string;
-    promptVersion: string;
-    /** [0,1] confidence the judge declared. Constant judge confidence
-     *  across many runs is a fallback signal (see `canary.ts`). */
-    confidence: number;
-    /** True if the judge degraded to a fallback path (rules-only,
-     *  prior-call cache, etc.). The canary uses this to alert. */
-    fallback: boolean;
-}
-/**
- * Per-judge / per-dimension breakdown for runs scored by an ensemble of
- * judges over a multi-dimensional rubric.
- *
- * The collapsed `outcome.searchScore` / `holdoutScore` carries the
- * composite the gate uses. The full breakdown belongs here so consumers
- * can answer "which judge disagreed?", "which dimension dragged the
- * composite down?", and "did half the panel fail?" without re-running.
- *
- * `perJudge[judgeId][dim]` is the canonical source; `perDimMean` and
- * `composite` are convenience projections — derivable but precomputed so
- * downstream IRR primitives (`interRaterReliability`,
- * `corpusInterRaterAgreement`) and reporters don't pay the same
- * aggregation twice.
- *
- * Fail-loud discipline: judges that errored out land in `failedJudges`
- * by id. A missing key in `perJudge` is ambiguous (silent zero vs not
- * run); the explicit list makes a partial-failure recorded as such.
- */
-interface JudgeScoresRecord {
-    /** Per-judge per-dimension scores. `{ "kimi-k2.6": { helpfulness: 0.8, clarity: 0.7 }, ... }`. */
-    perJudge: Record<string, Record<string, number>>;
-    /** Per-dim mean across judges. Convenience — derivable from `perJudge`. */
-    perDimMean: Record<string, number>;
-    /** Composite mean across all dims and judges. Mirrors the score
-     *  the gate sees on `outcome.searchScore` / `holdoutScore`. */
-    composite: number;
-    /** Judges that errored or returned an unparseable verdict. Recorded
-     *  by id (e.g. `['glm-5.1']`) so a partial-failure case is explicit,
-     *  not inferred from missing keys in `perJudge`. */
-    failedJudges?: string[];
-    /** Free-form notes the judges emitted (joined across judges or
-     *  first-judge only — consumer's choice). */
-    notes?: string;
-}
-interface RunOutcome {
-    /** Score on the search/optimization split. Optional because a
-     *  holdout-only evaluation only fills `holdoutScore`. */
-    searchScore?: number;
-    /** Score on the held-out split. Optional because a search-only run
-     *  only fills `searchScore`. At least one must be present. */
-    holdoutScore?: number;
-    /** Bag of any other metric the run produced — judge dimensions,
-     *  pass/fail counters, latency stats, etc. Numeric only — keeps
-     *  reporters honest. */
-    raw: Record<string, number>;
-    /** Per-judge / per-dim breakdown. Consumers writing ensemble
-     *  judgements populate this; substrate primitives like
-     *  `interRaterReliability` and `corpusInterRaterAgreement` accept
-     *  these records as input. Optional — single-judge or scalar-only
-     *  runs leave it unset. */
-    judgeScores?: JudgeScoresRecord;
-    /** Authenticity / realness verdict — did the run build the REAL thing on the
-     *  intended infra, or fake it (see `./authenticity`)? Optional: only domains
-     *  with an authenticity config populate it. Carried in the corpus so the
-     *  flywheel / off-policy learning can optimize for real completion, not gamed
-     *  pass-rate. `score` is 0-1; `gated` is the anti-Goodhart flag — a gated run
-     *  must not count as a real success regardless of `score`. */
-    realness?: {
-        score: number;
-        gated: boolean;
-        reason?: string;
-    };
-}
-/**
- * Mandatory paper-grade fields for a single evaluation run. Optional
- * fields are extension points; mandatory fields throw if missing.
- *
- * Hash discipline:
- *   - `promptHash` is the sha256 of the EFFECTIVE prompt sent to the
- *     model (after any steering bundle merge).
- *   - `configHash` is the sha256 of the effective run config (model,
- *     temperature, tools, judges, splits). The pair (promptHash,
- *     configHash) uniquely identifies an experiment cell.
- *
- * Model snapshot discipline:
- *   - `model` MUST encode a snapshot version. Bare aliases like
- *     `claude-sonnet-4` or `gpt-4o` are banned — they remap silently.
- *     Use `claude-sonnet-4-6@2025-04-15` or `gpt-4o-2024-11-20`.
- */
-interface RunRecord {
-    /** UUID for the run. */
-    runId: string;
-    /** Logical experiment grouping (a treatment vs a baseline within
-     *  the same sweep should share `experimentId`). */
-    experimentId: string;
-    /** Stable identifier for the candidate (variant) being run. The
-     *  promotion gate compares two `candidateId`s on matched items. */
-    candidateId: string;
-    /** RNG seed for the run. Always recorded — silent re-seeding is
-     *  the most common cause of non-reproducible numbers. */
-    seed: number;
-    /** Model identifier WITH snapshot version. */
-    model: string;
-    /** sha256 of the effective prompt (post-steering). */
-    promptHash: string;
-    /** sha256 of the effective config. */
-    configHash: string;
-    /** Git SHA the harness was run from. */
-    commitSha: string;
-    /** End-to-end wall-clock duration in milliseconds. */
-    wallMs: number;
-    /** Time spent queued before execution started, if known. */
-    queueMs?: number;
-    /** Total USD cost. Mandatory — runs without a cost number are
-     *  unbounded by definition and must not be admitted into the gate.
-     *  `0` is retained as the compatibility sentinel for an uncaptured amount;
-     *  inspect `costProvenance` before treating it as observed. */
-    costUsd: number;
-    /** Observed, model-priced estimate, or genuinely uncaptured USD amount.
-     *  Optional only so existing serialized RunRecords remain valid. */
-    costProvenance?: RunCostProvenance;
-    /** Token usage breakdown. */
-    tokenUsage: RunTokenUsage;
-    /** Judge-side metadata, if a judge was used. */
-    judgeMetadata?: RunJudgeMetadata;
-    /** Per-split scores + raw bag. */
-    outcome: RunOutcome;
-    /** Canonical, cross-agent failure class drawn from the shared
-     *  `FAILURE_CLASSES` taxonomy. This is the aggregation key that makes
-     *  "which failure dominates across the whole fleet" answerable in ONE
-     *  vocabulary — every agent classifies against the same enum. Producers
-     *  set it via the substrate classifier; leave unset only when the failure
-     *  genuinely can't be classified. */
-    failureClass?: FailureClass;
-    /** Free-form domain-specific failure detail, scoped UNDER `failureClass`
-     *  (e.g. failureClass='tool_recovery_failure', failureMode='forge_build_unsatisfied').
-     *  The within-agent drill-down; `failureClass` is the cross-agent key. */
-    failureMode?: string;
-    /** Which split this run was drawn from. */
-    splitTag: RunSplitTag;
-    /**
-     * Stable scenario identifier the run was scored against. Optional for
-     * backwards compatibility, but **strongly recommended**: every primitive
-     * that pairs runs by scenario (preferences, paired stats, BT tournament)
-     * keys on this. The campaign artifact populates it canonically; legacy
-     * runs without it fall back to inference from `outcome.raw.scenario_id`
-     * or `experimentId`.
-     */
-    scenarioId?: string;
-    /**
-     * Canonical identity for the agent profile cell that produced this row:
-     * profile artifact hash plus optional harness/model/prompt/reporting
-     * dimensions. Use `agentProfile.cellId` to group persona sweeps and
-     * longitudinal reports by the complete source profile, not by a loose
-     * candidate label or opaque config hash.
-     */
-    agentProfile?: AgentProfileCell;
-}
-
-type PrimeIntellectJson = null | boolean | number | string | PrimeIntellectJson[] | {
-    [key: string]: PrimeIntellectJson;
-};
-interface PrimeIntellectMessage {
-    role: 'system' | 'user' | 'assistant' | 'tool';
-    content: string;
-}
-interface PrimeIntellectScenario {
-    id: string;
-    prompt: string | readonly PrimeIntellectMessage[];
-    answer?: string;
-    requiredSubstrings?: readonly string[];
-    split?: string;
-    info?: Record<string, PrimeIntellectJson>;
-}
-interface PrimeIntellectDatasetRow {
-    id: string;
-    prompt: PrimeIntellectMessage[];
-    answer?: string;
-    required_substrings?: string[];
-    split?: string;
-    info: Record<string, PrimeIntellectJson>;
-}
-type PrimeIntellectScenarioMap = ReadonlyMap<string, PrimeIntellectScenario> | Record<string, PrimeIntellectScenario> | readonly PrimeIntellectScenario[];
-interface PrimeIntellectRowsFromRunRecordsOptions {
-    records: readonly RunRecord[];
-    scenarios: PrimeIntellectScenarioMap;
-    requireScorable?: boolean;
-    includeFailed?: boolean;
-}
-interface PrimeIntellectEnvironmentPackageInput {
-    name: string;
-    version?: string;
-    description?: string;
-    tags?: readonly string[];
-    moduleName?: string;
-    rows: readonly PrimeIntellectDatasetRow[];
-    runRecords?: readonly RunRecord[];
-    systemPrompt?: string;
-    verifiersVersion?: string;
-    dependencies?: readonly string[];
-    readme?: string;
-}
-interface PrimeIntellectPackageFile {
-    path: string;
-    content: string;
-}
-interface PrimeIntellectPackageManifest {
-    schemaVersion: 1;
-    name: string;
-    moduleName: string;
-    version: string;
-    rowCount: number;
-    runRecordCount: number;
-    artifactKinds: readonly ['environment', 'dataset', 'run_records'];
-}
-interface PrimeIntellectEnvironmentPackage {
-    files: PrimeIntellectPackageFile[];
-    manifest: PrimeIntellectPackageManifest;
-}
-declare class PrimeIntellectBridgeError extends Error {
-    constructor(message: string);
-}
-declare function primeIntellectRowsFromRunRecords(options: PrimeIntellectRowsFromRunRecordsOptions): PrimeIntellectDatasetRow[];
-declare function buildPrimeIntellectEnvironmentPackage(input: PrimeIntellectEnvironmentPackageInput): PrimeIntellectEnvironmentPackage;
-declare function writePrimeIntellectEnvironmentPackage(root: string, input: PrimeIntellectEnvironmentPackageInput): Promise<PrimeIntellectEnvironmentPackage>;
-
-export { PrimeIntellectBridgeError, type PrimeIntellectDatasetRow, type PrimeIntellectEnvironmentPackage, type PrimeIntellectEnvironmentPackageInput, type PrimeIntellectJson, type PrimeIntellectMessage, type PrimeIntellectPackageFile, type PrimeIntellectPackageManifest, type PrimeIntellectRowsFromRunRecordsOptions, type PrimeIntellectScenario, type PrimeIntellectScenarioMap, buildPrimeIntellectEnvironmentPackage, primeIntellectRowsFromRunRecords, writePrimeIntellectEnvironmentPackage };

package/dist/primeintellect/index.js

@@ -1,348 +0,0 @@
-import {
-  validateRunRecord
-} from "../chunk-S3UZOQ5Y.js";
-import "../chunk-MA6HLL3S.js";
-import "../chunk-XJYR7XFV.js";
-import "../chunk-VSMTAMNK.js";
-import "../chunk-ONWEPEDO.js";
-import "../chunk-PZ5AY32C.js";
-
-// src/primeintellect/index.ts
-import { mkdir, writeFile } from "fs/promises";
-import { dirname, join } from "path";
-var PrimeIntellectBridgeError = class extends Error {
-  constructor(message) {
-    super(message);
-    this.name = "PrimeIntellectBridgeError";
-  }
-};
-function primeIntellectRowsFromRunRecords(options) {
-  const scenarioById = normalizeScenarios(options.scenarios);
-  const requireScorable = options.requireScorable ?? true;
-  const includeFailed = options.includeFailed ?? true;
-  const rows = [];
-  for (const record of options.records) {
-    const validRecord = validateRunRecord(record);
-    if (!includeFailed && validRecord.failureMode) continue;
-    const scenarioId = validRecord.scenarioId;
-    if (!scenarioId) {
-      throw new PrimeIntellectBridgeError(
-        `RunRecord ${validRecord.runId} is missing scenarioId; PrimeIntellect rows need stable task ids`
-      );
-    }
-    const scenario = scenarioById.get(scenarioId);
-    if (!scenario) {
-      throw new PrimeIntellectBridgeError(
-        `RunRecord ${validRecord.runId} references unknown scenarioId ${JSON.stringify(scenarioId)}`
-      );
-    }
-    const hasAnswer = typeof scenario.answer === "string" && scenario.answer.trim().length > 0;
-    const hasRequiredSubstrings = (scenario.requiredSubstrings?.length ?? 0) > 0;
-    if (requireScorable && !hasAnswer && !hasRequiredSubstrings) {
-      throw new PrimeIntellectBridgeError(
-        `Scenario ${scenario.id} has no answer or requiredSubstrings; generated environments would always score 0`
-      );
-    }
-    const score = validRecord.outcome.holdoutScore ?? validRecord.outcome.searchScore ?? null;
-    const split = scenario.split ?? validRecord.splitTag;
-    const tangleInfo = {
-      run_id: validRecord.runId,
-      experiment_id: validRecord.experimentId,
-      candidate_id: validRecord.candidateId,
-      scenario_id: scenarioId,
-      split,
-      score,
-      cost_usd: validRecord.costUsd,
-      wall_ms: validRecord.wallMs,
-      failure_mode: validRecord.failureMode ?? null,
-      model: validRecord.model,
-      commit_sha: validRecord.commitSha
-    };
-    rows.push({
-      id: `${scenarioId}:${validRecord.runId}`,
-      prompt: normalizePrompt(scenario.prompt),
-      ...hasAnswer ? { answer: scenario.answer } : {},
-      ...hasRequiredSubstrings ? { required_substrings: [...scenario.requiredSubstrings ?? []] } : {},
-      split,
-      info: {
-        ...scenario.info ?? {},
-        tangle: tangleInfo
-      }
-    });
-  }
-  return rows;
-}
-function buildPrimeIntellectEnvironmentPackage(input) {
-  if (input.rows.length === 0) {
-    throw new PrimeIntellectBridgeError("cannot build a PrimeIntellect environment with zero rows");
-  }
-  for (const record of input.runRecords ?? []) {
-    validateRunRecord(record);
-  }
-  const moduleName = input.moduleName ?? toPythonModuleName(input.name);
-  if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(moduleName)) {
-    throw new PrimeIntellectBridgeError(
-      `moduleName ${JSON.stringify(moduleName)} is not a valid Python module name`
-    );
-  }
-  const version = input.version ?? "0.1.0";
-  const description = input.description ?? "PrimeIntellect Verifiers environment exported from Tangle runs.";
-  const verifiersVersion = input.verifiersVersion ?? ">=0.2.0,<0.3.0";
-  const dependencies = unique([
-    "datasets",
-    `verifiers${verifiersVersion}`,
-    ...input.dependencies ?? []
-  ]);
-  const tags = unique(["tangle", "agent-eval", ...input.tags ?? []]);
-  const manifest = {
-    schemaVersion: 1,
-    name: input.name,
-    moduleName,
-    version,
-    rowCount: input.rows.length,
-    runRecordCount: input.runRecords?.length ?? 0,
-    artifactKinds: ["environment", "dataset", "run_records"]
-  };
-  const files = [
-    {
-      path: "pyproject.toml",
-      content: renderPyproject({
-        name: input.name,
-        version,
-        description,
-        dependencies,
-        tags
-      })
-    },
-    {
-      path: "README.md",
-      content: input.readme ?? renderReadme({ name: input.name, description })
-    },
-    {
-      path: `${moduleName}.py`,
-      content: renderEnvironmentModule({
-        systemPrompt: input.systemPrompt
-      })
-    },
-    {
-      path: "data/dataset.jsonl",
-      content: toJsonl(input.rows)
-    },
-    {
-      path: "data/run_records.jsonl",
-      content: toJsonl(input.runRecords ?? [])
-    },
-    {
-      path: "tangle-primeintellect-manifest.json",
-      content: `${stableJson(manifest)}
-`
-    }
-  ];
-  return { files, manifest };
-}
-async function writePrimeIntellectEnvironmentPackage(root, input) {
-  const pkg = buildPrimeIntellectEnvironmentPackage(input);
-  await Promise.all(
-    pkg.files.map(async (file) => {
-      const target = join(root, file.path);
-      await mkdir(dirname(target), { recursive: true });
-      await writeFile(target, file.content, "utf8");
-    })
-  );
-  return pkg;
-}
-function normalizeScenarios(scenarios) {
-  const map = /* @__PURE__ */ new Map();
-  if (scenarios instanceof Map) {
-    for (const [id, scenario] of scenarios.entries()) {
-      map.set(id, { ...scenario, id: scenario.id ?? id });
-    }
-    return map;
-  }
-  const values = Array.isArray(scenarios) ? scenarios : Object.values(scenarios);
-  for (const scenario of values) {
-    if (!scenario.id) {
-      throw new PrimeIntellectBridgeError("every PrimeIntellect scenario needs an id");
-    }
-    if (map.has(scenario.id)) {
-      throw new PrimeIntellectBridgeError(`duplicate PrimeIntellect scenario id ${scenario.id}`);
-    }
-    map.set(scenario.id, scenario);
-  }
-  return map;
-}
-function normalizePrompt(prompt) {
-  if (typeof prompt === "string") {
-    return [{ role: "user", content: prompt }];
-  }
-  return prompt.map((message) => ({ role: message.role, content: message.content }));
-}
-function toPythonModuleName(name) {
-  return name.trim().toLowerCase().replace(/[^a-z0-9_]+/g, "_").replace(/^_+|_+$/g, "").replace(/^[0-9]/, "_$&");
-}
-function unique(values) {
-  return [...new Set(values)];
-}
-function toJsonl(values) {
-  if (values.length === 0) return "";
-  return `${values.map((value) => stableJson(value)).join("\n")}
-`;
-}
-function stableJson(value) {
-  return JSON.stringify(sortJson(value));
-}
-function sortJson(value) {
-  if (Array.isArray(value)) return value.map(sortJson);
-  if (value === null || typeof value !== "object") return value;
-  return Object.fromEntries(
-    Object.entries(value).sort(([a], [b]) => a.localeCompare(b)).map(([key, nested]) => [key, sortJson(nested)])
-  );
-}
-function renderPyproject(input) {
-  return `[project]
-name = ${tomlString(input.name)}
-version = ${tomlString(input.version)}
-description = ${tomlString(input.description)}
-readme = "README.md"
-requires-python = ">=3.11"
-license = "MIT"
-keywords = ${tomlArray(input.tags)}
-dependencies = ${tomlArray(input.dependencies)}
-
-[build-system]
-requires = ["hatchling"]
-build-backend = "hatchling.build"
-
-[tool.hatch.build]
-include = ["*.py", "README.md", "data/*.jsonl", "tangle-primeintellect-manifest.json"]
-
-[tool.uv]
-prerelease = "allow"
-
-[tool.verifiers.eval]
-num_examples = 5
-rollouts_per_example = 3
-`;
-}
-function renderReadme(input) {
-  return `# ${input.name}
-
-${input.description}
-
-Generated by \`@tangle-network/agent-eval/primeintellect\` from validated Tangle \`RunRecord\` rows.
-
-## Files
-
-- \`data/dataset.jsonl\`: prompts and scoring references for PrimeIntellect Verifiers.
-- \`data/run_records.jsonl\`: original Tangle run rows for provenance and analysis.
-- \`tangle-primeintellect-manifest.json\`: export metadata.
-
-## Local check
-
-\`\`\`sh
-uv pip install --prerelease=allow -e .
-uv run vf-eval ${input.name}
-\`\`\`
-
-## Upload
-
-\`\`\`sh
-prime login
-prime env push
-\`\`\`
-`;
-}
-function renderEnvironmentModule(input) {
-  const systemPrompt = input.systemPrompt === void 0 ? "None" : pyString(input.systemPrompt);
-  return `import json
-import re
-from pathlib import Path
-
-from datasets import Dataset
-import verifiers as vf
-
-
-DATA_PATH = Path(__file__).parent / "data" / "dataset.jsonl"
-DEFAULT_SYSTEM_PROMPT = ${systemPrompt}
-
-
-def _load_rows(dataset_path=None, dataset_split=None):
-    path = Path(dataset_path) if dataset_path is not None else DATA_PATH
-    rows = []
-    with path.open("r", encoding="utf-8") as handle:
-        for line in handle:
-            row = json.loads(line)
-            if dataset_split is None or row.get("split") == dataset_split:
-                rows.append(row)
-    if not rows:
-        raise ValueError(f"No dataset rows found in {path} for split={dataset_split!r}")
-    return rows
-
-
-def _message_text(message):
-    if isinstance(message, dict):
-        content = message.get("content", "")
-    else:
-        content = getattr(message, "content", "")
-    if isinstance(content, list):
-        return " ".join(
-            str(part.get("text", part)) if isinstance(part, dict) else str(part)
-            for part in content
-        )
-    return str(content)
-
-
-def _completion_text(completion):
-    if isinstance(completion, str):
-        return completion
-    if isinstance(completion, list):
-        return "\\n".join(_message_text(message) for message in completion)
-    return _message_text(completion)
-
-
-def _normalize(text):
-    return re.sub(r"\\s+", " ", str(text).strip().lower())
-
-
-async def tangle_reward(completion, answer=None, required_substrings=None, **kwargs):
-    response = _normalize(_completion_text(completion))
-    scores = []
-    if answer:
-        expected = _normalize(answer)
-        scores.append(1.0 if expected == response or expected in response else 0.0)
-    if required_substrings:
-        required = [_normalize(item) for item in required_substrings]
-        hits = sum(1 for item in required if item and item in response)
-        scores.append(hits / len(required))
-    return max(scores) if scores else 0.0
-
-
-def build_dataset(dataset_path=None, dataset_split=None):
-    return Dataset.from_list(_load_rows(dataset_path=dataset_path, dataset_split=dataset_split))
-
-
-def load_environment(dataset_path=None, dataset_split=None, system_prompt=DEFAULT_SYSTEM_PROMPT):
-    rubric = vf.Rubric(funcs=[tangle_reward])
-    return vf.SingleTurnEnv(
-        dataset=lambda: build_dataset(dataset_path=dataset_path, dataset_split=dataset_split),
-        system_prompt=system_prompt,
-        rubric=rubric,
-    )
-`;
-}
-function tomlString(value) {
-  return JSON.stringify(value);
-}
-function tomlArray(values) {
-  return `[${values.map(tomlString).join(", ")}]`;
-}
-function pyString(value) {
-  return JSON.stringify(value);
-}
-export {
-  PrimeIntellectBridgeError,
-  buildPrimeIntellectEnvironmentPackage,
-  primeIntellectRowsFromRunRecords,
-  writePrimeIntellectEnvironmentPackage
-};
-//# sourceMappingURL=index.js.map