@tangle-network/agent-eval 0.129.0 → 0.130.0

package/dist/schema-Cef2cFmb.d.ts

@@ -0,0 +1,408 @@
+//#region src/rollout/schema.d.ts
+/**
+ * `tangle.rollout.v1` — THE canonical rollout serialization, owned by
+ * agent-eval. One JSONL line per agent invocation (a solo eval run, a
+ * supervisor episode, a worker session, a proposer shot, a judge call, an
+ * analyst pass), labeled with its task/split coordinates and a single
+ * scalar reward, carrying the FULL message transcript inline.
+ *
+ * This schema is the reconciliation of two prior producers:
+ *   - agent-eval's RunRecord-joined rollout rows (PR #410): identity,
+ *     provenance hashes, the realness gate travelling into the reward,
+ *     trace-derived steps.
+ *   - the bench rollout-ledger (agent-runtime PR #591): the wire shape —
+ *     role, task.split/rep, parent_rollout_id, policy provenance, capture
+ *     provenance, inline canonical chat-with-tools messages.
+ * Where the two conflicted, RunRecord-derived semantics won; the wire
+ * field names follow the ledger (snake_case). See `docs/rollout.md` for
+ * the field-by-field decision table.
+ *
+ * Messages are inlined — never referenced — because every harness store a
+ * rollout can be recovered from is mutable or garbage-collected. A line
+ * must stay a complete training/eval example on its own.
+ *
+ * `outcome.reward` is THE single scalar (null = no verdict exists — a
+ * labeled gap, never 0). `outcome.realness_gated` is the anti-Goodhart
+ * flag: a gated line must never export as a positive training example.
+ *
+ * That last sentence is enforced here, by `validateRolloutLine`, not merely
+ * documented. Validating `reward` and `realness_gated` independently — each a
+ * well-typed field, their COMBINATION unchecked — is what let a line claiming
+ * `{reward: 0.95, realness_gated: true}` validate clean and walk into every
+ * training export. The relationship between the two IS the invariant, so it is
+ * checked where every other structural claim about a line is checked.
+ *
+ * The invariant is about the OUTCOME, not about one field of it. Zeroing
+ * `reward` while `outcome.metrics` still carried the per-layer scores that
+ * reward was computed from exported the gamed signal anyway, in the dict the
+ * verifiers format reads as its per-rubric scores. So `gateGamedOutcome`
+ * transforms the whole outcome once, at `assertMinted` — the funnel every
+ * minted line passes — and the reward-bearing components are relocated to
+ * `provenance.gated_evidence`, which no exporter projects.
+ *
+ * WHICH checks each door applies is not decided in this file. `./gate-checks`
+ * owns the canonical list and the total per-entry-point policy; the three doors
+ * below (`validateRolloutLine`, `assertRewardGate`, `assertMinted`) each call
+ * `gateErrors` with their declared policy, so a check added to that list applies
+ * here without anyone editing this file, and a check deliberately skipped has to
+ * name itself there.
+ */
+declare const ROLLOUT_SCHEMA = "tangle.rollout.v1";
+/** `agent` = a solo evaluation run (no multi-agent topology). */
+type RolloutRole = 'agent' | 'supervisor' | 'worker' | 'proposer' | 'judge' | 'analyst';
+declare const ROLLOUT_ROLES: readonly RolloutRole[];
+/** Split vocabulary follows `RunRecord.splitTag`, extended with `canary`. */
+type RolloutSplit = 'search' | 'dev' | 'holdout' | 'canary';
+declare const ROLLOUT_SPLITS: readonly RolloutSplit[];
+/** Splits that may ship in training exports. Everything else is fail-closed excluded. */
+declare const TRAINABLE_SPLITS: readonly RolloutSplit[];
+declare function isTrainableSplit(split: RolloutSplit): boolean;
+/** 'mint' = joined live from RunRecord + trace by `mintRolloutRows`. */
+type RolloutCapture = 'mint' | 'settle-time' | 'backfill';
+declare const ROLLOUT_CAPTURES: readonly RolloutCapture[];
+type ChatRole = 'system' | 'user' | 'assistant' | 'tool';
+declare const CHAT_ROLES: readonly ChatRole[];
+interface ChatToolCall {
+  id: string;
+  type: 'function';
+  function: {
+    name: string;
+    /** JSON-encoded argument object, exactly as the model emitted it. */
+    arguments: string;
+  };
+}
+interface ChatMessage {
+  role: ChatRole;
+  content: string | null;
+  /** Reasoning/thinking channel where the harness captured it (full fidelity). */
+  reasoning_content?: string;
+  tool_calls?: ChatToolCall[];
+  /** Required on role:"tool" — the ChatToolCall this result answers. */
+  tool_call_id?: string;
+  name?: string;
+  /**
+   * Harbor ATIF `is_copied_context` (RFC 0001 rule 7): this turn was COPIED IN
+   * from another trajectory's context, not produced by the agent on this line.
+   * The RFC makes excluding it from SFT a MUST, and `toSftRows` does — training
+   * on it teaches the model to author text it never authored, and credits this
+   * run for another one's work. Absent = false (authored here).
+   */
+  is_copied_context?: boolean;
+}
+interface ToolDef {
+  type: 'function';
+  function: {
+    name: string;
+    description?: string;
+    parameters?: Record<string, unknown>;
+  };
+}
+/**
+ * Compact trace-span projection (llm/tool step) carried alongside the
+ * conversation when the line was minted from a trace. Optional: lines
+ * recovered from harness stores have no span structure.
+ */
+interface RolloutStep {
+  kind: string;
+  name: string;
+  /** llm: last-message summary · tool: stringified args. Scrubbed. */
+  input?: string;
+  /** llm: output text · tool: stringified result. Scrubbed. */
+  output?: string;
+  status?: 'ok' | 'error';
+  durationMs?: number;
+  /**
+   * LLM inferences this span represents. 0 = deterministic dispatch with no
+   * model call — distinct from absent, which means the producer did not track it.
+   */
+  llm_call_count?: number;
+  /** Exact prompt tokenization. Removes the ambiguity of re-tokenizing text at train time. */
+  prompt_token_ids?: number[];
+  /** Exact completion tokenization; aligns index-wise with `logprobs`. */
+  completion_token_ids?: number[];
+  /**
+   * Per-completion-token log probabilities under the sampling policy. Required
+   * for off-policy correction (importance weighting) when the rollout was
+   * generated by a policy other than the one being trained.
+   */
+  logprobs?: number[];
+}
+interface RolloutTask {
+  /** Benchmark/suite id (e.g. "swe-bench-verified") or the experiment id. */
+  suite: string;
+  instance_id: string;
+  split: RolloutSplit;
+  /** Sampling seed the campaign pinned; null = not recorded. */
+  seed: number | null;
+  /** Replicate index (0-based). */
+  rep: number;
+}
+interface RolloutPolicy {
+  /** Harness that drove the invocation (e.g. "opencode", "claude", "pi-loops"). */
+  harness: string | null;
+  harness_version: string | null;
+  model: string | null;
+  provider: string | null;
+  /** Commit of the agent profile / candidate under evaluation. */
+  profile_commit: string | null;
+  /** sha256 of the effective prompt (post-steering), when recorded. */
+  prompt_hash?: string | null;
+  /** sha256 of the effective run config, when recorded. */
+  config_hash?: string | null;
+  /** Canonical agent-profile cell identity, when the run carries one. */
+  agent_profile_cell_id?: string | null;
+  /** Sampling params (temperature, top_p, max_tokens…); null = not recorded. */
+  sampling: Record<string, unknown> | null;
+}
+interface RolloutOutcome {
+  /**
+   * THE single scalar training signal — the official verdict.
+   * null = no verdict exists for this invocation (a labeled gap, never 0).
+   */
+  reward: number | null;
+  /** Where the reward came from (judge id; "/inherited" = parent episode's). */
+  reward_source: string | null;
+  /** Raw judge verdict record, verbatim. */
+  verdict: unknown;
+  /** Everything that is NOT the scalar reward. */
+  metrics: Record<string, unknown>;
+  is_completed: boolean;
+  is_truncated: boolean;
+  error: string | null;
+  /**
+   * Anti-Goodhart flag from `RunRecord.outcome.realness.gated`: the run faked
+   * its success signal. `true` requires `reward` to be 0 or null — the
+   * validator rejects the line otherwise — and the line never qualifies for
+   * SFT. Required on the wire: a line that does not state the flag does not
+   * validate, so no producer can dodge the gate by omitting it.
+   *
+   * `true` ALSO requires `metrics` to be empty and `verdict` to be null: the
+   * numbers the reward was computed from are relocated to
+   * `provenance.gated_evidence` by `gateGamedOutcome`. See that function for
+   * why zeroing the scalar alone was not enough.
+   */
+  realness_gated: boolean;
+  /**
+   * Whether an authenticity SCREEN ever RAN on this reward — a different claim
+   * from `realness_gated`, which is the screen's VERDICT.
+   *
+   * `realness_gated: false` reads as "we looked and nothing fired". A producer
+   * with no screen at all was emitting exactly that, so a never-screened reward
+   * was indistinguishable on the wire from a screened-clean one, and the whole
+   * anti-Goodhart apparatus silently treated the first as the second. The two
+   * claims are now separable:
+   *
+   *   - `true`   — a screen ran; `realness_gated` is its verdict.
+   *   - `false`  — the producer declares it HAS no screen (`unscreenedRewardFields`).
+   *                `assertMinted` REFUSES such a line when its reward is above
+   *                zero: an unscreened positive reward is precisely the signal
+   *                the gate exists to qualify, and nothing has qualified it.
+   *   - absent   — not stated. Pre-unification ledgers land here, as does a
+   *                `RunRecord` carrying no `outcome.realness` at all. Absent is
+   *                read as "unknown", never as `false` (which would refuse most
+   *                of the existing corpus) and never as `true`.
+   */
+  realness_screened?: boolean;
+}
+interface RolloutCostBlock {
+  usd: number | null;
+  tokens_in: number | null;
+  tokens_out: number | null;
+  tokens_reasoning: number | null;
+  cache_read: number | null;
+  cache_write: number | null;
+  wall_s: number | null;
+  /**
+   * Total LLM inferences across the invocation (ATIF `llm_call_count`,
+   * aggregated). Optional and additive: absent = not tracked, never 0.
+   */
+  llm_call_count?: number | null;
+}
+interface RolloutArtifacts {
+  patch_path: string | null;
+  run_dir: string | null;
+  /** Source-of-truth transcript pointer (session id / jsonl path) for audit. */
+  transcript_ref: string | null;
+}
+/**
+ * The reward-bearing half of a GATED line's outcome, moved off `outcome` and
+ * parked here verbatim. Diagnostics, never training input — see
+ * `gateGamedOutcome`.
+ */
+interface GatedEvidence {
+  /** `outcome.metrics` exactly as the producer measured it. */
+  metrics?: Record<string, unknown>;
+  /** `outcome.verdict` verbatim — the judge record that claimed the success. */
+  verdict?: unknown;
+  /**
+   * The per-step fields `tangle.rollout.v1` does not declare, parked here when
+   * the gate projected `steps[]` down to the schema's own key set.
+   *
+   * A per-step reward is training signal exactly like the scalar, and `steps`
+   * rides through `toRewardRows` verbatim — so a gated line was shipping its
+   * step-level credit assignment at full value beside a `reward` of 0.
+   */
+  steps?: unknown;
+}
+interface RolloutProvenance {
+  captured_at: string;
+  capture: RolloutCapture;
+  /**
+   * Why this line is incomplete. Required when `messages` is empty (the
+   * transcript could not be recovered); also set by interchange importers to
+   * name a MISSING LABEL — an imported trajectory carries no verdict, so
+   * `outcome.reward` is null and this says why.
+   */
+  gap?: string;
+  /**
+   * Present only on a realness-gated line: the outcome fields the gate
+   * relocated, kept so an auditor can still see WHY the run was gated and what
+   * it claimed. Deliberately OUTSIDE `outcome`, because every training exporter
+   * reads `outcome` and none reads `provenance`.
+   */
+  gated_evidence?: GatedEvidence;
+}
+interface RolloutLine {
+  schema: typeof ROLLOUT_SCHEMA;
+  rollout_id: string;
+  /** Spawning invocation within the same episode (worker → supervisor). */
+  parent_rollout_id: string | null;
+  run_id: string;
+  /** Logical experiment grouping from `RunRecord.experimentId`; null = not recorded. */
+  experiment_id: string | null;
+  /** Stable candidate identity from `RunRecord.candidateId`; null = not recorded. */
+  candidate_id: string | null;
+  /** Improvement-loop generation (-1 = baseline); null = not an improvement loop. */
+  generation: number | null;
+  /** Improvement-loop candidate index (-1 = baseline); null = not an improvement loop. */
+  candidate_index: number | null;
+  role: RolloutRole;
+  task: RolloutTask;
+  policy: RolloutPolicy;
+  /** Full transcript, inline. [] = gap line (see provenance.gap). */
+  messages: ChatMessage[];
+  tool_defs: ToolDef[];
+  /** Trace-span projections, when minted from a trace. */
+  steps?: RolloutStep[];
+  outcome: RolloutOutcome;
+  cost: RolloutCostBlock;
+  artifacts: RolloutArtifacts;
+  provenance: RolloutProvenance;
+}
+/**
+ * THE anti-Goodhart gate, applied to the WHOLE outcome as a TRANSFORMATION.
+ *
+ * Two prior rounds enforced the gate as a CHECK ON ONE FIELD at N call sites,
+ * and each round the next reward-bearing field leaked. The one that shipped:
+ * `mintRolloutRows` bulk-copied `RunRecord.outcome.raw` into `outcome.metrics`
+ * with no gate, so a gated run exported `reward: 0` (correct) while the
+ * deterministic per-layer scores that reward was COMPUTED FROM — the
+ * `layer.*` keys `rl/verifiable-reward.ts` calls the RL training signal —
+ * shipped at 1.0, in the top-level `metrics` dict of the Prime Intellect
+ * verifiers format, which IS that format's per-rubric score dict. `verdict`
+ * leaks the same way into `toRftItem`'s `reference.verdict`, where a grader
+ * author reads `resolved: true` off a run that faked it.
+ *
+ * So the rule is no longer "zero the field we remembered". It is: if the gate
+ * fired, the outcome that leaves here carries NOTHING positive that was derived
+ * from the reward, whichever field a present or future exporter decides to
+ * read. `reward` is already forced to 0 upstream (`trainingReward`) and
+ * REJECTED here if it is not; `metrics` and `verdict` are relocated.
+ *
+ * WHERE they go, and why relocation rather than deletion: zeroing destroys the
+ * audit trail that shows why the run was gated and what it claimed, which is
+ * the row an auditor most wants and the labeled example a gaming DETECTOR
+ * trains on. `provenance.gated_evidence` keeps every byte, at a path no
+ * training exporter reads — all four release configs and every `rl/exporters`
+ * shape project from `outcome`, `messages`, `cost` and `task`; none projects
+ * `provenance`. Auditability preserved, training signal removed, and a future
+ * exporter that reads a field nobody thought of is safe by construction because
+ * the field is empty rather than because the exporter remembered to check.
+ *
+ * Idempotent: a second application finds nothing left to move and returns the
+ * line unchanged, so re-minting a line read back off a ledger cannot clobber
+ * the evidence it already carries.
+ */
+declare function gateGamedOutcome(line: RolloutLine): RolloutLine;
+declare function validateRolloutLine(value: unknown): string[];
+declare function assertRolloutLine(value: unknown, context?: string): asserts value is RolloutLine;
+declare function isRolloutLine(value: unknown): value is RolloutLine;
+/**
+ * Phantom property. `declare const` means it exists only in the type system:
+ * nothing is written at runtime, so a branded line still serializes to exactly
+ * the same JSON as a plain one.
+ */
+declare const MINTED_ROLLOUT: unique symbol;
+/**
+ * A minted outcome states the gate verdict — it is not allowed to stay silent —
+ * and, when that verdict is `true`, carries nothing else the reward was derived
+ * from (`gateGamedOutcome` has run).
+ */
+interface MintedRolloutOutcome extends RolloutOutcome {
+  realness_gated: boolean;
+}
+/**
+ * A `RolloutLine` whose reward has been checked against the anti-Goodhart
+ * invariant. The type every training-data exporter takes.
+ *
+ * Why a brand and not just the interface: `RolloutLine` is structural, so any
+ * hand-built object literal of the right shape IS one — which is how a line
+ * declaring `{reward: 0.95, realness_gated: true}` reached the exporters
+ * despite them "only accepting a minted line". The phantom symbol makes the
+ * type nominal: it cannot be produced by writing an object literal, only by
+ * `mintRolloutRows` (which applies the gate), `readRolloutLedger` (which
+ * validates every line off disk), or an explicit, greppable `assertMinted`.
+ *
+ * Belt and braces on purpose. The brand closes first-party call sites at
+ * COMPILE time; `validateRolloutLine` closes data arriving at RUNTIME (ledger
+ * files, foreign imports, JSON from another process) where types are absent.
+ * Neither alone is enough.
+ *
+ * Assignable to `RolloutLine` in one direction only: readers, analysis, and
+ * the ledger writer keep taking the plain type.
+ */
+type MintedRolloutLine = Omit<RolloutLine, 'outcome'> & {
+  readonly [MINTED_ROLLOUT]: true;
+  outcome: MintedRolloutOutcome;
+};
+/**
+ * Promote a line to the type the training exporters accept, applying the
+ * anti-Goodhart gate to the WHOLE outcome on the way through. THE escape hatch
+ * — grep `assertMinted` to enumerate every place a line enters the training
+ * path without coming from mint or a ledger.
+ *
+ * The gate runs HERE, once, rather than at each producer, because this is the
+ * single funnel every minted line passes: `mintRolloutRows` calls it,
+ * `readRolloutLedger` calls it per line off disk, `scrubLines` calls it on the
+ * way out of a release, and a hand-built line has no other door. One
+ * transformation at the funnel means an already-published ledger holding a
+ * gated line with populated `metrics` is RE-GATED when it is read, instead of
+ * being rejected (which would make every such artifact unreadable) or trusted
+ * (which is the leak). Three steps, in this order:
+ *
+ *   1. VALIDATE the schema.
+ *   2. REFUSE every check `GATE_POLICIES.assertMinted` marks `enforce` — today
+ *      the reward relationship (which stays a REJECTION: a caller claiming
+ *      `{reward: 0.95, realness_gated: true}` is a producer defect and must fail
+ *      loudly, since laundering it into `reward: 0` here would hide the
+ *      producer) and a positive reward the producer declared it never screened.
+ *   3. TRANSFORM the one check that policy marks `repair` — relocate the
+ *      reward's components off `outcome` (`gateGamedOutcome`), so no exporter
+ *      can leak them whichever field it reads.
+ *
+ * Step 2 enumerates nothing by hand: a check added to `GATE_CHECKS` is enforced
+ * here the moment its disposition in that policy says so.
+ *
+ * Also normalizes the optional wire flag to an explicit boolean.
+ * `realness_gated` is absent on pre-unification ledgers and absent means "not
+ * flagged" per the schema, so filling it in states a claim the line was already
+ * making, and makes the flag readable on every published row instead of most of
+ * them. `realness_screened` is NOT filled in: absent means "unknown", and
+ * inventing either value there would be the same overclaim this round removed.
+ */
+declare function assertMinted(value: unknown, context?: string): MintedRolloutLine;
+/** `assertMinted` over a batch, naming the offending index in the error. */
+declare function assertMintedLines(values: readonly unknown[], context?: string): MintedRolloutLine[];
+//#endregion
+export { isTrainableSplit as A, TRAINABLE_SPLITS as C, assertRolloutLine as D, assertMintedLines as E, gateGamedOutcome as O, RolloutTask as S, assertMinted as T, RolloutPolicy as _, GatedEvidence as a, RolloutSplit as b, ROLLOUT_CAPTURES as c, ROLLOUT_SPLITS as d, RolloutArtifacts as f, RolloutOutcome as g, RolloutLine as h, ChatToolCall as i, validateRolloutLine as j, isRolloutLine as k, ROLLOUT_ROLES as l, RolloutCostBlock as m, ChatMessage as n, MintedRolloutLine as o, RolloutCapture as p, ChatRole as r, MintedRolloutOutcome as s, CHAT_ROLES as t, ROLLOUT_SCHEMA as u, RolloutProvenance as v, ToolDef as w, RolloutStep as x, RolloutRole as y };
+//# sourceMappingURL=schema-Cef2cFmb.d.ts.map

package/dist/schema-Cef2cFmb.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema-Cef2cFmb.d.ts","names":[],"sources":["../src/rollout/schema.ts"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;cAmDa;;KAGD;cACC,wBAAwB;;KAUzB;cACC,yBAAyB;;cAEzB,2BAA2B;iBAExB,iBAAiB,OAAO;;KAK5B;cACC,2BAA2B;KAM5B;cACC,qBAAqB;UAEjB;EACf;EACA;EACA;IACE;;IAEA;;;UAIa;EACf,MAAM;EACN;;EAEA;EACA,aAAa;;EAEb;EACA;;;;;;;;EAQA;;UAGe;EACf;EACA;IACE;IACA;IACA,aAAa;;;;;;;;UASA;EACf;EACA;;EAEA;;EAEA;EACA;EACA;;;;;EAUA;;EAEA;;EAEA;;;;;;EAMA;;UAOe;;EAEf;EACA;EACA,OAAO;;EAEP;;EAEA;;UAGe;;EAEf;EACA;EACA;EACA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA,UAAU;;UAGK;;;;;EAKf;;EAEA;;EAEA;;EAEA,SAAS;EACT;EACA;EACA;;;;;;;;;;;;;EAaA;;;;;;;;;;;;;;;;;;;;;EAqBA;;UAGe;EACf;EACA;EACA;EACA;EACA;EACA;EACA;;;;;EAKA;;UAGe;EACf;EACA;;EAEA;;;;;;;UAQe;;EAEf,UAAU;;EAEV;;;;;;;;;EASA;;UAGe;EACf;EACA,SAAS;;;;;;;EAOT;;;;;;;EAOA,iBAAiB;;UAGF;EACf,eAAe;EACf;;EAEA;EACA;;EAEA;;EAEA;;EAEA;;EAEA;EACA,MAAM;EACN,MAAM;EACN,QAAQ;;EAER,UAAU;EACV,WAAW;;EAEX,QAAQ;EACR,SAAS;EACT,MAAM;EACN,WAAW;EACX,YAAY;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBA6GE,iBAAiB,MAAM,cAAc;iBAqDrC,oBAAoB;iBAiNpB,kBACd,gBACA,2BACS,SAAS;iBAOJ,cAAc,iBAAiB,SAAS;;;;;;cAa1C;;;;;;UAOG,6BAA6B;EAC5C;;;;;;;;;;;;;;;;;;;;;;KAuBU,oBAAoB,KAAK;YACzB;EACV,SAAS;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAsCK,aAAa,gBAAgB,mBAA2B;;iBAiBxD,kBACd,4BACA,mBACC"}