@tangle-network/agent-eval 1.0.14 → 1.0.16

package/dist/{exporters-ByLcILYU.js.map → exporters-BKZLt1Px.js.map}

@@ -1 +1 @@
-{"version":3,"file":"exporters-ByLcILYU.js","names":[],"sources":["../src/rollout/exporters.ts"],"sourcesContent":["/**\n * Pure exporters over `tangle.rollout.v1` lines → the training-data shapes\n * the improvement loops feed:\n *   - SFT chat JSONL           (clean trainable successes, {messages, metadata})\n *   - reward rows              (every scored line, success or failure, with steps)\n *   - Prime Intellect verifiers RolloutOutput (prompt/completion split + reward)\n *   - OpenAI RFT items         (prompt turns + verdict reference fields)\n *\n * All exporters are pure functions of the lines — filtering (never train on\n * holdout, reward thresholds, the realness gate) happens HERE, on inline\n * labels, no joins.\n *\n * Every exporter takes `MintedRolloutLine[]`, not `RolloutLine[]`: the reward\n * on a minted line has been checked against the anti-Goodhart invariant, and\n * the brand is what stops a hand-built object literal claiming a positive\n * reward on a gamed run from being handed to an exporter that copies it\n * verbatim into training data.\n */\n\nimport { isLicenseEligible, type LicenseClass } from './license'\nimport type {\n  ChatMessage,\n  MintedRolloutLine,\n  RolloutLine,\n  RolloutSplit,\n  RolloutStep,\n  ToolDef,\n} from './schema'\nimport { assertRewardGate } from './schema'\n\n// ---------------------------------------------------------------------------\n// The realness claims every emitted row carries.\n// ---------------------------------------------------------------------------\n\n/**\n * The gate's two claims, which travel TOGETHER on every emitted row.\n *\n * `realness_gated` alone is ambiguous, and the ambiguity is exploitable:\n * `false` reads as \"we screened it and nothing fired\", so a producer that has no\n * screen at all emitted rows indistinguishable from screened-clean ones, and\n * every consumer of the published dataset read them as clean. The second field\n * is what separates the two claims, and it only removes the ambiguity if it\n * reaches the WIRE — for a round it existed on `RolloutOutcome` and on no\n * exported row shape at all, which left the published rows exactly as ambiguous\n * as before.\n *\n * So there is one helper and every row shape spreads it. A row that states one\n * claim without the other is not constructible by copying the pattern, and\n * `exporters.test.ts` walks every emitted shape to prove none does.\n */\nexport interface RealnessLabels {\n  /** The screen's VERDICT: the run faked its success signal. */\n  realness_gated: boolean\n  /**\n   * Whether a screen RAN at all. `true` = it ran, so `realness_gated` is its\n   * verdict. `false` = the producer declares it has none. `null` = not stated\n   * (pre-unification producers), which is \"unknown\" and never \"clean\".\n   */\n  realness_screened: boolean | null\n}\n\nfunction realnessLabels(line: MintedRolloutLine): RealnessLabels {\n  return {\n    realness_gated: line.outcome.realness_gated === true,\n    // `?? null` rather than `?? false`: absent means the producer never stated\n    // it, and `false` is a load-bearing claim (\"no screen exists\") that would be\n    // an invention here.\n    realness_screened: line.outcome.realness_screened ?? null,\n  }\n}\n\n// ---------------------------------------------------------------------------\n// (a) SFT chat JSONL\n// ---------------------------------------------------------------------------\n\nexport interface TrainingExportOptions {\n  /** Include held-out evaluation data in training output. Default false. */\n  allowHeldOutTrainingData?: boolean\n  /** Require reward to be strictly greater than this value. Default 0. */\n  minimumQualityExclusive?: number\n  /**\n   * Explicit split selection, replacing the default trainable-split rule.\n   * Use this only when producing a deliberately named non-training slice\n   * (e.g. `['holdout']` for a holdout-only evaluation bundle).\n   */\n  splitFilter?: RolloutSplit[]\n  /** Re-cut by license class: only these classes export. Default: every class. */\n  allowLicenseClasses?: readonly LicenseClass[]\n}\n\n/**\n * What a signed-signal exporter (verifiers, RFT) does with lines that are not\n * clean trainable successes — realness-gated lines above all.\n *\n *   - 'exclude'       — the default, the same fail-closed policy as every\n *                       other training export: positive, completed,\n *                       non-gated rows on a trainable split.\n *   - 'zero-and-flag' — keep them, at their non-positive (or null) reward,\n *                       with `RealnessLabels` on the row. The dataset release\n *                       sets this per `FORMAT_GATE_DISPOSITION`: in these\n *                       formats the reward is a signed learning signal, so a\n *                       gamed trajectory at reward 0 is a correct negative,\n *                       and dropping it would bias the negative population\n *                       toward honest failures and leave a trainer no example\n *                       of gaming being penalized. The split policy is NOT\n *                       relaxed: held-out lines still need the named opt-in.\n *\n * SFT deliberately has no such option — an SFT row is an imitation target and\n * a gamed trajectory must never appear in one at any weight.\n */\ntype GatedLineDisposition = 'exclude' | 'zero-and-flag'\n\ninterface SignedSignalExportOptions extends TrainingExportOptions {\n  /** Disposition for non-trainable lines. Default 'exclude'. */\n  gatedLines?: GatedLineDisposition\n}\n\nexport type SftExportOptions = TrainingExportOptions\n\nexport interface SftRow {\n  messages: ChatMessage[]\n  metadata: {\n    rollout_id: string\n    run_id: string\n    candidate_id: string | null\n    instance_id: string\n    reward: number\n  } & RealnessLabels\n}\n\n/**\n * Supervised fine-tune rows: the completed conversation of each qualifying\n * line. Fail-closed filters: trainable split only (never holdout/canary unless\n * an explicit `splitFilter` names a non-training slice),\n * reward strictly above `minimumQualityExclusive` (default 0), realness-gated\n * lines never qualify, gap lines carry no trainable content, and\n * copied-context turns are dropped from the transcript (Harbor ATIF RFC 0001\n * rule 7 — see `ChatMessage.is_copied_context`).\n *\n * `realness_gated` is therefore always `false` on an emitted row. It is carried\n * anyway: an SFT row is a pure imitation target, so the row states its realness\n * claims instead of making the reader know the format's policy, and carrying\n * both flags on all four shapes is what lets the release accounting measure\n * every config with one rule rather than skipping the one whose row shape\n * happened to omit the field.\n */\nexport function toSftRows(lines: MintedRolloutLine[], options: SftExportOptions = {}): SftRow[] {\n  for (const line of lines) assertRewardGate(line, 'SFT export')\n  return lines\n    .filter((line) => isTrainingLineEligible(line, options) && line.messages.length > 0)\n    .map((line) => ({\n      // Dropped, not kept-and-masked: a copied-context turn was authored by\n      // another agent, and an SFT trainer has no notion of \"present but not a\n      // target\" — every message it is handed is something to imitate.\n      messages: line.messages.filter((message) => message.is_copied_context !== true),\n      metadata: {\n        rollout_id: line.rollout_id,\n        run_id: line.run_id,\n        candidate_id: line.candidate_id ?? null,\n        instance_id: line.task.instance_id,\n        reward: line.outcome.reward as number,\n        ...realnessLabels(line),\n      },\n    }))\n    .filter((row) => row.messages.length > 0)\n}\n\n// ---------------------------------------------------------------------------\n// (b) Reward rows\n// ---------------------------------------------------------------------------\n\nexport interface RewardRow {\n  /** First user turn — the task prompt. */\n  prompt: string\n  steps: RolloutStep[]\n  reward: number\n  metadata: {\n    rollout_id: string\n    run_id: string\n    candidate_id: string | null\n    instance_id: string\n    split: RolloutSplit\n    /**\n     * The anti-Goodhart verdict AND whether a screen produced it, carried on the\n     * row rather than left implicit in a reward of 0. Zeroing alone is lossy: it\n     * makes a run that FAKED its success indistinguishable from one that\n     * honestly failed, so a consumer can neither drop the gamed population nor\n     * mine it (a labeled gamed trajectory is the training signal for a gaming\n     * detector). `realness_gated` alone is lossy the same way in the other\n     * direction — see `RealnessLabels`.\n     */\n  } & RealnessLabels\n}\n\n/**\n * Reward-labeled rows for completed, positive-quality training runs.\n */\nexport function toRewardRows(\n  lines: MintedRolloutLine[],\n  options: TrainingExportOptions = {},\n): RewardRow[] {\n  for (const line of lines) assertRewardGate(line, 'reward-row export')\n  return lines\n    .filter(\n      (line) =>\n        isTrainingLineEligible(line, options) &&\n        line.messages.some(\n          (message) =>\n            message.role === 'user' &&\n            typeof message.content === 'string' &&\n            message.content.length > 0,\n        ),\n    )\n    .map((line) => ({\n      prompt: line.messages.find((m) => m.role === 'user')?.content ?? '',\n      steps: line.steps ?? [],\n      reward: line.outcome.reward as number,\n      metadata: {\n        rollout_id: line.rollout_id,\n        run_id: line.run_id,\n        candidate_id: line.candidate_id ?? null,\n        instance_id: line.task.instance_id,\n        split: line.task.split,\n        ...realnessLabels(line),\n      },\n    }))\n}\n\n// ---------------------------------------------------------------------------\n// (c) Prime Intellect verifiers RolloutOutput\n// ---------------------------------------------------------------------------\n\nexport interface VerifiersTokenUsage {\n  input_tokens: number | null\n  output_tokens: number | null\n  reasoning_tokens: number | null\n  cache_read_tokens: number | null\n  cache_write_tokens: number | null\n}\n\nexport interface VerifiersRolloutOutput {\n  /** Messages through the last turn BEFORE the first assistant turn. */\n  prompt: ChatMessage[]\n  /** The first assistant turn onward — what the policy produced. */\n  completion: ChatMessage[]\n  reward: number | null\n  metrics: Record<string, unknown>\n  tool_defs: ToolDef[]\n  token_usage: VerifiersTokenUsage\n  info: {\n    task: RolloutLine['task']\n    policy: RolloutLine['policy']\n    rollout_id: string\n    run_id: string\n    experiment_id: string | null\n    candidate_id: string | null\n    generation: number | null\n    candidate_index: number | null\n    role: RolloutLine['role']\n  } & RealnessLabels\n}\n\n/** Index of the first assistant turn; messages.length when none exists. */\nfunction firstAssistantIndex(messages: ChatMessage[]): number {\n  const index = messages.findIndex((m) => m.role === 'assistant')\n  return index === -1 ? messages.length : index\n}\n\nfunction assertCompleteProjection(line: MintedRolloutLine, context: string): void {\n  if (line.provenance.lossy_projection === true) {\n    throw new Error(`${context}: rollout ${line.rollout_id} has a lossy projection`)\n  }\n}\n\nexport function toVerifiersRolloutOutput(line: MintedRolloutLine): VerifiersRolloutOutput {\n  assertRewardGate(line, 'verifiers export')\n  assertCompleteProjection(line, 'verifiers export')\n  const split = firstAssistantIndex(line.messages)\n  return {\n    prompt: line.messages.slice(0, split),\n    completion: line.messages.slice(split),\n    reward: line.outcome.reward,\n    metrics: line.outcome.metrics,\n    tool_defs: line.tool_defs,\n    token_usage: {\n      input_tokens: line.cost.tokens_in,\n      output_tokens: line.cost.tokens_out,\n      reasoning_tokens: line.cost.tokens_reasoning,\n      cache_read_tokens: line.cost.cache_read,\n      cache_write_tokens: line.cost.cache_write,\n    },\n    info: {\n      task: line.task,\n      policy: line.policy,\n      rollout_id: line.rollout_id,\n      run_id: line.run_id,\n      experiment_id: line.experiment_id ?? null,\n      candidate_id: line.candidate_id ?? null,\n      generation: line.generation,\n      candidate_index: line.candidate_index,\n      role: line.role,\n      ...realnessLabels(line),\n    },\n  }\n}\n\nexport function toVerifiersRolloutOutputs(\n  lines: MintedRolloutLine[],\n  options: SignedSignalExportOptions = {},\n): VerifiersRolloutOutput[] {\n  if (options.gatedLines === 'zero-and-flag') {\n    return lines\n      .filter(\n        (line) =>\n          isSplitEligible(line, options) &&\n          line.provenance.lossy_projection !== true &&\n          line.messages.length > 0,\n      )\n      .map(toVerifiersRolloutOutput)\n  }\n  return lines\n    .filter(\n      (line) =>\n        isTrainingLineEligible(line, options) &&\n        firstAssistantIndex(line.messages) > 0 &&\n        firstAssistantIndex(line.messages) < line.messages.length,\n    )\n    .map(toVerifiersRolloutOutput)\n}\n\n// ---------------------------------------------------------------------------\n// (d) OpenAI RFT items\n// ---------------------------------------------------------------------------\n\nexport interface RftItem {\n  /** Prompt turns only — the graded completion is re-sampled during RFT. */\n  messages: ChatMessage[]\n  /** Verdict/label fields the grader references as item.reference.* */\n  reference: {\n    reward: number | null\n    reward_source: string | null\n    verdict: unknown\n    instance_id: string\n    suite: string\n    split: RolloutSplit\n    rollout_id: string\n  } & RealnessLabels\n}\n\nexport function toRftItem(line: MintedRolloutLine): RftItem {\n  assertRewardGate(line, 'RFT export')\n  assertCompleteProjection(line, 'RFT export')\n  const split = firstAssistantIndex(line.messages)\n  return {\n    messages: line.messages.slice(0, split),\n    reference: {\n      reward: line.outcome.reward,\n      reward_source: line.outcome.reward_source,\n      verdict: line.outcome.verdict,\n      instance_id: line.task.instance_id,\n      suite: line.task.suite,\n      split: line.task.split,\n      rollout_id: line.rollout_id,\n      ...realnessLabels(line),\n    },\n  }\n}\n\n/** RFT needs a real prompt: lines whose transcript starts with prompt turns. */\nexport function toRftItems(\n  lines: MintedRolloutLine[],\n  options: SignedSignalExportOptions = {},\n): RftItem[] {\n  return lines\n    .filter(\n      (line) =>\n        (options.gatedLines === 'zero-and-flag'\n          ? isSplitEligible(line, options) && line.provenance.lossy_projection !== true\n          : isTrainingLineEligible(line, options)) &&\n        line.messages.length > 0 &&\n        firstAssistantIndex(line.messages) > 0,\n    )\n    .map(toRftItem)\n}\n\n// ---------------------------------------------------------------------------\n// Serialization — one JSON object per line, the interchange format for\n// every export. `tangle.rollout.v1` lines and export rows alike.\n// ---------------------------------------------------------------------------\n\nexport function toJsonl(rows: ReadonlyArray<unknown>): string {\n  return rows.map((r) => JSON.stringify(r)).join('\\n') + (rows.length ? '\\n' : '')\n}\n\n/**\n * The split half of the training policy alone: `search` is trainable, held-out\n * needs the named opt-in, `dev` and `canary` never ship. This is the ONE check\n * `'zero-and-flag'` does not relax — a gated line is shipped as a labeled\n * negative, not as a licence to train on evaluation data. An explicit\n * `splitFilter` names a non-training slice instead of the default rule.\n *\n * Exported as the single implementation of that rule: `rl/exporters` applies\n * it on its line paths too, so the two waists cannot drift on which splits are\n * trainable.\n */\nexport function isSplitEligible(\n  line: RolloutLine,\n  options: Pick<TrainingExportOptions, 'allowHeldOutTrainingData' | 'splitFilter'>,\n): boolean {\n  if (options.splitFilter !== undefined) return options.splitFilter.includes(line.task.split)\n  if (line.task.split === 'search') return true\n  return line.task.split === 'holdout' && options.allowHeldOutTrainingData === true\n}\n\n/**\n * The whole fail-closed training policy for one line: scored, positive above\n * the floor, completed, not gated, in the requested license classes, on an eligible split. Shared with\n * `rl/exporters` so its text-lookup adapter cannot drift from this rule.\n */\nexport function isTrainingLineEligible(\n  line: RolloutLine,\n  options: TrainingExportOptions,\n): line is RolloutLine & { outcome: RolloutLine['outcome'] & { reward: number } } {\n  const minimumQualityExclusive = options.minimumQualityExclusive ?? 0\n  if (!Number.isFinite(minimumQualityExclusive)) {\n    throw new Error('minimumQualityExclusive must be finite')\n  }\n\n  const reward = line.outcome.reward\n  if (reward === null) return false\n  if (!Number.isFinite(reward)) {\n    throw new Error(`training reward for rollout \"${line.rollout_id}\" must be finite`)\n  }\n  if (reward <= minimumQualityExclusive) return false\n  if (!line.outcome.is_completed || line.outcome.is_truncated || line.outcome.error !== null) {\n    return false\n  }\n  if (line.outcome.realness_gated === true) return false\n  if (line.provenance.lossy_projection === true) return false\n  if (!isLicenseEligible(line, options.allowLicenseClasses)) return false\n  return isSplitEligible(line, options)\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AA6DA,SAAS,eAAe,MAAyC;CAC/D,OAAO;EACL,gBAAgB,KAAK,QAAQ,mBAAmB;EAIhD,mBAAmB,KAAK,QAAQ,qBAAqB;CACvD;AACF;;;;;;;;;;;;;;;;;AA6EA,SAAgB,UAAU,OAA4B,UAA4B,CAAC,GAAa;CAC9F,KAAK,MAAM,QAAQ,OAAO,iBAAiB,MAAM,YAAY;CAC7D,OAAO,MACJ,QAAQ,SAAS,uBAAuB,MAAM,OAAO,KAAK,KAAK,SAAS,SAAS,CAAC,CAAC,CACnF,KAAK,UAAU;EAId,UAAU,KAAK,SAAS,QAAQ,YAAY,QAAQ,sBAAsB,IAAI;EAC9E,UAAU;GACR,YAAY,KAAK;GACjB,QAAQ,KAAK;GACb,cAAc,KAAK,gBAAgB;GACnC,aAAa,KAAK,KAAK;GACvB,QAAQ,KAAK,QAAQ;GACrB,GAAG,eAAe,IAAI;EACxB;CACF,EAAE,CAAC,CACF,QAAQ,QAAQ,IAAI,SAAS,SAAS,CAAC;AAC5C;;;;AAgCA,SAAgB,aACd,OACA,UAAiC,CAAC,GACrB;CACb,KAAK,MAAM,QAAQ,OAAO,iBAAiB,MAAM,mBAAmB;CACpE,OAAO,MACJ,QACE,SACC,uBAAuB,MAAM,OAAO,KACpC,KAAK,SAAS,MACX,YACC,QAAQ,SAAS,UACjB,OAAO,QAAQ,YAAY,YAC3B,QAAQ,QAAQ,SAAS,CAC7B,CACJ,CAAC,CACA,KAAK,UAAU;EACd,QAAQ,KAAK,SAAS,MAAM,MAAM,EAAE,SAAS,MAAM,CAAC,EAAE,WAAW;EACjE,OAAO,KAAK,SAAS,CAAC;EACtB,QAAQ,KAAK,QAAQ;EACrB,UAAU;GACR,YAAY,KAAK;GACjB,QAAQ,KAAK;GACb,cAAc,KAAK,gBAAgB;GACnC,aAAa,KAAK,KAAK;GACvB,OAAO,KAAK,KAAK;GACjB,GAAG,eAAe,IAAI;EACxB;CACF,EAAE;AACN;;AAqCA,SAAS,oBAAoB,UAAiC;CAC5D,MAAM,QAAQ,SAAS,WAAW,MAAM,EAAE,SAAS,WAAW;CAC9D,OAAO,UAAU,KAAK,SAAS,SAAS;AAC1C;AAEA,SAAS,yBAAyB,MAAyB,SAAuB;CAChF,IAAI,KAAK,WAAW,qBAAqB,MACvC,MAAM,IAAI,MAAM,GAAG,QAAQ,YAAY,KAAK,WAAW,wBAAwB;AAEnF;AAEA,SAAgB,yBAAyB,MAAiD;CACxF,iBAAiB,MAAM,kBAAkB;CACzC,yBAAyB,MAAM,kBAAkB;CACjD,MAAM,QAAQ,oBAAoB,KAAK,QAAQ;CAC/C,OAAO;EACL,QAAQ,KAAK,SAAS,MAAM,GAAG,KAAK;EACpC,YAAY,KAAK,SAAS,MAAM,KAAK;EACrC,QAAQ,KAAK,QAAQ;EACrB,SAAS,KAAK,QAAQ;EACtB,WAAW,KAAK;EAChB,aAAa;GACX,cAAc,KAAK,KAAK;GACxB,eAAe,KAAK,KAAK;GACzB,kBAAkB,KAAK,KAAK;GAC5B,mBAAmB,KAAK,KAAK;GAC7B,oBAAoB,KAAK,KAAK;EAChC;EACA,MAAM;GACJ,MAAM,KAAK;GACX,QAAQ,KAAK;GACb,YAAY,KAAK;GACjB,QAAQ,KAAK;GACb,eAAe,KAAK,iBAAiB;GACrC,cAAc,KAAK,gBAAgB;GACnC,YAAY,KAAK;GACjB,iBAAiB,KAAK;GACtB,MAAM,KAAK;GACX,GAAG,eAAe,IAAI;EACxB;CACF;AACF;AAEA,SAAgB,0BACd,OACA,UAAqC,CAAC,GACZ;CAC1B,IAAI,QAAQ,eAAe,iBACzB,OAAO,MACJ,QACE,SACC,gBAAgB,MAAM,OAAO,KAC7B,KAAK,WAAW,qBAAqB,QACrC,KAAK,SAAS,SAAS,CAC3B,CAAC,CACA,IAAI,wBAAwB;CAEjC,OAAO,MACJ,QACE,SACC,uBAAuB,MAAM,OAAO,KACpC,oBAAoB,KAAK,QAAQ,IAAI,KACrC,oBAAoB,KAAK,QAAQ,IAAI,KAAK,SAAS,MACvD,CAAC,CACA,IAAI,wBAAwB;AACjC;AAqBA,SAAgB,UAAU,MAAkC;CAC1D,iBAAiB,MAAM,YAAY;CACnC,yBAAyB,MAAM,YAAY;CAC3C,MAAM,QAAQ,oBAAoB,KAAK,QAAQ;CAC/C,OAAO;EACL,UAAU,KAAK,SAAS,MAAM,GAAG,KAAK;EACtC,WAAW;GACT,QAAQ,KAAK,QAAQ;GACrB,eAAe,KAAK,QAAQ;GAC5B,SAAS,KAAK,QAAQ;GACtB,aAAa,KAAK,KAAK;GACvB,OAAO,KAAK,KAAK;GACjB,OAAO,KAAK,KAAK;GACjB,YAAY,KAAK;GACjB,GAAG,eAAe,IAAI;EACxB;CACF;AACF;;AAGA,SAAgB,WACd,OACA,UAAqC,CAAC,GAC3B;CACX,OAAO,MACJ,QACE,UACE,QAAQ,eAAe,kBACpB,gBAAgB,MAAM,OAAO,KAAK,KAAK,WAAW,qBAAqB,OACvE,uBAAuB,MAAM,OAAO,MACxC,KAAK,SAAS,SAAS,KACvB,oBAAoB,KAAK,QAAQ,IAAI,CACzC,CAAC,CACA,IAAI,SAAS;AAClB;AAOA,SAAgB,QAAQ,MAAsC;CAC5D,OAAO,KAAK,KAAK,MAAM,KAAK,UAAU,CAAC,CAAC,CAAC,CAAC,KAAK,IAAI,KAAK,KAAK,SAAS,OAAO;AAC/E;;;;;;;;;;;;AAaA,SAAgB,gBACd,MACA,SACS;CACT,IAAI,QAAQ,gBAAgB,KAAA,GAAW,OAAO,QAAQ,YAAY,SAAS,KAAK,KAAK,KAAK;CAC1F,IAAI,KAAK,KAAK,UAAU,UAAU,OAAO;CACzC,OAAO,KAAK,KAAK,UAAU,aAAa,QAAQ,6BAA6B;AAC/E;;;;;;AAOA,SAAgB,uBACd,MACA,SACgF;CAChF,MAAM,0BAA0B,QAAQ,2BAA2B;CACnE,IAAI,CAAC,OAAO,SAAS,uBAAuB,GAC1C,MAAM,IAAI,MAAM,wCAAwC;CAG1D,MAAM,SAAS,KAAK,QAAQ;CAC5B,IAAI,WAAW,MAAM,OAAO;CAC5B,IAAI,CAAC,OAAO,SAAS,MAAM,GACzB,MAAM,IAAI,MAAM,gCAAgC,KAAK,WAAW,iBAAiB;CAEnF,IAAI,UAAU,yBAAyB,OAAO;CAC9C,IAAI,CAAC,KAAK,QAAQ,gBAAgB,KAAK,QAAQ,gBAAgB,KAAK,QAAQ,UAAU,MACpF,OAAO;CAET,IAAI,KAAK,QAAQ,mBAAmB,MAAM,OAAO;CACjD,IAAI,KAAK,WAAW,qBAAqB,MAAM,OAAO;CACtD,IAAI,CAAC,kBAAkB,MAAM,QAAQ,mBAAmB,GAAG,OAAO;CAClE,OAAO,gBAAgB,MAAM,OAAO;AACtC"}
+{"version":3,"file":"exporters-BKZLt1Px.js","names":[],"sources":["../src/rollout/exporters.ts"],"sourcesContent":["/**\n * Pure exporters over `tangle.rollout.v1` lines → the training-data shapes\n * the improvement loops feed:\n *   - SFT chat JSONL           (clean trainable successes, {messages, metadata})\n *   - reward rows              (every scored line, success or failure, with steps)\n *   - Prime Intellect verifiers RolloutOutput (prompt/completion split + reward)\n *   - OpenAI RFT items         (prompt turns + verdict reference fields)\n *\n * All exporters are pure functions of the lines — filtering (never train on\n * holdout, reward thresholds, the realness gate) happens HERE, on inline\n * labels, no joins.\n *\n * Every exporter takes `MintedRolloutLine[]`, not `RolloutLine[]`: the reward\n * on a minted line has been checked against the anti-Goodhart invariant, and\n * the brand is what stops a hand-built object literal claiming a positive\n * reward on a gamed run from being handed to an exporter that copies it\n * verbatim into training data.\n */\n\nimport { isLicenseEligible, type LicenseClass } from './license'\nimport type {\n  ChatMessage,\n  MintedRolloutLine,\n  RolloutLine,\n  RolloutSplit,\n  RolloutStep,\n  ToolDef,\n} from './schema'\nimport { assertRewardGate } from './schema'\n\n// ---------------------------------------------------------------------------\n// The realness claims every emitted row carries.\n// ---------------------------------------------------------------------------\n\n/**\n * The gate's two claims, which travel TOGETHER on every emitted row.\n *\n * `realness_gated` alone is ambiguous, and the ambiguity is exploitable:\n * `false` reads as \"we screened it and nothing fired\", so a producer that has no\n * screen at all emitted rows indistinguishable from screened-clean ones, and\n * every consumer of the published dataset read them as clean. The second field\n * is what separates the two claims, and it only removes the ambiguity if it\n * reaches the WIRE — for a round it existed on `RolloutOutcome` and on no\n * exported row shape at all, which left the published rows exactly as ambiguous\n * as before.\n *\n * So there is one helper and every row shape spreads it. A row that states one\n * claim without the other is not constructible by copying the pattern, and\n * `exporters.test.ts` walks every emitted shape to prove none does.\n */\nexport interface RealnessLabels {\n  /** The screen's VERDICT: the run faked its success signal. */\n  realness_gated: boolean\n  /**\n   * Whether a screen RAN at all. `true` = it ran, so `realness_gated` is its\n   * verdict. `false` = the producer declares it has none. `null` = not stated\n   * (pre-unification producers), which is \"unknown\" and never \"clean\".\n   */\n  realness_screened: boolean | null\n}\n\nfunction realnessLabels(line: MintedRolloutLine): RealnessLabels {\n  return {\n    realness_gated: line.outcome.realness_gated === true,\n    // `?? null` rather than `?? false`: absent means the producer never stated\n    // it, and `false` is a load-bearing claim (\"no screen exists\") that would be\n    // an invention here.\n    realness_screened: line.outcome.realness_screened ?? null,\n  }\n}\n\n// ---------------------------------------------------------------------------\n// (a) SFT chat JSONL\n// ---------------------------------------------------------------------------\n\nexport interface TrainingExportOptions {\n  /** Include held-out evaluation data in training output. Default false. */\n  allowHeldOutTrainingData?: boolean\n  /** Require reward to be strictly greater than this value. Default 0. */\n  minimumQualityExclusive?: number\n  /**\n   * Explicit split selection, replacing the default trainable-split rule.\n   * Use this only when producing a deliberately named non-training slice\n   * (e.g. `['holdout']` for a holdout-only evaluation bundle).\n   */\n  splitFilter?: RolloutSplit[]\n  /** Re-cut by license class: only these classes export. Default: every class. */\n  allowLicenseClasses?: readonly LicenseClass[]\n}\n\n/**\n * What a signed-signal exporter (verifiers, RFT) does with lines that are not\n * clean trainable successes — realness-gated lines above all.\n *\n *   - 'exclude'       — the default, the same fail-closed policy as every\n *                       other training export: positive, completed,\n *                       non-gated rows on a trainable split.\n *   - 'zero-and-flag' — keep them, at their non-positive (or null) reward,\n *                       with `RealnessLabels` on the row. The dataset release\n *                       sets this per `FORMAT_GATE_DISPOSITION`: in these\n *                       formats the reward is a signed learning signal, so a\n *                       gamed trajectory at reward 0 is a correct negative,\n *                       and dropping it would bias the negative population\n *                       toward honest failures and leave a trainer no example\n *                       of gaming being penalized. The split policy is NOT\n *                       relaxed: held-out lines still need the named opt-in.\n *\n * SFT deliberately has no such option — an SFT row is an imitation target and\n * a gamed trajectory must never appear in one at any weight.\n */\ntype GatedLineDisposition = 'exclude' | 'zero-and-flag'\n\ninterface SignedSignalExportOptions extends TrainingExportOptions {\n  /** Disposition for non-trainable lines. Default 'exclude'. */\n  gatedLines?: GatedLineDisposition\n}\n\nexport type SftExportOptions = TrainingExportOptions\n\nexport interface SftRow {\n  messages: ChatMessage[]\n  metadata: {\n    rollout_id: string\n    run_id: string\n    candidate_id: string | null\n    instance_id: string\n    reward: number\n  } & RealnessLabels\n}\n\n/**\n * Supervised fine-tune rows: the completed conversation of each qualifying\n * line. Fail-closed filters: trainable split only (never holdout/canary unless\n * an explicit `splitFilter` names a non-training slice),\n * reward strictly above `minimumQualityExclusive` (default 0), realness-gated\n * lines never qualify, gap lines carry no trainable content, and\n * copied-context turns are dropped from the transcript (Harbor ATIF RFC 0001\n * rule 7 — see `ChatMessage.is_copied_context`).\n *\n * `realness_gated` is therefore always `false` on an emitted row. It is carried\n * anyway: an SFT row is a pure imitation target, so the row states its realness\n * claims instead of making the reader know the format's policy, and carrying\n * both flags on all four shapes is what lets the release accounting measure\n * every config with one rule rather than skipping the one whose row shape\n * happened to omit the field.\n */\nexport function toSftRows(lines: MintedRolloutLine[], options: SftExportOptions = {}): SftRow[] {\n  for (const line of lines) assertRewardGate(line, 'SFT export')\n  return lines\n    .filter((line) => isTrainingLineEligible(line, options) && line.messages.length > 0)\n    .map((line) => ({\n      // Dropped, not kept-and-masked: a copied-context turn was authored by\n      // another agent, and an SFT trainer has no notion of \"present but not a\n      // target\" — every message it is handed is something to imitate.\n      messages: line.messages.filter((message) => message.is_copied_context !== true),\n      metadata: {\n        rollout_id: line.rollout_id,\n        run_id: line.run_id,\n        candidate_id: line.candidate_id ?? null,\n        instance_id: line.task.instance_id,\n        reward: line.outcome.reward as number,\n        ...realnessLabels(line),\n      },\n    }))\n    .filter((row) => row.messages.length > 0)\n}\n\n// ---------------------------------------------------------------------------\n// (b) Reward rows\n// ---------------------------------------------------------------------------\n\nexport interface RewardRow {\n  /** First user turn — the task prompt. */\n  prompt: string\n  steps: RolloutStep[]\n  reward: number\n  metadata: {\n    rollout_id: string\n    run_id: string\n    candidate_id: string | null\n    instance_id: string\n    split: RolloutSplit\n    /**\n     * The anti-Goodhart verdict AND whether a screen produced it, carried on the\n     * row rather than left implicit in a reward of 0. Zeroing alone is lossy: it\n     * makes a run that FAKED its success indistinguishable from one that\n     * honestly failed, so a consumer can neither drop the gamed population nor\n     * mine it (a labeled gamed trajectory is the training signal for a gaming\n     * detector). `realness_gated` alone is lossy the same way in the other\n     * direction — see `RealnessLabels`.\n     */\n  } & RealnessLabels\n}\n\n/**\n * Reward-labeled rows for completed, positive-quality training runs.\n */\nexport function toRewardRows(\n  lines: MintedRolloutLine[],\n  options: TrainingExportOptions = {},\n): RewardRow[] {\n  for (const line of lines) assertRewardGate(line, 'reward-row export')\n  return lines\n    .filter(\n      (line) =>\n        isTrainingLineEligible(line, options) &&\n        line.messages.some(\n          (message) =>\n            message.role === 'user' &&\n            typeof message.content === 'string' &&\n            message.content.length > 0,\n        ),\n    )\n    .map((line) => ({\n      prompt: line.messages.find((m) => m.role === 'user')?.content ?? '',\n      steps: line.steps ?? [],\n      reward: line.outcome.reward as number,\n      metadata: {\n        rollout_id: line.rollout_id,\n        run_id: line.run_id,\n        candidate_id: line.candidate_id ?? null,\n        instance_id: line.task.instance_id,\n        split: line.task.split,\n        ...realnessLabels(line),\n      },\n    }))\n}\n\n// ---------------------------------------------------------------------------\n// (c) Prime Intellect verifiers RolloutOutput\n// ---------------------------------------------------------------------------\n\nexport interface VerifiersTokenUsage {\n  input_tokens: number | null\n  output_tokens: number | null\n  reasoning_tokens: number | null\n  cache_read_tokens: number | null\n  cache_write_tokens: number | null\n}\n\nexport interface VerifiersRolloutOutput {\n  /** Messages through the last turn BEFORE the first assistant turn. */\n  prompt: ChatMessage[]\n  /** The first assistant turn onward — what the policy produced. */\n  completion: ChatMessage[]\n  reward: number | null\n  metrics: Record<string, unknown>\n  tool_defs: ToolDef[]\n  token_usage: VerifiersTokenUsage\n  info: {\n    task: RolloutLine['task']\n    policy: RolloutLine['policy']\n    rollout_id: string\n    run_id: string\n    experiment_id: string | null\n    candidate_id: string | null\n    generation: number | null\n    candidate_index: number | null\n    role: RolloutLine['role']\n  } & RealnessLabels\n}\n\n/** Index of the first assistant turn; messages.length when none exists. */\nfunction firstAssistantIndex(messages: ChatMessage[]): number {\n  const index = messages.findIndex((m) => m.role === 'assistant')\n  return index === -1 ? messages.length : index\n}\n\nfunction assertCompleteProjection(line: MintedRolloutLine, context: string): void {\n  if (line.provenance.lossy_projection === true) {\n    throw new Error(`${context}: rollout ${line.rollout_id} has a lossy projection`)\n  }\n}\n\nexport function toVerifiersRolloutOutput(line: MintedRolloutLine): VerifiersRolloutOutput {\n  assertRewardGate(line, 'verifiers export')\n  assertCompleteProjection(line, 'verifiers export')\n  const split = firstAssistantIndex(line.messages)\n  return {\n    prompt: line.messages.slice(0, split),\n    completion: line.messages.slice(split),\n    reward: line.outcome.reward,\n    metrics: line.outcome.metrics,\n    tool_defs: line.tool_defs,\n    token_usage: {\n      input_tokens: line.cost.tokens_in,\n      output_tokens: line.cost.tokens_out,\n      reasoning_tokens: line.cost.tokens_reasoning,\n      cache_read_tokens: line.cost.cache_read,\n      cache_write_tokens: line.cost.cache_write,\n    },\n    info: {\n      task: line.task,\n      policy: line.policy,\n      rollout_id: line.rollout_id,\n      run_id: line.run_id,\n      experiment_id: line.experiment_id ?? null,\n      candidate_id: line.candidate_id ?? null,\n      generation: line.generation,\n      candidate_index: line.candidate_index,\n      role: line.role,\n      ...realnessLabels(line),\n    },\n  }\n}\n\nexport function toVerifiersRolloutOutputs(\n  lines: MintedRolloutLine[],\n  options: SignedSignalExportOptions = {},\n): VerifiersRolloutOutput[] {\n  if (options.gatedLines === 'zero-and-flag') {\n    return lines\n      .filter(\n        (line) =>\n          isSplitEligible(line, options) &&\n          line.provenance.lossy_projection !== true &&\n          line.messages.length > 0,\n      )\n      .map(toVerifiersRolloutOutput)\n  }\n  return lines\n    .filter(\n      (line) =>\n        isTrainingLineEligible(line, options) &&\n        firstAssistantIndex(line.messages) > 0 &&\n        firstAssistantIndex(line.messages) < line.messages.length,\n    )\n    .map(toVerifiersRolloutOutput)\n}\n\n// ---------------------------------------------------------------------------\n// (d) OpenAI RFT items\n// ---------------------------------------------------------------------------\n\nexport interface RftItem {\n  /** Prompt turns only — the graded completion is re-sampled during RFT. */\n  messages: ChatMessage[]\n  /** Verdict/label fields the grader references as item.reference.* */\n  reference: {\n    reward: number | null\n    reward_source: string | null\n    verdict: unknown\n    instance_id: string\n    suite: string\n    split: RolloutSplit\n    rollout_id: string\n  } & RealnessLabels\n}\n\nexport function toRftItem(line: MintedRolloutLine): RftItem {\n  assertRewardGate(line, 'RFT export')\n  assertCompleteProjection(line, 'RFT export')\n  const split = firstAssistantIndex(line.messages)\n  return {\n    messages: line.messages.slice(0, split),\n    reference: {\n      reward: line.outcome.reward,\n      reward_source: line.outcome.reward_source,\n      verdict: line.outcome.verdict,\n      instance_id: line.task.instance_id,\n      suite: line.task.suite,\n      split: line.task.split,\n      rollout_id: line.rollout_id,\n      ...realnessLabels(line),\n    },\n  }\n}\n\n/** RFT needs a real prompt: lines whose transcript starts with prompt turns. */\nexport function toRftItems(\n  lines: MintedRolloutLine[],\n  options: SignedSignalExportOptions = {},\n): RftItem[] {\n  return lines\n    .filter(\n      (line) =>\n        (options.gatedLines === 'zero-and-flag'\n          ? isSplitEligible(line, options) && line.provenance.lossy_projection !== true\n          : isTrainingLineEligible(line, options)) &&\n        line.messages.length > 0 &&\n        firstAssistantIndex(line.messages) > 0,\n    )\n    .map(toRftItem)\n}\n\n// ---------------------------------------------------------------------------\n// Serialization — one JSON object per line, the interchange format for\n// every export. `tangle.rollout.v1` lines and export rows alike.\n// ---------------------------------------------------------------------------\n\nexport function toJsonl(rows: ReadonlyArray<unknown>): string {\n  return rows.map((r) => JSON.stringify(r)).join('\\n') + (rows.length ? '\\n' : '')\n}\n\n/**\n * The split half of the training policy alone: `search` is trainable, held-out\n * needs the named opt-in, `dev` and `canary` never ship. This is the ONE check\n * `'zero-and-flag'` does not relax — a gated line is shipped as a labeled\n * negative, not as a licence to train on evaluation data. An explicit\n * `splitFilter` names a non-training slice instead of the default rule.\n *\n * Exported as the single implementation of that rule: `rl/exporters` applies\n * it on its line paths too, so the two waists cannot drift on which splits are\n * trainable.\n */\nexport function isSplitEligible(\n  line: RolloutLine,\n  options: Pick<TrainingExportOptions, 'allowHeldOutTrainingData' | 'splitFilter'>,\n): boolean {\n  if (options.splitFilter !== undefined) return options.splitFilter.includes(line.task.split)\n  if (line.task.split === 'search') return true\n  return line.task.split === 'holdout' && options.allowHeldOutTrainingData === true\n}\n\n/**\n * The whole fail-closed training policy for one line: scored, positive above\n * the floor, completed, not gated, in the requested license classes, on an eligible split. Shared with\n * `rl/exporters` so its text-lookup adapter cannot drift from this rule.\n */\nexport function isTrainingLineEligible(\n  line: RolloutLine,\n  options: TrainingExportOptions,\n): line is RolloutLine & { outcome: RolloutLine['outcome'] & { reward: number } } {\n  const minimumQualityExclusive = options.minimumQualityExclusive ?? 0\n  if (!Number.isFinite(minimumQualityExclusive)) {\n    throw new Error('minimumQualityExclusive must be finite')\n  }\n\n  const reward = line.outcome.reward\n  if (reward === null) return false\n  if (!Number.isFinite(reward)) {\n    throw new Error(`training reward for rollout \"${line.rollout_id}\" must be finite`)\n  }\n  if (reward <= minimumQualityExclusive) return false\n  if (!line.outcome.is_completed || line.outcome.is_truncated || line.outcome.error !== null) {\n    return false\n  }\n  if (line.outcome.realness_gated === true) return false\n  if (line.provenance.lossy_projection === true) return false\n  if (!isLicenseEligible(line, options.allowLicenseClasses)) return false\n  return isSplitEligible(line, options)\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AA6DA,SAAS,eAAe,MAAyC;CAC/D,OAAO;EACL,gBAAgB,KAAK,QAAQ,mBAAmB;EAIhD,mBAAmB,KAAK,QAAQ,qBAAqB;CACvD;AACF;;;;;;;;;;;;;;;;;AA6EA,SAAgB,UAAU,OAA4B,UAA4B,CAAC,GAAa;CAC9F,KAAK,MAAM,QAAQ,OAAO,iBAAiB,MAAM,YAAY;CAC7D,OAAO,MACJ,QAAQ,SAAS,uBAAuB,MAAM,OAAO,KAAK,KAAK,SAAS,SAAS,CAAC,CAAC,CACnF,KAAK,UAAU;EAId,UAAU,KAAK,SAAS,QAAQ,YAAY,QAAQ,sBAAsB,IAAI;EAC9E,UAAU;GACR,YAAY,KAAK;GACjB,QAAQ,KAAK;GACb,cAAc,KAAK,gBAAgB;GACnC,aAAa,KAAK,KAAK;GACvB,QAAQ,KAAK,QAAQ;GACrB,GAAG,eAAe,IAAI;EACxB;CACF,EAAE,CAAC,CACF,QAAQ,QAAQ,IAAI,SAAS,SAAS,CAAC;AAC5C;;;;AAgCA,SAAgB,aACd,OACA,UAAiC,CAAC,GACrB;CACb,KAAK,MAAM,QAAQ,OAAO,iBAAiB,MAAM,mBAAmB;CACpE,OAAO,MACJ,QACE,SACC,uBAAuB,MAAM,OAAO,KACpC,KAAK,SAAS,MACX,YACC,QAAQ,SAAS,UACjB,OAAO,QAAQ,YAAY,YAC3B,QAAQ,QAAQ,SAAS,CAC7B,CACJ,CAAC,CACA,KAAK,UAAU;EACd,QAAQ,KAAK,SAAS,MAAM,MAAM,EAAE,SAAS,MAAM,CAAC,EAAE,WAAW;EACjE,OAAO,KAAK,SAAS,CAAC;EACtB,QAAQ,KAAK,QAAQ;EACrB,UAAU;GACR,YAAY,KAAK;GACjB,QAAQ,KAAK;GACb,cAAc,KAAK,gBAAgB;GACnC,aAAa,KAAK,KAAK;GACvB,OAAO,KAAK,KAAK;GACjB,GAAG,eAAe,IAAI;EACxB;CACF,EAAE;AACN;;AAqCA,SAAS,oBAAoB,UAAiC;CAC5D,MAAM,QAAQ,SAAS,WAAW,MAAM,EAAE,SAAS,WAAW;CAC9D,OAAO,UAAU,KAAK,SAAS,SAAS;AAC1C;AAEA,SAAS,yBAAyB,MAAyB,SAAuB;CAChF,IAAI,KAAK,WAAW,qBAAqB,MACvC,MAAM,IAAI,MAAM,GAAG,QAAQ,YAAY,KAAK,WAAW,wBAAwB;AAEnF;AAEA,SAAgB,yBAAyB,MAAiD;CACxF,iBAAiB,MAAM,kBAAkB;CACzC,yBAAyB,MAAM,kBAAkB;CACjD,MAAM,QAAQ,oBAAoB,KAAK,QAAQ;CAC/C,OAAO;EACL,QAAQ,KAAK,SAAS,MAAM,GAAG,KAAK;EACpC,YAAY,KAAK,SAAS,MAAM,KAAK;EACrC,QAAQ,KAAK,QAAQ;EACrB,SAAS,KAAK,QAAQ;EACtB,WAAW,KAAK;EAChB,aAAa;GACX,cAAc,KAAK,KAAK;GACxB,eAAe,KAAK,KAAK;GACzB,kBAAkB,KAAK,KAAK;GAC5B,mBAAmB,KAAK,KAAK;GAC7B,oBAAoB,KAAK,KAAK;EAChC;EACA,MAAM;GACJ,MAAM,KAAK;GACX,QAAQ,KAAK;GACb,YAAY,KAAK;GACjB,QAAQ,KAAK;GACb,eAAe,KAAK,iBAAiB;GACrC,cAAc,KAAK,gBAAgB;GACnC,YAAY,KAAK;GACjB,iBAAiB,KAAK;GACtB,MAAM,KAAK;GACX,GAAG,eAAe,IAAI;EACxB;CACF;AACF;AAEA,SAAgB,0BACd,OACA,UAAqC,CAAC,GACZ;CAC1B,IAAI,QAAQ,eAAe,iBACzB,OAAO,MACJ,QACE,SACC,gBAAgB,MAAM,OAAO,KAC7B,KAAK,WAAW,qBAAqB,QACrC,KAAK,SAAS,SAAS,CAC3B,CAAC,CACA,IAAI,wBAAwB;CAEjC,OAAO,MACJ,QACE,SACC,uBAAuB,MAAM,OAAO,KACpC,oBAAoB,KAAK,QAAQ,IAAI,KACrC,oBAAoB,KAAK,QAAQ,IAAI,KAAK,SAAS,MACvD,CAAC,CACA,IAAI,wBAAwB;AACjC;AAqBA,SAAgB,UAAU,MAAkC;CAC1D,iBAAiB,MAAM,YAAY;CACnC,yBAAyB,MAAM,YAAY;CAC3C,MAAM,QAAQ,oBAAoB,KAAK,QAAQ;CAC/C,OAAO;EACL,UAAU,KAAK,SAAS,MAAM,GAAG,KAAK;EACtC,WAAW;GACT,QAAQ,KAAK,QAAQ;GACrB,eAAe,KAAK,QAAQ;GAC5B,SAAS,KAAK,QAAQ;GACtB,aAAa,KAAK,KAAK;GACvB,OAAO,KAAK,KAAK;GACjB,OAAO,KAAK,KAAK;GACjB,YAAY,KAAK;GACjB,GAAG,eAAe,IAAI;EACxB;CACF;AACF;;AAGA,SAAgB,WACd,OACA,UAAqC,CAAC,GAC3B;CACX,OAAO,MACJ,QACE,UACE,QAAQ,eAAe,kBACpB,gBAAgB,MAAM,OAAO,KAAK,KAAK,WAAW,qBAAqB,OACvE,uBAAuB,MAAM,OAAO,MACxC,KAAK,SAAS,SAAS,KACvB,oBAAoB,KAAK,QAAQ,IAAI,CACzC,CAAC,CACA,IAAI,SAAS;AAClB;AAOA,SAAgB,QAAQ,MAAsC;CAC5D,OAAO,KAAK,KAAK,MAAM,KAAK,UAAU,CAAC,CAAC,CAAC,CAAC,KAAK,IAAI,KAAK,KAAK,SAAS,OAAO;AAC/E;;;;;;;;;;;;AAaA,SAAgB,gBACd,MACA,SACS;CACT,IAAI,QAAQ,gBAAgB,KAAA,GAAW,OAAO,QAAQ,YAAY,SAAS,KAAK,KAAK,KAAK;CAC1F,IAAI,KAAK,KAAK,UAAU,UAAU,OAAO;CACzC,OAAO,KAAK,KAAK,UAAU,aAAa,QAAQ,6BAA6B;AAC/E;;;;;;AAOA,SAAgB,uBACd,MACA,SACgF;CAChF,MAAM,0BAA0B,QAAQ,2BAA2B;CACnE,IAAI,CAAC,OAAO,SAAS,uBAAuB,GAC1C,MAAM,IAAI,MAAM,wCAAwC;CAG1D,MAAM,SAAS,KAAK,QAAQ;CAC5B,IAAI,WAAW,MAAM,OAAO;CAC5B,IAAI,CAAC,OAAO,SAAS,MAAM,GACzB,MAAM,IAAI,MAAM,gCAAgC,KAAK,WAAW,iBAAiB;CAEnF,IAAI,UAAU,yBAAyB,OAAO;CAC9C,IAAI,CAAC,KAAK,QAAQ,gBAAgB,KAAK,QAAQ,gBAAgB,KAAK,QAAQ,UAAU,MACpF,OAAO;CAET,IAAI,KAAK,QAAQ,mBAAmB,MAAM,OAAO;CACjD,IAAI,KAAK,WAAW,qBAAqB,MAAM,OAAO;CACtD,IAAI,CAAC,kBAAkB,MAAM,QAAQ,mBAAmB,GAAG,OAAO;CAClE,OAAO,gBAAgB,MAAM,OAAO;AACtC"}

package/dist/{hf-dataset-DPVQXW5n.js → hf-dataset-Bv114tzB.js}

@@ -1,5 +1,5 @@
-import { a as assertMinted, c as assertRolloutLine, d as isTrainableSplit, h as GATE_POLICIES, m as GATE_CHECK_IDS, n as ROLLOUT_ROLES, r as ROLLOUT_SCHEMA, v as undeclaredStepPayload, x as licenseClassOf } from "./schema-YUIXT8iW.js";
-import { a as toRftItems, o as toSftRows, r as toJsonl, s as toVerifiersRolloutOutputs } from "./exporters-ByLcILYU.js";
+import { a as assertMinted, c as assertRolloutLine, d as isTrainableSplit, g as gateErrors, h as GATE_POLICIES, m as GATE_CHECK_IDS, n as ROLLOUT_ROLES, r as ROLLOUT_SCHEMA, v as undeclaredStepPayload, x as licenseClassOf } from "./schema-ChIZfnqY.js";
+import { a as toRftItems, o as toSftRows, r as toJsonl, s as toVerifiersRolloutOutputs } from "./exporters-BKZLt1Px.js";
 import { basename, dirname, join } from "node:path";
 import { spawnSync } from "node:child_process";
 import { appendFile, mkdir, readFile, stat, writeFile } from "node:fs/promises";
@@ -719,7 +719,7 @@ const SCRUB_RULES = [
 	},
 	{
 		name: "bearer-token",
-		pattern: /\b(Bearer|Basic)\s+(?!\[REDACTED:)[A-Za-z0-9\-._~+/=]{8,}/g,
+		pattern: /\b(Bearer|Basic)\s+(?!\[REDACTED:)(?![A-Za-z]{1,16}\.?(?:[^\w\-.~+/=]|$))[A-Za-z0-9\-._~+/=]{8,}/g,
 		rewrite: (_match, scheme) => `${scheme} [REDACTED:bearer]`
 	},
 	{
@@ -808,7 +808,7 @@ function entropy(token) {
 	return bits;
 }
 const ENTROPY_TOKEN = /[A-Za-z0-9+=_-]{32,}/g;
-const NAMED = /(?:^|[-_])[a-z]{4,}(?:$|[-_])/;
+const NAMED = /(?:^|[-_])(?:[a-z]{4,}|[A-Z]{4,})(?:$|[-_])/;
 const highEntropy = (text) => {
 	for (const [token] of text.matchAll(ENTROPY_TOKEN)) {
 		if (NAMED.test(token)) continue;
@@ -848,7 +848,7 @@ const PRIVACY_RULES = [
 	},
 	{
 		name: "phone",
-		hit: (text) => /(?<![\d.])(?:\+?1[ .-]?)?\(?\d{3}\)?[ .-]\d{3}[ .-]\d{4}(?![\d.])/.test(text)
+		hit: (text) => /(?<![\d.]\s?)(?:\+?1[ .-]?)?\(?\d{3}\)?[ .-]\d{3}[ .-]\d{4}(?![\d.])/.test(text)
 	},
 	{
 		name: "postal-address",
@@ -912,10 +912,10 @@ async function buildHfDataset(inputs, options) {
 	const kept = [];
 	let read = 0;
 	for (const input of inputs) {
-		const lines = await readRolloutLedger(input);
+		const lines = await readRolloutJournal(input);
 		read += lines.length;
 		const scrubbed = scrubLines(lines.filter((line) => {
-			if (!isTrainableSplit(line.task.split)) {
+			if (gateErrors(line, GATE_POLICIES.assertMinted).length > 0 || !isTrainableSplit(line.task.split)) {
 				report.excluded.nonTrain += 1;
 				return false;
 			}
@@ -1041,9 +1041,9 @@ function parseRolloutReleaseArgs(argv) {
 		const arg = argv[i];
 		if (arg === "--out") args.out = argv[++i] ?? "";
 		else if (arg === "--formats") {
-			const raw = (argv[++i] ?? "").split(",").filter(Boolean);
-			for (const format of raw) if (!RELEASE_FORMATS.includes(format)) throw new Error(`unknown format "${format}" — expected one of ${RELEASE_FORMATS.join(",")}`);
-			args.formats = raw;
+			args.formats = (argv[++i] ?? "").split(",").filter(Boolean);
+			const unknown = args.formats.find((format) => !RELEASE_FORMATS.includes(format));
+			if (unknown) throw new Error(`unknown format "${unknown}" — expected one of ${RELEASE_FORMATS}`);
 		} else if (arg === "--include-proposers") args.includeProposers = true;
 		else if (arg === "--denylist") args.denylistPath = argv[++i] ?? null;
 		else if (arg === "--push") args.push = argv[++i] ?? null;
@@ -1081,4 +1081,4 @@ async function runRolloutReleaseCli(argv) {
 }
 export { writeRolloutLedger as _, privacyHit as a, buildDatasetCard as c, harnessSessionRolloutLine as d, harnessSessionRolloutLines as f, readRolloutLedger as g, readRolloutJournal as h, defaultRolloutScrubber as i, assertGateReport as l, appendRolloutLines as m, runRolloutReleaseCli as n, scrubLines as o, gateReceiptRolloutLines as p, PRIVACY_RULE_NAMES as r, scrubText as s, buildHfDataset as t, harnessSessionRefs as u };
 
-//# sourceMappingURL=hf-dataset-DPVQXW5n.js.map
+//# sourceMappingURL=hf-dataset-Bv114tzB.js.map