@tangle-network/agent-eval 0.128.2 → 0.129.0

package/dist/rollout/index.js

@@ -1,7 +1,11 @@
 import {
-  mintRolloutRows,
-  rolloutReward
-} from "../chunk-IHQDPH7D.js";
+  ATIF_SCHEMA_VERSION,
+  HARBOR_IMPORT_GAP,
+  fromHarborTrajectory,
+  relabelImportedSplit,
+  toHarborTrajectories,
+  toHarborTrajectory
+} from "../chunk-RXHCETDZ.js";
 import {
   DEFAULT_CLAUDE_PROJECTS_DIR,
   DEFAULT_OPENCODE_DB,
@@ -12,57 +16,91 @@ import {
   openOpencodeDb,
   readClaudeTranscript,
   readOpencodeSessionMessages
-} from "../chunk-VBQ3CRKH.js";
+} from "../chunk-HPWUNB47.js";
 import {
   FORMAT_FILES,
+  FORMAT_GATE_DISPOSITION,
   RELEASE_FORMATS,
   ROLLOUT_RELEASE_USAGE,
   SCRUB_RULES,
   addScrubCounts,
   appendRolloutLines,
+  assertGateReport,
   buildDatasetCard,
   buildHfDataset,
   defaultRolloutScrubber,
   emptyScrubCounts,
+  gatedRolloutIds,
+  measureFormatGate,
   parseRolloutReleaseArgs,
   planPushCommand,
   pushDataset,
+  readRolloutJournal,
   readRolloutLedger,
+  releaseRowRefs,
   runRolloutReleaseCli,
   scrubLines,
   scrubRolloutLine,
   scrubText,
+  writeRolloutLedger
+} from "../chunk-3OCR4R5I.js";
+import {
+  mintRolloutRows
+} from "../chunk-H23X7XKK.js";
+import {
+  realnessLabels,
   toJsonl,
   toRewardRows,
   toRftItem,
   toRftItems,
   toSftRows,
   toVerifiersRolloutOutput,
-  toVerifiersRolloutOutputs,
-  writeRolloutLedger
-} from "../chunk-EJGRPCO3.js";
+  toVerifiersRolloutOutputs
+} from "../chunk-OWN5NPMC.js";
 import {
   CHAT_ROLES,
+  GATE_CHECKS,
+  GATE_CHECK_IDS,
+  GATE_POLICIES,
   ROLLOUT_CAPTURES,
   ROLLOUT_ROLES,
   ROLLOUT_SCHEMA,
   ROLLOUT_SPLITS,
   TRAINABLE_SPLITS,
+  assertMinted,
+  assertMintedLines,
   assertRolloutLine,
+  gateErrors,
+  gateGamedOutcome,
+  gatedEvidenceOf,
   isRolloutLine,
   isTrainableSplit,
   validateRolloutLine
-} from "../chunk-UWZZKKU7.js";
+} from "../chunk-PC5DOSM7.js";
 import "../chunk-RZTMDUO7.js";
-import "../chunk-2JX3CFMB.js";
+import "../chunk-56TAVBOK.js";
 import "../chunk-MA6HLL3S.js";
+import {
+  isRealnessGated,
+  observedScore,
+  observedSplitScore,
+  scoreOrigin,
+  trainingReward,
+  trainingScore
+} from "../chunk-OIUOT4QD.js";
 import "../chunk-ONWEPEDO.js";
 import "../chunk-PZ5AY32C.js";
 export {
+  ATIF_SCHEMA_VERSION,
   CHAT_ROLES,
   DEFAULT_CLAUDE_PROJECTS_DIR,
   DEFAULT_OPENCODE_DB,
   FORMAT_FILES,
+  FORMAT_GATE_DISPOSITION,
+  GATE_CHECKS,
+  GATE_CHECK_IDS,
+  GATE_POLICIES,
+  HARBOR_IMPORT_GAP,
   RELEASE_FORMATS,
   ROLLOUT_CAPTURES,
   ROLLOUT_RELEASE_USAGE,
@@ -73,6 +111,9 @@ export {
   TRAINABLE_SPLITS,
   addScrubCounts,
   appendRolloutLines,
+  assertGateReport,
+  assertMinted,
+  assertMintedLines,
   assertRolloutLine,
   buildDatasetCard,
   buildHfDataset,
@@ -82,21 +123,36 @@ export {
   findClaudeTranscripts,
   findOpencodeSessionById,
   findOpencodeSessionsByDirectory,
+  fromHarborTrajectory,
+  gateErrors,
+  gateGamedOutcome,
+  gatedEvidenceOf,
+  gatedRolloutIds,
+  isRealnessGated,
   isRolloutLine,
   isTrainableSplit,
+  measureFormatGate,
   mintRolloutRows,
+  observedScore,
+  observedSplitScore,
   openOpencodeDb,
   parseRolloutReleaseArgs,
   planPushCommand,
   pushDataset,
   readClaudeTranscript,
   readOpencodeSessionMessages,
+  readRolloutJournal,
   readRolloutLedger,
-  rolloutReward,
+  realnessLabels,
+  relabelImportedSplit,
+  releaseRowRefs,
   runRolloutReleaseCli,
+  scoreOrigin,
   scrubLines,
   scrubRolloutLine,
   scrubText,
+  toHarborTrajectories,
+  toHarborTrajectory,
   toJsonl,
   toRewardRows,
   toRftItem,
@@ -104,6 +160,8 @@ export {
   toSftRows,
   toVerifiersRolloutOutput,
   toVerifiersRolloutOutputs,
+  trainingReward,
+  trainingScore,
   validateRolloutLine,
   writeRolloutLedger
 };

package/dist/run-campaign-OJJ7CZF4.js

@@ -0,0 +1,18 @@
+import {
+  planCampaignRun,
+  runCampaign
+} from "./chunk-C6LXANRU.js";
+import "./chunk-E7QXT7SX.js";
+import "./chunk-ZHTZ4EYI.js";
+import "./chunk-VCZ5FQYW.js";
+import "./chunk-VI2UW6B6.js";
+import "./chunk-56TAVBOK.js";
+import "./chunk-MA6HLL3S.js";
+import "./chunk-OIUOT4QD.js";
+import "./chunk-ONWEPEDO.js";
+import "./chunk-PZ5AY32C.js";
+export {
+  planCampaignRun,
+  runCampaign
+};
+//# sourceMappingURL=run-campaign-OJJ7CZF4.js.map

package/dist/supervisor-run/index.d.ts

@@ -23,6 +23,28 @@
  * `outcome.reward` is THE single scalar (null = no verdict exists — a
  * labeled gap, never 0). `outcome.realness_gated` is the anti-Goodhart
  * flag: a gated line must never export as a positive training example.
+ *
+ * That last sentence is enforced here, by `validateRolloutLine`, not merely
+ * documented. Validating `reward` and `realness_gated` independently — each a
+ * well-typed field, their COMBINATION unchecked — is what let a line claiming
+ * `{reward: 0.95, realness_gated: true}` validate clean and walk into every
+ * training export. The relationship between the two IS the invariant, so it is
+ * checked where every other structural claim about a line is checked.
+ *
+ * The invariant is about the OUTCOME, not about one field of it. Zeroing
+ * `reward` while `outcome.metrics` still carried the per-layer scores that
+ * reward was computed from exported the gamed signal anyway, in the dict the
+ * verifiers format reads as its per-rubric scores. So `gateGamedOutcome`
+ * transforms the whole outcome once, at `assertMinted` — the funnel every
+ * minted line passes — and the reward-bearing components are relocated to
+ * `provenance.gated_evidence`, which no exporter projects.
+ *
+ * WHICH checks each door applies is not decided in this file. `./gate-checks`
+ * owns the canonical list and the total per-entry-point policy; the three doors
+ * below (`validateRolloutLine`, `assertRewardGate`, `assertMinted`) each call
+ * `gateErrors` with their declared policy, so a check added to that list applies
+ * here without anyone editing this file, and a check deliberately skipped has to
+ * name itself there.
  */
 declare const ROLLOUT_SCHEMA = "tangle.rollout.v1";
 /** `agent` = a solo evaluation run (no multi-agent topology). */
@@ -50,6 +72,14 @@ interface ChatMessage {
     /** Required on role:"tool" — the ChatToolCall this result answers. */
     tool_call_id?: string;
     name?: string;
+    /**
+     * Harbor ATIF `is_copied_context` (RFC 0001 rule 7): this turn was COPIED IN
+     * from another trajectory's context, not produced by the agent on this line.
+     * The RFC makes excluding it from SFT a MUST, and `toSftRows` does — training
+     * on it teaches the model to author text it never authored, and credits this
+     * run for another one's work. Absent = false (authored here).
+     */
+    is_copied_context?: boolean;
 }
 interface ToolDef {
     type: 'function';
@@ -73,6 +103,21 @@ interface RolloutStep {
     output?: string;
     status?: 'ok' | 'error';
     durationMs?: number;
+    /**
+     * LLM inferences this span represents. 0 = deterministic dispatch with no
+     * model call — distinct from absent, which means the producer did not track it.
+     */
+    llm_call_count?: number;
+    /** Exact prompt tokenization. Removes the ambiguity of re-tokenizing text at train time. */
+    prompt_token_ids?: number[];
+    /** Exact completion tokenization; aligns index-wise with `logprobs`. */
+    completion_token_ids?: number[];
+    /**
+     * Per-completion-token log probabilities under the sampling policy. Required
+     * for off-policy correction (importance weighting) when the rollout was
+     * generated by a policy other than the one being trained.
+     */
+    logprobs?: number[];
 }
 interface RolloutTask {
     /** Benchmark/suite id (e.g. "swe-bench-verified") or the experiment id. */
@@ -117,11 +162,39 @@ interface RolloutOutcome {
     is_truncated: boolean;
     error: string | null;
     /**
-     * Anti-Goodhart flag from `RunRecord.outcome.realness.gated`: the run
-     * faked its success signal. Reward is forced to 0 at mint time and the
-     * line never qualifies for SFT.
+     * Anti-Goodhart flag from `RunRecord.outcome.realness.gated`: the run faked
+     * its success signal. `true` requires `reward` to be 0 or null — the
+     * validator rejects the line otherwise — and the line never qualifies for
+     * SFT. Required on the wire: a line that does not state the flag does not
+     * validate, so no producer can dodge the gate by omitting it.
+     *
+     * `true` ALSO requires `metrics` to be empty and `verdict` to be null: the
+     * numbers the reward was computed from are relocated to
+     * `provenance.gated_evidence` by `gateGamedOutcome`. See that function for
+     * why zeroing the scalar alone was not enough.
      */
     realness_gated: boolean;
+    /**
+     * Whether an authenticity SCREEN ever RAN on this reward — a different claim
+     * from `realness_gated`, which is the screen's VERDICT.
+     *
+     * `realness_gated: false` reads as "we looked and nothing fired". A producer
+     * with no screen at all was emitting exactly that, so a never-screened reward
+     * was indistinguishable on the wire from a screened-clean one, and the whole
+     * anti-Goodhart apparatus silently treated the first as the second. The two
+     * claims are now separable:
+     *
+     *   - `true`   — a screen ran; `realness_gated` is its verdict.
+     *   - `false`  — the producer declares it HAS no screen (`unscreenedRewardFields`).
+     *                `assertMinted` REFUSES such a line when its reward is above
+     *                zero: an unscreened positive reward is precisely the signal
+     *                the gate exists to qualify, and nothing has qualified it.
+     *   - absent   — not stated. Pre-unification ledgers land here, as does a
+     *                `RunRecord` carrying no `outcome.realness` at all. Absent is
+     *                read as "unknown", never as `false` (which would refuse most
+     *                of the existing corpus) and never as `true`.
+     */
+    realness_screened?: boolean;
 }
 interface RolloutCostBlock {
     usd: number | null;
@@ -131,6 +204,11 @@ interface RolloutCostBlock {
     cache_read: number | null;
     cache_write: number | null;
     wall_s: number | null;
+    /**
+     * Total LLM inferences across the invocation (ATIF `llm_call_count`,
+     * aggregated). Optional and additive: absent = not tracked, never 0.
+     */
+    llm_call_count?: number | null;
 }
 interface RolloutArtifacts {
     patch_path: string | null;
@@ -138,11 +216,43 @@ interface RolloutArtifacts {
     /** Source-of-truth transcript pointer (session id / jsonl path) for audit. */
     transcript_ref: string | null;
 }
+/**
+ * The reward-bearing half of a GATED line's outcome, moved off `outcome` and
+ * parked here verbatim. Diagnostics, never training input — see
+ * `gateGamedOutcome`.
+ */
+interface GatedEvidence {
+    /** `outcome.metrics` exactly as the producer measured it. */
+    metrics?: Record<string, unknown>;
+    /** `outcome.verdict` verbatim — the judge record that claimed the success. */
+    verdict?: unknown;
+    /**
+     * The per-step fields `tangle.rollout.v1` does not declare, parked here when
+     * the gate projected `steps[]` down to the schema's own key set.
+     *
+     * A per-step reward is training signal exactly like the scalar, and `steps`
+     * rides through `toRewardRows` verbatim — so a gated line was shipping its
+     * step-level credit assignment at full value beside a `reward` of 0.
+     */
+    steps?: unknown;
+}
 interface RolloutProvenance {
     captured_at: string;
     capture: RolloutCapture;
-    /** Present on gap lines: why `messages` could not be recovered. */
+    /**
+     * Why this line is incomplete. Required when `messages` is empty (the
+     * transcript could not be recovered); also set by interchange importers to
+     * name a MISSING LABEL — an imported trajectory carries no verdict, so
+     * `outcome.reward` is null and this says why.
+     */
     gap?: string;
+    /**
+     * Present only on a realness-gated line: the outcome fields the gate
+     * relocated, kept so an auditor can still see WHY the run was gated and what
+     * it claimed. Deliberately OUTSIDE `outcome`, because every training exporter
+     * reads `outcome` and none reads `provenance`.
+     */
+    gated_evidence?: GatedEvidence;
 }
 interface RolloutLine {
     schema: typeof ROLLOUT_SCHEMA;

package/dist/supervisor-run/index.js

@@ -27,9 +27,10 @@ import {
   unavailable,
   writeSupervisorRunReport,
   writeSupervisorRunReportSafe
-} from "../chunk-TSN7JT6D.js";
-import "../chunk-VBQ3CRKH.js";
-import "../chunk-UWZZKKU7.js";
+} from "../chunk-X4YIBDER.js";
+import "../chunk-HPWUNB47.js";
+import "../chunk-PC5DOSM7.js";
+import "../chunk-OIUOT4QD.js";
 import "../chunk-PZ5AY32C.js";
 export {
   DEFAULT_CANCEL_TOOLS,

package/dist/traces.d.ts

@@ -1972,7 +1972,7 @@ interface OtlpFlatLine {
     }>;
 }
 interface FlattenOtlpOptions {
-    /** `'openinference'` (default) mirrors legacy per-span attributes into the
+    /** `'openinference'` (default) maps source per-span attributes into the
      *  canonical OpenInference vocabulary the analyst readers consume. `'none'`
      *  passes attributes through untouched. */
     attributeVocabulary?: 'openinference' | 'none';

package/dist/traces.js

@@ -1,6 +1,4 @@
 import {
-  FileSystemTraceStore,
-  InMemoryTraceStore,
   OTEL_AGENT_EVAL_SCOPE,
   ReplayCache,
   ReplayCacheMissError,
@@ -27,7 +25,7 @@ import {
   scoreTraceInsightReadiness,
   tokenizeDomainWords,
   traceAnalystOnRunComplete
-} from "./chunk-EOSZT7PL.js";
+} from "./chunk-U4L7JRPZ.js";
 import "./chunk-7ZZMD7UK.js";
 import {
   extractUsage,
@@ -78,10 +76,12 @@ import {
   traceSpanKindToOpenInferenceKind
 } from "./chunk-P6FYH6K4.js";
 import {
+  FileSystemTraceStore,
+  InMemoryTraceStore,
   RunIntegrityError,
   assertRunCaptured,
   throwIfRunIncomplete
-} from "./chunk-TT4KNT67.js";
+} from "./chunk-U4PHLT2N.js";
 import {
   FileSystemRawProviderSink,
   InMemoryRawProviderSink,
@@ -93,7 +93,7 @@ import {
   TraceEmitter,
   llmSpanFromProvider
 } from "./chunk-VQMK5FMP.js";
-import "./chunk-2JX3CFMB.js";
+import "./chunk-56TAVBOK.js";
 import {
   FAILURE_CLASSES,
   TRACE_SCHEMA_VERSION,
@@ -103,6 +103,7 @@ import {
   isSandboxSpan,
   isToolSpan
 } from "./chunk-MA6HLL3S.js";
+import "./chunk-OIUOT4QD.js";
 import "./chunk-ONWEPEDO.js";
 import {
   INPUT_VALUE,

package/dist/wire/index.d.ts

@@ -652,9 +652,8 @@ type ProviderRedactor = (event: RawProviderEvent) => RawProviderEvent;
  *     { baseUrl: 'https://router.tangle.tools/v1', apiKey: process.env.KEY },
  *   )
  *
- * This is THE llm-calling seam for agent-eval primitives that need structured
- * output (semantic concept judge, reviewer directives, critic scores). Primitives
- * that need free-form text use `callLlm` and parse output themselves.
+ * `createChatClient` wraps this implementation for provider-neutral package
+ * entry points. Direct callers can use `callLlm` or `callLlmJson`.
  */
 
 type LlmThinkingMode = 'enabled' | 'disabled';
@@ -688,8 +687,8 @@ interface LlmClientOptions {
      * total attempts × `timeoutMs`.
      */
     deadlineMs?: number;
-    /** Total provider attempts. Legacy option name; default 3 (1 initial + 2 retries). */
-    maxRetries?: number;
+    /** Total provider attempts. Default 3. */
+    maximumAttempts?: number;
     /** Token rates used when the provider omits cost or package pricing does not cover the model. */
     customTokenPricing?: CustomTokenPricing;
     /**
@@ -909,8 +908,8 @@ declare const FeedbackLabelSchema: z.ZodObject<{
     id: z.ZodOptional<z.ZodString>;
     source: z.ZodEnum<{
         judge: "judge";
-        user: "user";
         system: "system";
+        user: "user";
         policy: "policy";
         environment: "environment";
         metric: "metric";
@@ -957,9 +956,9 @@ declare const FeedbackAttemptSchema: z.ZodObject<{
     proposedAction: z.ZodOptional<z.ZodObject<{
         type: z.ZodString;
         risk: z.ZodOptional<z.ZodEnum<{
-            medium: "medium";
             low: "low";
             high: "high";
+            medium: "medium";
         }>>;
         costUsd: z.ZodOptional<z.ZodNumber>;
         externalSideEffect: z.ZodOptional<z.ZodBoolean>;
@@ -970,8 +969,8 @@ declare const FeedbackAttemptSchema: z.ZodObject<{
         id: z.ZodOptional<z.ZodString>;
         source: z.ZodEnum<{
             judge: "judge";
-            user: "user";
             system: "system";
+            user: "user";
             policy: "policy";
             environment: "environment";
             metric: "metric";
@@ -1029,9 +1028,9 @@ declare const FeedbackTrajectorySchema: z.ZodObject<{
         proposedAction: z.ZodOptional<z.ZodObject<{
             type: z.ZodString;
             risk: z.ZodOptional<z.ZodEnum<{
-                medium: "medium";
                 low: "low";
                 high: "high";
+                medium: "medium";
             }>>;
             costUsd: z.ZodOptional<z.ZodNumber>;
             externalSideEffect: z.ZodOptional<z.ZodBoolean>;
@@ -1042,8 +1041,8 @@ declare const FeedbackTrajectorySchema: z.ZodObject<{
             id: z.ZodOptional<z.ZodString>;
             source: z.ZodEnum<{
                 judge: "judge";
-                user: "user";
                 system: "system";
+                user: "user";
                 policy: "policy";
                 environment: "environment";
                 metric: "metric";
@@ -1078,8 +1077,8 @@ declare const FeedbackTrajectorySchema: z.ZodObject<{
         id: z.ZodOptional<z.ZodString>;
         source: z.ZodEnum<{
             judge: "judge";
-            user: "user";
             system: "system";
+            user: "user";
             policy: "policy";
             environment: "environment";
             metric: "metric";

package/dist/wire/index.js

@@ -34,9 +34,9 @@ import {
   runRpcOnce,
   startServer,
   startServerAsync
-} from "../chunk-YJBNWCAA.js";
-import "../chunk-PBE2LOSS.js";
-import "../chunk-WS3NZZQQ.js";
+} from "../chunk-NY44NC4A.js";
+import "../chunk-SFLLL76A.js";
+import "../chunk-VCZ5FQYW.js";
 import "../chunk-VI2UW6B6.js";
 import "../chunk-PC4UYEBM.js";
 import "../chunk-ONWEPEDO.js";

package/docs/feature-guide.md

@@ -151,7 +151,7 @@ Store as `FeedbackTrajectory`, then derive:
 
 | Area | Key exports | Best for | Notes |
 | --- | --- | --- | --- |
-| Judging | `createCustomJudge`, semantic judges, anti-slop, wire rubrics | Content, voice, semantic quality | Pair with objective checks when possible. |
+| Judging | `llmJudge`, semantic judges, anti-slop, wire rubrics | Content, voice, semantic quality | Pair with objective checks when possible. |
 | Verification | `MultiLayerVerifier`, `JudgeRunner`, sandbox harness | Code and multi-step gates | Do not let semantic judges override failed builds. |
 | Control | `runAgentControlLoop`, `objectiveEval`, `subjectiveEval` | Long-running agent tasks | Supports budgets, cost, stop policies, trace spans. |
 | Propose/review | `runProposeReview`, `runProposeReviewAsControlLoop` | Iterative artifact repair | Good for code, docs, plans, briefs. |