bitfab-cli 0.2.308 → 0.2.310

package/dist/index.js

@@ -23782,6 +23782,8 @@ var dateOnlyStringSchema = external_exports.string().regex(DATE_PATTERN, "Must b
   return date5.getFullYear() === year && date5.getMonth() === month - 1 && date5.getDate() === day;
 }, "Must be a valid calendar date");
 var GET_TRACE_LABELS_MAX_IDS = 100;
+var GET_TRACE_ASSERTIONS_MAX_IDS = 100;
+var MAX_ASSERTIONS_PER_REQUEST = 50;
 var GET_GRADER_LABELS_MAX_IDS = 100;
 var DEFAULT_GRADER_LABEL_LIMIT = 50;
 var MAX_GRADER_LABEL_LIMIT = 200;
@@ -23932,7 +23934,7 @@ var getTraces = {
 var getTraceLabels = {
   name: "get_trace_labels",
   title: "Get Trace Labels",
-  description: `Read just the labels for one or more traces by ID: returns each trace's pass/fail verdict, its annotation (the reviewer's reasoning), and whether the label is human-validated (a human authored or approved it). No span content, inputs, or outputs are loaded, so the response is small and one call accepts up to ${GET_TRACE_LABELS_MAX_IDS} IDs. Use this to load a whole dataset's verdicts into context in a single call (e.g. when building or confirming a dataset). When you need a trace's actual inputs/outputs/spans, use get_traces instead.`,
+  description: `Read just the labels for one or more traces by ID: returns each trace's pass/fail verdict, its annotation (the reviewer's reasoning), and whether the label is human-validated (a human authored or approved it). No span content, inputs, or outputs are loaded, so the response is small and one call accepts up to ${GET_TRACE_LABELS_MAX_IDS} IDs. Use this to load a whole dataset's verdicts into context in a single call (e.g. when building or confirming a dataset). When you need a trace's actual inputs/outputs/spans, use get_traces instead. This returns verdicts on runs that already happened; for what a trace SHOULD do on its next replay, use get_trace_assertions.`,
   inputSchema: {
     traceIds: external_exports.preprocess(parseJsonString, external_exports.array(external_exports.uuid()).min(1).max(GET_TRACE_LABELS_MAX_IDS)).describe(`Trace IDs to read labels for (1-${GET_TRACE_LABELS_MAX_IDS})`)
   }
@@ -23951,7 +23953,7 @@ var getSpanField = {
 var saveAgentLabels = {
   name: "save_agent_labels",
   title: "Save Agent Labels",
-  description: "Set, skip, or archive the agent's pass/fail verdict on one or more traces (`labelSource=\"agent\"`). Use this AFTER you have read the traces with get_traces and decided yourself whether each one looks like a pass, a fail, or genuinely cannot be judged. To set a verdict, pass `label` (true=PASS, false=FAIL) and `annotation` (your reasoning, shown to the human reviewer in the labeling UI). Optionally pass `confidence` (`VeryLow|Low|Medium|High|VeryHigh`) to record how confident you are - surfaced in the labeling UI so reviewers can prioritize low-confidence verdicts. To explicitly skip a trace you cannot decide on (instead of leaving it unlabeled), pass `skip: true` and omit label/annotation - this records an explicit skip so coverage checks know you intentionally did not verdict it. To clear a previously-set agent verdict (e.g., you changed your mind or labeled the wrong trace), pass `archive: true` and omit label/annotation. New verdicts start unapproved (`approvedAt=null`); once a human approves one, it joins the validated dataset (queryable via `search_traces` with `validated: true`). Archiving is non-destructive - the row is hidden from all reads but kept for audit, and you can immediately re-label the trace from scratch. For replay results, key each label by the replay item's `originalTraceId` (the original trace it was replayed from; `sourceTraceId` is accepted as a deprecated alias) plus the top-level `testRunId` instead of a `traceId`: the server resolves it to the replay trace via lineage, so you never need a server-generated replay trace id. When the experiment ran each trace more than once, also pass the item's `attempt` so each attempt gets its own verdict. Returns an agent-readable summary with one parseable effective label line per updated trace, keyed by the id you supplied (`originalTraceId` for replay verdicts, otherwise `traceId`), so command callers can verify persistence.",
+  description: "Set, skip, or archive the agent's pass/fail verdict on one or more traces (`labelSource=\"agent\"`). Use this AFTER you have read the traces with get_traces and decided yourself whether each one looks like a pass, a fail, or genuinely cannot be judged. To set a verdict, pass `label` (true=PASS, false=FAIL) and `annotation` (your reasoning, shown to the human reviewer in the labeling UI). Optionally pass `confidence` (`VeryLow|Low|Medium|High|VeryHigh`) to record how confident you are - surfaced in the labeling UI so reviewers can prioritize low-confidence verdicts. To explicitly skip a trace you cannot decide on (instead of leaving it unlabeled), pass `skip: true` and omit label/annotation - this records an explicit skip so coverage checks know you intentionally did not verdict it. To clear a previously-set agent verdict (e.g., you changed your mind or labeled the wrong trace), pass `archive: true` and omit label/annotation. New verdicts start unapproved (`approvedAt=null`); once a human approves one, it joins the validated dataset (queryable via `search_traces` with `validated: true`). Archiving is non-destructive - the row is hidden from all reads but kept for audit, and you can immediately re-label the trace from scratch. For replay results, key each label by the replay item's `originalTraceId` (the original trace it was replayed from; `sourceTraceId` is accepted as a deprecated alias) plus the top-level `testRunId` instead of a `traceId`: the server resolves it to the replay trace via lineage, so you never need a server-generated replay trace id. When the experiment ran each trace more than once, also pass the item's `attempt` so each attempt gets its own verdict. Returns an agent-readable summary with one parseable effective label line per updated trace, keyed by the id you supplied (`originalTraceId` for replay verdicts, otherwise `traceId`), so command callers can verify persistence. Before judging a replay, call get_trace_assertions on the original trace ids: an expectation says what the user asked this specific case to do, so the verdict is measured against that rather than a guess. When an expectation's target cannot be found on the trace you are judging, the check never ran, so pass `skip: true` for that trace rather than recording a FAIL.",
   inputSchema: {
     testRunId: external_exports.uuid().optional().describe("The replay test run id. Required when any label targets a trace by originalTraceId (replay verdicts); ignored otherwise."),
     labels: external_exports.preprocess(parseJsonString, external_exports.array(external_exports.object({
@@ -23970,7 +23972,7 @@ var saveAgentLabels = {
 var saveHumanLabels = {
   name: "save_human_labels",
   title: "Save Human Labels",
-  description: 'Record one or more human-authored pass/fail verdicts that are VALIDATED immediately (`labelSource="human"`, no approval step). Unlike save_agent_labels (which writes agent suggestions that start unapproved), labels set here join the validated dataset the instant they are written and satisfy `search_traces` with `validated: true`. Pass `label` (true=PASS, false=FAIL), `annotation` (the reasoning behind the verdict), and optionally `confidence` (`VeryLow|Low|Medium|High|VeryHigh`). USE ONLY when a human has explicitly decided the verdict, for example saving a known production bug as a test case. Do NOT use this for the agent\'s own first-pass guesses on traces awaiting human review; use save_agent_labels for those so they go through the normal approve/edit loop.',
+  description: 'Record one or more human-authored pass/fail verdicts that are VALIDATED immediately (`labelSource="human"`, no approval step). Unlike save_agent_labels (which writes agent suggestions that start unapproved), labels set here join the validated dataset the instant they are written and satisfy `search_traces` with `validated: true`. Pass `label` (true=PASS, false=FAIL), `annotation` (the reasoning behind the verdict), and optionally `confidence` (`VeryLow|Low|Medium|High|VeryHigh`). USE ONLY when a human has explicitly decided the verdict, for example saving a known production bug as a test case. This records a verdict on a run that already happened; to record what a trace should do on its next replay, use save_trace_assertions. Do NOT use this for the agent\'s own first-pass guesses on traces awaiting human review; use save_agent_labels for those so they go through the normal approve/edit loop.',
   inputSchema: {
     labels: external_exports.preprocess(parseJsonString, external_exports.array(external_exports.object({
       traceId: external_exports.uuid().describe("The trace ID to label"),
@@ -23980,6 +23982,41 @@ var saveHumanLabels = {
     })).min(1).max(50)).describe("One verdict per trace (1-50)")
   }
 };
+var assertionTargetShape = external_exports.discriminatedUnion("kind", [
+  external_exports.object({ kind: external_exports.literal("output") }).describe("Check only the trace's final output, ignoring everything that happened on the way there."),
+  external_exports.object({
+    kind: external_exports.literal("span"),
+    name: external_exports.string().min(1).describe(`The span's name as it appears on the trace being evaluated, e.g. "search_flights". Names only, never span ids: an id belongs to one trace, so an id read off the original resolves to nothing on its replay and the check would never run.`),
+    occurrence: external_exports.union([
+      external_exports.literal("first"),
+      external_exports.literal("last"),
+      external_exports.number().int().min(0)
+    ]).optional().describe('Which call to check when the trace runs that span more than once, e.g. a retry loop. "first", "last" (the default), or a 0-based index. Leave it off unless the repetition matters.')
+  }).describe("Check one span by name rather than the whole trace, e.g. that a particular tool call was made with the right arguments.")
+]).describe('SCOPE: what part of the trace under evaluation this assertion is checked against. OMIT IT for the whole trace, which is the common case and the right default when unsure. Pass { "kind": "output" } to check only the final output, or { "kind": "span", "name": "..." } to check one span. Scoping narrows what a judge looks at, so a wrong scope hides real failures. A target naming something the evaluated trace does not contain makes the check ERRORED, never passed, because a check that could not run must never look like a check that succeeded.');
+var saveTraceAssertions = {
+  name: "save_trace_assertions",
+  title: "Save Trace Expectations",
+  description: `Record what SHOULD happen when this trace is replayed. Bitfab stores two kinds of label on a trace. A VERDICT says how a run that already happened turned out, and that is save_agent_labels / save_human_labels. An EXPECTATION, which is what this tool writes, says what a correct run looks like for THIS specific input, carries no pass/fail of its own, and is checked against a later replay. A trace holds at most one verdict per author and any number of expectations. Call it when the user describes the right answer for a particular case, for example "this booking should have picked the 6am flight, not the 9am". This is not a verdict on a run that already happened, which is save_agent_labels, and it is not a check that applies to every trace of a function, which is save_grader. Pass \`assertion\` plus optional \`passCriteria\` / \`failCriteria\`, the same trio save_grader takes, so an expectation that proves out across many traces can later be promoted into a grader with no rewriting. Pass an entry's \`id\` to edit an existing expectation, or omit it to add a new one, so two callers adding different expectations to one trace never overwrite each other. Read them back with get_trace_assertions before judging a replay. Up to ${MAX_ASSERTIONS_PER_REQUEST} per call.`,
+  inputSchema: {
+    traceId: external_exports.uuid().describe("The ORIGINAL trace to attach expectations to, never a replay trace. A replay reads its original's expectations automatically, so writing them onto a replay trace pins them to one run instead of to the case."),
+    assertions: external_exports.preprocess(parseJsonString, external_exports.array(external_exports.object({
+      id: external_exports.uuid().optional().describe("Id of an existing expectation to edit, from a previous save or from get_trace_assertions. Omit to add a new one."),
+      assertion: external_exports.string().min(1).describe("The single thing that must hold, in one sentence, stated so a reader who has never seen this trace could check it, e.g. 'The itinerary returned lands before 9am local time'. One claim per expectation: if you are about to write 'and', write two expectations instead, so each can pass or fail on its own."),
+      passCriteria: external_exports.string().optional().describe("How a judge should recognise a pass, when the assertion alone leaves room to argue, e.g. 'arrival timestamp is strictly before 09:00 in the destination timezone'. Optional, and only worth writing when it removes real ambiguity. Same field save_grader takes, so an expectation that proves out across many traces is promoted into a grader by copying it. On an edit, omit to keep the current value and pass an empty string to clear it."),
+      failCriteria: external_exports.string().optional().describe("How a judge should recognise a failure, for cases the pass criteria do not obviously exclude, e.g. 'any leg departing after 09:00, including connections'. Optional. On an edit, omit to keep the current value and pass an empty string to clear it."),
+      targetOnEvaluatedTrace: assertionTargetShape.optional()
+    })).min(1).max(MAX_ASSERTIONS_PER_REQUEST)).describe(`One entry per expectation (1-${MAX_ASSERTIONS_PER_REQUEST})`)
+  }
+};
+var getTraceAssertions = {
+  name: "get_trace_assertions",
+  title: "Get Trace Expectations",
+  description: `Read what SHOULD happen when one or more traces are replayed: each trace's assertions, their pass/fail criteria, and what part of the evaluated trace each one checks. Call this before judging a replay so the verdict is measured against what the user actually asked for rather than a guess. No span content is loaded, so one call accepts up to ${GET_TRACE_ASSERTIONS_MAX_IDS} ids. Accepts original trace ids and replay trace ids alike: a replay with no expectations of its own reads its original's, and the response says which original they came from. An expectation whose target cannot be found on the trace being evaluated is errored, never passed, so record it with save_agent_labels \`skip\` rather than a FAIL.`,
+  inputSchema: {
+    traceIds: external_exports.preprocess(parseJsonString, external_exports.array(external_exports.uuid()).min(1).max(GET_TRACE_ASSERTIONS_MAX_IDS)).describe(`Original trace IDs to read expectations for (1-${GET_TRACE_ASSERTIONS_MAX_IDS})`)
+  }
+};
 var saveGrader = {
   name: "save_grader",
   title: "Save Grader",
@@ -24304,6 +24341,8 @@ var ALL_TOOL_CONTRACTS = [
   getSpanField,
   saveAgentLabels,
   saveHumanLabels,
+  saveTraceAssertions,
+  getTraceAssertions,
   saveGrader,
   listGraders,
   saveDataset,
@@ -26936,10 +26975,10 @@ var semver3 = __toESM(require_semver2(), 1);
 
 // ../bitfab-plugin-lib/dist/bakedSdkVersions.js
 var BAKED_SDK_VERSIONS = {
-  typescript: "0.44.0",
-  python: "0.43.0",
-  ruby: "0.43.0",
-  go: "0.41.0"
+  typescript: "0.45.0",
+  python: "0.45.0",
+  ruby: "0.43.1",
+  go: "0.41.1"
 };
 
 // ../bitfab-plugin-lib/dist/installedSdk.js
@@ -32827,9 +32866,9 @@ cd <project-dir> && {{command:replayProgress}} --label <pipeline-name> --run-dir
 
 3. **Run the live replay/evaluation loop while the replay is still running.** Use the run directory you passed with \`--run-dir\` and poll \`<run-dir>/events.jsonl\` alongside the human output. Each \`type: "progress"\` JSONL row is a normalized SDK item-finish event; \`item.originalTraceId\` is the **original trace id, the key verdicts are persisted by** (every item file carries it under the same \`originalTraceId\` field, plus a deprecated \`sourceTraceId\` alias). On modern SDKs, \`item.traceId\` may already hold the server replay trace ID after the per-item flush; it is null only when that flush/readback could not confirm the ID, and the final persistence barrier/result fills it when possible. \`item.replayTraceId\` is only set by older SDKs. Treat a new successful progress row as "this original trace's replay item finished" and start evaluation for that item as soon as the row carries \`item.itemPath\` and an \`item.originalTraceId\`: read \`item.itemPath\` for the full input, replay output, original output, and metadata. Failed progress rows become unreplayable candidates immediately and should be carried forward with their error string. If the row is progress-only (old SDK/current basic reporter: no \`item.itemPath\`), do the cheap prep only and defer judging to the final \`type: "complete"\` row's item refs. If the JSONL file is missing or empty (old SDK, old replay script, unwired \`onItemFinish\` / \`on_item_finish\`, or an unwritable log path), do not block or treat it as a replay failure: continue from the final complete event exactly as older scripts did from the old result file.
 
-   For every item you can judge during the run, key the verdict by its **original trace id** (the \`originalTraceId\` field in each item file, the same \`item.originalTraceId\` on progress rows), the original trace the item was replayed from, which the server resolves to this run's replay trace, no local-to-server id mapping step. You need the run's \`testRunId\` to persist (enriched progress rows may carry \`event.testRunId\`; older scripts may only reveal it in the final \`ReplayResult\`). As soon as a small batch of judged items has its original trace ids and you know the \`testRunId\`, persist that batch with \`{{command:persistReplayLabels}}\`. Keep a set of original trace IDs already persisted so the final reconciliation never double-writes a verdict.
+   For every item you can judge during the run, key the verdict by its **original trace id** (the \`originalTraceId\` field in each item file, the same \`item.originalTraceId\` on progress rows), the original trace the item was replayed from, which the server resolves to this run's replay trace, no local-to-server id mapping step. You need the run's \`testRunId\` to persist (enriched progress rows may carry \`event.testRunId\`; older scripts may only reveal it in the final \`ReplayResult\`). As soon as a small batch of judged items has its original trace ids and you know the \`testRunId\`, persist that batch with \`{{command:persistReplayLabels}}\`. Keep a set of original trace IDs already persisted so the final reconciliation never double-writes a verdict. A verdict judged live was judged before the item's outlines existed (\`originalTraceOutline\` and \`traceOutline\` are \`null\` on progress item files), so also keep the set of original trace ids judged without outlines; step 4 re-checks those against the final item files.
 
-4. **When the background command finishes, read the final \`type: "complete"\` row from this run's \`events.jsonl\`**. Its \`result\` carries run metadata (\`testRunId\`, \`testRunUrl\`, \`itemCount\`) and its \`items\` array carries item refs. Read each needed \`items[].itemPath\` for the full replay item (trace ID, duration, tokens, model, and full original/new outputs). Read from the **files**, not from the captured command output, which the harness truncates in the middle.
+4. **When the background command finishes, read the final \`type: "complete"\` row from this run's \`events.jsonl\`**. Its \`result\` carries run metadata (\`testRunId\`, \`testRunUrl\`, \`itemCount\`) and its \`items\` array carries item refs. Read each needed \`items[].itemPath\` for the full replay item (trace ID, duration, tokens, model, full original/new outputs, and the two trace outlines). Read from the **files**, not from the captured command output, which the harness truncates in the middle. On current SDKs each completed item also carries \`originalTraceOutline\` and \`traceOutline\`: the original and the replayed trace's span tree (span names, types, nesting, order, durations, tokens, model, errors, and whether each span was mocked) with no inputs or outputs. The SDK fills both in at completion from the server, so they are \`null\` on live progress rows and on older SDKs. Use them to judge execution shape (did the replay call the same tools in the same order, did a span that used to run get mocked or skipped, did a child span error) straight from the item file, instead of reading both traces back from the server for that comparison. **Re-check every item you judged live:** its verdict was derived before the outlines existed, so re-read its final item file (the complete row's write fills both outlines) and run the execution-shape comparison now. When that comparison changes the verdict, include the corrected \`{ originalTraceId, label, annotation }\` in the next \`{{command:persistReplayLabels}}\` batch: a new agent verdict for the same original trace replaces the earlier one in place, so this correction is the one case where re-persisting an already-persisted id is right. When it does not change the verdict, leave the persisted verdict alone.
 
 **Before running: verify the replay script prints the full original and new output values AND at least one verdict persist key (\`item.originalTraceId\` or \`item.traceId\`) for every item** (not just lengths, counts, hashes, or truncated previews) so the run's \`items/*.json\` files carry them. Modern items may carry both: prefer lineage persistence by \`item.originalTraceId\` (older SDKs may print it under the deprecated \`sourceTraceId\` alias), and fall back to the server replay \`item.traceId\` only when original lineage is absent. After an item's replay trace is flushed, its \`item.traceId\` is available to lifecycle callbacks and in the final result. The oldest SDKs have no \`item.originalTraceId\` and persist by \`item.traceId\`. If the script prints neither, fix it first; the Replay Output Contract and example script live in the SDK reference at \`https://docs.bitfab.ai/<language>-sdk.md\`. Subagents can't evaluate an improvement from \`5 \u2192 7 (+2)\`, and an item that carries no persist key blocks verdict persistence for that item.
 
@@ -33050,7 +33089,7 @@ When the server reports \`completed\` with \`traceCount > 0\`, call {{tool:listE
 
 If \`replay-against-dataset\` already consumed a non-empty run \`events.jsonl\`, reuse that progress-derived work and any verdicts already produced from item files referenced by progress rows. Successful progress events identify which original traces finished, and failed events already define unreplayable candidates. Do not reclassify those failures as output regressions. If the progress file is missing or empty, fall back to the completed replay output; this is expected for old SDKs or replay scripts without \`onItemFinish\` / \`on_item_finish\`. Since no persist key is available on this path, verdicts can't be persisted and stay in working context only.
 
-For each completed (non-errored) replay item, derive a verdict by comparing the replay's new output against the original trace's label and annotation:
+For each completed (non-errored) replay item, derive a verdict by comparing the replay's new output against the original trace's label and annotation, and when the item file carries \`originalTraceOutline\` and \`traceOutline\`, also compare the two span trees so a replay that produced the right text by skipping a required tool call, erroring in a child span, or leaning on a mocked span that used to run real code does not pass on output alone:
 
 - **fail**-labeled original: does the replay's new output address the annotation? If yes, mark as PASS. If no, mark as FAIL.
 - **pass**-labeled original: preserved means PASS, regressed means FAIL.
@@ -33084,11 +33123,11 @@ Hold the verdicts in working context for the final report, the \`share-results\`
 
 **The verdict is decided by the output against its criteria, never by your opinion of how informative, useful, or production-like the run is.** Environmental doubts, a mock/stub model, no real gateway, flaky infra, an output that "looks like noise", a worry that the labels would pollute the dataset, are **caveats only**: write them into the \`annotation\` and lower the \`confidence\` (e.g. \`Low\` / \`VeryLow\`). They never suppress a verdict and never skip this step. A low-confidence verdict with a caveat is still a verdict and still gets persisted.
 
-**Finalize the live evaluation loop; do not start from scratch.** If \`replay-against-dataset\` produced a non-empty run \`events.jsonl\`, read it before judging and reuse any prep, unreplayable buckets, verdicts, and persisted original trace IDs already produced during the running replay. Each progress event's \`item.originalTraceId\` is the original trace id: use it both to identify which dataset item settled and as the key each verdict is persisted by (item files carry it under the same \`originalTraceId\` field, plus a deprecated \`sourceTraceId\` alias). Enriched progress rows carry \`item.itemPath\`; the item file has the full input, replay output, original output, and metadata needed to judge during replay. Progress-only rows are only a trigger/prep signal. If the file is missing or empty, continue from the final complete event; older scripts remain valid and simply do not get incremental evaluation. The final verdict still comes from the completed replay item files / experiment trace data, and persistence is still keyed by the **original trace** id.
+**Finalize the live evaluation loop; do not start from scratch.** If \`replay-against-dataset\` produced a non-empty run \`events.jsonl\`, read it before judging and reuse any prep, unreplayable buckets, verdicts, and persisted original trace IDs already produced during the running replay. Each progress event's \`item.originalTraceId\` is the original trace id: use it both to identify which dataset item settled and as the key each verdict is persisted by (item files carry it under the same \`originalTraceId\` field, plus a deprecated \`sourceTraceId\` alias). Enriched progress rows carry \`item.itemPath\`; the item file has the full input, replay output, original output, and metadata needed to judge during replay (its \`originalTraceOutline\` and \`traceOutline\` are still \`null\` mid-run and are filled by the final write, so an execution-shape comparison waits for the complete row). Progress-only rows are only a trigger/prep signal. If the file is missing or empty, continue from the final complete event; older scripts remain valid and simply do not get incremental evaluation. The final verdict still comes from the completed replay item files / experiment trace data, and persistence is still keyed by the **original trace** id.
 
-When possible, evaluate and persist in small completed batches rather than waiting to judge every item at once: every time a progress event gives you a replay item ref with an original trace id, full item file, and its original label/annotation, read the item file, derive that item's verdict, and append it to the pending verdict set. Once a batch has complete coverage for its expected original trace ids, call \`persistReplayLabels.js\` for that batch. If the current replay script/SDK only exposes replay outputs at the end, use progress during the run for preparation and failed-item bucketing, then do the first persist immediately after this run's \`complete\` row appears. At the end of the replay, run the same coverage check over the complete row's item refs; any item not already persisted must be judged and persisted before continuing. This keeps Studio's experiment view filling in as early as the available data allows while preserving the same final correctness checks.
+When possible, evaluate and persist in small completed batches rather than waiting to judge every item at once: every time a progress event gives you a replay item ref with an original trace id, full item file, and its original label/annotation, read the item file, derive that item's verdict, and append it to the pending verdict set. Once a batch has complete coverage for its expected original trace ids, call \`persistReplayLabels.js\` for that batch. If the current replay script/SDK only exposes replay outputs at the end, use progress during the run for preparation and failed-item bucketing, then do the first persist immediately after this run's \`complete\` row appears. At the end of the replay, run the same coverage check over the complete row's item refs; any item not already persisted must be judged and persisted before continuing, and any item judged live before its outlines existed must be re-checked against the final item file's \`originalTraceOutline\` and \`traceOutline\`, re-persisting only when that comparison changes the verdict. This keeps Studio's experiment view filling in as early as the available data allows while preserving the same final correctness checks.
 
-For each completed (non-errored) replay item, derive a verdict by comparing the replay's new output against the original trace's label and annotation (from Phase 3 in \`wizard\`/\`dataset\` modes, loaded by \`pick-dataset\` in Phase 5 Setup in \`experiment\` and \`benchmark\` modes; in \`fix\` mode's single-trace pass, the failure annotation held from Phase Fix \`resolve\` is the criterion, and on a later full-dataset run the added trace plus any pre-existing dataset siblings loaded by \`fix-add-to-dataset\`):
+For each completed (non-errored) replay item, derive a verdict by comparing the replay's new output against the original trace's label and annotation, and when the item file carries \`originalTraceOutline\` and \`traceOutline\`, also compare the two span trees so a replay that produced the right text by skipping a required tool call, erroring in a child span, or leaning on a mocked span that used to run real code does not pass on output alone (from Phase 3 in \`wizard\`/\`dataset\` modes, loaded by \`pick-dataset\` in Phase 5 Setup in \`experiment\` and \`benchmark\` modes; in \`fix\` mode's single-trace pass, the failure annotation held from Phase Fix \`resolve\` is the criterion, and on a later full-dataset run the added trace plus any pre-existing dataset siblings loaded by \`fix-add-to-dataset\`):
 
 - **fail**-labeled original: does the replay's new output address the annotation? If yes \u2192 \`label: true\` (PASS). If no \u2192 \`label: false\` (FAIL). Use the annotation as the acceptance criterion.
 - **pass**-labeled original: preserved \u2192 \`label: true\` (PASS). regressed \u2192 \`label: false\` (FAIL).

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bitfab-cli",
-  "version": "0.2.308",
+  "version": "0.2.310",
   "description": "Install and configure the Bitfab plugin in Claude Code, Codex, or Cursor, or the Bitfab skill pack in Amp (Alpha).",
   "type": "module",
   "main": "dist/index.js",