bitfab-cli 0.2.332 → 0.2.333

package/dist/index.js

@@ -23953,7 +23953,7 @@ var getSpanField = {
 var saveAgentLabels = {
   name: "save_agent_labels",
   title: "Save Agent Labels",
-  description: "Set, skip, or archive the agent's pass/fail verdict on one or more traces (`labelSource=\"agent\"`). Use this AFTER you have read the traces with get_traces and decided yourself whether each one looks like a pass, a fail, or genuinely cannot be judged. To set a verdict, pass `label` (true=PASS, false=FAIL) and `annotation` (your reasoning, shown to the human reviewer in the labeling UI). Optionally pass `confidence` (`VeryLow|Low|Medium|High|VeryHigh`) to record how confident you are - surfaced in the labeling UI so reviewers can prioritize low-confidence verdicts. To explicitly skip a trace you cannot decide on (instead of leaving it unlabeled), pass `skip: true` and omit label/annotation - this records an explicit skip so coverage checks know you intentionally did not verdict it. To clear a previously-set agent verdict (e.g., you changed your mind or labeled the wrong trace), pass `archive: true` and omit label/annotation. New verdicts start unapproved (`approvedAt=null`); once a human approves one, it joins the validated dataset (queryable via `search_traces` with `validated: true`). Archiving is non-destructive - the row is hidden from all reads but kept for audit, and you can immediately re-label the trace from scratch. For replay results, key each label by the replay item's `originalTraceId` (the original trace it was replayed from; `sourceTraceId` is accepted as a deprecated alias) plus the top-level `testRunId` instead of a `traceId`: the server resolves it to the replay trace via lineage, so you never need a server-generated replay trace id. When the experiment ran each trace more than once, also pass the item's `attempt` so each attempt gets its own verdict. Returns an agent-readable summary with one parseable effective label line per updated trace, keyed by the id you supplied (`originalTraceId` for replay verdicts, otherwise `traceId`), so command callers can verify persistence. Before judging a replay, call get_trace_assertions on the original trace ids: an expectation says what the user asked this specific case to do, so the verdict is measured against that rather than a guess. When an expectation's target cannot be found on the trace you are judging, the check never ran, so pass `skip: true` for that trace rather than recording a FAIL. Pass `assertionId` to score ONE assertion on the trace (the `[ID: ...]` get_trace_assertions prints), one entry per assertion, all in the same call. Omit it for the trace's single whole-trace verdict. get_trace_labels reads these back one line per assertion, keyed by the same assertionId, so a per-assertion write is verifiable per assertion.",
+  description: "Set, skip, or archive the agent's pass/fail verdict on one or more traces (`labelSource=\"agent\"`). Use this AFTER you have read the traces with get_traces and decided yourself whether each one looks like a pass, a fail, or genuinely cannot be judged. To set a verdict, pass `label` (true=PASS, false=FAIL) and `annotation` (your reasoning, shown to the human reviewer in the labeling UI). Optionally pass `confidence` (`VeryLow|Low|Medium|High|VeryHigh`) to record how confident you are - surfaced in the labeling UI so reviewers can prioritize low-confidence verdicts. To explicitly skip a trace you cannot decide on (instead of leaving it unlabeled), pass `skip: true` and omit label/annotation - this records an explicit skip so coverage checks know you intentionally did not verdict it. To clear a previously-set agent verdict (e.g., you changed your mind or labeled the wrong trace), pass `archive: true` and omit label/annotation. New verdicts start unapproved (`approvedAt=null`); once a human approves one, it joins the validated dataset (queryable via `search_traces` with `validated: true`). Archiving is non-destructive - the row is hidden from all reads but kept for audit, and you can immediately re-label the trace from scratch. For replay results, key each label by the replay item's `originalTraceId` (the original trace it was replayed from; `sourceTraceId` is accepted as a deprecated alias) plus the top-level `testRunId` instead of a `traceId`: the server resolves it to the replay trace via lineage, so you never need a server-generated replay trace id. When the experiment ran each trace more than once, also pass the item's `attempt` so each attempt gets its own verdict. Returns an agent-readable summary with one parseable effective label line per updated trace, keyed by the id you supplied (`originalTraceId` for replay verdicts, otherwise `traceId`), so command callers can verify persistence. Before judging a replay, call get_trace_assertions on the original trace ids: an assertion says what the user asked this specific case to do, so the verdict is measured against that rather than a guess. When an assertion's target cannot be found on the trace you are judging, the check never ran, so pass `skip: true` for that trace rather than recording a FAIL. Pass `assertionId` to score ONE assertion on the trace (the `[ID: ...]` get_trace_assertions prints), one entry per assertion, all in the same call. Omit it for the trace's single whole-trace verdict. get_trace_labels reads these back one line per assertion, keyed by the same assertionId, so a per-assertion write is verifiable per assertion.",
   inputSchema: {
     testRunId: external_exports.uuid().optional().describe("The replay test run id. Required when any label targets a trace by originalTraceId (replay verdicts); ignored otherwise."),
     labels: external_exports.preprocess(parseJsonString, external_exports.array(external_exports.object({
@@ -23998,34 +23998,34 @@ var assertionTargetShape = external_exports.discriminatedUnion("kind", [
 ]).describe('SCOPE: what part of the trace under evaluation this assertion is checked against. OMIT IT for the whole trace, which is the common case and the right default when unsure. Pass { "kind": "output" } to check only the final output, or { "kind": "span", "name": "..." } to check one span. Scoping narrows what a judge looks at, so a wrong scope hides real failures. A target naming something the evaluated trace does not contain makes the check ERRORED, never passed, because a check that could not run must never look like a check that succeeded.');
 var saveTraceAssertions = {
   name: "save_trace_assertions",
-  title: "Save Trace Expectations",
-  description: `Record what SHOULD happen when this trace is replayed. Bitfab stores two kinds of label on a trace. A VERDICT says how a run that already happened turned out, and that is save_agent_labels / save_human_labels. An EXPECTATION, which is what this tool writes, says what a correct run looks like for THIS specific input, carries no pass/fail of its own, and is checked against a later replay. A trace holds at most one verdict per author and any number of expectations. Call it when the user describes the right answer for a particular case, for example "this booking should have picked the 6am flight, not the 9am". This is not a verdict on a run that already happened, which is save_agent_labels, and it is not a check that applies to every trace of a function, which is save_grader. Pass \`assertion\` plus optional \`passCriteria\` / \`failCriteria\`, the same trio save_grader takes, so an expectation that proves out across many traces can later be promoted into a grader with no rewriting. Pass an entry's \`id\` to edit an existing expectation, or omit it to add a new one, so two callers adding different expectations to one trace never overwrite each other. Read them back with get_trace_assertions before judging a replay. Up to ${MAX_ASSERTIONS_PER_REQUEST} per call.`,
+  title: "Save Trace Assertions",
+  description: `Record what SHOULD happen when this trace is replayed. Bitfab stores two kinds of label on a trace. A VERDICT says how a run that already happened turned out, and that is save_agent_labels / save_human_labels. An ASSERTION, which is what this tool writes, says what a correct run looks like for THIS specific input, carries no pass/fail of its own, and is checked against a later replay. A trace holds at most one verdict per author and any number of assertions. Call it when the user describes the right answer for a particular case, for example "this booking should have picked the 6am flight, not the 9am". This is not a verdict on a run that already happened, which is save_agent_labels, and it is not a check that applies to every trace of a function, which is save_grader. Pass \`assertion\` plus optional \`passCriteria\` / \`failCriteria\`, the same trio save_grader takes, so an assertion that proves out across many traces can later be promoted into a grader with no rewriting. Pass an entry's \`id\` to edit an existing assertion, or omit it to add a new one, so two callers adding different assertions to one trace never overwrite each other. Call get_trace_assertions first to find the assertion to edit and to avoid creating a duplicate, and read them back before judging a replay. Up to ${MAX_ASSERTIONS_PER_REQUEST} per call.`,
   inputSchema: {
-    traceId: external_exports.uuid().describe("The ORIGINAL trace to attach expectations to, never a replay trace. A replay reads its original's expectations automatically, so writing them onto a replay trace pins them to one run instead of to the case."),
+    traceId: external_exports.uuid().describe("The ORIGINAL trace to attach assertions to. A replay trace id is refused, and the error names the original to retry with, because a replay reads its original's assertions automatically."),
     assertions: external_exports.preprocess(parseJsonString, external_exports.array(external_exports.object({
-      id: external_exports.uuid().optional().describe("Id of an existing expectation to edit, from a previous save or from get_trace_assertions. Omit to add a new one."),
-      assertion: external_exports.string().min(1).describe("The single thing that must hold, in one sentence, stated so a reader who has never seen this trace could check it, e.g. 'The itinerary returned lands before 9am local time'. One claim per expectation: if you are about to write 'and', write two expectations instead, so each can pass or fail on its own."),
-      passCriteria: external_exports.string().optional().describe("How a judge should recognise a pass, when the assertion alone leaves room to argue, e.g. 'arrival timestamp is strictly before 09:00 in the destination timezone'. Optional, and only worth writing when it removes real ambiguity. Same field save_grader takes, so an expectation that proves out across many traces is promoted into a grader by copying it. On an edit, omit to keep the current value and pass an empty string to clear it."),
+      id: external_exports.uuid().optional().describe("Id of an existing assertion to edit, from a previous save or from get_trace_assertions. Omit to add a new one."),
+      assertion: external_exports.string().min(1).describe("The single thing that must hold, in one sentence, stated so a reader who has never seen this trace could check it, e.g. 'The itinerary returned lands before 9am local time'. One claim per assertion: if you are about to write 'and', write two assertions instead, so each can pass or fail on its own."),
+      passCriteria: external_exports.string().optional().describe("How a judge should recognise a pass, when the assertion alone leaves room to argue, e.g. 'arrival timestamp is strictly before 09:00 in the destination timezone'. Optional, and only worth writing when it removes real ambiguity. Same field save_grader takes, so an assertion that proves out across many traces is promoted into a grader by copying it. On an edit, omit to keep the current value and pass an empty string to clear it."),
       failCriteria: external_exports.string().optional().describe("How a judge should recognise a failure, for cases the pass criteria do not obviously exclude, e.g. 'any leg departing after 09:00, including connections'. Optional. On an edit, omit to keep the current value and pass an empty string to clear it."),
       targetOnEvaluatedTrace: assertionTargetShape.optional()
-    })).min(1).max(MAX_ASSERTIONS_PER_REQUEST)).describe(`One entry per expectation (1-${MAX_ASSERTIONS_PER_REQUEST})`)
+    })).min(1).max(MAX_ASSERTIONS_PER_REQUEST)).describe(`One entry per assertion (1-${MAX_ASSERTIONS_PER_REQUEST})`)
   }
 };
 var getTraceAssertions = {
   name: "get_trace_assertions",
-  title: "Get Trace Expectations",
-  description: `Read what SHOULD happen when one or more traces are replayed: each trace's assertions, their pass/fail criteria, and what part of the evaluated trace each one checks. Call this before judging a replay so the verdict is measured against what the user actually asked for rather than a guess. No span content is loaded, so one call accepts up to ${GET_TRACE_ASSERTIONS_MAX_IDS} ids. Accepts original trace ids and replay trace ids alike: a replay with no expectations of its own reads its original's, and the response says which original they came from. An expectation whose target cannot be found on the trace being evaluated is errored, never passed, so record it with save_agent_labels \`skip\` rather than a FAIL.`,
+  title: "Get Trace Assertions",
+  description: `Read what SHOULD happen when one or more traces are replayed: each trace's assertions, their pass/fail criteria, and what part of the evaluated trace each one checks. Call this before judging a replay so the verdict is measured against what the user actually asked for rather than a guess, and before save_trace_assertions to find the assertion to edit or to avoid creating a duplicate. No span content is loaded, so one call accepts up to ${GET_TRACE_ASSERTIONS_MAX_IDS} ids. Accepts original trace ids and replay trace ids alike: a replay with no assertions of its own reads its original's, and the response says which original they came from. An assertion whose target cannot be found on the trace being evaluated is errored, never passed, so record it with save_agent_labels \`skip\` rather than a FAIL.`,
   inputSchema: {
-    traceIds: external_exports.preprocess(parseJsonString, external_exports.array(external_exports.uuid()).min(1).max(GET_TRACE_ASSERTIONS_MAX_IDS)).describe(`Original trace IDs to read expectations for (1-${GET_TRACE_ASSERTIONS_MAX_IDS})`)
+    traceIds: external_exports.preprocess(parseJsonString, external_exports.array(external_exports.uuid()).min(1).max(GET_TRACE_ASSERTIONS_MAX_IDS)).describe(`Original trace IDs to read assertions for (1-${GET_TRACE_ASSERTIONS_MAX_IDS})`)
   }
 };
 var archiveTraceAssertions = {
   name: "archive_trace_assertions",
-  title: "Archive Trace Expectations",
-  description: `Retire expectations on a trace so later replays stop checking them. Non-destructive: the row is hidden from get_trace_assertions and from replay judging but kept for audit, and verdicts already recorded against it stay. Use it for an expectation that was wrong or superseded; to reword one instead, pass its id back to save_trace_assertions, which edits in place. Pass the ids get_trace_assertions prints as \`[ID: ...]\`. All-or-nothing: one unknown, already-archived, or wrong-trace id fails the call naming that id and archives nothing, so re-read and retry with the active ids. Up to ${MAX_ASSERTIONS_PER_REQUEST} per call.`,
+  title: "Archive Trace Assertions",
+  description: `Retire assertions on a trace so later replays stop checking them. Non-destructive: the row is hidden from get_trace_assertions and from replay judging but kept for audit, and verdicts already recorded against it stay. Use it for an assertion that was wrong or superseded; to reword one instead, pass its id back to save_trace_assertions, which edits in place. Pass the ids get_trace_assertions prints as \`[ID: ...]\`. All-or-nothing: one unknown, already-archived, or wrong-trace id fails the call naming that id and archives nothing, so re-read and retry with the active ids. Up to ${MAX_ASSERTIONS_PER_REQUEST} per call.`,
   inputSchema: {
-    traceId: external_exports.uuid().describe("The trace the expectations are attached to. This is the ORIGINAL trace they were written on, not a replay that inherited them."),
-    assertionIds: external_exports.preprocess(parseJsonString, external_exports.array(external_exports.uuid()).min(1).max(MAX_ASSERTIONS_PER_REQUEST)).describe(`Ids of the expectations to archive (1-${MAX_ASSERTIONS_PER_REQUEST}), from get_trace_assertions`)
+    traceId: external_exports.uuid().describe("The trace the assertions are attached to. This is the ORIGINAL trace they were written on, not a replay that inherited them."),
+    assertionIds: external_exports.preprocess(parseJsonString, external_exports.array(external_exports.uuid()).min(1).max(MAX_ASSERTIONS_PER_REQUEST)).describe(`Ids of the assertions to archive (1-${MAX_ASSERTIONS_PER_REQUEST}), from get_trace_assertions`)
   }
 };
 var saveGrader = {
@@ -31096,7 +31096,7 @@ var FAN_OUT_JUDGING = `{{#claude}}**Scale the judging with fan-out when there ar
 Make each subagent prompt fully self-contained: its batch's per-item payloads (each item carries its own artifacts, enumerated below), the fixed rubric, and any shared context you gathered once (so no subagent re-derives it or touches the repo). Tell it to return one verdict entry per item in the exact shape this step persists, and nothing else.
 
 **Then collect and persist once.** Wait for every batch, concatenate their verdict arrays into the single full set covering all items, and make the one batched persist call this step already describes, unchanged. Fan-out changes only how you produce the verdicts, never how they are stored or routed: same call, same shape, same buckets, same downstream steps.{{/claude}}{{^claude}}**Judge serially.** This editor doesn't use subagents for judging, so judge every item yourself, inline in this agent, as described above, regardless of how many there are.{{/claude}}`;
-var PER_ASSERTION_VERDICTS = `**Read each trace's assertions first, then write one verdict per assertion.** An assertion says what a correct run looks like for that one case, and it is checked against the trace under evaluation. That is what the {{tool:getTraceAssertions}} call in this step is for, and one call covers up to 100 trace IDs, so make it once and before you judge anything. Each assertion comes back on its own line under its trace as \`[ID: <uuid>] checks <target>: <assertion>\`, with its pass and fail criteria in parentheses when it has them. Hold that \`[ID: <uuid>]\` value, it is the \`assertionId\` every verdict on that assertion carries. A trace with no assertions comes back as "no expectations recorded", which is that tool's wording for the same thing.
+var PER_ASSERTION_VERDICTS = `**Read each trace's assertions first, then write one verdict per assertion.** An assertion says what a correct run looks like for that one case, and it is checked against the trace under evaluation. That is what the {{tool:getTraceAssertions}} call in this step is for, and one call covers up to 100 trace IDs, so make it once and before you judge anything. Each assertion comes back on its own line under its trace as \`[ID: <uuid>] checks <target>: <assertion>\`, with its pass and fail criteria in parentheses when it has them. Hold that \`[ID: <uuid>]\` value, it is the \`assertionId\` every verdict on that assertion carries. A trace with no assertions comes back as "no assertions recorded", which is that tool's wording for the same thing.
 
 - **A trace that has assertions:** send one {{tool:saveAgentLabels}} entry per assertion, each carrying that assertion's \`assertionId\`, its own \`label\` (\`true\` for passed, \`false\` for failed), and its own \`annotation\`, which is the reasoning for that one assertion and nothing else. Judge each assertion on its own evidence. **Write no whole-trace verdict for that trace.** The trace verdict is derived from the per-assertion labels, so a whole-trace entry alongside them would compete with the rows it is derived from.
 - **A trace with no assertions:** nothing changes. Write the one whole-trace verdict this step has always written, \`{ traceId, label, annotation }\` with no \`assertionId\`.
@@ -33249,7 +33249,7 @@ ${FAN_OUT_JUDGING}
 
 In lineage keying, \`testRunId\` is this replay run's id (from the progress rows or the final \`ReplayResult\`). The server uses it to resolve each original trace to its replay trace within this run. The \`expected*\` list MUST be the full set of ids covered by this call's batch (and across all batches, every completed \`item.error\`-unset replay item must be persisted exactly once, no fewer, per the mandatory-coverage rule above). For the final end-of-run call, use only the ids not already successfully persisted by an earlier batch. \`verdicts\` MUST cover every id in the \`expected*\` list, keyed by the same id field as that list. A trace with no assertions gets exactly one entry, either a \`{label, annotation, confidence?}\` verdict or a \`{skip: true}\` explicit skip (skips allowed only for the three enumerated skip cases above, never for an environmental doubt). A trace that has assertions gets one entry per assertion instead, in the shapes the per-assertion block below fixes. \`confidence\` is optional but recommended (\`VeryLow|Low|Medium|High|VeryHigh\`). It surfaces in the labeling UI so reviewers can prioritize low-confidence verdicts. If any expected id gets no entry at all, the script returns \`status: "missing-coverage"\` and the verify step routes you back to fill the gaps.
 
-**A replay inherits the original trace's assertions, so score them one at a time.** Before you judge this batch, call {{tool:getTraceAssertions}} once with the batch's **original** trace ids (one call covers up to 100 ids, so make it once and before you judge anything). Each assertion comes back under its trace as \`[ID: <uuid>] checks <target>: <assertion>\`, and that \`[ID: <uuid>]\` value is the \`assertionId\` its verdict carries. A trace that comes back "no expectations recorded" has none.
+**A replay inherits the original trace's assertions, so score them one at a time.** Before you judge this batch, call {{tool:getTraceAssertions}} once with the batch's **original** trace ids (one call covers up to 100 ids, so make it once and before you judge anything). Each assertion comes back under its trace as \`[ID: <uuid>] checks <target>: <assertion>\`, and that \`[ID: <uuid>]\` value is the \`assertionId\` its verdict carries. A trace that comes back "no assertions recorded" has none.
 
 - **The original had assertions:** write one entry per assertion, each carrying that assertion's \`assertionId\`, its own \`label\`, and its own \`annotation\` covering that one assertion and nothing else. **Write no whole-trace entry for that trace.** The trace verdict is derived from the per-assertion rows, so an entry beside them contradicts the rows it comes from, and the script rejects the whole file with \`status: "invalid-input"\`.
 - **The original had none:** nothing changes. Write the single whole-trace entry with no \`assertionId\`, exactly as before.
@@ -33749,7 +33749,7 @@ This is a single-trace, in-chat path: run the replay directly, no progress-bar w
 
 There is no verdict to persist for an errored item. Offer a retry only after the diagnosed cause is addressed, or offer to stop.
 
-**If the replay completed**, call {{tool:getTraceAssertions}} with the ORIGINAL trace id first. An assertion says what the user asked this one case to do, and the replay inherits the original's assertions, so it is what the new output is measured against. Each one comes back as \`[ID: <uuid>] checks <target>: <assertion>\`, and that \`[ID: <uuid>]\` value is the \`assertionId\` its verdict carries. "no expectations recorded" means the trace has none, and everything below reads exactly as it always has.
+**If the replay completed**, call {{tool:getTraceAssertions}} with the ORIGINAL trace id first. An assertion says what the user asked this one case to do, and the replay inherits the original's assertions, so it is what the new output is measured against. Each one comes back as \`[ID: <uuid>] checks <target>: <assertion>\`, and that \`[ID: <uuid>]\` value is the \`assertionId\` its verdict carries. "no assertions recorded" means the trace has none, and everything below reads exactly as it always has.
 
 Then compare the new output against the original trace's assertions, label, and annotation, and report one line:
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bitfab-cli",
-  "version": "0.2.332",
+  "version": "0.2.333",
   "description": "Install and configure the Bitfab plugin in Claude Code, Codex, or Cursor, or the Bitfab skill pack in Amp (Alpha).",
   "type": "module",
   "main": "dist/index.js",