bitfab-cli 0.2.363 → 0.2.365

package/dist/index.js

@@ -23537,11 +23537,20 @@ var getTraces = {
 var getTraceLabels = {
   name: "get_trace_labels",
   title: "Get Trace Labels",
-  description: `Read just the labels for one or more traces by ID: returns each trace's pass/fail verdict, its annotation (the reviewer's reasoning), and whether the label is human-validated (a human authored or approved it). Per-assertion verdicts follow on indented lines, one per assertion, keyed by the assertionId the write used, each with its own verdict, annotation, confidence, and author, so a verdict written per assertion reads back per assertion rather than as a tally. No span content, inputs, or outputs are loaded, so the response is small and one call accepts up to ${GET_TRACE_LABELS_MAX_IDS} IDs. Use this to load a whole dataset's verdicts into context in a single call (e.g. when building or confirming a dataset). When you need a trace's actual inputs/outputs/spans, use get_traces instead. This returns verdicts on runs that already happened; for what a trace SHOULD do on its next replay, use get_trace_assertions.`,
+  description: `Read just the labels for one or more traces by ID: returns each trace's pass/fail verdict, its annotation (the reviewer's reasoning), its saved { spanId, text } evidence, and whether the label is human-validated (a human authored or approved it). Per-assertion verdicts follow on indented lines, one per assertion, keyed by the assertionId the write used, each with its own verdict, annotation, evidence, confidence, and author, so a verdict written per assertion reads back per assertion rather than as a tally. No span content, inputs, or outputs are loaded, so the response is small and one call accepts up to ${GET_TRACE_LABELS_MAX_IDS} IDs. Use this to load a whole dataset's verdicts into context in a single call (e.g. when building or confirming a dataset). When you need a trace's actual inputs/outputs/spans, use get_traces instead. This returns verdicts on runs that already happened; for what a trace SHOULD do on its next replay, use get_trace_assertions.`,
   inputSchema: {
     traceIds: external_exports.preprocess(parseJsonString, external_exports.array(external_exports.uuid()).min(1).max(GET_TRACE_LABELS_MAX_IDS)).describe(`Trace IDs to read labels for (1-${GET_TRACE_LABELS_MAX_IDS})`)
   }
 };
+var getLabelEvidence = {
+  name: "get_label_evidence",
+  title: "Get Label Evidence",
+  description: "Suggest span-by-span evidence that could support a label on a trace or one of its assertions. This discovery API is currently stubbed and returns an empty evidence list. Saved evidence is read with get_trace_labels instead.",
+  inputSchema: {
+    traceId: external_exports.uuid().describe("The trace whose spans could provide evidence"),
+    assertionId: external_exports.uuid().optional().describe("The assertion the potential evidence would support. Omit it for the whole-trace label.")
+  }
+};
 var getSpanField = {
   name: "get_span_field",
   title: "Get Span Field",
@@ -23553,10 +23562,14 @@ var getSpanField = {
     maxChars: external_exports.preprocess(parseJsonString, external_exports.number().int().positive()).optional().describe("Maximum characters to return (default 100000). Raise it to read further into an extremely large field.")
   }
 };
+var labelEvidenceShape = external_exports.preprocess(parseJsonString, external_exports.array(external_exports.object({
+  spanId: external_exports.uuid(),
+  text: external_exports.string().trim().min(1).max(1e4)
+})).min(1).max(50));
 var saveAgentLabels = {
   name: "save_agent_labels",
   title: "Save Agent Labels",
-  description: "Set, skip, or archive the agent's pass/fail verdict on one or more traces (`labelSource=\"agent\"`). Use this AFTER you have read the traces with get_traces and decided yourself whether each one looks like a pass, a fail, or genuinely cannot be judged. To set a verdict, pass `label` (true=PASS, false=FAIL) and `annotation` (your reasoning, shown to the human reviewer in the labeling UI). Optionally pass `confidence` (`VeryLow|Low|Medium|High|VeryHigh`) to record how confident you are - surfaced in the labeling UI so reviewers can prioritize low-confidence verdicts. To explicitly skip a trace you cannot decide on (instead of leaving it unlabeled), pass `skip: true` and omit label/annotation - this records an explicit skip so coverage checks know you intentionally did not verdict it. To clear a previously-set agent verdict (e.g., you changed your mind or labeled the wrong trace), pass `archive: true` and omit label/annotation. New verdicts start unapproved (`approvedAt=null`); once a human approves one, it joins the validated dataset (queryable via `search_traces` with `validated: true`). Archiving is non-destructive - the row is hidden from all reads but kept for audit, and you can immediately re-label the trace from scratch. For replay results, key each label by the replay item's `originalTraceId` (the original trace it was replayed from; `sourceTraceId` is accepted as a deprecated alias) plus the top-level `testRunId` instead of a `traceId`: the server resolves it to the replay trace via lineage, so you never need a server-generated replay trace id. When the experiment ran each trace more than once, also pass the item's `attempt` so each attempt gets its own verdict. Returns an agent-readable summary with one parseable effective label line per updated trace, keyed by the id you supplied (`originalTraceId` for replay verdicts, otherwise `traceId`), so command callers can verify persistence. Before judging a replay, call get_trace_assertions on the original trace ids: an assertion says what the user asked this specific case to do, so the verdict is measured against that rather than a guess. When an assertion's target cannot be found on the trace you are judging, the check never ran, so pass `skip: true` for that trace rather than recording a FAIL. Pass `assertionId` to score ONE assertion on the trace (the `[ID: ...]` get_trace_assertions prints), one entry per assertion, all in the same call. Omit it for the trace's single whole-trace verdict. get_trace_labels reads these back one line per assertion, keyed by the same assertionId, so a per-assertion write is verifiable per assertion.",
+  description: "Set, skip, or archive the agent's pass/fail verdict on one or more traces (`labelSource=\"agent\"`). Use this AFTER you have read the traces with get_traces and decided yourself whether each one looks like a pass, a fail, or genuinely cannot be judged. To set a verdict, pass `label` (true=PASS, false=FAIL) and `annotation` (your reasoning, shown to the human reviewer in the labeling UI). Add `evidence` as `{ spanId, text }` entries to ground that reasoning in spans on the evaluated trace. Optionally pass `confidence` (`VeryLow|Low|Medium|High|VeryHigh`) to record how confident you are - surfaced in the labeling UI so reviewers can prioritize low-confidence verdicts. To explicitly skip a trace you cannot decide on (instead of leaving it unlabeled), pass `skip: true` and omit label/annotation - this records an explicit skip so coverage checks know you intentionally did not verdict it. To clear a previously-set agent verdict (e.g., you changed your mind or labeled the wrong trace), pass `archive: true` and omit label/annotation. New verdicts start unapproved (`approvedAt=null`); once a human approves one, it joins the validated dataset (queryable via `search_traces` with `validated: true`). Archiving is non-destructive - the row is hidden from all reads but kept for audit, and you can immediately re-label the trace from scratch. For replay results, key each label by the replay item's `originalTraceId` (the original trace it was replayed from; `sourceTraceId` is accepted as a deprecated alias) plus the top-level `testRunId` instead of a `traceId`: the server resolves it to the replay trace via lineage, so you never need a server-generated replay trace id. When the experiment ran each trace more than once, also pass the item's `attempt` so each attempt gets its own verdict. Returns an agent-readable summary with one parseable effective label line per updated trace, keyed by the id you supplied (`originalTraceId` for replay verdicts, otherwise `traceId`), so command callers can verify persistence. Before judging a replay, call get_trace_assertions on the original trace ids: an assertion says what the user asked this specific case to do, so the verdict is measured against that rather than a guess. When an assertion's target cannot be found on the trace you are judging, the check never ran, so pass `skip: true` for that trace rather than recording a FAIL. Pass `assertionId` to score ONE assertion on the trace (the `[ID: ...]` get_trace_assertions prints), one entry per assertion, all in the same call. Omit it for the trace's single whole-trace verdict. get_trace_labels reads these back one line per assertion, keyed by the same assertionId, so a per-assertion write is verifiable per assertion.",
   inputSchema: {
     testRunId: external_exports.uuid().optional().describe("The replay test run id. Required when any label targets a trace by originalTraceId (replay verdicts); ignored otherwise."),
     labels: external_exports.preprocess(parseJsonString, external_exports.array(external_exports.object({
@@ -23569,21 +23582,23 @@ var saveAgentLabels = {
       skip: external_exports.boolean().optional().describe("If true, record an explicit skip (cannot judge) and ignore label/annotation/confidence. Default false."),
       label: external_exports.boolean().optional().describe("true for PASS, false for FAIL. Required when archive and skip are both false/omitted."),
       annotation: external_exports.string().min(1).optional().describe("Your reasoning for the verdict - shown to the human reviewer in the labeling UI. Required when archive and skip are both false/omitted."),
-      confidence: external_exports.enum(["VeryLow", "Low", "Medium", "High", "VeryHigh"]).optional().describe("How confident you are in this verdict: VeryLow / Low / Medium / High / VeryHigh. Surfaced as a chip in the labeling UI; low-confidence labels are prioritized for human review. Optional; only meaningful when paired with a label.")
+      confidence: external_exports.enum(["VeryLow", "Low", "Medium", "High", "VeryHigh"]).optional().describe("How confident you are in this verdict: VeryLow / Low / Medium / High / VeryHigh. Surfaced as a chip in the labeling UI; low-confidence labels are prioritized for human review. Optional; only meaningful when paired with a label."),
+      evidence: labelEvidenceShape.nullable().optional().describe("Span-by-span evidence for this verdict. Omit it to preserve existing evidence. Pass null to clear existing evidence.")
     })).min(1).max(200)).describe("One entry per assertion on traces that have assertions, otherwise one whole-trace entry (1-200)")
   }
 };
 var saveHumanLabels = {
   name: "save_human_labels",
   title: "Save Human Labels",
-  description: "Record one or more human-authored pass/fail verdicts that are VALIDATED immediately (`labelSource=\"human\"`, no approval step). Unlike save_agent_labels (which writes agent suggestions that start unapproved), labels set here join the validated dataset the instant they are written and satisfy `search_traces` with `validated: true`. Pass `label` (true=PASS, false=FAIL), `annotation` (the reasoning behind the verdict), and optionally `confidence` (`VeryLow|Low|Medium|High|VeryHigh`). USE ONLY when a human has explicitly decided the verdict, for example saving a known production bug as a test case. This records a verdict on a run that already happened; to record what a trace should do on its next replay, use save_trace_assertions. Do NOT use this for the agent's own first-pass guesses on traces awaiting human review; use save_agent_labels for those so they go through the normal approve/edit loop. Pass `assertionId` when the human judged ONE assertion (the `[ID: ...]` get_trace_assertions prints), one entry per assertion. Omit it for the trace's single whole-trace verdict. get_trace_labels reads these back one line per assertion, keyed by the same assertionId, so a per-assertion write is verifiable per assertion.",
+  description: "Record one or more human-authored pass/fail verdicts that are VALIDATED immediately (`labelSource=\"human\"`, no approval step). Unlike save_agent_labels (which writes agent suggestions that start unapproved), labels set here join the validated dataset the instant they are written and satisfy `search_traces` with `validated: true`. Pass `label` (true=PASS, false=FAIL), `annotation` (the reasoning behind the verdict), and optionally `evidence` as `{ spanId, text }` entries grounded in the evaluated trace. Optionally pass `confidence` (`VeryLow|Low|Medium|High|VeryHigh`). USE ONLY when a human has explicitly decided the verdict, for example saving a known production bug as a test case. This records a verdict on a run that already happened; to record what a trace should do on its next replay, use save_trace_assertions. Do NOT use this for the agent's own first-pass guesses on traces awaiting human review; use save_agent_labels for those so they go through the normal approve/edit loop. Pass `assertionId` when the human judged ONE assertion (the `[ID: ...]` get_trace_assertions prints), one entry per assertion. Omit it for the trace's single whole-trace verdict. get_trace_labels reads these back one line per assertion, keyed by the same assertionId, so a per-assertion write is verifiable per assertion.",
   inputSchema: {
     labels: external_exports.preprocess(parseJsonString, external_exports.array(external_exports.object({
       traceId: external_exports.uuid().describe("The trace ID to label"),
       assertionId: external_exports.uuid().optional().describe("The assertion this verdict scores, taken from the `[ID: ...]` get_trace_assertions prints. Omit it to set the trace's whole-trace verdict, of which there is one."),
       label: external_exports.boolean().describe("true for PASS, false for FAIL. Required."),
       annotation: external_exports.string().min(1).describe("The reasoning behind the verdict. Required."),
-      confidence: external_exports.enum(["VeryLow", "Low", "Medium", "High", "VeryHigh"]).optional().describe("How confident the verdict is: VeryLow / Low / Medium / High / VeryHigh. Optional.")
+      confidence: external_exports.enum(["VeryLow", "Low", "Medium", "High", "VeryHigh"]).optional().describe("How confident the verdict is: VeryLow / Low / Medium / High / VeryHigh. Optional."),
+      evidence: labelEvidenceShape.nullable().optional().describe("Span-by-span evidence for this verdict. Omit it to preserve existing evidence. Pass null to clear existing evidence.")
     })).min(1).max(200)).describe("One entry per assertion on traces that have assertions, otherwise one whole-trace entry (1-200)")
   }
 };
@@ -23950,6 +23965,7 @@ var ALL_TOOL_CONTRACTS = [
   searchTraces,
   getTraces,
   getTraceLabels,
+  getLabelEvidence,
   getSpanField,
   saveAgentLabels,
   saveHumanLabels,
@@ -24581,10 +24597,10 @@ var semver3 = __toESM(require_semver2(), 1);
 
 // ../bitfab-plugin-lib/dist/bakedSdkVersions.js
 var BAKED_SDK_VERSIONS = {
-  typescript: "0.57.3",
-  python: "0.57.3",
-  ruby: "0.57.3",
-  go: "0.57.3"
+  typescript: "0.57.4",
+  python: "0.57.5",
+  ruby: "0.57.4",
+  go: "0.57.4"
 };
 
 // ../bitfab-plugin-lib/dist/installedSdk.js
@@ -29401,7 +29417,7 @@ Hold the chosen mode in working context, the next steps branch on it.
           body: `**Seed dataset from existing validated labels.** Reachable only when the user picked Option B in \`ask-search-mode\`. Pull traces that already have a validated label (human-authored, or agent-authored and human-approved) for this function, attach them to the picked dataset, and route on whether the user also wants to add more.
 
 1. Call {{tool:searchTraces}} with \`traceFunctionKey: <key>\`, \`validated: true\`, and a generous \`limit\` (50 is the cap). Both \`labelResult: true\` and \`labelResult: false\` matter, failures are the hill-climbing signal, but passes anchor the regression boundary. If 50 isn't enough to cover the function's labeled history, run a second call with \`labelResult: false\` only to bias toward fails first, then a third with \`labelResult: true\`. De-dupe trace IDs across calls.
-2. Call {{tool:getTraceLabels}} on the resulting trace IDs so the labels + annotations are in working context. Don't re-label them, these are already validated. {{tool:getTraceLabels}} returns only label + annotation + approved (no span content) and takes up to 100 IDs per call, so the deduped set from step 1 is usually a single call; if it deduped more than 100 IDs, split them into chunks of 100 and call {{tool:getTraceLabels}} once per chunk (sequential is fine, the responses are small).
+2. Call {{tool:getTraceLabels}} on the resulting trace IDs so the labels, annotations, and saved evidence are in working context. Don't re-label them, these are already validated. {{tool:getTraceLabels}} returns only label + annotation + evidence + approved (no span content) and takes up to 100 IDs per call, so the deduped set from step 1 is usually a single call; if it deduped more than 100 IDs, split them into chunks of 100 and call {{tool:getTraceLabels}} once per chunk (sequential is fine, the responses are small).
 3. Call {{tool:addTracesToDataset}} once with \`datasetId\` (the one picked in \`list-datasets\`) and the full deduped trace ID array. The call is idempotent, so re-attaching IDs already in the dataset is a safe no-op.
 4. Briefly summarize for the user: "Seeded the dataset with N reused labels (M fails, K passes). Want me to find more candidates to label, or is this set enough to move on?"
 
@@ -29596,9 +29612,9 @@ Then return to the await-event step. If the user wants more edits, they'll ask a
           toolCalls: ["getTraceLabels", "bash", "read"],
           commandCalls: ["readTracesBatched"],
           title: "Build the dataset",
-          body: `**Build the dataset**: You already know the trace IDs in this dataset (you attached them in earlier steps and tracked any add/remove from modify rounds). Load each trace's label + annotation into context. **Which tool depends on \`costRun\`:**
+          body: `**Build the dataset**: You already know the trace IDs in this dataset (you attached them in earlier steps and tracked any add/remove from modify rounds). Load each trace's label + annotation + evidence into context. **Which tool depends on \`costRun\`:**
 
-- **\`costRun\` is false (the common case):** call {{tool:getTraceLabels}} with all of them. It returns only the verdict fields (label + annotation + approved, no span content) and takes up to 100 IDs per call, so for almost every dataset this is a single call. Only if the dataset has more than 100 traces, split the IDs into chunks of 100 and call {{tool:getTraceLabels}} once per chunk; the responses are small, so plain sequential calls are fine.
+- **\`costRun\` is false (the common case):** call {{tool:getTraceLabels}} with all of them. It returns only the verdict fields (label + annotation + evidence + approved, no span content) and takes up to 100 IDs per call, so for almost every dataset this is a single call. Only if the dataset has more than 100 traces, split the IDs into chunks of 100 and call {{tool:getTraceLabels}} once per chunk; the responses are small, so plain sequential calls are fine.
 - **\`costRun\` is set:** run \`{{command:readTracesBatched}} <trace-id...> --scope full\` **once** with all the trace IDs instead, because the Phase 5 cost-delta step reads each original trace's recorded token usage from this load and {{tool:getTraceLabels}} does not carry it. The command fans the \`scope: "full"\` reads out in parallel batches of 10 and writes the combined result to a temp file; it prints \`{"status":"ok","outputFile":"..."}\` as JSON, so \`Read\` that \`outputFile\`. **Use this command here, not the \`get_traces\` MCP tool directly:** \`get_traces\` caps at 10 IDs, so calling it for the dataset would re-introduce the serial per-batch fan-out \`readTracesBatched\` exists to replace.
 
 **Re-read them fresh here even if you read them earlier in this phase:** the Bitfab labeling review persists human approvals and label/annotation edits to the DB, so cached context from the find / label steps can be stale. This is the working set for confirm + every Phase 5 experiment.`,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bitfab-cli",
-  "version": "0.2.363",
+  "version": "0.2.365",
   "description": "Install and configure the Bitfab plugin in Claude Code, Codex, or Cursor, or the Bitfab skill pack in Amp (Alpha).",
   "type": "module",
   "main": "dist/index.js",