bitfab-cli 0.2.526 → 0.2.527

package/dist/index.js

@@ -29162,7 +29162,7 @@ var agentVerdictEntryShape = external_exports.object({
 var saveAgentVerdicts = {
   name: "save_agent_verdicts",
   title: "Save Agent Verdicts",
-  description: "Set, skip, abstain on, or archive the agent's pass/fail verdict on one or more of a trace's approved assertions (`verdictSource=\"agent\"`). Every verdict, skip, and abstention scores ONE assertion and must carry its `assertionId` (the `[ID: ...]` get_trace_assertions prints). Verdicts are only ever written on assertions, never directly on a trace: a trace with no approved assertions gets no verdict, so add assertions first (save_trace_assertions) or report your judgment in the conversation. Use this AFTER you have read the traces with get_traces and decided yourself whether each one looks like a pass, a fail, or genuinely cannot be judged. To set a verdict, pass `verdict` (\"pass\" or \"fail\") and `annotation` (your reasoning, shown to the human reviewer beside the verdict). Add `evidence` as `{ spanId, text }` entries to ground that reasoning in spans on the evaluated trace. Optionally pass `confidence` (`VeryLow|Low|Medium|High|VeryHigh`) to record how confident you are, shown beside the verdict so reviewers can look at low-confidence verdicts first. To explicitly skip an assertion you cannot decide on (instead of leaving it with no verdict), pass `skip: true` and omit verdict. That records an explicit skip so coverage checks know you intentionally did not judge it, and `annotation` still applies: pass it to say why the verdict was withheld (the check never ran, the tape was punctured, the run never terminated), which is the only place a reader learns that. A skip with no annotation clears the note the previous verdict carried, so a stale reason never sits under a withheld verdict. When you read the trace and the evidence does not show the part of the run the assertion checks at all (the message it sent, the call it made, the answer it gave), so nothing can confirm or deny the assertion, pass `verdict: \"abstained\"`. That is counted as a FAIL in every pass rate, since anything short of an evidenced pass is a fail, but it is stored and shown as an abstention so a reader sees judge coverage rather than the agent's behavior; `annotation` says what the evidence would have had to show. Abstaining is not a skip: a skip is for a check that never ran, an abstention is for a check that ran and found nothing to judge. To clear a previously-set agent verdict (e.g., you changed your mind or judged the wrong trace), pass `archive: true` and omit verdict/annotation; an archive may omit `assertionId` to clear an old whole-trace agent verdict. A verdict counts as soon as it is written; there is no approval step, and a person can override it with their own verdict. An assertion Bitfab's judge already decided (pass, fail, or abstained; shown with source `bitfab_judge`) is refused and the whole call writes nothing, so the judge's verdict stands; an assertion the judge skipped can still be judged. Archiving is non-destructive: the row is hidden from all reads but kept for audit, and you can immediately judge the assertion again from scratch. For replay results, key each verdict by the replay item's `originalTraceId` (the original trace it was replayed from; `sourceTraceId` is accepted as a deprecated alias) plus the top-level `experimentId` instead of a `traceId`: the server resolves it to the replay trace via lineage, so you never need a server-generated replay trace id. When the experiment ran each trace more than once, also pass the item's `attempt` so each attempt gets its own verdict. Returns an agent-readable summary with one parseable effective verdict line per entry you sent (so a trace scored on several assertions gets one line per assertion), keyed by the id you supplied (`originalTraceId` for replay verdicts, otherwise `traceId`), so command callers can verify persistence. Before judging a replay, call get_trace_assertions on the original trace ids: an assertion says what the user asked this specific case to do, so the verdict is measured against that rather than a guess. When an assertion's target cannot be found on the trace you are judging, the check never ran, so pass `skip: true` for that assertion rather than recording a FAIL. Send one entry per assertion, all in the same call. get_trace_verdicts reads these back one line per assertion, keyed by the same assertionId, so a per-assertion write is verifiable per assertion. The old `labels` array, with `label` (true or false) and `abstain` on each entry, is still accepted.",
+  description: "Set, skip, abstain on, or archive the agent's pass/fail verdict on one or more of a trace's approved assertions (`verdictSource=\"agent\"`). Every verdict, skip, and abstention scores ONE assertion and must carry its `assertionId` (the `[ID: ...]` get_trace_assertions prints). Verdicts are only ever written on assertions, never directly on a trace: a trace with no approved assertions gets no verdict, so add assertions first (save_trace_assertions) or report your judgment in the conversation. Use this AFTER you have read the traces with get_traces and decided yourself whether each one looks like a pass, a fail, or genuinely cannot be judged. To set a verdict, pass `verdict` (\"pass\" or \"fail\") and `annotation` (your reasoning, shown to the human reviewer beside the verdict). Add `evidence` as `{ spanId, text }` entries to ground that reasoning in spans on the evaluated trace. Optionally pass `confidence` (`VeryLow|Low|Medium|High|VeryHigh`) to record how confident you are, shown beside the verdict so reviewers can look at low-confidence verdicts first. To explicitly skip an assertion you cannot decide on (instead of leaving it with no verdict), pass `skip: true` and omit verdict. That records an explicit skip so coverage checks know you intentionally did not judge it, and `annotation` still applies: pass it to say why the verdict was withheld (the check never ran, the tape was punctured, the run never terminated), which is the only place a reader learns that. A skip with no annotation clears the note the previous verdict carried, so a stale reason never sits under a withheld verdict. When you read the trace and the evidence does not show the part of the run the assertion checks at all (the message it sent, the call it made, the answer it gave), so nothing can confirm or deny the assertion, pass `verdict: \"abstained\"`. That is counted as a FAIL in every pass rate, since anything short of an evidenced pass is a fail, but it is stored and shown as an abstention so a reader sees judge coverage rather than the agent's behavior; `annotation` says what the evidence would have had to show. Abstaining is not a skip: a skip is for a check that never ran, an abstention is for a check that ran and found nothing to judge. To clear a previously-set agent verdict (e.g., you changed your mind or judged the wrong trace), pass `archive: true` and omit verdict/annotation; an archive may omit `assertionId` to clear an old whole-trace agent verdict. A verdict counts as soon as it is written; there is no approval step, and a person can override it with their own verdict. You can also give a verdict on an assertion Bitfab's judge already decided (shown with source `bitfab_judge`): yours is saved beside the judge's and becomes the one counted, unless a person gave one, and archiving yours makes the judge's count again. Archiving is non-destructive: the row is hidden from all reads but kept for audit, and you can immediately judge the assertion again from scratch. For replay results, key each verdict by the replay item's `originalTraceId` (the original trace it was replayed from; `sourceTraceId` is accepted as a deprecated alias) plus the top-level `experimentId` instead of a `traceId`: the server resolves it to the replay trace via lineage, so you never need a server-generated replay trace id. When the experiment ran each trace more than once, also pass the item's `attempt` so each attempt gets its own verdict. Returns an agent-readable summary with one parseable effective verdict line per entry you sent (so a trace scored on several assertions gets one line per assertion), keyed by the id you supplied (`originalTraceId` for replay verdicts, otherwise `traceId`), so command callers can verify persistence. Before judging a replay, call get_trace_assertions on the original trace ids: an assertion says what the user asked this specific case to do, so the verdict is measured against that rather than a guess. When an assertion's target cannot be found on the trace you are judging, the check never ran, so pass `skip: true` for that assertion rather than recording a FAIL. Send one entry per assertion, all in the same call. get_trace_verdicts reads these back one line per assertion, keyed by the same assertionId, so a per-assertion write is verifiable per assertion. The old `labels` array, with `label` (true or false) and `abstain` on each entry, is still accepted.",
   inputSchema: {
     experimentId: external_exports.uuid().optional().describe("The experiment ID. Required when any verdict targets a trace by originalTraceId (replay verdicts); ignored otherwise."),
     verdicts: external_exports.preprocess(parseJsonString, external_exports.array(agentVerdictEntryShape).min(1).max(MAX_VERDICTS_PER_REQUEST)).optional().describe(`One entry per approved assertion, each carrying its assertionId; an archive entry may omit it (1-${MAX_VERDICTS_PER_REQUEST}). Required unless the deprecated labels is sent.`),
@@ -29181,12 +29181,29 @@ var humanVerdictEntryShape = external_exports.object({
 var saveHumanVerdicts = {
   name: "save_human_verdicts",
   title: "Save Human Verdicts",
-  description: 'Record one or more pass/fail verdicts a person gave, which are VALIDATED immediately (`verdictSource="human"`), on one or more of a trace\'s approved assertions. Unlike save_agent_verdicts (the agent\'s own verdicts), verdicts set here join the validated dataset the instant they are written and satisfy `search_traces` with `validated: true`. Pass `verdict` ("pass" or "fail"), `annotation` (the reasoning behind the verdict), and optionally `evidence` as `{ spanId, text }` entries grounded in the evaluated trace. Optionally pass `confidence` (`VeryLow|Low|Medium|High|VeryHigh`). USE ONLY when a person has explicitly decided the verdict, for example saving a known production bug as a test case. This records a verdict on a run that already happened; to record what a trace should do on its next replay, use save_trace_assertions. Do NOT use this for the agent\'s own first-pass guesses; use save_agent_verdicts for those. Every entry must carry the `assertionId` of the ONE assertion the person judged (the `[ID: ...]` get_trace_assertions prints), one entry per assertion. An assertion Bitfab\'s judge already decided (pass, fail, or abstained) is refused, so the judge\'s verdict stands. Verdicts are never written directly on a trace, so a trace with no approved assertions needs assertions first (save_trace_assertions). get_trace_verdicts reads these back one line per assertion, keyed by the same assertionId, so a per-assertion write is verifiable per assertion. The old `labels` array, with `label` (true or false) on each entry, is still accepted.',
+  description: "Record one or more pass/fail verdicts a person gave, which are VALIDATED immediately (`verdictSource=\"human\"`), on one or more of a trace's approved assertions. Unlike save_agent_verdicts (the agent's own verdicts), verdicts set here join the validated dataset the instant they are written and satisfy `search_traces` with `validated: true`. Pass `verdict` (\"pass\" or \"fail\"), `annotation` (the reasoning behind the verdict), and optionally `evidence` as `{ spanId, text }` entries grounded in the evaluated trace. Optionally pass `confidence` (`VeryLow|Low|Medium|High|VeryHigh`). USE ONLY when a person has explicitly decided the verdict, for example saving a known production bug as a test case. This records a verdict on a run that already happened; to record what a trace should do on its next replay, use save_trace_assertions. Do NOT use this for the agent's own first-pass guesses; use save_agent_verdicts for those. Every entry must carry the `assertionId` of the ONE assertion the person judged (the `[ID: ...]` get_trace_assertions prints), one entry per assertion. A person's verdict on an assertion Bitfab's judge already decided is saved beside the judge's and becomes the one counted; the judge's stays readable with source `bitfab_judge`. Verdicts are never written directly on a trace, so a trace with no approved assertions needs assertions first (save_trace_assertions). get_trace_verdicts reads these back one line per assertion, keyed by the same assertionId, so a per-assertion write is verifiable per assertion. The old `labels` array, with `label` (true or false) on each entry, is still accepted.",
   inputSchema: {
     verdicts: external_exports.preprocess(parseJsonString, external_exports.array(humanVerdictEntryShape).min(1).max(MAX_VERDICTS_PER_REQUEST)).optional().describe(`One entry per approved assertion, each carrying its assertionId (1-${MAX_VERDICTS_PER_REQUEST}). Required unless the deprecated labels is sent.`),
     labels: external_exports.preprocess(parseJsonString, external_exports.array(humanVerdictEntryShape).min(1).max(MAX_VERDICTS_PER_REQUEST)).optional().meta({ deprecated: true }).describe("Deprecated: the old name for verdicts. Refused when verdicts is also sent.")
   }
 };
+var externalAssertionVerdictEntryShape = external_exports.object({
+  assertionId: external_exports.uuid().describe("The external assertion this verdict scores, taken from the `[ID: ...]` get_trace_assertions prints. The trace is the one the assertion is on, so no trace ID is needed."),
+  attempt: external_exports.number().int().min(0).optional().describe("With experimentId, on an experiment that ran each trace more than once: which attempt this verdict is for (0-based). Defaults to 0."),
+  verdict: external_exports.enum(["pass", "fail"]).describe('"pass" or "fail".'),
+  annotation: external_exports.string().min(1).optional().describe("The reasoning behind the verdict. Optional."),
+  confidence: confidenceShape.optional().describe("How confident the verdict is: VeryLow / Low / Medium / High / VeryHigh. Optional."),
+  evidence: verdictEvidenceShape.nullable().optional().describe("Span-by-span evidence for this verdict. Omit it to preserve existing evidence. Pass null to clear existing evidence.")
+});
+var saveAssertionVerdicts = {
+  name: "save_assertion_verdicts",
+  title: "Save Assertion Verdicts",
+  description: `Record verdicts for EXTERNAL assertions: assertions created with save_trace_assertions \`kind: "external"\`, which Bitfab never judges because the user evaluates the trace somewhere else (their own eval harness, CI, a review queue). Key each verdict by \`assertionId\` alone: the server finds the trace the assertion is on. For a replay, also pass the top-level \`experimentId\` (and \`attempt\` when the experiment ran each trace more than once), and the verdict is saved on that experiment's replay of the assertion's trace. The verdict is stored where Bitfab's judge would store its own, so it counts in every pass rate as soon as it is written, with no approval step; a person's verdict (save_human_verdicts) or an agent's (save_agent_verdicts) on the same assertion still counts over it. Saving again for the same assertion and trace replaces the earlier verdict. Rejects any assertion that is not external, since Bitfab decides those itself, and any assertion that is not approved. Up to ${MAX_VERDICTS_PER_REQUEST} entries per call; the whole call is rejected if any entry is invalid.`,
+  inputSchema: {
+    experimentId: external_exports.uuid().optional().describe("For replay verdicts: the experiment whose replay of each assertion's trace the verdicts are on. Omit it for verdicts on the trace the assertion is on."),
+    verdicts: external_exports.preprocess(parseJsonString, external_exports.array(externalAssertionVerdictEntryShape).min(1).max(MAX_VERDICTS_PER_REQUEST)).describe(`One entry per external assertion (1-${MAX_VERDICTS_PER_REQUEST}).`)
+  }
+};
 var assertionTargetShape = external_exports.discriminatedUnion("kind", [
   external_exports.object({ kind: external_exports.literal("output") }).describe("Check only the trace's final output, ignoring everything that happened on the way there."),
   external_exports.object({
@@ -29199,6 +29216,7 @@ var assertionTargetShape = external_exports.discriminatedUnion("kind", [
     ]).optional().describe('Which call to check when the trace runs that span more than once, e.g. a retry loop. "first", "last" (the default), or a 0-based index. Leave it off unless the repetition matters.')
   }).describe("Check one span by name rather than the whole trace, e.g. that a particular tool call was made with the right arguments.")
 ]).describe('SCOPE: what part of the trace under evaluation this assertion is checked against. OMIT IT for the whole trace, which is the common case and the right default when unsure. On an edit, omitting it keeps the current scope, and passing null returns the assertion to the whole trace. Pass { "kind": "output" } to check only the final output, or { "kind": "span", "name": "..." } to check one span. Scoping narrows what a judge looks at, so a wrong scope hides real failures. A target naming something the evaluated trace does not contain makes the check ERRORED, never passed, because a check that could not run must never look like a check that succeeded.');
+var assertionKindShape = external_exports.enum(["bitfab", "external"]).describe(`KIND: optional, and only on the call that creates the assertion. Omit it (or send "bitfab") for the normal case: Bitfab decides the assertion on every replay, with its code where it has some and the LLM judge otherwise. Send "external" when the user's own system evaluates the trace (their eval harness, CI, a review queue): Bitfab never judges an external assertion, it takes no code, and the user posts each verdict with save_assertion_verdicts. An assertion's kind never changes after it is created, so sending a different kind on an edit is refused; omit it on an edit.`);
 var assertionCodeShape = external_exports.object({
   source: external_exports.string().trim().min(1).max(2e4).describe('Body of an async TypeScript function (plain JavaScript also works) called with `spans` (every span of the run in start order, each { name, index, hasData } without its data), `input`, and `output` (the run\'s root input and output). Read span data with `await named("span name")`, which returns the spans with that name with { name, index, input, output, metadata }, or `await span.data()`. Count or order spans from `spans` without loading data. It returns "pass", "fail", or "defer", or { result, reason } with a short reason. Return "pass" or "fail" when the facts the code read settle the assertion, and "defer" to hand the run to the LLM judge, for example when the part the code can read holds and the rest needs reading what text means. A defer may carry data, { result: "defer", reason, data }, with the values the code found that the judge needs to decide, such as the sent email\'s subject and body; the judge reads the reason and data alongside the run. A thrown error counts as "defer" with no data. Copy ids, recipients, and times from the trace as literals, never match the wording of prose, and guard every read so it never throws on a missing field. It runs in a sandbox with no network, files, or timers, gets one second of compute, and may load the data of at most 50 spans within ten seconds.'),
   spanNames: external_exports.array(external_exports.string().min(1)).max(50).describe("Every span name the code expects a correct run to produce. Leave out names it only checks did not run. When one of these is missing from a replay and from the other replays sampled for the same function, the code is reported as broken rather than failing the replay.")
@@ -29294,6 +29312,7 @@ var saveTraceAssertions = {
       passCriteria: external_exports.string().optional().describe("How a judge should recognise a pass, when the assertion alone leaves room to argue, e.g. 'arrival timestamp is strictly before 09:00 in the destination timezone'. Optional, and only worth writing when it removes real ambiguity. On an edit, omit to keep the current value and pass an empty string to clear it."),
       failCriteria: external_exports.string().optional().describe("How a judge should recognise a failure, for cases the pass criteria do not obviously exclude, e.g. 'any leg departing after 09:00, including connections'. Optional. On an edit, omit to keep the current value and pass an empty string to clear it."),
       targetOnEvaluatedTrace: assertionTargetShape.nullish(),
+      kind: assertionKindShape.nullish(),
       code: assertionCodeShape.nullish(),
       codeOff: external_exports.boolean().optional().describe("Pass true to stop this assertion's code from deciding its verdicts while keeping the code, so the LLM judge decides every replay. Use it when the code turns out to be wrong. Pass false to use the code again. Bitfab does not write new code for an assertion whose code is turned off, saving new code leaves the switch where it is, and removing the code (code: null) turns it back on. Omit to keep the current setting."),
       justification: justificationShape.nullish().describe("Why this assertion is right for THIS trace, cited span by span: the spans that show what the run actually did, each with its own reason. Omit on an edit to preserve it, or pass null to remove it. It is the evidence behind `approvalState`, so an assertion with no justification is one a reviewer has to reconstruct from scratch. Cite spans on this trace only."),
@@ -29711,6 +29730,7 @@ var generateScenarioAssertions = {
 var ALL_TOOL_CONTRACTS = [
   getGraderVerdicts,
   saveGraderVerdicts,
+  saveAssertionVerdicts,
   getGraderLabels,
   saveGraderLabels,
   getExperiment,
@@ -31141,10 +31161,10 @@ var semver2 = __toESM(require_semver2(), 1);
 
 // ../bitfab-plugin-lib/dist/bakedSdkVersions.js
 var BAKED_SDK_VERSIONS = {
-  typescript: "0.76.9",
-  python: "0.76.8",
-  ruby: "0.75.5",
-  go: "0.75.5"
+  typescript: "0.77.0",
+  python: "0.77.0",
+  ruby: "0.76.0",
+  go: "0.76.0"
 };
 
 // ../bitfab-plugin-lib/dist/installedSdk.js

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bitfab-cli",
-  "version": "0.2.526",
+  "version": "0.2.527",
   "description": "Install and configure the Bitfab plugin in Claude Code, Codex, or Cursor, or the Bitfab skill pack in Amp (Alpha).",
   "type": "module",
   "main": "dist/index.js",