bitfab-cli 0.2.521 → 0.2.522

package/dist/index.js

@@ -29282,7 +29282,7 @@ var regenerateAssertion = {
 var saveTraceAssertions = {
   name: "save_trace_assertions",
   title: "Save Trace Assertions",
-  description: `Record what SHOULD happen when this trace is replayed. Bitfab stores two kinds of record on a trace. A VERDICT says how a run that already happened turned out, and that is save_agent_verdicts / save_human_verdicts. An ASSERTION, which is what this tool writes, says what a correct run looks like for THIS specific input, carries no pass/fail of its own, and is checked against a later replay once a reviewer approves it. A trace holds any number of assertions, and each verdict is recorded on one of them. Call it when the user describes the right answer for a particular case, for example "this booking should have picked the 6am flight, not the 9am". This is not a verdict on a run that already happened, which is save_agent_verdicts, and it is not a check that applies to every trace of a function, which is save_grader. Pass \`assertion\` plus optional \`passCriteria\` / \`failCriteria\`, the same trio save_grader takes, so an assertion that proves out across many traces can later be promoted into a grader with no rewriting. \`humanNote\` stores people-only context on the assertion; preserve and return it, but never use it to judge whether a replay passed. \`assigneeEmail\` names the member who should review it, which routes the assertion to that person without deciding anything, so it never changes the approval state. Pass an entry's \`id\` to edit an existing assertion, or omit it to add a new one, so two callers adding different assertions to one trace never overwrite each other. What this tool writes is saved awaiting review, so nothing checks it yet; a reviewer approves it, in Bitfab or with \`approvalState\` on a later call here, and from then on every replay of the trace is judged against it. \`approvalState\` decides an assertion that already exists, so deciding one is \`{ id, approvalState }\` with nothing else: an omitted \`assertion\` keeps the current wording. It is refused on a new assertion, so the call that invents a claim is never the call that puts it into force. One decision covers the assertion and the category it is filed under, so set \`category_id\` and \`approvalState\` in the same entry when you are deciding both; recategorizing on a later call without deciding returns it to not reviewed, since the approval covered the old category. What it gates is whether replays check the assertion at all, so decide from the assertion and the spans it cites rather than from memory of having written it. Rewording an assertion, or changing its criteria, target, or category, returns it to not reviewed unless the same call says otherwise. Call get_trace_assertions first to find the assertion to edit and to avoid creating a duplicate, and read them back before judging a replay, where it lists the approved ones by default. Pass it \`approvalStates: ["pending"]\` to read back drafts awaiting review, including the ones you just wrote. Each assertion with new text is checked against the writing rules and rewritten when it breaks one, unless \`guidedRewrite\` is false, so read the response for what was saved and for any claim to save separately. Up to ${MAX_ASSERTIONS_PER_REQUEST} per call.`,
+  description: `Record what SHOULD happen when this trace is replayed. Bitfab stores two kinds of record on a trace. A VERDICT says how a run that already happened turned out, and that is save_agent_verdicts / save_human_verdicts. An ASSERTION, which is what this tool writes, says what a correct run looks like for THIS specific input, carries no pass/fail of its own, and is checked against a later replay once a reviewer approves it. A trace holds any number of assertions, and each verdict is recorded on one of them. Call it when the user describes the right answer for a particular case, for example "this booking should have picked the 6am flight, not the 9am". This is not a verdict on a run that already happened, which is save_agent_verdicts,. Pass \`assertion\` plus optional \`passCriteria\` / \`failCriteria\`. \`humanNote\` stores people-only context on the assertion; preserve and return it, but never use it to judge whether a replay passed. \`assigneeEmail\` names the member who should review it, which routes the assertion to that person without deciding anything, so it never changes the approval state. Pass an entry's \`id\` to edit an existing assertion, or omit it to add a new one, so two callers adding different assertions to one trace never overwrite each other. What this tool writes is saved awaiting review, so nothing checks it yet; a reviewer approves it, in Bitfab or with \`approvalState\` on a later call here, and from then on every replay of the trace is judged against it. \`approvalState\` decides an assertion that already exists, so deciding one is \`{ id, approvalState }\` with nothing else: an omitted \`assertion\` keeps the current wording. It is refused on a new assertion, so the call that invents a claim is never the call that puts it into force. One decision covers the assertion and the category it is filed under, so set \`category_id\` and \`approvalState\` in the same entry when you are deciding both; recategorizing on a later call without deciding returns it to not reviewed, since the approval covered the old category. What it gates is whether replays check the assertion at all, so decide from the assertion and the spans it cites rather than from memory of having written it. Rewording an assertion, or changing its criteria, target, or category, returns it to not reviewed unless the same call says otherwise. Call get_trace_assertions first to find the assertion to edit and to avoid creating a duplicate, and read them back before judging a replay, where it lists the approved ones by default. Pass it \`approvalStates: ["pending"]\` to read back drafts awaiting review, including the ones you just wrote. Each assertion with new text is checked against the writing rules and rewritten when it breaks one, unless \`guidedRewrite\` is false, so read the response for what was saved and for any claim to save separately. Up to ${MAX_ASSERTIONS_PER_REQUEST} per call.`,
   inputSchema: {
     traceId: external_exports.uuid().describe("The ORIGINAL trace to attach assertions to. A replay trace id is refused, and the error names the original to retry with, because a replay reads its original's assertions automatically."),
     assertions: external_exports.preprocess(parseJsonString, external_exports.array(external_exports.object({
@@ -29291,7 +29291,7 @@ var saveTraceAssertions = {
       category_assertion_id: external_exports.uuid().nullish().describe("Deprecated: the old name for category_id. Ignored when category_id is also sent."),
       assertion: external_exports.string().min(1).optional().describe("The single thing that must hold, in one sentence, stated so a reader who has never seen this trace could check it, e.g. 'The itinerary returned lands before 9am local time'. One claim per assertion: if you are about to write 'and', write two assertions instead, so each can pass or fail on its own. The replay gets this trace's input and world unchanged, so state the answer they settle, with its real names, dates, times, and amounts, never a condition such as 'if the meeting is booked' and never a pointer such as 'the times the user approved'. Never pin a value the recorded run chose where other choices are also correct, such as which of several open slots it offered or the channel it replied on; state the range or set instead. Answer the latest input, not the thread history. Required when adding an assertion. Omit on an edit to preserve the current wording, which is what a call that only records an approval wants."),
       humanNote: external_exports.string().max(1e4).optional().describe("People-only context attached to this assertion. It is returned by reads but must never be used as evidence when assessing a replay. On an edit, omit to preserve it and pass an empty string to clear it."),
-      passCriteria: external_exports.string().optional().describe("How a judge should recognise a pass, when the assertion alone leaves room to argue, e.g. 'arrival timestamp is strictly before 09:00 in the destination timezone'. Optional, and only worth writing when it removes real ambiguity. Same field save_grader takes, so an assertion that proves out across many traces is promoted into a grader by copying it. On an edit, omit to keep the current value and pass an empty string to clear it."),
+      passCriteria: external_exports.string().optional().describe("How a judge should recognise a pass, when the assertion alone leaves room to argue, e.g. 'arrival timestamp is strictly before 09:00 in the destination timezone'. Optional, and only worth writing when it removes real ambiguity. On an edit, omit to keep the current value and pass an empty string to clear it."),
       failCriteria: external_exports.string().optional().describe("How a judge should recognise a failure, for cases the pass criteria do not obviously exclude, e.g. 'any leg departing after 09:00, including connections'. Optional. On an edit, omit to keep the current value and pass an empty string to clear it."),
       targetOnEvaluatedTrace: assertionTargetShape.nullish(),
       code: assertionCodeShape.nullish(),
@@ -29336,23 +29336,21 @@ var archiveTraceAssertions = {
 var saveGrader = {
   name: "save_grader",
   title: "Save Grader",
-  description: `Create or edit a grader: a pass/fail check that describes what must hold for a traced function (e.g. 'The reply never invents order numbers'). Call when the user asks to add, define, update, rename, archive, or restore a grader or check for a traced function. By default the grader is an LLM-as-judge that Bitfab runs; pass external: true when the user evaluates traces themselves. To create, pass traceFunctionKey + name + evaluationFocus. To edit, pass graderId (from list_graders) plus only the fields to change; saving with the name of an existing grader on the same function updates that grader instead of creating a duplicate. Set status to "archived" to retire a grader; any later save that omits status (or sets "active") restores it. On edit, pass an empty string for passCriteria/failCriteria to clear them. After a successful create or update, present the returned grader definition to the user, including its evaluation focus and any pass/fail criteria; do not respond with only a generic success confirmation. Graders are definitions only for now: nothing auto-evaluates them on new traces yet, so do not promise live grading. Call list_graders first to find ids and avoid duplicates. An external grader is one the user runs themselves (their own eval harness, CI, a human review queue): Bitfab never runs it, and its verdicts are posted with save_grader_verdicts. A grader's external setting is fixed at creation and cannot be changed on edit, and external graders take no model.`,
+  description: 'Edit an existing grader: rename it, rewrite its criteria, switch its judge model, archive it, or restore it. Graders are deprecated and new ones can no longer be created; when the user wants a new check, write assertions with save_trace_assertions instead. Pass graderId (from list_graders) plus only the fields to change. Set status to "archived" to retire a grader; any later save that omits status (or sets "active") restores it. Pass an empty string for passCriteria/failCriteria to clear them. After a successful update, present the returned grader definition to the user, including its evaluation focus and any pass/fail criteria; do not respond with only a generic success confirmation. External graders (ones Bitfab never runs) take no model.',
   inputSchema: {
-    graderId: external_exports.uuid().optional().describe("Id of an existing grader to edit (from list_graders or a previous save). Omit to create a new grader (or update-by-name)."),
-    traceFunctionKey: external_exports.string().min(1).optional().describe("The traced function the grader applies to. Required when graderId is omitted; ignored when it is set."),
-    name: external_exports.string().min(1).optional().describe("Short human-readable name, unique per trace function (e.g. 'No fabricated order ids'). Required when graderId is omitted; when editing by id, pass only to rename."),
-    evaluationFocus: external_exports.string().min(1).optional().describe("What this grader evaluates: the property a trace of this function must satisfy, written as a concrete, checkable statement. Required when creating; on edit, omit to keep the current value."),
-    passCriteria: external_exports.string().optional().describe("Optional: what a PASSING trace looks like. On edit, omit to keep the stored value; pass an empty string to clear it."),
-    failCriteria: external_exports.string().optional().describe("Optional: what a FAILING trace looks like. On edit, omit to keep the stored value; pass an empty string to clear it."),
-    status: external_exports.enum(["active", "archived"]).optional().describe(`The grader's status: "active" (the default for a create or edit) or "archived" to retire it (hidden from list_graders by default). Omit on edit to keep it active / restore an archived grader.`),
-    model: external_exports.enum(["gemini-2-5-flash", "sol", "terra", "opus-4-8", "gemini-3-1-pro"]).optional().describe("Optional: the judge model this grader runs on. One of: gemini-2-5-flash, sol, terra, opus-4-8, gemini-3-1-pro. Omit to keep the current value (defaults to gemini-2-5-flash)."),
-    external: external_exports.boolean().optional().describe("Create an external grader: Bitfab never runs it, and its verdicts arrive through save_grader_verdicts. Only honored when creating; on edit it must match the grader or be omitted.")
+    graderId: external_exports.uuid().describe("Id of the grader to edit, from list_graders."),
+    name: external_exports.string().min(1).optional().describe("New name, unique per trace function (e.g. 'No fabricated order ids'). Pass only to rename."),
+    evaluationFocus: external_exports.string().min(1).optional().describe("What this grader evaluates: the property a trace of this function must satisfy, written as a concrete, checkable statement. Omit to keep the current value."),
+    passCriteria: external_exports.string().optional().describe("Optional: what a PASSING trace looks like. Omit to keep the stored value; pass an empty string to clear it."),
+    failCriteria: external_exports.string().optional().describe("Optional: what a FAILING trace looks like. Omit to keep the stored value; pass an empty string to clear it."),
+    status: external_exports.enum(["active", "archived"]).optional().describe('Set "archived" to retire the grader (hidden from list_graders by default). Omit, or set "active", to keep its current running status; on an archived grader it restores it, as live when it has a trained prompt and as active otherwise.'),
+    model: external_exports.enum(["gemini-2-5-flash", "sol", "terra", "opus-4-8", "gemini-3-1-pro"]).optional().describe("Optional: the judge model this grader runs on. One of: gemini-2-5-flash, sol, terra, opus-4-8, gemini-3-1-pro. Omit to keep the current value (defaults to gemini-2-5-flash).")
   }
 };
 var listGraders = {
   name: "list_graders",
   title: "List Graders",
-  description: "List the graders for a traced function: LLM-as-judge graders Bitfab runs, and external graders (marked 'Runs: externally') whose verdicts the user posts with save_grader_verdicts. Each comes with id, name, status, evaluation focus, and pass/fail criteria. Call before save_grader to find the grader to edit or to avoid creating a duplicate. Results are newest-first and paginated. Archived graders are hidden unless includeArchived is true.",
+  description: "List the graders for a traced function: LLM-as-judge graders Bitfab runs, and external graders (marked 'Runs: externally') whose verdicts the user posts with save_grader_verdicts. Each comes with id, name, status, evaluation focus, and pass/fail criteria. Call before save_grader to find the grader to edit. Results are newest-first and paginated. Archived graders are hidden unless includeArchived is true.",
   inputSchema: {
     traceFunctionKey: external_exports.string().min(1).describe("The trace function key to list graders for"),
     includeArchived: external_exports.preprocess(parseJsonString, external_exports.boolean()).optional().describe("Include archived graders (default false)"),
@@ -29583,7 +29581,7 @@ var getGraderVerdicts = {
   }
 };
 var graderVerdictEntryShape = external_exports.object({
-  graderId: external_exports.uuid().describe("The external grader that produced this verdict (from list_graders or save_grader)."),
+  graderId: external_exports.uuid().describe("The external grader that produced this verdict (from list_graders)."),
   traceId: external_exports.uuid().describe("The trace the verdict is about."),
   verdict: external_exports.enum(["pass", "fail"]).optional().describe('"pass" or "fail". Required unless the deprecated label is sent.'),
   label: external_exports.boolean().optional().meta({ deprecated: true }).describe('Deprecated: the old spelling of verdict. true is "pass", false is "fail". Refused when verdict is also sent.'),
@@ -29594,7 +29592,7 @@ var graderVerdictEntryShape = external_exports.object({
 var saveGraderVerdicts = {
   name: "save_grader_verdicts",
   title: "Save Grader Verdicts",
-  description: `Record verdicts for EXTERNAL graders: graders created with save_grader external: true, which Bitfab never runs because the user evaluates traces somewhere else (their own eval harness, CI, a review queue). Each entry is one grader's pass/fail on one trace, written as that grader's own verdict, so it shows up everywhere Bitfab-run grader verdicts do: get_grader_verdicts, dataset and experiment pass rates, and the trace's grader panel. Saving again for the same grader and trace replaces the earlier verdict. Rejects every grader that is not external: Bitfab runs its own graders, and a person's verdict on a trace belongs in save_human_verdicts. Up to ${SAVE_GRADER_VERDICTS_MAX_ENTRIES} entries per call; the whole call is rejected if any entry is invalid. The old \`labels\` array, with \`label\` (true or false) on each entry, is still accepted.`,
+  description: `Record verdicts for EXTERNAL graders: existing graders Bitfab never runs because the user evaluates traces somewhere else (their own eval harness, CI, a review queue). Each entry is one grader's pass/fail on one trace, written as that grader's own verdict, so it shows up everywhere Bitfab-run grader verdicts do: get_grader_verdicts, dataset and experiment pass rates, and the trace's grader panel. Saving again for the same grader and trace replaces the earlier verdict. Rejects every grader that is not external: Bitfab runs its own graders, and a person's verdict on a trace belongs in save_human_verdicts. Up to ${SAVE_GRADER_VERDICTS_MAX_ENTRIES} entries per call; the whole call is rejected if any entry is invalid. The old \`labels\` array, with \`label\` (true or false) on each entry, is still accepted.`,
   inputSchema: {
     verdicts: external_exports.preprocess(parseJsonString, external_exports.array(graderVerdictEntryShape).min(1).max(SAVE_GRADER_VERDICTS_MAX_ENTRIES)).optional().describe(`One entry per grader per trace (1-${SAVE_GRADER_VERDICTS_MAX_ENTRIES}). Required unless the deprecated labels is sent.`),
     labels: external_exports.preprocess(parseJsonString, external_exports.array(graderVerdictEntryShape).min(1).max(SAVE_GRADER_VERDICTS_MAX_ENTRIES)).optional().meta({ deprecated: true }).describe("Deprecated: the old name for verdicts. Refused when verdicts is also sent.")
@@ -37107,7 +37105,6 @@ Finding out what happened:
 Writing down what should happen:
 - An assertion says what should happen when one trace is replayed. Draft them from what the trace did with ${generateTraceAssertions.name}, or write them with ${saveTraceAssertions.name}. They wait for review until approved. Retire one with ${archiveTraceAssertions.name}.
 - A verdict says whether a run passed one assertion. Read a trace's assertions with ${getTraceAssertions.name}, then record your verdicts with ${saveAgentVerdicts.name}. Use ${saveHumanVerdicts.name} only when the user gives the verdict.
-- A grader is a pass/fail check for every trace of a function. Create one with ${saveGrader.name}.
 - A scenario describes the situation one run was in. Read it with ${getScenario.name} and build it with ${saveScenario.name}.
 
 Building datasets:

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bitfab-cli",
-  "version": "0.2.521",
+  "version": "0.2.522",
   "description": "Install and configure the Bitfab plugin in Claude Code, Codex, or Cursor, or the Bitfab skill pack in Amp (Alpha).",
   "type": "module",
   "main": "dist/index.js",