bitfab-cli 0.2.399 → 0.2.401

package/dist/index.js

@@ -23449,6 +23449,7 @@ var MAX_ASSIGNEE_EMAIL_LENGTH = 320;
 var GET_GRADER_LABELS_MAX_IDS = 100;
 var DEFAULT_GRADER_LABEL_LIMIT = 50;
 var MAX_GRADER_LABEL_LIMIT = 200;
+var SAVE_GRADER_LABELS_MAX_ENTRIES = 200;
 function parseJsonString(val) {
   if (typeof val !== "string") {
     return val;
@@ -23752,7 +23753,7 @@ var archiveTraceAssertions = {
 var saveGrader = {
   name: "save_grader",
   title: "Save Grader",
-  description: `Create or edit an automated grader (LLM-as-judge): a pass/fail check that describes what must hold for a traced function (e.g. 'The reply never invents order numbers'). Call when the user asks to add, define, update, rename, archive, or restore an automated grader or check for a traced function. To create, pass traceFunctionKey + name + evaluationFocus. To edit, pass graderId (from list_graders) plus only the fields to change; saving with the name of an existing grader on the same function updates that grader instead of creating a duplicate. Set status to "archived" to retire a grader; any later save that omits status (or sets "active") restores it. On edit, pass an empty string for passCriteria/failCriteria to clear them. After a successful create or update, present the returned grader definition to the user, including its evaluation focus and any pass/fail criteria; do not respond with only a generic success confirmation. Graders are definitions only for now: nothing auto-evaluates them on new traces yet, so do not promise live grading. Call list_graders first to find ids and avoid duplicates.`,
+  description: `Create or edit a grader: a pass/fail check that describes what must hold for a traced function (e.g. 'The reply never invents order numbers'). Call when the user asks to add, define, update, rename, archive, or restore a grader or check for a traced function. By default the grader is an LLM-as-judge that Bitfab runs; pass external: true when the user evaluates traces themselves. To create, pass traceFunctionKey + name + evaluationFocus. To edit, pass graderId (from list_graders) plus only the fields to change; saving with the name of an existing grader on the same function updates that grader instead of creating a duplicate. Set status to "archived" to retire a grader; any later save that omits status (or sets "active") restores it. On edit, pass an empty string for passCriteria/failCriteria to clear them. After a successful create or update, present the returned grader definition to the user, including its evaluation focus and any pass/fail criteria; do not respond with only a generic success confirmation. Graders are definitions only for now: nothing auto-evaluates them on new traces yet, so do not promise live grading. Call list_graders first to find ids and avoid duplicates. An external grader is one the user runs themselves (their own eval harness, CI, a human review queue): Bitfab never runs it, and its verdicts are posted with save_grader_labels. A grader's external setting is fixed at creation and cannot be changed on edit, and external graders take no model.`,
   inputSchema: {
     graderId: external_exports.uuid().optional().describe("Id of an existing grader to edit (from list_graders or a previous save). Omit to create a new grader (or update-by-name)."),
     traceFunctionKey: external_exports.string().min(1).optional().describe("The traced function the grader applies to. Required when graderId is omitted; ignored when it is set."),
@@ -23765,13 +23766,14 @@ var saveGrader = {
     status: external_exports.enum(["active", "archived"]).optional().describe(`The grader's status: "active" (the default for a create or edit) or "archived" to retire it (hidden from list_graders by default). Omit on edit to keep it active / restore an archived grader.`),
     // Keep the enum in sync with bitfab-web's save_grader route schema
     // (GRADER_MODEL_SLUGS in graderModels.ts).
-    model: external_exports.enum(["gemini-2-5-flash", "sol", "terra", "opus-4-8", "gemini-3-1-pro"]).optional().describe("Optional: the judge model this grader runs on. One of: gemini-2-5-flash, sol, terra, opus-4-8, gemini-3-1-pro. Omit to keep the current value (defaults to gemini-2-5-flash).")
+    model: external_exports.enum(["gemini-2-5-flash", "sol", "terra", "opus-4-8", "gemini-3-1-pro"]).optional().describe("Optional: the judge model this grader runs on. One of: gemini-2-5-flash, sol, terra, opus-4-8, gemini-3-1-pro. Omit to keep the current value (defaults to gemini-2-5-flash)."),
+    external: external_exports.boolean().optional().describe("Create an external grader: Bitfab never runs it, and its verdicts arrive through save_grader_labels. Only honored when creating; on edit it must match the grader or be omitted.")
   }
 };
 var listGraders = {
   name: "list_graders",
   title: "List Graders",
-  description: "List the automated graders (LLM-as-judge definitions) for a traced function, with id, name, status, and each one's evaluation focus and pass/fail criteria. Call before save_grader to find the grader to edit or to avoid creating a duplicate. Results are newest-first and paginated. Archived graders are hidden unless includeArchived is true.",
+  description: "List the graders for a traced function: LLM-as-judge graders Bitfab runs, and external graders (marked 'Runs: externally') whose verdicts the user posts with save_grader_labels. Each comes with id, name, status, evaluation focus, and pass/fail criteria. Call before save_grader to find the grader to edit or to avoid creating a duplicate. Results are newest-first and paginated. Archived graders are hidden unless includeArchived is true.",
   inputSchema: {
     traceFunctionKey: external_exports.string().min(1).describe("The trace function key to list graders for"),
     includeArchived: external_exports.preprocess(parseJsonString, external_exports.boolean()).optional().describe("Include archived graders (default false)"),
@@ -23992,13 +23994,28 @@ var getExperiment = {
 var getGraderLabels = {
   name: "get_grader_labels",
   title: "Get Grader Labels",
-  description: `Read the INDIVIDUAL verdicts each automated grader (LLM-as-judge) recorded, one row per grader per trace, with the grader's reason, its failure diagnostic, the confidence, and whether the verdict came from a human or from a grader run. This is the per-grader breakdown that get_trace_labels does not show: that tool returns one grader-agnostic verdict per trace, so use it for a dataset's overall pass/fail and use this one to see WHICH check failed and why. Pass traceIds to see every grader's verdict on those traces, graderId to see that grader's most recent verdicts across traces (newest first, use it to diagnose a grader that is misfiring), or both to narrow to one grader on specific traces. Call list_graders first if you need grader ids.`,
+  description: `Read the INDIVIDUAL verdicts each grader recorded, one row per grader per trace, with the grader's reason, its failure diagnostic, the confidence, and whether the verdict came from a human, a grader Bitfab ran, or an external grader whose verdict was posted from outside Bitfab with save_grader_labels. This is the per-grader breakdown that get_trace_labels does not show: that tool returns one grader-agnostic verdict per trace, so use it for a dataset's overall pass/fail and use this one to see WHICH check failed and why. Pass traceIds to see every grader's verdict on those traces, graderId to see that grader's most recent verdicts across traces (newest first, use it to diagnose a grader that is misfiring), or both to narrow to one grader on specific traces. Call list_graders first if you need grader ids.`,
   inputSchema: {
     traceIds: external_exports.preprocess(parseJsonString, external_exports.array(external_exports.uuid()).min(1).max(GET_GRADER_LABELS_MAX_IDS)).optional().describe(`Trace IDs to read grader verdicts for (1-${GET_GRADER_LABELS_MAX_IDS}). Omit to read by grader instead.`),
     graderId: external_exports.uuid().optional().describe("Restrict to one grader (from list_graders). On its own, returns that grader's most recent verdicts across all traces."),
     limit: external_exports.preprocess(parseJsonString, external_exports.number().int().min(1).max(MAX_GRADER_LABEL_LIMIT)).optional().describe(`Max labels to return when reading by grader alone (default ${DEFAULT_GRADER_LABEL_LIMIT}, max ${MAX_GRADER_LABEL_LIMIT}). Ignored when traceIds is given.`)
   }
 };
+var saveGraderLabels = {
+  name: "save_grader_labels",
+  title: "Save Grader Labels",
+  description: `Record verdicts for EXTERNAL graders: graders created with save_grader external: true, which Bitfab never runs because the user evaluates traces somewhere else (their own eval harness, CI, a review queue). Each entry is one grader's pass/fail on one trace, written as that grader's own verdict, so it shows up everywhere Bitfab-run grader verdicts do: get_grader_labels, dataset and experiment pass rates, and the trace's grader panel. Saving again for the same grader and trace replaces the earlier verdict. Rejects every grader that is not external: Bitfab runs its own graders, and a human's verdict on a trace belongs in save_human_labels. Up to ${SAVE_GRADER_LABELS_MAX_ENTRIES} entries per call; the whole call is rejected if any entry is invalid.`,
+  inputSchema: {
+    labels: external_exports.preprocess(parseJsonString, external_exports.array(external_exports.object({
+      graderId: external_exports.uuid().describe("The external grader that produced this verdict (from list_graders or save_grader)."),
+      traceId: external_exports.uuid().describe("The trace the verdict is about."),
+      label: external_exports.boolean().describe("true for PASS, false for FAIL. Required."),
+      reason: external_exports.string().min(1).optional().describe("Why the grader reached this verdict. Optional."),
+      failureDiagnostic: external_exports.string().min(1).optional().describe("For a FAIL: what went wrong in the trace, in a form someone fixing it can act on. Optional."),
+      confidence: external_exports.enum(["VeryLow", "Low", "Medium", "High", "VeryHigh"]).optional().describe("How confident the grader is: VeryLow / Low / Medium / High / VeryHigh. Optional.")
+    })).min(1).max(SAVE_GRADER_LABELS_MAX_ENTRIES)).describe(`One entry per grader per trace (1-${SAVE_GRADER_LABELS_MAX_ENTRIES}).`)
+  }
+};
 var getSimPlan = {
   name: "get_sim_plan",
   title: "Get Sim Plan",
@@ -24022,6 +24039,7 @@ var saveSimPlan = {
 };
 var ALL_TOOL_CONTRACTS = [
   getGraderLabels,
+  saveGraderLabels,
   getExperiment,
   getExperimentGroup,
   saveExperiment,
@@ -24679,10 +24697,10 @@ var semver3 = __toESM(require_semver2(), 1);
 
 // ../bitfab-plugin-lib/dist/bakedSdkVersions.js
 var BAKED_SDK_VERSIONS = {
-  typescript: "0.62.2",
-  python: "0.62.1",
-  ruby: "0.62.1",
-  go: "0.62.1"
+  typescript: "0.62.3",
+  python: "0.62.2",
+  ruby: "0.62.2",
+  go: "0.62.2"
 };
 
 // ../bitfab-plugin-lib/dist/installedSdk.js
@@ -26022,14 +26040,17 @@ Run ${platform2.loginHint} to re-authenticate.`);
 }
 
 // ../bitfab-plugin-lib/dist/commands/switchOrg.js
-async function requestSwitch(serviceUrl, apiKey, clerkOrganizationId) {
+async function requestSwitch(serviceUrl, apiKey, authOrganizationId) {
   const res = await fetch(`${serviceUrl}/api/plugin/organizations/switch`, {
     method: "POST",
     headers: {
       Authorization: `Bearer ${apiKey}`,
       "Content-Type": "application/json"
     },
-    body: JSON.stringify({ clerkOrganizationId })
+    body: JSON.stringify({
+      authOrganizationId,
+      clerkOrganizationId: authOrganizationId
+    })
   });
   if (!res.ok) {
     throw new Error(`Failed to switch organization (HTTP ${res.status})`);
@@ -26038,9 +26059,9 @@ async function requestSwitch(serviceUrl, apiKey, clerkOrganizationId) {
 }
 var ARGS = [
   {
-    name: "clerkOrganizationId",
+    name: "authOrganizationId",
     required: false,
-    description: "Clerk organization ID to make active. Discover IDs with the list_organizations MCP tool."
+    description: "Organization ID to make active. Discover IDs with the list_organizations MCP tool."
   }
 ];
 async function runSwitchOrg(platform2) {
@@ -26049,21 +26070,21 @@ async function runSwitchOrg(platform2) {
   })[0] ?? null;
   const { apiKey, serviceUrl } = requireAuth(platform2.loginHint);
   if (!target) {
-    console.error("Usage: switchOrg <clerkOrganizationId>");
+    console.error("Usage: switchOrg <authOrganizationId>");
     console.error("Discover org ids with the list_organizations MCP tool, then pass one here.");
     process.exit(1);
   }
   try {
     const result = await requestSwitch(serviceUrl, apiKey, target);
     if (result.status === "not-member") {
-      emitJson({ event: "not-member", clerkOrganizationId: target });
+      emitJson({ event: "not-member", authOrganizationId: target });
       return;
     }
     saveCredentials(result.apiKey);
     emitJson({
       event: "switched",
       status: result.status,
-      clerkOrganizationId: result.clerkOrganizationId,
+      authOrganizationId: result.authOrganizationId ?? result.clerkOrganizationId ?? target,
       organizationName: result.organizationName,
       apiKey: result.apiKey
     });
@@ -31220,8 +31241,8 @@ var setupFlow = Flow.parse({
       description: "Open a sign-in window and wait for authentication. Relay the printed sign-in link as a fallback."
     },
     switchOrg: {
-      description: "List the user's Bitfab orgs (no args), or switch the plugin's active org and replace the local API key (with a <clerkOrganizationId> arg)",
-      args: "[<clerkOrganizationId>]"
+      description: "List the user's Bitfab orgs (no args), or switch the plugin's active org and replace the local API key (with a <authOrganizationId> arg)",
+      args: "[<authOrganizationId>]"
     },
     startTemplatePreview: {
       description: "Print a template preview link and exit.",
@@ -32230,7 +32251,7 @@ Then, for anything not healthy, name the most likely cause and the fix:
 - **Root not replayable**, two failure modes, with the fix matched to each: **(a) the root takes unserializable inputs** (live SDK/DB clients, HTTP req/res, streams, opaque contexts), with or without a replay registry module: move the trace boundary inward to a serializable-input function or refactor to introduce one; **(b) a bare trace-processor-only key** ({{traceProcessorNames}}) whose root is the processor's empty-input span: add a \`withTrace\`/\`trace\` root that wraps the run and takes its input in a supported subtree runtime, or route the run through the run wrapper ({{traceProcessorRootWrappers}}). Use a manual \`withSpan\`/\`@span\` root only when opt-out is technically unavailable. Either way, re-instrument via \`{{cmd}}setup modify\` (or \`{{cmd}}setup instrument\` for a fresh boundary). This is a code change, recommended here, not applied blanket.
 - **Opt-out roots without subtree capture**: every trace is the root alone, so tracing looks healthy and the traces are useless for per-step diagnosis, replay mocking, and prompt iteration. For TypeScript, use the installed SDK and wire the matching adapter into the build behind the server entrypoint, preserving existing config wrappers such as Sentry. For Python, this is a runtime upgrade to 3.12+; report it rather than attempting it.
 - **Instrumented but no traces**: the app hasn't run with tracing enabled, or \`BITFAB_API_KEY\` isn't set in the run environment. Run the app with the key loaded.
-- **Key set but traces aren't visible in the browser**: the API key is bound to a different Clerk org/tenant than the browser session. A key resolves \`API key \u2192 organization_id \u2192 clerk_organization_id \u2192 Clerk tenant\` at creation time; browser visibility requires both to be the same tenant.
+- **Key set but traces aren't visible in the browser**: the API key belongs to a different organization than the one the browser is showing. A key is bound to one organization when it is created, and the browser shows the signed-in user's active organization. Have the user pick the key's organization in the in-app org switcher; if it isn't listed, the signed-in user isn't a member of it.
 - **Nothing instrumented**: run \`{{cmd}}setup instrument\`.
 - **Want to change what's captured**: run \`{{cmd}}setup modify\`.
 
@@ -32294,7 +32315,7 @@ For unusual monorepos or private registries, defer to \`{{cmd}}update\`. Report
       stepStyle: "list",
       intro: `Switch which Bitfab organization the plugin reads and writes. Triggered explicitly by \`{{cmd}}setup switch-org\` (or natural-language asks like "switch org" / "change org" / "switch to the <name> org" / "I'm in the wrong org"). The plugin's org is set by the API key in \`~/.config/bitfab/credentials.json\`; this lists the user's orgs, switches to the chosen one, and replaces that local key. Requires authentication. Does **not** open Bitfab.
 
-**The live browser does not follow on its own.** Switching persists the new active org server-side (so future sign-ins default to it) and replaces the plugin's key, but a browser tab that's already signed in keeps showing the old org until its session is re-minted. The org actually flips in the browser on the **next** Bitfab open (a fresh session whose org check runs Clerk's client-side \`setActive\`) or when the user picks the org from the in-app org switcher.
+**The live browser does not follow on its own.** Switching persists the new active org server-side (so future sign-ins default to it) and replaces the plugin's key, but a browser tab that's already signed in keeps showing the old org until its session is re-minted. The org actually flips in the browser on the **next** Bitfab open (a fresh session whose org check sets the active organization in the browser) or when the user picks the org from the in-app org switcher.
 
 **The plugin key and the app's runtime key are separate.** Switching replaces only the plugin's credential in \`~/.config/bitfab/credentials.json\`. The \`BITFAB_API_KEY\` your application reads at runtime (from a \`.env\`-style file) is untouched, so traces your code sends keep landing in the **old** org until that key is updated too. The last step offers to do that.`,
       steps: [
@@ -32327,7 +32348,7 @@ If **already authenticated**, continue to step {{step:switch-org/pick-org}}. If
           kind: "branch",
           toolCalls: ["listOrganizations", "ask"],
           title: "List the orgs and choose a target",
-          body: `Call {{tool:listOrganizations}} to list the organizations the signed-in user belongs to. Each entry has a name, the user's role, an \`id:\` (the \`clerkOrganizationId\`), and the org the plugin uses now is marked \`[current]\`.
+          body: `Call {{tool:listOrganizations}} to list the organizations the signed-in user belongs to. Each entry has a name, the user's role, an \`id:\` (the \`authOrganizationId\`), and the org the plugin uses now is marked \`[current]\`.
 
 Choose the target org:
 - **If the user already named an org** (in their request), match it case-insensitively by name against the list and use that org's \`id\`. If the name matches none, or matches more than one, fall through to asking.
@@ -32352,15 +32373,15 @@ Only ever use an \`id\` value returned by {{tool:listOrganizations}}; never inve
           toolCalls: ["bash"],
           commandCalls: ["switchOrg"],
           title: "Switch and confirm",
-          body: `Switch to the chosen org by passing its \`clerkOrganizationId\`:
+          body: `Switch to the chosen org by passing its \`authOrganizationId\`:
 
 \`\`\`bash
-{{command:switchOrg}} <clerkOrganizationId>
+{{command:switchOrg}} <authOrganizationId>
 \`\`\`
 
 The command prints one JSON line; act on it:
-- \`{"event":"switched","status":"switched"|"already-aligned","clerkOrganizationId":"...","organizationName":"...","apiKey":"..."}\`: success. The plugin now reads and writes that org and its API key has been replaced locally. Tell the user in one line: the plugin is now connected to **<organizationName>**. Then tell them to select **<organizationName>** with the in-app org switcher to align their browser with the plugin. Opening a plugin page link does not change the browser's active organization. Hold on to the \`apiKey\` value from this JSON; the next step uses it to sync the app's local key, and you must never echo that value to the user.
-- \`{"event":"not-member","clerkOrganizationId":"..."}\`: the user isn't a member of that org. Report it; do not retry.
+- \`{"event":"switched","status":"switched"|"already-aligned","authOrganizationId":"...","organizationName":"...","apiKey":"..."}\`: success. The plugin now reads and writes that org and its API key has been replaced locally. Tell the user in one line: the plugin is now connected to **<organizationName>**. Then tell them to select **<organizationName>** with the in-app org switcher to align their browser with the plugin. Opening a plugin page link does not change the browser's active organization. Hold on to the \`apiKey\` value from this JSON; the next step uses it to sync the app's local key, and you must never echo that value to the user.
+- \`{"event":"not-member","authOrganizationId":"..."}\`: the user isn't a member of that org. Report it; do not retry.
 - \`{"event":"error","reason":"..."}\`: report the reason.
 
 Do not print or ask for the API key, and do not surface the \`apiKey\` value to the user; the command replaces the plugin's copy for you and hands you that value solely for the next step.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bitfab-cli",
-  "version": "0.2.399",
+  "version": "0.2.401",
   "description": "Install and configure the Bitfab plugin in Claude Code, Codex, or Cursor, or the Bitfab skill pack in Amp (Alpha).",
   "type": "module",
   "main": "dist/index.js",