bitfab-cli 0.2.400 → 0.2.402

package/dist/index.js

@@ -23449,6 +23449,7 @@ var MAX_ASSIGNEE_EMAIL_LENGTH = 320;
 var GET_GRADER_LABELS_MAX_IDS = 100;
 var DEFAULT_GRADER_LABEL_LIMIT = 50;
 var MAX_GRADER_LABEL_LIMIT = 200;
+var SAVE_GRADER_LABELS_MAX_ENTRIES = 200;
 function parseJsonString(val) {
   if (typeof val !== "string") {
     return val;
@@ -23752,7 +23753,7 @@ var archiveTraceAssertions = {
 var saveGrader = {
   name: "save_grader",
   title: "Save Grader",
-  description: `Create or edit an automated grader (LLM-as-judge): a pass/fail check that describes what must hold for a traced function (e.g. 'The reply never invents order numbers'). Call when the user asks to add, define, update, rename, archive, or restore an automated grader or check for a traced function. To create, pass traceFunctionKey + name + evaluationFocus. To edit, pass graderId (from list_graders) plus only the fields to change; saving with the name of an existing grader on the same function updates that grader instead of creating a duplicate. Set status to "archived" to retire a grader; any later save that omits status (or sets "active") restores it. On edit, pass an empty string for passCriteria/failCriteria to clear them. After a successful create or update, present the returned grader definition to the user, including its evaluation focus and any pass/fail criteria; do not respond with only a generic success confirmation. Graders are definitions only for now: nothing auto-evaluates them on new traces yet, so do not promise live grading. Call list_graders first to find ids and avoid duplicates.`,
+  description: `Create or edit a grader: a pass/fail check that describes what must hold for a traced function (e.g. 'The reply never invents order numbers'). Call when the user asks to add, define, update, rename, archive, or restore a grader or check for a traced function. By default the grader is an LLM-as-judge that Bitfab runs; pass external: true when the user evaluates traces themselves. To create, pass traceFunctionKey + name + evaluationFocus. To edit, pass graderId (from list_graders) plus only the fields to change; saving with the name of an existing grader on the same function updates that grader instead of creating a duplicate. Set status to "archived" to retire a grader; any later save that omits status (or sets "active") restores it. On edit, pass an empty string for passCriteria/failCriteria to clear them. After a successful create or update, present the returned grader definition to the user, including its evaluation focus and any pass/fail criteria; do not respond with only a generic success confirmation. Graders are definitions only for now: nothing auto-evaluates them on new traces yet, so do not promise live grading. Call list_graders first to find ids and avoid duplicates. An external grader is one the user runs themselves (their own eval harness, CI, a human review queue): Bitfab never runs it, and its verdicts are posted with save_grader_labels. A grader's external setting is fixed at creation and cannot be changed on edit, and external graders take no model.`,
   inputSchema: {
     graderId: external_exports.uuid().optional().describe("Id of an existing grader to edit (from list_graders or a previous save). Omit to create a new grader (or update-by-name)."),
     traceFunctionKey: external_exports.string().min(1).optional().describe("The traced function the grader applies to. Required when graderId is omitted; ignored when it is set."),
@@ -23765,13 +23766,14 @@ var saveGrader = {
     status: external_exports.enum(["active", "archived"]).optional().describe(`The grader's status: "active" (the default for a create or edit) or "archived" to retire it (hidden from list_graders by default). Omit on edit to keep it active / restore an archived grader.`),
     // Keep the enum in sync with bitfab-web's save_grader route schema
     // (GRADER_MODEL_SLUGS in graderModels.ts).
-    model: external_exports.enum(["gemini-2-5-flash", "sol", "terra", "opus-4-8", "gemini-3-1-pro"]).optional().describe("Optional: the judge model this grader runs on. One of: gemini-2-5-flash, sol, terra, opus-4-8, gemini-3-1-pro. Omit to keep the current value (defaults to gemini-2-5-flash).")
+    model: external_exports.enum(["gemini-2-5-flash", "sol", "terra", "opus-4-8", "gemini-3-1-pro"]).optional().describe("Optional: the judge model this grader runs on. One of: gemini-2-5-flash, sol, terra, opus-4-8, gemini-3-1-pro. Omit to keep the current value (defaults to gemini-2-5-flash)."),
+    external: external_exports.boolean().optional().describe("Create an external grader: Bitfab never runs it, and its verdicts arrive through save_grader_labels. Only honored when creating; on edit it must match the grader or be omitted.")
   }
 };
 var listGraders = {
   name: "list_graders",
   title: "List Graders",
-  description: "List the automated graders (LLM-as-judge definitions) for a traced function, with id, name, status, and each one's evaluation focus and pass/fail criteria. Call before save_grader to find the grader to edit or to avoid creating a duplicate. Results are newest-first and paginated. Archived graders are hidden unless includeArchived is true.",
+  description: "List the graders for a traced function: LLM-as-judge graders Bitfab runs, and external graders (marked 'Runs: externally') whose verdicts the user posts with save_grader_labels. Each comes with id, name, status, evaluation focus, and pass/fail criteria. Call before save_grader to find the grader to edit or to avoid creating a duplicate. Results are newest-first and paginated. Archived graders are hidden unless includeArchived is true.",
   inputSchema: {
     traceFunctionKey: external_exports.string().min(1).describe("The trace function key to list graders for"),
     includeArchived: external_exports.preprocess(parseJsonString, external_exports.boolean()).optional().describe("Include archived graders (default false)"),
@@ -23992,13 +23994,28 @@ var getExperiment = {
 var getGraderLabels = {
   name: "get_grader_labels",
   title: "Get Grader Labels",
-  description: `Read the INDIVIDUAL verdicts each automated grader (LLM-as-judge) recorded, one row per grader per trace, with the grader's reason, its failure diagnostic, the confidence, and whether the verdict came from a human or from a grader run. This is the per-grader breakdown that get_trace_labels does not show: that tool returns one grader-agnostic verdict per trace, so use it for a dataset's overall pass/fail and use this one to see WHICH check failed and why. Pass traceIds to see every grader's verdict on those traces, graderId to see that grader's most recent verdicts across traces (newest first, use it to diagnose a grader that is misfiring), or both to narrow to one grader on specific traces. Call list_graders first if you need grader ids.`,
+  description: `Read the INDIVIDUAL verdicts each grader recorded, one row per grader per trace, with the grader's reason, its failure diagnostic, the confidence, and whether the verdict came from a human, a grader Bitfab ran, or an external grader whose verdict was posted from outside Bitfab with save_grader_labels. This is the per-grader breakdown that get_trace_labels does not show: that tool returns one grader-agnostic verdict per trace, so use it for a dataset's overall pass/fail and use this one to see WHICH check failed and why. Pass traceIds to see every grader's verdict on those traces, graderId to see that grader's most recent verdicts across traces (newest first, use it to diagnose a grader that is misfiring), or both to narrow to one grader on specific traces. Call list_graders first if you need grader ids.`,
   inputSchema: {
     traceIds: external_exports.preprocess(parseJsonString, external_exports.array(external_exports.uuid()).min(1).max(GET_GRADER_LABELS_MAX_IDS)).optional().describe(`Trace IDs to read grader verdicts for (1-${GET_GRADER_LABELS_MAX_IDS}). Omit to read by grader instead.`),
     graderId: external_exports.uuid().optional().describe("Restrict to one grader (from list_graders). On its own, returns that grader's most recent verdicts across all traces."),
     limit: external_exports.preprocess(parseJsonString, external_exports.number().int().min(1).max(MAX_GRADER_LABEL_LIMIT)).optional().describe(`Max labels to return when reading by grader alone (default ${DEFAULT_GRADER_LABEL_LIMIT}, max ${MAX_GRADER_LABEL_LIMIT}). Ignored when traceIds is given.`)
   }
 };
+var saveGraderLabels = {
+  name: "save_grader_labels",
+  title: "Save Grader Labels",
+  description: `Record verdicts for EXTERNAL graders: graders created with save_grader external: true, which Bitfab never runs because the user evaluates traces somewhere else (their own eval harness, CI, a review queue). Each entry is one grader's pass/fail on one trace, written as that grader's own verdict, so it shows up everywhere Bitfab-run grader verdicts do: get_grader_labels, dataset and experiment pass rates, and the trace's grader panel. Saving again for the same grader and trace replaces the earlier verdict. Rejects every grader that is not external: Bitfab runs its own graders, and a human's verdict on a trace belongs in save_human_labels. Up to ${SAVE_GRADER_LABELS_MAX_ENTRIES} entries per call; the whole call is rejected if any entry is invalid.`,
+  inputSchema: {
+    labels: external_exports.preprocess(parseJsonString, external_exports.array(external_exports.object({
+      graderId: external_exports.uuid().describe("The external grader that produced this verdict (from list_graders or save_grader)."),
+      traceId: external_exports.uuid().describe("The trace the verdict is about."),
+      label: external_exports.boolean().describe("true for PASS, false for FAIL. Required."),
+      reason: external_exports.string().min(1).optional().describe("Why the grader reached this verdict. Optional."),
+      failureDiagnostic: external_exports.string().min(1).optional().describe("For a FAIL: what went wrong in the trace, in a form someone fixing it can act on. Optional."),
+      confidence: external_exports.enum(["VeryLow", "Low", "Medium", "High", "VeryHigh"]).optional().describe("How confident the grader is: VeryLow / Low / Medium / High / VeryHigh. Optional.")
+    })).min(1).max(SAVE_GRADER_LABELS_MAX_ENTRIES)).describe(`One entry per grader per trace (1-${SAVE_GRADER_LABELS_MAX_ENTRIES}).`)
+  }
+};
 var getSimPlan = {
   name: "get_sim_plan",
   title: "Get Sim Plan",
@@ -24022,6 +24039,7 @@ var saveSimPlan = {
 };
 var ALL_TOOL_CONTRACTS = [
   getGraderLabels,
+  saveGraderLabels,
   getExperiment,
   getExperimentGroup,
   saveExperiment,
@@ -24679,10 +24697,10 @@ var semver3 = __toESM(require_semver2(), 1);
 
 // ../bitfab-plugin-lib/dist/bakedSdkVersions.js
 var BAKED_SDK_VERSIONS = {
-  typescript: "0.62.2",
-  python: "0.62.1",
-  ruby: "0.62.1",
-  go: "0.62.1"
+  typescript: "0.62.4",
+  python: "0.62.3",
+  ruby: "0.62.3",
+  go: "0.62.3"
 };
 
 // ../bitfab-plugin-lib/dist/installedSdk.js
@@ -31385,7 +31403,7 @@ Never modify existing code on a refactor path without completing this three-step
           toolCalls: ["bash", "read", "glob", "grep", "ask"],
           body: `Read the existing GitHub Actions, runtime manifests, lockfiles, replay registry, gitignore, and environment-variable NAMES. Never read or print secret values. Check git origin, Python 3.10+, and gh authentication. Local dispatch supports macOS/Linux and github.com. Inspect all push-triggered workflows and deployment integrations. Temporary branches use bitfab-replay/** and can trigger those systems. Offer the usual fixes: a push workflow can add bitfab-replay/** to branches-ignore, and a Vercel project can skip those branches with git.deploymentEnabled {"bitfab-replay/**": false} in its vercel.json or an Ignored Build Step in the dashboard. Ask before changing existing CI or deployment behavior. Do not set pushTriggersReviewed until these triggers have been reviewed with the user.
 
-Choose one existing replay registry and reproduce its runtime, dependency installation, build steps, working directory, caches, and required services from CI. Cover what every pipeline in that registry needs, since cloud replay allows all of them by default. Use the installed SDK's cloud-capable CLI: run its bitfab-replay --cloud --help now. If it does not print the cloud options, the project's SDK predates cloud replay, so upgrade it through bitfab:update before continuing. Never give the user .bitfab/cloudReplay.py as the command to run; the SDK command starts it. TypeScript/Python/Ruby registry imports happen on the runner only. Go must dispatch cloud flags before building its registry or initializing the application. For database reads, prefer database branching. When the Bitfab organization has a database connected, each replay item gets a branch at its trace's captured instant, and the runner needs no database secret. That works only when every registry entry that reads the database sets dbBranch, and the application takes its connection per call from the replay branch URL (getCurrentReplayBranch().databaseUrl, or the Python, Ruby, or Go equivalent), falling back to its normal URL. A pool or engine created once at import from DATABASE_URL never sees the branch, and the SDK does not change DATABASE_URL, so check how the application builds its database client. When either piece is missing, offer to run db-snapshot mode first. Without branching, a private database needs runner network access or a replay-safe service fixture. Do not silently give replay production side effects. Submodules/gitlinks and credential-like tracked files are rejected by the snapshot helper. A gitlink with no .gitmodules entry is usually a nested repository committed by accident: offer to untrack it with git rm --cached and ignore its folder. Surface blockers instead of claiming the environment is ready.
+Choose one existing replay registry and reproduce its runtime, dependency installation, build steps, working directory, caches, and required services from CI. Cover what every pipeline in that registry needs, since cloud replay allows all of them by default. Use the installed SDK's cloud-capable CLI: run its bitfab-replay --cloud --help now. If it does not list --cloud-init, the project's SDK predates this setup, so upgrade it through bitfab:update before continuing. The code that submits, watches, and runs cloud replays ships inside the SDK and updates with it; the repository keeps only settings, .bitfab/cloud.json and the workflow. An older setup also left .bitfab/cloudReplay.py in the repository. The SDK no longer reads that file, so offer to migrate: change the workflow's Replay step to run the SDK command with --cloud-execute (next step), then delete .bitfab/cloudReplay.py. TypeScript/Python/Ruby registry imports happen on the runner only. Go must dispatch cloud flags before building its registry or initializing the application. For database reads, prefer database branching. When the Bitfab organization has a database connected, each replay item gets a branch at its trace's captured instant, and the runner needs no database secret. That works only when every registry entry that reads the database sets dbBranch, and the application takes its connection per call from the replay branch URL (getCurrentReplayBranch().databaseUrl, or the Python, Ruby, or Go equivalent), falling back to its normal URL. A pool or engine created once at import from DATABASE_URL never sees the branch, and the SDK does not change DATABASE_URL, so check how the application builds its database client. When either piece is missing, offer to run db-snapshot mode first. Without branching, a private database needs runner network access or a replay-safe service fixture. Do not silently give replay production side effects. Submodules/gitlinks and credential-like tracked files are rejected by the snapshot helper. A gitlink with no .gitmodules entry is usually a nested repository committed by accident: offer to untrack it with git rm --cached and ignore its folder. Surface blockers instead of claiming the environment is ready.
 
 Present the proposed workflow, fixed replay command, required secret/variable names, and any required CI trigger exclusions. Ask only about unresolved project choices. Never ask for secret values in chat.`,
           next: "cloud/install"
@@ -31393,17 +31411,17 @@ Present the proposed workflow, fixed replay command, required secret/variable na
         {
           id: "install",
           kind: "action",
-          title: "Install the repository-owned runner",
+          title: "Install the cloud replay settings",
           toolCalls: ["bash", "read", "write", "edit"],
-          body: `Create a reviewed JSON setup specification containing version: 1, provider: "github", workflow: "bitfab-replay.yml", workingDirectory (repository-relative), registry (repository-relative path or null for Go), pipeline (null allows every pipeline in the registry and is the default; a name restricts replay to that one pipeline, for when the others need credentials or services the runner should not have), command (a JSON argv array without the pipeline argument or cloud flags), pushTriggersReviewed: true, setupSteps (GitHub Actions step objects), secrets (names including BITFAB_API_KEY), and variables (names only). The runner always provides BITFAB_API_KEY. Find the variable the registry's Bitfab client actually reads its key from, and when it is another name, map it in the command, for example ["sh", "-c", "APP_KEY_NAME=\\"$BITFAB_API_KEY\\" exec <command> \\"$@\\"", "sh"]. Optional secretPrefix (uppercase, ending in an underscore) names the GitHub secret each one is read from without changing the environment variable the replay sees; it defaults to BITFAB_CLOUD_, and only an explicit empty string turns prefixing off. Optional runsOn, services, and environment match the existing CI. Prefer a GitHub Environment with required reviewers when replay code receives sensitive credentials. Pin third-party actions to reviewed commit SHAs. Ensure Python 3.10+ exists on the runner. Commands run in workingDirectory. Registry in the command must be relative to that directory. Set step working-directory explicitly where needed. Do not copy deployment steps, local env values, or API keys into the specification.
+          body: `Create a reviewed JSON setup specification containing version: 1, provider: "github", workflow: "bitfab-replay.yml", workingDirectory (repository-relative), registry (repository-relative path or null for Go), pipeline (null allows every pipeline in the registry and is the default; a name restricts replay to that one pipeline, for when the others need credentials or services the runner should not have), command (a JSON argv array without the pipeline argument or cloud flags), cliCommand (the JSON argv array that starts the SDK's bitfab-replay command from workingDirectory, with no flags, such as ["npx", "--no-install", "bitfab-replay"] or ["node", "node_modules/@bitfab/sdk/dist/replayCli.js"]; the Replay step runs it with --cloud-execute), pushTriggersReviewed: true, setupSteps (GitHub Actions step objects), secrets (names including BITFAB_API_KEY), and variables (names only). The runner always provides BITFAB_API_KEY. Find the variable the registry's Bitfab client actually reads its key from, and when it is another name, map it in the command, for example ["sh", "-c", "APP_KEY_NAME=\\"$BITFAB_API_KEY\\" exec <command> \\"$@\\"", "sh"]. Optional secretPrefix (uppercase, ending in an underscore) names the GitHub secret each one is read from without changing the environment variable the replay sees; it defaults to BITFAB_CLOUD_, and only an explicit empty string turns prefixing off. Optional runsOn, services, and environment match the existing CI. Prefer a GitHub Environment with required reviewers when replay code receives sensitive credentials. Pin third-party actions to reviewed commit SHAs. Ensure Python 3.10+ exists on the runner. Commands run in workingDirectory. Registry in the command must be relative to that directory. Set step working-directory explicitly where needed. Do not copy deployment steps, local env values, or API keys into the specification.
 
-Run python3 "{{pluginRoot}}/scripts/cloudReplay.py" init --config <specification-path>. This creates .bitfab/cloud.json, .bitfab/cloudReplay.py, and .github/workflows/bitfab-replay.yml. The generated workflow is JSON, which is valid YAML. Existing files are never overwritten. For an existing setup, review the diff and edit project configuration deliberately. Remove only your temporary specification after successful installation. If .bitfab is ignored, propose a narrow gitignore exception for these two files. Never force-add secrets or arbitrary ignored files.
+From workingDirectory, run the SDK's bitfab-replay command with --cloud-init --config <specification-path>. This creates .bitfab/cloud.json and .github/workflows/bitfab-replay.yml. The generated workflow is JSON, which is valid YAML. Existing files are never overwritten. For an existing setup, review the diff and edit project configuration deliberately. Remove only your temporary specification after successful installation. If .bitfab is ignored, propose a narrow gitignore exception for .bitfab/cloud.json. Never force-add secrets or arbitrary ignored files.
 
 Tell the user which repository or Environment secrets to create in GitHub Settings \u2192 Secrets and variables \u2192 Actions. BITFAB_API_KEY belongs to the Bitfab organization that holds the registry's traces; confirm that with the user, because a key from another organization finds none of them. Other names come from their application, such as OPENAI_API_KEY. BITFAB_SERVICE_URL is optional. Do not override the SDK's default with an empty GitHub variable. Runtime secrets belong on the replay step. Any install-time credentials need explicit references on the relevant setup step. Local gh authentication is separate from these runner secrets. Do not create a Bitfab cloud target, install a GitHub App, or request a repository connection.
 
 Init records the secret names and the resolved prefix in .bitfab/cloud.json, so later lifecycle commands reuse them without being told again. Runner secrets are prefixed BITFAB_CLOUD_ unless the user asks otherwise, so the runner reads BITFAB_CLOUD_OPENAI_API_KEY and exposes it to the replay as OPENAI_API_KEY. Replay credentials then never collide with secrets the repository already uses, and can carry narrower scopes or a different Bitfab organization than production. Pass a different prefix, or an empty string to reuse the repository's existing bare-named secrets, only when the user wants that. Report the prefixed names from requiredSecrets as the ones to create.
 
-When the project keeps local environment files, list the NAMES each required secret could come from and ask whether to copy those local values into GitHub. Never read, print, or compare the values, and ask once for the whole set rather than per secret. On a yes, run python3 .bitfab/cloudReplay.py secrets --env-file <file> [--env-file <file>] [--environment <name>], which reads the recorded names and prefix, parses quoted, multi-line, and JSON values correctly, and pipes each value to gh on standard input so it never appears in arguments, process listings, or shell history. Never hand-extract a value with grep, cut, or a shell variable, and never pass one through gh secret set --body. Use --dry-run first to report which names the files define. The command reports set, missing, and empty names only; create the missing ones by hand. A name your files define as blank is skipped rather than uploaded, because an empty secret replaces a working default with nothing. Warn before copying a value whose own configuration points somewhere a runner cannot reach, such as a localhost service URL, or that grants more than the replay needs, and let the user decide. When the value for BITFAB_API_KEY lives locally under the client's other name, or only a development key sits under BITFAB_API_KEY, leave it out of the copy and have the user set it. Environment secrets need the Environment to exist: check with gh api repos/<owner>/<repo>/environments, and when it is missing, ask before creating it with gh api -X PUT repos/<owner>/<repo>/environments/<name>, offering the user as a required reviewer. Setting secrets requires the same explicit authorization as any other remote write, and a wrong value only surfaces when the first real replay runs, since the dry run never calls the application.`,
+When the project keeps local environment files, list the NAMES each required secret could come from and ask whether to copy those local values into GitHub. Never read, print, or compare the values, and ask once for the whole set rather than per secret. On a yes, run the SDK's bitfab-replay command with --cloud-secrets --env-file <file> [--env-file <file>] [--environment <name>], which reads the recorded names and prefix, parses quoted, multi-line, and JSON values correctly, and pipes each value to gh on standard input so it never appears in arguments, process listings, or shell history. Never hand-extract a value with grep, cut, or a shell variable, and never pass one through gh secret set --body. Use --dry-run first to report which names the files define. The command reports set, missing, and empty names only; create the missing ones by hand. A name your files define as blank is skipped rather than uploaded, because an empty secret replaces a working default with nothing. Warn before copying a value whose own configuration points somewhere a runner cannot reach, such as a localhost service URL, or that grants more than the replay needs, and let the user decide. When the value for BITFAB_API_KEY lives locally under the client's other name, or only a development key sits under BITFAB_API_KEY, leave it out of the copy and have the user set it. Environment secrets need the Environment to exist: check with gh api repos/<owner>/<repo>/environments, and when it is missing, ask before creating it with gh api -X PUT repos/<owner>/<repo>/environments/<name>, offering the user as a required reviewer. Setting secrets requires the same explicit authorization as any other remote write, and a wrong value only surfaces when the first real replay runs, since the dry run never calls the application.`,
           next: "cloud/verify"
         },
         {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bitfab-cli",
-  "version": "0.2.400",
+  "version": "0.2.402",
   "description": "Install and configure the Bitfab plugin in Claude Code, Codex, or Cursor, or the Bitfab skill pack in Amp (Alpha).",
   "type": "module",
   "main": "dist/index.js",