bitfab 0.65.11 → 0.66.0

package/dist/replayCli.js

@@ -1,5 +1,5 @@
 #!/usr/bin/env node
-import { a as runRegistryMain, c as runSeedCli, i as runIfEntrypoint, n as parseRegistryCommandArgs, o as checkReplayCliArgs, r as resolveRegistry, s as runReplayCli, t as importRegistry } from "./registryCommand-Cr7m6Vif.js";
+import { a as runRegistryMain, c as runSeedCli, i as runIfEntrypoint, n as parseRegistryCommandArgs, o as checkReplayCliArgs, r as resolveRegistry, s as runReplayCli, t as importRegistry } from "./registryCommand-Dk4Y1ZgW.js";
 //#region src/cloudReplayCli.ts
 const CLOUD_REPLAY_HINT = "Add --cloud to run the same replay on GitHub Actions; bitfab-replay --cloud --help explains it.";
 function isCloudReplayCommand(argv) {

package/dist/seedCli.js

@@ -1,5 +1,5 @@
 #!/usr/bin/env node
-import { a as runRegistryMain, c as runSeedCli, i as runIfEntrypoint, n as parseRegistryCommandArgs, r as resolveRegistry, t as importRegistry } from "./registryCommand-Cr7m6Vif.js";
+import { a as runRegistryMain, c as runSeedCli, i as runIfEntrypoint, n as parseRegistryCommandArgs, r as resolveRegistry, t as importRegistry } from "./registryCommand-Dk4Y1ZgW.js";
 //#region src/seedCli.ts
 const USAGE = "Usage: bitfab-seed --registry <path> <pipeline> --from-trace <id>[,<id>...]\n       bitfab-seed --registry <path> <pipeline> --cases <cases.jsonl> [--run]\nRun bitfab-seed --registry <path> <pipeline> --help for details.";
 async function runSeedCommand(argv, io = {}, importer = importRegistry) {

package/dist/{src-lTOsAt47.js → src-CL5eQlNk.js}

@@ -1,5 +1,5 @@
 import { t as __exportAll } from "./rolldown-runtime-D7D4PA-g.js";
-import { C as readEnv$1, S as MixedTracingError, _ as getReplayContext, b as gitRunner, c as recordCallerTraceMetadata, f as ROOT_TRACE_FUNCTION_KEY_FIELD, g as MAX_COMPRESSIBLE_SPAN_CARRIER_BYTES, h as makeSpanOrigin, i as flushTraces, l as retireTraceMetadata, m as isFrameworkInstrumentation, n as SPAN_TYPES, o as unrefTimer, p as SimulationPlan, r as experimentIdWireFields, s as callerTraceMetadata, t as HttpClient, v as replayContextReady, w as __version__, x as BitfabError, y as runWithReplayContext } from "./http-DdlELWcL.js";
+import { C as readEnv$1, S as MixedTracingError, _ as getReplayContext, b as gitRunner, c as recordCallerTraceMetadata, f as ROOT_TRACE_FUNCTION_KEY_FIELD, g as MAX_COMPRESSIBLE_SPAN_CARRIER_BYTES, h as makeSpanOrigin, i as flushTraces, l as retireTraceMetadata, m as isFrameworkInstrumentation, n as SPAN_TYPES, o as unrefTimer, p as SimulationPlan, r as experimentIdWireFields, s as callerTraceMetadata, t as HttpClient, v as replayContextReady, w as __version__, x as BitfabError, y as runWithReplayContext } from "./http-D91qP2J-.js";
 import { a as clearAutoTraceNodeConfiguration, b as isAsyncStorageInitDone, c as runAsAutoTraceRootFunction, d as runWithAutoTraceRootContext, g as runWithCaptureSuppressed, h as isCaptureSuppressed, l as runWithAutoTraceEntry, m as writeOnce, n as __bitfabAutoTraceActive, o as currentAutoTraceEntries, p as warnOnce, s as invokeThroughAutoTraceEntries, u as runWithAutoTraceNodeConfiguration, v as asyncStorageReady, y as createAsyncLocalStorage } from "./autoTrace-O40k8f5M.js";
 import { r as importOptionalPeer, t as loadBamlModule } from "./bamlPeer-CIVwS68E.js";
 import superjson from "superjson";
@@ -1240,52 +1240,63 @@ var ExperimentsClient = class {
 	}
 };
 //#endregion
-//#region src/graders.ts
-const GRADER_LABELS_PATH = "/api/sdk/graderLabels";
-var GradersClient = class {
-	constructor(httpClient) {
-		this.httpClient = httpClient;
-	}
-	async getLabels(params) {
-		const hasTraceIds = params.traceIds !== void 0 && params.traceIds.length > 0;
-		const hasGraderId = params.graderId !== void 0;
-		if (!hasTraceIds && !hasGraderId) throw new BitfabError("Pass traceIds, graderId, or both: there is nothing to read otherwise.");
-		const query = new URLSearchParams();
-		if (params.traceIds !== void 0 && params.traceIds.length > 0) query.set("traceIds", params.traceIds.join(","));
-		if (params.graderId !== void 0) query.set("graderId", params.graderId);
-		if (params.limit !== void 0) query.set("limit", String(params.limit));
-		return (await this.httpClient.get(`${GRADER_LABELS_PATH}?${query.toString()}`)).labels;
-	}
-	async saveLabel(update) {
-		const [outcome] = await this.saveLabelAll([update]);
-		return outcome;
+//#region src/verdicts.ts
+const VERDICTS_PATH = "/api/sdk/traces/verdicts";
+const HUMAN_VERDICTS_PATH = "/api/sdk/traces/verdicts/human";
+function isSkipOrArchive(fields) {
+	return fields.skip === true || fields.archive === true;
+}
+function toWireVerdict(update) {
+	const fields = update;
+	const legacyLabel = fields.label !== void 0;
+	const legacyAbstain = fields.abstain !== void 0;
+	if (!legacyLabel && !legacyAbstain) return update;
+	if (fields.verdict !== void 0) throw new BitfabError("Pass verdict, or the deprecated label or abstain, not both.");
+	if (legacyLabel && fields.abstain === true) throw new BitfabError("Pass the deprecated label or abstain, not both. Pass verdict: \"pass\" | \"fail\" | \"abstained\" instead.");
+	const { label, abstain, ...rest } = fields;
+	if (!legacyLabel && abstain !== true) {
+		if (isSkipOrArchive(fields)) return rest;
+		throw new BitfabError("abstain: false is not a verdict. Pass verdict: \"pass\" | \"fail\" | \"abstained\" instead.");
+	}
+	if (legacyLabel) {
+		warnOnce("deprecated-label-field", "The label field is deprecated; pass verdict: \"pass\" | \"fail\" instead.");
+		return {
+			...rest,
+			verdict: label === true ? "pass" : "fail"
+		};
 	}
-	async saveLabelAll(updates) {
-		return (await this.httpClient.request(GRADER_LABELS_PATH, { labels: updates })).labels;
+	warnOnce("deprecated-abstain-field", "The abstain field is deprecated; pass verdict: \"abstained\" or call verdicts.abstain instead.");
+	return {
+		...rest,
+		verdict: "abstained"
+	};
+}
+function assertAbstainedCarriesNoExtras(updates) {
+	for (const update of updates) {
+		const fields = update;
+		if (fields.verdict !== "abstained") continue;
+		const extra = fields.confidence !== void 0 ? "confidence" : fields.evidence !== void 0 ? "evidence" : null;
+		if (extra !== null) throw new BitfabError(`An "abstained" verdict carries no ${extra}, since nothing was found to judge.`);
 	}
-};
-//#endregion
-//#region src/labels.ts
-const LABELS_PATH = "/api/sdk/traces/labels";
-const HUMAN_LABELS_PATH = "/api/sdk/traces/labels/human";
+}
 function assertEveryVerdictNamesAnAssertion(updates) {
 	const unscoped = updates.filter((update) => !("archive" in update && update.archive === true) && (typeof update.assertionId !== "string" || update.assertionId === ""));
 	if (unscoped.length > 0) {
 		const targets = unscoped.map((update) => update.traceId ?? update.originalTraceId);
-		throw new BitfabError(`Agent verdicts must name an assertion: ${targets.join(", ")}. Pass the assertionId of the assertion each verdict scores to labels.save, saveAll, skip, and abstain. Only archive may omit it, to clear an old whole-trace verdict.`);
+		throw new BitfabError(`Agent verdicts must name an assertion: ${targets.join(", ")}. Pass the assertionId of the assertion each verdict scores to verdicts.save, saveAll, skip, and abstain. Only archive may omit it, to clear an old whole-trace verdict.`);
 	}
 }
 function assertEveryHumanVerdictNamesAnAssertion(updates) {
 	const unscoped = updates.filter((update) => typeof update.assertionId !== "string" || update.assertionId === "");
 	if (unscoped.length > 0) {
 		const targets = unscoped.map((update) => update.traceId);
-		throw new BitfabError(`Human verdicts must name an assertion: ${targets.join(", ")}. Pass the assertionId of the assertion each verdict scores to labels.saveHuman and saveHumanAll.`);
+		throw new BitfabError(`Human verdicts must name an assertion: ${targets.join(", ")}. Pass the assertionId of the assertion each verdict scores to verdicts.saveHuman and saveHumanAll.`);
 	}
 }
-function labelEvidencePath(traceId, assertionId) {
+function verdictEvidencePath(traceId, assertionId) {
 	return `/api/sdk/traces/${encodeURIComponent(traceId)}/assertions/${encodeURIComponent(assertionId)}/evidence`;
 }
-var LabelsClient = class {
+var VerdictsClient = class {
 	constructor(httpClient) {
 		this.httpClient = httpClient;
 	}
@@ -1295,10 +1306,12 @@ var LabelsClient = class {
 	}
 	async saveAll(updates, experimentId) {
 		assertEveryVerdictNamesAnAssertion(updates);
-		return (await this.httpClient.request(LABELS_PATH, {
-			labels: updates,
+		const verdicts = updates.map((update) => toWireVerdict(update));
+		assertAbstainedCarriesNoExtras(verdicts);
+		return (await this.httpClient.request(VERDICTS_PATH, {
+			verdicts,
 			...experimentId === void 0 ? {} : experimentIdWireFields(experimentId)
-		})).labels;
+		})).verdicts;
 	}
 	async skip(target, experimentId) {
 		return this.save({
@@ -1309,7 +1322,7 @@ var LabelsClient = class {
 	async abstain(target, experimentId) {
 		return this.save({
 			...target,
-			abstain: true
+			verdict: "abstained"
 		}, experimentId);
 	}
 	async archive(target, experimentId) {
@@ -1324,19 +1337,73 @@ var LabelsClient = class {
 	}
 	async saveHumanAll(updates) {
 		assertEveryHumanVerdictNamesAnAssertion(updates);
-		return (await this.httpClient.request(HUMAN_LABELS_PATH, { labels: updates })).labels;
+		const verdicts = updates.map((update) => toWireVerdict(update));
+		return (await this.httpClient.request(HUMAN_VERDICTS_PATH, { verdicts })).verdicts.map((outcome) => ({
+			...outcome,
+			label: outcome.verdict === "pass"
+		}));
 	}
 	async get(traceId) {
-		const [labels] = await this.getAll([traceId]);
-		return labels ?? null;
+		const [verdicts] = await this.getAll([traceId]);
+		return verdicts ?? null;
 	}
 	async getAll(traceIds) {
 		if (traceIds.length === 0) return [];
 		const query = new URLSearchParams({ traceIds: traceIds.join(",") });
-		return (await this.httpClient.get(`${LABELS_PATH}?${query.toString()}`)).labels;
+		return (await this.httpClient.get(`${VERDICTS_PATH}?${query.toString()}`)).verdicts;
+	}
+	async generateEvidence(traceId, assertionId) {
+		return (await this.httpClient.get(verdictEvidencePath(traceId, assertionId))).evidence;
 	}
 	async generateLabelEvidence(traceId, assertionId) {
-		return (await this.httpClient.get(labelEvidencePath(traceId, assertionId))).evidence;
+		return this.generateEvidence(traceId, assertionId);
+	}
+};
+const LabelsClient = VerdictsClient;
+//#endregion
+//#region src/graders.ts
+const GRADER_VERDICTS_PATH = "/api/sdk/graderVerdicts";
+function withDeprecatedGraderFields(row) {
+	return {
+		...row,
+		label: row.verdict === null ? null : row.verdict === "pass",
+		labelReason: row.reason,
+		labelConfidence: row.confidence
+	};
+}
+var GradersClient = class {
+	constructor(httpClient) {
+		this.httpClient = httpClient;
+	}
+	async getVerdicts(params) {
+		const hasTraceIds = params.traceIds !== void 0 && params.traceIds.length > 0;
+		const hasGraderId = params.graderId !== void 0;
+		if (!hasTraceIds && !hasGraderId) throw new BitfabError("Pass traceIds, graderId, or both: there is nothing to read otherwise.");
+		const query = new URLSearchParams();
+		if (params.traceIds !== void 0 && params.traceIds.length > 0) query.set("traceIds", params.traceIds.join(","));
+		if (params.graderId !== void 0) query.set("graderId", params.graderId);
+		if (params.limit !== void 0) query.set("limit", String(params.limit));
+		return (await this.httpClient.get(`${GRADER_VERDICTS_PATH}?${query.toString()}`)).verdicts.map(withDeprecatedGraderFields);
+	}
+	async saveVerdict(update) {
+		const [outcome] = await this.saveVerdictAll([update]);
+		return outcome;
+	}
+	async saveVerdictAll(updates) {
+		const verdicts = updates.map((update) => toWireVerdict(update));
+		return (await this.httpClient.request(GRADER_VERDICTS_PATH, { verdicts })).verdicts.map((outcome) => ({
+			...outcome,
+			label: outcome.verdict === "pass"
+		}));
+	}
+	async getLabels(params) {
+		return this.getVerdicts(params);
+	}
+	async saveLabel(update) {
+		return this.saveVerdict(update);
+	}
+	async saveLabelAll(updates) {
+		return this.saveVerdictAll(updates);
 	}
 };
 //#endregion
@@ -3465,7 +3532,7 @@ async function runReplaySession(httpClient, fn, session, fullExperimentUrl, maxC
 			throw new ReplayError(cause.message, resultItems, experimentId, fullExperimentUrl, cause);
 		}
 		if (missing.length > 0) try {
-			console.error(`Bitfab: server has no persisted trace for ${missing.length} of ${completedCount} completed replay item(s) (experiment ${experimentId}). Their replay token usage is unavailable and they cannot be labeled.`);
+			console.error(`Bitfab: server has no persisted trace for ${missing.length} of ${completedCount} completed replay item(s) (experiment ${experimentId}). Their replay token usage is unavailable and they cannot be given a verdict.`);
 		} catch {}
 	}
 	const result = {
@@ -4820,7 +4887,8 @@ var Bitfab = class {
 		this.organizationMembers = new OrganizationMembersClient(this.httpClient);
 		this.traces = new TracesClient(this.httpClient);
 		this.spans = new SpansClient(this.httpClient);
-		this.labels = new LabelsClient(this.httpClient);
+		this.verdicts = new VerdictsClient(this.httpClient);
+		this.labels = this.verdicts;
 		this.graders = new GradersClient(this.httpClient);
 		this.experiments = new ExperimentsClient(this.httpClient);
 	}
@@ -5910,7 +5978,7 @@ var Bitfab = class {
 			await runWithSeedContext({ traceId }, async () => target(...args));
 		} finally {
 			try {
-				const { flushTraces } = await import("./http-DdlELWcL.js").then((n) => n.a);
+				const { flushTraces } = await import("./http-D91qP2J-.js").then((n) => n.a);
 				await flushTraces(3e4);
 			} finally {
 				unrecorded = activeTraceStates.delete(traceId);
@@ -6098,7 +6166,7 @@ async function seedFromRegistry(registry, pipeline, cases, options = {}) {
 		metadata: seedCase.metadata,
 		sessionId: seedCase.sessionId
 	}));
-	const { flushTraces } = await import("./http-DdlELWcL.js").then((n) => n.a);
+	const { flushTraces } = await import("./http-D91qP2J-.js").then((n) => n.a);
 	await flushTraces(3e4);
 	return {
 		pipeline,
@@ -6113,6 +6181,6 @@ function resolveTraceFunctionKey(registration) {
 	return key;
 }
 //#endregion
-export { SUPPORTED_PROVIDERS as A, BitfabOpenAIAgentHandler as C, LabelsClient as D, BitfabLangGraphCallbackHandler as E, DEFAULT_SERVICE_URL as F, AssertionCategoriesClient as I, CategoriesClient as L, GraderRerunError as M, GraderRerunTimeoutError as N, GradersClient as O, BitfabClaudeAgentHandler as P, OrganizationMembersClient as S, BitfabLangGraphIntegration as T, BITFAB_PROGRESS_PREFIX as _, Bitfab as a, reportReplayProgress as b, getCurrentSpan as c, BitfabOpenAITracingProcessor as d, AssertionGenerationError as f, BitfabRivetIntegration as g, SpansClient as h, finalizers as i, DatasetsClient as j, ExperimentsClient as k, getCurrentTrace as l, TracesClient as m, reseedFromRegistry as n, BitfabFunction as o, AssertionGenerationTimeoutError as p, seedFromRegistry as r, getCurrentReplayBranch as s, defineReplayRegistry as t, BitfabVercelAiHandler as u, DbBranchReplayError as v, NO_MOCK_OVERRIDE as w, serializeReplayResult as x, ReplayError as y };
+export { ExperimentsClient as A, BitfabOpenAIAgentHandler as C, GradersClient as D, BitfabLangGraphCallbackHandler as E, BitfabClaudeAgentHandler as F, DEFAULT_SERVICE_URL as I, AssertionCategoriesClient as L, DatasetsClient as M, GraderRerunError as N, LabelsClient as O, GraderRerunTimeoutError as P, CategoriesClient as R, OrganizationMembersClient as S, BitfabLangGraphIntegration as T, BITFAB_PROGRESS_PREFIX as _, Bitfab as a, reportReplayProgress as b, getCurrentSpan as c, BitfabOpenAITracingProcessor as d, AssertionGenerationError as f, BitfabRivetIntegration as g, SpansClient as h, finalizers as i, SUPPORTED_PROVIDERS as j, VerdictsClient as k, getCurrentTrace as l, TracesClient as m, reseedFromRegistry as n, BitfabFunction as o, AssertionGenerationTimeoutError as p, seedFromRegistry as r, getCurrentReplayBranch as s, defineReplayRegistry as t, BitfabVercelAiHandler as u, DbBranchReplayError as v, NO_MOCK_OVERRIDE as w, serializeReplayResult as x, ReplayError as y };
 
-//# sourceMappingURL=src-lTOsAt47.js.map
+//# sourceMappingURL=src-CL5eQlNk.js.map