@bastani/atomic 0.9.14-alpha.4 → 0.9.14-alpha.5

package/dist/builtin/workflows/builtin/tournament.ts

@@ -4,36 +4,70 @@ import { withSteeringPropagationContext } from "./steering-context.js";
 import { runTournament } from "./tournament-runner.js";
 
 export default workflow({
-  name: "tournament",
-  description: "Run several independent whole-task attempts through a balanced pairwise judging bracket and return an auditable winner.",
-  // The 15-minute default, stated rather than inherited: this is a per-workflow
-  // product decision, so a future change to the global default must not silently
-  // re-cadence a long autonomous run.
-  heartbeatIntervalMinutes: 15,
-  inputs: {
-    prompt: Type.String({ description: "Task every competing agent must attempt independently." }),
-    num_attempts: Type.Integer({
-      minimum: 2,
-      maximum: 8,
-      default: 4,
-      description: "Number of independent whole-task attempts (2-8).",
-    }),
-    max_concurrency: Type.Integer({
-      minimum: 1,
-      maximum: 8,
-      default: 4,
-      description: "Maximum simultaneously active attempts or pairwise judges (1-8).",
-    }),
-  },
-  outputs: {
-    result: Type.String({ description: "Compact reference to the final reducer report holding the winning solution and decision trail; read `result_path` for the full report." }),
-    winner: Type.String({ description: "Stable attempt label selected as the tournament winner." }),
-    winner_artifact_path: Type.String({ description: "Path to the original winning attempt artifact." }),
-    result_path: Type.String({ description: "Path to the final reducer report artifact." }),
-    attempt_artifact_paths: Type.Array(Type.String(), { description: "Paths to every independent attempt artifact." }),
-    judge_artifact_paths: Type.Array(Type.String(), { description: "Paths to all structured pairwise judge artifacts." }),
-    bracket_path: Type.String({ description: "Path to the durable JSON bracket with matches, byes, rationales, and winner." }),
-    artifact_dir: Type.String({ description: "Run-specific directory containing tournament artifacts." }),
-  },
-  run: async (ctx) => await runTournament(withSteeringPropagationContext(ctx)),
+	name: "tournament",
+	description: "Run independent whole-task attempts through a soft-scored pivot-pairing schedule and return an auditable ranking.",
+	// The 15-minute default, stated rather than inherited: this is a per-workflow
+	// product decision, so a future change to the global default must not silently
+	// re-cadence a long autonomous run.
+	heartbeatIntervalMinutes: 15,
+	inputs: {
+		prompt: Type.String({ description: "Task every competing agent must attempt independently." }),
+		num_attempts: Type.Integer({
+			minimum: 2,
+			maximum: 8,
+			default: 4,
+			description: "Number of independent whole-task attempts (2-8).",
+		}),
+		max_concurrency: Type.Integer({
+			minimum: 1,
+			maximum: 8,
+			default: 4,
+			description: "Maximum simultaneously active attempts or pairwise judges (1-8).",
+		}),
+		n_evaluations: Type.Integer({
+			minimum: 1,
+			default: 2,
+			description: "Number of repeated evaluations per criterion and directed pair.",
+		}),
+		pivots: Type.Integer({
+			minimum: 1,
+			default: 1,
+			description: "Number of pivot candidates used for the second comparison phase.",
+		}),
+		seed: Type.Integer({
+			default: 0,
+			description: "Seed for the deterministic comparison schedule.",
+		}),
+		criteria: Type.Optional(Type.Union([
+			Type.String(),
+			Type.Record(Type.String(), Type.String()),
+			Type.Array(Type.String()),
+			Type.Array(Type.Object({
+				id: Type.Optional(Type.String()),
+				name: Type.Optional(Type.String()),
+				description: Type.String(),
+			}, { additionalProperties: true })),
+		], {
+			description: "Optional V1 judge criteria; accepts a markdown rubric, record, string list, or CriterionInput list; omitted uses the default three-criterion rubric.",
+		})),
+		models: Type.Optional(Type.Array(Type.String(), {
+			description: "Optional ordered model ids assigned round-robin to attempt slots.",
+		})),
+	},
+	outputs: {
+		result: Type.String({ description: "Compact reference to the final reducer report; read `result_path` for the full report." }),
+		winner: Type.String({ description: "Stable attempt label selected as the top-ranked tournament winner." }),
+		winner_artifact_path: Type.String({ description: "Path to the original winning attempt artifact." }),
+		result_path: Type.String({ description: "Path to the final reducer report artifact." }),
+		attempt_artifact_paths: Type.Array(Type.String(), { description: "Paths to every independent attempt artifact." }),
+		judge_artifact_paths: Type.Array(Type.String(), { description: "Paths to all per-job structured judge artifacts." }),
+		comparisons_path: Type.String({ description: "Path to the durable JSON soft-scored comparisons ledger." }),
+		ranking: Type.Array(Type.Object({
+			label: Type.String(),
+			meanPreference: Type.Number(),
+		}, { additionalProperties: false }), { description: "All attempts ordered by mean preference." }),
+		seed: Type.Integer({ description: "Seed used for the deterministic comparison schedule." }),
+		artifact_dir: Type.String({ description: "Run-specific directory containing tournament artifacts." }),
+	},
+	run: async (ctx) => await runTournament(withSteeringPropagationContext(ctx)),
 });

package/dist/builtin/workflows/builtin/verification-criteria.ts

@@ -0,0 +1,330 @@
+/**
+ * Shared verification rubric primitive.
+ *
+ * Doors: `parse_rubric`, `normalize_criteria`, `select_criteria`,
+ * `decide_verification`. A CriterionScore can only exist as a schema-valid
+ * structured report — an unparseable verifier output is unrepresentable as a
+ * vote, so it cannot shift the mean.
+ */
+import { Type, type TSchema } from "typebox";
+
+export interface Criterion {
+  readonly id: string;
+  readonly name: string;
+  readonly description: string;
+}
+
+export interface Criteria {
+  readonly groundTruthNote: string;
+  readonly criteria: readonly Criterion[];
+}
+
+export interface CriterionInput {
+  readonly id?: string;
+  readonly name?: string;
+  readonly description?: string;
+}
+
+export interface Finding {
+  readonly finding: string;
+  readonly severity: "veto" | "blocking" | "note";
+}
+
+export interface CriterionScore {
+  readonly criterionId: string;
+  readonly score: number;
+  readonly evidence: readonly string[];
+  readonly findings: readonly Finding[];
+}
+
+export type RubricErrorCode = "NoCriteria" | "EmptyCriterion";
+
+export class RubricError extends Error {
+  readonly code: RubricErrorCode;
+
+  constructor(message: string, code: RubricErrorCode) {
+    super(message);
+    this.name = code;
+    this.code = code;
+  }
+}
+
+export class NoCriteria extends RubricError {
+  constructor(message = "no criteria found — check the `## Criteria` section and its `### Name {#id}` headings") {
+    super(message, "NoCriteria");
+    this.name = "NoCriteria";
+  }
+}
+
+export class EmptyCriterion extends RubricError {
+  readonly ids: readonly string[];
+
+  constructor(ids: readonly string[]) {
+    super(`criteria have empty instructions: ${ids.join(", ")}`, "EmptyCriterion");
+    this.name = "EmptyCriterion";
+    this.ids = ids;
+  }
+}
+
+const CRITERION_ID_ANCHOR = /^(.*?)\s*\{#([A-Za-z0-9_-]+)\}\s*$/;
+const SLUG_MAX_LENGTH = 40;
+
+function slug_id(text: string): string {
+  const slug = text
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, "_")
+    .replace(/^_+|_+$/g, "");
+  return slug.slice(0, SLUG_MAX_LENGTH).replace(/_+$/g, "") || "criterion";
+}
+
+function dedup_id(cid: string, seen: Set<string>): string {
+  let out = cid;
+  let n = 1;
+  while (seen.has(out)) {
+    n += 1;
+    out = `${cid}_${n}`;
+  }
+  seen.add(out);
+  return out;
+}
+
+function criterion(id: string, name: string, description: string): Criterion {
+  return { id, name, description };
+}
+
+function strip_html_comments(markdown: string): string {
+  let sanitized = markdown;
+  while (true) {
+    const opener = sanitized.indexOf("<!--");
+    if (opener === -1) return sanitized;
+    const closer = sanitized.indexOf("-->", opener + 4);
+    if (closer === -1) return sanitized.slice(0, opener);
+    sanitized = sanitized.slice(0, opener) + sanitized.slice(closer + 3);
+  }
+}
+
+/**
+ * Parse a `criteria.md` document into normalized criteria.
+ *
+ * `# title` is ignored. `## Ground Truth Note` is optional; the first section
+ * wins. `## Criteria` owns `### Name {#id}` headings. HTML comments are stripped
+ * before parse. Ids slug to lowercase alnum/underscore, ≤40 chars, fallback
+ * `"criterion"`, with encounter-order `_2` / `_3` dedup.
+ *
+ * @throws {NoCriteria} when the Criteria section yields no headings
+ * @throws {EmptyCriterion} when any criterion has an empty body
+ */
+export function parse_rubric(markdown: string): Criteria {
+  const lines = strip_html_comments(markdown).split(/\r?\n/);
+
+  let groundTruthNote = "";
+  let seenGroundTruth = false;
+  const criteria: Criterion[] = [];
+  const seen = new Set<string>();
+
+  let section: "ground_truth" | "criteria" | null = null;
+  let currentName: string | null = null;
+  let currentId: string | null = null;
+  let buf: string[] = [];
+
+  const flush = (): void => {
+    const text = buf.join("\n").trim();
+    buf = [];
+    if (section === "ground_truth") {
+      if (!seenGroundTruth) {
+        groundTruthNote = text;
+        seenGroundTruth = true;
+      }
+      return;
+    }
+    if (currentName === null || currentId === null) return;
+    criteria.push(criterion(currentId, currentName, text));
+    currentName = null;
+    currentId = null;
+  };
+
+  for (const line of lines) {
+    if (line.startsWith("## ") && !line.startsWith("### ")) {
+      flush();
+      const heading = line.slice(3).trim().toLowerCase();
+      if (heading.includes("ground truth")) {
+        section = "ground_truth";
+      } else if (heading.includes("criteri")) {
+        section = "criteria";
+      } else {
+        section = null;
+      }
+      continue;
+    }
+    if (line.startsWith("### ") && section === "criteria") {
+      flush();
+      const heading = line.slice(4).trim();
+      const anchored = CRITERION_ID_ANCHOR.exec(heading);
+      const name = anchored ? anchored[1]!.trim() : heading;
+      const rawId = anchored ? anchored[2]!.trim() : slug_id(heading);
+      currentName = name;
+      currentId = dedup_id(rawId, seen);
+      continue;
+    }
+    if (line.startsWith("# ")) {
+      continue;
+    }
+    buf.push(line);
+  }
+  flush();
+
+  if (criteria.length === 0) {
+    throw new NoCriteria();
+  }
+  const empty = criteria.filter((item) => item.description.length === 0).map((item) => item.id);
+  if (empty.length > 0) {
+    throw new EmptyCriterion(empty);
+  }
+  return { groundTruthNote, criteria };
+}
+
+function isRecord(value: object): value is Record<string, string> {
+  return Object.values(value).every((entry) => typeof entry === "string");
+}
+
+function isOptionalString(value: object, key: "id" | "name" | "description"): boolean {
+  if (!(key in value)) return true;
+  return typeof (value as CriterionInput)[key] === "string";
+}
+
+function isCriterionInput(value: object): value is CriterionInput {
+  return isOptionalString(value, "id") && isOptionalString(value, "name") && isOptionalString(value, "description");
+}
+
+function normalize_one(raw: string | CriterionInput, index: number, seen: Set<string>): Criterion {
+  const cidRaw = typeof raw === "string" ? "" : String(raw.id ?? "");
+  const nameRaw = typeof raw === "string" ? raw : String(raw.name ?? "");
+  const desc = typeof raw === "string" ? raw : String(raw.description ?? "");
+  if (desc.length === 0) {
+    throw new EmptyCriterion([cidRaw || nameRaw || String(index)]);
+  }
+  const name = nameRaw || cidRaw || slug_id(desc);
+  const id = dedup_id(cidRaw || slug_id(name), seen);
+  return criterion(id, name, desc);
+}
+
+/**
+ * Canonicalize any accepted criteria shape into `{id,name,description}[]`.
+ *
+ * Same slug/dedup rules as `parse_rubric`. Empty descriptions throw
+ * `EmptyCriterion`; an empty collection throws `NoCriteria`.
+ */
+export function normalize_criteria(
+  input: Record<string, string> | readonly string[] | readonly CriterionInput[],
+): readonly Criterion[] {
+  if (input === null || typeof input !== "object") {
+    throw new TypeError(`criteria must be a record, string[], or CriterionInput[], got ${typeof input}`);
+  }
+
+  const rawItems: Array<string | CriterionInput> = [];
+  if (Array.isArray(input)) {
+    for (const [index, raw] of input.entries()) {
+      if (typeof raw === "string") {
+        rawItems.push(raw);
+        continue;
+      }
+      if (isCriterionInput(raw)) {
+        rawItems.push(raw);
+        continue;
+      }
+      throw new TypeError(`criteria[${index}] must be a string or CriterionInput`);
+    }
+  } else if (isRecord(input)) {
+    for (const [name, description] of Object.entries(input)) {
+      rawItems.push({ name, description });
+    }
+  } else {
+    throw new TypeError("criteria record values must be strings");
+  }
+
+  const seen = new Set<string>();
+  const out = rawItems.map((raw, index) => normalize_one(raw, index, seen));
+  if (out.length === 0) {
+    throw new NoCriteria("criteria is empty");
+  }
+  return out;
+}
+
+/**
+ * Return the subset of `criteria` in `ids` order.
+ *
+ * Omitted `ids` returns every criterion in the original order. An unknown id
+ * throws — nothing is silently dropped.
+ */
+export function select_criteria(criteria: readonly Criterion[], ids?: readonly string[]): readonly Criterion[] {
+  if (ids === undefined) return criteria;
+  const byId = new Map(criteria.map((item) => [item.id, item]));
+  const missing = ids.filter((id) => !byId.has(id));
+  if (missing.length > 0) {
+    throw new Error(`criteria not found: ${missing.join(", ")}`);
+  }
+  return ids.map((id) => byId.get(id)!);
+}
+
+export const VERIFICATION_SCALE: {
+  readonly min: 1;
+  readonly max: 20;
+  readonly anchors: string;
+  readonly schema: TSchema;
+} = {
+  min: 1,
+  max: 20,
+  anchors: "1 = certainly fails … 10 = borderline … 20 = verified correct",
+  schema: Type.Integer({ minimum: 1, maximum: 20 }),
+};
+
+export type Accept = { readonly kind: "accept"; readonly mean: number };
+export type Repair = { readonly kind: "repair"; readonly mean: number; readonly findings: readonly Finding[] };
+export type Indeterminate = { readonly kind: "indeterminate"; readonly missing: number };
+export type VerificationDecision = Accept | Repair | Indeterminate;
+
+export type VerificationRound = {
+  readonly scores: readonly CriterionScore[];
+  readonly invalidCount: number;
+  readonly expectedCount: number;
+};
+
+export type VerificationPolicy = {
+  readonly acceptMean: number;
+  readonly quorumFraction: number;
+};
+
+function mean_score(scores: readonly CriterionScore[]): number {
+  if (scores.length === 0) return 0;
+  let total = 0;
+  for (const item of scores) total += item.score;
+  return total / scores.length;
+}
+
+/**
+ * Map one round of schema-valid scores to accept / repair / indeterminate.
+ *
+ * Accept iff quorum holds AND mean ≥ acceptMean AND no `severity: "veto"`
+ * finding exists. Any veto forces Repair regardless of mean. Below quorum
+ * (`valid < ceil(expected × quorumFraction)`) is Indeterminate.
+ *
+ * `invalidCount` is metadata only: there is no constructor that turns an
+ * unparseable report into a CriterionScore, so a parse failure cannot shift
+ * the mean.
+ */
+export function decide_verification(round: VerificationRound, policy: VerificationPolicy): VerificationDecision {
+  // `invalidCount` is intentionally unread. It cannot enter the mean.
+  const { scores, expectedCount } = round;
+  const valid = scores.length;
+  const quorumNeeded = Math.ceil(expectedCount * policy.quorumFraction);
+  if (valid < quorumNeeded) {
+    return { kind: "indeterminate", missing: Math.max(0, expectedCount - valid) };
+  }
+
+  const mean = mean_score(scores);
+  const findings = scores.flatMap((item) => item.findings);
+  if (findings.some((item) => item.severity === "veto") || mean < policy.acceptMean) {
+    return { kind: "repair", mean, findings };
+  }
+  return { kind: "accept", mean };
+}

package/dist/builtin/workflows/builtin/verification-prompts.ts

@@ -0,0 +1,206 @@
+/**
+ * Prefix-cache-aware verification prompts and warm-first fan-out.
+ *
+ * A scoring-family prompt is always `SHARED HEAD ‖ VARYING TAIL`: the head
+ * contains the task statement, ground-truth note, candidate bodies (or read
+ * references when the family is too large), and scale anchors, in that order.
+ * The tail contains only the criterion name and description plus the output
+ * format instruction. Keeping this ordering invariant means sibling criteria
+ * can share the provider's cached prefix byte-for-byte.
+ *
+ * A pathless candidate is valid while the whole family is inline. If any body
+ * exceeds the inline bound, every candidate must provide its caller-bound path;
+ * this layer refuses the family rather than inventing a read path.
+ */
+import type {
+	WorkflowParallelOptions,
+	WorkflowRunContext,
+	WorkflowTaskResult,
+	WorkflowTaskStep,
+} from "../src/shared/types.js";
+import type { Criterion } from "./verification-criteria.js";
+import { VERIFICATION_SCALE } from "./verification-criteria.js";
+
+/** Maximum UTF-8 byte length of one candidate body eligible for inlining. */
+export const MAX_INLINE_CANDIDATE_BYTES = 32 * 1024;
+
+export interface ScoringCandidate {
+	/**
+	 * Optional only while every candidate body is inline. An oversized family
+	 * requires a caller-bound path for every candidate; paths are preserved
+	 * verbatim and are never synthesized.
+	 */
+	readonly path?: string;
+	readonly body: string;
+}
+
+export interface SharedHead {
+	readonly task: string;
+	readonly groundTruthNote: string;
+	readonly candidates: readonly ScoringCandidate[];
+	readonly scaleAnchors?: string;
+	readonly outputFormat?: string;
+}
+
+export interface WarmFirstFanOutOptions extends WorkflowParallelOptions {
+	/** Maximum number of distinct prefixes warmed at once. */
+	readonly warmConcurrency?: number;
+}
+
+export type WarmFirstFanOutContext = Pick<WorkflowRunContext, "parallel">;
+
+const DEFAULT_WARM_CONCURRENCY = 4;
+const DEFAULT_OUTPUT_FORMAT =
+	"Call structured_output with criterion_id, score (1–20), evidence, and findings containing finding and severity.";
+
+function inlineCandidates(head: SharedHead): boolean {
+	return head.candidates.every(
+		(candidate) => Buffer.byteLength(candidate.body, "utf8") <= MAX_INLINE_CANDIDATE_BYTES,
+	);
+}
+
+function fallbackReadPaths(head: SharedHead): readonly string[] | undefined {
+	if (inlineCandidates(head)) return undefined;
+	const paths: string[] = [];
+	for (const [index, candidate] of head.candidates.entries()) {
+		if (candidate.path === undefined) {
+			throw new TypeError(
+				`Oversized candidate family requires a caller-bound path for every candidate; candidate ${index + 1} has no path.`,
+			);
+		}
+		paths.push(candidate.path);
+	}
+	return paths;
+}
+
+function candidateSection(head: SharedHead, readPaths: readonly string[] | undefined): string {
+	return head.candidates
+		.map((candidate, index) => {
+			if (readPaths === undefined) {
+				return `<candidate index="${index + 1}">\n${candidate.body}\n</candidate>`;
+			}
+			return `<candidate index="${index + 1}" source="read">\nRead candidate from ${readPaths[index]!}.\n</candidate>`;
+		})
+		.join("\n");
+}
+
+/**
+ * Return candidate paths required by the read fallback.
+ *
+ * The whole family switches together: either every body is in the shared head,
+ * or every candidate is named as a caller-provided read. Order and duplicate
+ * paths are kept; an oversized pathless family throws instead of guessing.
+ */
+export function scoring_prompt_reads(head: SharedHead): readonly string[] {
+	return fallbackReadPaths(head) ?? [];
+}
+
+/**
+ * Build one scoring prompt with a byte-identical shared head and criterion tail.
+ * Candidate bodies are measured in UTF-8 bytes, not JavaScript code units. If
+ * any body exceeds `MAX_INLINE_CANDIDATE_BYTES`, no body is inlined for the
+ * family, preserving the same head for every sibling criterion.
+ */
+export function build_scoring_prompt(head: SharedHead, criterion: Criterion): string {
+	const readPaths = fallbackReadPaths(head);
+	const sharedHead = [
+		"<scoring_head>",
+		"<task_statement>",
+		head.task,
+		"</task_statement>",
+		"<ground_truth_note>",
+		head.groundTruthNote,
+		"</ground_truth_note>",
+		"<candidates>",
+		candidateSection(head, readPaths),
+		"</candidates>",
+		"<scale_anchors>",
+		head.scaleAnchors ?? VERIFICATION_SCALE.anchors,
+		"</scale_anchors>",
+	].join("\n");
+	const varyingTail = [
+		"<criterion>",
+		`<name>${criterion.name}</name>`,
+		`<description>${criterion.description}</description>`,
+		"</criterion>",
+		"<output_format>",
+		head.outputFormat ?? DEFAULT_OUTPUT_FORMAT,
+		"</output_format>",
+	].join("\n");
+	return `${sharedHead}\n\n${varyingTail}`;
+}
+
+function boundedConcurrency(
+	stepCount: number,
+	phaseCap: number | undefined,
+	inheritedCap: number | undefined,
+): number {
+	return Math.min(stepCount, phaseCap ?? stepCount, inheritedCap ?? stepCount);
+}
+
+/**
+ * Run one warm step per prefix, then release the remaining steps. Prefix keys
+ * are caller-owned; first-seen order is deterministic. Warm failures are
+ * observed without fail-fast, and the second phase is always attempted before
+ * the warm error is rethrown. Successful results are returned in input order.
+ */
+export async function warm_first_fan_out<K>(
+	ctx: WarmFirstFanOutContext,
+	steps: readonly WorkflowTaskStep[],
+	prefixKeyOf: (step: WorkflowTaskStep, index: number) => K,
+	options: WarmFirstFanOutOptions = {},
+): Promise<WorkflowTaskResult[]> {
+	const warmIndices: number[] = [];
+	const restIndices: number[] = [];
+	const seen = new Set<K>();
+	for (const [index, step] of steps.entries()) {
+		const prefixKey = prefixKeyOf(step, index);
+		if (seen.has(prefixKey)) restIndices.push(index);
+		else {
+			seen.add(prefixKey);
+			warmIndices.push(index);
+		}
+	}
+
+	const { warmConcurrency, ...parallelOptions } = options;
+	const inheritedConcurrency = parallelOptions.concurrency;
+	const resultsByIndex = new Map<number, WorkflowTaskResult>();
+	let warmFailure: Error | undefined;
+	if (warmIndices.length > 0) {
+		const warmSteps = warmIndices.map((index) => steps[index]!);
+		try {
+			const warmResults = await ctx.parallel(warmSteps, {
+				...parallelOptions,
+				concurrency: boundedConcurrency(
+					warmSteps.length,
+					warmConcurrency ?? DEFAULT_WARM_CONCURRENCY,
+					inheritedConcurrency,
+				),
+				failFast: false,
+			});
+			for (const [resultIndex, result] of warmResults.entries()) {
+				const originalIndex = warmIndices[resultIndex];
+				if (originalIndex !== undefined) resultsByIndex.set(originalIndex, result);
+			}
+		} catch (error) {
+			warmFailure = error instanceof Error ? error : new Error(String(error));
+		}
+	}
+
+	if (restIndices.length > 0) {
+		const restSteps = restIndices.map((index) => steps[index]!);
+		const restResults = await ctx.parallel(restSteps, {
+			...parallelOptions,
+			concurrency: boundedConcurrency(restSteps.length, undefined, inheritedConcurrency),
+		});
+		for (const [resultIndex, result] of restResults.entries()) {
+			const originalIndex = restIndices[resultIndex];
+			if (originalIndex !== undefined) resultsByIndex.set(originalIndex, result);
+		}
+	}
+
+	if (warmFailure !== undefined) throw warmFailure;
+	return steps
+		.map((_, index) => resultsByIndex.get(index))
+		.filter((result): result is WorkflowTaskResult => result !== undefined);
+}

package/dist/builtin/workflows/builtin/verification-usage.ts

@@ -0,0 +1,44 @@
+import type { WorkflowTaskResult } from "../src/shared/types.js";
+
+export type UsageTotals = {
+	calls: number;
+	input: number;
+	output: number;
+	cacheRead: number;
+	cacheWrite: number;
+	cost: number;
+	turns: number;
+	cacheHitRate: number;
+};
+
+export function fold_usage(results: readonly WorkflowTaskResult[]): UsageTotals {
+	const totals = {
+		calls: 0,
+		input: 0,
+		output: 0,
+		cacheRead: 0,
+		cacheWrite: 0,
+		cost: 0,
+		turns: 0,
+	};
+
+	for (const result of results) {
+		for (const attempt of result.modelAttempts ?? []) {
+			totals.calls += 1;
+			const usage = attempt.usage;
+			if (usage === undefined) continue;
+			totals.input += usage.input ?? 0;
+			totals.output += usage.output ?? 0;
+			totals.cacheRead += usage.cacheRead ?? 0;
+			totals.cacheWrite += usage.cacheWrite ?? 0;
+			totals.cost += usage.cost ?? 0;
+			totals.turns += usage.turns ?? 0;
+		}
+	}
+
+	const cacheDenominator = totals.input + totals.cacheRead;
+	return {
+		...totals,
+		cacheHitRate: cacheDenominator === 0 ? 0 : totals.cacheRead / cacheDenominator,
+	};
+}

package/dist/builtin/workflows/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bastani/workflows",
-  "version": "0.9.14-alpha.4",
+  "version": "0.9.14-alpha.5",
   "private": true,
   "description": "Atomic extension for multi-stage workflow authoring and execution.",
   "contributors": [