@bastani/atomic 0.9.16-alpha.6 → 0.9.16-alpha.8

package/dist/builtin/workflows/builtin/verification-criteria.ts

@@ -1,330 +0,0 @@
-/**
- * Shared verification rubric primitive.
- *
- * Doors: `parse_rubric`, `normalize_criteria`, `select_criteria`,
- * `decide_verification`. A CriterionScore can only exist as a schema-valid
- * structured report — an unparseable verifier output is unrepresentable as a
- * vote, so it cannot shift the mean.
- */
-import { Type, type TSchema } from "typebox";
-
-export interface Criterion {
-  readonly id: string;
-  readonly name: string;
-  readonly description: string;
-}
-
-export interface Criteria {
-  readonly groundTruthNote: string;
-  readonly criteria: readonly Criterion[];
-}
-
-export interface CriterionInput {
-  readonly id?: string;
-  readonly name?: string;
-  readonly description?: string;
-}
-
-export interface Finding {
-  readonly finding: string;
-  readonly severity: "veto" | "blocking" | "note";
-}
-
-export interface CriterionScore {
-  readonly criterionId: string;
-  readonly score: number;
-  readonly evidence: readonly string[];
-  readonly findings: readonly Finding[];
-}
-
-export type RubricErrorCode = "NoCriteria" | "EmptyCriterion";
-
-export class RubricError extends Error {
-  readonly code: RubricErrorCode;
-
-  constructor(message: string, code: RubricErrorCode) {
-    super(message);
-    this.name = code;
-    this.code = code;
-  }
-}
-
-export class NoCriteria extends RubricError {
-  constructor(message = "no criteria found — check the `## Criteria` section and its `### Name {#id}` headings") {
-    super(message, "NoCriteria");
-    this.name = "NoCriteria";
-  }
-}
-
-export class EmptyCriterion extends RubricError {
-  readonly ids: readonly string[];
-
-  constructor(ids: readonly string[]) {
-    super(`criteria have empty instructions: ${ids.join(", ")}`, "EmptyCriterion");
-    this.name = "EmptyCriterion";
-    this.ids = ids;
-  }
-}
-
-const CRITERION_ID_ANCHOR = /^(.*?)\s*\{#([A-Za-z0-9_-]+)\}\s*$/;
-const SLUG_MAX_LENGTH = 40;
-
-function slug_id(text: string): string {
-  const slug = text
-    .toLowerCase()
-    .replace(/[^a-z0-9]+/g, "_")
-    .replace(/^_+|_+$/g, "");
-  return slug.slice(0, SLUG_MAX_LENGTH).replace(/_+$/g, "") || "criterion";
-}
-
-function dedup_id(cid: string, seen: Set<string>): string {
-  let out = cid;
-  let n = 1;
-  while (seen.has(out)) {
-    n += 1;
-    out = `${cid}_${n}`;
-  }
-  seen.add(out);
-  return out;
-}
-
-function criterion(id: string, name: string, description: string): Criterion {
-  return { id, name, description };
-}
-
-function strip_html_comments(markdown: string): string {
-  let sanitized = markdown;
-  while (true) {
-    const opener = sanitized.indexOf("<!--");
-    if (opener === -1) return sanitized;
-    const closer = sanitized.indexOf("-->", opener + 4);
-    if (closer === -1) return sanitized.slice(0, opener);
-    sanitized = sanitized.slice(0, opener) + sanitized.slice(closer + 3);
-  }
-}
-
-/**
- * Parse a `criteria.md` document into normalized criteria.
- *
- * `# title` is ignored. `## Ground Truth Note` is optional; the first section
- * wins. `## Criteria` owns `### Name {#id}` headings. HTML comments are stripped
- * before parse. Ids slug to lowercase alnum/underscore, ≤40 chars, fallback
- * `"criterion"`, with encounter-order `_2` / `_3` dedup.
- *
- * @throws {NoCriteria} when the Criteria section yields no headings
- * @throws {EmptyCriterion} when any criterion has an empty body
- */
-export function parse_rubric(markdown: string): Criteria {
-  const lines = strip_html_comments(markdown).split(/\r?\n/);
-
-  let groundTruthNote = "";
-  let seenGroundTruth = false;
-  const criteria: Criterion[] = [];
-  const seen = new Set<string>();
-
-  let section: "ground_truth" | "criteria" | null = null;
-  let currentName: string | null = null;
-  let currentId: string | null = null;
-  let buf: string[] = [];
-
-  const flush = (): void => {
-    const text = buf.join("\n").trim();
-    buf = [];
-    if (section === "ground_truth") {
-      if (!seenGroundTruth) {
-        groundTruthNote = text;
-        seenGroundTruth = true;
-      }
-      return;
-    }
-    if (currentName === null || currentId === null) return;
-    criteria.push(criterion(currentId, currentName, text));
-    currentName = null;
-    currentId = null;
-  };
-
-  for (const line of lines) {
-    if (line.startsWith("## ") && !line.startsWith("### ")) {
-      flush();
-      const heading = line.slice(3).trim().toLowerCase();
-      if (heading.includes("ground truth")) {
-        section = "ground_truth";
-      } else if (heading.includes("criteri")) {
-        section = "criteria";
-      } else {
-        section = null;
-      }
-      continue;
-    }
-    if (line.startsWith("### ") && section === "criteria") {
-      flush();
-      const heading = line.slice(4).trim();
-      const anchored = CRITERION_ID_ANCHOR.exec(heading);
-      const name = anchored ? anchored[1]!.trim() : heading;
-      const rawId = anchored ? anchored[2]!.trim() : slug_id(heading);
-      currentName = name;
-      currentId = dedup_id(rawId, seen);
-      continue;
-    }
-    if (line.startsWith("# ")) {
-      continue;
-    }
-    buf.push(line);
-  }
-  flush();
-
-  if (criteria.length === 0) {
-    throw new NoCriteria();
-  }
-  const empty = criteria.filter((item) => item.description.length === 0).map((item) => item.id);
-  if (empty.length > 0) {
-    throw new EmptyCriterion(empty);
-  }
-  return { groundTruthNote, criteria };
-}
-
-function isRecord(value: object): value is Record<string, string> {
-  return Object.values(value).every((entry) => typeof entry === "string");
-}
-
-function isOptionalString(value: object, key: "id" | "name" | "description"): boolean {
-  if (!(key in value)) return true;
-  return typeof (value as CriterionInput)[key] === "string";
-}
-
-function isCriterionInput(value: object): value is CriterionInput {
-  return isOptionalString(value, "id") && isOptionalString(value, "name") && isOptionalString(value, "description");
-}
-
-function normalize_one(raw: string | CriterionInput, index: number, seen: Set<string>): Criterion {
-  const cidRaw = typeof raw === "string" ? "" : String(raw.id ?? "");
-  const nameRaw = typeof raw === "string" ? raw : String(raw.name ?? "");
-  const desc = typeof raw === "string" ? raw : String(raw.description ?? "");
-  if (desc.length === 0) {
-    throw new EmptyCriterion([cidRaw || nameRaw || String(index)]);
-  }
-  const name = nameRaw || cidRaw || slug_id(desc);
-  const id = dedup_id(cidRaw || slug_id(name), seen);
-  return criterion(id, name, desc);
-}
-
-/**
- * Canonicalize any accepted criteria shape into `{id,name,description}[]`.
- *
- * Same slug/dedup rules as `parse_rubric`. Empty descriptions throw
- * `EmptyCriterion`; an empty collection throws `NoCriteria`.
- */
-export function normalize_criteria(
-  input: Record<string, string> | readonly string[] | readonly CriterionInput[],
-): readonly Criterion[] {
-  if (input === null || typeof input !== "object") {
-    throw new TypeError(`criteria must be a record, string[], or CriterionInput[], got ${typeof input}`);
-  }
-
-  const rawItems: Array<string | CriterionInput> = [];
-  if (Array.isArray(input)) {
-    for (const [index, raw] of input.entries()) {
-      if (typeof raw === "string") {
-        rawItems.push(raw);
-        continue;
-      }
-      if (isCriterionInput(raw)) {
-        rawItems.push(raw);
-        continue;
-      }
-      throw new TypeError(`criteria[${index}] must be a string or CriterionInput`);
-    }
-  } else if (isRecord(input)) {
-    for (const [name, description] of Object.entries(input)) {
-      rawItems.push({ name, description });
-    }
-  } else {
-    throw new TypeError("criteria record values must be strings");
-  }
-
-  const seen = new Set<string>();
-  const out = rawItems.map((raw, index) => normalize_one(raw, index, seen));
-  if (out.length === 0) {
-    throw new NoCriteria("criteria is empty");
-  }
-  return out;
-}
-
-/**
- * Return the subset of `criteria` in `ids` order.
- *
- * Omitted `ids` returns every criterion in the original order. An unknown id
- * throws — nothing is silently dropped.
- */
-export function select_criteria(criteria: readonly Criterion[], ids?: readonly string[]): readonly Criterion[] {
-  if (ids === undefined) return criteria;
-  const byId = new Map(criteria.map((item) => [item.id, item]));
-  const missing = ids.filter((id) => !byId.has(id));
-  if (missing.length > 0) {
-    throw new Error(`criteria not found: ${missing.join(", ")}`);
-  }
-  return ids.map((id) => byId.get(id)!);
-}
-
-export const VERIFICATION_SCALE: {
-  readonly min: 1;
-  readonly max: 20;
-  readonly anchors: string;
-  readonly schema: TSchema;
-} = {
-  min: 1,
-  max: 20,
-  anchors: "1 = certainly fails … 10 = borderline … 20 = verified correct",
-  schema: Type.Integer({ minimum: 1, maximum: 20 }),
-};
-
-export type Accept = { readonly kind: "accept"; readonly mean: number };
-export type Repair = { readonly kind: "repair"; readonly mean: number; readonly findings: readonly Finding[] };
-export type Indeterminate = { readonly kind: "indeterminate"; readonly missing: number };
-export type VerificationDecision = Accept | Repair | Indeterminate;
-
-export type VerificationRound = {
-  readonly scores: readonly CriterionScore[];
-  readonly invalidCount: number;
-  readonly expectedCount: number;
-};
-
-export type VerificationPolicy = {
-  readonly acceptMean: number;
-  readonly quorumFraction: number;
-};
-
-function mean_score(scores: readonly CriterionScore[]): number {
-  if (scores.length === 0) return 0;
-  let total = 0;
-  for (const item of scores) total += item.score;
-  return total / scores.length;
-}
-
-/**
- * Map one round of schema-valid scores to accept / repair / indeterminate.
- *
- * Accept iff quorum holds AND mean ≥ acceptMean AND no `severity: "veto"`
- * finding exists. Any veto forces Repair regardless of mean. Below quorum
- * (`valid < ceil(expected × quorumFraction)`) is Indeterminate.
- *
- * `invalidCount` is metadata only: there is no constructor that turns an
- * unparseable report into a CriterionScore, so a parse failure cannot shift
- * the mean.
- */
-export function decide_verification(round: VerificationRound, policy: VerificationPolicy): VerificationDecision {
-  // `invalidCount` is intentionally unread. It cannot enter the mean.
-  const { scores, expectedCount } = round;
-  const valid = scores.length;
-  const quorumNeeded = Math.ceil(expectedCount * policy.quorumFraction);
-  if (valid < quorumNeeded) {
-    return { kind: "indeterminate", missing: Math.max(0, expectedCount - valid) };
-  }
-
-  const mean = mean_score(scores);
-  const findings = scores.flatMap((item) => item.findings);
-  if (findings.some((item) => item.severity === "veto") || mean < policy.acceptMean) {
-    return { kind: "repair", mean, findings };
-  }
-  return { kind: "accept", mean };
-}

package/dist/builtin/workflows/builtin/verification-prompts.ts

@@ -1,206 +0,0 @@
-/**
- * Prefix-cache-aware verification prompts and warm-first fan-out.
- *
- * A scoring-family prompt is always `SHARED HEAD ‖ VARYING TAIL`: the head
- * contains the task statement, ground-truth note, candidate bodies (or read
- * references when the family is too large), and scale anchors, in that order.
- * The tail contains only the criterion name and description plus the output
- * format instruction. Keeping this ordering invariant means sibling criteria
- * can share the provider's cached prefix byte-for-byte.
- *
- * A pathless candidate is valid while the whole family is inline. If any body
- * exceeds the inline bound, every candidate must provide its caller-bound path;
- * this layer refuses the family rather than inventing a read path.
- */
-import type {
-	WorkflowParallelOptions,
-	WorkflowRunContext,
-	WorkflowTaskResult,
-	WorkflowTaskStep,
-} from "../src/shared/types.js";
-import type { Criterion } from "./verification-criteria.js";
-import { VERIFICATION_SCALE } from "./verification-criteria.js";
-
-/** Maximum UTF-8 byte length of one candidate body eligible for inlining. */
-export const MAX_INLINE_CANDIDATE_BYTES = 32 * 1024;
-
-export interface ScoringCandidate {
-	/**
-	 * Optional only while every candidate body is inline. An oversized family
-	 * requires a caller-bound path for every candidate; paths are preserved
-	 * verbatim and are never synthesized.
-	 */
-	readonly path?: string;
-	readonly body: string;
-}
-
-export interface SharedHead {
-	readonly task: string;
-	readonly groundTruthNote: string;
-	readonly candidates: readonly ScoringCandidate[];
-	readonly scaleAnchors?: string;
-	readonly outputFormat?: string;
-}
-
-export interface WarmFirstFanOutOptions extends WorkflowParallelOptions {
-	/** Maximum number of distinct prefixes warmed at once. */
-	readonly warmConcurrency?: number;
-}
-
-export type WarmFirstFanOutContext = Pick<WorkflowRunContext, "parallel">;
-
-const DEFAULT_WARM_CONCURRENCY = 4;
-const DEFAULT_OUTPUT_FORMAT =
-	"Call structured_output with criterion_id, score (1–20), evidence, and findings containing finding and severity.";
-
-function inlineCandidates(head: SharedHead): boolean {
-	return head.candidates.every(
-		(candidate) => Buffer.byteLength(candidate.body, "utf8") <= MAX_INLINE_CANDIDATE_BYTES,
-	);
-}
-
-function fallbackReadPaths(head: SharedHead): readonly string[] | undefined {
-	if (inlineCandidates(head)) return undefined;
-	const paths: string[] = [];
-	for (const [index, candidate] of head.candidates.entries()) {
-		if (candidate.path === undefined) {
-			throw new TypeError(
-				`Oversized candidate family requires a caller-bound path for every candidate; candidate ${index + 1} has no path.`,
-			);
-		}
-		paths.push(candidate.path);
-	}
-	return paths;
-}
-
-function candidateSection(head: SharedHead, readPaths: readonly string[] | undefined): string {
-	return head.candidates
-		.map((candidate, index) => {
-			if (readPaths === undefined) {
-				return `<candidate index="${index + 1}">\n${candidate.body}\n</candidate>`;
-			}
-			return `<candidate index="${index + 1}" source="read">\nRead candidate from ${readPaths[index]!}.\n</candidate>`;
-		})
-		.join("\n");
-}
-
-/**
- * Return candidate paths required by the read fallback.
- *
- * The whole family switches together: either every body is in the shared head,
- * or every candidate is named as a caller-provided read. Order and duplicate
- * paths are kept; an oversized pathless family throws instead of guessing.
- */
-export function scoring_prompt_reads(head: SharedHead): readonly string[] {
-	return fallbackReadPaths(head) ?? [];
-}
-
-/**
- * Build one scoring prompt with a byte-identical shared head and criterion tail.
- * Candidate bodies are measured in UTF-8 bytes, not JavaScript code units. If
- * any body exceeds `MAX_INLINE_CANDIDATE_BYTES`, no body is inlined for the
- * family, preserving the same head for every sibling criterion.
- */
-export function build_scoring_prompt(head: SharedHead, criterion: Criterion): string {
-	const readPaths = fallbackReadPaths(head);
-	const sharedHead = [
-		"<scoring_head>",
-		"<task_statement>",
-		head.task,
-		"</task_statement>",
-		"<ground_truth_note>",
-		head.groundTruthNote,
-		"</ground_truth_note>",
-		"<candidates>",
-		candidateSection(head, readPaths),
-		"</candidates>",
-		"<scale_anchors>",
-		head.scaleAnchors ?? VERIFICATION_SCALE.anchors,
-		"</scale_anchors>",
-	].join("\n");
-	const varyingTail = [
-		"<criterion>",
-		`<name>${criterion.name}</name>`,
-		`<description>${criterion.description}</description>`,
-		"</criterion>",
-		"<output_format>",
-		head.outputFormat ?? DEFAULT_OUTPUT_FORMAT,
-		"</output_format>",
-	].join("\n");
-	return `${sharedHead}\n\n${varyingTail}`;
-}
-
-function boundedConcurrency(
-	stepCount: number,
-	phaseCap: number | undefined,
-	inheritedCap: number | undefined,
-): number {
-	return Math.min(stepCount, phaseCap ?? stepCount, inheritedCap ?? stepCount);
-}
-
-/**
- * Run one warm step per prefix, then release the remaining steps. Prefix keys
- * are caller-owned; first-seen order is deterministic. Warm failures are
- * observed without fail-fast, and the second phase is always attempted before
- * the warm error is rethrown. Successful results are returned in input order.
- */
-export async function warm_first_fan_out<K>(
-	ctx: WarmFirstFanOutContext,
-	steps: readonly WorkflowTaskStep[],
-	prefixKeyOf: (step: WorkflowTaskStep, index: number) => K,
-	options: WarmFirstFanOutOptions = {},
-): Promise<WorkflowTaskResult[]> {
-	const warmIndices: number[] = [];
-	const restIndices: number[] = [];
-	const seen = new Set<K>();
-	for (const [index, step] of steps.entries()) {
-		const prefixKey = prefixKeyOf(step, index);
-		if (seen.has(prefixKey)) restIndices.push(index);
-		else {
-			seen.add(prefixKey);
-			warmIndices.push(index);
-		}
-	}
-
-	const { warmConcurrency, ...parallelOptions } = options;
-	const inheritedConcurrency = parallelOptions.concurrency;
-	const resultsByIndex = new Map<number, WorkflowTaskResult>();
-	let warmFailure: Error | undefined;
-	if (warmIndices.length > 0) {
-		const warmSteps = warmIndices.map((index) => steps[index]!);
-		try {
-			const warmResults = await ctx.parallel(warmSteps, {
-				...parallelOptions,
-				concurrency: boundedConcurrency(
-					warmSteps.length,
-					warmConcurrency ?? DEFAULT_WARM_CONCURRENCY,
-					inheritedConcurrency,
-				),
-				failFast: false,
-			});
-			for (const [resultIndex, result] of warmResults.entries()) {
-				const originalIndex = warmIndices[resultIndex];
-				if (originalIndex !== undefined) resultsByIndex.set(originalIndex, result);
-			}
-		} catch (error) {
-			warmFailure = error instanceof Error ? error : new Error(String(error));
-		}
-	}
-
-	if (restIndices.length > 0) {
-		const restSteps = restIndices.map((index) => steps[index]!);
-		const restResults = await ctx.parallel(restSteps, {
-			...parallelOptions,
-			concurrency: boundedConcurrency(restSteps.length, undefined, inheritedConcurrency),
-		});
-		for (const [resultIndex, result] of restResults.entries()) {
-			const originalIndex = restIndices[resultIndex];
-			if (originalIndex !== undefined) resultsByIndex.set(originalIndex, result);
-		}
-	}
-
-	if (warmFailure !== undefined) throw warmFailure;
-	return steps
-		.map((_, index) => resultsByIndex.get(index))
-		.filter((result): result is WorkflowTaskResult => result !== undefined);
-}

package/dist/builtin/workflows/builtin/verification-usage.ts

@@ -1,44 +0,0 @@
-import type { WorkflowTaskResult } from "../src/shared/types.js";
-
-export type UsageTotals = {
-	calls: number;
-	input: number;
-	output: number;
-	cacheRead: number;
-	cacheWrite: number;
-	cost: number;
-	turns: number;
-	cacheHitRate: number;
-};
-
-export function fold_usage(results: readonly WorkflowTaskResult[]): UsageTotals {
-	const totals = {
-		calls: 0,
-		input: 0,
-		output: 0,
-		cacheRead: 0,
-		cacheWrite: 0,
-		cost: 0,
-		turns: 0,
-	};
-
-	for (const result of results) {
-		for (const attempt of result.modelAttempts ?? []) {
-			totals.calls += 1;
-			const usage = attempt.usage;
-			if (usage === undefined) continue;
-			totals.input += usage.input ?? 0;
-			totals.output += usage.output ?? 0;
-			totals.cacheRead += usage.cacheRead ?? 0;
-			totals.cacheWrite += usage.cacheWrite ?? 0;
-			totals.cost += usage.cost ?? 0;
-			totals.turns += usage.turns ?? 0;
-		}
-	}
-
-	const cacheDenominator = totals.input + totals.cacheRead;
-	return {
-		...totals,
-		cacheHitRate: cacheDenominator === 0 ? 0 : totals.cacheRead / cacheDenominator,
-	};
-}