@prismatic-io/lux 0.0.1

package/src/orchestrator/suite-runs.ts

@@ -0,0 +1,202 @@
+import { existsSync } from "node:fs";
+import { mkdir, readdir, rename, writeFile } from "node:fs/promises";
+import { basename, join, relative, resolve } from "node:path";
+import { z } from "zod";
+import { safeReadJson } from "./fs-read.js";
+
+const SUITES_DIR = ".suites";
+const SUITE_FILE = "suite.json";
+
+const SuiteItemSchema = z.object({
+  caseId: z.string(),
+  canonicalCaseId: z.string(),
+  status: z.enum(["pending", "running", "completed"]),
+  runDir: z.string().nullable(),
+  exitReason: z.union([z.literal("done"), z.string()]).nullable(),
+  casePassed: z.boolean().nullable(),
+  errorMessage: z.exactOptional(z.string()),
+});
+
+export const SuiteRunSchema = z.object({
+  schemaVersion: z.literal(1),
+  suiteId: z.string(),
+  startedAt: z.string(),
+  endedAt: z.string().nullable(),
+  status: z.enum(["running", "passed", "failed", "aborted"]),
+  aborted: z.boolean(),
+  concurrency: z.number().int().positive(),
+  profileId: z.exactOptional(z.string()),
+  experimentId: z.exactOptional(z.string()),
+  experimentUnitId: z.exactOptional(z.string()),
+  experimentAttemptId: z.exactOptional(z.string()),
+  variantId: z.exactOptional(z.string()),
+  candidateId: z.exactOptional(z.string()),
+  subjectHash: z.exactOptional(z.string()),
+  errorMessage: z.exactOptional(z.string()),
+  fixtures: z
+    .array(
+      z.object({
+        name: z.string(),
+        pluginVersion: z.string(),
+      }),
+    )
+    .optional(),
+  items: z.array(SuiteItemSchema),
+});
+export type SuiteRun = z.infer<typeof SuiteRunSchema>;
+export type SuiteRunItem = z.infer<typeof SuiteItemSchema>;
+
+const timestampId = (date: Date): string => date.toISOString().replace(/[:.]/g, "-");
+
+const writeJsonAtomic = async (path: string, value: unknown): Promise<void> => {
+  const temporary = `${path}.${process.pid}.tmp`;
+  await writeFile(temporary, `${JSON.stringify(value, null, 2)}\n`);
+  await rename(temporary, path);
+};
+
+export type OpenedSuiteRun = {
+  path: string;
+  manifest: SuiteRun;
+  write: () => Promise<void>;
+  relativeRunDir: (runDir: string) => string;
+};
+
+export const openSuiteRun = async (options: {
+  runsRoot: string;
+  concurrency: number;
+  items: { id: string; caseId?: string | undefined }[];
+  startedAt?: Date | undefined;
+  profileId?: string | undefined;
+  experimentId?: string | undefined;
+  experimentUnitId?: string | undefined;
+  experimentAttemptId?: string | undefined;
+  variantId?: string | undefined;
+  candidateId?: string | undefined;
+  subjectHash?: string | undefined;
+  fixtures?: Array<{ name: string; pluginVersion: string }> | undefined;
+}): Promise<OpenedSuiteRun> => {
+  const startedAt = options.startedAt ?? new Date();
+  const suitesRoot = join(resolve(options.runsRoot), SUITES_DIR);
+  await mkdir(suitesRoot, { recursive: true });
+  const baseId = timestampId(startedAt);
+  let suiteId = baseId;
+  let path = join(suitesRoot, suiteId);
+  for (let n = 2; ; n++) {
+    try {
+      await mkdir(path);
+      break;
+    } catch (err) {
+      if ((err as NodeJS.ErrnoException).code !== "EEXIST") throw err;
+      suiteId = `${baseId}-${n}`;
+      path = join(suitesRoot, suiteId);
+    }
+  }
+  const manifest: SuiteRun = {
+    schemaVersion: 1,
+    suiteId,
+    startedAt: startedAt.toISOString(),
+    endedAt: null,
+    status: "running",
+    aborted: false,
+    concurrency: options.concurrency,
+    ...(options.profileId ? { profileId: options.profileId } : {}),
+    ...(options.experimentId ? { experimentId: options.experimentId } : {}),
+    ...(options.experimentUnitId ? { experimentUnitId: options.experimentUnitId } : {}),
+    ...(options.experimentAttemptId ? { experimentAttemptId: options.experimentAttemptId } : {}),
+    ...(options.variantId ? { variantId: options.variantId } : {}),
+    ...(options.candidateId ? { candidateId: options.candidateId } : {}),
+    ...(options.subjectHash ? { subjectHash: options.subjectHash } : {}),
+    ...(options.fixtures && options.fixtures.length > 0 ? { fixtures: options.fixtures } : {}),
+    items: options.items.map((item) => ({
+      caseId: item.id,
+      canonicalCaseId: item.caseId ?? item.id,
+      status: "pending",
+      runDir: null,
+      exitReason: null,
+      casePassed: null,
+    })),
+  };
+  const write = (): Promise<void> => writeJsonAtomic(join(path, SUITE_FILE), manifest);
+  await write();
+  return { path, manifest, write, relativeRunDir: (runDir) => relative(path, runDir) };
+};
+
+export const loadSuiteRun = async (path: string): Promise<SuiteRun | null> =>
+  safeReadJson(join(path, SUITE_FILE), SuiteRunSchema);
+
+export const listSuiteRuns = async (
+  runsRoot: string,
+): Promise<Array<SuiteRun & { suiteDir: string }>> => {
+  const root = join(runsRoot, SUITES_DIR);
+  if (!existsSync(root)) return [];
+  const dirs = (await readdir(root, { withFileTypes: true }))
+    .filter((entry) => entry.isDirectory())
+    .map((entry) => join(root, entry.name));
+  const manifests = await Promise.all(
+    dirs.map(async (suiteDir) => ({ suiteDir, suite: await loadSuiteRun(suiteDir) })),
+  );
+  return manifests
+    .flatMap(({ suiteDir, suite }) => (suite ? [{ ...suite, suiteDir }] : []))
+    .sort((a, b) => b.suiteId.localeCompare(a.suiteId));
+};
+
+const resolveSuiteRunDir = (suiteDir: string, runDir: string): string => resolve(suiteDir, runDir);
+
+export const formatSuiteRunsTable = (suites: Array<SuiteRun & { suiteDir: string }>): string => {
+  if (suites.length === 0) return "(no suite runs found)";
+  const rows = [
+    ["status", "suite", "cases", "passed", "failed", "ungraded", "pending"],
+    ...suites.map((suite) => [
+      suite.status.toUpperCase(),
+      basename(suite.suiteDir),
+      String(suite.items.length),
+      String(suite.items.filter((item) => item.casePassed === true).length),
+      String(
+        suite.items.filter(
+          (item) =>
+            item.status === "completed" &&
+            (item.exitReason !== "done" || item.casePassed === false),
+        ).length,
+      ),
+      String(
+        suite.items.filter(
+          (item) =>
+            item.status === "completed" && item.exitReason === "done" && item.casePassed === null,
+        ).length,
+      ),
+      String(suite.items.filter((item) => item.status !== "completed").length),
+    ]),
+  ];
+  const widths = (rows[0] ?? []).map((_, column) =>
+    Math.max(...rows.map((row) => row[column]?.length ?? 0)),
+  );
+  return rows
+    .map((row) => row.map((cell, column) => cell.padEnd(widths[column] ?? 0)).join("  "))
+    .join("\n");
+};
+
+export const formatSuiteRunDetail = (suiteDir: string, suite: SuiteRun): string => {
+  const lines = [
+    `suiteDir: ${suiteDir}`,
+    `suite:    ${suite.suiteId}`,
+    `status:   ${suite.status}`,
+    `started:  ${suite.startedAt}`,
+    `ended:    ${suite.endedAt ?? "—"}`,
+    `cases:    ${suite.items.length}`,
+    ...(suite.errorMessage ? [`error:    ${suite.errorMessage}`] : []),
+    "",
+  ];
+  for (const item of suite.items) {
+    const status = (() => {
+      if (item.status !== "completed") return item.status.toUpperCase();
+      if (item.exitReason !== "done" || item.casePassed === false) return "FAIL";
+      if (item.casePassed === true) return "PASS";
+      return "UNGRADED";
+    })();
+    lines.push(
+      `${status.padEnd(8)} ${item.caseId}${item.runDir ? `  ${resolveSuiteRunDir(suiteDir, item.runDir)}` : ""}`,
+    );
+    if (item.errorMessage) lines.push(`         ${item.errorMessage}`);
+  }
+  return lines.join("\n");
+};

package/src/orchestrator/suite-summary.ts

@@ -0,0 +1,276 @@
+import { rename, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { z } from "zod";
+import { harnessUsage, isUsageEmpty, type RunMetadata } from "../core/index.js";
+import { runComparisonIdentity } from "./comparison-identity.js";
+import { safeReadJson } from "./fs-read.js";
+import type { SuiteCaseResult } from "./orchestrator.js";
+import type { SuiteRun } from "./suite-runs.js";
+
+const SUMMARY_FILE = "suite-summary.json";
+
+export const MetricSummarySchema = z.object({
+  n: z.number().int().nonnegative(),
+  min: z.number(),
+  max: z.number(),
+  mean: z.number(),
+  median: z.number(),
+  p90: z.number(),
+});
+export type MetricSummary = z.infer<typeof MetricSummarySchema>;
+
+const OutcomeSummarySchema = z.object({
+  attempts: z.number().int().nonnegative(),
+  completed: z.number().int().nonnegative(),
+  passed: z.number().int().nonnegative(),
+  failed: z.number().int().nonnegative(),
+  ungraded: z.number().int().nonnegative(),
+  pending: z.number().int().nonnegative(),
+  passRate: z.number().min(0).max(1),
+});
+export type OutcomeSummary = z.infer<typeof OutcomeSummarySchema>;
+
+const CaseSummarySchema = OutcomeSummarySchema.extend({
+  canonicalCaseId: z.string(),
+  metrics: z.record(z.string(), MetricSummarySchema),
+});
+export type CaseSummary = z.infer<typeof CaseSummarySchema>;
+
+export const SuiteSampleSchema = z.object({
+  index: z.number().int().nonnegative(),
+  caseId: z.string(),
+  canonicalCaseId: z.string(),
+  exitReason: z.string().nullable(),
+  passed: z.boolean().nullable(),
+  identityHash: z.string().nullable(),
+  score: z.exactOptional(z.number()),
+  metrics: z.record(z.string(), z.number()),
+});
+export type SuiteSample = z.infer<typeof SuiteSampleSchema>;
+
+export const SuiteSummarySchema = z.object({
+  schemaVersion: z.literal(1),
+  suiteId: z.string(),
+  generatedAt: z.string(),
+  totals: OutcomeSummarySchema,
+  metrics: z.record(z.string(), MetricSummarySchema),
+  cases: z.array(CaseSummarySchema),
+  samples: z.array(SuiteSampleSchema),
+});
+export type SuiteSummary = z.infer<typeof SuiteSummarySchema>;
+
+type MetricSamples = Map<string, number[]>;
+
+const add = (samples: MetricSamples, name: string, value: number | undefined): void => {
+  if (value === undefined || !Number.isFinite(value)) return;
+  const values = samples.get(name) ?? [];
+  values.push(value);
+  samples.set(name, values);
+};
+
+const cacheHitRate = (usage: {
+  input: number;
+  cacheRead: number;
+  cacheCreation: number;
+}): number | undefined => {
+  const promptTokens = usage.input + usage.cacheRead + usage.cacheCreation;
+  return promptTokens > 0 ? usage.cacheRead / promptTokens : undefined;
+};
+
+const collectMetadata = (samples: MetricSamples, metadata: RunMetadata): void => {
+  const started = Date.parse(metadata.startedAt);
+  const ended = Date.parse(metadata.endedAt);
+  if (!Number.isNaN(started) && !Number.isNaN(ended)) {
+    add(samples, "durationMs", Math.max(0, ended - started));
+  }
+
+  if (!isUsageEmpty(metadata.usage.agent)) {
+    add(samples, "agent.input", metadata.usage.agent.input);
+    add(samples, "agent.output", metadata.usage.agent.output);
+    add(samples, "agent.cacheRead", metadata.usage.agent.cacheRead);
+    add(samples, "agent.cacheCreation", metadata.usage.agent.cacheCreation);
+    add(samples, "agent.cacheHitRate", cacheHitRate(metadata.usage.agent));
+    add(
+      samples,
+      "agent.total",
+      metadata.usage.agent.input +
+        metadata.usage.agent.output +
+        metadata.usage.agent.cacheRead +
+        metadata.usage.agent.cacheCreation,
+    );
+  }
+  const harness = harnessUsage(metadata.usage);
+  if (!isUsageEmpty(harness)) {
+    add(samples, "harness.input", harness.input);
+    add(samples, "harness.output", harness.output);
+    add(samples, "harness.cacheRead", harness.cacheRead);
+    add(samples, "harness.cacheCreation", harness.cacheCreation);
+    add(samples, "harness.cacheHitRate", cacheHitRate(harness));
+    add(
+      samples,
+      "harness.total",
+      harness.input + harness.output + harness.cacheRead + harness.cacheCreation,
+    );
+  }
+  add(samples, "interrupts.ask", metadata.interruptCounts.ask);
+  add(samples, "interrupts.approve", metadata.interruptCounts.approve);
+  add(samples, "interrupts.custom", metadata.interruptCounts.custom);
+  add(samples, "toolCalls", metadata.toolOutcomes?.total ?? metadata.toolCallCount);
+  add(
+    samples,
+    "toolCalls.reported",
+    metadata.toolOutcomes?.reportedCalls ?? metadata.toolCallCount,
+  );
+  add(samples, "toolCalls.succeeded", metadata.toolOutcomes?.succeeded);
+  add(samples, "toolCalls.failed", metadata.toolOutcomes?.failed);
+  add(samples, "toolCalls.unresolved", metadata.toolOutcomes?.unresolved);
+  add(samples, "toolCalls.unmatchedResults", metadata.toolOutcomes?.unmatchedResults);
+  add(samples, "toolCalls.malformedEvents", metadata.toolOutcomes?.malformedEvents);
+  const settledToolCalls =
+    (metadata.toolOutcomes?.succeeded ?? 0) + (metadata.toolOutcomes?.failed ?? 0);
+  if (settledToolCalls > 0 && metadata.toolOutcomes) {
+    add(samples, "toolCalls.failureRate", metadata.toolOutcomes.failed / settledToolCalls);
+  }
+  add(samples, "artifacts.count", metadata.artifacts?.count);
+  add(samples, "artifacts.bytes", metadata.artifacts?.bytes);
+  add(samples, "cost.usd", metadata.cost?.usd);
+  add(samples, "phase.agentMs", metadata.phaseDurationMs?.agent);
+  add(samples, "phase.collectMs", metadata.phaseDurationMs?.collect);
+  add(samples, "phase.gradingMs", metadata.phaseDurationMs?.grading);
+};
+
+const percentile = (sorted: readonly number[], fraction: number): number => {
+  const index = Math.max(0, Math.ceil(sorted.length * fraction) - 1);
+  return sorted[index] ?? 0;
+};
+
+export const summarizeNumbers = (values: readonly number[]): MetricSummary => {
+  const sorted = [...values].sort((a, b) => a - b);
+  const middle = Math.floor(sorted.length / 2);
+  const median =
+    sorted.length % 2 === 0
+      ? ((sorted[middle - 1] ?? 0) + (sorted[middle] ?? 0)) / 2
+      : (sorted[middle] ?? 0);
+  return {
+    n: sorted.length,
+    min: sorted[0] ?? 0,
+    max: sorted.at(-1) ?? 0,
+    mean: sorted.reduce((sum, value) => sum + value, 0) / Math.max(sorted.length, 1),
+    median,
+    p90: percentile(sorted, 0.9),
+  };
+};
+
+const summarizeMetrics = (samples: MetricSamples): Record<string, MetricSummary> =>
+  Object.fromEntries(
+    [...samples.entries()]
+      .sort(([a], [b]) => a.localeCompare(b))
+      .map(([name, values]) => [name, summarizeNumbers(values)]),
+  );
+
+const summarizeOutcomes = (
+  entries: Array<{
+    item: SuiteRun["items"][number];
+    result: SuiteCaseResult | undefined;
+  }>,
+): OutcomeSummary => {
+  const completed = entries.filter((entry) => entry.item.status === "completed").length;
+  const passed = entries.filter((entry) => entry.result?.casePassed === true).length;
+  const failed = entries.filter(
+    (entry) =>
+      entry.item.status === "completed" &&
+      (entry.result?.exitReason !== "done" || entry.result.casePassed === false),
+  ).length;
+  const ungraded = entries.filter(
+    (entry) =>
+      entry.item.status === "completed" &&
+      entry.result?.exitReason === "done" &&
+      entry.result.casePassed === null,
+  ).length;
+  return {
+    attempts: entries.length,
+    completed,
+    passed,
+    failed,
+    ungraded,
+    pending: entries.length - completed,
+    passRate: entries.length === 0 ? 0 : passed / entries.length,
+  };
+};
+
+export const buildSuiteSummary = (
+  suite: SuiteRun,
+  results: readonly (SuiteCaseResult | undefined)[],
+  generatedAt: Date = new Date(),
+): SuiteSummary => {
+  const entries = suite.items.map((item, index) => ({ item, result: results[index] }));
+  const allMetrics: MetricSamples = new Map();
+  for (const entry of entries) {
+    const result = entry.result?.result;
+    if (!result) continue;
+    add(allMetrics, "score", result.grading.caseScore ?? undefined);
+    collectMetadata(allMetrics, result.metadata);
+  }
+
+  const byCase = new Map<string, typeof entries>();
+  for (const entry of entries) {
+    const group = byCase.get(entry.item.canonicalCaseId) ?? [];
+    group.push(entry);
+    byCase.set(entry.item.canonicalCaseId, group);
+  }
+
+  const cases = [...byCase.entries()]
+    .sort(([a], [b]) => a.localeCompare(b))
+    .map(([canonicalCaseId, caseEntries]) => {
+      const metrics: MetricSamples = new Map();
+      for (const entry of caseEntries) {
+        const result = entry.result?.result;
+        if (!result) continue;
+        add(metrics, "score", result.grading.caseScore ?? undefined);
+        collectMetadata(metrics, result.metadata);
+      }
+      return {
+        canonicalCaseId,
+        ...summarizeOutcomes(caseEntries),
+        metrics: summarizeMetrics(metrics),
+      };
+    });
+
+  return {
+    schemaVersion: 1,
+    suiteId: suite.suiteId,
+    generatedAt: generatedAt.toISOString(),
+    totals: summarizeOutcomes(entries),
+    metrics: summarizeMetrics(allMetrics),
+    cases,
+    samples: entries.map((entry, index) => {
+      const samples: MetricSamples = new Map();
+      const result = entry.result?.result;
+      if (result) collectMetadata(samples, result.metadata);
+      return {
+        index,
+        caseId: entry.item.caseId,
+        canonicalCaseId: entry.item.canonicalCaseId,
+        exitReason: entry.result?.exitReason ?? null,
+        passed: entry.result?.casePassed ?? null,
+        identityHash: result ? runComparisonIdentity(result.metadata) : null,
+        ...(result?.grading.caseScore !== null && result?.grading.caseScore !== undefined
+          ? { score: result.grading.caseScore }
+          : {}),
+        metrics: Object.fromEntries(
+          [...samples.entries()].map(([name, values]) => [name, values[0] ?? 0]),
+        ),
+      };
+    }),
+  };
+};
+
+export const writeSuiteSummary = async (suiteDir: string, summary: SuiteSummary): Promise<void> => {
+  const path = join(suiteDir, SUMMARY_FILE);
+  const temporary = `${path}.${process.pid}.tmp`;
+  await writeFile(temporary, `${JSON.stringify(summary, null, 2)}\n`);
+  await rename(temporary, path);
+};
+
+export const loadSuiteSummary = async (suiteDir: string): Promise<SuiteSummary | null> =>
+  safeReadJson(join(suiteDir, SUMMARY_FILE), SuiteSummarySchema);

package/src/orchestrator/trace-limits.ts

@@ -0,0 +1,65 @@
+import type { RunEvent } from "../core/index.js";
+
+export const MAX_RUN_TRACE_BYTES = 10 * 1024 * 1024;
+export const MAX_RUN_TRACE_EVENTS = 50_000;
+
+export type RunTraceLimits = {
+  maxBytes: number;
+  maxEvents: number;
+};
+
+export type RunTraceLimitFailure = {
+  code: "run-trace-limit";
+  limit: "bytes" | "events";
+  maxBytes: number;
+  maxEvents: number;
+  reason: string;
+};
+
+export const isRunTraceLimit = (value: unknown): value is Pick<RunTraceLimitFailure, "code"> =>
+  typeof value === "object" &&
+  value !== null &&
+  "code" in value &&
+  value.code === "run-trace-limit";
+
+export const DEFAULT_RUN_TRACE_LIMITS: RunTraceLimits = {
+  maxBytes: MAX_RUN_TRACE_BYTES,
+  maxEvents: MAX_RUN_TRACE_EVENTS,
+};
+
+export const runTraceLimitFailure = (
+  limit: RunTraceLimitFailure["limit"],
+  limits: RunTraceLimits,
+): RunTraceLimitFailure => ({
+  code: "run-trace-limit",
+  limit,
+  maxBytes: limits.maxBytes,
+  maxEvents: limits.maxEvents,
+  reason:
+    limit === "events"
+      ? `run trace exceeded the ${limits.maxEvents}-event limit`
+      : `run trace exceeded the ${limits.maxBytes}-byte limit`,
+});
+
+export const runTraceLimitDiagnostic = (
+  failure: RunTraceLimitFailure,
+  ts: number = Date.now(),
+): RunEvent => ({
+  type: "error",
+  exitReason: "error",
+  reason: failure.reason,
+  ts,
+});
+
+const serializedEventBytes = (event: RunEvent): number =>
+  Buffer.byteLength(`${JSON.stringify(event)}\n`);
+
+export const terminalDiagnosticReserveBytes = (limits: RunTraceLimits): number =>
+  Math.max(
+    serializedEventBytes(
+      runTraceLimitDiagnostic(runTraceLimitFailure("bytes", limits), Number.MAX_SAFE_INTEGER),
+    ),
+    serializedEventBytes(
+      runTraceLimitDiagnostic(runTraceLimitFailure("events", limits), Number.MAX_SAFE_INTEGER),
+    ),
+  );

package/src/orchestrator/validate-plugin-config.ts

@@ -0,0 +1,34 @@
+import type { z } from "zod";
+import { formatZodIssues } from "../core/index.js";
+
+/**
+ * Centralized plugin/assertion config validation. Both the orchestrator
+ * (driver/answerer config) and the grader (assertion config) need to
+ * validate untrusted config blobs against a plugin-supplied Zod schema
+ * with consistent error formatting; this is the one place that does it.
+ */
+export const validatePluginConfig = (
+  schema: z.ZodTypeAny | undefined,
+  config: unknown,
+  contextLabel: string,
+): unknown => {
+  if (!schema) return config;
+  const parsed = schema.safeParse(config ?? {});
+  if (!parsed.success) {
+    throw new Error(`invalid config for ${contextLabel}: ${formatZodIssues(parsed.error)}`);
+  }
+  return parsed.data;
+};
+
+export const validatePluginConfigAsync = async (
+  schema: z.ZodTypeAny | undefined,
+  config: unknown,
+  contextLabel: string,
+): Promise<unknown> => {
+  if (!schema) return config;
+  const parsed = await schema.safeParseAsync(config ?? {});
+  if (!parsed.success) {
+    throw new Error(`invalid config for ${contextLabel}: ${formatZodIssues(parsed.error)}`);
+  }
+  return parsed.data;
+};