@hassangameryt/freeflow 0.2.0 → 0.4.0

package/skills/evaluate-skill/scripts/lib/definitions.mjs

@@ -0,0 +1,296 @@
+import { readFile, realpath } from "node:fs/promises";
+import path from "node:path";
+
+const ID = /^[a-z][a-z0-9]*(?:-[a-z0-9]+)*$/;
+const EVALUATION_TYPES = new Set(["description", "body", "end-to-end"]);
+const VARIANTS = ["baseline", "candidate"];
+
+export class DefinitionError extends Error {
+  constructor(code, message, definitionPath = null) {
+    super(message);
+    this.name = "DefinitionError";
+    this.code = code;
+    this.definitionPath = definitionPath;
+  }
+}
+
+function fail(code, message, definitionPath) {
+  throw new DefinitionError(code, message, definitionPath);
+}
+
+function isPlainObject(value) {
+  return value !== null && typeof value === "object" && !Array.isArray(value);
+}
+
+function requireObject(value, label, definitionPath) {
+  if (!isPlainObject(value)) fail("invalid-definition", `${label} must be an object`, definitionPath);
+  return value;
+}
+
+function requireExactKeys(value, required, optional, label, definitionPath) {
+  const object = requireObject(value, label, definitionPath);
+  const allowed = new Set([...required, ...optional]);
+  for (const key of required) {
+    if (!Object.hasOwn(object, key)) fail("invalid-definition", `${label} is missing ${key}`, definitionPath);
+  }
+  for (const key of Object.keys(object)) {
+    if (!allowed.has(key)) fail("invalid-definition", `${label} contains unsupported field ${key}`, definitionPath);
+  }
+  return object;
+}
+
+function requireId(value, label, definitionPath) {
+  if (typeof value !== "string" || value.length > 64 || !ID.test(value)) {
+    fail(
+      "invalid-definition",
+      `${label} must start with a lowercase letter and use lowercase letters, numbers, or single hyphens up to 64 characters`,
+      definitionPath,
+    );
+  }
+  return value;
+}
+
+function requireString(value, label, definitionPath) {
+  if (typeof value !== "string" || value.trim() === "") {
+    fail("invalid-definition", `${label} must be a non-empty string`, definitionPath);
+  }
+  return value;
+}
+
+function requireStringArray(value, label, definitionPath, { paths = false, unique = false } = {}) {
+  if (!Array.isArray(value)) fail("invalid-definition", `${label} must be an array`, definitionPath);
+  const seen = new Set();
+  return value.map((entry, index) => {
+    const string = requireString(entry, `${label}[${index}]`, definitionPath);
+    if (paths) requireSafeRelativePath(string, `${label}[${index}]`, definitionPath);
+    if (unique && seen.has(string)) {
+      fail("invalid-definition", `${label} contains duplicate ${string}`, definitionPath);
+    }
+    seen.add(string);
+    return string;
+  });
+}
+
+function requireSafeRelativePath(value, label, definitionPath) {
+  const string = requireString(value, label, definitionPath);
+  const normalized = path.normalize(string);
+  if (path.isAbsolute(string) || normalized === "." || normalized === ".." || normalized.startsWith(`..${path.sep}`)) {
+    fail("unsafe-path", `${label} must stay inside the definition root`, definitionPath);
+  }
+  return string;
+}
+
+function requireSuiteReference(value, label, definitionPath) {
+  const string = requireString(value, label, definitionPath);
+  if (path.isAbsolute(string) || path.normalize(string) === ".") {
+    fail("unsafe-path", `${label} must be relative to the suite file`, definitionPath);
+  }
+  return string;
+}
+
+function validateInput(value, definitionPath) {
+  const input = requireExactKeys(value, [], ["prompt", "turns"], "group.input", definitionPath);
+  const hasPrompt = Object.hasOwn(input, "prompt");
+  const hasTurns = Object.hasOwn(input, "turns");
+  if (hasPrompt === hasTurns) {
+    fail("invalid-definition", "group.input must contain exactly one of prompt or turns", definitionPath);
+  }
+  if (hasPrompt) requireString(input.prompt, "group.input.prompt", definitionPath);
+  if (hasTurns) {
+    const turns = requireStringArray(input.turns, "group.input.turns", definitionPath);
+    if (turns.length === 0) fail("invalid-definition", "group.input.turns must not be empty", definitionPath);
+  }
+}
+
+function validateSource(value, label, definitionPath) {
+  const source = requireObject(value, label, definitionPath);
+  if (source.kind === "working-tree") {
+    requireExactKeys(source, ["kind"], [], label, definitionPath);
+    return;
+  }
+  if (source.kind === "git") {
+    requireExactKeys(source, ["kind", "ref"], [], label, definitionPath);
+    requireString(source.ref, `${label}.ref`, definitionPath);
+    return;
+  }
+  fail("invalid-definition", `${label}.kind must be working-tree or git`, definitionPath);
+}
+
+function validateEnvironment(value, label, definitionPath) {
+  const environment = requireExactKeys(value, ["source", "skills", "target", "context"], [], label, definitionPath);
+  validateSource(environment.source, `${label}.source`, definitionPath);
+  const skills = requireStringArray(environment.skills, `${label}.skills`, definitionPath, { paths: true });
+  requireStringArray(environment.context, `${label}.context`, definitionPath, { paths: true });
+  if (
+    environment.target !== null &&
+    (!Number.isInteger(environment.target) || environment.target < 0 || environment.target >= skills.length)
+  ) {
+    fail("invalid-definition", `${label}.target must be null or an index into skills`, definitionPath);
+  }
+}
+
+function validateExpectations(value, definitionPath) {
+  if (!Array.isArray(value)) fail("invalid-definition", "group.expectations must be an array", definitionPath);
+  const ids = new Set();
+  for (const [index, entry] of value.entries()) {
+    const expectation = requireObject(entry, `group.expectations[${index}]`, definitionPath);
+    requireId(expectation.id, `group.expectations[${index}].id`, definitionPath);
+    requireString(expectation.kind, `group.expectations[${index}].kind`, definitionPath);
+    if (ids.has(expectation.id)) {
+      fail("invalid-definition", `group.expectations contains duplicate ID ${expectation.id}`, definitionPath);
+    }
+    ids.add(expectation.id);
+  }
+}
+
+function validateModel(value, definitionPath) {
+  const model = requireExactKeys(value, ["model"], ["thinking"], "group.model", definitionPath);
+  requireString(model.model, "group.model.model", definitionPath);
+  if (Object.hasOwn(model, "thinking")) requireString(model.thinking, "group.model.thinking", definitionPath);
+}
+
+function validateGroup(value, definitionPath) {
+  const group = requireExactKeys(
+    value,
+    [
+      "schema_version",
+      "kind",
+      "id",
+      "type",
+      "input",
+      "fixture",
+      "tools",
+      "variants",
+      "expectations",
+      "review_questions",
+    ],
+    ["model"],
+    "group",
+    definitionPath,
+  );
+  if (group.schema_version !== 1) fail("unsupported-schema", "group.schema_version must be 1", definitionPath);
+  if (group.kind !== "group") fail("invalid-definition", "group.kind must be group", definitionPath);
+  requireId(group.id, "group.id", definitionPath);
+  if (!EVALUATION_TYPES.has(group.type)) {
+    fail("invalid-definition", "group.type must be description, body, or end-to-end", definitionPath);
+  }
+  validateInput(group.input, definitionPath);
+  if (group.fixture !== null) requireSafeRelativePath(group.fixture, "group.fixture", definitionPath);
+  requireStringArray(group.tools, "group.tools", definitionPath, { unique: true });
+  const variants = requireExactKeys(group.variants, VARIANTS, [], "group.variants", definitionPath);
+  for (const variant of VARIANTS) validateEnvironment(variants[variant], `group.variants.${variant}`, definitionPath);
+  if (variants.candidate.target === null) {
+    fail(
+      "invalid-definition",
+      "group.variants.candidate.target must identify the candidate target skill",
+      definitionPath,
+    );
+  }
+  validateExpectations(group.expectations, definitionPath);
+  requireStringArray(group.review_questions, "group.review_questions", definitionPath);
+  if (Object.hasOwn(group, "model")) validateModel(group.model, definitionPath);
+  return group;
+}
+
+function validateSuite(value, definitionPath) {
+  const suite = requireExactKeys(value, ["schema_version", "kind", "id", "groups"], [], "suite", definitionPath);
+  if (suite.schema_version !== 1) fail("unsupported-schema", "suite.schema_version must be 1", definitionPath);
+  if (suite.kind !== "suite") fail("invalid-definition", "suite.kind must be suite", definitionPath);
+  requireId(suite.id, "suite.id", definitionPath);
+  const groups = requireStringArray(suite.groups, "suite.groups", definitionPath, { unique: true });
+  for (const [index, reference] of groups.entries()) {
+    requireSuiteReference(reference, `suite.groups[${index}]`, definitionPath);
+  }
+  if (groups.length === 0) fail("invalid-definition", "suite.groups must not be empty", definitionPath);
+  return suite;
+}
+
+function isContained(root, target) {
+  const relative = path.relative(root, target);
+  return relative === "" || (!relative.startsWith(`..${path.sep}`) && relative !== ".." && !path.isAbsolute(relative));
+}
+
+async function readJsonFile(file, root) {
+  const lexicalFile = path.resolve(file);
+  if (!isContained(root, lexicalFile)) fail("unsafe-path", `Definition escapes root: ${file}`, lexicalFile);
+
+  let canonicalRoot;
+  let canonicalFile;
+  try {
+    [canonicalRoot, canonicalFile] = await Promise.all([realpath(root), realpath(lexicalFile)]);
+  } catch {
+    fail("unreadable-definition", `Cannot resolve definition: ${lexicalFile}`, lexicalFile);
+  }
+  if (!isContained(canonicalRoot, canonicalFile)) {
+    fail("unsafe-path", `Definition symlink escapes root: ${lexicalFile}`, lexicalFile);
+  }
+
+  let contents;
+  try {
+    contents = await readFile(canonicalFile, "utf8");
+  } catch {
+    fail("unreadable-definition", `Cannot read definition: ${lexicalFile}`, lexicalFile);
+  }
+  try {
+    return { value: JSON.parse(contents), file: lexicalFile };
+  } catch {
+    fail("invalid-json", `Definition is not valid JSON: ${lexicalFile}`, lexicalFile);
+  }
+}
+
+export async function loadDefinition(file, { root = process.cwd() } = {}) {
+  const definitionRoot = path.resolve(root);
+  const loaded = await readJsonFile(file, definitionRoot);
+  const object = requireObject(loaded.value, "definition", loaded.file);
+  if (object.kind === "group") return validateGroup(object, loaded.file);
+  if (object.kind !== "suite") fail("invalid-definition", "definition.kind must be group or suite", loaded.file);
+
+  const suite = validateSuite(object, loaded.file);
+  const groups = [];
+  const ids = new Set();
+  for (const reference of suite.groups) {
+    const groupFile = path.resolve(path.dirname(loaded.file), reference);
+    const groupLoaded = await readJsonFile(groupFile, definitionRoot);
+    const group = validateGroup(groupLoaded.value, groupLoaded.file);
+    if (ids.has(group.id)) fail("invalid-definition", `suite contains duplicate group ID ${group.id}`, loaded.file);
+    ids.add(group.id);
+    groups.push(group);
+  }
+  return { ...suite, groups };
+}
+
+function variantSelection(selected) {
+  return {
+    baseline: selected === null || selected === "baseline" ? "selected" : "not-selected",
+    candidate: selected === null || selected === "candidate" ? "selected" : "not-selected",
+  };
+}
+
+export function selectDefinition(definition, { group = null, variant = null } = {}) {
+  if (variant !== null && !VARIANTS.includes(variant)) {
+    fail("invalid-selector", "variant must be baseline or candidate", null);
+  }
+  if (definition.kind === "group" && group !== null) {
+    fail("invalid-selector", "--group cannot be used with a direct group definition", null);
+  }
+
+  const available = definition.kind === "group" ? [definition] : definition.groups;
+  let selected = available;
+  if (group !== null) {
+    const position = typeof group === "number" || /^\d+$/.test(group) ? Number(group) : null;
+    selected =
+      position === null ? available.filter((entry) => entry.id === group) : [available[position - 1]].filter(Boolean);
+    if (selected.length === 0) fail("unknown-group", `Unknown group selector: ${group}`, null);
+  }
+
+  return {
+    kind: "selection",
+    definitionKind: definition.kind,
+    id: definition.id,
+    groups: selected.map((entry) => ({
+      position: available.indexOf(entry) + 1,
+      group: entry,
+      variants: variantSelection(variant),
+    })),
+  };
+}

package/skills/evaluate-skill/scripts/lib/evidence.mjs

@@ -0,0 +1,46 @@
+import { createHash, randomBytes } from "node:crypto";
+import { mkdir, readFile, rename, rm, writeFile } from "node:fs/promises";
+import path from "node:path";
+
+export function createInvocationId(now = new Date()) {
+  const timestamp = now
+    .toISOString()
+    .replaceAll(/[-:.TZ]/g, "")
+    .slice(0, 14);
+  return `${timestamp}-${randomBytes(4).toString("hex")}`;
+}
+
+export async function writeJson(file, value) {
+  const contents = `${JSON.stringify(value, null, 2)}\n`;
+  await writeAtomic(file, contents);
+  return { contents, sha256: sha256(contents) };
+}
+
+export async function writeText(file, contents) {
+  await writeAtomic(file, contents);
+  return { contents, sha256: sha256(contents) };
+}
+
+export async function fileIdentity(file) {
+  const contents = await readFile(file);
+  return {
+    path: file,
+    sha256: createHash("sha256").update(contents).digest("hex"),
+  };
+}
+
+export function sha256(value) {
+  return createHash("sha256").update(value).digest("hex");
+}
+
+async function writeAtomic(file, contents) {
+  await mkdir(path.dirname(file), { recursive: true });
+  const temporary = `${file}.${process.pid}.${randomBytes(4).toString("hex")}.tmp`;
+  try {
+    await writeFile(temporary, contents);
+    await rename(temporary, file);
+  } catch (error) {
+    await rm(temporary, { force: true }).catch(() => {});
+    throw error;
+  }
+}

package/skills/evaluate-skill/scripts/lib/grade-comparison.mjs

@@ -0,0 +1,60 @@
+import { isDeepStrictEqual } from "node:util";
+
+import { hasOwn } from "./grade-support.mjs";
+
+export function gradeComparisons(expectations, checks) {
+  /** @type {Map<string, any[]>} */
+  const grouped = new Map();
+  const errors = /** @type {any[]} */ ([]);
+  for (const expectation of expectations) {
+    if (!hasOwn(expectation, "comparison")) continue;
+    if (typeof expectation.comparison !== "string" || !/^[a-z][a-z0-9]*(?:-[a-z0-9]+)*$/.test(expectation.comparison)) {
+      errors.push({ id: expectation.id, reason: "invalid comparison identity" });
+      continue;
+    }
+    const entries = grouped.get(expectation.comparison) ?? [];
+    entries.push(expectation);
+    grouped.set(expectation.comparison, entries);
+  }
+
+  /** @type {Map<string, any>} */
+  const byCheck = new Map(checks.map((check) => [check.id, check]));
+  const comparisons = /** @type {any[]} */ ([]);
+  for (const [id, entries] of grouped) {
+    const baselineExpectation = entries.find((entry) => entry.variant === "baseline");
+    const candidateExpectation = entries.find((entry) => entry.variant === "candidate");
+    if (entries.length !== 2 || !baselineExpectation || !candidateExpectation) {
+      errors.push({ id, reason: "comparison requires one baseline and one candidate expectation" });
+      continue;
+    }
+    const baseline = byCheck.get(baselineExpectation.id);
+    const candidate = byCheck.get(candidateExpectation.id);
+    if (!sameCheck(baseline, candidate)) {
+      errors.push({ id, reason: "comparison expectations must describe the same deterministic check" });
+      continue;
+    }
+    comparisons.push({
+      id,
+      kind: baseline.kind,
+      baseline: { check: baseline.id, state: baseline.state },
+      candidate: { check: candidate.id, state: candidate.state },
+      transition: transition(baseline.state, candidate.state),
+    });
+  }
+  return { comparisons, errors };
+}
+
+function sameCheck(baseline, candidate) {
+  return (
+    baseline !== undefined &&
+    candidate !== undefined &&
+    baseline.kind === candidate.kind &&
+    isDeepStrictEqual(baseline.expected, candidate.expected)
+  );
+}
+
+function transition(baseline, candidate) {
+  return [baseline, candidate].every((state) => state === "pass" || state === "fail")
+    ? `${baseline}-to-${candidate}`
+    : "unavailable";
+}

package/skills/evaluate-skill/scripts/lib/grade-interaction.mjs

@@ -0,0 +1,148 @@
+import { realpath } from "node:fs/promises";
+import path from "node:path";
+
+import {
+  completedCheck,
+  hasOwn,
+  isContained,
+  isSafeRelativePath,
+  readPaths,
+  RESPONSE_EXPECTATIONS,
+  textMatches,
+  unavailableCheck,
+  validBodyExpectation,
+  validTurn,
+  VARIANTS,
+} from "./grade-support.mjs";
+
+const ACTIVATION_EXPECTATIONS = new Set(["never", "on-turn", "by-turn", "not-before-turn"]);
+
+export async function gradeSkillRead(group, runs, expectation) {
+  const turn = expectation.turn ?? null;
+  const timingExpectation = expectation.expect !== "never";
+  if (
+    group.type !== "description" ||
+    !VARIANTS.includes(expectation.variant) ||
+    !ACTIVATION_EXPECTATIONS.has(expectation.expect) ||
+    (timingExpectation ? turn === null || !validTurn(group, turn) : hasOwn(expectation, "turn"))
+  ) {
+    return { error: "invalid skill-read expectation" };
+  }
+
+  const expected = { read: expectation.expect, turn };
+  const run = runs[expectation.variant];
+  if (!run || run.state !== "complete") return unavailableCheck(expectation, expected);
+
+  const observed = {
+    targetRead: run.activation.targetRead,
+    firstReadTurn: run.activation.firstReadTurn,
+    readTurns: run.activation.readTurns,
+  };
+  return completedCheck(expectation, expected, observed, activationMatches(expectation, observed));
+}
+
+export async function gradeResourceRead(group, runs, expectation) {
+  const turn = expectation.turn ?? null;
+  const indexed = expectation.resource === "skill" || expectation.resource === "context";
+  if (
+    !VARIANTS.includes(expectation.variant) ||
+    !["workspace", "skill", "context"].includes(expectation.resource) ||
+    !["read", "not-read"].includes(expectation.expect) ||
+    typeof expectation.path !== "string" ||
+    !isSafeRelativePath(expectation.path) ||
+    !validTurn(group, turn) ||
+    (indexed && (!Number.isInteger(expectation.index) || expectation.index < 0)) ||
+    (!indexed && hasOwn(expectation, "index"))
+  ) {
+    return { error: "invalid resource-read expectation" };
+  }
+  const expected = {
+    expect: expectation.expect,
+    resource: expectation.resource,
+    index: indexed ? expectation.index : null,
+    path: expectation.path,
+    turn,
+  };
+  const run = runs[expectation.variant];
+  if (!run || run.state !== "complete") return unavailableCheck(expectation, expected);
+  const reads = readPaths(run, turn);
+  if (reads === null) return unavailableCheck(expectation, expected);
+
+  const resources = /** @type {any} */ (run.resources);
+  let resource = null;
+  if (expectation.resource === "skill") resource = resources?.skills?.[expectation.index];
+  if (expectation.resource === "context") resource = resources?.context?.[expectation.index];
+  const root = expectation.resource === "workspace" ? run.workspace : resource?.path;
+  if (typeof root !== "string") return { error: "declared resource is unavailable" };
+  if (resource !== null && !resource.files?.some((file) => file.path === expectation.path)) {
+    return { error: "declared resource path is unavailable" };
+  }
+
+  const canonicalRoot = await realpath(root);
+  const declaredPath = path.resolve(canonicalRoot, expectation.path);
+  const canonicalPath = await realpath(declaredPath).catch(() => declaredPath);
+  if (!isContained(canonicalRoot, canonicalPath)) return { error: "resource-read path escapes its declared root" };
+  if (
+    expectation.resource === "workspace" &&
+    !run.effects?.before?.some((file) => file.path === expectation.path) &&
+    !run.effects?.after?.some((file) => file.path === expectation.path) &&
+    !reads.includes(canonicalPath)
+  ) {
+    return { error: "declared workspace resource path is unavailable" };
+  }
+
+  const observed = { path: canonicalPath, read: reads.includes(canonicalPath), turn };
+  return completedCheck(
+    expectation,
+    expected,
+    observed,
+    expectation.expect === "read" ? observed.read : !observed.read,
+  );
+}
+
+export async function gradeResponseText(group, runs, expectation) {
+  const turn = expectation.turn ?? null;
+  if (
+    !validBodyExpectation(group, expectation, turn) ||
+    !RESPONSE_EXPECTATIONS.has(expectation.expect) ||
+    typeof expectation.value !== "string"
+  ) {
+    return { error: "invalid response-text expectation" };
+  }
+
+  const expected = { expect: expectation.expect, value: expectation.value, turn };
+  const run = runs[expectation.variant];
+  if (!run || run.state !== "complete") return unavailableCheck(expectation, expected);
+  const observed = responseObservation(run, turn);
+  if (observed === null) return unavailableCheck(expectation, expected);
+  return completedCheck(
+    expectation,
+    expected,
+    observed,
+    textMatches(expectation.expect, expectation.value, observed.response),
+  );
+}
+
+function activationMatches(expectation, observed) {
+  if (expectation.expect === "never") return !observed.targetRead;
+  if (!Number.isInteger(expectation.turn) || expectation.turn < 1) return false;
+  if (expectation.expect === "on-turn") return observed.readTurns.includes(expectation.turn);
+  if (expectation.expect === "by-turn") {
+    return observed.firstReadTurn !== null && observed.firstReadTurn <= expectation.turn;
+  }
+  return observed.firstReadTurn === null || observed.firstReadTurn >= expectation.turn;
+}
+
+function responseObservation(run, turn) {
+  if (turn === null) {
+    const observedTurn = Array.isArray(run.turns) && run.turns.length > 0 ? run.turns.at(-1).turn : 1;
+    return { response: typeof run.response === "string" ? run.response : "", turn: observedTurn };
+  }
+  if (!Array.isArray(run.turns)) {
+    if (turn !== 1) return null;
+    return { response: typeof run.response === "string" ? run.response : "", turn };
+  }
+  const observedTurn = run.turns.find((entry) => entry.turn === turn);
+  if (!observedTurn) return null;
+  return { response: observedTurn.response ?? "", turn };
+}

package/skills/evaluate-skill/scripts/lib/grade-support.mjs

@@ -0,0 +1,126 @@
+import { createHash } from "node:crypto";
+import { lstat, readFile, realpath } from "node:fs/promises";
+import path from "node:path";
+
+export const VARIANTS = ["baseline", "candidate"];
+export const RESPONSE_EXPECTATIONS = new Set(["contains", "not-contains", "equals"]);
+
+export function hasOwn(value, key) {
+  return Object.getOwnPropertyDescriptor(value, key) !== undefined;
+}
+
+export function validBodyExpectation(group, expectation, turn) {
+  return group.type === "body" && VARIANTS.includes(expectation.variant) && validTurn(group, turn);
+}
+
+export function validTurn(group, turn) {
+  if (turn === null) return true;
+  const declaredTurnCount = group.input.prompt === undefined ? group.input.turns.length : 1;
+  return Number.isInteger(turn) && turn >= 1 && turn <= declaredTurnCount;
+}
+
+export function completedCheck(expectation, expected, observed, passed) {
+  return {
+    id: expectation.id,
+    kind: expectation.kind,
+    variant: expectation.variant,
+    state: passed ? "pass" : "fail",
+    expected,
+    observed,
+  };
+}
+
+export function unavailableCheck(expectation, expected) {
+  return {
+    id: expectation.id,
+    kind: expectation.kind,
+    variant: expectation.variant,
+    state: "unavailable",
+    expected,
+    observed: null,
+  };
+}
+
+export function workspaceEvidence(run, turn) {
+  if (turn === null || (!Array.isArray(run.turns) && turn === 1)) {
+    if (!run.workspace || !Array.isArray(run.effects?.after)) return null;
+    return { root: run.workspace, files: run.effects.after, changes: run.effects.changes ?? null };
+  }
+  const observedTurn = run.turns?.find((entry) => entry.turn === turn);
+  if (!observedTurn?.workspace?.path || !Array.isArray(observedTurn.workspace.files)) return null;
+  return {
+    root: observedTurn.workspace.path,
+    files: observedTurn.workspace.files,
+    changes: observedTurn.workspace.changes ?? null,
+  };
+}
+
+export function readPaths(run, turn) {
+  if (turn === null) {
+    if (Array.isArray(run.turns)) {
+      return run.turns.flatMap((entry) => (Array.isArray(entry.successfulReadPaths) ? entry.successfulReadPaths : []));
+    }
+    return Array.isArray(run.successfulReadPaths) ? run.successfulReadPaths : [];
+  }
+  if (!Array.isArray(run.turns)) {
+    return turn === 1 && Array.isArray(run.successfulReadPaths) ? run.successfulReadPaths : null;
+  }
+  const observedTurn = run.turns.find((entry) => entry.turn === turn);
+  return observedTurn && Array.isArray(observedTurn.successfulReadPaths) ? observedTurn.successfulReadPaths : null;
+}
+
+export function textMatches(expect, value, text) {
+  if (expect === "contains") return text.includes(value);
+  if (expect === "not-contains") return !text.includes(value);
+  return text === value;
+}
+
+export function validPathArray(value) {
+  return (
+    Array.isArray(value) &&
+    value.every((entry) => typeof entry === "string" && isSafeRelativePath(entry)) &&
+    new Set(value).size === value.length
+  );
+}
+
+export async function readWorkspaceFile(workspace, relativePath) {
+  const entry = workspace.files.find((file) => file.path === relativePath);
+  if (!entry) return { state: "missing", sha256: null };
+  if (entry.type !== undefined && entry.type !== "file") {
+    throw new Error(`file assertion requires a regular file: ${relativePath}`);
+  }
+
+  const root = await realpath(workspace.root);
+  const declaredPath = path.resolve(root, relativePath);
+  const declaredStat = await lstat(declaredPath);
+  if (declaredStat.isSymbolicLink()) {
+    throw new Error(`file assertion requires a regular file: ${relativePath}`);
+  }
+  const file = await realpath(declaredPath);
+  if (!isContained(root, file)) throw new Error(`workspace evidence path escapes its root: ${relativePath}`);
+  const contents = await readFile(file);
+  const actualHash = createHash("sha256").update(contents).digest("hex");
+  if (actualHash !== entry.sha256) throw new Error(`workspace evidence changed after capture: ${relativePath}`);
+  return { state: "present", sha256: actualHash, text: contents.toString("utf8") };
+}
+
+export function isSafeRelativePath(value) {
+  return (
+    value.length > 0 &&
+    !path.isAbsolute(value) &&
+    !value.includes("\\") &&
+    !value.includes("\0") &&
+    !value.split("/").some((part) => part === "" || part === "." || part === "..")
+  );
+}
+
+export function isContained(root, target) {
+  const relative = path.relative(root, target);
+  return relative === "" || (!relative.startsWith(`..${path.sep}`) && relative !== ".." && !path.isAbsolute(relative));
+}
+
+export function compareText(left, right) {
+  if (left < right) return -1;
+  if (left > right) return 1;
+  return 0;
+}