@prismatic-io/lux 0.0.1

package/src/assertions/core/file-checks.ts

@@ -0,0 +1,336 @@
+import { glob } from "tinyglobby";
+import { z } from "zod";
+import { type AssertionImpl, readInspectedFileBytes } from "../../core/index.js";
+import {
+  baseAssertionFields,
+  globEscapesAssertionRoot,
+  inRange,
+  inspectAssertionPath,
+  LocationSchema,
+  resolveLocationRoot,
+} from "./helpers.js";
+import {
+  countSafeRegex,
+  inspectSafeRegexes,
+  testSafeRegex,
+  testSafeRegexSet,
+} from "./safe-regex.js";
+
+const MAX_REGEX_FILE_BYTES = 5 * 1024 * 1024;
+const MAX_TEXT_FILE_BYTES = 10 * 1024 * 1024;
+const MAX_REGEX_PATTERNS = 50;
+const MAX_REGEX_PATTERN_CHARS = 2_000;
+
+export const FileExistsConfigSchema = z
+  .object({
+    type: z.literal("file-exists"),
+    path: z.string(),
+    in: LocationSchema,
+    ...baseAssertionFields,
+  })
+  .strict();
+export type FileExistsConfig = z.infer<typeof FileExistsConfigSchema>;
+
+export const fileExistsAssertion: AssertionImpl<FileExistsConfig> = {
+  type: "file-exists",
+  configSchema: FileExistsConfigSchema,
+  evaluate: async (config, run) => {
+    const { stats, issue } = await inspectAssertionPath(run, config.in, config.path);
+    const passed = stats?.isFile() ?? false;
+    return {
+      passed,
+      score: passed ? 1 : 0,
+      comment: passed
+        ? `${config.path} exists`
+        : (issue ?? (stats ? `${config.path} is not a file` : `${config.path} not found`)),
+    };
+  },
+};
+
+export const FileContainsConfigSchema = z
+  .object({
+    type: z.literal("file-contains"),
+    path: z.string(),
+    text: z.string(),
+    in: LocationSchema,
+    ...baseAssertionFields,
+  })
+  .strict();
+export type FileContainsConfig = z.infer<typeof FileContainsConfigSchema>;
+
+export const fileContainsAssertion: AssertionImpl<FileContainsConfig> = {
+  type: "file-contains",
+  configSchema: FileContainsConfigSchema,
+  evaluate: async (config, run) => {
+    const { target, stats, issue } = await inspectAssertionPath(run, config.in, config.path);
+    if (!stats?.isFile()) {
+      return {
+        passed: false,
+        score: 0,
+        comment: issue ?? (stats ? `${config.path} is not a file` : `${config.path} not found`),
+      };
+    }
+    if (stats.size > MAX_TEXT_FILE_BYTES) {
+      return {
+        passed: false,
+        score: 0,
+        comment: `${config.path} exceeds the ${MAX_TEXT_FILE_BYTES}-byte text assertion limit`,
+      };
+    }
+    const content = (
+      await readInspectedFileBytes(target, stats, MAX_TEXT_FILE_BYTES, config.path)
+    ).toString("utf8");
+    const passed = content.includes(config.text);
+    return {
+      passed,
+      score: passed ? 1 : 0,
+      comment: passed
+        ? `${config.path} contains '${config.text}'`
+        : `${config.path} does not contain '${config.text}'`,
+    };
+  },
+};
+
+export const FileMatchesConfigSchema = z
+  .object({
+    type: z.literal("file-matches"),
+    path: z.string(),
+    pattern: z.string().max(MAX_REGEX_PATTERN_CHARS),
+    flags: z.string().max(16).optional(),
+    in: LocationSchema,
+    ...baseAssertionFields,
+  })
+  .strict()
+  .superRefine(async (config, ctx) => {
+    const [inspection] = await inspectSafeRegexes([config.pattern], config.flags ?? "");
+    if (inspection && "error" in inspection) {
+      ctx.addIssue({ code: "custom", path: ["pattern"], message: inspection.error });
+    }
+  });
+export type FileMatchesConfig = z.infer<typeof FileMatchesConfigSchema>;
+
+export const fileMatchesAssertion: AssertionImpl<FileMatchesConfig> = {
+  type: "file-matches",
+  configSchema: FileMatchesConfigSchema,
+  evaluate: async (config, run, ctx) => {
+    const inspected = await readAssertionFile(config, run);
+    if ("failure" in inspected) return inspected.failure;
+    const passed = await testSafeRegex(config.pattern, config.flags ?? "", inspected.content, {
+      abortSignal: ctx?.abortSignal,
+    });
+    return {
+      passed,
+      score: passed ? 1 : 0,
+      comment: passed
+        ? `${config.path} matches /${config.pattern}/${config.flags ?? ""}`
+        : `${config.path} does not match /${config.pattern}/${config.flags ?? ""}`,
+    };
+  },
+};
+
+const readAssertionFile = async (
+  config: { in: z.infer<typeof LocationSchema>; path: string },
+  run: Parameters<AssertionImpl["evaluate"]>[1],
+): Promise<{ content: string } | { failure: { passed: false; score: 0; comment: string } }> => {
+  const { target, stats, issue } = await inspectAssertionPath(run, config.in, config.path);
+  if (!stats?.isFile()) {
+    return {
+      failure: {
+        passed: false,
+        score: 0,
+        comment: issue ?? (stats ? `${config.path} is not a file` : `${config.path} not found`),
+      },
+    };
+  }
+  if (stats.size > MAX_REGEX_FILE_BYTES) {
+    return {
+      failure: {
+        passed: false,
+        score: 0,
+        comment: `${config.path} exceeds the ${MAX_REGEX_FILE_BYTES}-byte regex assertion limit`,
+      },
+    };
+  }
+  return {
+    content: (
+      await readInspectedFileBytes(target, stats, MAX_REGEX_FILE_BYTES, config.path)
+    ).toString("utf8"),
+  };
+};
+
+const regexSetPassed = (
+  mode: FileRegexSetConfig["mode"],
+  matched: number,
+  total: number,
+): boolean => {
+  if (mode === "all") return matched === total;
+  if (mode === "any") return matched > 0;
+  return matched === 0;
+};
+
+export const FileRegexSetConfigSchema = z
+  .object({
+    type: z.literal("file-regex-set"),
+    path: z.string(),
+    in: LocationSchema,
+    patterns: z.array(z.string().max(MAX_REGEX_PATTERN_CHARS)).min(1).max(MAX_REGEX_PATTERNS),
+    mode: z.enum(["all", "any", "none"]),
+    flags: z.string().max(16).optional(),
+    ...baseAssertionFields,
+  })
+  .strict()
+  .superRefine(async (config, ctx) => {
+    const inspections = await inspectSafeRegexes(config.patterns, config.flags ?? "");
+    for (const [index, inspection] of inspections.entries()) {
+      if ("error" in inspection) {
+        ctx.addIssue({
+          code: "custom",
+          path: ["patterns", index],
+          message: inspection.error,
+        });
+      }
+    }
+  });
+export type FileRegexSetConfig = z.infer<typeof FileRegexSetConfigSchema>;
+
+export const fileRegexSetAssertion: AssertionImpl<FileRegexSetConfig> = {
+  type: "file-regex-set",
+  configSchema: FileRegexSetConfigSchema,
+  evaluate: async (config, run, ctx) => {
+    const inspected = await readAssertionFile(config, run);
+    if ("failure" in inspected) return inspected.failure;
+    const matches = await testSafeRegexSet(config.patterns, config.flags ?? "", inspected.content, {
+      abortSignal: ctx?.abortSignal,
+    });
+    const matched = matches.filter(Boolean).length;
+    const passed = regexSetPassed(config.mode, matched, matches.length);
+    return {
+      passed,
+      score: passed ? 1 : 0,
+      comment: `${matched}/${matches.length} regex patterns matched ${config.path}; expected ${config.mode}`,
+      detail: {
+        matched,
+        total: matches.length,
+        unmatchedPatterns: config.patterns.filter((_pattern, index) => !matches[index]),
+      },
+    };
+  },
+};
+
+export const FileRegexCountConfigSchema = z
+  .object({
+    type: z.literal("file-regex-count"),
+    path: z.string(),
+    in: LocationSchema,
+    pattern: z.string().max(MAX_REGEX_PATTERN_CHARS),
+    flags: z.string().max(16).optional(),
+    min: z.number().int().nonnegative().optional(),
+    max: z.number().int().nonnegative().optional(),
+    ...baseAssertionFields,
+  })
+  .strict()
+  .refine((config) => config.min !== undefined || config.max !== undefined, {
+    message: "set at least one of min or max",
+  })
+  .refine((config) => config.max !== undefined || (config.min ?? 0) > 0, {
+    message: "min alone must be greater than zero",
+  })
+  .refine(
+    (config) => config.min === undefined || config.max === undefined || config.min <= config.max,
+    { message: "min must be <= max" },
+  )
+  .superRefine(async (config, ctx) => {
+    const [inspection] = await inspectSafeRegexes([config.pattern], config.flags ?? "");
+    if (!inspection) return;
+    if ("error" in inspection) {
+      ctx.addIssue({ code: "custom", path: ["pattern"], message: inspection.error });
+      return;
+    }
+    if (inspection.matchesEmpty) {
+      ctx.addIssue({
+        code: "custom",
+        path: ["pattern"],
+        message: "count pattern must not match an empty string",
+      });
+    }
+  });
+export type FileRegexCountConfig = z.infer<typeof FileRegexCountConfigSchema>;
+
+export const fileRegexCountAssertion: AssertionImpl<FileRegexCountConfig> = {
+  type: "file-regex-count",
+  configSchema: FileRegexCountConfigSchema,
+  evaluate: async (config, run, ctx) => {
+    const inspected = await readAssertionFile(config, run);
+    if ("failure" in inspected) return inspected.failure;
+    const stopAfter = config.max !== undefined ? config.max + 1 : (config.min ?? 1);
+    const { count, exact } = await countSafeRegex(
+      config.pattern,
+      config.flags ?? "",
+      inspected.content,
+      stopAfter,
+      { abortSignal: ctx?.abortSignal },
+    );
+    const { passed, comment } = inRange(count, config.min, config.max);
+    return {
+      passed,
+      score: passed ? 1 : 0,
+      comment: `${config.path} has ${exact ? "" : "at least "}${count} matches for /${config.pattern}/${config.flags ?? ""} (${comment})`,
+      detail: { count, exact },
+    };
+  },
+};
+
+export const GlobCountConfigSchema = z
+  .object({
+    type: z.literal("glob-count"),
+    glob: z.string(),
+    min: z.number().int().nonnegative().optional(),
+    max: z.number().int().nonnegative().optional(),
+    in: LocationSchema,
+    ...baseAssertionFields,
+  })
+  .strict()
+  // With neither bound the assertion is vacuous — it would always pass.
+  .refine((c) => c.min !== undefined || c.max !== undefined, {
+    message: "set at least one of min or max",
+  })
+  .refine((c) => c.min === undefined || c.max === undefined || c.min <= c.max, {
+    message: "min must be <= max",
+  });
+export type GlobCountConfig = z.infer<typeof GlobCountConfigSchema>;
+
+export const globCountAssertion: AssertionImpl<GlobCountConfig> = {
+  type: "glob-count",
+  configSchema: GlobCountConfigSchema,
+  evaluate: async (config, run) => {
+    const root = resolveLocationRoot(run, config.in);
+    if (globEscapesAssertionRoot(config.glob)) {
+      return {
+        passed: false,
+        score: 0,
+        comment: `${config.glob} escapes the selected assertion root`,
+      };
+    }
+    const matches = await glob(config.glob, {
+      cwd: root,
+      onlyFiles: true,
+      followSymbolicLinks: false,
+    });
+    const inspected = await Promise.all(
+      matches.map(async (path) => ({
+        path,
+        inspection: await inspectAssertionPath(run, config.in, path),
+      })),
+    );
+    const files = inspected
+      .filter(({ inspection }) => inspection.stats?.isFile())
+      .map(({ path }) => path);
+    const { passed, comment } = inRange(files.length, config.min, config.max);
+    return {
+      passed,
+      score: passed ? 1 : 0,
+      comment: `${files.length} matches for ${config.glob} (${comment})`,
+      detail: { count: files.length, files: files.slice(0, 10) },
+    };
+  },
+};

package/src/assertions/core/helpers.ts

@@ -0,0 +1,203 @@
+import type { Stats } from "node:fs";
+import { lstat } from "node:fs/promises";
+import { isAbsolute, join, relative, resolve, sep } from "node:path";
+import { z } from "zod";
+import { inspectRunArtifact, runArtifactsRoot } from "../../core/artifact-evidence.js";
+import type { Run, RunEvent } from "../../core/index.js";
+
+const FsLocationSchema = z.enum(["artifacts", "cwd"]);
+export type FsLocation = z.infer<typeof FsLocationSchema>;
+
+export const resolveLocationRoot = (run: Run, where: FsLocation = "artifacts"): string =>
+  where === "cwd" ? run.runDir : runArtifactsRoot(run);
+
+export type AssertionPathInspection = {
+  target: string;
+  stats?: Stats;
+  issue?: string;
+};
+
+const pathEscapesRoot = (root: string, target: string): boolean => {
+  const rel = relative(root, target);
+  return rel === ".." || rel.startsWith(`..${sep}`) || isAbsolute(rel);
+};
+
+export const inspectAssertionPath = async (
+  run: Run,
+  where: FsLocation,
+  authoredPath: string,
+): Promise<AssertionPathInspection> => {
+  if (where === "artifacts") {
+    const { target, stats, issue } = await inspectRunArtifact(run, authoredPath);
+    return { target, ...(stats ? { stats } : {}), ...(issue ? { issue } : {}) };
+  }
+  const root = resolve(resolveLocationRoot(run, where));
+  const target = resolve(root, authoredPath);
+  if (pathEscapesRoot(root, target)) {
+    return { target, issue: `${authoredPath} escapes the selected assertion root` };
+  }
+
+  const rel = relative(root, target);
+  const components = [
+    root,
+    ...rel
+      .split(sep)
+      .filter(Boolean)
+      .map((_, index, parts) => join(root, ...parts.slice(0, index + 1))),
+  ];
+  for (const [index, component] of components.entries()) {
+    let stats: Stats;
+    try {
+      stats = await lstat(component);
+    } catch (err) {
+      const code = (err as NodeJS.ErrnoException).code;
+      if (code === "ENOENT" || code === "ENOTDIR") {
+        return { target, issue: `${authoredPath} not found` };
+      }
+      throw err;
+    }
+    if (stats.isSymbolicLink()) {
+      return { target, issue: `${authoredPath} resolves through a symlink` };
+    }
+    if (index < components.length - 1 && !stats.isDirectory()) {
+      return { target, issue: `${authoredPath} not found` };
+    }
+    if (index === components.length - 1) return { target, stats };
+  }
+  return { target, issue: `${authoredPath} not found` };
+};
+
+export const globEscapesAssertionRoot = (pattern: string): boolean =>
+  isAbsolute(pattern) || pattern.split(/[\\/]/).includes("..");
+
+const COMMAND_TOOLCHAIN_ENV = [
+  "PATH",
+  "PATHEXT",
+  "HOME",
+  "USERPROFILE",
+  "SYSTEMROOT",
+  "COMSPEC",
+  "SHELL",
+  "TMPDIR",
+  "TMP",
+  "TEMP",
+  "LANG",
+  "LANGUAGE",
+  "LC_ALL",
+  "LC_CTYPE",
+  "TERM",
+  "COLORTERM",
+  "NO_COLOR",
+  "FORCE_COLOR",
+] as const;
+
+export const commandEnvironment = (
+  parent: NodeJS.ProcessEnv,
+  inherit: true | readonly string[] = [],
+): NodeJS.ProcessEnv => {
+  if (inherit === true) return { ...parent };
+  const names = new Set<string>([...COMMAND_TOOLCHAIN_ENV, ...inherit]);
+  return Object.fromEntries(
+    [...names].flatMap((name) => {
+      const value = parent[name];
+      return value === undefined ? [] : [[name, value] as const];
+    }),
+  );
+};
+
+/**
+ * Resolve an RFC 6901 JSON pointer against a parsed value; undefined when it
+ * does not resolve. Deviates from the RFC for arrays: any own property
+ * resolves, so `/length` works even though the RFC only allows index tokens.
+ */
+export const resolvePointer = (root: unknown, pointer: string): unknown => {
+  if (pointer === "") return root;
+  if (!pointer.startsWith("/")) return undefined;
+  const parts = pointer
+    .slice(1)
+    .split("/")
+    .map((p) => p.replace(/~1/g, "/").replace(/~0/g, "~"));
+  let node: unknown = root;
+  for (const part of parts) {
+    if (node === null || typeof node !== "object") return undefined;
+    // Own properties only — a pointer like `/constructor` must not walk the
+    // prototype chain into inherited values.
+    if (!Object.hasOwn(node, part)) return undefined;
+    node = (node as Record<string, unknown>)[part];
+  }
+  return node;
+};
+
+export const inRange = (
+  value: number,
+  min?: number,
+  max?: number,
+): { passed: boolean; comment: string } => {
+  if (min !== undefined && value < min) {
+    return { passed: false, comment: `${value} < min ${min}` };
+  }
+  if (max !== undefined && value > max) {
+    return { passed: false, comment: `${value} > max ${max}` };
+  }
+  const bounds = [
+    min !== undefined ? `>= ${min}` : undefined,
+    max !== undefined ? `<= ${max}` : undefined,
+  ]
+    .filter((b) => b !== undefined)
+    .join(", ");
+  return {
+    passed: true,
+    comment: bounds ? `${value} ${bounds}` : `${value}`,
+  };
+};
+
+/** Structural equality for JSON-shaped values — object key order is irrelevant. */
+export const deepEquals = (a: unknown, b: unknown): boolean => {
+  if (Object.is(a, b)) return true;
+  if (typeof a !== "object" || typeof b !== "object" || a === null || b === null) return false;
+  if (Array.isArray(a) || Array.isArray(b)) {
+    if (!Array.isArray(a) || !Array.isArray(b) || a.length !== b.length) return false;
+    return a.every((v, i) => deepEquals(v, b[i]));
+  }
+  const aEntries = Object.entries(a);
+  if (aEntries.length !== Object.keys(b).length) return false;
+  return aEntries.every(
+    ([k, v]) => Object.hasOwn(b, k) && deepEquals(v, (b as Record<string, unknown>)[k]),
+  );
+};
+
+/**
+ * Zod fragment for the framework-level fields every assertion shares
+ * (weight, threshold, name). Spread into per-assertion config schemas:
+ *
+ *     z.object({ type: z.literal("foo"), ...baseAssertionFields, ... })
+ *
+ * Mirrors {@link Assertion} from the core module while letting each built-in
+ * assertion reject misspelled type-specific fields at its own schema boundary.
+ */
+export const baseAssertionFields = {
+  id: z.string().min(1).optional(),
+  weight: z.number().positive().optional(),
+  threshold: z.number().min(0).max(1).optional(),
+  name: z.string().optional(),
+} as const;
+
+/**
+ * Where on the run dir an assertion looks for files. Defaults to
+ * `artifacts` (the standard place drivers drop captured files).
+ */
+export const LocationSchema = FsLocationSchema.default("artifacts");
+
+// `name` is intentionally excluded: for tool assertions the `name` field is
+// the tool's name, which then doubles as the display label.
+export const weightThresholdFields = {
+  id: baseAssertionFields.id,
+  weight: baseAssertionFields.weight,
+  threshold: baseAssertionFields.threshold,
+} as const;
+
+export type ProgressEvent = Extract<RunEvent, { type: "progress" }>;
+export type ToolCallEvent = ProgressEvent & { kind: "tool-call" };
+
+export const isToolCallEvent = (e: RunEvent): e is ToolCallEvent =>
+  e.type === "progress" && e.kind === "tool-call";

package/src/assertions/core/index.ts

@@ -0,0 +1,80 @@
+import type { AssertionImpl, Registry } from "../../core/index.js";
+import { commandExitsZeroAssertion } from "./command-exits-zero.js";
+import {
+  interruptCountAssertion,
+  toolCallCountAssertion,
+  toolCalledAssertion,
+} from "./event-checks.js";
+import {
+  fileContainsAssertion,
+  fileExistsAssertion,
+  fileMatchesAssertion,
+  fileRegexCountAssertion,
+  fileRegexSetAssertion,
+  globCountAssertion,
+} from "./file-checks.js";
+import { jsonPointerEqualsAssertion } from "./json-pointer-equals.js";
+import { containsAssertion, notContainsAssertion, regexAssertion } from "./output-checks.js";
+import { predicateAssertion } from "./predicate.js";
+import {
+  artifactBytesUnderAssertion,
+  costUnderAssertion,
+  latencyUnderAssertion,
+  runSucceededAssertion,
+  tokensUnderAssertion,
+} from "./resource-checks.js";
+import { toolCalledWithAssertion } from "./tool-called-with.js";
+import { toolInvocationAssertion } from "./tool-invocation.js";
+import {
+  noErrorResultsAssertion,
+  toolResultContainsAssertion,
+  toolResultIsErrorAssertion,
+  toolResultJsonPointerEqualsAssertion,
+} from "./tool-result-checks.js";
+
+export * from "./command-exits-zero.js";
+export * from "./event-checks.js";
+export * from "./file-checks.js";
+export * from "./json-pointer-equals.js";
+export * from "./output-checks.js";
+export * from "./predicate.js";
+export * from "./resource-checks.js";
+export * from "./tool-called-with.js";
+export * from "./tool-calls.js";
+export * from "./tool-invocation.js";
+export * from "./tool-result-checks.js";
+
+export const allCoreAssertions: AssertionImpl[] = [
+  fileExistsAssertion,
+  fileContainsAssertion,
+  fileMatchesAssertion,
+  fileRegexSetAssertion,
+  fileRegexCountAssertion,
+  globCountAssertion,
+  toolCalledAssertion,
+  toolCalledWithAssertion,
+  toolInvocationAssertion,
+  toolCallCountAssertion,
+  toolResultIsErrorAssertion,
+  toolResultContainsAssertion,
+  toolResultJsonPointerEqualsAssertion,
+  noErrorResultsAssertion,
+  interruptCountAssertion,
+  commandExitsZeroAssertion,
+  containsAssertion,
+  notContainsAssertion,
+  regexAssertion,
+  jsonPointerEqualsAssertion,
+  latencyUnderAssertion,
+  artifactBytesUnderAssertion,
+  tokensUnderAssertion,
+  costUnderAssertion,
+  runSucceededAssertion,
+  predicateAssertion,
+];
+
+export const registerCoreAssertions = (registry: Registry): void => {
+  for (const a of allCoreAssertions) {
+    if (!registry.getAssertion(a.type)) registry.registerAssertion(a);
+  }
+};

package/src/assertions/core/json-pointer-equals.ts

@@ -0,0 +1,84 @@
+import { z } from "zod";
+import { type AssertionImpl, readInspectedFileBytes } from "../../core/index.js";
+import {
+  baseAssertionFields,
+  deepEquals,
+  inspectAssertionPath,
+  LocationSchema,
+  resolvePointer,
+} from "./helpers.js";
+
+const MAX_JSON_FILE_BYTES = 10 * 1024 * 1024;
+
+export const JsonPointerEqualsConfigSchema = z
+  .object({
+    type: z.literal("json-pointer-equals"),
+    /** File path to read (parsed as JSON). */
+    path: z.string(),
+    /** Where to look for the file. */
+    in: LocationSchema,
+    /** RFC 6901 JSON pointer. */
+    pointer: z.string(),
+    /** Expected value at the pointer. */
+    value: z.unknown(),
+    ...baseAssertionFields,
+  })
+  .strict()
+  // An omitted `value` would silently pass against any missing pointer
+  // (undefined === undefined), so its presence is required outright.
+  .refine((c) => "value" in c, { message: "value is required" });
+export type JsonPointerEqualsConfig = z.infer<typeof JsonPointerEqualsConfigSchema>;
+
+export const jsonPointerEqualsAssertion: AssertionImpl<JsonPointerEqualsConfig> = {
+  type: "json-pointer-equals",
+  configSchema: JsonPointerEqualsConfigSchema,
+  evaluate: async (config, run) => {
+    const { target, stats, issue } = await inspectAssertionPath(run, config.in, config.path);
+    if (!stats?.isFile()) {
+      return {
+        passed: false,
+        score: 0,
+        comment: issue ?? (stats ? `${config.path} is not a file` : `${config.path} not found`),
+      };
+    }
+    if (stats.size > MAX_JSON_FILE_BYTES) {
+      return {
+        passed: false,
+        score: 0,
+        comment: `${config.path} exceeds the ${MAX_JSON_FILE_BYTES}-byte JSON assertion limit`,
+      };
+    }
+    let parsed: unknown;
+    try {
+      const contents = await readInspectedFileBytes(
+        target,
+        stats,
+        MAX_JSON_FILE_BYTES,
+        config.path,
+      );
+      parsed = JSON.parse(contents.toString("utf8"));
+    } catch (err) {
+      return {
+        passed: false,
+        score: 0,
+        comment: `${config.path} is not valid JSON: ${err instanceof Error ? err.message : String(err)}`,
+      };
+    }
+    const resolved = resolvePointer(parsed, config.pointer);
+    if (resolved === undefined) {
+      return {
+        passed: false,
+        score: 0,
+        comment: `${config.pointer} does not resolve in ${config.path}`,
+      };
+    }
+    const passed = deepEquals(resolved, config.value);
+    return {
+      passed,
+      score: passed ? 1 : 0,
+      comment: passed
+        ? `${config.pointer} matches expected value`
+        : `${config.pointer}: expected ${JSON.stringify(config.value)}, got ${JSON.stringify(resolved)}`,
+    };
+  },
+};