@bjoernrochel/sevro 0.1.0-rc.2 → 0.1.0-rc.21

package/src/graders/output.ts

@@ -1,3 +1,4 @@
+import { isRecord, isUnknownArray } from "../value-guards";
 import { isDeepStrictEqual } from "node:util";
 import Ajv2020, { type ValidateFunction } from "ajv/dist/2020.js";
 import { canonicalJson } from "../identity";
@@ -83,144 +84,177 @@ function pointerValue(value: unknown, pointer: string): unknown {
     }, value);
 }
 
+function subsetRegexPattern(value: Record<string, unknown>): boolean {
+  return Object.keys(value).length === 1 && Object.hasOwn(value, "$regex");
+}
+function validateSubsetRegex(pattern: unknown): void {
+  if (typeof pattern !== "string") throw new Error("invalid subset regex");
+  try {
+    new RegExp(pattern);
+  } catch (cause) {
+    throw new Error("invalid subset regex", { cause });
+  }
+}
 function validateSubsetPatterns(value: unknown): void {
-  if (!value || typeof value !== "object") return;
-  if (Array.isArray(value)) {
+  if (isUnknownArray(value)) {
     value.forEach(validateSubsetPatterns);
     return;
   }
-  const record = value as Record<string, unknown>;
-  if (Object.keys(record).length === 1 && Object.hasOwn(record, "$regex")) {
-    if (typeof record.$regex !== "string")
-      throw new Error("invalid subset regex");
-    try {
-      new RegExp(record.$regex);
-    } catch {
-      throw new Error("invalid subset regex");
-    }
+  if (!isRecord(value)) return;
+  if (subsetRegexPattern(value)) {
+    validateSubsetRegex(value.$regex);
     return;
   }
-  Object.values(record).forEach(validateSubsetPatterns);
+  Object.values(value).forEach(validateSubsetPatterns);
 }
 
 function containsSubset(value: unknown, expected: unknown): boolean {
-  if (Array.isArray(expected))
-    return (
-      Array.isArray(value) &&
-      expected.every((part) => value.some((item) => containsSubset(item, part)))
-    );
-  if (expected && typeof expected === "object") {
-    const record = expected as Record<string, unknown>;
-    if (Object.keys(record).length === 1 && typeof record.$regex === "string")
-      return typeof value === "string" && new RegExp(record.$regex).test(value);
-    return Boolean(
-      value &&
-      typeof value === "object" &&
-      Object.entries(record).every(([key, part]) =>
-        containsSubset((value as Record<string, unknown>)[key], part),
-      ),
-    );
-  }
+  if (isUnknownArray(expected)) return arraySubset(value, expected);
+  if (isRecord(expected)) return recordSubset(value, expected);
   return Object.is(value, expected);
 }
+function arraySubset(value: unknown, expected: unknown[]): boolean {
+  return (
+    isUnknownArray(value) &&
+    expected.every((part) => value.some((item) => containsSubset(item, part)))
+  );
+}
+function recordSubset(
+  value: unknown,
+  expected: Record<string, unknown>,
+): boolean {
+  if (Object.keys(expected).length === 1 && typeof expected.$regex === "string")
+    return typeof value === "string" && new RegExp(expected.$regex).test(value);
+  return objectSubset(value, expected);
+}
+function objectSubset(
+  value: unknown,
+  expected: Record<string, unknown>,
+): boolean {
+  if (!value || typeof value !== "object") return false;
+  const actual = value as Record<string, unknown>;
+  return Object.entries(expected).every(([key, part]) =>
+    containsSubset(actual[key], part),
+  );
+}
+
+type JsonConfiguration = Record<string, unknown> & {
+  pointer?: string;
+  exactDocument?: boolean;
+};
+function validPointer(pointer: string): boolean {
+  return (
+    pointer === "" || (pointer.startsWith("/") && !/~(?![01])/.test(pointer))
+  );
+}
+function requireOptionalPointer(pointer: unknown): void {
+  if (pointer === undefined) return;
+  if (typeof pointer !== "string")
+    throw new Error("JSON pointer must be a string");
+  if (!validPointer(pointer)) throw new Error("invalid JSON pointer");
+}
+function requireExactDocument(value: unknown): void {
+  if (value !== undefined && typeof value !== "boolean")
+    throw new Error("exactDocument must be boolean");
+}
+function requireJsonConfiguration(
+  config: Record<string, unknown>,
+): asserts config is JsonConfiguration {
+  requireOptionalPointer(config.pointer);
+  requireExactDocument(config.exactDocument);
+}
 
 function compileJson(
   config: Record<string, unknown>,
 ): PreparedOutputCheck["evaluate"] {
   configKeys(config, ["pointer", "equals", "contains", "exactDocument"]);
-  if (config.pointer !== undefined && typeof config.pointer !== "string")
-    throw new Error("JSON pointer must be a string");
-  if (
-    config.pointer !== undefined &&
-    config.pointer !== "" &&
-    (!config.pointer.startsWith("/") || /~(?![01])/.test(config.pointer))
-  )
-    throw new Error("invalid JSON pointer");
-  if (
-    config.exactDocument !== undefined &&
-    typeof config.exactDocument !== "boolean"
-  )
-    throw new Error("exactDocument must be boolean");
+  requireJsonConfiguration(config);
   if (Object.hasOwn(config, "equals")) canonicalJson(config.equals);
   if (Object.hasOwn(config, "contains")) {
     canonicalJson(config.contains);
     validateSubsetPatterns(config.contains);
   }
-  return (_text, document) => {
-    if (document.error) return "output is not valid JSON";
-    if (config.exactDocument && !document.exact)
-      return "output is not exactly one JSON document";
-    let selected: unknown;
-    try {
-      selected = pointerValue(
-        document.value,
-        (config.pointer as string | undefined) ?? "",
-      );
-    } catch {
-      return "JSON pointer does not resolve";
-    }
-    if (
-      Object.hasOwn(config, "equals") &&
-      !isDeepStrictEqual(selected, config.equals)
-    )
-      return "JSON value did not equal expectation";
-    if (
-      Object.hasOwn(config, "contains") &&
-      (!Array.isArray(selected) ||
-        !selected.some((item) => containsSubset(item, config.contains)))
-    )
-      return "JSON array did not contain expected item";
-    return undefined;
-  };
+  return (_text, document) => evaluateJson(config, document);
+}
+function documentFailure(
+  document: ParsedDocument,
+  exactDocument: unknown,
+): string | undefined {
+  if (document.error) return "output is not valid JSON";
+  if (exactDocument && !document.exact)
+    return "output is not exactly one JSON document";
+  return undefined;
+}
+function selectedDocument(
+  config: JsonConfiguration,
+  document: ParsedDocument,
+): { value: unknown } | { failure: string } {
+  try {
+    return { value: pointerValue(document.value, config.pointer ?? "") };
+  } catch {
+    return { failure: "JSON pointer does not resolve" };
+  }
+}
+function equalsFailure(
+  selected: unknown,
+  config: Record<string, unknown>,
+): string | undefined {
+  return Object.hasOwn(config, "equals") &&
+    !isDeepStrictEqual(selected, config.equals)
+    ? "JSON value did not equal expectation"
+    : undefined;
+}
+function containsFailure(
+  selected: unknown,
+  config: Record<string, unknown>,
+): string | undefined {
+  if (!Object.hasOwn(config, "contains")) return undefined;
+  if (
+    !isUnknownArray(selected) ||
+    !selected.some((item) => containsSubset(item, config.contains))
+  )
+    return "JSON array did not contain expected item";
+  return undefined;
+}
+function evaluateJson(
+  config: JsonConfiguration,
+  document: ParsedDocument,
+): string | undefined {
+  const failure = documentFailure(document, config.exactDocument);
+  if (failure) return failure;
+  const selected = selectedDocument(config, document);
+  if ("failure" in selected) return selected.failure;
+  return (
+    equalsFailure(selected.value, config) ??
+    containsFailure(selected.value, config)
+  );
 }
 
+function requireSchema(value: unknown): asserts value is object {
+  if (!isRecord(value))
+    throw new Error("schema check requires an inline JSON Schema object");
+}
 function compileSchema(
   config: Record<string, unknown>,
 ): PreparedOutputCheck["evaluate"] {
   configKeys(config, ["schema", "exactDocument"]);
-  if (
-    !config.schema ||
-    typeof config.schema !== "object" ||
-    Array.isArray(config.schema)
-  )
-    throw new Error("schema check requires an inline JSON Schema object");
-  if (
-    config.exactDocument !== undefined &&
-    typeof config.exactDocument !== "boolean"
-  )
-    throw new Error("exactDocument must be boolean");
+  requireSchema(config.schema);
+  requireExactDocument(config.exactDocument);
   canonicalJson(config.schema);
   let validate: ValidateFunction;
   try {
     validate = new Ajv2020({ strict: false }).compile(config.schema);
-  } catch {
-    throw new Error("invalid inline JSON Schema");
+  } catch (cause) {
+    throw new Error("invalid inline JSON Schema", { cause });
   }
-  return (_text, document) => {
-    if (document.error) return "output is not valid JSON";
-    if (config.exactDocument && !document.exact)
-      return "output is not exactly one JSON document";
-    return validate(document.value)
-      ? undefined
-      : "output did not match JSON Schema";
-  };
+  return (_text, document) =>
+    documentFailure(document, config.exactDocument) ??
+    (validate(document.value) ? undefined : "output did not match JSON Schema");
 }
 
 /** One named assertion may combine independent final-message predicates. */
-function compileCompositeOutput(
-  config: Record<string, unknown>,
-): PreparedOutputCheck["evaluate"] {
-  configKeys(config, [
-    "validJson",
-    "schema",
-    "jsonPath",
-    "expectJson",
-    "containsJson",
-    "expectExact",
-    "expectRegex",
-    "notRegex",
-    "flags",
-  ]);
+
+function requireCompositeTypes(config: Record<string, unknown>): void {
   if (config.validJson !== undefined && typeof config.validJson !== "boolean")
     throw new Error("validJson must be a boolean");
   if (
@@ -228,34 +262,49 @@ function compileCompositeOutput(
     typeof config.expectExact !== "string"
   )
     throw new Error("expectExact must be a string");
+}
+function requireCompositePointer(config: Record<string, unknown>): void {
   if (
     (Object.hasOwn(config, "expectJson") ||
       Object.hasOwn(config, "containsJson")) &&
     config.jsonPath === undefined
   )
     throw new Error("JSON expectations require a pointer");
+}
+function requireCompositeRegex(config: Record<string, unknown>): void {
   if (
     config.flags !== undefined &&
     config.expectRegex === undefined &&
     config.notRegex === undefined
   )
     throw new Error("regex flags require a pattern");
+}
+function compositePointerConfiguration(config: Record<string, unknown>) {
+  return {
+    pointer: config.jsonPath,
+    ...(Object.hasOwn(config, "expectJson")
+      ? { equals: config.expectJson }
+      : {}),
+    ...(Object.hasOwn(config, "containsJson")
+      ? { contains: config.containsJson }
+      : {}),
+  };
+}
+function compositeJsonStages(
+  config: Record<string, unknown>,
+): PreparedOutputCheck["evaluate"][] {
   const stages: PreparedOutputCheck["evaluate"][] = [];
   if (config.validJson) stages.push(compileJson({ exactDocument: true }));
   if (config.schema !== undefined)
     stages.push(compileSchema({ schema: config.schema }));
   if (config.jsonPath !== undefined)
-    stages.push(
-      compileJson({
-        pointer: config.jsonPath,
-        ...(Object.hasOwn(config, "expectJson")
-          ? { equals: config.expectJson }
-          : {}),
-        ...(Object.hasOwn(config, "containsJson")
-          ? { contains: config.containsJson }
-          : {}),
-      }),
-    );
+    stages.push(compileJson(compositePointerConfiguration(config)));
+  return stages;
+}
+function compositeTextStages(
+  config: Record<string, unknown>,
+): PreparedOutputCheck["evaluate"][] {
+  const stages: PreparedOutputCheck["evaluate"][] = [];
   if (config.expectExact !== undefined) {
     const expected = config.expectExact;
     stages.push((text) =>
@@ -274,6 +323,29 @@ function compileCompositeOutput(
         flags: config.flags,
       }),
     );
+  return stages;
+}
+function compileCompositeOutput(
+  config: Record<string, unknown>,
+): PreparedOutputCheck["evaluate"] {
+  configKeys(config, [
+    "validJson",
+    "schema",
+    "jsonPath",
+    "expectJson",
+    "containsJson",
+    "expectExact",
+    "expectRegex",
+    "notRegex",
+    "flags",
+  ]);
+  requireCompositeTypes(config);
+  requireCompositePointer(config);
+  requireCompositeRegex(config);
+  const stages = [
+    ...compositeJsonStages(config),
+    ...compositeTextStages(config),
+  ];
   return (text, document) => {
     for (const stage of stages) {
       const failure = stage(text, document);
@@ -288,29 +360,37 @@ export function prepareOutputChecks(
   declarations: OutputCheckDeclaration[],
 ): PreparedOutputCheck[] {
   const seen = new Set<string>();
-  return declarations.map(({ id, grader, configuration }) => {
-    if (!id || seen.has(id))
-      throw new Error(`duplicate or empty check ID: ${id}`);
-    seen.add(id);
-    if (
-      !configuration ||
-      typeof configuration !== "object" ||
-      Array.isArray(configuration)
-    )
-      throw new Error(`invalid check configuration: ${id}`);
-    const evaluate =
-      grader === "sevro.regex"
-        ? compileRegex(configuration)
-        : grader === "sevro.json"
-          ? compileJson(configuration)
-          : grader === "sevro.schema"
-            ? compileSchema(configuration)
-            : grader === "sevro.output"
-              ? compileCompositeOutput(configuration)
-              : undefined;
-    if (!evaluate) throw new Error(`unsupported output grader: ${grader}`);
-    return { id, grader, evaluate };
-  });
+  return declarations.map((declaration) =>
+    prepareOutputCheck(declaration, seen),
+  );
+}
+
+function requireOutputConfiguration(
+  value: unknown,
+  id: string,
+): asserts value is Record<string, unknown> {
+  if (!isRecord(value)) throw new Error(`invalid check configuration: ${id}`);
+}
+const OUTPUT_COMPILERS = new Map<
+  string,
+  (config: Record<string, unknown>) => PreparedOutputCheck["evaluate"]
+>([
+  ["sevro.regex", compileRegex],
+  ["sevro.json", compileJson],
+  ["sevro.schema", compileSchema],
+  ["sevro.output", compileCompositeOutput],
+]);
+function prepareOutputCheck(
+  { id, grader, configuration }: OutputCheckDeclaration,
+  seen: Set<string>,
+): PreparedOutputCheck {
+  if (!id || seen.has(id))
+    throw new Error(`duplicate or empty check ID: ${id}`);
+  seen.add(id);
+  requireOutputConfiguration(configuration, id);
+  const compiler = OUTPUT_COMPILERS.get(grader);
+  if (!compiler) throw new Error(`unsupported output grader: ${grader}`);
+  return { id, grader, evaluate: compiler(configuration) };
 }
 
 function parseJsonDocument(text: string): ParsedDocument {
@@ -318,11 +398,13 @@ function parseJsonDocument(text: string): ParsedDocument {
     return { value: JSON.parse(text), exact: true, error: false };
   } catch {
     const fences = [...text.matchAll(/```json[ \t]*\r?\n([\s\S]*?)\r?\n```/gi)];
-    if (fences.length !== 1) return { value: null, exact: false, error: true };
+    const [fence] = fences;
+    if (fences.length !== 1 || fence === undefined)
+      return { value: null, exact: false, error: true };
     try {
       return {
-        value: JSON.parse(fences[0]![1]!),
-        exact: text.trim() === fences[0]![0],
+        value: JSON.parse(fence.slice(1, 2).join("")),
+        exact: text.trim() === fence[0],
         error: false,
       };
     } catch {

package/src/graders/semantic.ts

@@ -1,3 +1,4 @@
+import { isRecord, isUnknownArray } from "../value-guards";
 import { validateArtifactPath } from "./artifact";
 export interface SemanticCheckDeclaration {
   id: string;
@@ -29,22 +30,12 @@ export function prepareSemanticChecks(
     if (!id || ids.has(id))
       throw new Error("semantic check IDs must be unique");
     ids.add(id);
-    if (
-      !configuration ||
-      Object.keys(configuration).some(
-        (key) => key !== "proposition" && key !== "artifactPath",
-      ) ||
-      typeof configuration.proposition !== "string" ||
-      !configuration.proposition.trim() ||
-      Buffer.byteLength(configuration.proposition, "utf8") >
-        MAX_PROPOSITION_BYTES
-    )
-      throw new Error("invalid semantic proposition");
+    const proposition = semanticProposition(configuration);
     if (configuration.artifactPath !== undefined)
       validateArtifactPath(configuration.artifactPath);
     return {
       id,
-      proposition: configuration.proposition,
+      proposition,
       ...(configuration.artifactPath === undefined
         ? {}
         : { artifactPath: configuration.artifactPath }),
@@ -52,6 +43,26 @@ export function prepareSemanticChecks(
   });
 }
 
+function validProposition(value: unknown): value is string {
+  return (
+    typeof value === "string" &&
+    Boolean(value.trim()) &&
+    Buffer.byteLength(value, "utf8") <= MAX_PROPOSITION_BYTES
+  );
+}
+
+function semanticProposition(value: unknown): string {
+  if (
+    !isRecord(value) ||
+    Object.keys(value).some(
+      (key) => key !== "proposition" && key !== "artifactPath",
+    ) ||
+    !validProposition(value.proposition)
+  )
+    throw new Error("invalid semantic proposition");
+  return value.proposition;
+}
+
 function safeJson(value: unknown): string {
   return JSON.stringify(value)
     .replaceAll("&", "\\u0026")
@@ -84,7 +95,7 @@ Return exactly one JSON object with one result for each ID and no others:
 function parseDocument(text: string): unknown {
   const trimmed = text.trim();
   const fenced = /^```(?:json)?\s*([\s\S]*?)\s*```$/i.exec(trimmed);
-  return JSON.parse(fenced ? fenced[1]! : trimmed);
+  return JSON.parse(fenced?.[1] ?? trimmed);
 }
 
 /** Require one bounded verdict for each declared proposition. */
@@ -97,48 +108,96 @@ export function parseSemanticVerdicts(
   let value: unknown;
   try {
     value = parseDocument(text);
-  } catch {
-    throw new Error("semantic grader did not return JSON");
+  } catch (cause) {
+    throw new Error("semantic grader did not return JSON", { cause });
   }
-  if (!value || typeof value !== "object" || Array.isArray(value))
-    throw new Error("semantic grader result must be an object");
-  const record = value as Record<string, unknown>;
-  if (Object.keys(record).length !== 1 || !Array.isArray(record.checks))
-    throw new Error("semantic grader result must contain only checks");
+  const entries = semanticVerdictEntries(value);
   const declared = new Set(checks.map((item) => item.id));
   const seen = new Set<string>();
-  const verdicts: SemanticVerdict[] = record.checks.map((item: unknown) => {
-    if (!item || typeof item !== "object" || Array.isArray(item))
-      throw new Error("invalid semantic verdict");
-    const entry = item as Record<string, unknown>;
-    if (
-      Object.keys(entry).some(
-        (key) => key !== "id" && key !== "verdict" && key !== "reason",
-      ) ||
-      typeof entry.id !== "string" ||
-      !declared.has(entry.id) ||
-      seen.has(entry.id) ||
-      (entry.verdict !== "pass" && entry.verdict !== "fail") ||
-      typeof entry.reason !== "string" ||
-      !entry.reason.trim() ||
-      entry.reason.length > 4096
-    )
-      throw new Error("invalid semantic verdict");
-    seen.add(entry.id);
-    return entry as unknown as SemanticVerdict;
-  });
+  const verdicts = entries.map((item) =>
+    validatedSemanticVerdict(item, declared, seen),
+  );
   if (seen.size !== declared.size)
     throw new Error("semantic grader omitted a declared check");
   return verdicts;
 }
 
+function semanticVerdictEntries(value: unknown): unknown[] {
+  if (!isRecord(value))
+    throw new Error("semantic grader result must be an object");
+  if (Object.keys(value).length !== 1 || !isUnknownArray(value.checks))
+    throw new Error("semantic grader result must contain only checks");
+  return value.checks;
+}
+
+function semanticVerdictRecord(value: unknown): Record<string, unknown> {
+  if (!isRecord(value)) throw new Error("invalid semantic verdict");
+  return value;
+}
+
+function validVerdictKeys(entry: Record<string, unknown>): boolean {
+  return !Object.keys(entry).some(
+    (key) => key !== "id" && key !== "verdict" && key !== "reason",
+  );
+}
+
+function validVerdictId(
+  value: unknown,
+  declared: Set<string>,
+  seen: Set<string>,
+): value is string {
+  return typeof value === "string" && declared.has(value) && !seen.has(value);
+}
+
+function validVerdict(value: unknown): value is "pass" | "fail" {
+  return value === "pass" || value === "fail";
+}
+
+function validVerdictReason(value: unknown): value is string {
+  return (
+    typeof value === "string" && Boolean(value.trim()) && value.length <= 4096
+  );
+}
+
+function assertSemanticVerdict(
+  entry: Record<string, unknown>,
+  declared: Set<string>,
+  seen: Set<string>,
+): asserts entry is Record<string, unknown> & SemanticVerdict {
+  if (
+    !validVerdictKeys(entry) ||
+    !validVerdictId(entry.id, declared, seen) ||
+    !validVerdict(entry.verdict) ||
+    !validVerdictReason(entry.reason)
+  )
+    throw new Error("invalid semantic verdict");
+}
+
+function validatedSemanticVerdict(
+  value: unknown,
+  declared: Set<string>,
+  seen: Set<string>,
+): SemanticVerdict {
+  const entry = semanticVerdictRecord(value);
+  assertSemanticVerdict(entry, declared, seen);
+  seen.add(entry.id);
+  return entry;
+}
+
+export type SemanticCheckGroup = [
+  PreparedSemanticCheck,
+  ...PreparedSemanticCheck[],
+];
+
 export function semanticCheckGroups(
   checks: PreparedSemanticCheck[],
-): PreparedSemanticCheck[][] {
-  const groups = new Map<string, PreparedSemanticCheck[]>();
+): SemanticCheckGroup[] {
+  const groups = new Map<string, SemanticCheckGroup>();
   for (const check of checks) {
     const key = check.artifactPath ?? "";
-    groups.set(key, [...(groups.get(key) ?? []), check]);
+    const existing = groups.get(key);
+    if (existing) existing.push(check);
+    else groups.set(key, [check]);
   }
   return [...groups.values()];
 }