@supersuit/hyperspec 0.1.0 → 0.2.0

package/src/compare.mjs

@@ -0,0 +1,189 @@
+import { appendFileSync, readFileSync } from "node:fs";
+import { spawnSync } from "node:child_process";
+import { dirname, relative, resolve } from "node:path";
+import { sha256 } from "./hash.mjs";
+import { insideDir } from "./fsutil.mjs";
+import { loadSpec } from "./load.mjs";
+import { readRecipe } from "./recipe.mjs";
+
+const present = (v) => typeof v === "string" && v.trim().length > 0;
+const DOCTOR_MAX_BUFFER = 512 * 1024 * 1024;
+
+// Spec requirement `compare`: grade a regenerated output and its parent through ONE doctor
+// against ONE spec, so "the new one should be better" becomes a number, and flag a regression
+// naming the change as the suspect. hyperspec never calls a model: the doctor is a command the
+// caller supplies, run once per output via /bin/sh -c, with no recipe-derived value ever
+// interpolated into the command string itself (only passed on stdin).
+export function compare(childRecipePath, { doctor, parent: parentOption, spec: specOption } = {}) {
+  const empty = () => ({ ok: false, parent: null, child: null, delta: null, regressed: null, suspect: null, specChanged: null, ledger: null, warnings: [] });
+  const usage = (error) => ({ ...empty(), usage: true, error });
+  const failed = (error, extra = {}) => ({ ...empty(), error, ...extra });
+
+  if (!present(doctor)) return usage("a doctor command is required");
+  if (!present(childRecipePath)) return usage("a child recipe path is required");
+
+  // ---- Child -------------------------------------------------------------------------------------
+  const childLoaded = readRecipe(childRecipePath);
+  if (childLoaded.error) return usage(childLoaded.error);
+  const { data: child, dir: childDir } = childLoaded;
+  const childRecipeAbs = resolve(childRecipePath);
+
+  // ---- Parent: defaults to the child's recorded parent.path -------------------------------------
+  let parentPath = parentOption;
+  if (!present(parentPath)) {
+    if (!present(child.parent?.path)) return usage("the child recipe has no parent recorded, and none was given");
+    parentPath = resolve(childDir, child.parent.path);
+  } else {
+    parentPath = resolve(process.cwd(), parentPath);
+  }
+  const parentLoaded = readRecipe(parentPath);
+  if (parentLoaded.error) return usage(parentLoaded.error);
+  const { data: parentRecipe, dir: parentDir } = parentLoaded;
+  const parentRecipeAbs = resolve(parentPath);
+
+  const warnings = [];
+  // The child's recorded parent.sha256 is what the parent recipe's bytes were when the child was
+  // made. Whatever parent recipe is actually being graded against here (the default or an explicit
+  // override) may have moved since, which is worth a warning but never a refusal to compare.
+  if (child.parent && present(child.parent.sha256)) {
+    const parentBytes = readFileSync(parentRecipeAbs);
+    if (sha256(parentBytes) !== child.parent.sha256) {
+      warnings.push("parent recipe changed since the child was made");
+    }
+  }
+
+  // ---- Spec: defaults to the child's spec path, graded ONCE for both outputs --------------------
+  let specPath = specOption;
+  if (!present(specPath)) {
+    if (!present(child.spec?.path)) return usage("the child recipe has no spec path");
+    specPath = resolve(childDir, child.spec.path);
+  } else {
+    specPath = resolve(process.cwd(), specPath);
+  }
+  const specLoaded = loadSpec(specPath);
+  if (specLoaded.error) return usage(specLoaded.error);
+  const specAbs = specLoaded.path;
+  const specHash = sha256(readFileSync(specAbs));
+  // specChanged is about the PARENT's record, not the child's: it tells the caller the spec being
+  // graded against now differs from what the parent was made under. Both outputs still get graded
+  // against this one spec file either way — never each against its own.
+  const specChanged = parentRecipe.spec?.sha256 !== specHash;
+  if (specChanged) warnings.push("spec changed since the parent was made; both outputs graded against the current file");
+
+  // ---- Output files, resolved against each recipe's own directory --------------------------------
+  const childOutputPath = child.output?.path;
+  if (!present(childOutputPath)) return failed("the child recipe has no output path");
+  if (!insideDir(childDir, childOutputPath)) return failed("the child recipe's output path escapes its recipe directory");
+  const childOutputAbs = resolve(childDir, childOutputPath);
+  let childOutputBytes;
+  try { childOutputBytes = readFileSync(childOutputAbs); } catch { return failed(`cannot read the child's output: ${childOutputPath}`); }
+
+  const parentOutputPath = parentRecipe.output?.path;
+  if (!present(parentOutputPath)) return failed("the parent recipe has no output path");
+  if (!insideDir(parentDir, parentOutputPath)) return failed("the parent recipe's output path escapes its recipe directory");
+  const parentOutputAbs = resolve(parentDir, parentOutputPath);
+  let parentOutputBytes;
+  try { parentOutputBytes = readFileSync(parentOutputAbs); } catch { return failed(`cannot read the parent's output: ${parentOutputPath}`); }
+
+  // A score is attributed to a recipe, so the bytes graded must be the bytes that recipe records.
+  // A file edited by hand since (or never the recipe's output at all) would turn a hand edit into
+  // a regression or an improvement blamed on the change, and write that into the ledger. Refuse
+  // before the doctor runs; reproduce --restore puts the recorded bytes back.
+  const stale = (bytes, recipe) => sha256(bytes) !== recipe.output?.sha256;
+  if (stale(parentOutputBytes, parentRecipe)) return failed("parent output does not match its recipe; run hyperspec reproduce --restore");
+  if (stale(childOutputBytes, child)) return failed("child output does not match its recipe; run hyperspec reproduce --restore");
+
+  // ---- Grade, same doctor command for both, same spec for both -----------------------------------
+  const parentGraded = runDoctor(doctor, parentOutputAbs, specAbs);
+  if (parentGraded.error) return usage(`parent: ${parentGraded.error}`);
+  const childGraded = runDoctor(doctor, childOutputAbs, specAbs);
+  if (childGraded.error) return usage(`child: ${childGraded.error}`);
+
+  const delta = childGraded.score - parentGraded.score;
+  const regressed = childGraded.score < parentGraded.score;
+  const suspect = regressed ? (child.change ?? null) : null;
+
+  // ---- Ledger: one line of evidence for the self-upgrade loop, only if the spec declares one -----
+  let ledgerPath = null;
+  const ledgerDecl = specLoaded.data?.improvement?.ledger;
+  if (present(ledgerDecl)) {
+    const specDir = specLoaded.dir;
+    if (!insideDir(specDir, ledgerDecl)) {
+      warnings.push("improvement.ledger escapes the spec's directory; not appended");
+    } else {
+      const ledgerAbs = resolve(specDir, ledgerDecl);
+      const ledgerDir = dirname(ledgerAbs);
+      // The child's own `change` is null whenever it has no genealogical parent of its own (a root
+      // recipe compared against an explicit, unrelated --parent — compare's usage check allows
+      // this: it only requires *either* the child's recorded parent.path *or* an explicit
+      // override). Fall back to naming what was actually compared against, so `change` and the
+      // not-improved `reason` below are never "after null" — both a broken persisted record and,
+      // for the "improved" verdict, a lint failure (rules.mjs's verdict-change: `!str(v.change)`).
+      const childChange = child.change ?? `compared against ${relative(ledgerDir, parentRecipeAbs)}`;
+      // A compare line must satisfy test 9 ("it improves itself"), which only knows the
+      // verdict vocabulary one-shot/improved/not-improved — never a new "compare" verdict. A
+      // strictly higher child score is improved (and already carries change, which doubles as
+      // that verdict's required field). Equal or lower is not-improved, with a reason a later
+      // session can argue with; when it's a genuine regression (strictly lower, not merely tied)
+      // the reason is prefixed to say so.
+      const line = {
+        at: new Date().toISOString(),
+        kind: "compare",
+        parent: relative(ledgerDir, parentRecipeAbs),
+        child: relative(ledgerDir, childRecipeAbs),
+        scores: { parent: parentGraded.score, child: childGraded.score },
+        regressed,
+        change: childChange,
+      };
+      if (childGraded.score > parentGraded.score) {
+        line.verdict = "improved";
+      } else {
+        line.verdict = "not-improved";
+        const reasonBase = `compare: child scored ${childGraded.score} vs parent ${parentGraded.score} after ${childChange}`;
+        line.reason = regressed ? `regression: ${reasonBase}` : reasonBase;
+      }
+      appendFileSync(ledgerAbs, `${JSON.stringify(line)}\n`);
+      ledgerPath = ledgerAbs;
+    }
+  }
+
+  return {
+    ok: true,
+    parent: { recipe: parentRecipeAbs, output: parentOutputAbs, score: parentGraded.score, notes: parentGraded.notes },
+    child: { recipe: childRecipeAbs, output: childOutputAbs, score: childGraded.score, notes: childGraded.notes },
+    delta,
+    regressed,
+    suspect,
+    specChanged,
+    ledger: ledgerPath,
+    warnings,
+  };
+}
+
+// Runs the doctor command once, via /bin/sh -c, over one output against the one spec. stdin is
+// { output, spec } (absolute paths); the last non-empty stdout line must be JSON with a numeric
+// score. Nothing recipe-derived is ever interpolated into the command string — only passed on
+// stdin — so the same command string is reused verbatim for the parent and the child.
+function runDoctor(doctorCmd, outputAbs, specAbs) {
+  const res = spawnSync("/bin/sh", ["-c", doctorCmd], {
+    input: JSON.stringify({ output: outputAbs, spec: specAbs }),
+    maxBuffer: DOCTOR_MAX_BUFFER,
+  });
+  if (res.error) return { error: `doctor could not run: ${res.error.message}` };
+  const stderr = res.stderr ? res.stderr.toString("utf8") : "";
+  if (res.status !== 0) {
+    const how = res.signal ? `was killed by ${res.signal}` : `exited ${res.status}`;
+    const tail = stderr.trim().split("\n").slice(-5).join("\n");
+    return { error: `doctor ${how}${tail ? `: ${tail}` : ""}` };
+  }
+  const stdout = res.stdout ? res.stdout.toString("utf8") : "";
+  const lines = stdout.split(/\r?\n/).filter((l) => l.trim().length > 0);
+  if (!lines.length) return { error: "doctor produced no output" };
+  const last = lines[lines.length - 1].trim();
+  let value;
+  try { value = JSON.parse(last); } catch (e) { return { error: `doctor's last line is not JSON: ${e.message}` }; }
+  if (!value || typeof value !== "object" || Array.isArray(value) || typeof value.score !== "number" || !Number.isFinite(value.score)) {
+    return { error: "doctor's last line must be an object with a numeric score" };
+  }
+  return { score: value.score, notes: typeof value.notes === "string" ? value.notes : "" };
+}

package/src/fsutil.mjs

@@ -0,0 +1,33 @@
+import { renameSync, unlinkSync, writeFileSync } from "node:fs";
+import { randomBytes } from "node:crypto";
+import { relative, resolve, sep } from "node:path";
+
+// True when `target`, resolved against `dir`, stays inside `dir` — refuses a `..`-escape and an
+// absolute path pointing elsewhere. Both a relative `target` (including one that climbs out via
+// `../`) and an already-absolute `target` are handled the same way, since `path.resolve(dir,
+// target)` already treats an absolute second argument as overriding the first: either way, the
+// resolved path is compared against `dir` by `path.relative`, and anything that needs a leading
+// `..` segment to get there is outside.
+export function insideDir(dir, target) {
+  const dirAbs = resolve(dir);
+  const targetAbs = resolve(dir, target);
+  const rel = relative(dirAbs, targetAbs);
+  return rel === "" || (rel !== ".." && !rel.startsWith(`..${sep}`));
+}
+
+// Writes `bytes` to `path` atomically: a temp file in the same directory, then a single
+// renameSync (atomic on one filesystem) onto the final path. Same pattern as blobs.mjs's
+// putBlob, minus putBlob's content-addressed dedup (this always writes/overwrites the given
+// path; it isn't keyed by the bytes' own hash, so there's nothing to skip). Any failure along
+// the way removes the temp file before rethrowing, so a crash mid-write never leaves the target
+// path holding a partial write, and never leaves an orphaned temp file behind either.
+export function writeFileAtomic(path, bytes) {
+  const tmp = `${path}.tmp-${process.pid}-${randomBytes(6).toString("hex")}`;
+  try {
+    writeFileSync(tmp, bytes);
+    renameSync(tmp, path);
+  } catch (e) {
+    try { unlinkSync(tmp); } catch { /* nothing to clean up */ }
+    throw e;
+  }
+}

package/src/hash.mjs

@@ -0,0 +1,26 @@
+import { createHash } from "node:crypto";
+
+// SHA-256, lowercase hex, over exact bytes. Accepts a Buffer or a string.
+export function sha256(bufferOrString) {
+  return createHash("sha256").update(bufferOrString).digest("hex");
+}
+
+// Canonical JSON for anything about to be hashed: keys sorted recursively,
+// no whitespace, undefined keys dropped. Array order is preserved, since
+// order is meaningful there (inputs, stages, reads).
+export function canonical(value) {
+  return JSON.stringify(sortKeys(value));
+}
+
+function sortKeys(value) {
+  if (Array.isArray(value)) return value.map(sortKeys);
+  if (value && typeof value === "object") {
+    const out = {};
+    for (const key of Object.keys(value).sort()) {
+      if (value[key] === undefined) continue;
+      out[key] = sortKeys(value[key]);
+    }
+    return out;
+  }
+  return value;
+}

package/src/recipe.mjs

@@ -0,0 +1,164 @@
+import { readFileSync } from "node:fs";
+import { dirname, resolve } from "node:path";
+import { canonical, sha256 } from "./hash.mjs";
+import { writeFileAtomic } from "./fsutil.mjs";
+import { isSha256 } from "./blobs.mjs";
+
+const present = (v) => typeof v === "string" && v.trim().length > 0;
+const NOT_HEX = "is not a SHA-256 hash (64 lowercase hex characters)";
+
+export function readRecipe(path) {
+  const abs = resolve(path);
+  let text;
+  try { text = readFileSync(abs, "utf8"); } catch { return { error: `cannot read ${path}` }; }
+  let data;
+  try { data = JSON.parse(text); } catch (e) { return { error: `invalid JSON in ${path}: ${e.message}` }; }
+  return { data, dir: dirname(abs) };
+}
+
+// Pretty-printed (2-space indent) with a trailing newline, per the recipe file convention.
+// Atomic (temp file + rename), so a crash mid-write never leaves a half-written recipe.
+export function writeRecipe(path, data) {
+  writeFileAtomic(path, `${JSON.stringify(data, null, 2)}\n`);
+}
+
+// A stage with a declared, non-empty `reads` resolves exactly those refs. An empty (or missing)
+// `reads` means "read everything": every input in order, then every earlier stage in array
+// order, then spec (spec decision downstream-rerun-detection).
+export function resolveReads(recipe, stageIndex) {
+  const stages = Array.isArray(recipe.stages) ? recipe.stages : [];
+  const stage = stages[stageIndex];
+  if (!stage) throw new Error(`no stage at index ${stageIndex}`);
+  const declared = Array.isArray(stage.reads) && stage.reads.length ? stage.reads : expandReads(recipe, stageIndex);
+  return declared.map((ref) => [ref, resolveRef(recipe, stageIndex, ref)]);
+}
+
+function expandReads(recipe, stageIndex) {
+  const inputs = [...(Array.isArray(recipe.inputs) ? recipe.inputs : [])].sort((a, b) => (a.order ?? 0) - (b.order ?? 0));
+  const stages = Array.isArray(recipe.stages) ? recipe.stages : [];
+  return [
+    ...inputs.map((i) => `input:${i.name}`),
+    ...stages.slice(0, stageIndex).map((s) => `stage:${s.id}`),
+    "spec",
+  ];
+}
+
+function resolveRef(recipe, stageIndex, ref) {
+  if (ref === "spec") {
+    const hex = recipe.spec?.sha256;
+    if (!present(hex)) throw new Error(`stage ${stageIndex}: spec has no sha256`);
+    return hex;
+  }
+  if (typeof ref === "string" && ref.startsWith("input:")) {
+    const name = ref.slice("input:".length);
+    const input = (Array.isArray(recipe.inputs) ? recipe.inputs : []).find((i) => i.name === name);
+    if (!input) throw new Error(`stage ${stageIndex}: unknown read "${ref}"`);
+    return input.sha256;
+  }
+  if (typeof ref === "string" && ref.startsWith("stage:")) {
+    const id = ref.slice("stage:".length);
+    const stages = Array.isArray(recipe.stages) ? recipe.stages : [];
+    const idx = stages.findIndex((s) => s.id === id);
+    if (idx === -1) throw new Error(`stage ${stageIndex}: unknown read "${ref}"`);
+    if (idx >= stageIndex) throw new Error(`stage ${stageIndex}: "${ref}" is not an earlier stage`);
+    return stages[idx].output?.sha256;
+  }
+  throw new Error(`stage ${stageIndex}: unknown read "${ref}"`);
+}
+
+// sha256(canonical({ reads, spec, factory, model })) where reads is the resolved [ref, hex] list
+// in declared (or expanded) order, spec is spec.sha256, factory is factory.version, and model is
+// the stage's own model settings or null.
+export function stageKey(recipe, stageIndex) {
+  const stages = Array.isArray(recipe.stages) ? recipe.stages : [];
+  const stage = stages[stageIndex];
+  if (!stage) throw new Error(`no stage at index ${stageIndex}`);
+  const reads = resolveReads(recipe, stageIndex);
+  return sha256(canonical({
+    reads,
+    spec: recipe.spec?.sha256,
+    factory: recipe.factory?.version,
+    model: stage.model ?? null,
+  }));
+}
+
+// Implements spec requirement recipe-completeness: every fail below is a field the recipe must
+// carry for it to count as done; the one warn flags a stage that will rerun on every regeneration
+// because it declared nothing.
+export function checkRecipe(recipe) {
+  const out = [];
+  const fail = (field, message) => out.push({ severity: "fail", field, message });
+  const warn = (field, message) => out.push({ severity: "warn", field, message });
+
+  if (!present(recipe.factory?.name)) fail("factory.name", "factory.name is missing");
+  if (!present(recipe.factory?.version)) fail("factory.version", "factory.version is missing");
+  if (!present(recipe.spec?.sha256)) fail("spec.sha256", "spec.sha256 is missing");
+  else if (!isSha256(recipe.spec.sha256)) fail("spec.sha256", `spec.sha256 ${NOT_HEX}`);
+  if (present(recipe.output?.sha256) && !isSha256(recipe.output.sha256)) fail("output.sha256", `output.sha256 ${NOT_HEX}`);
+  const authors = recipe.spec?.authors;
+  if (!authors || typeof authors !== "object" || Array.isArray(authors) || Object.keys(authors).length === 0) {
+    fail("spec.authors", "spec.authors is missing");
+  } else {
+    // The writer records null for a spec entry with no author rather than dropping the id, so the
+    // gap stays visible here instead of disappearing from the map.
+    for (const [id, author] of Object.entries(authors)) {
+      if (!present(author)) warn(`spec.authors.${id}`, `spec id "${id}" has no author`);
+    }
+  }
+  if (!present(recipe.clicker)) fail("clicker", "clicker is missing");
+  if (!present(recipe.approver)) fail("approver", "approver is missing");
+
+  const parentSet = recipe.parent !== null && recipe.parent !== undefined;
+  const changeSet = recipe.change !== null && recipe.change !== undefined;
+  if (parentSet !== changeSet) fail("parent", "parent and change must both be null or both be set");
+  if (parentSet && present(recipe.parent?.sha256) && !isSha256(recipe.parent.sha256)) fail("parent.sha256", `parent.sha256 ${NOT_HEX}`);
+
+  const inputs = Array.isArray(recipe.inputs) ? recipe.inputs : [];
+  inputs.forEach((input, i) => {
+    if (!present(input?.sha256)) fail(`inputs[${i}].sha256`, `input "${input?.name ?? i}" has no sha256`);
+    else if (!isSha256(input.sha256)) fail(`inputs[${i}].sha256`, `input "${input?.name ?? i}" sha256 ${NOT_HEX}`);
+  });
+
+  const stages = Array.isArray(recipe.stages) ? recipe.stages : [];
+  if (stages.length === 0) fail("stages", "a recipe needs at least one stage");
+  stages.forEach((stage, i) => {
+    const id = stage?.id ?? `#${i}`;
+
+    if (!stage?.verdict || typeof stage.verdict !== "object") {
+      fail(`stages[${i}].verdict`, `stage "${id}" has no verdict`);
+    } else if (typeof stage.verdict.pass !== "boolean") {
+      fail(`stages[${i}].verdict.pass`, `stage "${id}" verdict.pass is not a boolean`);
+    }
+
+    // pending: true, or a null/incomplete output, makes the stage incomplete by definition.
+    if (stage?.pending === true || stage?.output == null || !present(stage.output?.sha256)) {
+      fail(`stages[${i}]`, `stage ${id} is pending`);
+    } else if (!isSha256(stage.output.sha256)) {
+      fail(`stages[${i}].output.sha256`, `stage "${id}" output.sha256 ${NOT_HEX}`);
+    }
+
+    if (!Array.isArray(stage?.reads) || stage.reads.length === 0) {
+      warn(`stages[${i}].reads`, `stage "${id}" declares no reads, so it reruns on every regeneration; declare what it reads`);
+    }
+
+    try {
+      const recomputed = stageKey(recipe, i);
+      if (stage?.key !== recomputed) fail(`stages[${i}].key`, `stage "${id}" key does not match its recomputed key`);
+    } catch (e) {
+      // A bad ref is why the key can't be recomputed, but this is still a key-completeness
+      // failure, not the reads-declaration warn: keep it on its own field so a caller grouping
+      // or deduping findings by field can't collapse a blocking fail into an informational warn.
+      fail(`stages[${i}].key`, `stage "${id}" key could not be recomputed: ${e.message}`);
+    }
+  });
+
+  if (stages.length) {
+    const last = stages[stages.length - 1];
+    const lastHash = last?.output?.sha256;
+    if (present(lastHash) && lastHash !== recipe.output?.sha256) {
+      fail("output.sha256", "the last stage's output does not match output.sha256");
+    }
+  }
+
+  return out;
+}