full-reverse 0.0.0-stage → 1.0.0

package/skills/full-reverse/scripts/lib/selectors.mjs

@@ -0,0 +1,256 @@
+// Evaluation of evidence selectors against saved files.
+// Outcomes are verified | failed | unverified; syntax-only checks are never "verified".
+import fsp from "node:fs/promises";
+import fs from "node:fs";
+import { scanFile } from "./jsonscan.mjs";
+import { pointerSegments, resolvePointer } from "./rea.mjs";
+import { canonicalDigest, jsonEqual } from "./util.mjs";
+
+const verified = (extra = {}) => ({ status: "verified", ...extra });
+const failed = (reason, extra = {}) => ({ status: "failed", reason, ...extra });
+const unverified = (reason, extra = {}) => ({ status: "unverified", reason, ...extra });
+
+/** Canonical-record pointer store: parses small files once, scans large ones for requested pointers. */
+export class PointerStore {
+  constructor({ maxParseBytes, maxCaptureBytes = 1 << 20 }) {
+    this.maxParseBytes = maxParseBytes;
+    this.maxCaptureBytes = maxCaptureBytes;
+    this.docs = new Map(); // abs -> { doc }
+    this.scans = new Map(); // abs -> { balanced, captures }
+    this.failures = new Map(); // abs -> reason
+  }
+
+  /** requests: Map(abs -> { bytes, pointers:Set<string> }) */
+  async prepare(requests) {
+    for (const [abs, request] of requests) {
+      if (request.bytes <= this.maxParseBytes) continue; // parsed lazily in get()
+      const pointers = [...request.pointers].filter((p) => {
+        const segments = pointerSegments(p);
+        return segments !== null && !segments.includes("*");
+      });
+      try {
+        const scan = await scanFile(abs, pointers, { maxCaptureBytes: this.maxCaptureBytes });
+        this.scans.set(abs, scan);
+      } catch (error) {
+        this.failures.set(abs, `scan_failed:${error.code ?? error.message}`);
+      }
+    }
+  }
+
+  async doc(abs) {
+    if (this.docs.has(abs)) return this.docs.get(abs);
+    let value;
+    try {
+      value = JSON.parse(await fsp.readFile(abs, "utf8"));
+    } catch (error) {
+      value = { error: error.message };
+    }
+    if (this.docs.size >= 2) this.docs.delete(this.docs.keys().next().value);
+    const holder = { doc: value };
+    this.docs.set(abs, holder);
+    return holder;
+  }
+
+  /** Look up the absolute pointer (record pointer + selector pointer) in a file. */
+  async lookup(abs, bytes, absolutePointer) {
+    if (pointerSegments(absolutePointer) === null) return { kind: "invalid" };
+    if (bytes <= this.maxParseBytes) {
+      const { doc } = await this.doc(abs);
+      if (doc && doc.error && Object.keys(doc).length === 1) return { kind: "unparsable" };
+      const result = resolvePointer(doc, absolutePointer);
+      return result.found ? { kind: "found", value: result.value } : { kind: "missing" };
+    }
+    if (this.failures.has(abs)) return { kind: "unverified", reason: this.failures.get(abs) };
+    const scan = this.scans.get(abs);
+    if (!scan) return { kind: "unverified", reason: "file_exceeds_parse_limit" };
+    if (pointerSegments(absolutePointer).includes("*")) return { kind: "unverified", reason: "wildcard_pointer_unsupported" };
+    if (!scan.balanced) return { kind: "unverified", reason: "json_structure_incomplete" };
+    const capture = scan.captures.get(absolutePointer);
+    if (!capture) return { kind: "missing" };
+    if (capture.oversized || capture.text === null) return { kind: "unverified", reason: "value_exceeds_capture_limit" };
+    try {
+      return { kind: "found", value: JSON.parse(capture.text) };
+    } catch {
+      return { kind: "unverified", reason: "captured_value_unparsable" };
+    }
+  }
+}
+
+/** Evaluate one json-pointer selector against a looked-up value. */
+export function judgePointer(selector, lookup) {
+  switch (lookup.kind) {
+    case "invalid":
+      return failed("invalid_rfc6901_pointer");
+    case "unparsable":
+      return unverified("evidence_file_unparsable");
+    case "unverified":
+      return unverified(lookup.reason);
+    case "missing":
+      return failed("pointer_not_found");
+    default:
+      break;
+  }
+  const expect = selector.expect;
+  if (!expect) return verified({ detail: "pointer resolves; no expectation stated" });
+  if ("exists" in expect) return verified({ detail: "pointer resolves" });
+  if ("equals" in expect) {
+    return jsonEqual(lookup.value, expect.equals) ? verified({ detail: "value equals expectation" }) : failed("expect_equals_mismatch");
+  }
+  if ("sha256" in expect) {
+    let digest;
+    try {
+      digest = canonicalDigest(lookup.value);
+    } catch {
+      return unverified("value_not_canonicalizable");
+    }
+    return digest === expect.sha256 ? verified({ detail: "value digest matches" }) : failed("expect_sha256_mismatch");
+  }
+  return unverified("unsupported_expectation");
+}
+
+// --- external selectors ------------------------------------------------------------
+
+async function countLines(abs) {
+  let lines = 0;
+  let nonEmpty = 0;
+  let lineHasContent = false;
+  let lastByte = 0x0a;
+  for await (const chunk of fs.createReadStream(abs, { highWaterMark: 1 << 20 })) {
+    for (let i = 0; i < chunk.length; i += 1) {
+      const b = chunk[i];
+      if (b === 0x0a) {
+        lines += 1;
+        if (lineHasContent) nonEmpty += 1;
+        lineHasContent = false;
+      } else if (b !== 0x0d && b !== 0x20 && b !== 0x09) lineHasContent = true;
+      lastByte = b;
+    }
+  }
+  if (lastByte !== 0x0a) {
+    lines += 1;
+    if (lineHasContent) nonEmpty += 1;
+  }
+  return { lines, nonEmpty };
+}
+
+async function imageSize(abs) {
+  const handle = await fsp.open(abs, "r");
+  try {
+    const head = Buffer.alloc(1 << 20);
+    const { bytesRead } = await handle.read(head, 0, head.length, 0);
+    const buf = head.subarray(0, bytesRead);
+    if (buf.length >= 24 && buf.subarray(0, 8).equals(Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a])) && buf.toString("ascii", 12, 16) === "IHDR") {
+      return { width: buf.readUInt32BE(16), height: buf.readUInt32BE(20), format: "png" };
+    }
+    if (buf.length >= 10 && (buf.toString("ascii", 0, 6) === "GIF87a" || buf.toString("ascii", 0, 6) === "GIF89a")) {
+      return { width: buf.readUInt16LE(6), height: buf.readUInt16LE(8), format: "gif" };
+    }
+    if (buf.length > 4 && buf[0] === 0xff && buf[1] === 0xd8) {
+      let offset = 2;
+      while (offset + 9 < buf.length) {
+        if (buf[offset] !== 0xff) {
+          offset += 1;
+          continue;
+        }
+        const marker = buf[offset + 1];
+        if (marker === 0xff) {
+          offset += 1;
+          continue;
+        }
+        if (marker === 0xd8 || marker === 0x01 || (marker >= 0xd0 && marker <= 0xd7)) {
+          offset += 2;
+          continue;
+        }
+        const length = buf.readUInt16BE(offset + 2);
+        if ((marker >= 0xc0 && marker <= 0xcf) && marker !== 0xc4 && marker !== 0xc8 && marker !== 0xcc) {
+          return { height: buf.readUInt16BE(offset + 5), width: buf.readUInt16BE(offset + 7), format: "jpeg" };
+        }
+        offset += 2 + length;
+      }
+    }
+    return null;
+  } finally {
+    await handle.close();
+  }
+}
+
+/** Lazily computed, cached facts about an external file. */
+export class ExternalFacts {
+  constructor({ maxParseBytes }) {
+    this.maxParseBytes = maxParseBytes;
+    this.cache = new Map();
+  }
+
+  async facts(abs, bytes, mediaType) {
+    if (this.cache.has(abs)) return this.cache.get(abs);
+    const facts = { bytes, mediaType, lines: null, image: null, jsonArrayLength: null, jsonlEventCount: null, jsonlReason: null };
+    const sniff = Buffer.alloc(Math.min(8192, bytes));
+    if (bytes > 0) {
+      const handle = await fsp.open(abs, "r");
+      try {
+        await handle.read(sniff, 0, sniff.length, 0);
+      } finally {
+        await handle.close();
+      }
+    }
+    facts.text = !sniff.includes(0);
+    if (facts.text) facts.lines = bytes > 0 ? await countLines(abs) : { lines: 0, nonEmpty: 0 };
+    facts.image = await imageSize(abs).catch(() => null);
+    if (facts.text && bytes > 0 && bytes <= this.maxParseBytes && /\.json$/i.test(abs)) {
+      try {
+        const doc = JSON.parse(await fsp.readFile(abs, "utf8"));
+        if (Array.isArray(doc)) facts.jsonArrayLength = doc.length;
+      } catch {
+        /* not JSON */
+      }
+    }
+    if (facts.text && /\.(jsonl|ndjson)$/i.test(abs)) {
+      if (bytes > this.maxParseBytes) facts.jsonlReason = "event_parse_limit_exceeded";
+      else {
+        try {
+          const lines = (await fsp.readFile(abs, "utf8")).split("\n").filter((line) => line.trim() !== "");
+          for (const line of lines) JSON.parse(line);
+          facts.jsonlEventCount = lines.length;
+        } catch {
+          facts.jsonlReason = "invalid_jsonl_event";
+        }
+      }
+    }
+    this.cache.set(abs, facts);
+    return facts;
+  }
+}
+
+export async function judgeExternal(selector, abs, entry, externalFacts) {
+  const facts = await externalFacts.facts(abs, entry.bytes, entry.external?.media_type ?? null);
+  switch (selector.type) {
+    case "whole-file":
+      return verified({ detail: "file digest matches" });
+    case "byte-range":
+      if (selector.start >= selector.end) return failed("empty_or_inverted_range");
+      return selector.end <= facts.bytes ? verified({ detail: `file has ${facts.bytes} bytes` }) : failed("range_beyond_file_size");
+    case "line-range":
+      if (!facts.text) return unverified("file_is_not_text");
+      if (selector.start > selector.end) return failed("inverted_range");
+      return selector.end <= facts.lines.lines ? verified({ detail: `file has ${facts.lines.lines} lines` }) : failed("range_beyond_line_count");
+    case "image-region":
+      if (!facts.image) return unverified("image_format_unsupported_for_bounds_check");
+      return selector.x + selector.width <= facts.image.width && selector.y + selector.height <= facts.image.height
+        ? verified({ detail: `${facts.image.format} ${facts.image.width}x${facts.image.height}` })
+        : failed("region_outside_image");
+    case "event-ordinal": {
+      if (/\.(jsonl|ndjson)$/i.test(abs) && facts.lines) {
+        if (facts.jsonlEventCount === null) return unverified(facts.jsonlReason ?? "event_structure_unknown");
+        return selector.ordinal < facts.jsonlEventCount ? verified({ detail: `${facts.jsonlEventCount} events` }) : failed("ordinal_beyond_event_count");
+      }
+      if (facts.jsonArrayLength !== null) {
+        return selector.ordinal < facts.jsonArrayLength ? verified({ detail: `${facts.jsonArrayLength} events` }) : failed("ordinal_beyond_event_count");
+      }
+      return unverified("event_structure_unknown");
+    }
+    case "time-range":
+      return unverified("no_time_base_for_file_type");
+    default:
+      return unverified("unsupported_selector_type");
+  }
+}

package/skills/full-reverse/scripts/lib/util.mjs

@@ -0,0 +1,333 @@
+// Shared low-level helpers: errors, hashing, canonical JSON, safe paths, atomic writes.
+import { createHash, randomBytes } from "node:crypto";
+import fs from "node:fs";
+import fsp from "node:fs/promises";
+import path from "node:path";
+
+export const EXIT = Object.freeze({
+  OK: 0,
+  FAILED: 1,
+  USAGE: 2,
+  PREREQUISITE: 3,
+  BLOCKED: 4,
+  IO: 5,
+});
+
+/** Error carrying a CLI exit code and a stable machine-readable code. */
+export class CliError extends Error {
+  constructor(message, { exit = EXIT.FAILED, code = "error", details } = {}) {
+    super(message);
+    this.name = "CliError";
+    this.exit = exit;
+    this.code = code;
+    this.details = details;
+  }
+}
+
+export const usage = (message) => new CliError(message, { exit: EXIT.USAGE, code: "usage" });
+
+export const nowIso = () => new Date().toISOString();
+
+/**
+ * Open a path for reading only if it is a regular file. The path is stat-ed
+ * first (no open side effects on devices) and the opened descriptor is checked
+ * again, so a FIFO/device swapped in after the first check is refused and
+ * O_NONBLOCK keeps the open itself from blocking on a FIFO.
+ */
+export async function openRegularFile(file) {
+  const before = await fsp.stat(file);
+  if (!before.isFile()) throw new CliError(`not a regular file: ${file}`, { code: "not_regular_file" });
+  const handle = await fsp.open(file, fs.constants.O_RDONLY | fs.constants.O_NONBLOCK);
+  try {
+    const stat = await handle.stat();
+    if (!stat.isFile()) throw new CliError(`not a regular file: ${file}`, { code: "not_regular_file" });
+    return { handle, stat };
+  } catch (error) {
+    await handle.close();
+    throw error;
+  }
+}
+
+/** Stream a regular file through SHA-256 without loading it; refuses FIFOs, devices and directories. */
+export async function hashFile(file) {
+  const { handle } = await openRegularFile(file);
+  const hash = createHash("sha256");
+  let bytes = 0;
+  const stream = handle.createReadStream({ highWaterMark: 1 << 20 });
+  for await (const chunk of stream) {
+    hash.update(chunk);
+    bytes += chunk.length;
+  }
+  return { sha256: hash.digest("hex"), bytes };
+}
+
+export const sha256Buffer = (buffer) => createHash("sha256").update(buffer).digest("hex");
+
+/**
+ * REA canonical JSON: object keys sorted by UTF-16 code unit, scalars through
+ * JSON.stringify, no whitespace. Matches REA's own digestCanonicalValue; values
+ * must be JSON data (finite numbers, no undefined).
+ */
+export function canonicalJson(value) {
+  if (value === null || typeof value !== "object") {
+    if (typeof value === "number" && !Number.isFinite(value)) throw new TypeError("non-finite number");
+    const encoded = JSON.stringify(value);
+    if (encoded === undefined) throw new TypeError("unsupported value");
+    return encoded;
+  }
+  if (Array.isArray(value)) return `[${value.map((item) => canonicalJson(item === undefined ? null : item)).join(",")}]`;
+  const parts = [];
+  for (const key of Object.keys(value).sort()) {
+    if (value[key] === undefined) continue;
+    parts.push(`${JSON.stringify(key)}:${canonicalJson(value[key])}`);
+  }
+  return `{${parts.join(",")}}`;
+}
+
+export const canonicalDigest = (value) => createHash("sha256").update(canonicalJson(value)).digest("hex");
+
+/** Deep JSON equality through canonical form. */
+export const jsonEqual = (a, b) => {
+  try {
+    return canonicalJson(a) === canonicalJson(b);
+  } catch {
+    return false;
+  }
+};
+
+const BAD_SEGMENT = /[\u0000\\]/;
+
+/**
+ * Normalize a workspace-relative path. Rejects absolute paths, drive letters,
+ * backslashes, NUL, empty and dot segments and `..` traversal. Pure string check.
+ */
+export function normalizeRelPath(rel) {
+  if (typeof rel !== "string" || rel.length === 0 || rel.length > 1024) return null;
+  if (rel.startsWith("/") || /^[A-Za-z]:/.test(rel) || BAD_SEGMENT.test(rel)) return null;
+  const segments = rel.split("/");
+  for (const segment of segments) {
+    if (segment === "" || segment === "." || segment === "..") return null;
+  }
+  return segments.join("/");
+}
+
+/**
+ * Resolve a relative path inside a workspace without following symlinks:
+ * every existing component is lstat-ed and a symlink anywhere is refused.
+ * Returns { abs, rel, exists, stat }.
+ */
+export async function resolveInWorkspace(workspaceAbs, rel, { allowMissing = true } = {}) {
+  const normalized = normalizeRelPath(rel);
+  if (normalized === null) {
+    throw new CliError(`unsafe workspace-relative path: ${JSON.stringify(rel)}`, { code: "unsafe_path" });
+  }
+  let current = workspaceAbs;
+  let stat = null;
+  const segments = normalized.split("/");
+  for (let index = 0; index < segments.length; index += 1) {
+    current = path.join(current, segments[index]);
+    try {
+      stat = await fsp.lstat(current);
+    } catch (error) {
+      if (error.code === "ENOENT" || error.code === "ENOTDIR") {
+        if (!allowMissing) throw new CliError(`path does not exist: ${normalized}`, { code: "missing" });
+        return { abs: path.join(workspaceAbs, ...segments), rel: normalized, exists: false, stat: null };
+      }
+      throw error;
+    }
+    if (stat.isSymbolicLink()) {
+      throw new CliError(`symbolic link refused inside workspace: ${segments.slice(0, index + 1).join("/")}`, {
+        code: "symlink_refused",
+      });
+    }
+    if (index < segments.length - 1 && !stat.isDirectory()) {
+      throw new CliError(`path component is not a directory: ${segments.slice(0, index + 1).join("/")}`, {
+        code: "not_directory",
+      });
+    }
+  }
+  return { abs: current, rel: normalized, exists: true, stat };
+}
+
+/**
+ * Resolve and verify the workspace root itself. The root (and any symlinked
+ * ancestor) is canonicalized once with realpath; every path below it is then
+ * refused if it is a symlink. Callers must compare paths against the returned
+ * canonical `abs`, never against the lexical argument.
+ */
+export async function openWorkspace(workspaceArg) {
+  if (!workspaceArg) throw usage("--workspace DIR is required");
+  const abs = path.resolve(workspaceArg);
+  let real;
+  try {
+    real = await fsp.realpath(abs);
+  } catch {
+    throw new CliError(`workspace does not exist: ${abs}`, { code: "workspace_missing" });
+  }
+  const stat = await fsp.stat(real);
+  if (!stat.isDirectory()) throw new CliError(`workspace is not a directory: ${abs}`, { code: "workspace_missing" });
+  return { abs: real };
+}
+
+const fsyncDir = async (dir) => {
+  let handle;
+  try {
+    handle = await fsp.open(dir, "r");
+    await handle.sync();
+  } catch {
+    /* directory fsync is best-effort on platforms that refuse it */
+  } finally {
+    await handle?.close();
+  }
+};
+
+const tempSibling = (dest) =>
+  path.join(path.dirname(dest), `.${path.basename(dest)}.tmp-${process.pid}-${randomBytes(4).toString("hex")}`);
+
+/**
+ * Atomically replace (or create) a mutable file: write a sibling temp file,
+ * fsync, rename. The published destination is never truncated first.
+ */
+export async function atomicReplace(dest, data, { mode = 0o644 } = {}) {
+  const temp = tempSibling(dest);
+  const handle = await fsp.open(temp, "wx", mode);
+  try {
+    await handle.writeFile(data);
+    await handle.sync();
+  } catch (error) {
+    await handle.close();
+    await fsp.rm(temp, { force: true });
+    throw error;
+  }
+  await handle.close();
+  try {
+    await fsp.rename(temp, dest);
+  } catch (error) {
+    await fsp.rm(temp, { force: true });
+    throw error;
+  }
+  await fsyncDir(path.dirname(dest));
+}
+
+/**
+ * Publish an immutable snapshot exactly once without changing or hard-linking
+ * the caller's source inode. The source is opened once as a regular file and
+ * streamed into a private sibling copy while hashing; with expectedSha256 the
+ * copy must match it (the source changed after inspection otherwise). Only the
+ * private copy is linked into place, atomically and exclusively.
+ */
+export async function publishExclusive(source, dest, { mode = 0o444, expectedSha256 = null } = {}) {
+  const temp = tempSibling(dest);
+  try {
+    const { handle: input } = await openRegularFile(source);
+    const hash = createHash("sha256");
+    try {
+      const output = await fsp.open(temp, "wx", 0o600);
+      try {
+        for await (const chunk of input.createReadStream({ highWaterMark: 1 << 20 })) {
+          hash.update(chunk);
+          for (let offset = 0; offset < chunk.length; ) {
+            offset += (await output.write(chunk, offset, chunk.length - offset)).bytesWritten;
+          }
+        }
+        await output.chmod(mode);
+        await output.sync();
+      } finally {
+        await output.close();
+      }
+    } finally {
+      await input.close().catch(() => {});
+    }
+    if (expectedSha256 && hash.digest("hex") !== expectedSha256) {
+      throw new CliError(`source ${source} changed after inspection; nothing was published`, { code: "source_changed", exit: EXIT.IO });
+    }
+    try {
+      await fsp.link(temp, dest);
+    } catch (error) {
+      if (error.code === "EEXIST") return "exists";
+      throw error;
+    }
+    await fsyncDir(path.dirname(dest));
+    return "created";
+  } finally {
+    await fsp.rm(temp, { force: true });
+  }
+}
+
+/** Run fn while holding an exclusive lock file; never breaks an existing lock. */
+export async function withLock(lockPath, fn) {
+  let handle;
+  try {
+    handle = await fsp.open(lockPath, "wx", 0o600);
+  } catch (error) {
+    if (error.code === "EEXIST") {
+      throw new CliError(`another helper operation holds ${lockPath}; remove it only after confirming no helper is running`, {
+        code: "locked",
+        exit: EXIT.IO,
+      });
+    }
+    throw error;
+  }
+  try {
+    await handle.writeFile(`${process.pid}\n`);
+    return await fn();
+  } finally {
+    await handle.close();
+    await fsp.rm(lockPath, { force: true });
+  }
+}
+
+export async function readJson(file) {
+  return JSON.parse(await fsp.readFile(file, "utf8"));
+}
+
+export const slugify = (text, fallback = "output") => {
+  const slug = String(text ?? "")
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, "-")
+    .replace(/^-+|-+$/g, "")
+    .slice(0, 40);
+  return slug || fallback;
+};
+
+export const stableStringify = (value) => `${JSON.stringify(value, null, 2)}\n`;
+
+/** Parse "--flag value" style arguments; flags listed in booleans take no value. */
+export function parseArgs(argv, { booleans = [], multi = [] } = {}) {
+  const flags = {};
+  const positionals = [];
+  for (let index = 0; index < argv.length; index += 1) {
+    const arg = argv[index];
+    if (arg === "--") {
+      positionals.push(...argv.slice(index + 1));
+      break;
+    }
+    if (arg.startsWith("--")) {
+      const eq = arg.indexOf("=");
+      const name = eq === -1 ? arg.slice(2) : arg.slice(2, eq);
+      let value;
+      if (booleans.includes(name)) {
+        value = true;
+      } else if (eq !== -1) {
+        value = arg.slice(eq + 1);
+      } else {
+        index += 1;
+        if (index >= argv.length) throw usage(`--${name} requires a value`);
+        value = argv[index];
+      }
+      if (multi.includes(name)) (flags[name] ??= []).push(value);
+      else if (name in flags) throw usage(`--${name} given more than once`);
+      else flags[name] = value;
+    } else {
+      positionals.push(arg);
+    }
+  }
+  return { flags, positionals };
+}
+
+export function toInt(value, name, { min = 0 } = {}) {
+  if (value === undefined) return undefined;
+  const n = Number(value);
+  if (!Number.isSafeInteger(n) || n < min) throw usage(`--${name} must be an integer >= ${min}`);
+  return n;
+}