@loomhelm/multi-task-flow 0.2.1 → 0.2.7

package/toolkit/skills/hardware-doc-ingest/scripts/_runtime/modules/docs/hardware_ingest.ts

@@ -0,0 +1,377 @@
+import { createHash } from "node:crypto";
+import {
+  chmodSync,
+  cpSync,
+  existsSync,
+  lstatSync,
+  mkdirSync,
+  readdirSync,
+  readFileSync,
+  rmSync,
+  statSync,
+  writeFileSync,
+} from "node:fs";
+import { dirname, join, resolve } from "node:path";
+
+function lstatOrUndefined(path: string) {
+  try {
+    return lstatSync(path);
+  } catch (error) {
+    if (
+      error instanceof Error &&
+      "code" in error &&
+      (error as NodeJS.ErrnoException).code === "ENOENT"
+    )
+      return undefined;
+    throw error;
+  }
+}
+
+export interface IngestOptions {
+  extractPath: string;
+  destDir: string;
+  slug: string;
+  overwrite?: boolean;
+  docSubdir?: string;
+  sourceUrl?: string;
+  docType?: string;
+  vendor?: string;
+  part?: string;
+  addHeader?: boolean;
+  keepMineruJson?: boolean;
+  writeLock?: boolean;
+  lockGenerated?: boolean;
+  createReadme?: boolean;
+}
+
+export interface IngestResult {
+  pdf: string;
+  markdown: string;
+  lock?: string;
+  notes?: string;
+  images?: string;
+  metadata?: string;
+  lines: string[];
+}
+
+function requireSingleOriginPdf(root: string): string {
+  const matches = readdirSync(root)
+    .filter((name) => name.endsWith("_origin.pdf"))
+    .sort();
+  if (matches.length === 0)
+    throw new Error(`No files matched '*_origin.pdf' under ${root}`);
+  if (matches.length > 1)
+    throw new Error(
+      `Multiple files matched '*_origin.pdf' under ${root}: ${matches.join(", ")}`,
+    );
+  const path = join(root, matches[0] as string);
+  assertSafeSourceEntry(path, "origin PDF");
+  return path;
+}
+
+function assertSafeSourceEntry(path: string, label: string): void {
+  const stat = lstatSync(path);
+  if (stat.isSymbolicLink() || stat.nlink > 1)
+    throw new Error(`${label} must not be a symlink or hardlink: ${path}`);
+  if (!stat.isFile())
+    throw new Error(`${label} must be a regular file: ${path}`);
+}
+
+function assertSafeSourceTree(root: string): void {
+  const stat = lstatSync(root);
+  if (stat.isSymbolicLink() || !stat.isDirectory())
+    throw new Error(`extract_path must be a real directory: ${root}`);
+  for (const entry of readdirSync(root)) {
+    const path = join(root, entry);
+    const child = lstatSync(path);
+    if (child.isSymbolicLink() || (!child.isDirectory() && child.nlink > 1))
+      throw new Error(`extract tree contains an unsafe link: ${path}`);
+    if (child.isDirectory()) assertSafeSourceTree(path);
+    else if (!child.isFile())
+      throw new Error(`extract tree entry is not a file: ${path}`);
+  }
+}
+
+function trustedSystemLinks(): Set<string> {
+  return process.platform === "darwin"
+    ? new Set(["/etc", "/tmp", "/var"])
+    : process.platform === "linux"
+      ? new Set(["/bin", "/lib", "/lib64", "/sbin"])
+      : new Set<string>();
+}
+
+function assertNoUnsafeSymlink(path: string, label: string): void {
+  const trustedLinks = trustedSystemLinks();
+  let current = resolve(path);
+  while (true) {
+    if (
+      lstatOrUndefined(current)?.isSymbolicLink() &&
+      !trustedLinks.has(current)
+    )
+      throw new Error(`${label} contains a symlink: ${current}`);
+    const parent = dirname(current);
+    if (parent === current) return;
+    current = parent;
+  }
+}
+
+function assertSafeDestinationPath(path: string): void {
+  assertNoUnsafeSymlink(path, "destination path");
+}
+
+function safeRelative(value: string, field: string, nested: boolean): string {
+  if (
+    !value ||
+    value.includes("\0") ||
+    value.startsWith("/") ||
+    value.includes("\\") ||
+    /^[A-Za-z]:/.test(value)
+  )
+    throw new Error(`${field} must be a relative path`);
+  const parts = value.split("/");
+  if (!nested && parts.length !== 1)
+    throw new Error(`${field} must be a single path component`);
+  if (parts.some((part) => !part || part === "." || part === ".."))
+    throw new Error(`${field} contains an unsafe path component`);
+  return parts.join("/");
+}
+
+function replaceExisting(
+  path: string,
+  overwrite: boolean,
+  directory = false,
+): void {
+  const existing = lstatOrUndefined(path);
+  if (!existing) return;
+  if (
+    existing.isSymbolicLink() ||
+    (!existing.isDirectory() && existing.nlink > 1)
+  )
+    throw new Error(
+      `Refusing to operate on an unsafe destination entry: ${path}`,
+    );
+  if (!overwrite)
+    throw new Error(
+      `Refusing to overwrite existing ${directory ? "directory" : "file"}: ${path}`,
+    );
+  rmSync(path, { recursive: directory, force: true });
+}
+
+function copyFile(
+  source: string,
+  destination: string,
+  overwrite: boolean,
+): void {
+  assertSafeDestinationPath(destination);
+  replaceExisting(destination, overwrite);
+  mkdirSync(resolve(destination, ".."), { recursive: true });
+  cpSync(source, destination, { preserveTimestamps: true });
+}
+
+function copyTree(
+  source: string,
+  destination: string,
+  overwrite: boolean,
+): void {
+  assertSafeDestinationPath(destination);
+  replaceExisting(destination, overwrite, true);
+  mkdirSync(resolve(destination, ".."), { recursive: true });
+  cpSync(source, destination, { recursive: true, preserveTimestamps: true });
+}
+
+export function rewriteImagePaths(markdown: string, slug: string): string {
+  const prefix = `${slug}.assets/images/`;
+  return markdown
+    .replaceAll("(./images/", `(${prefix}`)
+    .replaceAll("(images/", `(${prefix}`)
+    .replaceAll('src="./images/', `src="${prefix}`)
+    .replaceAll("src='./images/", `src='${prefix}`)
+    .replaceAll('src="images/', `src="${prefix}`)
+    .replaceAll("src='images/", `src='${prefix}`);
+}
+
+export function buildHeader(options: IngestOptions): string {
+  const lines: string[] = [];
+  for (const [label, value] of [
+    ["Type", options.docType],
+    ["Vendor", options.vendor],
+    ["Part", options.part],
+    ["Source", options.sourceUrl],
+  ] as const) {
+    if (value) lines.push(`> ${label}: ${value}`);
+  }
+  lines.push(`> PDF: \`${options.slug}.pdf\``);
+  return lines.length === 0 ? "" : `${lines.join("\n")}\n\n`;
+}
+
+export function buildNotesReadme(options: IngestOptions): string {
+  const title =
+    [options.vendor, options.part, options.docType].filter(Boolean).join(" ") ||
+    options.slug;
+  const lines = [
+    `# ${title}`,
+    "",
+    "This folder contains a **generated** Markdown mirror of the vendor document:",
+    "",
+    `- Generated Markdown (**DO NOT EDIT**): \`${options.slug}.md\``,
+    `- Source PDF (fidelity): \`${options.slug}.pdf\``,
+  ];
+  if (options.sourceUrl) lines.push(`- Source URL: ${options.sourceUrl}`);
+  lines.push(
+    "",
+    "## Notes (editable)",
+    "",
+    "- Summary:",
+    "- Key specs:",
+    "- Project decisions / remarks:",
+    "",
+  );
+  return lines.join("\n");
+}
+
+function sha256(path: string): string {
+  return createHash("sha256").update(readFileSync(path)).digest("hex");
+}
+
+function buildLock(
+  options: IngestOptions,
+  markdown: string,
+  pdf: string,
+): string {
+  return `${JSON.stringify(
+    {
+      artifacts: {
+        md: { path: `${options.slug}.md`, sha256: sha256(markdown) },
+        pdf: { path: `${options.slug}.pdf`, sha256: sha256(pdf) },
+      },
+      generated_at_utc: new Date().toISOString().replace(/\.\d{3}Z$/, "+00:00"),
+      generator: { tool: "ingest_mineru_extract.ts" },
+      schema: "codex.hardware-doc-ingest.lock.v1",
+      slug: options.slug,
+      source: {
+        doc_type: options.docType ?? null,
+        part: options.part ?? null,
+        url: options.sourceUrl ?? null,
+        vendor: options.vendor ?? null,
+      },
+    },
+    null,
+    2,
+  )}\n`;
+}
+
+export function ingestExtract(input: IngestOptions): IngestResult {
+  const options = {
+    overwrite: false,
+    addHeader: false,
+    keepMineruJson: false,
+    writeLock: true,
+    lockGenerated: true,
+    createReadme: true,
+    ...input,
+  };
+  const extract = resolve(options.extractPath);
+  const destination = resolve(options.destDir);
+  if (!existsSync(extract) || !statSync(extract).isDirectory())
+    throw new Error(
+      `extract_path does not exist or is not a directory: ${extract}`,
+    );
+  assertNoUnsafeSymlink(extract, "extract path");
+  assertSafeSourceTree(extract);
+  assertSafeDestinationPath(destination);
+  const slug = safeRelative(options.slug, "slug", false);
+  const docSubdir = options.docSubdir
+    ? safeRelative(options.docSubdir, "doc-subdir", true)
+    : undefined;
+  options.slug = slug;
+  if (docSubdir === undefined) delete options.docSubdir;
+  else options.docSubdir = docSubdir;
+  const fullMarkdown = join(extract, "full.md");
+  if (!existsSync(fullMarkdown))
+    throw new Error(`Missing full.md under ${extract}`);
+  assertSafeSourceEntry(fullMarkdown, "full.md");
+  const originPdf = requireSingleOriginPdf(extract);
+  const root = docSubdir ? join(destination, docSubdir) : destination;
+  const markdown = join(root, `${slug}.md`);
+  const pdf = join(root, `${slug}.pdf`);
+  const assets = join(root, `${slug}.assets`);
+  const notes = join(root, docSubdir ? "README.md" : `${slug}.README.md`);
+  const lock = join(root, `${slug}.lock.json`);
+  assertSafeDestinationPath(root);
+
+  copyFile(originPdf, pdf, options.overwrite);
+  let content = rewriteImagePaths(readFileSync(fullMarkdown, "utf8"), slug);
+  if (options.addHeader) content = buildHeader(options) + content;
+  replaceExisting(markdown, options.overwrite);
+  mkdirSync(resolve(markdown, ".."), { recursive: true });
+  writeFileSync(markdown, content, "utf8");
+  const images = join(extract, "images");
+  const destinationImages = join(assets, "images");
+  if (existsSync(images) && statSync(images).isDirectory())
+    copyTree(images, destinationImages, options.overwrite);
+  let metadata: string | undefined;
+  if (options.keepMineruJson) {
+    metadata = join(assets, "mineru");
+    assertSafeDestinationPath(metadata);
+    mkdirSync(metadata, { recursive: true });
+    for (const entry of readdirSync(extract)) {
+      if (entry.endsWith(".json")) {
+        assertSafeSourceEntry(join(extract, entry), `metadata ${entry}`);
+        copyFile(
+          join(extract, entry),
+          join(metadata, entry),
+          options.overwrite,
+        );
+      }
+    }
+  }
+  let notesPath: string | undefined;
+  if (options.createReadme) {
+    const existingNotes = lstatOrUndefined(notes);
+    if (
+      existingNotes?.isSymbolicLink() ||
+      (existingNotes && existingNotes.nlink > 1)
+    )
+      throw new Error(
+        `Refusing to operate on an unsafe destination entry: ${notes}`,
+      );
+    if (!existingNotes) {
+      assertSafeDestinationPath(notes);
+      mkdirSync(resolve(notes, ".."), { recursive: true });
+      writeFileSync(notes, buildNotesReadme(options), "utf8");
+    }
+    if (lstatOrUndefined(notes)) notesPath = notes;
+  }
+  let lockPath: string | undefined;
+  if (options.writeLock) {
+    lockPath = lock;
+    replaceExisting(lock, options.overwrite);
+    writeFileSync(lock, buildLock(options, markdown, pdf), "utf8");
+  }
+  if (options.lockGenerated) {
+    try {
+      chmodSync(markdown, 0o444);
+      chmodSync(pdf, 0o444);
+    } catch {
+      /* best effort */
+    }
+  }
+  const lines = [
+    `Staged mineru extract:`,
+    `- PDF: ${pdf}`,
+    `- MD:  ${markdown}`,
+  ];
+  if (lockPath) lines.push(`- LOCK: ${lockPath}`);
+  if (notesPath) lines.push(`- NOTES: ${notesPath}`);
+  if (existsSync(destinationImages)) lines.push(`- IMG: ${destinationImages}`);
+  if (metadata) lines.push(`- META: ${metadata}`);
+  return {
+    pdf,
+    markdown,
+    ...(lockPath ? { lock: lockPath } : {}),
+    ...(notesPath ? { notes: notesPath } : {}),
+    ...(existsSync(destinationImages) ? { images: destinationImages } : {}),
+    ...(metadata ? { metadata } : {}),
+    lines,
+  };
+}

package/toolkit/skills/hardware-doc-ingest/scripts/_runtime/modules/docs/hardware_locks.ts

@@ -0,0 +1,181 @@
+import {
+  accessSync,
+  constants,
+  readdirSync,
+  readFileSync,
+  statSync,
+} from "node:fs";
+import { basename, dirname, join, resolve } from "node:path";
+import { createHash } from "node:crypto";
+
+export const LOCK_SCHEMA = "codex.hardware-doc-ingest.lock.v1";
+
+interface JsonObject {
+  [key: string]: unknown;
+}
+
+export interface LockCheckOptions {
+  strictPdf?: boolean;
+  checkPerms?: boolean;
+}
+
+export interface LockCheckResult {
+  exitCode: number;
+  docs: number;
+  errors: number;
+  warnings: number;
+  lines: string[];
+}
+
+function object(value: unknown): JsonObject {
+  return value !== null && typeof value === "object" && !Array.isArray(value)
+    ? (value as JsonObject)
+    : {};
+}
+
+function sha256(path: string): string {
+  return createHash("sha256").update(readFileSync(path)).digest("hex");
+}
+
+function writable(path: string): boolean {
+  try {
+    accessSync(path, constants.W_OK);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+function lockFiles(root: string): string[] {
+  const found: string[] = [];
+  function visit(directory: string): void {
+    for (const entry of readdirSync(directory, { withFileTypes: true })) {
+      const path = join(directory, entry.name);
+      if (entry.isDirectory() && !entry.name.endsWith(".assets")) visit(path);
+      else if (entry.isFile() && entry.name.endsWith(".lock.json"))
+        found.push(path);
+    }
+  }
+  visit(root);
+  return found.sort();
+}
+
+function verifyLock(
+  path: string,
+  options: Required<LockCheckOptions>,
+): { errors: number; warnings: number; docs: number; lines: string[] } {
+  const lines: string[] = [];
+  let data: JsonObject;
+  try {
+    data = object(JSON.parse(readFileSync(path, "utf8")));
+  } catch (error) {
+    lines.push(`[ERROR] ${path}: invalid JSON (${String(error)})`);
+    return { errors: 1, warnings: 0, docs: 0, lines };
+  }
+  let errors = 0;
+  let warnings = 0;
+  if (data.schema !== LOCK_SCHEMA) {
+    lines.push(
+      `[WARN] ${path}: unknown schema ${JSON.stringify(data.schema)} (expected ${JSON.stringify(LOCK_SCHEMA)})`,
+    );
+    warnings += 1;
+  }
+  const artifacts = object(data.artifacts);
+  const md = object(artifacts.md);
+  const pdf = object(artifacts.pdf);
+  const mdRel = typeof md.path === "string" ? md.path : "";
+  const pdfRel = typeof pdf.path === "string" ? pdf.path : "";
+  const mdExpected = typeof md.sha256 === "string" ? md.sha256 : "";
+  const pdfExpected = typeof pdf.sha256 === "string" ? pdf.sha256 : "";
+  if (!mdRel || !mdExpected) {
+    lines.push(`[ERROR] ${path}: missing artifacts.md fields`);
+    return { errors: 1, warnings, docs: 0, lines };
+  }
+  const checkArtifact = (
+    relative: string,
+    expected: string,
+    kind: "Markdown" | "PDF",
+    strictMissing: boolean,
+  ): void => {
+    const artifact = resolve(dirname(path), relative);
+    if (!lstatSafe(artifact)) {
+      const prefix = strictMissing ? "[ERROR]" : "[WARN]";
+      const label = kind === "Markdown" ? "generated Markdown" : "source PDF";
+      lines.push(`${prefix} ${path}: missing ${label}: ${relative}`);
+      if (strictMissing) errors += 1;
+      else warnings += 1;
+      return;
+    }
+    const actual = sha256(artifact);
+    if (actual !== expected) {
+      lines.push(`[ERROR] ${path}: ${kind} SHA mismatch: ${relative}`);
+      errors += 1;
+    } else if (options.checkPerms && writable(artifact)) {
+      lines.push(
+        `[WARN] ${path}: ${kind} is writable (expected read-only): ${relative}`,
+      );
+      warnings += 1;
+    }
+  };
+  checkArtifact(mdRel, mdExpected, "Markdown", true);
+  if (pdfRel && pdfExpected)
+    checkArtifact(pdfRel, pdfExpected, "PDF", options.strictPdf);
+  return { errors, warnings, docs: 1, lines };
+}
+
+function lstatSafe(path: string): boolean {
+  try {
+    return statSync(path).isFile();
+  } catch {
+    return false;
+  }
+}
+
+export function verifyDocLocks(
+  rootInput = ".",
+  options: LockCheckOptions = {},
+): LockCheckResult {
+  const root = resolve(rootInput);
+  const lines: string[] = [];
+  if (!lstatSafe(root) && !directoryExists(root))
+    return {
+      exitCode: 2,
+      docs: 0,
+      errors: 1,
+      warnings: 0,
+      lines: [`[ERROR] root does not exist: ${rootInput}`],
+    };
+  const locks = lockFiles(root);
+  if (locks.length === 0)
+    return {
+      exitCode: 0,
+      docs: 0,
+      errors: 0,
+      warnings: 0,
+      lines: [`[INFO] No *.lock.json found under ${rootInput}`],
+    };
+  let errors = 0;
+  let warnings = 0;
+  let docs = 0;
+  const normalized = {
+    strictPdf: options.strictPdf ?? false,
+    checkPerms: options.checkPerms ?? true,
+  };
+  for (const lock of locks) {
+    const result = verifyLock(lock, normalized);
+    errors += result.errors;
+    warnings += result.warnings;
+    docs += result.docs;
+    lines.push(...result.lines);
+  }
+  lines.push(`[SUMMARY] docs=${docs} errors=${errors} warnings=${warnings}`);
+  return { exitCode: errors ? 1 : 0, docs, errors, warnings, lines };
+}
+
+function directoryExists(path: string): boolean {
+  try {
+    return statSync(path).isDirectory();
+  } catch {
+    return false;
+  }
+}

package/toolkit/skills/hardware-doc-ingest/scripts/_runtime/modules/docs/legacy_spec_id.ts

@@ -0,0 +1,131 @@
+import { randomInt } from "node:crypto";
+import { existsSync, readdirSync, readFileSync } from "node:fs";
+import { resolve } from "node:path";
+
+export const DEFAULT_ALPHABET = "23456789abcdefghjkmnpqrstuvwxyz";
+export const DEFAULT_LENGTH = 5;
+
+export interface LegacyIdOptions {
+  repoRoot?: string;
+  legacyCompat?: boolean;
+  count?: number;
+  length?: number;
+  alphabet?: string;
+  maxAttempts?: number;
+}
+
+export interface LegacyIdResult {
+  exitCode: number;
+  stdout: string;
+  stderr: string;
+}
+
+function idsInRoot(
+  root: string,
+  alphabet: string,
+  length: number,
+): Set<string> {
+  const ids = new Set<string>();
+  const idPattern = new RegExp(
+    `^(?:\\d{4}|[${escapeRegex(alphabet)}]{${length}})$`,
+  );
+  for (const relativeRoot of ["docs/specs", "docs/plan"]) {
+    const directory = resolve(root, relativeRoot);
+    if (existsSync(directory)) {
+      for (const entry of readdirSync(directory, { withFileTypes: true })) {
+        if (!entry.isDirectory()) continue;
+        const prefix = entry.name.split(/[-:]/, 1)[0]?.trim() ?? "";
+        if (idPattern.test(prefix)) ids.add(prefix);
+      }
+    }
+    const readme = resolve(directory, "README.md");
+    if (!existsSync(readme)) continue;
+    for (const line of readFileSync(readme, "utf8").split(/\r?\n/)) {
+      if (!line.startsWith("|")) continue;
+      const id =
+        line
+          .replace(/^\||\|$/g, "")
+          .split("|")[0]
+          ?.trim() ?? "";
+      if (idPattern.test(id)) ids.add(id);
+    }
+  }
+  return ids;
+}
+
+function escapeRegex(value: string): string {
+  return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+
+function oneId(alphabet: string, length: number): string {
+  return Array.from(
+    { length },
+    () => alphabet[randomInt(alphabet.length)] ?? alphabet[0],
+  ).join("");
+}
+
+export function generateLegacySpecIds(
+  options: LegacyIdOptions = {},
+): LegacyIdResult {
+  if (!options.legacyCompat)
+    return {
+      exitCode: 2,
+      stdout: "",
+      stderr:
+        "[ERROR] legacy ID generation is migration-only; create new specs with docs/specs/<topic>/ instead.\n",
+    };
+  const count = options.count ?? 1;
+  const length = options.length ?? DEFAULT_LENGTH;
+  const alphabet = (options.alphabet ?? DEFAULT_ALPHABET).trim();
+  const maxAttempts = options.maxAttempts ?? 10_000;
+  if (count < 1)
+    return {
+      exitCode: 2,
+      stdout: "",
+      stderr: "[ERROR] --count must be >= 1\n",
+    };
+  if (length < 1)
+    return {
+      exitCode: 2,
+      stdout: "",
+      stderr: "[ERROR] --length must be >= 1\n",
+    };
+  if (!alphabet)
+    return {
+      exitCode: 2,
+      stdout: "",
+      stderr: "[ERROR] --alphabet must be non-empty\n",
+    };
+  if (new Set(alphabet).size !== alphabet.length)
+    return {
+      exitCode: 2,
+      stdout: "",
+      stderr: "[ERROR] --alphabet must not contain duplicate characters\n",
+    };
+
+  const existing = idsInRoot(
+    resolve(options.repoRoot ?? "."),
+    alphabet,
+    length,
+  );
+  const generated: string[] = [];
+  for (let index = 0; index < count; index += 1) {
+    let candidate: string | undefined;
+    for (let attempt = 0; attempt < maxAttempts; attempt += 1) {
+      const next = oneId(alphabet, length);
+      if (!existing.has(next)) {
+        candidate = next;
+        existing.add(next);
+        break;
+      }
+    }
+    if (!candidate)
+      return {
+        exitCode: 1,
+        stdout: "",
+        stderr: `[ERROR] Failed to generate a unique id after ${maxAttempts} attempts.\n`,
+      };
+    generated.push(candidate);
+  }
+  return { exitCode: 0, stdout: `${generated.join("\n")}\n`, stderr: "" };
+}