@hadden-industries/markdown-quality 0.1.0-alpha.2

package/src/cli.js

@@ -0,0 +1,85 @@
+#!/usr/bin/env node
+// SPDX-License-Identifier: AGPL-3.0-only
+import { parseArgs } from "node:util";
+import { runQuality, createResult } from "./quality.js";
+import { terminalText } from "./contracts.js";
+const args = process.argv.slice(2);
+if (args.length === 0 || ["--help", "-h"].includes(args[0])) {
+  process.stdout.write(
+    "Usage: markdown-quality <check|format|inspect> [--root PATH] [--config PATH] [--json] [--files-json JSON] [-- literal.md ...]\nFull selection is the default. Comparison options are deferred. Exit: 0 clean, 1 findings, 2 operation failure.\n",
+  );
+} else {
+  const terminator = args.indexOf("--");
+  let json = args
+    .slice(0, terminator < 0 ? args.length : terminator)
+    .includes("--json");
+  try {
+    const { values, positionals } = parseArgs({
+      args: args.slice(1),
+      allowPositionals: true,
+      strict: true,
+      options: {
+        root: { type: "string" },
+        config: { type: "string" },
+        json: { type: "boolean" },
+        "files-json": { type: "string" },
+      },
+    });
+    const hasTerminator = args.includes("--");
+    if (
+      positionals.length !== (hasTerminator ? args.length - terminator - 1 : 0)
+    )
+      throw new Error("Literal filenames must follow --.");
+    json = values.json ?? false;
+    if (values["files-json"] !== undefined && hasTerminator)
+      throw new Error("Use only one explicit selection mode.");
+    const files =
+      values["files-json"] !== undefined
+        ? JSON.parse(values["files-json"])
+        : hasTerminator
+          ? positionals
+          : undefined;
+    const result = await runQuality({
+      mode: args[0],
+      root: values.root,
+      config: values.config,
+      files,
+    });
+    if (json) process.stdout.write(JSON.stringify(result) + "\n");
+    else {
+      for (const d of result.diagnostics)
+        process.stdout.write(
+          `${terminalText(d.path)}:${d.line}:${d.column}: ${d.source}/${d.rule}: ${terminalText(d.message)}\n`,
+        );
+      for (const e of result.errors)
+        process.stdout.write(`${e.code}: ${terminalText(e.message)}\n`);
+      process.stdout.write(
+        `${result.outcome}; ${result.selection.mode} selection; ${result.selection.files.length} documents; ${result.written.length} written.\n`,
+      );
+      if (args[0] === "inspect")
+        process.stdout.write(
+          JSON.stringify({
+            package: result.package,
+            preset: result.preset,
+            tools: result.tools,
+            configDigest: result.configDigest,
+            configuration: result.configuration,
+            selection: result.selection,
+          }) + "\n",
+        );
+    }
+    process.exitCode = result.exitCode;
+  } catch (error) {
+    const message = error.message.startsWith("Unexpected token")
+      ? "Invalid files JSON."
+      : error.message;
+    const result = createResult(args[0]);
+    result.errors = [{ code: "CLI_INPUT", message }];
+    process.stdout.write(
+      json
+        ? JSON.stringify(result) + "\n"
+        : `CLI_INPUT: ${terminalText(message)}\n`,
+    );
+    process.exitCode = 2;
+  }
+}

package/src/configuration.js

@@ -0,0 +1,123 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+import { existsSync, lstatSync, readFileSync, realpathSync } from "node:fs";
+import { resolve, dirname, join, relative, isAbsolute, sep } from "node:path";
+import Ajv from "ajv";
+import markdown from "@eslint/markdown";
+import { decode, digest, fail, limits } from "./contracts.js";
+const schema = JSON.parse(
+  readFileSync(
+    new URL("../schemas/configuration.schema.json", import.meta.url),
+    "utf8",
+  ),
+);
+const validate = new Ajv({ allErrors: true, strict: true }).compile(schema);
+export function contained(root, path) {
+  const rel = relative(root, path);
+  return (
+    rel === "" ||
+    (!isAbsolute(rel) && rel !== ".." && !rel.startsWith(".." + sep))
+  );
+}
+export function safePath(root, input, { missing = false, file = false } = {}) {
+  if (
+    typeof input !== "string" ||
+    input.includes("\0") ||
+    input.replace(/^[A-Za-z]:[\\/]/u, "").includes(":")
+  )
+    fail("UNSAFE_PATH", "Invalid path.");
+  const path = resolve(root, input);
+  if (!contained(root, path))
+    fail("UNSAFE_PATH", "Path escapes the consumer root.");
+  const parts = relative(root, path).split(sep).filter(Boolean);
+  if (parts.length === 0 && file && !lstatSync(root).isFile())
+    fail("INVALID_FILE", "Expected a regular file.");
+  let current = root;
+  for (let i = 0; i < parts.length; i++) {
+    current = join(current, parts[i]);
+    let stat;
+    try {
+      stat = lstatSync(current);
+    } catch (error) {
+      if (missing && error.code === "ENOENT") return path;
+      throw error;
+    }
+    if (stat.isSymbolicLink())
+      fail("UNSAFE_PATH", "Symbolic links and junctions are not supported.");
+    if (!contained(root, realpathSync(current)))
+      fail("UNSAFE_PATH", "Resolved path escapes the consumer root.");
+    if (i < parts.length - 1 && !stat.isDirectory())
+      fail("UNSAFE_PATH", "A path parent is not a directory.");
+    if (i === parts.length - 1 && file && !stat.isFile())
+      fail("INVALID_FILE", "Expected a regular file.");
+  }
+  return path;
+}
+export function loadConfiguration({ root, config } = {}) {
+  if (!root) {
+    root = resolve(process.cwd());
+    while (!existsSync(join(root, ".markdown-quality.json"))) {
+      const parent = dirname(root);
+      if (parent === root)
+        fail("MISSING_CONFIG", "No .markdown-quality.json found.");
+      root = parent;
+    }
+  }
+  root = resolve(root);
+  const rootStat = lstatSync(root);
+  if (
+    !rootStat.isDirectory() ||
+    rootStat.isSymbolicLink() ||
+    realpathSync(root) !== root
+  )
+    fail(
+      "UNSAFE_ROOT",
+      "Consumer root must be a real directory without linked parents.",
+    );
+  const path = safePath(root, config ?? ".markdown-quality.json", {
+    file: true,
+  });
+  if (lstatSync(path).size > limits.configBytes)
+    fail("CONFIG_LIMIT", "Configuration exceeds the size limit.");
+  const bytes = readFileSync(path);
+  let value;
+  try {
+    value = JSON.parse(decode(bytes));
+  } catch (error) {
+    if (error.code) throw error;
+    fail("INVALID_CONFIG", "Configuration must be JSON.");
+  }
+  if (!validate(value))
+    fail(
+      "INVALID_CONFIG",
+      "Configuration does not match schema version 1: " +
+        JSON.stringify(validate.errors),
+    );
+  for (const rule of Object.keys(value.lint ?? {})) {
+    if (
+      !rule.startsWith("markdown/") ||
+      !Object.hasOwn(markdown.rules, rule.slice(9))
+    )
+      fail("INVALID_RULE", "Unknown Markdown rule: " + rule);
+  }
+  for (const pattern of [...value.include, ...(value.exclude ?? [])]) {
+    if (
+      pattern.startsWith("/") ||
+      pattern.includes("\\") ||
+      pattern.split("/").includes("..") ||
+      pattern.includes("\0")
+    )
+      fail(
+        "INVALID_PATTERN",
+        "Patterns must be relative slash-separated globs.",
+      );
+  }
+  const effective = {
+    ...value,
+    exclude: value.exclude ?? [],
+    ignoreFiles: value.ignoreFiles ?? [".gitignore", ".prettierignore"],
+    lint: value.lint ?? {},
+    links: { localFiles: true, rootRelative: "reject", ...value.links },
+    layout: { endOfLine: "preserve", tabWidth: 2, ...value.layout },
+  };
+  return { root, config: effective, configDigest: digest(bytes) };
+}

package/src/contracts.js

@@ -0,0 +1,55 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+import { createHash } from "node:crypto";
+import { readFileSync } from "node:fs";
+export const packageRoot = new URL("../", import.meta.url);
+export const metadata = JSON.parse(
+  readFileSync(new URL("../package.json", import.meta.url), "utf8"),
+);
+export const limits = Object.freeze({
+  fileBytes: 2_097_152,
+  totalBytes: 33_554_432,
+  files: 10_000,
+  entries: 100_000,
+  configBytes: 262_144,
+  nativeMs: 15_000,
+  nativeOutputBytes: 8_388_608,
+  diagnostics: 10_000,
+  documentDiagnostics: 1_000,
+  diagnosticBytes: 4_194_304,
+  selectionBytes: 4_194_304,
+});
+export const digest = (bytes) =>
+  createHash("sha256").update(bytes).digest("hex");
+export class OperationError extends Error {
+  constructor(code, message) {
+    super(message);
+    this.code = code;
+  }
+}
+export function fail(code, message) {
+  throw new OperationError(code, message);
+}
+export function diagnosticBudget(diagnostics) {
+  if (
+    diagnostics.length > limits.documentDiagnostics ||
+    Buffer.byteLength(JSON.stringify(diagnostics)) > limits.diagnosticBytes
+  )
+    fail(
+      "DIAGNOSTIC_LIMIT",
+      "Diagnostic count or output bytes exceed the limit.",
+    );
+}
+export function terminalText(value) {
+  return String(value).replace(
+    /[\u0000-\u001f\u007f-\u009f\u2028-\u202e\u2066-\u2069]/gu,
+    (character) =>
+      "\\u" + character.codePointAt(0).toString(16).padStart(4, "0"),
+  );
+}
+export function decode(bytes) {
+  try {
+    return new TextDecoder("utf-8", { fatal: true }).decode(bytes);
+  } catch {
+    fail("INVALID_UTF8", "Input is not valid UTF-8.");
+  }
+}

package/src/document-analysis.js

@@ -0,0 +1,79 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+import { Worker } from "node:worker_threads";
+import { OperationError } from "./contracts.js";
+export function createDocumentAnalyzer(context, tool) {
+  const worker = new Worker(new URL("./document-worker.js", import.meta.url), {
+    workerData: { context, tool },
+    resourceLimits: { maxOldGenerationSizeMb: 128, stackSizeMb: 4 },
+    env: {},
+  });
+  let pending,
+    closed = false,
+    termination;
+  function settle(error, value) {
+    if (!pending) return;
+    const request = pending;
+    pending = undefined;
+    clearTimeout(request.timer);
+    if (error) request.reject(error);
+    else request.resolve(value);
+  }
+  function close() {
+    closed = true;
+    settle(
+      new OperationError("ANALYSIS_FAILURE", "Document analysis stopped."),
+    );
+    return (termination ??= worker.terminate());
+  }
+  worker.on("message", (value) =>
+    value.error
+      ? settle(new OperationError(value.error.code, value.error.message))
+      : settle(null, value),
+  );
+  worker.on("error", () => {
+    closed = true;
+    settle(
+      new OperationError(
+        "ANALYSIS_FAILURE",
+        "Document analysis failed or exceeded its memory limit.",
+      ),
+    );
+  });
+  worker.on("exit", () => {
+    closed = true;
+    settle(
+      new OperationError(
+        "ANALYSIS_FAILURE",
+        "Document worker exited without a result.",
+      ),
+    );
+  });
+  return {
+    analyze(data) {
+      if (closed || pending)
+        return Promise.reject(
+          new OperationError(
+            "ANALYSIS_FAILURE",
+            "Document analysis is unavailable.",
+          ),
+        );
+      return new Promise((resolve, reject) => {
+        pending = {
+          resolve,
+          reject,
+          timer: setTimeout(() => {
+            settle(
+              new OperationError(
+                "ANALYSIS_TIMEOUT",
+                "Document analysis exceeded 30 seconds.",
+              ),
+            );
+            void close();
+          }, 30_000),
+        };
+        worker.postMessage(data);
+      });
+    },
+    close,
+  };
+}

package/src/document-worker.js

@@ -0,0 +1,30 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+import { parentPort, workerData } from "node:worker_threads";
+import { formatDocument } from "./formatting.js";
+import { createLinter, lintDocument, checkLinks } from "./analysis.js";
+import { diagnosticBudget, OperationError } from "./contracts.js";
+const { context, tool } = workerData;
+const linter = createLinter(context);
+parentPort.on("message", async ({ text, file, mode }) => {
+  try {
+    const formatted = await formatDocument(text, context, tool);
+    const proposed = mode === "format" ? formatted.output : text;
+    const diagnostics = [
+      ...formatted.diagnostics,
+      ...(await lintDocument(linter, proposed, file)),
+      ...checkLinks(context, proposed, file),
+    ];
+    diagnosticBudget(diagnostics);
+    parentPort.postMessage({ output: formatted.output, diagnostics });
+  } catch (error) {
+    parentPort.postMessage({
+      error: {
+        code: error instanceof OperationError ? error.code : "ANALYSIS_FAILURE",
+        message:
+          error instanceof OperationError
+            ? error.message
+            : "Document analysis failed.",
+      },
+    });
+  }
+});

package/src/documents.js

@@ -0,0 +1,136 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+import {
+  lstatSync,
+  openSync,
+  closeSync,
+  fstatSync,
+  readSync,
+  opendirSync,
+} from "node:fs";
+import { join, relative, sep } from "node:path";
+import picomatch from "picomatch";
+import * as prettier from "prettier";
+import { safePath } from "./configuration.js";
+import { fail, limits } from "./contracts.js";
+const blocked = new Set([
+  ".git",
+  "node_modules",
+  ".venv",
+  "venv",
+  ".hi",
+  ".sdlc",
+]);
+export async function selectDocuments(context, files) {
+  const { root, config } = context;
+  if (
+    files !== undefined &&
+    (!Array.isArray(files) ||
+      files.some((p) => typeof p !== "string") ||
+      files.length > limits.files)
+  )
+    fail(
+      "INVALID_SELECTION",
+      "Explicit files must be a bounded JSON string array.",
+    );
+  const included = picomatch(config.include, { dot: true });
+  const excluded = picomatch(config.exclude, { dot: true });
+  const ignores = config.ignoreFiles.map((p) => {
+    const path = safePath(root, p, { missing: true, file: true });
+    try {
+      if (lstatSync(path).size > limits.configBytes)
+        fail("IGNORE_LIMIT", "Ignore file exceeds the size limit.");
+    } catch (error) {
+      if (error.code !== "ENOENT") throw error;
+    }
+    return path;
+  });
+  let visited = 0;
+  const candidates = [];
+  let candidateBytes = 0;
+  function addCandidate(path) {
+    candidateBytes += Buffer.byteLength(path);
+    if (candidateBytes > limits.selectionBytes)
+      fail("SELECTION_LIMIT", "Discovered path bytes exceed the limit.");
+    candidates.push(path);
+  }
+  function walk(dir) {
+    const directory = opendirSync(dir);
+    try {
+      for (let entry; (entry = directory.readSync()) !== null;) {
+        if (++visited > limits.entries)
+          fail("SELECTION_LIMIT", "Directory entry limit exceeded.");
+        if (blocked.has(entry.name)) continue;
+        const path = join(dir, entry.name);
+        if (entry.isSymbolicLink()) continue;
+        if (entry.isDirectory()) {
+          if (!excluded(relative(root, path).split(sep).join("/") + "/"))
+            walk(path);
+        } else if (entry.isFile() && entry.name.endsWith(".md"))
+          addCandidate(path);
+      }
+    } finally {
+      directory.closeSync();
+    }
+  }
+  if (files === undefined) walk(root);
+  else
+    for (const file of files)
+      addCandidate(safePath(root, file, { file: true }));
+  const selected = [],
+    exclusions = [];
+  for (const path of [...new Set(candidates)].sort()) {
+    const rel = relative(root, path).split(sep).join("/");
+    let reason;
+    if (rel.split("/").some((part) => blocked.has(part))) reason = "mandatory";
+    else if (!rel.endsWith(".md") || !included(rel)) reason = "not-included";
+    else if (excluded(rel)) reason = "excluded";
+    else if (
+      (
+        await prettier.getFileInfo(path, {
+          ignorePath: ignores,
+          resolveConfig: false,
+          withNodeModules: false,
+        })
+      ).ignored
+    )
+      reason = "ignored";
+    if (reason) exclusions.push({ path: rel, reason });
+    else {
+      safePath(root, path, { file: true });
+      selected.push(rel);
+    }
+  }
+  if (selected.length > limits.files)
+    fail("SELECTION_LIMIT", "Document count limit exceeded.");
+  return {
+    files: selected,
+    exclusions,
+    mode: files === undefined ? "full" : "explicit",
+  };
+}
+export function readDocument(root, file) {
+  const path = safePath(root, file, { file: true });
+  const fd = openSync(path, "r");
+  try {
+    const stat = fstatSync(fd, { bigint: true });
+    if (!stat.isFile()) fail("INVALID_FILE", "Expected a regular file.");
+    if (stat.size > BigInt(limits.fileBytes))
+      fail("DOCUMENT_LIMIT", "Document exceeds the size limit.");
+    const buffer = Buffer.alloc(limits.fileBytes + 1);
+    let size = 0,
+      count;
+    while (
+      size < buffer.length &&
+      (count = readSync(fd, buffer, size, buffer.length - size, null)) !== 0
+    )
+      size += count;
+    if (size > limits.fileBytes)
+      fail("DOCUMENT_LIMIT", "Document exceeds the size limit.");
+    // Own only the observed bytes, rather than retaining the bounded scratch buffer.
+    const bytes = Buffer.alloc(size);
+    buffer.copy(bytes, 0, 0, size);
+    return { path, bytes, stat };
+  } finally {
+    closeSync(fd);
+  }
+}

package/src/formatting.js

@@ -0,0 +1,44 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+import * as prettier from "prettier";
+import { parse } from "./analysis.js";
+import { runNative } from "./native-tool.js";
+import { checkProse } from "./prose-diagnostics.js";
+import { fail } from "./contracts.js";
+function semantic(node) {
+  const value = {};
+  for (const [key, item] of Object.entries(node)) {
+    if (key === "position") continue;
+    if (key === "children") value.children = item.map(semantic);
+    else if (key === "value" && node.type === "text")
+      value.value = item.replace(/\s+/gu, " ");
+    else value[key] = item;
+  }
+  return value;
+}
+export async function formatDocument(text, context, tool) {
+  const endOfLine =
+    context.config.layout.endOfLine === "preserve"
+      ? text.includes("\r\n")
+        ? "crlf"
+        : "lf"
+      : context.config.layout.endOfLine;
+  const options = {
+    parser: "markdown",
+    proseWrap: "preserve",
+    embeddedLanguageFormatting: "off",
+    endOfLine,
+    tabWidth: context.config.layout.tabWidth,
+    plugins: [],
+  };
+  const layout = await prettier.format(text, options);
+  const result = await prettier.format(runNative(tool, layout), options);
+  if (
+    JSON.stringify(semantic(parse(text))) !==
+    JSON.stringify(semantic(parse(result)))
+  )
+    fail("PRESERVATION", "Formatting changed parsed meaning or a literal.");
+  const second = await prettier.format(runNative(tool, result), options);
+  if (result !== second)
+    fail("CONVERGENCE", "Formatter pipeline did not converge.");
+  return { output: result, diagnostics: checkProse(tool, result) };
+}

package/src/native-tool.js

@@ -0,0 +1,135 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+import { createRequire } from "node:module";
+import { readFileSync, lstatSync, realpathSync } from "node:fs";
+import { join, dirname } from "node:path";
+import { fileURLToPath } from "node:url";
+import { spawnSync } from "node:child_process";
+import { decode, digest, fail, limits, metadata } from "./contracts.js";
+const require = createRequire(import.meta.url);
+export const manifest = JSON.parse(
+  readFileSync(
+    new URL("../assets/tool-manifest.json", import.meta.url),
+    "utf8",
+  ),
+);
+export function resolveTool() {
+  const key = `${process.platform}-${process.arch}`;
+  const spec = manifest.platforms[key];
+  if (
+    !spec ||
+    (process.platform === "linux" &&
+      !process.report.getReport().header.glibcVersionRuntime)
+  )
+    fail(
+      "UNSUPPORTED_PLATFORM",
+      "Only Windows x64 and glibc Linux x64 are supported.",
+    );
+  let pkg;
+  try {
+    pkg = require.resolve(spec.package + "/package.json");
+  } catch {
+    fail(
+      "MISSING_TOOL",
+      "Install the matching optional platform package with lifecycle scripts disabled.",
+    );
+  }
+  if (JSON.parse(readFileSync(pkg, "utf8")).version !== metadata.version)
+    fail("TOOL_VERSION", "Core and native package versions differ.");
+  const executable = join(dirname(pkg), spec.executable);
+  if (!lstatSync(executable).isFile() || lstatSync(executable).isSymbolicLink())
+    fail("TOOL_IDENTITY", "Native executable must be a regular file.");
+  if (digest(readFileSync(executable)) !== spec.sha256)
+    fail("TOOL_IDENTITY", "Native executable digest mismatch.");
+  return {
+    executable: realpathSync(executable),
+    key,
+    version: manifest.snapperVersion,
+    sha256: spec.sha256,
+  };
+}
+export function validateDiagnostics(result, text) {
+  if (result.error || result.signal || ![0, 1].includes(result.status))
+    fail("NATIVE_FAILURE", "Native check failed or exceeded resource limits.");
+  let reports;
+  try {
+    reports = JSON.parse(decode(result.stdout));
+  } catch {
+    fail("NATIVE_REPORT", "Malformed native JSON.");
+  }
+  if (Array.isArray(reports) && reports.length === 0 && result.status === 0)
+    return [];
+  if (
+    !Array.isArray(reports) ||
+    reports.length !== 1 ||
+    typeof reports[0]?.would_reformat !== "boolean" ||
+    !Array.isArray(reports[0].diagnostics)
+  )
+    fail("NATIVE_REPORT", "Invalid single-document native report.");
+  const lines = text.split("\n").length;
+  for (const d of reports[0].diagnostics)
+    if (
+      !d ||
+      !["fused", "wrap", "long"].includes(d.kind) ||
+      !Number.isSafeInteger(d.line) ||
+      d.line < 1 ||
+      d.line > lines ||
+      typeof d.excerpt !== "string"
+    )
+      fail("NATIVE_REPORT", "Invalid or unknown native diagnostic.");
+  if (
+    result.status === 1 &&
+    reports[0].diagnostics.length === 0 &&
+    !reports[0].would_reformat
+  )
+    fail("NATIVE_REPORT", "Inconsistent native failure report.");
+  // Snapper's documented advisory length threshold is independent of wrapping.
+  // This preset has no sentence-length policy; validate but omit that advisory.
+  return reports[0].diagnostics
+    .filter((d) => d.kind !== "long")
+    .map((d) => ({
+      source: "snapper",
+      rule: d.kind,
+      line: d.line,
+      column: 1,
+      message: `Sentence layout: ${d.kind}.`,
+      severity: "error",
+    }));
+}
+export function runNative(tool, text, check = false) {
+  const args = [
+    "--native",
+    "--config",
+    fileURLToPath(new URL("../assets/snapper.toml", import.meta.url)),
+    "--format",
+    "markdown",
+    "--stdin-filepath",
+    "document.md",
+    "--max-width",
+    "0",
+    "--color",
+    "never",
+  ];
+  if (check) args.push("--check", "--output-format", "json");
+  // A fixed working directory and virtual filename keep consumer configuration out.
+  const env = Object.fromEntries(
+    ["SystemRoot", "WINDIR", "TEMP", "TMP", "LANG", "LC_ALL"]
+      .filter((k) => process.env[k])
+      .map((k) => [k, process.env[k]]),
+  );
+  const result = spawnSync(tool.executable, args, {
+    input: Buffer.from(text),
+    cwd: fileURLToPath(new URL("../assets/", import.meta.url)),
+    env,
+    shell: false,
+    timeout: limits.nativeMs,
+    maxBuffer: limits.nativeOutputBytes,
+    windowsHide: true,
+  });
+  if (check) return validateDiagnostics(result, text);
+  if (result.error || result.signal || result.status !== 0)
+    fail(
+      "NATIVE_FAILURE",
+      "Native formatter failed or exceeded resource limits.",
+    );
+  return decode(result.stdout);
+}