@hadden-industries/markdown-quality 1.0.1 → 1.0.3

package/src/quality.js

@@ -4,6 +4,7 @@ import { selectDocuments, readDocument } from "./documents.js";
 import { resolveTool, manifest } from "./native-tool.js";
 import { createDocumentAnalyzer } from "./document-analysis.js";
 import { replaceDocument } from "./replacement.js";
+import { createNativeStaging } from "./native-staging.js";
 import {
   decode,
   fail,
@@ -44,7 +45,21 @@ export function createResult(mode = "check", explicit = false) {
 export async function runQuality(options = {}) {
   const mode = options.mode ?? "check";
   const result = createResult(mode, options.files !== undefined);
-  let analyzer;
+  let analyzer,
+    staging,
+    stagingAttempted = false;
+  async function closeAnalysis() {
+    if (analyzer) {
+      const owned = analyzer;
+      analyzer = undefined;
+      await owned.close();
+    }
+    if (staging) {
+      const owned = staging;
+      staging = undefined;
+      owned.close();
+    }
+  }
   try {
     if (!["check", "format", "inspect"].includes(mode))
       fail("INVALID_OPERATION", "Unknown operation.");
@@ -71,48 +86,157 @@ export async function runQuality(options = {}) {
     let total = 0,
       outputTotal = 0,
       diagnosticBytes = 0;
-    for (const file of result.selection.files) {
-      const item = readDocument(context.root, file);
-      if (mode === "format" && item.stat.nlink !== 1n)
-        fail("HARD_LINK", "Hard-linked documents cannot be formatted.");
-      total += item.bytes.length;
-      if (total > limits.totalBytes)
-        fail("BATCH_LIMIT", "Total document bytes exceed the limit.");
-      const original = decode(item.bytes);
-      const formatted = await analyzer.analyze({ text: original, file, mode });
-      const outputBytes = Buffer.byteLength(formatted.output);
-      if (outputBytes > limits.fileBytes)
-        fail("DOCUMENT_LIMIT", "Formatted document exceeds the size limit.");
-      outputTotal += outputBytes;
-      if (outputTotal > limits.totalBytes)
-        fail("BATCH_LIMIT", "Total formatted document bytes exceed the limit.");
-      const diagnostics = formatted.diagnostics;
-      if (mode === "check" && !item.bytes.equals(Buffer.from(formatted.output)))
-        diagnostics.push({
-          source: "formatter",
-          rule: "layout",
-          line: 1,
-          column: 1,
-          severity: "error",
-          message: "Document requires formatting.",
-        });
-      diagnosticBudget(diagnostics);
-      for (const d of diagnostics) {
-        const diagnostic = { path: file, ...d };
-        diagnosticBytes += Buffer.byteLength(JSON.stringify(diagnostic)) + 1;
+    let prepared = [],
+      preparedBytes = 0;
+    let inputs = [],
+      inputBytes = 0;
+    async function flushPrepared() {
+      if (!prepared.length) return;
+      const group = prepared;
+      prepared = [];
+      preparedBytes = 0;
+      if (group.length > 1 && !stagingAttempted) {
+        stagingAttempted = true;
+        const started = performance.now();
+        staging = createNativeStaging(context.root);
+        const elapsed = performance.now() - started;
+        for (const record of group) record.elapsedMs += elapsed;
+      }
+      const verified = await analyzer.verify(
+        group.map(({ formatted, elapsedMs, precheck }) => ({
+          output: formatted.output,
+          elapsedMs,
+          precheck: formatted.prechecked ? precheck : undefined,
+        })),
+        staging?.token,
+      );
+      for (const [index, { file, item, formatted }] of group.entries()) {
+        if (formatted.deferredError)
+          fail(formatted.deferredError.code, formatted.deferredError.message);
+        const outputBytes = Buffer.byteLength(formatted.output);
+        if (outputBytes > limits.fileBytes)
+          fail("DOCUMENT_LIMIT", "Formatted document exceeds the size limit.");
+        outputTotal += outputBytes;
+        if (outputTotal > limits.totalBytes)
+          fail(
+            "BATCH_LIMIT",
+            "Total formatted document bytes exceed the limit.",
+          );
+        const diagnostics = [
+          ...formatted.diagnostics.slice(0, formatted.whitespaceCount),
+          ...verified.diagnostics[index],
+          ...formatted.diagnostics.slice(formatted.whitespaceCount),
+        ];
         if (
-          result.diagnostics.length >= limits.diagnostics ||
-          diagnosticBytes > limits.diagnosticBytes
+          mode === "check" &&
+          !item.bytes.equals(Buffer.from(formatted.output))
         )
-          fail(
-            "DIAGNOSTIC_LIMIT",
-            "Diagnostic count or output bytes exceed the limit.",
+          diagnostics.push({
+            source: "formatter",
+            rule: "layout",
+            line: 1,
+            column: 1,
+            severity: "error",
+            message: "Document requires formatting.",
+          });
+        diagnosticBudget(diagnostics);
+        for (const d of diagnostics) {
+          const diagnostic = { path: file, ...d };
+          diagnosticBytes += Buffer.byteLength(JSON.stringify(diagnostic)) + 1;
+          if (
+            result.diagnostics.length >= limits.diagnostics ||
+            diagnosticBytes > limits.diagnosticBytes
+          )
+            fail(
+              "DIAGNOSTIC_LIMIT",
+              "Diagnostic count or output bytes exceed the limit.",
+            );
+          result.diagnostics.push(diagnostic);
+        }
+        if (mode === "format")
+          candidates.push({ file, item, output: formatted.output });
+      }
+    }
+    async function flushInputs() {
+      if (!inputs.length) return;
+      const group = inputs;
+      inputs = [];
+      inputBytes = 0;
+      let setupMs = 0;
+      if (group.length > 1 && !stagingAttempted) {
+        stagingAttempted = true;
+        const started = performance.now();
+        staging = createNativeStaging(context.root);
+        setupMs = performance.now() - started;
+      }
+      let checks;
+      let overheadMs = setupMs;
+      if (staging) {
+        const started = performance.now();
+        checks = await analyzer.precheck(
+          group.map((record) => record.original),
+          staging.token,
+        );
+        overheadMs += Math.max(
+          0,
+          performance.now() - started - checks.workerMs,
+        );
+      }
+      for (const [index, { file, item, original }] of group.entries()) {
+        const precheck = checks?.checks[index];
+        let elapsedMs = overheadMs + (checks?.elapsedMs[index] ?? 0);
+        let formatted;
+        try {
+          const started = performance.now();
+          formatted = await analyzer.prepare(
+            { text: original, file, mode, precheck },
+            30_000 - elapsedMs,
           );
-        result.diagnostics.push(diagnostic);
+          elapsedMs += performance.now() - started;
+        } catch (error) {
+          await flushPrepared();
+          throw error;
+        }
+        const retainedBytes =
+          item.bytes.length + Buffer.byteLength(formatted.output);
+        if (
+          prepared.length >= 32 ||
+          preparedBytes + retainedBytes > 4 * 1024 * 1024 ||
+          formatted.deferredError
+        )
+          await flushPrepared();
+        prepared.push({ file, item, formatted, elapsedMs, precheck });
+        preparedBytes += retainedBytes;
+        if (formatted.deferredError) await flushPrepared();
       }
-      if (mode === "format")
-        candidates.push({ file, item, output: formatted.output });
+      await flushPrepared();
     }
+    for (const file of result.selection.files) {
+      let item, original;
+      try {
+        item = readDocument(context.root, file);
+        if (mode === "format" && item.stat.nlink !== 1n)
+          fail("HARD_LINK", "Hard-linked documents cannot be formatted.");
+        total += item.bytes.length;
+        if (total > limits.totalBytes)
+          fail("BATCH_LIMIT", "Total document bytes exceed the limit.");
+        original = decode(item.bytes);
+      } catch (error) {
+        await flushInputs();
+        throw error;
+      }
+      if (
+        inputs.length >= 32 ||
+        inputBytes + item.bytes.length > 4 * 1024 * 1024
+      )
+        await flushInputs();
+      inputs.push({ file, item, original });
+      inputBytes += item.bytes.length;
+    }
+    await flushInputs();
+    await flushPrepared();
+    // Quiesce the worker and remove private payloads before admitting any write.
+    await closeAnalysis();
     result.diagnostics.sort((a, b) =>
       a.path < b.path
         ? -1
@@ -150,7 +274,17 @@ export async function runQuality(options = {}) {
         error instanceof OperationError ? error.message : "Operation failed.",
     });
   } finally {
-    if (analyzer) await analyzer.close();
+    try {
+      await closeAnalysis();
+    } catch (error) {
+      result.outcome = "error";
+      result.exitCode = 2;
+      result.errors.push({
+        code: error instanceof OperationError ? error.code : "OPERATION_FAILED",
+        message:
+          error instanceof OperationError ? error.message : "Operation failed.",
+      });
+    }
   }
   return result;
 }

package/src/whitespace.js

@@ -0,0 +1,57 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Authored Markdown whitespace policy; literal code bodies are the only exemption.
+import { parse } from "./analysis.js";
+import { codeBodyRows } from "./literal-layout.js";
+import { fail, limits } from "./contracts.js";
+
+/** Find every non-code trailing space/tab, or fail at the document diagnostic bound. */
+export function checkTrailingWhitespace(text, memo) {
+  if (!/[ \t](?:\r\n|\r|\n|$)/u.test(text)) return [];
+  const rows = text.split(/(\r\n|\r|\n)/u);
+  const codeRows = codeBodyRows(text, memo ? memo.parse(text) : parse(text));
+  const diagnostics = [];
+  for (let row = 0; row < rows.length; row += 2) {
+    if (codeRows.has(row)) continue;
+    const trailing = /[ \t]+$/u.exec(rows[row]);
+    if (!trailing) continue;
+    if (diagnostics.length >= limits.documentDiagnostics)
+      fail(
+        "DIAGNOSTIC_LIMIT",
+        "Diagnostic count or output bytes exceed the limit.",
+      );
+    diagnostics.push({
+      source: "formatter",
+      rule: "trailing-whitespace",
+      line: row / 2 + 1,
+      column: trailing.index + 1,
+      severity: "error",
+      message:
+        "Trailing spaces and tabs are forbidden outside code-block contents; literal-sensitive fixes require manual editing.",
+    });
+  }
+  return diagnostics;
+}
+
+/** Propose trimming with parsed hard breaks made explicit; caller must verify semantics. */
+export function normalizeTrailingWhitespace(text, memo) {
+  // Most documents need no policy repair. Avoid another Markdown parse there.
+  if (!/[ \t](?:\r\n|\r|\n|$)/u.test(text)) return text;
+  const tree = memo ? memo.parse(text) : parse(text);
+  const rows = text.split(/(\r\n|\r|\n)/u);
+  const codeRows = codeBodyRows(text, tree);
+  const hardBreakRows = new Set();
+  function visit(node) {
+    // Only the parser can distinguish an actual hard break from spaces at a
+    // paragraph end, in a literal, or beside escaped punctuation.
+    if (node.type === "break" && text[node.position.start.offset] === " ")
+      hardBreakRows.add((node.position.start.line - 1) * 2);
+    for (const child of node.children ?? []) visit(child);
+  }
+  visit(tree);
+  for (let row = 0; row < rows.length; row += 2) {
+    if (codeRows.has(row) || !/[ \t]+$/u.test(rows[row])) continue;
+    rows[row] = rows[row].replace(/[ \t]+$/u, "");
+    if (hardBreakRows.has(row)) rows[row] += "\\";
+  }
+  return rows.join("");
+}