@mintlify/cli 4.0.1315 → 4.0.1316

package/src/deslop/whitespace.ts

@@ -0,0 +1,155 @@
+export interface WhitespaceReport {
+  fixed: string;
+  changed: boolean;
+  counts: {
+    trailing: number;
+    blankRuns: number;
+    unicodeSpaces: number;
+    zeroWidth: number;
+    spaces: number;
+    finalNewline: boolean;
+  };
+}
+
+// Unusual space characters that should read as a normal space.
+const UNICODE_SPACES = /[\u00A0\u2000-\u200A\u202F\u205F\u3000]/g;
+// Zero-width and BOM characters that should be removed outright.
+const ZERO_WIDTH = /[\u200B-\u200D\u2060\uFEFF]/g;
+
+const FENCE = /^(\s*)(`{3,}|~{3,})/;
+// Splits a line so odd indices are inline-code spans (left untouched).
+const INLINE_CODE = /(`+[^`]*`+)/;
+
+function isFrontmatterFence(line: string, index: number): boolean {
+  return index === 0 && line.trim() === '---';
+}
+
+/**
+ * Normalize prose whitespace in an MDX document without touching the bytes of
+ * fenced code blocks or YAML frontmatter, where spacing is significant.
+ */
+export function normalizeWhitespace(content: string): WhitespaceReport {
+  const counts = {
+    trailing: 0,
+    blankRuns: 0,
+    unicodeSpaces: 0,
+    zeroWidth: 0,
+    spaces: 0,
+    finalNewline: false,
+  };
+
+  const lines = content.split('\n');
+  const out: string[] = [];
+  const isProtected: boolean[] = [];
+  let inFence = false;
+  let fenceMarker = '';
+  let inFrontmatter = false;
+
+  const pushProtected = (line: string) => {
+    out.push(line);
+    isProtected.push(true);
+  };
+
+  for (const [index, rawLine] of lines.entries()) {
+    if (isFrontmatterFence(rawLine, index)) {
+      inFrontmatter = true;
+      pushProtected(rawLine);
+      continue;
+    }
+    if (inFrontmatter) {
+      pushProtected(rawLine);
+      if (rawLine.trim() === '---') inFrontmatter = false;
+      continue;
+    }
+
+    const fence = rawLine.match(FENCE);
+    if (fence) {
+      const marker = fence[2] ?? '';
+      if (!inFence) {
+        inFence = true;
+        fenceMarker = marker;
+      } else if (marker[0] === fenceMarker[0] && marker.length >= fenceMarker.length) {
+        inFence = false;
+      }
+      pushProtected(rawLine);
+      continue;
+    }
+    if (inFence) {
+      pushProtected(rawLine);
+      continue;
+    }
+
+    let line = rawLine;
+    // Separate a trailing CR so CRLF files still get their trailing whitespace
+    // stripped (and keep the CR so line endings are preserved).
+    const cr = line.endsWith('\r') ? '\r' : '';
+    if (cr) line = line.slice(0, -1);
+
+    // Apply substitutions only outside inline-code spans (odd split indices),
+    // so backticked code keeps its exact spacing.
+    line = line
+      .split(INLINE_CODE)
+      .map((seg, i) => {
+        if (i % 2 === 1) return seg;
+        const uni = seg.match(UNICODE_SPACES);
+        if (uni) counts.unicodeSpaces += uni.length;
+        seg = seg.replace(UNICODE_SPACES, ' ');
+        const zw = seg.match(ZERO_WIDTH);
+        if (zw) counts.zeroWidth += zw.length;
+        return seg.replace(ZERO_WIDTH, '');
+      })
+      .join('');
+
+    const trimmed = line.replace(/[ \t]+$/, '');
+    if (trimmed !== line) counts.trailing += 1;
+    line = trimmed;
+
+    // Collapse runs of internal spaces (outside inline code), preserving leading indentation.
+    const leading = line.match(/^[ \t]*/)?.[0] ?? '';
+    const body = line.slice(leading.length);
+    const collapsed = body
+      .split(INLINE_CODE)
+      .map((seg, i) => (i % 2 === 1 ? seg : seg.replace(/ {2,}/g, ' ')))
+      .join('');
+    if (collapsed !== body) counts.spaces += 1;
+    line = leading + collapsed + cr;
+
+    out.push(line);
+    isProtected.push(false);
+  }
+
+  // Collapse runs of 2 or more consecutive blank lines down to a single blank
+  // line, but never inside protected regions (code fences, frontmatter).
+  const collapsed: string[] = [];
+  let blankRun = 0;
+  for (const [index, line] of out.entries()) {
+    if (!isProtected[index] && line.trim() === '') {
+      blankRun += 1;
+      if (blankRun <= 1) collapsed.push(line);
+      continue;
+    }
+    if (blankRun > 1) counts.blankRuns += 1;
+    blankRun = 0;
+    collapsed.push(line);
+  }
+  if (blankRun > 1) counts.blankRuns += 1;
+
+  let fixed = collapsed.join('\n').replace(/\n+$/, '');
+  if (content.length > 0) {
+    counts.finalNewline = !content.endsWith('\n') || content.endsWith('\n\n');
+    fixed += '\n';
+  }
+
+  return { fixed, changed: fixed !== content, counts };
+}
+
+export function summarizeWhitespace(counts: WhitespaceReport['counts']): string | null {
+  const parts: string[] = [];
+  if (counts.trailing) parts.push(`${counts.trailing} trailing`);
+  if (counts.blankRuns) parts.push(`${counts.blankRuns} blank-line runs`);
+  if (counts.unicodeSpaces) parts.push(`${counts.unicodeSpaces} unicode spaces`);
+  if (counts.zeroWidth) parts.push(`${counts.zeroWidth} zero-width chars`);
+  if (counts.spaces) parts.push(`${counts.spaces} collapsed spaces`);
+  if (counts.finalNewline) parts.push('final newline');
+  return parts.length ? parts.join(', ') : null;
+}