formatferry 1.0.29 → 1.0.31

package/README.md

@@ -44,11 +44,44 @@ already have real text. It is deliberately absent here — the CLI ships no WASM
 OCR engine, which keeps the install small and the conversion fully offline.
 
 For a PDF that mixes real text pages with scanned inserts, the conversion
-**succeeds** and the scanned pages come through empty. The document has enough
-text elsewhere to avoid the `IMAGE_ONLY` error above, so the exit code is 0 and
-the only sign is a page heading with nothing under it. Check the output when a
-PDF may contain scans — for this case the CLI cannot tell you which pages it
-could not read, because no OCR ran and so nothing reported a per-page failure.
+still **succeeds** with exit code 0 — the document has enough text elsewhere
+to avoid the `IMAGE_ONLY` error above — and the scanned pages come through
+empty. That loss is no longer silent: stderr carries one warning naming the
+pages that could not be fully read:
+
+```
+Warning: Page(s) 3 could not be fully read: they have little or no extractable text and OCR is not available in this environment, so their content may be missing or incomplete.
+```
+
+Pipelines that would rather fail than accept a warned conversion can pass
+`--strict`. The complete output is still written (to stdout or the `-o`
+file), and the process then exits 1 after a summary line:
+
+```
+$ formatferry -i mixed.pdf --strict; echo "exit=$?"
+Warning: Page(s) 3 could not be fully read: they have little or no extractable text and OCR is not available in this environment, so their content may be missing or incomplete.
+Error: --strict: conversion produced 1 warning(s) [pdf.warnings.ocrUnavailablePages].
+exit=1
+```
+
+The summary names the machine-readable warning code(s) in brackets, so
+pipelines can triage without parsing prose.
+
+Be aware that **any** warning trips `--strict`, not only page-loss warnings:
+advisory notices count too. For example, the xlsx warning "This spreadsheet
+has many columns" fires on wide-but-ordinary spreadsheets and would exit 1
+under `--strict`. This any-warning meaning of bare `--strict` is the permanent
+contract: if a curated data-loss-only mode is ever added, it will be a
+separate additive spelling (for example `--strict=loss`) — bare `--strict`
+will not change meaning underneath existing pipelines.
+
+Under `--batch`, each file's warnings are printed to stderr as
+`Warning (<file>): <message>`, and `--strict` exits 1 after the batch
+completes (all outputs still written) if any file warned.
+
+One caveat for MCP users: whether the external `formatferry-mcp` wrapper
+forwards `--strict` to the CLI has not been verified; if it filters unknown
+flags, the flag is inert there.
 
 Conversion warnings that *are* produced (for example unusual table structures)
 are written to stderr, never stdout, so `formatferry -i doc.pdf > out.md` keeps
@@ -72,6 +105,7 @@ formatferry [options]
 | `--url <url>` | Fetch and convert a web article (requires API key) |
 | `--batch <glob>` | Batch convert files matching a glob pattern (Premium) |
 | `--output-dir <dir>` | Output directory for batch mode |
+| `--strict` | Exit with code 1 if the conversion produced any warnings (including advisory ones) |
 
 #### Input Modes
 

package/dist/index.js

@@ -58500,6 +58500,25 @@ function normalizeAtlassianHTML(html, domParser) {
 
 // ../../src/shared/confluence-slack-services.ts
 var import_turndown = __toESM(require_turndown_cjs(), 1);
+function escapeSlackEntities(text) {
+  return text.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;");
+}
+var ZERO_WIDTH_SPACE = "\u200B";
+function escapeSlackText(text) {
+  return escapeSlackEntities(text).replace(
+    /(^|[^\p{L}\p{N}])([*_~`])/gu,
+    `$1${ZERO_WIDTH_SPACE}$2`
+  );
+}
+function slackLabel(text) {
+  return text.replace(/\|/g, "\uFF5C");
+}
+function escapeConfluenceText(text) {
+  return text.replace(/([{}[\]\\])/g, "\\$1").replace(/(^|\n)([*#-])(?=\s)/g, "$1\\$2").replace(/(^|[^\p{L}\p{N}\\])([*_+\-^~])(?=\S)/gu, "$1\\$2").replace(
+    /(?<=\S)([*_+\-^~])(?=$|[^\p{L}\p{N}])/gu,
+    (m, c, offset, str) => str[offset - 1] === "\\" ? m : `\\${c}`
+  );
+}
 function createSlackService(deps) {
   const service = new import_turndown.default({
     headingStyle: "atx",
@@ -58509,6 +58528,7 @@ function createSlackService(deps) {
     fence: "```",
     emDelimiter: "_"
   });
+  service.escape = escapeSlackText;
   deps.addListRules(service);
   deps.addPdfPageMarkerRules(service);
   service.addRule("slackBold", {
@@ -58557,12 +58577,33 @@ function createSlackService(deps) {
       const fontFamily = style.toLowerCase();
       return fontFamily.includes("monospace") || fontFamily.includes("courier") || fontFamily.includes("consolas") || fontFamily.includes("monaco") || fontFamily.includes("menlo") || fontFamily.includes("source code") || fontFamily.includes("fira code") || fontFamily.includes("roboto mono") || fontFamily.includes("ubuntu mono");
     },
-    replacement: (content) => {
-      const trimmed = content.trim();
+    replacement: (content, node) => {
+      const trimmed = escapeSlackEntities((node.textContent || "").trim());
       if (!trimmed) return content;
       return trimmed.includes("`") ? `\`\` ${trimmed} \`\`` : `\`${trimmed}\``;
     }
   });
+  service.addRule("slackCodeBlock", {
+    filter: (node) => node.nodeName === "PRE",
+    replacement: (_content, node) => {
+      const code = (node.textContent || "").replace(/\n$/, "");
+      return `
+
+\`\`\`
+${escapeSlackEntities(code)}
+\`\`\`
+
+`;
+    }
+  });
+  service.addRule("slackInlineCode", {
+    filter: (node) => node.nodeName === "CODE" && node.parentNode?.nodeName !== "PRE",
+    replacement: (content, node) => {
+      const code = escapeSlackEntities(node.textContent || "");
+      if (!code) return content;
+      return code.includes("`") ? `\`\` ${code} \`\`` : `\`${code}\``;
+    }
+  });
   service.addRule("slackHeading", {
     filter: ["h1", "h2", "h3", "h4", "h5", "h6"],
     replacement: (content) => content.trim() ? `*${content.trim()}*
@@ -58599,8 +58640,8 @@ function createSlackService(deps) {
   service.addRule("slackLink", {
     filter: (node) => node.nodeName === "A" && !!node.getAttribute("href"),
     replacement: (content, node) => {
-      const href = node.getAttribute("href") || "";
-      const text = content.trim().replace(/\|/g, " ");
+      const href = escapeSlackEntities(node.getAttribute("href") || "");
+      const text = slackLabel(content.trim());
       if (!text || text === href) return `<${href}>`;
       return `<${href}|${text}>`;
     }
@@ -58608,15 +58649,15 @@ function createSlackService(deps) {
   service.addRule("slackImage", {
     filter: "img",
     replacement: (_content, node) => {
-      const src = node.getAttribute("src") || "";
-      const alt = (node.getAttribute("alt") || "image").replace(/\|/g, " ");
+      const src = escapeSlackEntities(node.getAttribute("src") || "");
+      const alt = slackLabel(escapeSlackEntities(node.getAttribute("alt") || "image"));
       return `<${src}|${alt}>`;
     }
   });
   service.addRule("slackTable", {
     filter: "table",
     replacement: (_content, node) => {
-      const plainText = deps.tableToPlainText(node);
+      const plainText = escapeSlackEntities(deps.tableToPlainText(node));
       return `\`\`\`
 ${plainText}
 \`\`\`
@@ -58664,6 +58705,7 @@ function createConfluenceService(deps) {
     codeBlockStyle: "fenced",
     emDelimiter: "_"
   });
+  service.escape = escapeConfluenceText;
   deps.addPdfPageMarkerRules(service);
   service.addRule("confluenceHeading", {
     filter: ["h1", "h2", "h3", "h4", "h5", "h6"],
@@ -58754,7 +58796,10 @@ ${codeContent}
       const fontFamily = style.toLowerCase();
       return fontFamily.includes("monospace") || fontFamily.includes("courier") || fontFamily.includes("consolas") || fontFamily.includes("monaco") || fontFamily.includes("menlo") || fontFamily.includes("source code") || fontFamily.includes("fira code") || fontFamily.includes("roboto mono") || fontFamily.includes("ubuntu mono");
     },
-    replacement: (content) => content.trim() ? `{{${content.trim().replace(/}+/g, (m) => m.split("").join(" "))}}}` : content
+    replacement: (content, node) => {
+      const code = (node.textContent || "").trim();
+      return code ? `{{${code.replace(/}+/g, (m) => m.split("").join(" "))}}}` : content;
+    }
   });
   service.addRule("confluenceLink", {
     filter: (node) => node.nodeName === "A" && !!node.getAttribute("href"),
@@ -61636,6 +61681,9 @@ function localize2(translate, key, vars) {
   if (key === "pdf.warnings.ocrPageSkippedTooLarge") {
     return `Page(s) ${vars.pages} are too large to OCR and were skipped.`;
   }
+  if (key === "pdf.warnings.ocrUnavailablePages") {
+    return `Page(s) ${vars.pages} could not be fully read: they have little or no extractable text and OCR is not available in this environment, so their content may be missing or incomplete.`;
+  }
   return key;
 }
 var PdfAdapter = class {
@@ -62039,6 +62087,9 @@ var PdfAdapter = class {
           }
         }
       }
+      if (routedPages.length > 0 && !fullyScanned && !ocrRan) {
+        pushCodedWarning("pdf.warnings.ocrUnavailablePages", routedPages);
+      }
       let documentHasUsableText = false;
       if (ocrRan) {
         const { isUsableText: isUsableText2 } = await Promise.resolve().then(() => (init_pdf_ocr_adapter(), pdf_ocr_adapter_exports));
@@ -63941,7 +63992,7 @@ async function binaryToMarkdown(buffer, type, format) {
         preserveLists: true,
         customRules: []
       });
-      return { markdown: converted.markdown, warnings: result.warnings };
+      return { markdown: converted.markdown, warnings: result.warnings, warningCodes: result.warningCodes };
     }
     case "pdf": {
       const result = await pdfAdapter.parseToText(toArrayBuffer());
@@ -63952,7 +64003,7 @@ async function binaryToMarkdown(buffer, type, format) {
         preserveLists: true,
         customRules: []
       });
-      return { markdown: converted.markdown, warnings: result.warnings };
+      return { markdown: converted.markdown, warnings: result.warnings, warningCodes: result.warningCodes };
     }
     case "xlsx":
     case "csv": {
@@ -63964,7 +64015,7 @@ async function binaryToMarkdown(buffer, type, format) {
         preserveLists: true,
         customRules: []
       });
-      return { markdown: converted.markdown, warnings: result.warnings };
+      return { markdown: converted.markdown, warnings: result.warnings, warningCodes: result.warningCodes };
     }
     case "json": {
       const result = await dataAdapter.parseJsonToHtml(toArrayBuffer());
@@ -63975,7 +64026,7 @@ async function binaryToMarkdown(buffer, type, format) {
         preserveLists: true,
         customRules: []
       });
-      return { markdown: converted.markdown, warnings: result.warnings };
+      return { markdown: converted.markdown, warnings: result.warnings, warningCodes: result.warningCodes };
     }
     case "xml": {
       const result = await dataAdapter.parseXmlToHtml(toArrayBuffer());
@@ -63986,7 +64037,7 @@ async function binaryToMarkdown(buffer, type, format) {
         preserveLists: true,
         customRules: []
       });
-      return { markdown: converted.markdown, warnings: result.warnings };
+      return { markdown: converted.markdown, warnings: result.warnings, warningCodes: result.warningCodes };
     }
     case "pptx": {
       const result = await pptxAdapter.parseToHtml(toArrayBuffer());
@@ -63997,7 +64048,7 @@ async function binaryToMarkdown(buffer, type, format) {
         preserveLists: true,
         customRules: []
       });
-      return { markdown: converted.markdown, warnings: result.warnings };
+      return { markdown: converted.markdown, warnings: result.warnings, warningCodes: result.warningCodes };
     }
   }
 }
@@ -64111,6 +64162,7 @@ async function convertCommand(options) {
   }
   let markdown;
   let conversionWarnings = [];
+  let conversionWarningCodes = [];
   if (options.input) {
     let fileBuffer;
     try {
@@ -64139,6 +64191,7 @@ async function convertCommand(options) {
       const binary = await binaryToMarkdown(fileBuffer, inputType, format);
       markdown = binary.markdown;
       conversionWarnings = binary.warnings;
+      conversionWarningCodes = binary.warningCodes ?? [];
     }
   } else {
     const html = await readStdin();
@@ -64170,6 +64223,13 @@ async function convertCommand(options) {
   } else {
     process.stdout.write(markdown);
   }
+  if (options.strict && conversionWarnings.length > 0) {
+    const codes = [...new Set(conversionWarningCodes.map((w) => w.code))];
+    const codeList = codes.length > 0 ? ` [${codes.join(", ")}]` : "";
+    console.error(`Error: --strict: conversion produced ${conversionWarnings.length} warning(s)${codeList}.`);
+    process.exitCode = 1;
+    return;
+  }
 }
 
 // src/commands/batch.ts
@@ -64195,38 +64255,38 @@ async function convertFile(filePath, format) {
       preserveLists: true,
       customRules: []
     });
-    return result.markdown;
+    return { markdown: result.markdown, warnings: [] };
   }
   const arrayBuffer = buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength);
   if (ext === ".docx") {
     const r = await docxAdapter.parseToHtml(arrayBuffer);
     const c = await engine.convert(sanitizeHTML(r.html), { format, preserveTables: true, preserveLists: true, customRules: [] });
-    return c.markdown;
+    return { markdown: c.markdown, warnings: r.warnings ?? [] };
   }
   if (ext === ".pdf") {
     const r = await pdfAdapter.parseToText(arrayBuffer);
     const c = await engine.convert(sanitizeHTML(r.html), { format, preserveTables: true, preserveLists: true, customRules: [] });
-    return c.markdown;
+    return { markdown: c.markdown, warnings: r.warnings ?? [] };
   }
   if (ext === ".xlsx" || ext === ".csv") {
     const r = await spreadsheetAdapter.parseToHtml(arrayBuffer, ext.slice(1), 0);
     const c = await engine.convert(sanitizeHTML(r.html), { format, preserveTables: true, preserveLists: true, customRules: [] });
-    return c.markdown;
+    return { markdown: c.markdown, warnings: r.warnings ?? [] };
   }
   if (ext === ".json") {
     const r = await dataAdapter.parseJsonToHtml(arrayBuffer);
     const c = await engine.convert(sanitizeHTML(r.html), { format, preserveTables: true, preserveLists: true, customRules: [] });
-    return c.markdown;
+    return { markdown: c.markdown, warnings: r.warnings ?? [] };
   }
   if (ext === ".xml") {
     const r = await dataAdapter.parseXmlToHtml(arrayBuffer);
     const c = await engine.convert(sanitizeHTML(r.html), { format, preserveTables: true, preserveLists: true, customRules: [] });
-    return c.markdown;
+    return { markdown: c.markdown, warnings: r.warnings ?? [] };
   }
   if (ext === ".pptx") {
     const r = await pptxAdapter.parseToHtml(arrayBuffer);
     const c = await engine.convert(sanitizeHTML(r.html), { format, preserveTables: true, preserveLists: true, customRules: [] });
-    return c.markdown;
+    return { markdown: c.markdown, warnings: r.warnings ?? [] };
   }
   throw new Error(`Unsupported file type: ${ext}`);
 }
@@ -64268,11 +64328,17 @@ async function batchCommand(options) {
   }
   const format = options.format ?? "github";
   if (options.outputDir) mkdirSync2(options.outputDir, { recursive: true });
-  let successful = 0, failed = 0;
+  let successful = 0, failed = 0, batchWarnings = 0;
   let totalOriginal = 0, totalCompressed = 0, toonifiedCount = 0;
   for (const filePath of files) {
     try {
-      let markdown = await convertFile(filePath, format);
+      const converted = await convertFile(filePath, format);
+      for (const w of converted.warnings) {
+        process.stderr.write(`Warning (${basename(filePath)}): ${w}
+`);
+      }
+      batchWarnings += converted.warnings.length;
+      let markdown = converted.markdown;
       if (options.toonify) {
         if (markdown.trim()) {
           if (Buffer.byteLength(markdown, "utf8") > TOONIFY_MAX_INPUT_BYTES) {
@@ -64311,6 +64377,11 @@ async function batchCommand(options) {
   process.stderr.write(`
 Batch complete: ${successful} succeeded, ${failed} failed, ${files.length} total
 `);
+  if (options.strict && batchWarnings > 0) {
+    process.stderr.write(`Error: --strict: conversion produced ${batchWarnings} warning(s) across the batch.
+`);
+    process.exitCode = 1;
+  }
 }
 
 // src/commands/auth.ts
@@ -64518,8 +64589,8 @@ import { readFileSync as readFileSync5 } from "fs";
 import { resolve, dirname as dirname3 } from "path";
 import { fileURLToPath } from "url";
 var getVersion = () => {
-  if ("1.0.29") {
-    return "1.0.29";
+  if ("1.0.31") {
+    return "1.0.31";
   }
   try {
     const __dirname2 = dirname3(fileURLToPath(import.meta.url));
@@ -64532,7 +64603,7 @@ var getVersion = () => {
   }
 };
 var VERSION = getVersion();
-program.name("formatferry").version(VERSION).description("Convert HTML, DOCX, PDF, XLSX, CSV, JSON, XML, PPTX to Markdown").option("-i, --input <path>", "Input file path").option("-o, --output <path>", "Output file path").option("-f, --format <format>", "Markdown format (github, commonmark, slack, discord, reddit, confluence, gdocs, rmarkdown)", "github").option("--url <url>", "Fetch article from URL").option("--batch <glob>", "Batch convert files (Premium)").option("--output-dir <dir>", "Output directory for batch").option("--toonify", "Compress output to TOON format (Premium)").option("--offline", "Skip entitlement check (use cached or free tier)").option("--timeout <ms>", "License server timeout in milliseconds (default: 2000)", "2000").action(async (options) => {
+program.name("formatferry").version(VERSION).description("Convert HTML, DOCX, PDF, XLSX, CSV, JSON, XML, PPTX to Markdown").option("-i, --input <path>", "Input file path").option("-o, --output <path>", "Output file path").option("-f, --format <format>", "Markdown format (github, commonmark, slack, discord, reddit, confluence, gdocs, rmarkdown)", "github").option("--url <url>", "Fetch article from URL").option("--batch <glob>", "Batch convert files (Premium)").option("--output-dir <dir>", "Output directory for batch").option("--toonify", "Compress output to TOON format (Premium)").option("--offline", "Skip entitlement check (use cached or free tier)").option("--timeout <ms>", "License server timeout in milliseconds (default: 2000)", "2000").option("--strict", "Exit with code 1 if the conversion produced any warnings (including advisory ones)").action(async (options) => {
   if (options.batch) {
     await batchCommand(options);
   } else {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "formatferry",
-  "version": "1.0.29",
+  "version": "1.0.31",
   "description": "FormatFerry CLI — Convert HTML, DOCX, PDF, XLSX, CSV, JSON, XML, PPTX to Markdown",
   "keywords": [
     "markdown",