formatferry 1.0.40 → 1.0.41

package/dist/index.js

@@ -58011,6 +58011,153 @@ import DOMPurifyImport from "dompurify";
 // ../../src/shared/constants.ts
 var MAX_INTERMEDIATE_HTML_CHARS = 256 * 1024 * 1024;
 
+// ../../src/shared/linkedom-sanitizer.ts
+var FORBID_CONTENTS = /* @__PURE__ */ new Set([
+  "annotation-xml",
+  "audio",
+  "colgroup",
+  "desc",
+  "foreignobject",
+  "head",
+  "iframe",
+  "math",
+  "mi",
+  "mn",
+  "mo",
+  "ms",
+  "mtext",
+  "noembed",
+  "noframes",
+  "noscript",
+  "plaintext",
+  "script",
+  "selectedcontent",
+  "style",
+  "svg",
+  "template",
+  "thead",
+  "title",
+  "video",
+  "xmp"
+]);
+var BLOCK_UNWRAP_TAGS = /* @__PURE__ */ new Set([
+  "address",
+  "article",
+  "aside",
+  "center",
+  "details",
+  "dialog",
+  "dir",
+  "dd",
+  "dl",
+  "dt",
+  "fieldset",
+  "figcaption",
+  "figure",
+  "footer",
+  "form",
+  "header",
+  "hgroup",
+  "legend",
+  "main",
+  "menu",
+  "nav",
+  "search",
+  "section",
+  "summary"
+]);
+var DATA_URI_TAGS = /* @__PURE__ */ new Set(["audio", "video", "img", "source", "image", "track"]);
+var URI_SAFE_ATTRIBUTES = /* @__PURE__ */ new Set([
+  "alt",
+  "class",
+  "for",
+  "id",
+  "label",
+  "name",
+  "pattern",
+  "placeholder",
+  "role",
+  "summary",
+  "title",
+  "value",
+  "style",
+  "xmlns"
+]);
+var IS_ALLOWED_URI = /^(?:(?:(?:f|ht)tps?|mailto|tel|callto|sms|cid|xmpp|matrix):|[^a-z]|[a-z+.\-]+(?:[^a-z+.\-:]|$))/i;
+var ATTR_WHITESPACE = /[\u0000-\u0020\u00A0\u1680\u180E\u2000-\u2029\u205F\u3000]/g;
+var ELEMENT_NODE = 1;
+var COMMENT_NODE = 8;
+function toSet(list) {
+  return new Set((list ?? []).map((s) => s.toLowerCase()));
+}
+function attrAllowed(name, allowed, wildcards) {
+  return allowed.has(name) || wildcards.some((prefix) => name.startsWith(prefix));
+}
+function isSafeValue(tag, name, value) {
+  if (URI_SAFE_ATTRIBUTES.has(name) || !value) return true;
+  if (IS_ALLOWED_URI.test(value.replace(ATTR_WHITESPACE, ""))) return true;
+  return (name === "src" || name === "href" || name === "xlink:href") && value.startsWith("data:") && DATA_URI_TAGS.has(tag);
+}
+function sanitizeWithLinkedom(html, config, parse) {
+  if (!html) return "";
+  const allowedTags = toSet(config.ALLOWED_TAGS);
+  const forbidTags = toSet(config.FORBID_TAGS);
+  const allowedAttrList = (config.ALLOWED_ATTR ?? []).map((s) => s.toLowerCase());
+  const allowedAttrs = new Set(allowedAttrList.filter((a) => !a.endsWith("*")));
+  const attrWildcards = allowedAttrList.filter((a) => a.endsWith("*")).map((a) => a.slice(0, -1));
+  const forbidAttrs = toSet(config.FORBID_ATTR);
+  const doc = parse(`<!DOCTYPE html><html><head></head><body>${html}</body></html>`);
+  const body = doc.body;
+  if (!body) return "";
+  const toUnwrap = [];
+  const hadChildren = /* @__PURE__ */ new Set();
+  const stack = [body];
+  while (stack.length > 0) {
+    const parent = stack.pop();
+    for (const child of Array.from(parent.childNodes)) {
+      if (child.nodeType === COMMENT_NODE) {
+        parent.removeChild(child);
+        continue;
+      }
+      if (child.nodeType !== ELEMENT_NODE) continue;
+      const el = child;
+      const tag = el.nodeName.toLowerCase();
+      if (!allowedTags.has(tag) || forbidTags.has(tag)) {
+        if (FORBID_CONTENTS.has(tag)) {
+          parent.removeChild(el);
+        } else {
+          toUnwrap.push(el);
+          if (el.firstChild) hadChildren.add(el);
+          stack.push(el);
+        }
+        continue;
+      }
+      for (const attr of Array.from(el.attributes)) {
+        const name = attr.name.toLowerCase();
+        if (name.startsWith("on") || forbidAttrs.has(name) || !attrAllowed(name, allowedAttrs, attrWildcards) || !isSafeValue(tag, name, attr.value)) {
+          el.removeAttribute(attr.name);
+        }
+      }
+      stack.push(el);
+    }
+  }
+  const wrapBlocks = allowedTags.has("div") && !forbidTags.has("div");
+  for (let i = toUnwrap.length - 1; i >= 0; i--) {
+    const el = toUnwrap[i];
+    const parent = el.parentNode;
+    if (!parent) continue;
+    if (wrapBlocks && hadChildren.has(el) && BLOCK_UNWRAP_TAGS.has(el.nodeName.toLowerCase())) {
+      const div = doc.createElement("div");
+      while (el.firstChild) div.appendChild(el.firstChild);
+      parent.replaceChild(div, el);
+      continue;
+    }
+    while (el.firstChild) parent.insertBefore(el.firstChild, el);
+    parent.removeChild(el);
+  }
+  return body.innerHTML;
+}
+
 // ../../src/shared/input-sanitizer.ts
 function resolveDOMPurify() {
   const shimInstance = globalThis.__FF_DOMPurify;
@@ -58056,12 +58203,42 @@ var InputSanitizer = class {
     if (!html || typeof html !== "string") {
       return "";
     }
-    return this.domEnv.DOMPurify.sanitize(html, {
-      ALLOWED_TAGS: this.purifyConfig.ALLOWED_TAGS,
-      ALLOWED_ATTR: this.purifyConfig.ALLOWED_ATTR,
-      FORBID_TAGS: this.purifyConfig.FORBID_TAGS,
-      FORBID_ATTR: this.purifyConfig.FORBID_ATTR
-    });
+    const purify = this.domEnv.DOMPurify;
+    const hook = this.blockUnwrapHook();
+    const useHook = hook && purify.addHook && purify.removeHook;
+    if (useHook) purify.addHook("uponSanitizeElement", hook);
+    try {
+      return purify.sanitize(html, {
+        ALLOWED_TAGS: this.purifyConfig.ALLOWED_TAGS,
+        ALLOWED_ATTR: this.purifyConfig.ALLOWED_ATTR,
+        FORBID_TAGS: this.purifyConfig.FORBID_TAGS,
+        FORBID_ATTR: this.purifyConfig.FORBID_ATTR
+      });
+    } finally {
+      if (useHook) purify.removeHook("uponSanitizeElement", hook);
+    }
+  }
+  /**
+   * DOMPurify unwraps a disallowed element by moving its children up, so
+   * `<header>Head</header><nav>Nav</nav>` came out as "HeadNav". For a
+   * block-level element, move its children into a <div> first: DOMPurify then
+   * hoists that <div>, and Turndown gives the content its own paragraph. Same
+   * rule as the linkedom sanitiser (BLOCK_UNWRAP_TAGS, #177).
+   */
+  blockUnwrapHook() {
+    const allowed = new Set((this.purifyConfig.ALLOWED_TAGS ?? []).map((t) => t.toLowerCase()));
+    const forbidden = new Set((this.purifyConfig.FORBID_TAGS ?? []).map((t) => t.toLowerCase()));
+    if (!allowed.has("div") || forbidden.has("div")) return void 0;
+    return (node, data) => {
+      const tag = data.tagName.toLowerCase();
+      if (!BLOCK_UNWRAP_TAGS.has(tag) || allowed.has(tag) && !forbidden.has(tag)) return;
+      const el = node;
+      const doc = el.ownerDocument;
+      if (!doc || !el.firstChild) return;
+      const div = doc.createElement("div");
+      while (el.firstChild) div.appendChild(el.firstChild);
+      el.appendChild(div);
+    };
   }
   /**
    * Sanitize HTML content to prevent XSS attacks
@@ -58238,6 +58415,13 @@ var InputSanitizer = class {
         "th",
         "figure",
         "figcaption",
+        // Definition lists and disclosure widgets (#177): unwrapped, their
+        // text ran together ("TermDefinition", "MoreHidden").
+        "dl",
+        "dt",
+        "dd",
+        "details",
+        "summary",
         "a",
         "img",
         "sup",
@@ -59128,7 +59312,21 @@ var ConversionEngine = class {
       this.rmarkdownService,
       this.slackService,
       this.confluenceService
-    ].forEach((service) => this.addGoogleDocsWrapperRule(service));
+    ].forEach((service) => {
+      this.addGoogleDocsWrapperRule(service);
+      this.addDisclosureBlockRule(service);
+    });
+  }
+  /**
+   * Turndown's block list has no DETAILS or SUMMARY, so it treated them as
+   * inline and `<details><summary>More</summary>Hidden</details>` came out as
+   * "MoreHidden". Give them block spacing like a <div> (#177).
+   */
+  addDisclosureBlockRule(service) {
+    service.addRule("disclosureBlock", {
+      filter: ["details", "summary"],
+      replacement: (content) => "\n\n" + content + "\n\n"
+    });
   }
   /**
    * Google Docs wraps every copied selection in
@@ -63814,118 +64012,6 @@ function buildCachedProvider(config, lastValidated) {
   return new CachedEntitlementProvider(cachedState);
 }
 
-// ../../src/shared/linkedom-sanitizer.ts
-var FORBID_CONTENTS = /* @__PURE__ */ new Set([
-  "annotation-xml",
-  "audio",
-  "colgroup",
-  "desc",
-  "foreignobject",
-  "head",
-  "iframe",
-  "math",
-  "mi",
-  "mn",
-  "mo",
-  "ms",
-  "mtext",
-  "noembed",
-  "noframes",
-  "noscript",
-  "plaintext",
-  "script",
-  "selectedcontent",
-  "style",
-  "svg",
-  "template",
-  "thead",
-  "title",
-  "video",
-  "xmp"
-]);
-var DATA_URI_TAGS = /* @__PURE__ */ new Set(["audio", "video", "img", "source", "image", "track"]);
-var URI_SAFE_ATTRIBUTES = /* @__PURE__ */ new Set([
-  "alt",
-  "class",
-  "for",
-  "id",
-  "label",
-  "name",
-  "pattern",
-  "placeholder",
-  "role",
-  "summary",
-  "title",
-  "value",
-  "style",
-  "xmlns"
-]);
-var IS_ALLOWED_URI = /^(?:(?:(?:f|ht)tps?|mailto|tel|callto|sms|cid|xmpp|matrix):|[^a-z]|[a-z+.\-]+(?:[^a-z+.\-:]|$))/i;
-var ATTR_WHITESPACE = /[\u0000-\u0020\u00A0\u1680\u180E\u2000-\u2029\u205F\u3000]/g;
-var ELEMENT_NODE = 1;
-var COMMENT_NODE = 8;
-function toSet(list) {
-  return new Set((list ?? []).map((s) => s.toLowerCase()));
-}
-function attrAllowed(name, allowed, wildcards) {
-  return allowed.has(name) || wildcards.some((prefix) => name.startsWith(prefix));
-}
-function isSafeValue(tag, name, value) {
-  if (URI_SAFE_ATTRIBUTES.has(name) || !value) return true;
-  if (IS_ALLOWED_URI.test(value.replace(ATTR_WHITESPACE, ""))) return true;
-  return (name === "src" || name === "href" || name === "xlink:href") && value.startsWith("data:") && DATA_URI_TAGS.has(tag);
-}
-function sanitizeWithLinkedom(html, config, parse) {
-  if (!html) return "";
-  const allowedTags = toSet(config.ALLOWED_TAGS);
-  const forbidTags = toSet(config.FORBID_TAGS);
-  const allowedAttrList = (config.ALLOWED_ATTR ?? []).map((s) => s.toLowerCase());
-  const allowedAttrs = new Set(allowedAttrList.filter((a) => !a.endsWith("*")));
-  const attrWildcards = allowedAttrList.filter((a) => a.endsWith("*")).map((a) => a.slice(0, -1));
-  const forbidAttrs = toSet(config.FORBID_ATTR);
-  const doc = parse(`<!DOCTYPE html><html><head></head><body>${html}</body></html>`);
-  const body = doc.body;
-  if (!body) return "";
-  const toUnwrap = [];
-  const stack = [body];
-  while (stack.length > 0) {
-    const parent = stack.pop();
-    for (const child of Array.from(parent.childNodes)) {
-      if (child.nodeType === COMMENT_NODE) {
-        parent.removeChild(child);
-        continue;
-      }
-      if (child.nodeType !== ELEMENT_NODE) continue;
-      const el = child;
-      const tag = el.nodeName.toLowerCase();
-      if (!allowedTags.has(tag) || forbidTags.has(tag)) {
-        if (FORBID_CONTENTS.has(tag)) {
-          parent.removeChild(el);
-        } else {
-          toUnwrap.push(el);
-          stack.push(el);
-        }
-        continue;
-      }
-      for (const attr of Array.from(el.attributes)) {
-        const name = attr.name.toLowerCase();
-        if (name.startsWith("on") || forbidAttrs.has(name) || !attrAllowed(name, allowedAttrs, attrWildcards) || !isSafeValue(tag, name, attr.value)) {
-          el.removeAttribute(attr.name);
-        }
-      }
-      stack.push(el);
-    }
-  }
-  for (let i = toUnwrap.length - 1; i >= 0; i--) {
-    const el = toUnwrap[i];
-    const parent = el.parentNode;
-    if (!parent) continue;
-    while (el.firstChild) parent.insertBefore(el.firstChild, el);
-    parent.removeChild(el);
-  }
-  return body.innerHTML;
-}
-
 // ../../src/shared/token-estimator.ts
 var CJK_REGEX3 = /[\u4e00-\u9fff\u3400-\u4dbf\u3000-\u303f\u3040-\u309f\u30a0-\u30ff\uac00-\ud7af]/g;
 var MAX_INPUT_LENGTH = 1e6;
@@ -64390,7 +64476,7 @@ function readStdin() {
 }
 async function htmlToMarkdown(html, format, applyAtlassianNormalization = false) {
   const processedHtml = applyAtlassianNormalization ? normalizeAtlassianHTML(html) : html;
-  const engine = new ConversionEngine();
+  const engine = new ConversionEngine({ dom: linkedomDomEnvironment() });
   const result = await engine.convert(processedHtml, {
     format,
     preserveTables: true,
@@ -64404,12 +64490,11 @@ async function binaryToMarkdown(buffer, type, format) {
     buffer.byteOffset,
     buffer.byteOffset + buffer.byteLength
   );
-  const engine = new ConversionEngine();
+  const engine = new ConversionEngine({ dom: linkedomDomEnvironment() });
   switch (type) {
     case "docx": {
       const result = await docxAdapter.parseToHtml(buffer);
-      const sanitized = sanitizeHTML(result.html);
-      const converted = await engine.convert(sanitized, {
+      const converted = await engine.convert(result.html, {
         format,
         preserveTables: true,
         preserveLists: true,
@@ -64419,8 +64504,7 @@ async function binaryToMarkdown(buffer, type, format) {
     }
     case "pdf": {
       const result = await pdfAdapter.parseToText(toArrayBuffer());
-      const sanitized = sanitizeHTML(result.html);
-      const converted = await engine.convert(sanitized, {
+      const converted = await engine.convert(result.html, {
         format,
         preserveTables: true,
         preserveLists: true,
@@ -64431,8 +64515,7 @@ async function binaryToMarkdown(buffer, type, format) {
     case "xlsx":
     case "csv": {
       const result = await spreadsheetAdapter.parseToHtml(toArrayBuffer(), type, 0);
-      const sanitized = sanitizeHTML(result.html);
-      const converted = await engine.convert(sanitized, {
+      const converted = await engine.convert(result.html, {
         format,
         preserveTables: true,
         preserveLists: true,
@@ -64442,8 +64525,7 @@ async function binaryToMarkdown(buffer, type, format) {
     }
     case "json": {
       const result = await dataAdapter.parseJsonToHtml(toArrayBuffer());
-      const sanitized = sanitizeHTML(result.html);
-      const converted = await engine.convert(sanitized, {
+      const converted = await engine.convert(result.html, {
         format,
         preserveTables: true,
         preserveLists: true,
@@ -64453,8 +64535,7 @@ async function binaryToMarkdown(buffer, type, format) {
     }
     case "xml": {
       const result = await dataAdapter.parseXmlToHtml(toArrayBuffer());
-      const sanitized = sanitizeHTML(result.html);
-      const converted = await engine.convert(sanitized, {
+      const converted = await engine.convert(result.html, {
         format,
         preserveTables: true,
         preserveLists: true,
@@ -64464,8 +64545,7 @@ async function binaryToMarkdown(buffer, type, format) {
     }
     case "pptx": {
       const result = await pptxAdapter.parseToHtml(toArrayBuffer());
-      const sanitized = sanitizeHTML(result.html);
-      const converted = await engine.convert(sanitized, {
+      const converted = await engine.convert(result.html, {
         format,
         preserveTables: true,
         preserveLists: true,
@@ -64477,7 +64557,7 @@ async function binaryToMarkdown(buffer, type, format) {
 }
 function linkedomDomEnvironment() {
   const parseHTML2 = globalThis.__FF_parseHTML;
-  if (!parseHTML2) throw new Error("The CLI shim must be loaded before converting a URL");
+  if (!parseHTML2) throw new Error("The CLI shim must be loaded before converting");
   const parseDocument = (html) => parseHTML2(html).document;
   const { window: window3 } = parseHTML2("<!DOCTYPE html><html><head></head><body></body></html>");
   return {
@@ -64685,9 +64765,9 @@ async function convertFile(filePath, format) {
   const ext = extname2(filePath).toLowerCase();
   if (!SUPPORTED_EXTENSIONS.has(ext)) throw new Error(`Unsupported file type: ${ext}`);
   const buffer = readFileSync4(filePath);
-  const engine = new ConversionEngine();
+  const engine = new ConversionEngine({ dom: linkedomDomEnvironment() });
   if (ext === ".html" || ext === ".htm") {
-    const result = await engine.convert(sanitizeHTML(buffer.toString("utf8")), {
+    const result = await engine.convert(buffer.toString("utf8"), {
       format,
       preserveTables: true,
       preserveLists: true,
@@ -64698,32 +64778,32 @@ async function convertFile(filePath, format) {
   const arrayBuffer = buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength);
   if (ext === ".docx") {
     const r = await docxAdapter.parseToHtml(arrayBuffer);
-    const c = await engine.convert(sanitizeHTML(r.html), { format, preserveTables: true, preserveLists: true, customRules: [] });
+    const c = await engine.convert(r.html, { format, preserveTables: true, preserveLists: true, customRules: [] });
     return { markdown: c.markdown, warnings: r.warnings ?? [] };
   }
   if (ext === ".pdf") {
     const r = await pdfAdapter.parseToText(arrayBuffer);
-    const c = await engine.convert(sanitizeHTML(r.html), { format, preserveTables: true, preserveLists: true, customRules: [] });
+    const c = await engine.convert(r.html, { format, preserveTables: true, preserveLists: true, customRules: [] });
     return { markdown: c.markdown, warnings: r.warnings ?? [] };
   }
   if (ext === ".xlsx" || ext === ".csv") {
     const r = await spreadsheetAdapter.parseToHtml(arrayBuffer, ext.slice(1), 0);
-    const c = await engine.convert(sanitizeHTML(r.html), { format, preserveTables: true, preserveLists: true, customRules: [] });
+    const c = await engine.convert(r.html, { format, preserveTables: true, preserveLists: true, customRules: [] });
     return { markdown: c.markdown, warnings: r.warnings ?? [] };
   }
   if (ext === ".json") {
     const r = await dataAdapter.parseJsonToHtml(arrayBuffer);
-    const c = await engine.convert(sanitizeHTML(r.html), { format, preserveTables: true, preserveLists: true, customRules: [] });
+    const c = await engine.convert(r.html, { format, preserveTables: true, preserveLists: true, customRules: [] });
     return { markdown: c.markdown, warnings: r.warnings ?? [] };
   }
   if (ext === ".xml") {
     const r = await dataAdapter.parseXmlToHtml(arrayBuffer);
-    const c = await engine.convert(sanitizeHTML(r.html), { format, preserveTables: true, preserveLists: true, customRules: [] });
+    const c = await engine.convert(r.html, { format, preserveTables: true, preserveLists: true, customRules: [] });
     return { markdown: c.markdown, warnings: r.warnings ?? [] };
   }
   if (ext === ".pptx") {
     const r = await pptxAdapter.parseToHtml(arrayBuffer);
-    const c = await engine.convert(sanitizeHTML(r.html), { format, preserveTables: true, preserveLists: true, customRules: [] });
+    const c = await engine.convert(r.html, { format, preserveTables: true, preserveLists: true, customRules: [] });
     return { markdown: c.markdown, warnings: r.warnings ?? [] };
   }
   throw new Error(`Unsupported file type: ${ext}`);
@@ -65027,8 +65107,8 @@ import { readFileSync as readFileSync5 } from "fs";
 import { resolve, dirname as dirname3 } from "path";
 import { fileURLToPath } from "url";
 var getVersion = () => {
-  if ("1.0.40") {
-    return "1.0.40";
+  if ("1.0.41") {
+    return "1.0.41";
   }
   try {
     const __dirname2 = dirname3(fileURLToPath(import.meta.url));

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "formatferry",
-  "version": "1.0.40",
+  "version": "1.0.41",
   "description": "Format Ferry CLI — Convert HTML, DOCX, PDF, XLSX, CSV, JSON, XML, PPTX to Markdown",
   "keywords": [
     "markdown",