formatferry 1.0.33 → 1.0.34

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (3) hide show
  1. package/README.md +2 -0
  2. package/dist/index.js +237 -51
  3. package/package.json +1 -1
package/README.md CHANGED
@@ -103,6 +103,8 @@ formatferry [options]
103
103
  | `-o, --output <path>` | Output file path (defaults to stdout) |
104
104
  | `-f, --format <format>` | Markdown format (default: `github`) |
105
105
  | `--url <url>` | Fetch and convert a web article (requires API key) |
106
+
107
+ `--url` asks the proxy for the page HTML and converts it locally, so `--format` chooses the Markdown flavour exactly as it does for a local file. The proxy also offers the finished Markdown directly: `POST /v1/extract` with `{ "format": "markdown" }` (see the root README).
106
108
  | `--batch <glob>` | Batch convert files matching a glob pattern (Premium) |
107
109
  | `--output-dir <dir>` | Output directory for batch mode |
108
110
  | `--strict` | Exit with code 1 if the conversion produced any warnings (including advisory ones) |
package/dist/index.js CHANGED
@@ -948,14 +948,14 @@ var require_Node = __commonJS({
948
948
  this._nextSibling = this._previousSibling = this;
949
949
  this._index = void 0;
950
950
  }
951
- var ELEMENT_NODE = Node2.ELEMENT_NODE = 1;
951
+ var ELEMENT_NODE2 = Node2.ELEMENT_NODE = 1;
952
952
  var ATTRIBUTE_NODE = Node2.ATTRIBUTE_NODE = 2;
953
953
  var TEXT_NODE = Node2.TEXT_NODE = 3;
954
954
  var CDATA_SECTION_NODE = Node2.CDATA_SECTION_NODE = 4;
955
955
  var ENTITY_REFERENCE_NODE = Node2.ENTITY_REFERENCE_NODE = 5;
956
956
  var ENTITY_NODE = Node2.ENTITY_NODE = 6;
957
957
  var PROCESSING_INSTRUCTION_NODE = Node2.PROCESSING_INSTRUCTION_NODE = 7;
958
- var COMMENT_NODE = Node2.COMMENT_NODE = 8;
958
+ var COMMENT_NODE2 = Node2.COMMENT_NODE = 8;
959
959
  var DOCUMENT_NODE = Node2.DOCUMENT_NODE = 9;
960
960
  var DOCUMENT_TYPE_NODE = Node2.DOCUMENT_TYPE_NODE = 10;
961
961
  var DOCUMENT_FRAGMENT_NODE = Node2.DOCUMENT_FRAGMENT_NODE = 11;
@@ -973,7 +973,7 @@ var require_Node = __commonJS({
973
973
  // we'll come back to this later.
974
974
  baseURI: { get: utils2.nyi },
975
975
  parentElement: { get: function() {
976
- return this.parentNode && this.parentNode.nodeType === ELEMENT_NODE ? this.parentNode : null;
976
+ return this.parentNode && this.parentNode.nodeType === ELEMENT_NODE2 ? this.parentNode : null;
977
977
  } },
978
978
  hasChildNodes: { value: utils2.shouldOverride },
979
979
  firstChild: { get: utils2.shouldOverride },
@@ -1034,7 +1034,7 @@ var require_Node = __commonJS({
1034
1034
  switch (parent.nodeType) {
1035
1035
  case DOCUMENT_NODE:
1036
1036
  case DOCUMENT_FRAGMENT_NODE:
1037
- case ELEMENT_NODE:
1037
+ case ELEMENT_NODE2:
1038
1038
  break;
1039
1039
  default:
1040
1040
  utils2.HierarchyRequestError();
@@ -1046,10 +1046,10 @@ var require_Node = __commonJS({
1046
1046
  switch (node.nodeType) {
1047
1047
  case DOCUMENT_FRAGMENT_NODE:
1048
1048
  case DOCUMENT_TYPE_NODE:
1049
- case ELEMENT_NODE:
1049
+ case ELEMENT_NODE2:
1050
1050
  case TEXT_NODE:
1051
1051
  case PROCESSING_INSTRUCTION_NODE:
1052
- case COMMENT_NODE:
1052
+ case COMMENT_NODE2:
1053
1053
  break;
1054
1054
  default:
1055
1055
  utils2.HierarchyRequestError();
@@ -1062,7 +1062,7 @@ var require_Node = __commonJS({
1062
1062
  case DOCUMENT_FRAGMENT_NODE:
1063
1063
  if (node._countChildrenOfType(TEXT_NODE) > 0)
1064
1064
  utils2.HierarchyRequestError();
1065
- switch (node._countChildrenOfType(ELEMENT_NODE)) {
1065
+ switch (node._countChildrenOfType(ELEMENT_NODE2)) {
1066
1066
  case 0:
1067
1067
  break;
1068
1068
  case 1:
@@ -1074,12 +1074,12 @@ var require_Node = __commonJS({
1074
1074
  utils2.HierarchyRequestError();
1075
1075
  }
1076
1076
  }
1077
- i = parent._countChildrenOfType(ELEMENT_NODE);
1077
+ i = parent._countChildrenOfType(ELEMENT_NODE2);
1078
1078
  if (isPreinsert) {
1079
1079
  if (i > 0)
1080
1080
  utils2.HierarchyRequestError();
1081
1081
  } else {
1082
- if (i > 1 || i === 1 && child.nodeType !== ELEMENT_NODE)
1082
+ if (i > 1 || i === 1 && child.nodeType !== ELEMENT_NODE2)
1083
1083
  utils2.HierarchyRequestError();
1084
1084
  }
1085
1085
  break;
@@ -1087,7 +1087,7 @@ var require_Node = __commonJS({
1087
1087
  utils2.HierarchyRequestError();
1088
1088
  }
1089
1089
  break;
1090
- case ELEMENT_NODE:
1090
+ case ELEMENT_NODE2:
1091
1091
  if (child !== null) {
1092
1092
  if (isPreinsert && child.nodeType === DOCUMENT_TYPE_NODE)
1093
1093
  utils2.HierarchyRequestError();
@@ -1096,23 +1096,23 @@ var require_Node = __commonJS({
1096
1096
  utils2.HierarchyRequestError();
1097
1097
  }
1098
1098
  }
1099
- i = parent._countChildrenOfType(ELEMENT_NODE);
1099
+ i = parent._countChildrenOfType(ELEMENT_NODE2);
1100
1100
  if (isPreinsert) {
1101
1101
  if (i > 0)
1102
1102
  utils2.HierarchyRequestError();
1103
1103
  } else {
1104
- if (i > 1 || i === 1 && child.nodeType !== ELEMENT_NODE)
1104
+ if (i > 1 || i === 1 && child.nodeType !== ELEMENT_NODE2)
1105
1105
  utils2.HierarchyRequestError();
1106
1106
  }
1107
1107
  break;
1108
1108
  case DOCUMENT_TYPE_NODE:
1109
1109
  if (child === null) {
1110
- if (parent._countChildrenOfType(ELEMENT_NODE))
1110
+ if (parent._countChildrenOfType(ELEMENT_NODE2))
1111
1111
  utils2.HierarchyRequestError();
1112
1112
  } else {
1113
1113
  for (kid = parent.firstChild; kid !== null; kid = kid.nextSibling) {
1114
1114
  if (kid === child) break;
1115
- if (kid.nodeType === ELEMENT_NODE)
1115
+ if (kid.nodeType === ELEMENT_NODE2)
1116
1116
  utils2.HierarchyRequestError();
1117
1117
  }
1118
1118
  }
@@ -1229,7 +1229,7 @@ var require_Node = __commonJS({
1229
1229
  var e;
1230
1230
  if (ns === "" || ns === null || ns === void 0) return null;
1231
1231
  switch (this.nodeType) {
1232
- case ELEMENT_NODE:
1232
+ case ELEMENT_NODE2:
1233
1233
  return this._lookupNamespacePrefix(ns, this);
1234
1234
  case DOCUMENT_NODE:
1235
1235
  e = this.documentElement;
@@ -1253,7 +1253,7 @@ var require_Node = __commonJS({
1253
1253
  }
1254
1254
  var e;
1255
1255
  switch (this.nodeType) {
1256
- case ELEMENT_NODE:
1256
+ case ELEMENT_NODE2:
1257
1257
  return utils2.shouldOverride();
1258
1258
  case DOCUMENT_NODE:
1259
1259
  e = this.documentElement;
@@ -1511,14 +1511,14 @@ var require_Node = __commonJS({
1511
1511
  },
1512
1512
  // mirror node type properties in the prototype, so they are present
1513
1513
  // in instances of Node (and subclasses)
1514
- ELEMENT_NODE: { value: ELEMENT_NODE },
1514
+ ELEMENT_NODE: { value: ELEMENT_NODE2 },
1515
1515
  ATTRIBUTE_NODE: { value: ATTRIBUTE_NODE },
1516
1516
  TEXT_NODE: { value: TEXT_NODE },
1517
1517
  CDATA_SECTION_NODE: { value: CDATA_SECTION_NODE },
1518
1518
  ENTITY_REFERENCE_NODE: { value: ENTITY_REFERENCE_NODE },
1519
1519
  ENTITY_NODE: { value: ENTITY_NODE },
1520
1520
  PROCESSING_INSTRUCTION_NODE: { value: PROCESSING_INSTRUCTION_NODE },
1521
- COMMENT_NODE: { value: COMMENT_NODE },
1521
+ COMMENT_NODE: { value: COMMENT_NODE2 },
1522
1522
  DOCUMENT_NODE: { value: DOCUMENT_NODE },
1523
1523
  DOCUMENT_TYPE_NODE: { value: DOCUMENT_TYPE_NODE },
1524
1524
  DOCUMENT_FRAGMENT_NODE: { value: DOCUMENT_FRAGMENT_NODE },
@@ -58046,20 +58046,29 @@ var InputSanitizer = class {
58046
58046
  this.purifyConfig = this.createPurifyConfig();
58047
58047
  }
58048
58048
  /**
58049
- * Sanitize HTML content to prevent XSS attacks
58049
+ * Like sanitizeHTML, but a sanitiser failure is thrown instead of being
58050
+ * turned into an empty string. Use it wherever an empty result would be
58051
+ * read as "the page had no content" (the ConversionEngine on an injected
58052
+ * DOM: the Worker's server-side Markdown and the CLI), so a failure is
58053
+ * reported as a failure (#170).
58050
58054
  */
58051
- sanitizeHTML(html) {
58055
+ sanitizeHTMLOrThrow(html) {
58052
58056
  if (!html || typeof html !== "string") {
58053
58057
  return "";
58054
58058
  }
58059
+ return this.domEnv.DOMPurify.sanitize(html, {
58060
+ ALLOWED_TAGS: this.purifyConfig.ALLOWED_TAGS,
58061
+ ALLOWED_ATTR: this.purifyConfig.ALLOWED_ATTR,
58062
+ FORBID_TAGS: this.purifyConfig.FORBID_TAGS,
58063
+ FORBID_ATTR: this.purifyConfig.FORBID_ATTR
58064
+ });
58065
+ }
58066
+ /**
58067
+ * Sanitize HTML content to prevent XSS attacks
58068
+ */
58069
+ sanitizeHTML(html) {
58055
58070
  try {
58056
- const sanitized = this.domEnv.DOMPurify.sanitize(html, {
58057
- ALLOWED_TAGS: this.purifyConfig.ALLOWED_TAGS,
58058
- ALLOWED_ATTR: this.purifyConfig.ALLOWED_ATTR,
58059
- FORBID_TAGS: this.purifyConfig.FORBID_TAGS,
58060
- FORBID_ATTR: this.purifyConfig.FORBID_ATTR
58061
- });
58062
- return sanitized;
58071
+ return this.sanitizeHTMLOrThrow(html);
58063
58072
  } catch (error) {
58064
58073
  console.warn("HTML sanitization failed:", error);
58065
58074
  return "";
@@ -58312,7 +58321,11 @@ var InputSanitizer = class {
58312
58321
  return { ...this.config };
58313
58322
  }
58314
58323
  };
58315
- var defaultSanitizer = new InputSanitizer();
58324
+ var defaultSanitizerInstance;
58325
+ function getDefaultSanitizer() {
58326
+ defaultSanitizerInstance ??= new InputSanitizer();
58327
+ return defaultSanitizerInstance;
58328
+ }
58316
58329
  function normalizeSlackHTML(html, domParser) {
58317
58330
  if (!html || !html.includes("indent=")) return html;
58318
58331
  const doc = (domParser ?? new globalThis.DOMParser()).parseFromString(html, "text/html");
@@ -58343,7 +58356,7 @@ function normalizeSlackHTML(html, domParser) {
58343
58356
  }
58344
58357
  return root;
58345
58358
  };
58346
- const isListEl = (n) => n.nodeType === Node.ELEMENT_NODE && (n.nodeName === "UL" || n.nodeName === "OL");
58359
+ const isListEl = (n) => n.nodeType === 1 && (n.nodeName === "UL" || n.nodeName === "OL");
58347
58360
  const snapshot = Array.from(body.childNodes);
58348
58361
  let runStart = -1;
58349
58362
  const flushRun = (endIdx) => {
@@ -58377,7 +58390,7 @@ function sanitizeHTML(html, config) {
58377
58390
  const sanitizer = new InputSanitizer(config);
58378
58391
  return sanitizer.sanitizeHTML(html);
58379
58392
  }
58380
- return defaultSanitizer.sanitizeHTML(html);
58393
+ return getDefaultSanitizer().sanitizeHTML(html);
58381
58394
  }
58382
58395
  var PANEL_EMOJI = {
58383
58396
  info: "\u2139\uFE0F Info:",
@@ -58463,11 +58476,11 @@ function normalizeAtlassianTables(doc) {
58463
58476
  }
58464
58477
  }
58465
58478
  function stripUnrecognizedMacros(node) {
58466
- const ELEMENT_NODE = 1;
58479
+ const ELEMENT_NODE2 = 1;
58467
58480
  let i = 0;
58468
58481
  while (i < node.childNodes.length) {
58469
58482
  const child = node.childNodes[i];
58470
- if (child.nodeType === ELEMENT_NODE) {
58483
+ if (child.nodeType === ELEMENT_NODE2) {
58471
58484
  const el = child;
58472
58485
  const tagName = el.tagName.toLowerCase();
58473
58486
  if (tagName.startsWith("ac:") || tagName.startsWith("ri:")) {
@@ -58682,11 +58695,11 @@ function processConfluenceList(list, service, depth) {
58682
58695
  function processConfluenceListItem(item, service, depth) {
58683
58696
  let content = "";
58684
58697
  const TEXT_NODE = 3;
58685
- const ELEMENT_NODE = 1;
58698
+ const ELEMENT_NODE2 = 1;
58686
58699
  Array.from(item.childNodes).forEach((child) => {
58687
58700
  if (child.nodeType === TEXT_NODE) {
58688
58701
  content += child.textContent || "";
58689
- } else if (child.nodeType === ELEMENT_NODE) {
58702
+ } else if (child.nodeType === ELEMENT_NODE2) {
58690
58703
  const el = child;
58691
58704
  if (el.nodeName === "UL" || el.nodeName === "OL") {
58692
58705
  content += "\n" + processConfluenceList(el, service, depth + 1);
@@ -58904,8 +58917,46 @@ var ConversionEngine = class {
58904
58917
  rmarkdownService;
58905
58918
  slackService;
58906
58919
  confluenceService;
58907
- constructor() {
58920
+ dom;
58921
+ domSanitizer;
58922
+ constructor(options = {}) {
58923
+ this.dom = options.dom;
58908
58924
  this.initializeServices();
58925
+ if (this.dom) {
58926
+ for (const service of [
58927
+ this.githubService,
58928
+ this.commonmarkService,
58929
+ this.customService,
58930
+ this.discordService,
58931
+ this.redditService,
58932
+ this.rmarkdownService,
58933
+ this.slackService,
58934
+ this.confluenceService
58935
+ ]) {
58936
+ this.parseWithInjectedDom(service, this.dom);
58937
+ }
58938
+ }
58939
+ }
58940
+ /**
58941
+ * Make `service.turndown(html)` parse with the injected DOM. Turndown parses
58942
+ * a string by wrapping it in `<x-turndown id="turndown-root">` and handing
58943
+ * that element to its converter; this does the same with `dom.DOMParser` and
58944
+ * passes the element, which Turndown accepts in place of a string. Covers
58945
+ * the nested `service.turndown(el.outerHTML)` calls in the list rules too.
58946
+ */
58947
+ parseWithInjectedDom(service, dom) {
58948
+ const turndown = service.turndown.bind(service);
58949
+ service.turndown = (input) => {
58950
+ if (typeof input !== "string") return turndown(input);
58951
+ const doc = new dom.DOMParser().parseFromString(
58952
+ '<x-turndown id="turndown-root">' + input + "</x-turndown>",
58953
+ "text/html"
58954
+ );
58955
+ const root = doc.getElementById("turndown-root");
58956
+ root?.normalize();
58957
+ if (!root) throw new Error("Injected DOM did not produce the turndown root element");
58958
+ return turndown(root);
58959
+ };
58909
58960
  }
58910
58961
  initializeServices() {
58911
58962
  this.githubService = new import_turndown2.default({
@@ -59017,7 +59068,7 @@ ${plainText}
59017
59068
  return node.nodeName === "INPUT" && node.type === "checkbox";
59018
59069
  },
59019
59070
  replacement: (_content, node) => {
59020
- const checked = node.checked;
59071
+ const checked = node.checked || node.hasAttribute("checked");
59021
59072
  return checked ? "\u2611 " : "\u2610 ";
59022
59073
  }
59023
59074
  });
@@ -59298,8 +59349,9 @@ ${plainText}
59298
59349
  return lines.join("\n");
59299
59350
  }
59300
59351
  flattenTables(html) {
59301
- if (typeof DOMParser === "undefined") return html;
59302
- const parser = new DOMParser();
59352
+ const Parser = this.dom?.DOMParser ?? (typeof DOMParser === "undefined" ? void 0 : DOMParser);
59353
+ if (!Parser) return html;
59354
+ const parser = new Parser();
59303
59355
  const doc = parser.parseFromString(`<div>${html}</div>`, "text/html");
59304
59356
  Array.from(doc.querySelectorAll("table")).forEach((table) => {
59305
59357
  const replacement = doc.createElement("div");
@@ -59459,10 +59511,11 @@ ${plainText}
59459
59511
  });
59460
59512
  }
59461
59513
  renderTableCell(cell) {
59514
+ const ELEMENT_NODE2 = 1;
59462
59515
  const blocks = [];
59463
59516
  let inlineBuffer = "";
59464
59517
  cell.childNodes.forEach((child) => {
59465
- if (child.nodeType === Node.ELEMENT_NODE && this.isTableCellBlock(child)) {
59518
+ if (child.nodeType === ELEMENT_NODE2 && this.isTableCellBlock(child)) {
59466
59519
  if (inlineBuffer.trim()) {
59467
59520
  blocks.push(inlineBuffer.trim());
59468
59521
  inlineBuffer = "";
@@ -59486,8 +59539,10 @@ ${plainText}
59486
59539
  return this.renderInlineNode(element).trim();
59487
59540
  }
59488
59541
  renderInlineNode(node) {
59489
- if (node.nodeType === Node.TEXT_NODE) return node.textContent || "";
59490
- if (node.nodeType !== Node.ELEMENT_NODE) return "";
59542
+ const TEXT_NODE = 3;
59543
+ const ELEMENT_NODE2 = 1;
59544
+ if (node.nodeType === TEXT_NODE) return node.textContent || "";
59545
+ if (node.nodeType !== ELEMENT_NODE2) return "";
59491
59546
  const element = node;
59492
59547
  const content = Array.from(element.childNodes).map((child) => this.renderInlineNode(child)).join("");
59493
59548
  const trimmed = content.trim();
@@ -59619,11 +59674,11 @@ ${plainText}
59619
59674
  processListItem(item, service, depth = 0) {
59620
59675
  let content = "";
59621
59676
  const TEXT_NODE = 3;
59622
- const ELEMENT_NODE = 1;
59677
+ const ELEMENT_NODE2 = 1;
59623
59678
  Array.from(item.childNodes).forEach((child) => {
59624
59679
  if (child.nodeType === TEXT_NODE) {
59625
59680
  content += child.textContent || "";
59626
- } else if (child.nodeType === ELEMENT_NODE) {
59681
+ } else if (child.nodeType === ELEMENT_NODE2) {
59627
59682
  const element = child;
59628
59683
  if (element.nodeName.toLowerCase() === "ul" || element.nodeName.toLowerCase() === "ol") {
59629
59684
  const nestedContent = this.processNestedList(element, service, depth + 1);
@@ -59764,6 +59819,10 @@ ${plainText}
59764
59819
  };
59765
59820
  }
59766
59821
  sanitize(html) {
59822
+ if (this.dom) {
59823
+ this.domSanitizer ??= new InputSanitizer({ level: "moderate" }, this.dom);
59824
+ return this.domSanitizer.sanitizeHTMLOrThrow(html);
59825
+ }
59767
59826
  return sanitizeHTML(html, {
59768
59827
  level: "moderate"
59769
59828
  // Use moderate level for conversion
@@ -63205,7 +63264,7 @@ var PptxAdapter = class {
63205
63264
  };
63206
63265
  var pptxAdapter = new PptxAdapter();
63207
63266
 
63208
- // ../../src/shared/readability-processor.ts
63267
+ // ../../src/shared/readability-core.ts
63209
63268
  var import_readability = __toESM(require_readability(), 1);
63210
63269
  var LinkedomProvider = class {
63211
63270
  createDocument(html) {
@@ -63503,6 +63562,118 @@ function buildCachedProvider(config, lastValidated) {
63503
63562
  return new CachedEntitlementProvider(cachedState);
63504
63563
  }
63505
63564
 
63565
+ // ../../src/shared/linkedom-sanitizer.ts
63566
+ var FORBID_CONTENTS = /* @__PURE__ */ new Set([
63567
+ "annotation-xml",
63568
+ "audio",
63569
+ "colgroup",
63570
+ "desc",
63571
+ "foreignobject",
63572
+ "head",
63573
+ "iframe",
63574
+ "math",
63575
+ "mi",
63576
+ "mn",
63577
+ "mo",
63578
+ "ms",
63579
+ "mtext",
63580
+ "noembed",
63581
+ "noframes",
63582
+ "noscript",
63583
+ "plaintext",
63584
+ "script",
63585
+ "selectedcontent",
63586
+ "style",
63587
+ "svg",
63588
+ "template",
63589
+ "thead",
63590
+ "title",
63591
+ "video",
63592
+ "xmp"
63593
+ ]);
63594
+ var DATA_URI_TAGS = /* @__PURE__ */ new Set(["audio", "video", "img", "source", "image", "track"]);
63595
+ var URI_SAFE_ATTRIBUTES = /* @__PURE__ */ new Set([
63596
+ "alt",
63597
+ "class",
63598
+ "for",
63599
+ "id",
63600
+ "label",
63601
+ "name",
63602
+ "pattern",
63603
+ "placeholder",
63604
+ "role",
63605
+ "summary",
63606
+ "title",
63607
+ "value",
63608
+ "style",
63609
+ "xmlns"
63610
+ ]);
63611
+ var IS_ALLOWED_URI = /^(?:(?:(?:f|ht)tps?|mailto|tel|callto|sms|cid|xmpp|matrix):|[^a-z]|[a-z+.\-]+(?:[^a-z+.\-:]|$))/i;
63612
+ var ATTR_WHITESPACE = /[\u0000-\u0020\u00A0\u1680\u180E\u2000-\u2029\u205F\u3000]/g;
63613
+ var ELEMENT_NODE = 1;
63614
+ var COMMENT_NODE = 8;
63615
+ function toSet(list) {
63616
+ return new Set((list ?? []).map((s) => s.toLowerCase()));
63617
+ }
63618
+ function attrAllowed(name, allowed, wildcards) {
63619
+ return allowed.has(name) || wildcards.some((prefix) => name.startsWith(prefix));
63620
+ }
63621
+ function isSafeValue(tag, name, value) {
63622
+ if (URI_SAFE_ATTRIBUTES.has(name) || !value) return true;
63623
+ if (IS_ALLOWED_URI.test(value.replace(ATTR_WHITESPACE, ""))) return true;
63624
+ return (name === "src" || name === "href" || name === "xlink:href") && value.startsWith("data:") && DATA_URI_TAGS.has(tag);
63625
+ }
63626
+ function sanitizeWithLinkedom(html, config, parse) {
63627
+ if (!html) return "";
63628
+ const allowedTags = toSet(config.ALLOWED_TAGS);
63629
+ const forbidTags = toSet(config.FORBID_TAGS);
63630
+ const allowedAttrList = (config.ALLOWED_ATTR ?? []).map((s) => s.toLowerCase());
63631
+ const allowedAttrs = new Set(allowedAttrList.filter((a) => !a.endsWith("*")));
63632
+ const attrWildcards = allowedAttrList.filter((a) => a.endsWith("*")).map((a) => a.slice(0, -1));
63633
+ const forbidAttrs = toSet(config.FORBID_ATTR);
63634
+ const doc = parse(`<!DOCTYPE html><html><head></head><body>${html}</body></html>`);
63635
+ const body = doc.body;
63636
+ if (!body) return "";
63637
+ const toUnwrap = [];
63638
+ const stack = [body];
63639
+ while (stack.length > 0) {
63640
+ const parent = stack.pop();
63641
+ for (const child of Array.from(parent.childNodes)) {
63642
+ if (child.nodeType === COMMENT_NODE) {
63643
+ parent.removeChild(child);
63644
+ continue;
63645
+ }
63646
+ if (child.nodeType !== ELEMENT_NODE) continue;
63647
+ const el = child;
63648
+ const tag = el.nodeName.toLowerCase();
63649
+ if (!allowedTags.has(tag) || forbidTags.has(tag)) {
63650
+ if (FORBID_CONTENTS.has(tag)) {
63651
+ parent.removeChild(el);
63652
+ } else {
63653
+ toUnwrap.push(el);
63654
+ stack.push(el);
63655
+ }
63656
+ continue;
63657
+ }
63658
+ for (const attr of Array.from(el.attributes)) {
63659
+ const name = attr.name.toLowerCase();
63660
+ if (name.startsWith("on") || forbidAttrs.has(name) || !attrAllowed(name, allowedAttrs, attrWildcards) || !isSafeValue(tag, name, attr.value)) {
63661
+ el.removeAttribute(attr.name);
63662
+ }
63663
+ }
63664
+ stack.push(el);
63665
+ }
63666
+ }
63667
+ for (let i = toUnwrap.length - 1; i >= 0; i--) {
63668
+ const el = toUnwrap[i];
63669
+ const parent = el.parentNode;
63670
+ if (!parent) continue;
63671
+ while (el.firstChild) parent.insertBefore(el.firstChild, el);
63672
+ parent.removeChild(el);
63673
+ }
63674
+ return body.innerHTML;
63675
+ }
63676
+
63506
63677
  // ../../src/shared/token-estimator.ts
63507
63678
  var CJK_REGEX3 = /[\u4e00-\u9fff\u3400-\u4dbf\u3000-\u303f\u3040-\u309f\u30a0-\u30ff\uac00-\ud7af]/g;
63508
63679
  var MAX_INPUT_LENGTH = 1e6;
@@ -64052,6 +64223,18 @@ async function binaryToMarkdown(buffer, type, format) {
64052
64223
  }
64053
64224
  }
64054
64225
  }
64226
+ function linkedomDomEnvironment() {
64227
+ const parseHTML2 = globalThis.__FF_parseHTML;
64228
+ if (!parseHTML2) throw new Error("The CLI shim must be loaded before converting a URL");
64229
+ const parseDocument = (html) => parseHTML2(html).document;
64230
+ const { window: window3 } = parseHTML2("<!DOCTYPE html><html><head></head><body></body></html>");
64231
+ return {
64232
+ DOMParser: window3.DOMParser,
64233
+ DOMPurify: {
64234
+ sanitize: (html, config) => sanitizeWithLinkedom(html, config ?? {}, parseDocument)
64235
+ }
64236
+ };
64237
+ }
64055
64238
  var DEFAULT_PROXY_URL = "https://formatferry.vibingfun.com";
64056
64239
  async function urlToMarkdown(url, apiKey, format, proxyBaseUrl = process.env.FORMATFERRY_PROXY_URL ?? DEFAULT_PROXY_URL) {
64057
64240
  const extractEndpoint = `${proxyBaseUrl}/v1/extract`;
@@ -64063,7 +64246,9 @@ async function urlToMarkdown(url, apiKey, format, proxyBaseUrl = process.env.FOR
64063
64246
  "Content-Type": "application/json",
64064
64247
  "X-API-Key": apiKey
64065
64248
  },
64066
- body: JSON.stringify({ url })
64249
+ // Readability and the chosen flavour run locally, so ask for the page's
64250
+ // HTML (the API's default is Markdown).
64251
+ body: JSON.stringify({ url, format: "html" })
64067
64252
  });
64068
64253
  } catch (err) {
64069
64254
  const message = err instanceof Error ? err.message : String(err);
@@ -64074,18 +64259,19 @@ async function urlToMarkdown(url, apiKey, format, proxyBaseUrl = process.env.FOR
64074
64259
  if (!response.ok) {
64075
64260
  let errorMessage = `Proxy returned ${response.status}`;
64076
64261
  try {
64077
- const body = await response.json();
64078
- errorMessage = body.error ?? body.message ?? errorMessage;
64262
+ const body2 = await response.json();
64263
+ errorMessage = body2.error ?? body2.message ?? errorMessage;
64079
64264
  } catch {
64080
64265
  }
64081
64266
  process.stderr.write(`Error: ${errorMessage}
64082
64267
  `);
64083
64268
  process.exit(1);
64084
64269
  }
64085
- const { html } = await response.json();
64270
+ const body = await response.json();
64271
+ const html = body.content ?? body.html ?? "";
64086
64272
  const domProvider = new LinkedomProvider();
64087
64273
  const article = extractArticle(html, domProvider);
64088
- const engine = new ConversionEngine();
64274
+ const engine = new ConversionEngine({ dom: linkedomDomEnvironment() });
64089
64275
  const result = await engine.convert(article.content, {
64090
64276
  format,
64091
64277
  preserveTables: true,
@@ -64589,8 +64775,8 @@ import { readFileSync as readFileSync5 } from "fs";
64589
64775
  import { resolve, dirname as dirname3 } from "path";
64590
64776
  import { fileURLToPath } from "url";
64591
64777
  var getVersion = () => {
64592
- if ("1.0.33") {
64593
- return "1.0.33";
64778
+ if ("1.0.34") {
64779
+ return "1.0.34";
64594
64780
  }
64595
64781
  try {
64596
64782
  const __dirname2 = dirname3(fileURLToPath(import.meta.url));
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "formatferry",
3
- "version": "1.0.33",
3
+ "version": "1.0.34",
4
4
  "description": "Format Ferry CLI — Convert HTML, DOCX, PDF, XLSX, CSV, JSON, XML, PPTX to Markdown",
5
5
  "keywords": [
6
6
  "markdown",