formatferry 1.0.33 → 1.0.34
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -0
- package/dist/index.js +237 -51
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -103,6 +103,8 @@ formatferry [options]
|
|
|
103
103
|
| `-o, --output <path>` | Output file path (defaults to stdout) |
|
|
104
104
|
| `-f, --format <format>` | Markdown format (default: `github`) |
|
|
105
105
|
| `--url <url>` | Fetch and convert a web article (requires API key) |
|
|
106
|
+
|
|
107
|
+
`--url` asks the proxy for the page HTML and converts it locally, so `--format` chooses the Markdown flavour exactly as it does for a local file. The proxy also offers the finished Markdown directly: `POST /v1/extract` with `{ "format": "markdown" }` (see the root README).
|
|
106
108
|
| `--batch <glob>` | Batch convert files matching a glob pattern (Premium) |
|
|
107
109
|
| `--output-dir <dir>` | Output directory for batch mode |
|
|
108
110
|
| `--strict` | Exit with code 1 if the conversion produced any warnings (including advisory ones) |
|
package/dist/index.js
CHANGED
|
@@ -948,14 +948,14 @@ var require_Node = __commonJS({
|
|
|
948
948
|
this._nextSibling = this._previousSibling = this;
|
|
949
949
|
this._index = void 0;
|
|
950
950
|
}
|
|
951
|
-
var
|
|
951
|
+
var ELEMENT_NODE2 = Node2.ELEMENT_NODE = 1;
|
|
952
952
|
var ATTRIBUTE_NODE = Node2.ATTRIBUTE_NODE = 2;
|
|
953
953
|
var TEXT_NODE = Node2.TEXT_NODE = 3;
|
|
954
954
|
var CDATA_SECTION_NODE = Node2.CDATA_SECTION_NODE = 4;
|
|
955
955
|
var ENTITY_REFERENCE_NODE = Node2.ENTITY_REFERENCE_NODE = 5;
|
|
956
956
|
var ENTITY_NODE = Node2.ENTITY_NODE = 6;
|
|
957
957
|
var PROCESSING_INSTRUCTION_NODE = Node2.PROCESSING_INSTRUCTION_NODE = 7;
|
|
958
|
-
var
|
|
958
|
+
var COMMENT_NODE2 = Node2.COMMENT_NODE = 8;
|
|
959
959
|
var DOCUMENT_NODE = Node2.DOCUMENT_NODE = 9;
|
|
960
960
|
var DOCUMENT_TYPE_NODE = Node2.DOCUMENT_TYPE_NODE = 10;
|
|
961
961
|
var DOCUMENT_FRAGMENT_NODE = Node2.DOCUMENT_FRAGMENT_NODE = 11;
|
|
@@ -973,7 +973,7 @@ var require_Node = __commonJS({
|
|
|
973
973
|
// we'll come back to this later.
|
|
974
974
|
baseURI: { get: utils2.nyi },
|
|
975
975
|
parentElement: { get: function() {
|
|
976
|
-
return this.parentNode && this.parentNode.nodeType ===
|
|
976
|
+
return this.parentNode && this.parentNode.nodeType === ELEMENT_NODE2 ? this.parentNode : null;
|
|
977
977
|
} },
|
|
978
978
|
hasChildNodes: { value: utils2.shouldOverride },
|
|
979
979
|
firstChild: { get: utils2.shouldOverride },
|
|
@@ -1034,7 +1034,7 @@ var require_Node = __commonJS({
|
|
|
1034
1034
|
switch (parent.nodeType) {
|
|
1035
1035
|
case DOCUMENT_NODE:
|
|
1036
1036
|
case DOCUMENT_FRAGMENT_NODE:
|
|
1037
|
-
case
|
|
1037
|
+
case ELEMENT_NODE2:
|
|
1038
1038
|
break;
|
|
1039
1039
|
default:
|
|
1040
1040
|
utils2.HierarchyRequestError();
|
|
@@ -1046,10 +1046,10 @@ var require_Node = __commonJS({
|
|
|
1046
1046
|
switch (node.nodeType) {
|
|
1047
1047
|
case DOCUMENT_FRAGMENT_NODE:
|
|
1048
1048
|
case DOCUMENT_TYPE_NODE:
|
|
1049
|
-
case
|
|
1049
|
+
case ELEMENT_NODE2:
|
|
1050
1050
|
case TEXT_NODE:
|
|
1051
1051
|
case PROCESSING_INSTRUCTION_NODE:
|
|
1052
|
-
case
|
|
1052
|
+
case COMMENT_NODE2:
|
|
1053
1053
|
break;
|
|
1054
1054
|
default:
|
|
1055
1055
|
utils2.HierarchyRequestError();
|
|
@@ -1062,7 +1062,7 @@ var require_Node = __commonJS({
|
|
|
1062
1062
|
case DOCUMENT_FRAGMENT_NODE:
|
|
1063
1063
|
if (node._countChildrenOfType(TEXT_NODE) > 0)
|
|
1064
1064
|
utils2.HierarchyRequestError();
|
|
1065
|
-
switch (node._countChildrenOfType(
|
|
1065
|
+
switch (node._countChildrenOfType(ELEMENT_NODE2)) {
|
|
1066
1066
|
case 0:
|
|
1067
1067
|
break;
|
|
1068
1068
|
case 1:
|
|
@@ -1074,12 +1074,12 @@ var require_Node = __commonJS({
|
|
|
1074
1074
|
utils2.HierarchyRequestError();
|
|
1075
1075
|
}
|
|
1076
1076
|
}
|
|
1077
|
-
i = parent._countChildrenOfType(
|
|
1077
|
+
i = parent._countChildrenOfType(ELEMENT_NODE2);
|
|
1078
1078
|
if (isPreinsert) {
|
|
1079
1079
|
if (i > 0)
|
|
1080
1080
|
utils2.HierarchyRequestError();
|
|
1081
1081
|
} else {
|
|
1082
|
-
if (i > 1 || i === 1 && child.nodeType !==
|
|
1082
|
+
if (i > 1 || i === 1 && child.nodeType !== ELEMENT_NODE2)
|
|
1083
1083
|
utils2.HierarchyRequestError();
|
|
1084
1084
|
}
|
|
1085
1085
|
break;
|
|
@@ -1087,7 +1087,7 @@ var require_Node = __commonJS({
|
|
|
1087
1087
|
utils2.HierarchyRequestError();
|
|
1088
1088
|
}
|
|
1089
1089
|
break;
|
|
1090
|
-
case
|
|
1090
|
+
case ELEMENT_NODE2:
|
|
1091
1091
|
if (child !== null) {
|
|
1092
1092
|
if (isPreinsert && child.nodeType === DOCUMENT_TYPE_NODE)
|
|
1093
1093
|
utils2.HierarchyRequestError();
|
|
@@ -1096,23 +1096,23 @@ var require_Node = __commonJS({
|
|
|
1096
1096
|
utils2.HierarchyRequestError();
|
|
1097
1097
|
}
|
|
1098
1098
|
}
|
|
1099
|
-
i = parent._countChildrenOfType(
|
|
1099
|
+
i = parent._countChildrenOfType(ELEMENT_NODE2);
|
|
1100
1100
|
if (isPreinsert) {
|
|
1101
1101
|
if (i > 0)
|
|
1102
1102
|
utils2.HierarchyRequestError();
|
|
1103
1103
|
} else {
|
|
1104
|
-
if (i > 1 || i === 1 && child.nodeType !==
|
|
1104
|
+
if (i > 1 || i === 1 && child.nodeType !== ELEMENT_NODE2)
|
|
1105
1105
|
utils2.HierarchyRequestError();
|
|
1106
1106
|
}
|
|
1107
1107
|
break;
|
|
1108
1108
|
case DOCUMENT_TYPE_NODE:
|
|
1109
1109
|
if (child === null) {
|
|
1110
|
-
if (parent._countChildrenOfType(
|
|
1110
|
+
if (parent._countChildrenOfType(ELEMENT_NODE2))
|
|
1111
1111
|
utils2.HierarchyRequestError();
|
|
1112
1112
|
} else {
|
|
1113
1113
|
for (kid = parent.firstChild; kid !== null; kid = kid.nextSibling) {
|
|
1114
1114
|
if (kid === child) break;
|
|
1115
|
-
if (kid.nodeType ===
|
|
1115
|
+
if (kid.nodeType === ELEMENT_NODE2)
|
|
1116
1116
|
utils2.HierarchyRequestError();
|
|
1117
1117
|
}
|
|
1118
1118
|
}
|
|
@@ -1229,7 +1229,7 @@ var require_Node = __commonJS({
|
|
|
1229
1229
|
var e;
|
|
1230
1230
|
if (ns === "" || ns === null || ns === void 0) return null;
|
|
1231
1231
|
switch (this.nodeType) {
|
|
1232
|
-
case
|
|
1232
|
+
case ELEMENT_NODE2:
|
|
1233
1233
|
return this._lookupNamespacePrefix(ns, this);
|
|
1234
1234
|
case DOCUMENT_NODE:
|
|
1235
1235
|
e = this.documentElement;
|
|
@@ -1253,7 +1253,7 @@ var require_Node = __commonJS({
|
|
|
1253
1253
|
}
|
|
1254
1254
|
var e;
|
|
1255
1255
|
switch (this.nodeType) {
|
|
1256
|
-
case
|
|
1256
|
+
case ELEMENT_NODE2:
|
|
1257
1257
|
return utils2.shouldOverride();
|
|
1258
1258
|
case DOCUMENT_NODE:
|
|
1259
1259
|
e = this.documentElement;
|
|
@@ -1511,14 +1511,14 @@ var require_Node = __commonJS({
|
|
|
1511
1511
|
},
|
|
1512
1512
|
// mirror node type properties in the prototype, so they are present
|
|
1513
1513
|
// in instances of Node (and subclasses)
|
|
1514
|
-
ELEMENT_NODE: { value:
|
|
1514
|
+
ELEMENT_NODE: { value: ELEMENT_NODE2 },
|
|
1515
1515
|
ATTRIBUTE_NODE: { value: ATTRIBUTE_NODE },
|
|
1516
1516
|
TEXT_NODE: { value: TEXT_NODE },
|
|
1517
1517
|
CDATA_SECTION_NODE: { value: CDATA_SECTION_NODE },
|
|
1518
1518
|
ENTITY_REFERENCE_NODE: { value: ENTITY_REFERENCE_NODE },
|
|
1519
1519
|
ENTITY_NODE: { value: ENTITY_NODE },
|
|
1520
1520
|
PROCESSING_INSTRUCTION_NODE: { value: PROCESSING_INSTRUCTION_NODE },
|
|
1521
|
-
COMMENT_NODE: { value:
|
|
1521
|
+
COMMENT_NODE: { value: COMMENT_NODE2 },
|
|
1522
1522
|
DOCUMENT_NODE: { value: DOCUMENT_NODE },
|
|
1523
1523
|
DOCUMENT_TYPE_NODE: { value: DOCUMENT_TYPE_NODE },
|
|
1524
1524
|
DOCUMENT_FRAGMENT_NODE: { value: DOCUMENT_FRAGMENT_NODE },
|
|
@@ -58046,20 +58046,29 @@ var InputSanitizer = class {
|
|
|
58046
58046
|
this.purifyConfig = this.createPurifyConfig();
|
|
58047
58047
|
}
|
|
58048
58048
|
/**
|
|
58049
|
-
*
|
|
58049
|
+
* Like sanitizeHTML, but a sanitiser failure is thrown instead of being
|
|
58050
|
+
* turned into an empty string. Use it wherever an empty result would be
|
|
58051
|
+
* read as "the page had no content" (the ConversionEngine on an injected
|
|
58052
|
+
* DOM: the Worker's server-side Markdown and the CLI), so a failure is
|
|
58053
|
+
* reported as a failure (#170).
|
|
58050
58054
|
*/
|
|
58051
|
-
|
|
58055
|
+
sanitizeHTMLOrThrow(html) {
|
|
58052
58056
|
if (!html || typeof html !== "string") {
|
|
58053
58057
|
return "";
|
|
58054
58058
|
}
|
|
58059
|
+
return this.domEnv.DOMPurify.sanitize(html, {
|
|
58060
|
+
ALLOWED_TAGS: this.purifyConfig.ALLOWED_TAGS,
|
|
58061
|
+
ALLOWED_ATTR: this.purifyConfig.ALLOWED_ATTR,
|
|
58062
|
+
FORBID_TAGS: this.purifyConfig.FORBID_TAGS,
|
|
58063
|
+
FORBID_ATTR: this.purifyConfig.FORBID_ATTR
|
|
58064
|
+
});
|
|
58065
|
+
}
|
|
58066
|
+
/**
|
|
58067
|
+
* Sanitize HTML content to prevent XSS attacks
|
|
58068
|
+
*/
|
|
58069
|
+
sanitizeHTML(html) {
|
|
58055
58070
|
try {
|
|
58056
|
-
|
|
58057
|
-
ALLOWED_TAGS: this.purifyConfig.ALLOWED_TAGS,
|
|
58058
|
-
ALLOWED_ATTR: this.purifyConfig.ALLOWED_ATTR,
|
|
58059
|
-
FORBID_TAGS: this.purifyConfig.FORBID_TAGS,
|
|
58060
|
-
FORBID_ATTR: this.purifyConfig.FORBID_ATTR
|
|
58061
|
-
});
|
|
58062
|
-
return sanitized;
|
|
58071
|
+
return this.sanitizeHTMLOrThrow(html);
|
|
58063
58072
|
} catch (error) {
|
|
58064
58073
|
console.warn("HTML sanitization failed:", error);
|
|
58065
58074
|
return "";
|
|
@@ -58312,7 +58321,11 @@ var InputSanitizer = class {
|
|
|
58312
58321
|
return { ...this.config };
|
|
58313
58322
|
}
|
|
58314
58323
|
};
|
|
58315
|
-
var
|
|
58324
|
+
var defaultSanitizerInstance;
|
|
58325
|
+
function getDefaultSanitizer() {
|
|
58326
|
+
defaultSanitizerInstance ??= new InputSanitizer();
|
|
58327
|
+
return defaultSanitizerInstance;
|
|
58328
|
+
}
|
|
58316
58329
|
function normalizeSlackHTML(html, domParser) {
|
|
58317
58330
|
if (!html || !html.includes("indent=")) return html;
|
|
58318
58331
|
const doc = (domParser ?? new globalThis.DOMParser()).parseFromString(html, "text/html");
|
|
@@ -58343,7 +58356,7 @@ function normalizeSlackHTML(html, domParser) {
|
|
|
58343
58356
|
}
|
|
58344
58357
|
return root;
|
|
58345
58358
|
};
|
|
58346
|
-
const isListEl = (n) => n.nodeType ===
|
|
58359
|
+
const isListEl = (n) => n.nodeType === 1 && (n.nodeName === "UL" || n.nodeName === "OL");
|
|
58347
58360
|
const snapshot = Array.from(body.childNodes);
|
|
58348
58361
|
let runStart = -1;
|
|
58349
58362
|
const flushRun = (endIdx) => {
|
|
@@ -58377,7 +58390,7 @@ function sanitizeHTML(html, config) {
|
|
|
58377
58390
|
const sanitizer = new InputSanitizer(config);
|
|
58378
58391
|
return sanitizer.sanitizeHTML(html);
|
|
58379
58392
|
}
|
|
58380
|
-
return
|
|
58393
|
+
return getDefaultSanitizer().sanitizeHTML(html);
|
|
58381
58394
|
}
|
|
58382
58395
|
var PANEL_EMOJI = {
|
|
58383
58396
|
info: "\u2139\uFE0F Info:",
|
|
@@ -58463,11 +58476,11 @@ function normalizeAtlassianTables(doc) {
|
|
|
58463
58476
|
}
|
|
58464
58477
|
}
|
|
58465
58478
|
function stripUnrecognizedMacros(node) {
|
|
58466
|
-
const
|
|
58479
|
+
const ELEMENT_NODE2 = 1;
|
|
58467
58480
|
let i = 0;
|
|
58468
58481
|
while (i < node.childNodes.length) {
|
|
58469
58482
|
const child = node.childNodes[i];
|
|
58470
|
-
if (child.nodeType ===
|
|
58483
|
+
if (child.nodeType === ELEMENT_NODE2) {
|
|
58471
58484
|
const el = child;
|
|
58472
58485
|
const tagName = el.tagName.toLowerCase();
|
|
58473
58486
|
if (tagName.startsWith("ac:") || tagName.startsWith("ri:")) {
|
|
@@ -58682,11 +58695,11 @@ function processConfluenceList(list, service, depth) {
|
|
|
58682
58695
|
function processConfluenceListItem(item, service, depth) {
|
|
58683
58696
|
let content = "";
|
|
58684
58697
|
const TEXT_NODE = 3;
|
|
58685
|
-
const
|
|
58698
|
+
const ELEMENT_NODE2 = 1;
|
|
58686
58699
|
Array.from(item.childNodes).forEach((child) => {
|
|
58687
58700
|
if (child.nodeType === TEXT_NODE) {
|
|
58688
58701
|
content += child.textContent || "";
|
|
58689
|
-
} else if (child.nodeType ===
|
|
58702
|
+
} else if (child.nodeType === ELEMENT_NODE2) {
|
|
58690
58703
|
const el = child;
|
|
58691
58704
|
if (el.nodeName === "UL" || el.nodeName === "OL") {
|
|
58692
58705
|
content += "\n" + processConfluenceList(el, service, depth + 1);
|
|
@@ -58904,8 +58917,46 @@ var ConversionEngine = class {
|
|
|
58904
58917
|
rmarkdownService;
|
|
58905
58918
|
slackService;
|
|
58906
58919
|
confluenceService;
|
|
58907
|
-
|
|
58920
|
+
dom;
|
|
58921
|
+
domSanitizer;
|
|
58922
|
+
constructor(options = {}) {
|
|
58923
|
+
this.dom = options.dom;
|
|
58908
58924
|
this.initializeServices();
|
|
58925
|
+
if (this.dom) {
|
|
58926
|
+
for (const service of [
|
|
58927
|
+
this.githubService,
|
|
58928
|
+
this.commonmarkService,
|
|
58929
|
+
this.customService,
|
|
58930
|
+
this.discordService,
|
|
58931
|
+
this.redditService,
|
|
58932
|
+
this.rmarkdownService,
|
|
58933
|
+
this.slackService,
|
|
58934
|
+
this.confluenceService
|
|
58935
|
+
]) {
|
|
58936
|
+
this.parseWithInjectedDom(service, this.dom);
|
|
58937
|
+
}
|
|
58938
|
+
}
|
|
58939
|
+
}
|
|
58940
|
+
/**
|
|
58941
|
+
* Make `service.turndown(html)` parse with the injected DOM. Turndown parses
|
|
58942
|
+
* a string by wrapping it in `<x-turndown id="turndown-root">` and handing
|
|
58943
|
+
* that element to its converter; this does the same with `dom.DOMParser` and
|
|
58944
|
+
* passes the element, which Turndown accepts in place of a string. Covers
|
|
58945
|
+
* the nested `service.turndown(el.outerHTML)` calls in the list rules too.
|
|
58946
|
+
*/
|
|
58947
|
+
parseWithInjectedDom(service, dom) {
|
|
58948
|
+
const turndown = service.turndown.bind(service);
|
|
58949
|
+
service.turndown = (input) => {
|
|
58950
|
+
if (typeof input !== "string") return turndown(input);
|
|
58951
|
+
const doc = new dom.DOMParser().parseFromString(
|
|
58952
|
+
'<x-turndown id="turndown-root">' + input + "</x-turndown>",
|
|
58953
|
+
"text/html"
|
|
58954
|
+
);
|
|
58955
|
+
const root = doc.getElementById("turndown-root");
|
|
58956
|
+
root?.normalize();
|
|
58957
|
+
if (!root) throw new Error("Injected DOM did not produce the turndown root element");
|
|
58958
|
+
return turndown(root);
|
|
58959
|
+
};
|
|
58909
58960
|
}
|
|
58910
58961
|
initializeServices() {
|
|
58911
58962
|
this.githubService = new import_turndown2.default({
|
|
@@ -59017,7 +59068,7 @@ ${plainText}
|
|
|
59017
59068
|
return node.nodeName === "INPUT" && node.type === "checkbox";
|
|
59018
59069
|
},
|
|
59019
59070
|
replacement: (_content, node) => {
|
|
59020
|
-
const checked = node.checked;
|
|
59071
|
+
const checked = node.checked || node.hasAttribute("checked");
|
|
59021
59072
|
return checked ? "\u2611 " : "\u2610 ";
|
|
59022
59073
|
}
|
|
59023
59074
|
});
|
|
@@ -59298,8 +59349,9 @@ ${plainText}
|
|
|
59298
59349
|
return lines.join("\n");
|
|
59299
59350
|
}
|
|
59300
59351
|
flattenTables(html) {
|
|
59301
|
-
|
|
59302
|
-
|
|
59352
|
+
const Parser = this.dom?.DOMParser ?? (typeof DOMParser === "undefined" ? void 0 : DOMParser);
|
|
59353
|
+
if (!Parser) return html;
|
|
59354
|
+
const parser = new Parser();
|
|
59303
59355
|
const doc = parser.parseFromString(`<div>${html}</div>`, "text/html");
|
|
59304
59356
|
Array.from(doc.querySelectorAll("table")).forEach((table) => {
|
|
59305
59357
|
const replacement = doc.createElement("div");
|
|
@@ -59459,10 +59511,11 @@ ${plainText}
|
|
|
59459
59511
|
});
|
|
59460
59512
|
}
|
|
59461
59513
|
renderTableCell(cell) {
|
|
59514
|
+
const ELEMENT_NODE2 = 1;
|
|
59462
59515
|
const blocks = [];
|
|
59463
59516
|
let inlineBuffer = "";
|
|
59464
59517
|
cell.childNodes.forEach((child) => {
|
|
59465
|
-
if (child.nodeType ===
|
|
59518
|
+
if (child.nodeType === ELEMENT_NODE2 && this.isTableCellBlock(child)) {
|
|
59466
59519
|
if (inlineBuffer.trim()) {
|
|
59467
59520
|
blocks.push(inlineBuffer.trim());
|
|
59468
59521
|
inlineBuffer = "";
|
|
@@ -59486,8 +59539,10 @@ ${plainText}
|
|
|
59486
59539
|
return this.renderInlineNode(element).trim();
|
|
59487
59540
|
}
|
|
59488
59541
|
renderInlineNode(node) {
|
|
59489
|
-
|
|
59490
|
-
|
|
59542
|
+
const TEXT_NODE = 3;
|
|
59543
|
+
const ELEMENT_NODE2 = 1;
|
|
59544
|
+
if (node.nodeType === TEXT_NODE) return node.textContent || "";
|
|
59545
|
+
if (node.nodeType !== ELEMENT_NODE2) return "";
|
|
59491
59546
|
const element = node;
|
|
59492
59547
|
const content = Array.from(element.childNodes).map((child) => this.renderInlineNode(child)).join("");
|
|
59493
59548
|
const trimmed = content.trim();
|
|
@@ -59619,11 +59674,11 @@ ${plainText}
|
|
|
59619
59674
|
processListItem(item, service, depth = 0) {
|
|
59620
59675
|
let content = "";
|
|
59621
59676
|
const TEXT_NODE = 3;
|
|
59622
|
-
const
|
|
59677
|
+
const ELEMENT_NODE2 = 1;
|
|
59623
59678
|
Array.from(item.childNodes).forEach((child) => {
|
|
59624
59679
|
if (child.nodeType === TEXT_NODE) {
|
|
59625
59680
|
content += child.textContent || "";
|
|
59626
|
-
} else if (child.nodeType ===
|
|
59681
|
+
} else if (child.nodeType === ELEMENT_NODE2) {
|
|
59627
59682
|
const element = child;
|
|
59628
59683
|
if (element.nodeName.toLowerCase() === "ul" || element.nodeName.toLowerCase() === "ol") {
|
|
59629
59684
|
const nestedContent = this.processNestedList(element, service, depth + 1);
|
|
@@ -59764,6 +59819,10 @@ ${plainText}
|
|
|
59764
59819
|
};
|
|
59765
59820
|
}
|
|
59766
59821
|
sanitize(html) {
|
|
59822
|
+
if (this.dom) {
|
|
59823
|
+
this.domSanitizer ??= new InputSanitizer({ level: "moderate" }, this.dom);
|
|
59824
|
+
return this.domSanitizer.sanitizeHTMLOrThrow(html);
|
|
59825
|
+
}
|
|
59767
59826
|
return sanitizeHTML(html, {
|
|
59768
59827
|
level: "moderate"
|
|
59769
59828
|
// Use moderate level for conversion
|
|
@@ -63205,7 +63264,7 @@ var PptxAdapter = class {
|
|
|
63205
63264
|
};
|
|
63206
63265
|
var pptxAdapter = new PptxAdapter();
|
|
63207
63266
|
|
|
63208
|
-
// ../../src/shared/readability-
|
|
63267
|
+
// ../../src/shared/readability-core.ts
|
|
63209
63268
|
var import_readability = __toESM(require_readability(), 1);
|
|
63210
63269
|
var LinkedomProvider = class {
|
|
63211
63270
|
createDocument(html) {
|
|
@@ -63503,6 +63562,118 @@ function buildCachedProvider(config, lastValidated) {
|
|
|
63503
63562
|
return new CachedEntitlementProvider(cachedState);
|
|
63504
63563
|
}
|
|
63505
63564
|
|
|
63565
|
+
// ../../src/shared/linkedom-sanitizer.ts
|
|
63566
|
+
var FORBID_CONTENTS = /* @__PURE__ */ new Set([
|
|
63567
|
+
"annotation-xml",
|
|
63568
|
+
"audio",
|
|
63569
|
+
"colgroup",
|
|
63570
|
+
"desc",
|
|
63571
|
+
"foreignobject",
|
|
63572
|
+
"head",
|
|
63573
|
+
"iframe",
|
|
63574
|
+
"math",
|
|
63575
|
+
"mi",
|
|
63576
|
+
"mn",
|
|
63577
|
+
"mo",
|
|
63578
|
+
"ms",
|
|
63579
|
+
"mtext",
|
|
63580
|
+
"noembed",
|
|
63581
|
+
"noframes",
|
|
63582
|
+
"noscript",
|
|
63583
|
+
"plaintext",
|
|
63584
|
+
"script",
|
|
63585
|
+
"selectedcontent",
|
|
63586
|
+
"style",
|
|
63587
|
+
"svg",
|
|
63588
|
+
"template",
|
|
63589
|
+
"thead",
|
|
63590
|
+
"title",
|
|
63591
|
+
"video",
|
|
63592
|
+
"xmp"
|
|
63593
|
+
]);
|
|
63594
|
+
var DATA_URI_TAGS = /* @__PURE__ */ new Set(["audio", "video", "img", "source", "image", "track"]);
|
|
63595
|
+
var URI_SAFE_ATTRIBUTES = /* @__PURE__ */ new Set([
|
|
63596
|
+
"alt",
|
|
63597
|
+
"class",
|
|
63598
|
+
"for",
|
|
63599
|
+
"id",
|
|
63600
|
+
"label",
|
|
63601
|
+
"name",
|
|
63602
|
+
"pattern",
|
|
63603
|
+
"placeholder",
|
|
63604
|
+
"role",
|
|
63605
|
+
"summary",
|
|
63606
|
+
"title",
|
|
63607
|
+
"value",
|
|
63608
|
+
"style",
|
|
63609
|
+
"xmlns"
|
|
63610
|
+
]);
|
|
63611
|
+
var IS_ALLOWED_URI = /^(?:(?:(?:f|ht)tps?|mailto|tel|callto|sms|cid|xmpp|matrix):|[^a-z]|[a-z+.\-]+(?:[^a-z+.\-:]|$))/i;
|
|
63612
|
+
var ATTR_WHITESPACE = /[\u0000-\u0020\u00A0\u1680\u180E\u2000-\u2029\u205F\u3000]/g;
|
|
63613
|
+
var ELEMENT_NODE = 1;
|
|
63614
|
+
var COMMENT_NODE = 8;
|
|
63615
|
+
function toSet(list) {
|
|
63616
|
+
return new Set((list ?? []).map((s) => s.toLowerCase()));
|
|
63617
|
+
}
|
|
63618
|
+
function attrAllowed(name, allowed, wildcards) {
|
|
63619
|
+
return allowed.has(name) || wildcards.some((prefix) => name.startsWith(prefix));
|
|
63620
|
+
}
|
|
63621
|
+
function isSafeValue(tag, name, value) {
|
|
63622
|
+
if (URI_SAFE_ATTRIBUTES.has(name) || !value) return true;
|
|
63623
|
+
if (IS_ALLOWED_URI.test(value.replace(ATTR_WHITESPACE, ""))) return true;
|
|
63624
|
+
return (name === "src" || name === "href" || name === "xlink:href") && value.startsWith("data:") && DATA_URI_TAGS.has(tag);
|
|
63625
|
+
}
|
|
63626
|
+
function sanitizeWithLinkedom(html, config, parse) {
|
|
63627
|
+
if (!html) return "";
|
|
63628
|
+
const allowedTags = toSet(config.ALLOWED_TAGS);
|
|
63629
|
+
const forbidTags = toSet(config.FORBID_TAGS);
|
|
63630
|
+
const allowedAttrList = (config.ALLOWED_ATTR ?? []).map((s) => s.toLowerCase());
|
|
63631
|
+
const allowedAttrs = new Set(allowedAttrList.filter((a) => !a.endsWith("*")));
|
|
63632
|
+
const attrWildcards = allowedAttrList.filter((a) => a.endsWith("*")).map((a) => a.slice(0, -1));
|
|
63633
|
+
const forbidAttrs = toSet(config.FORBID_ATTR);
|
|
63634
|
+
const doc = parse(`<!DOCTYPE html><html><head></head><body>${html}</body></html>`);
|
|
63635
|
+
const body = doc.body;
|
|
63636
|
+
if (!body) return "";
|
|
63637
|
+
const toUnwrap = [];
|
|
63638
|
+
const stack = [body];
|
|
63639
|
+
while (stack.length > 0) {
|
|
63640
|
+
const parent = stack.pop();
|
|
63641
|
+
for (const child of Array.from(parent.childNodes)) {
|
|
63642
|
+
if (child.nodeType === COMMENT_NODE) {
|
|
63643
|
+
parent.removeChild(child);
|
|
63644
|
+
continue;
|
|
63645
|
+
}
|
|
63646
|
+
if (child.nodeType !== ELEMENT_NODE) continue;
|
|
63647
|
+
const el = child;
|
|
63648
|
+
const tag = el.nodeName.toLowerCase();
|
|
63649
|
+
if (!allowedTags.has(tag) || forbidTags.has(tag)) {
|
|
63650
|
+
if (FORBID_CONTENTS.has(tag)) {
|
|
63651
|
+
parent.removeChild(el);
|
|
63652
|
+
} else {
|
|
63653
|
+
toUnwrap.push(el);
|
|
63654
|
+
stack.push(el);
|
|
63655
|
+
}
|
|
63656
|
+
continue;
|
|
63657
|
+
}
|
|
63658
|
+
for (const attr of Array.from(el.attributes)) {
|
|
63659
|
+
const name = attr.name.toLowerCase();
|
|
63660
|
+
if (name.startsWith("on") || forbidAttrs.has(name) || !attrAllowed(name, allowedAttrs, attrWildcards) || !isSafeValue(tag, name, attr.value)) {
|
|
63661
|
+
el.removeAttribute(attr.name);
|
|
63662
|
+
}
|
|
63663
|
+
}
|
|
63664
|
+
stack.push(el);
|
|
63665
|
+
}
|
|
63666
|
+
}
|
|
63667
|
+
for (let i = toUnwrap.length - 1; i >= 0; i--) {
|
|
63668
|
+
const el = toUnwrap[i];
|
|
63669
|
+
const parent = el.parentNode;
|
|
63670
|
+
if (!parent) continue;
|
|
63671
|
+
while (el.firstChild) parent.insertBefore(el.firstChild, el);
|
|
63672
|
+
parent.removeChild(el);
|
|
63673
|
+
}
|
|
63674
|
+
return body.innerHTML;
|
|
63675
|
+
}
|
|
63676
|
+
|
|
63506
63677
|
// ../../src/shared/token-estimator.ts
|
|
63507
63678
|
var CJK_REGEX3 = /[\u4e00-\u9fff\u3400-\u4dbf\u3000-\u303f\u3040-\u309f\u30a0-\u30ff\uac00-\ud7af]/g;
|
|
63508
63679
|
var MAX_INPUT_LENGTH = 1e6;
|
|
@@ -64052,6 +64223,18 @@ async function binaryToMarkdown(buffer, type, format) {
|
|
|
64052
64223
|
}
|
|
64053
64224
|
}
|
|
64054
64225
|
}
|
|
64226
|
+
function linkedomDomEnvironment() {
|
|
64227
|
+
const parseHTML2 = globalThis.__FF_parseHTML;
|
|
64228
|
+
if (!parseHTML2) throw new Error("The CLI shim must be loaded before converting a URL");
|
|
64229
|
+
const parseDocument = (html) => parseHTML2(html).document;
|
|
64230
|
+
const { window: window3 } = parseHTML2("<!DOCTYPE html><html><head></head><body></body></html>");
|
|
64231
|
+
return {
|
|
64232
|
+
DOMParser: window3.DOMParser,
|
|
64233
|
+
DOMPurify: {
|
|
64234
|
+
sanitize: (html, config) => sanitizeWithLinkedom(html, config ?? {}, parseDocument)
|
|
64235
|
+
}
|
|
64236
|
+
};
|
|
64237
|
+
}
|
|
64055
64238
|
var DEFAULT_PROXY_URL = "https://formatferry.vibingfun.com";
|
|
64056
64239
|
async function urlToMarkdown(url, apiKey, format, proxyBaseUrl = process.env.FORMATFERRY_PROXY_URL ?? DEFAULT_PROXY_URL) {
|
|
64057
64240
|
const extractEndpoint = `${proxyBaseUrl}/v1/extract`;
|
|
@@ -64063,7 +64246,9 @@ async function urlToMarkdown(url, apiKey, format, proxyBaseUrl = process.env.FOR
|
|
|
64063
64246
|
"Content-Type": "application/json",
|
|
64064
64247
|
"X-API-Key": apiKey
|
|
64065
64248
|
},
|
|
64066
|
-
|
|
64249
|
+
// Readability and the chosen flavour run locally, so ask for the page's
|
|
64250
|
+
// HTML (the API's default is Markdown).
|
|
64251
|
+
body: JSON.stringify({ url, format: "html" })
|
|
64067
64252
|
});
|
|
64068
64253
|
} catch (err) {
|
|
64069
64254
|
const message = err instanceof Error ? err.message : String(err);
|
|
@@ -64074,18 +64259,19 @@ async function urlToMarkdown(url, apiKey, format, proxyBaseUrl = process.env.FOR
|
|
|
64074
64259
|
if (!response.ok) {
|
|
64075
64260
|
let errorMessage = `Proxy returned ${response.status}`;
|
|
64076
64261
|
try {
|
|
64077
|
-
const
|
|
64078
|
-
errorMessage =
|
|
64262
|
+
const body2 = await response.json();
|
|
64263
|
+
errorMessage = body2.error ?? body2.message ?? errorMessage;
|
|
64079
64264
|
} catch {
|
|
64080
64265
|
}
|
|
64081
64266
|
process.stderr.write(`Error: ${errorMessage}
|
|
64082
64267
|
`);
|
|
64083
64268
|
process.exit(1);
|
|
64084
64269
|
}
|
|
64085
|
-
const
|
|
64270
|
+
const body = await response.json();
|
|
64271
|
+
const html = body.content ?? body.html ?? "";
|
|
64086
64272
|
const domProvider = new LinkedomProvider();
|
|
64087
64273
|
const article = extractArticle(html, domProvider);
|
|
64088
|
-
const engine = new ConversionEngine();
|
|
64274
|
+
const engine = new ConversionEngine({ dom: linkedomDomEnvironment() });
|
|
64089
64275
|
const result = await engine.convert(article.content, {
|
|
64090
64276
|
format,
|
|
64091
64277
|
preserveTables: true,
|
|
@@ -64589,8 +64775,8 @@ import { readFileSync as readFileSync5 } from "fs";
|
|
|
64589
64775
|
import { resolve, dirname as dirname3 } from "path";
|
|
64590
64776
|
import { fileURLToPath } from "url";
|
|
64591
64777
|
var getVersion = () => {
|
|
64592
|
-
if ("1.0.
|
|
64593
|
-
return "1.0.
|
|
64778
|
+
if ("1.0.34") {
|
|
64779
|
+
return "1.0.34";
|
|
64594
64780
|
}
|
|
64595
64781
|
try {
|
|
64596
64782
|
const __dirname2 = dirname3(fileURLToPath(import.meta.url));
|