guidelinescraper 1.0.6 → 1.0.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/crawl.mjs +7 -1
- package/package.json +1 -1
- package/purge-html.mjs +23 -1
package/crawl.mjs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import { PlaywrightCrawler } from "crawlee";
|
|
2
|
+
import { PlaywrightCrawler, Configuration } from "crawlee";
|
|
3
3
|
import fs from "node:fs";
|
|
4
4
|
import path from "node:path";
|
|
5
5
|
import os from "node:os";
|
|
@@ -124,6 +124,11 @@ for (const { pdfPath } of pages) {
|
|
|
124
124
|
const cookieConsentValue =
|
|
125
125
|
'{"isCookieConsentOpen":false,"preferencesScriptsEnabled":true,"statisticsScriptsEnabled":true}';
|
|
126
126
|
|
|
127
|
+
const config = new Configuration({
|
|
128
|
+
persistStorage: false,
|
|
129
|
+
purgeOnStart: true,
|
|
130
|
+
});
|
|
131
|
+
|
|
127
132
|
const crawler = new PlaywrightCrawler({
|
|
128
133
|
headless: true,
|
|
129
134
|
launchContext: {
|
|
@@ -132,6 +137,7 @@ const crawler = new PlaywrightCrawler({
|
|
|
132
137
|
maxConcurrency: 8,
|
|
133
138
|
maxRequestRetries: 2,
|
|
134
139
|
navigationTimeoutSecs: 120,
|
|
140
|
+
configuration: config,
|
|
135
141
|
|
|
136
142
|
preNavigationHooks: [
|
|
137
143
|
async ({ page, request }) => {
|
package/package.json
CHANGED
package/purge-html.mjs
CHANGED
|
@@ -83,9 +83,31 @@ export function purge(html) {
|
|
|
83
83
|
|
|
84
84
|
for (const child of [...main.childNodes]) cleanNode(child);
|
|
85
85
|
|
|
86
|
+
// Frontify code blocks: a <pre> with only line numbers followed by <pre><code>
|
|
87
|
+
for (const pre of [...main.querySelectorAll("pre")]) {
|
|
88
|
+
if (pre.querySelector("code")) continue;
|
|
89
|
+
const text = pre.textContent.trim();
|
|
90
|
+
if (/^(\d+\s*)+$/.test(text)) {
|
|
91
|
+
pre.remove();
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
// Remove stray button labels left behind from copy/download widgets
|
|
96
|
+
const strayLabels = /^(Copy|Download)\s+(markdown|code|link|text)$/i;
|
|
97
|
+
const walk = (node) => {
|
|
98
|
+
for (const child of [...node.childNodes]) {
|
|
99
|
+
if (child.nodeType === 3 && strayLabels.test(child.textContent.trim())) {
|
|
100
|
+
child.remove();
|
|
101
|
+
} else if (child.nodeType === 1) {
|
|
102
|
+
walk(child);
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
};
|
|
106
|
+
walk(main);
|
|
107
|
+
|
|
86
108
|
const title = document.querySelector("title")?.textContent?.trim() || "";
|
|
87
109
|
const cleanHtml =
|
|
88
|
-
`<!DOCTYPE html>\n<html lang="${document.documentElement?.getAttribute("lang") || "en"}">\n<head>\n<meta charset="utf-8">\n<title>${title}</title>\n<style>body{max-width:72ch;margin:2rem auto;padding:0 1rem;font:1rem/1.6 monospace}img{max-width:100%;height:auto}</style>\n</head>\n<body>\n${main.innerHTML.trim()}\n</body>\n</html>`;
|
|
110
|
+
`<!DOCTYPE html>\n<html lang="${document.documentElement?.getAttribute("lang") || "en"}">\n<head>\n<meta charset="utf-8">\n<title>${title}</title>\n<style>body{max-width:72ch;margin:2rem auto;padding:0 1rem;font:1rem/1.6 monospace}img{max-width:100%;height:auto}pre{background:#f5f5f5;padding:1rem;overflow-x:auto;border-radius:4px}code{font-family:monospace}table{border-collapse:collapse;width:100%}th,td{border:1px solid #ccc;padding:.5rem;text-align:left}th{background:#f5f5f5}</style>\n</head>\n<body>\n${main.innerHTML.trim()}\n</body>\n</html>`;
|
|
89
111
|
|
|
90
112
|
return cleanHtml.replace(/\n{3,}/g, "\n\n").replace(/[ \t]+\n/g, "\n");
|
|
91
113
|
}
|