guidelinescraper 1.0.7 → 1.0.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/crawl.mjs +4 -1
- package/package.json +1 -1
- package/purge-html.mjs +1 -1
package/crawl.mjs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import { PlaywrightCrawler } from "crawlee";
|
|
2
|
+
import { PlaywrightCrawler, Configuration } from "crawlee";
|
|
3
3
|
import fs from "node:fs";
|
|
4
4
|
import path from "node:path";
|
|
5
5
|
import os from "node:os";
|
|
@@ -124,6 +124,9 @@ for (const { pdfPath } of pages) {
|
|
|
124
124
|
const cookieConsentValue =
|
|
125
125
|
'{"isCookieConsentOpen":false,"preferencesScriptsEnabled":true,"statisticsScriptsEnabled":true}';
|
|
126
126
|
|
|
127
|
+
Configuration.getGlobalConfig().set("persistStorage", false);
|
|
128
|
+
Configuration.getGlobalConfig().set("purgeOnStart", true);
|
|
129
|
+
|
|
127
130
|
const crawler = new PlaywrightCrawler({
|
|
128
131
|
headless: true,
|
|
129
132
|
launchContext: {
|
package/package.json
CHANGED
package/purge-html.mjs
CHANGED
|
@@ -107,7 +107,7 @@ export function purge(html) {
|
|
|
107
107
|
|
|
108
108
|
const title = document.querySelector("title")?.textContent?.trim() || "";
|
|
109
109
|
const cleanHtml =
|
|
110
|
-
`<!DOCTYPE html>\n<html lang="${document.documentElement?.getAttribute("lang") || "en"}">\n<head>\n<meta charset="utf-8">\n<title>${title}</title>\n<style>body{max-width:72ch;margin:2rem auto;padding:0 1rem;font:1rem/1.6 monospace}img{max-width:100%;height:auto}pre{background:#f5f5f5;padding:1rem;overflow-x:auto;border-radius:4px}code{font-family:monospace}</style>\n</head>\n<body>\n${main.innerHTML.trim()}\n</body>\n</html>`;
|
|
110
|
+
`<!DOCTYPE html>\n<html lang="${document.documentElement?.getAttribute("lang") || "en"}">\n<head>\n<meta charset="utf-8">\n<title>${title}</title>\n<style>body{max-width:72ch;margin:2rem auto;padding:0 1rem;font:1rem/1.6 monospace}img{max-width:100%;height:auto}pre{background:#f5f5f5;padding:1rem;overflow-x:auto;border-radius:4px}code{font-family:monospace}table{border-collapse:collapse;width:100%}th,td{border:1px solid #ccc;padding:.5rem;text-align:left}th{background:#f5f5f5}</style>\n</head>\n<body>\n${main.innerHTML.trim()}\n</body>\n</html>`;
|
|
111
111
|
|
|
112
112
|
return cleanHtml.replace(/\n{3,}/g, "\n\n").replace(/[ \t]+\n/g, "\n");
|
|
113
113
|
}
|