guidelinescraper 1.0.7 → 1.0.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/crawl.mjs +7 -1
- package/package.json +1 -1
- package/purge-html.mjs +1 -1
package/crawl.mjs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import { PlaywrightCrawler } from "crawlee";
|
|
2
|
+
import { PlaywrightCrawler, Configuration } from "crawlee";
|
|
3
3
|
import fs from "node:fs";
|
|
4
4
|
import path from "node:path";
|
|
5
5
|
import os from "node:os";
|
|
@@ -124,6 +124,11 @@ for (const { pdfPath } of pages) {
|
|
|
124
124
|
const cookieConsentValue =
|
|
125
125
|
'{"isCookieConsentOpen":false,"preferencesScriptsEnabled":true,"statisticsScriptsEnabled":true}';
|
|
126
126
|
|
|
127
|
+
const config = new Configuration({
|
|
128
|
+
persistStorage: false,
|
|
129
|
+
purgeOnStart: true,
|
|
130
|
+
});
|
|
131
|
+
|
|
127
132
|
const crawler = new PlaywrightCrawler({
|
|
128
133
|
headless: true,
|
|
129
134
|
launchContext: {
|
|
@@ -132,6 +137,7 @@ const crawler = new PlaywrightCrawler({
|
|
|
132
137
|
maxConcurrency: 8,
|
|
133
138
|
maxRequestRetries: 2,
|
|
134
139
|
navigationTimeoutSecs: 120,
|
|
140
|
+
configuration: config,
|
|
135
141
|
|
|
136
142
|
preNavigationHooks: [
|
|
137
143
|
async ({ page, request }) => {
|
package/package.json
CHANGED
package/purge-html.mjs
CHANGED
|
@@ -107,7 +107,7 @@ export function purge(html) {
|
|
|
107
107
|
|
|
108
108
|
const title = document.querySelector("title")?.textContent?.trim() || "";
|
|
109
109
|
const cleanHtml =
|
|
110
|
-
`<!DOCTYPE html>\n<html lang="${document.documentElement?.getAttribute("lang") || "en"}">\n<head>\n<meta charset="utf-8">\n<title>${title}</title>\n<style>body{max-width:72ch;margin:2rem auto;padding:0 1rem;font:1rem/1.6 monospace}img{max-width:100%;height:auto}pre{background:#f5f5f5;padding:1rem;overflow-x:auto;border-radius:4px}code{font-family:monospace}</style>\n</head>\n<body>\n${main.innerHTML.trim()}\n</body>\n</html>`;
|
|
110
|
+
`<!DOCTYPE html>\n<html lang="${document.documentElement?.getAttribute("lang") || "en"}">\n<head>\n<meta charset="utf-8">\n<title>${title}</title>\n<style>body{max-width:72ch;margin:2rem auto;padding:0 1rem;font:1rem/1.6 monospace}img{max-width:100%;height:auto}pre{background:#f5f5f5;padding:1rem;overflow-x:auto;border-radius:4px}code{font-family:monospace}table{border-collapse:collapse;width:100%}th,td{border:1px solid #ccc;padding:.5rem;text-align:left}th{background:#f5f5f5}</style>\n</head>\n<body>\n${main.innerHTML.trim()}\n</body>\n</html>`;
|
|
111
111
|
|
|
112
112
|
return cleanHtml.replace(/\n{3,}/g, "\n\n").replace(/[ \t]+\n/g, "\n");
|
|
113
113
|
}
|