@rendela/vite 1.0.3 → 1.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/crawler.js +42 -34
- package/dist/crawler.js.map +1 -1
- package/dist/package.json +1 -1
- package/dist/types/crawler.d.ts +1 -1
- package/package.json +1 -1
package/dist/crawler.js
CHANGED
|
@@ -1,12 +1,13 @@
|
|
|
1
|
-
import { chromium } from
|
|
2
|
-
import fs from
|
|
3
|
-
import path from
|
|
4
|
-
import { LogUtils } from
|
|
5
|
-
import { execSync } from
|
|
1
|
+
import { chromium } from "playwright";
|
|
2
|
+
import fs from "fs";
|
|
3
|
+
import path from "path";
|
|
4
|
+
import { LogUtils } from "./logUtils.js";
|
|
5
|
+
import { execSync } from "child_process";
|
|
6
6
|
export async function startCrawler(config) {
|
|
7
|
-
await askToInstallChromium(config);
|
|
7
|
+
const executablePath = await askToInstallChromium(config);
|
|
8
8
|
const browser = await chromium.launch({
|
|
9
|
-
headless: true
|
|
9
|
+
headless: true,
|
|
10
|
+
executablePath,
|
|
10
11
|
});
|
|
11
12
|
const projectSavePath = path.join(config.buildDir, config.savePath);
|
|
12
13
|
const savePath = path.join(process.cwd(), projectSavePath);
|
|
@@ -20,7 +21,7 @@ export async function startCrawler(config) {
|
|
|
20
21
|
const pages = [...config.pages];
|
|
21
22
|
while (pages.length > 0) {
|
|
22
23
|
if (config.debug) {
|
|
23
|
-
LogUtils.info(
|
|
24
|
+
LogUtils.info("");
|
|
24
25
|
}
|
|
25
26
|
const batch = pages.splice(0, concurrencyLimit);
|
|
26
27
|
await Promise.all(batch.map((url) => processPage(browser, config, url, savePath, projectSavePath)));
|
|
@@ -28,20 +29,20 @@ export async function startCrawler(config) {
|
|
|
28
29
|
await browser.close();
|
|
29
30
|
}
|
|
30
31
|
async function processPage(browser, config, url, savePath, projectSavePath) {
|
|
31
|
-
if (!url.url.startsWith(
|
|
32
|
+
if (!url.url.startsWith("/")) {
|
|
32
33
|
url.url = `/${url.url}`;
|
|
33
34
|
}
|
|
34
35
|
const html = await crawlPage(browser, config, url);
|
|
35
36
|
if (html) {
|
|
36
|
-
const fileName = url.url && url.url !==
|
|
37
|
+
const fileName = url.url && url.url !== "/" ? url.url : "";
|
|
37
38
|
const filePath = path.join(savePath, fileName);
|
|
38
39
|
if (!fs.existsSync(filePath)) {
|
|
39
40
|
fs.mkdirSync(filePath, { recursive: true });
|
|
40
41
|
}
|
|
41
|
-
await fs.promises.writeFile(path.join(filePath,
|
|
42
|
+
await fs.promises.writeFile(path.join(filePath, "index.html"), html);
|
|
42
43
|
if (config.debug) {
|
|
43
|
-
const signalPreSignal = fileName[0] ===
|
|
44
|
-
const signalPostSignal = fileName ===
|
|
44
|
+
const signalPreSignal = fileName[0] === "/" ? "" : "/";
|
|
45
|
+
const signalPostSignal = fileName === "" || fileName[fileName.length - 1] === "/" ? "" : "/";
|
|
45
46
|
LogUtils.info(`Crawled: ${LogUtils.startPrintingGray()}${url.url}${LogUtils.endPrintingGray()} > ${LogUtils.startPrintingGray()}${projectSavePath}${signalPreSignal}${LogUtils.endPrintingGray()}${fileName}${signalPostSignal}index.html`);
|
|
46
47
|
}
|
|
47
48
|
}
|
|
@@ -51,9 +52,9 @@ async function crawlPage(browser, config, pageConfig) {
|
|
|
51
52
|
const page = await context.newPage();
|
|
52
53
|
try {
|
|
53
54
|
// Block unnecessary resources
|
|
54
|
-
await page.route(
|
|
55
|
+
await page.route("**/*", (route) => {
|
|
55
56
|
const resourceType = route.request().resourceType();
|
|
56
|
-
if ([
|
|
57
|
+
if (["image", "font", "media"].includes(resourceType)) {
|
|
57
58
|
route.abort();
|
|
58
59
|
}
|
|
59
60
|
else {
|
|
@@ -65,7 +66,7 @@ async function crawlPage(browser, config, pageConfig) {
|
|
|
65
66
|
page.setDefaultNavigationTimeout(timeout + config.pageTimeout);
|
|
66
67
|
}
|
|
67
68
|
const response = await page.goto(`http://localhost:${config.port}${pageConfig.url}`, {
|
|
68
|
-
waitUntil:
|
|
69
|
+
waitUntil: "networkidle",
|
|
69
70
|
});
|
|
70
71
|
if (response && response.status() === 404) {
|
|
71
72
|
LogUtils.error(`Skipping 404: ${pageConfig.url}`);
|
|
@@ -85,38 +86,45 @@ async function crawlPage(browser, config, pageConfig) {
|
|
|
85
86
|
}
|
|
86
87
|
}
|
|
87
88
|
async function askToInstallChromium(config) {
|
|
88
|
-
if (config?.chromiumExecutablePath)
|
|
89
|
-
return;
|
|
89
|
+
if (config?.chromiumExecutablePath) {
|
|
90
|
+
return config.chromiumExecutablePath;
|
|
91
|
+
}
|
|
90
92
|
let browserInstalled = false;
|
|
93
|
+
let path = undefined;
|
|
91
94
|
try {
|
|
92
|
-
|
|
95
|
+
path = await chromium.executablePath(); // will throw if not installed
|
|
93
96
|
// Check if the executable path exists
|
|
94
97
|
if (fs.existsSync(path)) {
|
|
95
98
|
browserInstalled = true;
|
|
99
|
+
path = path;
|
|
96
100
|
}
|
|
97
101
|
}
|
|
98
102
|
catch { }
|
|
99
|
-
if (browserInstalled)
|
|
100
|
-
return;
|
|
103
|
+
if (browserInstalled && path) {
|
|
104
|
+
return path;
|
|
105
|
+
}
|
|
101
106
|
if (config.debug) {
|
|
102
|
-
LogUtils.log(
|
|
107
|
+
LogUtils.log("Installing Chromium...");
|
|
103
108
|
}
|
|
104
109
|
const packageManager = getPackageManager();
|
|
105
|
-
execSync(`${packageManager} playwright install chromium`, {
|
|
110
|
+
execSync(`${packageManager} playwright install chromium`, {
|
|
111
|
+
stdio: "inherit",
|
|
112
|
+
});
|
|
106
113
|
if (config.debug) {
|
|
107
|
-
LogUtils.success(
|
|
114
|
+
LogUtils.success("Chromium installed successfully");
|
|
108
115
|
}
|
|
116
|
+
return await chromium.executablePath();
|
|
109
117
|
}
|
|
110
118
|
function getPackageManager() {
|
|
111
|
-
const userAgent = process.env.npm_config_user_agent ||
|
|
112
|
-
if (process.env.BUN_INSTALL ===
|
|
113
|
-
return
|
|
114
|
-
if (userAgent.includes(
|
|
115
|
-
return
|
|
116
|
-
if (userAgent.includes(
|
|
117
|
-
return
|
|
118
|
-
if (userAgent.includes(
|
|
119
|
-
return
|
|
120
|
-
return
|
|
119
|
+
const userAgent = process.env.npm_config_user_agent || "";
|
|
120
|
+
if (process.env.BUN_INSTALL === "1")
|
|
121
|
+
return "bunx";
|
|
122
|
+
if (userAgent.includes("pnpm"))
|
|
123
|
+
return "pnpx";
|
|
124
|
+
if (userAgent.includes("yarn"))
|
|
125
|
+
return "yarn dlx";
|
|
126
|
+
if (userAgent.includes("npm"))
|
|
127
|
+
return "npx";
|
|
128
|
+
return "npx";
|
|
121
129
|
}
|
|
122
130
|
//# sourceMappingURL=crawler.js.map
|
package/dist/crawler.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"crawler.js","sourceRoot":"","sources":["../src/crawler.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,
|
|
1
|
+
{"version":3,"file":"crawler.js","sourceRoot":"","sources":["../src/crawler.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAW,MAAM,YAAY,CAAC;AAC/C,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAEzC,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAEzC,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,MAAyB;IAC1D,MAAM,cAAc,GAAG,MAAM,oBAAoB,CAAC,MAAM,CAAC,CAAC;IAC1D,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC;QACpC,QAAQ,EAAE,IAAI;QACd,cAAc;KACf,CAAC,CAAC;IAEH,MAAM,eAAe,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,QAAS,CAAC,CAAC;IACrE,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,eAAe,CAAC,CAAC;IAE3D,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC7B,EAAE,CAAC,SAAS,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC9C,CAAC;IAED,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;QACjB,QAAQ,CAAC,GAAG,CACV,KACE,MAAM,CAAC,KAAM,CAAC,MAAM,IAAI,CAC1B,sBAAsB,QAAQ,CAAC,iBAAiB,EAAE,GAAG,eAAe,GAAG,QAAQ,CAAC,eAAe,EAAE,EAAE,CACpG,CAAC;IACJ,CAAC;IAED,MAAM,gBAAgB,GAAG,MAAM,CAAC,gBAAgB,IAAI,CAAC,CAAC;IACtD,MAAM,KAAK,GAAG,CAAC,GAAG,MAAM,CAAC,KAAM,CAAC,CAAC;IAEjC,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxB,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;YACjB,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACpB,CAAC;QACD,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,EAAE,gBAAgB,CAAC,CAAC;QAChD,MAAM,OAAO,CAAC,GAAG,CACf,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAChB,WAAW,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,QAAQ,EAAE,eAAe,CAAC,CAC7D,CACF,CAAC;IACJ,CAAC;IAED,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;AACxB,CAAC;AAED,KAAK,UAAU,WAAW,CACxB,OAAgB,EAChB,MAAyB,EACzB,GAAoB,EACpB,QAAgB,EAChB,eAAuB;IAEvB,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QAC7B,GAAG,CAAC,GAAG,GAAG,IAAI,GAAG,CAAC,GAAG,EAAE,CAAC;IAC1B,CAAC;IAED,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,CAAC,CAAC;IACnD,IAAI,IAAI,EAAE,CAAC;QACT,MAAM,QAAQ,GAAG,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;QAC3D,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;QAC/C,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC7B,EAAE,CAAC,SAAS,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC9C,CAAC;QAED,MAAM,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,YAAY,CAAC,EAAE,IAAI,CAAC,CAAC;QACrE,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;YACjB,MAAM,eAAe,GAAG,QAAQ,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;YACvD,MAAM,gBAAgB,GACpB,QAAQ,KAAK,EAAE,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;YACtE,QAAQ,CAAC,IAAI,CACX,YAAY,QAAQ,CAAC,iBAAiB,EAAE,GACtC,GAAG,CAAC,GACN,GAAG,QAAQ,CAAC,eAAe,EAAE,MAAM,QAAQ,CAAC,iBAAiB,EAAE,GAAG,eAAe,GAAG,eAAe,GAAG,QAAQ,CAAC,eAAe,EAAE,GAAG,QAAQ,GAAG,gBAAgB,YAAY,CAC3K,CAAC;QACJ,CAAC;IACH,CAAC;AACH,CAAC;AAED,KAAK,UAAU,SAAS,CACtB,OAAgB,EAChB,MAAyB,EACzB,UAA2B;IAE3B,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,EAAE,CAAC;IAC3C,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;IAErC,IAAI,CAAC;QACH,8BAA8B;QAC9B,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE;YACjC,MAAM,YAAY,GAAG,KAAK,CAAC,OAAO,EAAE,CAAC,YAAY,EAAE,CAAC;YACpD,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC;gBACtD,KAAK,CAAC,KAAK,EAAE,CAAC;YAChB,CAAC;iBAAM,CAAC;gBACN,KAAK,CAAC,QAAQ,EAAE,CAAC;YACnB,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,MAAM,OAAO,GAAG,MAAM,EAAE,YAAY,IAAI,CAAC,CAAC;QAC1C,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC;YACvB,IAAI,CAAC,2BAA2B,CAAC,OAAO,GAAG,MAAM,CAAC,WAAW,CAAC,CAAC;QACjE,CAAC;QAED,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,IAAI,CAC9B,oBAAoB,MAAM,CAAC,IAAI,GAAG,UAAU,CAAC,GAAG,EAAE,EAClD;YACE,SAAS,EAAE,aAAa;SACzB,CACF,CAAC;QAEF,IAAI,QAAQ,IAAI,QAAQ,CAAC,MAAM,EAAE,KAAK,GAAG,EAAE,CAAC;YAC1C,QAAQ,CAAC,KAAK,CAAC,iBAAiB,UAAU,CAAC,GAAG,EAAE,CAAC,CAAC;YAClD,OAAO,IAAI,CAAC;QACd,CAAC;QAED,MAAM,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;QAEnC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;QAClC,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,QAAQ,CAAC,KAAK,CAAC,kBAAkB,UAAU,CAAC,GAAG,KAAK,KAAK,EAAE,CAAC,CAAC;QAC7D,OAAO,IAAI,CAAC;IACd,CAAC;YAAS,CAAC;QACT,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QACnB,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;IACxB,CAAC;AACH,CAAC;AAED,KAAK,UAAU,oBAAoB,CAAC,MAAyB;IAC3D,IAAI,MAAM,EAAE,sBAAsB,EAAE,CAAC;QACnC,OAAO,MAAM,CAAC,sBAAsB,CAAC;IACvC,CAAC;IACD,IAAI,gBAAgB,GAAG,KAAK,CAAC;IAC7B,IAAI,IAAI,GAAG,SAAS,CAAC;IACrB,IAAI,CAAC;QACH,IAAI,GAAG,MAAM,QAAQ,CAAC,cAAc,EAAE,CAAC,CAAC,8BAA8B;QACtE,sCAAsC;QACtC,IAAI,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;YACxB,gBAAgB,GAAG,IAAI,CAAC;YACxB,IAAI,GAAG,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAAC,MAAM,CAAC,CAAA,CAAC;IACV,IAAI,gBAAgB,IAAI,IAAI,EAAE,CAAC;QAC7B,OAAO,IAAI,CAAC;IACd,CAAC;IACD,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;QACjB,QAAQ,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAC;IACzC,CAAC;IACD,MAAM,cAAc,GAAG,iBAAiB,EAAE,CAAC;IAC3C,QAAQ,CAAC,GAAG,cAAc,8BAA8B,EAAE;QACxD,KAAK,EAAE,SAAS;KACjB,CAAC,CAAC;IACH,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;QACjB,QAAQ,CAAC,OAAO,CAAC,iCAAiC,CAAC,CAAC;IACtD,CAAC;IACD,OAAO,MAAM,QAAQ,CAAC,cAAc,EAAE,CAAC;AACzC,CAAC;AAED,SAAS,iBAAiB;IACxB,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,qBAAqB,IAAI,EAAE,CAAC;IAC1D,IAAI,OAAO,CAAC,GAAG,CAAC,WAAW,KAAK,GAAG;QAAE,OAAO,MAAM,CAAC;IACnD,IAAI,SAAS,CAAC,QAAQ,CAAC,MAAM,CAAC;QAAE,OAAO,MAAM,CAAC;IAC9C,IAAI,SAAS,CAAC,QAAQ,CAAC,MAAM,CAAC;QAAE,OAAO,UAAU,CAAC;IAClD,IAAI,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAC5C,OAAO,KAAK,CAAC;AACf,CAAC"}
|
package/dist/package.json
CHANGED
package/dist/types/crawler.d.ts
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
import { RendelaConfigType } from
|
|
1
|
+
import { RendelaConfigType } from "./config.js";
|
|
2
2
|
export declare function startCrawler(config: RendelaConfigType): Promise<void>;
|