@xbrowser/cli 1.6.3 → 1.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-O3FLVCUU.js → chunk-24SY6PE5.js} +12 -1
- package/dist/{chunk-L35D5DAY.js → chunk-53LRNYPG.js} +23 -2
- package/dist/cli.js +76 -45
- package/dist/{daemon-client-WT7PTGYQ.js → daemon-client-PZX2KOLQ.js} +20 -4
- package/dist/{daemon-client-S3EUTRC6.js → daemon-client-TOUDMIY5.js} +1 -1
- package/dist/daemon-main.js +41 -4
- package/dist/index.js +76 -45
- package/package.json +1 -1
|
@@ -98,6 +98,9 @@ async function startDaemonProcess(port = 9224) {
|
|
|
98
98
|
});
|
|
99
99
|
});
|
|
100
100
|
}
|
|
101
|
+
async function stopDaemonProcess() {
|
|
102
|
+
await xcliStopDaemon(getDaemonConfig());
|
|
103
|
+
}
|
|
101
104
|
function getDaemonProcessStatus() {
|
|
102
105
|
const config = getDaemonConfig();
|
|
103
106
|
const running = isDaemonRunning(config);
|
|
@@ -113,8 +116,16 @@ function getDaemonProcessStatus() {
|
|
|
113
116
|
};
|
|
114
117
|
}
|
|
115
118
|
|
|
119
|
+
// src/version.ts
|
|
120
|
+
import { createRequire } from "module";
|
|
121
|
+
var require2 = createRequire(import.meta.url);
|
|
122
|
+
var pkg = require2("../package.json");
|
|
123
|
+
var version = pkg.version;
|
|
124
|
+
|
|
116
125
|
export {
|
|
117
126
|
getDaemonConfig,
|
|
118
127
|
startDaemonProcess,
|
|
119
|
-
|
|
128
|
+
stopDaemonProcess,
|
|
129
|
+
getDaemonProcessStatus,
|
|
130
|
+
version
|
|
120
131
|
};
|
|
@@ -123,6 +123,12 @@ function getDaemonProcessStatus() {
|
|
|
123
123
|
};
|
|
124
124
|
}
|
|
125
125
|
|
|
126
|
+
// src/version.ts
|
|
127
|
+
import { createRequire } from "module";
|
|
128
|
+
var require2 = createRequire(import.meta.url);
|
|
129
|
+
var pkg = require2("../package.json");
|
|
130
|
+
var version = pkg.version;
|
|
131
|
+
|
|
126
132
|
// src/client/daemon-client.ts
|
|
127
133
|
var DAEMON_PORT = 9224;
|
|
128
134
|
var DAEMON_BASE = `http://localhost:${DAEMON_PORT}`;
|
|
@@ -136,8 +142,22 @@ async function ensureDaemonRunning() {
|
|
|
136
142
|
_ensurePromise = null;
|
|
137
143
|
}
|
|
138
144
|
for (let attempt = 0; attempt < 3; attempt++) {
|
|
139
|
-
const healthOk = await fetch(`${DAEMON_BASE}/health`, { signal: AbortSignal.timeout(2e3) }).then((r) => r.ok ? r.json() : null).then((d) =>
|
|
140
|
-
|
|
145
|
+
const healthOk = await fetch(`${DAEMON_BASE}/health`, { signal: AbortSignal.timeout(2e3) }).then((r) => r.ok ? r.json() : null).then((d) => {
|
|
146
|
+
if (d?.status === "ok") {
|
|
147
|
+
if (d.version && d.version !== version) {
|
|
148
|
+
return { needsRestart: true };
|
|
149
|
+
}
|
|
150
|
+
return { needsRestart: false };
|
|
151
|
+
}
|
|
152
|
+
return null;
|
|
153
|
+
}).catch(() => null);
|
|
154
|
+
if (healthOk?.needsRestart === false) return;
|
|
155
|
+
if (healthOk?.needsRestart === true) {
|
|
156
|
+
console.error(`\u26A0\uFE0F Daemon version mismatch. Restarting...`);
|
|
157
|
+
await stopDaemonProcess().catch(() => {
|
|
158
|
+
});
|
|
159
|
+
break;
|
|
160
|
+
}
|
|
141
161
|
if (attempt < 2) await new Promise((r) => setTimeout(r, 500));
|
|
142
162
|
}
|
|
143
163
|
console.error("\u{1F504} Starting daemon...");
|
|
@@ -308,6 +328,7 @@ async function forwardViewerCheckSelector(name, selector) {
|
|
|
308
328
|
}
|
|
309
329
|
|
|
310
330
|
export {
|
|
331
|
+
version,
|
|
311
332
|
getDaemonConfig,
|
|
312
333
|
startDaemonProcess,
|
|
313
334
|
stopDaemonProcess,
|
package/dist/cli.js
CHANGED
|
@@ -53,8 +53,9 @@ import {
|
|
|
53
53
|
getDaemonProcessStatus,
|
|
54
54
|
killAllDaemonProcesses,
|
|
55
55
|
startDaemonProcess,
|
|
56
|
-
stopDaemonProcess
|
|
57
|
-
|
|
56
|
+
stopDaemonProcess,
|
|
57
|
+
version
|
|
58
|
+
} from "./chunk-53LRNYPG.js";
|
|
58
59
|
import {
|
|
59
60
|
errMsg
|
|
60
61
|
} from "./chunk-GDKLH7ZY.js";
|
|
@@ -178,12 +179,6 @@ function asZodSchema(value) {
|
|
|
178
179
|
return value;
|
|
179
180
|
}
|
|
180
181
|
|
|
181
|
-
// src/version.ts
|
|
182
|
-
import { createRequire } from "module";
|
|
183
|
-
var require2 = createRequire(import.meta.url);
|
|
184
|
-
var pkg = require2("../package.json");
|
|
185
|
-
var version = pkg.version;
|
|
186
|
-
|
|
187
182
|
// src/executor.ts
|
|
188
183
|
import {
|
|
189
184
|
ok as ok25,
|
|
@@ -2799,6 +2794,24 @@ async function extractLinks(page, origin) {
|
|
|
2799
2794
|
}).filter(Boolean);
|
|
2800
2795
|
}, origin);
|
|
2801
2796
|
}
|
|
2797
|
+
async function detectSpaRoutes(page, origin) {
|
|
2798
|
+
return page.evaluate((evalOrigin) => {
|
|
2799
|
+
const routeSet = /* @__PURE__ */ new Set();
|
|
2800
|
+
try {
|
|
2801
|
+
const scripts = document.querySelectorAll("script");
|
|
2802
|
+
const allContent = Array.from(scripts).map((s) => s.textContent || "").join("\n");
|
|
2803
|
+
const pathRegex = /['"`](\/[a-zA-Z0-9_\-/]+)['"`]/g;
|
|
2804
|
+
let match;
|
|
2805
|
+
while ((match = pathRegex.exec(allContent)) !== null) {
|
|
2806
|
+
const path3 = match[1];
|
|
2807
|
+
if (path3.includes(":") || path3.includes("*") || routeSet.has(path3)) continue;
|
|
2808
|
+
routeSet.add(path3);
|
|
2809
|
+
}
|
|
2810
|
+
} catch {
|
|
2811
|
+
}
|
|
2812
|
+
return Array.from(routeSet).map((path3) => `${evalOrigin.replace(/\/$/, "")}${path3}`);
|
|
2813
|
+
}, origin);
|
|
2814
|
+
}
|
|
2802
2815
|
function parseRobotsTxt(text) {
|
|
2803
2816
|
const rules = [];
|
|
2804
2817
|
let inRelevantBlock = false;
|
|
@@ -2938,6 +2951,7 @@ var crawlCommand = registerCommand({
|
|
|
2938
2951
|
allowSubdomains: z17.boolean().default(false),
|
|
2939
2952
|
allowExternalLinks: z17.boolean().default(false),
|
|
2940
2953
|
allowBackwardCrawling: z17.boolean().default(false),
|
|
2954
|
+
enableSpa: z17.boolean().default(false).describe("Detect SPA (Vue/React) routes from router config"),
|
|
2941
2955
|
format: z17.enum(["markdown", "html"]).default("markdown"),
|
|
2942
2956
|
onlyMainContent: z17.boolean().default(true),
|
|
2943
2957
|
concurrency: z17.number().default(3),
|
|
@@ -2954,6 +2968,7 @@ var crawlCommand = registerCommand({
|
|
|
2954
2968
|
allowSubdomains: p.allowSubdomains,
|
|
2955
2969
|
allowExternalLinks: p.allowExternalLinks,
|
|
2956
2970
|
allowBackwardCrawling: p.allowBackwardCrawling,
|
|
2971
|
+
enableSpa: p.enableSpa,
|
|
2957
2972
|
format: p.format,
|
|
2958
2973
|
onlyMainContent: p.onlyMainContent,
|
|
2959
2974
|
concurrency: Math.min(Math.max(p.concurrency, 1), 10),
|
|
@@ -2986,6 +3001,22 @@ var crawlCommand = registerCommand({
|
|
|
2986
3001
|
const content = options.format === "html" ? html : htmlToMarkdown(html, { onlyMainContent: options.onlyMainContent });
|
|
2987
3002
|
results.push({ url: seedPage.url(), title, content });
|
|
2988
3003
|
const firstLinks = await extractLinks(seedPage, startUrl.origin);
|
|
3004
|
+
if (options.enableSpa) {
|
|
3005
|
+
const spaRoutes = await detectSpaRoutes(seedPage, startUrl.origin);
|
|
3006
|
+
for (const route2 of spaRoutes) {
|
|
3007
|
+
try {
|
|
3008
|
+
const absNorm = normalizeUrl(stripHashAnchorQuery(route2));
|
|
3009
|
+
if (!visited.has(absNorm) && !shouldSkipUrl(route2)) {
|
|
3010
|
+
queue.push({ url: stripHashAnchorQuery(route2), depth: 1 });
|
|
3011
|
+
}
|
|
3012
|
+
} catch {
|
|
3013
|
+
}
|
|
3014
|
+
}
|
|
3015
|
+
if (options.verbose && spaRoutes.length > 0) {
|
|
3016
|
+
process.stderr.write(`[SPA] Detected ${spaRoutes.length} SPA routes
|
|
3017
|
+
`);
|
|
3018
|
+
}
|
|
3019
|
+
}
|
|
2989
3020
|
for (const link of firstLinks) {
|
|
2990
3021
|
try {
|
|
2991
3022
|
const absolute = new URL(link, seedPage.url()).href;
|
|
@@ -6119,14 +6150,14 @@ function ensurePluginDependencies(pluginsDir) {
|
|
|
6119
6150
|
if (existsSync3(zodPath)) return;
|
|
6120
6151
|
mkdirSync4(pluginsDir, { recursive: true });
|
|
6121
6152
|
const pkgPath = join5(pluginsDir, "package.json");
|
|
6122
|
-
let
|
|
6153
|
+
let pkg = {};
|
|
6123
6154
|
if (existsSync3(pkgPath)) {
|
|
6124
6155
|
try {
|
|
6125
|
-
|
|
6156
|
+
pkg = readJsonFile(pkgPath, {});
|
|
6126
6157
|
} catch {
|
|
6127
6158
|
}
|
|
6128
6159
|
}
|
|
6129
|
-
const existingDeps =
|
|
6160
|
+
const existingDeps = pkg.dependencies || {};
|
|
6130
6161
|
let needsInstall = false;
|
|
6131
6162
|
for (const [dep, version2] of Object.entries(SHARED_PLUGIN_DEPENDENCIES)) {
|
|
6132
6163
|
if (!existingDeps[dep]) {
|
|
@@ -6135,10 +6166,10 @@ function ensurePluginDependencies(pluginsDir) {
|
|
|
6135
6166
|
}
|
|
6136
6167
|
}
|
|
6137
6168
|
if (!needsInstall && existsSync3(join5(pluginsDir, "node_modules"))) return;
|
|
6138
|
-
|
|
6139
|
-
|
|
6140
|
-
|
|
6141
|
-
writeFileSync5(pkgPath, JSON.stringify(
|
|
6169
|
+
pkg.dependencies = existingDeps;
|
|
6170
|
+
pkg.private = true;
|
|
6171
|
+
pkg.description = pkg.description || "xbrowser plugins \u2014 shared dependencies";
|
|
6172
|
+
writeFileSync5(pkgPath, JSON.stringify(pkg, null, 2) + "\n", "utf-8");
|
|
6142
6173
|
try {
|
|
6143
6174
|
execSync("npm install --production --no-package-lock --no-fund --no-audit", {
|
|
6144
6175
|
cwd: pluginsDir,
|
|
@@ -7186,7 +7217,7 @@ async function executeCommand(commandName, params, sessionName = "default", extr
|
|
|
7186
7217
|
params = result.data;
|
|
7187
7218
|
}
|
|
7188
7219
|
if (command.scope !== "cli" && !process.env.XBROWSER_DAEMON_WORKER) {
|
|
7189
|
-
const { forwardExec } = await import("./daemon-client-
|
|
7220
|
+
const { forwardExec } = await import("./daemon-client-TOUDMIY5.js");
|
|
7190
7221
|
const result = await forwardExec(commandName, params, sessionName, extraOpts?.cdpEndpoint);
|
|
7191
7222
|
if (result) return result;
|
|
7192
7223
|
}
|
|
@@ -8030,10 +8061,10 @@ async function installFromNpm(packageName, name, targetDir) {
|
|
|
8030
8061
|
cpSync2(extractDir, targetDir, { recursive: true, force: true });
|
|
8031
8062
|
const pkgPath = resolve4(targetDir, "package.json");
|
|
8032
8063
|
if (existsSync6(pkgPath)) {
|
|
8033
|
-
const
|
|
8034
|
-
if (!
|
|
8035
|
-
|
|
8036
|
-
writeFileSync6(pkgPath, JSON.stringify(
|
|
8064
|
+
const pkg = JSON.parse(readFileSync4(pkgPath, "utf-8"));
|
|
8065
|
+
if (!pkg._npmSource) {
|
|
8066
|
+
pkg._npmSource = { name: packageName, version: latestVersion };
|
|
8067
|
+
writeFileSync6(pkgPath, JSON.stringify(pkg, null, 2));
|
|
8037
8068
|
}
|
|
8038
8069
|
}
|
|
8039
8070
|
} finally {
|
|
@@ -8072,10 +8103,10 @@ async function installFromGit(gitUrl, name, targetDir) {
|
|
|
8072
8103
|
rmSync3(resolve5(targetDir, ".git"), { recursive: true, force: true });
|
|
8073
8104
|
const pkgPath = resolve5(targetDir, "package.json");
|
|
8074
8105
|
if (existsSync7(pkgPath)) {
|
|
8075
|
-
const
|
|
8076
|
-
if (!
|
|
8077
|
-
|
|
8078
|
-
writeFileSync7(pkgPath, JSON.stringify(
|
|
8106
|
+
const pkg = JSON.parse(readFileSync5(pkgPath, "utf-8"));
|
|
8107
|
+
if (!pkg._gitSource) {
|
|
8108
|
+
pkg._gitSource = { url: gitUrl };
|
|
8109
|
+
writeFileSync7(pkgPath, JSON.stringify(pkg, null, 2));
|
|
8079
8110
|
}
|
|
8080
8111
|
}
|
|
8081
8112
|
} finally {
|
|
@@ -8124,10 +8155,10 @@ async function installFromUrl(url, name, targetDir) {
|
|
|
8124
8155
|
cpSync4(extractDir, targetDir, { recursive: true, force: true });
|
|
8125
8156
|
const pkgPath = resolve6(targetDir, "package.json");
|
|
8126
8157
|
if (existsSync8(pkgPath)) {
|
|
8127
|
-
const
|
|
8128
|
-
if (!
|
|
8129
|
-
|
|
8130
|
-
writeFileSync8(pkgPath, JSON.stringify(
|
|
8158
|
+
const pkg = JSON.parse(readFileSync6(pkgPath, "utf-8"));
|
|
8159
|
+
if (!pkg._urlSource) {
|
|
8160
|
+
pkg._urlSource = { url };
|
|
8161
|
+
writeFileSync8(pkgPath, JSON.stringify(pkg, null, 2));
|
|
8131
8162
|
}
|
|
8132
8163
|
}
|
|
8133
8164
|
} finally {
|
|
@@ -8537,11 +8568,11 @@ var PluginInstaller = class {
|
|
|
8537
8568
|
if (!existsSync10(indexPath) && !existsSync10(indexJsPath)) continue;
|
|
8538
8569
|
const metadata = PluginMetadataParser.parseFromPackageJson(pluginPath);
|
|
8539
8570
|
let source = "local";
|
|
8540
|
-
const
|
|
8541
|
-
if (
|
|
8542
|
-
else if (
|
|
8543
|
-
else if (
|
|
8544
|
-
else if (
|
|
8571
|
+
const pkg = readJsonFile(resolve8(pluginPath, "package.json"), {});
|
|
8572
|
+
if (pkg._marketplace) source = "marketplace";
|
|
8573
|
+
else if (pkg._npmSource) source = "npm";
|
|
8574
|
+
else if (pkg._gitSource) source = "git";
|
|
8575
|
+
else if (pkg._urlSource) source = "url";
|
|
8545
8576
|
plugins.push({
|
|
8546
8577
|
id: entry.name,
|
|
8547
8578
|
name: entry.name,
|
|
@@ -8881,8 +8912,8 @@ var NPMSearcher = class {
|
|
|
8881
8912
|
}
|
|
8882
8913
|
return parts.join(" ");
|
|
8883
8914
|
}
|
|
8884
|
-
static parseNPMPackage(
|
|
8885
|
-
const data =
|
|
8915
|
+
static parseNPMPackage(pkg) {
|
|
8916
|
+
const data = pkg;
|
|
8886
8917
|
const author = this.parseAuthor(data.author);
|
|
8887
8918
|
const links = this.parseLinks(data);
|
|
8888
8919
|
const time = data.time;
|
|
@@ -10442,19 +10473,19 @@ async function handlePluginInfo(args, options, mode) {
|
|
|
10442
10473
|
const distTags = data["dist-tags"];
|
|
10443
10474
|
const latest = distTags?.latest;
|
|
10444
10475
|
const versions = data.versions;
|
|
10445
|
-
const
|
|
10446
|
-
if (
|
|
10476
|
+
const pkg = latest && versions?.[latest];
|
|
10477
|
+
if (pkg) {
|
|
10447
10478
|
if (mode === "json") {
|
|
10448
|
-
outputResult({ source: "npm", name:
|
|
10479
|
+
outputResult({ source: "npm", name: pkg.name, version: latest, description: pkg.description }, mode);
|
|
10449
10480
|
return;
|
|
10450
10481
|
}
|
|
10451
|
-
console.log(`\u540D\u79F0: ${
|
|
10482
|
+
console.log(`\u540D\u79F0: ${pkg.name || ""}`);
|
|
10452
10483
|
console.log(`\u7248\u672C: ${latest}`);
|
|
10453
|
-
console.log(`\u63CF\u8FF0: ${
|
|
10454
|
-
const author =
|
|
10484
|
+
console.log(`\u63CF\u8FF0: ${pkg.description || ""}`);
|
|
10485
|
+
const author = pkg.author;
|
|
10455
10486
|
console.log(`\u4F5C\u8005: ${typeof author === "string" ? author : author?.name || ""}`);
|
|
10456
|
-
console.log(`\u5173\u952E\u8BCD: ${(
|
|
10457
|
-
console.log(`\u8BB8\u53EF\u8BC1: ${
|
|
10487
|
+
console.log(`\u5173\u952E\u8BCD: ${(pkg.keywords || []).join(", ")}`);
|
|
10488
|
+
console.log(`\u8BB8\u53EF\u8BC1: ${pkg.license || ""}`);
|
|
10458
10489
|
return;
|
|
10459
10490
|
}
|
|
10460
10491
|
}
|
|
@@ -10541,7 +10572,7 @@ async function handlePlugin(args, options, mode) {
|
|
|
10541
10572
|
} catch {
|
|
10542
10573
|
}
|
|
10543
10574
|
try {
|
|
10544
|
-
const { daemonPing } = await import("./daemon-client-
|
|
10575
|
+
const { daemonPing } = await import("./daemon-client-TOUDMIY5.js");
|
|
10545
10576
|
if (await daemonPing()) {
|
|
10546
10577
|
await fetch("http://localhost:9224/rpc", {
|
|
10547
10578
|
method: "POST",
|
|
@@ -12948,7 +12979,7 @@ Run "xbrowser ${command} ${subCommand} --help" to see available parameters.`
|
|
|
12948
12979
|
}
|
|
12949
12980
|
const needsBrowser = cmdEntry.scope === "page" || cmdEntry.scope === "browser";
|
|
12950
12981
|
if (needsBrowser && !process.env.XBROWSER_DAEMON_WORKER) {
|
|
12951
|
-
const { forwardExec } = await import("./daemon-client-
|
|
12982
|
+
const { forwardExec } = await import("./daemon-client-TOUDMIY5.js");
|
|
12952
12983
|
const userTimeout = typeof params.timeout === "number" && params.timeout > 0 ? params.timeout * 1e3 + 3e4 : void 0;
|
|
12953
12984
|
const result = await forwardExec(`${command}.${subCommand}`, params, sessionName, cdpEndpoint, userTimeout);
|
|
12954
12985
|
const resultData = result && typeof result === "object" && "data" in result ? result.data : void 0;
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import {
|
|
2
|
-
startDaemonProcess
|
|
3
|
-
|
|
2
|
+
startDaemonProcess,
|
|
3
|
+
stopDaemonProcess,
|
|
4
|
+
version
|
|
5
|
+
} from "./chunk-24SY6PE5.js";
|
|
4
6
|
import {
|
|
5
7
|
errMsg
|
|
6
8
|
} from "./chunk-GDKLH7ZY.js";
|
|
@@ -19,8 +21,22 @@ async function ensureDaemonRunning() {
|
|
|
19
21
|
_ensurePromise = null;
|
|
20
22
|
}
|
|
21
23
|
for (let attempt = 0; attempt < 3; attempt++) {
|
|
22
|
-
const healthOk = await fetch(`${DAEMON_BASE}/health`, { signal: AbortSignal.timeout(2e3) }).then((r) => r.ok ? r.json() : null).then((d) =>
|
|
23
|
-
|
|
24
|
+
const healthOk = await fetch(`${DAEMON_BASE}/health`, { signal: AbortSignal.timeout(2e3) }).then((r) => r.ok ? r.json() : null).then((d) => {
|
|
25
|
+
if (d?.status === "ok") {
|
|
26
|
+
if (d.version && d.version !== version) {
|
|
27
|
+
return { needsRestart: true };
|
|
28
|
+
}
|
|
29
|
+
return { needsRestart: false };
|
|
30
|
+
}
|
|
31
|
+
return null;
|
|
32
|
+
}).catch(() => null);
|
|
33
|
+
if (healthOk?.needsRestart === false) return;
|
|
34
|
+
if (healthOk?.needsRestart === true) {
|
|
35
|
+
console.error(`\u26A0\uFE0F Daemon version mismatch. Restarting...`);
|
|
36
|
+
await stopDaemonProcess().catch(() => {
|
|
37
|
+
});
|
|
38
|
+
break;
|
|
39
|
+
}
|
|
24
40
|
if (attempt < 2) await new Promise((r) => setTimeout(r, 500));
|
|
25
41
|
}
|
|
26
42
|
console.error("\u{1F504} Starting daemon...");
|
package/dist/daemon-main.js
CHANGED
|
@@ -29,8 +29,9 @@ import {
|
|
|
29
29
|
} from "./chunk-2QQDTXDL.js";
|
|
30
30
|
import {
|
|
31
31
|
getDaemonConfig,
|
|
32
|
-
getDaemonProcessStatus
|
|
33
|
-
|
|
32
|
+
getDaemonProcessStatus,
|
|
33
|
+
version
|
|
34
|
+
} from "./chunk-24SY6PE5.js";
|
|
34
35
|
import {
|
|
35
36
|
errMsg
|
|
36
37
|
} from "./chunk-GDKLH7ZY.js";
|
|
@@ -2760,6 +2761,24 @@ async function extractLinks(page, origin) {
|
|
|
2760
2761
|
}).filter(Boolean);
|
|
2761
2762
|
}, origin);
|
|
2762
2763
|
}
|
|
2764
|
+
async function detectSpaRoutes(page, origin) {
|
|
2765
|
+
return page.evaluate((evalOrigin) => {
|
|
2766
|
+
const routeSet = /* @__PURE__ */ new Set();
|
|
2767
|
+
try {
|
|
2768
|
+
const scripts = document.querySelectorAll("script");
|
|
2769
|
+
const allContent = Array.from(scripts).map((s) => s.textContent || "").join("\n");
|
|
2770
|
+
const pathRegex = /['"`](\/[a-zA-Z0-9_\-/]+)['"`]/g;
|
|
2771
|
+
let match;
|
|
2772
|
+
while ((match = pathRegex.exec(allContent)) !== null) {
|
|
2773
|
+
const path2 = match[1];
|
|
2774
|
+
if (path2.includes(":") || path2.includes("*") || routeSet.has(path2)) continue;
|
|
2775
|
+
routeSet.add(path2);
|
|
2776
|
+
}
|
|
2777
|
+
} catch {
|
|
2778
|
+
}
|
|
2779
|
+
return Array.from(routeSet).map((path2) => `${evalOrigin.replace(/\/$/, "")}${path2}`);
|
|
2780
|
+
}, origin);
|
|
2781
|
+
}
|
|
2763
2782
|
function parseRobotsTxt(text) {
|
|
2764
2783
|
const rules = [];
|
|
2765
2784
|
let inRelevantBlock = false;
|
|
@@ -2899,6 +2918,7 @@ var crawlCommand = registerCommand({
|
|
|
2899
2918
|
allowSubdomains: z17.boolean().default(false),
|
|
2900
2919
|
allowExternalLinks: z17.boolean().default(false),
|
|
2901
2920
|
allowBackwardCrawling: z17.boolean().default(false),
|
|
2921
|
+
enableSpa: z17.boolean().default(false).describe("Detect SPA (Vue/React) routes from router config"),
|
|
2902
2922
|
format: z17.enum(["markdown", "html"]).default("markdown"),
|
|
2903
2923
|
onlyMainContent: z17.boolean().default(true),
|
|
2904
2924
|
concurrency: z17.number().default(3),
|
|
@@ -2915,6 +2935,7 @@ var crawlCommand = registerCommand({
|
|
|
2915
2935
|
allowSubdomains: p.allowSubdomains,
|
|
2916
2936
|
allowExternalLinks: p.allowExternalLinks,
|
|
2917
2937
|
allowBackwardCrawling: p.allowBackwardCrawling,
|
|
2938
|
+
enableSpa: p.enableSpa,
|
|
2918
2939
|
format: p.format,
|
|
2919
2940
|
onlyMainContent: p.onlyMainContent,
|
|
2920
2941
|
concurrency: Math.min(Math.max(p.concurrency, 1), 10),
|
|
@@ -2947,6 +2968,22 @@ var crawlCommand = registerCommand({
|
|
|
2947
2968
|
const content = options.format === "html" ? html : htmlToMarkdown(html, { onlyMainContent: options.onlyMainContent });
|
|
2948
2969
|
results.push({ url: seedPage.url(), title, content });
|
|
2949
2970
|
const firstLinks = await extractLinks(seedPage, startUrl.origin);
|
|
2971
|
+
if (options.enableSpa) {
|
|
2972
|
+
const spaRoutes = await detectSpaRoutes(seedPage, startUrl.origin);
|
|
2973
|
+
for (const route of spaRoutes) {
|
|
2974
|
+
try {
|
|
2975
|
+
const absNorm = normalizeUrl(stripHashAnchorQuery(route));
|
|
2976
|
+
if (!visited.has(absNorm) && !shouldSkipUrl(route)) {
|
|
2977
|
+
queue.push({ url: stripHashAnchorQuery(route), depth: 1 });
|
|
2978
|
+
}
|
|
2979
|
+
} catch {
|
|
2980
|
+
}
|
|
2981
|
+
}
|
|
2982
|
+
if (options.verbose && spaRoutes.length > 0) {
|
|
2983
|
+
process.stderr.write(`[SPA] Detected ${spaRoutes.length} SPA routes
|
|
2984
|
+
`);
|
|
2985
|
+
}
|
|
2986
|
+
}
|
|
2950
2987
|
for (const link of firstLinks) {
|
|
2951
2988
|
try {
|
|
2952
2989
|
const absolute = new URL(link, seedPage.url()).href;
|
|
@@ -6717,7 +6754,7 @@ async function executeCommand(commandName, params, sessionName = "default", extr
|
|
|
6717
6754
|
params = result.data;
|
|
6718
6755
|
}
|
|
6719
6756
|
if (command.scope !== "cli" && !process.env.XBROWSER_DAEMON_WORKER) {
|
|
6720
|
-
const { forwardExec } = await import("./daemon-client-
|
|
6757
|
+
const { forwardExec } = await import("./daemon-client-PZX2KOLQ.js");
|
|
6721
6758
|
const result = await forwardExec(commandName, params, sessionName, extraOpts?.cdpEndpoint);
|
|
6722
6759
|
if (result) return result;
|
|
6723
6760
|
}
|
|
@@ -11208,7 +11245,7 @@ async function main() {
|
|
|
11208
11245
|
pathname: "/health",
|
|
11209
11246
|
handler: (_req, res) => {
|
|
11210
11247
|
res.writeHead(200, { "Content-Type": "application/json" });
|
|
11211
|
-
res.end(JSON.stringify({ status: "ok", pid: process.pid }));
|
|
11248
|
+
res.end(JSON.stringify({ status: "ok", pid: process.pid, version }));
|
|
11212
11249
|
}
|
|
11213
11250
|
}
|
|
11214
11251
|
]
|
package/dist/index.js
CHANGED
|
@@ -24,8 +24,9 @@ import {
|
|
|
24
24
|
getDaemonProcessStatus,
|
|
25
25
|
killAllDaemonProcesses,
|
|
26
26
|
startDaemonProcess,
|
|
27
|
-
stopDaemonProcess
|
|
28
|
-
|
|
27
|
+
stopDaemonProcess,
|
|
28
|
+
version
|
|
29
|
+
} from "./chunk-53LRNYPG.js";
|
|
29
30
|
import {
|
|
30
31
|
CaptchaDetector,
|
|
31
32
|
HumanInteractionManager,
|
|
@@ -105,12 +106,6 @@ import {
|
|
|
105
106
|
__require
|
|
106
107
|
} from "./chunk-KFQGP6VL.js";
|
|
107
108
|
|
|
108
|
-
// src/version.ts
|
|
109
|
-
import { createRequire } from "module";
|
|
110
|
-
var require2 = createRequire(import.meta.url);
|
|
111
|
-
var pkg = require2("../package.json");
|
|
112
|
-
var version = pkg.version;
|
|
113
|
-
|
|
114
109
|
// src/executor.ts
|
|
115
110
|
import {
|
|
116
111
|
ok as ok25,
|
|
@@ -2839,6 +2834,24 @@ async function extractLinks(page, origin) {
|
|
|
2839
2834
|
}).filter(Boolean);
|
|
2840
2835
|
}, origin);
|
|
2841
2836
|
}
|
|
2837
|
+
async function detectSpaRoutes(page, origin) {
|
|
2838
|
+
return page.evaluate((evalOrigin) => {
|
|
2839
|
+
const routeSet = /* @__PURE__ */ new Set();
|
|
2840
|
+
try {
|
|
2841
|
+
const scripts = document.querySelectorAll("script");
|
|
2842
|
+
const allContent = Array.from(scripts).map((s) => s.textContent || "").join("\n");
|
|
2843
|
+
const pathRegex = /['"`](\/[a-zA-Z0-9_\-/]+)['"`]/g;
|
|
2844
|
+
let match;
|
|
2845
|
+
while ((match = pathRegex.exec(allContent)) !== null) {
|
|
2846
|
+
const path5 = match[1];
|
|
2847
|
+
if (path5.includes(":") || path5.includes("*") || routeSet.has(path5)) continue;
|
|
2848
|
+
routeSet.add(path5);
|
|
2849
|
+
}
|
|
2850
|
+
} catch {
|
|
2851
|
+
}
|
|
2852
|
+
return Array.from(routeSet).map((path5) => `${evalOrigin.replace(/\/$/, "")}${path5}`);
|
|
2853
|
+
}, origin);
|
|
2854
|
+
}
|
|
2842
2855
|
function parseRobotsTxt(text) {
|
|
2843
2856
|
const rules = [];
|
|
2844
2857
|
let inRelevantBlock = false;
|
|
@@ -2978,6 +2991,7 @@ var crawlCommand = registerCommand({
|
|
|
2978
2991
|
allowSubdomains: z17.boolean().default(false),
|
|
2979
2992
|
allowExternalLinks: z17.boolean().default(false),
|
|
2980
2993
|
allowBackwardCrawling: z17.boolean().default(false),
|
|
2994
|
+
enableSpa: z17.boolean().default(false).describe("Detect SPA (Vue/React) routes from router config"),
|
|
2981
2995
|
format: z17.enum(["markdown", "html"]).default("markdown"),
|
|
2982
2996
|
onlyMainContent: z17.boolean().default(true),
|
|
2983
2997
|
concurrency: z17.number().default(3),
|
|
@@ -2994,6 +3008,7 @@ var crawlCommand = registerCommand({
|
|
|
2994
3008
|
allowSubdomains: p.allowSubdomains,
|
|
2995
3009
|
allowExternalLinks: p.allowExternalLinks,
|
|
2996
3010
|
allowBackwardCrawling: p.allowBackwardCrawling,
|
|
3011
|
+
enableSpa: p.enableSpa,
|
|
2997
3012
|
format: p.format,
|
|
2998
3013
|
onlyMainContent: p.onlyMainContent,
|
|
2999
3014
|
concurrency: Math.min(Math.max(p.concurrency, 1), 10),
|
|
@@ -3026,6 +3041,22 @@ var crawlCommand = registerCommand({
|
|
|
3026
3041
|
const content = options.format === "html" ? html : htmlToMarkdown(html, { onlyMainContent: options.onlyMainContent });
|
|
3027
3042
|
results.push({ url: seedPage.url(), title, content });
|
|
3028
3043
|
const firstLinks = await extractLinks(seedPage, startUrl.origin);
|
|
3044
|
+
if (options.enableSpa) {
|
|
3045
|
+
const spaRoutes = await detectSpaRoutes(seedPage, startUrl.origin);
|
|
3046
|
+
for (const route2 of spaRoutes) {
|
|
3047
|
+
try {
|
|
3048
|
+
const absNorm = normalizeUrl(stripHashAnchorQuery(route2));
|
|
3049
|
+
if (!visited.has(absNorm) && !shouldSkipUrl(route2)) {
|
|
3050
|
+
queue.push({ url: stripHashAnchorQuery(route2), depth: 1 });
|
|
3051
|
+
}
|
|
3052
|
+
} catch {
|
|
3053
|
+
}
|
|
3054
|
+
}
|
|
3055
|
+
if (options.verbose && spaRoutes.length > 0) {
|
|
3056
|
+
process.stderr.write(`[SPA] Detected ${spaRoutes.length} SPA routes
|
|
3057
|
+
`);
|
|
3058
|
+
}
|
|
3059
|
+
}
|
|
3029
3060
|
for (const link of firstLinks) {
|
|
3030
3061
|
try {
|
|
3031
3062
|
const absolute = new URL(link, seedPage.url()).href;
|
|
@@ -6436,14 +6467,14 @@ function ensurePluginDependencies(pluginsDir) {
|
|
|
6436
6467
|
if (existsSync3(zodPath)) return;
|
|
6437
6468
|
mkdirSync4(pluginsDir, { recursive: true });
|
|
6438
6469
|
const pkgPath = join5(pluginsDir, "package.json");
|
|
6439
|
-
let
|
|
6470
|
+
let pkg = {};
|
|
6440
6471
|
if (existsSync3(pkgPath)) {
|
|
6441
6472
|
try {
|
|
6442
|
-
|
|
6473
|
+
pkg = readJsonFile(pkgPath, {});
|
|
6443
6474
|
} catch {
|
|
6444
6475
|
}
|
|
6445
6476
|
}
|
|
6446
|
-
const existingDeps =
|
|
6477
|
+
const existingDeps = pkg.dependencies || {};
|
|
6447
6478
|
let needsInstall = false;
|
|
6448
6479
|
for (const [dep, version2] of Object.entries(SHARED_PLUGIN_DEPENDENCIES)) {
|
|
6449
6480
|
if (!existingDeps[dep]) {
|
|
@@ -6452,10 +6483,10 @@ function ensurePluginDependencies(pluginsDir) {
|
|
|
6452
6483
|
}
|
|
6453
6484
|
}
|
|
6454
6485
|
if (!needsInstall && existsSync3(join5(pluginsDir, "node_modules"))) return;
|
|
6455
|
-
|
|
6456
|
-
|
|
6457
|
-
|
|
6458
|
-
writeFileSync5(pkgPath, JSON.stringify(
|
|
6486
|
+
pkg.dependencies = existingDeps;
|
|
6487
|
+
pkg.private = true;
|
|
6488
|
+
pkg.description = pkg.description || "xbrowser plugins \u2014 shared dependencies";
|
|
6489
|
+
writeFileSync5(pkgPath, JSON.stringify(pkg, null, 2) + "\n", "utf-8");
|
|
6459
6490
|
try {
|
|
6460
6491
|
execSync("npm install --production --no-package-lock --no-fund --no-audit", {
|
|
6461
6492
|
cwd: pluginsDir,
|
|
@@ -7506,7 +7537,7 @@ async function executeCommand(commandName, params, sessionName = "default", extr
|
|
|
7506
7537
|
params = result.data;
|
|
7507
7538
|
}
|
|
7508
7539
|
if (command.scope !== "cli" && !process.env.XBROWSER_DAEMON_WORKER) {
|
|
7509
|
-
const { forwardExec } = await import("./daemon-client-
|
|
7540
|
+
const { forwardExec } = await import("./daemon-client-TOUDMIY5.js");
|
|
7510
7541
|
const result = await forwardExec(commandName, params, sessionName, extraOpts?.cdpEndpoint);
|
|
7511
7542
|
if (result) return result;
|
|
7512
7543
|
}
|
|
@@ -8348,10 +8379,10 @@ async function installFromNpm(packageName, name, targetDir) {
|
|
|
8348
8379
|
cpSync2(extractDir, targetDir, { recursive: true, force: true });
|
|
8349
8380
|
const pkgPath = resolve4(targetDir, "package.json");
|
|
8350
8381
|
if (existsSync6(pkgPath)) {
|
|
8351
|
-
const
|
|
8352
|
-
if (!
|
|
8353
|
-
|
|
8354
|
-
writeFileSync6(pkgPath, JSON.stringify(
|
|
8382
|
+
const pkg = JSON.parse(readFileSync4(pkgPath, "utf-8"));
|
|
8383
|
+
if (!pkg._npmSource) {
|
|
8384
|
+
pkg._npmSource = { name: packageName, version: latestVersion };
|
|
8385
|
+
writeFileSync6(pkgPath, JSON.stringify(pkg, null, 2));
|
|
8355
8386
|
}
|
|
8356
8387
|
}
|
|
8357
8388
|
} finally {
|
|
@@ -8390,10 +8421,10 @@ async function installFromGit(gitUrl, name, targetDir) {
|
|
|
8390
8421
|
rmSync3(resolve5(targetDir, ".git"), { recursive: true, force: true });
|
|
8391
8422
|
const pkgPath = resolve5(targetDir, "package.json");
|
|
8392
8423
|
if (existsSync7(pkgPath)) {
|
|
8393
|
-
const
|
|
8394
|
-
if (!
|
|
8395
|
-
|
|
8396
|
-
writeFileSync7(pkgPath, JSON.stringify(
|
|
8424
|
+
const pkg = JSON.parse(readFileSync5(pkgPath, "utf-8"));
|
|
8425
|
+
if (!pkg._gitSource) {
|
|
8426
|
+
pkg._gitSource = { url: gitUrl };
|
|
8427
|
+
writeFileSync7(pkgPath, JSON.stringify(pkg, null, 2));
|
|
8397
8428
|
}
|
|
8398
8429
|
}
|
|
8399
8430
|
} finally {
|
|
@@ -8442,10 +8473,10 @@ async function installFromUrl(url, name, targetDir) {
|
|
|
8442
8473
|
cpSync4(extractDir, targetDir, { recursive: true, force: true });
|
|
8443
8474
|
const pkgPath = resolve6(targetDir, "package.json");
|
|
8444
8475
|
if (existsSync8(pkgPath)) {
|
|
8445
|
-
const
|
|
8446
|
-
if (!
|
|
8447
|
-
|
|
8448
|
-
writeFileSync8(pkgPath, JSON.stringify(
|
|
8476
|
+
const pkg = JSON.parse(readFileSync6(pkgPath, "utf-8"));
|
|
8477
|
+
if (!pkg._urlSource) {
|
|
8478
|
+
pkg._urlSource = { url };
|
|
8479
|
+
writeFileSync8(pkgPath, JSON.stringify(pkg, null, 2));
|
|
8449
8480
|
}
|
|
8450
8481
|
}
|
|
8451
8482
|
} finally {
|
|
@@ -8855,11 +8886,11 @@ var PluginInstaller = class {
|
|
|
8855
8886
|
if (!existsSync10(indexPath) && !existsSync10(indexJsPath)) continue;
|
|
8856
8887
|
const metadata = PluginMetadataParser.parseFromPackageJson(pluginPath);
|
|
8857
8888
|
let source = "local";
|
|
8858
|
-
const
|
|
8859
|
-
if (
|
|
8860
|
-
else if (
|
|
8861
|
-
else if (
|
|
8862
|
-
else if (
|
|
8889
|
+
const pkg = readJsonFile(resolve8(pluginPath, "package.json"), {});
|
|
8890
|
+
if (pkg._marketplace) source = "marketplace";
|
|
8891
|
+
else if (pkg._npmSource) source = "npm";
|
|
8892
|
+
else if (pkg._gitSource) source = "git";
|
|
8893
|
+
else if (pkg._urlSource) source = "url";
|
|
8863
8894
|
plugins.push({
|
|
8864
8895
|
id: entry.name,
|
|
8865
8896
|
name: entry.name,
|
|
@@ -9199,8 +9230,8 @@ var NPMSearcher = class {
|
|
|
9199
9230
|
}
|
|
9200
9231
|
return parts.join(" ");
|
|
9201
9232
|
}
|
|
9202
|
-
static parseNPMPackage(
|
|
9203
|
-
const data =
|
|
9233
|
+
static parseNPMPackage(pkg) {
|
|
9234
|
+
const data = pkg;
|
|
9204
9235
|
const author = this.parseAuthor(data.author);
|
|
9205
9236
|
const links = this.parseLinks(data);
|
|
9206
9237
|
const time = data.time;
|
|
@@ -10765,19 +10796,19 @@ async function handlePluginInfo(args, options, mode) {
|
|
|
10765
10796
|
const distTags = data["dist-tags"];
|
|
10766
10797
|
const latest = distTags?.latest;
|
|
10767
10798
|
const versions = data.versions;
|
|
10768
|
-
const
|
|
10769
|
-
if (
|
|
10799
|
+
const pkg = latest && versions?.[latest];
|
|
10800
|
+
if (pkg) {
|
|
10770
10801
|
if (mode === "json") {
|
|
10771
|
-
outputResult({ source: "npm", name:
|
|
10802
|
+
outputResult({ source: "npm", name: pkg.name, version: latest, description: pkg.description }, mode);
|
|
10772
10803
|
return;
|
|
10773
10804
|
}
|
|
10774
|
-
console.log(`\u540D\u79F0: ${
|
|
10805
|
+
console.log(`\u540D\u79F0: ${pkg.name || ""}`);
|
|
10775
10806
|
console.log(`\u7248\u672C: ${latest}`);
|
|
10776
|
-
console.log(`\u63CF\u8FF0: ${
|
|
10777
|
-
const author =
|
|
10807
|
+
console.log(`\u63CF\u8FF0: ${pkg.description || ""}`);
|
|
10808
|
+
const author = pkg.author;
|
|
10778
10809
|
console.log(`\u4F5C\u8005: ${typeof author === "string" ? author : author?.name || ""}`);
|
|
10779
|
-
console.log(`\u5173\u952E\u8BCD: ${(
|
|
10780
|
-
console.log(`\u8BB8\u53EF\u8BC1: ${
|
|
10810
|
+
console.log(`\u5173\u952E\u8BCD: ${(pkg.keywords || []).join(", ")}`);
|
|
10811
|
+
console.log(`\u8BB8\u53EF\u8BC1: ${pkg.license || ""}`);
|
|
10781
10812
|
return;
|
|
10782
10813
|
}
|
|
10783
10814
|
}
|
|
@@ -10864,7 +10895,7 @@ async function handlePlugin(args, options, mode) {
|
|
|
10864
10895
|
} catch {
|
|
10865
10896
|
}
|
|
10866
10897
|
try {
|
|
10867
|
-
const { daemonPing } = await import("./daemon-client-
|
|
10898
|
+
const { daemonPing } = await import("./daemon-client-TOUDMIY5.js");
|
|
10868
10899
|
if (await daemonPing()) {
|
|
10869
10900
|
await fetch("http://localhost:9224/rpc", {
|
|
10870
10901
|
method: "POST",
|
|
@@ -13271,7 +13302,7 @@ Run "xbrowser ${command} ${subCommand} --help" to see available parameters.`
|
|
|
13271
13302
|
}
|
|
13272
13303
|
const needsBrowser = cmdEntry.scope === "page" || cmdEntry.scope === "browser";
|
|
13273
13304
|
if (needsBrowser && !process.env.XBROWSER_DAEMON_WORKER) {
|
|
13274
|
-
const { forwardExec } = await import("./daemon-client-
|
|
13305
|
+
const { forwardExec } = await import("./daemon-client-TOUDMIY5.js");
|
|
13275
13306
|
const userTimeout = typeof params.timeout === "number" && params.timeout > 0 ? params.timeout * 1e3 + 3e4 : void 0;
|
|
13276
13307
|
const result = await forwardExec(`${command}.${subCommand}`, params, sessionName, cdpEndpoint, userTimeout);
|
|
13277
13308
|
const resultData = result && typeof result === "object" && "data" in result ? result.data : void 0;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@xbrowser/cli",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.7.1",
|
|
4
4
|
"description": "Browser automation CLI for web scraping, headless browsing, SEO analysis, and AI agent workflows. A command-line alternative to Playwright, Puppeteer, and Selenium.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|