@taodigital/agent-cli 1.2.0 → 1.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +1076 -49
- package/dist/index.js.map +4 -4
- package/package.json +2 -2
package/dist/index.js
CHANGED
|
@@ -6226,9 +6226,9 @@ var require_websocket = __commonJS({
|
|
|
6226
6226
|
emitErrorAndClose(websocket, err);
|
|
6227
6227
|
});
|
|
6228
6228
|
req.on("response", (res) => {
|
|
6229
|
-
const
|
|
6229
|
+
const location2 = res.headers.location;
|
|
6230
6230
|
const statusCode = res.statusCode;
|
|
6231
|
-
if (
|
|
6231
|
+
if (location2 && opts.followRedirects && statusCode >= 300 && statusCode < 400) {
|
|
6232
6232
|
if (++websocket._redirects > opts.maxRedirects) {
|
|
6233
6233
|
abortHandshake(websocket, req, "Maximum redirects exceeded");
|
|
6234
6234
|
return;
|
|
@@ -6236,9 +6236,9 @@ var require_websocket = __commonJS({
|
|
|
6236
6236
|
req.abort();
|
|
6237
6237
|
let addr;
|
|
6238
6238
|
try {
|
|
6239
|
-
addr = new URL2(
|
|
6239
|
+
addr = new URL2(location2, address);
|
|
6240
6240
|
} catch (e) {
|
|
6241
|
-
const err = new SyntaxError(`Invalid URL: ${
|
|
6241
|
+
const err = new SyntaxError(`Invalid URL: ${location2}`);
|
|
6242
6242
|
emitErrorAndClose(websocket, err);
|
|
6243
6243
|
return;
|
|
6244
6244
|
}
|
|
@@ -7579,10 +7579,83 @@ async function runPostman(message, send, signal) {
|
|
|
7579
7579
|
send({ ...result, duration: Date.now() - started });
|
|
7580
7580
|
}
|
|
7581
7581
|
|
|
7582
|
+
// src/proxy.ts
|
|
7583
|
+
var import_node_net = require("node:net");
|
|
7584
|
+
var PROXY_HDR_LEN = 37;
|
|
7585
|
+
var PROXY_DATA_TO_AGENT = 1;
|
|
7586
|
+
var PROXY_DATA_FROM_AGENT = 2;
|
|
7587
|
+
var PROXY_CLOSE = 3;
|
|
7588
|
+
var tunnels = /* @__PURE__ */ new Map();
|
|
7589
|
+
function handleProxyConnect(ws, tunnelId, host, port) {
|
|
7590
|
+
const tcp = (0, import_node_net.createConnection)({ host, port }, () => {
|
|
7591
|
+
ws.send(JSON.stringify({ type: "proxy_connected", tunnelId }));
|
|
7592
|
+
tcp.on("data", (data) => {
|
|
7593
|
+
if (ws.readyState !== ws.OPEN)
|
|
7594
|
+
return;
|
|
7595
|
+
const hdr = Buffer.allocUnsafe(PROXY_HDR_LEN);
|
|
7596
|
+
hdr[0] = PROXY_DATA_FROM_AGENT;
|
|
7597
|
+
hdr.write(tunnelId, 1, "ascii");
|
|
7598
|
+
ws.send(Buffer.concat([hdr, data]));
|
|
7599
|
+
});
|
|
7600
|
+
});
|
|
7601
|
+
tcp.on("error", (err) => {
|
|
7602
|
+
ws.send(JSON.stringify({ type: "proxy_error", tunnelId, error: err.message }));
|
|
7603
|
+
tunnels.delete(tunnelId);
|
|
7604
|
+
});
|
|
7605
|
+
tcp.on("close", () => {
|
|
7606
|
+
if (!tunnels.has(tunnelId))
|
|
7607
|
+
return;
|
|
7608
|
+
tunnels.delete(tunnelId);
|
|
7609
|
+
if (ws.readyState === ws.OPEN) {
|
|
7610
|
+
const hdr = Buffer.allocUnsafe(PROXY_HDR_LEN);
|
|
7611
|
+
hdr[0] = PROXY_CLOSE;
|
|
7612
|
+
hdr.write(tunnelId, 1, "ascii");
|
|
7613
|
+
ws.send(hdr);
|
|
7614
|
+
}
|
|
7615
|
+
});
|
|
7616
|
+
tunnels.set(tunnelId, { socket: tcp });
|
|
7617
|
+
}
|
|
7618
|
+
function handleProxyBinaryFrame(ws, raw) {
|
|
7619
|
+
if (raw.length < PROXY_HDR_LEN)
|
|
7620
|
+
return;
|
|
7621
|
+
const msgType = raw[0];
|
|
7622
|
+
if (msgType !== PROXY_DATA_TO_AGENT && msgType !== PROXY_CLOSE)
|
|
7623
|
+
return;
|
|
7624
|
+
const tunnelId = raw.slice(1, PROXY_HDR_LEN).toString("ascii");
|
|
7625
|
+
const tunnel = tunnels.get(tunnelId);
|
|
7626
|
+
if (!tunnel)
|
|
7627
|
+
return;
|
|
7628
|
+
if (msgType === PROXY_DATA_TO_AGENT) {
|
|
7629
|
+
tunnel.socket.write(raw.slice(PROXY_HDR_LEN));
|
|
7630
|
+
} else {
|
|
7631
|
+
tunnel.socket.destroy();
|
|
7632
|
+
tunnels.delete(tunnelId);
|
|
7633
|
+
}
|
|
7634
|
+
}
|
|
7635
|
+
function closeAllTunnels() {
|
|
7636
|
+
for (const [id, tunnel] of tunnels) {
|
|
7637
|
+
tunnel.socket.destroy();
|
|
7638
|
+
tunnels.delete(id);
|
|
7639
|
+
}
|
|
7640
|
+
}
|
|
7641
|
+
|
|
7582
7642
|
// src/index.ts
|
|
7583
7643
|
var _pkg = (0, import_node_module2.createRequire)(__filename)("../package.json");
|
|
7584
7644
|
var PKG_VERSION = _pkg.version;
|
|
7585
7645
|
var API_URL = process.env.ZETA_API_URL ?? "https://zeta.taodigitalsolutions.com/api";
|
|
7646
|
+
var PROFILE_API_URLS = {
|
|
7647
|
+
default: "https://zeta.taodigitalsolutions.com/api",
|
|
7648
|
+
prod: "https://zeta.taodigitalsolutions.com/api",
|
|
7649
|
+
dev: "https://dev.zeta.taodigitalsolutions.com/api",
|
|
7650
|
+
preview: "https://pre.zeta.taodigitalsolutions.com/api",
|
|
7651
|
+
pre: "https://pre.zeta.taodigitalsolutions.com/api",
|
|
7652
|
+
local: "http://localhost:3001/api"
|
|
7653
|
+
};
|
|
7654
|
+
function resolveApiUrl(explicitFlag, profile) {
|
|
7655
|
+
if (explicitFlag)
|
|
7656
|
+
return explicitFlag;
|
|
7657
|
+
return PROFILE_API_URLS[profile] ?? API_URL;
|
|
7658
|
+
}
|
|
7586
7659
|
var DEFAULT_MAX_SCREENS = process.env.ZETA_DEFAULT_MAX_SCREENS ?? "50";
|
|
7587
7660
|
var DEFAULT_STRATEGY = process.env.ZETA_DEFAULT_STRATEGY ?? "BFS";
|
|
7588
7661
|
var LOGIN_TIMEOUT_MS = parseInt(process.env.ZETA_LOGIN_TIMEOUT_MS ?? "") || 1e4;
|
|
@@ -7603,7 +7676,8 @@ var WsMessageType = {
|
|
|
7603
7676
|
RUN_COMPLETE: "run_complete",
|
|
7604
7677
|
SETUP: "setup",
|
|
7605
7678
|
DOCTOR: "doctor",
|
|
7606
|
-
RUN_POSTMAN: "run_postman"
|
|
7679
|
+
RUN_POSTMAN: "run_postman",
|
|
7680
|
+
LOCAL_CRAWL: "local_crawl"
|
|
7607
7681
|
};
|
|
7608
7682
|
var TERMINAL_JOB_STATUSES = /* @__PURE__ */ new Set([
|
|
7609
7683
|
"DONE",
|
|
@@ -7612,20 +7686,82 @@ var TERMINAL_JOB_STATUSES = /* @__PURE__ */ new Set([
|
|
|
7612
7686
|
"TIMED_OUT",
|
|
7613
7687
|
"ABORTED"
|
|
7614
7688
|
]);
|
|
7615
|
-
var IDENTITY_PATH = "/
|
|
7689
|
+
var IDENTITY_PATH = "/agent/ping";
|
|
7616
7690
|
var CREDS_DIR = (0, import_node_path2.join)((0, import_node_os2.homedir)(), ".config", "zeta");
|
|
7617
7691
|
var CREDS_FILE = (0, import_node_path2.join)(CREDS_DIR, "credentials.json");
|
|
7618
|
-
|
|
7692
|
+
function resolveProfile(explicit) {
|
|
7693
|
+
return explicit ?? process.env.ZETA_PROFILE ?? "default";
|
|
7694
|
+
}
|
|
7695
|
+
async function loadCreds(profile) {
|
|
7619
7696
|
try {
|
|
7620
7697
|
const raw = await (0, import_promises2.readFile)(CREDS_FILE, "utf-8");
|
|
7621
|
-
|
|
7698
|
+
const data = JSON.parse(raw);
|
|
7699
|
+
if (data._version === 2) {
|
|
7700
|
+
const name = resolveProfile(profile) === "default" && !profile && !process.env.ZETA_PROFILE ? data._default ?? "default" : resolveProfile(profile);
|
|
7701
|
+
return data.profiles?.[name] ?? null;
|
|
7702
|
+
}
|
|
7703
|
+
if (profile && profile !== "default")
|
|
7704
|
+
return null;
|
|
7705
|
+
return data;
|
|
7622
7706
|
} catch {
|
|
7623
7707
|
return null;
|
|
7624
7708
|
}
|
|
7625
7709
|
}
|
|
7626
|
-
async function
|
|
7710
|
+
async function listAllProfiles() {
|
|
7711
|
+
try {
|
|
7712
|
+
const raw = await (0, import_promises2.readFile)(CREDS_FILE, "utf-8");
|
|
7713
|
+
const data = JSON.parse(raw);
|
|
7714
|
+
if (data._version === 2)
|
|
7715
|
+
return data.profiles ?? {};
|
|
7716
|
+
return { default: data };
|
|
7717
|
+
} catch {
|
|
7718
|
+
return {};
|
|
7719
|
+
}
|
|
7720
|
+
}
|
|
7721
|
+
async function saveCreds(creds, profile) {
|
|
7627
7722
|
await (0, import_promises2.mkdir)(CREDS_DIR, { recursive: true });
|
|
7628
|
-
|
|
7723
|
+
let data = { _version: 2, _default: "default", profiles: {} };
|
|
7724
|
+
try {
|
|
7725
|
+
const existing = JSON.parse(await (0, import_promises2.readFile)(CREDS_FILE, "utf-8"));
|
|
7726
|
+
if (existing._version === 2) {
|
|
7727
|
+
data = existing;
|
|
7728
|
+
} else {
|
|
7729
|
+
data.profiles.default = existing;
|
|
7730
|
+
}
|
|
7731
|
+
} catch {
|
|
7732
|
+
}
|
|
7733
|
+
const name = resolveProfile(profile);
|
|
7734
|
+
data.profiles[name] = creds;
|
|
7735
|
+
if (!data._default)
|
|
7736
|
+
data._default = name;
|
|
7737
|
+
await (0, import_promises2.writeFile)(CREDS_FILE, JSON.stringify(data, null, 2), { encoding: "utf-8", mode: 384 });
|
|
7738
|
+
}
|
|
7739
|
+
async function deleteCredsProfile(profile) {
|
|
7740
|
+
try {
|
|
7741
|
+
const raw = await (0, import_promises2.readFile)(CREDS_FILE, "utf-8");
|
|
7742
|
+
const data = JSON.parse(raw);
|
|
7743
|
+
if (data._version === 2) {
|
|
7744
|
+
const name = resolveProfile(profile);
|
|
7745
|
+
if (!data.profiles[name])
|
|
7746
|
+
return false;
|
|
7747
|
+
delete data.profiles[name];
|
|
7748
|
+
if (data._default === name) {
|
|
7749
|
+
const remaining = Object.keys(data.profiles);
|
|
7750
|
+
data._default = remaining[0] ?? "default";
|
|
7751
|
+
}
|
|
7752
|
+
if (Object.keys(data.profiles).length === 0) {
|
|
7753
|
+
await (0, import_promises2.unlink)(CREDS_FILE).catch(() => {
|
|
7754
|
+
});
|
|
7755
|
+
} else {
|
|
7756
|
+
await (0, import_promises2.writeFile)(CREDS_FILE, JSON.stringify(data, null, 2), { encoding: "utf-8", mode: 384 });
|
|
7757
|
+
}
|
|
7758
|
+
return true;
|
|
7759
|
+
}
|
|
7760
|
+
await (0, import_promises2.unlink)(CREDS_FILE);
|
|
7761
|
+
return true;
|
|
7762
|
+
} catch {
|
|
7763
|
+
return false;
|
|
7764
|
+
}
|
|
7629
7765
|
}
|
|
7630
7766
|
function executeCommand(command, logger, timeoutMs) {
|
|
7631
7767
|
return new Promise((resolve, reject) => {
|
|
@@ -7821,13 +7957,15 @@ function parseEnvironmentOptions(value) {
|
|
|
7821
7957
|
}
|
|
7822
7958
|
var program2 = new Command();
|
|
7823
7959
|
program2.version(PKG_VERSION).description("ZeTa-AI Local Agent \u2014 run tests and crawls on private networks").enablePositionalOptions();
|
|
7824
|
-
program2.command("login").description("Authenticate with ZeTa and store credentials locally (~/.config/zeta/credentials.json)").argument("[token]", "Agent token (zeta_...) \u2014 skip interactive prompt").option("-a, --api <url>", "ZeTa API URL").option("-t, --token <token>", "ZeTa Agent Token (zeta_...)").action(async (tokenArg, options) => {
|
|
7960
|
+
program2.command("login").description("Authenticate with ZeTa and store credentials locally (~/.config/zeta/credentials.json)").argument("[token]", "Agent token (zeta_...) \u2014 skip interactive prompt").option("-a, --api <url>", "ZeTa API URL").option("-t, --token <token>", "ZeTa Agent Token (zeta_...)").option("--profile <name>", 'Credential profile to save under (default: $ZETA_PROFILE or "default")').action(async (tokenArg, options) => {
|
|
7961
|
+
const profile = resolveProfile(options.profile);
|
|
7825
7962
|
const iface = rl.createInterface({ input: process.stdin, output: process.stdout });
|
|
7963
|
+
const profileDefault = resolveApiUrl(void 0, profile);
|
|
7826
7964
|
let apiUrl = options.api ?? "";
|
|
7827
7965
|
if (!apiUrl) {
|
|
7828
|
-
apiUrl = (await iface.question(`API URL [${
|
|
7966
|
+
apiUrl = (await iface.question(`API URL [${profileDefault}]: `)).trim();
|
|
7829
7967
|
if (!apiUrl)
|
|
7830
|
-
apiUrl =
|
|
7968
|
+
apiUrl = profileDefault;
|
|
7831
7969
|
}
|
|
7832
7970
|
let token = tokenArg ?? options.token ?? "";
|
|
7833
7971
|
if (!token) {
|
|
@@ -7871,9 +8009,9 @@ program2.command("login").description("Authenticate with ZeTa and store credenti
|
|
|
7871
8009
|
}
|
|
7872
8010
|
} catch {
|
|
7873
8011
|
}
|
|
7874
|
-
await saveCreds({ apiUrl, token, workspaceId, workspaceName, projects });
|
|
8012
|
+
await saveCreds({ apiUrl, token, workspaceId, workspaceName, projects }, profile);
|
|
7875
8013
|
console.log(`
|
|
7876
|
-
[ZeTa] \u2713 Logged in to "${workspaceName ?? workspaceId}"`);
|
|
8014
|
+
[ZeTa] \u2713 Logged in to "${workspaceName ?? workspaceId}" (profile: ${profile})`);
|
|
7877
8015
|
console.log(`[ZeTa] Credentials stored: ${CREDS_FILE}`);
|
|
7878
8016
|
if (projects.length > 0) {
|
|
7879
8017
|
console.log(`
|
|
@@ -7883,32 +8021,64 @@ Projects (${projects.length}):`);
|
|
|
7883
8021
|
const serviceProject = projects[0];
|
|
7884
8022
|
if (serviceProject) {
|
|
7885
8023
|
console.log(`
|
|
7886
|
-
[ZeTa] Installing background service for "${serviceProject.name}"...`);
|
|
8024
|
+
[ZeTa] Installing background service for "${serviceProject.name}" (profile: ${profile})...`);
|
|
7887
8025
|
try {
|
|
7888
|
-
await serviceInstall(serviceProject.id);
|
|
8026
|
+
await serviceInstall(serviceProject.id, profile);
|
|
7889
8027
|
if (projects.length > 1) {
|
|
7890
8028
|
console.log(`[ZeTa] \u2139 To switch project: zeta service uninstall && zeta service install -p <project-id>`);
|
|
7891
8029
|
}
|
|
7892
8030
|
} catch (e) {
|
|
7893
8031
|
console.warn(`[ZeTa] \u26A0 Background service install failed: ${e instanceof Error ? e.message : String(e)}`);
|
|
7894
|
-
console.warn(`[ZeTa] Start manually: zeta -p ${serviceProject.id}`);
|
|
8032
|
+
console.warn(`[ZeTa] Start manually: zeta --profile ${profile} -p ${serviceProject.id}`);
|
|
7895
8033
|
}
|
|
7896
8034
|
}
|
|
7897
8035
|
});
|
|
7898
|
-
program2.command("logout").description("Remove stored ZeTa credentials").action(async () => {
|
|
7899
|
-
|
|
7900
|
-
|
|
7901
|
-
|
|
7902
|
-
|
|
7903
|
-
|
|
8036
|
+
program2.command("logout").description("Remove stored ZeTa credentials").option("--profile <name>", 'Remove only this profile (default: $ZETA_PROFILE or "default"); use --all to remove everything').option("--all", "Remove all profiles and the credentials file").action(async (options) => {
|
|
8037
|
+
if (options.all) {
|
|
8038
|
+
try {
|
|
8039
|
+
await (0, import_promises2.unlink)(CREDS_FILE);
|
|
8040
|
+
console.log("[ZeTa] \u2713 All credentials removed.");
|
|
8041
|
+
} catch {
|
|
8042
|
+
console.log("[ZeTa] No stored credentials found.");
|
|
8043
|
+
}
|
|
8044
|
+
return;
|
|
8045
|
+
}
|
|
8046
|
+
const profile = resolveProfile(options.profile);
|
|
8047
|
+
const removed = await deleteCredsProfile(profile);
|
|
8048
|
+
if (removed) {
|
|
8049
|
+
console.log(`[ZeTa] \u2713 Profile "${profile}" removed.`);
|
|
8050
|
+
} else {
|
|
8051
|
+
console.log(`[ZeTa] Profile "${profile}" not found.`);
|
|
7904
8052
|
}
|
|
7905
8053
|
});
|
|
7906
|
-
program2.command("whoami").description("Show currently stored credentials").action(async () => {
|
|
7907
|
-
|
|
8054
|
+
program2.command("whoami").description("Show currently stored credentials").option("--profile <name>", 'Profile to inspect (default: $ZETA_PROFILE or "default")').option("--all", "Show all configured profiles").action(async (options) => {
|
|
8055
|
+
if (options.all) {
|
|
8056
|
+
const all = await listAllProfiles();
|
|
8057
|
+
const names = Object.keys(all);
|
|
8058
|
+
if (names.length === 0) {
|
|
8059
|
+
console.log("[ZeTa] No profiles configured. Run `zeta login`.");
|
|
8060
|
+
process.exit(1);
|
|
8061
|
+
}
|
|
8062
|
+
names.forEach((name) => {
|
|
8063
|
+
const c = all[name];
|
|
8064
|
+
console.log(`
|
|
8065
|
+
Profile: ${name}`);
|
|
8066
|
+
console.log(` Workspace : ${c.workspaceName ?? c.workspaceId ?? "(unknown)"}`);
|
|
8067
|
+
console.log(` API : ${c.apiUrl}`);
|
|
8068
|
+
console.log(` Token : ${c.token.slice(0, 10)}...`);
|
|
8069
|
+
if (c.projects?.length) {
|
|
8070
|
+
console.log(` Projects : ${c.projects.map((p) => p.name).join(", ")}`);
|
|
8071
|
+
}
|
|
8072
|
+
});
|
|
8073
|
+
return;
|
|
8074
|
+
}
|
|
8075
|
+
const profile = resolveProfile(options.profile);
|
|
8076
|
+
const creds = await loadCreds(profile);
|
|
7908
8077
|
if (!creds) {
|
|
7909
|
-
console.log(
|
|
8078
|
+
console.log(`[ZeTa] Profile "${profile}" not found. Run \`zeta login --profile ${profile}\`.`);
|
|
7910
8079
|
process.exit(1);
|
|
7911
8080
|
}
|
|
8081
|
+
console.log(`Profile : ${profile}`);
|
|
7912
8082
|
console.log(`Workspace : ${creds.workspaceName ?? creds.workspaceId ?? "(unknown)"}`);
|
|
7913
8083
|
console.log(`API : ${creds.apiUrl}`);
|
|
7914
8084
|
console.log(`Token : ${creds.token.slice(0, 10)}...`);
|
|
@@ -7918,6 +8088,20 @@ Projects (${creds.projects.length}):`);
|
|
|
7918
8088
|
creds.projects.forEach((p) => console.log(` ${p.id} ${p.name}`));
|
|
7919
8089
|
}
|
|
7920
8090
|
});
|
|
8091
|
+
program2.command("profiles").description("List all configured credential profiles").action(async () => {
|
|
8092
|
+
const all = await listAllProfiles();
|
|
8093
|
+
const names = Object.keys(all);
|
|
8094
|
+
if (names.length === 0) {
|
|
8095
|
+
console.log("[ZeTa] No profiles configured. Run `zeta login`.");
|
|
8096
|
+
return;
|
|
8097
|
+
}
|
|
8098
|
+
console.log("Configured profiles:");
|
|
8099
|
+
names.forEach((name) => {
|
|
8100
|
+
const c = all[name];
|
|
8101
|
+
console.log(` ${name.padEnd(12)} ${c.apiUrl} (${c.workspaceName ?? c.workspaceId ?? "unknown"})`);
|
|
8102
|
+
});
|
|
8103
|
+
console.log("\nSwitch profiles with --profile <name> or ZETA_PROFILE=<name>");
|
|
8104
|
+
});
|
|
7921
8105
|
program2.command("crawl").description("Trigger a crawl for a ZeTa project").requiredOption("-p, --project <id>", "ZeTa Project ID").option("-t, --token <token>", "ZeTa API Token (falls back to stored credentials)").option("-a, --api <url>", "ZeTa API URL (falls back to stored credentials)").option("-u, --url <url>", "App URL to crawl (overrides project setting)").option("--max-screens <n>", "Maximum screens to crawl", DEFAULT_MAX_SCREENS).option("--strategy <s>", "Traversal strategy: BFS | DFS | BEST_FIRST", DEFAULT_STRATEGY).option("--url-filter <regex>", "Regex \u2014 only crawl matching URLs").option("--no-wait", "Start crawl and exit without waiting for completion").action(async (options) => {
|
|
7922
8106
|
const creds = await loadCreds();
|
|
7923
8107
|
const token = options.token ?? creds?.token;
|
|
@@ -8117,8 +8301,11 @@ var LOG_FILE = (0, import_node_path2.join)(CREDS_DIR, "agent.log");
|
|
|
8117
8301
|
function resolveZetaBinary() {
|
|
8118
8302
|
return { node: process.execPath, script: process.argv[1] };
|
|
8119
8303
|
}
|
|
8120
|
-
function getMacPlist(projectId) {
|
|
8304
|
+
function getMacPlist(projectId, profile) {
|
|
8121
8305
|
const { node, script } = resolveZetaBinary();
|
|
8306
|
+
const profileEntry = profile && profile !== "default" ? `
|
|
8307
|
+
<key>ZETA_PROFILE</key>
|
|
8308
|
+
<string>${profile}</string>` : "";
|
|
8122
8309
|
return `<?xml version="1.0" encoding="UTF-8"?>
|
|
8123
8310
|
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
|
8124
8311
|
<plist version="1.0">
|
|
@@ -8145,13 +8332,15 @@ function getMacPlist(projectId) {
|
|
|
8145
8332
|
<key>EnvironmentVariables</key>
|
|
8146
8333
|
<dict>
|
|
8147
8334
|
<key>PATH</key>
|
|
8148
|
-
<string>/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin</string
|
|
8335
|
+
<string>/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin</string>${profileEntry}
|
|
8149
8336
|
</dict>
|
|
8150
8337
|
</dict>
|
|
8151
8338
|
</plist>`;
|
|
8152
8339
|
}
|
|
8153
|
-
function getLinuxUnit(projectId) {
|
|
8340
|
+
function getLinuxUnit(projectId, profile) {
|
|
8154
8341
|
const { node, script } = resolveZetaBinary();
|
|
8342
|
+
const profileEnv = profile && profile !== "default" ? `
|
|
8343
|
+
Environment=ZETA_PROFILE=${profile}` : "";
|
|
8155
8344
|
return `[Unit]
|
|
8156
8345
|
Description=ZeTa AI Local Agent (project ${projectId})
|
|
8157
8346
|
After=network-online.target
|
|
@@ -8163,7 +8352,7 @@ ExecStart=${node} ${script} -p ${projectId}
|
|
|
8163
8352
|
Restart=always
|
|
8164
8353
|
RestartSec=10
|
|
8165
8354
|
StandardOutput=append:${LOG_FILE}
|
|
8166
|
-
StandardError=append:${LOG_FILE}
|
|
8355
|
+
StandardError=append:${LOG_FILE}${profileEnv}
|
|
8167
8356
|
|
|
8168
8357
|
[Install]
|
|
8169
8358
|
WantedBy=default.target
|
|
@@ -8175,12 +8364,12 @@ async function getMacPlistPath() {
|
|
|
8175
8364
|
function getLinuxUnitPath() {
|
|
8176
8365
|
return (0, import_node_path2.join)((0, import_node_os2.homedir)(), ".config", "systemd", "user", `${SERVICE_LABEL.replace(/\./g, "-")}.service`);
|
|
8177
8366
|
}
|
|
8178
|
-
async function serviceInstall(projectId) {
|
|
8367
|
+
async function serviceInstall(projectId, profile) {
|
|
8179
8368
|
if (process.platform === "darwin") {
|
|
8180
8369
|
const plistPath = await getMacPlistPath();
|
|
8181
8370
|
await (0, import_promises2.mkdir)((0, import_node_path2.join)((0, import_node_os2.homedir)(), "Library", "LaunchAgents"), { recursive: true });
|
|
8182
8371
|
await (0, import_promises2.mkdir)(CREDS_DIR, { recursive: true });
|
|
8183
|
-
await (0, import_promises2.writeFile)(plistPath, getMacPlist(projectId), { encoding: "utf-8", mode: 420 });
|
|
8372
|
+
await (0, import_promises2.writeFile)(plistPath, getMacPlist(projectId, profile), { encoding: "utf-8", mode: 420 });
|
|
8184
8373
|
await new Promise((resolve, reject) => {
|
|
8185
8374
|
(0, import_node_child_process2.exec)(`launchctl unload "${plistPath}" 2>/dev/null; launchctl load -w "${plistPath}"`, (err) => {
|
|
8186
8375
|
if (err)
|
|
@@ -8197,7 +8386,7 @@ async function serviceInstall(projectId) {
|
|
|
8197
8386
|
const unitPath = getLinuxUnitPath();
|
|
8198
8387
|
await (0, import_promises2.mkdir)((0, import_node_path2.join)((0, import_node_os2.homedir)(), ".config", "systemd", "user"), { recursive: true });
|
|
8199
8388
|
await (0, import_promises2.mkdir)(CREDS_DIR, { recursive: true });
|
|
8200
|
-
await (0, import_promises2.writeFile)(unitPath, getLinuxUnit(projectId), { encoding: "utf-8", mode: 420 });
|
|
8389
|
+
await (0, import_promises2.writeFile)(unitPath, getLinuxUnit(projectId, profile), { encoding: "utf-8", mode: 420 });
|
|
8201
8390
|
await new Promise((resolve, reject) => {
|
|
8202
8391
|
(0, import_node_child_process2.exec)("systemctl --user daemon-reload", (err) => {
|
|
8203
8392
|
if (err)
|
|
@@ -8306,12 +8495,13 @@ async function serviceStatus() {
|
|
|
8306
8495
|
}
|
|
8307
8496
|
}
|
|
8308
8497
|
program2.command("service").description("Manage the ZeTa agent as a background system service").addCommand(
|
|
8309
|
-
new Command("install").description("Install and start the agent as a service (auto-starts on login)").option("-p, --project <id>", "ZeTa Project ID (uses stored credentials if omitted)").action(async (opts) => {
|
|
8310
|
-
const
|
|
8498
|
+
new Command("install").description("Install and start the agent as a service (auto-starts on login)").option("-p, --project <id>", "ZeTa Project ID (uses stored credentials if omitted)").option("--profile <name>", 'Credential profile to use (default: $ZETA_PROFILE or "default")').action(async (opts) => {
|
|
8499
|
+
const profile = resolveProfile(opts.profile);
|
|
8500
|
+
const creds = await loadCreds(profile);
|
|
8311
8501
|
let projectId = opts.project;
|
|
8312
8502
|
if (!projectId) {
|
|
8313
8503
|
if (!creds?.projects?.length) {
|
|
8314
|
-
console.error(
|
|
8504
|
+
console.error(`[ZeTa Service] No project ID. Run \`zeta <api-key> --profile ${profile}\` first, then \`zeta service install -p <project-id>\``);
|
|
8315
8505
|
process.exit(1);
|
|
8316
8506
|
}
|
|
8317
8507
|
if (creds.projects.length === 1) {
|
|
@@ -8324,11 +8514,11 @@ program2.command("service").description("Manage the ZeTa agent as a background s
|
|
|
8324
8514
|
}
|
|
8325
8515
|
}
|
|
8326
8516
|
if (!creds?.token) {
|
|
8327
|
-
console.error(
|
|
8517
|
+
console.error(`[ZeTa Service] No credentials. Run \`zeta <api-key> --profile ${profile}\` first.`);
|
|
8328
8518
|
process.exit(1);
|
|
8329
8519
|
}
|
|
8330
8520
|
try {
|
|
8331
|
-
await serviceInstall(projectId);
|
|
8521
|
+
await serviceInstall(projectId, profile);
|
|
8332
8522
|
} catch (e) {
|
|
8333
8523
|
console.error(`[ZeTa Service] Install failed: ${e instanceof Error ? e.message : String(e)}`);
|
|
8334
8524
|
process.exitCode = 1;
|
|
@@ -8437,9 +8627,10 @@ program2.command("run_postman").description("Run a local Postman collection and
|
|
|
8437
8627
|
process.exitCode = 1;
|
|
8438
8628
|
}
|
|
8439
8629
|
});
|
|
8440
|
-
program2.argument("[apiKey]", "API key from the ZeTa dashboard \u2014 saves credentials and exits (run `zeta -p <id>` afterwards)").option("-p, --project <id>", "ZeTa Project ID").option("-t, --token <token>", "ZeTa Agent Token (starts with zeta_)").option("-a, --api <url>", "ZeTa API base URL", process.env.ZETA_API_URL ?? "").action(async (apiKey, opts) => {
|
|
8630
|
+
program2.argument("[apiKey]", "API key from the ZeTa dashboard \u2014 saves credentials and exits (run `zeta -p <id>` afterwards)").option("-p, --project <id>", "ZeTa Project ID").option("-t, --token <token>", "ZeTa Agent Token (starts with zeta_)").option("-a, --api <url>", "ZeTa API base URL", process.env.ZETA_API_URL ?? "").option("--profile <name>", 'Credential profile to use/save (default: $ZETA_PROFILE or "default")').action(async (apiKey, opts) => {
|
|
8631
|
+
const profile = resolveProfile(opts.profile);
|
|
8441
8632
|
if (apiKey) {
|
|
8442
|
-
const api2 = opts.api ||
|
|
8633
|
+
const api2 = resolveApiUrl(opts.api || void 0, profile);
|
|
8443
8634
|
const token2 = apiKey;
|
|
8444
8635
|
console.log("[ZeTa] Validating token...");
|
|
8445
8636
|
const headers = { Authorization: `Bearer ${token2}` };
|
|
@@ -8474,9 +8665,9 @@ program2.argument("[apiKey]", "API key from the ZeTa dashboard \u2014 saves cred
|
|
|
8474
8665
|
}
|
|
8475
8666
|
} catch {
|
|
8476
8667
|
}
|
|
8477
|
-
await saveCreds({ apiUrl: api2, token: token2, workspaceId, workspaceName, projects });
|
|
8668
|
+
await saveCreds({ apiUrl: api2, token: token2, workspaceId, workspaceName, projects }, profile);
|
|
8478
8669
|
console.log(`
|
|
8479
|
-
[ZeTa] \u2713 API key saved for "${workspaceName ?? workspaceId}"`);
|
|
8670
|
+
[ZeTa] \u2713 API key saved for "${workspaceName ?? workspaceId}" (profile: ${profile})`);
|
|
8480
8671
|
console.log(`[ZeTa] Credentials stored: ${CREDS_FILE}`);
|
|
8481
8672
|
if (projects.length > 0) {
|
|
8482
8673
|
console.log(`
|
|
@@ -8486,16 +8677,16 @@ Projects (${projects.length}):`);
|
|
|
8486
8677
|
const serviceProject = projects[0];
|
|
8487
8678
|
if (serviceProject) {
|
|
8488
8679
|
console.log(`
|
|
8489
|
-
[ZeTa] Installing background service for "${serviceProject.name}"...`);
|
|
8680
|
+
[ZeTa] Installing background service for "${serviceProject.name}" (profile: ${profile})...`);
|
|
8490
8681
|
try {
|
|
8491
|
-
await serviceInstall(serviceProject.id);
|
|
8682
|
+
await serviceInstall(serviceProject.id, profile);
|
|
8492
8683
|
if (projects.length > 1) {
|
|
8493
8684
|
console.log(`[ZeTa] \u2139 Multiple projects found. Service running for "${serviceProject.name}".`);
|
|
8494
8685
|
console.log(`[ZeTa] To switch: zeta service uninstall && zeta service install -p <project-id>`);
|
|
8495
8686
|
}
|
|
8496
8687
|
} catch (e) {
|
|
8497
8688
|
console.warn(`[ZeTa] \u26A0 Background service install failed: ${e instanceof Error ? e.message : String(e)}`);
|
|
8498
|
-
console.warn(`[ZeTa] Start manually: zeta -p ${serviceProject.id}`);
|
|
8689
|
+
console.warn(`[ZeTa] Start manually: zeta --profile ${profile} -p ${serviceProject.id}`);
|
|
8499
8690
|
}
|
|
8500
8691
|
} else {
|
|
8501
8692
|
console.log(`
|
|
@@ -8504,13 +8695,13 @@ Projects (${projects.length}):`);
|
|
|
8504
8695
|
}
|
|
8505
8696
|
return;
|
|
8506
8697
|
}
|
|
8507
|
-
const creds = await loadCreds();
|
|
8698
|
+
const creds = await loadCreds(profile);
|
|
8508
8699
|
const token = opts.token ?? creds?.token;
|
|
8509
|
-
const api = opts.api || creds?.apiUrl ||
|
|
8700
|
+
const api = resolveApiUrl(opts.api || creds?.apiUrl || void 0, profile);
|
|
8510
8701
|
const project = opts.project;
|
|
8511
8702
|
if (!token) {
|
|
8512
8703
|
console.error("[ZeTa Agent] No API key found.");
|
|
8513
|
-
console.error(
|
|
8704
|
+
console.error(` Configure once: zeta <your-api-key>${profile !== "default" ? ` --profile ${profile}` : ""}`);
|
|
8514
8705
|
console.error(" Or pass inline: zeta -t <token> -p <project-id>");
|
|
8515
8706
|
process.exit(1);
|
|
8516
8707
|
}
|
|
@@ -8583,6 +8774,7 @@ Projects (${projects.length}):`);
|
|
|
8583
8774
|
}
|
|
8584
8775
|
}
|
|
8585
8776
|
function scheduleReconnect() {
|
|
8777
|
+
closeAllTunnels();
|
|
8586
8778
|
const delay = Math.min(
|
|
8587
8779
|
WS_RECONNECT_BASE_MS * Math.pow(2, reconnectAttempt) * (0.5 + Math.random() * 0.5),
|
|
8588
8780
|
WS_RECONNECT_MAX_MS
|
|
@@ -8595,6 +8787,7 @@ Projects (${projects.length}):`);
|
|
|
8595
8787
|
ws = new wrapper_default(wsUrl, {
|
|
8596
8788
|
headers: { Authorization: `Bearer ${token}` }
|
|
8597
8789
|
});
|
|
8790
|
+
ws.binaryType = "nodebuffer";
|
|
8598
8791
|
const connection = ws;
|
|
8599
8792
|
ws.on("open", () => {
|
|
8600
8793
|
console.log("[ZeTa Agent] Connected to ZeTa Cloud.");
|
|
@@ -8605,6 +8798,10 @@ Projects (${projects.length}):`);
|
|
|
8605
8798
|
}
|
|
8606
8799
|
});
|
|
8607
8800
|
ws.on("message", async (raw) => {
|
|
8801
|
+
if (Buffer.isBuffer(raw) && raw.length > 0 && raw[0] !== 123) {
|
|
8802
|
+
handleProxyBinaryFrame(connection, raw);
|
|
8803
|
+
return;
|
|
8804
|
+
}
|
|
8608
8805
|
let msg;
|
|
8609
8806
|
try {
|
|
8610
8807
|
msg = JSON.parse(raw.toString());
|
|
@@ -8625,6 +8822,11 @@ Projects (${projects.length}):`);
|
|
|
8625
8822
|
case WsMessageType.DOCTOR:
|
|
8626
8823
|
await handleEnvironmentRequest(msg, connection);
|
|
8627
8824
|
break;
|
|
8825
|
+
case "proxy_connect":
|
|
8826
|
+
if (typeof msg.tunnelId === "string" && typeof msg.host === "string" && typeof msg.port === "number") {
|
|
8827
|
+
handleProxyConnect(connection, msg.tunnelId, msg.host, msg.port);
|
|
8828
|
+
}
|
|
8829
|
+
break;
|
|
8628
8830
|
case WsMessageType.RUN_POSTMAN: {
|
|
8629
8831
|
const runId = typeof msg.runId === "string" ? msg.runId : null;
|
|
8630
8832
|
const logPrefix = `[ZeTa Postman] Run ${runId}`;
|
|
@@ -8676,6 +8878,28 @@ Projects (${projects.length}):`);
|
|
|
8676
8878
|
}
|
|
8677
8879
|
break;
|
|
8678
8880
|
}
|
|
8881
|
+
case WsMessageType.LOCAL_CRAWL: {
|
|
8882
|
+
const { crawlId, url: crawlUrl, maxScreens = 20, storageState, deviceProfiles, httpBasicUsername, httpBasicPassword, traversalStrategy, bestFirstKeywords } = msg;
|
|
8883
|
+
if (!crawlId || !crawlUrl)
|
|
8884
|
+
break;
|
|
8885
|
+
if (postmanRunRunning || environmentOperationRunning) {
|
|
8886
|
+
if (ws.readyState === wrapper_default.OPEN) {
|
|
8887
|
+
ws.send(JSON.stringify({ type: "crawl_done", crawlId, totalScreens: 0, error: "Another operation is running; retry later" }));
|
|
8888
|
+
}
|
|
8889
|
+
break;
|
|
8890
|
+
}
|
|
8891
|
+
handleLocalCrawl({ crawlId, url: crawlUrl, maxScreens, storageState, deviceProfiles, httpBasicUsername, httpBasicPassword, traversalStrategy, bestFirstKeywords }, ws).catch(() => {
|
|
8892
|
+
});
|
|
8893
|
+
break;
|
|
8894
|
+
}
|
|
8895
|
+
case "local_discover": {
|
|
8896
|
+
const { discoverJobId, url: discoverUrl, maxDiscoverUrls = 2e3, storageState: discoverStorageState } = msg;
|
|
8897
|
+
if (!discoverJobId || !discoverUrl)
|
|
8898
|
+
break;
|
|
8899
|
+
handleLocalDiscover({ discoverJobId, url: discoverUrl, maxDiscoverUrls, storageState: discoverStorageState }, ws).catch(() => {
|
|
8900
|
+
});
|
|
8901
|
+
break;
|
|
8902
|
+
}
|
|
8679
8903
|
default:
|
|
8680
8904
|
break;
|
|
8681
8905
|
}
|
|
@@ -8723,6 +8947,809 @@ async function handleRunTests(msg, ws) {
|
|
|
8723
8947
|
await (0, import_promises2.rm)(tmpDir, { recursive: true, force: true });
|
|
8724
8948
|
}
|
|
8725
8949
|
}
|
|
8950
|
+
var DEVICE_VIEWPORTS = {
|
|
8951
|
+
DESKTOP: { width: 1440, height: 900 },
|
|
8952
|
+
MOBILE_PORTRAIT: { width: 393, height: 852, isMobile: true, deviceScaleFactor: 3 },
|
|
8953
|
+
TABLET: { width: 834, height: 1194, isMobile: true, deviceScaleFactor: 2 }
|
|
8954
|
+
};
|
|
8955
|
+
async function handleLocalCrawl(msg, ws) {
|
|
8956
|
+
const { crawlId, url, maxScreens = 20 } = msg;
|
|
8957
|
+
const deviceProfiles = Array.isArray(msg.deviceProfiles) && msg.deviceProfiles.length > 0 ? msg.deviceProfiles : ["DESKTOP"];
|
|
8958
|
+
console.log(`[ZeTa Agent] Starting local crawl: ${url} (id: ${crawlId}, maxScreens: ${maxScreens})`);
|
|
8959
|
+
const send = (event) => {
|
|
8960
|
+
if (ws.readyState === 1)
|
|
8961
|
+
ws.send(JSON.stringify({ ...event, crawlId }));
|
|
8962
|
+
};
|
|
8963
|
+
try {
|
|
8964
|
+
let chromium;
|
|
8965
|
+
try {
|
|
8966
|
+
chromium = (await import("playwright")).chromium;
|
|
8967
|
+
} catch {
|
|
8968
|
+
send({ type: "crawl_progress", message: "Installing playwright \u2014 this only happens once\u2026" });
|
|
8969
|
+
const { execFileSync } = await import("child_process");
|
|
8970
|
+
const { resolve } = await import("path");
|
|
8971
|
+
const { platform } = await import("os");
|
|
8972
|
+
const agentDir = resolve(__dirname, "..");
|
|
8973
|
+
const isWin = platform() === "win32";
|
|
8974
|
+
const npm = isWin ? "npm.cmd" : "npm";
|
|
8975
|
+
const npx = isWin ? "npx.cmd" : "npx";
|
|
8976
|
+
execFileSync(npm, ["install", "playwright"], { cwd: agentDir, stdio: "pipe", timeout: 12e4 });
|
|
8977
|
+
send({ type: "crawl_progress", message: "Downloading Chromium browser \u2014 this only happens once\u2026" });
|
|
8978
|
+
execFileSync(npx, ["playwright", "install", "chromium"], { cwd: agentDir, stdio: "pipe", timeout: 3e5 });
|
|
8979
|
+
chromium = (await import("playwright")).chromium;
|
|
8980
|
+
}
|
|
8981
|
+
console.log("[ZeTa Agent] Playwright ready \u2014 launching browser");
|
|
8982
|
+
const browser = await chromium.launch({ headless: true });
|
|
8983
|
+
const contextOptions = {
|
|
8984
|
+
viewport: { width: 1440, height: 900 },
|
|
8985
|
+
userAgent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
|
|
8986
|
+
};
|
|
8987
|
+
if (msg.storageState) {
|
|
8988
|
+
try {
|
|
8989
|
+
contextOptions.storageState = JSON.parse(msg.storageState);
|
|
8990
|
+
} catch {
|
|
8991
|
+
}
|
|
8992
|
+
}
|
|
8993
|
+
const context = await browser.newContext(contextOptions);
|
|
8994
|
+
if (msg.httpBasicUsername && msg.httpBasicPassword) {
|
|
8995
|
+
const basicCreds = Buffer.from(`${msg.httpBasicUsername}:${msg.httpBasicPassword}`).toString("base64");
|
|
8996
|
+
await context.setExtraHTTPHeaders({ "Authorization": `Basic ${basicCreds}` });
|
|
8997
|
+
}
|
|
8998
|
+
if (msg.httpAuthToken) {
|
|
8999
|
+
await context.setExtraHTTPHeaders({ Authorization: `Bearer ${msg.httpAuthToken}` });
|
|
9000
|
+
}
|
|
9001
|
+
if (msg.httpAuthHeaders && typeof msg.httpAuthHeaders === "object") {
|
|
9002
|
+
await context.setExtraHTTPHeaders(msg.httpAuthHeaders);
|
|
9003
|
+
}
|
|
9004
|
+
context.on("page", async (popup) => {
|
|
9005
|
+
try {
|
|
9006
|
+
await popup.waitForLoadState("domcontentloaded", { timeout: 1e4 }).catch(() => {
|
|
9007
|
+
});
|
|
9008
|
+
const popupUrl = popup.url();
|
|
9009
|
+
if (!popupUrl || popupUrl === "about:blank")
|
|
9010
|
+
return;
|
|
9011
|
+
const popupTitle = await popup.title().catch(() => popupUrl);
|
|
9012
|
+
const popupScreenshot = await popup.screenshot({ type: "jpeg", quality: 80, fullPage: true }).then((b) => b.toString("base64")).catch(() => null);
|
|
9013
|
+
const popupSnapshot = await popup.content().catch(() => null);
|
|
9014
|
+
send({
|
|
9015
|
+
type: "crawl_screen",
|
|
9016
|
+
screen: {
|
|
9017
|
+
url: popupUrl,
|
|
9018
|
+
title: `[Popup] ${popupTitle}`,
|
|
9019
|
+
screenshot: popupScreenshot,
|
|
9020
|
+
domSnapshot: popupSnapshot,
|
|
9021
|
+
inputs: [],
|
|
9022
|
+
links: [],
|
|
9023
|
+
elements: [],
|
|
9024
|
+
apiEndpoints: [],
|
|
9025
|
+
pageText: "",
|
|
9026
|
+
viewportScreenshots: {}
|
|
9027
|
+
}
|
|
9028
|
+
});
|
|
9029
|
+
screensCount++;
|
|
9030
|
+
} catch {
|
|
9031
|
+
}
|
|
9032
|
+
});
|
|
9033
|
+
const visited = /* @__PURE__ */ new Set();
|
|
9034
|
+
const queue = [url];
|
|
9035
|
+
let screensCount = 0;
|
|
9036
|
+
if (msg.seedUrls && Array.isArray(msg.seedUrls)) {
|
|
9037
|
+
for (const su of msg.seedUrls) {
|
|
9038
|
+
if (!queue.includes(su))
|
|
9039
|
+
queue.push(su);
|
|
9040
|
+
}
|
|
9041
|
+
}
|
|
9042
|
+
const traversalStrategy = msg.traversalStrategy ?? "BFS";
|
|
9043
|
+
const bestFirstKeywords = msg.bestFirstKeywords ?? [];
|
|
9044
|
+
const scoreUrl = (u) => {
|
|
9045
|
+
if (traversalStrategy !== "BEST_FIRST" || bestFirstKeywords.length === 0)
|
|
9046
|
+
return 0;
|
|
9047
|
+
const lower = u.toLowerCase();
|
|
9048
|
+
let score = 0;
|
|
9049
|
+
for (const kw of bestFirstKeywords) {
|
|
9050
|
+
if (lower.includes(kw.toLowerCase()))
|
|
9051
|
+
score -= 10;
|
|
9052
|
+
}
|
|
9053
|
+
return score;
|
|
9054
|
+
};
|
|
9055
|
+
const enqueue = (u) => {
|
|
9056
|
+
if (traversalStrategy === "DFS") {
|
|
9057
|
+
queue.unshift(u);
|
|
9058
|
+
} else if (traversalStrategy === "BEST_FIRST") {
|
|
9059
|
+
const score = scoreUrl(u);
|
|
9060
|
+
const idx = queue.findIndex((q) => scoreUrl(q) > score);
|
|
9061
|
+
if (idx === -1)
|
|
9062
|
+
queue.push(u);
|
|
9063
|
+
else
|
|
9064
|
+
queue.splice(idx, 0, u);
|
|
9065
|
+
} else {
|
|
9066
|
+
queue.push(u);
|
|
9067
|
+
}
|
|
9068
|
+
};
|
|
9069
|
+
const normalizeUrl = (u) => {
|
|
9070
|
+
try {
|
|
9071
|
+
const parsed = new URL(u);
|
|
9072
|
+
parsed.hash = "";
|
|
9073
|
+
let path = parsed.pathname.replace(/\/$/, "") || "/";
|
|
9074
|
+
path = path.replace(/\/[0-9a-f]{20,}(\/|$)/gi, "/:id$1").replace(/\/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}(\/|$)/gi, "/:id$1");
|
|
9075
|
+
parsed.pathname = path;
|
|
9076
|
+
return parsed.href;
|
|
9077
|
+
} catch {
|
|
9078
|
+
return u;
|
|
9079
|
+
}
|
|
9080
|
+
};
|
|
9081
|
+
try {
|
|
9082
|
+
const baseOrigin = new URL(url).origin;
|
|
9083
|
+
const sitemapRes = await fetch(`${baseOrigin}/sitemap.xml`, {
|
|
9084
|
+
signal: AbortSignal.timeout(5e3),
|
|
9085
|
+
headers: { "User-Agent": "Mozilla/5.0 (compatible; ZeTaBot/1.0)" }
|
|
9086
|
+
});
|
|
9087
|
+
if (sitemapRes.ok) {
|
|
9088
|
+
const xml = await sitemapRes.text();
|
|
9089
|
+
const locs = [...xml.matchAll(/<loc>([^<]+)<\/loc>/g)].map((m) => m[1].trim());
|
|
9090
|
+
for (const loc of locs) {
|
|
9091
|
+
try {
|
|
9092
|
+
const locUrl = new URL(loc);
|
|
9093
|
+
if (locUrl.origin === baseOrigin && !queue.includes(loc)) {
|
|
9094
|
+
enqueue(loc);
|
|
9095
|
+
}
|
|
9096
|
+
} catch {
|
|
9097
|
+
}
|
|
9098
|
+
}
|
|
9099
|
+
send({ type: "crawl_progress", message: `Sitemap: found ${locs.length} URLs` });
|
|
9100
|
+
}
|
|
9101
|
+
} catch {
|
|
9102
|
+
}
|
|
9103
|
+
const disallowedPaths = /* @__PURE__ */ new Set();
|
|
9104
|
+
try {
|
|
9105
|
+
const origin = new URL(url).origin;
|
|
9106
|
+
const robotsPage = await context.newPage();
|
|
9107
|
+
try {
|
|
9108
|
+
const robotsResp = await robotsPage.goto(`${origin}/robots.txt`, { waitUntil: "domcontentloaded", timeout: 5e3 });
|
|
9109
|
+
if (robotsResp?.ok()) {
|
|
9110
|
+
const robotsText = await robotsPage.content();
|
|
9111
|
+
let userAgentAll = false;
|
|
9112
|
+
for (const line of robotsText.split("\n")) {
|
|
9113
|
+
const trimmed = line.trim();
|
|
9114
|
+
if (trimmed.toLowerCase().startsWith("user-agent: *"))
|
|
9115
|
+
userAgentAll = true;
|
|
9116
|
+
else if (trimmed.toLowerCase().startsWith("user-agent:"))
|
|
9117
|
+
userAgentAll = false;
|
|
9118
|
+
else if (userAgentAll && trimmed.toLowerCase().startsWith("disallow:")) {
|
|
9119
|
+
const path = trimmed.slice("disallow:".length).trim();
|
|
9120
|
+
if (path)
|
|
9121
|
+
disallowedPaths.add(path);
|
|
9122
|
+
}
|
|
9123
|
+
}
|
|
9124
|
+
}
|
|
9125
|
+
} finally {
|
|
9126
|
+
await robotsPage.close().catch(() => {
|
|
9127
|
+
});
|
|
9128
|
+
}
|
|
9129
|
+
} catch {
|
|
9130
|
+
}
|
|
9131
|
+
const contentHashes = /* @__PURE__ */ new Set();
|
|
9132
|
+
while (queue.length > 0 && screensCount < maxScreens) {
|
|
9133
|
+
const pageUrl = queue.shift();
|
|
9134
|
+
const normalizedUrl = normalizeUrl(pageUrl);
|
|
9135
|
+
if (visited.has(normalizedUrl))
|
|
9136
|
+
continue;
|
|
9137
|
+
visited.add(normalizedUrl);
|
|
9138
|
+
const urlPath = new URL(pageUrl).pathname;
|
|
9139
|
+
if ([...disallowedPaths].some((d) => d !== "/" && urlPath.startsWith(d))) {
|
|
9140
|
+
send({ type: "crawl_progress", message: `Skipped (robots.txt): ${pageUrl}` });
|
|
9141
|
+
continue;
|
|
9142
|
+
}
|
|
9143
|
+
const page = await context.newPage();
|
|
9144
|
+
const observedApiEndpoints = [];
|
|
9145
|
+
try {
|
|
9146
|
+
page.on("request", (req) => {
|
|
9147
|
+
try {
|
|
9148
|
+
const reqUrl = req.url();
|
|
9149
|
+
if (req.resourceType() === "fetch" || req.resourceType() === "xhr") {
|
|
9150
|
+
const parsed = new URL(reqUrl);
|
|
9151
|
+
const path = parsed.pathname;
|
|
9152
|
+
if (!observedApiEndpoints.includes(path))
|
|
9153
|
+
observedApiEndpoints.push(path);
|
|
9154
|
+
}
|
|
9155
|
+
} catch {
|
|
9156
|
+
}
|
|
9157
|
+
});
|
|
9158
|
+
await page.addInitScript(() => {
|
|
9159
|
+
window.__ZETA_VITALS__ = { lcp: null, cls: 0, fcp: null };
|
|
9160
|
+
try {
|
|
9161
|
+
new PerformanceObserver((list) => {
|
|
9162
|
+
for (const entry of list.getEntries()) {
|
|
9163
|
+
window.__ZETA_VITALS__.lcp = entry.startTime;
|
|
9164
|
+
}
|
|
9165
|
+
}).observe({ type: "largest-contentful-paint", buffered: true });
|
|
9166
|
+
new PerformanceObserver((list) => {
|
|
9167
|
+
for (const entry of list.getEntries()) {
|
|
9168
|
+
window.__ZETA_VITALS__.cls += entry.value ?? 0;
|
|
9169
|
+
}
|
|
9170
|
+
}).observe({ type: "layout-shift", buffered: true });
|
|
9171
|
+
new PerformanceObserver((list) => {
|
|
9172
|
+
for (const entry of list.getEntries()) {
|
|
9173
|
+
if (entry.name === "first-contentful-paint")
|
|
9174
|
+
window.__ZETA_VITALS__.fcp = entry.startTime;
|
|
9175
|
+
}
|
|
9176
|
+
}).observe({ type: "paint", buffered: true });
|
|
9177
|
+
} catch {
|
|
9178
|
+
}
|
|
9179
|
+
}).catch(() => {
|
|
9180
|
+
});
|
|
9181
|
+
await page.goto(pageUrl, { waitUntil: "domcontentloaded", timeout: 25e3 });
|
|
9182
|
+
await page.waitForLoadState("networkidle", { timeout: 8e3 }).catch(() => {
|
|
9183
|
+
});
|
|
9184
|
+
await page.evaluate(() => {
|
|
9185
|
+
const patterns = [
|
|
9186
|
+
'button[id*="accept"i]',
|
|
9187
|
+
'button[class*="accept"i]',
|
|
9188
|
+
'button[id*="cookie"i][id*="agree"i]',
|
|
9189
|
+
'button[class*="cookie"i][class*="agree"i]',
|
|
9190
|
+
'button[data-testid*="accept"i]',
|
|
9191
|
+
'button[aria-label*="accept"i]',
|
|
9192
|
+
'[id*="onetrust"i] button[class*="accept"i]',
|
|
9193
|
+
"#onetrust-accept-btn-handler",
|
|
9194
|
+
".cc-accept-all",
|
|
9195
|
+
".consent-accept"
|
|
9196
|
+
];
|
|
9197
|
+
for (const sel of patterns) {
|
|
9198
|
+
const el = document.querySelector(sel);
|
|
9199
|
+
if (el) {
|
|
9200
|
+
el.click();
|
|
9201
|
+
return;
|
|
9202
|
+
}
|
|
9203
|
+
}
|
|
9204
|
+
}).catch(() => {
|
|
9205
|
+
});
|
|
9206
|
+
await page.waitForTimeout(500).catch(() => {
|
|
9207
|
+
});
|
|
9208
|
+
await page.addStyleTag({ content: `
|
|
9209
|
+
*, *::before, *::after {
|
|
9210
|
+
animation-duration: 0s !important;
|
|
9211
|
+
animation-delay: 0s !important;
|
|
9212
|
+
transition-duration: 0s !important;
|
|
9213
|
+
transition-delay: 0s !important;
|
|
9214
|
+
}
|
|
9215
|
+
` }).catch(() => {
|
|
9216
|
+
});
|
|
9217
|
+
await page.waitForFunction(() => {
|
|
9218
|
+
const busy = document.querySelector('[aria-busy="true"]');
|
|
9219
|
+
const skeletons = document.querySelectorAll('[class*="skeleton"],[class*="Skeleton"],[class*="shimmer"],[class*="loading-pulse"]');
|
|
9220
|
+
return !busy && skeletons.length === 0;
|
|
9221
|
+
}, { timeout: 4e3 }).catch(() => {
|
|
9222
|
+
});
|
|
9223
|
+
await page.evaluate(async () => {
|
|
9224
|
+
await new Promise((resolve) => {
|
|
9225
|
+
let last = document.documentElement.scrollHeight;
|
|
9226
|
+
let iters = 0;
|
|
9227
|
+
const timer = setInterval(() => {
|
|
9228
|
+
window.scrollTo(0, document.documentElement.scrollHeight);
|
|
9229
|
+
if (document.documentElement.scrollHeight === last || iters++ > 5) {
|
|
9230
|
+
clearInterval(timer);
|
|
9231
|
+
window.scrollTo(0, 0);
|
|
9232
|
+
resolve();
|
|
9233
|
+
}
|
|
9234
|
+
last = document.documentElement.scrollHeight;
|
|
9235
|
+
}, 400);
|
|
9236
|
+
});
|
|
9237
|
+
}).catch(() => {
|
|
9238
|
+
});
|
|
9239
|
+
const title = await page.title().catch(() => pageUrl);
|
|
9240
|
+
const metaTags = await page.evaluate(() => {
|
|
9241
|
+
const getMeta = (name) => document.querySelector(`meta[name="${name}"], meta[property="${name}"]`)?.getAttribute("content") ?? null;
|
|
9242
|
+
return { description: getMeta("description"), ogImage: getMeta("og:image"), ogType: getMeta("og:type") };
|
|
9243
|
+
}).catch(() => null);
|
|
9244
|
+
const isErrorPage = /^\s*(404|not found|page not found|error|forbidden|403|500|unauthorized|401)\s*$/i.test(title);
|
|
9245
|
+
if (isErrorPage) {
|
|
9246
|
+
send({ type: "crawl_progress", message: `Skipped error page: ${title} (${pageUrl})` });
|
|
9247
|
+
continue;
|
|
9248
|
+
}
|
|
9249
|
+
const webVitals = await page.evaluate(() => {
|
|
9250
|
+
const nav = performance.getEntriesByType("navigation")[0];
|
|
9251
|
+
return {
|
|
9252
|
+
ttfb: nav ? Math.round(nav.responseStart - nav.requestStart) : null,
|
|
9253
|
+
domInteractive: nav ? Math.round(nav.domInteractive) : null,
|
|
9254
|
+
...window.__ZETA_VITALS__
|
|
9255
|
+
};
|
|
9256
|
+
}).catch(() => null);
|
|
9257
|
+
const dpr = await page.evaluate(() => window.devicePixelRatio || 1).catch(() => 1);
|
|
9258
|
+
const viewport = await page.evaluate(() => ({ width: window.innerWidth, height: window.innerHeight })).catch(() => ({ width: 1440, height: 900 }));
|
|
9259
|
+
const screenshot = await page.screenshot({ type: "jpeg", quality: 85, fullPage: true }).then((b) => b.toString("base64")).catch(() => null);
|
|
9260
|
+
const domSnapshot = await page.content().catch(() => null);
|
|
9261
|
+
const inputs = await page.evaluate(() => {
|
|
9262
|
+
return Array.from(document.querySelectorAll("input,select,textarea")).slice(0, 150).map((el) => {
|
|
9263
|
+
const labelEl = el.id ? document.querySelector(`label[for="${el.id}"]`) : el.closest("label") || el.previousElementSibling?.tagName === "LABEL" ? el.previousElementSibling : null;
|
|
9264
|
+
return {
|
|
9265
|
+
type: el.type || el.tagName.toLowerCase(),
|
|
9266
|
+
name: el.name || "",
|
|
9267
|
+
id: el.id || "",
|
|
9268
|
+
placeholder: el.placeholder || "",
|
|
9269
|
+
label: labelEl?.textContent?.trim() || el.getAttribute("aria-label") || el.getAttribute("aria-labelledby") ? document.getElementById(el.getAttribute("aria-labelledby") || "")?.textContent?.trim() || "" : "",
|
|
9270
|
+
required: el.required || el.getAttribute("aria-required") === "true" || false,
|
|
9271
|
+
value: el.tagName === "SELECT" ? Array.from(el.options).map((o) => o.value).join(",") : el.value || ""
|
|
9272
|
+
};
|
|
9273
|
+
});
|
|
9274
|
+
}).catch(() => []);
|
|
9275
|
+
const links = await page.evaluate((baseUrl) => {
|
|
9276
|
+
const base = new URL(baseUrl);
|
|
9277
|
+
const hrefs = /* @__PURE__ */ new Set();
|
|
9278
|
+
const visited2 = /* @__PURE__ */ new Set();
|
|
9279
|
+
const queue2 = [document.documentElement];
|
|
9280
|
+
while (queue2.length > 0) {
|
|
9281
|
+
const root = queue2.shift();
|
|
9282
|
+
if (!root || visited2.has(root))
|
|
9283
|
+
continue;
|
|
9284
|
+
visited2.add(root);
|
|
9285
|
+
root.querySelectorAll("a[href]").forEach((el) => {
|
|
9286
|
+
try {
|
|
9287
|
+
const href = new URL(el.href, baseUrl).href;
|
|
9288
|
+
if (new URL(href).hostname === base.hostname && !href.includes("#"))
|
|
9289
|
+
hrefs.add(href);
|
|
9290
|
+
} catch {
|
|
9291
|
+
}
|
|
9292
|
+
});
|
|
9293
|
+
root.querySelectorAll("*").forEach((el) => {
|
|
9294
|
+
if (el.shadowRoot)
|
|
9295
|
+
queue2.push(el.shadowRoot);
|
|
9296
|
+
});
|
|
9297
|
+
}
|
|
9298
|
+
try {
|
|
9299
|
+
const nextData = window.__NEXT_DATA__;
|
|
9300
|
+
if (nextData?.buildManifest?.pages) {
|
|
9301
|
+
Object.keys(nextData.buildManifest.pages).forEach((route) => {
|
|
9302
|
+
if (!route.includes("[") && route !== "/_app" && route !== "/_error") {
|
|
9303
|
+
hrefs.add(base.origin + route);
|
|
9304
|
+
}
|
|
9305
|
+
});
|
|
9306
|
+
}
|
|
9307
|
+
} catch {
|
|
9308
|
+
}
|
|
9309
|
+
try {
|
|
9310
|
+
const nuxt = window.__NUXT__;
|
|
9311
|
+
if (Array.isArray(nuxt?.ssrContext?.payload?.modules)) {
|
|
9312
|
+
}
|
|
9313
|
+
} catch {
|
|
9314
|
+
}
|
|
9315
|
+
return Array.from(hrefs).slice(0, 80);
|
|
9316
|
+
}, pageUrl).catch(() => []);
|
|
9317
|
+
const elements = await page.evaluate(() => {
|
|
9318
|
+
const els = [];
|
|
9319
|
+
const seen = /* @__PURE__ */ new Set();
|
|
9320
|
+
const add = (el, role, text, isInteractive, expectedData) => {
|
|
9321
|
+
const t = text.trim().slice(0, 200);
|
|
9322
|
+
if (!t || t.length < 2)
|
|
9323
|
+
return;
|
|
9324
|
+
const key = `${role}::${t.toLowerCase()}`;
|
|
9325
|
+
if (seen.has(key))
|
|
9326
|
+
return;
|
|
9327
|
+
seen.add(key);
|
|
9328
|
+
const rect = el.getBoundingClientRect();
|
|
9329
|
+
const landmark = (() => {
|
|
9330
|
+
let p = el.parentElement;
|
|
9331
|
+
while (p) {
|
|
9332
|
+
const tag = p.tagName?.toLowerCase();
|
|
9333
|
+
const role2 = p.getAttribute("role");
|
|
9334
|
+
if (tag === "main" || role2 === "main")
|
|
9335
|
+
return "main";
|
|
9336
|
+
if (tag === "nav" || role2 === "navigation")
|
|
9337
|
+
return "nav";
|
|
9338
|
+
if (tag === "form" || role2 === "form")
|
|
9339
|
+
return "form";
|
|
9340
|
+
if (tag === "aside" || role2 === "complementary")
|
|
9341
|
+
return "aside";
|
|
9342
|
+
if (tag === "footer" || role2 === "contentinfo")
|
|
9343
|
+
return "footer";
|
|
9344
|
+
if (tag === "header" || role2 === "banner")
|
|
9345
|
+
return "header";
|
|
9346
|
+
if (tag === "dialog" || role2 === "dialog")
|
|
9347
|
+
return "dialog";
|
|
9348
|
+
p = p.parentElement;
|
|
9349
|
+
}
|
|
9350
|
+
return void 0;
|
|
9351
|
+
})();
|
|
9352
|
+
const ariaState = (() => {
|
|
9353
|
+
const attrs = [];
|
|
9354
|
+
const names = ["aria-expanded", "aria-selected", "aria-checked", "aria-pressed", "aria-disabled", "aria-current"];
|
|
9355
|
+
names.forEach((a) => {
|
|
9356
|
+
const v = el.getAttribute(a);
|
|
9357
|
+
if (v)
|
|
9358
|
+
attrs.push(`${a}=${v}`);
|
|
9359
|
+
});
|
|
9360
|
+
return attrs.join(";") || void 0;
|
|
9361
|
+
})();
|
|
9362
|
+
els.push({
|
|
9363
|
+
role,
|
|
9364
|
+
text: t,
|
|
9365
|
+
isInteractive,
|
|
9366
|
+
meaning: t,
|
|
9367
|
+
...expectedData ? { expectedData } : {},
|
|
9368
|
+
...ariaState ? { ariaState } : {},
|
|
9369
|
+
...landmark ? { landmark } : {},
|
|
9370
|
+
boundingRect: rect.width > 0 ? { x: Math.round(rect.x), y: Math.round(rect.y), w: Math.round(rect.width), h: Math.round(rect.height) } : void 0
|
|
9371
|
+
});
|
|
9372
|
+
};
|
|
9373
|
+
const getLabel = (el) => {
|
|
9374
|
+
const ariaLabel = el.getAttribute("aria-label");
|
|
9375
|
+
if (ariaLabel)
|
|
9376
|
+
return ariaLabel;
|
|
9377
|
+
const labelledBy = el.getAttribute("aria-labelledby");
|
|
9378
|
+
if (labelledBy)
|
|
9379
|
+
return document.getElementById(labelledBy)?.textContent?.trim() || "";
|
|
9380
|
+
if (el.id)
|
|
9381
|
+
return document.querySelector(`label[for="${el.id}"]`)?.textContent?.trim() || "";
|
|
9382
|
+
const closestLabel = el.closest("label");
|
|
9383
|
+
if (closestLabel)
|
|
9384
|
+
return closestLabel.textContent?.trim() || "";
|
|
9385
|
+
return el.textContent?.trim() || el.placeholder || el.name || el.value || "";
|
|
9386
|
+
};
|
|
9387
|
+
const inferExpectedData = (el) => {
|
|
9388
|
+
const type = el.type?.toLowerCase() || "";
|
|
9389
|
+
const label = getLabel(el).toLowerCase();
|
|
9390
|
+
if (type === "email" || label.includes("email"))
|
|
9391
|
+
return "valid_email";
|
|
9392
|
+
if (type === "password" || label.includes("password") || label.includes("passwd"))
|
|
9393
|
+
return "password";
|
|
9394
|
+
if (type === "tel" || label.includes("phone") || label.includes("mobile"))
|
|
9395
|
+
return "phone";
|
|
9396
|
+
if (type === "date" || label.includes("date") || label.includes("dob"))
|
|
9397
|
+
return "date";
|
|
9398
|
+
if (label.includes("otp") || label.includes("pin") || label.includes("code"))
|
|
9399
|
+
return "otp";
|
|
9400
|
+
if (type === "search" || label.includes("search") || label.includes("query"))
|
|
9401
|
+
return "search_term";
|
|
9402
|
+
if (type === "number" || label.includes("amount") || label.includes("price"))
|
|
9403
|
+
return "number";
|
|
9404
|
+
return void 0;
|
|
9405
|
+
};
|
|
9406
|
+
document.querySelectorAll('button,[role="button"],[type="submit"],[type="reset"],[type="button"]').forEach((el) => {
|
|
9407
|
+
const text = getLabel(el);
|
|
9408
|
+
if (text)
|
|
9409
|
+
add(el, "button", text, true);
|
|
9410
|
+
});
|
|
9411
|
+
document.querySelectorAll("h1,h2,h3,h4,h5,h6").forEach((el) => {
|
|
9412
|
+
const text = el.textContent?.trim() || "";
|
|
9413
|
+
if (text)
|
|
9414
|
+
add(el, el.tagName.toLowerCase(), text, false);
|
|
9415
|
+
});
|
|
9416
|
+
document.querySelectorAll("a[href]").forEach((el) => {
|
|
9417
|
+
const text = (el.textContent?.trim() || el.getAttribute("aria-label") || "").slice(0, 150);
|
|
9418
|
+
if (text && text.length > 1)
|
|
9419
|
+
add(el, "nav-link", text, true);
|
|
9420
|
+
});
|
|
9421
|
+
document.querySelectorAll('input:not([type="hidden"]):not([type="submit"]):not([type="button"]):not([type="reset"])').forEach((el) => {
|
|
9422
|
+
const label = getLabel(el);
|
|
9423
|
+
if (label)
|
|
9424
|
+
add(el, `input-${el.type || "text"}`, label, true, inferExpectedData(el));
|
|
9425
|
+
});
|
|
9426
|
+
document.querySelectorAll("textarea").forEach((el) => {
|
|
9427
|
+
const label = getLabel(el);
|
|
9428
|
+
if (label)
|
|
9429
|
+
add(el, "textarea", label, true);
|
|
9430
|
+
});
|
|
9431
|
+
document.querySelectorAll("select").forEach((el) => {
|
|
9432
|
+
const label = getLabel(el);
|
|
9433
|
+
if (label)
|
|
9434
|
+
add(el, "select", label, true);
|
|
9435
|
+
});
|
|
9436
|
+
document.querySelectorAll('input[type="checkbox"],input[type="radio"]').forEach((el) => {
|
|
9437
|
+
const label = getLabel(el);
|
|
9438
|
+
if (label)
|
|
9439
|
+
add(el, el.type, label, true);
|
|
9440
|
+
});
|
|
9441
|
+
document.querySelectorAll('[role="tab"],[role="menuitem"],[role="option"],[role="switch"],[role="combobox"],[role="listbox"],[role="slider"],[role="spinbutton"]').forEach((el) => {
|
|
9442
|
+
const text = getLabel(el);
|
|
9443
|
+
if (text)
|
|
9444
|
+
add(el, el.getAttribute("role") || "widget", text, true);
|
|
9445
|
+
});
|
|
9446
|
+
document.querySelectorAll('[class*="card"],[class*="chip"],[class*="badge"],[class*="tag"],[class*="pill"],[class*="metric"],[class*="stat"]').forEach((el) => {
|
|
9447
|
+
const text = el.textContent?.trim()?.slice(0, 100);
|
|
9448
|
+
if (text && text.length > 2)
|
|
9449
|
+
add(el, "card", text, false);
|
|
9450
|
+
});
|
|
9451
|
+
document.querySelectorAll('[role="alert"],[role="status"],[class*="alert"],[class*="toast"],[class*="banner"],[class*="notification"]').forEach((el) => {
|
|
9452
|
+
const text = el.textContent?.trim()?.slice(0, 150);
|
|
9453
|
+
if (text)
|
|
9454
|
+
add(el, "alert", text, false);
|
|
9455
|
+
});
|
|
9456
|
+
document.querySelectorAll('img[alt]:not([alt=""])').forEach((el) => {
|
|
9457
|
+
const text = (el.alt || "").trim().slice(0, 100);
|
|
9458
|
+
if (text && text.length > 2)
|
|
9459
|
+
add(el, "image", text, false);
|
|
9460
|
+
});
|
|
9461
|
+
const shadowQueue = [document.documentElement];
|
|
9462
|
+
const shadowSeen = /* @__PURE__ */ new Set();
|
|
9463
|
+
while (shadowQueue.length) {
|
|
9464
|
+
const root = shadowQueue.shift();
|
|
9465
|
+
if (!root || shadowSeen.has(root))
|
|
9466
|
+
continue;
|
|
9467
|
+
shadowSeen.add(root);
|
|
9468
|
+
root.querySelectorAll('button,[role="button"]').forEach((el) => {
|
|
9469
|
+
const text = getLabel(el);
|
|
9470
|
+
if (text)
|
|
9471
|
+
add(el, "button", text, true);
|
|
9472
|
+
});
|
|
9473
|
+
root.querySelectorAll("*").forEach((el) => {
|
|
9474
|
+
if (el.shadowRoot)
|
|
9475
|
+
shadowQueue.push(el.shadowRoot);
|
|
9476
|
+
});
|
|
9477
|
+
}
|
|
9478
|
+
return els.slice(0, 400);
|
|
9479
|
+
}).catch(() => []);
|
|
9480
|
+
const a11ySnapshot = await page.accessibility.snapshot({ interestingOnly: false }).catch(() => null);
|
|
9481
|
+
let a11yViolations = [];
|
|
9482
|
+
if (a11ySnapshot) {
|
|
9483
|
+
const flattenTree = (node, out = []) => {
|
|
9484
|
+
if (node) {
|
|
9485
|
+
out.push(node);
|
|
9486
|
+
(node.children ?? []).forEach((c) => flattenTree(c, out));
|
|
9487
|
+
}
|
|
9488
|
+
return out;
|
|
9489
|
+
};
|
|
9490
|
+
const nodes = flattenTree(a11ySnapshot);
|
|
9491
|
+
const unlabeled = nodes.filter(
|
|
9492
|
+
(n) => ["textbox", "combobox", "listbox", "spinbutton", "searchbox"].includes(n.role) && !n.name
|
|
9493
|
+
);
|
|
9494
|
+
if (unlabeled.length > 0) {
|
|
9495
|
+
a11yViolations.push({
|
|
9496
|
+
id: "label-missing",
|
|
9497
|
+
impact: "critical",
|
|
9498
|
+
description: `${unlabeled.length} form control(s) have no accessible name`,
|
|
9499
|
+
nodes: unlabeled.length
|
|
9500
|
+
});
|
|
9501
|
+
}
|
|
9502
|
+
const imgNoAlt = nodes.filter((n) => n.role === "img" && !n.name);
|
|
9503
|
+
if (imgNoAlt.length > 0) {
|
|
9504
|
+
a11yViolations.push({
|
|
9505
|
+
id: "image-alt",
|
|
9506
|
+
impact: "critical",
|
|
9507
|
+
description: `${imgNoAlt.length} image(s) missing alt text`,
|
|
9508
|
+
nodes: imgNoAlt.length
|
|
9509
|
+
});
|
|
9510
|
+
}
|
|
9511
|
+
const hasH1 = nodes.some((n) => n.role === "heading" && n.level === 1);
|
|
9512
|
+
if (!hasH1) {
|
|
9513
|
+
a11yViolations.push({
|
|
9514
|
+
id: "page-has-heading-one",
|
|
9515
|
+
impact: "moderate",
|
|
9516
|
+
description: "Page does not contain an h1 heading",
|
|
9517
|
+
nodes: 1
|
|
9518
|
+
});
|
|
9519
|
+
}
|
|
9520
|
+
const noName = nodes.filter(
|
|
9521
|
+
(n) => ["button", "link"].includes(n.role) && !n.name
|
|
9522
|
+
);
|
|
9523
|
+
if (noName.length > 0) {
|
|
9524
|
+
a11yViolations.push({
|
|
9525
|
+
id: "interactive-no-name",
|
|
9526
|
+
impact: "serious",
|
|
9527
|
+
description: `${noName.length} interactive element(s) have no accessible name`,
|
|
9528
|
+
nodes: noName.length
|
|
9529
|
+
});
|
|
9530
|
+
}
|
|
9531
|
+
}
|
|
9532
|
+
const pageText = await page.evaluate(() => {
|
|
9533
|
+
const lines = [];
|
|
9534
|
+
const walk = (node, depth = 0) => {
|
|
9535
|
+
const tag = node.tagName?.toLowerCase();
|
|
9536
|
+
if (!tag)
|
|
9537
|
+
return;
|
|
9538
|
+
if (["script", "style", "noscript", "svg", "iframe"].includes(tag))
|
|
9539
|
+
return;
|
|
9540
|
+
if (["h1", "h2", "h3", "h4", "h5", "h6"].includes(tag)) {
|
|
9541
|
+
const t = node.textContent?.trim();
|
|
9542
|
+
if (t)
|
|
9543
|
+
lines.push(`${"#".repeat(parseInt(tag[1]))} ${t}`);
|
|
9544
|
+
} else if (tag === "p" || tag === "li") {
|
|
9545
|
+
const t = node.textContent?.trim()?.slice(0, 300);
|
|
9546
|
+
if (t && t.length > 10)
|
|
9547
|
+
lines.push(t);
|
|
9548
|
+
} else {
|
|
9549
|
+
Array.from(node.children).forEach((c) => walk(c, depth + 1));
|
|
9550
|
+
return;
|
|
9551
|
+
}
|
|
9552
|
+
};
|
|
9553
|
+
document.querySelectorAll('main, article, [role="main"], body > div').forEach((root) => walk(root));
|
|
9554
|
+
return lines.slice(0, 200).join("\n");
|
|
9555
|
+
}).catch(() => "");
|
|
9556
|
+
const contentHash = pageText ? pageText.slice(0, 500).replace(/\s+/g, " ") : "";
|
|
9557
|
+
if (contentHash && contentHash.length > 50 && contentHashes.has(contentHash)) {
|
|
9558
|
+
send({ type: "crawl_progress", message: `Skipped duplicate content: ${pageUrl}` });
|
|
9559
|
+
continue;
|
|
9560
|
+
}
|
|
9561
|
+
if (contentHash && contentHash.length > 50) {
|
|
9562
|
+
contentHashes.add(contentHash);
|
|
9563
|
+
}
|
|
9564
|
+
const viewportScreenshots = {};
|
|
9565
|
+
const extraProfiles = deviceProfiles.filter((p) => p !== "DESKTOP" && DEVICE_VIEWPORTS[p]);
|
|
9566
|
+
for (const profile of extraProfiles) {
|
|
9567
|
+
const vp = DEVICE_VIEWPORTS[profile];
|
|
9568
|
+
const vpCtx = await browser.newContext({
|
|
9569
|
+
viewport: { width: vp.width, height: vp.height },
|
|
9570
|
+
isMobile: vp.isMobile ?? false,
|
|
9571
|
+
deviceScaleFactor: vp.deviceScaleFactor ?? 1,
|
|
9572
|
+
userAgent: vp.isMobile ? "Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1" : contextOptions.userAgent
|
|
9573
|
+
}).catch(() => null);
|
|
9574
|
+
if (!vpCtx)
|
|
9575
|
+
continue;
|
|
9576
|
+
const vpPage = await vpCtx.newPage().catch(() => null);
|
|
9577
|
+
if (!vpPage) {
|
|
9578
|
+
await vpCtx.close().catch(() => {
|
|
9579
|
+
});
|
|
9580
|
+
continue;
|
|
9581
|
+
}
|
|
9582
|
+
try {
|
|
9583
|
+
await vpPage.goto(pageUrl, { waitUntil: "domcontentloaded", timeout: 2e4 });
|
|
9584
|
+
await vpPage.waitForLoadState("networkidle", { timeout: 5e3 }).catch(() => {
|
|
9585
|
+
});
|
|
9586
|
+
await vpPage.addStyleTag({ content: "*, *::before, *::after { animation-duration: 0s !important; transition-duration: 0s !important; }" }).catch(() => {
|
|
9587
|
+
});
|
|
9588
|
+
const vpShot = await vpPage.screenshot({ type: "jpeg", quality: 80, fullPage: true }).then((b) => b.toString("base64")).catch(() => null);
|
|
9589
|
+
viewportScreenshots[profile] = vpShot;
|
|
9590
|
+
} finally {
|
|
9591
|
+
await vpPage.close().catch(() => {
|
|
9592
|
+
});
|
|
9593
|
+
await vpCtx.close().catch(() => {
|
|
9594
|
+
});
|
|
9595
|
+
}
|
|
9596
|
+
}
|
|
9597
|
+
screensCount++;
|
|
9598
|
+
send({
|
|
9599
|
+
type: "crawl_screen",
|
|
9600
|
+
screen: { url: pageUrl, title, screenshot, domSnapshot, inputs, links, elements, apiEndpoints: observedApiEndpoints, dpr, viewport, pageText, viewportScreenshots, a11yViolations, webVitals, meta: metaTags }
|
|
9601
|
+
});
|
|
9602
|
+
for (const link of links) {
|
|
9603
|
+
const norm = normalizeUrl(link);
|
|
9604
|
+
if (!visited.has(norm) && !queue.some((q) => normalizeUrl(q) === norm)) {
|
|
9605
|
+
enqueue(link);
|
|
9606
|
+
}
|
|
9607
|
+
}
|
|
9608
|
+
send({ type: "crawl_progress", message: `Crawled ${screensCount}/${maxScreens}: ${title || pageUrl}` });
|
|
9609
|
+
} finally {
|
|
9610
|
+
await page.close().catch(() => {
|
|
9611
|
+
});
|
|
9612
|
+
}
|
|
9613
|
+
}
|
|
9614
|
+
try {
|
|
9615
|
+
const exportedState = await context.storageState();
|
|
9616
|
+
if (exportedState.cookies.length > 0 || exportedState.origins.length > 0) {
|
|
9617
|
+
send({ type: "crawl_session_state", storageState: JSON.stringify(exportedState) });
|
|
9618
|
+
}
|
|
9619
|
+
} catch {
|
|
9620
|
+
}
|
|
9621
|
+
await browser.close().catch(() => {
|
|
9622
|
+
});
|
|
9623
|
+
send({ type: "crawl_done", totalScreens: screensCount });
|
|
9624
|
+
} catch (err) {
|
|
9625
|
+
console.error("[ZeTa Agent] Crawl error:", err?.message ?? err);
|
|
9626
|
+
send({ type: "crawl_done", totalScreens: 0, error: err?.message ?? "Local crawl failed" });
|
|
9627
|
+
}
|
|
9628
|
+
}
|
|
9629
|
+
async function handleLocalDiscover(msg, ws) {
|
|
9630
|
+
const { discoverJobId, url, maxDiscoverUrls = 2e3, storageState } = msg;
|
|
9631
|
+
const send = (payload) => {
|
|
9632
|
+
if (ws.readyState === 1)
|
|
9633
|
+
ws.send(JSON.stringify({ ...payload, discoverJobId }));
|
|
9634
|
+
};
|
|
9635
|
+
try {
|
|
9636
|
+
let chromium;
|
|
9637
|
+
try {
|
|
9638
|
+
chromium = (await import("playwright")).chromium;
|
|
9639
|
+
} catch {
|
|
9640
|
+
const { execFileSync: execFileSync2 } = await import("child_process");
|
|
9641
|
+
const { resolve: resolve2 } = await import("path");
|
|
9642
|
+
const agentDir2 = resolve2(__dirname, "..");
|
|
9643
|
+
const npx = process.platform === "win32" ? "npx.cmd" : "npx";
|
|
9644
|
+
execFileSync2(npx, ["playwright", "install", "chromium"], { cwd: agentDir2, stdio: "pipe", timeout: 3e5 });
|
|
9645
|
+
chromium = (await import("playwright")).chromium;
|
|
9646
|
+
}
|
|
9647
|
+
const browser = await chromium.launch({ headless: true });
|
|
9648
|
+
const contextOptions = {
|
|
9649
|
+
viewport: { width: 1440, height: 900 },
|
|
9650
|
+
userAgent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
|
|
9651
|
+
};
|
|
9652
|
+
if (storageState) {
|
|
9653
|
+
try {
|
|
9654
|
+
contextOptions.storageState = JSON.parse(storageState);
|
|
9655
|
+
} catch {
|
|
9656
|
+
}
|
|
9657
|
+
}
|
|
9658
|
+
const context = await browser.newContext(contextOptions);
|
|
9659
|
+
const page = await context.newPage();
|
|
9660
|
+
const origin = new URL(url).origin;
|
|
9661
|
+
const visited = /* @__PURE__ */ new Set();
|
|
9662
|
+
const queue = [url];
|
|
9663
|
+
const allUrls = /* @__PURE__ */ new Set([url]);
|
|
9664
|
+
let pagesVisited = 0;
|
|
9665
|
+
const normalizeDiscoverUrl = (u) => {
|
|
9666
|
+
try {
|
|
9667
|
+
const p = new URL(u);
|
|
9668
|
+
p.hash = "";
|
|
9669
|
+
p.search = p.search.replace(/[?&](utm_[^&]*|ref|source|medium|campaign|fbclid|gclid)=[^&]*/g, "").replace(/^[?&]/, "?");
|
|
9670
|
+
return p.href.replace(/\/$/, "") || p.origin;
|
|
9671
|
+
} catch {
|
|
9672
|
+
return u;
|
|
9673
|
+
}
|
|
9674
|
+
};
|
|
9675
|
+
try {
|
|
9676
|
+
const sitemapResp = await page.goto(`${origin}/sitemap.xml`, { waitUntil: "domcontentloaded", timeout: 8e3 });
|
|
9677
|
+
if (sitemapResp?.ok()) {
|
|
9678
|
+
const sitemapContent = await page.content();
|
|
9679
|
+
const locs = [...sitemapContent.matchAll(/<loc[^>]*>([^<]+)<\/loc>/gi)];
|
|
9680
|
+
for (const match of locs) {
|
|
9681
|
+
const loc = match[1].trim();
|
|
9682
|
+
try {
|
|
9683
|
+
const locUrl = new URL(loc);
|
|
9684
|
+
if (locUrl.origin === origin) {
|
|
9685
|
+
allUrls.add(loc);
|
|
9686
|
+
if (!visited.has(normalizeDiscoverUrl(loc)) && !queue.some((q) => normalizeDiscoverUrl(q) === normalizeDiscoverUrl(loc))) {
|
|
9687
|
+
queue.push(loc);
|
|
9688
|
+
}
|
|
9689
|
+
}
|
|
9690
|
+
} catch {
|
|
9691
|
+
}
|
|
9692
|
+
}
|
|
9693
|
+
if (allUrls.size > 1) {
|
|
9694
|
+
send({ type: "discover_progress", partialUrls: Array.from(allUrls), pct: 5 });
|
|
9695
|
+
}
|
|
9696
|
+
}
|
|
9697
|
+
} catch {
|
|
9698
|
+
}
|
|
9699
|
+
while (queue.length > 0 && allUrls.size < maxDiscoverUrls) {
|
|
9700
|
+
const pageUrl = queue.shift();
|
|
9701
|
+
const norm = normalizeDiscoverUrl(pageUrl);
|
|
9702
|
+
if (visited.has(norm))
|
|
9703
|
+
continue;
|
|
9704
|
+
visited.add(norm);
|
|
9705
|
+
try {
|
|
9706
|
+
await page.goto(pageUrl, { waitUntil: "domcontentloaded", timeout: 2e4 });
|
|
9707
|
+
await page.waitForTimeout(500);
|
|
9708
|
+
pagesVisited++;
|
|
9709
|
+
const links = await page.evaluate((origin2) => {
|
|
9710
|
+
const hrefs = [];
|
|
9711
|
+
const queue2 = [document.documentElement];
|
|
9712
|
+
while (queue2.length > 0) {
|
|
9713
|
+
const node = queue2.shift();
|
|
9714
|
+
if (node.tagName === "A") {
|
|
9715
|
+
try {
|
|
9716
|
+
const abs = new URL(node.href, location.href).href;
|
|
9717
|
+
if (abs.startsWith(origin2))
|
|
9718
|
+
hrefs.push(abs);
|
|
9719
|
+
} catch {
|
|
9720
|
+
}
|
|
9721
|
+
}
|
|
9722
|
+
queue2.push(...Array.from(node.children));
|
|
9723
|
+
if (node.shadowRoot) {
|
|
9724
|
+
queue2.push(...Array.from(node.shadowRoot.children));
|
|
9725
|
+
}
|
|
9726
|
+
}
|
|
9727
|
+
return hrefs;
|
|
9728
|
+
}, origin).catch(() => []);
|
|
9729
|
+
for (const link of links) {
|
|
9730
|
+
const n = normalizeDiscoverUrl(link);
|
|
9731
|
+
if (!visited.has(n) && !queue.some((q) => normalizeDiscoverUrl(q) === n)) {
|
|
9732
|
+
allUrls.add(link);
|
|
9733
|
+
queue.push(link);
|
|
9734
|
+
}
|
|
9735
|
+
}
|
|
9736
|
+
if (pagesVisited % 10 === 0) {
|
|
9737
|
+
const pct = Math.min(Math.round(allUrls.size / maxDiscoverUrls * 90), 90);
|
|
9738
|
+
send({ type: "discover_progress", partialUrls: Array.from(allUrls), pct });
|
|
9739
|
+
}
|
|
9740
|
+
} catch {
|
|
9741
|
+
}
|
|
9742
|
+
}
|
|
9743
|
+
await page.close().catch(() => {
|
|
9744
|
+
});
|
|
9745
|
+
await browser.close().catch(() => {
|
|
9746
|
+
});
|
|
9747
|
+
send({ type: "discover_done", discoveredUrls: Array.from(allUrls) });
|
|
9748
|
+
} catch (err) {
|
|
9749
|
+
console.error("[ZeTa Agent] Discover error:", err?.message ?? err);
|
|
9750
|
+
send({ type: "discover_done", discoveredUrls: [], error: err?.message ?? "Local discovery failed" });
|
|
9751
|
+
}
|
|
9752
|
+
}
|
|
8726
9753
|
function buildRunCommand(language, scriptPath, _baseUrl) {
|
|
8727
9754
|
if (language.startsWith("playwright")) {
|
|
8728
9755
|
return { cmd: "npx", args: ["playwright", "test", scriptPath, "--reporter=json", "--reporter=line"] };
|