@cyberstrike-io/cyberstrike 1.1.17-beta.5 → 1.1.17-beta.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/hackbrowser-worker.js +390 -248
- package/package.json +12 -12
package/hackbrowser-worker.js
CHANGED
|
@@ -8766,13 +8766,13 @@ async function fetch2(url2, options_) {
|
|
|
8766
8766
|
request_.setTimeout(0);
|
|
8767
8767
|
const headers = fromRawHeaders(response_.rawHeaders);
|
|
8768
8768
|
if (isRedirect(response_.statusCode)) {
|
|
8769
|
-
const
|
|
8769
|
+
const location2 = headers.get("Location");
|
|
8770
8770
|
let locationURL = null;
|
|
8771
8771
|
try {
|
|
8772
|
-
locationURL =
|
|
8772
|
+
locationURL = location2 === null ? null : new URL(location2, request.url);
|
|
8773
8773
|
} catch {
|
|
8774
8774
|
if (request.redirect !== "manual") {
|
|
8775
|
-
reject(new FetchError(`uri requested responds with an invalid redirect URL: ${
|
|
8775
|
+
reject(new FetchError(`uri requested responds with an invalid redirect URL: ${location2}`, "invalid-redirect"));
|
|
8776
8776
|
finalize();
|
|
8777
8777
|
return;
|
|
8778
8778
|
}
|
|
@@ -17063,25 +17063,25 @@ var require_googleauth = __commonJS((exports) => {
|
|
|
17063
17063
|
}
|
|
17064
17064
|
}
|
|
17065
17065
|
async _tryGetApplicationCredentialsFromWellKnownFile(options) {
|
|
17066
|
-
let
|
|
17066
|
+
let location2 = null;
|
|
17067
17067
|
if (this._isWindows()) {
|
|
17068
|
-
|
|
17068
|
+
location2 = process.env["APPDATA"];
|
|
17069
17069
|
} else {
|
|
17070
17070
|
const home = process.env["HOME"];
|
|
17071
17071
|
if (home) {
|
|
17072
|
-
|
|
17072
|
+
location2 = path.join(home, ".config");
|
|
17073
17073
|
}
|
|
17074
17074
|
}
|
|
17075
|
-
if (
|
|
17076
|
-
|
|
17077
|
-
if (!fs2.existsSync(
|
|
17078
|
-
|
|
17075
|
+
if (location2) {
|
|
17076
|
+
location2 = path.join(location2, "gcloud", "application_default_credentials.json");
|
|
17077
|
+
if (!fs2.existsSync(location2)) {
|
|
17078
|
+
location2 = null;
|
|
17079
17079
|
}
|
|
17080
17080
|
}
|
|
17081
|
-
if (!
|
|
17081
|
+
if (!location2) {
|
|
17082
17082
|
return null;
|
|
17083
17083
|
}
|
|
17084
|
-
const client = await this._getApplicationCredentialsFromFilePath(
|
|
17084
|
+
const client = await this._getApplicationCredentialsFromFilePath(location2, options);
|
|
17085
17085
|
return client;
|
|
17086
17086
|
}
|
|
17087
17087
|
async _getApplicationCredentialsFromFilePath(filePath, options = {}) {
|
|
@@ -56573,7 +56573,7 @@ async function generateAuthToken2(options) {
|
|
|
56573
56573
|
}
|
|
56574
56574
|
function createVertexAnthropic(options = {}) {
|
|
56575
56575
|
var _a16;
|
|
56576
|
-
const
|
|
56576
|
+
const location2 = loadOptionalSetting({
|
|
56577
56577
|
settingValue: options.location,
|
|
56578
56578
|
environmentVariableName: "GOOGLE_VERTEX_LOCATION"
|
|
56579
56579
|
});
|
|
@@ -56581,7 +56581,7 @@ function createVertexAnthropic(options = {}) {
|
|
|
56581
56581
|
settingValue: options.project,
|
|
56582
56582
|
environmentVariableName: "GOOGLE_VERTEX_PROJECT"
|
|
56583
56583
|
});
|
|
56584
|
-
const baseURL = (_a16 = withoutTrailingSlash(options.baseURL)) != null ? _a16 : `https://${
|
|
56584
|
+
const baseURL = (_a16 = withoutTrailingSlash(options.baseURL)) != null ? _a16 : `https://${location2 === "global" ? "" : location2 + "-"}aiplatform.googleapis.com/v1/projects/${project}/locations/${location2}/publishers/anthropic/models`;
|
|
56585
56585
|
const createChatModel = (modelId) => {
|
|
56586
56586
|
var _a22;
|
|
56587
56587
|
return new AnthropicMessagesLanguageModel2(modelId, {
|
|
@@ -106258,6 +106258,80 @@ import { chromium as chromium2 } from "playwright";
|
|
|
106258
106258
|
// ../hackbrowser/src/stealth.ts
|
|
106259
106259
|
import { existsSync as existsSync2 } from "fs";
|
|
106260
106260
|
import { chromium } from "playwright";
|
|
106261
|
+
|
|
106262
|
+
// ../hackbrowser/src/log.ts
|
|
106263
|
+
var LEVEL_PRIORITY = {
|
|
106264
|
+
DEBUG: 0,
|
|
106265
|
+
INFO: 1,
|
|
106266
|
+
WARN: 2,
|
|
106267
|
+
ERROR: 3
|
|
106268
|
+
};
|
|
106269
|
+
var currentLevel = "INFO";
|
|
106270
|
+
var sink = defaultStderrSink;
|
|
106271
|
+
function defaultStderrSink(record3) {
|
|
106272
|
+
const ts = new Date(record3.timestamp).toISOString();
|
|
106273
|
+
let line = `${ts} ${record3.level} ${record3.service} ${record3.message}`;
|
|
106274
|
+
if (record3.extra && Object.keys(record3.extra).length > 0) {
|
|
106275
|
+
try {
|
|
106276
|
+
line += " " + JSON.stringify(record3.extra);
|
|
106277
|
+
} catch {
|
|
106278
|
+
line += " [unserializable extra]";
|
|
106279
|
+
}
|
|
106280
|
+
}
|
|
106281
|
+
process.stderr.write(line + `
|
|
106282
|
+
`);
|
|
106283
|
+
}
|
|
106284
|
+
function shouldLog(level) {
|
|
106285
|
+
return LEVEL_PRIORITY[level] >= LEVEL_PRIORITY[currentLevel];
|
|
106286
|
+
}
|
|
106287
|
+
function emit(level, service, message, extra) {
|
|
106288
|
+
if (!shouldLog(level))
|
|
106289
|
+
return;
|
|
106290
|
+
sink({
|
|
106291
|
+
level,
|
|
106292
|
+
service,
|
|
106293
|
+
message,
|
|
106294
|
+
extra,
|
|
106295
|
+
timestamp: Date.now()
|
|
106296
|
+
});
|
|
106297
|
+
}
|
|
106298
|
+
var Log;
|
|
106299
|
+
((Log) => {
|
|
106300
|
+
function init(options) {
|
|
106301
|
+
if (options.level)
|
|
106302
|
+
currentLevel = options.level;
|
|
106303
|
+
}
|
|
106304
|
+
Log.init = init;
|
|
106305
|
+
function setSink(custom4) {
|
|
106306
|
+
sink = custom4;
|
|
106307
|
+
}
|
|
106308
|
+
Log.setSink = setSink;
|
|
106309
|
+
function resetSink() {
|
|
106310
|
+
sink = defaultStderrSink;
|
|
106311
|
+
}
|
|
106312
|
+
Log.resetSink = resetSink;
|
|
106313
|
+
function create(options) {
|
|
106314
|
+
const { service } = options;
|
|
106315
|
+
return {
|
|
106316
|
+
debug(message, extra) {
|
|
106317
|
+
emit("DEBUG", service, message, extra);
|
|
106318
|
+
},
|
|
106319
|
+
info(message, extra) {
|
|
106320
|
+
emit("INFO", service, message, extra);
|
|
106321
|
+
},
|
|
106322
|
+
warn(message, extra) {
|
|
106323
|
+
emit("WARN", service, message, extra);
|
|
106324
|
+
},
|
|
106325
|
+
error(message, extra) {
|
|
106326
|
+
emit("ERROR", service, message, extra);
|
|
106327
|
+
}
|
|
106328
|
+
};
|
|
106329
|
+
}
|
|
106330
|
+
Log.create = create;
|
|
106331
|
+
})(Log ||= {});
|
|
106332
|
+
|
|
106333
|
+
// ../hackbrowser/src/stealth.ts
|
|
106334
|
+
var log2 = Log.create({ service: "hackbrowser:stealth" });
|
|
106261
106335
|
var PLATFORM_ARGS = process.platform === "linux" ? ["--no-sandbox", "--disable-setuid-sandbox", "--disable-dev-shm-usage", "--no-zygote"] : [];
|
|
106262
106336
|
var LAUNCH_ARGS = [
|
|
106263
106337
|
"--disable-blink-features=AutomationControlled",
|
|
@@ -106284,20 +106358,31 @@ function findSystemChrome() {
|
|
|
106284
106358
|
return candidates.find((p) => existsSync2(p));
|
|
106285
106359
|
}
|
|
106286
106360
|
async function connect(opts) {
|
|
106287
|
-
|
|
106361
|
+
const proxy = opts.network?.proxy;
|
|
106362
|
+
if (opts.cdp) {
|
|
106363
|
+
if (proxy) {
|
|
106364
|
+
log2.warn("proxy config ignored: connecting to an already-running browser over CDP", {
|
|
106365
|
+
server: proxy.server,
|
|
106366
|
+
hint: "start that browser with its own --proxy-server, or drop the cdp option to let hackbrowser launch one"
|
|
106367
|
+
});
|
|
106368
|
+
}
|
|
106288
106369
|
return chromium.connectOverCDP(opts.cdp);
|
|
106370
|
+
}
|
|
106371
|
+
const launch = { ...launchOptions(opts.headless), ...proxy ? { proxy } : {} };
|
|
106289
106372
|
const chrome = findSystemChrome();
|
|
106290
106373
|
if (chrome)
|
|
106291
|
-
return chromium.launch({ ...
|
|
106292
|
-
return chromium.launch(
|
|
106374
|
+
return chromium.launch({ ...launch, executablePath: chrome });
|
|
106375
|
+
return chromium.launch(launch);
|
|
106293
106376
|
}
|
|
106294
|
-
function contextOptions(headless = true) {
|
|
106377
|
+
function contextOptions(headless = true, network) {
|
|
106295
106378
|
return {
|
|
106296
106379
|
viewport: headless ? { width: 1920, height: 1080 } : null,
|
|
106297
106380
|
screen: { width: 1920, height: 1080 },
|
|
106298
106381
|
locale: "en-US",
|
|
106299
106382
|
timezoneId: "America/New_York",
|
|
106300
|
-
extraHTTPHeaders: { "Accept-Language": "en-US,en;q=0.9" }
|
|
106383
|
+
extraHTTPHeaders: { "Accept-Language": "en-US,en;q=0.9" },
|
|
106384
|
+
...network?.proxy ? { proxy: network.proxy } : {},
|
|
106385
|
+
ignoreHTTPSErrors: network?.ignoreHTTPSErrors ?? true
|
|
106301
106386
|
};
|
|
106302
106387
|
}
|
|
106303
106388
|
var INIT_SCRIPT = `
|
|
@@ -106375,77 +106460,6 @@ if (!window.chrome) window.chrome = {};
|
|
|
106375
106460
|
if (!window.chrome.runtime) window.chrome.runtime = { connect: () => {}, sendMessage: () => {} };
|
|
106376
106461
|
`;
|
|
106377
106462
|
|
|
106378
|
-
// ../hackbrowser/src/log.ts
|
|
106379
|
-
var LEVEL_PRIORITY = {
|
|
106380
|
-
DEBUG: 0,
|
|
106381
|
-
INFO: 1,
|
|
106382
|
-
WARN: 2,
|
|
106383
|
-
ERROR: 3
|
|
106384
|
-
};
|
|
106385
|
-
var currentLevel = "INFO";
|
|
106386
|
-
var sink = defaultStderrSink;
|
|
106387
|
-
function defaultStderrSink(record3) {
|
|
106388
|
-
const ts = new Date(record3.timestamp).toISOString();
|
|
106389
|
-
let line = `${ts} ${record3.level} ${record3.service} ${record3.message}`;
|
|
106390
|
-
if (record3.extra && Object.keys(record3.extra).length > 0) {
|
|
106391
|
-
try {
|
|
106392
|
-
line += " " + JSON.stringify(record3.extra);
|
|
106393
|
-
} catch {
|
|
106394
|
-
line += " [unserializable extra]";
|
|
106395
|
-
}
|
|
106396
|
-
}
|
|
106397
|
-
process.stderr.write(line + `
|
|
106398
|
-
`);
|
|
106399
|
-
}
|
|
106400
|
-
function shouldLog(level) {
|
|
106401
|
-
return LEVEL_PRIORITY[level] >= LEVEL_PRIORITY[currentLevel];
|
|
106402
|
-
}
|
|
106403
|
-
function emit(level, service, message, extra) {
|
|
106404
|
-
if (!shouldLog(level))
|
|
106405
|
-
return;
|
|
106406
|
-
sink({
|
|
106407
|
-
level,
|
|
106408
|
-
service,
|
|
106409
|
-
message,
|
|
106410
|
-
extra,
|
|
106411
|
-
timestamp: Date.now()
|
|
106412
|
-
});
|
|
106413
|
-
}
|
|
106414
|
-
var Log;
|
|
106415
|
-
((Log) => {
|
|
106416
|
-
function init(options) {
|
|
106417
|
-
if (options.level)
|
|
106418
|
-
currentLevel = options.level;
|
|
106419
|
-
}
|
|
106420
|
-
Log.init = init;
|
|
106421
|
-
function setSink(custom4) {
|
|
106422
|
-
sink = custom4;
|
|
106423
|
-
}
|
|
106424
|
-
Log.setSink = setSink;
|
|
106425
|
-
function resetSink() {
|
|
106426
|
-
sink = defaultStderrSink;
|
|
106427
|
-
}
|
|
106428
|
-
Log.resetSink = resetSink;
|
|
106429
|
-
function create(options) {
|
|
106430
|
-
const { service } = options;
|
|
106431
|
-
return {
|
|
106432
|
-
debug(message, extra) {
|
|
106433
|
-
emit("DEBUG", service, message, extra);
|
|
106434
|
-
},
|
|
106435
|
-
info(message, extra) {
|
|
106436
|
-
emit("INFO", service, message, extra);
|
|
106437
|
-
},
|
|
106438
|
-
warn(message, extra) {
|
|
106439
|
-
emit("WARN", service, message, extra);
|
|
106440
|
-
},
|
|
106441
|
-
error(message, extra) {
|
|
106442
|
-
emit("ERROR", service, message, extra);
|
|
106443
|
-
}
|
|
106444
|
-
};
|
|
106445
|
-
}
|
|
106446
|
-
Log.create = create;
|
|
106447
|
-
})(Log ||= {});
|
|
106448
|
-
|
|
106449
106463
|
// ../hackbrowser/src/capture.ts
|
|
106450
106464
|
async function snapshotPageUI(page, trigger, componentPath = "") {
|
|
106451
106465
|
const pageUrl = page.url();
|
|
@@ -106705,7 +106719,7 @@ function flattenObject(obj, prefix, out) {
|
|
|
106705
106719
|
}
|
|
106706
106720
|
|
|
106707
106721
|
// ../hackbrowser/src/ingest.ts
|
|
106708
|
-
var
|
|
106722
|
+
var log3 = Log.create({ service: "hackbrowser:ingest" });
|
|
106709
106723
|
var authHeader = "";
|
|
106710
106724
|
function initAuth(username, password) {
|
|
106711
106725
|
if (!password) {
|
|
@@ -106727,18 +106741,18 @@ async function initSession(serverUrl, targetUrl, credentialId) {
|
|
|
106727
106741
|
body: JSON.stringify(payload)
|
|
106728
106742
|
});
|
|
106729
106743
|
if (!res.ok) {
|
|
106730
|
-
|
|
106744
|
+
log3.error("initSession failed", { status: res.status, body: await res.text().catch(() => "") });
|
|
106731
106745
|
return null;
|
|
106732
106746
|
}
|
|
106733
106747
|
const data = await res.json();
|
|
106734
106748
|
if (!data.sessionID) {
|
|
106735
|
-
|
|
106749
|
+
log3.error("initSession: no sessionID in response");
|
|
106736
106750
|
return null;
|
|
106737
106751
|
}
|
|
106738
|
-
|
|
106752
|
+
log3.info("session created", { sessionID: data.sessionID });
|
|
106739
106753
|
return data.sessionID;
|
|
106740
106754
|
} catch (err) {
|
|
106741
|
-
|
|
106755
|
+
log3.error("initSession network error", { err: String(err) });
|
|
106742
106756
|
return null;
|
|
106743
106757
|
}
|
|
106744
106758
|
}
|
|
@@ -106767,7 +106781,7 @@ function buildIngestPayload(captured, sessionID, credentialId, accessContext) {
|
|
|
106767
106781
|
async function sendIngest(captured, serverUrl, sessionID, credentialId) {
|
|
106768
106782
|
const payload = buildIngestPayload(captured, sessionID, credentialId);
|
|
106769
106783
|
const body = JSON.stringify(payload);
|
|
106770
|
-
|
|
106784
|
+
log3.debug("sending ingest", {
|
|
106771
106785
|
request: captured.raw.split(`\r
|
|
106772
106786
|
`)[0],
|
|
106773
106787
|
uiFields: captured.uiContext?.fields.length ?? 0,
|
|
@@ -106784,14 +106798,14 @@ async function sendIngest(captured, serverUrl, sessionID, credentialId) {
|
|
|
106784
106798
|
body
|
|
106785
106799
|
});
|
|
106786
106800
|
if (!res.ok) {
|
|
106787
|
-
|
|
106801
|
+
log3.error("ingest server error", { status: res.status, body: await res.text().catch(() => "") });
|
|
106788
106802
|
return null;
|
|
106789
106803
|
}
|
|
106790
106804
|
const data = await res.json();
|
|
106791
|
-
|
|
106805
|
+
log3.debug("ingest accepted", { sessionID: data.sessionID ?? sessionID });
|
|
106792
106806
|
return data;
|
|
106793
106807
|
} catch (err) {
|
|
106794
|
-
|
|
106808
|
+
log3.error("ingest network error", { err: String(err) });
|
|
106795
106809
|
return null;
|
|
106796
106810
|
}
|
|
106797
106811
|
}
|
|
@@ -106849,14 +106863,14 @@ async function registerCredential(serverUrl, sessionID, label) {
|
|
|
106849
106863
|
body: JSON.stringify({ label, headers: {} })
|
|
106850
106864
|
});
|
|
106851
106865
|
if (!res.ok) {
|
|
106852
|
-
|
|
106866
|
+
log3.error("registerCredential failed", { status: res.status, label });
|
|
106853
106867
|
return null;
|
|
106854
106868
|
}
|
|
106855
106869
|
const data = await res.json();
|
|
106856
|
-
|
|
106870
|
+
log3.info("credential registered", { sessionID, credentialID: data.id, label });
|
|
106857
106871
|
return data.id ?? null;
|
|
106858
106872
|
} catch (err) {
|
|
106859
|
-
|
|
106873
|
+
log3.error("registerCredential network error", { err: String(err), label });
|
|
106860
106874
|
return null;
|
|
106861
106875
|
}
|
|
106862
106876
|
}
|
|
@@ -106868,22 +106882,22 @@ async function syncCredentialHeaders(serverUrl, sessionID, credentialID, headers
|
|
|
106868
106882
|
body: JSON.stringify({ headers })
|
|
106869
106883
|
});
|
|
106870
106884
|
if (!res.ok) {
|
|
106871
|
-
|
|
106885
|
+
log3.error("syncCredentialHeaders failed", { status: res.status, credentialID });
|
|
106872
106886
|
} else {
|
|
106873
|
-
|
|
106887
|
+
log3.debug("credential headers synced", { credentialID });
|
|
106874
106888
|
}
|
|
106875
106889
|
} catch (err) {
|
|
106876
|
-
|
|
106890
|
+
log3.error("syncCredentialHeaders network error", { err: String(err), credentialID });
|
|
106877
106891
|
}
|
|
106878
106892
|
}
|
|
106879
106893
|
|
|
106880
106894
|
// ../hackbrowser/src/auth.ts
|
|
106881
106895
|
import fs3 from "fs";
|
|
106882
106896
|
import readline from "readline";
|
|
106883
|
-
var
|
|
106897
|
+
var log4 = Log.create({ service: "hackbrowser:auth" });
|
|
106884
106898
|
async function loadSession(context, filePath) {
|
|
106885
106899
|
if (!fs3.existsSync(filePath)) {
|
|
106886
|
-
|
|
106900
|
+
log4.info("no session file found", { file: filePath });
|
|
106887
106901
|
return false;
|
|
106888
106902
|
}
|
|
106889
106903
|
try {
|
|
@@ -106891,10 +106905,10 @@ async function loadSession(context, filePath) {
|
|
|
106891
106905
|
if (data.cookies) {
|
|
106892
106906
|
await context.addCookies(data.cookies);
|
|
106893
106907
|
}
|
|
106894
|
-
|
|
106908
|
+
log4.info("session loaded", { file: filePath });
|
|
106895
106909
|
return true;
|
|
106896
106910
|
} catch (err) {
|
|
106897
|
-
|
|
106911
|
+
log4.error("failed to load session", { file: filePath, err: String(err) });
|
|
106898
106912
|
return false;
|
|
106899
106913
|
}
|
|
106900
106914
|
}
|
|
@@ -106924,7 +106938,7 @@ async function handle2FA(page) {
|
|
|
106924
106938
|
for (const sel of mfaSelectors) {
|
|
106925
106939
|
const el = await page.$(sel);
|
|
106926
106940
|
if (el) {
|
|
106927
|
-
|
|
106941
|
+
log4.info("2FA detected", { selector: sel });
|
|
106928
106942
|
const code = await waitForHuman("Enter your 2FA code and press Enter:");
|
|
106929
106943
|
await el.fill(code);
|
|
106930
106944
|
const form = await page.$("form");
|
|
@@ -107169,13 +107183,13 @@ async function waitForManualLogin(page, label, progress) {
|
|
|
107169
107183
|
};
|
|
107170
107184
|
await injectButton();
|
|
107171
107185
|
page.on("load", injectButton);
|
|
107172
|
-
|
|
107186
|
+
log4.info("waiting for manual login", { label: label ?? "default" });
|
|
107173
107187
|
await readyPromise;
|
|
107174
107188
|
page.removeListener("load", injectButton);
|
|
107175
107189
|
await page.evaluate((btnId) => {
|
|
107176
107190
|
document.getElementById(btnId)?.remove();
|
|
107177
107191
|
}, buttonId).catch(() => {});
|
|
107178
|
-
|
|
107192
|
+
log4.info("manual login confirmed", { label: label ?? "default" });
|
|
107179
107193
|
}
|
|
107180
107194
|
async function autoLogin(page, credentials) {
|
|
107181
107195
|
const userSel = credentials.usernameSelector ?? 'input[type="text"], input[type="email"], input[name*="user"], input[name*="email"], input[id*="user"], input[id*="email"]';
|
|
@@ -107191,10 +107205,10 @@ async function autoLogin(page, credentials) {
|
|
|
107191
107205
|
await page.keyboard.press("Enter");
|
|
107192
107206
|
}
|
|
107193
107207
|
await page.waitForLoadState("domcontentloaded").catch(() => {});
|
|
107194
|
-
|
|
107208
|
+
log4.info("auto-login attempted", { username: credentials.username });
|
|
107195
107209
|
await handle2FA(page);
|
|
107196
107210
|
} catch (err) {
|
|
107197
|
-
|
|
107211
|
+
log4.warn("auto-login failed, falling back to manual login", { err: String(err) });
|
|
107198
107212
|
await waitForManualLogin(page);
|
|
107199
107213
|
}
|
|
107200
107214
|
}
|
|
@@ -115904,7 +115918,7 @@ Always respond with a valid JSON object. If there is nothing to explore, return:
|
|
|
115904
115918
|
`;
|
|
115905
115919
|
|
|
115906
115920
|
// ../hackbrowser/src/navigator.ts
|
|
115907
|
-
var
|
|
115921
|
+
var log5 = Log.create({ service: "hackbrowser:navigator" });
|
|
115908
115922
|
function isAuthError(err) {
|
|
115909
115923
|
if (!err || typeof err !== "object")
|
|
115910
115924
|
return false;
|
|
@@ -115924,20 +115938,20 @@ var cachedModel = null;
|
|
|
115924
115938
|
async function resolveModel(override) {
|
|
115925
115939
|
if (override) {
|
|
115926
115940
|
cachedModel = override;
|
|
115927
|
-
|
|
115941
|
+
log5.info("model resolved via opts.model (cyberstrike injection)");
|
|
115928
115942
|
return cachedModel;
|
|
115929
115943
|
}
|
|
115930
115944
|
if (cachedModel)
|
|
115931
115945
|
return cachedModel;
|
|
115932
115946
|
if (process.env.ANTHROPIC_API_KEY) {
|
|
115933
115947
|
const model = process.env.BROWSER_AGENT_MODEL ?? "claude-sonnet-4-6";
|
|
115934
|
-
|
|
115948
|
+
log5.info("model resolved via ANTHROPIC_API_KEY", { model });
|
|
115935
115949
|
cachedModel = createAnthropic2({ apiKey: process.env.ANTHROPIC_API_KEY })(model);
|
|
115936
115950
|
return cachedModel;
|
|
115937
115951
|
}
|
|
115938
115952
|
if (process.env.OPENAI_API_KEY) {
|
|
115939
115953
|
const model = process.env.BROWSER_AGENT_MODEL ?? "gpt-4o";
|
|
115940
|
-
|
|
115954
|
+
log5.info("model resolved via OPENAI_API_KEY", { model });
|
|
115941
115955
|
cachedModel = createOpenAI2({ apiKey: process.env.OPENAI_API_KEY })(model);
|
|
115942
115956
|
return cachedModel;
|
|
115943
115957
|
}
|
|
@@ -115963,7 +115977,7 @@ async function planPage(snapshot, model, usageAcc) {
|
|
|
115963
115977
|
usageAcc.cacheReadTokens += result.usage.cachedInputTokens ?? 0;
|
|
115964
115978
|
}
|
|
115965
115979
|
const raw = result.text.trim();
|
|
115966
|
-
|
|
115980
|
+
log5.debug("planner response", { length: raw.length, raw: raw.slice(0, 500) });
|
|
115967
115981
|
const start = raw.indexOf("{");
|
|
115968
115982
|
const end = raw.lastIndexOf("}");
|
|
115969
115983
|
if (start === -1 || end === -1)
|
|
@@ -115973,18 +115987,18 @@ async function planPage(snapshot, model, usageAcc) {
|
|
|
115973
115987
|
};
|
|
115974
115988
|
try {
|
|
115975
115989
|
const plan = await attempt();
|
|
115976
|
-
|
|
115990
|
+
log5.debug("page plan", { tasks: plan.tasks.length });
|
|
115977
115991
|
return plan;
|
|
115978
115992
|
} catch (err) {
|
|
115979
115993
|
if (isAuthError(err))
|
|
115980
115994
|
throw err;
|
|
115981
|
-
|
|
115995
|
+
log5.warn("planPage failed, retrying once", { err: String(err) });
|
|
115982
115996
|
try {
|
|
115983
115997
|
return await attempt();
|
|
115984
115998
|
} catch (err2) {
|
|
115985
115999
|
if (isAuthError(err2))
|
|
115986
116000
|
throw err2;
|
|
115987
|
-
|
|
116001
|
+
log5.error("planPage failed after retry, returning empty plan", { err: String(err2) });
|
|
115988
116002
|
return { tasks: [] };
|
|
115989
116003
|
}
|
|
115990
116004
|
}
|
|
@@ -116020,7 +116034,7 @@ async function planUnexploredElements(snapshot, unexploredLabels, model, usageAc
|
|
|
116020
116034
|
usageAcc.cacheReadTokens += result.usage.cachedInputTokens ?? 0;
|
|
116021
116035
|
}
|
|
116022
116036
|
const raw = result.text.trim();
|
|
116023
|
-
|
|
116037
|
+
log5.debug("unexplored planner response", { length: raw.length, raw: raw.slice(0, 500) });
|
|
116024
116038
|
const start = raw.indexOf("{");
|
|
116025
116039
|
const end = raw.lastIndexOf("}");
|
|
116026
116040
|
if (start === -1 || end === -1)
|
|
@@ -116030,12 +116044,12 @@ async function planUnexploredElements(snapshot, unexploredLabels, model, usageAc
|
|
|
116030
116044
|
};
|
|
116031
116045
|
try {
|
|
116032
116046
|
const plan = await attempt();
|
|
116033
|
-
|
|
116047
|
+
log5.debug("unexplored plan", { tasks: plan.tasks.length });
|
|
116034
116048
|
return plan;
|
|
116035
116049
|
} catch (err) {
|
|
116036
116050
|
if (isAuthError(err))
|
|
116037
116051
|
throw err;
|
|
116038
|
-
|
|
116052
|
+
log5.warn("planUnexploredElements failed", { err: String(err) });
|
|
116039
116053
|
return { tasks: [] };
|
|
116040
116054
|
}
|
|
116041
116055
|
}
|
|
@@ -116092,7 +116106,7 @@ function validateIntelligence(raw) {
|
|
|
116092
116106
|
out.revisitOn = on;
|
|
116093
116107
|
}
|
|
116094
116108
|
if (out.pageState === "empty" && !out.revisitReason) {
|
|
116095
|
-
|
|
116109
|
+
log5.warn("pageState='empty' without revisitReason — downgrading to 'unknown'");
|
|
116096
116110
|
out.pageState = "unknown";
|
|
116097
116111
|
out.revisitAfter = null;
|
|
116098
116112
|
out.revisitOn = undefined;
|
|
@@ -116285,7 +116299,7 @@ function buildPlannerSnapshot(url3, elements, globalState, credId, viewportCente
|
|
|
116285
116299
|
}
|
|
116286
116300
|
|
|
116287
116301
|
// ../hackbrowser/src/scanner.ts
|
|
116288
|
-
var
|
|
116302
|
+
var log6 = Log.create({ service: "hackbrowser:scanner" });
|
|
116289
116303
|
var MAX_ELEMENTS = 50;
|
|
116290
116304
|
var MAX_PER_TEMPLATE = 5;
|
|
116291
116305
|
var REVEAL_MAX_STEPS = 10;
|
|
@@ -117007,14 +117021,14 @@ function pickSample(accept) {
|
|
|
117007
117021
|
}
|
|
117008
117022
|
|
|
117009
117023
|
// ../hackbrowser/src/executor.ts
|
|
117010
|
-
var
|
|
117024
|
+
var log7 = Log.create({ service: "hackbrowser:executor" });
|
|
117011
117025
|
var CLICK_TIMEOUT = 2000;
|
|
117012
117026
|
var FILL_TIMEOUT = 3000;
|
|
117013
117027
|
var STABILIZE_TIMEOUT = 3000;
|
|
117014
117028
|
var STABILIZE_WAIT = 200;
|
|
117015
117029
|
async function execute(page, element, action, value, setPendingUI, elementsBefore) {
|
|
117016
117030
|
const urlBefore = page.url();
|
|
117017
|
-
|
|
117031
|
+
log7.info("executing", { action, elementId: element.id, label: element.label, selector: element.selector });
|
|
117018
117032
|
let error49;
|
|
117019
117033
|
switch (action) {
|
|
117020
117034
|
case "fill":
|
|
@@ -117062,7 +117076,7 @@ async function executeFill(page, element, value) {
|
|
|
117062
117076
|
`)[0]);
|
|
117063
117077
|
if (!typeErr)
|
|
117064
117078
|
return;
|
|
117065
|
-
|
|
117079
|
+
log7.warn("fill failed", { selector, error: fillErr });
|
|
117066
117080
|
return fillErr;
|
|
117067
117081
|
}
|
|
117068
117082
|
async function executeFileSelect(page, selector) {
|
|
@@ -117071,10 +117085,10 @@ async function executeFileSelect(page, selector) {
|
|
|
117071
117085
|
const err = await page.setInputFiles(selector, { name: sample.name, mimeType: sample.mimeType, buffer: sample.buffer }, { timeout: FILL_TIMEOUT }).then(() => null).catch((e2) => e2.message.split(`
|
|
117072
117086
|
`)[0]);
|
|
117073
117087
|
if (err) {
|
|
117074
|
-
|
|
117088
|
+
log7.warn("file select failed", { selector, accept, error: err });
|
|
117075
117089
|
return err;
|
|
117076
117090
|
}
|
|
117077
|
-
|
|
117091
|
+
log7.info("file selected", { selector, file: sample.name });
|
|
117078
117092
|
return;
|
|
117079
117093
|
}
|
|
117080
117094
|
async function executeSliderFill(page, selector, value) {
|
|
@@ -117108,7 +117122,7 @@ async function executeClick(page, element) {
|
|
|
117108
117122
|
const err = await page.click(element.selector, { timeout: CLICK_TIMEOUT }).then(() => null).catch((e2) => e2.message.split(`
|
|
117109
117123
|
`)[0]);
|
|
117110
117124
|
if (err) {
|
|
117111
|
-
|
|
117125
|
+
log7.warn("click failed", { selector: element.selector, error: err });
|
|
117112
117126
|
return err;
|
|
117113
117127
|
}
|
|
117114
117128
|
return;
|
|
@@ -117121,14 +117135,14 @@ async function executeSelect(page, element, value) {
|
|
|
117121
117135
|
const clickErr = await page.click(element.selector, { timeout: CLICK_TIMEOUT }).then(() => null).catch((e2) => e2.message.split(`
|
|
117122
117136
|
`)[0]);
|
|
117123
117137
|
if (clickErr) {
|
|
117124
|
-
|
|
117138
|
+
log7.warn("select failed", { selector: element.selector, error: nativeErr });
|
|
117125
117139
|
return nativeErr;
|
|
117126
117140
|
}
|
|
117127
117141
|
await page.waitForTimeout(300);
|
|
117128
117142
|
const optionErr = await page.click(`role=option[name="${value}"]`, { timeout: CLICK_TIMEOUT }).then(() => null).catch(() => page.click(`text="${value}"`, { timeout: CLICK_TIMEOUT }).then(() => null).catch((e2) => e2.message.split(`
|
|
117129
117143
|
`)[0]));
|
|
117130
117144
|
if (optionErr) {
|
|
117131
|
-
|
|
117145
|
+
log7.warn("select option failed", { value, error: optionErr });
|
|
117132
117146
|
return optionErr;
|
|
117133
117147
|
}
|
|
117134
117148
|
return;
|
|
@@ -117139,7 +117153,7 @@ async function executeNavigate(page, element) {
|
|
|
117139
117153
|
const err = await page.goto(element.href, { waitUntil: "domcontentloaded", timeout: 15000 }).then(() => null).catch((e2) => e2.message.split(`
|
|
117140
117154
|
`)[0]);
|
|
117141
117155
|
if (err) {
|
|
117142
|
-
|
|
117156
|
+
log7.warn("navigate failed", { url: element.href, error: err });
|
|
117143
117157
|
return err;
|
|
117144
117158
|
}
|
|
117145
117159
|
await page.waitForTimeout(400);
|
|
@@ -128177,7 +128191,7 @@ function makeMatcher(scopes) {
|
|
|
128177
128191
|
}
|
|
128178
128192
|
|
|
128179
128193
|
// ../hackbrowser/src/agent.ts
|
|
128180
|
-
var
|
|
128194
|
+
var log8 = Log.create({ service: "hackbrowser:agent" });
|
|
128181
128195
|
var MAX_STEPS_PER_PAGE = 30;
|
|
128182
128196
|
var MAX_UNPLANNED_ITERATIONS = 2;
|
|
128183
128197
|
var OVERLAY_ESCAPE_WAIT = 400;
|
|
@@ -128187,6 +128201,12 @@ var POST_GOTO_WAIT = 400;
|
|
|
128187
128201
|
var NETWORK_IDLE_TIMEOUT = 1500;
|
|
128188
128202
|
var MAX_INLINE_DEPTH = 2;
|
|
128189
128203
|
var LOGIN_SUCCESS_PATTERN = /POST\s+.*\/(login|signin|authenticate)\S*\s+\[200\]/i;
|
|
128204
|
+
var CERT_ERROR = /ERR_CERT|CERT_AUTHORITY|ERR_SSL|SSL_ERROR/i;
|
|
128205
|
+
function certHint(message, config3) {
|
|
128206
|
+
if (!config3.network?.proxy || !CERT_ERROR.test(message))
|
|
128207
|
+
return "";
|
|
128208
|
+
return " — this request went through the configured proxy. If the proxy intercepts TLS, the browser must trust its CA" + " (install it in the OS certificate store) or be told to accept untrusted certificates.";
|
|
128209
|
+
}
|
|
128190
128210
|
async function stabilizeAfterGoto(page) {
|
|
128191
128211
|
await page.waitForTimeout(POST_GOTO_WAIT);
|
|
128192
128212
|
await page.waitForLoadState("networkidle", { timeout: NETWORK_IDLE_TIMEOUT }).catch(() => {});
|
|
@@ -128194,21 +128214,36 @@ async function stabilizeAfterGoto(page) {
|
|
|
128194
128214
|
function createBrowserHealth() {
|
|
128195
128215
|
return { dead: false, reason: "" };
|
|
128196
128216
|
}
|
|
128217
|
+
function lifecycleContext(browser, page) {
|
|
128218
|
+
try {
|
|
128219
|
+
const ctx = { connected: browser.isConnected() };
|
|
128220
|
+
if (page) {
|
|
128221
|
+
ctx.url = page.url();
|
|
128222
|
+
ctx.openPages = page.context().pages().length;
|
|
128223
|
+
}
|
|
128224
|
+
return ctx;
|
|
128225
|
+
} catch {
|
|
128226
|
+
return {};
|
|
128227
|
+
}
|
|
128228
|
+
}
|
|
128197
128229
|
function attachLifecycleHandlers(browser, page, health) {
|
|
128198
128230
|
browser.on("disconnected", () => {
|
|
128199
128231
|
health.dead = true;
|
|
128200
128232
|
health.reason = "browser process disconnected";
|
|
128201
|
-
|
|
128233
|
+
log8.error("browser disconnected — crawl will terminate", lifecycleContext(browser, page));
|
|
128202
128234
|
});
|
|
128203
128235
|
page.on("close", () => {
|
|
128204
128236
|
health.dead = true;
|
|
128205
128237
|
health.reason = "page closed unexpectedly";
|
|
128206
|
-
|
|
128238
|
+
log8.error("page closed — crawl will terminate", lifecycleContext(browser, page));
|
|
128207
128239
|
});
|
|
128208
128240
|
page.on("crash", () => {
|
|
128209
128241
|
health.dead = true;
|
|
128210
128242
|
health.reason = "page renderer crashed";
|
|
128211
|
-
|
|
128243
|
+
log8.error("page crashed — crawl will terminate", lifecycleContext(browser, page));
|
|
128244
|
+
});
|
|
128245
|
+
page.context().on("page", (p) => {
|
|
128246
|
+
log8.debug("new page/popup opened", lifecycleContext(browser, p));
|
|
128212
128247
|
});
|
|
128213
128248
|
}
|
|
128214
128249
|
function isBrowserDead(health) {
|
|
@@ -128243,7 +128278,7 @@ function triggerReDiscovery(globalState) {
|
|
|
128243
128278
|
return;
|
|
128244
128279
|
globalState.authPhase = "authenticated";
|
|
128245
128280
|
globalState.pendingReDiscovery = true;
|
|
128246
|
-
|
|
128281
|
+
log8.info("login detected, re-discovery pending");
|
|
128247
128282
|
}
|
|
128248
128283
|
function flushReDiscovery(globalState) {
|
|
128249
128284
|
if (!globalState.pendingReDiscovery)
|
|
@@ -128255,7 +128290,7 @@ function flushReDiscovery(globalState) {
|
|
|
128255
128290
|
globalState.pageQueue.push(url3);
|
|
128256
128291
|
}
|
|
128257
128292
|
}
|
|
128258
|
-
|
|
128293
|
+
log8.info("post-login re-discovery flushed", {
|
|
128259
128294
|
pagesToRevisit: pagesToRevisit.length,
|
|
128260
128295
|
queueSize: globalState.pageQueue.length
|
|
128261
128296
|
});
|
|
@@ -128265,7 +128300,7 @@ function processAuthPhase(globalState) {
|
|
|
128265
128300
|
if (globalState.authPhase === "anonymous") {
|
|
128266
128301
|
const registerPage = deferred.find((d) => d.type === "register");
|
|
128267
128302
|
if (registerPage) {
|
|
128268
|
-
|
|
128303
|
+
log8.info("phase transition: anonymous → register", { url: registerPage.url });
|
|
128269
128304
|
globalState.authPhase = "registered";
|
|
128270
128305
|
globalState.visitedPages.add(normalizeUrl(registerPage.url));
|
|
128271
128306
|
globalState.pageQueue.push(registerPage.url);
|
|
@@ -128277,12 +128312,12 @@ function processAuthPhase(globalState) {
|
|
|
128277
128312
|
const loginNormalized = normalizeUrl(loginPage.url);
|
|
128278
128313
|
const alreadyExplored = globalState.visitedPages.has(loginNormalized);
|
|
128279
128314
|
if (!alreadyExplored) {
|
|
128280
|
-
|
|
128315
|
+
log8.info("exploring login page to discover register link", { url: loginPage.url });
|
|
128281
128316
|
globalState.visitedPages.add(loginNormalized);
|
|
128282
128317
|
globalState.pageQueue.push(loginPage.url);
|
|
128283
128318
|
return;
|
|
128284
128319
|
}
|
|
128285
|
-
|
|
128320
|
+
log8.info("phase transition: anonymous → login (no register found)", { url: loginPage.url });
|
|
128286
128321
|
globalState.pageQueue.push(loginPage.url);
|
|
128287
128322
|
globalState.deferredAuthPages = deferred.filter((d) => d.type !== "login");
|
|
128288
128323
|
return;
|
|
@@ -128291,7 +128326,7 @@ function processAuthPhase(globalState) {
|
|
|
128291
128326
|
if (globalState.authPhase === "registered") {
|
|
128292
128327
|
const loginPage = deferred.find((d) => d.type === "login");
|
|
128293
128328
|
if (loginPage) {
|
|
128294
|
-
|
|
128329
|
+
log8.info("phase transition: registered → login", { url: loginPage.url });
|
|
128295
128330
|
getIntelligence(globalState, SINGLE_CRED).pageFingerprints.delete(normalizeUrl(loginPage.url));
|
|
128296
128331
|
globalState.pageQueue.push(loginPage.url);
|
|
128297
128332
|
globalState.deferredAuthPages = deferred.filter((d) => d.type !== "login");
|
|
@@ -128301,7 +128336,7 @@ function processAuthPhase(globalState) {
|
|
|
128301
128336
|
if (globalState.authPhase === "authenticated") {
|
|
128302
128337
|
const logoutPage = deferred.find((d) => d.type === "logout");
|
|
128303
128338
|
if (logoutPage) {
|
|
128304
|
-
|
|
128339
|
+
log8.info("phase transition: authenticated → logout", { url: logoutPage.url });
|
|
128305
128340
|
globalState.visitedPages.add(normalizeUrl(logoutPage.url));
|
|
128306
128341
|
globalState.pageQueue.push(logoutPage.url);
|
|
128307
128342
|
globalState.deferredAuthPages = deferred.filter((d) => d.type !== "logout");
|
|
@@ -128309,7 +128344,7 @@ function processAuthPhase(globalState) {
|
|
|
128309
128344
|
}
|
|
128310
128345
|
}
|
|
128311
128346
|
if (globalState.deferredAuthPages.length > 0) {
|
|
128312
|
-
|
|
128347
|
+
log8.debug("clearing remaining deferred auth pages", {
|
|
128313
128348
|
remaining: globalState.deferredAuthPages.map((d) => `${d.type}:${d.url}`)
|
|
128314
128349
|
});
|
|
128315
128350
|
globalState.deferredAuthPages = [];
|
|
@@ -128322,7 +128357,7 @@ function resolveScopePatterns(config3) {
|
|
|
128322
128357
|
}
|
|
128323
128358
|
function logScope(targetUrl, patterns, userProvided) {
|
|
128324
128359
|
const normalized = patterns.map(normalizeScope).filter(Boolean);
|
|
128325
|
-
|
|
128360
|
+
log8.info("network scope", {
|
|
128326
128361
|
target: targetUrl,
|
|
128327
128362
|
scope: normalized,
|
|
128328
128363
|
source: userProvided ? "explicit (--scope)" : "auto-derived"
|
|
@@ -128342,7 +128377,7 @@ function createDryRunHandler() {
|
|
|
128342
128377
|
const roles = captured.elementRoles;
|
|
128343
128378
|
const page = captured.pageUrl;
|
|
128344
128379
|
const visitedBy = captured.pageVisitedBy;
|
|
128345
|
-
|
|
128380
|
+
log8.info(`→ ${requestLine} [${status}]${uiFields > 0 ? ` ui_fields:${uiFields}` : ""}${trigger ? ` trigger:${trigger}` : ""}${roles ? ` roles:[${roles.join(",")}]` : ""}${page ? ` page:${page}` : ""}${visitedBy ? ` visited_by:[${visitedBy.join(",")}]` : ""}`);
|
|
128346
128381
|
if (captured.uiContext) {
|
|
128347
128382
|
for (const f3 of captured.uiContext.fields) {
|
|
128348
128383
|
const flags = [
|
|
@@ -128351,10 +128386,10 @@ function createDryRunHandler() {
|
|
|
128351
128386
|
f3.isDisabled ? "disabled" : "",
|
|
128352
128387
|
f3.isDisplayOnly ? "display-only" : ""
|
|
128353
128388
|
].filter(Boolean).join(",");
|
|
128354
|
-
|
|
128389
|
+
log8.debug(`field: ${f3.name || f3.label || "(unnamed)"} type:${f3.type} value:${JSON.stringify(f3.value)}${flags ? ` [${flags}]` : ""}`);
|
|
128355
128390
|
}
|
|
128356
128391
|
if (hiddenParams.length > 0) {
|
|
128357
|
-
|
|
128392
|
+
log8.debug(`hidden params (not in UI): ${hiddenParams.join(", ")}`);
|
|
128358
128393
|
}
|
|
128359
128394
|
}
|
|
128360
128395
|
};
|
|
@@ -128426,7 +128461,13 @@ function setupRequestInterceptor(page, inScope, onCapture, credentialId = SINGLE
|
|
|
128426
128461
|
return;
|
|
128427
128462
|
}
|
|
128428
128463
|
const method = request.method();
|
|
128429
|
-
|
|
128464
|
+
let headers;
|
|
128465
|
+
try {
|
|
128466
|
+
headers = await request.allHeaders();
|
|
128467
|
+
} catch {
|
|
128468
|
+
log8.debug("request capture dropped — target closed mid-flight", { url: url3 });
|
|
128469
|
+
return;
|
|
128470
|
+
}
|
|
128430
128471
|
const postData = request.postData() ?? null;
|
|
128431
128472
|
const raw = buildRawRequest(method, url3, headers, postData);
|
|
128432
128473
|
let uiContext = null;
|
|
@@ -128525,7 +128566,7 @@ function enqueueUrl(url3, globalState, inScope) {
|
|
|
128525
128566
|
if (patternKey) {
|
|
128526
128567
|
const count = globalState.pathPatternCounts.get(patternKey) ?? 0;
|
|
128527
128568
|
if (count >= MAX_PER_PATH_PATTERN) {
|
|
128528
|
-
|
|
128569
|
+
log8.debug("path pattern limit reached, skipping", { url: normalized, patternKey, count });
|
|
128529
128570
|
return false;
|
|
128530
128571
|
}
|
|
128531
128572
|
globalState.pathPatternCounts.set(patternKey, count + 1);
|
|
@@ -128536,14 +128577,14 @@ function enqueueUrl(url3, globalState, inScope) {
|
|
|
128536
128577
|
const alreadyDeferred = globalState.deferredAuthPages.some((d) => d.url === normalized);
|
|
128537
128578
|
if (!alreadyDeferred) {
|
|
128538
128579
|
globalState.deferredAuthPages.push({ url: normalized, type: authType });
|
|
128539
|
-
|
|
128580
|
+
log8.debug("deferred auth URL", { url: normalized, type: authType });
|
|
128540
128581
|
}
|
|
128541
128582
|
return false;
|
|
128542
128583
|
}
|
|
128543
128584
|
}
|
|
128544
128585
|
globalState.visitedPages.add(normalized);
|
|
128545
128586
|
globalState.pageQueue.push(url3);
|
|
128546
|
-
|
|
128587
|
+
log8.debug("enqueued URL", { url: normalized, queueSize: globalState.pageQueue.length });
|
|
128547
128588
|
return true;
|
|
128548
128589
|
}
|
|
128549
128590
|
function getPathPatternKey(url3) {
|
|
@@ -128570,17 +128611,17 @@ async function explorePageWithAI(page, pageUrl, interceptor, model, globalState,
|
|
|
128570
128611
|
});
|
|
128571
128612
|
let elements = await collectElements(page);
|
|
128572
128613
|
if (elements.length === 0) {
|
|
128573
|
-
|
|
128614
|
+
log8.debug("no elements on first collection, waiting for render");
|
|
128574
128615
|
await page.waitForTimeout(SPA_RENDER_RETRY_WAIT);
|
|
128575
128616
|
elements = await collectElements(page);
|
|
128576
128617
|
}
|
|
128577
128618
|
elements = filterVisitedLinks(elements, pageUrl, globalState.visitedPages);
|
|
128578
|
-
|
|
128619
|
+
log8.debug("initial scan", { elements: elements.length });
|
|
128579
128620
|
const vcBlocked = await isViewportCenterBlocked(page);
|
|
128580
128621
|
const snapshot = buildPlannerSnapshot(pageUrl, elements, globalState, credentialId, vcBlocked);
|
|
128581
128622
|
csEmit(page, { type: "llm-thinking", reason: "page-plan", elements: elements.length, credential: credentialId });
|
|
128582
128623
|
const plan = await planPage(snapshot, model, usageAcc);
|
|
128583
|
-
|
|
128624
|
+
log8.info("page plan received", {
|
|
128584
128625
|
tasks: plan.tasks.length,
|
|
128585
128626
|
pageState: plan.pageState ?? "unknown",
|
|
128586
128627
|
revisitAfter: plan.revisitAfter ?? null
|
|
@@ -128608,12 +128649,12 @@ async function explorePageWithAI(page, pageUrl, interceptor, model, globalState,
|
|
|
128608
128649
|
const lookupRole = task.type === "form" ? task.fields[0]?.role ?? task.submit.role : task.role;
|
|
128609
128650
|
const lookupLabel = task.type === "form" ? task.fields[0]?.label ?? task.submit.label : task.label;
|
|
128610
128651
|
if (!resolveElement(elements, lookupRole, lookupLabel) && await isViewportCenterBlocked(page)) {
|
|
128611
|
-
|
|
128652
|
+
log8.debug("task target blocked by overlay, closing overlay first", { role: lookupRole, label: lookupLabel });
|
|
128612
128653
|
await closeOverlay(page);
|
|
128613
128654
|
elements = filterVisitedLinks(await collectElements(page), pageUrl, globalState.visitedPages);
|
|
128614
128655
|
}
|
|
128615
128656
|
}
|
|
128616
|
-
|
|
128657
|
+
log8.info("executing task", {
|
|
128617
128658
|
step: `${steps}/${MAX_STEPS_PER_PAGE}`,
|
|
128618
128659
|
type: task.type,
|
|
128619
128660
|
label: task.type === "form" ? `form(${task.fields.length} fields → ${task.submit.label.slice(0, 30)})` : task.label,
|
|
@@ -128648,7 +128689,7 @@ async function explorePageWithAI(page, pageUrl, interceptor, model, globalState,
|
|
|
128648
128689
|
const newPlan = await planPage(snapshot2, model, usageAcc);
|
|
128649
128690
|
applyPlanIntelligence(newPlan, pageUrl, globalState, credentialId, page);
|
|
128650
128691
|
if (newPlan.tasks.length > 0) {
|
|
128651
|
-
|
|
128692
|
+
log8.debug("re-plan after state change", {
|
|
128652
128693
|
reason: hasStaleTargets ? hasNewElements ? "new+stale" : "stale" : "new",
|
|
128653
128694
|
tasks: newPlan.tasks.length
|
|
128654
128695
|
});
|
|
@@ -128666,7 +128707,7 @@ async function explorePageWithAI(page, pageUrl, interceptor, model, globalState,
|
|
|
128666
128707
|
}
|
|
128667
128708
|
const dropped = pruneStaleTasks(taskQueue, freshElements);
|
|
128668
128709
|
if (dropped > 0) {
|
|
128669
|
-
|
|
128710
|
+
log8.debug("pruned stale tasks", { dropped });
|
|
128670
128711
|
csEmit(page, {
|
|
128671
128712
|
type: "intelligence",
|
|
128672
128713
|
kind: "stale-prune",
|
|
@@ -128677,7 +128718,7 @@ async function explorePageWithAI(page, pageUrl, interceptor, model, globalState,
|
|
|
128677
128718
|
}
|
|
128678
128719
|
}
|
|
128679
128720
|
if (taskQueue.length === 0 && await isViewportCenterBlocked(page)) {
|
|
128680
|
-
|
|
128721
|
+
log8.debug("queue empty, overlay open — closing overlay");
|
|
128681
128722
|
await closeOverlay(page);
|
|
128682
128723
|
if (!await isViewportCenterBlocked(page)) {
|
|
128683
128724
|
elements = filterVisitedLinks(await collectElements(page), pageUrl, globalState.visitedPages);
|
|
@@ -128696,7 +128737,7 @@ async function explorePageWithAI(page, pageUrl, interceptor, model, globalState,
|
|
|
128696
128737
|
const unexplored = findUnexploredElements(currentElements, semanticActionsDone, seenKeys);
|
|
128697
128738
|
if (unexplored.length === 0)
|
|
128698
128739
|
break;
|
|
128699
|
-
|
|
128740
|
+
log8.info("unexplored elements found", {
|
|
128700
128741
|
count: unexplored.length,
|
|
128701
128742
|
iteration: iteration + 1,
|
|
128702
128743
|
labels: unexplored.slice(0, 5)
|
|
@@ -128713,7 +128754,7 @@ async function explorePageWithAI(page, pageUrl, interceptor, model, globalState,
|
|
|
128713
128754
|
applyPlanIntelligence(additionalPlan, pageUrl, globalState, credentialId, page);
|
|
128714
128755
|
if (additionalPlan.tasks.length === 0)
|
|
128715
128756
|
break;
|
|
128716
|
-
|
|
128757
|
+
log8.info("additional plan for unexplored elements", { tasks: additionalPlan.tasks.length });
|
|
128717
128758
|
csEmit(page, {
|
|
128718
128759
|
type: "plan-received",
|
|
128719
128760
|
tasks: additionalPlan.tasks.length,
|
|
@@ -128737,7 +128778,7 @@ async function explorePageWithAI(page, pageUrl, interceptor, model, globalState,
|
|
|
128737
128778
|
elements = filterVisitedLinks(await collectElements(page), pageUrl, globalState.visitedPages);
|
|
128738
128779
|
}
|
|
128739
128780
|
}
|
|
128740
|
-
|
|
128781
|
+
log8.info("executing additional task", {
|
|
128741
128782
|
step: `${steps}/${MAX_STEPS_PER_PAGE}`,
|
|
128742
128783
|
type: task.type,
|
|
128743
128784
|
label: task.type === "form" ? `form(${task.fields.length} fields → ${task.submit.label.slice(0, 30)})` : task.label,
|
|
@@ -128785,7 +128826,7 @@ async function explorePageWithAI(page, pageUrl, interceptor, model, globalState,
|
|
|
128785
128826
|
}
|
|
128786
128827
|
const dropped = pruneStaleTasks(additionalQueue, freshElements);
|
|
128787
128828
|
if (dropped > 0) {
|
|
128788
|
-
|
|
128829
|
+
log8.debug("pruned stale additional tasks", { dropped });
|
|
128789
128830
|
csEmit(page, {
|
|
128790
128831
|
type: "intelligence",
|
|
128791
128832
|
kind: "stale-prune",
|
|
@@ -128799,7 +128840,7 @@ async function explorePageWithAI(page, pageUrl, interceptor, model, globalState,
|
|
|
128799
128840
|
}
|
|
128800
128841
|
const finalElements = await collectElements(page);
|
|
128801
128842
|
getIntelligence(globalState, credentialId).pageFingerprints.set(pageUrl, generateFingerprint(finalElements));
|
|
128802
|
-
|
|
128843
|
+
log8.debug("fingerprint stored", { url: pageUrl, credential: credentialId });
|
|
128803
128844
|
try {
|
|
128804
128845
|
const domLinks = await collectDOMLinks(page, pageUrl, inScope);
|
|
128805
128846
|
for (const url3 of domLinks) {
|
|
@@ -128807,7 +128848,7 @@ async function explorePageWithAI(page, pageUrl, interceptor, model, globalState,
|
|
|
128807
128848
|
linksToEnqueue.push(url3);
|
|
128808
128849
|
}
|
|
128809
128850
|
} catch {
|
|
128810
|
-
|
|
128851
|
+
log8.debug("collectDOMLinks failed (page may have navigated)");
|
|
128811
128852
|
}
|
|
128812
128853
|
return linksToEnqueue;
|
|
128813
128854
|
}
|
|
@@ -128816,7 +128857,7 @@ async function recoverFromOcclusion(page, el, original, retry) {
|
|
|
128816
128857
|
return original;
|
|
128817
128858
|
if ((await probeClickPoint(page, el.selector)).status !== "occluded")
|
|
128818
128859
|
return original;
|
|
128819
|
-
|
|
128860
|
+
log8.debug("click occluded — clearing overlay + waiting, then retrying once", { label: el.label });
|
|
128820
128861
|
await closeOverlay(page);
|
|
128821
128862
|
await page.waitForTimeout(OVERLAY_CLEAR_WAIT);
|
|
128822
128863
|
return retry();
|
|
@@ -128832,7 +128873,7 @@ async function recoverFromClosedContainer(page, el, original, opener, intercepto
|
|
|
128832
128873
|
const openerEl = resolveElement(fresh, opener.role, opener.label);
|
|
128833
128874
|
if (!openerEl)
|
|
128834
128875
|
return original;
|
|
128835
|
-
|
|
128876
|
+
log8.debug("target offscreen — re-opening its container via opener, retrying once", {
|
|
128836
128877
|
target: el.label,
|
|
128837
128878
|
opener: opener.label
|
|
128838
128879
|
});
|
|
@@ -128845,7 +128886,7 @@ async function diagnoseOcclusion(page, el, occ) {
|
|
|
128845
128886
|
if (probe.status === "occluded" && !occ.signaled.has(probe.occluder.text)) {
|
|
128846
128887
|
occ.signaled.add(probe.occluder.text);
|
|
128847
128888
|
occ.pending.push({ label: el.label, role: el.role, occluderText: probe.occluder.text });
|
|
128848
|
-
|
|
128889
|
+
log8.debug("action target occluded — signalling planner to dismiss overlay first", {
|
|
128849
128890
|
label: el.label,
|
|
128850
128891
|
occluder: probe.occluder.text
|
|
128851
128892
|
});
|
|
@@ -128875,7 +128916,7 @@ async function executeFormTask(task, page, elements, interceptor, semanticAction
|
|
|
128875
128916
|
el = resolveElement(elements, field.role, field.label);
|
|
128876
128917
|
}
|
|
128877
128918
|
if (!el) {
|
|
128878
|
-
|
|
128919
|
+
log8.debug("form field not found", { role: field.role, label: field.label });
|
|
128879
128920
|
continue;
|
|
128880
128921
|
}
|
|
128881
128922
|
const action = fieldAction(field.role);
|
|
@@ -128903,7 +128944,7 @@ async function executeFormTask(task, page, elements, interceptor, semanticAction
|
|
|
128903
128944
|
elements = await collectElements(page);
|
|
128904
128945
|
const submitEl = resolveElement(elements, task.submit.role, task.submit.label);
|
|
128905
128946
|
if (!submitEl) {
|
|
128906
|
-
|
|
128947
|
+
log8.debug("submit button not found", task.submit);
|
|
128907
128948
|
return;
|
|
128908
128949
|
}
|
|
128909
128950
|
const fieldsHash = JSON.stringify(task.fields.map((f3) => [f3.label, f3.value]));
|
|
@@ -128934,7 +128975,7 @@ async function executeFormTask(task, page, elements, interceptor, semanticAction
|
|
|
128934
128975
|
async function executeClickTask(task, page, elements, interceptor, semanticActionsDone, globalState, credentialId, linksToEnqueue, pageUrl, inScope, occ, nav, opener) {
|
|
128935
128976
|
const el = resolveElement(elements, task.role, task.label);
|
|
128936
128977
|
if (!el) {
|
|
128937
|
-
|
|
128978
|
+
log8.debug("click target not found", { role: task.role, label: task.label });
|
|
128938
128979
|
return;
|
|
128939
128980
|
}
|
|
128940
128981
|
const key = `${el.selector}::click::`;
|
|
@@ -129019,7 +129060,7 @@ function discoverNewElements(elements, seenKeys) {
|
|
|
129019
129060
|
found = true;
|
|
129020
129061
|
}
|
|
129021
129062
|
if (found)
|
|
129022
|
-
|
|
129063
|
+
log8.debug("new elements discovered after action");
|
|
129023
129064
|
return found;
|
|
129024
129065
|
}
|
|
129025
129066
|
function findUnexploredElements(elements, semanticActionsDone, seenKeys) {
|
|
@@ -129079,7 +129120,7 @@ function applyPlanIntelligence(plan, pageUrl, globalState, credentialId, page) {
|
|
|
129079
129120
|
const queued = markPageEmpty(globalState, credentialId, pageUrl, plan.revisitOn);
|
|
129080
129121
|
const intel = getIntelligence(globalState, credentialId);
|
|
129081
129122
|
if (queued) {
|
|
129082
|
-
|
|
129123
|
+
log8.info("page marked empty for mutation-triggered revisit", {
|
|
129083
129124
|
url: pageUrl,
|
|
129084
129125
|
credential: credentialId,
|
|
129085
129126
|
reason: plan.revisitReason,
|
|
@@ -129096,7 +129137,7 @@ function applyPlanIntelligence(plan, pageUrl, globalState, credentialId, page) {
|
|
|
129096
129137
|
});
|
|
129097
129138
|
}
|
|
129098
129139
|
} else {
|
|
129099
|
-
|
|
129140
|
+
log8.debug("empty-state revisit rejected by hard limit", {
|
|
129100
129141
|
url: pageUrl,
|
|
129101
129142
|
credential: credentialId,
|
|
129102
129143
|
count: intel.revisitCount.get(pageUrl) ?? 0
|
|
@@ -129121,7 +129162,7 @@ function reconcileQueue(queue, freshTasks) {
|
|
|
129121
129162
|
}
|
|
129122
129163
|
}
|
|
129123
129164
|
if (dropped > 0)
|
|
129124
|
-
|
|
129165
|
+
log8.debug("reconcileQueue: superseded stale tasks", { dropped });
|
|
129125
129166
|
queue.unshift(...freshTasks);
|
|
129126
129167
|
}
|
|
129127
129168
|
function trackResult(result, interceptor, globalState, credentialId, taskMutation, page) {
|
|
@@ -129134,7 +129175,7 @@ function trackResult(result, interceptor, globalState, credentialId, taskMutatio
|
|
|
129134
129175
|
if (hasSuccessfulMutation(result.httpRequests) && intel.emptyStateQueue.size > 0) {
|
|
129135
129176
|
const drained = drainOnMutation(globalState, credentialId, taskMutation);
|
|
129136
129177
|
if (drained.length > 0) {
|
|
129137
|
-
|
|
129178
|
+
log8.info("mutation triggered empty-state revisit drain", {
|
|
129138
129179
|
drained: drained.length,
|
|
129139
129180
|
urls: drained,
|
|
129140
129181
|
credential: credentialId,
|
|
@@ -129161,7 +129202,7 @@ async function closeOverlay(page) {
|
|
|
129161
129202
|
await page.keyboard.press("Escape").catch(() => {});
|
|
129162
129203
|
await page.waitForTimeout(OVERLAY_ESCAPE_WAIT);
|
|
129163
129204
|
if (await isViewportCenterBlocked(page)) {
|
|
129164
|
-
|
|
129205
|
+
log8.debug("Escape failed, clicking backdrop");
|
|
129165
129206
|
const dims = await page.evaluate(() => ({ w: window.innerWidth, h: window.innerHeight })).catch(() => null);
|
|
129166
129207
|
if (dims) {
|
|
129167
129208
|
await page.mouse.click(dims.w / 2, dims.h / 2).catch(() => {});
|
|
@@ -129191,13 +129232,38 @@ async function dismissCookieBanner(page) {
|
|
|
129191
129232
|
if (btn) {
|
|
129192
129233
|
await btn.click().catch(() => {});
|
|
129193
129234
|
await page.waitForTimeout(300);
|
|
129194
|
-
|
|
129235
|
+
log8.debug("cookie banner dismissed", { selector });
|
|
129195
129236
|
return;
|
|
129196
129237
|
}
|
|
129197
129238
|
}
|
|
129198
129239
|
}
|
|
129199
129240
|
async function collectDOMLinks(page, pageUrl, inScope) {
|
|
129200
|
-
const hrefs = await page.$$eval("a[href]", (els) =>
|
|
129241
|
+
const hrefs = await page.$$eval("a[href], [data-href], [onclick]", (els) => {
|
|
129242
|
+
const out = [];
|
|
129243
|
+
for (const el of els) {
|
|
129244
|
+
const a = el;
|
|
129245
|
+
if (a.tagName === "A" && a.href) {
|
|
129246
|
+
out.push(a.href);
|
|
129247
|
+
continue;
|
|
129248
|
+
}
|
|
129249
|
+
const dataHref = el.getAttribute("data-href");
|
|
129250
|
+
if (dataHref)
|
|
129251
|
+
out.push(dataHref);
|
|
129252
|
+
const onclick = el.getAttribute("onclick");
|
|
129253
|
+
if (onclick) {
|
|
129254
|
+
const m2 = onclick.match(/(?:window\.)?location(?:\.href)?\s*=\s*['"]([^'"]+)['"]/) || onclick.match(/location\.assign\s*\(\s*['"]([^'"]+)['"]/) || onclick.match(/window\.open\s*\(\s*['"]([^'"]+)['"]/);
|
|
129255
|
+
if (m2)
|
|
129256
|
+
out.push(m2[1]);
|
|
129257
|
+
}
|
|
129258
|
+
}
|
|
129259
|
+
return out.map((h2) => {
|
|
129260
|
+
try {
|
|
129261
|
+
return new URL(h2, location.href).href;
|
|
129262
|
+
} catch {
|
|
129263
|
+
return "";
|
|
129264
|
+
}
|
|
129265
|
+
});
|
|
129266
|
+
});
|
|
129201
129267
|
const results = [];
|
|
129202
129268
|
const seen = new Set;
|
|
129203
129269
|
for (const href of hrefs) {
|
|
@@ -129280,7 +129346,7 @@ async function runMultiCredential(config3, credentials) {
|
|
|
129280
129346
|
setPanelEnabled(panelOn);
|
|
129281
129347
|
const usageAcc = { inputTokens: 0, outputTokens: 0, cacheReadTokens: 0, cacheWriteTokens: 0 };
|
|
129282
129348
|
logScope(targetUrl, scopePatterns, !!config3.scope);
|
|
129283
|
-
|
|
129349
|
+
log8.info(`starting multi-credential crawl${dryRun ? " (DRY-RUN)" : ""}`, {
|
|
129284
129350
|
target: targetUrl,
|
|
129285
129351
|
credentials: credentials.map((c) => c.id),
|
|
129286
129352
|
maxPages,
|
|
@@ -129289,16 +129355,20 @@ async function runMultiCredential(config3, credentials) {
|
|
|
129289
129355
|
let model;
|
|
129290
129356
|
try {
|
|
129291
129357
|
model = await resolveModel(config3.model);
|
|
129292
|
-
|
|
129358
|
+
log8.info("AI navigation enabled");
|
|
129293
129359
|
} catch (err) {
|
|
129294
129360
|
throw new Error(`AI model required: ${String(err)}`);
|
|
129295
129361
|
}
|
|
129296
|
-
const browser = await connect({
|
|
129362
|
+
const browser = await connect({
|
|
129363
|
+
cdp: config3.cdp,
|
|
129364
|
+
headless: config3.headless ?? false,
|
|
129365
|
+
network: config3.network
|
|
129366
|
+
});
|
|
129297
129367
|
const health = createBrowserHealth();
|
|
129298
129368
|
browser.on("disconnected", () => {
|
|
129299
129369
|
health.dead = true;
|
|
129300
129370
|
health.reason = "browser process disconnected";
|
|
129301
|
-
|
|
129371
|
+
log8.error("browser disconnected — multi-credential crawl will terminate", lifecycleContext(browser));
|
|
129302
129372
|
});
|
|
129303
129373
|
let sessionId = config3.cyberstrike.sessionID ?? "";
|
|
129304
129374
|
if (!dryRun && !sessionId) {
|
|
@@ -129313,7 +129383,7 @@ async function runMultiCredential(config3, credentials) {
|
|
|
129313
129383
|
const captureQueues = new Map;
|
|
129314
129384
|
const lastAuthHeaders = new Map;
|
|
129315
129385
|
for (const [credIndex, cred] of credentials.entries()) {
|
|
129316
|
-
const browserContext = await browser.newContext(contextOptions(config3.headless ?? false));
|
|
129386
|
+
const browserContext = await browser.newContext(contextOptions(config3.headless ?? false, config3.network));
|
|
129317
129387
|
await browserContext.addInitScript(INIT_SCRIPT);
|
|
129318
129388
|
if (panelOn)
|
|
129319
129389
|
await browserContext.addInitScript(PANEL_INIT_SCRIPT);
|
|
@@ -129321,25 +129391,34 @@ async function runMultiCredential(config3, credentials) {
|
|
|
129321
129391
|
page.on("close", () => {
|
|
129322
129392
|
health.dead = true;
|
|
129323
129393
|
health.reason = `page closed (credential: ${cred.id})`;
|
|
129324
|
-
|
|
129394
|
+
log8.error("page closed — multi-credential crawl will terminate", {
|
|
129395
|
+
credential: cred.id,
|
|
129396
|
+
...lifecycleContext(browser, page)
|
|
129397
|
+
});
|
|
129325
129398
|
});
|
|
129326
129399
|
page.on("crash", () => {
|
|
129327
129400
|
health.dead = true;
|
|
129328
129401
|
health.reason = `page crashed (credential: ${cred.id})`;
|
|
129329
|
-
|
|
129402
|
+
log8.error("page crashed — multi-credential crawl will terminate", {
|
|
129403
|
+
credential: cred.id,
|
|
129404
|
+
...lifecycleContext(browser, page)
|
|
129405
|
+
});
|
|
129406
|
+
});
|
|
129407
|
+
browserContext.on("page", (p) => {
|
|
129408
|
+
log8.debug("new page/popup opened", { credential: cred.id, ...lifecycleContext(browser, p) });
|
|
129330
129409
|
});
|
|
129331
129410
|
attachDialogAutoAccept(page);
|
|
129332
129411
|
attachFileChooserAutoFill(page);
|
|
129333
129412
|
const mcInitNavErr = await page.goto(targetUrl, { waitUntil: "domcontentloaded", timeout: 30000 }).then(() => null).catch((e2) => e2);
|
|
129334
129413
|
if (mcInitNavErr) {
|
|
129335
|
-
|
|
129414
|
+
log8.warn("initial navigation failed", {
|
|
129336
129415
|
credential: cred.id,
|
|
129337
129416
|
url: targetUrl,
|
|
129338
129417
|
err: mcInitNavErr.message.split(`
|
|
129339
129418
|
`)[0]
|
|
129340
129419
|
});
|
|
129341
129420
|
if (!config3.headless && browser.isConnected()) {
|
|
129342
|
-
|
|
129421
|
+
log8.info("browser stays open — navigate manually or close the window to finish");
|
|
129343
129422
|
await waitForBrowserClose(browser, config3.signal);
|
|
129344
129423
|
} else {
|
|
129345
129424
|
await browser.close().catch(() => {});
|
|
@@ -129368,7 +129447,7 @@ async function runMultiCredential(config3, credentials) {
|
|
|
129368
129447
|
if (registeredId) {
|
|
129369
129448
|
credentialId = registeredId;
|
|
129370
129449
|
} else {
|
|
129371
|
-
|
|
129450
|
+
log8.warn("credential registration failed, using label as ID", { credential: cred.id });
|
|
129372
129451
|
}
|
|
129373
129452
|
}
|
|
129374
129453
|
const captureQueue = [];
|
|
@@ -129394,7 +129473,7 @@ async function runMultiCredential(config3, credentials) {
|
|
|
129394
129473
|
credentialId,
|
|
129395
129474
|
sessionId
|
|
129396
129475
|
});
|
|
129397
|
-
|
|
129476
|
+
log8.info("credential context ready", {
|
|
129398
129477
|
credential: cred.id,
|
|
129399
129478
|
credentialId,
|
|
129400
129479
|
sessionId: dryRun ? "dry-run" : sessionId
|
|
@@ -129419,7 +129498,7 @@ async function runMultiCredential(config3, credentials) {
|
|
|
129419
129498
|
let pagesExplored = 0;
|
|
129420
129499
|
while (pageQueue.length > 0 && pagesExplored < maxPages) {
|
|
129421
129500
|
if (isBrowserDead(health)) {
|
|
129422
|
-
|
|
129501
|
+
log8.error("browser died, terminating multi-credential crawl", {
|
|
129423
129502
|
reason: health.reason,
|
|
129424
129503
|
pagesExplored,
|
|
129425
129504
|
captured: globalState.capturedEndpoints.size
|
|
@@ -129427,12 +129506,12 @@ async function runMultiCredential(config3, credentials) {
|
|
|
129427
129506
|
break;
|
|
129428
129507
|
}
|
|
129429
129508
|
if (config3.signal?.aborted) {
|
|
129430
|
-
|
|
129509
|
+
log8.info("multi-credential crawl cancelled by signal", { pagesExplored, queued: pageQueue.length });
|
|
129431
129510
|
break;
|
|
129432
129511
|
}
|
|
129433
129512
|
const entry = pageQueue.shift();
|
|
129434
129513
|
pagesExplored++;
|
|
129435
|
-
|
|
129514
|
+
log8.info("processing URL", {
|
|
129436
129515
|
page: `${pagesExplored}/${maxPages}`,
|
|
129437
129516
|
url: entry.url,
|
|
129438
129517
|
contexts: entry.contexts,
|
|
@@ -129445,11 +129524,11 @@ async function runMultiCredential(config3, credentials) {
|
|
|
129445
129524
|
await stabilizeAfterGoto(ctx.page);
|
|
129446
129525
|
return { ctx, success: true, redirected: normalizeUrl(ctx.page.url()) !== normalizeUrl(entry.url) };
|
|
129447
129526
|
} catch (err) {
|
|
129448
|
-
|
|
129527
|
+
log8.warn("navigation failed", { credential: ctx.id, url: entry.url, err: String(err) });
|
|
129449
129528
|
return { ctx, success: false, redirected: false };
|
|
129450
129529
|
}
|
|
129451
129530
|
}));
|
|
129452
|
-
const ACCESS_DENIED_PATTERNS = /\/(login|signin|sign-in|unauthorized|forbidden|access-denied|
|
|
129531
|
+
const ACCESS_DENIED_PATTERNS = /\/(login|signin|sign-in|unauthorized|forbidden|access-denied|authenticate)/i;
|
|
129453
129532
|
const visitableContexts = [];
|
|
129454
129533
|
for (const r2 of navigateResults) {
|
|
129455
129534
|
if (!r2.success)
|
|
@@ -129457,7 +129536,7 @@ async function runMultiCredential(config3, credentials) {
|
|
|
129457
129536
|
if (r2.redirected) {
|
|
129458
129537
|
const actualPath = new URL(r2.ctx.page.url()).pathname;
|
|
129459
129538
|
if (ACCESS_DENIED_PATTERNS.test(actualPath)) {
|
|
129460
|
-
|
|
129539
|
+
log8.info("access denied redirect, skipping explore", {
|
|
129461
129540
|
credential: r2.ctx.id,
|
|
129462
129541
|
intended: entry.url,
|
|
129463
129542
|
redirectedTo: r2.ctx.page.url()
|
|
@@ -129465,7 +129544,7 @@ async function runMultiCredential(config3, credentials) {
|
|
|
129465
129544
|
continue;
|
|
129466
129545
|
}
|
|
129467
129546
|
const landed = normalizeUrl(r2.ctx.page.url());
|
|
129468
|
-
|
|
129547
|
+
log8.info("redirect to different page, enqueueing landed URL", {
|
|
129469
129548
|
credential: r2.ctx.id,
|
|
129470
129549
|
intended: entry.url,
|
|
129471
129550
|
landed
|
|
@@ -129476,7 +129555,7 @@ async function runMultiCredential(config3, credentials) {
|
|
|
129476
129555
|
visitableContexts.push(r2.ctx);
|
|
129477
129556
|
}
|
|
129478
129557
|
if (visitableContexts.length === 0) {
|
|
129479
|
-
|
|
129558
|
+
log8.info("no contexts could access page, skipping", { url: entry.url });
|
|
129480
129559
|
continue;
|
|
129481
129560
|
}
|
|
129482
129561
|
const normalizedEntryUrl = normalizeUrl(entry.url);
|
|
@@ -129500,7 +129579,7 @@ async function runMultiCredential(config3, credentials) {
|
|
|
129500
129579
|
const fingerprints = [...fingerprintsByContext.values()];
|
|
129501
129580
|
const allSameFingerprint = fingerprints.length > 0 && fingerprints.every((fp) => fp === fingerprints[0]);
|
|
129502
129581
|
if (dryRun) {
|
|
129503
|
-
|
|
129582
|
+
log8.info("page-diff", {
|
|
129504
129583
|
url: entry.url,
|
|
129505
129584
|
fingerprintMatch: allSameFingerprint,
|
|
129506
129585
|
elementCount: availability.size,
|
|
@@ -129508,14 +129587,14 @@ async function runMultiCredential(config3, credentials) {
|
|
|
129508
129587
|
});
|
|
129509
129588
|
}
|
|
129510
129589
|
if (allSameFingerprint && visitableContexts.length > 1) {
|
|
129511
|
-
|
|
129590
|
+
log8.info("same fingerprint across contexts", {
|
|
129512
129591
|
contexts: visitableContexts.map((c) => c.id)
|
|
129513
129592
|
});
|
|
129514
129593
|
}
|
|
129515
129594
|
for (const ctx of visitableContexts) {
|
|
129516
129595
|
if (isBrowserDead(health))
|
|
129517
129596
|
break;
|
|
129518
|
-
|
|
129597
|
+
log8.info("exploring", { credential: ctx.id, url: entry.url });
|
|
129519
129598
|
const exploreInline = async (p, url3, depth) => {
|
|
129520
129599
|
if (isBrowserDead(health))
|
|
129521
129600
|
return;
|
|
@@ -129530,14 +129609,14 @@ async function runMultiCredential(config3, credentials) {
|
|
|
129530
129609
|
}
|
|
129531
129610
|
} catch (err) {
|
|
129532
129611
|
if (isBrowserDead(health)) {
|
|
129533
|
-
|
|
129612
|
+
log8.error("exploration aborted — browser died mid-page", {
|
|
129534
129613
|
credential: ctx.id,
|
|
129535
129614
|
url: entry.url,
|
|
129536
129615
|
reason: health.reason
|
|
129537
129616
|
});
|
|
129538
129617
|
break;
|
|
129539
129618
|
}
|
|
129540
|
-
|
|
129619
|
+
log8.warn("exploration error", { credential: ctx.id, url: entry.url, err: String(err) });
|
|
129541
129620
|
}
|
|
129542
129621
|
}
|
|
129543
129622
|
for (const ctx of visitableContexts) {
|
|
@@ -129551,7 +129630,7 @@ async function runMultiCredential(config3, credentials) {
|
|
|
129551
129630
|
await visitableContexts[0]?.page.waitForTimeout(300);
|
|
129552
129631
|
}
|
|
129553
129632
|
const browserDied = isBrowserDead(health);
|
|
129554
|
-
|
|
129633
|
+
log8.info(browserDied ? "browser died, draining captured requests" : "multi-credential exploration complete, draining remaining requests", {
|
|
129555
129634
|
pagesExplored,
|
|
129556
129635
|
captured: globalState.capturedEndpoints.size,
|
|
129557
129636
|
...browserDied ? { reason: health.reason } : {}
|
|
@@ -129559,7 +129638,7 @@ async function runMultiCredential(config3, credentials) {
|
|
|
129559
129638
|
if (!browserDied)
|
|
129560
129639
|
await contexts[0]?.page.waitForTimeout(2000).catch(() => {});
|
|
129561
129640
|
await drainPageCaptures(captureQueues, captureHandlers, new Map, null, null);
|
|
129562
|
-
|
|
129641
|
+
log8.info("multi-credential crawl done", {
|
|
129563
129642
|
pagesExplored,
|
|
129564
129643
|
credentials: contexts.map((c) => c.id),
|
|
129565
129644
|
totalSteps: globalState.totalSteps,
|
|
@@ -129583,7 +129662,7 @@ async function runMultiCredential(config3, credentials) {
|
|
|
129583
129662
|
await contexts[0]?.page.waitForTimeout(600).catch(() => {});
|
|
129584
129663
|
}
|
|
129585
129664
|
if (!config3.headless && browser.isConnected()) {
|
|
129586
|
-
|
|
129665
|
+
log8.info("crawl complete — browser stays open, close the window to finish");
|
|
129587
129666
|
await waitForBrowserClose(browser, config3.signal);
|
|
129588
129667
|
} else if (!browserDied) {
|
|
129589
129668
|
await browser.close().catch(() => {});
|
|
@@ -129613,7 +129692,7 @@ async function run(config3) {
|
|
|
129613
129692
|
const panelOn = config3.panel ?? true;
|
|
129614
129693
|
setPanelEnabled(panelOn);
|
|
129615
129694
|
logScope(targetUrl, scopePatterns, !!config3.scope);
|
|
129616
|
-
|
|
129695
|
+
log8.info(`starting browser agent v2${dryRun ? " (DRY-RUN)" : ""}`, {
|
|
129617
129696
|
target: targetUrl,
|
|
129618
129697
|
maxPages,
|
|
129619
129698
|
server: dryRun ? undefined : serverUrl,
|
|
@@ -129622,7 +129701,7 @@ async function run(config3) {
|
|
|
129622
129701
|
let model = null;
|
|
129623
129702
|
try {
|
|
129624
129703
|
model = await resolveModel(config3.model);
|
|
129625
|
-
|
|
129704
|
+
log8.info("AI navigation enabled");
|
|
129626
129705
|
} catch (err) {
|
|
129627
129706
|
throw new Error(`AI model required for v2 architecture: ${String(err)}`);
|
|
129628
129707
|
}
|
|
@@ -129634,9 +129713,13 @@ async function run(config3) {
|
|
|
129634
129713
|
}
|
|
129635
129714
|
sessionID = created;
|
|
129636
129715
|
}
|
|
129637
|
-
const browser = await connect({
|
|
129716
|
+
const browser = await connect({
|
|
129717
|
+
cdp: config3.cdp,
|
|
129718
|
+
headless: config3.headless ?? false,
|
|
129719
|
+
network: config3.network
|
|
129720
|
+
});
|
|
129638
129721
|
const health = createBrowserHealth();
|
|
129639
|
-
const context = await browser.newContext(contextOptions(config3.headless ?? false));
|
|
129722
|
+
const context = await browser.newContext(contextOptions(config3.headless ?? false, config3.network));
|
|
129640
129723
|
await context.addInitScript(INIT_SCRIPT);
|
|
129641
129724
|
if (panelOn)
|
|
129642
129725
|
await context.addInitScript(PANEL_INIT_SCRIPT);
|
|
@@ -129669,10 +129752,10 @@ async function run(config3) {
|
|
|
129669
129752
|
try {
|
|
129670
129753
|
const initNavErr = await page.goto(targetUrl, { waitUntil: "domcontentloaded", timeout: 30000 }).then(() => null).catch((e2) => e2);
|
|
129671
129754
|
if (initNavErr) {
|
|
129672
|
-
|
|
129755
|
+
log8.warn("initial navigation failed", { url: targetUrl, err: initNavErr.message.split(`
|
|
129673
129756
|
`)[0] });
|
|
129674
129757
|
if (!config3.headless && browser.isConnected()) {
|
|
129675
|
-
|
|
129758
|
+
log8.info("browser stays open — navigate manually or close the window to finish");
|
|
129676
129759
|
await waitForBrowserClose(browser, config3.signal);
|
|
129677
129760
|
} else {
|
|
129678
129761
|
await browser.close().catch(() => {});
|
|
@@ -129682,7 +129765,7 @@ async function run(config3) {
|
|
|
129682
129765
|
capturedEndpoints: 0,
|
|
129683
129766
|
pagesExplored: 0,
|
|
129684
129767
|
totalSteps: 0,
|
|
129685
|
-
errors: [`Initial navigation failed: ${initNavErr.message}`],
|
|
129768
|
+
errors: [`Initial navigation failed: ${initNavErr.message}${certHint(initNavErr.message, config3)}`],
|
|
129686
129769
|
usage: usageAcc
|
|
129687
129770
|
};
|
|
129688
129771
|
}
|
|
@@ -129710,7 +129793,7 @@ async function run(config3) {
|
|
|
129710
129793
|
}
|
|
129711
129794
|
const globalState = createGlobalState({ outOfScope: config3.outOfScope });
|
|
129712
129795
|
if (config3.outOfScope?.length) {
|
|
129713
|
-
|
|
129796
|
+
log8.info("out-of-scope labels", { count: config3.outOfScope.length, labels: config3.outOfScope });
|
|
129714
129797
|
}
|
|
129715
129798
|
if (isAuthenticated) {
|
|
129716
129799
|
globalState.authPhase = "authenticated";
|
|
@@ -129726,7 +129809,7 @@ async function run(config3) {
|
|
|
129726
129809
|
try {
|
|
129727
129810
|
const pathname = new URL(req.url()).pathname;
|
|
129728
129811
|
if (LOGIN_SUCCESS_PATTERN.test(`${method} ${pathname} [${status}]`)) {
|
|
129729
|
-
|
|
129812
|
+
log8.info("login success detected via HTTP response");
|
|
129730
129813
|
triggerReDiscovery(globalState);
|
|
129731
129814
|
}
|
|
129732
129815
|
} catch {}
|
|
@@ -129734,7 +129817,7 @@ async function run(config3) {
|
|
|
129734
129817
|
let pagesExplored = 0;
|
|
129735
129818
|
while (globalState.pageQueue.length > 0 && pagesExplored < maxPages) {
|
|
129736
129819
|
if (isBrowserDead(health)) {
|
|
129737
|
-
|
|
129820
|
+
log8.error("browser died, terminating crawl", {
|
|
129738
129821
|
reason: health.reason,
|
|
129739
129822
|
pagesExplored,
|
|
129740
129823
|
captured: globalState.capturedEndpoints.size
|
|
@@ -129742,7 +129825,7 @@ async function run(config3) {
|
|
|
129742
129825
|
break;
|
|
129743
129826
|
}
|
|
129744
129827
|
if (config3.signal?.aborted) {
|
|
129745
|
-
|
|
129828
|
+
log8.info("crawl cancelled by signal", { pagesExplored, queued: globalState.pageQueue.length });
|
|
129746
129829
|
break;
|
|
129747
129830
|
}
|
|
129748
129831
|
const nextUrl = globalState.pageQueue.shift();
|
|
@@ -129750,28 +129833,33 @@ async function run(config3) {
|
|
|
129750
129833
|
if (page.url() !== nextUrl) {
|
|
129751
129834
|
const navErr = await page.goto(nextUrl, { waitUntil: "domcontentloaded", timeout: 15000 }).then(() => null).catch((e2) => e2);
|
|
129752
129835
|
if (navErr) {
|
|
129753
|
-
|
|
129754
|
-
`)[0]
|
|
129836
|
+
const head = navErr.message.split(`
|
|
129837
|
+
`)[0];
|
|
129838
|
+
const hint = certHint(navErr.message, config3);
|
|
129839
|
+
if (hint)
|
|
129840
|
+
log8.error("navigation failed", { url: nextUrl, err: head + hint });
|
|
129841
|
+
else
|
|
129842
|
+
log8.warn("navigation failed", { url: nextUrl, err: head });
|
|
129755
129843
|
continue;
|
|
129756
129844
|
}
|
|
129757
129845
|
await stabilizeAfterGoto(page);
|
|
129758
129846
|
}
|
|
129759
129847
|
if (!isInScope(page.url(), inScope)) {
|
|
129760
|
-
|
|
129848
|
+
log8.info("off-host redirect, skipping", { url: page.url() });
|
|
129761
129849
|
await page.goto(targetUrl, { waitUntil: "domcontentloaded", timeout: 15000 }).catch(() => {});
|
|
129762
129850
|
continue;
|
|
129763
129851
|
}
|
|
129764
129852
|
const currentUrl = normalizeUrl(page.url());
|
|
129765
129853
|
if (currentUrl !== normalizeUrl(nextUrl) && globalState.visitedPages.has(currentUrl)) {
|
|
129766
|
-
|
|
129854
|
+
log8.info("redirect to already-explored page, skipping", {
|
|
129767
129855
|
requested: normalizeUrl(nextUrl),
|
|
129768
129856
|
landed: currentUrl
|
|
129769
129857
|
});
|
|
129770
129858
|
continue;
|
|
129771
129859
|
}
|
|
129772
129860
|
globalState.visitedPages.add(currentUrl);
|
|
129773
|
-
|
|
129774
|
-
|
|
129861
|
+
log8.info("exploring page", { page: `${pagesExplored}/${maxPages}`, url: currentUrl });
|
|
129862
|
+
log8.debug("state", {
|
|
129775
129863
|
phase: globalState.authPhase,
|
|
129776
129864
|
queueSize: globalState.pageQueue.length,
|
|
129777
129865
|
queue: globalState.pageQueue.slice(0, 5).map((u) => {
|
|
@@ -129798,18 +129886,18 @@ async function run(config3) {
|
|
|
129798
129886
|
await expandDisclosures(page);
|
|
129799
129887
|
const intel = getIntelligence(globalState, SINGLE_CRED);
|
|
129800
129888
|
const oldFingerprint = intel.pageFingerprints.get(currentUrl);
|
|
129801
|
-
|
|
129889
|
+
log8.debug("fingerprint check", { url: currentUrl, hasOld: !!oldFingerprint });
|
|
129802
129890
|
if (oldFingerprint !== undefined) {
|
|
129803
129891
|
const currentElements = await collectElements(page);
|
|
129804
129892
|
const newFingerprint = generateFingerprint(currentElements);
|
|
129805
|
-
|
|
129893
|
+
log8.debug("fingerprint compare", {
|
|
129806
129894
|
url: currentUrl,
|
|
129807
129895
|
match: newFingerprint === oldFingerprint,
|
|
129808
129896
|
oldFp: oldFingerprint.slice(0, 80),
|
|
129809
129897
|
newFp: newFingerprint.slice(0, 80)
|
|
129810
129898
|
});
|
|
129811
129899
|
if (newFingerprint === oldFingerprint) {
|
|
129812
|
-
|
|
129900
|
+
log8.info("page unchanged after auth, skipping exploration", { url: currentUrl });
|
|
129813
129901
|
const domLinks = await collectDOMLinks(page, currentUrl, inScope);
|
|
129814
129902
|
for (const url3 of domLinks) {
|
|
129815
129903
|
enqueueUrl(url3, globalState, inScope);
|
|
@@ -129818,7 +129906,7 @@ async function run(config3) {
|
|
|
129818
129906
|
await page.waitForTimeout(300);
|
|
129819
129907
|
continue;
|
|
129820
129908
|
}
|
|
129821
|
-
|
|
129909
|
+
log8.info("page changed after auth, re-exploring", { url: currentUrl });
|
|
129822
129910
|
}
|
|
129823
129911
|
const exploreInline = async (p, url3, depth) => {
|
|
129824
129912
|
if (isBrowserDead(health))
|
|
@@ -129832,10 +129920,10 @@ async function run(config3) {
|
|
|
129832
129920
|
discovered = await explorePageWithAI(page, currentUrl, interceptor, model, globalState, inScope, SINGLE_CRED, maxPages, usageAcc, { explore: exploreInline, depth: 0 });
|
|
129833
129921
|
} catch (err) {
|
|
129834
129922
|
if (isBrowserDead(health)) {
|
|
129835
|
-
|
|
129923
|
+
log8.error("exploration aborted — browser died mid-page", { url: currentUrl, reason: health.reason });
|
|
129836
129924
|
break;
|
|
129837
129925
|
}
|
|
129838
|
-
|
|
129926
|
+
log8.warn("exploration error", { url: currentUrl, err: String(err) });
|
|
129839
129927
|
}
|
|
129840
129928
|
let newEnqueued = 0;
|
|
129841
129929
|
for (const url3 of discovered) {
|
|
@@ -129843,7 +129931,7 @@ async function run(config3) {
|
|
|
129843
129931
|
newEnqueued++;
|
|
129844
129932
|
}
|
|
129845
129933
|
if (discovered.length > 0) {
|
|
129846
|
-
|
|
129934
|
+
log8.debug("discovered links", {
|
|
129847
129935
|
found: discovered.length,
|
|
129848
129936
|
enqueued: newEnqueued,
|
|
129849
129937
|
queueSize: globalState.pageQueue.length
|
|
@@ -129852,7 +129940,7 @@ async function run(config3) {
|
|
|
129852
129940
|
flushReDiscovery(globalState);
|
|
129853
129941
|
await page.waitForTimeout(300);
|
|
129854
129942
|
if (globalState.pageQueue.length === 0 && globalState.deferredAuthPages.length > 0) {
|
|
129855
|
-
|
|
129943
|
+
log8.debug("queue empty, processing auth phase", {
|
|
129856
129944
|
phase: globalState.authPhase,
|
|
129857
129945
|
deferred: globalState.deferredAuthPages.map((d) => d.type)
|
|
129858
129946
|
});
|
|
@@ -129860,7 +129948,7 @@ async function run(config3) {
|
|
|
129860
129948
|
}
|
|
129861
129949
|
}
|
|
129862
129950
|
const browserDied = isBrowserDead(health);
|
|
129863
|
-
|
|
129951
|
+
log8.info(browserDied ? "browser died, draining captured requests" : "exploration complete, draining remaining requests", {
|
|
129864
129952
|
pagesExplored,
|
|
129865
129953
|
captured: globalState.capturedEndpoints.size,
|
|
129866
129954
|
...browserDied ? { reason: health.reason } : {}
|
|
@@ -129871,7 +129959,7 @@ async function run(config3) {
|
|
|
129871
129959
|
const captured = captureQueue.shift();
|
|
129872
129960
|
await handleCapture(captured);
|
|
129873
129961
|
}
|
|
129874
|
-
|
|
129962
|
+
log8.info("done", {
|
|
129875
129963
|
pagesExplored,
|
|
129876
129964
|
totalSteps: globalState.totalSteps,
|
|
129877
129965
|
capturedEndpoints: globalState.capturedEndpoints.size,
|
|
@@ -129895,7 +129983,7 @@ async function run(config3) {
|
|
|
129895
129983
|
}
|
|
129896
129984
|
}
|
|
129897
129985
|
if (!config3.headless && browser.isConnected()) {
|
|
129898
|
-
|
|
129986
|
+
log8.info("crawl complete — browser stays open, close the window to finish");
|
|
129899
129987
|
const postCrawlDrain = setInterval(async () => {
|
|
129900
129988
|
while (captureQueue.length > 0) {
|
|
129901
129989
|
const captured = captureQueue.shift();
|
|
@@ -129921,7 +130009,7 @@ async function run(config3) {
|
|
|
129921
130009
|
}
|
|
129922
130010
|
|
|
129923
130011
|
// ../hackbrowser/src/api.ts
|
|
129924
|
-
var
|
|
130012
|
+
var log9 = Log.create({ service: "hackbrowser:api" });
|
|
129925
130013
|
function preflightCheck() {
|
|
129926
130014
|
if (findSystemChrome())
|
|
129927
130015
|
return;
|
|
@@ -129963,6 +130051,7 @@ function toAgentConfig(opts) {
|
|
|
129963
130051
|
dryRun: opts.dryRun,
|
|
129964
130052
|
panel: opts.panel,
|
|
129965
130053
|
model: opts.model,
|
|
130054
|
+
network: opts.network,
|
|
129966
130055
|
cdp: opts.cdp,
|
|
129967
130056
|
signal: opts.signal
|
|
129968
130057
|
};
|
|
@@ -129979,7 +130068,7 @@ async function runCrawl(opts) {
|
|
|
129979
130068
|
setEventSink(opts.eventSink);
|
|
129980
130069
|
const config3 = toAgentConfig(opts);
|
|
129981
130070
|
try {
|
|
129982
|
-
|
|
130071
|
+
log9.info("runCrawl starting", {
|
|
129983
130072
|
url: opts.url,
|
|
129984
130073
|
multiCred: !!opts.multiCredentials?.length,
|
|
129985
130074
|
headless: opts.headless,
|
|
@@ -129987,7 +130076,7 @@ async function runCrawl(opts) {
|
|
|
129987
130076
|
eventSink: !!opts.eventSink
|
|
129988
130077
|
});
|
|
129989
130078
|
const result = await run(config3);
|
|
129990
|
-
|
|
130079
|
+
log9.info("runCrawl done", {
|
|
129991
130080
|
sessionID: result.sessionID,
|
|
129992
130081
|
capturedEndpoints: result.capturedEndpoints,
|
|
129993
130082
|
pagesExplored: result.pagesExplored
|
|
@@ -129995,7 +130084,7 @@ async function runCrawl(opts) {
|
|
|
129995
130084
|
return result;
|
|
129996
130085
|
} catch (err) {
|
|
129997
130086
|
const message = err instanceof Error ? err.message : String(err);
|
|
129998
|
-
|
|
130087
|
+
log9.error("runCrawl failed", { err: message });
|
|
129999
130088
|
return {
|
|
130000
130089
|
sessionID: opts.sessionID ?? "",
|
|
130001
130090
|
capturedEndpoints: 0,
|
|
@@ -130014,6 +130103,7 @@ async function runCrawl(opts) {
|
|
|
130014
130103
|
|
|
130015
130104
|
// src/hackbrowser-subprocess/hackbrowser-worker.ts
|
|
130016
130105
|
import readline2 from "readline";
|
|
130106
|
+
import tls from "node:tls";
|
|
130017
130107
|
function send(msg) {
|
|
130018
130108
|
process.stdout.write(JSON.stringify(msg) + `
|
|
130019
130109
|
`);
|
|
@@ -130058,9 +130148,35 @@ function applyAnthropicBearerBody(body, opts) {
|
|
|
130058
130148
|
return body;
|
|
130059
130149
|
}
|
|
130060
130150
|
}
|
|
130151
|
+
function applyTransport(desc, init) {
|
|
130152
|
+
const n = desc.network;
|
|
130153
|
+
if (!n)
|
|
130154
|
+
return init;
|
|
130155
|
+
const out = { ...init ?? {} };
|
|
130156
|
+
if (n.proxy)
|
|
130157
|
+
out.proxy = n.proxy;
|
|
130158
|
+
const tls2 = {};
|
|
130159
|
+
if (n.rejectUnauthorized !== undefined)
|
|
130160
|
+
tls2.rejectUnauthorized = n.rejectUnauthorized;
|
|
130161
|
+
if (n.ca)
|
|
130162
|
+
tls2.ca = n.ca;
|
|
130163
|
+
if (n.cert)
|
|
130164
|
+
tls2.cert = n.cert;
|
|
130165
|
+
if (n.key)
|
|
130166
|
+
tls2.key = n.key;
|
|
130167
|
+
if (n.passphrase)
|
|
130168
|
+
tls2.passphrase = n.passphrase;
|
|
130169
|
+
if (Object.keys(tls2).length > 0)
|
|
130170
|
+
out.tls = tls2;
|
|
130171
|
+
return out;
|
|
130172
|
+
}
|
|
130173
|
+
function shouldUseCopilotResponsesApi(modelID) {
|
|
130174
|
+
const match = /^gpt-(\d+)/.exec(modelID);
|
|
130175
|
+
return match !== null && Number(match[1]) >= 5 && !modelID.startsWith("gpt-5-mini");
|
|
130176
|
+
}
|
|
130061
130177
|
function createModelFromDescriptor(desc) {
|
|
130062
130178
|
const stripSampling = desc.supportsTemperature === false;
|
|
130063
|
-
const samplingFetch = stripSampling ? (input, init) => fetch(input, init ? { ...init, body: stripSamplingParams(init.body) } : init) : undefined;
|
|
130179
|
+
const samplingFetch = stripSampling || desc.network ? (input, init) => fetch(input, applyTransport(desc, init && stripSampling ? { ...init, body: stripSamplingParams(init.body) } : init)) : undefined;
|
|
130064
130180
|
if (desc.npm.includes("anthropic")) {
|
|
130065
130181
|
if (desc.authToken) {
|
|
130066
130182
|
const token = desc.authToken;
|
|
@@ -130078,7 +130194,7 @@ function createModelFromDescriptor(desc) {
|
|
|
130078
130194
|
userId: desc.anthropicUserId,
|
|
130079
130195
|
systemPrefix: desc.anthropicSystemPrefix
|
|
130080
130196
|
});
|
|
130081
|
-
return fetch(url3, { ...init, headers, body });
|
|
130197
|
+
return fetch(url3, applyTransport(desc, { ...init, headers, body }));
|
|
130082
130198
|
}
|
|
130083
130199
|
};
|
|
130084
130200
|
if (desc.baseURL)
|
|
@@ -130114,11 +130230,11 @@ function createModelFromDescriptor(desc) {
|
|
|
130114
130230
|
const version3 = typeof CYBERSTRIKE_VERSION === "string" ? CYBERSTRIKE_VERSION : "local";
|
|
130115
130231
|
headers.set("User-Agent", `cyberstrike/${version3}`);
|
|
130116
130232
|
headers.set("Openai-Intent", "conversation-edits");
|
|
130117
|
-
return fetch(url3, {
|
|
130233
|
+
return fetch(url3, applyTransport(desc, {
|
|
130118
130234
|
...init,
|
|
130119
130235
|
headers,
|
|
130120
130236
|
body: stripSampling ? stripSamplingParams(init?.body) : init?.body
|
|
130121
|
-
});
|
|
130237
|
+
}));
|
|
130122
130238
|
}
|
|
130123
130239
|
};
|
|
130124
130240
|
if (desc.baseURL)
|
|
@@ -130126,7 +130242,17 @@ function createModelFromDescriptor(desc) {
|
|
|
130126
130242
|
if (desc.headers)
|
|
130127
130243
|
opts2.headers = desc.headers;
|
|
130128
130244
|
const sdk2 = factory10(opts2);
|
|
130129
|
-
|
|
130245
|
+
const api2 = sdk2.responses === undefined && sdk2.chat === undefined ? "languageModel" : shouldUseCopilotResponsesApi(desc.modelApiId) ? "responses" : "chat";
|
|
130246
|
+
send({
|
|
130247
|
+
type: "log",
|
|
130248
|
+
level: "info",
|
|
130249
|
+
service: "hackbrowser:worker",
|
|
130250
|
+
message: `copilot model "${desc.modelApiId}" -> ${api2} API`,
|
|
130251
|
+
extra: { model: desc.modelApiId, api: api2 }
|
|
130252
|
+
});
|
|
130253
|
+
if (api2 === "languageModel")
|
|
130254
|
+
return sdk2.languageModel(desc.modelApiId);
|
|
130255
|
+
return api2 === "responses" ? sdk2.responses(desc.modelApiId) : sdk2.chat(desc.modelApiId);
|
|
130130
130256
|
}
|
|
130131
130257
|
const factory7 = BUNDLED_PROVIDERS[desc.npm];
|
|
130132
130258
|
if (!factory7) {
|
|
@@ -130178,13 +130304,26 @@ function buildCrawlOptions(opts, signal) {
|
|
|
130178
130304
|
cyberstrikeUrl: opts.cyberstrikeUrl,
|
|
130179
130305
|
model,
|
|
130180
130306
|
cdp: opts.cdp,
|
|
130307
|
+
network: opts.network,
|
|
130181
130308
|
logSink,
|
|
130182
130309
|
eventSink: eventSink2,
|
|
130183
130310
|
signal,
|
|
130184
130311
|
...credentialFields
|
|
130185
130312
|
};
|
|
130186
130313
|
}
|
|
130314
|
+
function trustSystemCertificates() {
|
|
130315
|
+
try {
|
|
130316
|
+
const t2 = tls;
|
|
130317
|
+
if (typeof t2.getCACertificates !== "function" || typeof t2.setDefaultCACertificates !== "function")
|
|
130318
|
+
return;
|
|
130319
|
+
const system = t2.getCACertificates("system");
|
|
130320
|
+
if (system.length === 0)
|
|
130321
|
+
return;
|
|
130322
|
+
t2.setDefaultCACertificates([...t2.getCACertificates("default"), ...system]);
|
|
130323
|
+
} catch {}
|
|
130324
|
+
}
|
|
130187
130325
|
async function main() {
|
|
130326
|
+
trustSystemCertificates();
|
|
130188
130327
|
const controller = new AbortController;
|
|
130189
130328
|
const rl = readline2.createInterface({ input: process.stdin, terminal: false });
|
|
130190
130329
|
let started = false;
|
|
@@ -130239,3 +130378,6 @@ main().catch((err) => {
|
|
|
130239
130378
|
`);
|
|
130240
130379
|
process.exitCode = 1;
|
|
130241
130380
|
});
|
|
130381
|
+
export {
|
|
130382
|
+
applyTransport
|
|
130383
|
+
};
|