@skrillex1224/playwright-toolkit 2.1.166 → 2.1.168
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser.js +1 -1
- package/dist/browser.js.map +1 -1
- package/dist/index.cjs +450 -794
- package/dist/index.cjs.map +4 -4
- package/dist/index.js +447 -792
- package/dist/index.js.map +4 -4
- package/dist/proxy-meter.js +549 -0
- package/package.json +1 -1
package/dist/index.cjs
CHANGED
|
@@ -120,10 +120,10 @@ var createActorInfo = (info) => {
|
|
|
120
120
|
xurl
|
|
121
121
|
};
|
|
122
122
|
};
|
|
123
|
-
const buildLandingUrl = ({ protocol: protocol2, domain: domain2, path:
|
|
123
|
+
const buildLandingUrl = ({ protocol: protocol2, domain: domain2, path: path3 }) => {
|
|
124
124
|
const safeProtocol = String(protocol2).trim();
|
|
125
125
|
const safeDomain = normalizeDomain(domain2);
|
|
126
|
-
const safePath = normalizePath(
|
|
126
|
+
const safePath = normalizePath(path3);
|
|
127
127
|
return `${safeProtocol}://${safeDomain}${safePath}`;
|
|
128
128
|
};
|
|
129
129
|
const buildIcon = ({ key }) => {
|
|
@@ -131,13 +131,13 @@ var createActorInfo = (info) => {
|
|
|
131
131
|
};
|
|
132
132
|
const protocol = info.protocol || "https";
|
|
133
133
|
const domain = normalizeDomain(info.domain);
|
|
134
|
-
const
|
|
134
|
+
const path2 = normalizePath(info.path);
|
|
135
135
|
const share = normalizeShare2(info.share);
|
|
136
136
|
return {
|
|
137
137
|
...info,
|
|
138
138
|
protocol,
|
|
139
139
|
domain,
|
|
140
|
-
path,
|
|
140
|
+
path: path2,
|
|
141
141
|
share,
|
|
142
142
|
get icon() {
|
|
143
143
|
if (info.icon) return info.icon;
|
|
@@ -151,7 +151,7 @@ var createActorInfo = (info) => {
|
|
|
151
151
|
var ActorInfo = {
|
|
152
152
|
qbot: createActorInfo({
|
|
153
153
|
key: "qbot",
|
|
154
|
-
name: "\u641C
|
|
154
|
+
name: "QQ\u6D4F\u89C8\u5668AI\u641C",
|
|
155
155
|
domain: "sogou.com",
|
|
156
156
|
path: "/web",
|
|
157
157
|
share: {
|
|
@@ -573,20 +573,362 @@ var CrawlerError = class _CrawlerError extends Error {
|
|
|
573
573
|
// src/apify-kit.js
|
|
574
574
|
var import_serialize_error2 = require("serialize-error");
|
|
575
575
|
|
|
576
|
-
// src/
|
|
577
|
-
var
|
|
578
|
-
var
|
|
579
|
-
|
|
576
|
+
// src/proxy-meter-runtime.js
|
|
577
|
+
var import_child_process = require("child_process");
|
|
578
|
+
var import_fs = require("fs");
|
|
579
|
+
var import_os = require("os");
|
|
580
|
+
var import_path = __toESM(require("path"), 1);
|
|
581
|
+
var import_url = require("url");
|
|
582
|
+
var import_meta = {};
|
|
583
|
+
var logger2 = createInternalLogger("ProxyMeter");
|
|
584
|
+
var MAX_TOP_DOMAINS = 20;
|
|
585
|
+
var FLUSH_INTERVAL_MS = 2e3;
|
|
586
|
+
var DEFAULT_DEBUG_MAX_EVENTS = 400;
|
|
587
|
+
var runtime = null;
|
|
588
|
+
var cleanupInstalled = false;
|
|
589
|
+
var observedDomainResourceTypes = /* @__PURE__ */ new Map();
|
|
590
|
+
var toSafeInt = (value) => {
|
|
591
|
+
const num = Number(value);
|
|
592
|
+
if (!Number.isFinite(num) || num <= 0) return 0;
|
|
593
|
+
return Math.round(num);
|
|
594
|
+
};
|
|
595
|
+
var toSafeFloat = (value) => {
|
|
596
|
+
const num = Number(value);
|
|
597
|
+
if (!Number.isFinite(num)) return 0;
|
|
598
|
+
return num;
|
|
599
|
+
};
|
|
600
|
+
var normalizeResourceType = (value) => {
|
|
601
|
+
const type = String(value || "").trim().toLowerCase();
|
|
602
|
+
if (!type) return "other";
|
|
603
|
+
return type;
|
|
604
|
+
};
|
|
605
|
+
var resolveHostname = (requestUrl) => {
|
|
606
|
+
const raw = String(requestUrl || "").trim();
|
|
607
|
+
if (!raw) return "";
|
|
608
|
+
try {
|
|
609
|
+
return String(new URL(raw).hostname || "").trim().toLowerCase();
|
|
610
|
+
} catch {
|
|
611
|
+
return "";
|
|
612
|
+
}
|
|
613
|
+
};
|
|
614
|
+
var recordDomainResourceType = (domain, resourceType) => {
|
|
615
|
+
const host = String(domain || "").trim().toLowerCase();
|
|
616
|
+
if (!host) return;
|
|
617
|
+
const type = normalizeResourceType(resourceType);
|
|
618
|
+
let byType = observedDomainResourceTypes.get(host);
|
|
619
|
+
if (!byType) {
|
|
620
|
+
byType = /* @__PURE__ */ new Map();
|
|
621
|
+
observedDomainResourceTypes.set(host, byType);
|
|
622
|
+
}
|
|
623
|
+
byType.set(type, (byType.get(type) || 0) + 1);
|
|
624
|
+
};
|
|
625
|
+
var resolveResourceTypeMeta = (domain) => {
|
|
626
|
+
const host = String(domain || "").trim().toLowerCase();
|
|
627
|
+
if (!host) return { resourceType: "", resourceTypeCounts: void 0 };
|
|
628
|
+
const byType = observedDomainResourceTypes.get(host);
|
|
629
|
+
if (!byType || byType.size === 0) {
|
|
630
|
+
return { resourceType: "", resourceTypeCounts: void 0 };
|
|
631
|
+
}
|
|
632
|
+
const entries = Array.from(byType.entries()).filter(([type, count]) => type && Number(count) > 0).sort((a, b) => {
|
|
633
|
+
if (Number(b[1]) !== Number(a[1])) return Number(b[1]) - Number(a[1]);
|
|
634
|
+
return String(a[0]).localeCompare(String(b[0]));
|
|
635
|
+
});
|
|
636
|
+
if (entries.length === 0) {
|
|
637
|
+
return { resourceType: "", resourceTypeCounts: void 0 };
|
|
638
|
+
}
|
|
639
|
+
return {
|
|
640
|
+
resourceType: entries[0][0],
|
|
641
|
+
resourceTypeCounts: Object.fromEntries(entries.map(([type, count]) => [type, toSafeInt(count)]))
|
|
642
|
+
};
|
|
580
643
|
};
|
|
581
|
-
var
|
|
582
|
-
|
|
644
|
+
var resolveScriptPath = () => {
|
|
645
|
+
const baseDir = typeof __dirname !== "undefined" ? __dirname : import_path.default.dirname((0, import_url.fileURLToPath)(import_meta.url));
|
|
646
|
+
return import_path.default.join(baseDir, "proxy-meter.js");
|
|
647
|
+
};
|
|
648
|
+
var pickFreePort = () => {
|
|
649
|
+
const script = [
|
|
650
|
+
'const net=require("net");',
|
|
651
|
+
"const server=net.createServer();",
|
|
652
|
+
'server.listen(0,"127.0.0.1",()=>{',
|
|
653
|
+
"const port=server.address().port;",
|
|
654
|
+
"server.close(()=>{console.log(port);});",
|
|
655
|
+
"});"
|
|
656
|
+
].join("");
|
|
657
|
+
const result = (0, import_child_process.spawnSync)(process.execPath, ["-e", script], { encoding: "utf8" });
|
|
658
|
+
const port = Number(String(result.stdout || "").trim());
|
|
659
|
+
if (result.status === 0 && Number.isFinite(port) && port > 0) {
|
|
660
|
+
return port;
|
|
661
|
+
}
|
|
662
|
+
return 2e4 + Math.floor(Math.random() * 2e4);
|
|
663
|
+
};
|
|
664
|
+
var resolveLogDir = () => {
|
|
665
|
+
const storageDir = process.env.APIFY_STORAGE_DIR || process.env.APIFY_LOCAL_STORAGE_DIR || "";
|
|
666
|
+
if (storageDir) {
|
|
667
|
+
return import_path.default.join(storageDir, "proxy-meter");
|
|
668
|
+
}
|
|
669
|
+
return import_path.default.join((0, import_os.tmpdir)(), "proxy-meter");
|
|
670
|
+
};
|
|
671
|
+
var ensureLogPath = () => {
|
|
672
|
+
const baseDir = resolveLogDir();
|
|
673
|
+
if (!(0, import_fs.existsSync)(baseDir)) {
|
|
674
|
+
try {
|
|
675
|
+
(0, import_fs.mkdirSync)(baseDir, { recursive: true });
|
|
676
|
+
} catch {
|
|
677
|
+
}
|
|
678
|
+
}
|
|
679
|
+
const runId = String(process.env.APIFY_ACTOR_RUN_ID || "").trim();
|
|
680
|
+
const suffix = `${Date.now()}-${Math.floor(Math.random() * 1e6)}`;
|
|
681
|
+
const label = runId ? `proxy-meter-${runId}-${suffix}.json` : `proxy-meter-${process.pid}-${suffix}.json`;
|
|
682
|
+
return import_path.default.join(baseDir, label);
|
|
683
|
+
};
|
|
684
|
+
var readSnapshot = (logPath) => {
|
|
685
|
+
if (!logPath || !(0, import_fs.existsSync)(logPath)) return null;
|
|
686
|
+
try {
|
|
687
|
+
const raw = (0, import_fs.readFileSync)(logPath, "utf8");
|
|
688
|
+
if (!raw) return null;
|
|
689
|
+
return JSON.parse(raw);
|
|
690
|
+
} catch {
|
|
583
691
|
return null;
|
|
584
692
|
}
|
|
585
|
-
|
|
693
|
+
};
|
|
694
|
+
var normalizeDomainRows = (hosts) => {
|
|
695
|
+
const rows = [];
|
|
696
|
+
let requestCount = 0;
|
|
697
|
+
let connectCount = 0;
|
|
698
|
+
for (const [domain, stat] of Object.entries(hosts || {})) {
|
|
699
|
+
if (!stat || typeof stat !== "object") continue;
|
|
700
|
+
const inBytes = toSafeInt(stat.inBytes);
|
|
701
|
+
const outBytes = toSafeInt(stat.outBytes);
|
|
702
|
+
const requests = toSafeInt(stat.requests);
|
|
703
|
+
const connections = toSafeInt(stat.connections);
|
|
704
|
+
const totalBytes = inBytes + outBytes;
|
|
705
|
+
requestCount += requests;
|
|
706
|
+
connectCount += connections;
|
|
707
|
+
if (totalBytes <= 0 && requests <= 0 && connections <= 0) continue;
|
|
708
|
+
const resourceTypeMeta = resolveResourceTypeMeta(domain);
|
|
709
|
+
rows.push({
|
|
710
|
+
domain,
|
|
711
|
+
inBytes,
|
|
712
|
+
outBytes,
|
|
713
|
+
totalBytes,
|
|
714
|
+
requests,
|
|
715
|
+
connections,
|
|
716
|
+
resourceType: resourceTypeMeta.resourceType || void 0,
|
|
717
|
+
resourceTypeCounts: resourceTypeMeta.resourceTypeCounts || void 0
|
|
718
|
+
});
|
|
719
|
+
}
|
|
720
|
+
rows.sort((a, b) => {
|
|
721
|
+
if (b.totalBytes !== a.totalBytes) return b.totalBytes - a.totalBytes;
|
|
722
|
+
if (b.requests !== a.requests) return b.requests - a.requests;
|
|
723
|
+
return String(a.domain).localeCompare(String(b.domain));
|
|
724
|
+
});
|
|
725
|
+
return {
|
|
726
|
+
topDomains: rows.slice(0, MAX_TOP_DOMAINS),
|
|
727
|
+
requestCount,
|
|
728
|
+
connectCount
|
|
729
|
+
};
|
|
730
|
+
};
|
|
731
|
+
var normalizeDebugRows = (rows, type) => {
|
|
732
|
+
if (!Array.isArray(rows) || rows.length === 0) return void 0;
|
|
733
|
+
const normalized = rows.map((row) => {
|
|
734
|
+
if (!row || typeof row !== "object") return null;
|
|
735
|
+
return {
|
|
736
|
+
domain: String(row.domain || "").trim(),
|
|
737
|
+
status: String(row.status || "").trim(),
|
|
738
|
+
count: toSafeInt(row.count),
|
|
739
|
+
failedCount: toSafeInt(row.failedCount),
|
|
740
|
+
inBytes: toSafeInt(row.inBytes),
|
|
741
|
+
outBytes: toSafeInt(row.outBytes),
|
|
742
|
+
totalBytes: toSafeInt(row.totalBytes),
|
|
743
|
+
avgDurationMs: toSafeFloat(row.avgDurationMs),
|
|
744
|
+
reconnectCount: toSafeInt(row.reconnectCount),
|
|
745
|
+
largeResponseCount: toSafeInt(row.largeResponseCount),
|
|
746
|
+
failureRatePct: toSafeFloat(row.failureRatePct),
|
|
747
|
+
type
|
|
748
|
+
};
|
|
749
|
+
}).filter((row) => row && row.domain && row.count > 0);
|
|
750
|
+
return normalized.length > 0 ? normalized : void 0;
|
|
751
|
+
};
|
|
752
|
+
var normalizeDebugEvent = (row) => {
|
|
753
|
+
if (!row || typeof row !== "object") return null;
|
|
754
|
+
const host = String(row.host || "").trim();
|
|
755
|
+
if (!host) return null;
|
|
756
|
+
return {
|
|
757
|
+
ts: String(row.ts || ""),
|
|
758
|
+
runId: String(row.runId || ""),
|
|
759
|
+
channel: String(row.channel || ""),
|
|
760
|
+
host,
|
|
761
|
+
method: String(row.method || ""),
|
|
762
|
+
path: String(row.path || ""),
|
|
763
|
+
statusCode: toSafeInt(row.statusCode),
|
|
764
|
+
durationMs: toSafeInt(row.durationMs),
|
|
765
|
+
inBytes: toSafeInt(row.inBytes),
|
|
766
|
+
outBytes: toSafeInt(row.outBytes),
|
|
767
|
+
totalBytes: toSafeInt(row.totalBytes),
|
|
768
|
+
error: String(row.error || "")
|
|
769
|
+
};
|
|
770
|
+
};
|
|
771
|
+
var normalizeDebugSummary = (debug) => {
|
|
772
|
+
if (!debug || typeof debug !== "object" || !debug.summary || typeof debug.summary !== "object") {
|
|
773
|
+
return null;
|
|
774
|
+
}
|
|
775
|
+
const summary = debug.summary;
|
|
776
|
+
const domainStatus = normalizeDebugRows(summary.domainStatus, "domain_status");
|
|
777
|
+
const topFailureDomains = normalizeDebugRows(summary.topFailureDomains, "top_failure");
|
|
778
|
+
const topLargeDomains = normalizeDebugRows(summary.topLargeDomains, "top_large");
|
|
779
|
+
const topReconnectDomains = normalizeDebugRows(summary.topReconnectDomains, "top_reconnect");
|
|
780
|
+
return {
|
|
781
|
+
enabled: Boolean(debug.enabled),
|
|
782
|
+
sampledEvents: toSafeInt(debug.sampledEvents),
|
|
783
|
+
droppedEvents: toSafeInt(debug.droppedEvents),
|
|
784
|
+
totalEvents: toSafeInt(debug.totalEvents),
|
|
785
|
+
largeResponseThresholdBytes: toSafeInt(summary.largeResponseThresholdBytes),
|
|
786
|
+
requestCount: toSafeInt(summary.requestCount),
|
|
787
|
+
failedCount: toSafeInt(summary.failedCount),
|
|
788
|
+
failureRatePct: toSafeFloat(summary.failureRatePct),
|
|
789
|
+
largeResponseCount: toSafeInt(summary.largeResponseCount),
|
|
790
|
+
reconnectCount: toSafeInt(summary.reconnectCount),
|
|
791
|
+
domainStatus,
|
|
792
|
+
topFailureDomains,
|
|
793
|
+
topLargeDomains,
|
|
794
|
+
topReconnectDomains
|
|
795
|
+
};
|
|
796
|
+
};
|
|
797
|
+
var normalizeSnapshot = (raw) => {
|
|
798
|
+
if (!raw || typeof raw !== "object") return null;
|
|
799
|
+
const totalIn = toSafeInt(raw.totalInBytes);
|
|
800
|
+
const totalOut = toSafeInt(raw.totalOutBytes);
|
|
801
|
+
const domains = normalizeDomainRows(raw.hosts && typeof raw.hosts === "object" ? raw.hosts : {});
|
|
802
|
+
const normalized = {
|
|
803
|
+
meter: "proxy-meter-v1",
|
|
804
|
+
totalBytes: totalIn + totalOut,
|
|
805
|
+
uploadBytes: totalOut,
|
|
806
|
+
downloadBytes: totalIn,
|
|
807
|
+
requestCount: domains.requestCount,
|
|
808
|
+
connectCount: domains.connectCount,
|
|
809
|
+
topDomains: domains.topDomains
|
|
810
|
+
};
|
|
811
|
+
const debug = raw.debug && typeof raw.debug === "object" ? raw.debug : null;
|
|
812
|
+
const summary = normalizeDebugSummary(debug);
|
|
813
|
+
const events = Array.isArray(debug?.events) ? debug.events.map(normalizeDebugEvent).filter(Boolean) : [];
|
|
814
|
+
if (summary) {
|
|
815
|
+
normalized.trafficDebugSummary = summary;
|
|
816
|
+
}
|
|
817
|
+
if (events.length > 0) {
|
|
818
|
+
normalized.trafficDebugEvents = events;
|
|
819
|
+
}
|
|
820
|
+
const hasSignal = normalized.totalBytes > 0 || normalized.uploadBytes > 0 || normalized.downloadBytes > 0 || normalized.requestCount > 0 || normalized.connectCount > 0 || normalized.topDomains && normalized.topDomains.length > 0 || Boolean(summary) || events.length > 0;
|
|
821
|
+
if (!hasSignal) return null;
|
|
822
|
+
return normalized;
|
|
823
|
+
};
|
|
824
|
+
var registerCleanup = () => {
|
|
825
|
+
if (cleanupInstalled) return;
|
|
826
|
+
cleanupInstalled = true;
|
|
827
|
+
const shutdown = () => {
|
|
828
|
+
if (!runtime || !runtime.proc) return;
|
|
829
|
+
try {
|
|
830
|
+
runtime.proc.kill("SIGTERM");
|
|
831
|
+
} catch {
|
|
832
|
+
}
|
|
833
|
+
};
|
|
834
|
+
process.once("exit", shutdown);
|
|
835
|
+
process.once("SIGINT", shutdown);
|
|
836
|
+
process.once("SIGTERM", shutdown);
|
|
837
|
+
};
|
|
838
|
+
var startProxyMeter = (options = {}) => {
|
|
839
|
+
const upstreamUrl = String(options.proxyUrl || "").trim();
|
|
840
|
+
if (!upstreamUrl) return null;
|
|
841
|
+
if (runtime && runtime.proc) {
|
|
842
|
+
try {
|
|
843
|
+
runtime.proc.kill("SIGTERM");
|
|
844
|
+
} catch {
|
|
845
|
+
}
|
|
846
|
+
runtime = null;
|
|
847
|
+
}
|
|
848
|
+
observedDomainResourceTypes = /* @__PURE__ */ new Map();
|
|
849
|
+
const port = pickFreePort();
|
|
850
|
+
const logPath = ensureLogPath();
|
|
851
|
+
const scriptPath = resolveScriptPath();
|
|
852
|
+
const debugMode = Boolean(options.debugMode);
|
|
853
|
+
const debugMaxEvents = Math.max(10, toSafeInt(options.debugMaxEvents) || DEFAULT_DEBUG_MAX_EVENTS);
|
|
854
|
+
const env = {
|
|
855
|
+
...process.env,
|
|
856
|
+
PROXY_METER_PORT: String(port),
|
|
857
|
+
PROXY_METER_LOG: logPath,
|
|
858
|
+
PROXY_METER_UPSTREAM: upstreamUrl,
|
|
859
|
+
PROXY_METER_FLUSH_MS: String(FLUSH_INTERVAL_MS),
|
|
860
|
+
PROXY_METER_DEBUG: debugMode ? "1" : "0",
|
|
861
|
+
PROXY_METER_DEBUG_MAX_EVENTS: String(debugMaxEvents)
|
|
862
|
+
};
|
|
863
|
+
const child = (0, import_child_process.spawn)(process.execPath, [scriptPath], {
|
|
864
|
+
env,
|
|
865
|
+
stdio: ["ignore", "ignore", "ignore"]
|
|
866
|
+
});
|
|
867
|
+
child.once("exit", (code) => {
|
|
868
|
+
if (code && code !== 0) {
|
|
869
|
+
logger2.warn(`[proxy-meter] exited with code ${code}`);
|
|
870
|
+
}
|
|
871
|
+
});
|
|
872
|
+
runtime = {
|
|
873
|
+
proc: child,
|
|
874
|
+
port,
|
|
875
|
+
logPath,
|
|
876
|
+
startedAt: Date.now()
|
|
877
|
+
};
|
|
878
|
+
registerCleanup();
|
|
879
|
+
return { server: `http://127.0.0.1:${port}` };
|
|
880
|
+
};
|
|
881
|
+
var recordProxyMeterResourceType = (requestUrl, resourceType) => {
|
|
882
|
+
if (!runtime) return;
|
|
883
|
+
const host = resolveHostname(requestUrl);
|
|
884
|
+
if (!host) return;
|
|
885
|
+
recordDomainResourceType(host, resourceType);
|
|
886
|
+
};
|
|
887
|
+
var stopProxyMeter = async () => {
|
|
888
|
+
if (!runtime) return null;
|
|
889
|
+
const { proc, logPath } = runtime;
|
|
890
|
+
if (!proc || proc.killed) {
|
|
891
|
+
runtime = null;
|
|
892
|
+
return logPath || null;
|
|
893
|
+
}
|
|
894
|
+
await new Promise((resolve) => {
|
|
895
|
+
const timeout = setTimeout(() => {
|
|
896
|
+
try {
|
|
897
|
+
proc.kill("SIGKILL");
|
|
898
|
+
} catch {
|
|
899
|
+
}
|
|
900
|
+
resolve();
|
|
901
|
+
}, 2e3);
|
|
902
|
+
proc.once("exit", () => {
|
|
903
|
+
clearTimeout(timeout);
|
|
904
|
+
resolve();
|
|
905
|
+
});
|
|
906
|
+
try {
|
|
907
|
+
proc.kill("SIGTERM");
|
|
908
|
+
} catch {
|
|
909
|
+
resolve();
|
|
910
|
+
}
|
|
911
|
+
});
|
|
912
|
+
runtime = null;
|
|
913
|
+
return logPath || null;
|
|
914
|
+
};
|
|
915
|
+
var getProxyMeterSnapshot = async (options = {}) => {
|
|
916
|
+
if (!runtime) return null;
|
|
917
|
+
const finalize = Boolean(options.finalize);
|
|
918
|
+
const logPath = finalize ? await stopProxyMeter() : runtime.logPath;
|
|
919
|
+
const raw = readSnapshot(logPath);
|
|
920
|
+
const snapshot = normalizeSnapshot(raw);
|
|
921
|
+
if (finalize && logPath) {
|
|
922
|
+
try {
|
|
923
|
+
(0, import_fs.rmSync)(logPath, { force: true });
|
|
924
|
+
} catch {
|
|
925
|
+
}
|
|
926
|
+
}
|
|
927
|
+
return snapshot;
|
|
586
928
|
};
|
|
587
929
|
|
|
588
930
|
// src/apify-kit.js
|
|
589
|
-
var
|
|
931
|
+
var logger3 = createInternalLogger("ApifyKit");
|
|
590
932
|
async function createApifyKit() {
|
|
591
933
|
let apify = null;
|
|
592
934
|
try {
|
|
@@ -614,29 +956,29 @@ async function createApifyKit() {
|
|
|
614
956
|
const { times: retryTimes = 0, mode: retryMode = "direct", before: beforeRetry } = retry;
|
|
615
957
|
const executeAction = async (attemptNumber) => {
|
|
616
958
|
const attemptLabel = attemptNumber > 0 ? ` (\u91CD\u8BD5 #${attemptNumber})` : "";
|
|
617
|
-
|
|
959
|
+
logger3.start(`[Step] ${step}${attemptLabel}`);
|
|
618
960
|
try {
|
|
619
961
|
const result = await actionFn();
|
|
620
|
-
|
|
962
|
+
logger3.success(`[Step] ${step}${attemptLabel}`);
|
|
621
963
|
return { success: true, result };
|
|
622
964
|
} catch (error) {
|
|
623
|
-
|
|
965
|
+
logger3.fail(`[Step] ${step}${attemptLabel}`, error);
|
|
624
966
|
return { success: false, error };
|
|
625
967
|
}
|
|
626
968
|
};
|
|
627
969
|
const prepareForRetry = async (attemptNumber) => {
|
|
628
970
|
if (typeof beforeRetry === "function") {
|
|
629
|
-
|
|
971
|
+
logger3.start(`[RetryStep] \u6267\u884C\u81EA\u5B9A\u4E49 before \u94A9\u5B50 (\u7B2C ${attemptNumber} \u6B21\u91CD\u8BD5)`);
|
|
630
972
|
await beforeRetry(page, attemptNumber);
|
|
631
|
-
|
|
973
|
+
logger3.success(`[RetryStep] before \u94A9\u5B50\u5B8C\u6210`);
|
|
632
974
|
} else if (retryMode === "refresh") {
|
|
633
|
-
|
|
975
|
+
logger3.start(`[RetryStep] \u5237\u65B0\u9875\u9762 (\u7B2C ${attemptNumber} \u6B21\u91CD\u8BD5)`);
|
|
634
976
|
await page.reload({ waitUntil: "commit" });
|
|
635
|
-
|
|
977
|
+
logger3.success(`[RetryStep] \u9875\u9762\u5237\u65B0\u5B8C\u6210`);
|
|
636
978
|
} else {
|
|
637
|
-
|
|
979
|
+
logger3.start(`[RetryStep] \u7B49\u5F85 3 \u79D2 (\u7B2C ${attemptNumber} \u6B21\u91CD\u8BD5)`);
|
|
638
980
|
await new Promise((resolve) => setTimeout(resolve, 3e3));
|
|
639
|
-
|
|
981
|
+
logger3.success(`[RetryStep] \u7B49\u5F85\u5B8C\u6210`);
|
|
640
982
|
}
|
|
641
983
|
};
|
|
642
984
|
let lastResult = await executeAction(0);
|
|
@@ -644,11 +986,11 @@ async function createApifyKit() {
|
|
|
644
986
|
return lastResult.result;
|
|
645
987
|
}
|
|
646
988
|
for (let attempt = 1; attempt <= retryTimes; attempt++) {
|
|
647
|
-
|
|
989
|
+
logger3.start(`[RetryStep] \u51C6\u5907\u7B2C ${attempt}/${retryTimes} \u6B21\u91CD\u8BD5: ${step}`);
|
|
648
990
|
try {
|
|
649
991
|
await prepareForRetry(attempt);
|
|
650
992
|
} catch (prepareError) {
|
|
651
|
-
|
|
993
|
+
logger3.warn(`[RetryStep] \u91CD\u8BD5\u51C6\u5907\u5931\u8D25: ${prepareError.message}`);
|
|
652
994
|
continue;
|
|
653
995
|
}
|
|
654
996
|
lastResult = await executeAction(attempt);
|
|
@@ -669,7 +1011,7 @@ async function createApifyKit() {
|
|
|
669
1011
|
base64 = `data:image/jpeg;base64,${buffer.toString("base64")}`;
|
|
670
1012
|
}
|
|
671
1013
|
} catch (snapErr) {
|
|
672
|
-
|
|
1014
|
+
logger3.warn(`\u622A\u56FE\u751F\u6210\u5931\u8D25: ${snapErr.message}`);
|
|
673
1015
|
}
|
|
674
1016
|
await this.pushFailed(finalError, {
|
|
675
1017
|
step,
|
|
@@ -703,7 +1045,7 @@ async function createApifyKit() {
|
|
|
703
1045
|
* @param {Object} data - 要推送的数据对象
|
|
704
1046
|
*/
|
|
705
1047
|
async pushSuccess(data) {
|
|
706
|
-
const traffic =
|
|
1048
|
+
const traffic = await getProxyMeterSnapshot({ finalize: true });
|
|
707
1049
|
await Actor2.pushData({
|
|
708
1050
|
// 固定为0
|
|
709
1051
|
code: Code.Success,
|
|
@@ -712,7 +1054,7 @@ async function createApifyKit() {
|
|
|
712
1054
|
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
713
1055
|
data
|
|
714
1056
|
});
|
|
715
|
-
|
|
1057
|
+
logger3.success("pushSuccess", "Data pushed");
|
|
716
1058
|
},
|
|
717
1059
|
/**
|
|
718
1060
|
* 推送失败数据的通用方法(私有方法,仅供runStep内部使用)
|
|
@@ -725,7 +1067,7 @@ async function createApifyKit() {
|
|
|
725
1067
|
const isCrawlerError = CrawlerError.isCrawlerError(error);
|
|
726
1068
|
const code = isCrawlerError ? error.code : Code.UnknownError;
|
|
727
1069
|
const context = isCrawlerError ? error.context : {};
|
|
728
|
-
const traffic =
|
|
1070
|
+
const traffic = await getProxyMeterSnapshot({ finalize: true });
|
|
729
1071
|
await Actor2.pushData({
|
|
730
1072
|
// 如果是 CrawlerError,使用其 code,否则使用默认 Failed code
|
|
731
1073
|
code,
|
|
@@ -736,7 +1078,7 @@ async function createApifyKit() {
|
|
|
736
1078
|
context,
|
|
737
1079
|
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
738
1080
|
});
|
|
739
|
-
|
|
1081
|
+
logger3.success("pushFailed", "Error data pushed");
|
|
740
1082
|
}
|
|
741
1083
|
};
|
|
742
1084
|
}
|
|
@@ -752,7 +1094,7 @@ var ApifyKit = {
|
|
|
752
1094
|
};
|
|
753
1095
|
|
|
754
1096
|
// src/internals/utils.js
|
|
755
|
-
var
|
|
1097
|
+
var logger4 = createInternalLogger("InternalUtils");
|
|
756
1098
|
var parseCookies = (cookieString, domain) => {
|
|
757
1099
|
const cookies = [];
|
|
758
1100
|
const pairs = cookieString.split(";").map((c) => c.trim());
|
|
@@ -770,7 +1112,7 @@ var parseCookies = (cookieString, domain) => {
|
|
|
770
1112
|
cookies.push(cookie);
|
|
771
1113
|
}
|
|
772
1114
|
}
|
|
773
|
-
|
|
1115
|
+
logger4.success("parseCookies", `parsed ${cookies.length} cookies`);
|
|
774
1116
|
return cookies;
|
|
775
1117
|
};
|
|
776
1118
|
|
|
@@ -813,7 +1155,7 @@ var Utils = {
|
|
|
813
1155
|
};
|
|
814
1156
|
|
|
815
1157
|
// src/anti-cheat.js
|
|
816
|
-
var
|
|
1158
|
+
var logger5 = createInternalLogger("AntiCheat");
|
|
817
1159
|
var BASE_CONFIG = Object.freeze({
|
|
818
1160
|
locale: "zh-CN",
|
|
819
1161
|
acceptLanguage: "zh-CN,zh;q=0.9",
|
|
@@ -875,7 +1217,7 @@ var AntiCheat = {
|
|
|
875
1217
|
// src/humanize.js
|
|
876
1218
|
var import_delay = __toESM(require("delay"), 1);
|
|
877
1219
|
var import_ghost_cursor_playwright = require("ghost-cursor-playwright");
|
|
878
|
-
var
|
|
1220
|
+
var logger6 = createInternalLogger("Humanize");
|
|
879
1221
|
var $CursorWeakMap = /* @__PURE__ */ new WeakMap();
|
|
880
1222
|
function $GetCursor(page) {
|
|
881
1223
|
const cursor = $CursorWeakMap.get(page);
|
|
@@ -903,13 +1245,13 @@ var Humanize = {
|
|
|
903
1245
|
*/
|
|
904
1246
|
async initializeCursor(page) {
|
|
905
1247
|
if ($CursorWeakMap.has(page)) {
|
|
906
|
-
|
|
1248
|
+
logger6.debug("initializeCursor: cursor already exists, skipping");
|
|
907
1249
|
return;
|
|
908
1250
|
}
|
|
909
|
-
|
|
1251
|
+
logger6.start("initializeCursor", "creating cursor");
|
|
910
1252
|
const cursor = await (0, import_ghost_cursor_playwright.createCursor)(page);
|
|
911
1253
|
$CursorWeakMap.set(page, cursor);
|
|
912
|
-
|
|
1254
|
+
logger6.success("initializeCursor", "cursor initialized");
|
|
913
1255
|
},
|
|
914
1256
|
/**
|
|
915
1257
|
* 人类化鼠标移动 - 使用 ghost-cursor 移动到指定位置或元素
|
|
@@ -919,17 +1261,17 @@ var Humanize = {
|
|
|
919
1261
|
*/
|
|
920
1262
|
async humanMove(page, target) {
|
|
921
1263
|
const cursor = $GetCursor(page);
|
|
922
|
-
|
|
1264
|
+
logger6.start("humanMove", `target=${typeof target === "string" ? target : "element/coords"}`);
|
|
923
1265
|
try {
|
|
924
1266
|
if (typeof target === "string") {
|
|
925
1267
|
const element = await page.$(target);
|
|
926
1268
|
if (!element) {
|
|
927
|
-
|
|
1269
|
+
logger6.warn(`humanMove: \u5143\u7D20\u4E0D\u5B58\u5728 ${target}`);
|
|
928
1270
|
return false;
|
|
929
1271
|
}
|
|
930
1272
|
const box = await element.boundingBox();
|
|
931
1273
|
if (!box) {
|
|
932
|
-
|
|
1274
|
+
logger6.warn(`humanMove: \u65E0\u6CD5\u83B7\u53D6\u4F4D\u7F6E ${target}`);
|
|
933
1275
|
return false;
|
|
934
1276
|
}
|
|
935
1277
|
const x = box.x + box.width / 2 + (Math.random() - 0.5) * box.width * 0.2;
|
|
@@ -945,10 +1287,10 @@ var Humanize = {
|
|
|
945
1287
|
await cursor.actions.move({ x, y });
|
|
946
1288
|
}
|
|
947
1289
|
}
|
|
948
|
-
|
|
1290
|
+
logger6.success("humanMove");
|
|
949
1291
|
return true;
|
|
950
1292
|
} catch (error) {
|
|
951
|
-
|
|
1293
|
+
logger6.fail("humanMove", error);
|
|
952
1294
|
throw error;
|
|
953
1295
|
}
|
|
954
1296
|
},
|
|
@@ -972,12 +1314,12 @@ var Humanize = {
|
|
|
972
1314
|
maxDurationMs = maxSteps * 220 + 800
|
|
973
1315
|
} = options;
|
|
974
1316
|
const targetDesc = typeof target === "string" ? target : "ElementHandle";
|
|
975
|
-
|
|
1317
|
+
logger6.start("humanScroll", `target=${targetDesc}`);
|
|
976
1318
|
let element;
|
|
977
1319
|
if (typeof target === "string") {
|
|
978
1320
|
element = await page.$(target);
|
|
979
1321
|
if (!element) {
|
|
980
|
-
|
|
1322
|
+
logger6.warn(`humanScroll | \u5143\u7D20\u672A\u627E\u5230: ${target}`);
|
|
981
1323
|
return { element: null, didScroll: false };
|
|
982
1324
|
}
|
|
983
1325
|
} else {
|
|
@@ -1052,26 +1394,26 @@ var Humanize = {
|
|
|
1052
1394
|
try {
|
|
1053
1395
|
for (let i = 0; i < maxSteps; i++) {
|
|
1054
1396
|
if (Date.now() - startTime > maxDurationMs) {
|
|
1055
|
-
|
|
1397
|
+
logger6.warn(`humanScroll | \u8D85\u65F6\u4FDD\u62A4\u89E6\u53D1 (${maxDurationMs}ms)`);
|
|
1056
1398
|
return { element, didScroll };
|
|
1057
1399
|
}
|
|
1058
1400
|
const status = await checkVisibility();
|
|
1059
1401
|
if (status.code === "VISIBLE") {
|
|
1060
1402
|
if (status.isFixed) {
|
|
1061
|
-
|
|
1403
|
+
logger6.info("humanScroll | fixed \u5BB9\u5668\u5185\uFF0C\u8DF3\u8FC7\u6EDA\u52A8");
|
|
1062
1404
|
} else {
|
|
1063
|
-
|
|
1405
|
+
logger6.debug("humanScroll | \u5143\u7D20\u53EF\u89C1\u4E14\u65E0\u906E\u6321");
|
|
1064
1406
|
}
|
|
1065
|
-
|
|
1407
|
+
logger6.success("humanScroll", didScroll ? "\u5DF2\u6EDA\u52A8" : "\u65E0\u9700\u6EDA\u52A8");
|
|
1066
1408
|
return { element, didScroll };
|
|
1067
1409
|
}
|
|
1068
|
-
|
|
1410
|
+
logger6.debug(`humanScroll | \u6B65\u9AA4 ${i + 1}/${maxSteps}: ${status.reason} ${status.direction ? `(${status.direction})` : ""}`);
|
|
1069
1411
|
if (status.code === "OBSTRUCTED" && status.obstruction) {
|
|
1070
|
-
|
|
1412
|
+
logger6.debug(`humanScroll | \u88AB\u4EE5\u4E0B\u5143\u7D20\u906E\u6321 <${status.obstruction.tag} id="${status.obstruction.id}">`);
|
|
1071
1413
|
}
|
|
1072
1414
|
const scrollRect = await getScrollableRect();
|
|
1073
1415
|
if (!scrollRect && status.isFixed) {
|
|
1074
|
-
|
|
1416
|
+
logger6.warn("humanScroll | fixed \u5BB9\u5668\u5185\u4E14\u65E0\u53EF\u6EDA\u52A8\u7956\u5148\uFF0C\u8DF3\u8FC7\u6EDA\u52A8");
|
|
1075
1417
|
return { element, didScroll };
|
|
1076
1418
|
}
|
|
1077
1419
|
const stepMin = scrollRect ? Math.min(minStep, Math.max(60, scrollRect.height * 0.4)) : minStep;
|
|
@@ -1107,10 +1449,10 @@ var Humanize = {
|
|
|
1107
1449
|
didScroll = true;
|
|
1108
1450
|
await (0, import_delay.default)(this.jitterMs(20 + Math.random() * 40, 0.2));
|
|
1109
1451
|
}
|
|
1110
|
-
|
|
1452
|
+
logger6.warn(`humanScroll | \u5728 ${maxSteps} \u6B65\u540E\u65E0\u6CD5\u786E\u4FDD\u53EF\u89C1\u6027`);
|
|
1111
1453
|
return { element, didScroll };
|
|
1112
1454
|
} catch (error) {
|
|
1113
|
-
|
|
1455
|
+
logger6.fail("humanScroll", error);
|
|
1114
1456
|
throw error;
|
|
1115
1457
|
}
|
|
1116
1458
|
},
|
|
@@ -1128,7 +1470,7 @@ var Humanize = {
|
|
|
1128
1470
|
const cursor = $GetCursor(page);
|
|
1129
1471
|
const { reactionDelay = 250, throwOnMissing = true, scrollIfNeeded = true, restore = false } = options;
|
|
1130
1472
|
const targetDesc = target == null ? "Current Position" : typeof target === "string" ? target : "ElementHandle";
|
|
1131
|
-
|
|
1473
|
+
logger6.start("humanClick", `target=${targetDesc}`);
|
|
1132
1474
|
const restoreOnce = async () => {
|
|
1133
1475
|
if (restoreOnce.restored) return;
|
|
1134
1476
|
restoreOnce.restored = true;
|
|
@@ -1137,14 +1479,14 @@ var Humanize = {
|
|
|
1137
1479
|
await (0, import_delay.default)(this.jitterMs(1e3));
|
|
1138
1480
|
await restoreOnce.do();
|
|
1139
1481
|
} catch (restoreError) {
|
|
1140
|
-
|
|
1482
|
+
logger6.warn(`humanClick: \u6062\u590D\u6EDA\u52A8\u4F4D\u7F6E\u5931\u8D25: ${restoreError.message}`);
|
|
1141
1483
|
}
|
|
1142
1484
|
};
|
|
1143
1485
|
try {
|
|
1144
1486
|
if (target == null) {
|
|
1145
1487
|
await (0, import_delay.default)(this.jitterMs(reactionDelay, 0.4));
|
|
1146
1488
|
await cursor.actions.click();
|
|
1147
|
-
|
|
1489
|
+
logger6.success("humanClick", "Clicked current position");
|
|
1148
1490
|
return true;
|
|
1149
1491
|
}
|
|
1150
1492
|
let element;
|
|
@@ -1154,7 +1496,7 @@ var Humanize = {
|
|
|
1154
1496
|
if (throwOnMissing) {
|
|
1155
1497
|
throw new Error(`\u627E\u4E0D\u5230\u5143\u7D20 ${target}`);
|
|
1156
1498
|
}
|
|
1157
|
-
|
|
1499
|
+
logger6.warn(`humanClick: \u5143\u7D20\u4E0D\u5B58\u5728\uFF0C\u8DF3\u8FC7\u70B9\u51FB ${target}`);
|
|
1158
1500
|
return false;
|
|
1159
1501
|
}
|
|
1160
1502
|
} else {
|
|
@@ -1170,7 +1512,7 @@ var Humanize = {
|
|
|
1170
1512
|
if (throwOnMissing) {
|
|
1171
1513
|
throw new Error("\u65E0\u6CD5\u83B7\u53D6\u5143\u7D20\u4F4D\u7F6E");
|
|
1172
1514
|
}
|
|
1173
|
-
|
|
1515
|
+
logger6.warn("humanClick: \u65E0\u6CD5\u83B7\u53D6\u4F4D\u7F6E\uFF0C\u8DF3\u8FC7\u70B9\u51FB");
|
|
1174
1516
|
return false;
|
|
1175
1517
|
}
|
|
1176
1518
|
const x = box.x + box.width / 2 + (Math.random() - 0.5) * box.width * 0.3;
|
|
@@ -1179,11 +1521,11 @@ var Humanize = {
|
|
|
1179
1521
|
await (0, import_delay.default)(this.jitterMs(reactionDelay, 0.4));
|
|
1180
1522
|
await cursor.actions.click();
|
|
1181
1523
|
await restoreOnce();
|
|
1182
|
-
|
|
1524
|
+
logger6.success("humanClick");
|
|
1183
1525
|
return true;
|
|
1184
1526
|
} catch (error) {
|
|
1185
1527
|
await restoreOnce();
|
|
1186
|
-
|
|
1528
|
+
logger6.fail("humanClick", error);
|
|
1187
1529
|
throw error;
|
|
1188
1530
|
}
|
|
1189
1531
|
},
|
|
@@ -1194,9 +1536,9 @@ var Humanize = {
|
|
|
1194
1536
|
*/
|
|
1195
1537
|
async randomSleep(baseMs, jitterPercent = 0.3) {
|
|
1196
1538
|
const ms = this.jitterMs(baseMs, jitterPercent);
|
|
1197
|
-
|
|
1539
|
+
logger6.start("randomSleep", `base=${baseMs}, actual=${ms}ms`);
|
|
1198
1540
|
await (0, import_delay.default)(ms);
|
|
1199
|
-
|
|
1541
|
+
logger6.success("randomSleep");
|
|
1200
1542
|
},
|
|
1201
1543
|
/**
|
|
1202
1544
|
* 模拟人类"注视"或"阅读"行为:鼠标在页面上随机微动
|
|
@@ -1206,7 +1548,7 @@ var Humanize = {
|
|
|
1206
1548
|
async simulateGaze(page, baseDurationMs = 2500) {
|
|
1207
1549
|
const cursor = $GetCursor(page);
|
|
1208
1550
|
const durationMs = this.jitterMs(baseDurationMs, 0.4);
|
|
1209
|
-
|
|
1551
|
+
logger6.start("simulateGaze", `duration=${durationMs}ms`);
|
|
1210
1552
|
const startTime = Date.now();
|
|
1211
1553
|
const viewportSize = page.viewportSize() || { width: 1920, height: 1080 };
|
|
1212
1554
|
while (Date.now() - startTime < durationMs) {
|
|
@@ -1215,7 +1557,7 @@ var Humanize = {
|
|
|
1215
1557
|
await cursor.actions.move({ x, y });
|
|
1216
1558
|
await (0, import_delay.default)(this.jitterMs(600, 0.5));
|
|
1217
1559
|
}
|
|
1218
|
-
|
|
1560
|
+
logger6.success("simulateGaze");
|
|
1219
1561
|
},
|
|
1220
1562
|
/**
|
|
1221
1563
|
* 人类化输入 - 带节奏变化(快-慢-停顿-偶尔加速)
|
|
@@ -1228,7 +1570,7 @@ var Humanize = {
|
|
|
1228
1570
|
* @param {number} [options.pauseBase=800] - 停顿时长基础值 (ms),实际 ±50% 抖动
|
|
1229
1571
|
*/
|
|
1230
1572
|
async humanType(page, selector, text, options = {}) {
|
|
1231
|
-
|
|
1573
|
+
logger6.start("humanType", `selector=${selector}, textLen=${text.length}`);
|
|
1232
1574
|
const {
|
|
1233
1575
|
baseDelay = 180,
|
|
1234
1576
|
pauseProbability = 0.08,
|
|
@@ -1252,13 +1594,13 @@ var Humanize = {
|
|
|
1252
1594
|
await (0, import_delay.default)(charDelay);
|
|
1253
1595
|
if (Math.random() < pauseProbability && i < text.length - 1) {
|
|
1254
1596
|
const pauseTime = this.jitterMs(pauseBase, 0.5);
|
|
1255
|
-
|
|
1597
|
+
logger6.debug(`\u505C\u987F ${pauseTime}ms...`);
|
|
1256
1598
|
await (0, import_delay.default)(pauseTime);
|
|
1257
1599
|
}
|
|
1258
1600
|
}
|
|
1259
|
-
|
|
1601
|
+
logger6.success("humanType");
|
|
1260
1602
|
} catch (error) {
|
|
1261
|
-
|
|
1603
|
+
logger6.fail("humanType", error);
|
|
1262
1604
|
throw error;
|
|
1263
1605
|
}
|
|
1264
1606
|
},
|
|
@@ -1268,22 +1610,22 @@ var Humanize = {
|
|
|
1268
1610
|
* @param {string} selector - 输入框选择器
|
|
1269
1611
|
*/
|
|
1270
1612
|
async humanClear(page, selector) {
|
|
1271
|
-
|
|
1613
|
+
logger6.start("humanClear", `selector=${selector}`);
|
|
1272
1614
|
try {
|
|
1273
1615
|
const locator = page.locator(selector);
|
|
1274
1616
|
await locator.click();
|
|
1275
1617
|
await (0, import_delay.default)(this.jitterMs(200, 0.4));
|
|
1276
1618
|
const currentValue = await locator.inputValue();
|
|
1277
1619
|
if (!currentValue || currentValue.length === 0) {
|
|
1278
|
-
|
|
1620
|
+
logger6.success("humanClear", "already empty");
|
|
1279
1621
|
return;
|
|
1280
1622
|
}
|
|
1281
1623
|
await page.keyboard.press("Meta+A");
|
|
1282
1624
|
await (0, import_delay.default)(this.jitterMs(100, 0.4));
|
|
1283
1625
|
await page.keyboard.press("Backspace");
|
|
1284
|
-
|
|
1626
|
+
logger6.success("humanClear");
|
|
1285
1627
|
} catch (error) {
|
|
1286
|
-
|
|
1628
|
+
logger6.fail("humanClear", error);
|
|
1287
1629
|
throw error;
|
|
1288
1630
|
}
|
|
1289
1631
|
},
|
|
@@ -1295,7 +1637,7 @@ var Humanize = {
|
|
|
1295
1637
|
async warmUpBrowsing(page, baseDuration = 3500) {
|
|
1296
1638
|
const cursor = $GetCursor(page);
|
|
1297
1639
|
const durationMs = this.jitterMs(baseDuration, 0.4);
|
|
1298
|
-
|
|
1640
|
+
logger6.start("warmUpBrowsing", `duration=${durationMs}ms`);
|
|
1299
1641
|
const startTime = Date.now();
|
|
1300
1642
|
const viewportSize = page.viewportSize() || { width: 1920, height: 1080 };
|
|
1301
1643
|
try {
|
|
@@ -1314,9 +1656,9 @@ var Humanize = {
|
|
|
1314
1656
|
await (0, import_delay.default)(this.jitterMs(800, 0.5));
|
|
1315
1657
|
}
|
|
1316
1658
|
}
|
|
1317
|
-
|
|
1659
|
+
logger6.success("warmUpBrowsing");
|
|
1318
1660
|
} catch (error) {
|
|
1319
|
-
|
|
1661
|
+
logger6.fail("warmUpBrowsing", error);
|
|
1320
1662
|
throw error;
|
|
1321
1663
|
}
|
|
1322
1664
|
},
|
|
@@ -1330,7 +1672,7 @@ var Humanize = {
|
|
|
1330
1672
|
async naturalScroll(page, direction = "down", distance = 300, baseSteps = 5) {
|
|
1331
1673
|
const steps = Math.max(3, baseSteps + Math.floor(Math.random() * 3) - 1);
|
|
1332
1674
|
const actualDistance = this.jitterMs(distance, 0.15);
|
|
1333
|
-
|
|
1675
|
+
logger6.start("naturalScroll", `dir=${direction}, dist=${actualDistance}, steps=${steps}`);
|
|
1334
1676
|
const sign = direction === "down" ? 1 : -1;
|
|
1335
1677
|
const stepDistance = actualDistance / steps;
|
|
1336
1678
|
try {
|
|
@@ -1342,9 +1684,9 @@ var Humanize = {
|
|
|
1342
1684
|
const baseDelay = 60 + i * 25;
|
|
1343
1685
|
await (0, import_delay.default)(this.jitterMs(baseDelay, 0.3));
|
|
1344
1686
|
}
|
|
1345
|
-
|
|
1687
|
+
logger6.success("naturalScroll");
|
|
1346
1688
|
} catch (error) {
|
|
1347
|
-
|
|
1689
|
+
logger6.fail("naturalScroll", error);
|
|
1348
1690
|
throw error;
|
|
1349
1691
|
}
|
|
1350
1692
|
}
|
|
@@ -1393,693 +1735,10 @@ var findMatchedByPassRule = (rules = [], requestUrl = "") => {
|
|
|
1393
1735
|
hostname
|
|
1394
1736
|
};
|
|
1395
1737
|
};
|
|
1396
|
-
var resolveRouteByProxy = ({
|
|
1397
|
-
requestUrl = "",
|
|
1398
|
-
enableProxy = false,
|
|
1399
|
-
byPassRules = []
|
|
1400
|
-
}) => {
|
|
1401
|
-
if (!enableProxy) {
|
|
1402
|
-
return { route: "direct", matchedRule: null, hostname: "" };
|
|
1403
|
-
}
|
|
1404
|
-
const matched = findMatchedByPassRule(byPassRules, requestUrl);
|
|
1405
|
-
if (!matched) {
|
|
1406
|
-
return { route: "proxy", matchedRule: null, hostname: "" };
|
|
1407
|
-
}
|
|
1408
|
-
if (matched.rule) {
|
|
1409
|
-
return { route: "direct", matchedRule: matched.rule, hostname: matched.hostname };
|
|
1410
|
-
}
|
|
1411
|
-
return { route: "proxy", matchedRule: null, hostname: matched.hostname };
|
|
1412
|
-
};
|
|
1413
|
-
|
|
1414
|
-
// src/traffic-meter.js
|
|
1415
|
-
var logger6 = createInternalLogger("TrafficMeter");
|
|
1416
|
-
var encoder = new TextEncoder();
|
|
1417
|
-
var MAX_DOMAIN_BUCKETS = 160;
|
|
1418
|
-
var MAX_REASON_BUCKETS = 64;
|
|
1419
|
-
var MAX_TOP_ITEMS = 12;
|
|
1420
|
-
var MAX_HINT_ITEMS = 8;
|
|
1421
|
-
var UNKNOWN_DOMAIN = "(unknown)";
|
|
1422
|
-
var OTHER_DOMAINS = "(other-domains)";
|
|
1423
|
-
var OTHER_REASONS = "(other-reasons)";
|
|
1424
|
-
var STATIC_RESOURCE_TYPES = /* @__PURE__ */ new Set([
|
|
1425
|
-
"script",
|
|
1426
|
-
"stylesheet",
|
|
1427
|
-
"image",
|
|
1428
|
-
"font",
|
|
1429
|
-
"media",
|
|
1430
|
-
"manifest"
|
|
1431
|
-
]);
|
|
1432
|
-
var BACKEND_RESOURCE_TYPES = /* @__PURE__ */ new Set([
|
|
1433
|
-
"xhr",
|
|
1434
|
-
"fetch",
|
|
1435
|
-
"websocket",
|
|
1436
|
-
"eventsource"
|
|
1437
|
-
]);
|
|
1438
|
-
var toSafeNumber = (value) => {
|
|
1439
|
-
if (typeof value !== "number" || !Number.isFinite(value) || value <= 0) return 0;
|
|
1440
|
-
return Math.round(value);
|
|
1441
|
-
};
|
|
1442
|
-
var byteLength = (value) => {
|
|
1443
|
-
const text = String(value || "");
|
|
1444
|
-
if (!text) return 0;
|
|
1445
|
-
try {
|
|
1446
|
-
return encoder.encode(text).length;
|
|
1447
|
-
} catch {
|
|
1448
|
-
return text.length;
|
|
1449
|
-
}
|
|
1450
|
-
};
|
|
1451
|
-
var normalizeHeaderValue = (value) => {
|
|
1452
|
-
if (Array.isArray(value)) return value.join(",");
|
|
1453
|
-
if (value === null || value === void 0) return "";
|
|
1454
|
-
return String(value);
|
|
1455
|
-
};
|
|
1456
|
-
var estimateRequestBytes = (request = {}) => {
|
|
1457
|
-
const method = String(request.method || "GET");
|
|
1458
|
-
const url = String(request.url || "");
|
|
1459
|
-
let total = byteLength(`${method} ${url} HTTP/1.1\r
|
|
1460
|
-
`);
|
|
1461
|
-
const headers = request.headers && typeof request.headers === "object" ? request.headers : {};
|
|
1462
|
-
Object.entries(headers).forEach(([key, value]) => {
|
|
1463
|
-
total += byteLength(`${key}: ${normalizeHeaderValue(value)}\r
|
|
1464
|
-
`);
|
|
1465
|
-
});
|
|
1466
|
-
total += 2;
|
|
1467
|
-
if (typeof request.postData === "string" && request.postData) {
|
|
1468
|
-
total += byteLength(request.postData);
|
|
1469
|
-
}
|
|
1470
|
-
return total;
|
|
1471
|
-
};
|
|
1472
|
-
var createTrafficState = () => ({
|
|
1473
|
-
totalRequests: 0,
|
|
1474
|
-
proxyRequests: 0,
|
|
1475
|
-
directRequests: 0,
|
|
1476
|
-
totalUploadBytes: 0,
|
|
1477
|
-
proxyUploadBytes: 0,
|
|
1478
|
-
directUploadBytes: 0,
|
|
1479
|
-
totalDownloadBytes: 0,
|
|
1480
|
-
proxyDownloadBytes: 0,
|
|
1481
|
-
directDownloadBytes: 0,
|
|
1482
|
-
totalFailedRequests: 0,
|
|
1483
|
-
proxyFailedRequests: 0,
|
|
1484
|
-
directFailedRequests: 0,
|
|
1485
|
-
totalCanceledRequests: 0,
|
|
1486
|
-
proxyCanceledRequests: 0,
|
|
1487
|
-
directCanceledRequests: 0,
|
|
1488
|
-
orphanDataReceivedBytes: 0,
|
|
1489
|
-
orphanProxyDataReceivedBytes: 0,
|
|
1490
|
-
orphanFinishDeltaBytes: 0,
|
|
1491
|
-
orphanProxyFinishDeltaBytes: 0,
|
|
1492
|
-
domainStats: /* @__PURE__ */ new Map(),
|
|
1493
|
-
typeStats: /* @__PURE__ */ new Map(),
|
|
1494
|
-
failedReasonStats: /* @__PURE__ */ new Map()
|
|
1495
|
-
});
|
|
1496
|
-
var ensureRoute = (route) => route === "proxy" ? "proxy" : "direct";
|
|
1497
|
-
var normalizeResourceType = (value) => {
|
|
1498
|
-
const type = String(value || "").trim().toLowerCase();
|
|
1499
|
-
if (!type) return "other";
|
|
1500
|
-
if (type === "ws") return "websocket";
|
|
1501
|
-
return type;
|
|
1502
|
-
};
|
|
1503
|
-
var parseHostname = (url = "") => {
|
|
1504
|
-
try {
|
|
1505
|
-
const hostname = new URL(String(url || "")).hostname.toLowerCase();
|
|
1506
|
-
return hostname || "";
|
|
1507
|
-
} catch {
|
|
1508
|
-
return "";
|
|
1509
|
-
}
|
|
1510
|
-
};
|
|
1511
|
-
var normalizeDomainKey = (domain = "") => {
|
|
1512
|
-
const key = String(domain || "").trim().toLowerCase();
|
|
1513
|
-
return key || UNKNOWN_DOMAIN;
|
|
1514
|
-
};
|
|
1515
|
-
var isStaticType = (resourceType = "") => STATIC_RESOURCE_TYPES.has(normalizeResourceType(resourceType));
|
|
1516
|
-
var isBackendType = (resourceType = "") => BACKEND_RESOURCE_TYPES.has(normalizeResourceType(resourceType));
|
|
1517
|
-
var createDomainBucket = (domain) => ({
|
|
1518
|
-
domain,
|
|
1519
|
-
requests: 0,
|
|
1520
|
-
proxyRequests: 0,
|
|
1521
|
-
directRequests: 0,
|
|
1522
|
-
uploadBytes: 0,
|
|
1523
|
-
downloadBytes: 0,
|
|
1524
|
-
totalBytes: 0,
|
|
1525
|
-
proxyBytes: 0,
|
|
1526
|
-
directBytes: 0,
|
|
1527
|
-
failedRequests: 0,
|
|
1528
|
-
canceledRequests: 0,
|
|
1529
|
-
staticBytes: 0,
|
|
1530
|
-
backendBytes: 0
|
|
1531
|
-
});
|
|
1532
|
-
var createTypeBucket = (resourceType) => ({
|
|
1533
|
-
resourceType,
|
|
1534
|
-
requests: 0,
|
|
1535
|
-
proxyRequests: 0,
|
|
1536
|
-
directRequests: 0,
|
|
1537
|
-
uploadBytes: 0,
|
|
1538
|
-
downloadBytes: 0,
|
|
1539
|
-
totalBytes: 0,
|
|
1540
|
-
proxyBytes: 0,
|
|
1541
|
-
directBytes: 0,
|
|
1542
|
-
failedRequests: 0,
|
|
1543
|
-
canceledRequests: 0
|
|
1544
|
-
});
|
|
1545
|
-
var createReasonBucket = (reason) => ({
|
|
1546
|
-
reason,
|
|
1547
|
-
count: 0,
|
|
1548
|
-
canceledCount: 0,
|
|
1549
|
-
proxyCount: 0,
|
|
1550
|
-
directCount: 0
|
|
1551
|
-
});
|
|
1552
|
-
var formatBytes = (bytes = 0) => {
|
|
1553
|
-
const value = toSafeNumber(bytes);
|
|
1554
|
-
if (value <= 0) return "0B";
|
|
1555
|
-
const units = ["B", "KB", "MB", "GB", "TB"];
|
|
1556
|
-
let size = value;
|
|
1557
|
-
let unit = 0;
|
|
1558
|
-
while (size >= 1024 && unit < units.length - 1) {
|
|
1559
|
-
size /= 1024;
|
|
1560
|
-
unit += 1;
|
|
1561
|
-
}
|
|
1562
|
-
const precision = size >= 100 || unit === 0 ? 0 : 2;
|
|
1563
|
-
return `${size.toFixed(precision)}${units[unit]}`;
|
|
1564
|
-
};
|
|
1565
|
-
var addRequests = (state, route, count = 1) => {
|
|
1566
|
-
const c = toSafeNumber(count);
|
|
1567
|
-
if (c <= 0) return;
|
|
1568
|
-
const normalizedRoute = ensureRoute(route);
|
|
1569
|
-
state.totalRequests += c;
|
|
1570
|
-
if (normalizedRoute === "proxy") {
|
|
1571
|
-
state.proxyRequests += c;
|
|
1572
|
-
return;
|
|
1573
|
-
}
|
|
1574
|
-
state.directRequests += c;
|
|
1575
|
-
};
|
|
1576
|
-
var addUploadBytes = (state, route, bytes = 0) => {
|
|
1577
|
-
const b = toSafeNumber(bytes);
|
|
1578
|
-
if (b <= 0) return;
|
|
1579
|
-
const normalizedRoute = ensureRoute(route);
|
|
1580
|
-
state.totalUploadBytes += b;
|
|
1581
|
-
if (normalizedRoute === "proxy") {
|
|
1582
|
-
state.proxyUploadBytes += b;
|
|
1583
|
-
return;
|
|
1584
|
-
}
|
|
1585
|
-
state.directUploadBytes += b;
|
|
1586
|
-
};
|
|
1587
|
-
var addDownloadBytes = (state, route, bytes = 0) => {
|
|
1588
|
-
const b = toSafeNumber(bytes);
|
|
1589
|
-
if (b <= 0) return;
|
|
1590
|
-
const normalizedRoute = ensureRoute(route);
|
|
1591
|
-
state.totalDownloadBytes += b;
|
|
1592
|
-
if (normalizedRoute === "proxy") {
|
|
1593
|
-
state.proxyDownloadBytes += b;
|
|
1594
|
-
return;
|
|
1595
|
-
}
|
|
1596
|
-
state.directDownloadBytes += b;
|
|
1597
|
-
};
|
|
1598
|
-
var incrementBucketRequests = (bucket, route) => {
|
|
1599
|
-
if (!bucket) return;
|
|
1600
|
-
bucket.requests += 1;
|
|
1601
|
-
if (ensureRoute(route) === "proxy") {
|
|
1602
|
-
bucket.proxyRequests += 1;
|
|
1603
|
-
return;
|
|
1604
|
-
}
|
|
1605
|
-
bucket.directRequests += 1;
|
|
1606
|
-
};
|
|
1607
|
-
var incrementBucketBytes = (bucket, route, bytes = 0, direction = "download", resourceType = "other") => {
|
|
1608
|
-
if (!bucket) return;
|
|
1609
|
-
const b = toSafeNumber(bytes);
|
|
1610
|
-
if (b <= 0) return;
|
|
1611
|
-
if (direction === "upload") {
|
|
1612
|
-
bucket.uploadBytes += b;
|
|
1613
|
-
} else {
|
|
1614
|
-
bucket.downloadBytes += b;
|
|
1615
|
-
}
|
|
1616
|
-
bucket.totalBytes += b;
|
|
1617
|
-
if (ensureRoute(route) === "proxy") {
|
|
1618
|
-
bucket.proxyBytes += b;
|
|
1619
|
-
} else {
|
|
1620
|
-
bucket.directBytes += b;
|
|
1621
|
-
}
|
|
1622
|
-
if (bucket.domain !== void 0) {
|
|
1623
|
-
if (isStaticType(resourceType)) {
|
|
1624
|
-
bucket.staticBytes += b;
|
|
1625
|
-
} else if (isBackendType(resourceType)) {
|
|
1626
|
-
bucket.backendBytes += b;
|
|
1627
|
-
}
|
|
1628
|
-
}
|
|
1629
|
-
};
|
|
1630
|
-
var incrementBucketFailed = (bucket, canceled = false) => {
|
|
1631
|
-
if (!bucket) return;
|
|
1632
|
-
bucket.failedRequests += 1;
|
|
1633
|
-
if (canceled) {
|
|
1634
|
-
bucket.canceledRequests += 1;
|
|
1635
|
-
}
|
|
1636
|
-
};
|
|
1637
|
-
var ensureReasonText = (reason) => {
|
|
1638
|
-
const normalized = String(reason || "").trim().toLowerCase();
|
|
1639
|
-
return normalized || "unknown";
|
|
1640
|
-
};
|
|
1641
|
-
var pickDomainBucket = (state, domain = "") => {
|
|
1642
|
-
const domainKey = normalizeDomainKey(domain);
|
|
1643
|
-
const knownBucket = state.domainStats.get(domainKey);
|
|
1644
|
-
if (knownBucket) return knownBucket;
|
|
1645
|
-
if (state.domainStats.size < MAX_DOMAIN_BUCKETS) {
|
|
1646
|
-
const bucket2 = createDomainBucket(domainKey);
|
|
1647
|
-
state.domainStats.set(domainKey, bucket2);
|
|
1648
|
-
return bucket2;
|
|
1649
|
-
}
|
|
1650
|
-
const overflow = state.domainStats.get(OTHER_DOMAINS);
|
|
1651
|
-
if (overflow) return overflow;
|
|
1652
|
-
const bucket = createDomainBucket(OTHER_DOMAINS);
|
|
1653
|
-
state.domainStats.set(OTHER_DOMAINS, bucket);
|
|
1654
|
-
return bucket;
|
|
1655
|
-
};
|
|
1656
|
-
var pickTypeBucket = (state, resourceType = "other") => {
|
|
1657
|
-
const typeKey = normalizeResourceType(resourceType);
|
|
1658
|
-
const knownBucket = state.typeStats.get(typeKey);
|
|
1659
|
-
if (knownBucket) return knownBucket;
|
|
1660
|
-
const bucket = createTypeBucket(typeKey);
|
|
1661
|
-
state.typeStats.set(typeKey, bucket);
|
|
1662
|
-
return bucket;
|
|
1663
|
-
};
|
|
1664
|
-
var pickReasonBucket = (state, reason = "") => {
|
|
1665
|
-
const reasonKey = ensureReasonText(reason);
|
|
1666
|
-
const knownBucket = state.failedReasonStats.get(reasonKey);
|
|
1667
|
-
if (knownBucket) return knownBucket;
|
|
1668
|
-
if (state.failedReasonStats.size < MAX_REASON_BUCKETS) {
|
|
1669
|
-
const bucket2 = createReasonBucket(reasonKey);
|
|
1670
|
-
state.failedReasonStats.set(reasonKey, bucket2);
|
|
1671
|
-
return bucket2;
|
|
1672
|
-
}
|
|
1673
|
-
const overflow = state.failedReasonStats.get(OTHER_REASONS);
|
|
1674
|
-
if (overflow) return overflow;
|
|
1675
|
-
const bucket = createReasonBucket(OTHER_REASONS);
|
|
1676
|
-
state.failedReasonStats.set(OTHER_REASONS, bucket);
|
|
1677
|
-
return bucket;
|
|
1678
|
-
};
|
|
1679
|
-
var addRequestProfile = (state, route, domain, resourceType, uploadBytes = 0) => {
|
|
1680
|
-
const domainBucket = pickDomainBucket(state, domain);
|
|
1681
|
-
const typeBucket = pickTypeBucket(state, resourceType);
|
|
1682
|
-
incrementBucketRequests(domainBucket, route);
|
|
1683
|
-
incrementBucketRequests(typeBucket, route);
|
|
1684
|
-
incrementBucketBytes(domainBucket, route, uploadBytes, "upload", resourceType);
|
|
1685
|
-
incrementBucketBytes(typeBucket, route, uploadBytes, "upload", resourceType);
|
|
1686
|
-
};
|
|
1687
|
-
var addUploadProfile = (state, route, domain, resourceType, uploadBytes = 0) => {
|
|
1688
|
-
const domainBucket = pickDomainBucket(state, domain);
|
|
1689
|
-
const typeBucket = pickTypeBucket(state, resourceType);
|
|
1690
|
-
incrementBucketBytes(domainBucket, route, uploadBytes, "upload", resourceType);
|
|
1691
|
-
incrementBucketBytes(typeBucket, route, uploadBytes, "upload", resourceType);
|
|
1692
|
-
};
|
|
1693
|
-
var addDownloadProfile = (state, route, domain, resourceType, downloadBytes = 0) => {
|
|
1694
|
-
const domainBucket = pickDomainBucket(state, domain);
|
|
1695
|
-
const typeBucket = pickTypeBucket(state, resourceType);
|
|
1696
|
-
incrementBucketBytes(domainBucket, route, downloadBytes, "download", resourceType);
|
|
1697
|
-
incrementBucketBytes(typeBucket, route, downloadBytes, "download", resourceType);
|
|
1698
|
-
};
|
|
1699
|
-
var addFailedProfile = (state, route, domain, resourceType, reason = "unknown", canceled = false) => {
|
|
1700
|
-
const domainBucket = pickDomainBucket(state, domain);
|
|
1701
|
-
const typeBucket = pickTypeBucket(state, resourceType);
|
|
1702
|
-
const reasonBucket = pickReasonBucket(state, reason);
|
|
1703
|
-
incrementBucketFailed(domainBucket, canceled);
|
|
1704
|
-
incrementBucketFailed(typeBucket, canceled);
|
|
1705
|
-
reasonBucket.count += 1;
|
|
1706
|
-
if (ensureRoute(route) === "proxy") {
|
|
1707
|
-
reasonBucket.proxyCount += 1;
|
|
1708
|
-
} else {
|
|
1709
|
-
reasonBucket.directCount += 1;
|
|
1710
|
-
}
|
|
1711
|
-
if (canceled) {
|
|
1712
|
-
reasonBucket.canceledCount += 1;
|
|
1713
|
-
}
|
|
1714
|
-
};
|
|
1715
|
-
var toRoundedRatio = (numerator, denominator) => {
|
|
1716
|
-
if (denominator <= 0) return 0;
|
|
1717
|
-
return Number((numerator / denominator * 100).toFixed(2));
|
|
1718
|
-
};
|
|
1719
|
-
var sortByBytesAndRequests = (left, right) => {
|
|
1720
|
-
if (right.totalBytes !== left.totalBytes) return right.totalBytes - left.totalBytes;
|
|
1721
|
-
if (right.requests !== left.requests) return right.requests - left.requests;
|
|
1722
|
-
return String(left.domain || left.resourceType || "").localeCompare(String(right.domain || right.resourceType || ""));
|
|
1723
|
-
};
|
|
1724
|
-
var buildTopDomains = (state) => {
|
|
1725
|
-
return Array.from(state.domainStats.values()).filter((item) => item && item.totalBytes > 0).sort(sortByBytesAndRequests).slice(0, MAX_TOP_ITEMS).map((item) => ({
|
|
1726
|
-
domain: item.domain,
|
|
1727
|
-
requests: item.requests,
|
|
1728
|
-
proxyRequests: item.proxyRequests,
|
|
1729
|
-
directRequests: item.directRequests,
|
|
1730
|
-
uploadBytes: item.uploadBytes,
|
|
1731
|
-
downloadBytes: item.downloadBytes,
|
|
1732
|
-
totalBytes: item.totalBytes,
|
|
1733
|
-
proxyBytes: item.proxyBytes,
|
|
1734
|
-
directBytes: item.directBytes,
|
|
1735
|
-
failedRequests: item.failedRequests,
|
|
1736
|
-
canceledRequests: item.canceledRequests,
|
|
1737
|
-
staticBytes: item.staticBytes,
|
|
1738
|
-
backendBytes: item.backendBytes
|
|
1739
|
-
}));
|
|
1740
|
-
};
|
|
1741
|
-
var buildTopResourceTypes = (state) => {
|
|
1742
|
-
return Array.from(state.typeStats.values()).filter((item) => item && (item.totalBytes > 0 || item.requests > 0)).sort(sortByBytesAndRequests).slice(0, MAX_TOP_ITEMS).map((item) => ({
|
|
1743
|
-
resourceType: item.resourceType,
|
|
1744
|
-
requests: item.requests,
|
|
1745
|
-
proxyRequests: item.proxyRequests,
|
|
1746
|
-
directRequests: item.directRequests,
|
|
1747
|
-
uploadBytes: item.uploadBytes,
|
|
1748
|
-
downloadBytes: item.downloadBytes,
|
|
1749
|
-
totalBytes: item.totalBytes,
|
|
1750
|
-
proxyBytes: item.proxyBytes,
|
|
1751
|
-
directBytes: item.directBytes,
|
|
1752
|
-
failedRequests: item.failedRequests,
|
|
1753
|
-
canceledRequests: item.canceledRequests
|
|
1754
|
-
}));
|
|
1755
|
-
};
|
|
1756
|
-
var buildFailedReasons = (state) => {
|
|
1757
|
-
return Array.from(state.failedReasonStats.values()).filter((item) => item && item.count > 0).sort((left, right) => {
|
|
1758
|
-
if (right.count !== left.count) return right.count - left.count;
|
|
1759
|
-
return String(left.reason || "").localeCompare(String(right.reason || ""));
|
|
1760
|
-
}).slice(0, MAX_TOP_ITEMS).map((item) => ({
|
|
1761
|
-
reason: item.reason,
|
|
1762
|
-
count: item.count,
|
|
1763
|
-
canceledCount: item.canceledCount,
|
|
1764
|
-
proxyCount: item.proxyCount,
|
|
1765
|
-
directCount: item.directCount
|
|
1766
|
-
}));
|
|
1767
|
-
};
|
|
1768
|
-
var buildProxyByPassHints = (state) => {
|
|
1769
|
-
return Array.from(state.domainStats.values()).filter((item) => item && item.domain !== UNKNOWN_DOMAIN && item.domain !== OTHER_DOMAINS).filter((item) => item.proxyBytes >= 128 * 1024 && item.proxyRequests >= 2 && item.totalBytes > 0).filter((item) => item.staticBytes > item.backendBytes && toRoundedRatio(item.staticBytes, item.totalBytes) >= 80).sort((left, right) => {
|
|
1770
|
-
if (right.proxyBytes !== left.proxyBytes) return right.proxyBytes - left.proxyBytes;
|
|
1771
|
-
return right.proxyRequests - left.proxyRequests;
|
|
1772
|
-
}).slice(0, MAX_HINT_ITEMS).map((item) => ({
|
|
1773
|
-
domain: item.domain,
|
|
1774
|
-
proxyBytes: item.proxyBytes,
|
|
1775
|
-
proxyRequests: item.proxyRequests,
|
|
1776
|
-
totalBytes: item.totalBytes,
|
|
1777
|
-
staticBytes: item.staticBytes,
|
|
1778
|
-
backendBytes: item.backendBytes,
|
|
1779
|
-
staticRatioPct: toRoundedRatio(item.staticBytes, item.totalBytes)
|
|
1780
|
-
}));
|
|
1781
|
-
};
|
|
1782
|
-
var createTrafficMeter = ({
|
|
1783
|
-
enableProxy = false,
|
|
1784
|
-
byPassRules = [],
|
|
1785
|
-
debugMode = false
|
|
1786
|
-
} = {}) => {
|
|
1787
|
-
const state = createTrafficState();
|
|
1788
|
-
const requestMap = /* @__PURE__ */ new Map();
|
|
1789
|
-
const orphanReceivedMap = /* @__PURE__ */ new Map();
|
|
1790
|
-
const wsRouteMap = /* @__PURE__ */ new Map();
|
|
1791
|
-
const attachedPages = /* @__PURE__ */ new WeakSet();
|
|
1792
|
-
const attachedContexts = /* @__PURE__ */ new WeakSet();
|
|
1793
|
-
const resolveRoute = (url = "") => {
|
|
1794
|
-
return resolveRouteByProxy({
|
|
1795
|
-
requestUrl: url,
|
|
1796
|
-
enableProxy,
|
|
1797
|
-
byPassRules
|
|
1798
|
-
}).route;
|
|
1799
|
-
};
|
|
1800
|
-
const fallbackRoute = () => enableProxy ? "proxy" : "direct";
|
|
1801
|
-
const debugLog = (message) => {
|
|
1802
|
-
if (!debugMode) return;
|
|
1803
|
-
logger6.info(`[\u9010\u8BF7\u6C42\u8C03\u8BD5] ${message}`);
|
|
1804
|
-
};
|
|
1805
|
-
const addFailed = (route, canceled = false) => {
|
|
1806
|
-
const normalizedRoute = ensureRoute(route);
|
|
1807
|
-
state.totalFailedRequests += 1;
|
|
1808
|
-
if (normalizedRoute === "proxy") {
|
|
1809
|
-
state.proxyFailedRequests += 1;
|
|
1810
|
-
} else {
|
|
1811
|
-
state.directFailedRequests += 1;
|
|
1812
|
-
}
|
|
1813
|
-
if (!canceled) return;
|
|
1814
|
-
state.totalCanceledRequests += 1;
|
|
1815
|
-
if (normalizedRoute === "proxy") {
|
|
1816
|
-
state.proxyCanceledRequests += 1;
|
|
1817
|
-
} else {
|
|
1818
|
-
state.directCanceledRequests += 1;
|
|
1819
|
-
}
|
|
1820
|
-
};
|
|
1821
|
-
const finalizeByEncodedLength = (requestId, encodedDataLength, source = "finished") => {
|
|
1822
|
-
const safeRequestId = String(requestId || "");
|
|
1823
|
-
if (!safeRequestId) return;
|
|
1824
|
-
const requestState = requestMap.get(safeRequestId);
|
|
1825
|
-
const orphanReceived = toSafeNumber(orphanReceivedMap.get(safeRequestId));
|
|
1826
|
-
const encoded = toSafeNumber(encodedDataLength);
|
|
1827
|
-
if (requestState) {
|
|
1828
|
-
const routed2 = ensureRoute(requestState.route);
|
|
1829
|
-
const downloaded = toSafeNumber(requestState.downloadBytes);
|
|
1830
|
-
const delta2 = Math.max(0, encoded - downloaded);
|
|
1831
|
-
if (delta2 > 0) {
|
|
1832
|
-
addDownloadBytes(state, routed2, delta2);
|
|
1833
|
-
addDownloadProfile(state, routed2, requestState.domain, requestState.resourceType, delta2);
|
|
1834
|
-
requestState.downloadBytes = downloaded + delta2;
|
|
1835
|
-
}
|
|
1836
|
-
const uploadBytes = toSafeNumber(requestState.uploadBytes);
|
|
1837
|
-
const total = uploadBytes + toSafeNumber(requestState.downloadBytes);
|
|
1838
|
-
debugLog(
|
|
1839
|
-
`final id=${safeRequestId} source=${source} status=ok route=${routed2} type=${requestState.resourceType || "other"} upload=${formatBytes(uploadBytes)} (${uploadBytes}) download=${formatBytes(requestState.downloadBytes)} (${requestState.downloadBytes}) total=${formatBytes(total)} (${total}) url=${requestState.url || "-"}`
|
|
1840
|
-
);
|
|
1841
|
-
requestMap.delete(safeRequestId);
|
|
1842
|
-
orphanReceivedMap.delete(safeRequestId);
|
|
1843
|
-
return;
|
|
1844
|
-
}
|
|
1845
|
-
const routed = fallbackRoute();
|
|
1846
|
-
const delta = Math.max(0, encoded - orphanReceived);
|
|
1847
|
-
if (delta > 0) {
|
|
1848
|
-
addDownloadBytes(state, routed, delta);
|
|
1849
|
-
addDownloadProfile(state, routed, UNKNOWN_DOMAIN, "other", delta);
|
|
1850
|
-
}
|
|
1851
|
-
state.orphanFinishDeltaBytes += delta;
|
|
1852
|
-
if (routed === "proxy") {
|
|
1853
|
-
state.orphanProxyFinishDeltaBytes += delta;
|
|
1854
|
-
}
|
|
1855
|
-
debugLog(
|
|
1856
|
-
`final id=${safeRequestId} source=${source} status=orphan route=${routed} encoded=${formatBytes(encoded)} (${encoded}) delta=${formatBytes(delta)} (${delta})`
|
|
1857
|
-
);
|
|
1858
|
-
orphanReceivedMap.delete(safeRequestId);
|
|
1859
|
-
};
|
|
1860
|
-
const recordRequest = (params = {}) => {
|
|
1861
|
-
const requestId = String(params.requestId || "");
|
|
1862
|
-
const request = params.request && typeof params.request === "object" ? params.request : {};
|
|
1863
|
-
const url = String(request.url || "");
|
|
1864
|
-
const route = resolveRoute(url);
|
|
1865
|
-
const resourceType = normalizeResourceType(params.type || request.type || "other");
|
|
1866
|
-
const domain = normalizeDomainKey(parseHostname(url));
|
|
1867
|
-
if (requestId && requestMap.has(requestId)) {
|
|
1868
|
-
const redirectResponse = params.redirectResponse && typeof params.redirectResponse === "object" ? params.redirectResponse : null;
|
|
1869
|
-
finalizeByEncodedLength(
|
|
1870
|
-
requestId,
|
|
1871
|
-
redirectResponse ? redirectResponse.encodedDataLength : 0,
|
|
1872
|
-
"redirect"
|
|
1873
|
-
);
|
|
1874
|
-
}
|
|
1875
|
-
addRequests(state, route, 1);
|
|
1876
|
-
const uploadBytes = estimateRequestBytes(request);
|
|
1877
|
-
addUploadBytes(state, route, uploadBytes);
|
|
1878
|
-
addRequestProfile(state, route, domain, resourceType, uploadBytes);
|
|
1879
|
-
if (requestId) {
|
|
1880
|
-
requestMap.set(requestId, {
|
|
1881
|
-
route: ensureRoute(route),
|
|
1882
|
-
resourceType,
|
|
1883
|
-
domain,
|
|
1884
|
-
url,
|
|
1885
|
-
uploadBytes,
|
|
1886
|
-
downloadBytes: 0
|
|
1887
|
-
});
|
|
1888
|
-
}
|
|
1889
|
-
debugLog(
|
|
1890
|
-
`request id=${requestId || "-"} route=${route} type=${resourceType} upload=${formatBytes(uploadBytes)} (${uploadBytes}) method=${String(request.method || "GET")} url=${url || "-"}`
|
|
1891
|
-
);
|
|
1892
|
-
};
|
|
1893
|
-
const recordDataReceived = (params = {}) => {
|
|
1894
|
-
const requestId = String(params.requestId || "");
|
|
1895
|
-
const bytes = toSafeNumber(params.encodedDataLength);
|
|
1896
|
-
if (bytes <= 0) return;
|
|
1897
|
-
if (!requestId) {
|
|
1898
|
-
const routed2 = fallbackRoute();
|
|
1899
|
-
addDownloadBytes(state, routed2, bytes);
|
|
1900
|
-
addDownloadProfile(state, routed2, UNKNOWN_DOMAIN, "other", bytes);
|
|
1901
|
-
state.orphanDataReceivedBytes += bytes;
|
|
1902
|
-
if (routed2 === "proxy") {
|
|
1903
|
-
state.orphanProxyDataReceivedBytes += bytes;
|
|
1904
|
-
}
|
|
1905
|
-
return;
|
|
1906
|
-
}
|
|
1907
|
-
const requestState = requestMap.get(requestId);
|
|
1908
|
-
if (requestState) {
|
|
1909
|
-
requestState.downloadBytes = toSafeNumber(requestState.downloadBytes) + bytes;
|
|
1910
|
-
addDownloadBytes(state, requestState.route, bytes);
|
|
1911
|
-
addDownloadProfile(state, requestState.route, requestState.domain, requestState.resourceType, bytes);
|
|
1912
|
-
return;
|
|
1913
|
-
}
|
|
1914
|
-
const prev = toSafeNumber(orphanReceivedMap.get(requestId));
|
|
1915
|
-
orphanReceivedMap.set(requestId, prev + bytes);
|
|
1916
|
-
const routed = fallbackRoute();
|
|
1917
|
-
addDownloadBytes(state, routed, bytes);
|
|
1918
|
-
addDownloadProfile(state, routed, UNKNOWN_DOMAIN, "other", bytes);
|
|
1919
|
-
state.orphanDataReceivedBytes += bytes;
|
|
1920
|
-
if (routed === "proxy") {
|
|
1921
|
-
state.orphanProxyDataReceivedBytes += bytes;
|
|
1922
|
-
}
|
|
1923
|
-
};
|
|
1924
|
-
const recordLoadingFinished = (params = {}) => {
|
|
1925
|
-
finalizeByEncodedLength(params.requestId, params.encodedDataLength, "loadingFinished");
|
|
1926
|
-
};
|
|
1927
|
-
const recordRequestFailed = (params = {}) => {
|
|
1928
|
-
const requestId = String(params.requestId || "");
|
|
1929
|
-
const requestState = requestId ? requestMap.get(requestId) : null;
|
|
1930
|
-
const canceled = Boolean(params.canceled);
|
|
1931
|
-
const reason = ensureReasonText(params.errorText);
|
|
1932
|
-
const routed = ensureRoute(requestState?.route || fallbackRoute());
|
|
1933
|
-
const resourceType = normalizeResourceType(requestState?.resourceType || "other");
|
|
1934
|
-
const domain = normalizeDomainKey(requestState?.domain || "");
|
|
1935
|
-
if (requestState) {
|
|
1936
|
-
addFailed(routed, canceled);
|
|
1937
|
-
addFailedProfile(state, routed, domain, resourceType, reason, canceled);
|
|
1938
|
-
const uploadBytes = toSafeNumber(requestState.uploadBytes);
|
|
1939
|
-
const downloadBytes = toSafeNumber(requestState.downloadBytes);
|
|
1940
|
-
const totalBytes = uploadBytes + downloadBytes;
|
|
1941
|
-
debugLog(
|
|
1942
|
-
`final id=${requestId || "-"} source=loadingFailed status=failed route=${routed} type=${requestState.resourceType || "other"} upload=${formatBytes(uploadBytes)} (${uploadBytes}) download=${formatBytes(downloadBytes)} (${downloadBytes}) total=${formatBytes(totalBytes)} (${totalBytes}) canceled=${canceled} reason=${reason} url=${requestState.url || "-"}`
|
|
1943
|
-
);
|
|
1944
|
-
} else {
|
|
1945
|
-
const orphanDownload = toSafeNumber(orphanReceivedMap.get(requestId));
|
|
1946
|
-
addFailed(routed, canceled);
|
|
1947
|
-
addFailedProfile(state, routed, UNKNOWN_DOMAIN, "other", reason, canceled);
|
|
1948
|
-
debugLog(
|
|
1949
|
-
`final id=${requestId || "-"} source=loadingFailed status=orphan-failed route=${routed} upload=0B (0) download=${formatBytes(orphanDownload)} (${orphanDownload}) total=${formatBytes(orphanDownload)} (${orphanDownload}) canceled=${canceled} reason=${reason} url=-`
|
|
1950
|
-
);
|
|
1951
|
-
}
|
|
1952
|
-
if (requestId) {
|
|
1953
|
-
requestMap.delete(requestId);
|
|
1954
|
-
orphanReceivedMap.delete(requestId);
|
|
1955
|
-
}
|
|
1956
|
-
};
|
|
1957
|
-
const bindWebSocketRoute = (params = {}) => {
|
|
1958
|
-
const requestId = String(params.requestId || "");
|
|
1959
|
-
if (!requestId) return;
|
|
1960
|
-
const url = String(params.url || "");
|
|
1961
|
-
const route = resolveRoute(url);
|
|
1962
|
-
wsRouteMap.set(requestId, { route, url, domain: normalizeDomainKey(parseHostname(url)) });
|
|
1963
|
-
};
|
|
1964
|
-
const clearWebSocketRoute = (params = {}) => {
|
|
1965
|
-
const requestId = String(params.requestId || "");
|
|
1966
|
-
if (!requestId) return;
|
|
1967
|
-
wsRouteMap.delete(requestId);
|
|
1968
|
-
};
|
|
1969
|
-
const getWebSocketMeta = (requestId = "") => {
|
|
1970
|
-
if (!requestId) return { route: fallbackRoute(), url: "", domain: UNKNOWN_DOMAIN };
|
|
1971
|
-
const meta = wsRouteMap.get(requestId);
|
|
1972
|
-
if (!meta || typeof meta !== "object") {
|
|
1973
|
-
return { route: fallbackRoute(), url: "", domain: UNKNOWN_DOMAIN };
|
|
1974
|
-
}
|
|
1975
|
-
return {
|
|
1976
|
-
route: ensureRoute(meta.route),
|
|
1977
|
-
url: String(meta.url || ""),
|
|
1978
|
-
domain: normalizeDomainKey(meta.domain)
|
|
1979
|
-
};
|
|
1980
|
-
};
|
|
1981
|
-
const recordWebSocketFrameSent = (params = {}) => {
|
|
1982
|
-
const requestId = String(params.requestId || "");
|
|
1983
|
-
const { route, url, domain } = getWebSocketMeta(requestId);
|
|
1984
|
-
const payload = params.response && typeof params.response === "object" ? params.response.payloadData : "";
|
|
1985
|
-
const bytes = byteLength(payload || "");
|
|
1986
|
-
addUploadBytes(state, route, bytes);
|
|
1987
|
-
addUploadProfile(state, route, domain, "websocket", bytes);
|
|
1988
|
-
debugLog(`ws-send id=${requestId || "-"} route=${route} bytes=${formatBytes(bytes)} (${bytes}) url=${url || "-"}`);
|
|
1989
|
-
};
|
|
1990
|
-
const recordWebSocketFrameReceived = (params = {}) => {
|
|
1991
|
-
const requestId = String(params.requestId || "");
|
|
1992
|
-
const { route, url, domain } = getWebSocketMeta(requestId);
|
|
1993
|
-
const payload = params.response && typeof params.response === "object" ? params.response.payloadData : "";
|
|
1994
|
-
const bytes = byteLength(payload || "");
|
|
1995
|
-
addDownloadBytes(state, route, bytes);
|
|
1996
|
-
addDownloadProfile(state, route, domain, "websocket", bytes);
|
|
1997
|
-
debugLog(`ws-recv id=${requestId || "-"} route=${route} bytes=${formatBytes(bytes)} (${bytes}) url=${url || "-"}`);
|
|
1998
|
-
};
|
|
1999
|
-
const attachPage = async (page) => {
|
|
2000
|
-
if (!page || typeof page.context !== "function") return;
|
|
2001
|
-
if (attachedPages.has(page)) return;
|
|
2002
|
-
attachedPages.add(page);
|
|
2003
|
-
try {
|
|
2004
|
-
const context = page.context();
|
|
2005
|
-
if (!context || typeof context.newCDPSession !== "function") {
|
|
2006
|
-
return;
|
|
2007
|
-
}
|
|
2008
|
-
if (!attachedContexts.has(context) && typeof context.on === "function") {
|
|
2009
|
-
attachedContexts.add(context);
|
|
2010
|
-
context.on("page", (nextPage) => {
|
|
2011
|
-
if (!nextPage) return;
|
|
2012
|
-
attachPage(nextPage).catch((error) => {
|
|
2013
|
-
logger6.warn(`\u5B50\u9875\u9762 CDP \u76D1\u542C\u6CE8\u518C\u5931\u8D25: ${error?.message || error}`);
|
|
2014
|
-
});
|
|
2015
|
-
});
|
|
2016
|
-
}
|
|
2017
|
-
const session = await context.newCDPSession(page);
|
|
2018
|
-
await session.send("Network.enable");
|
|
2019
|
-
session.on("Network.requestWillBeSent", recordRequest);
|
|
2020
|
-
session.on("Network.dataReceived", recordDataReceived);
|
|
2021
|
-
session.on("Network.loadingFinished", recordLoadingFinished);
|
|
2022
|
-
session.on("Network.loadingFailed", recordRequestFailed);
|
|
2023
|
-
session.on("Network.webSocketCreated", bindWebSocketRoute);
|
|
2024
|
-
session.on("Network.webSocketClosed", clearWebSocketRoute);
|
|
2025
|
-
session.on("Network.webSocketFrameSent", recordWebSocketFrameSent);
|
|
2026
|
-
session.on("Network.webSocketFrameReceived", recordWebSocketFrameReceived);
|
|
2027
|
-
debugLog("CDP \u76D1\u542C\u5DF2\u6CE8\u518C");
|
|
2028
|
-
} catch (error) {
|
|
2029
|
-
logger6.warn(`CDP \u76D1\u542C\u6CE8\u518C\u5931\u8D25: ${error?.message || error}`);
|
|
2030
|
-
}
|
|
2031
|
-
};
|
|
2032
|
-
const snapshot = () => {
|
|
2033
|
-
const totalBytes = state.totalUploadBytes + state.totalDownloadBytes;
|
|
2034
|
-
const proxyBytes = state.proxyUploadBytes + state.proxyDownloadBytes;
|
|
2035
|
-
const directBytes = state.directUploadBytes + state.directDownloadBytes;
|
|
2036
|
-
return {
|
|
2037
|
-
meter: "cdp-data-received-v3",
|
|
2038
|
-
totalRequests: state.totalRequests,
|
|
2039
|
-
proxyRequests: state.proxyRequests,
|
|
2040
|
-
directRequests: state.directRequests,
|
|
2041
|
-
totalUploadBytes: state.totalUploadBytes,
|
|
2042
|
-
proxyUploadBytes: state.proxyUploadBytes,
|
|
2043
|
-
directUploadBytes: state.directUploadBytes,
|
|
2044
|
-
totalDownloadBytes: state.totalDownloadBytes,
|
|
2045
|
-
proxyDownloadBytes: state.proxyDownloadBytes,
|
|
2046
|
-
directDownloadBytes: state.directDownloadBytes,
|
|
2047
|
-
totalBytes,
|
|
2048
|
-
proxyBytes,
|
|
2049
|
-
directBytes,
|
|
2050
|
-
totalFailedRequests: state.totalFailedRequests,
|
|
2051
|
-
proxyFailedRequests: state.proxyFailedRequests,
|
|
2052
|
-
directFailedRequests: state.directFailedRequests,
|
|
2053
|
-
totalCanceledRequests: state.totalCanceledRequests,
|
|
2054
|
-
proxyCanceledRequests: state.proxyCanceledRequests,
|
|
2055
|
-
directCanceledRequests: state.directCanceledRequests,
|
|
2056
|
-
orphanDataReceivedBytes: state.orphanDataReceivedBytes,
|
|
2057
|
-
orphanProxyDataReceivedBytes: state.orphanProxyDataReceivedBytes,
|
|
2058
|
-
orphanFinishDeltaBytes: state.orphanFinishDeltaBytes,
|
|
2059
|
-
orphanProxyFinishDeltaBytes: state.orphanProxyFinishDeltaBytes,
|
|
2060
|
-
openRequestCount: requestMap.size,
|
|
2061
|
-
orphanOpenCount: orphanReceivedMap.size,
|
|
2062
|
-
topDomains: buildTopDomains(state),
|
|
2063
|
-
topResourceTypes: buildTopResourceTypes(state),
|
|
2064
|
-
failedReasons: buildFailedReasons(state),
|
|
2065
|
-
proxyBypassHints: buildProxyByPassHints(state)
|
|
2066
|
-
};
|
|
2067
|
-
};
|
|
2068
|
-
const reset = () => {
|
|
2069
|
-
Object.assign(state, createTrafficState());
|
|
2070
|
-
requestMap.clear();
|
|
2071
|
-
orphanReceivedMap.clear();
|
|
2072
|
-
wsRouteMap.clear();
|
|
2073
|
-
};
|
|
2074
|
-
return {
|
|
2075
|
-
attachPage,
|
|
2076
|
-
snapshot,
|
|
2077
|
-
reset
|
|
2078
|
-
};
|
|
2079
|
-
};
|
|
2080
1738
|
|
|
2081
1739
|
// src/launch.js
|
|
2082
1740
|
var logger7 = createInternalLogger("Launch");
|
|
1741
|
+
var REQUEST_HOOK_FLAG = Symbol("playwright-toolkit-request-hook");
|
|
2083
1742
|
var DEFAULT_CRAWLER_BASE_OPTIONS = Object.freeze({
|
|
2084
1743
|
maxConcurrency: 1,
|
|
2085
1744
|
maxRequestRetries: 0,
|
|
@@ -2091,24 +1750,10 @@ var resolveProxyLaunchOptions = (proxyConfiguration = {}) => {
|
|
|
2091
1750
|
const proxyUrl = String(config.proxy_url || "").trim();
|
|
2092
1751
|
const enableProxy = typeof config.enable_proxy === "boolean" ? config.enable_proxy : proxyUrl !== "";
|
|
2093
1752
|
if (!enableProxy || !proxyUrl) {
|
|
2094
|
-
return {
|
|
1753
|
+
return { byPassDomains: [], enableProxy, proxyUrl };
|
|
2095
1754
|
}
|
|
2096
1755
|
const byPassDomains = normalizeByPassDomains(config.by_pass_domains);
|
|
2097
|
-
|
|
2098
|
-
parsedProxyUrl = new URL(proxyUrl);
|
|
2099
|
-
const launchProxy = {
|
|
2100
|
-
server: `${parsedProxyUrl.protocol}//${parsedProxyUrl.host}`
|
|
2101
|
-
};
|
|
2102
|
-
if (parsedProxyUrl.username) {
|
|
2103
|
-
launchProxy.username = decodeURIComponent(parsedProxyUrl.username);
|
|
2104
|
-
}
|
|
2105
|
-
if (parsedProxyUrl.password) {
|
|
2106
|
-
launchProxy.password = decodeURIComponent(parsedProxyUrl.password);
|
|
2107
|
-
}
|
|
2108
|
-
if (byPassDomains.length > 0) {
|
|
2109
|
-
launchProxy.bypass = byPassDomains.join(",");
|
|
2110
|
-
}
|
|
2111
|
-
return { launchProxy, byPassDomains, enableProxy, proxyUrl };
|
|
1756
|
+
return { byPassDomains, enableProxy, proxyUrl };
|
|
2112
1757
|
};
|
|
2113
1758
|
var Launch = {
|
|
2114
1759
|
getPlaywrightCrawlerOptions(options = {}) {
|
|
@@ -2124,14 +1769,13 @@ var Launch = {
|
|
|
2124
1769
|
preNavigationHooks = [],
|
|
2125
1770
|
postNavigationHooks = []
|
|
2126
1771
|
} = normalizedOptions;
|
|
2127
|
-
const {
|
|
1772
|
+
const { byPassDomains, enableProxy, proxyUrl } = resolveProxyLaunchOptions(proxyConfiguration);
|
|
2128
1773
|
const byPassRules = buildByPassDomainRules(byPassDomains);
|
|
2129
|
-
const
|
|
2130
|
-
|
|
2131
|
-
|
|
2132
|
-
|
|
2133
|
-
}
|
|
2134
|
-
setTrafficMeter(trafficMeter2);
|
|
1774
|
+
const proxyMeter = enableProxy && proxyUrl ? startProxyMeter({ proxyUrl, debugMode }) : null;
|
|
1775
|
+
const launchProxy = proxyMeter ? { server: proxyMeter.server } : null;
|
|
1776
|
+
if (launchProxy && byPassDomains.length > 0) {
|
|
1777
|
+
launchProxy.bypass = byPassDomains.join(",");
|
|
1778
|
+
}
|
|
2135
1779
|
const launchOptions = {
|
|
2136
1780
|
args: [
|
|
2137
1781
|
...AntiCheat.getLaunchArgs(),
|
|
@@ -2144,8 +1788,14 @@ var Launch = {
|
|
|
2144
1788
|
}
|
|
2145
1789
|
const enableByPassLogger = Boolean(logOptions && logOptions.enable);
|
|
2146
1790
|
if (enableByPassLogger && launchProxy) {
|
|
1791
|
+
let upstreamLabel = "";
|
|
1792
|
+
try {
|
|
1793
|
+
const parsedProxyUrl = new URL(proxyUrl);
|
|
1794
|
+
upstreamLabel = `${parsedProxyUrl.protocol}//${parsedProxyUrl.host}`;
|
|
1795
|
+
} catch {
|
|
1796
|
+
}
|
|
2147
1797
|
logger7.info(
|
|
2148
|
-
`[\u4EE3\u7406\u5DF2\u542F\u7528] \
|
|
1798
|
+
`[\u4EE3\u7406\u5DF2\u542F\u7528] \u672C\u5730=${launchProxy.server} \u4E0A\u6E38=${upstreamLabel || "-"} \u76F4\u8FDE\u57DF\u540D=${(byPassDomains || []).join(",")}`
|
|
2149
1799
|
);
|
|
2150
1800
|
logger7.info(`[\u6D41\u91CF\u89C2\u6D4B] \u9010\u8BF7\u6C42\u8C03\u8BD5=${Boolean(debugMode) ? "\u5F00\u542F" : "\u5173\u95ED"}\uFF08\u6C47\u603B\u59CB\u7EC8\u5F00\u542F\uFF09`);
|
|
2151
1801
|
} else if (enableByPassLogger && enableProxy && !launchProxy) {
|
|
@@ -2165,17 +1815,23 @@ var Launch = {
|
|
|
2165
1815
|
const recommendedGotoOptions = {
|
|
2166
1816
|
waitUntil: "commit"
|
|
2167
1817
|
};
|
|
2168
|
-
if (page
|
|
2169
|
-
|
|
1818
|
+
if (!page || typeof page.on !== "function") {
|
|
1819
|
+
return recommendedGotoOptions;
|
|
2170
1820
|
}
|
|
2171
|
-
if (
|
|
1821
|
+
if (page[REQUEST_HOOK_FLAG]) {
|
|
2172
1822
|
return recommendedGotoOptions;
|
|
2173
1823
|
}
|
|
1824
|
+
page[REQUEST_HOOK_FLAG] = true;
|
|
2174
1825
|
const requestHandler = (req) => {
|
|
2175
1826
|
const requestUrl = req.url();
|
|
1827
|
+
const resourceType = req.resourceType();
|
|
1828
|
+
if (launchProxy) {
|
|
1829
|
+
recordProxyMeterResourceType(requestUrl, resourceType);
|
|
1830
|
+
}
|
|
1831
|
+
if (!enableByPassLogger || byPassDomains.length === 0) return;
|
|
2176
1832
|
const matched = findMatchedByPassRule(byPassRules, requestUrl);
|
|
2177
1833
|
if (!matched || !matched.rule) return;
|
|
2178
|
-
logger7.info(`[\u76F4\u8FDE\u547D\u4E2D] \u89C4\u5219=${matched.rule.pattern} \u57DF\u540D=${matched.hostname} \u8D44\u6E90\u7C7B\u578B=${
|
|
1834
|
+
logger7.info(`[\u76F4\u8FDE\u547D\u4E2D] \u89C4\u5219=${matched.rule.pattern} \u57DF\u540D=${matched.hostname} \u8D44\u6E90\u7C7B\u578B=${resourceType} \u65B9\u6CD5=${req.method()} \u5730\u5740=${requestUrl}`);
|
|
2179
1835
|
};
|
|
2180
1836
|
page.on("request", requestHandler);
|
|
2181
1837
|
return recommendedGotoOptions;
|
|
@@ -2395,7 +2051,7 @@ var Captcha = {
|
|
|
2395
2051
|
|
|
2396
2052
|
// src/sse.js
|
|
2397
2053
|
var import_https = __toESM(require("https"), 1);
|
|
2398
|
-
var
|
|
2054
|
+
var import_url2 = require("url");
|
|
2399
2055
|
var logger10 = createInternalLogger("Sse");
|
|
2400
2056
|
var Sse = {
|
|
2401
2057
|
/**
|
|
@@ -2505,7 +2161,7 @@ var Sse = {
|
|
|
2505
2161
|
try {
|
|
2506
2162
|
const headers = await request.allHeaders();
|
|
2507
2163
|
const postData = request.postData();
|
|
2508
|
-
const urlObj = new
|
|
2164
|
+
const urlObj = new import_url2.URL(request.url());
|
|
2509
2165
|
delete headers["accept-encoding"];
|
|
2510
2166
|
delete headers["content-length"];
|
|
2511
2167
|
const reqOptions = {
|
|
@@ -2860,10 +2516,10 @@ var Mutation = {
|
|
|
2860
2516
|
let text = "";
|
|
2861
2517
|
let html = "";
|
|
2862
2518
|
let source = "main";
|
|
2863
|
-
let
|
|
2519
|
+
let path2 = `${selector}[${index}]`;
|
|
2864
2520
|
if (isIframe) {
|
|
2865
2521
|
source = "iframe";
|
|
2866
|
-
|
|
2522
|
+
path2 = `${selector}[${index}]::iframe(${safeFrameId(node)})`;
|
|
2867
2523
|
try {
|
|
2868
2524
|
const frameDoc = node.contentDocument;
|
|
2869
2525
|
const frameRoot = frameDoc?.body || frameDoc?.documentElement;
|
|
@@ -2881,7 +2537,7 @@ var Mutation = {
|
|
|
2881
2537
|
items.push({
|
|
2882
2538
|
selector,
|
|
2883
2539
|
source,
|
|
2884
|
-
path,
|
|
2540
|
+
path: path2,
|
|
2885
2541
|
text,
|
|
2886
2542
|
html,
|
|
2887
2543
|
snapshot
|