@skrillex1224/playwright-toolkit 2.1.170 → 2.1.180
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -1
- package/dist/browser.js +96 -15
- package/dist/browser.js.map +3 -3
- package/dist/index.cjs +942 -348
- package/dist/index.cjs.map +4 -4
- package/dist/index.js +942 -348
- package/dist/index.js.map +4 -4
- package/index.d.ts +89 -39
- package/package.json +3 -1
- /package/dist/{proxy-meter.js → internals/proxy-meter.js} +0 -0
package/dist/index.cjs
CHANGED
|
@@ -335,18 +335,18 @@ var fallbackLog = {
|
|
|
335
335
|
error: (...args) => console.error(...args),
|
|
336
336
|
debug: (...args) => console.debug ? console.debug(...args) : console.log(...args)
|
|
337
337
|
};
|
|
338
|
-
var resolveLogMethod = (
|
|
339
|
-
if (
|
|
340
|
-
return
|
|
338
|
+
var resolveLogMethod = (logger12, name) => {
|
|
339
|
+
if (logger12 && typeof logger12[name] === "function") {
|
|
340
|
+
return logger12[name].bind(logger12);
|
|
341
341
|
}
|
|
342
|
-
if (name === "warning" &&
|
|
343
|
-
return
|
|
342
|
+
if (name === "warning" && logger12 && typeof logger12.warn === "function") {
|
|
343
|
+
return logger12.warn.bind(logger12);
|
|
344
344
|
}
|
|
345
345
|
return fallbackLog[name];
|
|
346
346
|
};
|
|
347
347
|
var defaultLogger = null;
|
|
348
|
-
var setDefaultLogger = (
|
|
349
|
-
defaultLogger =
|
|
348
|
+
var setDefaultLogger = (logger12) => {
|
|
349
|
+
defaultLogger = logger12;
|
|
350
350
|
};
|
|
351
351
|
var resolveLogger = (explicitLogger) => {
|
|
352
352
|
if (explicitLogger && typeof explicitLogger.info === "function") {
|
|
@@ -373,8 +373,8 @@ var colorize = (text, color) => {
|
|
|
373
373
|
var createBaseLogger = (prefix = "", explicitLogger) => {
|
|
374
374
|
const name = prefix ? String(prefix) : "";
|
|
375
375
|
const dispatch = (methodName, icon, message, color) => {
|
|
376
|
-
const
|
|
377
|
-
const logFn = resolveLogMethod(
|
|
376
|
+
const logger12 = resolveLogger(explicitLogger);
|
|
377
|
+
const logFn = resolveLogMethod(logger12, methodName);
|
|
378
378
|
const timestamp = colorize(`[${formatTimestamp()}]`, ANSI.gray);
|
|
379
379
|
const line = formatLine(name, icon, message);
|
|
380
380
|
const coloredLine = colorize(line, color);
|
|
@@ -573,7 +573,7 @@ var CrawlerError = class _CrawlerError extends Error {
|
|
|
573
573
|
// src/apify-kit.js
|
|
574
574
|
var import_serialize_error2 = require("serialize-error");
|
|
575
575
|
|
|
576
|
-
// src/proxy-meter-runtime.js
|
|
576
|
+
// src/internals/proxy-meter-runtime.js
|
|
577
577
|
var import_child_process = require("child_process");
|
|
578
578
|
var import_fs = require("fs");
|
|
579
579
|
var import_os = require("os");
|
|
@@ -643,6 +643,10 @@ var resolveResourceTypeMeta = (domain) => {
|
|
|
643
643
|
};
|
|
644
644
|
var resolveScriptPath = () => {
|
|
645
645
|
const baseDir = typeof __dirname !== "undefined" ? __dirname : import_path.default.dirname((0, import_url.fileURLToPath)(import_meta.url));
|
|
646
|
+
const internalCandidate = import_path.default.join(baseDir, "internals", "proxy-meter.js");
|
|
647
|
+
if ((0, import_fs.existsSync)(internalCandidate)) {
|
|
648
|
+
return internalCandidate;
|
|
649
|
+
}
|
|
646
650
|
return import_path.default.join(baseDir, "proxy-meter.js");
|
|
647
651
|
};
|
|
648
652
|
var pickFreePort = () => {
|
|
@@ -926,9 +930,589 @@ var getProxyMeterSnapshot = async (options = {}) => {
|
|
|
926
930
|
}
|
|
927
931
|
return snapshot;
|
|
928
932
|
};
|
|
933
|
+
var ProxyMeterRuntime = {
|
|
934
|
+
startProxyMeter,
|
|
935
|
+
recordProxyMeterResourceType,
|
|
936
|
+
stopProxyMeter,
|
|
937
|
+
getProxyMeterSnapshot
|
|
938
|
+
};
|
|
939
|
+
|
|
940
|
+
// src/runtime-env.js
|
|
941
|
+
var BROWSER_PROFILE_SCHEMA_VERSION = 1;
|
|
942
|
+
var rememberedRuntimeState = null;
|
|
943
|
+
var isPlainObject = (value) => value && typeof value === "object" && !Array.isArray(value);
|
|
944
|
+
var deepClone = (value) => {
|
|
945
|
+
if (value == null) return value;
|
|
946
|
+
try {
|
|
947
|
+
return JSON.parse(JSON.stringify(value));
|
|
948
|
+
} catch {
|
|
949
|
+
return value;
|
|
950
|
+
}
|
|
951
|
+
};
|
|
952
|
+
var tryParseJSON = (value) => {
|
|
953
|
+
if (value == null) return null;
|
|
954
|
+
if (typeof value === "object") return value;
|
|
955
|
+
const raw = String(value || "").trim();
|
|
956
|
+
if (!raw) return null;
|
|
957
|
+
try {
|
|
958
|
+
return JSON.parse(raw);
|
|
959
|
+
} catch {
|
|
960
|
+
return null;
|
|
961
|
+
}
|
|
962
|
+
};
|
|
963
|
+
var normalizeLocalStorage = (value) => {
|
|
964
|
+
const source = value && typeof value === "object" && !Array.isArray(value) ? value : {};
|
|
965
|
+
return Object.entries(source).reduce((acc, [key, item]) => {
|
|
966
|
+
const safeKey = String(key || "").trim();
|
|
967
|
+
const safeValue = String(item ?? "").trim();
|
|
968
|
+
if (!safeKey || !safeValue || safeValue === "<nil>") return acc;
|
|
969
|
+
acc[safeKey] = safeValue;
|
|
970
|
+
return acc;
|
|
971
|
+
}, {});
|
|
972
|
+
};
|
|
973
|
+
var normalizeSessionStorage = (value) => {
|
|
974
|
+
const source = value && typeof value === "object" && !Array.isArray(value) ? value : {};
|
|
975
|
+
return Object.entries(source).reduce((acc, [key, item]) => {
|
|
976
|
+
const safeKey = String(key || "").trim();
|
|
977
|
+
const safeValue = String(item ?? "").trim();
|
|
978
|
+
if (!safeKey || !safeValue || safeValue === "<nil>") return acc;
|
|
979
|
+
acc[safeKey] = safeValue;
|
|
980
|
+
return acc;
|
|
981
|
+
}, {});
|
|
982
|
+
};
|
|
983
|
+
var normalizeAuth = (value) => {
|
|
984
|
+
const source = value && typeof value === "object" && !Array.isArray(value) ? value : {};
|
|
985
|
+
return Object.entries(source).reduce((acc, [key, item]) => {
|
|
986
|
+
const safeKey = String(key || "").trim();
|
|
987
|
+
const safeValue = String(item ?? "").trim();
|
|
988
|
+
if (!safeKey || !safeValue || safeValue === "<nil>") return acc;
|
|
989
|
+
acc[safeKey] = safeValue;
|
|
990
|
+
return acc;
|
|
991
|
+
}, {});
|
|
992
|
+
};
|
|
993
|
+
var normalizeCookies = (value) => {
|
|
994
|
+
if (!Array.isArray(value)) return [];
|
|
995
|
+
return value.map((item) => {
|
|
996
|
+
const raw = item && typeof item === "object" ? item : {};
|
|
997
|
+
const name = String(raw.name || "").trim();
|
|
998
|
+
const cookieValue = String(raw.value ?? "").trim();
|
|
999
|
+
if (!name || !cookieValue || cookieValue === "<nil>") return null;
|
|
1000
|
+
const domain = String(raw.domain || "").trim();
|
|
1001
|
+
const path2 = String(raw.path || "").trim() || "/";
|
|
1002
|
+
const sameSiteRaw = String(raw.sameSite || "").trim();
|
|
1003
|
+
return {
|
|
1004
|
+
...raw,
|
|
1005
|
+
name,
|
|
1006
|
+
value: cookieValue,
|
|
1007
|
+
domain,
|
|
1008
|
+
path: path2,
|
|
1009
|
+
...sameSiteRaw ? { sameSite: sameSiteRaw } : {}
|
|
1010
|
+
};
|
|
1011
|
+
}).filter(Boolean);
|
|
1012
|
+
};
|
|
1013
|
+
var buildCookieMap = (cookies) => cookies.reduce((acc, item) => {
|
|
1014
|
+
acc[item.name] = item.value;
|
|
1015
|
+
return acc;
|
|
1016
|
+
}, {});
|
|
1017
|
+
var normalizeObservedBrowserProfile = (value) => {
|
|
1018
|
+
const source = isPlainObject(value) ? value : {};
|
|
1019
|
+
const profile = {};
|
|
1020
|
+
const stringFields = {
|
|
1021
|
+
user_agent: source.user_agent,
|
|
1022
|
+
platform: source.platform,
|
|
1023
|
+
language: source.language,
|
|
1024
|
+
timezone: source.timezone
|
|
1025
|
+
};
|
|
1026
|
+
Object.entries(stringFields).forEach(([key, item]) => {
|
|
1027
|
+
const safeValue = String(item || "").trim();
|
|
1028
|
+
if (safeValue) {
|
|
1029
|
+
profile[key] = safeValue;
|
|
1030
|
+
}
|
|
1031
|
+
});
|
|
1032
|
+
if (Array.isArray(source.languages)) {
|
|
1033
|
+
const languages = source.languages.map((item) => String(item || "").trim()).filter(Boolean);
|
|
1034
|
+
if (languages.length > 0) {
|
|
1035
|
+
profile.languages = languages;
|
|
1036
|
+
}
|
|
1037
|
+
}
|
|
1038
|
+
const numericFields = {
|
|
1039
|
+
hardware_concurrency: Number(source.hardware_concurrency || 0),
|
|
1040
|
+
device_memory: Number(source.device_memory || 0)
|
|
1041
|
+
};
|
|
1042
|
+
Object.entries(numericFields).forEach(([key, item]) => {
|
|
1043
|
+
if (Number.isFinite(item) && item > 0) {
|
|
1044
|
+
profile[key] = item;
|
|
1045
|
+
}
|
|
1046
|
+
});
|
|
1047
|
+
if (isPlainObject(source.viewport)) {
|
|
1048
|
+
const width = Number(source.viewport.width || 0);
|
|
1049
|
+
const height = Number(source.viewport.height || 0);
|
|
1050
|
+
if (width > 0 && height > 0) {
|
|
1051
|
+
profile.viewport = { width, height };
|
|
1052
|
+
}
|
|
1053
|
+
}
|
|
1054
|
+
if (isPlainObject(source.screen)) {
|
|
1055
|
+
const screenProfile = {};
|
|
1056
|
+
const screenFields = {
|
|
1057
|
+
width: Number(source.screen.width || 0),
|
|
1058
|
+
height: Number(source.screen.height || 0),
|
|
1059
|
+
avail_width: Number(source.screen.avail_width || 0),
|
|
1060
|
+
avail_height: Number(source.screen.avail_height || 0),
|
|
1061
|
+
color_depth: Number(source.screen.color_depth || 0),
|
|
1062
|
+
pixel_depth: Number(source.screen.pixel_depth || 0)
|
|
1063
|
+
};
|
|
1064
|
+
Object.entries(screenFields).forEach(([key, item]) => {
|
|
1065
|
+
if (Number.isFinite(item) && item > 0) {
|
|
1066
|
+
screenProfile[key] = item;
|
|
1067
|
+
}
|
|
1068
|
+
});
|
|
1069
|
+
if (Object.keys(screenProfile).length > 0) {
|
|
1070
|
+
profile.screen = screenProfile;
|
|
1071
|
+
}
|
|
1072
|
+
}
|
|
1073
|
+
return profile;
|
|
1074
|
+
};
|
|
1075
|
+
var normalizeBrowserProfileCore = (value) => {
|
|
1076
|
+
const source = isPlainObject(value) ? value : {};
|
|
1077
|
+
const profile = {};
|
|
1078
|
+
if (isPlainObject(source.fingerprint) && Object.keys(source.fingerprint).length > 0) {
|
|
1079
|
+
profile.fingerprint = deepClone(source.fingerprint);
|
|
1080
|
+
}
|
|
1081
|
+
const timezoneId = String(source.timezone_id || "").trim();
|
|
1082
|
+
if (timezoneId) {
|
|
1083
|
+
profile.timezone_id = timezoneId;
|
|
1084
|
+
}
|
|
1085
|
+
const locale = String(source.locale || "").trim();
|
|
1086
|
+
if (locale) {
|
|
1087
|
+
profile.locale = locale;
|
|
1088
|
+
}
|
|
1089
|
+
const profileKey = String(source.profile_key || "").trim();
|
|
1090
|
+
if (profileKey) {
|
|
1091
|
+
profile.profile_key = profileKey;
|
|
1092
|
+
}
|
|
1093
|
+
const browserMajorVersion = Number(source.browser_major_version || 0);
|
|
1094
|
+
if (Number.isFinite(browserMajorVersion) && browserMajorVersion > 0) {
|
|
1095
|
+
profile.browser_major_version = browserMajorVersion;
|
|
1096
|
+
}
|
|
1097
|
+
const schemaVersion = Number(source.schema_version || 0);
|
|
1098
|
+
if (Number.isFinite(schemaVersion) && schemaVersion > 0) {
|
|
1099
|
+
profile.schema_version = schemaVersion;
|
|
1100
|
+
} else if (Object.keys(profile).length > 0) {
|
|
1101
|
+
profile.schema_version = BROWSER_PROFILE_SCHEMA_VERSION;
|
|
1102
|
+
}
|
|
1103
|
+
return profile;
|
|
1104
|
+
};
|
|
1105
|
+
var buildBrowserProfilePayload = (core = {}, observed = {}) => {
|
|
1106
|
+
const payload = {};
|
|
1107
|
+
if (isPlainObject(core) && Object.keys(core).length > 0) {
|
|
1108
|
+
payload.core = core;
|
|
1109
|
+
}
|
|
1110
|
+
if (isPlainObject(observed) && Object.keys(observed).length > 0) {
|
|
1111
|
+
payload.observed = observed;
|
|
1112
|
+
}
|
|
1113
|
+
return payload;
|
|
1114
|
+
};
|
|
1115
|
+
var mergePlainObject = (target = {}, source = {}) => {
|
|
1116
|
+
const next = isPlainObject(target) ? deepClone(target) : {};
|
|
1117
|
+
if (!isPlainObject(source)) return next;
|
|
1118
|
+
Object.entries(source).forEach(([key, value]) => {
|
|
1119
|
+
if (!key) return;
|
|
1120
|
+
if (isPlainObject(value) && isPlainObject(next[key])) {
|
|
1121
|
+
next[key] = mergePlainObject(next[key], value);
|
|
1122
|
+
return;
|
|
1123
|
+
}
|
|
1124
|
+
next[key] = deepClone(value);
|
|
1125
|
+
});
|
|
1126
|
+
return next;
|
|
1127
|
+
};
|
|
1128
|
+
var mergeBrowserProfilePayload = (target = {}, source = {}) => {
|
|
1129
|
+
const current = isPlainObject(target) ? target : {};
|
|
1130
|
+
const incoming = isPlainObject(source) ? source : {};
|
|
1131
|
+
const currentCore = normalizeBrowserProfileCore(current.core);
|
|
1132
|
+
const incomingCore = normalizeBrowserProfileCore(incoming.core);
|
|
1133
|
+
const currentObserved = normalizeObservedBrowserProfile(current.observed);
|
|
1134
|
+
const incomingObserved = normalizeObservedBrowserProfile(incoming.observed);
|
|
1135
|
+
let mergedCore = currentCore;
|
|
1136
|
+
if (Object.keys(mergedCore).length === 0 && Object.keys(incomingCore).length > 0) {
|
|
1137
|
+
mergedCore = incomingCore;
|
|
1138
|
+
} else if (Object.keys(incomingCore).length > 0) {
|
|
1139
|
+
const currentVersion = Number(currentCore.browser_major_version || 0);
|
|
1140
|
+
const incomingVersion = Number(incomingCore.browser_major_version || 0);
|
|
1141
|
+
if (currentVersion > 0 && incomingVersion > 0 && currentVersion !== incomingVersion) {
|
|
1142
|
+
mergedCore = incomingCore;
|
|
1143
|
+
}
|
|
1144
|
+
}
|
|
1145
|
+
const mergedObserved = mergePlainObject(currentObserved, incomingObserved);
|
|
1146
|
+
return buildBrowserProfilePayload(mergedCore, mergedObserved);
|
|
1147
|
+
};
|
|
1148
|
+
var mergeEnvPatchObjects = (...patches) => {
|
|
1149
|
+
const merged = {};
|
|
1150
|
+
patches.forEach((patch) => {
|
|
1151
|
+
if (!isPlainObject(patch)) return;
|
|
1152
|
+
Object.entries(patch).forEach(([key, value]) => {
|
|
1153
|
+
if (!key) return;
|
|
1154
|
+
if (key === "browser_profile") {
|
|
1155
|
+
const browserProfile = mergeBrowserProfilePayload(merged.browser_profile, value);
|
|
1156
|
+
if (Object.keys(browserProfile).length > 0) {
|
|
1157
|
+
merged.browser_profile = browserProfile;
|
|
1158
|
+
}
|
|
1159
|
+
return;
|
|
1160
|
+
}
|
|
1161
|
+
if (isPlainObject(value) && isPlainObject(merged[key])) {
|
|
1162
|
+
merged[key] = mergePlainObject(merged[key], value);
|
|
1163
|
+
return;
|
|
1164
|
+
}
|
|
1165
|
+
merged[key] = deepClone(value);
|
|
1166
|
+
});
|
|
1167
|
+
});
|
|
1168
|
+
return Object.keys(merged).length > 0 ? merged : null;
|
|
1169
|
+
};
|
|
1170
|
+
var normalizeBrowserProfile = (value) => {
|
|
1171
|
+
const source = isPlainObject(value) ? value : {};
|
|
1172
|
+
const coreSource = isPlainObject(source.core) ? source.core : {};
|
|
1173
|
+
const observedSource = isPlainObject(source.observed) ? source.observed : {};
|
|
1174
|
+
const core = normalizeBrowserProfileCore(coreSource);
|
|
1175
|
+
const observed = normalizeObservedBrowserProfile(observedSource);
|
|
1176
|
+
return {
|
|
1177
|
+
core,
|
|
1178
|
+
observed,
|
|
1179
|
+
payload: buildBrowserProfilePayload(core, observed)
|
|
1180
|
+
};
|
|
1181
|
+
};
|
|
1182
|
+
var rememberRuntimeState = (state) => {
|
|
1183
|
+
rememberedRuntimeState = deepClone(state);
|
|
1184
|
+
return rememberedRuntimeState;
|
|
1185
|
+
};
|
|
1186
|
+
var normalizeRuntimeState = (source = {}, actor = "") => {
|
|
1187
|
+
if (source && typeof source === "object" && source.runtime && typeof source.runtime === "object" && !Array.isArray(source.runtime) && Object.prototype.hasOwnProperty.call(source, "browserProfileCore")) {
|
|
1188
|
+
return source;
|
|
1189
|
+
}
|
|
1190
|
+
return RuntimeEnv.parseInput(source, actor);
|
|
1191
|
+
};
|
|
1192
|
+
var RuntimeEnv = {
|
|
1193
|
+
tryParseJSON,
|
|
1194
|
+
normalizeCookies,
|
|
1195
|
+
normalizeLocalStorage,
|
|
1196
|
+
normalizeSessionStorage,
|
|
1197
|
+
normalizeAuth,
|
|
1198
|
+
normalizeBrowserProfileCore,
|
|
1199
|
+
normalizeObservedBrowserProfile,
|
|
1200
|
+
mergeEnvPatches(...patches) {
|
|
1201
|
+
return mergeEnvPatchObjects(...patches);
|
|
1202
|
+
},
|
|
1203
|
+
// parseInput 把 Actor 输入里的 runtime 字段标准化成 toolkit 内部统一结构。
|
|
1204
|
+
// 后续 visitor 只和这个 state 交互,不再自己拼 token/cookie 逻辑。
|
|
1205
|
+
parseInput(input = {}, actor = "") {
|
|
1206
|
+
const runtime2 = tryParseJSON(input?.runtime) || {};
|
|
1207
|
+
const resolvedActor = String(actor || input?.actor || "").trim();
|
|
1208
|
+
const cookies = normalizeCookies(runtime2?.cookies);
|
|
1209
|
+
const cookieMap = buildCookieMap(cookies);
|
|
1210
|
+
const localStorage = normalizeLocalStorage(runtime2?.local_storage);
|
|
1211
|
+
const sessionStorage = normalizeSessionStorage(runtime2?.session_storage);
|
|
1212
|
+
const auth = normalizeAuth(runtime2?.auth);
|
|
1213
|
+
const browserProfile = normalizeBrowserProfile(runtime2?.browser_profile);
|
|
1214
|
+
const envId = String(input?.env_id || "").trim();
|
|
1215
|
+
const normalizedRuntime = {
|
|
1216
|
+
...runtime2,
|
|
1217
|
+
...cookies.length > 0 ? { cookies } : {},
|
|
1218
|
+
...Object.keys(localStorage).length > 0 ? { local_storage: localStorage } : {},
|
|
1219
|
+
...Object.keys(sessionStorage).length > 0 ? { session_storage: sessionStorage } : {},
|
|
1220
|
+
...Object.keys(auth).length > 0 ? { auth } : {}
|
|
1221
|
+
};
|
|
1222
|
+
delete normalizedRuntime.actor;
|
|
1223
|
+
delete normalizedRuntime.env_id;
|
|
1224
|
+
if (Object.keys(browserProfile.payload).length > 0) {
|
|
1225
|
+
normalizedRuntime.browser_profile = browserProfile.payload;
|
|
1226
|
+
} else {
|
|
1227
|
+
delete normalizedRuntime.browser_profile;
|
|
1228
|
+
}
|
|
1229
|
+
const state = {
|
|
1230
|
+
actor: resolvedActor,
|
|
1231
|
+
runtime: normalizedRuntime,
|
|
1232
|
+
envId,
|
|
1233
|
+
auth,
|
|
1234
|
+
cookies,
|
|
1235
|
+
cookieMap,
|
|
1236
|
+
localStorage,
|
|
1237
|
+
sessionStorage,
|
|
1238
|
+
browserProfile: browserProfile.payload,
|
|
1239
|
+
browserProfileCore: browserProfile.core,
|
|
1240
|
+
browserProfileObserved: browserProfile.observed
|
|
1241
|
+
};
|
|
1242
|
+
rememberRuntimeState(state);
|
|
1243
|
+
return state;
|
|
1244
|
+
},
|
|
1245
|
+
// buildEnvPatch 只构造允许回写到后端 env 的字段集合。
|
|
1246
|
+
buildEnvPatch(source = {}, actor = "") {
|
|
1247
|
+
const state = normalizeRuntimeState(source, actor);
|
|
1248
|
+
const browserProfile = buildBrowserProfilePayload(state.browserProfileCore, state.browserProfileObserved);
|
|
1249
|
+
const envPatch = {
|
|
1250
|
+
...Object.keys(state.auth || {}).length > 0 ? { auth: state.auth } : {},
|
|
1251
|
+
...Array.isArray(state.cookies) && state.cookies.length > 0 ? { cookies: state.cookies } : {},
|
|
1252
|
+
...Object.keys(state.localStorage || {}).length > 0 ? { local_storage: state.localStorage } : {},
|
|
1253
|
+
...Object.keys(state.sessionStorage || {}).length > 0 ? { session_storage: state.sessionStorage } : {},
|
|
1254
|
+
...Object.keys(browserProfile).length > 0 ? { browser_profile: browserProfile } : {}
|
|
1255
|
+
};
|
|
1256
|
+
return Object.keys(envPatch).length > 0 ? envPatch : null;
|
|
1257
|
+
},
|
|
1258
|
+
// hasLoginState 用来区分“必须鉴权的平台”和“可匿名运行的平台”。
|
|
1259
|
+
hasLoginState(source = {}, actor = "") {
|
|
1260
|
+
const state = normalizeRuntimeState(source, actor);
|
|
1261
|
+
return Array.isArray(state.cookies) && state.cookies.length > 0 || Object.keys(state.localStorage || {}).length > 0 || Object.keys(state.sessionStorage || {}).length > 0 || Object.keys(state.auth || {}).length > 0;
|
|
1262
|
+
},
|
|
1263
|
+
getAuthValue(source = {}, key = "", actor = "") {
|
|
1264
|
+
const state = normalizeRuntimeState(source, actor);
|
|
1265
|
+
const safeKey = String(key || "").trim();
|
|
1266
|
+
if (!safeKey) return "";
|
|
1267
|
+
return String(state.auth?.[safeKey] ?? "").trim();
|
|
1268
|
+
},
|
|
1269
|
+
rememberState(source = {}) {
|
|
1270
|
+
const state = normalizeRuntimeState(source);
|
|
1271
|
+
rememberRuntimeState(state);
|
|
1272
|
+
return RuntimeEnv.peekRememberedState();
|
|
1273
|
+
},
|
|
1274
|
+
peekRememberedState() {
|
|
1275
|
+
return rememberedRuntimeState ? deepClone(rememberedRuntimeState) : null;
|
|
1276
|
+
},
|
|
1277
|
+
getBrowserProfileCore(source = {}, actor = "") {
|
|
1278
|
+
const state = normalizeRuntimeState(source, actor);
|
|
1279
|
+
return deepClone(state.browserProfileCore || {});
|
|
1280
|
+
},
|
|
1281
|
+
setBrowserProfileCore(source = {}, core = {}, actor = "") {
|
|
1282
|
+
const state = normalizeRuntimeState(source, actor);
|
|
1283
|
+
const normalizedCore = normalizeBrowserProfileCore(core);
|
|
1284
|
+
state.browserProfileCore = normalizedCore;
|
|
1285
|
+
state.browserProfile = buildBrowserProfilePayload(normalizedCore, state.browserProfileObserved);
|
|
1286
|
+
if (Object.keys(state.browserProfile).length > 0) {
|
|
1287
|
+
state.runtime.browser_profile = state.browserProfile;
|
|
1288
|
+
} else {
|
|
1289
|
+
delete state.runtime.browser_profile;
|
|
1290
|
+
}
|
|
1291
|
+
rememberRuntimeState(state);
|
|
1292
|
+
return state;
|
|
1293
|
+
},
|
|
1294
|
+
// applyToPage 只负责把登录态相关字段注入页面:
|
|
1295
|
+
// cookies / localStorage / sessionStorage。
|
|
1296
|
+
// 指纹、时区、UA、viewport 的回放发生在 launch.js 启动阶段,不在这里做。
|
|
1297
|
+
async applyToPage(page, source = {}, options = {}) {
|
|
1298
|
+
if (!page) return;
|
|
1299
|
+
const state = normalizeRuntimeState(source, options?.actor || "");
|
|
1300
|
+
const localStorage = state.localStorage || {};
|
|
1301
|
+
const sessionStorage = state.sessionStorage || {};
|
|
1302
|
+
const cookies = (state.cookies || []).map((cookie) => {
|
|
1303
|
+
const normalized = { ...cookie };
|
|
1304
|
+
if (!normalized.path) {
|
|
1305
|
+
normalized.path = "/";
|
|
1306
|
+
}
|
|
1307
|
+
if (!normalized.domain) {
|
|
1308
|
+
return null;
|
|
1309
|
+
}
|
|
1310
|
+
return normalized;
|
|
1311
|
+
}).filter(Boolean);
|
|
1312
|
+
if (cookies.length > 0) {
|
|
1313
|
+
await page.context().addCookies(cookies);
|
|
1314
|
+
}
|
|
1315
|
+
if (Object.keys(localStorage).length > 0) {
|
|
1316
|
+
await page.addInitScript((payload) => {
|
|
1317
|
+
try {
|
|
1318
|
+
Object.entries(payload || {}).forEach(([key, value]) => {
|
|
1319
|
+
if (!key) return;
|
|
1320
|
+
window.localStorage.setItem(key, String(value ?? ""));
|
|
1321
|
+
});
|
|
1322
|
+
} catch (error) {
|
|
1323
|
+
}
|
|
1324
|
+
}, localStorage);
|
|
1325
|
+
}
|
|
1326
|
+
if (Object.keys(sessionStorage).length > 0) {
|
|
1327
|
+
await page.addInitScript((payload) => {
|
|
1328
|
+
try {
|
|
1329
|
+
Object.entries(payload || {}).forEach(([key, value]) => {
|
|
1330
|
+
if (!key) return;
|
|
1331
|
+
window.sessionStorage.setItem(key, String(value ?? ""));
|
|
1332
|
+
});
|
|
1333
|
+
} catch (error) {
|
|
1334
|
+
}
|
|
1335
|
+
}, sessionStorage);
|
|
1336
|
+
}
|
|
1337
|
+
},
|
|
1338
|
+
// captureEnvPatch 在任务结束时采集最新环境快照,用于 pushSuccess / pushFailed 自动回写。
|
|
1339
|
+
async captureEnvPatch(page, source = {}, options = {}) {
|
|
1340
|
+
const state = normalizeRuntimeState(source, options?.actor || "");
|
|
1341
|
+
const baseline = RuntimeEnv.buildEnvPatch(state) || {};
|
|
1342
|
+
const patch = {
|
|
1343
|
+
...baseline
|
|
1344
|
+
};
|
|
1345
|
+
if (!page || typeof page.evaluate !== "function" || typeof page.context !== "function") {
|
|
1346
|
+
return Object.keys(patch).length > 0 ? patch : null;
|
|
1347
|
+
}
|
|
1348
|
+
try {
|
|
1349
|
+
const contextCookies = await page.context().cookies();
|
|
1350
|
+
const normalizedCookies = normalizeCookies(contextCookies || []);
|
|
1351
|
+
if (normalizedCookies.length > 0) {
|
|
1352
|
+
patch.cookies = normalizedCookies;
|
|
1353
|
+
}
|
|
1354
|
+
} catch (error) {
|
|
1355
|
+
}
|
|
1356
|
+
try {
|
|
1357
|
+
const snapshot = await page.evaluate(() => {
|
|
1358
|
+
const localStorage2 = {};
|
|
1359
|
+
const sessionStorage2 = {};
|
|
1360
|
+
try {
|
|
1361
|
+
for (let i = 0; i < window.localStorage.length; i += 1) {
|
|
1362
|
+
const key = window.localStorage.key(i);
|
|
1363
|
+
if (!key) continue;
|
|
1364
|
+
const value = window.localStorage.getItem(key);
|
|
1365
|
+
if (value !== null) localStorage2[key] = value;
|
|
1366
|
+
}
|
|
1367
|
+
} catch (error) {
|
|
1368
|
+
}
|
|
1369
|
+
try {
|
|
1370
|
+
for (let i = 0; i < window.sessionStorage.length; i += 1) {
|
|
1371
|
+
const key = window.sessionStorage.key(i);
|
|
1372
|
+
if (!key) continue;
|
|
1373
|
+
const value = window.sessionStorage.getItem(key);
|
|
1374
|
+
if (value !== null) sessionStorage2[key] = value;
|
|
1375
|
+
}
|
|
1376
|
+
} catch (error) {
|
|
1377
|
+
}
|
|
1378
|
+
const nav = window.navigator || {};
|
|
1379
|
+
const screen = window.screen || {};
|
|
1380
|
+
const timezone = Intl.DateTimeFormat().resolvedOptions().timeZone || "";
|
|
1381
|
+
return {
|
|
1382
|
+
localStorage: localStorage2,
|
|
1383
|
+
sessionStorage: sessionStorage2,
|
|
1384
|
+
browserProfileObserved: {
|
|
1385
|
+
user_agent: nav.userAgent || "",
|
|
1386
|
+
platform: nav.platform || "",
|
|
1387
|
+
language: nav.language || "",
|
|
1388
|
+
languages: Array.isArray(nav.languages) ? nav.languages : [],
|
|
1389
|
+
hardware_concurrency: Number(nav.hardwareConcurrency || 0),
|
|
1390
|
+
device_memory: Number(nav.deviceMemory || 0),
|
|
1391
|
+
timezone,
|
|
1392
|
+
viewport: {
|
|
1393
|
+
width: Number(window.innerWidth || 0),
|
|
1394
|
+
height: Number(window.innerHeight || 0)
|
|
1395
|
+
},
|
|
1396
|
+
screen: {
|
|
1397
|
+
width: Number(screen.width || 0),
|
|
1398
|
+
height: Number(screen.height || 0),
|
|
1399
|
+
avail_width: Number(screen.availWidth || 0),
|
|
1400
|
+
avail_height: Number(screen.availHeight || 0),
|
|
1401
|
+
color_depth: Number(screen.colorDepth || 0),
|
|
1402
|
+
pixel_depth: Number(screen.pixelDepth || 0)
|
|
1403
|
+
}
|
|
1404
|
+
}
|
|
1405
|
+
};
|
|
1406
|
+
});
|
|
1407
|
+
const localStorage = normalizeLocalStorage(snapshot?.localStorage);
|
|
1408
|
+
if (Object.keys(localStorage).length > 0) {
|
|
1409
|
+
patch.local_storage = localStorage;
|
|
1410
|
+
}
|
|
1411
|
+
const sessionStorage = normalizeSessionStorage(snapshot?.sessionStorage);
|
|
1412
|
+
if (Object.keys(sessionStorage).length > 0) {
|
|
1413
|
+
patch.session_storage = sessionStorage;
|
|
1414
|
+
}
|
|
1415
|
+
const observedProfile = normalizeObservedBrowserProfile(snapshot?.browserProfileObserved);
|
|
1416
|
+
const browserProfile = buildBrowserProfilePayload(state.browserProfileCore, observedProfile);
|
|
1417
|
+
if (Object.keys(browserProfile).length > 0) {
|
|
1418
|
+
patch.browser_profile = browserProfile;
|
|
1419
|
+
}
|
|
1420
|
+
} catch (error) {
|
|
1421
|
+
}
|
|
1422
|
+
return Object.keys(patch).length > 0 ? patch : null;
|
|
1423
|
+
}
|
|
1424
|
+
};
|
|
929
1425
|
|
|
930
1426
|
// src/apify-kit.js
|
|
931
1427
|
var logger3 = createInternalLogger("ApifyKit");
|
|
1428
|
+
var resolveRuntimeContext = (input) => {
|
|
1429
|
+
const rememberedState = RuntimeEnv.peekRememberedState();
|
|
1430
|
+
const state = rememberedState || RuntimeEnv.parseInput(input || {});
|
|
1431
|
+
const envPatch = RuntimeEnv.buildEnvPatch(state) || null;
|
|
1432
|
+
return {
|
|
1433
|
+
actor: state.actor,
|
|
1434
|
+
runtime: state.runtime,
|
|
1435
|
+
envPatch
|
|
1436
|
+
};
|
|
1437
|
+
};
|
|
1438
|
+
var isPageLike = (value) => {
|
|
1439
|
+
if (!value || typeof value !== "object") return false;
|
|
1440
|
+
return typeof value.evaluate === "function" && typeof value.context === "function";
|
|
1441
|
+
};
|
|
1442
|
+
var pickPage = (...candidates) => {
|
|
1443
|
+
for (const candidate of candidates) {
|
|
1444
|
+
if (isPageLike(candidate)) {
|
|
1445
|
+
return candidate;
|
|
1446
|
+
}
|
|
1447
|
+
}
|
|
1448
|
+
return null;
|
|
1449
|
+
};
|
|
1450
|
+
var toSerializable = (value, depth = 0, seen = /* @__PURE__ */ new WeakSet()) => {
|
|
1451
|
+
if (depth > 6) return "[MaxDepth]";
|
|
1452
|
+
if (value == null) return value;
|
|
1453
|
+
const valueType = typeof value;
|
|
1454
|
+
if (valueType === "string" || valueType === "number" || valueType === "boolean") {
|
|
1455
|
+
return value;
|
|
1456
|
+
}
|
|
1457
|
+
if (valueType === "bigint") {
|
|
1458
|
+
return value.toString();
|
|
1459
|
+
}
|
|
1460
|
+
if (valueType === "function" || valueType === "symbol") {
|
|
1461
|
+
return void 0;
|
|
1462
|
+
}
|
|
1463
|
+
if (value instanceof Error) {
|
|
1464
|
+
return (0, import_serialize_error2.serializeError)(value);
|
|
1465
|
+
}
|
|
1466
|
+
if (Array.isArray(value)) {
|
|
1467
|
+
return value.map((item) => toSerializable(item, depth + 1, seen)).filter((item) => item !== void 0);
|
|
1468
|
+
}
|
|
1469
|
+
if (valueType !== "object") {
|
|
1470
|
+
return String(value);
|
|
1471
|
+
}
|
|
1472
|
+
if (seen.has(value)) {
|
|
1473
|
+
return "[Circular]";
|
|
1474
|
+
}
|
|
1475
|
+
seen.add(value);
|
|
1476
|
+
const ctorName = String(value?.constructor?.name || "");
|
|
1477
|
+
if (ctorName === "Page" || ctorName === "BrowserContext" || ctorName === "Browser" || ctorName === "Frame" || ctorName === "ElementHandle" || ctorName === "JSHandle") {
|
|
1478
|
+
return `[${ctorName}]`;
|
|
1479
|
+
}
|
|
1480
|
+
const output = {};
|
|
1481
|
+
Object.entries(value).forEach(([key, item]) => {
|
|
1482
|
+
const safeValue = toSerializable(item, depth + 1, seen);
|
|
1483
|
+
if (safeValue !== void 0) {
|
|
1484
|
+
output[key] = safeValue;
|
|
1485
|
+
}
|
|
1486
|
+
});
|
|
1487
|
+
return output;
|
|
1488
|
+
};
|
|
1489
|
+
var sanitizeMeta = (meta) => {
|
|
1490
|
+
if (!meta || typeof meta !== "object" || Array.isArray(meta)) {
|
|
1491
|
+
return {};
|
|
1492
|
+
}
|
|
1493
|
+
const result = toSerializable(meta) || {};
|
|
1494
|
+
if (result && typeof result === "object" && !Array.isArray(result)) {
|
|
1495
|
+
delete result.page;
|
|
1496
|
+
}
|
|
1497
|
+
return result;
|
|
1498
|
+
};
|
|
1499
|
+
var captureAutoEnvPatch = async (page, runtimeContext) => {
|
|
1500
|
+
const currentPage = pickPage(page);
|
|
1501
|
+
if (!currentPage) return null;
|
|
1502
|
+
try {
|
|
1503
|
+
return await RuntimeEnv.captureEnvPatch(
|
|
1504
|
+
currentPage,
|
|
1505
|
+
{
|
|
1506
|
+
actor: runtimeContext.actor,
|
|
1507
|
+
runtime: runtimeContext.runtime
|
|
1508
|
+
},
|
|
1509
|
+
{ actor: runtimeContext.actor }
|
|
1510
|
+
);
|
|
1511
|
+
} catch (error) {
|
|
1512
|
+
logger3.warn(`\u81EA\u52A8\u91C7\u96C6 env_patch \u5931\u8D25: ${error?.message || error}`);
|
|
1513
|
+
return null;
|
|
1514
|
+
}
|
|
1515
|
+
};
|
|
932
1516
|
async function createApifyKit() {
|
|
933
1517
|
let apify = null;
|
|
934
1518
|
try {
|
|
@@ -937,6 +1521,9 @@ async function createApifyKit() {
|
|
|
937
1521
|
throw new Error("\u26A0\uFE0F apify \u5E93\u672A\u5B89\u88C5\uFF0CApifyKit \u7684 Actor \u76F8\u5173\u529F\u80FD\u4E0D\u53EF\u7528");
|
|
938
1522
|
}
|
|
939
1523
|
const { Actor: Actor2 } = apify;
|
|
1524
|
+
const actorInput = await Actor2.getInput() || {};
|
|
1525
|
+
let lastPage = null;
|
|
1526
|
+
const getRuntimeContext = () => resolveRuntimeContext(actorInput);
|
|
940
1527
|
return {
|
|
941
1528
|
/**
|
|
942
1529
|
* 核心封装:执行步骤,带自动日志确认、失败截图处理和重试机制
|
|
@@ -952,6 +1539,9 @@ async function createApifyKit() {
|
|
|
952
1539
|
* @param {Function} [options.retry.before] - 重试前钩子,可覆盖默认等待行为
|
|
953
1540
|
*/
|
|
954
1541
|
async runStep(step, page, actionFn, options = {}) {
|
|
1542
|
+
if (isPageLike(page)) {
|
|
1543
|
+
lastPage = page;
|
|
1544
|
+
}
|
|
955
1545
|
const { failActor = true, retry = {} } = options;
|
|
956
1546
|
const { times: retryTimes = 0, mode: retryMode = "direct", before: beforeRetry } = retry;
|
|
957
1547
|
const executeAction = async (attemptNumber) => {
|
|
@@ -1013,13 +1603,18 @@ async function createApifyKit() {
|
|
|
1013
1603
|
} catch (snapErr) {
|
|
1014
1604
|
logger3.warn(`\u622A\u56FE\u751F\u6210\u5931\u8D25: ${snapErr.message}`);
|
|
1015
1605
|
}
|
|
1016
|
-
await this.pushFailed(
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1606
|
+
await this.pushFailed(
|
|
1607
|
+
finalError,
|
|
1608
|
+
{
|
|
1609
|
+
step,
|
|
1610
|
+
base64,
|
|
1611
|
+
retryAttempts: retryTimes
|
|
1612
|
+
},
|
|
1613
|
+
{
|
|
1614
|
+
page,
|
|
1615
|
+
options
|
|
1616
|
+
}
|
|
1617
|
+
);
|
|
1023
1618
|
await Actor2.fail(`Run Step ${step} \u5931\u8D25 (\u5DF2\u91CD\u8BD5 ${retryTimes} \u6B21): ${finalError.message}`);
|
|
1024
1619
|
} else {
|
|
1025
1620
|
throw finalError;
|
|
@@ -1041,42 +1636,83 @@ async function createApifyKit() {
|
|
|
1041
1636
|
});
|
|
1042
1637
|
},
|
|
1043
1638
|
/**
|
|
1044
|
-
*
|
|
1639
|
+
* 推送成功数据的通用方法。
|
|
1640
|
+
* 这里会自动做三件事:
|
|
1641
|
+
* 1. 采集当前页面的 env_patch;
|
|
1642
|
+
* 2. 合并显式传入的 env_patch;
|
|
1643
|
+
* 3. 一并写入 dataset,供后端回写 runtime_env_json。
|
|
1045
1644
|
* @param {Object} data - 要推送的数据对象
|
|
1046
1645
|
*/
|
|
1047
|
-
async pushSuccess(data) {
|
|
1048
|
-
const
|
|
1049
|
-
|
|
1050
|
-
|
|
1646
|
+
async pushSuccess(data, options = {}) {
|
|
1647
|
+
const runtimeContext = getRuntimeContext();
|
|
1648
|
+
const page = pickPage(options?.page, data?.page, lastPage);
|
|
1649
|
+
if (page) {
|
|
1650
|
+
lastPage = page;
|
|
1651
|
+
}
|
|
1652
|
+
let payloadData = data;
|
|
1653
|
+
let explicitPatch = null;
|
|
1654
|
+
if (data && typeof data === "object" && !Array.isArray(data)) {
|
|
1655
|
+
payloadData = { ...data };
|
|
1656
|
+
explicitPatch = payloadData.env_patch;
|
|
1657
|
+
delete payloadData.env_patch;
|
|
1658
|
+
delete payloadData.page;
|
|
1659
|
+
}
|
|
1660
|
+
const capturedPatch = await captureAutoEnvPatch(page, runtimeContext);
|
|
1661
|
+
const envPatch = RuntimeEnv.mergeEnvPatches(runtimeContext.envPatch, capturedPatch, explicitPatch);
|
|
1662
|
+
const traffic = await ProxyMeterRuntime.getProxyMeterSnapshot({ finalize: true });
|
|
1663
|
+
const payload = {
|
|
1051
1664
|
code: Code.Success,
|
|
1052
1665
|
status: Status.Success,
|
|
1053
1666
|
...traffic ? { traffic } : {},
|
|
1054
1667
|
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1055
|
-
data
|
|
1668
|
+
data: payloadData
|
|
1669
|
+
};
|
|
1670
|
+
if (envPatch) {
|
|
1671
|
+
payload.env_patch = envPatch;
|
|
1672
|
+
}
|
|
1673
|
+
await Actor2.pushData({
|
|
1674
|
+
...payload
|
|
1056
1675
|
});
|
|
1057
1676
|
logger3.success("pushSuccess", "Data pushed");
|
|
1058
1677
|
},
|
|
1059
1678
|
/**
|
|
1060
|
-
* 推送失败数据的通用方法(私有方法,仅供runStep
|
|
1061
|
-
*
|
|
1679
|
+
* 推送失败数据的通用方法(私有方法,仅供 runStep 内部使用)。
|
|
1680
|
+
* 即便任务失败,这里也会自动回收 env_patch,确保 cookies/storage 的最新状态不会丢。
|
|
1062
1681
|
* @param {Error|CrawlerError} error - 错误对象
|
|
1063
|
-
* @param {Object} [meta] - 额外的数据(如failedStep
|
|
1682
|
+
* @param {Object} [meta] - 额外的数据(如 failedStep、screenshotBase64 等)
|
|
1683
|
+
* @param {Object} [options] - 附加选项(如 page)
|
|
1064
1684
|
* @private
|
|
1065
1685
|
*/
|
|
1066
|
-
async pushFailed(error, meta = {}) {
|
|
1686
|
+
async pushFailed(error, meta = {}, options = {}) {
|
|
1687
|
+
const runtimeContext = getRuntimeContext();
|
|
1688
|
+
const page = pickPage(options?.page, meta?.page, lastPage);
|
|
1689
|
+
if (page) {
|
|
1690
|
+
lastPage = page;
|
|
1691
|
+
}
|
|
1692
|
+
const capturedPatch = await captureAutoEnvPatch(page, runtimeContext);
|
|
1693
|
+
const explicitPatch = meta?.env_patch;
|
|
1694
|
+
const envPatch = RuntimeEnv.mergeEnvPatches(runtimeContext.envPatch, capturedPatch, explicitPatch);
|
|
1695
|
+
const safeMeta = sanitizeMeta(meta);
|
|
1696
|
+
delete safeMeta.env_patch;
|
|
1067
1697
|
const isCrawlerError = CrawlerError.isCrawlerError(error);
|
|
1068
1698
|
const code = isCrawlerError ? error.code : Code.UnknownError;
|
|
1069
1699
|
const context = isCrawlerError ? error.context : {};
|
|
1070
|
-
const traffic = await getProxyMeterSnapshot({ finalize: true });
|
|
1071
|
-
|
|
1700
|
+
const traffic = await ProxyMeterRuntime.getProxyMeterSnapshot({ finalize: true });
|
|
1701
|
+
const payload = {
|
|
1072
1702
|
// 如果是 CrawlerError,使用其 code,否则使用默认 Failed code
|
|
1073
1703
|
code,
|
|
1074
1704
|
status: Status.Failed,
|
|
1075
1705
|
...traffic ? { traffic } : {},
|
|
1076
1706
|
error: (0, import_serialize_error2.serializeError)(error),
|
|
1077
|
-
meta,
|
|
1707
|
+
meta: safeMeta,
|
|
1078
1708
|
context,
|
|
1079
1709
|
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
1710
|
+
};
|
|
1711
|
+
if (envPatch) {
|
|
1712
|
+
payload.env_patch = envPatch;
|
|
1713
|
+
}
|
|
1714
|
+
await Actor2.pushData({
|
|
1715
|
+
...payload
|
|
1080
1716
|
});
|
|
1081
1717
|
logger3.success("pushFailed", "Error data pushed");
|
|
1082
1718
|
}
|
|
@@ -1167,12 +1803,13 @@ var DEFAULT_LAUNCH_ARGS = [
|
|
|
1167
1803
|
// '--disable-blink-features=AutomationControlled', // Crawlee 可能会自动处理,过多干预反而会被识别
|
|
1168
1804
|
"--no-sandbox",
|
|
1169
1805
|
"--disable-setuid-sandbox",
|
|
1170
|
-
"--window-position=0,0"
|
|
1171
|
-
`--lang=${BASE_CONFIG.locale}`
|
|
1806
|
+
"--window-position=0,0"
|
|
1172
1807
|
];
|
|
1173
|
-
function buildFingerprintOptions(locale) {
|
|
1808
|
+
function buildFingerprintOptions({ locale = BASE_CONFIG.locale, browserMajorVersion = 0 } = {}) {
|
|
1809
|
+
const normalizedBrowserMajorVersion = Number(browserMajorVersion || 0);
|
|
1810
|
+
const browserDescriptor = normalizedBrowserMajorVersion > 0 ? [{ name: "chrome", minVersion: normalizedBrowserMajorVersion, maxVersion: normalizedBrowserMajorVersion }] : [{ name: "chrome", minVersion: 110 }];
|
|
1174
1811
|
return {
|
|
1175
|
-
browsers:
|
|
1812
|
+
browsers: browserDescriptor,
|
|
1176
1813
|
devices: ["desktop"],
|
|
1177
1814
|
operatingSystems: ["windows", "macos", "linux"],
|
|
1178
1815
|
locales: [locale]
|
|
@@ -1188,20 +1825,21 @@ var AntiCheat = {
|
|
|
1188
1825
|
/**
|
|
1189
1826
|
* 用于 Crawlee fingerprint generator 的统一配置(桌面端)。
|
|
1190
1827
|
*/
|
|
1191
|
-
getFingerprintGeneratorOptions() {
|
|
1192
|
-
return buildFingerprintOptions(
|
|
1828
|
+
getFingerprintGeneratorOptions(options = {}) {
|
|
1829
|
+
return buildFingerprintOptions(options);
|
|
1193
1830
|
},
|
|
1194
1831
|
/**
|
|
1195
1832
|
* 获取基础启动参数。
|
|
1196
1833
|
*/
|
|
1197
|
-
getLaunchArgs() {
|
|
1198
|
-
|
|
1834
|
+
getLaunchArgs(options = {}) {
|
|
1835
|
+
const locale = String(options?.locale || BASE_CONFIG.locale).trim() || BASE_CONFIG.locale;
|
|
1836
|
+
return [...DEFAULT_LAUNCH_ARGS, `--lang=${locale}`];
|
|
1199
1837
|
},
|
|
1200
1838
|
/**
|
|
1201
1839
|
* 为 got-scraping 生成与浏览器一致的 TLS 指纹配置(桌面端)。
|
|
1202
1840
|
*/
|
|
1203
1841
|
getTlsFingerprintOptions(userAgent = "", acceptLanguage = "") {
|
|
1204
|
-
return buildFingerprintOptions(
|
|
1842
|
+
return buildFingerprintOptions();
|
|
1205
1843
|
},
|
|
1206
1844
|
/**
|
|
1207
1845
|
* 规范化请求头
|
|
@@ -1692,12 +2330,18 @@ var Humanize = {
|
|
|
1692
2330
|
}
|
|
1693
2331
|
};
|
|
1694
2332
|
|
|
2333
|
+
// src/launch.js
|
|
2334
|
+
var import_node_child_process = require("node:child_process");
|
|
2335
|
+
var import_fingerprint_generator = require("fingerprint-generator");
|
|
2336
|
+
var import_fingerprint_injector = require("fingerprint-injector");
|
|
2337
|
+
|
|
1695
2338
|
// src/proxy-bypass.js
|
|
1696
2339
|
var import_picomatch = __toESM(require("picomatch"), 1);
|
|
1697
2340
|
var normalizeByPassDomains = (domains) => {
|
|
1698
2341
|
if (!Array.isArray(domains)) return [];
|
|
1699
2342
|
return domains.map((item) => String(item || "").trim()).filter(Boolean);
|
|
1700
2343
|
};
|
|
2344
|
+
var normalizeHostname = (value = "") => String(value || "").trim().toLowerCase();
|
|
1701
2345
|
var buildByPassDomainRule = (rawPattern) => {
|
|
1702
2346
|
const pattern = String(rawPattern || "").trim().toLowerCase();
|
|
1703
2347
|
if (!pattern) return null;
|
|
@@ -1718,7 +2362,7 @@ var buildByPassDomainRules = (domains = []) => {
|
|
|
1718
2362
|
var findMatchedByPassRule = (rules = [], requestUrl = "") => {
|
|
1719
2363
|
let hostname = "";
|
|
1720
2364
|
try {
|
|
1721
|
-
hostname = new URL(String(requestUrl || "")).hostname
|
|
2365
|
+
hostname = normalizeHostname(new URL(String(requestUrl || "")).hostname);
|
|
1722
2366
|
} catch {
|
|
1723
2367
|
return null;
|
|
1724
2368
|
}
|
|
@@ -1735,16 +2379,56 @@ var findMatchedByPassRule = (rules = [], requestUrl = "") => {
|
|
|
1735
2379
|
hostname
|
|
1736
2380
|
};
|
|
1737
2381
|
};
|
|
2382
|
+
var isDomainCoveredByByPass = (domain = "", rules = []) => {
|
|
2383
|
+
const hostname = normalizeHostname(domain);
|
|
2384
|
+
if (!hostname) return false;
|
|
2385
|
+
for (const rule of rules || []) {
|
|
2386
|
+
if (!rule || typeof rule.test !== "function") continue;
|
|
2387
|
+
if (rule.test(hostname)) return true;
|
|
2388
|
+
}
|
|
2389
|
+
return false;
|
|
2390
|
+
};
|
|
2391
|
+
var resolveRouteByProxy = ({
|
|
2392
|
+
requestUrl = "",
|
|
2393
|
+
enableProxy = false,
|
|
2394
|
+
byPassRules = []
|
|
2395
|
+
}) => {
|
|
2396
|
+
if (!enableProxy) {
|
|
2397
|
+
return { route: "direct", matchedRule: null, hostname: "" };
|
|
2398
|
+
}
|
|
2399
|
+
const matched = findMatchedByPassRule(byPassRules, requestUrl);
|
|
2400
|
+
if (!matched) {
|
|
2401
|
+
return { route: "proxy", matchedRule: null, hostname: "" };
|
|
2402
|
+
}
|
|
2403
|
+
if (matched.rule) {
|
|
2404
|
+
return { route: "direct", matchedRule: matched.rule, hostname: matched.hostname };
|
|
2405
|
+
}
|
|
2406
|
+
return { route: "proxy", matchedRule: null, hostname: matched.hostname };
|
|
2407
|
+
};
|
|
2408
|
+
var ByPass = {
|
|
2409
|
+
normalizeByPassDomains,
|
|
2410
|
+
normalizeHostname,
|
|
2411
|
+
buildByPassDomainRule,
|
|
2412
|
+
buildByPassDomainRules,
|
|
2413
|
+
findMatchedByPassRule,
|
|
2414
|
+
isDomainCoveredByByPass,
|
|
2415
|
+
resolveRouteByProxy
|
|
2416
|
+
};
|
|
1738
2417
|
|
|
1739
2418
|
// src/launch.js
|
|
1740
2419
|
var logger7 = createInternalLogger("Launch");
|
|
1741
2420
|
var REQUEST_HOOK_FLAG = Symbol("playwright-toolkit-request-hook");
|
|
2421
|
+
var injectedContexts = /* @__PURE__ */ new WeakSet();
|
|
2422
|
+
var browserMajorVersionCache = /* @__PURE__ */ new Map();
|
|
2423
|
+
var DEFAULT_BROWSER_PROFILE_SCHEMA_VERSION = 1;
|
|
2424
|
+
var DEFAULT_LOCALE = "zh-CN";
|
|
1742
2425
|
var DEFAULT_CRAWLER_BASE_OPTIONS = Object.freeze({
|
|
1743
2426
|
maxConcurrency: 1,
|
|
1744
2427
|
maxRequestRetries: 0,
|
|
1745
2428
|
requestHandlerTimeoutSecs: 240,
|
|
1746
2429
|
navigationTimeoutSecs: 120
|
|
1747
2430
|
});
|
|
2431
|
+
var fingerprintInjector = new import_fingerprint_injector.FingerprintInjector();
|
|
1748
2432
|
var resolveProxyLaunchOptions = (proxyConfiguration = {}) => {
|
|
1749
2433
|
const config = proxyConfiguration && typeof proxyConfiguration === "object" && !Array.isArray(proxyConfiguration) ? proxyConfiguration : {};
|
|
1750
2434
|
const proxyUrl = String(config.proxy_url || "").trim();
|
|
@@ -1752,9 +2436,149 @@ var resolveProxyLaunchOptions = (proxyConfiguration = {}) => {
|
|
|
1752
2436
|
if (!enableProxy || !proxyUrl) {
|
|
1753
2437
|
return { byPassDomains: [], enableProxy, proxyUrl };
|
|
1754
2438
|
}
|
|
1755
|
-
const byPassDomains = normalizeByPassDomains(config.by_pass_domains);
|
|
2439
|
+
const byPassDomains = ByPass.normalizeByPassDomains(config.by_pass_domains);
|
|
1756
2440
|
return { byPassDomains, enableProxy, proxyUrl };
|
|
1757
2441
|
};
|
|
2442
|
+
var parseChromeMajorVersion = (rawValue = "") => {
|
|
2443
|
+
const match = String(rawValue || "").match(/Chrome\/(\d+)/i);
|
|
2444
|
+
return match ? Number(match[1] || 0) : 0;
|
|
2445
|
+
};
|
|
2446
|
+
var resolveLauncherExecutablePath = (launcher) => {
|
|
2447
|
+
if (!launcher || typeof launcher !== "object") return "";
|
|
2448
|
+
if (typeof launcher.executablePath === "function") {
|
|
2449
|
+
try {
|
|
2450
|
+
return String(launcher.executablePath() || "").trim();
|
|
2451
|
+
} catch {
|
|
2452
|
+
return "";
|
|
2453
|
+
}
|
|
2454
|
+
}
|
|
2455
|
+
return "";
|
|
2456
|
+
};
|
|
2457
|
+
var detectBrowserMajorVersion = (launcher) => {
|
|
2458
|
+
const executablePath = resolveLauncherExecutablePath(launcher);
|
|
2459
|
+
if (!executablePath) return 0;
|
|
2460
|
+
if (browserMajorVersionCache.has(executablePath)) {
|
|
2461
|
+
return browserMajorVersionCache.get(executablePath);
|
|
2462
|
+
}
|
|
2463
|
+
let detectedVersion = 0;
|
|
2464
|
+
try {
|
|
2465
|
+
const rawVersion = (0, import_node_child_process.execFileSync)(executablePath, ["--version"], {
|
|
2466
|
+
encoding: "utf8",
|
|
2467
|
+
stdio: ["ignore", "pipe", "ignore"]
|
|
2468
|
+
});
|
|
2469
|
+
detectedVersion = parseChromeMajorVersion(rawVersion);
|
|
2470
|
+
} catch (error) {
|
|
2471
|
+
logger7.warn(`\u8BFB\u53D6\u6D4F\u89C8\u5668\u7248\u672C\u5931\u8D25: ${error?.message || error}`);
|
|
2472
|
+
}
|
|
2473
|
+
browserMajorVersionCache.set(executablePath, detectedVersion);
|
|
2474
|
+
return detectedVersion;
|
|
2475
|
+
};
|
|
2476
|
+
var buildFingerprintGenerator = ({ locale, browserMajorVersion }) => {
|
|
2477
|
+
return new import_fingerprint_generator.FingerprintGenerator(
|
|
2478
|
+
AntiCheat.getFingerprintGeneratorOptions({
|
|
2479
|
+
locale,
|
|
2480
|
+
browserMajorVersion
|
|
2481
|
+
})
|
|
2482
|
+
);
|
|
2483
|
+
};
|
|
2484
|
+
var buildReplayableBrowserProfile = (runtimeState, launcher) => {
|
|
2485
|
+
if (!runtimeState || !RuntimeEnv.hasLoginState(runtimeState)) {
|
|
2486
|
+
return { runtimeState, browserProfileCore: null };
|
|
2487
|
+
}
|
|
2488
|
+
let nextState = RuntimeEnv.rememberState(runtimeState);
|
|
2489
|
+
let browserProfileCore = RuntimeEnv.getBrowserProfileCore(nextState);
|
|
2490
|
+
const timezoneId = String(browserProfileCore?.timezone_id || "").trim() || AntiCheat.getBaseConfig().timezoneId;
|
|
2491
|
+
const locale = DEFAULT_LOCALE;
|
|
2492
|
+
const currentBrowserMajorVersion = detectBrowserMajorVersion(launcher);
|
|
2493
|
+
const storedBrowserMajorVersion = Number(browserProfileCore?.browser_major_version || 0);
|
|
2494
|
+
const needsRebuild = !browserProfileCore?.fingerprint || Object.keys(browserProfileCore.fingerprint || {}).length === 0 || currentBrowserMajorVersion > 0 && storedBrowserMajorVersion > 0 && storedBrowserMajorVersion !== currentBrowserMajorVersion;
|
|
2495
|
+
if (needsRebuild) {
|
|
2496
|
+
const generator = buildFingerprintGenerator({
|
|
2497
|
+
locale,
|
|
2498
|
+
browserMajorVersion: currentBrowserMajorVersion
|
|
2499
|
+
});
|
|
2500
|
+
const fingerprint = generator.getFingerprint();
|
|
2501
|
+
const fingerprintBrowserMajorVersion = parseChromeMajorVersion(fingerprint?.fingerprint?.navigator?.userAgent || "");
|
|
2502
|
+
browserProfileCore = {
|
|
2503
|
+
fingerprint,
|
|
2504
|
+
timezone_id: timezoneId,
|
|
2505
|
+
locale,
|
|
2506
|
+
browser_major_version: currentBrowserMajorVersion > 0 ? currentBrowserMajorVersion : fingerprintBrowserMajorVersion,
|
|
2507
|
+
profile_key: String(nextState.envId || "").trim(),
|
|
2508
|
+
schema_version: DEFAULT_BROWSER_PROFILE_SCHEMA_VERSION
|
|
2509
|
+
};
|
|
2510
|
+
nextState = RuntimeEnv.setBrowserProfileCore(nextState, browserProfileCore);
|
|
2511
|
+
logger7.info(
|
|
2512
|
+
`\u5DF2\u751F\u6210\u6D4F\u89C8\u5668\u6307\u7EB9\u771F\u6E90 | env=${String(nextState.envId || "-")} | version=${browserProfileCore.browser_major_version || "-"} | timezone=${timezoneId}`
|
|
2513
|
+
);
|
|
2514
|
+
return { runtimeState: nextState, browserProfileCore };
|
|
2515
|
+
}
|
|
2516
|
+
const normalizedBrowserProfileCore = {
|
|
2517
|
+
...browserProfileCore,
|
|
2518
|
+
timezone_id: timezoneId,
|
|
2519
|
+
locale,
|
|
2520
|
+
profile_key: String(browserProfileCore.profile_key || nextState.envId || "").trim(),
|
|
2521
|
+
schema_version: Number(browserProfileCore.schema_version || 0) > 0 ? Number(browserProfileCore.schema_version || 0) : DEFAULT_BROWSER_PROFILE_SCHEMA_VERSION,
|
|
2522
|
+
browser_major_version: currentBrowserMajorVersion > 0 ? currentBrowserMajorVersion : Number(browserProfileCore.browser_major_version || 0)
|
|
2523
|
+
};
|
|
2524
|
+
const currentCoreRaw = JSON.stringify(browserProfileCore || {});
|
|
2525
|
+
const nextCoreRaw = JSON.stringify(normalizedBrowserProfileCore);
|
|
2526
|
+
if (currentCoreRaw !== nextCoreRaw) {
|
|
2527
|
+
nextState = RuntimeEnv.setBrowserProfileCore(nextState, normalizedBrowserProfileCore);
|
|
2528
|
+
}
|
|
2529
|
+
return {
|
|
2530
|
+
runtimeState: nextState,
|
|
2531
|
+
browserProfileCore: normalizedBrowserProfileCore
|
|
2532
|
+
};
|
|
2533
|
+
};
|
|
2534
|
+
var buildReplayBrowserPoolOptions = (browserProfileCore) => {
|
|
2535
|
+
const fingerprintWithHeaders = browserProfileCore?.fingerprint;
|
|
2536
|
+
const fingerprint = fingerprintWithHeaders?.fingerprint;
|
|
2537
|
+
if (!fingerprintWithHeaders || !fingerprint) {
|
|
2538
|
+
return null;
|
|
2539
|
+
}
|
|
2540
|
+
return {
|
|
2541
|
+
useFingerprints: false,
|
|
2542
|
+
prePageCreateHooks: [
|
|
2543
|
+
(_pageId, _browserController, pageOptions = {}) => {
|
|
2544
|
+
if (!pageOptions || typeof pageOptions !== "object") return;
|
|
2545
|
+
const screen = fingerprint.screen || {};
|
|
2546
|
+
const userAgent = String(fingerprint.navigator?.userAgent || "").trim();
|
|
2547
|
+
if (userAgent) {
|
|
2548
|
+
pageOptions.userAgent = userAgent;
|
|
2549
|
+
}
|
|
2550
|
+
if (Number(screen.width || 0) > 0 && Number(screen.height || 0) > 0) {
|
|
2551
|
+
pageOptions.viewport = {
|
|
2552
|
+
width: Number(screen.width),
|
|
2553
|
+
height: Number(screen.height)
|
|
2554
|
+
};
|
|
2555
|
+
pageOptions.screen = {
|
|
2556
|
+
width: Number(screen.width),
|
|
2557
|
+
height: Number(screen.height)
|
|
2558
|
+
};
|
|
2559
|
+
}
|
|
2560
|
+
if (browserProfileCore.locale) {
|
|
2561
|
+
pageOptions.locale = browserProfileCore.locale;
|
|
2562
|
+
}
|
|
2563
|
+
if (browserProfileCore.timezone_id) {
|
|
2564
|
+
pageOptions.timezoneId = browserProfileCore.timezone_id;
|
|
2565
|
+
}
|
|
2566
|
+
}
|
|
2567
|
+
],
|
|
2568
|
+
postPageCreateHooks: [
|
|
2569
|
+
async (page) => {
|
|
2570
|
+
const context = page?.context?.();
|
|
2571
|
+
if (!context) return;
|
|
2572
|
+
if (!injectedContexts.has(context)) {
|
|
2573
|
+
await fingerprintInjector.attachFingerprintToPlaywright(context, fingerprintWithHeaders);
|
|
2574
|
+
injectedContexts.add(context);
|
|
2575
|
+
}
|
|
2576
|
+
await page.emulateMedia({ colorScheme: "dark" }).catch(() => {
|
|
2577
|
+
});
|
|
2578
|
+
}
|
|
2579
|
+
]
|
|
2580
|
+
};
|
|
2581
|
+
};
|
|
1758
2582
|
var Launch = {
|
|
1759
2583
|
getPlaywrightCrawlerOptions(options = {}) {
|
|
1760
2584
|
const normalizedOptions = Array.isArray(options) ? { customArgs: options } : options || {};
|
|
@@ -1767,18 +2591,22 @@ var Launch = {
|
|
|
1767
2591
|
isRunningOnApify = false,
|
|
1768
2592
|
launcher = null,
|
|
1769
2593
|
preNavigationHooks = [],
|
|
1770
|
-
postNavigationHooks = []
|
|
2594
|
+
postNavigationHooks = [],
|
|
2595
|
+
runtimeState = null
|
|
1771
2596
|
} = normalizedOptions;
|
|
1772
2597
|
const { byPassDomains, enableProxy, proxyUrl } = resolveProxyLaunchOptions(proxyConfiguration);
|
|
1773
|
-
const byPassRules = buildByPassDomainRules(byPassDomains);
|
|
1774
|
-
const proxyMeter = enableProxy && proxyUrl ? startProxyMeter({ proxyUrl, debugMode }) : null;
|
|
2598
|
+
const byPassRules = ByPass.buildByPassDomainRules(byPassDomains);
|
|
2599
|
+
const proxyMeter = enableProxy && proxyUrl ? ProxyMeterRuntime.startProxyMeter({ proxyUrl, debugMode }) : null;
|
|
1775
2600
|
const launchProxy = proxyMeter ? { server: proxyMeter.server } : null;
|
|
1776
2601
|
if (launchProxy && byPassDomains.length > 0) {
|
|
1777
2602
|
launchProxy.bypass = byPassDomains.join(",");
|
|
1778
2603
|
}
|
|
2604
|
+
const replayContext = buildReplayableBrowserProfile(runtimeState, launcher);
|
|
2605
|
+
const replayBrowserPoolOptions = buildReplayBrowserPoolOptions(replayContext.browserProfileCore);
|
|
2606
|
+
const launchLocale = String(replayContext.browserProfileCore?.locale || DEFAULT_LOCALE).trim() || DEFAULT_LOCALE;
|
|
1779
2607
|
const launchOptions = {
|
|
1780
2608
|
args: [
|
|
1781
|
-
...AntiCheat.getLaunchArgs(),
|
|
2609
|
+
...AntiCheat.getLaunchArgs({ locale: launchLocale }),
|
|
1782
2610
|
...customArgs
|
|
1783
2611
|
],
|
|
1784
2612
|
ignoreDefaultArgs: ["--enable-automation"]
|
|
@@ -1790,7 +2618,7 @@ var Launch = {
|
|
|
1790
2618
|
if (enableByPassLogger && launchProxy) {
|
|
1791
2619
|
let upstreamLabel = "";
|
|
1792
2620
|
try {
|
|
1793
|
-
const parsedProxyUrl = new URL(proxyUrl);
|
|
2621
|
+
const parsedProxyUrl = new URL(proxyUrl.includes("://") ? proxyUrl : `http://${proxyUrl}`);
|
|
1794
2622
|
upstreamLabel = `${parsedProxyUrl.protocol}//${parsedProxyUrl.host}`;
|
|
1795
2623
|
} catch {
|
|
1796
2624
|
}
|
|
@@ -1799,14 +2627,10 @@ var Launch = {
|
|
|
1799
2627
|
);
|
|
1800
2628
|
logger7.info(`[\u6D41\u91CF\u89C2\u6D4B] \u9010\u8BF7\u6C42\u8C03\u8BD5=${Boolean(debugMode) ? "\u5F00\u542F" : "\u5173\u95ED"}\uFF08\u6C47\u603B\u59CB\u7EC8\u5F00\u542F\uFF09`);
|
|
1801
2629
|
} else if (enableByPassLogger && enableProxy && !launchProxy) {
|
|
1802
|
-
logger7.info(
|
|
1803
|
-
`[\u4EE3\u7406\u672A\u542F\u7528] enable_proxy=true \u4F46 proxy_url \u4E3A\u7A7A`
|
|
1804
|
-
);
|
|
2630
|
+
logger7.info("[\u4EE3\u7406\u672A\u542F\u7528] enable_proxy=true \u4F46 proxy_url \u4E3A\u7A7A");
|
|
1805
2631
|
logger7.info(`[\u6D41\u91CF\u89C2\u6D4B] \u9010\u8BF7\u6C42\u8C03\u8BD5=${Boolean(debugMode) ? "\u5F00\u542F" : "\u5173\u95ED"}\uFF08\u6C47\u603B\u59CB\u7EC8\u5F00\u542F\uFF09`);
|
|
1806
2632
|
} else if (enableByPassLogger && !enableProxy && proxyUrl) {
|
|
1807
|
-
logger7.info(
|
|
1808
|
-
`[\u4EE3\u7406\u672A\u542F\u7528] enable_proxy=false \u4E14 proxy_url \u5DF2\u914D\u7F6E`
|
|
1809
|
-
);
|
|
2633
|
+
logger7.info("[\u4EE3\u7406\u672A\u542F\u7528] enable_proxy=false \u4E14 proxy_url \u5DF2\u914D\u7F6E");
|
|
1810
2634
|
logger7.info(`[\u6D41\u91CF\u89C2\u6D4B] \u9010\u8BF7\u6C42\u8C03\u8BD5=${Boolean(debugMode) ? "\u5F00\u542F" : "\u5173\u95ED"}\uFF08\u6C47\u603B\u59CB\u7EC8\u5F00\u542F\uFF09`);
|
|
1811
2635
|
} else if (enableByPassLogger) {
|
|
1812
2636
|
logger7.info(`[\u6D41\u91CF\u89C2\u6D4B] \u9010\u8BF7\u6C42\u8C03\u8BD5=${Boolean(debugMode) ? "\u5F00\u542F" : "\u5173\u95ED"}\uFF08\u6C47\u603B\u59CB\u7EC8\u5F00\u542F\uFF09`);
|
|
@@ -1825,9 +2649,9 @@ var Launch = {
|
|
|
1825
2649
|
const requestHandler = (req) => {
|
|
1826
2650
|
const requestUrl = req.url();
|
|
1827
2651
|
const resourceType = req.resourceType();
|
|
1828
|
-
const matched = byPassDomains.length > 0 ? findMatchedByPassRule(byPassRules, requestUrl) : null;
|
|
1829
|
-
if (launchProxy
|
|
1830
|
-
recordProxyMeterResourceType(requestUrl, resourceType);
|
|
2652
|
+
const matched = byPassDomains.length > 0 ? ByPass.findMatchedByPassRule(byPassRules, requestUrl) : null;
|
|
2653
|
+
if (launchProxy) {
|
|
2654
|
+
ProxyMeterRuntime.recordProxyMeterResourceType(requestUrl, resourceType);
|
|
1831
2655
|
}
|
|
1832
2656
|
if (!enableByPassLogger || byPassDomains.length === 0) return;
|
|
1833
2657
|
if (!matched || !matched.rule) return;
|
|
@@ -1846,7 +2670,8 @@ var Launch = {
|
|
|
1846
2670
|
const crawlerBaseOptions = {
|
|
1847
2671
|
...DEFAULT_CRAWLER_BASE_OPTIONS,
|
|
1848
2672
|
headless: !runInHeadfulMode || isRunningOnApify,
|
|
1849
|
-
|
|
2673
|
+
// 有 core.fingerprint 时走固定回放;否则退回 Crawlee 默认指纹模式。
|
|
2674
|
+
browserPoolOptions: replayBrowserPoolOptions || {
|
|
1850
2675
|
useFingerprints: true,
|
|
1851
2676
|
fingerprintOptions: {
|
|
1852
2677
|
fingerprintGeneratorOptions: AntiCheat.getFingerprintGeneratorOptions()
|
|
@@ -2049,241 +2874,9 @@ var Captcha = {
|
|
|
2049
2874
|
useCaptchaMonitor
|
|
2050
2875
|
};
|
|
2051
2876
|
|
|
2052
|
-
// src/sse.js
|
|
2053
|
-
var import_https = __toESM(require("https"), 1);
|
|
2054
|
-
var import_url2 = require("url");
|
|
2055
|
-
var logger10 = createInternalLogger("Sse");
|
|
2056
|
-
var Sse = {
|
|
2057
|
-
/**
|
|
2058
|
-
* 解析 SSE 流文本
|
|
2059
|
-
* 支持 `data: {...}` 和 `data:{...}` 两种格式
|
|
2060
|
-
* @param {string} sseStreamText
|
|
2061
|
-
* @returns {Array<Object>} events
|
|
2062
|
-
*/
|
|
2063
|
-
parseSseStream(sseStreamText) {
|
|
2064
|
-
const events = [];
|
|
2065
|
-
const lines = sseStreamText.split("\n");
|
|
2066
|
-
for (const line of lines) {
|
|
2067
|
-
if (line.startsWith("data:")) {
|
|
2068
|
-
try {
|
|
2069
|
-
const jsonContent = line.substring(5).trim();
|
|
2070
|
-
if (jsonContent) {
|
|
2071
|
-
events.push(JSON.parse(jsonContent));
|
|
2072
|
-
}
|
|
2073
|
-
} catch (e) {
|
|
2074
|
-
logger10.debug("parseSseStream", `JSON \u89E3\u6790\u5931\u8D25: ${e.message}, line: ${line.substring(0, 100)}...`);
|
|
2075
|
-
}
|
|
2076
|
-
}
|
|
2077
|
-
}
|
|
2078
|
-
logger10.success("parseSseStream", `\u89E3\u6790\u5B8C\u6210, events \u6570\u91CF: ${events.length}`);
|
|
2079
|
-
return events;
|
|
2080
|
-
},
|
|
2081
|
-
/**
|
|
2082
|
-
* 拦截网络请求并使用 Node.js 原生 https 模块转发,以解决流式数据捕获问题。
|
|
2083
|
-
* @param {import('playwright').Page} page
|
|
2084
|
-
* @param {string|RegExp} urlPattern - 拦截的 URL 模式
|
|
2085
|
-
* @param {object} options
|
|
2086
|
-
* @param {function(string, function, string): void} [options.onData] - (textChunk, resolve, accumulatedText) => void
|
|
2087
|
-
* @param {function(string, function): void} [options.onEnd] - (fullText, resolve) => void
|
|
2088
|
-
* @param {function(Error, function): void} [options.onTimeout] - (error, reject) => void
|
|
2089
|
-
* @param {number} [options.initialTimeout=90000] - 初始数据接收超时 (ms),默认 90s
|
|
2090
|
-
* @param {number} [options.overallTimeout=180000] - 整体请求超时时间 (ms),默认 180s
|
|
2091
|
-
* @param {boolean} [options.autoUnroute=true] - resolve/reject 后是否自动取消拦截
|
|
2092
|
-
* @param {boolean} [options.firstMatchOnly=true] - 只拦截第一个命中的请求,后续请求全部放行
|
|
2093
|
-
* @returns {Promise<any>} - 返回 Promise,当流满足条件时 resolve
|
|
2094
|
-
*/
|
|
2095
|
-
async intercept(page, urlPattern, options = {}) {
|
|
2096
|
-
const {
|
|
2097
|
-
onData,
|
|
2098
|
-
onEnd,
|
|
2099
|
-
onTimeout,
|
|
2100
|
-
initialTimeout = 9e4,
|
|
2101
|
-
overallTimeout = 18e4,
|
|
2102
|
-
autoUnroute = true,
|
|
2103
|
-
firstMatchOnly = true
|
|
2104
|
-
} = options;
|
|
2105
|
-
let initialTimer = null;
|
|
2106
|
-
let overallTimer = null;
|
|
2107
|
-
let hasReceivedInitialData = false;
|
|
2108
|
-
const clearAllTimers = () => {
|
|
2109
|
-
if (initialTimer) clearTimeout(initialTimer);
|
|
2110
|
-
if (overallTimer) clearTimeout(overallTimer);
|
|
2111
|
-
initialTimer = null;
|
|
2112
|
-
overallTimer = null;
|
|
2113
|
-
};
|
|
2114
|
-
let safeResolve = () => {
|
|
2115
|
-
};
|
|
2116
|
-
let safeReject = () => {
|
|
2117
|
-
};
|
|
2118
|
-
let safeUnroute = () => {
|
|
2119
|
-
};
|
|
2120
|
-
const workPromise = new Promise((resolve, reject) => {
|
|
2121
|
-
let finished = false;
|
|
2122
|
-
let unrouteRequested = false;
|
|
2123
|
-
let hasMatchedOnce = false;
|
|
2124
|
-
safeUnroute = () => {
|
|
2125
|
-
if (!autoUnroute) return;
|
|
2126
|
-
if (unrouteRequested) return;
|
|
2127
|
-
unrouteRequested = true;
|
|
2128
|
-
logger10.info("[MITM] autoUnroute: \u53D6\u6D88\u540E\u7EED\u62E6\u622A");
|
|
2129
|
-
page.unroute(urlPattern, routeHandler).catch(() => {
|
|
2130
|
-
});
|
|
2131
|
-
};
|
|
2132
|
-
safeResolve = (value) => {
|
|
2133
|
-
if (finished) return;
|
|
2134
|
-
finished = true;
|
|
2135
|
-
clearAllTimers();
|
|
2136
|
-
safeUnroute();
|
|
2137
|
-
resolve(value);
|
|
2138
|
-
};
|
|
2139
|
-
safeReject = (error) => {
|
|
2140
|
-
if (finished) return;
|
|
2141
|
-
finished = true;
|
|
2142
|
-
clearAllTimers();
|
|
2143
|
-
safeUnroute();
|
|
2144
|
-
reject(error);
|
|
2145
|
-
};
|
|
2146
|
-
const routeHandler = async (route) => {
|
|
2147
|
-
if (firstMatchOnly && hasMatchedOnce) {
|
|
2148
|
-
logger10.info(`[MITM] firstMatchOnly: \u653E\u884C\u540E\u7EED\u8BF7\u6C42: ${route.request().url()}`);
|
|
2149
|
-
route.continue().catch(() => {
|
|
2150
|
-
});
|
|
2151
|
-
return;
|
|
2152
|
-
}
|
|
2153
|
-
if (firstMatchOnly && !hasMatchedOnce) {
|
|
2154
|
-
hasMatchedOnce = true;
|
|
2155
|
-
logger10.info("[MITM] firstMatchOnly: \u547D\u4E2D\u9996\u4E2A\u8BF7\u6C42\uFF0C\u53D6\u6D88\u540E\u7EED\u62E6\u622A");
|
|
2156
|
-
page.unroute(urlPattern, routeHandler).catch(() => {
|
|
2157
|
-
});
|
|
2158
|
-
}
|
|
2159
|
-
const request = route.request();
|
|
2160
|
-
logger10.info(`[MITM] \u5DF2\u62E6\u622A\u8BF7\u6C42: ${request.url()}`);
|
|
2161
|
-
try {
|
|
2162
|
-
const headers = await request.allHeaders();
|
|
2163
|
-
const postData = request.postData();
|
|
2164
|
-
const urlObj = new import_url2.URL(request.url());
|
|
2165
|
-
delete headers["accept-encoding"];
|
|
2166
|
-
delete headers["content-length"];
|
|
2167
|
-
const reqOptions = {
|
|
2168
|
-
hostname: urlObj.hostname,
|
|
2169
|
-
port: 443,
|
|
2170
|
-
path: urlObj.pathname + urlObj.search,
|
|
2171
|
-
method: request.method(),
|
|
2172
|
-
headers,
|
|
2173
|
-
timeout: overallTimeout
|
|
2174
|
-
};
|
|
2175
|
-
const req = import_https.default.request(reqOptions, (res) => {
|
|
2176
|
-
const chunks = [];
|
|
2177
|
-
let accumulatedText = "";
|
|
2178
|
-
res.on("data", (chunk) => {
|
|
2179
|
-
if (!hasReceivedInitialData) {
|
|
2180
|
-
hasReceivedInitialData = true;
|
|
2181
|
-
if (initialTimer) {
|
|
2182
|
-
clearTimeout(initialTimer);
|
|
2183
|
-
initialTimer = null;
|
|
2184
|
-
}
|
|
2185
|
-
logger10.debug("[Intercept] \u5DF2\u63A5\u6536\u521D\u59CB\u6570\u636E");
|
|
2186
|
-
}
|
|
2187
|
-
chunks.push(chunk);
|
|
2188
|
-
const textChunk = chunk.toString("utf-8");
|
|
2189
|
-
accumulatedText += textChunk;
|
|
2190
|
-
if (onData) {
|
|
2191
|
-
try {
|
|
2192
|
-
onData(textChunk, safeResolve, accumulatedText);
|
|
2193
|
-
} catch (e) {
|
|
2194
|
-
logger10.fail(`onData \u9519\u8BEF`, e);
|
|
2195
|
-
}
|
|
2196
|
-
}
|
|
2197
|
-
});
|
|
2198
|
-
res.on("end", () => {
|
|
2199
|
-
logger10.info("[MITM] \u4E0A\u6E38\u54CD\u5E94\u7ED3\u675F");
|
|
2200
|
-
clearAllTimers();
|
|
2201
|
-
if (onEnd) {
|
|
2202
|
-
try {
|
|
2203
|
-
onEnd(accumulatedText, safeResolve);
|
|
2204
|
-
} catch (e) {
|
|
2205
|
-
logger10.fail(`onEnd \u9519\u8BEF`, e);
|
|
2206
|
-
}
|
|
2207
|
-
} else if (!onData) {
|
|
2208
|
-
safeResolve(accumulatedText);
|
|
2209
|
-
}
|
|
2210
|
-
route.fulfill({
|
|
2211
|
-
status: res.statusCode,
|
|
2212
|
-
headers: res.headers,
|
|
2213
|
-
body: Buffer.concat(chunks)
|
|
2214
|
-
}).catch(() => {
|
|
2215
|
-
});
|
|
2216
|
-
});
|
|
2217
|
-
});
|
|
2218
|
-
req.on("error", (e) => {
|
|
2219
|
-
clearAllTimers();
|
|
2220
|
-
route.abort().catch(() => {
|
|
2221
|
-
});
|
|
2222
|
-
safeReject(e);
|
|
2223
|
-
});
|
|
2224
|
-
if (postData) req.write(postData);
|
|
2225
|
-
req.end();
|
|
2226
|
-
} catch (e) {
|
|
2227
|
-
clearAllTimers();
|
|
2228
|
-
route.continue().catch(() => {
|
|
2229
|
-
});
|
|
2230
|
-
safeReject(e);
|
|
2231
|
-
}
|
|
2232
|
-
};
|
|
2233
|
-
page.route(urlPattern, routeHandler).catch(safeReject);
|
|
2234
|
-
});
|
|
2235
|
-
const timeoutPromise = new Promise((_, reject) => {
|
|
2236
|
-
initialTimer = setTimeout(() => {
|
|
2237
|
-
if (!hasReceivedInitialData) {
|
|
2238
|
-
const error = new CrawlerError({
|
|
2239
|
-
message: `\u521D\u59CB\u6570\u636E\u63A5\u6536\u8D85\u65F6 (${initialTimeout}ms)`,
|
|
2240
|
-
code: Code.InitialTimeout,
|
|
2241
|
-
context: { timeout: initialTimeout }
|
|
2242
|
-
});
|
|
2243
|
-
clearAllTimers();
|
|
2244
|
-
if (onTimeout) {
|
|
2245
|
-
try {
|
|
2246
|
-
onTimeout(error, (err) => safeReject(err));
|
|
2247
|
-
} catch (e) {
|
|
2248
|
-
safeReject(e);
|
|
2249
|
-
}
|
|
2250
|
-
} else {
|
|
2251
|
-
safeReject(error);
|
|
2252
|
-
}
|
|
2253
|
-
}
|
|
2254
|
-
}, initialTimeout);
|
|
2255
|
-
overallTimer = setTimeout(() => {
|
|
2256
|
-
const error = new CrawlerError({
|
|
2257
|
-
message: `\u6574\u4F53\u8BF7\u6C42\u8D85\u65F6 (${overallTimeout}ms)`,
|
|
2258
|
-
code: Code.OverallTimeout,
|
|
2259
|
-
context: { timeout: overallTimeout }
|
|
2260
|
-
});
|
|
2261
|
-
clearAllTimers();
|
|
2262
|
-
if (onTimeout) {
|
|
2263
|
-
try {
|
|
2264
|
-
onTimeout(error, (err) => safeReject(err));
|
|
2265
|
-
} catch (e) {
|
|
2266
|
-
safeReject(e);
|
|
2267
|
-
}
|
|
2268
|
-
} else {
|
|
2269
|
-
safeReject(error);
|
|
2270
|
-
}
|
|
2271
|
-
}, overallTimeout);
|
|
2272
|
-
});
|
|
2273
|
-
workPromise.catch(() => {
|
|
2274
|
-
});
|
|
2275
|
-
timeoutPromise.catch(() => {
|
|
2276
|
-
});
|
|
2277
|
-
const racePromise = Promise.race([workPromise, timeoutPromise]);
|
|
2278
|
-
racePromise.catch(() => {
|
|
2279
|
-
});
|
|
2280
|
-
return racePromise;
|
|
2281
|
-
}
|
|
2282
|
-
};
|
|
2283
|
-
|
|
2284
2877
|
// src/mutation.js
|
|
2285
2878
|
var import_uuid2 = require("uuid");
|
|
2286
|
-
var
|
|
2879
|
+
var logger10 = createInternalLogger("Mutation");
|
|
2287
2880
|
var MUTATION_MONITOR_MODE = Object.freeze({
|
|
2288
2881
|
Added: "added",
|
|
2289
2882
|
Changed: "changed",
|
|
@@ -2316,14 +2909,14 @@ var Mutation = {
|
|
|
2316
2909
|
const stableTime = options.stableTime ?? 5 * 1e3;
|
|
2317
2910
|
const timeout = options.timeout ?? 120 * 1e3;
|
|
2318
2911
|
const onMutation = options.onMutation;
|
|
2319
|
-
|
|
2912
|
+
logger10.start("waitForStable", `\u76D1\u63A7 ${selectorList.length} \u4E2A\u9009\u62E9\u5668, \u7A33\u5B9A\u65F6\u95F4=${stableTime}ms`);
|
|
2320
2913
|
if (initialTimeout > 0) {
|
|
2321
2914
|
const selectorQuery = selectorList.join(",");
|
|
2322
2915
|
try {
|
|
2323
2916
|
await page.waitForSelector(selectorQuery, { timeout: initialTimeout });
|
|
2324
|
-
|
|
2917
|
+
logger10.info(`waitForStable \u5DF2\u68C0\u6D4B\u5230\u5143\u7D20: ${selectorQuery}`);
|
|
2325
2918
|
} catch (e) {
|
|
2326
|
-
|
|
2919
|
+
logger10.warning(`waitForStable \u521D\u59CB\u7B49\u5F85\u8D85\u65F6 (${initialTimeout}ms): ${selectorQuery}`);
|
|
2327
2920
|
throw e;
|
|
2328
2921
|
}
|
|
2329
2922
|
}
|
|
@@ -2339,7 +2932,7 @@ var Mutation = {
|
|
|
2339
2932
|
return "__CONTINUE__";
|
|
2340
2933
|
}
|
|
2341
2934
|
});
|
|
2342
|
-
|
|
2935
|
+
logger10.info("waitForStable \u5DF2\u542F\u7528 onMutation \u56DE\u8C03");
|
|
2343
2936
|
} catch (e) {
|
|
2344
2937
|
}
|
|
2345
2938
|
}
|
|
@@ -2454,9 +3047,9 @@ var Mutation = {
|
|
|
2454
3047
|
{ selectorList, stableTime, timeout, callbackName, hasCallback: !!onMutation }
|
|
2455
3048
|
);
|
|
2456
3049
|
if (result.mutationCount === 0 && result.stableTime === 0) {
|
|
2457
|
-
|
|
3050
|
+
logger10.warning("waitForStable \u672A\u627E\u5230\u53EF\u76D1\u63A7\u7684\u5143\u7D20");
|
|
2458
3051
|
}
|
|
2459
|
-
|
|
3052
|
+
logger10.success("waitForStable", `DOM \u7A33\u5B9A, \u603B\u5171 ${result.mutationCount} \u6B21\u53D8\u5316${result.wasPaused ? ", \u66FE\u6682\u505C\u8BA1\u65F6" : ""}`);
|
|
2460
3053
|
return result;
|
|
2461
3054
|
},
|
|
2462
3055
|
/**
|
|
@@ -2574,22 +3167,22 @@ var Mutation = {
|
|
|
2574
3167
|
return "__CONTINUE__";
|
|
2575
3168
|
}
|
|
2576
3169
|
};
|
|
2577
|
-
|
|
3170
|
+
logger10.start(
|
|
2578
3171
|
"waitForStableAcrossRoots",
|
|
2579
3172
|
`\u76D1\u63A7 ${selectorList.length} \u4E2A\u9009\u62E9\u5668(\u8DE8 root), \u7A33\u5B9A\u65F6\u95F4=${waitForStableTime}ms`
|
|
2580
3173
|
);
|
|
2581
3174
|
if (initialTimeout > 0) {
|
|
2582
3175
|
try {
|
|
2583
3176
|
await page.waitForSelector(selectorQuery, { timeout: initialTimeout });
|
|
2584
|
-
|
|
3177
|
+
logger10.info(`waitForStableAcrossRoots \u5DF2\u68C0\u6D4B\u5230\u5143\u7D20: ${selectorQuery}`);
|
|
2585
3178
|
} catch (e) {
|
|
2586
|
-
|
|
3179
|
+
logger10.warning(`waitForStableAcrossRoots \u521D\u59CB\u7B49\u5F85\u8D85\u65F6 (${initialTimeout}ms): ${selectorQuery}`);
|
|
2587
3180
|
throw e;
|
|
2588
3181
|
}
|
|
2589
3182
|
}
|
|
2590
3183
|
let state = await buildState();
|
|
2591
3184
|
if (!state?.hasMatched) {
|
|
2592
|
-
|
|
3185
|
+
logger10.warning("waitForStableAcrossRoots \u672A\u627E\u5230\u53EF\u76D1\u63A7\u7684\u5143\u7D20");
|
|
2593
3186
|
return { mutationCount: 0, stableTime: 0, wasPaused: false };
|
|
2594
3187
|
}
|
|
2595
3188
|
let mutationCount = 0;
|
|
@@ -2626,7 +3219,7 @@ var Mutation = {
|
|
|
2626
3219
|
if (lastState.snapshotKey !== lastSnapshotKey) {
|
|
2627
3220
|
lastSnapshotKey = lastState.snapshotKey;
|
|
2628
3221
|
mutationCount += 1;
|
|
2629
|
-
|
|
3222
|
+
logger10.info(
|
|
2630
3223
|
`waitForStableAcrossRoots \u53D8\u5316#${mutationCount}, len=${lastState.snapshotLength}, path=${lastState.primaryPath || "unknown"}, preview="${truncate(lastState.text, 120)}"`
|
|
2631
3224
|
);
|
|
2632
3225
|
const signal = await invokeMutationCallback({
|
|
@@ -2639,7 +3232,7 @@ var Mutation = {
|
|
|
2639
3232
|
continue;
|
|
2640
3233
|
}
|
|
2641
3234
|
if (!isPaused && stableSince > 0 && Date.now() - stableSince >= waitForStableTime) {
|
|
2642
|
-
|
|
3235
|
+
logger10.success("waitForStableAcrossRoots", `DOM \u7A33\u5B9A, \u603B\u5171 ${mutationCount} \u6B21\u53D8\u5316${wasPaused ? ", \u66FE\u6682\u505C\u8BA1\u65F6" : ""}`);
|
|
2643
3236
|
return {
|
|
2644
3237
|
mutationCount,
|
|
2645
3238
|
stableTime: waitForStableTime,
|
|
@@ -2666,7 +3259,7 @@ var Mutation = {
|
|
|
2666
3259
|
const onMutation = options.onMutation;
|
|
2667
3260
|
const rawMode = String(options.mode || MUTATION_MONITOR_MODE.Added).toLowerCase();
|
|
2668
3261
|
const mode = [MUTATION_MONITOR_MODE.Added, MUTATION_MONITOR_MODE.Changed, MUTATION_MONITOR_MODE.All].includes(rawMode) ? rawMode : MUTATION_MONITOR_MODE.Added;
|
|
2669
|
-
|
|
3262
|
+
logger10.start("useMonitor", `\u76D1\u63A7 ${selectorList.length} \u4E2A\u9009\u62E9\u5668, mode=${mode}`);
|
|
2670
3263
|
const monitorKey = generateKey("pk_mon");
|
|
2671
3264
|
const callbackName = generateKey("pk_mon_cb");
|
|
2672
3265
|
const cleanerName = generateKey("pk_mon_clean");
|
|
@@ -2809,7 +3402,7 @@ var Mutation = {
|
|
|
2809
3402
|
return total;
|
|
2810
3403
|
};
|
|
2811
3404
|
}, { selectorList, monitorKey, callbackName, cleanerName, hasCallback: !!onMutation, mode });
|
|
2812
|
-
|
|
3405
|
+
logger10.success("useMonitor", "\u76D1\u63A7\u5668\u5DF2\u542F\u52A8");
|
|
2813
3406
|
return {
|
|
2814
3407
|
stop: async () => {
|
|
2815
3408
|
let totalMutations = 0;
|
|
@@ -2822,7 +3415,7 @@ var Mutation = {
|
|
|
2822
3415
|
}, cleanerName);
|
|
2823
3416
|
} catch (e) {
|
|
2824
3417
|
}
|
|
2825
|
-
|
|
3418
|
+
logger10.success("useMonitor.stop", `\u76D1\u63A7\u5DF2\u505C\u6B62, \u5171 ${totalMutations} \u6B21\u53D8\u5316`);
|
|
2826
3419
|
return { totalMutations };
|
|
2827
3420
|
}
|
|
2828
3421
|
};
|
|
@@ -2833,28 +3426,28 @@ var Mutation = {
|
|
|
2833
3426
|
var Display = {
|
|
2834
3427
|
parseTokenDisplayName(value) {
|
|
2835
3428
|
if (!value) {
|
|
2836
|
-
return { owner: "",
|
|
3429
|
+
return { owner: "", envType: "", note: "" };
|
|
2837
3430
|
}
|
|
2838
3431
|
const parts = String(value).split(":");
|
|
2839
3432
|
const owner = (parts[0] || "").trim();
|
|
2840
|
-
const
|
|
3433
|
+
const envType = (parts[1] || "").trim();
|
|
2841
3434
|
const note = parts.length > 2 ? parts.slice(2).join(":").trim() : "";
|
|
2842
|
-
return { owner,
|
|
3435
|
+
return { owner, envType, note };
|
|
2843
3436
|
},
|
|
2844
|
-
|
|
3437
|
+
parseEnvDisplayName(value) {
|
|
2845
3438
|
if (!value) {
|
|
2846
|
-
return {
|
|
3439
|
+
return { env: "", owner: "", envType: "", note: "" };
|
|
2847
3440
|
}
|
|
2848
3441
|
const parts = String(value).split(":");
|
|
2849
|
-
const
|
|
3442
|
+
const env = (parts[0] || "").trim();
|
|
2850
3443
|
const owner = (parts[1] || "").trim();
|
|
2851
|
-
const
|
|
3444
|
+
const envType = (parts[2] || "").trim();
|
|
2852
3445
|
const note = parts.length > 3 ? parts.slice(3).join(":").trim() : "";
|
|
2853
|
-
return {
|
|
3446
|
+
return { env, owner, envType, note };
|
|
2854
3447
|
},
|
|
2855
|
-
buildTokenDisplayName({ owner,
|
|
3448
|
+
buildTokenDisplayName({ owner, envType, note } = {}) {
|
|
2856
3449
|
const trimmedOwner = owner?.trim() || "";
|
|
2857
|
-
const trimmedType =
|
|
3450
|
+
const trimmedType = envType?.trim() || "";
|
|
2858
3451
|
const trimmedNote = note?.trim() || "";
|
|
2859
3452
|
const parts = [trimmedOwner, trimmedType];
|
|
2860
3453
|
if (trimmedNote) {
|
|
@@ -2877,12 +3470,12 @@ var Display = {
|
|
|
2877
3470
|
const secondary = hasName && short ? short : "";
|
|
2878
3471
|
return { primary, secondary, fullId: cleanId, shortId: short };
|
|
2879
3472
|
},
|
|
2880
|
-
|
|
2881
|
-
const parts = this.
|
|
2882
|
-
const cleanId = String(
|
|
3473
|
+
resolveEnvIdentity(displayName, envId) {
|
|
3474
|
+
const parts = this.parseEnvDisplayName(displayName);
|
|
3475
|
+
const cleanId = String(envId || "").trim();
|
|
2883
3476
|
const short = this.shortId(cleanId, 8);
|
|
2884
|
-
const hasName = parts.
|
|
2885
|
-
const primary = hasName ? parts.
|
|
3477
|
+
const hasName = parts.env !== "" && parts.env !== cleanId;
|
|
3478
|
+
const primary = hasName ? parts.env : short || "-";
|
|
2886
3479
|
const secondary = hasName && short ? short : "";
|
|
2887
3480
|
return { primary, secondary, parts, fullId: cleanId, shortId: short };
|
|
2888
3481
|
}
|
|
@@ -3691,7 +4284,7 @@ var createTemplateLogger = (baseLogger = createBaseLogger()) => {
|
|
|
3691
4284
|
};
|
|
3692
4285
|
var getDefaultBaseLogger = () => createBaseLogger("");
|
|
3693
4286
|
var Logger = {
|
|
3694
|
-
setLogger: (
|
|
4287
|
+
setLogger: (logger12) => setDefaultLogger(logger12),
|
|
3695
4288
|
info: (message) => getDefaultBaseLogger().info(message),
|
|
3696
4289
|
success: (message) => getDefaultBaseLogger().success(message),
|
|
3697
4290
|
warning: (message) => getDefaultBaseLogger().warning(message),
|
|
@@ -3699,15 +4292,15 @@ var Logger = {
|
|
|
3699
4292
|
error: (message) => getDefaultBaseLogger().error(message),
|
|
3700
4293
|
debug: (message) => getDefaultBaseLogger().debug(message),
|
|
3701
4294
|
start: (message) => getDefaultBaseLogger().start(message),
|
|
3702
|
-
useTemplate: (
|
|
3703
|
-
if (
|
|
4295
|
+
useTemplate: (logger12) => {
|
|
4296
|
+
if (logger12) return createTemplateLogger(createBaseLogger("", logger12));
|
|
3704
4297
|
return createTemplateLogger();
|
|
3705
4298
|
}
|
|
3706
4299
|
};
|
|
3707
4300
|
|
|
3708
4301
|
// src/share.js
|
|
3709
4302
|
var import_delay2 = __toESM(require("delay"), 1);
|
|
3710
|
-
var
|
|
4303
|
+
var logger11 = createInternalLogger("Share");
|
|
3711
4304
|
var DEFAULT_TIMEOUT_MS2 = 50 * 1e3;
|
|
3712
4305
|
var DEFAULT_PAYLOAD_SNAPSHOT_MAX_LEN = 500;
|
|
3713
4306
|
var DEFAULT_POLL_INTERVAL_MS = 120;
|
|
@@ -3827,7 +4420,7 @@ var createDomShareMonitor = async (page, options = {}) => {
|
|
|
3827
4420
|
const onMatch = typeof options.onMatch === "function" ? options.onMatch : null;
|
|
3828
4421
|
const onTelemetry = typeof options.onTelemetry === "function" ? options.onTelemetry : null;
|
|
3829
4422
|
let matched = false;
|
|
3830
|
-
|
|
4423
|
+
logger11.info(`DOM \u76D1\u542C\u51C6\u5907\u6302\u8F7D: selectors=${toJsonInline(selectors, 120)}, mode=${mode}`);
|
|
3831
4424
|
const monitor = await Mutation.useMonitor(page, selectors, {
|
|
3832
4425
|
mode,
|
|
3833
4426
|
onMutation: (context = {}) => {
|
|
@@ -3845,12 +4438,12 @@ ${text}`;
|
|
|
3845
4438
|
});
|
|
3846
4439
|
}
|
|
3847
4440
|
if (mutationCount <= 5 || mutationCount % 50 === 0) {
|
|
3848
|
-
|
|
4441
|
+
logger11.info(`DOM \u53D8\u5316\u5DF2\u6355\u83B7: mutationCount=${mutationCount}, mutationNodes=${mutationNodes.length}`);
|
|
3849
4442
|
}
|
|
3850
4443
|
const [candidate] = Utils.parseLinks(rawDom, { prefix }) || [];
|
|
3851
4444
|
if (!candidate) return;
|
|
3852
4445
|
matched = true;
|
|
3853
|
-
|
|
4446
|
+
logger11.success("captureLink.domHit", `DOM \u547D\u4E2D\u5206\u4EAB\u94FE\u63A5: mutationCount=${mutationCount}, link=${candidate}`);
|
|
3854
4447
|
if (onMatch) {
|
|
3855
4448
|
onMatch({
|
|
3856
4449
|
link: candidate,
|
|
@@ -3866,7 +4459,7 @@ ${text}`;
|
|
|
3866
4459
|
return {
|
|
3867
4460
|
stop: async () => {
|
|
3868
4461
|
const result = await monitor.stop();
|
|
3869
|
-
|
|
4462
|
+
logger11.info(`DOM \u76D1\u542C\u5DF2\u505C\u6B62: totalMutations=${result?.totalMutations || 0}`);
|
|
3870
4463
|
return result;
|
|
3871
4464
|
}
|
|
3872
4465
|
};
|
|
@@ -3906,8 +4499,8 @@ var Share = {
|
|
|
3906
4499
|
if (share.mode === "response" && apiMatchers.length === 0) {
|
|
3907
4500
|
throw new Error("Share.captureLink requires share.xurl[0] api matcher when mode=response");
|
|
3908
4501
|
}
|
|
3909
|
-
|
|
3910
|
-
|
|
4502
|
+
logger11.start("captureLink", `mode=${share.mode}, timeoutMs=${timeoutMs}, prefix=${share.prefix}`);
|
|
4503
|
+
logger11.info(`captureLink \u914D\u7F6E: xurl=${toJsonInline(share.xurl)}, domMode=${domMode}, domSelectors=${toJsonInline(domSelectors, 120)}`);
|
|
3911
4504
|
const stats = {
|
|
3912
4505
|
actionTimedOut: false,
|
|
3913
4506
|
domMutationCount: 0,
|
|
@@ -3932,7 +4525,7 @@ var Share = {
|
|
|
3932
4525
|
link: validated,
|
|
3933
4526
|
payloadText: String(payloadText || "")
|
|
3934
4527
|
};
|
|
3935
|
-
|
|
4528
|
+
logger11.info(`\u5019\u9009\u94FE\u63A5\u5DF2\u786E\u8BA4: source=${source}, link=${validated}`);
|
|
3936
4529
|
return true;
|
|
3937
4530
|
};
|
|
3938
4531
|
const resolveResponseCandidate = (responseText) => {
|
|
@@ -3967,7 +4560,7 @@ var Share = {
|
|
|
3967
4560
|
try {
|
|
3968
4561
|
await monitor.stop();
|
|
3969
4562
|
} catch (error) {
|
|
3970
|
-
|
|
4563
|
+
logger11.warning(`\u505C\u6B62 DOM \u76D1\u542C\u5931\u8D25: ${error instanceof Error ? error.message : String(error)}`);
|
|
3971
4564
|
}
|
|
3972
4565
|
};
|
|
3973
4566
|
const onResponse = async (response) => {
|
|
@@ -3980,29 +4573,29 @@ var Share = {
|
|
|
3980
4573
|
stats.responseSampleUrls.push(url);
|
|
3981
4574
|
}
|
|
3982
4575
|
if (stats.responseObserved <= 5) {
|
|
3983
|
-
|
|
4576
|
+
logger11.info(`\u63A5\u53E3\u54CD\u5E94\u91C7\u6837(${stats.responseObserved}): ${url}`);
|
|
3984
4577
|
}
|
|
3985
4578
|
if (!apiMatchers.some((matcher) => url.includes(matcher))) return;
|
|
3986
4579
|
stats.responseMatched += 1;
|
|
3987
4580
|
stats.lastMatchedUrl = url;
|
|
3988
|
-
|
|
4581
|
+
logger11.info(`\u63A5\u53E3\u547D\u4E2D\u5339\u914D(${stats.responseMatched}): ${url}`);
|
|
3989
4582
|
const text = await response.text();
|
|
3990
4583
|
const hit = resolveResponseCandidate(text);
|
|
3991
4584
|
if (!hit?.link) {
|
|
3992
4585
|
if (stats.responseMatched <= 3) {
|
|
3993
|
-
|
|
4586
|
+
logger11.info(`\u63A5\u53E3\u89E3\u6790\u5B8C\u6210\u4F46\u672A\u63D0\u53D6\u5230\u5206\u4EAB\u94FE\u63A5: payloadSize=${text.length}`);
|
|
3994
4587
|
}
|
|
3995
4588
|
return;
|
|
3996
4589
|
}
|
|
3997
4590
|
stats.responseResolved += 1;
|
|
3998
|
-
|
|
4591
|
+
logger11.success("captureLink.responseHit", `\u63A5\u53E3\u547D\u4E2D\u5206\u4EAB\u94FE\u63A5: url=${url}, link=${hit.link}`);
|
|
3999
4592
|
setCandidate("response", hit.link, hit.payloadText);
|
|
4000
4593
|
} catch (error) {
|
|
4001
|
-
|
|
4594
|
+
logger11.warning(`\u63A5\u53E3\u54CD\u5E94\u5904\u7406\u5F02\u5E38: ${error instanceof Error ? error.message : String(error)}`);
|
|
4002
4595
|
}
|
|
4003
4596
|
};
|
|
4004
4597
|
if (share.mode === "dom") {
|
|
4005
|
-
|
|
4598
|
+
logger11.info("\u5F53\u524D\u4E3A DOM \u6A21\u5F0F\uFF0C\u4EC5\u542F\u7528 DOM \u76D1\u542C");
|
|
4006
4599
|
domMonitor = await createDomShareMonitor(page, {
|
|
4007
4600
|
prefix: share.prefix,
|
|
4008
4601
|
selectors: domSelectors,
|
|
@@ -4017,14 +4610,14 @@ var Share = {
|
|
|
4017
4610
|
});
|
|
4018
4611
|
}
|
|
4019
4612
|
if (share.mode === "response") {
|
|
4020
|
-
|
|
4613
|
+
logger11.info(`\u5F53\u524D\u4E3A\u63A5\u53E3\u6A21\u5F0F\uFF0C\u6302\u8F7D response \u76D1\u542C: apiMatchers=${toJsonInline(apiMatchers, 160)}`);
|
|
4021
4614
|
page.on("response", onResponse);
|
|
4022
4615
|
}
|
|
4023
4616
|
const deadline = Date.now() + timeoutMs;
|
|
4024
4617
|
const getRemainingMs = () => Math.max(0, deadline - Date.now());
|
|
4025
4618
|
try {
|
|
4026
4619
|
const actionTimeout = getRemainingMs();
|
|
4027
|
-
|
|
4620
|
+
logger11.start("captureLink.performActions", `\u6267\u884C\u52A8\u4F5C\u9884\u7B97=${actionTimeout}ms`);
|
|
4028
4621
|
if (actionTimeout > 0) {
|
|
4029
4622
|
let timer = null;
|
|
4030
4623
|
let actionError = null;
|
|
@@ -4038,21 +4631,21 @@ var Share = {
|
|
|
4038
4631
|
const actionResult = await Promise.race([actionPromise, timeoutPromise]);
|
|
4039
4632
|
if (timer) clearTimeout(timer);
|
|
4040
4633
|
if (actionResult === "__ACTION_ERROR__") {
|
|
4041
|
-
|
|
4634
|
+
logger11.fail("captureLink.performActions", actionError);
|
|
4042
4635
|
throw actionError;
|
|
4043
4636
|
}
|
|
4044
4637
|
if (actionResult === "__ACTION_TIMEOUT__") {
|
|
4045
4638
|
stats.actionTimedOut = true;
|
|
4046
|
-
|
|
4639
|
+
logger11.warning(`performActions \u5DF2\u8D85\u65F6 (${actionTimeout}ms)\uFF0C\u52A8\u4F5C\u53EF\u80FD\u4ECD\u5728\u5F02\u6B65\u6267\u884C`);
|
|
4047
4640
|
} else {
|
|
4048
|
-
|
|
4641
|
+
logger11.success("captureLink.performActions", "\u6267\u884C\u52A8\u4F5C\u5B8C\u6210");
|
|
4049
4642
|
}
|
|
4050
4643
|
}
|
|
4051
4644
|
let nextProgressLogTs = Date.now() + 3e3;
|
|
4052
4645
|
while (true) {
|
|
4053
4646
|
const selected = share.mode === "dom" ? candidates.dom : candidates.response;
|
|
4054
4647
|
if (selected?.link) {
|
|
4055
|
-
|
|
4648
|
+
logger11.success("captureLink", `\u6355\u83B7\u6210\u529F: source=${share.mode}, link=${selected.link}`);
|
|
4056
4649
|
return {
|
|
4057
4650
|
link: selected.link,
|
|
4058
4651
|
payloadText: selected.payloadText,
|
|
@@ -4064,7 +4657,7 @@ var Share = {
|
|
|
4064
4657
|
if (remaining <= 0) break;
|
|
4065
4658
|
const now = Date.now();
|
|
4066
4659
|
if (now >= nextProgressLogTs) {
|
|
4067
|
-
|
|
4660
|
+
logger11.info(
|
|
4068
4661
|
`captureLink \u7B49\u5F85\u4E2D: remaining=${remaining}ms, domMutationCount=${stats.domMutationCount}, responseMatched=${stats.responseMatched}`
|
|
4069
4662
|
);
|
|
4070
4663
|
nextProgressLogTs = now + 5e3;
|
|
@@ -4072,11 +4665,11 @@ var Share = {
|
|
|
4072
4665
|
await (0, import_delay2.default)(Math.max(0, Math.min(DEFAULT_POLL_INTERVAL_MS, remaining)));
|
|
4073
4666
|
}
|
|
4074
4667
|
if (share.mode === "response" && stats.responseMatched === 0) {
|
|
4075
|
-
|
|
4668
|
+
logger11.warning(
|
|
4076
4669
|
`\u63A5\u53E3\u76D1\u542C\u672A\u547D\u4E2D: apiMatchers=${toJsonInline(apiMatchers, 220)}, \u54CD\u5E94\u6837\u672CURLs=${toJsonInline(stats.responseSampleUrls, 420)}`
|
|
4077
4670
|
);
|
|
4078
4671
|
}
|
|
4079
|
-
|
|
4672
|
+
logger11.warning(
|
|
4080
4673
|
`captureLink \u8D85\u65F6\u672A\u62FF\u5230\u94FE\u63A5: mode=${share.mode}, actionTimedOut=${stats.actionTimedOut}, domMutationCount=${stats.domMutationCount}, responseObserved=${stats.responseObserved}, responseMatched=${stats.responseMatched}, lastMatchedUrl=${stats.lastMatchedUrl || "none"}`
|
|
4081
4674
|
);
|
|
4082
4675
|
return {
|
|
@@ -4088,7 +4681,7 @@ var Share = {
|
|
|
4088
4681
|
} finally {
|
|
4089
4682
|
if (share.mode === "response") {
|
|
4090
4683
|
page.off("response", onResponse);
|
|
4091
|
-
|
|
4684
|
+
logger11.info("response \u76D1\u542C\u5DF2\u5378\u8F7D");
|
|
4092
4685
|
}
|
|
4093
4686
|
await stopDomMonitor();
|
|
4094
4687
|
}
|
|
@@ -4174,13 +4767,14 @@ var usePlaywrightToolKit = () => {
|
|
|
4174
4767
|
LiveView,
|
|
4175
4768
|
Constants: constants_exports,
|
|
4176
4769
|
Utils,
|
|
4770
|
+
RuntimeEnv,
|
|
4177
4771
|
Captcha,
|
|
4178
|
-
Sse,
|
|
4179
4772
|
Errors: errors_exports,
|
|
4180
4773
|
Mutation,
|
|
4181
4774
|
Display,
|
|
4182
4775
|
Logger,
|
|
4183
4776
|
Share,
|
|
4777
|
+
ByPass,
|
|
4184
4778
|
$Internals: { LOG_TEMPLATES, stripAnsi }
|
|
4185
4779
|
};
|
|
4186
4780
|
};
|