@skrillex1224/playwright-toolkit 2.1.170 → 2.1.180
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -1
- package/dist/browser.js +96 -15
- package/dist/browser.js.map +3 -3
- package/dist/index.cjs +942 -348
- package/dist/index.cjs.map +4 -4
- package/dist/index.js +942 -348
- package/dist/index.js.map +4 -4
- package/index.d.ts +89 -39
- package/package.json +3 -1
- /package/dist/{proxy-meter.js → internals/proxy-meter.js} +0 -0
package/dist/index.js
CHANGED
|
@@ -308,18 +308,18 @@ var fallbackLog = {
|
|
|
308
308
|
error: (...args) => console.error(...args),
|
|
309
309
|
debug: (...args) => console.debug ? console.debug(...args) : console.log(...args)
|
|
310
310
|
};
|
|
311
|
-
var resolveLogMethod = (
|
|
312
|
-
if (
|
|
313
|
-
return
|
|
311
|
+
var resolveLogMethod = (logger12, name) => {
|
|
312
|
+
if (logger12 && typeof logger12[name] === "function") {
|
|
313
|
+
return logger12[name].bind(logger12);
|
|
314
314
|
}
|
|
315
|
-
if (name === "warning" &&
|
|
316
|
-
return
|
|
315
|
+
if (name === "warning" && logger12 && typeof logger12.warn === "function") {
|
|
316
|
+
return logger12.warn.bind(logger12);
|
|
317
317
|
}
|
|
318
318
|
return fallbackLog[name];
|
|
319
319
|
};
|
|
320
320
|
var defaultLogger = null;
|
|
321
|
-
var setDefaultLogger = (
|
|
322
|
-
defaultLogger =
|
|
321
|
+
var setDefaultLogger = (logger12) => {
|
|
322
|
+
defaultLogger = logger12;
|
|
323
323
|
};
|
|
324
324
|
var resolveLogger = (explicitLogger) => {
|
|
325
325
|
if (explicitLogger && typeof explicitLogger.info === "function") {
|
|
@@ -346,8 +346,8 @@ var colorize = (text, color) => {
|
|
|
346
346
|
var createBaseLogger = (prefix = "", explicitLogger) => {
|
|
347
347
|
const name = prefix ? String(prefix) : "";
|
|
348
348
|
const dispatch = (methodName, icon, message, color) => {
|
|
349
|
-
const
|
|
350
|
-
const logFn = resolveLogMethod(
|
|
349
|
+
const logger12 = resolveLogger(explicitLogger);
|
|
350
|
+
const logFn = resolveLogMethod(logger12, methodName);
|
|
351
351
|
const timestamp = colorize(`[${formatTimestamp()}]`, ANSI.gray);
|
|
352
352
|
const line = formatLine(name, icon, message);
|
|
353
353
|
const coloredLine = colorize(line, color);
|
|
@@ -546,7 +546,7 @@ var CrawlerError = class _CrawlerError extends Error {
|
|
|
546
546
|
// src/apify-kit.js
|
|
547
547
|
import { serializeError as serializeError2 } from "serialize-error";
|
|
548
548
|
|
|
549
|
-
// src/proxy-meter-runtime.js
|
|
549
|
+
// src/internals/proxy-meter-runtime.js
|
|
550
550
|
import { spawn, spawnSync } from "child_process";
|
|
551
551
|
import { existsSync, mkdirSync, readFileSync, rmSync } from "fs";
|
|
552
552
|
import { tmpdir } from "os";
|
|
@@ -615,6 +615,10 @@ var resolveResourceTypeMeta = (domain) => {
|
|
|
615
615
|
};
|
|
616
616
|
var resolveScriptPath = () => {
|
|
617
617
|
const baseDir = typeof __dirname !== "undefined" ? __dirname : path.dirname(fileURLToPath(import.meta.url));
|
|
618
|
+
const internalCandidate = path.join(baseDir, "internals", "proxy-meter.js");
|
|
619
|
+
if (existsSync(internalCandidate)) {
|
|
620
|
+
return internalCandidate;
|
|
621
|
+
}
|
|
618
622
|
return path.join(baseDir, "proxy-meter.js");
|
|
619
623
|
};
|
|
620
624
|
var pickFreePort = () => {
|
|
@@ -898,9 +902,589 @@ var getProxyMeterSnapshot = async (options = {}) => {
|
|
|
898
902
|
}
|
|
899
903
|
return snapshot;
|
|
900
904
|
};
|
|
905
|
+
var ProxyMeterRuntime = {
|
|
906
|
+
startProxyMeter,
|
|
907
|
+
recordProxyMeterResourceType,
|
|
908
|
+
stopProxyMeter,
|
|
909
|
+
getProxyMeterSnapshot
|
|
910
|
+
};
|
|
911
|
+
|
|
912
|
+
// src/runtime-env.js
|
|
913
|
+
var BROWSER_PROFILE_SCHEMA_VERSION = 1;
|
|
914
|
+
var rememberedRuntimeState = null;
|
|
915
|
+
var isPlainObject = (value) => value && typeof value === "object" && !Array.isArray(value);
|
|
916
|
+
var deepClone = (value) => {
|
|
917
|
+
if (value == null) return value;
|
|
918
|
+
try {
|
|
919
|
+
return JSON.parse(JSON.stringify(value));
|
|
920
|
+
} catch {
|
|
921
|
+
return value;
|
|
922
|
+
}
|
|
923
|
+
};
|
|
924
|
+
var tryParseJSON = (value) => {
|
|
925
|
+
if (value == null) return null;
|
|
926
|
+
if (typeof value === "object") return value;
|
|
927
|
+
const raw = String(value || "").trim();
|
|
928
|
+
if (!raw) return null;
|
|
929
|
+
try {
|
|
930
|
+
return JSON.parse(raw);
|
|
931
|
+
} catch {
|
|
932
|
+
return null;
|
|
933
|
+
}
|
|
934
|
+
};
|
|
935
|
+
var normalizeLocalStorage = (value) => {
|
|
936
|
+
const source = value && typeof value === "object" && !Array.isArray(value) ? value : {};
|
|
937
|
+
return Object.entries(source).reduce((acc, [key, item]) => {
|
|
938
|
+
const safeKey = String(key || "").trim();
|
|
939
|
+
const safeValue = String(item ?? "").trim();
|
|
940
|
+
if (!safeKey || !safeValue || safeValue === "<nil>") return acc;
|
|
941
|
+
acc[safeKey] = safeValue;
|
|
942
|
+
return acc;
|
|
943
|
+
}, {});
|
|
944
|
+
};
|
|
945
|
+
var normalizeSessionStorage = (value) => {
|
|
946
|
+
const source = value && typeof value === "object" && !Array.isArray(value) ? value : {};
|
|
947
|
+
return Object.entries(source).reduce((acc, [key, item]) => {
|
|
948
|
+
const safeKey = String(key || "").trim();
|
|
949
|
+
const safeValue = String(item ?? "").trim();
|
|
950
|
+
if (!safeKey || !safeValue || safeValue === "<nil>") return acc;
|
|
951
|
+
acc[safeKey] = safeValue;
|
|
952
|
+
return acc;
|
|
953
|
+
}, {});
|
|
954
|
+
};
|
|
955
|
+
var normalizeAuth = (value) => {
|
|
956
|
+
const source = value && typeof value === "object" && !Array.isArray(value) ? value : {};
|
|
957
|
+
return Object.entries(source).reduce((acc, [key, item]) => {
|
|
958
|
+
const safeKey = String(key || "").trim();
|
|
959
|
+
const safeValue = String(item ?? "").trim();
|
|
960
|
+
if (!safeKey || !safeValue || safeValue === "<nil>") return acc;
|
|
961
|
+
acc[safeKey] = safeValue;
|
|
962
|
+
return acc;
|
|
963
|
+
}, {});
|
|
964
|
+
};
|
|
965
|
+
var normalizeCookies = (value) => {
|
|
966
|
+
if (!Array.isArray(value)) return [];
|
|
967
|
+
return value.map((item) => {
|
|
968
|
+
const raw = item && typeof item === "object" ? item : {};
|
|
969
|
+
const name = String(raw.name || "").trim();
|
|
970
|
+
const cookieValue = String(raw.value ?? "").trim();
|
|
971
|
+
if (!name || !cookieValue || cookieValue === "<nil>") return null;
|
|
972
|
+
const domain = String(raw.domain || "").trim();
|
|
973
|
+
const path2 = String(raw.path || "").trim() || "/";
|
|
974
|
+
const sameSiteRaw = String(raw.sameSite || "").trim();
|
|
975
|
+
return {
|
|
976
|
+
...raw,
|
|
977
|
+
name,
|
|
978
|
+
value: cookieValue,
|
|
979
|
+
domain,
|
|
980
|
+
path: path2,
|
|
981
|
+
...sameSiteRaw ? { sameSite: sameSiteRaw } : {}
|
|
982
|
+
};
|
|
983
|
+
}).filter(Boolean);
|
|
984
|
+
};
|
|
985
|
+
var buildCookieMap = (cookies) => cookies.reduce((acc, item) => {
|
|
986
|
+
acc[item.name] = item.value;
|
|
987
|
+
return acc;
|
|
988
|
+
}, {});
|
|
989
|
+
var normalizeObservedBrowserProfile = (value) => {
|
|
990
|
+
const source = isPlainObject(value) ? value : {};
|
|
991
|
+
const profile = {};
|
|
992
|
+
const stringFields = {
|
|
993
|
+
user_agent: source.user_agent,
|
|
994
|
+
platform: source.platform,
|
|
995
|
+
language: source.language,
|
|
996
|
+
timezone: source.timezone
|
|
997
|
+
};
|
|
998
|
+
Object.entries(stringFields).forEach(([key, item]) => {
|
|
999
|
+
const safeValue = String(item || "").trim();
|
|
1000
|
+
if (safeValue) {
|
|
1001
|
+
profile[key] = safeValue;
|
|
1002
|
+
}
|
|
1003
|
+
});
|
|
1004
|
+
if (Array.isArray(source.languages)) {
|
|
1005
|
+
const languages = source.languages.map((item) => String(item || "").trim()).filter(Boolean);
|
|
1006
|
+
if (languages.length > 0) {
|
|
1007
|
+
profile.languages = languages;
|
|
1008
|
+
}
|
|
1009
|
+
}
|
|
1010
|
+
const numericFields = {
|
|
1011
|
+
hardware_concurrency: Number(source.hardware_concurrency || 0),
|
|
1012
|
+
device_memory: Number(source.device_memory || 0)
|
|
1013
|
+
};
|
|
1014
|
+
Object.entries(numericFields).forEach(([key, item]) => {
|
|
1015
|
+
if (Number.isFinite(item) && item > 0) {
|
|
1016
|
+
profile[key] = item;
|
|
1017
|
+
}
|
|
1018
|
+
});
|
|
1019
|
+
if (isPlainObject(source.viewport)) {
|
|
1020
|
+
const width = Number(source.viewport.width || 0);
|
|
1021
|
+
const height = Number(source.viewport.height || 0);
|
|
1022
|
+
if (width > 0 && height > 0) {
|
|
1023
|
+
profile.viewport = { width, height };
|
|
1024
|
+
}
|
|
1025
|
+
}
|
|
1026
|
+
if (isPlainObject(source.screen)) {
|
|
1027
|
+
const screenProfile = {};
|
|
1028
|
+
const screenFields = {
|
|
1029
|
+
width: Number(source.screen.width || 0),
|
|
1030
|
+
height: Number(source.screen.height || 0),
|
|
1031
|
+
avail_width: Number(source.screen.avail_width || 0),
|
|
1032
|
+
avail_height: Number(source.screen.avail_height || 0),
|
|
1033
|
+
color_depth: Number(source.screen.color_depth || 0),
|
|
1034
|
+
pixel_depth: Number(source.screen.pixel_depth || 0)
|
|
1035
|
+
};
|
|
1036
|
+
Object.entries(screenFields).forEach(([key, item]) => {
|
|
1037
|
+
if (Number.isFinite(item) && item > 0) {
|
|
1038
|
+
screenProfile[key] = item;
|
|
1039
|
+
}
|
|
1040
|
+
});
|
|
1041
|
+
if (Object.keys(screenProfile).length > 0) {
|
|
1042
|
+
profile.screen = screenProfile;
|
|
1043
|
+
}
|
|
1044
|
+
}
|
|
1045
|
+
return profile;
|
|
1046
|
+
};
|
|
1047
|
+
var normalizeBrowserProfileCore = (value) => {
|
|
1048
|
+
const source = isPlainObject(value) ? value : {};
|
|
1049
|
+
const profile = {};
|
|
1050
|
+
if (isPlainObject(source.fingerprint) && Object.keys(source.fingerprint).length > 0) {
|
|
1051
|
+
profile.fingerprint = deepClone(source.fingerprint);
|
|
1052
|
+
}
|
|
1053
|
+
const timezoneId = String(source.timezone_id || "").trim();
|
|
1054
|
+
if (timezoneId) {
|
|
1055
|
+
profile.timezone_id = timezoneId;
|
|
1056
|
+
}
|
|
1057
|
+
const locale = String(source.locale || "").trim();
|
|
1058
|
+
if (locale) {
|
|
1059
|
+
profile.locale = locale;
|
|
1060
|
+
}
|
|
1061
|
+
const profileKey = String(source.profile_key || "").trim();
|
|
1062
|
+
if (profileKey) {
|
|
1063
|
+
profile.profile_key = profileKey;
|
|
1064
|
+
}
|
|
1065
|
+
const browserMajorVersion = Number(source.browser_major_version || 0);
|
|
1066
|
+
if (Number.isFinite(browserMajorVersion) && browserMajorVersion > 0) {
|
|
1067
|
+
profile.browser_major_version = browserMajorVersion;
|
|
1068
|
+
}
|
|
1069
|
+
const schemaVersion = Number(source.schema_version || 0);
|
|
1070
|
+
if (Number.isFinite(schemaVersion) && schemaVersion > 0) {
|
|
1071
|
+
profile.schema_version = schemaVersion;
|
|
1072
|
+
} else if (Object.keys(profile).length > 0) {
|
|
1073
|
+
profile.schema_version = BROWSER_PROFILE_SCHEMA_VERSION;
|
|
1074
|
+
}
|
|
1075
|
+
return profile;
|
|
1076
|
+
};
|
|
1077
|
+
var buildBrowserProfilePayload = (core = {}, observed = {}) => {
|
|
1078
|
+
const payload = {};
|
|
1079
|
+
if (isPlainObject(core) && Object.keys(core).length > 0) {
|
|
1080
|
+
payload.core = core;
|
|
1081
|
+
}
|
|
1082
|
+
if (isPlainObject(observed) && Object.keys(observed).length > 0) {
|
|
1083
|
+
payload.observed = observed;
|
|
1084
|
+
}
|
|
1085
|
+
return payload;
|
|
1086
|
+
};
|
|
1087
|
+
var mergePlainObject = (target = {}, source = {}) => {
|
|
1088
|
+
const next = isPlainObject(target) ? deepClone(target) : {};
|
|
1089
|
+
if (!isPlainObject(source)) return next;
|
|
1090
|
+
Object.entries(source).forEach(([key, value]) => {
|
|
1091
|
+
if (!key) return;
|
|
1092
|
+
if (isPlainObject(value) && isPlainObject(next[key])) {
|
|
1093
|
+
next[key] = mergePlainObject(next[key], value);
|
|
1094
|
+
return;
|
|
1095
|
+
}
|
|
1096
|
+
next[key] = deepClone(value);
|
|
1097
|
+
});
|
|
1098
|
+
return next;
|
|
1099
|
+
};
|
|
1100
|
+
var mergeBrowserProfilePayload = (target = {}, source = {}) => {
|
|
1101
|
+
const current = isPlainObject(target) ? target : {};
|
|
1102
|
+
const incoming = isPlainObject(source) ? source : {};
|
|
1103
|
+
const currentCore = normalizeBrowserProfileCore(current.core);
|
|
1104
|
+
const incomingCore = normalizeBrowserProfileCore(incoming.core);
|
|
1105
|
+
const currentObserved = normalizeObservedBrowserProfile(current.observed);
|
|
1106
|
+
const incomingObserved = normalizeObservedBrowserProfile(incoming.observed);
|
|
1107
|
+
let mergedCore = currentCore;
|
|
1108
|
+
if (Object.keys(mergedCore).length === 0 && Object.keys(incomingCore).length > 0) {
|
|
1109
|
+
mergedCore = incomingCore;
|
|
1110
|
+
} else if (Object.keys(incomingCore).length > 0) {
|
|
1111
|
+
const currentVersion = Number(currentCore.browser_major_version || 0);
|
|
1112
|
+
const incomingVersion = Number(incomingCore.browser_major_version || 0);
|
|
1113
|
+
if (currentVersion > 0 && incomingVersion > 0 && currentVersion !== incomingVersion) {
|
|
1114
|
+
mergedCore = incomingCore;
|
|
1115
|
+
}
|
|
1116
|
+
}
|
|
1117
|
+
const mergedObserved = mergePlainObject(currentObserved, incomingObserved);
|
|
1118
|
+
return buildBrowserProfilePayload(mergedCore, mergedObserved);
|
|
1119
|
+
};
|
|
1120
|
+
var mergeEnvPatchObjects = (...patches) => {
|
|
1121
|
+
const merged = {};
|
|
1122
|
+
patches.forEach((patch) => {
|
|
1123
|
+
if (!isPlainObject(patch)) return;
|
|
1124
|
+
Object.entries(patch).forEach(([key, value]) => {
|
|
1125
|
+
if (!key) return;
|
|
1126
|
+
if (key === "browser_profile") {
|
|
1127
|
+
const browserProfile = mergeBrowserProfilePayload(merged.browser_profile, value);
|
|
1128
|
+
if (Object.keys(browserProfile).length > 0) {
|
|
1129
|
+
merged.browser_profile = browserProfile;
|
|
1130
|
+
}
|
|
1131
|
+
return;
|
|
1132
|
+
}
|
|
1133
|
+
if (isPlainObject(value) && isPlainObject(merged[key])) {
|
|
1134
|
+
merged[key] = mergePlainObject(merged[key], value);
|
|
1135
|
+
return;
|
|
1136
|
+
}
|
|
1137
|
+
merged[key] = deepClone(value);
|
|
1138
|
+
});
|
|
1139
|
+
});
|
|
1140
|
+
return Object.keys(merged).length > 0 ? merged : null;
|
|
1141
|
+
};
|
|
1142
|
+
var normalizeBrowserProfile = (value) => {
|
|
1143
|
+
const source = isPlainObject(value) ? value : {};
|
|
1144
|
+
const coreSource = isPlainObject(source.core) ? source.core : {};
|
|
1145
|
+
const observedSource = isPlainObject(source.observed) ? source.observed : {};
|
|
1146
|
+
const core = normalizeBrowserProfileCore(coreSource);
|
|
1147
|
+
const observed = normalizeObservedBrowserProfile(observedSource);
|
|
1148
|
+
return {
|
|
1149
|
+
core,
|
|
1150
|
+
observed,
|
|
1151
|
+
payload: buildBrowserProfilePayload(core, observed)
|
|
1152
|
+
};
|
|
1153
|
+
};
|
|
1154
|
+
var rememberRuntimeState = (state) => {
|
|
1155
|
+
rememberedRuntimeState = deepClone(state);
|
|
1156
|
+
return rememberedRuntimeState;
|
|
1157
|
+
};
|
|
1158
|
+
var normalizeRuntimeState = (source = {}, actor = "") => {
|
|
1159
|
+
if (source && typeof source === "object" && source.runtime && typeof source.runtime === "object" && !Array.isArray(source.runtime) && Object.prototype.hasOwnProperty.call(source, "browserProfileCore")) {
|
|
1160
|
+
return source;
|
|
1161
|
+
}
|
|
1162
|
+
return RuntimeEnv.parseInput(source, actor);
|
|
1163
|
+
};
|
|
1164
|
+
var RuntimeEnv = {
|
|
1165
|
+
tryParseJSON,
|
|
1166
|
+
normalizeCookies,
|
|
1167
|
+
normalizeLocalStorage,
|
|
1168
|
+
normalizeSessionStorage,
|
|
1169
|
+
normalizeAuth,
|
|
1170
|
+
normalizeBrowserProfileCore,
|
|
1171
|
+
normalizeObservedBrowserProfile,
|
|
1172
|
+
mergeEnvPatches(...patches) {
|
|
1173
|
+
return mergeEnvPatchObjects(...patches);
|
|
1174
|
+
},
|
|
1175
|
+
// parseInput 把 Actor 输入里的 runtime 字段标准化成 toolkit 内部统一结构。
|
|
1176
|
+
// 后续 visitor 只和这个 state 交互,不再自己拼 token/cookie 逻辑。
|
|
1177
|
+
parseInput(input = {}, actor = "") {
|
|
1178
|
+
const runtime2 = tryParseJSON(input?.runtime) || {};
|
|
1179
|
+
const resolvedActor = String(actor || input?.actor || "").trim();
|
|
1180
|
+
const cookies = normalizeCookies(runtime2?.cookies);
|
|
1181
|
+
const cookieMap = buildCookieMap(cookies);
|
|
1182
|
+
const localStorage = normalizeLocalStorage(runtime2?.local_storage);
|
|
1183
|
+
const sessionStorage = normalizeSessionStorage(runtime2?.session_storage);
|
|
1184
|
+
const auth = normalizeAuth(runtime2?.auth);
|
|
1185
|
+
const browserProfile = normalizeBrowserProfile(runtime2?.browser_profile);
|
|
1186
|
+
const envId = String(input?.env_id || "").trim();
|
|
1187
|
+
const normalizedRuntime = {
|
|
1188
|
+
...runtime2,
|
|
1189
|
+
...cookies.length > 0 ? { cookies } : {},
|
|
1190
|
+
...Object.keys(localStorage).length > 0 ? { local_storage: localStorage } : {},
|
|
1191
|
+
...Object.keys(sessionStorage).length > 0 ? { session_storage: sessionStorage } : {},
|
|
1192
|
+
...Object.keys(auth).length > 0 ? { auth } : {}
|
|
1193
|
+
};
|
|
1194
|
+
delete normalizedRuntime.actor;
|
|
1195
|
+
delete normalizedRuntime.env_id;
|
|
1196
|
+
if (Object.keys(browserProfile.payload).length > 0) {
|
|
1197
|
+
normalizedRuntime.browser_profile = browserProfile.payload;
|
|
1198
|
+
} else {
|
|
1199
|
+
delete normalizedRuntime.browser_profile;
|
|
1200
|
+
}
|
|
1201
|
+
const state = {
|
|
1202
|
+
actor: resolvedActor,
|
|
1203
|
+
runtime: normalizedRuntime,
|
|
1204
|
+
envId,
|
|
1205
|
+
auth,
|
|
1206
|
+
cookies,
|
|
1207
|
+
cookieMap,
|
|
1208
|
+
localStorage,
|
|
1209
|
+
sessionStorage,
|
|
1210
|
+
browserProfile: browserProfile.payload,
|
|
1211
|
+
browserProfileCore: browserProfile.core,
|
|
1212
|
+
browserProfileObserved: browserProfile.observed
|
|
1213
|
+
};
|
|
1214
|
+
rememberRuntimeState(state);
|
|
1215
|
+
return state;
|
|
1216
|
+
},
|
|
1217
|
+
// buildEnvPatch 只构造允许回写到后端 env 的字段集合。
|
|
1218
|
+
buildEnvPatch(source = {}, actor = "") {
|
|
1219
|
+
const state = normalizeRuntimeState(source, actor);
|
|
1220
|
+
const browserProfile = buildBrowserProfilePayload(state.browserProfileCore, state.browserProfileObserved);
|
|
1221
|
+
const envPatch = {
|
|
1222
|
+
...Object.keys(state.auth || {}).length > 0 ? { auth: state.auth } : {},
|
|
1223
|
+
...Array.isArray(state.cookies) && state.cookies.length > 0 ? { cookies: state.cookies } : {},
|
|
1224
|
+
...Object.keys(state.localStorage || {}).length > 0 ? { local_storage: state.localStorage } : {},
|
|
1225
|
+
...Object.keys(state.sessionStorage || {}).length > 0 ? { session_storage: state.sessionStorage } : {},
|
|
1226
|
+
...Object.keys(browserProfile).length > 0 ? { browser_profile: browserProfile } : {}
|
|
1227
|
+
};
|
|
1228
|
+
return Object.keys(envPatch).length > 0 ? envPatch : null;
|
|
1229
|
+
},
|
|
1230
|
+
// hasLoginState 用来区分“必须鉴权的平台”和“可匿名运行的平台”。
|
|
1231
|
+
hasLoginState(source = {}, actor = "") {
|
|
1232
|
+
const state = normalizeRuntimeState(source, actor);
|
|
1233
|
+
return Array.isArray(state.cookies) && state.cookies.length > 0 || Object.keys(state.localStorage || {}).length > 0 || Object.keys(state.sessionStorage || {}).length > 0 || Object.keys(state.auth || {}).length > 0;
|
|
1234
|
+
},
|
|
1235
|
+
getAuthValue(source = {}, key = "", actor = "") {
|
|
1236
|
+
const state = normalizeRuntimeState(source, actor);
|
|
1237
|
+
const safeKey = String(key || "").trim();
|
|
1238
|
+
if (!safeKey) return "";
|
|
1239
|
+
return String(state.auth?.[safeKey] ?? "").trim();
|
|
1240
|
+
},
|
|
1241
|
+
rememberState(source = {}) {
|
|
1242
|
+
const state = normalizeRuntimeState(source);
|
|
1243
|
+
rememberRuntimeState(state);
|
|
1244
|
+
return RuntimeEnv.peekRememberedState();
|
|
1245
|
+
},
|
|
1246
|
+
peekRememberedState() {
|
|
1247
|
+
return rememberedRuntimeState ? deepClone(rememberedRuntimeState) : null;
|
|
1248
|
+
},
|
|
1249
|
+
getBrowserProfileCore(source = {}, actor = "") {
|
|
1250
|
+
const state = normalizeRuntimeState(source, actor);
|
|
1251
|
+
return deepClone(state.browserProfileCore || {});
|
|
1252
|
+
},
|
|
1253
|
+
setBrowserProfileCore(source = {}, core = {}, actor = "") {
|
|
1254
|
+
const state = normalizeRuntimeState(source, actor);
|
|
1255
|
+
const normalizedCore = normalizeBrowserProfileCore(core);
|
|
1256
|
+
state.browserProfileCore = normalizedCore;
|
|
1257
|
+
state.browserProfile = buildBrowserProfilePayload(normalizedCore, state.browserProfileObserved);
|
|
1258
|
+
if (Object.keys(state.browserProfile).length > 0) {
|
|
1259
|
+
state.runtime.browser_profile = state.browserProfile;
|
|
1260
|
+
} else {
|
|
1261
|
+
delete state.runtime.browser_profile;
|
|
1262
|
+
}
|
|
1263
|
+
rememberRuntimeState(state);
|
|
1264
|
+
return state;
|
|
1265
|
+
},
|
|
1266
|
+
// applyToPage 只负责把登录态相关字段注入页面:
|
|
1267
|
+
// cookies / localStorage / sessionStorage。
|
|
1268
|
+
// 指纹、时区、UA、viewport 的回放发生在 launch.js 启动阶段,不在这里做。
|
|
1269
|
+
async applyToPage(page, source = {}, options = {}) {
|
|
1270
|
+
if (!page) return;
|
|
1271
|
+
const state = normalizeRuntimeState(source, options?.actor || "");
|
|
1272
|
+
const localStorage = state.localStorage || {};
|
|
1273
|
+
const sessionStorage = state.sessionStorage || {};
|
|
1274
|
+
const cookies = (state.cookies || []).map((cookie) => {
|
|
1275
|
+
const normalized = { ...cookie };
|
|
1276
|
+
if (!normalized.path) {
|
|
1277
|
+
normalized.path = "/";
|
|
1278
|
+
}
|
|
1279
|
+
if (!normalized.domain) {
|
|
1280
|
+
return null;
|
|
1281
|
+
}
|
|
1282
|
+
return normalized;
|
|
1283
|
+
}).filter(Boolean);
|
|
1284
|
+
if (cookies.length > 0) {
|
|
1285
|
+
await page.context().addCookies(cookies);
|
|
1286
|
+
}
|
|
1287
|
+
if (Object.keys(localStorage).length > 0) {
|
|
1288
|
+
await page.addInitScript((payload) => {
|
|
1289
|
+
try {
|
|
1290
|
+
Object.entries(payload || {}).forEach(([key, value]) => {
|
|
1291
|
+
if (!key) return;
|
|
1292
|
+
window.localStorage.setItem(key, String(value ?? ""));
|
|
1293
|
+
});
|
|
1294
|
+
} catch (error) {
|
|
1295
|
+
}
|
|
1296
|
+
}, localStorage);
|
|
1297
|
+
}
|
|
1298
|
+
if (Object.keys(sessionStorage).length > 0) {
|
|
1299
|
+
await page.addInitScript((payload) => {
|
|
1300
|
+
try {
|
|
1301
|
+
Object.entries(payload || {}).forEach(([key, value]) => {
|
|
1302
|
+
if (!key) return;
|
|
1303
|
+
window.sessionStorage.setItem(key, String(value ?? ""));
|
|
1304
|
+
});
|
|
1305
|
+
} catch (error) {
|
|
1306
|
+
}
|
|
1307
|
+
}, sessionStorage);
|
|
1308
|
+
}
|
|
1309
|
+
},
|
|
1310
|
+
// captureEnvPatch 在任务结束时采集最新环境快照,用于 pushSuccess / pushFailed 自动回写。
|
|
1311
|
+
async captureEnvPatch(page, source = {}, options = {}) {
|
|
1312
|
+
const state = normalizeRuntimeState(source, options?.actor || "");
|
|
1313
|
+
const baseline = RuntimeEnv.buildEnvPatch(state) || {};
|
|
1314
|
+
const patch = {
|
|
1315
|
+
...baseline
|
|
1316
|
+
};
|
|
1317
|
+
if (!page || typeof page.evaluate !== "function" || typeof page.context !== "function") {
|
|
1318
|
+
return Object.keys(patch).length > 0 ? patch : null;
|
|
1319
|
+
}
|
|
1320
|
+
try {
|
|
1321
|
+
const contextCookies = await page.context().cookies();
|
|
1322
|
+
const normalizedCookies = normalizeCookies(contextCookies || []);
|
|
1323
|
+
if (normalizedCookies.length > 0) {
|
|
1324
|
+
patch.cookies = normalizedCookies;
|
|
1325
|
+
}
|
|
1326
|
+
} catch (error) {
|
|
1327
|
+
}
|
|
1328
|
+
try {
|
|
1329
|
+
const snapshot = await page.evaluate(() => {
|
|
1330
|
+
const localStorage2 = {};
|
|
1331
|
+
const sessionStorage2 = {};
|
|
1332
|
+
try {
|
|
1333
|
+
for (let i = 0; i < window.localStorage.length; i += 1) {
|
|
1334
|
+
const key = window.localStorage.key(i);
|
|
1335
|
+
if (!key) continue;
|
|
1336
|
+
const value = window.localStorage.getItem(key);
|
|
1337
|
+
if (value !== null) localStorage2[key] = value;
|
|
1338
|
+
}
|
|
1339
|
+
} catch (error) {
|
|
1340
|
+
}
|
|
1341
|
+
try {
|
|
1342
|
+
for (let i = 0; i < window.sessionStorage.length; i += 1) {
|
|
1343
|
+
const key = window.sessionStorage.key(i);
|
|
1344
|
+
if (!key) continue;
|
|
1345
|
+
const value = window.sessionStorage.getItem(key);
|
|
1346
|
+
if (value !== null) sessionStorage2[key] = value;
|
|
1347
|
+
}
|
|
1348
|
+
} catch (error) {
|
|
1349
|
+
}
|
|
1350
|
+
const nav = window.navigator || {};
|
|
1351
|
+
const screen = window.screen || {};
|
|
1352
|
+
const timezone = Intl.DateTimeFormat().resolvedOptions().timeZone || "";
|
|
1353
|
+
return {
|
|
1354
|
+
localStorage: localStorage2,
|
|
1355
|
+
sessionStorage: sessionStorage2,
|
|
1356
|
+
browserProfileObserved: {
|
|
1357
|
+
user_agent: nav.userAgent || "",
|
|
1358
|
+
platform: nav.platform || "",
|
|
1359
|
+
language: nav.language || "",
|
|
1360
|
+
languages: Array.isArray(nav.languages) ? nav.languages : [],
|
|
1361
|
+
hardware_concurrency: Number(nav.hardwareConcurrency || 0),
|
|
1362
|
+
device_memory: Number(nav.deviceMemory || 0),
|
|
1363
|
+
timezone,
|
|
1364
|
+
viewport: {
|
|
1365
|
+
width: Number(window.innerWidth || 0),
|
|
1366
|
+
height: Number(window.innerHeight || 0)
|
|
1367
|
+
},
|
|
1368
|
+
screen: {
|
|
1369
|
+
width: Number(screen.width || 0),
|
|
1370
|
+
height: Number(screen.height || 0),
|
|
1371
|
+
avail_width: Number(screen.availWidth || 0),
|
|
1372
|
+
avail_height: Number(screen.availHeight || 0),
|
|
1373
|
+
color_depth: Number(screen.colorDepth || 0),
|
|
1374
|
+
pixel_depth: Number(screen.pixelDepth || 0)
|
|
1375
|
+
}
|
|
1376
|
+
}
|
|
1377
|
+
};
|
|
1378
|
+
});
|
|
1379
|
+
const localStorage = normalizeLocalStorage(snapshot?.localStorage);
|
|
1380
|
+
if (Object.keys(localStorage).length > 0) {
|
|
1381
|
+
patch.local_storage = localStorage;
|
|
1382
|
+
}
|
|
1383
|
+
const sessionStorage = normalizeSessionStorage(snapshot?.sessionStorage);
|
|
1384
|
+
if (Object.keys(sessionStorage).length > 0) {
|
|
1385
|
+
patch.session_storage = sessionStorage;
|
|
1386
|
+
}
|
|
1387
|
+
const observedProfile = normalizeObservedBrowserProfile(snapshot?.browserProfileObserved);
|
|
1388
|
+
const browserProfile = buildBrowserProfilePayload(state.browserProfileCore, observedProfile);
|
|
1389
|
+
if (Object.keys(browserProfile).length > 0) {
|
|
1390
|
+
patch.browser_profile = browserProfile;
|
|
1391
|
+
}
|
|
1392
|
+
} catch (error) {
|
|
1393
|
+
}
|
|
1394
|
+
return Object.keys(patch).length > 0 ? patch : null;
|
|
1395
|
+
}
|
|
1396
|
+
};
|
|
901
1397
|
|
|
902
1398
|
// src/apify-kit.js
|
|
903
1399
|
var logger3 = createInternalLogger("ApifyKit");
|
|
1400
|
+
var resolveRuntimeContext = (input) => {
|
|
1401
|
+
const rememberedState = RuntimeEnv.peekRememberedState();
|
|
1402
|
+
const state = rememberedState || RuntimeEnv.parseInput(input || {});
|
|
1403
|
+
const envPatch = RuntimeEnv.buildEnvPatch(state) || null;
|
|
1404
|
+
return {
|
|
1405
|
+
actor: state.actor,
|
|
1406
|
+
runtime: state.runtime,
|
|
1407
|
+
envPatch
|
|
1408
|
+
};
|
|
1409
|
+
};
|
|
1410
|
+
var isPageLike = (value) => {
|
|
1411
|
+
if (!value || typeof value !== "object") return false;
|
|
1412
|
+
return typeof value.evaluate === "function" && typeof value.context === "function";
|
|
1413
|
+
};
|
|
1414
|
+
var pickPage = (...candidates) => {
|
|
1415
|
+
for (const candidate of candidates) {
|
|
1416
|
+
if (isPageLike(candidate)) {
|
|
1417
|
+
return candidate;
|
|
1418
|
+
}
|
|
1419
|
+
}
|
|
1420
|
+
return null;
|
|
1421
|
+
};
|
|
1422
|
+
var toSerializable = (value, depth = 0, seen = /* @__PURE__ */ new WeakSet()) => {
|
|
1423
|
+
if (depth > 6) return "[MaxDepth]";
|
|
1424
|
+
if (value == null) return value;
|
|
1425
|
+
const valueType = typeof value;
|
|
1426
|
+
if (valueType === "string" || valueType === "number" || valueType === "boolean") {
|
|
1427
|
+
return value;
|
|
1428
|
+
}
|
|
1429
|
+
if (valueType === "bigint") {
|
|
1430
|
+
return value.toString();
|
|
1431
|
+
}
|
|
1432
|
+
if (valueType === "function" || valueType === "symbol") {
|
|
1433
|
+
return void 0;
|
|
1434
|
+
}
|
|
1435
|
+
if (value instanceof Error) {
|
|
1436
|
+
return serializeError2(value);
|
|
1437
|
+
}
|
|
1438
|
+
if (Array.isArray(value)) {
|
|
1439
|
+
return value.map((item) => toSerializable(item, depth + 1, seen)).filter((item) => item !== void 0);
|
|
1440
|
+
}
|
|
1441
|
+
if (valueType !== "object") {
|
|
1442
|
+
return String(value);
|
|
1443
|
+
}
|
|
1444
|
+
if (seen.has(value)) {
|
|
1445
|
+
return "[Circular]";
|
|
1446
|
+
}
|
|
1447
|
+
seen.add(value);
|
|
1448
|
+
const ctorName = String(value?.constructor?.name || "");
|
|
1449
|
+
if (ctorName === "Page" || ctorName === "BrowserContext" || ctorName === "Browser" || ctorName === "Frame" || ctorName === "ElementHandle" || ctorName === "JSHandle") {
|
|
1450
|
+
return `[${ctorName}]`;
|
|
1451
|
+
}
|
|
1452
|
+
const output = {};
|
|
1453
|
+
Object.entries(value).forEach(([key, item]) => {
|
|
1454
|
+
const safeValue = toSerializable(item, depth + 1, seen);
|
|
1455
|
+
if (safeValue !== void 0) {
|
|
1456
|
+
output[key] = safeValue;
|
|
1457
|
+
}
|
|
1458
|
+
});
|
|
1459
|
+
return output;
|
|
1460
|
+
};
|
|
1461
|
+
var sanitizeMeta = (meta) => {
|
|
1462
|
+
if (!meta || typeof meta !== "object" || Array.isArray(meta)) {
|
|
1463
|
+
return {};
|
|
1464
|
+
}
|
|
1465
|
+
const result = toSerializable(meta) || {};
|
|
1466
|
+
if (result && typeof result === "object" && !Array.isArray(result)) {
|
|
1467
|
+
delete result.page;
|
|
1468
|
+
}
|
|
1469
|
+
return result;
|
|
1470
|
+
};
|
|
1471
|
+
var captureAutoEnvPatch = async (page, runtimeContext) => {
|
|
1472
|
+
const currentPage = pickPage(page);
|
|
1473
|
+
if (!currentPage) return null;
|
|
1474
|
+
try {
|
|
1475
|
+
return await RuntimeEnv.captureEnvPatch(
|
|
1476
|
+
currentPage,
|
|
1477
|
+
{
|
|
1478
|
+
actor: runtimeContext.actor,
|
|
1479
|
+
runtime: runtimeContext.runtime
|
|
1480
|
+
},
|
|
1481
|
+
{ actor: runtimeContext.actor }
|
|
1482
|
+
);
|
|
1483
|
+
} catch (error) {
|
|
1484
|
+
logger3.warn(`\u81EA\u52A8\u91C7\u96C6 env_patch \u5931\u8D25: ${error?.message || error}`);
|
|
1485
|
+
return null;
|
|
1486
|
+
}
|
|
1487
|
+
};
|
|
904
1488
|
async function createApifyKit() {
|
|
905
1489
|
let apify = null;
|
|
906
1490
|
try {
|
|
@@ -909,6 +1493,9 @@ async function createApifyKit() {
|
|
|
909
1493
|
throw new Error("\u26A0\uFE0F apify \u5E93\u672A\u5B89\u88C5\uFF0CApifyKit \u7684 Actor \u76F8\u5173\u529F\u80FD\u4E0D\u53EF\u7528");
|
|
910
1494
|
}
|
|
911
1495
|
const { Actor: Actor2 } = apify;
|
|
1496
|
+
const actorInput = await Actor2.getInput() || {};
|
|
1497
|
+
let lastPage = null;
|
|
1498
|
+
const getRuntimeContext = () => resolveRuntimeContext(actorInput);
|
|
912
1499
|
return {
|
|
913
1500
|
/**
|
|
914
1501
|
* 核心封装:执行步骤,带自动日志确认、失败截图处理和重试机制
|
|
@@ -924,6 +1511,9 @@ async function createApifyKit() {
|
|
|
924
1511
|
* @param {Function} [options.retry.before] - 重试前钩子,可覆盖默认等待行为
|
|
925
1512
|
*/
|
|
926
1513
|
async runStep(step, page, actionFn, options = {}) {
|
|
1514
|
+
if (isPageLike(page)) {
|
|
1515
|
+
lastPage = page;
|
|
1516
|
+
}
|
|
927
1517
|
const { failActor = true, retry = {} } = options;
|
|
928
1518
|
const { times: retryTimes = 0, mode: retryMode = "direct", before: beforeRetry } = retry;
|
|
929
1519
|
const executeAction = async (attemptNumber) => {
|
|
@@ -985,13 +1575,18 @@ async function createApifyKit() {
|
|
|
985
1575
|
} catch (snapErr) {
|
|
986
1576
|
logger3.warn(`\u622A\u56FE\u751F\u6210\u5931\u8D25: ${snapErr.message}`);
|
|
987
1577
|
}
|
|
988
|
-
await this.pushFailed(
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
1578
|
+
await this.pushFailed(
|
|
1579
|
+
finalError,
|
|
1580
|
+
{
|
|
1581
|
+
step,
|
|
1582
|
+
base64,
|
|
1583
|
+
retryAttempts: retryTimes
|
|
1584
|
+
},
|
|
1585
|
+
{
|
|
1586
|
+
page,
|
|
1587
|
+
options
|
|
1588
|
+
}
|
|
1589
|
+
);
|
|
995
1590
|
await Actor2.fail(`Run Step ${step} \u5931\u8D25 (\u5DF2\u91CD\u8BD5 ${retryTimes} \u6B21): ${finalError.message}`);
|
|
996
1591
|
} else {
|
|
997
1592
|
throw finalError;
|
|
@@ -1013,42 +1608,83 @@ async function createApifyKit() {
|
|
|
1013
1608
|
});
|
|
1014
1609
|
},
|
|
1015
1610
|
/**
|
|
1016
|
-
*
|
|
1611
|
+
* 推送成功数据的通用方法。
|
|
1612
|
+
* 这里会自动做三件事:
|
|
1613
|
+
* 1. 采集当前页面的 env_patch;
|
|
1614
|
+
* 2. 合并显式传入的 env_patch;
|
|
1615
|
+
* 3. 一并写入 dataset,供后端回写 runtime_env_json。
|
|
1017
1616
|
* @param {Object} data - 要推送的数据对象
|
|
1018
1617
|
*/
|
|
1019
|
-
async pushSuccess(data) {
|
|
1020
|
-
const
|
|
1021
|
-
|
|
1022
|
-
|
|
1618
|
+
async pushSuccess(data, options = {}) {
|
|
1619
|
+
const runtimeContext = getRuntimeContext();
|
|
1620
|
+
const page = pickPage(options?.page, data?.page, lastPage);
|
|
1621
|
+
if (page) {
|
|
1622
|
+
lastPage = page;
|
|
1623
|
+
}
|
|
1624
|
+
let payloadData = data;
|
|
1625
|
+
let explicitPatch = null;
|
|
1626
|
+
if (data && typeof data === "object" && !Array.isArray(data)) {
|
|
1627
|
+
payloadData = { ...data };
|
|
1628
|
+
explicitPatch = payloadData.env_patch;
|
|
1629
|
+
delete payloadData.env_patch;
|
|
1630
|
+
delete payloadData.page;
|
|
1631
|
+
}
|
|
1632
|
+
const capturedPatch = await captureAutoEnvPatch(page, runtimeContext);
|
|
1633
|
+
const envPatch = RuntimeEnv.mergeEnvPatches(runtimeContext.envPatch, capturedPatch, explicitPatch);
|
|
1634
|
+
const traffic = await ProxyMeterRuntime.getProxyMeterSnapshot({ finalize: true });
|
|
1635
|
+
const payload = {
|
|
1023
1636
|
code: Code.Success,
|
|
1024
1637
|
status: Status.Success,
|
|
1025
1638
|
...traffic ? { traffic } : {},
|
|
1026
1639
|
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1027
|
-
data
|
|
1640
|
+
data: payloadData
|
|
1641
|
+
};
|
|
1642
|
+
if (envPatch) {
|
|
1643
|
+
payload.env_patch = envPatch;
|
|
1644
|
+
}
|
|
1645
|
+
await Actor2.pushData({
|
|
1646
|
+
...payload
|
|
1028
1647
|
});
|
|
1029
1648
|
logger3.success("pushSuccess", "Data pushed");
|
|
1030
1649
|
},
|
|
1031
1650
|
/**
|
|
1032
|
-
* 推送失败数据的通用方法(私有方法,仅供runStep
|
|
1033
|
-
*
|
|
1651
|
+
* 推送失败数据的通用方法(私有方法,仅供 runStep 内部使用)。
|
|
1652
|
+
* 即便任务失败,这里也会自动回收 env_patch,确保 cookies/storage 的最新状态不会丢。
|
|
1034
1653
|
* @param {Error|CrawlerError} error - 错误对象
|
|
1035
|
-
* @param {Object} [meta] - 额外的数据(如failedStep
|
|
1654
|
+
* @param {Object} [meta] - 额外的数据(如 failedStep、screenshotBase64 等)
|
|
1655
|
+
* @param {Object} [options] - 附加选项(如 page)
|
|
1036
1656
|
* @private
|
|
1037
1657
|
*/
|
|
1038
|
-
async pushFailed(error, meta = {}) {
|
|
1658
|
+
async pushFailed(error, meta = {}, options = {}) {
|
|
1659
|
+
const runtimeContext = getRuntimeContext();
|
|
1660
|
+
const page = pickPage(options?.page, meta?.page, lastPage);
|
|
1661
|
+
if (page) {
|
|
1662
|
+
lastPage = page;
|
|
1663
|
+
}
|
|
1664
|
+
const capturedPatch = await captureAutoEnvPatch(page, runtimeContext);
|
|
1665
|
+
const explicitPatch = meta?.env_patch;
|
|
1666
|
+
const envPatch = RuntimeEnv.mergeEnvPatches(runtimeContext.envPatch, capturedPatch, explicitPatch);
|
|
1667
|
+
const safeMeta = sanitizeMeta(meta);
|
|
1668
|
+
delete safeMeta.env_patch;
|
|
1039
1669
|
const isCrawlerError = CrawlerError.isCrawlerError(error);
|
|
1040
1670
|
const code = isCrawlerError ? error.code : Code.UnknownError;
|
|
1041
1671
|
const context = isCrawlerError ? error.context : {};
|
|
1042
|
-
const traffic = await getProxyMeterSnapshot({ finalize: true });
|
|
1043
|
-
|
|
1672
|
+
const traffic = await ProxyMeterRuntime.getProxyMeterSnapshot({ finalize: true });
|
|
1673
|
+
const payload = {
|
|
1044
1674
|
// 如果是 CrawlerError,使用其 code,否则使用默认 Failed code
|
|
1045
1675
|
code,
|
|
1046
1676
|
status: Status.Failed,
|
|
1047
1677
|
...traffic ? { traffic } : {},
|
|
1048
1678
|
error: serializeError2(error),
|
|
1049
|
-
meta,
|
|
1679
|
+
meta: safeMeta,
|
|
1050
1680
|
context,
|
|
1051
1681
|
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
1682
|
+
};
|
|
1683
|
+
if (envPatch) {
|
|
1684
|
+
payload.env_patch = envPatch;
|
|
1685
|
+
}
|
|
1686
|
+
await Actor2.pushData({
|
|
1687
|
+
...payload
|
|
1052
1688
|
});
|
|
1053
1689
|
logger3.success("pushFailed", "Error data pushed");
|
|
1054
1690
|
}
|
|
@@ -1139,12 +1775,13 @@ var DEFAULT_LAUNCH_ARGS = [
|
|
|
1139
1775
|
// '--disable-blink-features=AutomationControlled', // Crawlee 可能会自动处理,过多干预反而会被识别
|
|
1140
1776
|
"--no-sandbox",
|
|
1141
1777
|
"--disable-setuid-sandbox",
|
|
1142
|
-
"--window-position=0,0"
|
|
1143
|
-
`--lang=${BASE_CONFIG.locale}`
|
|
1778
|
+
"--window-position=0,0"
|
|
1144
1779
|
];
|
|
1145
|
-
function buildFingerprintOptions(locale) {
|
|
1780
|
+
function buildFingerprintOptions({ locale = BASE_CONFIG.locale, browserMajorVersion = 0 } = {}) {
|
|
1781
|
+
const normalizedBrowserMajorVersion = Number(browserMajorVersion || 0);
|
|
1782
|
+
const browserDescriptor = normalizedBrowserMajorVersion > 0 ? [{ name: "chrome", minVersion: normalizedBrowserMajorVersion, maxVersion: normalizedBrowserMajorVersion }] : [{ name: "chrome", minVersion: 110 }];
|
|
1146
1783
|
return {
|
|
1147
|
-
browsers:
|
|
1784
|
+
browsers: browserDescriptor,
|
|
1148
1785
|
devices: ["desktop"],
|
|
1149
1786
|
operatingSystems: ["windows", "macos", "linux"],
|
|
1150
1787
|
locales: [locale]
|
|
@@ -1160,20 +1797,21 @@ var AntiCheat = {
|
|
|
1160
1797
|
/**
|
|
1161
1798
|
* 用于 Crawlee fingerprint generator 的统一配置(桌面端)。
|
|
1162
1799
|
*/
|
|
1163
|
-
getFingerprintGeneratorOptions() {
|
|
1164
|
-
return buildFingerprintOptions(
|
|
1800
|
+
getFingerprintGeneratorOptions(options = {}) {
|
|
1801
|
+
return buildFingerprintOptions(options);
|
|
1165
1802
|
},
|
|
1166
1803
|
/**
|
|
1167
1804
|
* 获取基础启动参数。
|
|
1168
1805
|
*/
|
|
1169
|
-
getLaunchArgs() {
|
|
1170
|
-
|
|
1806
|
+
getLaunchArgs(options = {}) {
|
|
1807
|
+
const locale = String(options?.locale || BASE_CONFIG.locale).trim() || BASE_CONFIG.locale;
|
|
1808
|
+
return [...DEFAULT_LAUNCH_ARGS, `--lang=${locale}`];
|
|
1171
1809
|
},
|
|
1172
1810
|
/**
|
|
1173
1811
|
* 为 got-scraping 生成与浏览器一致的 TLS 指纹配置(桌面端)。
|
|
1174
1812
|
*/
|
|
1175
1813
|
getTlsFingerprintOptions(userAgent = "", acceptLanguage = "") {
|
|
1176
|
-
return buildFingerprintOptions(
|
|
1814
|
+
return buildFingerprintOptions();
|
|
1177
1815
|
},
|
|
1178
1816
|
/**
|
|
1179
1817
|
* 规范化请求头
|
|
@@ -1664,12 +2302,18 @@ var Humanize = {
|
|
|
1664
2302
|
}
|
|
1665
2303
|
};
|
|
1666
2304
|
|
|
2305
|
+
// src/launch.js
|
|
2306
|
+
import { execFileSync } from "node:child_process";
|
|
2307
|
+
import { FingerprintGenerator } from "fingerprint-generator";
|
|
2308
|
+
import { FingerprintInjector } from "fingerprint-injector";
|
|
2309
|
+
|
|
1667
2310
|
// src/proxy-bypass.js
|
|
1668
2311
|
import picomatch from "picomatch";
|
|
1669
2312
|
var normalizeByPassDomains = (domains) => {
|
|
1670
2313
|
if (!Array.isArray(domains)) return [];
|
|
1671
2314
|
return domains.map((item) => String(item || "").trim()).filter(Boolean);
|
|
1672
2315
|
};
|
|
2316
|
+
var normalizeHostname = (value = "") => String(value || "").trim().toLowerCase();
|
|
1673
2317
|
var buildByPassDomainRule = (rawPattern) => {
|
|
1674
2318
|
const pattern = String(rawPattern || "").trim().toLowerCase();
|
|
1675
2319
|
if (!pattern) return null;
|
|
@@ -1690,7 +2334,7 @@ var buildByPassDomainRules = (domains = []) => {
|
|
|
1690
2334
|
var findMatchedByPassRule = (rules = [], requestUrl = "") => {
|
|
1691
2335
|
let hostname = "";
|
|
1692
2336
|
try {
|
|
1693
|
-
hostname = new URL(String(requestUrl || "")).hostname
|
|
2337
|
+
hostname = normalizeHostname(new URL(String(requestUrl || "")).hostname);
|
|
1694
2338
|
} catch {
|
|
1695
2339
|
return null;
|
|
1696
2340
|
}
|
|
@@ -1707,16 +2351,56 @@ var findMatchedByPassRule = (rules = [], requestUrl = "") => {
|
|
|
1707
2351
|
hostname
|
|
1708
2352
|
};
|
|
1709
2353
|
};
|
|
2354
|
+
var isDomainCoveredByByPass = (domain = "", rules = []) => {
|
|
2355
|
+
const hostname = normalizeHostname(domain);
|
|
2356
|
+
if (!hostname) return false;
|
|
2357
|
+
for (const rule of rules || []) {
|
|
2358
|
+
if (!rule || typeof rule.test !== "function") continue;
|
|
2359
|
+
if (rule.test(hostname)) return true;
|
|
2360
|
+
}
|
|
2361
|
+
return false;
|
|
2362
|
+
};
|
|
2363
|
+
var resolveRouteByProxy = ({
|
|
2364
|
+
requestUrl = "",
|
|
2365
|
+
enableProxy = false,
|
|
2366
|
+
byPassRules = []
|
|
2367
|
+
}) => {
|
|
2368
|
+
if (!enableProxy) {
|
|
2369
|
+
return { route: "direct", matchedRule: null, hostname: "" };
|
|
2370
|
+
}
|
|
2371
|
+
const matched = findMatchedByPassRule(byPassRules, requestUrl);
|
|
2372
|
+
if (!matched) {
|
|
2373
|
+
return { route: "proxy", matchedRule: null, hostname: "" };
|
|
2374
|
+
}
|
|
2375
|
+
if (matched.rule) {
|
|
2376
|
+
return { route: "direct", matchedRule: matched.rule, hostname: matched.hostname };
|
|
2377
|
+
}
|
|
2378
|
+
return { route: "proxy", matchedRule: null, hostname: matched.hostname };
|
|
2379
|
+
};
|
|
2380
|
+
var ByPass = {
|
|
2381
|
+
normalizeByPassDomains,
|
|
2382
|
+
normalizeHostname,
|
|
2383
|
+
buildByPassDomainRule,
|
|
2384
|
+
buildByPassDomainRules,
|
|
2385
|
+
findMatchedByPassRule,
|
|
2386
|
+
isDomainCoveredByByPass,
|
|
2387
|
+
resolveRouteByProxy
|
|
2388
|
+
};
|
|
1710
2389
|
|
|
1711
2390
|
// src/launch.js
|
|
1712
2391
|
var logger7 = createInternalLogger("Launch");
|
|
1713
2392
|
var REQUEST_HOOK_FLAG = Symbol("playwright-toolkit-request-hook");
|
|
2393
|
+
var injectedContexts = /* @__PURE__ */ new WeakSet();
|
|
2394
|
+
var browserMajorVersionCache = /* @__PURE__ */ new Map();
|
|
2395
|
+
var DEFAULT_BROWSER_PROFILE_SCHEMA_VERSION = 1;
|
|
2396
|
+
var DEFAULT_LOCALE = "zh-CN";
|
|
1714
2397
|
var DEFAULT_CRAWLER_BASE_OPTIONS = Object.freeze({
|
|
1715
2398
|
maxConcurrency: 1,
|
|
1716
2399
|
maxRequestRetries: 0,
|
|
1717
2400
|
requestHandlerTimeoutSecs: 240,
|
|
1718
2401
|
navigationTimeoutSecs: 120
|
|
1719
2402
|
});
|
|
2403
|
+
var fingerprintInjector = new FingerprintInjector();
|
|
1720
2404
|
var resolveProxyLaunchOptions = (proxyConfiguration = {}) => {
|
|
1721
2405
|
const config = proxyConfiguration && typeof proxyConfiguration === "object" && !Array.isArray(proxyConfiguration) ? proxyConfiguration : {};
|
|
1722
2406
|
const proxyUrl = String(config.proxy_url || "").trim();
|
|
@@ -1724,9 +2408,149 @@ var resolveProxyLaunchOptions = (proxyConfiguration = {}) => {
|
|
|
1724
2408
|
if (!enableProxy || !proxyUrl) {
|
|
1725
2409
|
return { byPassDomains: [], enableProxy, proxyUrl };
|
|
1726
2410
|
}
|
|
1727
|
-
const byPassDomains = normalizeByPassDomains(config.by_pass_domains);
|
|
2411
|
+
const byPassDomains = ByPass.normalizeByPassDomains(config.by_pass_domains);
|
|
1728
2412
|
return { byPassDomains, enableProxy, proxyUrl };
|
|
1729
2413
|
};
|
|
2414
|
+
var parseChromeMajorVersion = (rawValue = "") => {
|
|
2415
|
+
const match = String(rawValue || "").match(/Chrome\/(\d+)/i);
|
|
2416
|
+
return match ? Number(match[1] || 0) : 0;
|
|
2417
|
+
};
|
|
2418
|
+
var resolveLauncherExecutablePath = (launcher) => {
|
|
2419
|
+
if (!launcher || typeof launcher !== "object") return "";
|
|
2420
|
+
if (typeof launcher.executablePath === "function") {
|
|
2421
|
+
try {
|
|
2422
|
+
return String(launcher.executablePath() || "").trim();
|
|
2423
|
+
} catch {
|
|
2424
|
+
return "";
|
|
2425
|
+
}
|
|
2426
|
+
}
|
|
2427
|
+
return "";
|
|
2428
|
+
};
|
|
2429
|
+
var detectBrowserMajorVersion = (launcher) => {
|
|
2430
|
+
const executablePath = resolveLauncherExecutablePath(launcher);
|
|
2431
|
+
if (!executablePath) return 0;
|
|
2432
|
+
if (browserMajorVersionCache.has(executablePath)) {
|
|
2433
|
+
return browserMajorVersionCache.get(executablePath);
|
|
2434
|
+
}
|
|
2435
|
+
let detectedVersion = 0;
|
|
2436
|
+
try {
|
|
2437
|
+
const rawVersion = execFileSync(executablePath, ["--version"], {
|
|
2438
|
+
encoding: "utf8",
|
|
2439
|
+
stdio: ["ignore", "pipe", "ignore"]
|
|
2440
|
+
});
|
|
2441
|
+
detectedVersion = parseChromeMajorVersion(rawVersion);
|
|
2442
|
+
} catch (error) {
|
|
2443
|
+
logger7.warn(`\u8BFB\u53D6\u6D4F\u89C8\u5668\u7248\u672C\u5931\u8D25: ${error?.message || error}`);
|
|
2444
|
+
}
|
|
2445
|
+
browserMajorVersionCache.set(executablePath, detectedVersion);
|
|
2446
|
+
return detectedVersion;
|
|
2447
|
+
};
|
|
2448
|
+
var buildFingerprintGenerator = ({ locale, browserMajorVersion }) => {
|
|
2449
|
+
return new FingerprintGenerator(
|
|
2450
|
+
AntiCheat.getFingerprintGeneratorOptions({
|
|
2451
|
+
locale,
|
|
2452
|
+
browserMajorVersion
|
|
2453
|
+
})
|
|
2454
|
+
);
|
|
2455
|
+
};
|
|
2456
|
+
var buildReplayableBrowserProfile = (runtimeState, launcher) => {
|
|
2457
|
+
if (!runtimeState || !RuntimeEnv.hasLoginState(runtimeState)) {
|
|
2458
|
+
return { runtimeState, browserProfileCore: null };
|
|
2459
|
+
}
|
|
2460
|
+
let nextState = RuntimeEnv.rememberState(runtimeState);
|
|
2461
|
+
let browserProfileCore = RuntimeEnv.getBrowserProfileCore(nextState);
|
|
2462
|
+
const timezoneId = String(browserProfileCore?.timezone_id || "").trim() || AntiCheat.getBaseConfig().timezoneId;
|
|
2463
|
+
const locale = DEFAULT_LOCALE;
|
|
2464
|
+
const currentBrowserMajorVersion = detectBrowserMajorVersion(launcher);
|
|
2465
|
+
const storedBrowserMajorVersion = Number(browserProfileCore?.browser_major_version || 0);
|
|
2466
|
+
const needsRebuild = !browserProfileCore?.fingerprint || Object.keys(browserProfileCore.fingerprint || {}).length === 0 || currentBrowserMajorVersion > 0 && storedBrowserMajorVersion > 0 && storedBrowserMajorVersion !== currentBrowserMajorVersion;
|
|
2467
|
+
if (needsRebuild) {
|
|
2468
|
+
const generator = buildFingerprintGenerator({
|
|
2469
|
+
locale,
|
|
2470
|
+
browserMajorVersion: currentBrowserMajorVersion
|
|
2471
|
+
});
|
|
2472
|
+
const fingerprint = generator.getFingerprint();
|
|
2473
|
+
const fingerprintBrowserMajorVersion = parseChromeMajorVersion(fingerprint?.fingerprint?.navigator?.userAgent || "");
|
|
2474
|
+
browserProfileCore = {
|
|
2475
|
+
fingerprint,
|
|
2476
|
+
timezone_id: timezoneId,
|
|
2477
|
+
locale,
|
|
2478
|
+
browser_major_version: currentBrowserMajorVersion > 0 ? currentBrowserMajorVersion : fingerprintBrowserMajorVersion,
|
|
2479
|
+
profile_key: String(nextState.envId || "").trim(),
|
|
2480
|
+
schema_version: DEFAULT_BROWSER_PROFILE_SCHEMA_VERSION
|
|
2481
|
+
};
|
|
2482
|
+
nextState = RuntimeEnv.setBrowserProfileCore(nextState, browserProfileCore);
|
|
2483
|
+
logger7.info(
|
|
2484
|
+
`\u5DF2\u751F\u6210\u6D4F\u89C8\u5668\u6307\u7EB9\u771F\u6E90 | env=${String(nextState.envId || "-")} | version=${browserProfileCore.browser_major_version || "-"} | timezone=${timezoneId}`
|
|
2485
|
+
);
|
|
2486
|
+
return { runtimeState: nextState, browserProfileCore };
|
|
2487
|
+
}
|
|
2488
|
+
const normalizedBrowserProfileCore = {
|
|
2489
|
+
...browserProfileCore,
|
|
2490
|
+
timezone_id: timezoneId,
|
|
2491
|
+
locale,
|
|
2492
|
+
profile_key: String(browserProfileCore.profile_key || nextState.envId || "").trim(),
|
|
2493
|
+
schema_version: Number(browserProfileCore.schema_version || 0) > 0 ? Number(browserProfileCore.schema_version || 0) : DEFAULT_BROWSER_PROFILE_SCHEMA_VERSION,
|
|
2494
|
+
browser_major_version: currentBrowserMajorVersion > 0 ? currentBrowserMajorVersion : Number(browserProfileCore.browser_major_version || 0)
|
|
2495
|
+
};
|
|
2496
|
+
const currentCoreRaw = JSON.stringify(browserProfileCore || {});
|
|
2497
|
+
const nextCoreRaw = JSON.stringify(normalizedBrowserProfileCore);
|
|
2498
|
+
if (currentCoreRaw !== nextCoreRaw) {
|
|
2499
|
+
nextState = RuntimeEnv.setBrowserProfileCore(nextState, normalizedBrowserProfileCore);
|
|
2500
|
+
}
|
|
2501
|
+
return {
|
|
2502
|
+
runtimeState: nextState,
|
|
2503
|
+
browserProfileCore: normalizedBrowserProfileCore
|
|
2504
|
+
};
|
|
2505
|
+
};
|
|
2506
|
+
var buildReplayBrowserPoolOptions = (browserProfileCore) => {
|
|
2507
|
+
const fingerprintWithHeaders = browserProfileCore?.fingerprint;
|
|
2508
|
+
const fingerprint = fingerprintWithHeaders?.fingerprint;
|
|
2509
|
+
if (!fingerprintWithHeaders || !fingerprint) {
|
|
2510
|
+
return null;
|
|
2511
|
+
}
|
|
2512
|
+
return {
|
|
2513
|
+
useFingerprints: false,
|
|
2514
|
+
prePageCreateHooks: [
|
|
2515
|
+
(_pageId, _browserController, pageOptions = {}) => {
|
|
2516
|
+
if (!pageOptions || typeof pageOptions !== "object") return;
|
|
2517
|
+
const screen = fingerprint.screen || {};
|
|
2518
|
+
const userAgent = String(fingerprint.navigator?.userAgent || "").trim();
|
|
2519
|
+
if (userAgent) {
|
|
2520
|
+
pageOptions.userAgent = userAgent;
|
|
2521
|
+
}
|
|
2522
|
+
if (Number(screen.width || 0) > 0 && Number(screen.height || 0) > 0) {
|
|
2523
|
+
pageOptions.viewport = {
|
|
2524
|
+
width: Number(screen.width),
|
|
2525
|
+
height: Number(screen.height)
|
|
2526
|
+
};
|
|
2527
|
+
pageOptions.screen = {
|
|
2528
|
+
width: Number(screen.width),
|
|
2529
|
+
height: Number(screen.height)
|
|
2530
|
+
};
|
|
2531
|
+
}
|
|
2532
|
+
if (browserProfileCore.locale) {
|
|
2533
|
+
pageOptions.locale = browserProfileCore.locale;
|
|
2534
|
+
}
|
|
2535
|
+
if (browserProfileCore.timezone_id) {
|
|
2536
|
+
pageOptions.timezoneId = browserProfileCore.timezone_id;
|
|
2537
|
+
}
|
|
2538
|
+
}
|
|
2539
|
+
],
|
|
2540
|
+
postPageCreateHooks: [
|
|
2541
|
+
async (page) => {
|
|
2542
|
+
const context = page?.context?.();
|
|
2543
|
+
if (!context) return;
|
|
2544
|
+
if (!injectedContexts.has(context)) {
|
|
2545
|
+
await fingerprintInjector.attachFingerprintToPlaywright(context, fingerprintWithHeaders);
|
|
2546
|
+
injectedContexts.add(context);
|
|
2547
|
+
}
|
|
2548
|
+
await page.emulateMedia({ colorScheme: "dark" }).catch(() => {
|
|
2549
|
+
});
|
|
2550
|
+
}
|
|
2551
|
+
]
|
|
2552
|
+
};
|
|
2553
|
+
};
|
|
1730
2554
|
var Launch = {
|
|
1731
2555
|
getPlaywrightCrawlerOptions(options = {}) {
|
|
1732
2556
|
const normalizedOptions = Array.isArray(options) ? { customArgs: options } : options || {};
|
|
@@ -1739,18 +2563,22 @@ var Launch = {
|
|
|
1739
2563
|
isRunningOnApify = false,
|
|
1740
2564
|
launcher = null,
|
|
1741
2565
|
preNavigationHooks = [],
|
|
1742
|
-
postNavigationHooks = []
|
|
2566
|
+
postNavigationHooks = [],
|
|
2567
|
+
runtimeState = null
|
|
1743
2568
|
} = normalizedOptions;
|
|
1744
2569
|
const { byPassDomains, enableProxy, proxyUrl } = resolveProxyLaunchOptions(proxyConfiguration);
|
|
1745
|
-
const byPassRules = buildByPassDomainRules(byPassDomains);
|
|
1746
|
-
const proxyMeter = enableProxy && proxyUrl ? startProxyMeter({ proxyUrl, debugMode }) : null;
|
|
2570
|
+
const byPassRules = ByPass.buildByPassDomainRules(byPassDomains);
|
|
2571
|
+
const proxyMeter = enableProxy && proxyUrl ? ProxyMeterRuntime.startProxyMeter({ proxyUrl, debugMode }) : null;
|
|
1747
2572
|
const launchProxy = proxyMeter ? { server: proxyMeter.server } : null;
|
|
1748
2573
|
if (launchProxy && byPassDomains.length > 0) {
|
|
1749
2574
|
launchProxy.bypass = byPassDomains.join(",");
|
|
1750
2575
|
}
|
|
2576
|
+
const replayContext = buildReplayableBrowserProfile(runtimeState, launcher);
|
|
2577
|
+
const replayBrowserPoolOptions = buildReplayBrowserPoolOptions(replayContext.browserProfileCore);
|
|
2578
|
+
const launchLocale = String(replayContext.browserProfileCore?.locale || DEFAULT_LOCALE).trim() || DEFAULT_LOCALE;
|
|
1751
2579
|
const launchOptions = {
|
|
1752
2580
|
args: [
|
|
1753
|
-
...AntiCheat.getLaunchArgs(),
|
|
2581
|
+
...AntiCheat.getLaunchArgs({ locale: launchLocale }),
|
|
1754
2582
|
...customArgs
|
|
1755
2583
|
],
|
|
1756
2584
|
ignoreDefaultArgs: ["--enable-automation"]
|
|
@@ -1762,7 +2590,7 @@ var Launch = {
|
|
|
1762
2590
|
if (enableByPassLogger && launchProxy) {
|
|
1763
2591
|
let upstreamLabel = "";
|
|
1764
2592
|
try {
|
|
1765
|
-
const parsedProxyUrl = new URL(proxyUrl);
|
|
2593
|
+
const parsedProxyUrl = new URL(proxyUrl.includes("://") ? proxyUrl : `http://${proxyUrl}`);
|
|
1766
2594
|
upstreamLabel = `${parsedProxyUrl.protocol}//${parsedProxyUrl.host}`;
|
|
1767
2595
|
} catch {
|
|
1768
2596
|
}
|
|
@@ -1771,14 +2599,10 @@ var Launch = {
|
|
|
1771
2599
|
);
|
|
1772
2600
|
logger7.info(`[\u6D41\u91CF\u89C2\u6D4B] \u9010\u8BF7\u6C42\u8C03\u8BD5=${Boolean(debugMode) ? "\u5F00\u542F" : "\u5173\u95ED"}\uFF08\u6C47\u603B\u59CB\u7EC8\u5F00\u542F\uFF09`);
|
|
1773
2601
|
} else if (enableByPassLogger && enableProxy && !launchProxy) {
|
|
1774
|
-
logger7.info(
|
|
1775
|
-
`[\u4EE3\u7406\u672A\u542F\u7528] enable_proxy=true \u4F46 proxy_url \u4E3A\u7A7A`
|
|
1776
|
-
);
|
|
2602
|
+
logger7.info("[\u4EE3\u7406\u672A\u542F\u7528] enable_proxy=true \u4F46 proxy_url \u4E3A\u7A7A");
|
|
1777
2603
|
logger7.info(`[\u6D41\u91CF\u89C2\u6D4B] \u9010\u8BF7\u6C42\u8C03\u8BD5=${Boolean(debugMode) ? "\u5F00\u542F" : "\u5173\u95ED"}\uFF08\u6C47\u603B\u59CB\u7EC8\u5F00\u542F\uFF09`);
|
|
1778
2604
|
} else if (enableByPassLogger && !enableProxy && proxyUrl) {
|
|
1779
|
-
logger7.info(
|
|
1780
|
-
`[\u4EE3\u7406\u672A\u542F\u7528] enable_proxy=false \u4E14 proxy_url \u5DF2\u914D\u7F6E`
|
|
1781
|
-
);
|
|
2605
|
+
logger7.info("[\u4EE3\u7406\u672A\u542F\u7528] enable_proxy=false \u4E14 proxy_url \u5DF2\u914D\u7F6E");
|
|
1782
2606
|
logger7.info(`[\u6D41\u91CF\u89C2\u6D4B] \u9010\u8BF7\u6C42\u8C03\u8BD5=${Boolean(debugMode) ? "\u5F00\u542F" : "\u5173\u95ED"}\uFF08\u6C47\u603B\u59CB\u7EC8\u5F00\u542F\uFF09`);
|
|
1783
2607
|
} else if (enableByPassLogger) {
|
|
1784
2608
|
logger7.info(`[\u6D41\u91CF\u89C2\u6D4B] \u9010\u8BF7\u6C42\u8C03\u8BD5=${Boolean(debugMode) ? "\u5F00\u542F" : "\u5173\u95ED"}\uFF08\u6C47\u603B\u59CB\u7EC8\u5F00\u542F\uFF09`);
|
|
@@ -1797,9 +2621,9 @@ var Launch = {
|
|
|
1797
2621
|
const requestHandler = (req) => {
|
|
1798
2622
|
const requestUrl = req.url();
|
|
1799
2623
|
const resourceType = req.resourceType();
|
|
1800
|
-
const matched = byPassDomains.length > 0 ? findMatchedByPassRule(byPassRules, requestUrl) : null;
|
|
1801
|
-
if (launchProxy
|
|
1802
|
-
recordProxyMeterResourceType(requestUrl, resourceType);
|
|
2624
|
+
const matched = byPassDomains.length > 0 ? ByPass.findMatchedByPassRule(byPassRules, requestUrl) : null;
|
|
2625
|
+
if (launchProxy) {
|
|
2626
|
+
ProxyMeterRuntime.recordProxyMeterResourceType(requestUrl, resourceType);
|
|
1803
2627
|
}
|
|
1804
2628
|
if (!enableByPassLogger || byPassDomains.length === 0) return;
|
|
1805
2629
|
if (!matched || !matched.rule) return;
|
|
@@ -1818,7 +2642,8 @@ var Launch = {
|
|
|
1818
2642
|
const crawlerBaseOptions = {
|
|
1819
2643
|
...DEFAULT_CRAWLER_BASE_OPTIONS,
|
|
1820
2644
|
headless: !runInHeadfulMode || isRunningOnApify,
|
|
1821
|
-
|
|
2645
|
+
// 有 core.fingerprint 时走固定回放;否则退回 Crawlee 默认指纹模式。
|
|
2646
|
+
browserPoolOptions: replayBrowserPoolOptions || {
|
|
1822
2647
|
useFingerprints: true,
|
|
1823
2648
|
fingerprintOptions: {
|
|
1824
2649
|
fingerprintGeneratorOptions: AntiCheat.getFingerprintGeneratorOptions()
|
|
@@ -2021,241 +2846,9 @@ var Captcha = {
|
|
|
2021
2846
|
useCaptchaMonitor
|
|
2022
2847
|
};
|
|
2023
2848
|
|
|
2024
|
-
// src/sse.js
|
|
2025
|
-
import https from "https";
|
|
2026
|
-
import { URL as URL2 } from "url";
|
|
2027
|
-
var logger10 = createInternalLogger("Sse");
|
|
2028
|
-
var Sse = {
|
|
2029
|
-
/**
|
|
2030
|
-
* 解析 SSE 流文本
|
|
2031
|
-
* 支持 `data: {...}` 和 `data:{...}` 两种格式
|
|
2032
|
-
* @param {string} sseStreamText
|
|
2033
|
-
* @returns {Array<Object>} events
|
|
2034
|
-
*/
|
|
2035
|
-
parseSseStream(sseStreamText) {
|
|
2036
|
-
const events = [];
|
|
2037
|
-
const lines = sseStreamText.split("\n");
|
|
2038
|
-
for (const line of lines) {
|
|
2039
|
-
if (line.startsWith("data:")) {
|
|
2040
|
-
try {
|
|
2041
|
-
const jsonContent = line.substring(5).trim();
|
|
2042
|
-
if (jsonContent) {
|
|
2043
|
-
events.push(JSON.parse(jsonContent));
|
|
2044
|
-
}
|
|
2045
|
-
} catch (e) {
|
|
2046
|
-
logger10.debug("parseSseStream", `JSON \u89E3\u6790\u5931\u8D25: ${e.message}, line: ${line.substring(0, 100)}...`);
|
|
2047
|
-
}
|
|
2048
|
-
}
|
|
2049
|
-
}
|
|
2050
|
-
logger10.success("parseSseStream", `\u89E3\u6790\u5B8C\u6210, events \u6570\u91CF: ${events.length}`);
|
|
2051
|
-
return events;
|
|
2052
|
-
},
|
|
2053
|
-
/**
|
|
2054
|
-
* 拦截网络请求并使用 Node.js 原生 https 模块转发,以解决流式数据捕获问题。
|
|
2055
|
-
* @param {import('playwright').Page} page
|
|
2056
|
-
* @param {string|RegExp} urlPattern - 拦截的 URL 模式
|
|
2057
|
-
* @param {object} options
|
|
2058
|
-
* @param {function(string, function, string): void} [options.onData] - (textChunk, resolve, accumulatedText) => void
|
|
2059
|
-
* @param {function(string, function): void} [options.onEnd] - (fullText, resolve) => void
|
|
2060
|
-
* @param {function(Error, function): void} [options.onTimeout] - (error, reject) => void
|
|
2061
|
-
* @param {number} [options.initialTimeout=90000] - 初始数据接收超时 (ms),默认 90s
|
|
2062
|
-
* @param {number} [options.overallTimeout=180000] - 整体请求超时时间 (ms),默认 180s
|
|
2063
|
-
* @param {boolean} [options.autoUnroute=true] - resolve/reject 后是否自动取消拦截
|
|
2064
|
-
* @param {boolean} [options.firstMatchOnly=true] - 只拦截第一个命中的请求,后续请求全部放行
|
|
2065
|
-
* @returns {Promise<any>} - 返回 Promise,当流满足条件时 resolve
|
|
2066
|
-
*/
|
|
2067
|
-
async intercept(page, urlPattern, options = {}) {
|
|
2068
|
-
const {
|
|
2069
|
-
onData,
|
|
2070
|
-
onEnd,
|
|
2071
|
-
onTimeout,
|
|
2072
|
-
initialTimeout = 9e4,
|
|
2073
|
-
overallTimeout = 18e4,
|
|
2074
|
-
autoUnroute = true,
|
|
2075
|
-
firstMatchOnly = true
|
|
2076
|
-
} = options;
|
|
2077
|
-
let initialTimer = null;
|
|
2078
|
-
let overallTimer = null;
|
|
2079
|
-
let hasReceivedInitialData = false;
|
|
2080
|
-
const clearAllTimers = () => {
|
|
2081
|
-
if (initialTimer) clearTimeout(initialTimer);
|
|
2082
|
-
if (overallTimer) clearTimeout(overallTimer);
|
|
2083
|
-
initialTimer = null;
|
|
2084
|
-
overallTimer = null;
|
|
2085
|
-
};
|
|
2086
|
-
let safeResolve = () => {
|
|
2087
|
-
};
|
|
2088
|
-
let safeReject = () => {
|
|
2089
|
-
};
|
|
2090
|
-
let safeUnroute = () => {
|
|
2091
|
-
};
|
|
2092
|
-
const workPromise = new Promise((resolve, reject) => {
|
|
2093
|
-
let finished = false;
|
|
2094
|
-
let unrouteRequested = false;
|
|
2095
|
-
let hasMatchedOnce = false;
|
|
2096
|
-
safeUnroute = () => {
|
|
2097
|
-
if (!autoUnroute) return;
|
|
2098
|
-
if (unrouteRequested) return;
|
|
2099
|
-
unrouteRequested = true;
|
|
2100
|
-
logger10.info("[MITM] autoUnroute: \u53D6\u6D88\u540E\u7EED\u62E6\u622A");
|
|
2101
|
-
page.unroute(urlPattern, routeHandler).catch(() => {
|
|
2102
|
-
});
|
|
2103
|
-
};
|
|
2104
|
-
safeResolve = (value) => {
|
|
2105
|
-
if (finished) return;
|
|
2106
|
-
finished = true;
|
|
2107
|
-
clearAllTimers();
|
|
2108
|
-
safeUnroute();
|
|
2109
|
-
resolve(value);
|
|
2110
|
-
};
|
|
2111
|
-
safeReject = (error) => {
|
|
2112
|
-
if (finished) return;
|
|
2113
|
-
finished = true;
|
|
2114
|
-
clearAllTimers();
|
|
2115
|
-
safeUnroute();
|
|
2116
|
-
reject(error);
|
|
2117
|
-
};
|
|
2118
|
-
const routeHandler = async (route) => {
|
|
2119
|
-
if (firstMatchOnly && hasMatchedOnce) {
|
|
2120
|
-
logger10.info(`[MITM] firstMatchOnly: \u653E\u884C\u540E\u7EED\u8BF7\u6C42: ${route.request().url()}`);
|
|
2121
|
-
route.continue().catch(() => {
|
|
2122
|
-
});
|
|
2123
|
-
return;
|
|
2124
|
-
}
|
|
2125
|
-
if (firstMatchOnly && !hasMatchedOnce) {
|
|
2126
|
-
hasMatchedOnce = true;
|
|
2127
|
-
logger10.info("[MITM] firstMatchOnly: \u547D\u4E2D\u9996\u4E2A\u8BF7\u6C42\uFF0C\u53D6\u6D88\u540E\u7EED\u62E6\u622A");
|
|
2128
|
-
page.unroute(urlPattern, routeHandler).catch(() => {
|
|
2129
|
-
});
|
|
2130
|
-
}
|
|
2131
|
-
const request = route.request();
|
|
2132
|
-
logger10.info(`[MITM] \u5DF2\u62E6\u622A\u8BF7\u6C42: ${request.url()}`);
|
|
2133
|
-
try {
|
|
2134
|
-
const headers = await request.allHeaders();
|
|
2135
|
-
const postData = request.postData();
|
|
2136
|
-
const urlObj = new URL2(request.url());
|
|
2137
|
-
delete headers["accept-encoding"];
|
|
2138
|
-
delete headers["content-length"];
|
|
2139
|
-
const reqOptions = {
|
|
2140
|
-
hostname: urlObj.hostname,
|
|
2141
|
-
port: 443,
|
|
2142
|
-
path: urlObj.pathname + urlObj.search,
|
|
2143
|
-
method: request.method(),
|
|
2144
|
-
headers,
|
|
2145
|
-
timeout: overallTimeout
|
|
2146
|
-
};
|
|
2147
|
-
const req = https.request(reqOptions, (res) => {
|
|
2148
|
-
const chunks = [];
|
|
2149
|
-
let accumulatedText = "";
|
|
2150
|
-
res.on("data", (chunk) => {
|
|
2151
|
-
if (!hasReceivedInitialData) {
|
|
2152
|
-
hasReceivedInitialData = true;
|
|
2153
|
-
if (initialTimer) {
|
|
2154
|
-
clearTimeout(initialTimer);
|
|
2155
|
-
initialTimer = null;
|
|
2156
|
-
}
|
|
2157
|
-
logger10.debug("[Intercept] \u5DF2\u63A5\u6536\u521D\u59CB\u6570\u636E");
|
|
2158
|
-
}
|
|
2159
|
-
chunks.push(chunk);
|
|
2160
|
-
const textChunk = chunk.toString("utf-8");
|
|
2161
|
-
accumulatedText += textChunk;
|
|
2162
|
-
if (onData) {
|
|
2163
|
-
try {
|
|
2164
|
-
onData(textChunk, safeResolve, accumulatedText);
|
|
2165
|
-
} catch (e) {
|
|
2166
|
-
logger10.fail(`onData \u9519\u8BEF`, e);
|
|
2167
|
-
}
|
|
2168
|
-
}
|
|
2169
|
-
});
|
|
2170
|
-
res.on("end", () => {
|
|
2171
|
-
logger10.info("[MITM] \u4E0A\u6E38\u54CD\u5E94\u7ED3\u675F");
|
|
2172
|
-
clearAllTimers();
|
|
2173
|
-
if (onEnd) {
|
|
2174
|
-
try {
|
|
2175
|
-
onEnd(accumulatedText, safeResolve);
|
|
2176
|
-
} catch (e) {
|
|
2177
|
-
logger10.fail(`onEnd \u9519\u8BEF`, e);
|
|
2178
|
-
}
|
|
2179
|
-
} else if (!onData) {
|
|
2180
|
-
safeResolve(accumulatedText);
|
|
2181
|
-
}
|
|
2182
|
-
route.fulfill({
|
|
2183
|
-
status: res.statusCode,
|
|
2184
|
-
headers: res.headers,
|
|
2185
|
-
body: Buffer.concat(chunks)
|
|
2186
|
-
}).catch(() => {
|
|
2187
|
-
});
|
|
2188
|
-
});
|
|
2189
|
-
});
|
|
2190
|
-
req.on("error", (e) => {
|
|
2191
|
-
clearAllTimers();
|
|
2192
|
-
route.abort().catch(() => {
|
|
2193
|
-
});
|
|
2194
|
-
safeReject(e);
|
|
2195
|
-
});
|
|
2196
|
-
if (postData) req.write(postData);
|
|
2197
|
-
req.end();
|
|
2198
|
-
} catch (e) {
|
|
2199
|
-
clearAllTimers();
|
|
2200
|
-
route.continue().catch(() => {
|
|
2201
|
-
});
|
|
2202
|
-
safeReject(e);
|
|
2203
|
-
}
|
|
2204
|
-
};
|
|
2205
|
-
page.route(urlPattern, routeHandler).catch(safeReject);
|
|
2206
|
-
});
|
|
2207
|
-
const timeoutPromise = new Promise((_, reject) => {
|
|
2208
|
-
initialTimer = setTimeout(() => {
|
|
2209
|
-
if (!hasReceivedInitialData) {
|
|
2210
|
-
const error = new CrawlerError({
|
|
2211
|
-
message: `\u521D\u59CB\u6570\u636E\u63A5\u6536\u8D85\u65F6 (${initialTimeout}ms)`,
|
|
2212
|
-
code: Code.InitialTimeout,
|
|
2213
|
-
context: { timeout: initialTimeout }
|
|
2214
|
-
});
|
|
2215
|
-
clearAllTimers();
|
|
2216
|
-
if (onTimeout) {
|
|
2217
|
-
try {
|
|
2218
|
-
onTimeout(error, (err) => safeReject(err));
|
|
2219
|
-
} catch (e) {
|
|
2220
|
-
safeReject(e);
|
|
2221
|
-
}
|
|
2222
|
-
} else {
|
|
2223
|
-
safeReject(error);
|
|
2224
|
-
}
|
|
2225
|
-
}
|
|
2226
|
-
}, initialTimeout);
|
|
2227
|
-
overallTimer = setTimeout(() => {
|
|
2228
|
-
const error = new CrawlerError({
|
|
2229
|
-
message: `\u6574\u4F53\u8BF7\u6C42\u8D85\u65F6 (${overallTimeout}ms)`,
|
|
2230
|
-
code: Code.OverallTimeout,
|
|
2231
|
-
context: { timeout: overallTimeout }
|
|
2232
|
-
});
|
|
2233
|
-
clearAllTimers();
|
|
2234
|
-
if (onTimeout) {
|
|
2235
|
-
try {
|
|
2236
|
-
onTimeout(error, (err) => safeReject(err));
|
|
2237
|
-
} catch (e) {
|
|
2238
|
-
safeReject(e);
|
|
2239
|
-
}
|
|
2240
|
-
} else {
|
|
2241
|
-
safeReject(error);
|
|
2242
|
-
}
|
|
2243
|
-
}, overallTimeout);
|
|
2244
|
-
});
|
|
2245
|
-
workPromise.catch(() => {
|
|
2246
|
-
});
|
|
2247
|
-
timeoutPromise.catch(() => {
|
|
2248
|
-
});
|
|
2249
|
-
const racePromise = Promise.race([workPromise, timeoutPromise]);
|
|
2250
|
-
racePromise.catch(() => {
|
|
2251
|
-
});
|
|
2252
|
-
return racePromise;
|
|
2253
|
-
}
|
|
2254
|
-
};
|
|
2255
|
-
|
|
2256
2849
|
// src/mutation.js
|
|
2257
2850
|
import { v4 as uuidv42 } from "uuid";
|
|
2258
|
-
var
|
|
2851
|
+
var logger10 = createInternalLogger("Mutation");
|
|
2259
2852
|
var MUTATION_MONITOR_MODE = Object.freeze({
|
|
2260
2853
|
Added: "added",
|
|
2261
2854
|
Changed: "changed",
|
|
@@ -2288,14 +2881,14 @@ var Mutation = {
|
|
|
2288
2881
|
const stableTime = options.stableTime ?? 5 * 1e3;
|
|
2289
2882
|
const timeout = options.timeout ?? 120 * 1e3;
|
|
2290
2883
|
const onMutation = options.onMutation;
|
|
2291
|
-
|
|
2884
|
+
logger10.start("waitForStable", `\u76D1\u63A7 ${selectorList.length} \u4E2A\u9009\u62E9\u5668, \u7A33\u5B9A\u65F6\u95F4=${stableTime}ms`);
|
|
2292
2885
|
if (initialTimeout > 0) {
|
|
2293
2886
|
const selectorQuery = selectorList.join(",");
|
|
2294
2887
|
try {
|
|
2295
2888
|
await page.waitForSelector(selectorQuery, { timeout: initialTimeout });
|
|
2296
|
-
|
|
2889
|
+
logger10.info(`waitForStable \u5DF2\u68C0\u6D4B\u5230\u5143\u7D20: ${selectorQuery}`);
|
|
2297
2890
|
} catch (e) {
|
|
2298
|
-
|
|
2891
|
+
logger10.warning(`waitForStable \u521D\u59CB\u7B49\u5F85\u8D85\u65F6 (${initialTimeout}ms): ${selectorQuery}`);
|
|
2299
2892
|
throw e;
|
|
2300
2893
|
}
|
|
2301
2894
|
}
|
|
@@ -2311,7 +2904,7 @@ var Mutation = {
|
|
|
2311
2904
|
return "__CONTINUE__";
|
|
2312
2905
|
}
|
|
2313
2906
|
});
|
|
2314
|
-
|
|
2907
|
+
logger10.info("waitForStable \u5DF2\u542F\u7528 onMutation \u56DE\u8C03");
|
|
2315
2908
|
} catch (e) {
|
|
2316
2909
|
}
|
|
2317
2910
|
}
|
|
@@ -2426,9 +3019,9 @@ var Mutation = {
|
|
|
2426
3019
|
{ selectorList, stableTime, timeout, callbackName, hasCallback: !!onMutation }
|
|
2427
3020
|
);
|
|
2428
3021
|
if (result.mutationCount === 0 && result.stableTime === 0) {
|
|
2429
|
-
|
|
3022
|
+
logger10.warning("waitForStable \u672A\u627E\u5230\u53EF\u76D1\u63A7\u7684\u5143\u7D20");
|
|
2430
3023
|
}
|
|
2431
|
-
|
|
3024
|
+
logger10.success("waitForStable", `DOM \u7A33\u5B9A, \u603B\u5171 ${result.mutationCount} \u6B21\u53D8\u5316${result.wasPaused ? ", \u66FE\u6682\u505C\u8BA1\u65F6" : ""}`);
|
|
2432
3025
|
return result;
|
|
2433
3026
|
},
|
|
2434
3027
|
/**
|
|
@@ -2546,22 +3139,22 @@ var Mutation = {
|
|
|
2546
3139
|
return "__CONTINUE__";
|
|
2547
3140
|
}
|
|
2548
3141
|
};
|
|
2549
|
-
|
|
3142
|
+
logger10.start(
|
|
2550
3143
|
"waitForStableAcrossRoots",
|
|
2551
3144
|
`\u76D1\u63A7 ${selectorList.length} \u4E2A\u9009\u62E9\u5668(\u8DE8 root), \u7A33\u5B9A\u65F6\u95F4=${waitForStableTime}ms`
|
|
2552
3145
|
);
|
|
2553
3146
|
if (initialTimeout > 0) {
|
|
2554
3147
|
try {
|
|
2555
3148
|
await page.waitForSelector(selectorQuery, { timeout: initialTimeout });
|
|
2556
|
-
|
|
3149
|
+
logger10.info(`waitForStableAcrossRoots \u5DF2\u68C0\u6D4B\u5230\u5143\u7D20: ${selectorQuery}`);
|
|
2557
3150
|
} catch (e) {
|
|
2558
|
-
|
|
3151
|
+
logger10.warning(`waitForStableAcrossRoots \u521D\u59CB\u7B49\u5F85\u8D85\u65F6 (${initialTimeout}ms): ${selectorQuery}`);
|
|
2559
3152
|
throw e;
|
|
2560
3153
|
}
|
|
2561
3154
|
}
|
|
2562
3155
|
let state = await buildState();
|
|
2563
3156
|
if (!state?.hasMatched) {
|
|
2564
|
-
|
|
3157
|
+
logger10.warning("waitForStableAcrossRoots \u672A\u627E\u5230\u53EF\u76D1\u63A7\u7684\u5143\u7D20");
|
|
2565
3158
|
return { mutationCount: 0, stableTime: 0, wasPaused: false };
|
|
2566
3159
|
}
|
|
2567
3160
|
let mutationCount = 0;
|
|
@@ -2598,7 +3191,7 @@ var Mutation = {
|
|
|
2598
3191
|
if (lastState.snapshotKey !== lastSnapshotKey) {
|
|
2599
3192
|
lastSnapshotKey = lastState.snapshotKey;
|
|
2600
3193
|
mutationCount += 1;
|
|
2601
|
-
|
|
3194
|
+
logger10.info(
|
|
2602
3195
|
`waitForStableAcrossRoots \u53D8\u5316#${mutationCount}, len=${lastState.snapshotLength}, path=${lastState.primaryPath || "unknown"}, preview="${truncate(lastState.text, 120)}"`
|
|
2603
3196
|
);
|
|
2604
3197
|
const signal = await invokeMutationCallback({
|
|
@@ -2611,7 +3204,7 @@ var Mutation = {
|
|
|
2611
3204
|
continue;
|
|
2612
3205
|
}
|
|
2613
3206
|
if (!isPaused && stableSince > 0 && Date.now() - stableSince >= waitForStableTime) {
|
|
2614
|
-
|
|
3207
|
+
logger10.success("waitForStableAcrossRoots", `DOM \u7A33\u5B9A, \u603B\u5171 ${mutationCount} \u6B21\u53D8\u5316${wasPaused ? ", \u66FE\u6682\u505C\u8BA1\u65F6" : ""}`);
|
|
2615
3208
|
return {
|
|
2616
3209
|
mutationCount,
|
|
2617
3210
|
stableTime: waitForStableTime,
|
|
@@ -2638,7 +3231,7 @@ var Mutation = {
|
|
|
2638
3231
|
const onMutation = options.onMutation;
|
|
2639
3232
|
const rawMode = String(options.mode || MUTATION_MONITOR_MODE.Added).toLowerCase();
|
|
2640
3233
|
const mode = [MUTATION_MONITOR_MODE.Added, MUTATION_MONITOR_MODE.Changed, MUTATION_MONITOR_MODE.All].includes(rawMode) ? rawMode : MUTATION_MONITOR_MODE.Added;
|
|
2641
|
-
|
|
3234
|
+
logger10.start("useMonitor", `\u76D1\u63A7 ${selectorList.length} \u4E2A\u9009\u62E9\u5668, mode=${mode}`);
|
|
2642
3235
|
const monitorKey = generateKey("pk_mon");
|
|
2643
3236
|
const callbackName = generateKey("pk_mon_cb");
|
|
2644
3237
|
const cleanerName = generateKey("pk_mon_clean");
|
|
@@ -2781,7 +3374,7 @@ var Mutation = {
|
|
|
2781
3374
|
return total;
|
|
2782
3375
|
};
|
|
2783
3376
|
}, { selectorList, monitorKey, callbackName, cleanerName, hasCallback: !!onMutation, mode });
|
|
2784
|
-
|
|
3377
|
+
logger10.success("useMonitor", "\u76D1\u63A7\u5668\u5DF2\u542F\u52A8");
|
|
2785
3378
|
return {
|
|
2786
3379
|
stop: async () => {
|
|
2787
3380
|
let totalMutations = 0;
|
|
@@ -2794,7 +3387,7 @@ var Mutation = {
|
|
|
2794
3387
|
}, cleanerName);
|
|
2795
3388
|
} catch (e) {
|
|
2796
3389
|
}
|
|
2797
|
-
|
|
3390
|
+
logger10.success("useMonitor.stop", `\u76D1\u63A7\u5DF2\u505C\u6B62, \u5171 ${totalMutations} \u6B21\u53D8\u5316`);
|
|
2798
3391
|
return { totalMutations };
|
|
2799
3392
|
}
|
|
2800
3393
|
};
|
|
@@ -2805,28 +3398,28 @@ var Mutation = {
|
|
|
2805
3398
|
var Display = {
|
|
2806
3399
|
parseTokenDisplayName(value) {
|
|
2807
3400
|
if (!value) {
|
|
2808
|
-
return { owner: "",
|
|
3401
|
+
return { owner: "", envType: "", note: "" };
|
|
2809
3402
|
}
|
|
2810
3403
|
const parts = String(value).split(":");
|
|
2811
3404
|
const owner = (parts[0] || "").trim();
|
|
2812
|
-
const
|
|
3405
|
+
const envType = (parts[1] || "").trim();
|
|
2813
3406
|
const note = parts.length > 2 ? parts.slice(2).join(":").trim() : "";
|
|
2814
|
-
return { owner,
|
|
3407
|
+
return { owner, envType, note };
|
|
2815
3408
|
},
|
|
2816
|
-
|
|
3409
|
+
parseEnvDisplayName(value) {
|
|
2817
3410
|
if (!value) {
|
|
2818
|
-
return {
|
|
3411
|
+
return { env: "", owner: "", envType: "", note: "" };
|
|
2819
3412
|
}
|
|
2820
3413
|
const parts = String(value).split(":");
|
|
2821
|
-
const
|
|
3414
|
+
const env = (parts[0] || "").trim();
|
|
2822
3415
|
const owner = (parts[1] || "").trim();
|
|
2823
|
-
const
|
|
3416
|
+
const envType = (parts[2] || "").trim();
|
|
2824
3417
|
const note = parts.length > 3 ? parts.slice(3).join(":").trim() : "";
|
|
2825
|
-
return {
|
|
3418
|
+
return { env, owner, envType, note };
|
|
2826
3419
|
},
|
|
2827
|
-
buildTokenDisplayName({ owner,
|
|
3420
|
+
buildTokenDisplayName({ owner, envType, note } = {}) {
|
|
2828
3421
|
const trimmedOwner = owner?.trim() || "";
|
|
2829
|
-
const trimmedType =
|
|
3422
|
+
const trimmedType = envType?.trim() || "";
|
|
2830
3423
|
const trimmedNote = note?.trim() || "";
|
|
2831
3424
|
const parts = [trimmedOwner, trimmedType];
|
|
2832
3425
|
if (trimmedNote) {
|
|
@@ -2849,12 +3442,12 @@ var Display = {
|
|
|
2849
3442
|
const secondary = hasName && short ? short : "";
|
|
2850
3443
|
return { primary, secondary, fullId: cleanId, shortId: short };
|
|
2851
3444
|
},
|
|
2852
|
-
|
|
2853
|
-
const parts = this.
|
|
2854
|
-
const cleanId = String(
|
|
3445
|
+
resolveEnvIdentity(displayName, envId) {
|
|
3446
|
+
const parts = this.parseEnvDisplayName(displayName);
|
|
3447
|
+
const cleanId = String(envId || "").trim();
|
|
2855
3448
|
const short = this.shortId(cleanId, 8);
|
|
2856
|
-
const hasName = parts.
|
|
2857
|
-
const primary = hasName ? parts.
|
|
3449
|
+
const hasName = parts.env !== "" && parts.env !== cleanId;
|
|
3450
|
+
const primary = hasName ? parts.env : short || "-";
|
|
2858
3451
|
const secondary = hasName && short ? short : "";
|
|
2859
3452
|
return { primary, secondary, parts, fullId: cleanId, shortId: short };
|
|
2860
3453
|
}
|
|
@@ -3663,7 +4256,7 @@ var createTemplateLogger = (baseLogger = createBaseLogger()) => {
|
|
|
3663
4256
|
};
|
|
3664
4257
|
var getDefaultBaseLogger = () => createBaseLogger("");
|
|
3665
4258
|
var Logger = {
|
|
3666
|
-
setLogger: (
|
|
4259
|
+
setLogger: (logger12) => setDefaultLogger(logger12),
|
|
3667
4260
|
info: (message) => getDefaultBaseLogger().info(message),
|
|
3668
4261
|
success: (message) => getDefaultBaseLogger().success(message),
|
|
3669
4262
|
warning: (message) => getDefaultBaseLogger().warning(message),
|
|
@@ -3671,15 +4264,15 @@ var Logger = {
|
|
|
3671
4264
|
error: (message) => getDefaultBaseLogger().error(message),
|
|
3672
4265
|
debug: (message) => getDefaultBaseLogger().debug(message),
|
|
3673
4266
|
start: (message) => getDefaultBaseLogger().start(message),
|
|
3674
|
-
useTemplate: (
|
|
3675
|
-
if (
|
|
4267
|
+
useTemplate: (logger12) => {
|
|
4268
|
+
if (logger12) return createTemplateLogger(createBaseLogger("", logger12));
|
|
3676
4269
|
return createTemplateLogger();
|
|
3677
4270
|
}
|
|
3678
4271
|
};
|
|
3679
4272
|
|
|
3680
4273
|
// src/share.js
|
|
3681
4274
|
import delay2 from "delay";
|
|
3682
|
-
var
|
|
4275
|
+
var logger11 = createInternalLogger("Share");
|
|
3683
4276
|
var DEFAULT_TIMEOUT_MS2 = 50 * 1e3;
|
|
3684
4277
|
var DEFAULT_PAYLOAD_SNAPSHOT_MAX_LEN = 500;
|
|
3685
4278
|
var DEFAULT_POLL_INTERVAL_MS = 120;
|
|
@@ -3799,7 +4392,7 @@ var createDomShareMonitor = async (page, options = {}) => {
|
|
|
3799
4392
|
const onMatch = typeof options.onMatch === "function" ? options.onMatch : null;
|
|
3800
4393
|
const onTelemetry = typeof options.onTelemetry === "function" ? options.onTelemetry : null;
|
|
3801
4394
|
let matched = false;
|
|
3802
|
-
|
|
4395
|
+
logger11.info(`DOM \u76D1\u542C\u51C6\u5907\u6302\u8F7D: selectors=${toJsonInline(selectors, 120)}, mode=${mode}`);
|
|
3803
4396
|
const monitor = await Mutation.useMonitor(page, selectors, {
|
|
3804
4397
|
mode,
|
|
3805
4398
|
onMutation: (context = {}) => {
|
|
@@ -3817,12 +4410,12 @@ ${text}`;
|
|
|
3817
4410
|
});
|
|
3818
4411
|
}
|
|
3819
4412
|
if (mutationCount <= 5 || mutationCount % 50 === 0) {
|
|
3820
|
-
|
|
4413
|
+
logger11.info(`DOM \u53D8\u5316\u5DF2\u6355\u83B7: mutationCount=${mutationCount}, mutationNodes=${mutationNodes.length}`);
|
|
3821
4414
|
}
|
|
3822
4415
|
const [candidate] = Utils.parseLinks(rawDom, { prefix }) || [];
|
|
3823
4416
|
if (!candidate) return;
|
|
3824
4417
|
matched = true;
|
|
3825
|
-
|
|
4418
|
+
logger11.success("captureLink.domHit", `DOM \u547D\u4E2D\u5206\u4EAB\u94FE\u63A5: mutationCount=${mutationCount}, link=${candidate}`);
|
|
3826
4419
|
if (onMatch) {
|
|
3827
4420
|
onMatch({
|
|
3828
4421
|
link: candidate,
|
|
@@ -3838,7 +4431,7 @@ ${text}`;
|
|
|
3838
4431
|
return {
|
|
3839
4432
|
stop: async () => {
|
|
3840
4433
|
const result = await monitor.stop();
|
|
3841
|
-
|
|
4434
|
+
logger11.info(`DOM \u76D1\u542C\u5DF2\u505C\u6B62: totalMutations=${result?.totalMutations || 0}`);
|
|
3842
4435
|
return result;
|
|
3843
4436
|
}
|
|
3844
4437
|
};
|
|
@@ -3878,8 +4471,8 @@ var Share = {
|
|
|
3878
4471
|
if (share.mode === "response" && apiMatchers.length === 0) {
|
|
3879
4472
|
throw new Error("Share.captureLink requires share.xurl[0] api matcher when mode=response");
|
|
3880
4473
|
}
|
|
3881
|
-
|
|
3882
|
-
|
|
4474
|
+
logger11.start("captureLink", `mode=${share.mode}, timeoutMs=${timeoutMs}, prefix=${share.prefix}`);
|
|
4475
|
+
logger11.info(`captureLink \u914D\u7F6E: xurl=${toJsonInline(share.xurl)}, domMode=${domMode}, domSelectors=${toJsonInline(domSelectors, 120)}`);
|
|
3883
4476
|
const stats = {
|
|
3884
4477
|
actionTimedOut: false,
|
|
3885
4478
|
domMutationCount: 0,
|
|
@@ -3904,7 +4497,7 @@ var Share = {
|
|
|
3904
4497
|
link: validated,
|
|
3905
4498
|
payloadText: String(payloadText || "")
|
|
3906
4499
|
};
|
|
3907
|
-
|
|
4500
|
+
logger11.info(`\u5019\u9009\u94FE\u63A5\u5DF2\u786E\u8BA4: source=${source}, link=${validated}`);
|
|
3908
4501
|
return true;
|
|
3909
4502
|
};
|
|
3910
4503
|
const resolveResponseCandidate = (responseText) => {
|
|
@@ -3939,7 +4532,7 @@ var Share = {
|
|
|
3939
4532
|
try {
|
|
3940
4533
|
await monitor.stop();
|
|
3941
4534
|
} catch (error) {
|
|
3942
|
-
|
|
4535
|
+
logger11.warning(`\u505C\u6B62 DOM \u76D1\u542C\u5931\u8D25: ${error instanceof Error ? error.message : String(error)}`);
|
|
3943
4536
|
}
|
|
3944
4537
|
};
|
|
3945
4538
|
const onResponse = async (response) => {
|
|
@@ -3952,29 +4545,29 @@ var Share = {
|
|
|
3952
4545
|
stats.responseSampleUrls.push(url);
|
|
3953
4546
|
}
|
|
3954
4547
|
if (stats.responseObserved <= 5) {
|
|
3955
|
-
|
|
4548
|
+
logger11.info(`\u63A5\u53E3\u54CD\u5E94\u91C7\u6837(${stats.responseObserved}): ${url}`);
|
|
3956
4549
|
}
|
|
3957
4550
|
if (!apiMatchers.some((matcher) => url.includes(matcher))) return;
|
|
3958
4551
|
stats.responseMatched += 1;
|
|
3959
4552
|
stats.lastMatchedUrl = url;
|
|
3960
|
-
|
|
4553
|
+
logger11.info(`\u63A5\u53E3\u547D\u4E2D\u5339\u914D(${stats.responseMatched}): ${url}`);
|
|
3961
4554
|
const text = await response.text();
|
|
3962
4555
|
const hit = resolveResponseCandidate(text);
|
|
3963
4556
|
if (!hit?.link) {
|
|
3964
4557
|
if (stats.responseMatched <= 3) {
|
|
3965
|
-
|
|
4558
|
+
logger11.info(`\u63A5\u53E3\u89E3\u6790\u5B8C\u6210\u4F46\u672A\u63D0\u53D6\u5230\u5206\u4EAB\u94FE\u63A5: payloadSize=${text.length}`);
|
|
3966
4559
|
}
|
|
3967
4560
|
return;
|
|
3968
4561
|
}
|
|
3969
4562
|
stats.responseResolved += 1;
|
|
3970
|
-
|
|
4563
|
+
logger11.success("captureLink.responseHit", `\u63A5\u53E3\u547D\u4E2D\u5206\u4EAB\u94FE\u63A5: url=${url}, link=${hit.link}`);
|
|
3971
4564
|
setCandidate("response", hit.link, hit.payloadText);
|
|
3972
4565
|
} catch (error) {
|
|
3973
|
-
|
|
4566
|
+
logger11.warning(`\u63A5\u53E3\u54CD\u5E94\u5904\u7406\u5F02\u5E38: ${error instanceof Error ? error.message : String(error)}`);
|
|
3974
4567
|
}
|
|
3975
4568
|
};
|
|
3976
4569
|
if (share.mode === "dom") {
|
|
3977
|
-
|
|
4570
|
+
logger11.info("\u5F53\u524D\u4E3A DOM \u6A21\u5F0F\uFF0C\u4EC5\u542F\u7528 DOM \u76D1\u542C");
|
|
3978
4571
|
domMonitor = await createDomShareMonitor(page, {
|
|
3979
4572
|
prefix: share.prefix,
|
|
3980
4573
|
selectors: domSelectors,
|
|
@@ -3989,14 +4582,14 @@ var Share = {
|
|
|
3989
4582
|
});
|
|
3990
4583
|
}
|
|
3991
4584
|
if (share.mode === "response") {
|
|
3992
|
-
|
|
4585
|
+
logger11.info(`\u5F53\u524D\u4E3A\u63A5\u53E3\u6A21\u5F0F\uFF0C\u6302\u8F7D response \u76D1\u542C: apiMatchers=${toJsonInline(apiMatchers, 160)}`);
|
|
3993
4586
|
page.on("response", onResponse);
|
|
3994
4587
|
}
|
|
3995
4588
|
const deadline = Date.now() + timeoutMs;
|
|
3996
4589
|
const getRemainingMs = () => Math.max(0, deadline - Date.now());
|
|
3997
4590
|
try {
|
|
3998
4591
|
const actionTimeout = getRemainingMs();
|
|
3999
|
-
|
|
4592
|
+
logger11.start("captureLink.performActions", `\u6267\u884C\u52A8\u4F5C\u9884\u7B97=${actionTimeout}ms`);
|
|
4000
4593
|
if (actionTimeout > 0) {
|
|
4001
4594
|
let timer = null;
|
|
4002
4595
|
let actionError = null;
|
|
@@ -4010,21 +4603,21 @@ var Share = {
|
|
|
4010
4603
|
const actionResult = await Promise.race([actionPromise, timeoutPromise]);
|
|
4011
4604
|
if (timer) clearTimeout(timer);
|
|
4012
4605
|
if (actionResult === "__ACTION_ERROR__") {
|
|
4013
|
-
|
|
4606
|
+
logger11.fail("captureLink.performActions", actionError);
|
|
4014
4607
|
throw actionError;
|
|
4015
4608
|
}
|
|
4016
4609
|
if (actionResult === "__ACTION_TIMEOUT__") {
|
|
4017
4610
|
stats.actionTimedOut = true;
|
|
4018
|
-
|
|
4611
|
+
logger11.warning(`performActions \u5DF2\u8D85\u65F6 (${actionTimeout}ms)\uFF0C\u52A8\u4F5C\u53EF\u80FD\u4ECD\u5728\u5F02\u6B65\u6267\u884C`);
|
|
4019
4612
|
} else {
|
|
4020
|
-
|
|
4613
|
+
logger11.success("captureLink.performActions", "\u6267\u884C\u52A8\u4F5C\u5B8C\u6210");
|
|
4021
4614
|
}
|
|
4022
4615
|
}
|
|
4023
4616
|
let nextProgressLogTs = Date.now() + 3e3;
|
|
4024
4617
|
while (true) {
|
|
4025
4618
|
const selected = share.mode === "dom" ? candidates.dom : candidates.response;
|
|
4026
4619
|
if (selected?.link) {
|
|
4027
|
-
|
|
4620
|
+
logger11.success("captureLink", `\u6355\u83B7\u6210\u529F: source=${share.mode}, link=${selected.link}`);
|
|
4028
4621
|
return {
|
|
4029
4622
|
link: selected.link,
|
|
4030
4623
|
payloadText: selected.payloadText,
|
|
@@ -4036,7 +4629,7 @@ var Share = {
|
|
|
4036
4629
|
if (remaining <= 0) break;
|
|
4037
4630
|
const now = Date.now();
|
|
4038
4631
|
if (now >= nextProgressLogTs) {
|
|
4039
|
-
|
|
4632
|
+
logger11.info(
|
|
4040
4633
|
`captureLink \u7B49\u5F85\u4E2D: remaining=${remaining}ms, domMutationCount=${stats.domMutationCount}, responseMatched=${stats.responseMatched}`
|
|
4041
4634
|
);
|
|
4042
4635
|
nextProgressLogTs = now + 5e3;
|
|
@@ -4044,11 +4637,11 @@ var Share = {
|
|
|
4044
4637
|
await delay2(Math.max(0, Math.min(DEFAULT_POLL_INTERVAL_MS, remaining)));
|
|
4045
4638
|
}
|
|
4046
4639
|
if (share.mode === "response" && stats.responseMatched === 0) {
|
|
4047
|
-
|
|
4640
|
+
logger11.warning(
|
|
4048
4641
|
`\u63A5\u53E3\u76D1\u542C\u672A\u547D\u4E2D: apiMatchers=${toJsonInline(apiMatchers, 220)}, \u54CD\u5E94\u6837\u672CURLs=${toJsonInline(stats.responseSampleUrls, 420)}`
|
|
4049
4642
|
);
|
|
4050
4643
|
}
|
|
4051
|
-
|
|
4644
|
+
logger11.warning(
|
|
4052
4645
|
`captureLink \u8D85\u65F6\u672A\u62FF\u5230\u94FE\u63A5: mode=${share.mode}, actionTimedOut=${stats.actionTimedOut}, domMutationCount=${stats.domMutationCount}, responseObserved=${stats.responseObserved}, responseMatched=${stats.responseMatched}, lastMatchedUrl=${stats.lastMatchedUrl || "none"}`
|
|
4053
4646
|
);
|
|
4054
4647
|
return {
|
|
@@ -4060,7 +4653,7 @@ var Share = {
|
|
|
4060
4653
|
} finally {
|
|
4061
4654
|
if (share.mode === "response") {
|
|
4062
4655
|
page.off("response", onResponse);
|
|
4063
|
-
|
|
4656
|
+
logger11.info("response \u76D1\u542C\u5DF2\u5378\u8F7D");
|
|
4064
4657
|
}
|
|
4065
4658
|
await stopDomMonitor();
|
|
4066
4659
|
}
|
|
@@ -4146,13 +4739,14 @@ var usePlaywrightToolKit = () => {
|
|
|
4146
4739
|
LiveView,
|
|
4147
4740
|
Constants: constants_exports,
|
|
4148
4741
|
Utils,
|
|
4742
|
+
RuntimeEnv,
|
|
4149
4743
|
Captcha,
|
|
4150
|
-
Sse,
|
|
4151
4744
|
Errors: errors_exports,
|
|
4152
4745
|
Mutation,
|
|
4153
4746
|
Display,
|
|
4154
4747
|
Logger,
|
|
4155
4748
|
Share,
|
|
4749
|
+
ByPass,
|
|
4156
4750
|
$Internals: { LOG_TEMPLATES, stripAnsi }
|
|
4157
4751
|
};
|
|
4158
4752
|
};
|