octoparse-cli 0.1.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +271 -0
- package/RUNTIME_SECURITY_NOTICE.txt +41 -0
- package/SECURITY.md +24 -0
- package/dist/cli/args.js +34 -0
- package/dist/cli/help.js +214 -0
- package/dist/cli/output.js +39 -0
- package/dist/commands/auth.js +283 -0
- package/dist/commands/capabilities.js +118 -0
- package/dist/commands/cloud.js +241 -0
- package/dist/commands/data.js +220 -0
- package/dist/commands/doctor.js +73 -0
- package/dist/commands/env.js +63 -0
- package/dist/commands/local.js +251 -0
- package/dist/commands/run.js +622 -0
- package/dist/commands/runs.js +171 -0
- package/dist/commands/task.js +101 -0
- package/dist/index.js +133 -0
- package/dist/runtime/account-capabilities.js +71 -0
- package/dist/runtime/api-client.js +290 -0
- package/dist/runtime/artifacts.js +33 -0
- package/dist/runtime/auth.js +94 -0
- package/dist/runtime/bridge-hub.js +173 -0
- package/dist/runtime/client-headers.js +23 -0
- package/dist/runtime/cloud-data.js +75 -0
- package/dist/runtime/config.js +48 -0
- package/dist/runtime/data-exporter.js +267 -0
- package/dist/runtime/engine-host.js +449 -0
- package/dist/runtime/local-runs.js +92 -0
- package/dist/runtime/naming.js +13 -0
- package/dist/runtime/run-control.js +363 -0
- package/dist/runtime/run-services.js +380 -0
- package/dist/runtime/security-notice.js +78 -0
- package/dist/runtime/task-definition-provider.js +282 -0
- package/dist/types.js +4 -0
- package/examples/minimal-task.json +6 -0
- package/examples/navigate-example-task.json +6 -0
- package/node_modules/@octopus/bpmn/index.js +3 -0
- package/node_modules/@octopus/bpmn/lib/Context.js +245 -0
- package/node_modules/@octopus/bpmn/lib/Definition.js +258 -0
- package/node_modules/@octopus/bpmn/lib/Engine.js +275 -0
- package/node_modules/@octopus/bpmn/lib/PrematureStopError.js +7 -0
- package/node_modules/@octopus/bpmn/lib/activities/Activity.js +202 -0
- package/node_modules/@octopus/bpmn/lib/activities/BaseProcess.js +308 -0
- package/node_modules/@octopus/bpmn/lib/activities/BaseTask.js +145 -0
- package/node_modules/@octopus/bpmn/lib/activities/BoundaryEvent.js +12 -0
- package/node_modules/@octopus/bpmn/lib/activities/Dummy.js +10 -0
- package/node_modules/@octopus/bpmn/lib/activities/EventDefinition.js +99 -0
- package/node_modules/@octopus/bpmn/lib/activities/Flow.js +52 -0
- package/node_modules/@octopus/bpmn/lib/activities/Form.js +67 -0
- package/node_modules/@octopus/bpmn/lib/activities/InputOutput.js +53 -0
- package/node_modules/@octopus/bpmn/lib/activities/IntermediateCatchEvent.js +12 -0
- package/node_modules/@octopus/bpmn/lib/activities/MessageFlow.js +19 -0
- package/node_modules/@octopus/bpmn/lib/activities/MultiInstanceLoopCharacteristics.js +160 -0
- package/node_modules/@octopus/bpmn/lib/activities/Properties.js +27 -0
- package/node_modules/@octopus/bpmn/lib/activities/SequenceFlow.js +56 -0
- package/node_modules/@octopus/bpmn/lib/activities/ServiceConnector.js +71 -0
- package/node_modules/@octopus/bpmn/lib/context-helper.js +198 -0
- package/node_modules/@octopus/bpmn/lib/events/EndEvent.js +22 -0
- package/node_modules/@octopus/bpmn/lib/events/ErrorEvent.js +41 -0
- package/node_modules/@octopus/bpmn/lib/events/MessageEvent.js +19 -0
- package/node_modules/@octopus/bpmn/lib/events/StartEvent.js +55 -0
- package/node_modules/@octopus/bpmn/lib/events/TimerEvent.js +75 -0
- package/node_modules/@octopus/bpmn/lib/expressions.js +41 -0
- package/node_modules/@octopus/bpmn/lib/gateways/ExclusiveGateway.js +86 -0
- package/node_modules/@octopus/bpmn/lib/gateways/InclusiveGateway.js +56 -0
- package/node_modules/@octopus/bpmn/lib/gateways/ParallelGateway.js +195 -0
- package/node_modules/@octopus/bpmn/lib/getPropertyValue.js +83 -0
- package/node_modules/@octopus/bpmn/lib/index.js +6 -0
- package/node_modules/@octopus/bpmn/lib/mapper.js +55 -0
- package/node_modules/@octopus/bpmn/lib/parameter.js +119 -0
- package/node_modules/@octopus/bpmn/lib/script-helper.js +45 -0
- package/node_modules/@octopus/bpmn/lib/tasks/ManualTask.js +31 -0
- package/node_modules/@octopus/bpmn/lib/tasks/ReceiveTask.js +31 -0
- package/node_modules/@octopus/bpmn/lib/tasks/ScriptTask.js +35 -0
- package/node_modules/@octopus/bpmn/lib/tasks/SendTask.js +16 -0
- package/node_modules/@octopus/bpmn/lib/tasks/ServiceTask.js +68 -0
- package/node_modules/@octopus/bpmn/lib/tasks/SubProcess.js +17 -0
- package/node_modules/@octopus/bpmn/lib/tasks/Task.js +16 -0
- package/node_modules/@octopus/bpmn/lib/tasks/UserTask.js +47 -0
- package/node_modules/@octopus/bpmn/lib/transformer.js +13 -0
- package/node_modules/@octopus/bpmn/lib/validation.js +111 -0
- package/node_modules/@octopus/bpmn/package.json +17 -0
- package/node_modules/@octopus/bpmn/types/bpmn.d.ts +85 -0
- package/node_modules/@octopus/engine/README.md +370 -0
- package/node_modules/@octopus/engine/dist/actions/BackPreWebPageAction.d.ts +4 -0
- package/node_modules/@octopus/engine/dist/actions/BackPreWebPageAction.js +1 -0
- package/node_modules/@octopus/engine/dist/actions/BaseAction.d.ts +339 -0
- package/node_modules/@octopus/engine/dist/actions/BaseAction.js +1559 -0
- package/node_modules/@octopus/engine/dist/actions/BranchAction.d.ts +9 -0
- package/node_modules/@octopus/engine/dist/actions/BranchAction.js +1 -0
- package/node_modules/@octopus/engine/dist/actions/ClickAction.d.ts +22 -0
- package/node_modules/@octopus/engine/dist/actions/ClickAction.js +1 -0
- package/node_modules/@octopus/engine/dist/actions/ConditionAction.d.ts +4 -0
- package/node_modules/@octopus/engine/dist/actions/ConditionAction.js +1 -0
- package/node_modules/@octopus/engine/dist/actions/EmptyAction.d.ts +4 -0
- package/node_modules/@octopus/engine/dist/actions/EmptyAction.js +12 -0
- package/node_modules/@octopus/engine/dist/actions/EnterCaptchaAction.d.ts +28 -0
- package/node_modules/@octopus/engine/dist/actions/EnterCaptchaAction.js +1 -0
- package/node_modules/@octopus/engine/dist/actions/EnterTextAction.d.ts +20 -0
- package/node_modules/@octopus/engine/dist/actions/EnterTextAction.js +1 -0
- package/node_modules/@octopus/engine/dist/actions/ExtractDataAction.d.ts +40 -0
- package/node_modules/@octopus/engine/dist/actions/ExtractDataAction.js +1 -0
- package/node_modules/@octopus/engine/dist/actions/LoopAction.d.ts +41 -0
- package/node_modules/@octopus/engine/dist/actions/LoopAction.js +526 -0
- package/node_modules/@octopus/engine/dist/actions/LoopStartAction.d.ts +47 -0
- package/node_modules/@octopus/engine/dist/actions/LoopStartAction.js +607 -0
- package/node_modules/@octopus/engine/dist/actions/MouseOverAction.d.ts +8 -0
- package/node_modules/@octopus/engine/dist/actions/MouseOverAction.js +34 -0
- package/node_modules/@octopus/engine/dist/actions/NavigateAction.d.ts +38 -0
- package/node_modules/@octopus/engine/dist/actions/NavigateAction.js +535 -0
- package/node_modules/@octopus/engine/dist/actions/SwitchComboAction.d.ts +13 -0
- package/node_modules/@octopus/engine/dist/actions/SwitchComboAction.js +69 -0
- package/node_modules/@octopus/engine/dist/browser.d.ts +17 -0
- package/node_modules/@octopus/engine/dist/browser.js +157 -0
- package/node_modules/@octopus/engine/dist/browserProxy.d.ts +90 -0
- package/node_modules/@octopus/engine/dist/browserProxy.js +1 -0
- package/node_modules/@octopus/engine/dist/configs/BaseConfig.d.ts +20 -0
- package/node_modules/@octopus/engine/dist/configs/BaseConfig.js +88 -0
- package/node_modules/@octopus/engine/dist/configs/BranchConfig.d.ts +7 -0
- package/node_modules/@octopus/engine/dist/configs/BranchConfig.js +1 -0
- package/node_modules/@octopus/engine/dist/configs/ClickConfig.d.ts +36 -0
- package/node_modules/@octopus/engine/dist/configs/ClickConfig.js +65 -0
- package/node_modules/@octopus/engine/dist/configs/EnterCaptchaConfig.d.ts +19 -0
- package/node_modules/@octopus/engine/dist/configs/EnterCaptchaConfig.js +25 -0
- package/node_modules/@octopus/engine/dist/configs/EnterTextConfig.d.ts +24 -0
- package/node_modules/@octopus/engine/dist/configs/EnterTextConfig.js +36 -0
- package/node_modules/@octopus/engine/dist/configs/ExtractDataConfig.d.ts +12 -0
- package/node_modules/@octopus/engine/dist/configs/ExtractDataConfig.js +1 -0
- package/node_modules/@octopus/engine/dist/configs/LoopConfig.d.ts +25 -0
- package/node_modules/@octopus/engine/dist/configs/LoopConfig.js +40 -0
- package/node_modules/@octopus/engine/dist/configs/LoopStartConfig.d.ts +4 -0
- package/node_modules/@octopus/engine/dist/configs/LoopStartConfig.js +12 -0
- package/node_modules/@octopus/engine/dist/configs/MouseOverConfig.d.ts +8 -0
- package/node_modules/@octopus/engine/dist/configs/MouseOverConfig.js +15 -0
- package/node_modules/@octopus/engine/dist/configs/NavigateConfig.d.ts +41 -0
- package/node_modules/@octopus/engine/dist/configs/NavigateConfig.js +121 -0
- package/node_modules/@octopus/engine/dist/configs/SwitchComboConfig.d.ts +8 -0
- package/node_modules/@octopus/engine/dist/configs/SwitchComboConfig.js +15 -0
- package/node_modules/@octopus/engine/dist/enums/index.d.ts +419 -0
- package/node_modules/@octopus/engine/dist/enums/index.js +314 -0
- package/node_modules/@octopus/engine/dist/extension/BrowserWebSocketTransport-D_zAGZMQ.js +1 -0
- package/node_modules/@octopus/engine/dist/extension/LaunchOptions-DxvePrV4.js +6 -0
- package/node_modules/@octopus/engine/dist/extension/NodeWebSocketTransport-BTgRVB7Z.js +6 -0
- package/node_modules/@octopus/engine/dist/extension/background.js +396 -0
- package/node_modules/@octopus/engine/dist/extension/bidi-C_GIZ8Uz.js +131 -0
- package/node_modules/@octopus/engine/dist/extension/manifest.json +27 -0
- package/node_modules/@octopus/engine/dist/extension/src/content/anti-detection.js +1 -0
- package/node_modules/@octopus/engine/dist/extension-bridge/BaseExtensionBridge.d.ts +21 -0
- package/node_modules/@octopus/engine/dist/extension-bridge/BaseExtensionBridge.js +117 -0
- package/node_modules/@octopus/engine/dist/extension-bridge/SessionExtensionBridge.d.ts +17 -0
- package/node_modules/@octopus/engine/dist/extension-bridge/SessionExtensionBridge.js +29 -0
- package/node_modules/@octopus/engine/dist/extension-bridge/index.d.ts +2 -0
- package/node_modules/@octopus/engine/dist/extension-bridge/index.js +5 -0
- package/node_modules/@octopus/engine/dist/extension-bridge/types.d.ts +159 -0
- package/node_modules/@octopus/engine/dist/extension-bridge/types.js +5 -0
- package/node_modules/@octopus/engine/dist/extensions/ublock-origin/uBlock0.chromium.tar.xz +0 -0
- package/node_modules/@octopus/engine/dist/extensions/ublock-origin-lite/uBOLite.chromium.tar.xz +0 -0
- package/node_modules/@octopus/engine/dist/index.d.ts +169 -0
- package/node_modules/@octopus/engine/dist/index.js +1 -0
- package/node_modules/@octopus/engine/dist/models/actionItem.d.ts +16 -0
- package/node_modules/@octopus/engine/dist/models/actionItem.js +15 -0
- package/node_modules/@octopus/engine/dist/models/conditionCheckArgs.d.ts +11 -0
- package/node_modules/@octopus/engine/dist/models/conditionCheckArgs.js +11 -0
- package/node_modules/@octopus/engine/dist/models/customizeCookie.d.ts +14 -0
- package/node_modules/@octopus/engine/dist/models/customizeCookie.js +6 -0
- package/node_modules/@octopus/engine/dist/models/downloadFileConfig.d.ts +17 -0
- package/node_modules/@octopus/engine/dist/models/downloadFileConfig.js +26 -0
- package/node_modules/@octopus/engine/dist/models/elementNotFoundArgs.d.ts +8 -0
- package/node_modules/@octopus/engine/dist/models/elementNotFoundArgs.js +12 -0
- package/node_modules/@octopus/engine/dist/models/elementNotFoundError.d.ts +2 -0
- package/node_modules/@octopus/engine/dist/models/elementNotFoundError.js +6 -0
- package/node_modules/@octopus/engine/dist/models/extractItem.d.ts +37 -0
- package/node_modules/@octopus/engine/dist/models/extractItem.js +35 -0
- package/node_modules/@octopus/engine/dist/models/extractTemplate.d.ts +11 -0
- package/node_modules/@octopus/engine/dist/models/extractTemplate.js +48 -0
- package/node_modules/@octopus/engine/dist/models/extractTextItem.d.ts +10 -0
- package/node_modules/@octopus/engine/dist/models/extractTextItem.js +17 -0
- package/node_modules/@octopus/engine/dist/models/globalConfig.d.ts +5 -0
- package/node_modules/@octopus/engine/dist/models/globalConfig.js +1 -0
- package/node_modules/@octopus/engine/dist/models/httpHeader.d.ts +4 -0
- package/node_modules/@octopus/engine/dist/models/httpHeader.js +10 -0
- package/node_modules/@octopus/engine/dist/models/operation.d.ts +27 -0
- package/node_modules/@octopus/engine/dist/models/operation.js +242 -0
- package/node_modules/@octopus/engine/dist/models/retryCondition.d.ts +7 -0
- package/node_modules/@octopus/engine/dist/models/retryCondition.js +10 -0
- package/node_modules/@octopus/engine/dist/models/task.d.ts +89 -0
- package/node_modules/@octopus/engine/dist/models/task.js +120 -0
- package/node_modules/@octopus/engine/dist/models/trigger.d.ts +66 -0
- package/node_modules/@octopus/engine/dist/models/trigger.js +117 -0
- package/node_modules/@octopus/engine/dist/package.json +26 -0
- package/node_modules/@octopus/engine/dist/public-types.d.ts +13 -0
- package/node_modules/@octopus/engine/dist/public-types.js +2 -0
- package/node_modules/@octopus/engine/dist/settings.d.ts +41 -0
- package/node_modules/@octopus/engine/dist/settings.js +20 -0
- package/node_modules/@octopus/engine/dist/solvers/captcha/ClickCaptchaSolver.d.ts +6 -0
- package/node_modules/@octopus/engine/dist/solvers/captcha/ClickCaptchaSolver.js +1 -0
- package/node_modules/@octopus/engine/dist/solvers/captcha/HCaptchaSolver.d.ts +4 -0
- package/node_modules/@octopus/engine/dist/solvers/captcha/HCaptchaSolver.js +73 -0
- package/node_modules/@octopus/engine/dist/solvers/captcha/ImageCaptchaSolver.d.ts +2 -0
- package/node_modules/@octopus/engine/dist/solvers/captcha/ImageCaptchaSolver.js +74 -0
- package/node_modules/@octopus/engine/dist/solvers/captcha/RecaptchaSolver.d.ts +9 -0
- package/node_modules/@octopus/engine/dist/solvers/captcha/RecaptchaSolver.js +371 -0
- package/node_modules/@octopus/engine/dist/solvers/captcha/SliderCaptchaSolver.d.ts +6 -0
- package/node_modules/@octopus/engine/dist/solvers/captcha/SliderCaptchaSolver.js +184 -0
- package/node_modules/@octopus/engine/dist/solvers/captcha/SlidingTrajectory.d.ts +50 -0
- package/node_modules/@octopus/engine/dist/solvers/captcha/SlidingTrajectory.js +125 -0
- package/node_modules/@octopus/engine/dist/solvers/captcha/types.d.ts +68 -0
- package/node_modules/@octopus/engine/dist/solvers/captcha/types.js +34 -0
- package/node_modules/@octopus/engine/dist/solvers/captcha/utils.d.ts +2 -0
- package/node_modules/@octopus/engine/dist/solvers/captcha/utils.js +15 -0
- package/node_modules/@octopus/engine/dist/translator/actionFactory.d.ts +6 -0
- package/node_modules/@octopus/engine/dist/translator/actionFactory.js +1 -0
- package/node_modules/@octopus/engine/dist/translator/activityTypeEnum.d.ts +22 -0
- package/node_modules/@octopus/engine/dist/translator/activityTypeEnum.js +1 -0
- package/node_modules/@octopus/engine/dist/translator/backPreWebPageAction.d.ts +5 -0
- package/node_modules/@octopus/engine/dist/translator/backPreWebPageAction.js +1 -0
- package/node_modules/@octopus/engine/dist/translator/baseAction.d.ts +31 -0
- package/node_modules/@octopus/engine/dist/translator/baseAction.js +1 -0
- package/node_modules/@octopus/engine/dist/translator/breakActivity.d.ts +5 -0
- package/node_modules/@octopus/engine/dist/translator/breakActivity.js +1 -0
- package/node_modules/@octopus/engine/dist/translator/clickAction.d.ts +5 -0
- package/node_modules/@octopus/engine/dist/translator/clickAction.js +1 -0
- package/node_modules/@octopus/engine/dist/translator/completeWF.d.ts +5 -0
- package/node_modules/@octopus/engine/dist/translator/completeWF.js +1 -0
- package/node_modules/@octopus/engine/dist/translator/conditionAction.d.ts +6 -0
- package/node_modules/@octopus/engine/dist/translator/conditionAction.js +1 -0
- package/node_modules/@octopus/engine/dist/translator/emptyAction.d.ts +5 -0
- package/node_modules/@octopus/engine/dist/translator/emptyAction.js +1 -0
- package/node_modules/@octopus/engine/dist/translator/enterCapachaAction.d.ts +5 -0
- package/node_modules/@octopus/engine/dist/translator/enterCapachaAction.js +1 -0
- package/node_modules/@octopus/engine/dist/translator/enterTextAction.d.ts +5 -0
- package/node_modules/@octopus/engine/dist/translator/enterTextAction.js +1 -0
- package/node_modules/@octopus/engine/dist/translator/extractDataAction.d.ts +13 -0
- package/node_modules/@octopus/engine/dist/translator/extractDataAction.js +1 -0
- package/node_modules/@octopus/engine/dist/translator/loopAction.d.ts +5 -0
- package/node_modules/@octopus/engine/dist/translator/loopAction.js +1 -0
- package/node_modules/@octopus/engine/dist/translator/mouseOverAction.d.ts +5 -0
- package/node_modules/@octopus/engine/dist/translator/mouseOverAction.js +19 -0
- package/node_modules/@octopus/engine/dist/translator/navigateAction.d.ts +5 -0
- package/node_modules/@octopus/engine/dist/translator/navigateAction.js +117 -0
- package/node_modules/@octopus/engine/dist/translator/rootAction.d.ts +6 -0
- package/node_modules/@octopus/engine/dist/translator/rootAction.js +80 -0
- package/node_modules/@octopus/engine/dist/translator/switchCombo2Action.d.ts +5 -0
- package/node_modules/@octopus/engine/dist/translator/switchCombo2Action.js +19 -0
- package/node_modules/@octopus/engine/dist/translator/translator.d.ts +1 -0
- package/node_modules/@octopus/engine/dist/translator/translator.js +36 -0
- package/node_modules/@octopus/engine/dist/type.d.ts +25 -0
- package/node_modules/@octopus/engine/dist/type.js +2 -0
- package/node_modules/@octopus/engine/dist/types/browser.d.ts +191 -0
- package/node_modules/@octopus/engine/dist/types/browser.js +1 -0
- package/node_modules/@octopus/engine/dist/types/browserManager.d.ts +41 -0
- package/node_modules/@octopus/engine/dist/types/browserManager.js +1 -0
- package/node_modules/@octopus/engine/dist/types/index.d.ts +40 -0
- package/node_modules/@octopus/engine/dist/types/index.js +2 -0
- package/node_modules/@octopus/engine/dist/types/plugin.d.ts +29 -0
- package/node_modules/@octopus/engine/dist/types/plugin.js +2 -0
- package/node_modules/@octopus/engine/dist/utils/AsyncEmitter.d.ts +15 -0
- package/node_modules/@octopus/engine/dist/utils/AsyncEmitter.js +1 -0
- package/node_modules/@octopus/engine/dist/utils/DataStore.d.ts +58 -0
- package/node_modules/@octopus/engine/dist/utils/DataStore.js +1 -0
- package/node_modules/@octopus/engine/dist/utils/DateTimeFormatHelper.d.ts +22 -0
- package/node_modules/@octopus/engine/dist/utils/DateTimeFormatHelper.js +173 -0
- package/node_modules/@octopus/engine/dist/utils/FileDownloader.d.ts +108 -0
- package/node_modules/@octopus/engine/dist/utils/FileDownloader.js +1 -0
- package/node_modules/@octopus/engine/dist/utils/HttpRequester.d.ts +43 -0
- package/node_modules/@octopus/engine/dist/utils/HttpRequester.js +174 -0
- package/node_modules/@octopus/engine/dist/utils/JsonParser.d.ts +95 -0
- package/node_modules/@octopus/engine/dist/utils/JsonParser.js +439 -0
- package/node_modules/@octopus/engine/dist/utils/Operations.d.ts +27 -0
- package/node_modules/@octopus/engine/dist/utils/Operations.js +115 -0
- package/node_modules/@octopus/engine/dist/utils/index.d.ts +28 -0
- package/node_modules/@octopus/engine/dist/utils/index.js +356 -0
- package/node_modules/@octopus/engine/package.json +58 -0
- package/package.json +79 -0
- package/schemas/capabilities-v1.schema.json +234 -0
- package/schemas/detached-bootstrap-v1.schema.json +42 -0
- package/schemas/json-envelope-v1.schema.json +39 -0
- package/schemas/run-event-v1.schema.json +47 -0
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import NavigateConfig from '../configs/NavigateConfig';
|
|
2
|
+
import BaseAction from './BaseAction';
|
|
3
|
+
export default class NavigateAction extends BaseAction {
|
|
4
|
+
config: NavigateConfig;
|
|
5
|
+
private page;
|
|
6
|
+
private url;
|
|
7
|
+
private retryTimes;
|
|
8
|
+
private retryIsNeed;
|
|
9
|
+
executeTimeMax: number;
|
|
10
|
+
get isAutoCloudflare(): boolean;
|
|
11
|
+
protected parseConfig(config: any): NavigateConfig;
|
|
12
|
+
/** 设置 Turnstile 拦截器 */
|
|
13
|
+
private setupTurnstileInterceptor;
|
|
14
|
+
private isTransientRiskControlStatus;
|
|
15
|
+
/**
|
|
16
|
+
* 某些站点会先返回 412/403 风控挑战页,再由页面脚本自动刷新为 200 正常页。
|
|
17
|
+
* 独立浏览器能看到最终页面,但如果只看首个 goto 响应会被误判成“打开网页失败”。
|
|
18
|
+
*/
|
|
19
|
+
private resolveFinalNavigationResponse;
|
|
20
|
+
private getErrorMessage;
|
|
21
|
+
private execute;
|
|
22
|
+
/** 清理当前网站的缓存 */
|
|
23
|
+
private clearCache;
|
|
24
|
+
/** 设置已保存的 Cookies */
|
|
25
|
+
private setCookies;
|
|
26
|
+
/** 判定是否需要重试 */
|
|
27
|
+
private checkRetry;
|
|
28
|
+
private parseCookies;
|
|
29
|
+
start(): void;
|
|
30
|
+
wait(): Promise<void>;
|
|
31
|
+
onPageLoading(url: string): void;
|
|
32
|
+
onPageLoaded(url: string): Promise<void>;
|
|
33
|
+
retry(): Promise<true>;
|
|
34
|
+
fixCloudflare(): Promise<void>;
|
|
35
|
+
judgeHasTurnstile(): Promise<unknown>;
|
|
36
|
+
onJsonResponse(): Promise<void>;
|
|
37
|
+
onJsonError(): void;
|
|
38
|
+
}
|
|
@@ -0,0 +1,535 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
const NavigateConfig_1 = __importDefault(require("../configs/NavigateConfig"));
|
|
7
|
+
const enums_1 = require("../enums");
|
|
8
|
+
const conditionCheckArgs_1 = require("../models/conditionCheckArgs");
|
|
9
|
+
const utils_1 = require("../utils");
|
|
10
|
+
const BaseAction_1 = __importDefault(require("./BaseAction"));
|
|
11
|
+
class NavigateAction extends BaseAction_1.default {
|
|
12
|
+
constructor() {
|
|
13
|
+
super(...arguments);
|
|
14
|
+
this.url = '';
|
|
15
|
+
this.retryTimes = 0;
|
|
16
|
+
this.retryIsNeed = false;
|
|
17
|
+
this.executeTimeMax = -1;
|
|
18
|
+
}
|
|
19
|
+
get isAutoCloudflare() {
|
|
20
|
+
var _a, _b;
|
|
21
|
+
return (_b = (_a = this.browserProxy.brokerSettings) === null || _a === void 0 ? void 0 : _a.captchaSettings) === null || _b === void 0 ? void 0 : _b.isAutoCloudflare;
|
|
22
|
+
}
|
|
23
|
+
parseConfig(config) {
|
|
24
|
+
return new NavigateConfig_1.default(config, this.activity.activity.extensionElements);
|
|
25
|
+
}
|
|
26
|
+
/** 设置 Turnstile 拦截器 */
|
|
27
|
+
async setupTurnstileInterceptor() {
|
|
28
|
+
await this.page.evaluateOnNewDocument(`
|
|
29
|
+
let i = setInterval(() => {
|
|
30
|
+
if (window.turnstile) {
|
|
31
|
+
clearInterval(i)
|
|
32
|
+
window.turnstile.render = (a, b) => {
|
|
33
|
+
let params = {
|
|
34
|
+
sitekey: b.sitekey,
|
|
35
|
+
pageurl: window.location.href,
|
|
36
|
+
data: b.cData,
|
|
37
|
+
pagedata: b.chlPageData,
|
|
38
|
+
action: b.action,
|
|
39
|
+
userAgent: navigator.userAgent,
|
|
40
|
+
json: 1
|
|
41
|
+
}
|
|
42
|
+
window.turnstileFreezeParams = params;
|
|
43
|
+
window.turnstileCallback = b.callback;
|
|
44
|
+
return
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
}, 1)
|
|
48
|
+
`);
|
|
49
|
+
}
|
|
50
|
+
isTransientRiskControlStatus(statusCode) {
|
|
51
|
+
return statusCode === 403 || statusCode === 412 || statusCode === 429;
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* 某些站点会先返回 412/403 风控挑战页,再由页面脚本自动刷新为 200 正常页。
|
|
55
|
+
* 独立浏览器能看到最终页面,但如果只看首个 goto 响应会被误判成“打开网页失败”。
|
|
56
|
+
*/
|
|
57
|
+
async resolveFinalNavigationResponse(response) {
|
|
58
|
+
var _a;
|
|
59
|
+
const statusCode = (_a = response === null || response === void 0 ? void 0 : response.status) === null || _a === void 0 ? void 0 : _a.call(response);
|
|
60
|
+
if (!response || response.ok() || !this.isTransientRiskControlStatus(statusCode)) {
|
|
61
|
+
return response;
|
|
62
|
+
}
|
|
63
|
+
try {
|
|
64
|
+
const followUpTimeout = Math.min(15000, Math.max(3000, (this.config.timeOut || 60) * 1000));
|
|
65
|
+
const followUpResponse = await this.page.waitForNavigation({
|
|
66
|
+
waitUntil: 'load',
|
|
67
|
+
timeout: followUpTimeout
|
|
68
|
+
});
|
|
69
|
+
return followUpResponse || response;
|
|
70
|
+
}
|
|
71
|
+
catch {
|
|
72
|
+
return response;
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
getErrorMessage(error) {
|
|
76
|
+
if (error instanceof Error) {
|
|
77
|
+
return error.message || error.toString();
|
|
78
|
+
}
|
|
79
|
+
return String(error);
|
|
80
|
+
}
|
|
81
|
+
async execute() {
|
|
82
|
+
var _a, _b;
|
|
83
|
+
try {
|
|
84
|
+
await this.clearCache();
|
|
85
|
+
await this.setCookies();
|
|
86
|
+
if (this.config.navigateType === "OpenJson" /* NavigateType.OpenJson */) {
|
|
87
|
+
const { requestMethod, requestHeaders, requestBodyContent, timeOut } = this.config;
|
|
88
|
+
const headers = new Map();
|
|
89
|
+
let extraHeaders = '';
|
|
90
|
+
for (const { name, value } of requestHeaders) {
|
|
91
|
+
headers.set(name, value);
|
|
92
|
+
extraHeaders += `${name}: ${value}\n`;
|
|
93
|
+
}
|
|
94
|
+
// 使用 jsonParser 发起 HTTP 请求并加载数据
|
|
95
|
+
// timeout 单位是毫秒
|
|
96
|
+
const timeoutMs = (timeOut || 60) * 1000;
|
|
97
|
+
await this.browserProxy.jsonParser.startRequest(this.url, requestMethod || 'GET', headers, requestBodyContent || '', timeoutMs);
|
|
98
|
+
}
|
|
99
|
+
else {
|
|
100
|
+
// 拦截cloudflare
|
|
101
|
+
if (this.isAutoCloudflare) {
|
|
102
|
+
// 打开网页之前注入代码
|
|
103
|
+
await this.setupTurnstileInterceptor();
|
|
104
|
+
}
|
|
105
|
+
const [, response] = await Promise.all([
|
|
106
|
+
this.page.waitForNavigation({
|
|
107
|
+
waitUntil: 'load',
|
|
108
|
+
timeout: this.config.timeOut * 1000 || 60000
|
|
109
|
+
}),
|
|
110
|
+
this.page.goto(this.url, {
|
|
111
|
+
waitUntil: 'load',
|
|
112
|
+
timeout: this.config.timeOut * 1000 || 60000
|
|
113
|
+
})
|
|
114
|
+
]);
|
|
115
|
+
const finalResponse = await this.resolveFinalNavigationResponse(response);
|
|
116
|
+
const statusText = ' status = ' + ((_a = finalResponse === null || finalResponse === void 0 ? void 0 : finalResponse.status) === null || _a === void 0 ? void 0 : _a.call(finalResponse));
|
|
117
|
+
// 检查是否被风控
|
|
118
|
+
const statusCode = (_b = finalResponse === null || finalResponse === void 0 ? void 0 : finalResponse.status) === null || _b === void 0 ? void 0 : _b.call(finalResponse);
|
|
119
|
+
const isLoadFailed = !(finalResponse === null || finalResponse === void 0 ? void 0 : finalResponse.ok());
|
|
120
|
+
// 保存页面加载状态到 BrowserProxy
|
|
121
|
+
this.browserProxy.pageLoadStatus = {
|
|
122
|
+
url: this.url,
|
|
123
|
+
statusCode,
|
|
124
|
+
isLoadFailed,
|
|
125
|
+
timestamp: Date.now()
|
|
126
|
+
};
|
|
127
|
+
if (statusCode &&
|
|
128
|
+
!/^[123]/.test(statusCode.toString()) &&
|
|
129
|
+
statusCode.toString() !== '404') {
|
|
130
|
+
this.debug('RiskControlBlocked', `httpErrorStatus=${statusCode}`);
|
|
131
|
+
this.collectProxyLog(`[${this.name}] RiskControlBlocked httpErrorStatus=${statusCode}`, false);
|
|
132
|
+
}
|
|
133
|
+
if (finalResponse && finalResponse.ok()) {
|
|
134
|
+
this.log(enums_1.LogKey.Navigate.Goto.Loading, this.name, this.url);
|
|
135
|
+
await this.onPageLoaded(this.page.url() + statusText);
|
|
136
|
+
}
|
|
137
|
+
else {
|
|
138
|
+
this.error(enums_1.LogKey.Navigate.Goto.Failure, this.name, this.url + statusText);
|
|
139
|
+
/** 上传代理IP失败的接口 */
|
|
140
|
+
this.collectProxyLog(`[${this.name}] Failed to load webpage,page=${this.url} status=${statusText}`, false);
|
|
141
|
+
await this.retry();
|
|
142
|
+
}
|
|
143
|
+
await this.fixCloudflare();
|
|
144
|
+
await this.checkPause();
|
|
145
|
+
// await this.scroll()
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
catch (error) {
|
|
149
|
+
const errorMessage = this.getErrorMessage(error);
|
|
150
|
+
const isTimeout = /timeout/i.test(errorMessage);
|
|
151
|
+
await this.checkPause();
|
|
152
|
+
// 保存页面加载失败状态到 BrowserProxy
|
|
153
|
+
this.browserProxy.pageLoadStatus = {
|
|
154
|
+
url: this.url,
|
|
155
|
+
statusCode: undefined,
|
|
156
|
+
isLoadFailed: true,
|
|
157
|
+
error: errorMessage,
|
|
158
|
+
timestamp: Date.now()
|
|
159
|
+
};
|
|
160
|
+
if (isTimeout) {
|
|
161
|
+
this.error(enums_1.LogKey.Navigate.Error.Timeout, this.name, this.page.url());
|
|
162
|
+
console.log('打开网页超时', error, this.retryIsNeed);
|
|
163
|
+
this.collectProxyLog(`[${this.name}] Page load timeout,page=${this.url} error=${errorMessage}`, false);
|
|
164
|
+
}
|
|
165
|
+
else {
|
|
166
|
+
this.error(enums_1.LogKey.Navigate.Goto.Failure, this.name, `${this.url} error = ${errorMessage}`);
|
|
167
|
+
console.log('打开网页失败', error, this.retryIsNeed);
|
|
168
|
+
this.collectProxyLog(`[${this.name}] Page load failed,page=${this.url} error=${errorMessage}`, false);
|
|
169
|
+
}
|
|
170
|
+
this.retryIsNeed = false;
|
|
171
|
+
}
|
|
172
|
+
finally {
|
|
173
|
+
if (!this.retryIsNeed) {
|
|
174
|
+
this.finish();
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
/** 清理当前网站的缓存 */
|
|
179
|
+
async clearCache() {
|
|
180
|
+
// JSON 任务不需要清除浏览器缓存
|
|
181
|
+
if (this.config.navigateType === "OpenJson" /* NavigateType.OpenJson */) {
|
|
182
|
+
return;
|
|
183
|
+
}
|
|
184
|
+
if (this.config.clearCache) {
|
|
185
|
+
// 清除缓存
|
|
186
|
+
const client = await this.page.createCDPSession();
|
|
187
|
+
await client.send('Network.clearBrowserCache');
|
|
188
|
+
// 清除cookie
|
|
189
|
+
const context = this.getContext();
|
|
190
|
+
const allCookies = await context.cookies();
|
|
191
|
+
await context.deleteCookie(...allCookies);
|
|
192
|
+
this.log(enums_1.LogKey.Navigate.ClearCache.Success, this.name);
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
/** 设置已保存的 Cookies */
|
|
196
|
+
async setCookies() {
|
|
197
|
+
// context.setCookie()
|
|
198
|
+
const url = (0, utils_1.correctUrl)(this.url);
|
|
199
|
+
/** 设置全局cookie */
|
|
200
|
+
if (this.browserProxy.isSetGlobalCookie) {
|
|
201
|
+
if (this.browserProxy.globalCookie) {
|
|
202
|
+
const cookies = this.parseCookies(this.browserProxy.globalCookie, url);
|
|
203
|
+
if (this.config.navigateType == "OpenJson" /* NavigateType.OpenJson */) {
|
|
204
|
+
//
|
|
205
|
+
}
|
|
206
|
+
else {
|
|
207
|
+
const context = this.getContext();
|
|
208
|
+
await context.setCookie(...cookies);
|
|
209
|
+
this.log(enums_1.LogKey.GlobalCookies.Success, this.name);
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
else {
|
|
213
|
+
this.error(enums_1.LogKey.GlobalCookies.Failure, this.name);
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
/** 设置打开网页步骤指定cookie */
|
|
217
|
+
if (this.config.useCustomizeCookie) {
|
|
218
|
+
if (!this.config.customizeCookies ||
|
|
219
|
+
this.config.customizeCookies.length === 0) {
|
|
220
|
+
this.error(enums_1.LogKey.Navigate.Cookies.Failure, this.name);
|
|
221
|
+
return;
|
|
222
|
+
}
|
|
223
|
+
if (this.config.navigateType == "OpenJson" /* NavigateType.OpenJson */) {
|
|
224
|
+
// HttpRequester.cookies = this.config.customizeCookies
|
|
225
|
+
}
|
|
226
|
+
else {
|
|
227
|
+
const cookies = [];
|
|
228
|
+
for (const { name, value, path, host: domain, isHttpOnly: httpOnly, expiry: expirationDate } of this.config.customizeCookies) {
|
|
229
|
+
const cookie = {
|
|
230
|
+
name,
|
|
231
|
+
value: `${value}`,
|
|
232
|
+
path,
|
|
233
|
+
domain,
|
|
234
|
+
httpOnly,
|
|
235
|
+
expires: expirationDate
|
|
236
|
+
};
|
|
237
|
+
cookies.push({
|
|
238
|
+
...cookie,
|
|
239
|
+
domain: domain.startsWith('.') ? domain : '.' + domain
|
|
240
|
+
});
|
|
241
|
+
if (!this.browserProxy.cookiesRule.excludeSetTopDomainList.includes(domain)) {
|
|
242
|
+
const items = domain.split('.');
|
|
243
|
+
const domains = [items.pop()];
|
|
244
|
+
while (items.length > 0) {
|
|
245
|
+
domains.unshift(items.pop());
|
|
246
|
+
const host = domains.join('.');
|
|
247
|
+
if (host === domain) {
|
|
248
|
+
break;
|
|
249
|
+
}
|
|
250
|
+
cookies.push({
|
|
251
|
+
...cookie,
|
|
252
|
+
domain: host.startsWith('.') ? host : '.' + host
|
|
253
|
+
});
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
const context = this.getContext();
|
|
258
|
+
await context.setCookie(...cookies);
|
|
259
|
+
this.log(enums_1.LogKey.Navigate.Cookies.Success, this.name);
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
/** 判定是否需要重试 */
|
|
264
|
+
async checkRetry() {
|
|
265
|
+
let isNeedRetry = false;
|
|
266
|
+
if (!this.config.enableRetry)
|
|
267
|
+
return isNeedRetry;
|
|
268
|
+
if (++this.retryTimes > this.config.maxRetry)
|
|
269
|
+
return isNeedRetry;
|
|
270
|
+
/** @todo 下面这三个判断逻辑在 8 版本里面应该是废弃了,待确定看是否可以删除 */
|
|
271
|
+
if (this.config.textContain) {
|
|
272
|
+
// this.log(LogKey.Common.Retry.Text.Found, this.name, this.config.textContain)
|
|
273
|
+
const args = new conditionCheckArgs_1.ConditionCheckArgs(this.config.textContain, "ContainText" /* RetryConditionType.ContainText */);
|
|
274
|
+
args.actionConfig = this.config;
|
|
275
|
+
args.navigateType = "OpenWebpage" /* NavigateType.OpenWebpage */;
|
|
276
|
+
isNeedRetry = await this.conditionCheck(args, this.page);
|
|
277
|
+
}
|
|
278
|
+
else if (this.config.textNotContain) {
|
|
279
|
+
// this.log(LogKey.Common.Retry.XPath.NotFound, this.name, this.config.textNotContain)
|
|
280
|
+
const args = new conditionCheckArgs_1.ConditionCheckArgs(this.config.textNotContain, "ContainText" /* RetryConditionType.ContainText */);
|
|
281
|
+
args.actionConfig = this.config;
|
|
282
|
+
args.navigateType = "OpenWebpage" /* NavigateType.OpenWebpage */;
|
|
283
|
+
isNeedRetry = !(await this.conditionCheck(args, this.page));
|
|
284
|
+
}
|
|
285
|
+
else if (this.config.urlContain) {
|
|
286
|
+
// this.log(LogKey.Common.Retry.URL.Found, this.name, this.config.urlContain)
|
|
287
|
+
const args = new conditionCheckArgs_1.ConditionCheckArgs(this.config.urlContain, "URLContain" /* RetryConditionType.URLContain */);
|
|
288
|
+
args.actionConfig = this.config;
|
|
289
|
+
args.navigateType = "OpenWebpage" /* NavigateType.OpenWebpage */;
|
|
290
|
+
isNeedRetry = await this.conditionCheck(args, this.page);
|
|
291
|
+
}
|
|
292
|
+
if (this.config.retryConditions && this.config.retryConditions.length > 0) {
|
|
293
|
+
const args = new conditionCheckArgs_1.ConditionCheckArgs(this.config.waitItem, "Multiple" /* RetryConditionType.Multiple */);
|
|
294
|
+
args.actionConfig = this.config;
|
|
295
|
+
args.navigateType = this.config.navigateType;
|
|
296
|
+
args.retryConditions = this.config.retryConditions;
|
|
297
|
+
isNeedRetry = await this.conditionCheck(args, this.page);
|
|
298
|
+
}
|
|
299
|
+
if (isNeedRetry)
|
|
300
|
+
this.log(enums_1.LogKey.Common.Retry.Success, this.name, this.config.retryInterval, this.retryTimes);
|
|
301
|
+
else
|
|
302
|
+
this.error(enums_1.LogKey.Common.Retry.Failure, this.name);
|
|
303
|
+
await this.checkPause();
|
|
304
|
+
return isNeedRetry;
|
|
305
|
+
}
|
|
306
|
+
parseCookies(cookieStr, correctUrl) {
|
|
307
|
+
const customizeCookies = [];
|
|
308
|
+
try {
|
|
309
|
+
if (!(0, utils_1.isNullOrEmpty)(cookieStr)) {
|
|
310
|
+
let url;
|
|
311
|
+
if (!(0, utils_1.isNullOrEmpty)(correctUrl)) {
|
|
312
|
+
url = new URL(correctUrl);
|
|
313
|
+
}
|
|
314
|
+
const cookieLines = cookieStr.split(';');
|
|
315
|
+
cookieLines.forEach((l) => {
|
|
316
|
+
const kv = l.split(/:|=/);
|
|
317
|
+
// 存在特殊情况
|
|
318
|
+
if (kv.length >= 2) {
|
|
319
|
+
const cookie = { name: '', value: '', domain: '' };
|
|
320
|
+
cookie.name = kv[0].trim();
|
|
321
|
+
cookie.path = '/';
|
|
322
|
+
cookie.domain = url === undefined ? '' : '.' + url.host;
|
|
323
|
+
cookie.httpOnly = false;
|
|
324
|
+
// 设置一个长的客户端的cookie过期时间,防止被浏览器认为无效
|
|
325
|
+
cookie.expires = new Date().getTime() / 1000 + 3600 * 24 * 365;
|
|
326
|
+
// cookie.sameSite = 'None'
|
|
327
|
+
if (kv.length === 2) {
|
|
328
|
+
cookie.value = kv[1].trim();
|
|
329
|
+
}
|
|
330
|
+
else {
|
|
331
|
+
const ind = l.indexOf('=');
|
|
332
|
+
cookie.value = l.substring(ind + 1);
|
|
333
|
+
}
|
|
334
|
+
customizeCookies.push(cookie);
|
|
335
|
+
const items = cookie.domain.split('.');
|
|
336
|
+
const domains = [items.pop()];
|
|
337
|
+
while (items.length > 0) {
|
|
338
|
+
domains.unshift(items.pop());
|
|
339
|
+
const host = domains.join('.');
|
|
340
|
+
if (host === cookie.domain) {
|
|
341
|
+
break;
|
|
342
|
+
}
|
|
343
|
+
customizeCookies.push({ ...cookie, domain: '.' + host });
|
|
344
|
+
}
|
|
345
|
+
}
|
|
346
|
+
});
|
|
347
|
+
}
|
|
348
|
+
}
|
|
349
|
+
catch (error) {
|
|
350
|
+
// this.workflow.error(LogKey.GlobalCookies.Failure, this.name)
|
|
351
|
+
}
|
|
352
|
+
return customizeCookies;
|
|
353
|
+
}
|
|
354
|
+
start() {
|
|
355
|
+
if (!this.getIsSkipCurrentLoop() ||
|
|
356
|
+
!this.getIsSkipActionUtilNextLoop(this.getNodeId())) {
|
|
357
|
+
// const currentIndex = this.workflow.browser.getCurrentPageIndex()
|
|
358
|
+
// currentIndex === -1 && this.workflow.browser.switchPage(this.config.pageIndex)
|
|
359
|
+
}
|
|
360
|
+
}
|
|
361
|
+
async wait() {
|
|
362
|
+
this.browserProxy.navigateURL = this.url;
|
|
363
|
+
if (this.config.navigateType === "OpenJson" /* NavigateType.OpenJson */) {
|
|
364
|
+
this.debug('JsonTask', 'json任务');
|
|
365
|
+
}
|
|
366
|
+
if (this.getIsSkipCurrentLoop() ||
|
|
367
|
+
this.getIsSkipActionUtilNextLoop(this.getNodeId())) {
|
|
368
|
+
this.finish();
|
|
369
|
+
return;
|
|
370
|
+
}
|
|
371
|
+
this.page = await this.getCurrentPage();
|
|
372
|
+
await this.checkPause();
|
|
373
|
+
if (this.config.navigateType !== "OpenJson" /* NavigateType.OpenJson */) {
|
|
374
|
+
await super.wait();
|
|
375
|
+
}
|
|
376
|
+
// 得到任务类型,赋值全局变量 todo
|
|
377
|
+
this.browserProxy.setNavigateType(this.config.navigateType);
|
|
378
|
+
if (this.config.useLoopItem) {
|
|
379
|
+
this.log(enums_1.LogKey.Navigate.BeginWithLoop, this.name);
|
|
380
|
+
this.url = this.getCurrentLoopValue();
|
|
381
|
+
}
|
|
382
|
+
else {
|
|
383
|
+
this.log(enums_1.LogKey.Navigate.Begin, this.name);
|
|
384
|
+
this.url = this.config.url;
|
|
385
|
+
}
|
|
386
|
+
// this.workflow.navigateURL = this.url
|
|
387
|
+
if (!this.url) {
|
|
388
|
+
this.error(enums_1.LogKey.Navigate.Error.NotFondURL, this.name);
|
|
389
|
+
this.finish();
|
|
390
|
+
return;
|
|
391
|
+
}
|
|
392
|
+
this.execute();
|
|
393
|
+
}
|
|
394
|
+
onPageLoading(url) {
|
|
395
|
+
if (!this.activity.waiting)
|
|
396
|
+
return;
|
|
397
|
+
if (this.config.navigateType === "OpenJson" /* NavigateType.OpenJson */)
|
|
398
|
+
return;
|
|
399
|
+
// this.log(LogKey.Navigate.Goto.Loading, this.name, url)
|
|
400
|
+
}
|
|
401
|
+
async onPageLoaded(url) {
|
|
402
|
+
if (!this.activity.waiting)
|
|
403
|
+
return;
|
|
404
|
+
if (this.config.navigateType === "OpenJson" /* NavigateType.OpenJson */)
|
|
405
|
+
return;
|
|
406
|
+
this.log(enums_1.LogKey.Navigate.Goto.Success, this.name, url);
|
|
407
|
+
/**
|
|
408
|
+
* 只上传一次
|
|
409
|
+
*/
|
|
410
|
+
if (!this.browserProxy.isUploadedProxyInfoOpenedPage) {
|
|
411
|
+
this.collectProxyLog(`[${this.name}] Success load webpage,page=${this.url}`, true);
|
|
412
|
+
this.browserProxy.isUploadedProxyInfoOpenedPage = true;
|
|
413
|
+
}
|
|
414
|
+
await this.checkPause();
|
|
415
|
+
await this.waitRequest();
|
|
416
|
+
const isNeedRetry = await this.retry();
|
|
417
|
+
if (isNeedRetry) {
|
|
418
|
+
return;
|
|
419
|
+
}
|
|
420
|
+
await this.scroll();
|
|
421
|
+
}
|
|
422
|
+
async retry() {
|
|
423
|
+
const isNeedRetry = await this.checkRetry();
|
|
424
|
+
this.retryIsNeed = isNeedRetry;
|
|
425
|
+
if (isNeedRetry) {
|
|
426
|
+
if (this.config.retryInterval > 0) {
|
|
427
|
+
await this.sleep(this.config.retryInterval * 1000);
|
|
428
|
+
await this.checkPause();
|
|
429
|
+
}
|
|
430
|
+
await this.switchIP(this.page);
|
|
431
|
+
this.switchUA();
|
|
432
|
+
this.execute();
|
|
433
|
+
return isNeedRetry;
|
|
434
|
+
}
|
|
435
|
+
}
|
|
436
|
+
async fixCloudflare() {
|
|
437
|
+
// 暂时先走刷新的方式拦截cloudflare
|
|
438
|
+
// 这里卡了太长时间,后面继续优化
|
|
439
|
+
// 刷新网页最多三次
|
|
440
|
+
try {
|
|
441
|
+
if (!this.isAutoCloudflare)
|
|
442
|
+
return;
|
|
443
|
+
const hasTurnstile = await this.judgeHasTurnstile();
|
|
444
|
+
if (!hasTurnstile)
|
|
445
|
+
return;
|
|
446
|
+
let retryGetParamsTimes = 0;
|
|
447
|
+
const maxRetryGetParamsTimes = 3;
|
|
448
|
+
let params = null;
|
|
449
|
+
while (!params && retryGetParamsTimes < maxRetryGetParamsTimes) {
|
|
450
|
+
await this.page.reload();
|
|
451
|
+
params = await this.page.evaluate(() => {
|
|
452
|
+
return new Promise((resolve, reject) => {
|
|
453
|
+
let checkTurnstileParamsI = 0;
|
|
454
|
+
const checkTurnstileParamsTimer = setInterval(() => {
|
|
455
|
+
if (window.turnstileFreezeParams) {
|
|
456
|
+
clearInterval(checkTurnstileParamsTimer);
|
|
457
|
+
resolve(window.turnstileFreezeParams);
|
|
458
|
+
}
|
|
459
|
+
checkTurnstileParamsI++;
|
|
460
|
+
if (checkTurnstileParamsI >= 5) {
|
|
461
|
+
clearInterval(checkTurnstileParamsTimer);
|
|
462
|
+
resolve(null);
|
|
463
|
+
}
|
|
464
|
+
}, 1000);
|
|
465
|
+
});
|
|
466
|
+
});
|
|
467
|
+
retryGetParamsTimes++;
|
|
468
|
+
}
|
|
469
|
+
if (params) {
|
|
470
|
+
this.resource(enums_1.LogKey.EnterCapacha.Begin, this.name);
|
|
471
|
+
this.resource(enums_1.LogKey.EnterCapacha.Action, this.name);
|
|
472
|
+
const token = await this.getCaptchaToken({
|
|
473
|
+
...params,
|
|
474
|
+
captchaType: enums_1.LocalCaptchaType.Cloudflare
|
|
475
|
+
});
|
|
476
|
+
await this.page.evaluate((token) => {
|
|
477
|
+
window.turnstileCallback && window.turnstileCallback(token);
|
|
478
|
+
}, token);
|
|
479
|
+
if (token) {
|
|
480
|
+
this.resource(enums_1.LogKey.EnterCapacha.Complete, this.name);
|
|
481
|
+
}
|
|
482
|
+
}
|
|
483
|
+
}
|
|
484
|
+
catch (error) {
|
|
485
|
+
console.log('解决cloudflare验证码出错:', error);
|
|
486
|
+
}
|
|
487
|
+
return;
|
|
488
|
+
}
|
|
489
|
+
async judgeHasTurnstile() {
|
|
490
|
+
try {
|
|
491
|
+
const hasTurnstile = await this.page.evaluate(`
|
|
492
|
+
new Promise(resolve => {
|
|
493
|
+
const start = Date.now();
|
|
494
|
+
const check = () => !!window.turnstile;
|
|
495
|
+
if (check()) return resolve(true);
|
|
496
|
+
const interval = setInterval(() => {
|
|
497
|
+
if (check()) {
|
|
498
|
+
clearInterval(interval);
|
|
499
|
+
resolve(true);
|
|
500
|
+
} else if (Date.now() - start > 5000) {
|
|
501
|
+
clearInterval(interval);
|
|
502
|
+
resolve(false);
|
|
503
|
+
}
|
|
504
|
+
}, 100);
|
|
505
|
+
});
|
|
506
|
+
`);
|
|
507
|
+
return hasTurnstile;
|
|
508
|
+
}
|
|
509
|
+
catch (error) {
|
|
510
|
+
console.log('judgeHasTurnstile error', error);
|
|
511
|
+
return false;
|
|
512
|
+
}
|
|
513
|
+
}
|
|
514
|
+
async onJsonResponse() {
|
|
515
|
+
// this.log(LogKey.Navigate.Goto.Success, this.name, this.url)
|
|
516
|
+
await this.checkPause();
|
|
517
|
+
const isNeedRetry = await this.checkRetry();
|
|
518
|
+
if (isNeedRetry) {
|
|
519
|
+
if (this.config.retryInterval > 0) {
|
|
520
|
+
await this.sleep(this.config.retryInterval * 1000);
|
|
521
|
+
await this.checkPause();
|
|
522
|
+
}
|
|
523
|
+
await this.switchIP(this.page);
|
|
524
|
+
this.switchUA();
|
|
525
|
+
this.execute();
|
|
526
|
+
return;
|
|
527
|
+
}
|
|
528
|
+
this.finish();
|
|
529
|
+
}
|
|
530
|
+
onJsonError() {
|
|
531
|
+
// this.workflow.error(LogKey.Navigate.Goto.Failure, this.name, this.url)
|
|
532
|
+
this.finish();
|
|
533
|
+
}
|
|
534
|
+
}
|
|
535
|
+
exports.default = NavigateAction;
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import SwitchComboConfig from '../configs/SwitchComboConfig';
|
|
2
|
+
import BaseAction from './BaseAction';
|
|
3
|
+
export default class SwitchComboAction extends BaseAction {
|
|
4
|
+
config: SwitchComboConfig;
|
|
5
|
+
private page;
|
|
6
|
+
protected parseConfig(config: Record<string, unknown>): SwitchComboConfig;
|
|
7
|
+
wait(): Promise<void>;
|
|
8
|
+
/**
|
|
9
|
+
* 使用项目通用方法processElement实现selectOptions功能
|
|
10
|
+
* 选择option元素并触发change事件
|
|
11
|
+
*/
|
|
12
|
+
private selectOptions;
|
|
13
|
+
}
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
const SwitchComboConfig_1 = __importDefault(require("../configs/SwitchComboConfig"));
|
|
7
|
+
const enums_1 = require("../enums");
|
|
8
|
+
const actionItem_1 = require("../models/actionItem");
|
|
9
|
+
const BaseAction_1 = __importDefault(require("./BaseAction"));
|
|
10
|
+
const Operations_1 = require("../utils/Operations");
|
|
11
|
+
class SwitchComboAction extends BaseAction_1.default {
|
|
12
|
+
parseConfig(config) {
|
|
13
|
+
return new SwitchComboConfig_1.default(config);
|
|
14
|
+
}
|
|
15
|
+
async wait() {
|
|
16
|
+
if (this.getIsSkipCurrentLoop() || this.getIsSkipActionUtilNextLoop(this.getNodeId())) {
|
|
17
|
+
this.finish();
|
|
18
|
+
return;
|
|
19
|
+
}
|
|
20
|
+
await super.wait();
|
|
21
|
+
let actionItem = this.config.elementXPath;
|
|
22
|
+
if (this.config.useLoopItem) {
|
|
23
|
+
const { AbsXpath, IsIFrame, IFrameAbsXPath } = this.getCurrentLoopValue();
|
|
24
|
+
actionItem = new actionItem_1.ActionItem(AbsXpath + actionItem.AbsXpath, IsIFrame, IFrameAbsXPath);
|
|
25
|
+
}
|
|
26
|
+
this.log(enums_1.LogKey.SwitchCombo.Begin, this.name, actionItem.AbsXpath);
|
|
27
|
+
this.page = await this.getCurrentPage();
|
|
28
|
+
const timeout = (this.config.ajaxTimeout || 1.5) * 1000;
|
|
29
|
+
const [, select] = await Promise.allSettled([
|
|
30
|
+
this.page.waitForNetworkIdle({ timeout, idleTime: 300, concurrency: 3 }),
|
|
31
|
+
await this.selectOptions(actionItem),
|
|
32
|
+
]);
|
|
33
|
+
if (select.status == 'rejected') {
|
|
34
|
+
console.log('select options error');
|
|
35
|
+
throw new Error(select.reason);
|
|
36
|
+
}
|
|
37
|
+
await this.ajax();
|
|
38
|
+
this.finish();
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* 使用项目通用方法processElement实现selectOptions功能
|
|
42
|
+
* 选择option元素并触发change事件
|
|
43
|
+
*/
|
|
44
|
+
async selectOptions(actionItem) {
|
|
45
|
+
try {
|
|
46
|
+
await (0, Operations_1.processElement)(actionItem, this.page, '$eval', (element) => {
|
|
47
|
+
if (element && element.nodeName === 'OPTION') {
|
|
48
|
+
element.selected = true;
|
|
49
|
+
// 找到父级select元素
|
|
50
|
+
let selectElement = element.parentElement;
|
|
51
|
+
while (selectElement && selectElement.nodeName !== 'SELECT') {
|
|
52
|
+
selectElement = selectElement.parentElement;
|
|
53
|
+
}
|
|
54
|
+
if (selectElement) {
|
|
55
|
+
// 触发change事件
|
|
56
|
+
const event = new Event('change', { bubbles: true });
|
|
57
|
+
selectElement.dispatchEvent(event);
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
});
|
|
61
|
+
}
|
|
62
|
+
catch (error) {
|
|
63
|
+
await this.checkPause();
|
|
64
|
+
console.error('选择选项时出错:', error.message);
|
|
65
|
+
throw error;
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
exports.default = SwitchComboAction;
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
export interface ResolveStatus {
|
|
2
|
+
state: 'checking' | 'downloading' | 'retrying' | 'completed' | 'failed';
|
|
3
|
+
progress?: number;
|
|
4
|
+
}
|
|
5
|
+
export interface ChromeInstallConfig {
|
|
6
|
+
baseUrl: string;
|
|
7
|
+
buildId: string;
|
|
8
|
+
cacheDir?: string;
|
|
9
|
+
}
|
|
10
|
+
export interface ResolveOptions {
|
|
11
|
+
onStatus?: (status: ResolveStatus) => void;
|
|
12
|
+
installOptions?: ChromeInstallConfig;
|
|
13
|
+
}
|
|
14
|
+
export declare function cleanupChromeCache(installOptions?: ChromeInstallConfig): Promise<void>;
|
|
15
|
+
export declare function resolveChrome(options?: ResolveOptions): Promise<{
|
|
16
|
+
executablePath: string;
|
|
17
|
+
}>;
|