octoparse-cli 0.1.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +271 -0
- package/RUNTIME_SECURITY_NOTICE.txt +41 -0
- package/SECURITY.md +24 -0
- package/dist/cli/args.js +34 -0
- package/dist/cli/help.js +214 -0
- package/dist/cli/output.js +39 -0
- package/dist/commands/auth.js +283 -0
- package/dist/commands/capabilities.js +118 -0
- package/dist/commands/cloud.js +241 -0
- package/dist/commands/data.js +220 -0
- package/dist/commands/doctor.js +73 -0
- package/dist/commands/env.js +63 -0
- package/dist/commands/local.js +251 -0
- package/dist/commands/run.js +622 -0
- package/dist/commands/runs.js +171 -0
- package/dist/commands/task.js +101 -0
- package/dist/index.js +133 -0
- package/dist/runtime/account-capabilities.js +71 -0
- package/dist/runtime/api-client.js +290 -0
- package/dist/runtime/artifacts.js +33 -0
- package/dist/runtime/auth.js +94 -0
- package/dist/runtime/bridge-hub.js +173 -0
- package/dist/runtime/client-headers.js +23 -0
- package/dist/runtime/cloud-data.js +75 -0
- package/dist/runtime/config.js +48 -0
- package/dist/runtime/data-exporter.js +267 -0
- package/dist/runtime/engine-host.js +449 -0
- package/dist/runtime/local-runs.js +92 -0
- package/dist/runtime/naming.js +13 -0
- package/dist/runtime/run-control.js +363 -0
- package/dist/runtime/run-services.js +380 -0
- package/dist/runtime/security-notice.js +78 -0
- package/dist/runtime/task-definition-provider.js +282 -0
- package/dist/types.js +4 -0
- package/examples/minimal-task.json +6 -0
- package/examples/navigate-example-task.json +6 -0
- package/node_modules/@octopus/bpmn/index.js +3 -0
- package/node_modules/@octopus/bpmn/lib/Context.js +245 -0
- package/node_modules/@octopus/bpmn/lib/Definition.js +258 -0
- package/node_modules/@octopus/bpmn/lib/Engine.js +275 -0
- package/node_modules/@octopus/bpmn/lib/PrematureStopError.js +7 -0
- package/node_modules/@octopus/bpmn/lib/activities/Activity.js +202 -0
- package/node_modules/@octopus/bpmn/lib/activities/BaseProcess.js +308 -0
- package/node_modules/@octopus/bpmn/lib/activities/BaseTask.js +145 -0
- package/node_modules/@octopus/bpmn/lib/activities/BoundaryEvent.js +12 -0
- package/node_modules/@octopus/bpmn/lib/activities/Dummy.js +10 -0
- package/node_modules/@octopus/bpmn/lib/activities/EventDefinition.js +99 -0
- package/node_modules/@octopus/bpmn/lib/activities/Flow.js +52 -0
- package/node_modules/@octopus/bpmn/lib/activities/Form.js +67 -0
- package/node_modules/@octopus/bpmn/lib/activities/InputOutput.js +53 -0
- package/node_modules/@octopus/bpmn/lib/activities/IntermediateCatchEvent.js +12 -0
- package/node_modules/@octopus/bpmn/lib/activities/MessageFlow.js +19 -0
- package/node_modules/@octopus/bpmn/lib/activities/MultiInstanceLoopCharacteristics.js +160 -0
- package/node_modules/@octopus/bpmn/lib/activities/Properties.js +27 -0
- package/node_modules/@octopus/bpmn/lib/activities/SequenceFlow.js +56 -0
- package/node_modules/@octopus/bpmn/lib/activities/ServiceConnector.js +71 -0
- package/node_modules/@octopus/bpmn/lib/context-helper.js +198 -0
- package/node_modules/@octopus/bpmn/lib/events/EndEvent.js +22 -0
- package/node_modules/@octopus/bpmn/lib/events/ErrorEvent.js +41 -0
- package/node_modules/@octopus/bpmn/lib/events/MessageEvent.js +19 -0
- package/node_modules/@octopus/bpmn/lib/events/StartEvent.js +55 -0
- package/node_modules/@octopus/bpmn/lib/events/TimerEvent.js +75 -0
- package/node_modules/@octopus/bpmn/lib/expressions.js +41 -0
- package/node_modules/@octopus/bpmn/lib/gateways/ExclusiveGateway.js +86 -0
- package/node_modules/@octopus/bpmn/lib/gateways/InclusiveGateway.js +56 -0
- package/node_modules/@octopus/bpmn/lib/gateways/ParallelGateway.js +195 -0
- package/node_modules/@octopus/bpmn/lib/getPropertyValue.js +83 -0
- package/node_modules/@octopus/bpmn/lib/index.js +6 -0
- package/node_modules/@octopus/bpmn/lib/mapper.js +55 -0
- package/node_modules/@octopus/bpmn/lib/parameter.js +119 -0
- package/node_modules/@octopus/bpmn/lib/script-helper.js +45 -0
- package/node_modules/@octopus/bpmn/lib/tasks/ManualTask.js +31 -0
- package/node_modules/@octopus/bpmn/lib/tasks/ReceiveTask.js +31 -0
- package/node_modules/@octopus/bpmn/lib/tasks/ScriptTask.js +35 -0
- package/node_modules/@octopus/bpmn/lib/tasks/SendTask.js +16 -0
- package/node_modules/@octopus/bpmn/lib/tasks/ServiceTask.js +68 -0
- package/node_modules/@octopus/bpmn/lib/tasks/SubProcess.js +17 -0
- package/node_modules/@octopus/bpmn/lib/tasks/Task.js +16 -0
- package/node_modules/@octopus/bpmn/lib/tasks/UserTask.js +47 -0
- package/node_modules/@octopus/bpmn/lib/transformer.js +13 -0
- package/node_modules/@octopus/bpmn/lib/validation.js +111 -0
- package/node_modules/@octopus/bpmn/package.json +17 -0
- package/node_modules/@octopus/bpmn/types/bpmn.d.ts +85 -0
- package/node_modules/@octopus/engine/README.md +370 -0
- package/node_modules/@octopus/engine/dist/actions/BackPreWebPageAction.d.ts +4 -0
- package/node_modules/@octopus/engine/dist/actions/BackPreWebPageAction.js +1 -0
- package/node_modules/@octopus/engine/dist/actions/BaseAction.d.ts +339 -0
- package/node_modules/@octopus/engine/dist/actions/BaseAction.js +1559 -0
- package/node_modules/@octopus/engine/dist/actions/BranchAction.d.ts +9 -0
- package/node_modules/@octopus/engine/dist/actions/BranchAction.js +1 -0
- package/node_modules/@octopus/engine/dist/actions/ClickAction.d.ts +22 -0
- package/node_modules/@octopus/engine/dist/actions/ClickAction.js +1 -0
- package/node_modules/@octopus/engine/dist/actions/ConditionAction.d.ts +4 -0
- package/node_modules/@octopus/engine/dist/actions/ConditionAction.js +1 -0
- package/node_modules/@octopus/engine/dist/actions/EmptyAction.d.ts +4 -0
- package/node_modules/@octopus/engine/dist/actions/EmptyAction.js +12 -0
- package/node_modules/@octopus/engine/dist/actions/EnterCaptchaAction.d.ts +28 -0
- package/node_modules/@octopus/engine/dist/actions/EnterCaptchaAction.js +1 -0
- package/node_modules/@octopus/engine/dist/actions/EnterTextAction.d.ts +20 -0
- package/node_modules/@octopus/engine/dist/actions/EnterTextAction.js +1 -0
- package/node_modules/@octopus/engine/dist/actions/ExtractDataAction.d.ts +40 -0
- package/node_modules/@octopus/engine/dist/actions/ExtractDataAction.js +1 -0
- package/node_modules/@octopus/engine/dist/actions/LoopAction.d.ts +41 -0
- package/node_modules/@octopus/engine/dist/actions/LoopAction.js +526 -0
- package/node_modules/@octopus/engine/dist/actions/LoopStartAction.d.ts +47 -0
- package/node_modules/@octopus/engine/dist/actions/LoopStartAction.js +607 -0
- package/node_modules/@octopus/engine/dist/actions/MouseOverAction.d.ts +8 -0
- package/node_modules/@octopus/engine/dist/actions/MouseOverAction.js +34 -0
- package/node_modules/@octopus/engine/dist/actions/NavigateAction.d.ts +38 -0
- package/node_modules/@octopus/engine/dist/actions/NavigateAction.js +535 -0
- package/node_modules/@octopus/engine/dist/actions/SwitchComboAction.d.ts +13 -0
- package/node_modules/@octopus/engine/dist/actions/SwitchComboAction.js +69 -0
- package/node_modules/@octopus/engine/dist/browser.d.ts +17 -0
- package/node_modules/@octopus/engine/dist/browser.js +157 -0
- package/node_modules/@octopus/engine/dist/browserProxy.d.ts +90 -0
- package/node_modules/@octopus/engine/dist/browserProxy.js +1 -0
- package/node_modules/@octopus/engine/dist/configs/BaseConfig.d.ts +20 -0
- package/node_modules/@octopus/engine/dist/configs/BaseConfig.js +88 -0
- package/node_modules/@octopus/engine/dist/configs/BranchConfig.d.ts +7 -0
- package/node_modules/@octopus/engine/dist/configs/BranchConfig.js +1 -0
- package/node_modules/@octopus/engine/dist/configs/ClickConfig.d.ts +36 -0
- package/node_modules/@octopus/engine/dist/configs/ClickConfig.js +65 -0
- package/node_modules/@octopus/engine/dist/configs/EnterCaptchaConfig.d.ts +19 -0
- package/node_modules/@octopus/engine/dist/configs/EnterCaptchaConfig.js +25 -0
- package/node_modules/@octopus/engine/dist/configs/EnterTextConfig.d.ts +24 -0
- package/node_modules/@octopus/engine/dist/configs/EnterTextConfig.js +36 -0
- package/node_modules/@octopus/engine/dist/configs/ExtractDataConfig.d.ts +12 -0
- package/node_modules/@octopus/engine/dist/configs/ExtractDataConfig.js +1 -0
- package/node_modules/@octopus/engine/dist/configs/LoopConfig.d.ts +25 -0
- package/node_modules/@octopus/engine/dist/configs/LoopConfig.js +40 -0
- package/node_modules/@octopus/engine/dist/configs/LoopStartConfig.d.ts +4 -0
- package/node_modules/@octopus/engine/dist/configs/LoopStartConfig.js +12 -0
- package/node_modules/@octopus/engine/dist/configs/MouseOverConfig.d.ts +8 -0
- package/node_modules/@octopus/engine/dist/configs/MouseOverConfig.js +15 -0
- package/node_modules/@octopus/engine/dist/configs/NavigateConfig.d.ts +41 -0
- package/node_modules/@octopus/engine/dist/configs/NavigateConfig.js +121 -0
- package/node_modules/@octopus/engine/dist/configs/SwitchComboConfig.d.ts +8 -0
- package/node_modules/@octopus/engine/dist/configs/SwitchComboConfig.js +15 -0
- package/node_modules/@octopus/engine/dist/enums/index.d.ts +419 -0
- package/node_modules/@octopus/engine/dist/enums/index.js +314 -0
- package/node_modules/@octopus/engine/dist/extension/BrowserWebSocketTransport-D_zAGZMQ.js +1 -0
- package/node_modules/@octopus/engine/dist/extension/LaunchOptions-DxvePrV4.js +6 -0
- package/node_modules/@octopus/engine/dist/extension/NodeWebSocketTransport-BTgRVB7Z.js +6 -0
- package/node_modules/@octopus/engine/dist/extension/background.js +396 -0
- package/node_modules/@octopus/engine/dist/extension/bidi-C_GIZ8Uz.js +131 -0
- package/node_modules/@octopus/engine/dist/extension/manifest.json +27 -0
- package/node_modules/@octopus/engine/dist/extension/src/content/anti-detection.js +1 -0
- package/node_modules/@octopus/engine/dist/extension-bridge/BaseExtensionBridge.d.ts +21 -0
- package/node_modules/@octopus/engine/dist/extension-bridge/BaseExtensionBridge.js +117 -0
- package/node_modules/@octopus/engine/dist/extension-bridge/SessionExtensionBridge.d.ts +17 -0
- package/node_modules/@octopus/engine/dist/extension-bridge/SessionExtensionBridge.js +29 -0
- package/node_modules/@octopus/engine/dist/extension-bridge/index.d.ts +2 -0
- package/node_modules/@octopus/engine/dist/extension-bridge/index.js +5 -0
- package/node_modules/@octopus/engine/dist/extension-bridge/types.d.ts +159 -0
- package/node_modules/@octopus/engine/dist/extension-bridge/types.js +5 -0
- package/node_modules/@octopus/engine/dist/extensions/ublock-origin/uBlock0.chromium.tar.xz +0 -0
- package/node_modules/@octopus/engine/dist/extensions/ublock-origin-lite/uBOLite.chromium.tar.xz +0 -0
- package/node_modules/@octopus/engine/dist/index.d.ts +169 -0
- package/node_modules/@octopus/engine/dist/index.js +1 -0
- package/node_modules/@octopus/engine/dist/models/actionItem.d.ts +16 -0
- package/node_modules/@octopus/engine/dist/models/actionItem.js +15 -0
- package/node_modules/@octopus/engine/dist/models/conditionCheckArgs.d.ts +11 -0
- package/node_modules/@octopus/engine/dist/models/conditionCheckArgs.js +11 -0
- package/node_modules/@octopus/engine/dist/models/customizeCookie.d.ts +14 -0
- package/node_modules/@octopus/engine/dist/models/customizeCookie.js +6 -0
- package/node_modules/@octopus/engine/dist/models/downloadFileConfig.d.ts +17 -0
- package/node_modules/@octopus/engine/dist/models/downloadFileConfig.js +26 -0
- package/node_modules/@octopus/engine/dist/models/elementNotFoundArgs.d.ts +8 -0
- package/node_modules/@octopus/engine/dist/models/elementNotFoundArgs.js +12 -0
- package/node_modules/@octopus/engine/dist/models/elementNotFoundError.d.ts +2 -0
- package/node_modules/@octopus/engine/dist/models/elementNotFoundError.js +6 -0
- package/node_modules/@octopus/engine/dist/models/extractItem.d.ts +37 -0
- package/node_modules/@octopus/engine/dist/models/extractItem.js +35 -0
- package/node_modules/@octopus/engine/dist/models/extractTemplate.d.ts +11 -0
- package/node_modules/@octopus/engine/dist/models/extractTemplate.js +48 -0
- package/node_modules/@octopus/engine/dist/models/extractTextItem.d.ts +10 -0
- package/node_modules/@octopus/engine/dist/models/extractTextItem.js +17 -0
- package/node_modules/@octopus/engine/dist/models/globalConfig.d.ts +5 -0
- package/node_modules/@octopus/engine/dist/models/globalConfig.js +1 -0
- package/node_modules/@octopus/engine/dist/models/httpHeader.d.ts +4 -0
- package/node_modules/@octopus/engine/dist/models/httpHeader.js +10 -0
- package/node_modules/@octopus/engine/dist/models/operation.d.ts +27 -0
- package/node_modules/@octopus/engine/dist/models/operation.js +242 -0
- package/node_modules/@octopus/engine/dist/models/retryCondition.d.ts +7 -0
- package/node_modules/@octopus/engine/dist/models/retryCondition.js +10 -0
- package/node_modules/@octopus/engine/dist/models/task.d.ts +89 -0
- package/node_modules/@octopus/engine/dist/models/task.js +120 -0
- package/node_modules/@octopus/engine/dist/models/trigger.d.ts +66 -0
- package/node_modules/@octopus/engine/dist/models/trigger.js +117 -0
- package/node_modules/@octopus/engine/dist/package.json +26 -0
- package/node_modules/@octopus/engine/dist/public-types.d.ts +13 -0
- package/node_modules/@octopus/engine/dist/public-types.js +2 -0
- package/node_modules/@octopus/engine/dist/settings.d.ts +41 -0
- package/node_modules/@octopus/engine/dist/settings.js +20 -0
- package/node_modules/@octopus/engine/dist/solvers/captcha/ClickCaptchaSolver.d.ts +6 -0
- package/node_modules/@octopus/engine/dist/solvers/captcha/ClickCaptchaSolver.js +1 -0
- package/node_modules/@octopus/engine/dist/solvers/captcha/HCaptchaSolver.d.ts +4 -0
- package/node_modules/@octopus/engine/dist/solvers/captcha/HCaptchaSolver.js +73 -0
- package/node_modules/@octopus/engine/dist/solvers/captcha/ImageCaptchaSolver.d.ts +2 -0
- package/node_modules/@octopus/engine/dist/solvers/captcha/ImageCaptchaSolver.js +74 -0
- package/node_modules/@octopus/engine/dist/solvers/captcha/RecaptchaSolver.d.ts +9 -0
- package/node_modules/@octopus/engine/dist/solvers/captcha/RecaptchaSolver.js +371 -0
- package/node_modules/@octopus/engine/dist/solvers/captcha/SliderCaptchaSolver.d.ts +6 -0
- package/node_modules/@octopus/engine/dist/solvers/captcha/SliderCaptchaSolver.js +184 -0
- package/node_modules/@octopus/engine/dist/solvers/captcha/SlidingTrajectory.d.ts +50 -0
- package/node_modules/@octopus/engine/dist/solvers/captcha/SlidingTrajectory.js +125 -0
- package/node_modules/@octopus/engine/dist/solvers/captcha/types.d.ts +68 -0
- package/node_modules/@octopus/engine/dist/solvers/captcha/types.js +34 -0
- package/node_modules/@octopus/engine/dist/solvers/captcha/utils.d.ts +2 -0
- package/node_modules/@octopus/engine/dist/solvers/captcha/utils.js +15 -0
- package/node_modules/@octopus/engine/dist/translator/actionFactory.d.ts +6 -0
- package/node_modules/@octopus/engine/dist/translator/actionFactory.js +1 -0
- package/node_modules/@octopus/engine/dist/translator/activityTypeEnum.d.ts +22 -0
- package/node_modules/@octopus/engine/dist/translator/activityTypeEnum.js +1 -0
- package/node_modules/@octopus/engine/dist/translator/backPreWebPageAction.d.ts +5 -0
- package/node_modules/@octopus/engine/dist/translator/backPreWebPageAction.js +1 -0
- package/node_modules/@octopus/engine/dist/translator/baseAction.d.ts +31 -0
- package/node_modules/@octopus/engine/dist/translator/baseAction.js +1 -0
- package/node_modules/@octopus/engine/dist/translator/breakActivity.d.ts +5 -0
- package/node_modules/@octopus/engine/dist/translator/breakActivity.js +1 -0
- package/node_modules/@octopus/engine/dist/translator/clickAction.d.ts +5 -0
- package/node_modules/@octopus/engine/dist/translator/clickAction.js +1 -0
- package/node_modules/@octopus/engine/dist/translator/completeWF.d.ts +5 -0
- package/node_modules/@octopus/engine/dist/translator/completeWF.js +1 -0
- package/node_modules/@octopus/engine/dist/translator/conditionAction.d.ts +6 -0
- package/node_modules/@octopus/engine/dist/translator/conditionAction.js +1 -0
- package/node_modules/@octopus/engine/dist/translator/emptyAction.d.ts +5 -0
- package/node_modules/@octopus/engine/dist/translator/emptyAction.js +1 -0
- package/node_modules/@octopus/engine/dist/translator/enterCapachaAction.d.ts +5 -0
- package/node_modules/@octopus/engine/dist/translator/enterCapachaAction.js +1 -0
- package/node_modules/@octopus/engine/dist/translator/enterTextAction.d.ts +5 -0
- package/node_modules/@octopus/engine/dist/translator/enterTextAction.js +1 -0
- package/node_modules/@octopus/engine/dist/translator/extractDataAction.d.ts +13 -0
- package/node_modules/@octopus/engine/dist/translator/extractDataAction.js +1 -0
- package/node_modules/@octopus/engine/dist/translator/loopAction.d.ts +5 -0
- package/node_modules/@octopus/engine/dist/translator/loopAction.js +1 -0
- package/node_modules/@octopus/engine/dist/translator/mouseOverAction.d.ts +5 -0
- package/node_modules/@octopus/engine/dist/translator/mouseOverAction.js +19 -0
- package/node_modules/@octopus/engine/dist/translator/navigateAction.d.ts +5 -0
- package/node_modules/@octopus/engine/dist/translator/navigateAction.js +117 -0
- package/node_modules/@octopus/engine/dist/translator/rootAction.d.ts +6 -0
- package/node_modules/@octopus/engine/dist/translator/rootAction.js +80 -0
- package/node_modules/@octopus/engine/dist/translator/switchCombo2Action.d.ts +5 -0
- package/node_modules/@octopus/engine/dist/translator/switchCombo2Action.js +19 -0
- package/node_modules/@octopus/engine/dist/translator/translator.d.ts +1 -0
- package/node_modules/@octopus/engine/dist/translator/translator.js +36 -0
- package/node_modules/@octopus/engine/dist/type.d.ts +25 -0
- package/node_modules/@octopus/engine/dist/type.js +2 -0
- package/node_modules/@octopus/engine/dist/types/browser.d.ts +191 -0
- package/node_modules/@octopus/engine/dist/types/browser.js +1 -0
- package/node_modules/@octopus/engine/dist/types/browserManager.d.ts +41 -0
- package/node_modules/@octopus/engine/dist/types/browserManager.js +1 -0
- package/node_modules/@octopus/engine/dist/types/index.d.ts +40 -0
- package/node_modules/@octopus/engine/dist/types/index.js +2 -0
- package/node_modules/@octopus/engine/dist/types/plugin.d.ts +29 -0
- package/node_modules/@octopus/engine/dist/types/plugin.js +2 -0
- package/node_modules/@octopus/engine/dist/utils/AsyncEmitter.d.ts +15 -0
- package/node_modules/@octopus/engine/dist/utils/AsyncEmitter.js +1 -0
- package/node_modules/@octopus/engine/dist/utils/DataStore.d.ts +58 -0
- package/node_modules/@octopus/engine/dist/utils/DataStore.js +1 -0
- package/node_modules/@octopus/engine/dist/utils/DateTimeFormatHelper.d.ts +22 -0
- package/node_modules/@octopus/engine/dist/utils/DateTimeFormatHelper.js +173 -0
- package/node_modules/@octopus/engine/dist/utils/FileDownloader.d.ts +108 -0
- package/node_modules/@octopus/engine/dist/utils/FileDownloader.js +1 -0
- package/node_modules/@octopus/engine/dist/utils/HttpRequester.d.ts +43 -0
- package/node_modules/@octopus/engine/dist/utils/HttpRequester.js +174 -0
- package/node_modules/@octopus/engine/dist/utils/JsonParser.d.ts +95 -0
- package/node_modules/@octopus/engine/dist/utils/JsonParser.js +439 -0
- package/node_modules/@octopus/engine/dist/utils/Operations.d.ts +27 -0
- package/node_modules/@octopus/engine/dist/utils/Operations.js +115 -0
- package/node_modules/@octopus/engine/dist/utils/index.d.ts +28 -0
- package/node_modules/@octopus/engine/dist/utils/index.js +356 -0
- package/node_modules/@octopus/engine/package.json +58 -0
- package/package.json +79 -0
- package/schemas/capabilities-v1.schema.json +234 -0
- package/schemas/detached-bootstrap-v1.schema.json +42 -0
- package/schemas/json-envelope-v1.schema.json +39 -0
- package/schemas/run-event-v1.schema.json +47 -0
|
@@ -0,0 +1,622 @@
|
|
|
1
|
+
import { spawn } from 'node:child_process';
|
|
2
|
+
import { mkdir, open, readFile, writeFile } from 'node:fs/promises';
|
|
3
|
+
import { join, resolve } from 'node:path';
|
|
4
|
+
import { hasFlag, parsePositiveInt, valueAfter } from '../cli/args.js';
|
|
5
|
+
import { printEnvelope, printUsageError } from '../cli/output.js';
|
|
6
|
+
import { resolveOPLocalRunPolicy } from '../runtime/account-capabilities.js';
|
|
7
|
+
import { ApiRequestError, fetchAccountInfo, fetchQuantityLimitSettings } from '../runtime/api-client.js';
|
|
8
|
+
import { appendJsonLine, ensureRunDir, writeRunSummary } from '../runtime/artifacts.js';
|
|
9
|
+
import { resolveAuth } from '../runtime/auth.js';
|
|
10
|
+
import { EngineHost } from '../runtime/engine-host.js';
|
|
11
|
+
import { defaultRunsDir } from '../runtime/local-runs.js';
|
|
12
|
+
import { safeFileName } from '../runtime/naming.js';
|
|
13
|
+
import { isRunControlReachable, listActiveTaskControlStates, readTaskControlState, startRunControlServer } from '../runtime/run-control.js';
|
|
14
|
+
import { TaskDefinitionProvider } from '../runtime/task-definition-provider.js';
|
|
15
|
+
import { EXIT_OK, EXIT_OPERATION_FAILED, EXIT_RUNTIME_FAILED } from '../types.js';
|
|
16
|
+
const DETACHED_CHILD_ENV = 'OCTO_ENGINE_DETACHED_CHILD';
|
|
17
|
+
const DETACHED_BOOTSTRAP_DIR_ENV = 'OCTO_ENGINE_DETACHED_BOOTSTRAP_DIR';
|
|
18
|
+
export async function runTask(taskId, args) {
|
|
19
|
+
const json = hasFlag([taskId ?? '', ...args], '--json') || hasFlag([taskId ?? '', ...args], '--jsonl');
|
|
20
|
+
if (!taskId || taskId.startsWith('-')) {
|
|
21
|
+
return printUsageError(json, 'Error: missing taskId', 'Usage: octoparse run <taskId> [--task-file <file.json|file.xml|file.otd>] [--output <dir>] [--chrome-path <path>] [--max-rows <n>] [--detach] [--json|--jsonl]');
|
|
22
|
+
}
|
|
23
|
+
if (taskId === 'export') {
|
|
24
|
+
return printUsageError(json, 'run has no export subcommand; run only starts local extraction.', 'Export data with: octoparse data export <taskId> [--source local|cloud] [--lot-id <lotId>] [--file <result.xlsx>] [--format xlsx|csv|html|json|xml]');
|
|
25
|
+
}
|
|
26
|
+
if (hasFlag(args, '--format')) {
|
|
27
|
+
const message = 'run does not support --format; use --json or --jsonl. Export data files with data export --format.';
|
|
28
|
+
if (hasFlag(args, '--json') || hasFlag(args, '--jsonl')) {
|
|
29
|
+
printEnvelope(false, undefined, 'RUN_FORMAT_UNSUPPORTED', message);
|
|
30
|
+
}
|
|
31
|
+
else {
|
|
32
|
+
console.error(message);
|
|
33
|
+
}
|
|
34
|
+
return EXIT_OPERATION_FAILED;
|
|
35
|
+
}
|
|
36
|
+
const options = parseRunOptions(taskId, args);
|
|
37
|
+
const maxRowsError = validateMaxRows(args);
|
|
38
|
+
if (maxRowsError) {
|
|
39
|
+
return printUsageError(options.json || options.jsonl, maxRowsError, 'Usage: octoparse run <taskId> [--max-rows <positive integer>] [--json|--jsonl]', 'RUN_MAX_ROWS_INVALID');
|
|
40
|
+
}
|
|
41
|
+
const active = await readTaskControlState(taskId);
|
|
42
|
+
if (await isRunControlReachable(active)) {
|
|
43
|
+
const message = `Local extraction is already running: taskId=${taskId}, status=${active?.status}`;
|
|
44
|
+
if (options.json || options.jsonl)
|
|
45
|
+
printEnvelope(false, undefined, 'LOCAL_RUN_ALREADY_RUNNING', message);
|
|
46
|
+
else
|
|
47
|
+
console.error(message);
|
|
48
|
+
return EXIT_OPERATION_FAILED;
|
|
49
|
+
}
|
|
50
|
+
if (process.env[DETACHED_CHILD_ENV] !== '1') {
|
|
51
|
+
const limitExitCode = await enforceLocalRunLimit(args, options);
|
|
52
|
+
if (limitExitCode !== EXIT_OK)
|
|
53
|
+
return limitExitCode;
|
|
54
|
+
}
|
|
55
|
+
if (options.detach && process.env[DETACHED_CHILD_ENV] !== '1') {
|
|
56
|
+
return startDetachedRun(taskId, args, options);
|
|
57
|
+
}
|
|
58
|
+
const provider = new TaskDefinitionProvider();
|
|
59
|
+
return executeTask(taskId, options, () => provider.getTask(taskId, options.taskFile));
|
|
60
|
+
}
|
|
61
|
+
async function enforceLocalRunLimit(args, options) {
|
|
62
|
+
try {
|
|
63
|
+
const auth = await resolveAuth();
|
|
64
|
+
if (!auth.apiKey)
|
|
65
|
+
return EXIT_OK;
|
|
66
|
+
const baseUrl = valueAfter(args, '--api-base-url');
|
|
67
|
+
const [account, limits] = await Promise.all([
|
|
68
|
+
fetchAccountInfo({
|
|
69
|
+
apiKey: auth.apiKey,
|
|
70
|
+
baseUrl
|
|
71
|
+
}),
|
|
72
|
+
fetchQuantityLimitSettings({
|
|
73
|
+
apiKey: auth.apiKey,
|
|
74
|
+
baseUrl
|
|
75
|
+
})
|
|
76
|
+
]);
|
|
77
|
+
const policy = resolveOPLocalRunPolicy(account.data, limits.data);
|
|
78
|
+
if (policy.maxActiveLocalRuns === undefined || policy.maxActiveLocalRuns === null) {
|
|
79
|
+
return EXIT_OK;
|
|
80
|
+
}
|
|
81
|
+
const activeRuns = await listActiveTaskControlStates();
|
|
82
|
+
if (activeRuns.length < policy.maxActiveLocalRuns) {
|
|
83
|
+
return EXIT_OK;
|
|
84
|
+
}
|
|
85
|
+
const message = `Local extraction concurrency reached the current account limit (${policy.maxActiveLocalRuns}). Stop a running task before starting another.`;
|
|
86
|
+
if (options.json || options.jsonl) {
|
|
87
|
+
printEnvelope(false, undefined, 'LOCAL_RUN_LIMIT_EXCEEDED', message);
|
|
88
|
+
}
|
|
89
|
+
else {
|
|
90
|
+
console.error(message);
|
|
91
|
+
}
|
|
92
|
+
return EXIT_OPERATION_FAILED;
|
|
93
|
+
}
|
|
94
|
+
catch (error) {
|
|
95
|
+
if (error instanceof ApiRequestError && error.code === 'AUTH_INVALID') {
|
|
96
|
+
const message = error.message;
|
|
97
|
+
if (options.json || options.jsonl)
|
|
98
|
+
printEnvelope(false, undefined, error.code, message);
|
|
99
|
+
else
|
|
100
|
+
console.error(`Authentication failed: ${message}`);
|
|
101
|
+
return EXIT_OPERATION_FAILED;
|
|
102
|
+
}
|
|
103
|
+
return EXIT_OK;
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
async function startDetachedRun(taskId, args, options) {
|
|
107
|
+
const bootstrap = await createDetachedBootstrap(taskId, options.outputDir);
|
|
108
|
+
const childArgs = [
|
|
109
|
+
process.argv[1],
|
|
110
|
+
'run',
|
|
111
|
+
taskId,
|
|
112
|
+
...args.filter((arg) => arg !== '--detach')
|
|
113
|
+
];
|
|
114
|
+
const child = spawn(process.execPath, childArgs, {
|
|
115
|
+
detached: true,
|
|
116
|
+
stdio: ['ignore', bootstrap.stdout.fd, bootstrap.stderr.fd],
|
|
117
|
+
env: {
|
|
118
|
+
...process.env,
|
|
119
|
+
[DETACHED_CHILD_ENV]: '1',
|
|
120
|
+
[DETACHED_BOOTSTRAP_DIR_ENV]: bootstrap.dir
|
|
121
|
+
},
|
|
122
|
+
cwd: process.cwd()
|
|
123
|
+
});
|
|
124
|
+
await closeFileHandles(bootstrap.stdout, bootstrap.stderr);
|
|
125
|
+
child.unref();
|
|
126
|
+
await writeDetachedBootstrap(bootstrap.dir, {
|
|
127
|
+
pid: child.pid,
|
|
128
|
+
status: 'spawned',
|
|
129
|
+
updatedAt: new Date().toISOString()
|
|
130
|
+
});
|
|
131
|
+
const startup = await waitForDetachedStartup(taskId, child, bootstrap.dir, 10_000);
|
|
132
|
+
if (!startup.ok) {
|
|
133
|
+
const message = `${startup.error ?? 'detached run failed before control channel became ready'}; bootstrap=${bootstrap.dir}`;
|
|
134
|
+
if (options.json)
|
|
135
|
+
printEnvelope(false, undefined, 'DETACHED_RUN_FAILED', message);
|
|
136
|
+
else {
|
|
137
|
+
console.error(`Detached run failed: ${startup.error ?? 'startup failed'}`);
|
|
138
|
+
console.error(`Bootstrap: ${bootstrap.dir}`);
|
|
139
|
+
}
|
|
140
|
+
return EXIT_RUNTIME_FAILED;
|
|
141
|
+
}
|
|
142
|
+
const state = startup.state;
|
|
143
|
+
if (!state) {
|
|
144
|
+
await writeDetachedBootstrap(bootstrap.dir, {
|
|
145
|
+
pid: child.pid,
|
|
146
|
+
status: 'starting',
|
|
147
|
+
updatedAt: new Date().toISOString()
|
|
148
|
+
});
|
|
149
|
+
}
|
|
150
|
+
const data = {
|
|
151
|
+
taskId,
|
|
152
|
+
detached: true,
|
|
153
|
+
pid: child.pid,
|
|
154
|
+
status: state?.status ?? 'starting',
|
|
155
|
+
outputDir: state?.outputDir,
|
|
156
|
+
bootstrapDir: bootstrap.dir,
|
|
157
|
+
stdout: bootstrap.stdoutPath,
|
|
158
|
+
stderr: bootstrap.stderrPath
|
|
159
|
+
};
|
|
160
|
+
if (options.json) {
|
|
161
|
+
printEnvelope(true, data);
|
|
162
|
+
}
|
|
163
|
+
else {
|
|
164
|
+
console.log(`Local run started: ${taskId}`);
|
|
165
|
+
console.log(`PID: ${child.pid}`);
|
|
166
|
+
console.log(`Status: ${data.status}`);
|
|
167
|
+
if (data.outputDir)
|
|
168
|
+
console.log(`Output: ${data.outputDir}`);
|
|
169
|
+
console.log(`Bootstrap: ${bootstrap.dir}`);
|
|
170
|
+
console.log(`Control: octoparse local status ${taskId}`);
|
|
171
|
+
}
|
|
172
|
+
return EXIT_OK;
|
|
173
|
+
}
|
|
174
|
+
async function executeTask(taskId, options, loadTask) {
|
|
175
|
+
const host = new EngineHost();
|
|
176
|
+
let currentRunDir = '';
|
|
177
|
+
let currentRunId = '';
|
|
178
|
+
let currentLotId = '';
|
|
179
|
+
let currentTaskName = '';
|
|
180
|
+
let runDirReady = null;
|
|
181
|
+
let runStatus = 'running';
|
|
182
|
+
let controlServer = null;
|
|
183
|
+
let controlServerReady = null;
|
|
184
|
+
let artifactQueue = Promise.resolve();
|
|
185
|
+
let signalHandler = null;
|
|
186
|
+
let savedRows = 0;
|
|
187
|
+
let rowLimitReached = false;
|
|
188
|
+
let stopReason;
|
|
189
|
+
const runtimeConsole = maybeSuppressRuntimeConsole(options);
|
|
190
|
+
const detachedBootstrapDir = process.env[DETACHED_BOOTSTRAP_DIR_ENV];
|
|
191
|
+
const startedAt = new Date().toISOString();
|
|
192
|
+
const appendRunArtifact = (fileName, value) => {
|
|
193
|
+
if (!runDirReady)
|
|
194
|
+
return;
|
|
195
|
+
artifactQueue = artifactQueue
|
|
196
|
+
.then(async () => {
|
|
197
|
+
const runDir = await runDirReady;
|
|
198
|
+
if (!runDir)
|
|
199
|
+
return;
|
|
200
|
+
await appendJsonLine(join(runDir, fileName), value);
|
|
201
|
+
})
|
|
202
|
+
.catch(() => undefined);
|
|
203
|
+
};
|
|
204
|
+
const waitControlServer = async () => {
|
|
205
|
+
const ready = controlServerReady;
|
|
206
|
+
if (ready)
|
|
207
|
+
await ready.catch(() => undefined);
|
|
208
|
+
};
|
|
209
|
+
const updateControlStatus = async (status) => {
|
|
210
|
+
const server = controlServer;
|
|
211
|
+
if (server)
|
|
212
|
+
await server.updateStatus(status).catch(() => undefined);
|
|
213
|
+
};
|
|
214
|
+
const closeControlServer = async () => {
|
|
215
|
+
const server = controlServer;
|
|
216
|
+
if (server)
|
|
217
|
+
await server.close().catch(() => undefined);
|
|
218
|
+
};
|
|
219
|
+
host.on('run.started', (event) => {
|
|
220
|
+
currentRunId = event.runId;
|
|
221
|
+
currentLotId = event.lotId;
|
|
222
|
+
currentTaskName = event.taskName;
|
|
223
|
+
currentRunDir = join(options.outputDir, event.runId);
|
|
224
|
+
runDirReady = ensureRunDir(options.outputDir, event.runId);
|
|
225
|
+
if (detachedBootstrapDir) {
|
|
226
|
+
void writeDetachedBootstrap(detachedBootstrapDir, {
|
|
227
|
+
status: 'running',
|
|
228
|
+
runId: event.runId,
|
|
229
|
+
lotId: event.lotId,
|
|
230
|
+
taskId: event.taskId,
|
|
231
|
+
taskName: event.taskName,
|
|
232
|
+
outputDir: currentRunDir,
|
|
233
|
+
updatedAt: new Date().toISOString()
|
|
234
|
+
});
|
|
235
|
+
}
|
|
236
|
+
controlServerReady = runDirReady.then(async (runDir) => {
|
|
237
|
+
const server = await startRunControlServer({
|
|
238
|
+
runDir,
|
|
239
|
+
outputDir: options.outputDir,
|
|
240
|
+
runId: event.runId,
|
|
241
|
+
lotId: event.lotId,
|
|
242
|
+
taskId: event.taskId,
|
|
243
|
+
taskName: event.taskName,
|
|
244
|
+
onCommand: async (command) => {
|
|
245
|
+
if (command === 'status')
|
|
246
|
+
return runStatus;
|
|
247
|
+
if (command === 'pause') {
|
|
248
|
+
host.pause();
|
|
249
|
+
runStatus = 'paused';
|
|
250
|
+
appendRunArtifact('events.jsonl', { event: 'run.paused', runId: event.runId, taskId: event.taskId });
|
|
251
|
+
if (options.jsonl)
|
|
252
|
+
printRunJsonLine(runtimeConsole, { event: 'run.paused', runId: event.runId, taskId: event.taskId });
|
|
253
|
+
return runStatus;
|
|
254
|
+
}
|
|
255
|
+
if (command === 'resume') {
|
|
256
|
+
host.resume();
|
|
257
|
+
runStatus = 'running';
|
|
258
|
+
appendRunArtifact('events.jsonl', { event: 'run.resumed', runId: event.runId, taskId: event.taskId });
|
|
259
|
+
if (options.jsonl)
|
|
260
|
+
printRunJsonLine(runtimeConsole, { event: 'run.resumed', runId: event.runId, taskId: event.taskId });
|
|
261
|
+
return runStatus;
|
|
262
|
+
}
|
|
263
|
+
host.stop();
|
|
264
|
+
runStatus = 'stopping';
|
|
265
|
+
appendRunArtifact('events.jsonl', { event: 'run.stopping', runId: event.runId, taskId: event.taskId });
|
|
266
|
+
if (options.jsonl)
|
|
267
|
+
printRunJsonLine(runtimeConsole, { event: 'run.stopping', runId: event.runId, taskId: event.taskId });
|
|
268
|
+
return runStatus;
|
|
269
|
+
}
|
|
270
|
+
});
|
|
271
|
+
controlServer = server;
|
|
272
|
+
return server;
|
|
273
|
+
});
|
|
274
|
+
void controlServerReady.catch(() => undefined);
|
|
275
|
+
appendRunArtifact('events.jsonl', { event: 'run.started', ...event });
|
|
276
|
+
if (options.jsonl)
|
|
277
|
+
printRunJsonLine(runtimeConsole, { event: 'run.started', ...event });
|
|
278
|
+
});
|
|
279
|
+
host.on('row', (event) => {
|
|
280
|
+
if (options.maxRows !== undefined && savedRows >= options.maxRows) {
|
|
281
|
+
if (!rowLimitReached) {
|
|
282
|
+
rowLimitReached = true;
|
|
283
|
+
stopReason = 'max_rows';
|
|
284
|
+
runStatus = 'stopping';
|
|
285
|
+
void updateControlStatus(runStatus);
|
|
286
|
+
host.stop();
|
|
287
|
+
}
|
|
288
|
+
return;
|
|
289
|
+
}
|
|
290
|
+
savedRows += 1;
|
|
291
|
+
const rowEvent = { ...event, total: savedRows };
|
|
292
|
+
appendRunArtifact('rows.jsonl', event.data);
|
|
293
|
+
appendRunArtifact('events.jsonl', { event: 'row', ...rowEvent });
|
|
294
|
+
if (options.jsonl)
|
|
295
|
+
printRunJsonLine(runtimeConsole, { event: 'row', ...rowEvent });
|
|
296
|
+
if (options.maxRows !== undefined && savedRows >= options.maxRows && !rowLimitReached) {
|
|
297
|
+
rowLimitReached = true;
|
|
298
|
+
stopReason = 'max_rows';
|
|
299
|
+
runStatus = 'stopping';
|
|
300
|
+
void updateControlStatus(runStatus);
|
|
301
|
+
appendRunArtifact('events.jsonl', {
|
|
302
|
+
event: 'run.stopping',
|
|
303
|
+
runId: event.runId,
|
|
304
|
+
taskId,
|
|
305
|
+
reason: stopReason,
|
|
306
|
+
maxRows: options.maxRows,
|
|
307
|
+
total: savedRows
|
|
308
|
+
});
|
|
309
|
+
if (options.jsonl) {
|
|
310
|
+
printRunJsonLine(runtimeConsole, {
|
|
311
|
+
event: 'run.stopping',
|
|
312
|
+
runId: event.runId,
|
|
313
|
+
taskId,
|
|
314
|
+
reason: stopReason,
|
|
315
|
+
maxRows: options.maxRows,
|
|
316
|
+
total: savedRows
|
|
317
|
+
});
|
|
318
|
+
}
|
|
319
|
+
host.stop();
|
|
320
|
+
}
|
|
321
|
+
});
|
|
322
|
+
host.on('log', (event) => {
|
|
323
|
+
appendRunArtifact('logs.jsonl', event);
|
|
324
|
+
appendRunArtifact('events.jsonl', { event: 'log', ...event });
|
|
325
|
+
if (options.jsonl)
|
|
326
|
+
printRunJsonLine(runtimeConsole, { event: 'log', ...event });
|
|
327
|
+
else if (!options.json && (options.debugBridge || event.message.startsWith('runtime.'))) {
|
|
328
|
+
runtimeConsole.stderr(event.message);
|
|
329
|
+
}
|
|
330
|
+
});
|
|
331
|
+
host.on('captcha', (event) => {
|
|
332
|
+
appendRunArtifact('events.jsonl', { event: 'captcha', ...event });
|
|
333
|
+
if (options.jsonl)
|
|
334
|
+
printRunJsonLine(runtimeConsole, { event: 'captcha', ...event });
|
|
335
|
+
});
|
|
336
|
+
host.on('proxy', (event) => {
|
|
337
|
+
appendRunArtifact('events.jsonl', { event: 'proxy', ...event });
|
|
338
|
+
if (options.jsonl)
|
|
339
|
+
printRunJsonLine(runtimeConsole, { event: 'proxy', ...event });
|
|
340
|
+
});
|
|
341
|
+
try {
|
|
342
|
+
const task = await loadTask();
|
|
343
|
+
let interruptCount = 0;
|
|
344
|
+
const interrupted = new Promise((resolveInterrupted) => {
|
|
345
|
+
signalHandler = () => {
|
|
346
|
+
interruptCount += 1;
|
|
347
|
+
if (interruptCount > 1) {
|
|
348
|
+
process.exit(130);
|
|
349
|
+
}
|
|
350
|
+
if (!options.json && !options.jsonl) {
|
|
351
|
+
runtimeConsole.stderr('\nReceived Ctrl+C, stopping extraction...');
|
|
352
|
+
}
|
|
353
|
+
runStatus = 'stopping';
|
|
354
|
+
stopReason = 'interrupt';
|
|
355
|
+
void updateControlStatus(runStatus);
|
|
356
|
+
host.stop();
|
|
357
|
+
setTimeout(() => {
|
|
358
|
+
resolveInterrupted({
|
|
359
|
+
runId: currentRunId || `run_${safeFileName(task.taskId)}_interrupted`,
|
|
360
|
+
lotId: currentLotId || 'lot_interrupted',
|
|
361
|
+
taskId: task.taskId,
|
|
362
|
+
taskName: currentTaskName || task.taskName,
|
|
363
|
+
status: 'stopped',
|
|
364
|
+
total: savedRows,
|
|
365
|
+
outputDir: options.outputDir,
|
|
366
|
+
startedAt,
|
|
367
|
+
stoppedAt: new Date().toISOString(),
|
|
368
|
+
stopReason
|
|
369
|
+
});
|
|
370
|
+
}, 3_000);
|
|
371
|
+
};
|
|
372
|
+
process.once('SIGINT', signalHandler);
|
|
373
|
+
});
|
|
374
|
+
const runPromise = withTimeout(host.start(task, options), options.runTimeoutMs, () => {
|
|
375
|
+
runStatus = 'stopping';
|
|
376
|
+
stopReason = 'timeout';
|
|
377
|
+
void updateControlStatus(runStatus);
|
|
378
|
+
host.stop();
|
|
379
|
+
return `Run timeout after ${options.runTimeoutMs}ms`;
|
|
380
|
+
});
|
|
381
|
+
const summary = await Promise.race([runPromise, interrupted]);
|
|
382
|
+
runPromise.catch(() => undefined);
|
|
383
|
+
if (signalHandler) {
|
|
384
|
+
process.off('SIGINT', signalHandler);
|
|
385
|
+
signalHandler = null;
|
|
386
|
+
}
|
|
387
|
+
const runDir = currentRunDir || await ensureRunDir(options.outputDir, summary.runId);
|
|
388
|
+
const finalSummary = {
|
|
389
|
+
...summary,
|
|
390
|
+
total: savedRows || summary.total,
|
|
391
|
+
...(stopReason ? { stopReason } : {}),
|
|
392
|
+
...(options.maxRows !== undefined ? { maxRows: options.maxRows } : {})
|
|
393
|
+
};
|
|
394
|
+
runStatus = finalSummary.status;
|
|
395
|
+
await waitControlServer();
|
|
396
|
+
await updateControlStatus(runStatus);
|
|
397
|
+
await artifactQueue;
|
|
398
|
+
await writeRunSummary(runDir, { ...finalSummary, outputDir: runDir });
|
|
399
|
+
await appendJsonLine(join(runDir, 'events.jsonl'), { event: 'run.stopped', ...finalSummary, outputDir: runDir });
|
|
400
|
+
await closeControlServer();
|
|
401
|
+
await host.close();
|
|
402
|
+
if (detachedBootstrapDir) {
|
|
403
|
+
await writeDetachedBootstrap(detachedBootstrapDir, {
|
|
404
|
+
status: finalSummary.status,
|
|
405
|
+
runId: finalSummary.runId,
|
|
406
|
+
lotId: finalSummary.lotId,
|
|
407
|
+
outputDir: runDir,
|
|
408
|
+
total: finalSummary.total,
|
|
409
|
+
stoppedAt: finalSummary.stoppedAt,
|
|
410
|
+
updatedAt: new Date().toISOString()
|
|
411
|
+
});
|
|
412
|
+
}
|
|
413
|
+
if (options.jsonl) {
|
|
414
|
+
printRunJsonLine(runtimeConsole, { event: 'run.stopped', ...finalSummary, outputDir: runDir });
|
|
415
|
+
}
|
|
416
|
+
else if (options.json) {
|
|
417
|
+
printRunEnvelope(runtimeConsole, true, { ...finalSummary, outputDir: runDir });
|
|
418
|
+
}
|
|
419
|
+
else {
|
|
420
|
+
runtimeConsole.stdout(`Run completed: ${finalSummary.runId}`);
|
|
421
|
+
runtimeConsole.stdout(`Task: ${finalSummary.taskId}`);
|
|
422
|
+
runtimeConsole.stdout(`Rows: ${finalSummary.total}`);
|
|
423
|
+
if (finalSummary.stopReason === 'max_rows')
|
|
424
|
+
runtimeConsole.stdout(`Stop reason: max_rows (${finalSummary.maxRows})`);
|
|
425
|
+
runtimeConsole.stdout(`View data: ${localDataExportCommand(finalSummary)}`);
|
|
426
|
+
}
|
|
427
|
+
return EXIT_OK;
|
|
428
|
+
}
|
|
429
|
+
catch (error) {
|
|
430
|
+
if (signalHandler) {
|
|
431
|
+
process.off('SIGINT', signalHandler);
|
|
432
|
+
signalHandler = null;
|
|
433
|
+
}
|
|
434
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
435
|
+
if (currentRunDir && currentRunId) {
|
|
436
|
+
runStatus = 'failed';
|
|
437
|
+
await waitControlServer();
|
|
438
|
+
await updateControlStatus(runStatus);
|
|
439
|
+
await artifactQueue;
|
|
440
|
+
await appendJsonLine(join(currentRunDir, 'events.jsonl'), {
|
|
441
|
+
event: 'run.failed',
|
|
442
|
+
runId: currentRunId,
|
|
443
|
+
taskId,
|
|
444
|
+
error: message
|
|
445
|
+
}).catch(() => undefined);
|
|
446
|
+
await closeControlServer();
|
|
447
|
+
}
|
|
448
|
+
await host.close();
|
|
449
|
+
if (detachedBootstrapDir) {
|
|
450
|
+
await writeDetachedBootstrap(detachedBootstrapDir, {
|
|
451
|
+
status: 'failed',
|
|
452
|
+
runId: currentRunId || undefined,
|
|
453
|
+
taskId,
|
|
454
|
+
outputDir: currentRunDir || undefined,
|
|
455
|
+
error: message,
|
|
456
|
+
updatedAt: new Date().toISOString()
|
|
457
|
+
}).catch(() => undefined);
|
|
458
|
+
}
|
|
459
|
+
if (options.json || options.jsonl) {
|
|
460
|
+
printRunEnvelope(runtimeConsole, false, undefined, 'ENGINE_RUN_FAILED', message);
|
|
461
|
+
}
|
|
462
|
+
else {
|
|
463
|
+
runtimeConsole.stderr(`Run failed: ${message}`);
|
|
464
|
+
}
|
|
465
|
+
return EXIT_RUNTIME_FAILED;
|
|
466
|
+
}
|
|
467
|
+
}
|
|
468
|
+
export function localDataExportCommand(summary) {
|
|
469
|
+
return `octoparse data export ${summary.taskId} --source local --lot-id ${summary.lotId}`;
|
|
470
|
+
}
|
|
471
|
+
function parseRunOptions(taskId, args) {
|
|
472
|
+
return {
|
|
473
|
+
taskId,
|
|
474
|
+
taskFile: valueAfter(args, '--task-file'),
|
|
475
|
+
outputDir: resolve(valueAfter(args, '--output') ?? defaultRunsDir()),
|
|
476
|
+
headless: hasFlag(args, '--headless'),
|
|
477
|
+
json: hasFlag(args, '--json'),
|
|
478
|
+
jsonl: hasFlag(args, '--jsonl'),
|
|
479
|
+
chromePath: valueAfter(args, '--chrome-path'),
|
|
480
|
+
disableImage: hasFlag(args, '--disable-image'),
|
|
481
|
+
disableAD: hasFlag(args, '--disable-ad'),
|
|
482
|
+
runTimeoutMs: parsePositiveInt(valueAfter(args, '--timeout-ms'), 10 * 60 * 1000),
|
|
483
|
+
extensionTimeoutMs: parsePositiveInt(valueAfter(args, '--extension-timeout-ms'), 15 * 1000),
|
|
484
|
+
debugBridge: hasFlag(args, '--debug-bridge'),
|
|
485
|
+
detach: hasFlag(args, '--detach'),
|
|
486
|
+
maxRows: parseOptionalPositiveInt(valueAfter(args, '--max-rows'))
|
|
487
|
+
};
|
|
488
|
+
}
|
|
489
|
+
function parseOptionalPositiveInt(value) {
|
|
490
|
+
if (!value)
|
|
491
|
+
return undefined;
|
|
492
|
+
const parsed = Number.parseInt(value, 10);
|
|
493
|
+
return Number.isFinite(parsed) && parsed > 0 ? parsed : undefined;
|
|
494
|
+
}
|
|
495
|
+
function validateMaxRows(args) {
|
|
496
|
+
if (!hasFlag(args, '--max-rows'))
|
|
497
|
+
return null;
|
|
498
|
+
const raw = valueAfter(args, '--max-rows');
|
|
499
|
+
if (!raw || raw.startsWith('-'))
|
|
500
|
+
return '--max-rows requires a positive integer';
|
|
501
|
+
const parsed = Number.parseInt(raw, 10);
|
|
502
|
+
if (!Number.isFinite(parsed) || parsed <= 0 || String(parsed) !== raw.trim()) {
|
|
503
|
+
return '--max-rows requires a positive integer';
|
|
504
|
+
}
|
|
505
|
+
return null;
|
|
506
|
+
}
|
|
507
|
+
async function withTimeout(promise, timeoutMs, onTimeout) {
|
|
508
|
+
let timeout;
|
|
509
|
+
try {
|
|
510
|
+
return await Promise.race([
|
|
511
|
+
promise,
|
|
512
|
+
new Promise((_resolve, reject) => {
|
|
513
|
+
timeout = setTimeout(() => reject(new Error(onTimeout())), timeoutMs);
|
|
514
|
+
})
|
|
515
|
+
]);
|
|
516
|
+
}
|
|
517
|
+
finally {
|
|
518
|
+
if (timeout)
|
|
519
|
+
clearTimeout(timeout);
|
|
520
|
+
}
|
|
521
|
+
}
|
|
522
|
+
async function waitForDetachedStartup(taskId, child, bootstrapDir, timeoutMs) {
|
|
523
|
+
let childExited = false;
|
|
524
|
+
let childExitCode = null;
|
|
525
|
+
let childExitSignal = null;
|
|
526
|
+
child.once('exit', (code, signal) => {
|
|
527
|
+
childExited = true;
|
|
528
|
+
childExitCode = code;
|
|
529
|
+
childExitSignal = signal;
|
|
530
|
+
});
|
|
531
|
+
const deadline = Date.now() + timeoutMs;
|
|
532
|
+
while (Date.now() < deadline) {
|
|
533
|
+
const state = await readTaskControlState(taskId);
|
|
534
|
+
if (await isRunControlReachable(state))
|
|
535
|
+
return { ok: true, state };
|
|
536
|
+
const bootstrap = await readDetachedBootstrap(bootstrapDir);
|
|
537
|
+
if (bootstrap?.status === 'failed') {
|
|
538
|
+
return { ok: false, error: String(bootstrap.error ?? 'child failed') };
|
|
539
|
+
}
|
|
540
|
+
if (childExited) {
|
|
541
|
+
return {
|
|
542
|
+
ok: false,
|
|
543
|
+
error: `child exited early code=${childExitCode ?? ''} signal=${childExitSignal ?? ''}`.trim()
|
|
544
|
+
};
|
|
545
|
+
}
|
|
546
|
+
await sleep(200);
|
|
547
|
+
}
|
|
548
|
+
return { ok: true, state: null };
|
|
549
|
+
}
|
|
550
|
+
async function createDetachedBootstrap(taskId, outputDir) {
|
|
551
|
+
const stamp = new Date().toISOString().replace(/[-:.TZ]/g, '').slice(0, 14);
|
|
552
|
+
const dir = join(outputDir, `.detach_${safeFileName(taskId)}_${stamp}`);
|
|
553
|
+
await mkdir(dir, { recursive: true });
|
|
554
|
+
const stdoutPath = join(dir, 'stdout.log');
|
|
555
|
+
const stderrPath = join(dir, 'stderr.log');
|
|
556
|
+
const stdout = await open(stdoutPath, 'a');
|
|
557
|
+
const stderr = await open(stderrPath, 'a');
|
|
558
|
+
await writeDetachedBootstrap(dir, {
|
|
559
|
+
taskId,
|
|
560
|
+
status: 'spawning',
|
|
561
|
+
stdout: stdoutPath,
|
|
562
|
+
stderr: stderrPath,
|
|
563
|
+
createdAt: new Date().toISOString(),
|
|
564
|
+
updatedAt: new Date().toISOString()
|
|
565
|
+
});
|
|
566
|
+
return { dir, stdoutPath, stderrPath, stdout, stderr };
|
|
567
|
+
}
|
|
568
|
+
async function closeFileHandles(...handles) {
|
|
569
|
+
await Promise.all(handles.map((handle) => handle.close().catch(() => undefined)));
|
|
570
|
+
}
|
|
571
|
+
async function readDetachedBootstrap(dir) {
|
|
572
|
+
try {
|
|
573
|
+
return JSON.parse(await readFile(join(dir, 'bootstrap.json'), 'utf8'));
|
|
574
|
+
}
|
|
575
|
+
catch {
|
|
576
|
+
return null;
|
|
577
|
+
}
|
|
578
|
+
}
|
|
579
|
+
async function writeDetachedBootstrap(dir, patch) {
|
|
580
|
+
await mkdir(dir, { recursive: true });
|
|
581
|
+
const existing = await readDetachedBootstrap(dir) ?? {};
|
|
582
|
+
const next = { ...existing, ...patch };
|
|
583
|
+
await writeFile(join(dir, 'bootstrap.json'), `${JSON.stringify(next, null, 2)}\n`, 'utf8');
|
|
584
|
+
}
|
|
585
|
+
function sleep(ms) {
|
|
586
|
+
return new Promise((resolveSleep) => setTimeout(resolveSleep, ms));
|
|
587
|
+
}
|
|
588
|
+
function maybeSuppressRuntimeConsole(options) {
|
|
589
|
+
if (options.debugBridge || process.env.OCTOPARSE_SHOW_RUNTIME_STDIO === '1')
|
|
590
|
+
return nativeRuntimeConsole();
|
|
591
|
+
const originalStdoutWrite = process.stdout.write.bind(process.stdout);
|
|
592
|
+
const originalStderrWrite = process.stderr.write.bind(process.stderr);
|
|
593
|
+
process.stdout.write = ((..._args) => true);
|
|
594
|
+
process.stderr.write = ((..._args) => true);
|
|
595
|
+
return {
|
|
596
|
+
stdout(line) {
|
|
597
|
+
originalStdoutWrite(`${line}\n`);
|
|
598
|
+
},
|
|
599
|
+
stderr(line) {
|
|
600
|
+
originalStderrWrite(`${line}\n`);
|
|
601
|
+
}
|
|
602
|
+
};
|
|
603
|
+
}
|
|
604
|
+
function printRunJsonLine(runtimeConsole, value) {
|
|
605
|
+
runtimeConsole.stdout(JSON.stringify(value));
|
|
606
|
+
}
|
|
607
|
+
function nativeRuntimeConsole() {
|
|
608
|
+
return {
|
|
609
|
+
stdout(line) {
|
|
610
|
+
console.log(line);
|
|
611
|
+
},
|
|
612
|
+
stderr(line) {
|
|
613
|
+
console.error(line);
|
|
614
|
+
}
|
|
615
|
+
};
|
|
616
|
+
}
|
|
617
|
+
function printRunEnvelope(runtimeConsole, ok, data, code, message) {
|
|
618
|
+
const payload = ok
|
|
619
|
+
? { ok: true, data }
|
|
620
|
+
: { ok: false, error: { code: code ?? 'ERROR', message: message ?? 'Unknown error' } };
|
|
621
|
+
runtimeConsole.stdout(JSON.stringify(payload));
|
|
622
|
+
}
|