octoparse-cli 0.1.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +271 -0
- package/RUNTIME_SECURITY_NOTICE.txt +41 -0
- package/SECURITY.md +24 -0
- package/dist/cli/args.js +34 -0
- package/dist/cli/help.js +214 -0
- package/dist/cli/output.js +39 -0
- package/dist/commands/auth.js +283 -0
- package/dist/commands/capabilities.js +118 -0
- package/dist/commands/cloud.js +241 -0
- package/dist/commands/data.js +220 -0
- package/dist/commands/doctor.js +73 -0
- package/dist/commands/env.js +63 -0
- package/dist/commands/local.js +251 -0
- package/dist/commands/run.js +622 -0
- package/dist/commands/runs.js +171 -0
- package/dist/commands/task.js +101 -0
- package/dist/index.js +133 -0
- package/dist/runtime/account-capabilities.js +71 -0
- package/dist/runtime/api-client.js +290 -0
- package/dist/runtime/artifacts.js +33 -0
- package/dist/runtime/auth.js +94 -0
- package/dist/runtime/bridge-hub.js +173 -0
- package/dist/runtime/client-headers.js +23 -0
- package/dist/runtime/cloud-data.js +75 -0
- package/dist/runtime/config.js +48 -0
- package/dist/runtime/data-exporter.js +267 -0
- package/dist/runtime/engine-host.js +449 -0
- package/dist/runtime/local-runs.js +92 -0
- package/dist/runtime/naming.js +13 -0
- package/dist/runtime/run-control.js +363 -0
- package/dist/runtime/run-services.js +380 -0
- package/dist/runtime/security-notice.js +78 -0
- package/dist/runtime/task-definition-provider.js +282 -0
- package/dist/types.js +4 -0
- package/examples/minimal-task.json +6 -0
- package/examples/navigate-example-task.json +6 -0
- package/node_modules/@octopus/bpmn/index.js +3 -0
- package/node_modules/@octopus/bpmn/lib/Context.js +245 -0
- package/node_modules/@octopus/bpmn/lib/Definition.js +258 -0
- package/node_modules/@octopus/bpmn/lib/Engine.js +275 -0
- package/node_modules/@octopus/bpmn/lib/PrematureStopError.js +7 -0
- package/node_modules/@octopus/bpmn/lib/activities/Activity.js +202 -0
- package/node_modules/@octopus/bpmn/lib/activities/BaseProcess.js +308 -0
- package/node_modules/@octopus/bpmn/lib/activities/BaseTask.js +145 -0
- package/node_modules/@octopus/bpmn/lib/activities/BoundaryEvent.js +12 -0
- package/node_modules/@octopus/bpmn/lib/activities/Dummy.js +10 -0
- package/node_modules/@octopus/bpmn/lib/activities/EventDefinition.js +99 -0
- package/node_modules/@octopus/bpmn/lib/activities/Flow.js +52 -0
- package/node_modules/@octopus/bpmn/lib/activities/Form.js +67 -0
- package/node_modules/@octopus/bpmn/lib/activities/InputOutput.js +53 -0
- package/node_modules/@octopus/bpmn/lib/activities/IntermediateCatchEvent.js +12 -0
- package/node_modules/@octopus/bpmn/lib/activities/MessageFlow.js +19 -0
- package/node_modules/@octopus/bpmn/lib/activities/MultiInstanceLoopCharacteristics.js +160 -0
- package/node_modules/@octopus/bpmn/lib/activities/Properties.js +27 -0
- package/node_modules/@octopus/bpmn/lib/activities/SequenceFlow.js +56 -0
- package/node_modules/@octopus/bpmn/lib/activities/ServiceConnector.js +71 -0
- package/node_modules/@octopus/bpmn/lib/context-helper.js +198 -0
- package/node_modules/@octopus/bpmn/lib/events/EndEvent.js +22 -0
- package/node_modules/@octopus/bpmn/lib/events/ErrorEvent.js +41 -0
- package/node_modules/@octopus/bpmn/lib/events/MessageEvent.js +19 -0
- package/node_modules/@octopus/bpmn/lib/events/StartEvent.js +55 -0
- package/node_modules/@octopus/bpmn/lib/events/TimerEvent.js +75 -0
- package/node_modules/@octopus/bpmn/lib/expressions.js +41 -0
- package/node_modules/@octopus/bpmn/lib/gateways/ExclusiveGateway.js +86 -0
- package/node_modules/@octopus/bpmn/lib/gateways/InclusiveGateway.js +56 -0
- package/node_modules/@octopus/bpmn/lib/gateways/ParallelGateway.js +195 -0
- package/node_modules/@octopus/bpmn/lib/getPropertyValue.js +83 -0
- package/node_modules/@octopus/bpmn/lib/index.js +6 -0
- package/node_modules/@octopus/bpmn/lib/mapper.js +55 -0
- package/node_modules/@octopus/bpmn/lib/parameter.js +119 -0
- package/node_modules/@octopus/bpmn/lib/script-helper.js +45 -0
- package/node_modules/@octopus/bpmn/lib/tasks/ManualTask.js +31 -0
- package/node_modules/@octopus/bpmn/lib/tasks/ReceiveTask.js +31 -0
- package/node_modules/@octopus/bpmn/lib/tasks/ScriptTask.js +35 -0
- package/node_modules/@octopus/bpmn/lib/tasks/SendTask.js +16 -0
- package/node_modules/@octopus/bpmn/lib/tasks/ServiceTask.js +68 -0
- package/node_modules/@octopus/bpmn/lib/tasks/SubProcess.js +17 -0
- package/node_modules/@octopus/bpmn/lib/tasks/Task.js +16 -0
- package/node_modules/@octopus/bpmn/lib/tasks/UserTask.js +47 -0
- package/node_modules/@octopus/bpmn/lib/transformer.js +13 -0
- package/node_modules/@octopus/bpmn/lib/validation.js +111 -0
- package/node_modules/@octopus/bpmn/package.json +17 -0
- package/node_modules/@octopus/bpmn/types/bpmn.d.ts +85 -0
- package/node_modules/@octopus/engine/README.md +370 -0
- package/node_modules/@octopus/engine/dist/actions/BackPreWebPageAction.d.ts +4 -0
- package/node_modules/@octopus/engine/dist/actions/BackPreWebPageAction.js +1 -0
- package/node_modules/@octopus/engine/dist/actions/BaseAction.d.ts +339 -0
- package/node_modules/@octopus/engine/dist/actions/BaseAction.js +1559 -0
- package/node_modules/@octopus/engine/dist/actions/BranchAction.d.ts +9 -0
- package/node_modules/@octopus/engine/dist/actions/BranchAction.js +1 -0
- package/node_modules/@octopus/engine/dist/actions/ClickAction.d.ts +22 -0
- package/node_modules/@octopus/engine/dist/actions/ClickAction.js +1 -0
- package/node_modules/@octopus/engine/dist/actions/ConditionAction.d.ts +4 -0
- package/node_modules/@octopus/engine/dist/actions/ConditionAction.js +1 -0
- package/node_modules/@octopus/engine/dist/actions/EmptyAction.d.ts +4 -0
- package/node_modules/@octopus/engine/dist/actions/EmptyAction.js +12 -0
- package/node_modules/@octopus/engine/dist/actions/EnterCaptchaAction.d.ts +28 -0
- package/node_modules/@octopus/engine/dist/actions/EnterCaptchaAction.js +1 -0
- package/node_modules/@octopus/engine/dist/actions/EnterTextAction.d.ts +20 -0
- package/node_modules/@octopus/engine/dist/actions/EnterTextAction.js +1 -0
- package/node_modules/@octopus/engine/dist/actions/ExtractDataAction.d.ts +40 -0
- package/node_modules/@octopus/engine/dist/actions/ExtractDataAction.js +1 -0
- package/node_modules/@octopus/engine/dist/actions/LoopAction.d.ts +41 -0
- package/node_modules/@octopus/engine/dist/actions/LoopAction.js +526 -0
- package/node_modules/@octopus/engine/dist/actions/LoopStartAction.d.ts +47 -0
- package/node_modules/@octopus/engine/dist/actions/LoopStartAction.js +607 -0
- package/node_modules/@octopus/engine/dist/actions/MouseOverAction.d.ts +8 -0
- package/node_modules/@octopus/engine/dist/actions/MouseOverAction.js +34 -0
- package/node_modules/@octopus/engine/dist/actions/NavigateAction.d.ts +38 -0
- package/node_modules/@octopus/engine/dist/actions/NavigateAction.js +535 -0
- package/node_modules/@octopus/engine/dist/actions/SwitchComboAction.d.ts +13 -0
- package/node_modules/@octopus/engine/dist/actions/SwitchComboAction.js +69 -0
- package/node_modules/@octopus/engine/dist/browser.d.ts +17 -0
- package/node_modules/@octopus/engine/dist/browser.js +157 -0
- package/node_modules/@octopus/engine/dist/browserProxy.d.ts +90 -0
- package/node_modules/@octopus/engine/dist/browserProxy.js +1 -0
- package/node_modules/@octopus/engine/dist/configs/BaseConfig.d.ts +20 -0
- package/node_modules/@octopus/engine/dist/configs/BaseConfig.js +88 -0
- package/node_modules/@octopus/engine/dist/configs/BranchConfig.d.ts +7 -0
- package/node_modules/@octopus/engine/dist/configs/BranchConfig.js +1 -0
- package/node_modules/@octopus/engine/dist/configs/ClickConfig.d.ts +36 -0
- package/node_modules/@octopus/engine/dist/configs/ClickConfig.js +65 -0
- package/node_modules/@octopus/engine/dist/configs/EnterCaptchaConfig.d.ts +19 -0
- package/node_modules/@octopus/engine/dist/configs/EnterCaptchaConfig.js +25 -0
- package/node_modules/@octopus/engine/dist/configs/EnterTextConfig.d.ts +24 -0
- package/node_modules/@octopus/engine/dist/configs/EnterTextConfig.js +36 -0
- package/node_modules/@octopus/engine/dist/configs/ExtractDataConfig.d.ts +12 -0
- package/node_modules/@octopus/engine/dist/configs/ExtractDataConfig.js +1 -0
- package/node_modules/@octopus/engine/dist/configs/LoopConfig.d.ts +25 -0
- package/node_modules/@octopus/engine/dist/configs/LoopConfig.js +40 -0
- package/node_modules/@octopus/engine/dist/configs/LoopStartConfig.d.ts +4 -0
- package/node_modules/@octopus/engine/dist/configs/LoopStartConfig.js +12 -0
- package/node_modules/@octopus/engine/dist/configs/MouseOverConfig.d.ts +8 -0
- package/node_modules/@octopus/engine/dist/configs/MouseOverConfig.js +15 -0
- package/node_modules/@octopus/engine/dist/configs/NavigateConfig.d.ts +41 -0
- package/node_modules/@octopus/engine/dist/configs/NavigateConfig.js +121 -0
- package/node_modules/@octopus/engine/dist/configs/SwitchComboConfig.d.ts +8 -0
- package/node_modules/@octopus/engine/dist/configs/SwitchComboConfig.js +15 -0
- package/node_modules/@octopus/engine/dist/enums/index.d.ts +419 -0
- package/node_modules/@octopus/engine/dist/enums/index.js +314 -0
- package/node_modules/@octopus/engine/dist/extension/BrowserWebSocketTransport-D_zAGZMQ.js +1 -0
- package/node_modules/@octopus/engine/dist/extension/LaunchOptions-DxvePrV4.js +6 -0
- package/node_modules/@octopus/engine/dist/extension/NodeWebSocketTransport-BTgRVB7Z.js +6 -0
- package/node_modules/@octopus/engine/dist/extension/background.js +396 -0
- package/node_modules/@octopus/engine/dist/extension/bidi-C_GIZ8Uz.js +131 -0
- package/node_modules/@octopus/engine/dist/extension/manifest.json +27 -0
- package/node_modules/@octopus/engine/dist/extension/src/content/anti-detection.js +1 -0
- package/node_modules/@octopus/engine/dist/extension-bridge/BaseExtensionBridge.d.ts +21 -0
- package/node_modules/@octopus/engine/dist/extension-bridge/BaseExtensionBridge.js +117 -0
- package/node_modules/@octopus/engine/dist/extension-bridge/SessionExtensionBridge.d.ts +17 -0
- package/node_modules/@octopus/engine/dist/extension-bridge/SessionExtensionBridge.js +29 -0
- package/node_modules/@octopus/engine/dist/extension-bridge/index.d.ts +2 -0
- package/node_modules/@octopus/engine/dist/extension-bridge/index.js +5 -0
- package/node_modules/@octopus/engine/dist/extension-bridge/types.d.ts +159 -0
- package/node_modules/@octopus/engine/dist/extension-bridge/types.js +5 -0
- package/node_modules/@octopus/engine/dist/extensions/ublock-origin/uBlock0.chromium.tar.xz +0 -0
- package/node_modules/@octopus/engine/dist/extensions/ublock-origin-lite/uBOLite.chromium.tar.xz +0 -0
- package/node_modules/@octopus/engine/dist/index.d.ts +169 -0
- package/node_modules/@octopus/engine/dist/index.js +1 -0
- package/node_modules/@octopus/engine/dist/models/actionItem.d.ts +16 -0
- package/node_modules/@octopus/engine/dist/models/actionItem.js +15 -0
- package/node_modules/@octopus/engine/dist/models/conditionCheckArgs.d.ts +11 -0
- package/node_modules/@octopus/engine/dist/models/conditionCheckArgs.js +11 -0
- package/node_modules/@octopus/engine/dist/models/customizeCookie.d.ts +14 -0
- package/node_modules/@octopus/engine/dist/models/customizeCookie.js +6 -0
- package/node_modules/@octopus/engine/dist/models/downloadFileConfig.d.ts +17 -0
- package/node_modules/@octopus/engine/dist/models/downloadFileConfig.js +26 -0
- package/node_modules/@octopus/engine/dist/models/elementNotFoundArgs.d.ts +8 -0
- package/node_modules/@octopus/engine/dist/models/elementNotFoundArgs.js +12 -0
- package/node_modules/@octopus/engine/dist/models/elementNotFoundError.d.ts +2 -0
- package/node_modules/@octopus/engine/dist/models/elementNotFoundError.js +6 -0
- package/node_modules/@octopus/engine/dist/models/extractItem.d.ts +37 -0
- package/node_modules/@octopus/engine/dist/models/extractItem.js +35 -0
- package/node_modules/@octopus/engine/dist/models/extractTemplate.d.ts +11 -0
- package/node_modules/@octopus/engine/dist/models/extractTemplate.js +48 -0
- package/node_modules/@octopus/engine/dist/models/extractTextItem.d.ts +10 -0
- package/node_modules/@octopus/engine/dist/models/extractTextItem.js +17 -0
- package/node_modules/@octopus/engine/dist/models/globalConfig.d.ts +5 -0
- package/node_modules/@octopus/engine/dist/models/globalConfig.js +1 -0
- package/node_modules/@octopus/engine/dist/models/httpHeader.d.ts +4 -0
- package/node_modules/@octopus/engine/dist/models/httpHeader.js +10 -0
- package/node_modules/@octopus/engine/dist/models/operation.d.ts +27 -0
- package/node_modules/@octopus/engine/dist/models/operation.js +242 -0
- package/node_modules/@octopus/engine/dist/models/retryCondition.d.ts +7 -0
- package/node_modules/@octopus/engine/dist/models/retryCondition.js +10 -0
- package/node_modules/@octopus/engine/dist/models/task.d.ts +89 -0
- package/node_modules/@octopus/engine/dist/models/task.js +120 -0
- package/node_modules/@octopus/engine/dist/models/trigger.d.ts +66 -0
- package/node_modules/@octopus/engine/dist/models/trigger.js +117 -0
- package/node_modules/@octopus/engine/dist/package.json +26 -0
- package/node_modules/@octopus/engine/dist/public-types.d.ts +13 -0
- package/node_modules/@octopus/engine/dist/public-types.js +2 -0
- package/node_modules/@octopus/engine/dist/settings.d.ts +41 -0
- package/node_modules/@octopus/engine/dist/settings.js +20 -0
- package/node_modules/@octopus/engine/dist/solvers/captcha/ClickCaptchaSolver.d.ts +6 -0
- package/node_modules/@octopus/engine/dist/solvers/captcha/ClickCaptchaSolver.js +1 -0
- package/node_modules/@octopus/engine/dist/solvers/captcha/HCaptchaSolver.d.ts +4 -0
- package/node_modules/@octopus/engine/dist/solvers/captcha/HCaptchaSolver.js +73 -0
- package/node_modules/@octopus/engine/dist/solvers/captcha/ImageCaptchaSolver.d.ts +2 -0
- package/node_modules/@octopus/engine/dist/solvers/captcha/ImageCaptchaSolver.js +74 -0
- package/node_modules/@octopus/engine/dist/solvers/captcha/RecaptchaSolver.d.ts +9 -0
- package/node_modules/@octopus/engine/dist/solvers/captcha/RecaptchaSolver.js +371 -0
- package/node_modules/@octopus/engine/dist/solvers/captcha/SliderCaptchaSolver.d.ts +6 -0
- package/node_modules/@octopus/engine/dist/solvers/captcha/SliderCaptchaSolver.js +184 -0
- package/node_modules/@octopus/engine/dist/solvers/captcha/SlidingTrajectory.d.ts +50 -0
- package/node_modules/@octopus/engine/dist/solvers/captcha/SlidingTrajectory.js +125 -0
- package/node_modules/@octopus/engine/dist/solvers/captcha/types.d.ts +68 -0
- package/node_modules/@octopus/engine/dist/solvers/captcha/types.js +34 -0
- package/node_modules/@octopus/engine/dist/solvers/captcha/utils.d.ts +2 -0
- package/node_modules/@octopus/engine/dist/solvers/captcha/utils.js +15 -0
- package/node_modules/@octopus/engine/dist/translator/actionFactory.d.ts +6 -0
- package/node_modules/@octopus/engine/dist/translator/actionFactory.js +1 -0
- package/node_modules/@octopus/engine/dist/translator/activityTypeEnum.d.ts +22 -0
- package/node_modules/@octopus/engine/dist/translator/activityTypeEnum.js +1 -0
- package/node_modules/@octopus/engine/dist/translator/backPreWebPageAction.d.ts +5 -0
- package/node_modules/@octopus/engine/dist/translator/backPreWebPageAction.js +1 -0
- package/node_modules/@octopus/engine/dist/translator/baseAction.d.ts +31 -0
- package/node_modules/@octopus/engine/dist/translator/baseAction.js +1 -0
- package/node_modules/@octopus/engine/dist/translator/breakActivity.d.ts +5 -0
- package/node_modules/@octopus/engine/dist/translator/breakActivity.js +1 -0
- package/node_modules/@octopus/engine/dist/translator/clickAction.d.ts +5 -0
- package/node_modules/@octopus/engine/dist/translator/clickAction.js +1 -0
- package/node_modules/@octopus/engine/dist/translator/completeWF.d.ts +5 -0
- package/node_modules/@octopus/engine/dist/translator/completeWF.js +1 -0
- package/node_modules/@octopus/engine/dist/translator/conditionAction.d.ts +6 -0
- package/node_modules/@octopus/engine/dist/translator/conditionAction.js +1 -0
- package/node_modules/@octopus/engine/dist/translator/emptyAction.d.ts +5 -0
- package/node_modules/@octopus/engine/dist/translator/emptyAction.js +1 -0
- package/node_modules/@octopus/engine/dist/translator/enterCapachaAction.d.ts +5 -0
- package/node_modules/@octopus/engine/dist/translator/enterCapachaAction.js +1 -0
- package/node_modules/@octopus/engine/dist/translator/enterTextAction.d.ts +5 -0
- package/node_modules/@octopus/engine/dist/translator/enterTextAction.js +1 -0
- package/node_modules/@octopus/engine/dist/translator/extractDataAction.d.ts +13 -0
- package/node_modules/@octopus/engine/dist/translator/extractDataAction.js +1 -0
- package/node_modules/@octopus/engine/dist/translator/loopAction.d.ts +5 -0
- package/node_modules/@octopus/engine/dist/translator/loopAction.js +1 -0
- package/node_modules/@octopus/engine/dist/translator/mouseOverAction.d.ts +5 -0
- package/node_modules/@octopus/engine/dist/translator/mouseOverAction.js +19 -0
- package/node_modules/@octopus/engine/dist/translator/navigateAction.d.ts +5 -0
- package/node_modules/@octopus/engine/dist/translator/navigateAction.js +117 -0
- package/node_modules/@octopus/engine/dist/translator/rootAction.d.ts +6 -0
- package/node_modules/@octopus/engine/dist/translator/rootAction.js +80 -0
- package/node_modules/@octopus/engine/dist/translator/switchCombo2Action.d.ts +5 -0
- package/node_modules/@octopus/engine/dist/translator/switchCombo2Action.js +19 -0
- package/node_modules/@octopus/engine/dist/translator/translator.d.ts +1 -0
- package/node_modules/@octopus/engine/dist/translator/translator.js +36 -0
- package/node_modules/@octopus/engine/dist/type.d.ts +25 -0
- package/node_modules/@octopus/engine/dist/type.js +2 -0
- package/node_modules/@octopus/engine/dist/types/browser.d.ts +191 -0
- package/node_modules/@octopus/engine/dist/types/browser.js +1 -0
- package/node_modules/@octopus/engine/dist/types/browserManager.d.ts +41 -0
- package/node_modules/@octopus/engine/dist/types/browserManager.js +1 -0
- package/node_modules/@octopus/engine/dist/types/index.d.ts +40 -0
- package/node_modules/@octopus/engine/dist/types/index.js +2 -0
- package/node_modules/@octopus/engine/dist/types/plugin.d.ts +29 -0
- package/node_modules/@octopus/engine/dist/types/plugin.js +2 -0
- package/node_modules/@octopus/engine/dist/utils/AsyncEmitter.d.ts +15 -0
- package/node_modules/@octopus/engine/dist/utils/AsyncEmitter.js +1 -0
- package/node_modules/@octopus/engine/dist/utils/DataStore.d.ts +58 -0
- package/node_modules/@octopus/engine/dist/utils/DataStore.js +1 -0
- package/node_modules/@octopus/engine/dist/utils/DateTimeFormatHelper.d.ts +22 -0
- package/node_modules/@octopus/engine/dist/utils/DateTimeFormatHelper.js +173 -0
- package/node_modules/@octopus/engine/dist/utils/FileDownloader.d.ts +108 -0
- package/node_modules/@octopus/engine/dist/utils/FileDownloader.js +1 -0
- package/node_modules/@octopus/engine/dist/utils/HttpRequester.d.ts +43 -0
- package/node_modules/@octopus/engine/dist/utils/HttpRequester.js +174 -0
- package/node_modules/@octopus/engine/dist/utils/JsonParser.d.ts +95 -0
- package/node_modules/@octopus/engine/dist/utils/JsonParser.js +439 -0
- package/node_modules/@octopus/engine/dist/utils/Operations.d.ts +27 -0
- package/node_modules/@octopus/engine/dist/utils/Operations.js +115 -0
- package/node_modules/@octopus/engine/dist/utils/index.d.ts +28 -0
- package/node_modules/@octopus/engine/dist/utils/index.js +356 -0
- package/node_modules/@octopus/engine/package.json +58 -0
- package/package.json +79 -0
- package/schemas/capabilities-v1.schema.json +234 -0
- package/schemas/detached-bootstrap-v1.schema.json +42 -0
- package/schemas/json-envelope-v1.schema.json +39 -0
- package/schemas/run-event-v1.schema.json +47 -0
|
@@ -0,0 +1,283 @@
|
|
|
1
|
+
import { spawn } from 'node:child_process';
|
|
2
|
+
import { homedir } from 'node:os';
|
|
3
|
+
import { join } from 'node:path';
|
|
4
|
+
import { firstPositionalArg, hasFlag, valueAfter } from '../cli/args.js';
|
|
5
|
+
import { printEnvelope, printUsageError } from '../cli/output.js';
|
|
6
|
+
import { API_BASE_URL_ENV, ApiRequestError, validateApiKey } from '../runtime/api-client.js';
|
|
7
|
+
import { API_KEY_ENV, maskApiKey, normalizeApiKey, removeApiKey, resolveAuth, saveApiKey } from '../runtime/auth.js';
|
|
8
|
+
import { EXIT_OK, EXIT_OPERATION_FAILED } from '../types.js';
|
|
9
|
+
export const API_KEYS_URL = 'https://www.octoparse.com/console/account-center/api-keys';
|
|
10
|
+
export async function authCommand(subcommand, args) {
|
|
11
|
+
const json = hasFlag([subcommand ?? '', ...args], '--json');
|
|
12
|
+
if (subcommand === 'login') {
|
|
13
|
+
return authLogin(args);
|
|
14
|
+
}
|
|
15
|
+
if (subcommand === 'status') {
|
|
16
|
+
return authStatus(args);
|
|
17
|
+
}
|
|
18
|
+
if (subcommand === 'logout') {
|
|
19
|
+
return authLogout(args);
|
|
20
|
+
}
|
|
21
|
+
return printUsageError(json, 'Error: invalid auth subcommand', 'Usage: octoparse auth <login|status|logout> [--json]');
|
|
22
|
+
}
|
|
23
|
+
export async function ensureAuthenticated(json) {
|
|
24
|
+
const auth = await resolveAuth();
|
|
25
|
+
if (auth.authenticated)
|
|
26
|
+
return EXIT_OK;
|
|
27
|
+
return printAuthRequired(json);
|
|
28
|
+
}
|
|
29
|
+
export function printAuthRequired(json) {
|
|
30
|
+
const message = [
|
|
31
|
+
'API key required.',
|
|
32
|
+
`Create one at ${API_KEYS_URL}, then run "octoparse auth login".`,
|
|
33
|
+
`For CI, set ${API_KEY_ENV}.`
|
|
34
|
+
].join(' ');
|
|
35
|
+
if (json) {
|
|
36
|
+
printEnvelope(false, undefined, 'AUTH_REQUIRED', message);
|
|
37
|
+
}
|
|
38
|
+
else {
|
|
39
|
+
console.error('Authentication failed: an API key is required.');
|
|
40
|
+
console.error('');
|
|
41
|
+
console.error('Create an API key:');
|
|
42
|
+
console.error(` ${API_KEYS_URL}`);
|
|
43
|
+
console.error('');
|
|
44
|
+
console.error('Then run:');
|
|
45
|
+
console.error(' octoparse auth login');
|
|
46
|
+
console.error('');
|
|
47
|
+
console.error(`For CI or scripts, set ${API_KEY_ENV}.`);
|
|
48
|
+
}
|
|
49
|
+
return EXIT_OPERATION_FAILED;
|
|
50
|
+
}
|
|
51
|
+
async function authLogin(args) {
|
|
52
|
+
const json = hasFlag(args, '--json');
|
|
53
|
+
const readFromStdin = hasFlag(args, '--stdin');
|
|
54
|
+
const providedApiKey = normalizeApiKey(firstPositionalArg(args, ['--api-base-url']) ?? '');
|
|
55
|
+
const shouldOpen = shouldOpenApiKeyPage(args, json, readFromStdin, Boolean(providedApiKey));
|
|
56
|
+
try {
|
|
57
|
+
if (!providedApiKey && !readFromStdin && !json) {
|
|
58
|
+
printLoginInstructions(shouldOpen);
|
|
59
|
+
}
|
|
60
|
+
if (shouldOpen) {
|
|
61
|
+
await openUrl(API_KEYS_URL);
|
|
62
|
+
}
|
|
63
|
+
const apiKey = providedApiKey
|
|
64
|
+
? providedApiKey
|
|
65
|
+
: readFromStdin
|
|
66
|
+
? await readApiKeyFromStdin()
|
|
67
|
+
: await readSecretFromTty('Paste API key: ');
|
|
68
|
+
const validation = await validateApiKey({ apiKey, baseUrl: valueAfter(args, '--api-base-url') });
|
|
69
|
+
const credentials = await saveApiKey(apiKey);
|
|
70
|
+
const status = {
|
|
71
|
+
authenticated: true,
|
|
72
|
+
source: 'file',
|
|
73
|
+
keyPreview: maskApiKey(credentials.apiKey),
|
|
74
|
+
credentialsFile: join(homedir(), '.octoparse', 'credentials.json'),
|
|
75
|
+
verified: true,
|
|
76
|
+
apiBaseUrl: validation.baseUrl
|
|
77
|
+
};
|
|
78
|
+
if (json) {
|
|
79
|
+
printEnvelope(true, status);
|
|
80
|
+
}
|
|
81
|
+
else {
|
|
82
|
+
console.log(`API key verified and saved: ${status.keyPreview}`);
|
|
83
|
+
console.log(`API: ${status.apiBaseUrl}`);
|
|
84
|
+
console.log(`Credentials: ${status.credentialsFile}`);
|
|
85
|
+
console.log('');
|
|
86
|
+
console.log('Next:');
|
|
87
|
+
console.log(' octoparse task list');
|
|
88
|
+
}
|
|
89
|
+
return EXIT_OK;
|
|
90
|
+
}
|
|
91
|
+
catch (error) {
|
|
92
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
93
|
+
const code = error instanceof ApiRequestError ? error.code : 'AUTH_LOGIN_FAILED';
|
|
94
|
+
if (json)
|
|
95
|
+
printEnvelope(false, undefined, code, message);
|
|
96
|
+
else {
|
|
97
|
+
console.error(`Login failed: ${message}`);
|
|
98
|
+
console.error('API key was not saved.');
|
|
99
|
+
if (code === 'AUTH_INVALID') {
|
|
100
|
+
console.error('');
|
|
101
|
+
console.error('Check:');
|
|
102
|
+
console.error(' 1. Whether the full API key was copied');
|
|
103
|
+
console.error(` 2. Whether the API key belongs to the current API environment, or check ${API_BASE_URL_ENV} / env status`);
|
|
104
|
+
console.error(` 3. Create a new API key: ${API_KEYS_URL}`);
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
return EXIT_OPERATION_FAILED;
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
function printLoginInstructions(willOpenBrowser) {
|
|
111
|
+
console.log('Octo Engine uses your Octoparse API key to verify your account and access tasks.');
|
|
112
|
+
console.log('');
|
|
113
|
+
if (willOpenBrowser) {
|
|
114
|
+
console.log('Opening API key page:');
|
|
115
|
+
}
|
|
116
|
+
else {
|
|
117
|
+
console.log('Create API key:');
|
|
118
|
+
}
|
|
119
|
+
console.log(` ${API_KEYS_URL}`);
|
|
120
|
+
console.log('');
|
|
121
|
+
if (willOpenBrowser) {
|
|
122
|
+
console.log('If the browser did not open, copy the URL above.');
|
|
123
|
+
}
|
|
124
|
+
console.log('Create an API key in the browser, then paste it here.');
|
|
125
|
+
console.log('The key will be verified before it is saved locally.');
|
|
126
|
+
console.log('');
|
|
127
|
+
}
|
|
128
|
+
function shouldOpenApiKeyPage(args, json, readFromStdin, hasProvidedApiKey) {
|
|
129
|
+
if (json || readFromStdin || hasProvidedApiKey || hasFlag(args, '--no-open'))
|
|
130
|
+
return false;
|
|
131
|
+
if (process.env.CI === 'true')
|
|
132
|
+
return false;
|
|
133
|
+
return Boolean(process.stdin.isTTY && process.stdout.isTTY);
|
|
134
|
+
}
|
|
135
|
+
function openUrl(url) {
|
|
136
|
+
const command = process.platform === 'darwin'
|
|
137
|
+
? 'open'
|
|
138
|
+
: process.platform === 'win32'
|
|
139
|
+
? 'cmd'
|
|
140
|
+
: 'xdg-open';
|
|
141
|
+
const args = process.platform === 'win32'
|
|
142
|
+
? ['/c', 'start', '', url]
|
|
143
|
+
: [url];
|
|
144
|
+
return new Promise((resolveOpen) => {
|
|
145
|
+
const child = spawn(command, args, {
|
|
146
|
+
detached: true,
|
|
147
|
+
stdio: 'ignore'
|
|
148
|
+
});
|
|
149
|
+
child.on('error', () => resolveOpen());
|
|
150
|
+
child.unref();
|
|
151
|
+
resolveOpen();
|
|
152
|
+
});
|
|
153
|
+
}
|
|
154
|
+
async function authStatus(args) {
|
|
155
|
+
const json = hasFlag(args, '--json');
|
|
156
|
+
const auth = await resolveAuth();
|
|
157
|
+
if (!auth.authenticated || !auth.apiKey) {
|
|
158
|
+
if (json) {
|
|
159
|
+
return printAuthRequired(true);
|
|
160
|
+
}
|
|
161
|
+
console.log('Not authenticated');
|
|
162
|
+
console.log('Run: octoparse auth login');
|
|
163
|
+
return EXIT_OPERATION_FAILED;
|
|
164
|
+
}
|
|
165
|
+
try {
|
|
166
|
+
const validation = await validateApiKey({
|
|
167
|
+
apiKey: auth.apiKey,
|
|
168
|
+
baseUrl: valueAfter(args, '--api-base-url')
|
|
169
|
+
});
|
|
170
|
+
const { apiKey: _apiKey, ...status } = auth;
|
|
171
|
+
const result = {
|
|
172
|
+
...status,
|
|
173
|
+
verified: true,
|
|
174
|
+
apiBaseUrl: validation.baseUrl
|
|
175
|
+
};
|
|
176
|
+
if (json) {
|
|
177
|
+
printEnvelope(true, result);
|
|
178
|
+
return EXIT_OK;
|
|
179
|
+
}
|
|
180
|
+
console.log(`Authenticated: yes (${status.source})`);
|
|
181
|
+
console.log('Verified: yes');
|
|
182
|
+
console.log(`API: ${validation.baseUrl}`);
|
|
183
|
+
console.log(`API key: ${status.keyPreview}`);
|
|
184
|
+
if (status.source === 'env') {
|
|
185
|
+
console.log(`Source: ${API_KEY_ENV}`);
|
|
186
|
+
}
|
|
187
|
+
else {
|
|
188
|
+
console.log(`Credentials: ${status.credentialsFile}`);
|
|
189
|
+
}
|
|
190
|
+
return EXIT_OK;
|
|
191
|
+
}
|
|
192
|
+
catch (error) {
|
|
193
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
194
|
+
const code = error instanceof ApiRequestError ? error.code : 'AUTH_STATUS_FAILED';
|
|
195
|
+
if (json) {
|
|
196
|
+
printEnvelope(false, undefined, code, message);
|
|
197
|
+
}
|
|
198
|
+
else {
|
|
199
|
+
console.error(`Authentication failed: ${message}`);
|
|
200
|
+
if (code === 'AUTH_INVALID') {
|
|
201
|
+
console.error('Re-login with: octoparse auth login');
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
return EXIT_OPERATION_FAILED;
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
async function authLogout(args) {
|
|
208
|
+
const json = hasFlag(args, '--json');
|
|
209
|
+
const removed = await removeApiKey();
|
|
210
|
+
const { apiKey: _apiKey, ...status } = await resolveAuth();
|
|
211
|
+
const result = { removed, ...status };
|
|
212
|
+
if (json) {
|
|
213
|
+
printEnvelope(true, result);
|
|
214
|
+
return EXIT_OK;
|
|
215
|
+
}
|
|
216
|
+
console.log(removed ? 'Stored API key removed' : 'No stored API key found');
|
|
217
|
+
if (status.authenticated && status.source === 'env') {
|
|
218
|
+
console.log(`${API_KEY_ENV} is still set and will continue to be used for this shell.`);
|
|
219
|
+
}
|
|
220
|
+
return EXIT_OK;
|
|
221
|
+
}
|
|
222
|
+
async function readApiKeyFromStdin() {
|
|
223
|
+
if (process.stdin.isTTY) {
|
|
224
|
+
throw new Error('Pass the API key through stdin when using --stdin');
|
|
225
|
+
}
|
|
226
|
+
let value = '';
|
|
227
|
+
process.stdin.setEncoding('utf8');
|
|
228
|
+
for await (const chunk of process.stdin) {
|
|
229
|
+
value += chunk;
|
|
230
|
+
}
|
|
231
|
+
const apiKey = value.trim();
|
|
232
|
+
if (!apiKey)
|
|
233
|
+
throw new Error('API key cannot be empty');
|
|
234
|
+
return apiKey;
|
|
235
|
+
}
|
|
236
|
+
function readSecretFromTty(prompt) {
|
|
237
|
+
const stdin = process.stdin;
|
|
238
|
+
const stdout = process.stdout;
|
|
239
|
+
if (!stdin.isTTY || !stdout.isTTY) {
|
|
240
|
+
return Promise.reject(new Error('This is not an interactive terminal; use --stdin'));
|
|
241
|
+
}
|
|
242
|
+
return new Promise((resolveSecret, rejectSecret) => {
|
|
243
|
+
let value = '';
|
|
244
|
+
const wasRaw = stdin.isRaw;
|
|
245
|
+
const cleanup = () => {
|
|
246
|
+
stdin.off('data', handleData);
|
|
247
|
+
if (stdin.setRawMode)
|
|
248
|
+
stdin.setRawMode(wasRaw);
|
|
249
|
+
stdin.pause();
|
|
250
|
+
stdout.write('\n');
|
|
251
|
+
};
|
|
252
|
+
const finish = () => {
|
|
253
|
+
cleanup();
|
|
254
|
+
const apiKey = value.trim();
|
|
255
|
+
apiKey ? resolveSecret(apiKey) : rejectSecret(new Error('API key cannot be empty'));
|
|
256
|
+
};
|
|
257
|
+
const handleData = (chunk) => {
|
|
258
|
+
const input = chunk.toString('utf8');
|
|
259
|
+
for (const char of input) {
|
|
260
|
+
if (char === '\u0003') {
|
|
261
|
+
cleanup();
|
|
262
|
+
rejectSecret(new Error('Cancelled'));
|
|
263
|
+
return;
|
|
264
|
+
}
|
|
265
|
+
if (char === '\r' || char === '\n') {
|
|
266
|
+
finish();
|
|
267
|
+
return;
|
|
268
|
+
}
|
|
269
|
+
if (char === '\u007f' || char === '\b') {
|
|
270
|
+
value = value.slice(0, -1);
|
|
271
|
+
continue;
|
|
272
|
+
}
|
|
273
|
+
if (char >= ' ') {
|
|
274
|
+
value += char;
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
};
|
|
278
|
+
stdout.write(prompt);
|
|
279
|
+
stdin.setRawMode(true);
|
|
280
|
+
stdin.resume();
|
|
281
|
+
stdin.on('data', handleData);
|
|
282
|
+
});
|
|
283
|
+
}
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
import { homedir } from 'node:os';
|
|
2
|
+
import { join } from 'node:path';
|
|
3
|
+
import { printEnvelope } from '../cli/output.js';
|
|
4
|
+
import { API_BASE_URL_ENV } from '../runtime/api-client.js';
|
|
5
|
+
import { API_KEY_ENV } from '../runtime/auth.js';
|
|
6
|
+
import { EXIT_OK } from '../types.js';
|
|
7
|
+
export async function capabilitiesCommand(version, json) {
|
|
8
|
+
const data = {
|
|
9
|
+
name: 'octoparse',
|
|
10
|
+
version,
|
|
11
|
+
agentContractVersion: 1,
|
|
12
|
+
authentication: {
|
|
13
|
+
requiredForUse: true,
|
|
14
|
+
loginVerifiesKeyBeforeSaving: true,
|
|
15
|
+
setupCommandsWithoutAuth: ['auth login', 'auth status', 'auth logout', 'env status', 'env prod', 'env online'],
|
|
16
|
+
diagnosticCommandsWithoutAuth: ['--help', '--version', 'capabilities', 'doctor', 'browser doctor'],
|
|
17
|
+
env: API_KEY_ENV,
|
|
18
|
+
file: join(homedir(), '.octoparse', 'credentials.json')
|
|
19
|
+
},
|
|
20
|
+
output: {
|
|
21
|
+
jsonEnvelope: { success: { ok: true, data: {} }, failure: { ok: false, error: { code: 'ERROR_CODE', message: 'message' } } },
|
|
22
|
+
jsonlEvents: ['run.started', 'row', 'log', 'run.paused', 'run.resumed', 'run.stopping', 'run.stopped'],
|
|
23
|
+
detachedBootstrap: ['bootstrap.json', 'stdout.log', 'stderr.log'],
|
|
24
|
+
stdout: 'machine data only in --json/--jsonl mode',
|
|
25
|
+
stderr: 'human diagnostics and failures'
|
|
26
|
+
},
|
|
27
|
+
machineContract: {
|
|
28
|
+
stable: true,
|
|
29
|
+
defaultOutput: 'human',
|
|
30
|
+
schemas: {
|
|
31
|
+
capabilities: 'schemas/capabilities-v1.schema.json',
|
|
32
|
+
jsonEnvelope: 'schemas/json-envelope-v1.schema.json',
|
|
33
|
+
runEvent: 'schemas/run-event-v1.schema.json',
|
|
34
|
+
detachedBootstrap: 'schemas/detached-bootstrap-v1.schema.json'
|
|
35
|
+
},
|
|
36
|
+
json: {
|
|
37
|
+
flag: '--json',
|
|
38
|
+
envelope: {
|
|
39
|
+
successRequiredFields: ['ok', 'data'],
|
|
40
|
+
failureRequiredFields: ['ok', 'error.code', 'error.message']
|
|
41
|
+
},
|
|
42
|
+
usageErrorsUseEnvelope: true,
|
|
43
|
+
commonErrorCodes: [
|
|
44
|
+
'AUTH_REQUIRED',
|
|
45
|
+
'AUTH_INVALID',
|
|
46
|
+
'USAGE_ERROR',
|
|
47
|
+
'UNKNOWN_COMMAND',
|
|
48
|
+
'TASK_INVALID',
|
|
49
|
+
'RUN_FORMAT_UNSUPPORTED',
|
|
50
|
+
'DETACHED_RUN_FAILED',
|
|
51
|
+
'ENGINE_RUN_FAILED',
|
|
52
|
+
'LOCAL_RUN_ALREADY_RUNNING',
|
|
53
|
+
'LOCAL_RUN_LIMIT_EXCEEDED',
|
|
54
|
+
'LOCAL_RUN_CONTROL_FAILED',
|
|
55
|
+
'RUN_CONTROL_FAILED',
|
|
56
|
+
'RUN_NOT_FOUND',
|
|
57
|
+
'LOCAL_LOT_NOT_FOUND',
|
|
58
|
+
'UNSUPPORTED_EXPORT_FORMAT'
|
|
59
|
+
]
|
|
60
|
+
},
|
|
61
|
+
jsonl: {
|
|
62
|
+
flag: '--jsonl',
|
|
63
|
+
command: 'run <taskId>',
|
|
64
|
+
eventField: 'event',
|
|
65
|
+
stableEvents: ['run.started', 'row', 'log', 'run.paused', 'run.resumed', 'run.stopping', 'run.stopped'],
|
|
66
|
+
rowLimitFlag: '--max-rows'
|
|
67
|
+
},
|
|
68
|
+
artifacts: {
|
|
69
|
+
localRunDir: ['meta.json', 'control.json', 'events.jsonl', 'logs.jsonl', 'rows.jsonl'],
|
|
70
|
+
detachedBootstrapDir: ['bootstrap.json', 'stdout.log', 'stderr.log']
|
|
71
|
+
},
|
|
72
|
+
lifecycle: {
|
|
73
|
+
detachModel: 'child-process',
|
|
74
|
+
daemonRequired: false,
|
|
75
|
+
activeRunIdentity: 'taskId',
|
|
76
|
+
artifactRunIdentity: 'runId',
|
|
77
|
+
maxActiveLocalRunsPerTaskId: 1,
|
|
78
|
+
orphanDetection: true,
|
|
79
|
+
cleanupCommands: ['local cleanup', 'runs cleanup']
|
|
80
|
+
}
|
|
81
|
+
},
|
|
82
|
+
exitCodes: {
|
|
83
|
+
0: 'success',
|
|
84
|
+
1: 'operation failed',
|
|
85
|
+
2: 'runtime/environment failure',
|
|
86
|
+
3: 'unsupported task definition'
|
|
87
|
+
},
|
|
88
|
+
commands: [
|
|
89
|
+
{ command: 'doctor', risk: 'low', json: true, authRequired: false },
|
|
90
|
+
{ command: 'auth login/status/logout', risk: 'medium', json: true, authRequired: false },
|
|
91
|
+
{ command: 'env prod/online/status', risk: 'medium', json: true, hidden: true, authRequired: false },
|
|
92
|
+
{ command: 'task list', risk: 'low', json: true, authRequired: true },
|
|
93
|
+
{ command: 'task inspect/validate', risk: 'low', json: true, authRequired: true },
|
|
94
|
+
{ command: 'run <taskId>', risk: 'medium', json: true, jsonl: true, authRequired: true },
|
|
95
|
+
{ command: 'cloud start/stop <taskId>', risk: 'medium', json: true, authRequired: true },
|
|
96
|
+
{ command: 'cloud status/history <taskId>', risk: 'low', json: true, authRequired: true },
|
|
97
|
+
{ command: 'local status/history <taskId>', risk: 'low', json: true, authRequired: true },
|
|
98
|
+
{ command: 'local cleanup', risk: 'low', json: true, authRequired: true },
|
|
99
|
+
{ command: 'local export <taskId>', risk: 'medium', json: true, authRequired: true },
|
|
100
|
+
{ command: 'local pause/resume/stop <taskId>', risk: 'medium', json: true, authRequired: true },
|
|
101
|
+
{ command: 'data history <taskId>', risk: 'low', json: true, authRequired: true },
|
|
102
|
+
{ command: 'data export <taskId>', risk: 'medium', json: true, authRequired: true },
|
|
103
|
+
{ command: 'runs list/status/logs/data', risk: 'low', json: true, internal: true, authRequired: true },
|
|
104
|
+
{ command: 'runs cleanup', risk: 'low', json: true, internal: true, authRequired: true }
|
|
105
|
+
],
|
|
106
|
+
dataSources: ['local', 'cloud'],
|
|
107
|
+
exportFormats: ['xlsx', 'csv', 'html', 'json', 'xml'],
|
|
108
|
+
env: {
|
|
109
|
+
apiKey: API_KEY_ENV,
|
|
110
|
+
apiBaseUrl: API_BASE_URL_ENV
|
|
111
|
+
}
|
|
112
|
+
};
|
|
113
|
+
if (json)
|
|
114
|
+
printEnvelope(true, data);
|
|
115
|
+
else
|
|
116
|
+
console.log(JSON.stringify(data, null, 2));
|
|
117
|
+
return EXIT_OK;
|
|
118
|
+
}
|
|
@@ -0,0 +1,241 @@
|
|
|
1
|
+
import { firstPositionalArg, hasFlag, valueAfter } from '../cli/args.js';
|
|
2
|
+
import { printEnvelope, printUsageError } from '../cli/output.js';
|
|
3
|
+
import { printAuthRequired } from './auth.js';
|
|
4
|
+
import { ApiRequestError, fetchCloudDataBatch, fetchCloudHistory, fetchCloudStatus, startCloudTask, stopCloudTask } from '../runtime/api-client.js';
|
|
5
|
+
import { resolveAuth } from '../runtime/auth.js';
|
|
6
|
+
import { EXIT_OK, EXIT_OPERATION_FAILED } from '../types.js';
|
|
7
|
+
export async function cloudCommand(subcommand, args) {
|
|
8
|
+
const json = hasFlag([subcommand ?? '', ...args], '--json');
|
|
9
|
+
if (subcommand === 'start' || subcommand === 'stop') {
|
|
10
|
+
return cloudAction(subcommand, args);
|
|
11
|
+
}
|
|
12
|
+
if (subcommand === 'status') {
|
|
13
|
+
return cloudStatus(args);
|
|
14
|
+
}
|
|
15
|
+
if (subcommand === 'history') {
|
|
16
|
+
return cloudHistory(args);
|
|
17
|
+
}
|
|
18
|
+
return printUsageError(json, 'Error: invalid cloud subcommand; cloud extraction supports start/stop, not pause/resume.', 'Usage: octoparse cloud <start|stop|status|history> <taskId> [--json]');
|
|
19
|
+
}
|
|
20
|
+
async function cloudAction(command, args) {
|
|
21
|
+
const taskId = firstPositionalArg(args, ['--api-base-url']);
|
|
22
|
+
const json = hasFlag(args, '--json');
|
|
23
|
+
if (!taskId) {
|
|
24
|
+
return printUsageError(json, 'Error: missing taskId', `Usage: octoparse cloud ${command} <taskId> [--json]`);
|
|
25
|
+
}
|
|
26
|
+
const auth = await resolveAuth();
|
|
27
|
+
if (!auth.authenticated || !auth.apiKey) {
|
|
28
|
+
return printAuthRequired(json);
|
|
29
|
+
}
|
|
30
|
+
try {
|
|
31
|
+
const result = command === 'start'
|
|
32
|
+
? await startCloudTask({ apiKey: auth.apiKey, taskId, baseUrl: valueAfter(args, '--api-base-url') })
|
|
33
|
+
: await stopCloudTask({ apiKey: auth.apiKey, taskId, baseUrl: valueAfter(args, '--api-base-url') });
|
|
34
|
+
if (json) {
|
|
35
|
+
printEnvelope(true, { taskId, action: command, ...result });
|
|
36
|
+
}
|
|
37
|
+
else {
|
|
38
|
+
console.log(`Cloud ${command}: ${taskId}`);
|
|
39
|
+
printCloudApiResult(result);
|
|
40
|
+
}
|
|
41
|
+
return EXIT_OK;
|
|
42
|
+
}
|
|
43
|
+
catch (error) {
|
|
44
|
+
return printApiError(json, `Failed to ${command} cloud extraction`, error);
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
async function cloudStatus(args) {
|
|
48
|
+
const taskId = firstPositionalArg(args, ['--api-base-url']);
|
|
49
|
+
const json = hasFlag(args, '--json');
|
|
50
|
+
if (!taskId) {
|
|
51
|
+
return printUsageError(json, 'Error: missing taskId', 'Usage: octoparse cloud status <taskId> [--json]');
|
|
52
|
+
}
|
|
53
|
+
const auth = await resolveAuth();
|
|
54
|
+
if (!auth.authenticated || !auth.apiKey) {
|
|
55
|
+
return printAuthRequired(json);
|
|
56
|
+
}
|
|
57
|
+
try {
|
|
58
|
+
const result = await fetchCloudStatus({ apiKey: auth.apiKey, taskId, baseUrl: valueAfter(args, '--api-base-url') });
|
|
59
|
+
if (json) {
|
|
60
|
+
printEnvelope(true, { taskId, ...result });
|
|
61
|
+
}
|
|
62
|
+
else {
|
|
63
|
+
printCloudLiveInfo(taskId, result.data);
|
|
64
|
+
}
|
|
65
|
+
return EXIT_OK;
|
|
66
|
+
}
|
|
67
|
+
catch (error) {
|
|
68
|
+
return printApiError(json, 'Failed to fetch cloud extraction status', error);
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
export async function cloudHistory(args) {
|
|
72
|
+
const taskId = firstPositionalArg(args, ['--api-base-url', '--source']);
|
|
73
|
+
const json = hasFlag(args, '--json');
|
|
74
|
+
if (!taskId) {
|
|
75
|
+
return printUsageError(json, 'Error: missing taskId', 'Usage: octoparse cloud history <taskId> [--json]');
|
|
76
|
+
}
|
|
77
|
+
const auth = await resolveAuth();
|
|
78
|
+
if (!auth.authenticated || !auth.apiKey) {
|
|
79
|
+
return printAuthRequired(json);
|
|
80
|
+
}
|
|
81
|
+
try {
|
|
82
|
+
const result = await fetchCloudHistory({ apiKey: auth.apiKey, taskId, baseUrl: valueAfter(args, '--api-base-url') });
|
|
83
|
+
const items = await withCloudExportStats({
|
|
84
|
+
apiKey: auth.apiKey,
|
|
85
|
+
taskId,
|
|
86
|
+
baseUrl: valueAfter(args, '--api-base-url'),
|
|
87
|
+
items: result.data
|
|
88
|
+
});
|
|
89
|
+
if (json) {
|
|
90
|
+
printEnvelope(true, { taskId, ...result, data: items });
|
|
91
|
+
}
|
|
92
|
+
else {
|
|
93
|
+
if (!items.length) {
|
|
94
|
+
console.log(`No cloud extraction history found: ${taskId}`);
|
|
95
|
+
return EXIT_OK;
|
|
96
|
+
}
|
|
97
|
+
console.log(`Cloud extraction history: ${taskId}\n`);
|
|
98
|
+
for (const item of items) {
|
|
99
|
+
const record = asRecord(item);
|
|
100
|
+
console.log(formatCloudHistoryLine(record));
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
return EXIT_OK;
|
|
104
|
+
}
|
|
105
|
+
catch (error) {
|
|
106
|
+
return printApiError(json, 'Failed to fetch cloud extraction history', error);
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
async function withCloudExportStats(options) {
|
|
110
|
+
return Promise.all(options.items.map(async (item) => {
|
|
111
|
+
const record = asRecord(item);
|
|
112
|
+
const lotId = stringValue(record.lot);
|
|
113
|
+
if (!lotId)
|
|
114
|
+
return item;
|
|
115
|
+
const stats = await fetchCloudExportStats({
|
|
116
|
+
apiKey: options.apiKey,
|
|
117
|
+
taskId: options.taskId,
|
|
118
|
+
lotId,
|
|
119
|
+
baseUrl: options.baseUrl
|
|
120
|
+
});
|
|
121
|
+
return stats ? { ...record, ...stats } : item;
|
|
122
|
+
}));
|
|
123
|
+
}
|
|
124
|
+
async function fetchCloudExportStats(options) {
|
|
125
|
+
try {
|
|
126
|
+
const result = await fetchCloudDataBatch({
|
|
127
|
+
apiKey: options.apiKey,
|
|
128
|
+
taskId: options.taskId,
|
|
129
|
+
lotId: options.lotId,
|
|
130
|
+
baseUrl: options.baseUrl,
|
|
131
|
+
offset: 0,
|
|
132
|
+
size: 1
|
|
133
|
+
});
|
|
134
|
+
const data = asRecord(result.data);
|
|
135
|
+
const uniqueRows = numberValue(data.total);
|
|
136
|
+
const duplicateRows = numberValue(data.duplicate);
|
|
137
|
+
if (uniqueRows === null && duplicateRows === null)
|
|
138
|
+
return null;
|
|
139
|
+
const resolvedUniqueRows = uniqueRows ?? 0;
|
|
140
|
+
return {
|
|
141
|
+
uniqueRows: resolvedUniqueRows,
|
|
142
|
+
duplicateRows: duplicateRows ?? 0,
|
|
143
|
+
exportRows: resolvedUniqueRows
|
|
144
|
+
};
|
|
145
|
+
}
|
|
146
|
+
catch {
|
|
147
|
+
return null;
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
function formatCloudHistoryLine(record) {
|
|
151
|
+
const rows = String(record.extCnt ?? record.dataCnt ?? 0);
|
|
152
|
+
const uniqueRows = numberValue(record.uniqueRows);
|
|
153
|
+
const duplicateRows = numberValue(record.duplicateRows);
|
|
154
|
+
const exportStats = uniqueRows === null
|
|
155
|
+
? ''
|
|
156
|
+
: ` uniqueRows=${uniqueRows}${duplicateRows && duplicateRows > 0 ? ` duplicateRows=${duplicateRows}` : ''}`;
|
|
157
|
+
return ` ${String(record.lot ?? '')} ${cloudStatusName(record.status)} rows=${rows}${exportStats} ${String(record.startTime ?? record.startExtractTime ?? '')}`;
|
|
158
|
+
}
|
|
159
|
+
function printCloudApiResult(result) {
|
|
160
|
+
console.log(`API: ${result.baseUrl}${result.endpoint}`);
|
|
161
|
+
const data = asRecord(result.data);
|
|
162
|
+
if (!Object.keys(data).length) {
|
|
163
|
+
console.log('Result: ok');
|
|
164
|
+
return;
|
|
165
|
+
}
|
|
166
|
+
for (const [key, value] of Object.entries(data)) {
|
|
167
|
+
console.log(`${key}: ${formatValue(value)}`);
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
function printCloudLiveInfo(taskId, data) {
|
|
171
|
+
const info = asRecord(data);
|
|
172
|
+
if (!Object.keys(info).length) {
|
|
173
|
+
console.log(`${taskId} no_status`);
|
|
174
|
+
return;
|
|
175
|
+
}
|
|
176
|
+
console.log(`${taskId} ${cloudStatusName(info.status)}`);
|
|
177
|
+
if (info.lot !== undefined)
|
|
178
|
+
console.log(`Lot: ${String(info.lot)}`);
|
|
179
|
+
if (info.extCnt !== undefined || info.dataCnt !== undefined)
|
|
180
|
+
console.log(`Rows: ${String(info.extCnt ?? info.dataCnt)}`);
|
|
181
|
+
if (info.startTime !== undefined || info.startExtractTime !== undefined)
|
|
182
|
+
console.log(`Start: ${String(info.startTime ?? info.startExtractTime)}`);
|
|
183
|
+
if (info.endTime !== undefined)
|
|
184
|
+
console.log(`End: ${String(info.endTime)}`);
|
|
185
|
+
const progress = asRecord(info.stProg);
|
|
186
|
+
if (Object.keys(progress).length) {
|
|
187
|
+
console.log(`Subtasks: executing=${String(progress.executingCnt ?? 0)} finished=${String(progress.finishedCnt ?? 0)} stopped=${String(progress.stoppedCnt ?? 0)} waiting=${String(progress.waittingCnt ?? 0)}`);
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
function cloudStatusName(status) {
|
|
191
|
+
const value = Number(status);
|
|
192
|
+
if (value === -1)
|
|
193
|
+
return 'initializing';
|
|
194
|
+
if (value === 0 || value === 1)
|
|
195
|
+
return 'waiting';
|
|
196
|
+
if (value === 2 || value === 3)
|
|
197
|
+
return 'running';
|
|
198
|
+
if (value === 4)
|
|
199
|
+
return 'stopped';
|
|
200
|
+
if (value === 5)
|
|
201
|
+
return 'completed';
|
|
202
|
+
return status === undefined || status === null ? 'unknown' : String(status);
|
|
203
|
+
}
|
|
204
|
+
function asRecord(value) {
|
|
205
|
+
return value && typeof value === 'object' && !Array.isArray(value) ? value : {};
|
|
206
|
+
}
|
|
207
|
+
function stringValue(value) {
|
|
208
|
+
return typeof value === 'string' ? value.trim() : value === undefined || value === null ? '' : String(value).trim();
|
|
209
|
+
}
|
|
210
|
+
function numberValue(value) {
|
|
211
|
+
if (typeof value === 'number' && Number.isFinite(value))
|
|
212
|
+
return value;
|
|
213
|
+
if (typeof value === 'string' && value.trim()) {
|
|
214
|
+
const parsed = Number.parseInt(value, 10);
|
|
215
|
+
return Number.isFinite(parsed) ? parsed : null;
|
|
216
|
+
}
|
|
217
|
+
return null;
|
|
218
|
+
}
|
|
219
|
+
function formatValue(value) {
|
|
220
|
+
if (value === null || value === undefined)
|
|
221
|
+
return '';
|
|
222
|
+
if (typeof value === 'string')
|
|
223
|
+
return value;
|
|
224
|
+
if (typeof value === 'number' || typeof value === 'boolean' || typeof value === 'bigint')
|
|
225
|
+
return String(value);
|
|
226
|
+
return JSON.stringify(value);
|
|
227
|
+
}
|
|
228
|
+
function printApiError(json, prefix, error) {
|
|
229
|
+
const code = error instanceof ApiRequestError ? error.code : 'API_REQUEST_FAILED';
|
|
230
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
231
|
+
if (json) {
|
|
232
|
+
printEnvelope(false, undefined, code, message);
|
|
233
|
+
}
|
|
234
|
+
else {
|
|
235
|
+
console.error(`${code === 'AUTH_INVALID' ? 'Authentication failed' : prefix}: ${message}`);
|
|
236
|
+
if (error instanceof ApiRequestError && error.body && code !== 'AUTH_INVALID') {
|
|
237
|
+
console.error(`Response: ${error.body}`);
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
return EXIT_OPERATION_FAILED;
|
|
241
|
+
}
|