octoparse-cli 0.1.14

package/dist/commands/data.js

@@ -0,0 +1,220 @@
+import { join, resolve } from 'node:path';
+import { firstPositionalArg, hasFlag, parsePositiveInt, valueAfter } from '../cli/args.js';
+import { printEnvelope, printUsageError } from '../cli/output.js';
+import { cloudHistory } from './cloud.js';
+import { printAuthRequired } from './auth.js';
+import { ApiRequestError, fetchTaskInfo } from '../runtime/api-client.js';
+import { resolveAuth } from '../runtime/auth.js';
+import { fetchCloudRows } from '../runtime/cloud-data.js';
+import { exportRowsToFile, normalizeDataExportFormat } from '../runtime/data-exporter.js';
+import { listRuns } from '../runtime/artifacts.js';
+import { countRunRows, defaultRunsDir, listActiveRuns, readJsonLines } from '../runtime/local-runs.js';
+import { defaultExportFileName } from '../runtime/naming.js';
+import { EXIT_OK, EXIT_OPERATION_FAILED } from '../types.js';
+export async function localHistory(args) {
+    const taskId = firstPositionalArg(args, ['--output']);
+    const json = hasFlag(args, '--json');
+    const outputDir = resolve(valueAfter(args, '--output') ?? defaultRunsDir());
+    if (!taskId) {
+        return printUsageError(json, 'Error: missing taskId', 'Usage: octoparse local history <taskId> [--output <dir>] [--json]');
+    }
+    const lots = await listLocalLots(outputDir, taskId);
+    if (json) {
+        printEnvelope(true, lots.map(localLotToPublic));
+        return EXIT_OK;
+    }
+    if (!lots.length) {
+        console.log(`No local extraction lots found: ${taskId}`);
+        return EXIT_OK;
+    }
+    console.log(`Local extraction lots: ${taskId}\n`);
+    for (const lot of lots) {
+        console.log(`  ${lot.lotId}  ${lot.status}  rows=${lot.total}  ${lot.startedAt}`);
+    }
+    return EXIT_OK;
+}
+export async function localExport(args) {
+    const taskId = firstPositionalArg(args, ['--file', '--lot-id', '--lot', '--output', '--format']);
+    const json = hasFlag(args, '--json');
+    const outputDir = resolve(valueAfter(args, '--output') ?? defaultRunsDir());
+    const lotId = valueAfter(args, '--lot-id') ?? valueAfter(args, '--lot');
+    const targetFile = valueAfter(args, '--file');
+    if (!taskId) {
+        return printUsageError(json, 'Error: missing taskId', 'Usage: octoparse local export <taskId> [--file <result.xlsx>] [--lot-id <lotId>] [--output <dir>] [--format xlsx|csv|html|json|xml] [--json]');
+    }
+    const format = normalizeDataExportFormat(valueAfter(args, '--format'), targetFile);
+    if (!format) {
+        return printUsageError(json, '--format supports xlsx, csv, html, json, and xml', undefined, 'UNSUPPORTED_EXPORT_FORMAT');
+    }
+    const lot = await findLocalLot(outputDir, taskId, lotId);
+    if (!lot) {
+        const message = lotId
+            ? `Local extraction lot not found: taskId=${taskId}, lotId=${lotId}`
+            : `No local extraction history found for task ${taskId}`;
+        if (json)
+            printEnvelope(false, undefined, 'LOCAL_LOT_NOT_FOUND', message);
+        else
+            console.error(message);
+        return EXIT_OPERATION_FAILED;
+    }
+    const runDir = join(outputDir, lot.runId);
+    const rows = await readJsonLines(join(runDir, 'rows.jsonl'), Number.MAX_SAFE_INTEGER);
+    const taskName = lot.taskName ?? await resolveTaskName(taskId);
+    const exportFile = targetFile ?? defaultExportFileName(taskName, format);
+    const exported = await exportRowsToFile(rows, exportFile, format);
+    const result = {
+        taskId,
+        taskName,
+        lotId: lot.lotId,
+        rows: exported.rows,
+        file: exported.file,
+        format: exported.format
+    };
+    if (json) {
+        printEnvelope(true, result);
+    }
+    else {
+        console.log(`Exported ${result.rows} rows -> ${result.file}`);
+        console.log(`Task: ${result.taskId}`);
+        console.log(`Lot: ${result.lotId}`);
+        console.log(`Format: ${result.format}`);
+    }
+    return EXIT_OK;
+}
+export async function dataHistory(args) {
+    const source = parseDataSource(args);
+    return source === 'cloud' ? cloudHistory(args) : localHistory(args);
+}
+export async function dataExport(args) {
+    const source = parseDataSource(args);
+    return source === 'cloud' ? cloudDataExport(args) : localExport(args);
+}
+async function cloudDataExport(args) {
+    const taskId = firstPositionalArg(args, ['--source', '--file', '--lot-id', '--lot', '--format', '--api-base-url', '--batch-size']);
+    const json = hasFlag(args, '--json');
+    const lotId = valueAfter(args, '--lot-id') ?? valueAfter(args, '--lot');
+    const targetFile = valueAfter(args, '--file');
+    if (!taskId) {
+        return printUsageError(json, 'Error: missing taskId', 'Usage: octoparse data export <taskId> --source cloud [--file <result.xlsx>] [--lot-id <lotId>] [--format xlsx|csv|html|json|xml] [--json]');
+    }
+    const auth = await resolveAuth();
+    if (!auth.authenticated || !auth.apiKey) {
+        return printAuthRequired(json);
+    }
+    const format = normalizeDataExportFormat(valueAfter(args, '--format'), targetFile);
+    if (!format) {
+        return printUsageError(json, '--format supports xlsx, csv, html, json, and xml', undefined, 'UNSUPPORTED_EXPORT_FORMAT');
+    }
+    try {
+        const rows = await fetchCloudRows({
+            apiKey: auth.apiKey,
+            taskId,
+            lotId,
+            baseUrl: valueAfter(args, '--api-base-url'),
+            batchSize: parsePositiveInt(valueAfter(args, '--batch-size'), 100)
+        });
+        const taskName = await resolveTaskName(taskId);
+        const exportFile = targetFile ?? defaultExportFileName(taskName, format);
+        const exported = await exportRowsToFile(rows, exportFile, format);
+        const result = {
+            taskId,
+            taskName,
+            source: 'cloud',
+            lotId,
+            rows: exported.rows,
+            file: exported.file,
+            format: exported.format
+        };
+        if (json) {
+            printEnvelope(true, result);
+        }
+        else {
+            console.log(`Exported ${result.rows} cloud rows -> ${result.file}`);
+            console.log(`Task: ${result.taskId}`);
+            if (result.lotId)
+                console.log(`Lot: ${result.lotId}`);
+            console.log(`Format: ${result.format}`);
+        }
+        return EXIT_OK;
+    }
+    catch (error) {
+        return printApiError(json, 'Failed to export cloud data', error);
+    }
+}
+function parseDataSource(args) {
+    if (hasFlag(args, '--cloud'))
+        return 'cloud';
+    if (hasFlag(args, '--local'))
+        return 'local';
+    const source = valueAfter(args, '--source');
+    return source === 'cloud' ? 'cloud' : 'local';
+}
+async function listLocalLots(outputDir, taskId) {
+    const runs = await listRuns(outputDir);
+    const activeRuns = await listActiveRuns(outputDir);
+    const byRunId = new Map();
+    for (const run of runs)
+        byRunId.set(run.runId, withLotId(run));
+    for (const run of activeRuns)
+        byRunId.set(run.runId, withLotId(run));
+    const matched = [...byRunId.values()]
+        .filter((run) => run.taskId === taskId)
+        .sort((a, b) => b.startedAt.localeCompare(a.startedAt));
+    return Promise.all(matched.map((run) => withActualRowCount(outputDir, run)));
+}
+async function findLocalLot(outputDir, taskId, lotId) {
+    const lots = await listLocalLots(outputDir, taskId);
+    if (!lots.length)
+        return null;
+    if (!lotId)
+        return lots[0];
+    return lots.find((lot) => lot.lotId === lotId) ?? null;
+}
+function withLotId(summary) {
+    return summary.lotId ? summary : { ...summary, lotId: deriveLotId(summary.runId) };
+}
+async function withActualRowCount(outputDir, summary) {
+    const total = await countRunRows(outputDir, summary.runId);
+    return total === summary.total ? summary : { ...summary, total };
+}
+function localLotToPublic(summary) {
+    return {
+        taskId: summary.taskId,
+        taskName: summary.taskName,
+        lotId: summary.lotId,
+        status: summary.status,
+        total: summary.total,
+        startedAt: summary.startedAt,
+        stoppedAt: summary.stoppedAt
+    };
+}
+function deriveLotId(runId) {
+    const stamp = runId.match(/_(\d{14})$/)?.[1];
+    return stamp ? `lot_${stamp}` : runId;
+}
+async function resolveTaskName(taskId) {
+    const auth = await resolveAuth();
+    if (!auth.apiKey)
+        return taskId;
+    try {
+        const info = await fetchTaskInfo({ apiKey: auth.apiKey, taskId });
+        return String(info.taskName ?? info.TaskName ?? taskId).trim() || taskId;
+    }
+    catch {
+        return taskId;
+    }
+}
+function printApiError(json, prefix, error) {
+    const code = error instanceof ApiRequestError ? error.code : 'API_REQUEST_FAILED';
+    const message = error instanceof Error ? error.message : String(error);
+    if (json) {
+        printEnvelope(false, undefined, code, message);
+    }
+    else {
+        console.error(`${code === 'AUTH_INVALID' ? 'Authentication failed' : prefix}: ${message}`);
+        if (error instanceof ApiRequestError && error.body && code !== 'AUTH_INVALID') {
+            console.error(`Response: ${error.body}`);
+        }
+    }
+    return EXIT_OPERATION_FAILED;
+}

package/dist/commands/doctor.js

@@ -0,0 +1,73 @@
+import { existsSync } from 'node:fs';
+import { createRequire } from 'node:module';
+import { dirname, join } from 'node:path';
+import { hasFlag, valueAfter } from '../cli/args.js';
+import { printResult } from '../cli/output.js';
+import { EXIT_OK, EXIT_RUNTIME_FAILED } from '../types.js';
+const require = createRequire(import.meta.url);
+const EngineModule = require('@octopus/engine');
+const resolveChrome = EngineModule.resolveChrome;
+export async function doctorCommand(args) {
+    const json = hasFlag(args, '--json');
+    const chromePath = valueAfter(args, '--chrome-path');
+    let chrome = { ok: false, message: 'not checked' };
+    if (chromePath) {
+        chrome = existsSync(chromePath)
+            ? { ok: true, message: chromePath }
+            : { ok: false, message: `Chrome executable not found: ${chromePath}` };
+    }
+    else {
+        try {
+            const resolved = await resolveChrome();
+            chrome = { ok: true, message: resolved.executablePath };
+        }
+        catch (error) {
+            chrome = { ok: false, message: error instanceof Error ? error.message : String(error) };
+        }
+    }
+    const engineDist = dirname(require.resolve('@octopus/engine'));
+    const extensionPath = join(engineDist, 'extension', 'manifest.json');
+    const ublockPath = join(engineDist, 'extensions', 'ublock-origin-lite', 'uBOLite.chromium.tar.xz');
+    const data = {
+        ok: chrome.ok && existsSync(extensionPath),
+        checks: [
+            { name: 'node', ok: true, message: process.version },
+            { name: 'engine', ok: true, message: require.resolve('@octopus/engine') },
+            { name: 'runtime-extension', ok: existsSync(extensionPath), message: extensionPath },
+            { name: 'adblock-extension-archive', ok: existsSync(ublockPath), message: ublockPath },
+            { name: 'chrome', ok: chrome.ok, message: chrome.message },
+            { name: 'electron-client', ok: true, message: 'not required' },
+            { name: 'browser-mode', ok: true, message: 'independent Chrome only' }
+        ]
+    };
+    printResult(json, data);
+    return EXIT_OK;
+}
+export async function browserDoctorCommand(args) {
+    const json = hasFlag(args, '--json');
+    const chromePath = valueAfter(args, '--chrome-path');
+    let executablePath = chromePath;
+    let ok = true;
+    let error = '';
+    try {
+        if (!executablePath) {
+            executablePath = (await resolveChrome()).executablePath;
+        }
+        ok = Boolean(executablePath) && existsSync(executablePath);
+        if (!ok)
+            error = `Chrome executable not found: ${executablePath}`;
+    }
+    catch (caught) {
+        ok = false;
+        error = caught instanceof Error ? caught.message : String(caught);
+    }
+    const data = {
+        ok,
+        supported: ['chrome'],
+        unsupported: ['kernel', 'legacy-workflow'],
+        executablePath,
+        message: ok ? 'independent Chrome is ready' : error
+    };
+    printResult(json, data);
+    return ok ? EXIT_OK : EXIT_RUNTIME_FAILED;
+}

package/dist/commands/env.js

@@ -0,0 +1,63 @@
+import { hasFlag, valueAfter } from '../cli/args.js';
+import { printEnvelope } from '../cli/output.js';
+import { API_BASE_URL_ENV, PROD_API_BASE_URL } from '../runtime/api-client.js';
+import { configFilePath, readCliConfig, saveCliConfig } from '../runtime/config.js';
+import { EXIT_OK, EXIT_OPERATION_FAILED } from '../types.js';
+export async function hiddenEnvCommand(subcommand, args) {
+    const json = hasFlag([subcommand ?? '', ...args], '--json');
+    if (subcommand === 'status' || !subcommand) {
+        const config = await readCliConfig();
+        const effectiveBaseUrl = valueAfter(args, '--api-base-url') ?? process.env[API_BASE_URL_ENV] ?? config.apiBaseUrl ?? PROD_API_BASE_URL;
+        const data = {
+            apiEnv: config.apiEnv ?? 'prod',
+            apiBaseUrl: effectiveBaseUrl,
+            configFile: configFilePath(),
+            envOverride: Boolean(process.env[API_BASE_URL_ENV])
+        };
+        if (json)
+            printEnvelope(true, data);
+        else {
+            console.log(`API env: ${data.apiEnv}`);
+            console.log(`API base URL: ${data.apiBaseUrl}`);
+            console.log(`Config: ${data.configFile}`);
+            if (data.envOverride)
+                console.log(`${API_BASE_URL_ENV} is set and overrides local config.`);
+        }
+        return EXIT_OK;
+    }
+    const next = resolveEnvTarget(subcommand);
+    if (!next) {
+        const message = 'Usage: octoparse env <prod|online|status> [--json]';
+        if (json)
+            printEnvelope(false, undefined, 'INVALID_ENV', message);
+        else
+            console.error(message);
+        return EXIT_OPERATION_FAILED;
+    }
+    const config = await saveCliConfig({
+        apiEnv: next.apiEnv,
+        apiBaseUrl: next.apiBaseUrl
+    });
+    const data = {
+        apiEnv: config.apiEnv,
+        apiBaseUrl: config.apiBaseUrl,
+        configFile: configFilePath()
+    };
+    if (json)
+        printEnvelope(true, data);
+    else {
+        console.log(`API env switched to: ${data.apiEnv}`);
+        console.log(`API base URL: ${data.apiBaseUrl}`);
+        console.log(`Config: ${data.configFile}`);
+        if (process.env[API_BASE_URL_ENV]) {
+            console.log(`${API_BASE_URL_ENV} is currently set and will still override this config.`);
+        }
+    }
+    return EXIT_OK;
+}
+function resolveEnvTarget(value) {
+    if (value === 'prod' || value === 'online' || value === 'production') {
+        return { apiEnv: 'prod', apiBaseUrl: PROD_API_BASE_URL };
+    }
+    return undefined;
+}

package/dist/commands/local.js

@@ -0,0 +1,251 @@
+import { join, resolve } from 'node:path';
+import { readFile, readdir } from 'node:fs/promises';
+import { firstPositionalArg, hasFlag, valueAfter } from '../cli/args.js';
+import { printEnvelope, printUsageError } from '../cli/output.js';
+import { localExport, localHistory } from './data.js';
+import { ensureRunDir, listRuns, writeRunSummary } from '../runtime/artifacts.js';
+import { cleanupTaskControlState, cleanupTaskControlStates, isRunControlReachable, readTaskControlState, resolveRunControlSocketPath, sendTaskControlCommand } from '../runtime/run-control.js';
+import { countRunRows, defaultRunsDir } from '../runtime/local-runs.js';
+import { EXIT_OK, EXIT_OPERATION_FAILED } from '../types.js';
+export async function localCommand(subcommand, args) {
+    const json = hasFlag([subcommand ?? '', ...args], '--json');
+    if (subcommand === 'status') {
+        return localStatus(args);
+    }
+    if (subcommand === 'history') {
+        return localHistory(args);
+    }
+    if (subcommand === 'export') {
+        return localExport(args);
+    }
+    if (subcommand === 'cleanup') {
+        return localCleanup(args);
+    }
+    if (subcommand === 'pause' || subcommand === 'resume' || subcommand === 'stop') {
+        return localControl(subcommand, args);
+    }
+    return printUsageError(json, 'Error: invalid local subcommand', 'Usage: octoparse local <status|pause|resume|stop|history|export|cleanup> <taskId> [--json]');
+}
+async function localStatus(args) {
+    const taskId = firstPositionalArg(args, ['--output']);
+    const json = hasFlag(args, '--json');
+    if (!taskId) {
+        return printUsageError(json, 'Error: missing taskId', 'Usage: octoparse local status <taskId> [--output <dir>] [--json]');
+    }
+    const state = await readTaskControlState(taskId);
+    const alive = await isRunControlReachable(state);
+    const actualSocketPath = state ? resolveRunControlSocketPath(state) : null;
+    const total = state ? await countRunRows(state.outputDir, state.runId) : 0;
+    const outputDir = resolve(state?.outputDir ?? valueAfter(args, '--output') ?? defaultRunsDir());
+    const bootstrap = !alive ? await findLatestStartingDetachedBootstrap(outputDir, taskId) : null;
+    const staleRun = state && !alive ? await preserveStaleRunSummary(state, total) : null;
+    const lastRun = staleRun ?? await findLastLocalRun(outputDir, taskId);
+    const currentRun = alive && state ? controlStateToPublicRun(state, total, actualSocketPath) : null;
+    const data = alive && state
+        ? {
+            ...state,
+            active: true,
+            status: state.status,
+            total,
+            controlSocketPath: actualSocketPath,
+            currentRun,
+            lastRun
+        }
+        : state
+            ? {
+                taskId,
+                active: false,
+                status: 'not_running',
+                currentRun: null,
+                lastRun,
+                cleanedStaleState: true,
+                lastStatus: state.status,
+                lastRunId: state.runId,
+                total
+            }
+            : bootstrap
+                ? {
+                    taskId,
+                    active: true,
+                    status: 'starting',
+                    currentRun: null,
+                    lastRun,
+                    detached: true,
+                    pid: bootstrap.pid,
+                    bootstrapDir: bootstrap.dir,
+                    stdout: bootstrap.stdout,
+                    stderr: bootstrap.stderr,
+                    updatedAt: bootstrap.updatedAt
+                }
+                : {
+                    taskId,
+                    active: false,
+                    status: 'not_running',
+                    currentRun: null,
+                    lastRun
+                };
+    if (state && !alive) {
+        await cleanupTaskControlState(taskId);
+    }
+    if (json) {
+        printEnvelope(true, data);
+    }
+    else if (alive && state) {
+        console.log(`${taskId}  ${state?.status}`);
+        console.log(`PID: ${state?.pid}`);
+        console.log(`Rows: ${total}`);
+        console.log(`Output: ${state?.outputDir}`);
+        if (actualSocketPath && actualSocketPath !== state.socketPath) {
+            console.log(`Control socket: ${actualSocketPath}`);
+        }
+    }
+    else if (state) {
+        console.log(`${taskId}  idle`);
+        console.log(`Cleaned stale local state from previous run: ${state.runId}`);
+        printLastRun(lastRun);
+        console.log(`Rows: ${total}`);
+    }
+    else if (bootstrap) {
+        console.log(`${taskId}  starting`);
+        if (bootstrap.pid)
+            console.log(`PID: ${bootstrap.pid}`);
+        console.log(`Bootstrap: ${bootstrap.dir}`);
+        if (bootstrap.stdout)
+            console.log(`Stdout: ${bootstrap.stdout}`);
+        if (bootstrap.stderr)
+            console.log(`Stderr: ${bootstrap.stderr}`);
+    }
+    else {
+        console.log(`${taskId}  idle`);
+        printLastRun(lastRun);
+    }
+    return EXIT_OK;
+}
+async function findLatestStartingDetachedBootstrap(outputDir, taskId) {
+    let entries = [];
+    try {
+        entries = await readdir(outputDir);
+    }
+    catch {
+        return null;
+    }
+    const candidates = [];
+    for (const entry of entries) {
+        if (!entry.startsWith('.detach_'))
+            continue;
+        const dir = join(outputDir, entry);
+        const bootstrap = await readDetachedBootstrap(dir);
+        if (!bootstrap || bootstrap.taskId !== taskId)
+            continue;
+        if (bootstrap.status !== 'spawning' && bootstrap.status !== 'spawned' && bootstrap.status !== 'starting')
+            continue;
+        if (!isProcessAlive(bootstrap.pid))
+            continue;
+        candidates.push(bootstrap);
+    }
+    candidates.sort((a, b) => String(b.updatedAt ?? '').localeCompare(String(a.updatedAt ?? '')));
+    return candidates[0] ?? null;
+}
+async function readDetachedBootstrap(dir) {
+    try {
+        const bootstrap = JSON.parse(await readFile(join(dir, 'bootstrap.json'), 'utf8'));
+        return { ...bootstrap, dir };
+    }
+    catch {
+        return null;
+    }
+}
+function isProcessAlive(pid) {
+    if (typeof pid !== 'number' || !Number.isInteger(pid) || pid <= 0)
+        return false;
+    try {
+        process.kill(pid, 0);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+async function preserveStaleRunSummary(state, total) {
+    if (!state)
+        return null;
+    const runDir = await ensureRunDir(state.outputDir, state.runId);
+    const summary = {
+        runId: state.runId,
+        lotId: state.lotId,
+        taskId: state.taskId,
+        taskName: state.taskName,
+        status: 'stopped',
+        total,
+        outputDir: runDir,
+        startedAt: state.updatedAt,
+        stoppedAt: new Date().toISOString()
+    };
+    await writeRunSummary(runDir, summary).catch(() => undefined);
+    return summary;
+}
+async function findLastLocalRun(outputDir, taskId) {
+    const runs = await listRuns(outputDir);
+    return runs.find((run) => run.taskId === taskId) ?? null;
+}
+function controlStateToPublicRun(state, total, controlSocketPath) {
+    return {
+        runId: state.runId,
+        lotId: state.lotId,
+        taskId: state.taskId,
+        taskName: state.taskName,
+        status: state.status,
+        total,
+        outputDir: state.outputDir,
+        pid: state.pid,
+        controlSocketPath,
+        updatedAt: state.updatedAt
+    };
+}
+function printLastRun(lastRun) {
+    if (!lastRun)
+        return;
+    const lot = lastRun.lotId ? `  lot=${lastRun.lotId}` : '';
+    console.log(`Last run: ${lastRun.status}  rows=${lastRun.total}${lot}`);
+}
+async function localControl(command, args) {
+    const taskId = firstPositionalArg(args);
+    const json = hasFlag(args, '--json');
+    if (!taskId) {
+        return printUsageError(json, 'Error: missing taskId', `Usage: octoparse local ${command} <taskId> [--json]`);
+    }
+    try {
+        const state = await sendTaskControlCommand(taskId, command);
+        if (json) {
+            printEnvelope(true, state);
+        }
+        else {
+            console.log(`${taskId} ${command} -> ${state.status}`);
+        }
+        return EXIT_OK;
+    }
+    catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        if (json)
+            printEnvelope(false, undefined, 'LOCAL_RUN_CONTROL_FAILED', message);
+        else
+            console.error(`Local extraction control failed: ${message}`);
+        return EXIT_OPERATION_FAILED;
+    }
+}
+async function localCleanup(args) {
+    const json = hasFlag(args, '--json');
+    const result = await cleanupTaskControlStates();
+    if (json) {
+        printEnvelope(true, result);
+    }
+    else {
+        console.log(`Checked: ${result.checked}`);
+        console.log(`Alive: ${result.alive}`);
+        console.log(`Removed orphaned: ${result.removed}`);
+        for (const item of result.orphaned) {
+            console.log(`  ${item.taskId ?? 'unknown-task'} ${item.runId ?? 'unknown-run'} ${item.filePath}`);
+        }
+    }
+    return EXIT_OK;
+}