@jackwener/opencli 1.5.2 → 1.5.3

package/dist/clis/xiaohongshu/search.test.js

@@ -28,6 +28,7 @@ function createPageMock(evaluateResults) {
         getInterceptedRequests: vi.fn().mockResolvedValue([]),
         getCookies: vi.fn().mockResolvedValue([]),
         screenshot: vi.fn().mockResolvedValue(''),
+        waitForCapture: vi.fn().mockResolvedValue(undefined),
     };
 }
 describe('xiaohongshu search', () => {

package/dist/download/index.js

@@ -4,14 +4,13 @@
 import { spawn } from 'node:child_process';
 import * as fs from 'node:fs';
 import * as path from 'node:path';
-import * as https from 'node:https';
-import * as http from 'node:http';
 import * as os from 'node:os';
-import { Transform } from 'node:stream';
+import { Readable, Transform } from 'node:stream';
 import { pipeline } from 'node:stream/promises';
 import { URL } from 'node:url';
 import { isBinaryInstalled } from '../external.js';
 import { getErrorMessage } from '../errors.js';
+import { fetchWithNodeNetwork } from '../node-network.js';
 /** Check if yt-dlp is available in PATH. */
 export function checkYtdlp() {
     return isBinaryInstalled('yt-dlp');
@@ -61,8 +60,6 @@ export function requiresYtdlp(url) {
 export async function httpDownload(url, destPath, options = {}, redirectCount = 0) {
     const { cookies, headers = {}, timeout = 30000, onProgress, maxRedirects = 10 } = options;
     return new Promise((resolve) => {
-        const parsedUrl = new URL(url);
-        const protocol = parsedUrl.protocol === 'https:' ? https : http;
         const requestHeaders = {
             'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36',
             ...headers,
@@ -86,30 +83,43 @@ export async function httpDownload(url, destPath, options = {}, redirectCount =
                 // Ignore cleanup errors so the original failure is preserved.
             }
         };
-        const request = protocol.get(url, { headers: requestHeaders, timeout }, (response) => {
-            void (async () => {
+        void (async () => {
+            const controller = new AbortController();
+            const timer = setTimeout(() => controller.abort(), timeout);
+            try {
+                const response = await fetchWithNodeNetwork(url, {
+                    headers: requestHeaders,
+                    signal: controller.signal,
+                    redirect: 'manual',
+                });
+                clearTimeout(timer);
                 // Handle redirects before creating any file handles.
-                if (response.statusCode && response.statusCode >= 300 && response.statusCode < 400 && response.headers.location) {
-                    response.resume();
-                    if (redirectCount >= maxRedirects) {
-                        finish({ success: false, size: 0, error: `Too many redirects (> ${maxRedirects})` });
+                if (response.status >= 300 && response.status < 400) {
+                    const location = response.headers.get('location');
+                    if (location) {
+                        if (redirectCount >= maxRedirects) {
+                            finish({ success: false, size: 0, error: `Too many redirects (> ${maxRedirects})` });
+                            return;
+                        }
+                        const redirectUrl = resolveRedirectUrl(url, location);
+                        const originalHost = new URL(url).hostname;
+                        const redirectHost = new URL(redirectUrl).hostname;
+                        const redirectOptions = originalHost === redirectHost
+                            ? options
+                            : { ...options, cookies: undefined, headers: stripCookieHeaders(options.headers) };
+                        finish(await httpDownload(redirectUrl, destPath, redirectOptions, redirectCount + 1));
                         return;
                     }
-                    const redirectUrl = resolveRedirectUrl(url, response.headers.location);
-                    const originalHost = new URL(url).hostname;
-                    const redirectHost = new URL(redirectUrl).hostname;
-                    const redirectOptions = originalHost === redirectHost
-                        ? options
-                        : { ...options, cookies: undefined, headers: stripCookieHeaders(options.headers) };
-                    finish(await httpDownload(redirectUrl, destPath, redirectOptions, redirectCount + 1));
+                }
+                if (response.status !== 200) {
+                    finish({ success: false, size: 0, error: `HTTP ${response.status}` });
                     return;
                 }
-                if (response.statusCode !== 200) {
-                    response.resume();
-                    finish({ success: false, size: 0, error: `HTTP ${response.statusCode}` });
+                if (!response.body) {
+                    finish({ success: false, size: 0, error: 'Empty response body' });
                     return;
                 }
-                const totalSize = parseInt(response.headers['content-length'] || '0', 10);
+                const totalSize = parseInt(response.headers.get('content-length') || '0', 10);
                 let received = 0;
                 const progressStream = new Transform({
                     transform(chunk, _encoding, callback) {
@@ -121,7 +131,7 @@ export async function httpDownload(url, destPath, options = {}, redirectCount =
                 });
                 try {
                     await fs.promises.mkdir(path.dirname(destPath), { recursive: true });
-                    await pipeline(response, progressStream, fs.createWriteStream(tempPath));
+                    await pipeline(Readable.fromWeb(response.body), progressStream, fs.createWriteStream(tempPath));
                     await fs.promises.rename(tempPath, destPath);
                     finish({ success: true, size: received });
                 }
@@ -129,17 +139,13 @@ export async function httpDownload(url, destPath, options = {}, redirectCount =
                     await cleanupTempFile();
                     finish({ success: false, size: 0, error: getErrorMessage(err) });
                 }
-            })();
-        });
-        request.on('error', (err) => {
-            void (async () => {
+            }
+            catch (err) {
+                clearTimeout(timer);
                 await cleanupTempFile();
-                finish({ success: false, size: 0, error: err.message });
-            })();
-        });
-        request.on('timeout', () => {
-            request.destroy(new Error('Timeout'));
-        });
+                finish({ success: false, size: 0, error: err instanceof Error ? err.message : String(err) });
+            }
+        })();
     });
 }
 export function resolveRedirectUrl(currentUrl, location) {

package/dist/download/index.test.js

@@ -2,11 +2,12 @@ import * as fs from 'node:fs';
 import * as http from 'node:http';
 import * as os from 'node:os';
 import * as path from 'node:path';
-import { afterEach, describe, expect, it } from 'vitest';
+import { afterEach, describe, expect, it, vi } from 'vitest';
 import { formatCookieHeader, httpDownload, resolveRedirectUrl } from './index.js';
 const servers = [];
 const tempDirs = [];
 afterEach(async () => {
+    vi.unstubAllEnvs();
     await Promise.all(servers.map((server) => new Promise((resolve, reject) => {
         server.close((err) => (err ? reject(err) : resolve()));
     })));
@@ -101,4 +102,17 @@ describe('download helpers', { retry: process.platform === 'win32' ? 2 : 0 }, ()
         expect(forwardedCookie).toBeUndefined();
         expect(fs.readFileSync(destPath, 'utf8')).toBe('ok');
     });
+    it('bypasses proxy settings for loopback downloads', async () => {
+        vi.stubEnv('HTTP_PROXY', 'http://127.0.0.1:9');
+        const baseUrl = await startServer((_req, res) => {
+            res.statusCode = 200;
+            res.end('ok');
+        });
+        const tempDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'opencli-dl-'));
+        tempDirs.push(tempDir);
+        const destPath = path.join(tempDir, 'loopback.txt');
+        const result = await httpDownload(`${baseUrl}/ok`, destPath);
+        expect(result).toEqual({ success: true, size: 2 });
+        expect(fs.readFileSync(destPath, 'utf8')).toBe('ok');
+    });
 });

package/dist/execution.js

@@ -17,6 +17,7 @@ import { shouldUseBrowserSession } from './capabilityRouting.js';
 import { getBrowserFactory, browserSession, runWithTimeout, DEFAULT_BROWSER_COMMAND_TIMEOUT } from './runtime.js';
 import { emitHook } from './hooks.js';
 import { checkDaemonStatus } from './browser/discover.js';
+import { log } from './logger.js';
 const _loadedModules = new Set();
 export function coerceAndValidateArgs(cmdArgs, kwargs) {
     const result = { ...kwargs };
@@ -167,7 +168,7 @@ export async function executeCommand(cmd, rawKwargs, debug = false) {
                     const skip = await isAlreadyOnDomain(page, preNavUrl);
                     if (skip) {
                         if (debug)
-                            console.error(`[pre-nav] Already on target domain, skipping navigation`);
+                            log.debug('[pre-nav] Already on target domain, skipping navigation');
                     }
                     else {
                         try {
@@ -177,7 +178,7 @@ export async function executeCommand(cmd, rawKwargs, debug = false) {
                         }
                         catch (err) {
                             if (debug)
-                                console.error(`[pre-nav] Failed to navigate to ${preNavUrl}: ${err instanceof Error ? err.message : err}`);
+                                log.debug(`[pre-nav] Failed to navigate to ${preNavUrl}: ${err instanceof Error ? err.message : err}`);
                         }
                     }
                 }

package/dist/main.js

@@ -19,7 +19,9 @@ import { discoverClis, discoverPlugins } from './discovery.js';
 import { getCompletions } from './completion.js';
 import { runCli } from './cli.js';
 import { emitHook } from './hooks.js';
+import { installNodeNetwork } from './node-network.js';
 import { registerUpdateNoticeOnExit, checkForUpdateBackground } from './update-check.js';
+installNodeNetwork();
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);
 const BUILTIN_CLIS = path.resolve(__dirname, 'clis');

package/dist/node-network.d.ts

@@ -0,0 +1,10 @@
+import { type Dispatcher } from 'undici';
+export interface ProxyDecision {
+    mode: 'direct' | 'proxy';
+    proxyUrl?: string;
+}
+export declare function hasProxyEnv(env?: NodeJS.ProcessEnv): boolean;
+export declare function decideProxy(url: URL, env?: NodeJS.ProcessEnv): ProxyDecision;
+export declare function getDispatcherForUrl(url: URL, env?: NodeJS.ProcessEnv): Dispatcher;
+export declare function fetchWithNodeNetwork(input: RequestInfo | URL, init?: RequestInit): Promise<Response>;
+export declare function installNodeNetwork(): void;

package/dist/node-network.js

@@ -0,0 +1,174 @@
+import { Agent, EnvHttpProxyAgent, fetch as undiciFetch } from 'undici';
+const LOOPBACK_NO_PROXY_ENTRIES = ['127.0.0.1', 'localhost', '::1'];
+const PROXY_ENV_BY_PROTOCOL = {
+    'http:': ['http_proxy', 'HTTP_PROXY', 'all_proxy', 'ALL_PROXY'],
+    'https:': ['https_proxy', 'HTTPS_PROXY', 'all_proxy', 'ALL_PROXY'],
+};
+const DEFAULT_PORT_BY_PROTOCOL = {
+    'http:': '80',
+    'https:': '443',
+};
+let installed = false;
+const directDispatcher = new Agent();
+const proxyDispatcherCache = new Map();
+const nativeFetch = globalThis.fetch.bind(globalThis);
+function readEnv(env, lower, upper) {
+    const lowerValue = env[lower];
+    if (typeof lowerValue === 'string' && lowerValue.trim() !== '')
+        return lowerValue;
+    const upperValue = env[upper];
+    if (typeof upperValue === 'string' && upperValue.trim() !== '')
+        return upperValue;
+    return undefined;
+}
+function readProxyEnv(env, keys) {
+    for (const key of keys) {
+        const value = env[key];
+        if (typeof value === 'string' && value.trim() !== '')
+            return value;
+    }
+    return undefined;
+}
+function normalizeHostname(hostname) {
+    return hostname.replace(/^\[(.*)\]$/, '$1').toLowerCase();
+}
+function splitNoProxy(raw) {
+    return (raw ?? '')
+        .split(/[,\s]+/)
+        .map((token) => token.trim())
+        .filter(Boolean);
+}
+function parseNoProxyEntry(entry) {
+    if (entry === '*')
+        return { host: '*' };
+    const trimmed = entry.trim().replace(/^\*?\./, '');
+    if (trimmed.startsWith('[')) {
+        const end = trimmed.indexOf(']');
+        if (end !== -1) {
+            const host = trimmed.slice(1, end);
+            const rest = trimmed.slice(end + 1);
+            if (rest.startsWith(':'))
+                return { host: normalizeHostname(host), port: rest.slice(1) };
+            return { host: normalizeHostname(host) };
+        }
+    }
+    const colonCount = (trimmed.match(/:/g) ?? []).length;
+    if (colonCount === 1) {
+        const [host, port] = trimmed.split(':');
+        return { host: normalizeHostname(host), port };
+    }
+    return { host: normalizeHostname(trimmed) };
+}
+function effectiveNoProxyEntries(env) {
+    const raw = readEnv(env, 'no_proxy', 'NO_PROXY');
+    const entries = splitNoProxy(raw).map(parseNoProxyEntry);
+    const seen = new Set(entries.map((entry) => `${entry.host}:${entry.port ?? ''}`));
+    for (const rawEntry of LOOPBACK_NO_PROXY_ENTRIES) {
+        const entry = parseNoProxyEntry(rawEntry);
+        const key = `${entry.host}:${entry.port ?? ''}`;
+        if (seen.has(key))
+            continue;
+        entries.push(entry);
+        seen.add(key);
+    }
+    return entries;
+}
+function serializeNoProxyEntry(entry) {
+    if (entry.host === '*')
+        return '*';
+    const host = entry.host.includes(':') ? `[${entry.host}]` : entry.host;
+    return entry.port ? `${host}:${entry.port}` : host;
+}
+function effectiveNoProxyValue(entries) {
+    if (entries.length === 0)
+        return undefined;
+    return entries.map(serializeNoProxyEntry).join(',');
+}
+function matchesNoProxyEntry(url, entry) {
+    const { host, port } = entry;
+    if (host === '*')
+        return true;
+    const hostname = normalizeHostname(url.hostname);
+    const urlPort = url.port || DEFAULT_PORT_BY_PROTOCOL[url.protocol] || undefined;
+    if (port && port !== urlPort)
+        return false;
+    return hostname === host || hostname.endsWith(`.${host}`);
+}
+function resolveProxyConfig(env = process.env) {
+    const noProxyEntries = effectiveNoProxyEntries(env);
+    return {
+        httpProxy: readProxyEnv(env, PROXY_ENV_BY_PROTOCOL['http:']),
+        httpsProxy: readProxyEnv(env, [
+            'https_proxy',
+            'HTTPS_PROXY',
+            'http_proxy',
+            'HTTP_PROXY',
+            'all_proxy',
+            'ALL_PROXY',
+        ]),
+        noProxy: effectiveNoProxyValue(noProxyEntries),
+        noProxyEntries,
+    };
+}
+function createProxyDispatcher(config) {
+    const cacheKey = JSON.stringify([
+        config.httpProxy ?? '',
+        config.httpsProxy ?? '',
+        config.noProxy ?? '',
+    ]);
+    const cached = proxyDispatcherCache.get(cacheKey);
+    if (cached)
+        return cached;
+    const dispatcher = new EnvHttpProxyAgent({
+        httpProxy: config.httpProxy,
+        httpsProxy: config.httpsProxy,
+        noProxy: config.noProxy,
+    });
+    proxyDispatcherCache.set(cacheKey, dispatcher);
+    return dispatcher;
+}
+function resolveUrl(input) {
+    if (typeof input === 'string')
+        return new URL(input);
+    if (input instanceof URL)
+        return input;
+    if (typeof Request !== 'undefined' && input instanceof Request)
+        return new URL(input.url);
+    return null;
+}
+export function hasProxyEnv(env = process.env) {
+    const config = resolveProxyConfig(env);
+    return Boolean(config.httpProxy || config.httpsProxy);
+}
+export function decideProxy(url, env = process.env) {
+    const config = resolveProxyConfig(env);
+    if (config.noProxyEntries.some((entry) => matchesNoProxyEntry(url, entry))) {
+        return { mode: 'direct' };
+    }
+    const proxyUrl = url.protocol === 'https:' ? config.httpsProxy : config.httpProxy;
+    if (!proxyUrl)
+        return { mode: 'direct' };
+    return { mode: 'proxy', proxyUrl };
+}
+export function getDispatcherForUrl(url, env = process.env) {
+    const config = resolveProxyConfig(env);
+    if (!config.httpProxy && !config.httpsProxy)
+        return directDispatcher;
+    return createProxyDispatcher(config);
+}
+export async function fetchWithNodeNetwork(input, init = {}) {
+    const url = resolveUrl(input);
+    if (!url || !hasProxyEnv()) {
+        return nativeFetch(input, init);
+    }
+    return (await undiciFetch(input, {
+        ...init,
+        dispatcher: getDispatcherForUrl(url),
+    }));
+}
+export function installNodeNetwork() {
+    if (installed)
+        return;
+    globalThis.fetch = ((input, init) => (fetchWithNodeNetwork(input, init)));
+    installed = true;
+}

package/dist/node-network.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/node-network.test.js

@@ -0,0 +1,55 @@
+import { describe, expect, it } from 'vitest';
+import { decideProxy, hasProxyEnv } from './node-network.js';
+describe('node network proxy decisions', () => {
+    it('detects common proxy env variables', () => {
+        expect(hasProxyEnv({ https_proxy: 'http://127.0.0.1:7897' })).toBe(true);
+        expect(hasProxyEnv({ HTTP_PROXY: 'http://proxy.example:8080' })).toBe(true);
+        expect(hasProxyEnv({})).toBe(false);
+    });
+    it('routes external https traffic through https_proxy', () => {
+        const decision = decideProxy(new URL('https://www.v2ex.com/api/topics/latest.json'), { https_proxy: 'http://127.0.0.1:7897' });
+        expect(decision).toEqual({
+            mode: 'proxy',
+            proxyUrl: 'http://127.0.0.1:7897',
+        });
+    });
+    it('falls back to HTTP_PROXY for https traffic when HTTPS_PROXY is absent', () => {
+        const decision = decideProxy(new URL('https://www.v2ex.com/api/topics/latest.json'), { HTTP_PROXY: 'http://127.0.0.1:7897' });
+        expect(decision).toEqual({
+            mode: 'proxy',
+            proxyUrl: 'http://127.0.0.1:7897',
+        });
+    });
+    it('bypasses proxies for loopback addresses', () => {
+        const env = { https_proxy: 'http://127.0.0.1:7897', http_proxy: 'http://127.0.0.1:7897' };
+        expect(decideProxy(new URL('http://127.0.0.1:19825/status'), env)).toEqual({ mode: 'direct' });
+        expect(decideProxy(new URL('http://localhost:19825/status'), env)).toEqual({ mode: 'direct' });
+        expect(decideProxy(new URL('http://[::1]:19825/status'), env)).toEqual({ mode: 'direct' });
+    });
+    it('honors NO_PROXY domain matches', () => {
+        const decision = decideProxy(new URL('https://api.example.com/v1/items'), {
+            https_proxy: 'http://127.0.0.1:7897',
+            no_proxy: '.example.com',
+        });
+        expect(decision).toEqual({ mode: 'direct' });
+    });
+    it('supports wildcard-style NO_PROXY subdomain entries', () => {
+        const decision = decideProxy(new URL('https://api.example.com/v1/items'), {
+            https_proxy: 'http://127.0.0.1:7897',
+            no_proxy: '*.example.com',
+        });
+        expect(decision).toEqual({ mode: 'direct' });
+    });
+    it('matches NO_PROXY entries that rely on the default URL port', () => {
+        const env = { https_proxy: 'http://127.0.0.1:7897', http_proxy: 'http://127.0.0.1:7897' };
+        expect(decideProxy(new URL('https://example.com/'), { ...env, NO_PROXY: 'example.com:443' })).toEqual({ mode: 'direct' });
+        expect(decideProxy(new URL('http://example.com/health'), { ...env, NO_PROXY: 'example.com:80' })).toEqual({ mode: 'direct' });
+    });
+    it('falls back to ALL_PROXY when protocol-specific settings are absent', () => {
+        const decision = decideProxy(new URL('http://example.net/data'), { ALL_PROXY: 'socks5://127.0.0.1:1080' });
+        expect(decision).toEqual({
+            mode: 'proxy',
+            proxyUrl: 'socks5://127.0.0.1:1080',
+        });
+    });
+});

package/dist/pipeline/executor.test.js

@@ -28,6 +28,7 @@ function createMockPage(overrides = {}) {
         installInterceptor: vi.fn(),
         getInterceptedRequests: vi.fn().mockResolvedValue([]),
         screenshot: vi.fn().mockResolvedValue(''),
+        waitForCapture: vi.fn().mockResolvedValue(undefined),
         ...overrides,
     };
 }

package/dist/pipeline/steps/download.test.js

@@ -39,6 +39,7 @@ function createMockPage(getCookies) {
         installInterceptor: vi.fn(),
         getInterceptedRequests: vi.fn().mockResolvedValue([]),
         screenshot: vi.fn().mockResolvedValue(''),
+        waitForCapture: vi.fn().mockResolvedValue(undefined),
     };
 }
 describe('stepDownload', () => {

package/dist/pipeline/steps/intercept.js

@@ -2,7 +2,6 @@
  * Pipeline step: intercept — declarative XHR interception.
  */
 import { render, normalizeEvaluateSource } from '../template.js';
-import { generateInterceptorJs, generateReadInterceptedJs } from '../../interceptor.js';
 export async function stepIntercept(page, params, data, args) {
     const cfg = typeof params === 'object' ? params : {};
     const trigger = cfg.trigger ?? '';
@@ -12,7 +11,7 @@ export async function stepIntercept(page, params, data, args) {
     if (!capturePattern)
         return data;
     // Step 1: Inject fetch/XHR interceptor BEFORE trigger
-    await page.evaluate(generateInterceptorJs(JSON.stringify(capturePattern)));
+    await page.installInterceptor(capturePattern);
     // Step 2: Execute the trigger action
     if (trigger.startsWith('navigate:')) {
         const url = render(trigger.slice('navigate:'.length), { args, data });
@@ -29,10 +28,10 @@ export async function stepIntercept(page, params, data, args) {
     else if (trigger === 'scroll') {
         await page.scroll('down');
     }
-    // Step 3: Wait a bit for network requests to fire
-    await page.wait(Math.min(timeout, 3));
+    // Step 3: Wait for network capture (event-driven, not fixed sleep)
+    await page.waitForCapture(timeout);
     // Step 4: Retrieve captured data
-    const matchingResponses = await page.evaluate(generateReadInterceptedJs());
+    const matchingResponses = await page.getInterceptedRequests();
     // Step 5: Select from response if specified
     let result = matchingResponses.length === 1 ? matchingResponses[0] :
         matchingResponses.length > 1 ? matchingResponses : data;

package/dist/types.d.ts

@@ -23,6 +23,7 @@ export interface SnapshotOptions {
 }
 export interface WaitOptions {
     text?: string;
+    selector?: string;
     time?: number;
     timeout?: number;
 }
@@ -67,6 +68,7 @@ export interface IPage {
     }): Promise<void>;
     installInterceptor(pattern: string): Promise<void>;
     getInterceptedRequests(): Promise<any[]>;
+    waitForCapture(timeout?: number): Promise<void>;
     screenshot(options?: ScreenshotOptions): Promise<string>;
     closeWindow?(): Promise<void>;
     /** Returns the current page URL, or null if unavailable. */

package/dist/utils.d.ts

@@ -5,5 +5,7 @@
 export declare function isRecord(value: unknown): value is Record<string, unknown>;
 /** Simple async concurrency limiter. */
 export declare function mapConcurrent<T, R>(items: T[], limit: number, fn: (item: T, index: number) => Promise<R>): Promise<R[]>;
+/** Pause for the given number of milliseconds. */
+export declare function sleep(ms: number): Promise<void>;
 /** Save a base64-encoded string to a file, creating parent directories as needed. */
 export declare function saveBase64ToFile(base64: string, filePath: string): Promise<void>;

package/dist/utils.js

@@ -21,6 +21,10 @@ export async function mapConcurrent(items, limit, fn) {
     await Promise.all(workers);
     return results;
 }
+/** Pause for the given number of milliseconds. */
+export function sleep(ms) {
+    return new Promise(resolve => setTimeout(resolve, ms));
+}
 /** Save a base64-encoded string to a file, creating parent directories as needed. */
 export async function saveBase64ToFile(base64, filePath) {
     const dir = path.dirname(filePath);