@jackwener/opencli 1.5.2 → 1.5.4

package/dist/node-network.d.ts

@@ -0,0 +1,10 @@
+import { type Dispatcher } from 'undici';
+export interface ProxyDecision {
+    mode: 'direct' | 'proxy';
+    proxyUrl?: string;
+}
+export declare function hasProxyEnv(env?: NodeJS.ProcessEnv): boolean;
+export declare function decideProxy(url: URL, env?: NodeJS.ProcessEnv): ProxyDecision;
+export declare function getDispatcherForUrl(url: URL, env?: NodeJS.ProcessEnv): Dispatcher;
+export declare function fetchWithNodeNetwork(input: RequestInfo | URL, init?: RequestInit): Promise<Response>;
+export declare function installNodeNetwork(): void;

package/dist/node-network.js

@@ -0,0 +1,174 @@
+import { Agent, EnvHttpProxyAgent, fetch as undiciFetch } from 'undici';
+const LOOPBACK_NO_PROXY_ENTRIES = ['127.0.0.1', 'localhost', '::1'];
+const PROXY_ENV_BY_PROTOCOL = {
+    'http:': ['http_proxy', 'HTTP_PROXY', 'all_proxy', 'ALL_PROXY'],
+    'https:': ['https_proxy', 'HTTPS_PROXY', 'all_proxy', 'ALL_PROXY'],
+};
+const DEFAULT_PORT_BY_PROTOCOL = {
+    'http:': '80',
+    'https:': '443',
+};
+let installed = false;
+const directDispatcher = new Agent();
+const proxyDispatcherCache = new Map();
+const nativeFetch = globalThis.fetch.bind(globalThis);
+function readEnv(env, lower, upper) {
+    const lowerValue = env[lower];
+    if (typeof lowerValue === 'string' && lowerValue.trim() !== '')
+        return lowerValue;
+    const upperValue = env[upper];
+    if (typeof upperValue === 'string' && upperValue.trim() !== '')
+        return upperValue;
+    return undefined;
+}
+function readProxyEnv(env, keys) {
+    for (const key of keys) {
+        const value = env[key];
+        if (typeof value === 'string' && value.trim() !== '')
+            return value;
+    }
+    return undefined;
+}
+function normalizeHostname(hostname) {
+    return hostname.replace(/^\[(.*)\]$/, '$1').toLowerCase();
+}
+function splitNoProxy(raw) {
+    return (raw ?? '')
+        .split(/[,\s]+/)
+        .map((token) => token.trim())
+        .filter(Boolean);
+}
+function parseNoProxyEntry(entry) {
+    if (entry === '*')
+        return { host: '*' };
+    const trimmed = entry.trim().replace(/^\*?\./, '');
+    if (trimmed.startsWith('[')) {
+        const end = trimmed.indexOf(']');
+        if (end !== -1) {
+            const host = trimmed.slice(1, end);
+            const rest = trimmed.slice(end + 1);
+            if (rest.startsWith(':'))
+                return { host: normalizeHostname(host), port: rest.slice(1) };
+            return { host: normalizeHostname(host) };
+        }
+    }
+    const colonCount = (trimmed.match(/:/g) ?? []).length;
+    if (colonCount === 1) {
+        const [host, port] = trimmed.split(':');
+        return { host: normalizeHostname(host), port };
+    }
+    return { host: normalizeHostname(trimmed) };
+}
+function effectiveNoProxyEntries(env) {
+    const raw = readEnv(env, 'no_proxy', 'NO_PROXY');
+    const entries = splitNoProxy(raw).map(parseNoProxyEntry);
+    const seen = new Set(entries.map((entry) => `${entry.host}:${entry.port ?? ''}`));
+    for (const rawEntry of LOOPBACK_NO_PROXY_ENTRIES) {
+        const entry = parseNoProxyEntry(rawEntry);
+        const key = `${entry.host}:${entry.port ?? ''}`;
+        if (seen.has(key))
+            continue;
+        entries.push(entry);
+        seen.add(key);
+    }
+    return entries;
+}
+function serializeNoProxyEntry(entry) {
+    if (entry.host === '*')
+        return '*';
+    const host = entry.host.includes(':') ? `[${entry.host}]` : entry.host;
+    return entry.port ? `${host}:${entry.port}` : host;
+}
+function effectiveNoProxyValue(entries) {
+    if (entries.length === 0)
+        return undefined;
+    return entries.map(serializeNoProxyEntry).join(',');
+}
+function matchesNoProxyEntry(url, entry) {
+    const { host, port } = entry;
+    if (host === '*')
+        return true;
+    const hostname = normalizeHostname(url.hostname);
+    const urlPort = url.port || DEFAULT_PORT_BY_PROTOCOL[url.protocol] || undefined;
+    if (port && port !== urlPort)
+        return false;
+    return hostname === host || hostname.endsWith(`.${host}`);
+}
+function resolveProxyConfig(env = process.env) {
+    const noProxyEntries = effectiveNoProxyEntries(env);
+    return {
+        httpProxy: readProxyEnv(env, PROXY_ENV_BY_PROTOCOL['http:']),
+        httpsProxy: readProxyEnv(env, [
+            'https_proxy',
+            'HTTPS_PROXY',
+            'http_proxy',
+            'HTTP_PROXY',
+            'all_proxy',
+            'ALL_PROXY',
+        ]),
+        noProxy: effectiveNoProxyValue(noProxyEntries),
+        noProxyEntries,
+    };
+}
+function createProxyDispatcher(config) {
+    const cacheKey = JSON.stringify([
+        config.httpProxy ?? '',
+        config.httpsProxy ?? '',
+        config.noProxy ?? '',
+    ]);
+    const cached = proxyDispatcherCache.get(cacheKey);
+    if (cached)
+        return cached;
+    const dispatcher = new EnvHttpProxyAgent({
+        httpProxy: config.httpProxy,
+        httpsProxy: config.httpsProxy,
+        noProxy: config.noProxy,
+    });
+    proxyDispatcherCache.set(cacheKey, dispatcher);
+    return dispatcher;
+}
+function resolveUrl(input) {
+    if (typeof input === 'string')
+        return new URL(input);
+    if (input instanceof URL)
+        return input;
+    if (typeof Request !== 'undefined' && input instanceof Request)
+        return new URL(input.url);
+    return null;
+}
+export function hasProxyEnv(env = process.env) {
+    const config = resolveProxyConfig(env);
+    return Boolean(config.httpProxy || config.httpsProxy);
+}
+export function decideProxy(url, env = process.env) {
+    const config = resolveProxyConfig(env);
+    if (config.noProxyEntries.some((entry) => matchesNoProxyEntry(url, entry))) {
+        return { mode: 'direct' };
+    }
+    const proxyUrl = url.protocol === 'https:' ? config.httpsProxy : config.httpProxy;
+    if (!proxyUrl)
+        return { mode: 'direct' };
+    return { mode: 'proxy', proxyUrl };
+}
+export function getDispatcherForUrl(url, env = process.env) {
+    const config = resolveProxyConfig(env);
+    if (!config.httpProxy && !config.httpsProxy)
+        return directDispatcher;
+    return createProxyDispatcher(config);
+}
+export async function fetchWithNodeNetwork(input, init = {}) {
+    const url = resolveUrl(input);
+    if (!url || !hasProxyEnv()) {
+        return nativeFetch(input, init);
+    }
+    return (await undiciFetch(input, {
+        ...init,
+        dispatcher: getDispatcherForUrl(url),
+    }));
+}
+export function installNodeNetwork() {
+    if (installed)
+        return;
+    globalThis.fetch = ((input, init) => (fetchWithNodeNetwork(input, init)));
+    installed = true;
+}

package/dist/node-network.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/node-network.test.js

@@ -0,0 +1,55 @@
+import { describe, expect, it } from 'vitest';
+import { decideProxy, hasProxyEnv } from './node-network.js';
+describe('node network proxy decisions', () => {
+    it('detects common proxy env variables', () => {
+        expect(hasProxyEnv({ https_proxy: 'http://127.0.0.1:7897' })).toBe(true);
+        expect(hasProxyEnv({ HTTP_PROXY: 'http://proxy.example:8080' })).toBe(true);
+        expect(hasProxyEnv({})).toBe(false);
+    });
+    it('routes external https traffic through https_proxy', () => {
+        const decision = decideProxy(new URL('https://www.v2ex.com/api/topics/latest.json'), { https_proxy: 'http://127.0.0.1:7897' });
+        expect(decision).toEqual({
+            mode: 'proxy',
+            proxyUrl: 'http://127.0.0.1:7897',
+        });
+    });
+    it('falls back to HTTP_PROXY for https traffic when HTTPS_PROXY is absent', () => {
+        const decision = decideProxy(new URL('https://www.v2ex.com/api/topics/latest.json'), { HTTP_PROXY: 'http://127.0.0.1:7897' });
+        expect(decision).toEqual({
+            mode: 'proxy',
+            proxyUrl: 'http://127.0.0.1:7897',
+        });
+    });
+    it('bypasses proxies for loopback addresses', () => {
+        const env = { https_proxy: 'http://127.0.0.1:7897', http_proxy: 'http://127.0.0.1:7897' };
+        expect(decideProxy(new URL('http://127.0.0.1:19825/status'), env)).toEqual({ mode: 'direct' });
+        expect(decideProxy(new URL('http://localhost:19825/status'), env)).toEqual({ mode: 'direct' });
+        expect(decideProxy(new URL('http://[::1]:19825/status'), env)).toEqual({ mode: 'direct' });
+    });
+    it('honors NO_PROXY domain matches', () => {
+        const decision = decideProxy(new URL('https://api.example.com/v1/items'), {
+            https_proxy: 'http://127.0.0.1:7897',
+            no_proxy: '.example.com',
+        });
+        expect(decision).toEqual({ mode: 'direct' });
+    });
+    it('supports wildcard-style NO_PROXY subdomain entries', () => {
+        const decision = decideProxy(new URL('https://api.example.com/v1/items'), {
+            https_proxy: 'http://127.0.0.1:7897',
+            no_proxy: '*.example.com',
+        });
+        expect(decision).toEqual({ mode: 'direct' });
+    });
+    it('matches NO_PROXY entries that rely on the default URL port', () => {
+        const env = { https_proxy: 'http://127.0.0.1:7897', http_proxy: 'http://127.0.0.1:7897' };
+        expect(decideProxy(new URL('https://example.com/'), { ...env, NO_PROXY: 'example.com:443' })).toEqual({ mode: 'direct' });
+        expect(decideProxy(new URL('http://example.com/health'), { ...env, NO_PROXY: 'example.com:80' })).toEqual({ mode: 'direct' });
+    });
+    it('falls back to ALL_PROXY when protocol-specific settings are absent', () => {
+        const decision = decideProxy(new URL('http://example.net/data'), { ALL_PROXY: 'socks5://127.0.0.1:1080' });
+        expect(decision).toEqual({
+            mode: 'proxy',
+            proxyUrl: 'socks5://127.0.0.1:1080',
+        });
+    });
+});

package/dist/pipeline/executor.test.js

@@ -28,6 +28,7 @@ function createMockPage(overrides = {}) {
         installInterceptor: vi.fn(),
         getInterceptedRequests: vi.fn().mockResolvedValue([]),
         screenshot: vi.fn().mockResolvedValue(''),
+        waitForCapture: vi.fn().mockResolvedValue(undefined),
         ...overrides,
     };
 }

package/dist/pipeline/steps/download.test.js

@@ -39,6 +39,7 @@ function createMockPage(getCookies) {
         installInterceptor: vi.fn(),
         getInterceptedRequests: vi.fn().mockResolvedValue([]),
         screenshot: vi.fn().mockResolvedValue(''),
+        waitForCapture: vi.fn().mockResolvedValue(undefined),
     };
 }
 describe('stepDownload', () => {

package/dist/pipeline/steps/intercept.js

@@ -2,7 +2,6 @@
  * Pipeline step: intercept — declarative XHR interception.
  */
 import { render, normalizeEvaluateSource } from '../template.js';
-import { generateInterceptorJs, generateReadInterceptedJs } from '../../interceptor.js';
 export async function stepIntercept(page, params, data, args) {
     const cfg = typeof params === 'object' ? params : {};
     const trigger = cfg.trigger ?? '';
@@ -12,7 +11,7 @@ export async function stepIntercept(page, params, data, args) {
     if (!capturePattern)
         return data;
     // Step 1: Inject fetch/XHR interceptor BEFORE trigger
-    await page.evaluate(generateInterceptorJs(JSON.stringify(capturePattern)));
+    await page.installInterceptor(capturePattern);
     // Step 2: Execute the trigger action
     if (trigger.startsWith('navigate:')) {
         const url = render(trigger.slice('navigate:'.length), { args, data });
@@ -29,10 +28,10 @@ export async function stepIntercept(page, params, data, args) {
     else if (trigger === 'scroll') {
         await page.scroll('down');
     }
-    // Step 3: Wait a bit for network requests to fire
-    await page.wait(Math.min(timeout, 3));
+    // Step 3: Wait for network capture (event-driven, not fixed sleep)
+    await page.waitForCapture(timeout);
     // Step 4: Retrieve captured data
-    const matchingResponses = await page.evaluate(generateReadInterceptedJs());
+    const matchingResponses = await page.getInterceptedRequests();
     // Step 5: Select from response if specified
     let result = matchingResponses.length === 1 ? matchingResponses[0] :
         matchingResponses.length > 1 ? matchingResponses : data;

package/dist/serialization.js

@@ -44,6 +44,11 @@ export function formatArgSummary(args) {
     })
         .join(' ');
 }
+function summarizeChoices(choices) {
+    if (choices.length <= 4)
+        return choices.join(', ');
+    return `${choices.slice(0, 4).join(', ')}, ... (+${choices.length - 4} more)`;
+}
 /** Generate the --help appendix showing registry metadata not exposed by Commander. */
 export function formatRegistryHelpText(cmd) {
     const lines = [];
@@ -51,7 +56,7 @@ export function formatRegistryHelpText(cmd) {
     for (const a of choicesArgs) {
         const prefix = a.positional ? `<${a.name}>` : `--${a.name}`;
         const def = a.default != null ? `  (default: ${a.default})` : '';
-        lines.push(`  ${prefix}: ${a.choices.join(', ')}${def}`);
+        lines.push(`  ${prefix}: ${summarizeChoices(a.choices)}${def}`);
     }
     const meta = [];
     meta.push(`Strategy: ${strategyLabel(cmd)}`);

package/dist/serialization.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/serialization.test.js

@@ -0,0 +1,23 @@
+import { describe, expect, it } from 'vitest';
+import { Strategy } from './registry.js';
+import { formatRegistryHelpText } from './serialization.js';
+describe('formatRegistryHelpText', () => {
+    it('summarizes long choices lists so help text stays readable', () => {
+        const cmd = {
+            site: 'demo',
+            name: 'dynamic',
+            description: 'Demo command',
+            strategy: Strategy.PUBLIC,
+            browser: false,
+            args: [
+                {
+                    name: 'field',
+                    help: 'Field to use',
+                    choices: ['all-fields', 'topic', 'title', 'author', 'publication-titles', 'year-published', 'doi'],
+                },
+            ],
+            columns: ['field'],
+        };
+        expect(formatRegistryHelpText(cmd)).toContain('--field: all-fields, topic, title, author, ... (+3 more)');
+    });
+});

package/dist/types.d.ts

@@ -23,6 +23,7 @@ export interface SnapshotOptions {
 }
 export interface WaitOptions {
     text?: string;
+    selector?: string;
     time?: number;
     timeout?: number;
 }
@@ -67,6 +68,7 @@ export interface IPage {
     }): Promise<void>;
     installInterceptor(pattern: string): Promise<void>;
     getInterceptedRequests(): Promise<any[]>;
+    waitForCapture(timeout?: number): Promise<void>;
     screenshot(options?: ScreenshotOptions): Promise<string>;
     closeWindow?(): Promise<void>;
     /** Returns the current page URL, or null if unavailable. */

package/dist/utils.d.ts

@@ -5,5 +5,7 @@
 export declare function isRecord(value: unknown): value is Record<string, unknown>;
 /** Simple async concurrency limiter. */
 export declare function mapConcurrent<T, R>(items: T[], limit: number, fn: (item: T, index: number) => Promise<R>): Promise<R[]>;
+/** Pause for the given number of milliseconds. */
+export declare function sleep(ms: number): Promise<void>;
 /** Save a base64-encoded string to a file, creating parent directories as needed. */
 export declare function saveBase64ToFile(base64: string, filePath: string): Promise<void>;

package/dist/utils.js

@@ -21,6 +21,10 @@ export async function mapConcurrent(items, limit, fn) {
     await Promise.all(workers);
     return results;
 }
+/** Pause for the given number of milliseconds. */
+export function sleep(ms) {
+    return new Promise(resolve => setTimeout(resolve, ms));
+}
 /** Save a base64-encoded string to a file, creating parent directories as needed. */
 export async function saveBase64ToFile(base64, filePath) {
     const dir = path.dirname(filePath);