@jackwener/opencli 1.5.2 → 1.5.4

package/src/node-network.test.ts

@@ -0,0 +1,93 @@
+import { describe, expect, it } from 'vitest';
+
+import { decideProxy, hasProxyEnv } from './node-network.js';
+
+describe('node network proxy decisions', () => {
+  it('detects common proxy env variables', () => {
+    expect(hasProxyEnv({ https_proxy: 'http://127.0.0.1:7897' })).toBe(true);
+    expect(hasProxyEnv({ HTTP_PROXY: 'http://proxy.example:8080' })).toBe(true);
+    expect(hasProxyEnv({})).toBe(false);
+  });
+
+  it('routes external https traffic through https_proxy', () => {
+    const decision = decideProxy(
+      new URL('https://www.v2ex.com/api/topics/latest.json'),
+      { https_proxy: 'http://127.0.0.1:7897' },
+    );
+
+    expect(decision).toEqual({
+      mode: 'proxy',
+      proxyUrl: 'http://127.0.0.1:7897',
+    });
+  });
+
+  it('falls back to HTTP_PROXY for https traffic when HTTPS_PROXY is absent', () => {
+    const decision = decideProxy(
+      new URL('https://www.v2ex.com/api/topics/latest.json'),
+      { HTTP_PROXY: 'http://127.0.0.1:7897' },
+    );
+
+    expect(decision).toEqual({
+      mode: 'proxy',
+      proxyUrl: 'http://127.0.0.1:7897',
+    });
+  });
+
+  it('bypasses proxies for loopback addresses', () => {
+    const env = { https_proxy: 'http://127.0.0.1:7897', http_proxy: 'http://127.0.0.1:7897' };
+
+    expect(decideProxy(new URL('http://127.0.0.1:19825/status'), env)).toEqual({ mode: 'direct' });
+    expect(decideProxy(new URL('http://localhost:19825/status'), env)).toEqual({ mode: 'direct' });
+    expect(decideProxy(new URL('http://[::1]:19825/status'), env)).toEqual({ mode: 'direct' });
+  });
+
+  it('honors NO_PROXY domain matches', () => {
+    const decision = decideProxy(
+      new URL('https://api.example.com/v1/items'),
+      {
+        https_proxy: 'http://127.0.0.1:7897',
+        no_proxy: '.example.com',
+      },
+    );
+
+    expect(decision).toEqual({ mode: 'direct' });
+  });
+
+  it('supports wildcard-style NO_PROXY subdomain entries', () => {
+    const decision = decideProxy(
+      new URL('https://api.example.com/v1/items'),
+      {
+        https_proxy: 'http://127.0.0.1:7897',
+        no_proxy: '*.example.com',
+      },
+    );
+
+    expect(decision).toEqual({ mode: 'direct' });
+  });
+
+  it('matches NO_PROXY entries that rely on the default URL port', () => {
+    const env = { https_proxy: 'http://127.0.0.1:7897', http_proxy: 'http://127.0.0.1:7897' };
+
+    expect(decideProxy(
+      new URL('https://example.com/'),
+      { ...env, NO_PROXY: 'example.com:443' },
+    )).toEqual({ mode: 'direct' });
+
+    expect(decideProxy(
+      new URL('http://example.com/health'),
+      { ...env, NO_PROXY: 'example.com:80' },
+    )).toEqual({ mode: 'direct' });
+  });
+
+  it('falls back to ALL_PROXY when protocol-specific settings are absent', () => {
+    const decision = decideProxy(
+      new URL('http://example.net/data'),
+      { ALL_PROXY: 'socks5://127.0.0.1:1080' },
+    );
+
+    expect(decision).toEqual({
+      mode: 'proxy',
+      proxyUrl: 'socks5://127.0.0.1:1080',
+    });
+  });
+});

package/src/node-network.ts

@@ -0,0 +1,213 @@
+import { Agent, EnvHttpProxyAgent, fetch as undiciFetch, type Dispatcher } from 'undici';
+
+const LOOPBACK_NO_PROXY_ENTRIES = ['127.0.0.1', 'localhost', '::1'];
+
+type ProxyEnvKey =
+  | 'http_proxy'
+  | 'https_proxy'
+  | 'all_proxy'
+  | 'HTTP_PROXY'
+  | 'HTTPS_PROXY'
+  | 'ALL_PROXY';
+
+const PROXY_ENV_BY_PROTOCOL: Record<'http:' | 'https:', ProxyEnvKey[]> = {
+  'http:': ['http_proxy', 'HTTP_PROXY', 'all_proxy', 'ALL_PROXY'],
+  'https:': ['https_proxy', 'HTTPS_PROXY', 'all_proxy', 'ALL_PROXY'],
+};
+const DEFAULT_PORT_BY_PROTOCOL: Record<'http:' | 'https:', string> = {
+  'http:': '80',
+  'https:': '443',
+};
+
+export interface ProxyDecision {
+  mode: 'direct' | 'proxy';
+  proxyUrl?: string;
+}
+
+interface NoProxyEntry {
+  host: string;
+  port?: string;
+}
+
+interface ProxyConfig {
+  httpProxy?: string;
+  httpsProxy?: string;
+  noProxy?: string;
+  noProxyEntries: NoProxyEntry[];
+}
+
+let installed = false;
+const directDispatcher = new Agent();
+const proxyDispatcherCache = new Map<string, Dispatcher>();
+const nativeFetch = globalThis.fetch.bind(globalThis);
+
+function readEnv(env: NodeJS.ProcessEnv, lower: string, upper: string): string | undefined {
+  const lowerValue = env[lower];
+  if (typeof lowerValue === 'string' && lowerValue.trim() !== '') return lowerValue;
+  const upperValue = env[upper];
+  if (typeof upperValue === 'string' && upperValue.trim() !== '') return upperValue;
+  return undefined;
+}
+
+function readProxyEnv(env: NodeJS.ProcessEnv, keys: ProxyEnvKey[]): string | undefined {
+  for (const key of keys) {
+    const value = env[key];
+    if (typeof value === 'string' && value.trim() !== '') return value;
+  }
+  return undefined;
+}
+
+function normalizeHostname(hostname: string): string {
+  return hostname.replace(/^\[(.*)\]$/, '$1').toLowerCase();
+}
+
+function splitNoProxy(raw: string | undefined): string[] {
+  return (raw ?? '')
+    .split(/[,\s]+/)
+    .map((token) => token.trim())
+    .filter(Boolean);
+}
+
+function parseNoProxyEntry(entry: string): NoProxyEntry {
+  if (entry === '*') return { host: '*' };
+
+  const trimmed = entry.trim().replace(/^\*?\./, '');
+  if (trimmed.startsWith('[')) {
+    const end = trimmed.indexOf(']');
+    if (end !== -1) {
+      const host = trimmed.slice(1, end);
+      const rest = trimmed.slice(end + 1);
+      if (rest.startsWith(':')) return { host: normalizeHostname(host), port: rest.slice(1) };
+      return { host: normalizeHostname(host) };
+    }
+  }
+
+  const colonCount = (trimmed.match(/:/g) ?? []).length;
+  if (colonCount === 1) {
+    const [host, port] = trimmed.split(':');
+    return { host: normalizeHostname(host), port };
+  }
+
+  return { host: normalizeHostname(trimmed) };
+}
+
+function effectiveNoProxyEntries(env: NodeJS.ProcessEnv): NoProxyEntry[] {
+  const raw = readEnv(env, 'no_proxy', 'NO_PROXY');
+  const entries = splitNoProxy(raw).map(parseNoProxyEntry);
+  const seen = new Set(entries.map((entry) => `${entry.host}:${entry.port ?? ''}`));
+  for (const rawEntry of LOOPBACK_NO_PROXY_ENTRIES) {
+    const entry = parseNoProxyEntry(rawEntry);
+    const key = `${entry.host}:${entry.port ?? ''}`;
+    if (seen.has(key)) continue;
+    entries.push(entry);
+    seen.add(key);
+  }
+  return entries;
+}
+
+function serializeNoProxyEntry(entry: NoProxyEntry): string {
+  if (entry.host === '*') return '*';
+
+  const host = entry.host.includes(':') ? `[${entry.host}]` : entry.host;
+  return entry.port ? `${host}:${entry.port}` : host;
+}
+
+function effectiveNoProxyValue(entries: NoProxyEntry[]): string | undefined {
+  if (entries.length === 0) return undefined;
+
+  return entries.map(serializeNoProxyEntry).join(',');
+}
+
+function matchesNoProxyEntry(url: URL, entry: NoProxyEntry): boolean {
+  const { host, port } = entry;
+  if (host === '*') return true;
+
+  const hostname = normalizeHostname(url.hostname);
+  const urlPort = url.port || DEFAULT_PORT_BY_PROTOCOL[url.protocol as 'http:' | 'https:'] || undefined;
+  if (port && port !== urlPort) return false;
+  return hostname === host || hostname.endsWith(`.${host}`);
+}
+
+function resolveProxyConfig(env: NodeJS.ProcessEnv = process.env): ProxyConfig {
+  const noProxyEntries = effectiveNoProxyEntries(env);
+  return {
+    httpProxy: readProxyEnv(env, PROXY_ENV_BY_PROTOCOL['http:']),
+    httpsProxy: readProxyEnv(env, [
+      'https_proxy',
+      'HTTPS_PROXY',
+      'http_proxy',
+      'HTTP_PROXY',
+      'all_proxy',
+      'ALL_PROXY',
+    ]),
+    noProxy: effectiveNoProxyValue(noProxyEntries),
+    noProxyEntries,
+  };
+}
+
+function createProxyDispatcher(config: ProxyConfig): Dispatcher {
+  const cacheKey = JSON.stringify([
+    config.httpProxy ?? '',
+    config.httpsProxy ?? '',
+    config.noProxy ?? '',
+  ]);
+  const cached = proxyDispatcherCache.get(cacheKey);
+  if (cached) return cached;
+  const dispatcher = new EnvHttpProxyAgent({
+    httpProxy: config.httpProxy,
+    httpsProxy: config.httpsProxy,
+    noProxy: config.noProxy,
+  });
+  proxyDispatcherCache.set(cacheKey, dispatcher);
+  return dispatcher;
+}
+
+function resolveUrl(input: RequestInfo | URL): URL | null {
+  if (typeof input === 'string') return new URL(input);
+  if (input instanceof URL) return input;
+  if (typeof Request !== 'undefined' && input instanceof Request) return new URL(input.url);
+  return null;
+}
+
+export function hasProxyEnv(env: NodeJS.ProcessEnv = process.env): boolean {
+  const config = resolveProxyConfig(env);
+  return Boolean(config.httpProxy || config.httpsProxy);
+}
+
+export function decideProxy(url: URL, env: NodeJS.ProcessEnv = process.env): ProxyDecision {
+  const config = resolveProxyConfig(env);
+  if (config.noProxyEntries.some((entry) => matchesNoProxyEntry(url, entry))) {
+    return { mode: 'direct' };
+  }
+
+  const proxyUrl = url.protocol === 'https:' ? config.httpsProxy : config.httpProxy;
+  if (!proxyUrl) return { mode: 'direct' };
+  return { mode: 'proxy', proxyUrl };
+}
+
+export function getDispatcherForUrl(url: URL, env: NodeJS.ProcessEnv = process.env): Dispatcher {
+  const config = resolveProxyConfig(env);
+  if (!config.httpProxy && !config.httpsProxy) return directDispatcher;
+  return createProxyDispatcher(config);
+}
+
+export async function fetchWithNodeNetwork(input: RequestInfo | URL, init: RequestInit = {}): Promise<Response> {
+  const url = resolveUrl(input);
+  if (!url || !hasProxyEnv()) {
+    return nativeFetch(input, init);
+  }
+
+  return (await undiciFetch(input as Parameters<typeof undiciFetch>[0], {
+    ...init,
+    dispatcher: getDispatcherForUrl(url),
+  } as Parameters<typeof undiciFetch>[1])) as unknown as Response;
+}
+
+export function installNodeNetwork(): void {
+  if (installed) return;
+
+  globalThis.fetch = ((input: RequestInfo | URL, init?: RequestInit) => (
+    fetchWithNodeNetwork(input, init)
+  )) as typeof globalThis.fetch;
+  installed = true;
+}

package/src/pipeline/executor.test.ts

@@ -31,6 +31,7 @@ function createMockPage(overrides: Partial<IPage> = {}): IPage {
     installInterceptor: vi.fn(),
     getInterceptedRequests: vi.fn().mockResolvedValue([]),
     screenshot: vi.fn().mockResolvedValue(''),
+    waitForCapture: vi.fn().mockResolvedValue(undefined),
     ...overrides,
   };
 }

package/src/pipeline/steps/download.test.ts

@@ -44,6 +44,7 @@ function createMockPage(getCookies: IPage['getCookies']): IPage {
     installInterceptor: vi.fn(),
     getInterceptedRequests: vi.fn().mockResolvedValue([]),
     screenshot: vi.fn().mockResolvedValue(''),
+    waitForCapture: vi.fn().mockResolvedValue(undefined),
   };
 }
 

package/src/pipeline/steps/intercept.ts

@@ -4,7 +4,6 @@
 
 import type { IPage } from '../../types.js';
 import { render, normalizeEvaluateSource } from '../template.js';
-import { generateInterceptorJs, generateReadInterceptedJs } from '../../interceptor.js';
 
 export async function stepIntercept(page: IPage | null, params: any, data: any, args: Record<string, any>): Promise<any> {
   const cfg = typeof params === 'object' ? params : {};
@@ -16,7 +15,7 @@ export async function stepIntercept(page: IPage | null, params: any, data: any,
   if (!capturePattern) return data;
 
   // Step 1: Inject fetch/XHR interceptor BEFORE trigger
-  await page!.evaluate(generateInterceptorJs(JSON.stringify(capturePattern)));
+  await page!.installInterceptor(capturePattern);
 
   // Step 2: Execute the trigger action
   if (trigger.startsWith('navigate:')) {
@@ -32,11 +31,11 @@ export async function stepIntercept(page: IPage | null, params: any, data: any,
     await page!.scroll('down');
   }
 
-  // Step 3: Wait a bit for network requests to fire
-  await page!.wait(Math.min(timeout, 3));
+  // Step 3: Wait for network capture (event-driven, not fixed sleep)
+  await page!.waitForCapture(timeout);
 
   // Step 4: Retrieve captured data
-  const matchingResponses = await page!.evaluate(generateReadInterceptedJs());
+  const matchingResponses = await page!.getInterceptedRequests();
 
   // Step 5: Select from response if specified
   let result = matchingResponses.length === 1 ? matchingResponses[0] :

package/src/serialization.test.ts

@@ -0,0 +1,26 @@
+import { describe, expect, it } from 'vitest';
+import type { CliCommand } from './registry.js';
+import { Strategy } from './registry.js';
+import { formatRegistryHelpText } from './serialization.js';
+
+describe('formatRegistryHelpText', () => {
+  it('summarizes long choices lists so help text stays readable', () => {
+    const cmd: CliCommand = {
+      site: 'demo',
+      name: 'dynamic',
+      description: 'Demo command',
+      strategy: Strategy.PUBLIC,
+      browser: false,
+      args: [
+        {
+          name: 'field',
+          help: 'Field to use',
+          choices: ['all-fields', 'topic', 'title', 'author', 'publication-titles', 'year-published', 'doi'],
+        },
+      ],
+      columns: ['field'],
+    };
+
+    expect(formatRegistryHelpText(cmd)).toContain('--field: all-fields, topic, title, author, ... (+3 more)');
+  });
+});

package/src/serialization.ts

@@ -62,6 +62,11 @@ export function formatArgSummary(args: Arg[]): string {
     .join(' ');
 }
 
+function summarizeChoices(choices: string[]): string {
+  if (choices.length <= 4) return choices.join(', ');
+  return `${choices.slice(0, 4).join(', ')}, ... (+${choices.length - 4} more)`;
+}
+
 /** Generate the --help appendix showing registry metadata not exposed by Commander. */
 export function formatRegistryHelpText(cmd: CliCommand): string {
   const lines: string[] = [];
@@ -69,7 +74,7 @@ export function formatRegistryHelpText(cmd: CliCommand): string {
   for (const a of choicesArgs) {
     const prefix = a.positional ? `<${a.name}>` : `--${a.name}`;
     const def = a.default != null ? `  (default: ${a.default})` : '';
-    lines.push(`  ${prefix}: ${a.choices!.join(', ')}${def}`);
+    lines.push(`  ${prefix}: ${summarizeChoices(a.choices!)}${def}`);
   }
   const meta: string[] = [];
   meta.push(`Strategy: ${strategyLabel(cmd)}`);

package/src/types.ts

@@ -26,6 +26,7 @@ export interface SnapshotOptions {
 
 export interface WaitOptions {
   text?: string;
+  selector?: string;   // wait until document.querySelector(selector) matches
   time?: number;
   timeout?: number;
 }
@@ -64,6 +65,7 @@ export interface IPage {
   autoScroll(options?: { times?: number; delayMs?: number }): Promise<void>;
   installInterceptor(pattern: string): Promise<void>;
   getInterceptedRequests(): Promise<any[]>;
+  waitForCapture(timeout?: number): Promise<void>;
   screenshot(options?: ScreenshotOptions): Promise<string>;
   closeWindow?(): Promise<void>;
   /** Returns the current page URL, or null if unavailable. */

package/src/utils.ts

@@ -31,6 +31,11 @@ export async function mapConcurrent<T, R>(
   return results;
 }
 
+/** Pause for the given number of milliseconds. */
+export function sleep(ms: number): Promise<void> {
+  return new Promise(resolve => setTimeout(resolve, ms));
+}
+
 /** Save a base64-encoded string to a file, creating parent directories as needed. */
 export async function saveBase64ToFile(base64: string, filePath: string): Promise<void> {
   const dir = path.dirname(filePath);