@jackwener/opencli 1.5.2 → 1.5.3

package/src/clis/douban/utils.ts

@@ -4,17 +4,13 @@
 
 import { ArgumentError, CliError, EmptyResultError } from '../../errors.js';
 import type { IPage } from '../../types.js';
+import { clamp } from '../_shared/common.js';
 
 const DOUBAN_PHOTO_PAGE_SIZE = 30;
 const MAX_DOUBAN_PHOTOS = 500;
 
-function clampLimit(limit: number): number {
-  return Math.max(1, Math.min(limit || 20, 50));
-}
-
-function clampPhotoLimit(limit: number): number {
-  return Math.max(1, Math.min(limit || 120, MAX_DOUBAN_PHOTOS));
-}
+const clampLimit = (limit: number) => clamp(limit || 20, 1, 50);
+const clampPhotoLimit = (limit: number) => clamp(limit || 120, 1, MAX_DOUBAN_PHOTOS);
 
 async function ensureDoubanReady(page: IPage): Promise<void> {
   const state = await page.evaluate(`

package/src/clis/medium/utils.ts

@@ -16,7 +16,7 @@ export function buildMediumUserUrl(username: string): string {
 export async function loadMediumPosts(page: IPage, url: string, limit: number): Promise<any[]> {
   if (!page) throw new CommandExecutionError('Browser session required for medium posts');
   await page.goto(url);
-  await page.wait(5);
+  await page.wait({ selector: 'article', timeout: 5 });
   const data = await page.evaluate(`
     (async () => {
       await new Promise((resolve) => setTimeout(resolve, 3000));

package/src/clis/producthunt/browse.ts

@@ -32,7 +32,7 @@ cli({
 
     await page.installInterceptor('producthunt.com');
     await page.goto(`https://www.producthunt.com/categories/${slug}`);
-    await page.wait(5);
+    await page.waitForCapture(5);
 
     const domItems: any = await page.evaluate(`
       (() => {

package/src/clis/producthunt/hot.ts

@@ -23,7 +23,7 @@ cli({
 
     await page.installInterceptor('producthunt.com');
     await page.goto('https://www.producthunt.com');
-    await page.wait(5);
+    await page.waitForCapture(5);
 
     const domItems: any = await page.evaluate(`
       (() => {

package/src/clis/sinablog/utils.ts

@@ -1,8 +1,7 @@
 import type { IPage } from '../../types.js';
+import { clamp } from '../_shared/common.js';
 
-function clampLimit(limit: number): number {
-  return Math.max(1, Math.min(limit || 20, 50));
-}
+const clampLimit = (limit: number) => clamp(limit || 20, 1, 50);
 
 export function buildSinaBlogSearchUrl(keyword: string): string {
   return `https://search.sina.com.cn/search?q=${encodeURIComponent(keyword)}&tp=mix`;
@@ -14,7 +13,7 @@ export function buildSinaBlogUserUrl(uid: string): string {
 
 export async function loadSinaBlogArticle(page: IPage, url: string): Promise<any> {
   await page.goto(url);
-  await page.wait(3);
+  await page.wait({ selector: 'h1', timeout: 3 });
   return page.evaluate(`
     (async () => {
       await new Promise((resolve) => setTimeout(resolve, 1500));
@@ -52,7 +51,7 @@ export async function loadSinaBlogArticle(page: IPage, url: string): Promise<any
 export async function loadSinaBlogHot(page: IPage, limit: number): Promise<any[]> {
   const safeLimit = clampLimit(limit);
   await page.goto('https://blog.sina.com.cn/');
-  await page.wait(3);
+  await page.wait({ selector: 'h1', timeout: 3 });
   const data = await page.evaluate(`
     (async () => {
       await new Promise((resolve) => setTimeout(resolve, 1500));
@@ -122,7 +121,7 @@ export async function loadSinaBlogHot(page: IPage, limit: number): Promise<any[]
 export async function loadSinaBlogSearch(page: IPage, keyword: string, limit: number): Promise<any[]> {
   const safeLimit = clampLimit(limit);
   await page.goto(buildSinaBlogSearchUrl(keyword));
-  await page.wait(5);
+  await page.wait({ selector: '.result-item', timeout: 5 });
   const data = await page.evaluate(`
     (async () => {
       const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
@@ -159,7 +158,7 @@ export async function loadSinaBlogSearch(page: IPage, keyword: string, limit: nu
 export async function loadSinaBlogUser(page: IPage, uid: string, limit: number): Promise<any[]> {
   const safeLimit = clampLimit(limit);
   await page.goto(buildSinaBlogUserUrl(uid));
-  await page.wait(3);
+  await page.wait({ selector: 'h1', timeout: 3 });
   const data = await page.evaluate(`
     (async () => {
       await new Promise((resolve) => setTimeout(resolve, 1000));

package/src/clis/substack/utils.ts

@@ -10,7 +10,7 @@ export function buildSubstackBrowseUrl(category?: string): string {
 export async function loadSubstackFeed(page: IPage, url: string, limit: number): Promise<any[]> {
   if (!page) throw new CommandExecutionError('Browser session required for substack feed');
   await page.goto(url);
-  await page.wait(5);
+  await page.wait({ selector: 'article', timeout: 5 });
   const data = await page.evaluate(`
     (async () => {
       await new Promise((resolve) => setTimeout(resolve, 3000));
@@ -79,7 +79,7 @@ export async function loadSubstackFeed(page: IPage, url: string, limit: number):
 export async function loadSubstackArchive(page: IPage, baseUrl: string, limit: number): Promise<any[]> {
   if (!page) throw new CommandExecutionError('Browser session required for substack archive');
   await page.goto(`${baseUrl}/archive`);
-  await page.wait(5);
+  await page.wait({ selector: 'article', timeout: 5 });
   const data = await page.evaluate(`
     (async () => {
       await new Promise((resolve) => setTimeout(resolve, 3000));

package/src/clis/twitter/block.ts

@@ -18,7 +18,7 @@ cli({
     const username = kwargs.username.replace(/^@/, '');
 
     await page.goto(`https://x.com/${username}`);
-    await page.wait(5);
+    await page.wait({ selector: '[data-testid="primaryColumn"]' });
 
     const result = await page.evaluate(`(async () => {
         try {

package/src/clis/twitter/bookmark.ts

@@ -17,7 +17,7 @@ cli({
     if (!page) throw new CommandExecutionError('Browser session required for twitter bookmark');
 
     await page.goto(kwargs.url);
-    await page.wait(5);
+    await page.wait({ selector: '[data-testid="primaryColumn"]' });
 
     const result = await page.evaluate(`(async () => {
         try {

package/src/clis/twitter/delete.ts

@@ -17,7 +17,7 @@ cli({
     if (!page) throw new CommandExecutionError('Browser session required for twitter delete');
 
     await page.goto(kwargs.url);
-    await page.wait(5); // Wait for tweet to load completely
+    await page.wait({ selector: '[data-testid="primaryColumn"]' }); // Wait for tweet to load completely
 
     const result = await page.evaluate(`(async () => {
         try {

package/src/clis/twitter/follow.ts

@@ -18,7 +18,7 @@ cli({
     const username = kwargs.username.replace(/^@/, '');
 
     await page.goto(`https://x.com/${username}`);
-    await page.wait(5);
+    await page.wait({ selector: '[data-testid="primaryColumn"]' });
 
     const result = await page.evaluate(`(async () => {
         try {

package/src/clis/twitter/followers.ts

@@ -19,7 +19,7 @@ cli({
     // If no user is specified, figure out the logged-in user's handle
     if (!targetUser) {
         await page.goto('https://x.com/home');
-        await page.wait(5);
+        await page.wait({ selector: '[data-testid="primaryColumn"]' });
 
         const href = await page.evaluate(`() => {
             const link = document.querySelector('a[data-testid="AppTabBar_Profile_Link"]');
@@ -58,7 +58,7 @@ cli({
     if (!clicked) {
         throw new SelectorError('Twitter followers link', 'Twitter may have changed the layout.');
     }
-    await page.wait(5);
+    await page.waitForCapture(5);
 
     // 4. Scroll to trigger pagination API calls
     await page.autoScroll({ times: Math.ceil(kwargs.limit / 20), delayMs: 2000 });

package/src/clis/twitter/following.ts

@@ -19,7 +19,7 @@ cli({
     // If no user is specified, figure out the logged-in user's handle
     if (!targetUser) {
         await page.goto('https://x.com/home');
-        await page.wait(5);
+        await page.wait({ selector: '[data-testid="primaryColumn"]' });
 
         const href = await page.evaluate(`() => {
             const link = document.querySelector('a[data-testid="AppTabBar_Profile_Link"]');
@@ -51,7 +51,7 @@ cli({
     if (!clicked) {
         throw new SelectorError('Twitter following link', 'Twitter may have changed the layout.');
     }
-    await page.wait(5);
+    await page.waitForCapture(5);
 
     // 4. Scroll to trigger pagination API calls
     await page.autoScroll({ times: Math.ceil(kwargs.limit / 20), delayMs: 2000 });

package/src/clis/twitter/hide-reply.ts

@@ -17,7 +17,7 @@ cli({
     if (!page) throw new CommandExecutionError('Browser session required for twitter hide-reply');
 
     await page.goto(kwargs.url);
-    await page.wait(5);
+    await page.wait({ selector: '[data-testid="primaryColumn"]' });
 
     const result = await page.evaluate(`(async () => {
         try {

package/src/clis/twitter/like.ts

@@ -17,7 +17,7 @@ cli({
     if (!page) throw new CommandExecutionError('Browser session required for twitter like');
 
     await page.goto(kwargs.url);
-    await page.wait(5); // Wait for tweet to load completely
+    await page.wait({ selector: '[data-testid="primaryColumn"]' }); // Wait for tweet to load completely
 
     const result = await page.evaluate(`(async () => {
         try {

package/src/clis/twitter/notifications.ts

@@ -25,7 +25,7 @@ cli({
         window.history.pushState({}, '', '/notifications');
         window.dispatchEvent(new PopStateEvent('popstate', { state: {} }));
     }`);
-    await page.wait(5);
+    await page.waitForCapture(5);
 
     // Verify SPA navigation succeeded
     const currentUrl = await page.evaluate('() => window.location.pathname');

package/src/clis/twitter/profile.ts

@@ -21,7 +21,7 @@ cli({
     // If no username, detect the logged-in user
     if (!username) {
       await page.goto('https://x.com/home');
-      await page.wait(5);
+      await page.wait({ selector: '[data-testid="primaryColumn"]' });
       const href = await page.evaluate(`() => {
         const link = document.querySelector('a[data-testid="AppTabBar_Profile_Link"]');
         return link ? link.getAttribute('href') : null;

package/src/clis/twitter/reply-dm.ts

@@ -27,7 +27,7 @@ cli({
 
     // Step 1: Navigate to messages to get conversation list
     await page.goto('https://x.com/messages');
-    await page.wait(5);
+    await page.wait({ selector: '[data-testid="primaryColumn"]' });
 
     // Step 2: Collect conversations with scroll-to-load
     const needed = maxSend + 10; // extra buffer for skips

package/src/clis/twitter/reply.ts

@@ -19,7 +19,7 @@ cli({
 
     // 1. Navigate to the tweet page
     await page.goto(kwargs.url);
-    await page.wait(5); // Wait for the react application to hydrate
+    await page.wait({ selector: '[data-testid="primaryColumn"]' });
 
     // 2. Automate typing the reply and clicking reply
     const result = await page.evaluate(`(async () => {

package/src/clis/twitter/search.ts

@@ -20,7 +20,7 @@ async function navigateToSearch(page: Pick<IPage, 'evaluate' | 'wait'>, query: s
         window.dispatchEvent(new PopStateEvent('popstate', { state: {} }));
       })()
     `);
-    await page.wait(5);
+    await page.wait({ selector: '[data-testid="primaryColumn"]' });
 
     lastPath = String(await page.evaluate('() => window.location.pathname') || '');
     if (lastPath.startsWith('/search')) {

package/src/clis/twitter/unblock.ts

@@ -18,7 +18,7 @@ cli({
     const username = kwargs.username.replace(/^@/, '');
 
     await page.goto(`https://x.com/${username}`);
-    await page.wait(5);
+    await page.wait({ selector: '[data-testid="primaryColumn"]' });
 
     const result = await page.evaluate(`(async () => {
         try {

package/src/clis/twitter/unbookmark.ts

@@ -17,7 +17,7 @@ cli({
     if (!page) throw new CommandExecutionError('Browser session required for twitter unbookmark');
 
     await page.goto(kwargs.url);
-    await page.wait(5);
+    await page.wait({ selector: '[data-testid="primaryColumn"]' });
 
     const result = await page.evaluate(`(async () => {
         try {

package/src/clis/twitter/unfollow.ts

@@ -18,7 +18,7 @@ cli({
     const username = kwargs.username.replace(/^@/, '');
 
     await page.goto(`https://x.com/${username}`);
-    await page.wait(5);
+    await page.wait({ selector: '[data-testid="primaryColumn"]' });
 
     const result = await page.evaluate(`(async () => {
         try {

package/src/clis/xiaohongshu/comments.test.ts

@@ -26,6 +26,7 @@ function createPageMock(evaluateResult: any): IPage {
     getInterceptedRequests: vi.fn().mockResolvedValue([]),
     getCookies: vi.fn().mockResolvedValue([]),
     screenshot: vi.fn().mockResolvedValue(''),
+    waitForCapture: vi.fn().mockResolvedValue(undefined),
   };
 }
 

package/src/clis/xiaohongshu/creator-note-detail.test.ts

@@ -33,6 +33,7 @@ function createPageMock(evaluateResult: any): IPage {
     getInterceptedRequests: vi.fn().mockResolvedValue([]),
     getCookies: vi.fn().mockResolvedValue([]),
     screenshot: vi.fn().mockResolvedValue(''),
+    waitForCapture: vi.fn().mockResolvedValue(undefined),
   };
 }
 

package/src/clis/xiaohongshu/creator-notes.test.ts

@@ -37,6 +37,7 @@ function createPageMock(evaluateResult: any, interceptedRequests: any[] = []): I
     getInterceptedRequests,
     getCookies: vi.fn().mockResolvedValue([]),
     screenshot: vi.fn().mockResolvedValue(''),
+    waitForCapture: vi.fn().mockResolvedValue(undefined),
   };
 }
 

package/src/clis/xiaohongshu/publish.test.ts

@@ -36,6 +36,7 @@ function createPageMock(evaluateResults: any[]): IPage {
     getInterceptedRequests: vi.fn().mockResolvedValue([]),
     getCookies: vi.fn().mockResolvedValue([]),
     screenshot: vi.fn().mockResolvedValue(''),
+    waitForCapture: vi.fn().mockResolvedValue(undefined),
   };
 }
 

package/src/clis/xiaohongshu/search.test.ts

@@ -31,6 +31,7 @@ function createPageMock(evaluateResults: any[]): IPage {
     getInterceptedRequests: vi.fn().mockResolvedValue([]),
     getCookies: vi.fn().mockResolvedValue([]),
     screenshot: vi.fn().mockResolvedValue(''),
+    waitForCapture: vi.fn().mockResolvedValue(undefined),
   };
 }
 

package/src/download/index.test.ts

@@ -2,13 +2,14 @@ import * as fs from 'node:fs';
 import * as http from 'node:http';
 import * as os from 'node:os';
 import * as path from 'node:path';
-import { afterEach, describe, expect, it } from 'vitest';
+import { afterEach, describe, expect, it, vi } from 'vitest';
 import { formatCookieHeader, httpDownload, resolveRedirectUrl } from './index.js';
 
 const servers: http.Server[] = [];
 const tempDirs: string[] = [];
 
 afterEach(async () => {
+  vi.unstubAllEnvs();
   await Promise.all(servers.map((server) => new Promise<void>((resolve, reject) => {
     server.close((err) => (err ? reject(err) : resolve()));
   })));
@@ -114,4 +115,21 @@ describe('download helpers', { retry: process.platform === 'win32' ? 2 : 0 }, ()
     expect(forwardedCookie).toBeUndefined();
     expect(fs.readFileSync(destPath, 'utf8')).toBe('ok');
   });
+
+  it('bypasses proxy settings for loopback downloads', async () => {
+    vi.stubEnv('HTTP_PROXY', 'http://127.0.0.1:9');
+
+    const baseUrl = await startServer((_req, res) => {
+      res.statusCode = 200;
+      res.end('ok');
+    });
+
+    const tempDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'opencli-dl-'));
+    tempDirs.push(tempDir);
+    const destPath = path.join(tempDir, 'loopback.txt');
+    const result = await httpDownload(`${baseUrl}/ok`, destPath);
+
+    expect(result).toEqual({ success: true, size: 2 });
+    expect(fs.readFileSync(destPath, 'utf8')).toBe('ok');
+  });
 });

package/src/download/index.ts

@@ -5,16 +5,16 @@
 import { spawn } from 'node:child_process';
 import * as fs from 'node:fs';
 import * as path from 'node:path';
-import * as https from 'node:https';
-import * as http from 'node:http';
 import * as os from 'node:os';
-import { Transform } from 'node:stream';
+import { Readable, Transform } from 'node:stream';
+import type { ReadableStream as WebReadableStream } from 'node:stream/web';
 import { pipeline } from 'node:stream/promises';
 import { URL } from 'node:url';
 import type { ProgressBar } from './progress.js';
 import { isBinaryInstalled } from '../external.js';
 import type { BrowserCookie } from '../types.js';
 import { getErrorMessage } from '../errors.js';
+import { fetchWithNodeNetwork } from '../node-network.js';
 
 export type { BrowserCookie } from '../types.js';
 
@@ -89,9 +89,6 @@ export async function httpDownload(
   const { cookies, headers = {}, timeout = 30000, onProgress, maxRedirects = 10 } = options;
 
   return new Promise((resolve) => {
-    const parsedUrl = new URL(url);
-    const protocol = parsedUrl.protocol === 'https:' ? https : http;
-
     const requestHeaders: Record<string, string> = {
       'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36',
       ...headers,
@@ -118,37 +115,52 @@ export async function httpDownload(
       }
     };
 
-    const request = protocol.get(url, { headers: requestHeaders, timeout }, (response) => {
-      void (async () => {
+    void (async () => {
+      const controller = new AbortController();
+      const timer = setTimeout(() => controller.abort(), timeout);
+      try {
+        const response = await fetchWithNodeNetwork(url, {
+          headers: requestHeaders,
+          signal: controller.signal,
+          redirect: 'manual',
+        });
+        clearTimeout(timer);
+
         // Handle redirects before creating any file handles.
-        if (response.statusCode && response.statusCode >= 300 && response.statusCode < 400 && response.headers.location) {
-          response.resume();
-          if (redirectCount >= maxRedirects) {
-            finish({ success: false, size: 0, error: `Too many redirects (> ${maxRedirects})` });
+        if (response.status >= 300 && response.status < 400) {
+          const location = response.headers.get('location');
+          if (location) {
+            if (redirectCount >= maxRedirects) {
+              finish({ success: false, size: 0, error: `Too many redirects (> ${maxRedirects})` });
+              return;
+            }
+            const redirectUrl = resolveRedirectUrl(url, location);
+            const originalHost = new URL(url).hostname;
+            const redirectHost = new URL(redirectUrl).hostname;
+            const redirectOptions = originalHost === redirectHost
+              ? options
+              : { ...options, cookies: undefined, headers: stripCookieHeaders(options.headers) };
+            finish(await httpDownload(
+              redirectUrl,
+              destPath,
+              redirectOptions,
+              redirectCount + 1,
+            ));
             return;
           }
-          const redirectUrl = resolveRedirectUrl(url, response.headers.location);
-          const originalHost = new URL(url).hostname;
-          const redirectHost = new URL(redirectUrl).hostname;
-          const redirectOptions = originalHost === redirectHost
-            ? options
-            : { ...options, cookies: undefined, headers: stripCookieHeaders(options.headers) };
-          finish(await httpDownload(
-            redirectUrl,
-            destPath,
-            redirectOptions,
-            redirectCount + 1,
-          ));
+        }
+
+        if (response.status !== 200) {
+          finish({ success: false, size: 0, error: `HTTP ${response.status}` });
           return;
         }
 
-        if (response.statusCode !== 200) {
-          response.resume();
-          finish({ success: false, size: 0, error: `HTTP ${response.statusCode}` });
+        if (!response.body) {
+          finish({ success: false, size: 0, error: 'Empty response body' });
           return;
         }
 
-        const totalSize = parseInt(response.headers['content-length'] || '0', 10);
+        const totalSize = parseInt(response.headers.get('content-length') || '0', 10);
         let received = 0;
         const progressStream = new Transform({
           transform(chunk, _encoding, callback) {
@@ -160,26 +172,23 @@ export async function httpDownload(
 
         try {
           await fs.promises.mkdir(path.dirname(destPath), { recursive: true });
-          await pipeline(response, progressStream, fs.createWriteStream(tempPath));
+          await pipeline(
+            Readable.fromWeb(response.body as unknown as WebReadableStream),
+            progressStream,
+            fs.createWriteStream(tempPath),
+          );
           await fs.promises.rename(tempPath, destPath);
           finish({ success: true, size: received });
         } catch (err) {
           await cleanupTempFile();
           finish({ success: false, size: 0, error: getErrorMessage(err) });
         }
-      })();
-    });
-
-    request.on('error', (err) => {
-      void (async () => {
+      } catch (err) {
+        clearTimeout(timer);
         await cleanupTempFile();
-        finish({ success: false, size: 0, error: err.message });
-      })();
-    });
-
-    request.on('timeout', () => {
-      request.destroy(new Error('Timeout'));
-    });
+        finish({ success: false, size: 0, error: err instanceof Error ? err.message : String(err) });
+      }
+    })();
   });
 }
 

package/src/execution.ts

@@ -19,6 +19,7 @@ import { shouldUseBrowserSession } from './capabilityRouting.js';
 import { getBrowserFactory, browserSession, runWithTimeout, DEFAULT_BROWSER_COMMAND_TIMEOUT } from './runtime.js';
 import { emitHook, type HookContext } from './hooks.js';
 import { checkDaemonStatus } from './browser/discover.js';
+import { log } from './logger.js';
 
 const _loadedModules = new Set<string>();
 
@@ -191,14 +192,14 @@ export async function executeCommand(
         if (preNavUrl) {
           const skip = await isAlreadyOnDomain(page, preNavUrl);
           if (skip) {
-            if (debug) console.error(`[pre-nav] Already on target domain, skipping navigation`);
+            if (debug) log.debug('[pre-nav] Already on target domain, skipping navigation');
           } else {
             try {
               // goto() already includes smart DOM-settle detection (waitForDomStable).
               // No additional fixed sleep needed.
               await page.goto(preNavUrl);
             } catch (err) {
-              if (debug) console.error(`[pre-nav] Failed to navigate to ${preNavUrl}: ${err instanceof Error ? err.message : err}`);
+              if (debug) log.debug(`[pre-nav] Failed to navigate to ${preNavUrl}: ${err instanceof Error ? err.message : err}`);
             }
           }
         }

package/src/main.ts

@@ -20,8 +20,11 @@ import { discoverClis, discoverPlugins } from './discovery.js';
 import { getCompletions } from './completion.js';
 import { runCli } from './cli.js';
 import { emitHook } from './hooks.js';
+import { installNodeNetwork } from './node-network.js';
 import { registerUpdateNoticeOnExit, checkForUpdateBackground } from './update-check.js';
 
+installNodeNetwork();
+
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);
 const BUILTIN_CLIS = path.resolve(__dirname, 'clis');