@jackwener/opencli 0.1.2 → 0.3.0

package/src/clis/zhihu/hot.yaml

@@ -17,12 +17,16 @@ pipeline:
         const res = await fetch('https://www.zhihu.com/api/v3/feed/topstory/hot-lists/total?limit=50', {
           credentials: 'include'
         });
-        const d = await res.json();
+        const text = await res.text();
+        const d = JSON.parse(
+          text.replace(/("id"\s*:\s*)(\d{16,})/g, '$1"$2"')
+        );
         return (d?.data || []).map((item) => {
           const t = item.target || {};
+          const questionId = t.id == null ? '' : String(t.id);
           return {
             title: t.title,
-            url: 'https://www.zhihu.com/question/' + t.id,
+            url: 'https://www.zhihu.com/question/' + questionId,
             answer_count: t.answer_count,
             follower_count: t.follower_count,
             heat: item.detail_text || '',

package/src/clis/zhihu/search.yaml

@@ -19,7 +19,9 @@ pipeline:
   - evaluate: |
       (async () => {
         const strip = (html) => (html || '').replace(/<[^>]+>/g, '').replace(/&nbsp;/g, ' ').replace(/&lt;/g, '<').replace(/&gt;/g, '>').replace(/&amp;/g, '&').replace(/<em>/g, '').replace(/<\/em>/g, '').trim();
-        const res = await fetch('https://www.zhihu.com/api/v4/search_v3?q=' + encodeURIComponent('${{ args.keyword }}') + '&t=general&offset=0&limit=${{ args.limit }}', {
+        const keyword = ${{ args.keyword | json }};
+        const limit = ${{ args.limit }};
+        const res = await fetch('https://www.zhihu.com/api/v4/search_v3?q=' + encodeURIComponent(keyword) + '&t=general&offset=0&limit=' + limit, {
           credentials: 'include'
         });
         const d = await res.json();

package/src/engine.ts

@@ -9,7 +9,8 @@ import { type CliCommand, type Arg, Strategy, registerCommand } from './registry
 import type { IPage } from './types.js';
 import { executePipeline } from './pipeline.js';
 
-export function discoverClis(...dirs: string[]): void {
+export async function discoverClis(...dirs: string[]): Promise<void> {
+  const promises: Promise<any>[] = [];
   for (const dir of dirs) {
     if (!fs.existsSync(dir)) continue;
     for (const site of fs.readdirSync(dir)) {
@@ -19,10 +20,18 @@ export function discoverClis(...dirs: string[]): void {
         const filePath = path.join(siteDir, file);
         if (file.endsWith('.yaml') || file.endsWith('.yml')) {
           registerYamlCli(filePath, site);
+        } else if (file.endsWith('.js')) {
+          // Dynamic import of compiled adapter modules
+          promises.push(
+            import(`file://${filePath}`).catch((err: any) => {
+              process.stderr.write(`Warning: failed to load module ${filePath}: ${err.message}\n`);
+            })
+          );
         }
       }
     }
   }
+  await Promise.all(promises);
 }
 
 function registerYamlCli(filePath: string, defaultSite: string): void {

package/src/main.ts

@@ -12,7 +12,6 @@ import chalk from 'chalk';
 import { discoverClis, executeCommand } from './engine.js';
 import { type CliCommand, fullName, getRegistry, strategyLabel } from './registry.js';
 import { render as renderOutput } from './output.js';
-import './clis/index.js';
 import { PlaywrightMCP } from './browser.js';
 import { browserSession, DEFAULT_BROWSER_COMMAND_TIMEOUT, runWithTimeout } from './runtime.js';
 
@@ -25,7 +24,7 @@ const USER_CLIS = path.join(os.homedir(), '.opencli', 'clis');
 const pkgJsonPath = path.resolve(__dirname, '..', 'package.json');
 const PKG_VERSION = JSON.parse(fs.readFileSync(pkgJsonPath, 'utf-8')).version ?? '0.0.0';
 
-discoverClis(BUILTIN_CLIS, USER_CLIS);
+await discoverClis(BUILTIN_CLIS, USER_CLIS);
 
 const program = new Command();
 program.name('opencli').description('Make any website your CLI. Zero setup. AI-powered.').version(PKG_VERSION);
@@ -104,8 +103,15 @@ for (const [, cmd] of registry) {
       if (cmd.browser) {
         result = await browserSession(PlaywrightMCP, async (page) => runWithTimeout(executeCommand(cmd, page, kwargs, actionOpts.verbose), { timeout: cmd.timeoutSeconds ?? DEFAULT_BROWSER_COMMAND_TIMEOUT, label: fullName(cmd) }));
       } else { result = await executeCommand(cmd, null, kwargs, actionOpts.verbose); }
+      if (actionOpts.verbose && (!result || (Array.isArray(result) && result.length === 0))) {
+        console.error(chalk.yellow(`[Verbose] Warning: Command returned an empty result. If the website structural API changed or requires authentication, check the network or update the adapter.`));
+      }
       renderOutput(result, { fmt: actionOpts.format, columns: cmd.columns, title: `${cmd.site}/${cmd.name}`, elapsed: (Date.now() - startTime) / 1000, source: fullName(cmd) });
-    } catch (err: any) { console.error(chalk.red(`Error: ${err.message ?? err}`)); process.exitCode = 1; }
+    } catch (err: any) { 
+      if (actionOpts.verbose && err.stack) { console.error(chalk.red(err.stack)); }
+      else { console.error(chalk.red(`Error: ${err.message ?? err}`)); }
+      process.exitCode = 1; 
+    }
   });
 }
 

package/src/output.ts

@@ -1,9 +1,10 @@
 /**
- * Output formatting: table, JSON, Markdown, CSV.
+ * Output formatting: table, JSON, Markdown, CSV, YAML.
  */
 
 import chalk from 'chalk';
 import Table from 'cli-table3';
+import yaml from 'js-yaml';
 
 export interface RenderOptions {
   fmt?: string;
@@ -23,6 +24,7 @@ export function render(data: any, opts: RenderOptions = {}): void {
     case 'json': renderJson(data); break;
     case 'md': case 'markdown': renderMarkdown(data, opts); break;
     case 'csv': renderCsv(data, opts); break;
+    case 'yaml': case 'yml': renderYaml(data); break;
     default: renderTable(data, opts); break;
   }
 }
@@ -32,16 +34,14 @@ function renderTable(data: any, opts: RenderOptions): void {
   if (!rows.length) { console.log(chalk.dim('(no data)')); return; }
   const columns = opts.columns ?? Object.keys(rows[0]);
 
-  const header = columns.map((c, i) => i === 0 ? '#' : capitalize(c));
+  const header = columns.map(c => capitalize(c));
   const table = new Table({
     head: header.map(h => chalk.bold(h)),
     style: { head: [], border: [] },
     wordWrap: true,
     wrapOnWordBoundary: true,
-    colWidths: columns.map((c, i) => {
-      if (i === 0) return 4;
-      if (c === 'url' || c === 'description') return null as any;
-      if (c === 'title' || c === 'name' || c === 'repo') return null as any;
+    colWidths: columns.map((_c, i) => {
+      if (i === 0) return 6;
       return null as any;
     }).filter(() => true),
   });
@@ -91,6 +91,10 @@ function renderCsv(data: any, opts: RenderOptions): void {
   }
 }
 
+function renderYaml(data: any): void {
+  console.log(yaml.dump(data, { sortKeys: false, lineWidth: 120, noRefs: true }));
+}
+
 function capitalize(s: string): string {
   return s.charAt(0).toUpperCase() + s.slice(1);
 }

package/src/pipeline/steps/intercept.ts

@@ -14,7 +14,55 @@ export async function stepIntercept(page: IPage, params: any, data: any, args: R
 
   if (!capturePattern) return data;
 
-  // Step 1: Execute the trigger action
+  // Step 1: Inject fetch/XHR interceptor BEFORE trigger
+  await page.evaluate(`
+    () => {
+      window.__opencli_intercepted = window.__opencli_intercepted || [];
+      const pattern = ${JSON.stringify(capturePattern)};
+      
+      if (!window.__opencli_fetch_patched) {
+        const origFetch = window.fetch;
+        window.fetch = async function(...args) {
+          const reqUrl = typeof args[0] === 'string' ? args[0] : (args[0] && args[0].url) || '';
+          const response = await origFetch.apply(this, args);
+          setTimeout(async () => {
+            try {
+              if (reqUrl.includes(pattern)) {
+                const clone = response.clone();
+                const json = await clone.json();
+                window.__opencli_intercepted.push(json);
+              }
+            } catch(e) {}
+          }, 0);
+          return response;
+        };
+        window.__opencli_fetch_patched = true;
+      }
+
+      if (!window.__opencli_xhr_patched) {
+        const XHR = XMLHttpRequest.prototype;
+        const open = XHR.open;
+        const send = XHR.send;
+        XHR.open = function(method, url, ...args) {
+          this._reqUrl = url;
+          return open.call(this, method, url, ...args);
+        };
+        XHR.send = function(...args) {
+          this.addEventListener('load', function() {
+            try {
+              if (this._reqUrl && this._reqUrl.includes(pattern)) {
+                window.__opencli_intercepted.push(JSON.parse(this.responseText));
+              }
+            } catch(e) {}
+          });
+          return send.apply(this, args);
+        };
+        window.__opencli_xhr_patched = true;
+      }
+    }
+  `);
+
+  // Step 2: Execute the trigger action
   if (trigger.startsWith('navigate:')) {
     const url = render(trigger.slice('navigate:'.length), { args, data });
     await page.goto(String(url));
@@ -29,36 +77,18 @@ export async function stepIntercept(page: IPage, params: any, data: any, args: R
     await page.scroll('down');
   }
 
-  // Step 2: Wait a bit for network requests to fire
+  // Step 3: Wait a bit for network requests to fire
   await page.wait(Math.min(timeout, 3));
 
-  // Step 3: Get network requests and find matching ones
-  const rawNetwork = await page.networkRequests(false);
-  const matchingResponses: any[] = [];
-
-  if (typeof rawNetwork === 'string') {
-    const lines = rawNetwork.split('\n');
-    for (const line of lines) {
-      const match = line.match(/\[?(GET|POST)\]?\s+(\S+)\s*(?:=>|→)\s*\[?(\d+)\]?/i);
-      if (match) {
-        const [, , url, status] = match;
-        if (url.includes(capturePattern) && status === '200') {
-          try {
-            const body = await page.evaluate(`
-              async () => {
-                try {
-                  const resp = await fetch(${JSON.stringify(url)}, { credentials: 'include' });
-                  if (!resp.ok) return null;
-                  return await resp.json();
-                } catch { return null; }
-              }
-            `);
-            if (body) matchingResponses.push(body);
-          } catch {}
-        }
-      }
+  // Step 4: Retrieve captured data
+  const matchingResponses = await page.evaluate(`
+    () => {
+      const data = window.__opencli_intercepted || [];
+      window.__opencli_intercepted = []; // clear after reading
+      return data;
     }
-  }
+  `);
+
 
   // Step 4: Select from response if specified
   let result = matchingResponses.length === 1 ? matchingResponses[0] :

package/src/pipeline/template.test.ts

@@ -57,6 +57,30 @@ describe('evalExpr', () => {
   it('resolves simple path', () => {
     expect(evalExpr('item.title', { item: { title: 'Test' } })).toBe('Test');
   });
+  it('applies join filter', () => {
+    expect(evalExpr('item.tags | join(,)', { item: { tags: ['a', 'b', 'c'] } })).toBe('a,b,c');
+  });
+  it('applies upper filter', () => {
+    expect(evalExpr('item.name | upper', { item: { name: 'hello' } })).toBe('HELLO');
+  });
+  it('applies lower filter', () => {
+    expect(evalExpr('item.name | lower', { item: { name: 'HELLO' } })).toBe('hello');
+  });
+  it('applies truncate filter', () => {
+    expect(evalExpr('item.text | truncate(5)', { item: { text: 'Hello World!' } })).toBe('Hello...');
+  });
+  it('chains filters', () => {
+    expect(evalExpr('item.name | upper | truncate(3)', { item: { name: 'hello' } })).toBe('HEL...');
+  });
+  it('applies length filter', () => {
+    expect(evalExpr('item.items | length', { item: { items: [1, 2, 3] } })).toBe(3);
+  });
+  it('applies json filter to strings with quotes', () => {
+    expect(evalExpr('args.keyword | json', { args: { keyword: "O'Reilly" } })).toBe('"O\'Reilly"');
+  });
+  it('applies json filter to nullish values', () => {
+    expect(evalExpr('args.keyword | json', { args: {} })).toBe('null');
+  });
 });
 
 describe('render', () => {

package/src/pipeline/template.ts

@@ -11,8 +11,17 @@ export interface RenderContext {
 
 export function render(template: any, ctx: RenderContext): any {
   if (typeof template !== 'string') return template;
-  const fullMatch = template.match(/^\$\{\{\s*(.*?)\s*\}\}$/);
-  if (fullMatch) return evalExpr(fullMatch[1].trim(), ctx);
+  // Full expression: entire string is a single ${{ ... }}
+  // Use [^}] to prevent matching across }} boundaries (e.g. "${{ a }}-${{ b }}")
+  const fullMatch = template.match(/^\$\{\{\s*([^}]*(?:\}[^}][^}]*)*)\s*\}\}$/);
+  if (fullMatch && !template.includes('}}-') && !template.includes('}}${{')) return evalExpr(fullMatch[1].trim(), ctx);
+  // Check if the entire string is a single expression (no other text around it)
+  const singleExpr = template.match(/^\$\{\{\s*([\s\S]*?)\s*\}\}$/);
+  if (singleExpr) {
+    // Verify it's truly a single expression (no other ${{ inside)
+    const inner = singleExpr[1];
+    if (!inner.includes('${{')) return evalExpr(inner.trim(), ctx);
+  }
   return template.replace(/\$\{\{\s*(.*?)\s*\}\}/g, (_m, expr) => String(evalExpr(expr.trim(), ctx)));
 }
 
@@ -22,18 +31,14 @@ export function evalExpr(expr: string, ctx: RenderContext): any {
   const data = ctx.data;
   const index = ctx.index ?? 0;
 
-  // Default filter: args.limit | default(20)
-  if (expr.includes('|') && expr.includes('default(')) {
-    const [mainExpr, rest] = expr.split('|', 2);
-    const defaultMatch = rest.match(/default\((.+?)\)/);
-    const defaultVal = defaultMatch ? defaultMatch[1] : null;
-    const result = resolvePath(mainExpr.trim(), { args, item, data, index });
-    if (result === null || result === undefined) {
-      if (defaultVal !== null) {
-        const intVal = parseInt(defaultVal!, 10);
-        if (!isNaN(intVal) && String(intVal) === defaultVal!.trim()) return intVal;
-        return defaultVal!.replace(/^['"]|['"]$/g, '');
-      }
+  // ── Pipe filters: expr | filter1(arg) | filter2 ──
+  // Supports: default(val), join(sep), upper, lower, truncate(n), trim, replace(old,new)
+  if (expr.includes('|') && !expr.includes('||')) {
+    const segments = expr.split('|').map(s => s.trim());
+    const mainExpr = segments[0];
+    let result = resolvePath(mainExpr, { args, item, data, index });
+    for (let i = 1; i < segments.length; i++) {
+      result = applyFilter(segments[i], result);
     }
     return result;
   }
@@ -66,6 +71,59 @@ export function evalExpr(expr: string, ctx: RenderContext): any {
   return resolvePath(expr, { args, item, data, index });
 }
 
+/**
+ * Apply a named filter to a value.
+ * Supported filters:
+ *   default(val), join(sep), upper, lower, truncate(n), trim,
+ *   replace(old,new), keys, length, first, last, json
+ */
+function applyFilter(filterExpr: string, value: any): any {
+  const match = filterExpr.match(/^(\w+)(?:\((.+)\))?$/);
+  if (!match) return value;
+  const [, name, rawArgs] = match;
+  const filterArg = rawArgs?.replace(/^['"]|['"]$/g, '') ?? '';
+
+  switch (name) {
+    case 'default': {
+      if (value === null || value === undefined || value === '') {
+        const intVal = parseInt(filterArg, 10);
+        if (!isNaN(intVal) && String(intVal) === filterArg.trim()) return intVal;
+        return filterArg;
+      }
+      return value;
+    }
+    case 'join':
+      return Array.isArray(value) ? value.join(filterArg || ', ') : value;
+    case 'upper':
+      return typeof value === 'string' ? value.toUpperCase() : value;
+    case 'lower':
+      return typeof value === 'string' ? value.toLowerCase() : value;
+    case 'trim':
+      return typeof value === 'string' ? value.trim() : value;
+    case 'truncate': {
+      const n = parseInt(filterArg, 10) || 50;
+      return typeof value === 'string' && value.length > n ? value.slice(0, n) + '...' : value;
+    }
+    case 'replace': {
+      if (typeof value !== 'string') return value;
+      const parts = rawArgs?.split(',').map(s => s.trim().replace(/^['"]|['"]$/g, '')) ?? [];
+      return parts.length >= 2 ? value.replaceAll(parts[0], parts[1]) : value;
+    }
+    case 'keys':
+      return value && typeof value === 'object' ? Object.keys(value) : value;
+    case 'length':
+      return Array.isArray(value) ? value.length : typeof value === 'string' ? value.length : value;
+    case 'first':
+      return Array.isArray(value) ? value[0] : value;
+    case 'last':
+      return Array.isArray(value) ? value[value.length - 1] : value;
+    case 'json':
+      return JSON.stringify(value ?? null);
+    default:
+      return value;
+  }
+}
+
 export function resolvePath(pathStr: string, ctx: RenderContext): any {
   const args = ctx.args ?? {};
   const item = ctx.item ?? {};

package/src/types.ts

@@ -20,4 +20,7 @@ export interface IPage {
   networkRequests(includeStatic?: boolean): Promise<any>;
   consoleMessages(level?: string): Promise<any>;
   scroll(direction?: string, amount?: number): Promise<void>;
+  autoScroll(options?: { times?: number; delayMs?: number }): Promise<void>;
+  installInterceptor(pattern: string): Promise<void>;
+  getInterceptedRequests(): Promise<any[]>;
 }

package/dist/clis/index.d.ts

@@ -1,22 +0,0 @@
-/**
- * Import all TypeScript CLI adapters so they self-register.
- *
- * Each TS adapter calls cli() on import, which adds itself to the global registry.
- */
-import './bilibili/search.js';
-import './bilibili/me.js';
-import './bilibili/favorite.js';
-import './bilibili/history.js';
-import './bilibili/feed.js';
-import './bilibili/user-videos.js';
-import './github/search.js';
-import './zhihu/question.js';
-import './xiaohongshu/search.js';
-import './bbc/news.js';
-import './weibo/hot.js';
-import './boss/search.js';
-import './yahoo-finance/quote.js';
-import './reuters/search.js';
-import './smzdm/search.js';
-import './ctrip/search.js';
-import './youtube/search.js';

package/dist/clis/index.js

@@ -1,34 +0,0 @@
-/**
- * Import all TypeScript CLI adapters so they self-register.
- *
- * Each TS adapter calls cli() on import, which adds itself to the global registry.
- */
-// bilibili
-import './bilibili/search.js';
-import './bilibili/me.js';
-import './bilibili/favorite.js';
-import './bilibili/history.js';
-import './bilibili/feed.js';
-import './bilibili/user-videos.js';
-// github
-import './github/search.js';
-// zhihu
-import './zhihu/question.js';
-// xiaohongshu
-import './xiaohongshu/search.js';
-// bbc
-import './bbc/news.js';
-// weibo
-import './weibo/hot.js';
-// boss
-import './boss/search.js';
-// yahoo-finance
-import './yahoo-finance/quote.js';
-// reuters
-import './reuters/search.js';
-// smzdm
-import './smzdm/search.js';
-// ctrip
-import './ctrip/search.js';
-// youtube
-import './youtube/search.js';

package/src/clis/index.ts

@@ -1,46 +0,0 @@
-/**
- * Import all TypeScript CLI adapters so they self-register.
- *
- * Each TS adapter calls cli() on import, which adds itself to the global registry.
- */
-
-// bilibili
-import './bilibili/search.js';
-import './bilibili/me.js';
-import './bilibili/favorite.js';
-import './bilibili/history.js';
-import './bilibili/feed.js';
-import './bilibili/user-videos.js';
-
-// github
-import './github/search.js';
-
-// zhihu
-import './zhihu/question.js';
-
-// xiaohongshu
-import './xiaohongshu/search.js';
-
-// bbc
-import './bbc/news.js';
-
-// weibo
-import './weibo/hot.js';
-
-// boss
-import './boss/search.js';
-
-// yahoo-finance
-import './yahoo-finance/quote.js';
-
-// reuters
-import './reuters/search.js';
-
-// smzdm
-import './smzdm/search.js';
-
-// ctrip
-import './ctrip/search.js';
-
-// youtube
-import './youtube/search.js';