@jackwener/opencli 0.1.0 → 0.1.2

package/src/cascade.ts

@@ -0,0 +1,218 @@
+/**
+ * Strategy Cascade: automatic strategy downgrade chain.
+ *
+ * Probes an API endpoint starting from the simplest strategy (PUBLIC)
+ * and automatically downgrades through the strategy tiers until one works:
+ *
+ *   PUBLIC → COOKIE → HEADER → INTERCEPT → UI
+ *
+ * This eliminates the need for manual strategy selection — the system
+ * automatically finds the minimum-privilege strategy that works.
+ */
+
+import { Strategy } from './registry.js';
+import type { IPage } from './types.js';
+
+/** Strategy cascade order (simplest → most complex) */
+const CASCADE_ORDER: Strategy[] = [
+  Strategy.PUBLIC,
+  Strategy.COOKIE,
+  Strategy.HEADER,
+  Strategy.INTERCEPT,
+  Strategy.UI,
+];
+
+interface ProbeResult {
+  strategy: Strategy;
+  success: boolean;
+  statusCode?: number;
+  hasData?: boolean;
+  error?: string;
+  responsePreview?: string;
+}
+
+interface CascadeResult {
+  bestStrategy: Strategy;
+  probes: ProbeResult[];
+  confidence: number;
+}
+
+/**
+ * Probe an endpoint with a specific strategy.
+ * Returns whether the probe succeeded and basic response info.
+ */
+export async function probeEndpoint(
+  page: IPage,
+  url: string,
+  strategy: Strategy,
+  opts: { timeout?: number } = {},
+): Promise<ProbeResult> {
+  const result: ProbeResult = { strategy, success: false };
+
+  try {
+    switch (strategy) {
+      case Strategy.PUBLIC: {
+        // Try direct fetch without browser (no credentials)
+        const js = `
+          async () => {
+            try {
+              const resp = await fetch(${JSON.stringify(url)});
+              const status = resp.status;
+              if (!resp.ok) return { status, ok: false };
+              const text = await resp.text();
+              let hasData = false;
+              try {
+                const json = JSON.parse(text);
+                hasData = !!json && (Array.isArray(json) ? json.length > 0 :
+                  typeof json === 'object' && Object.keys(json).length > 0);
+              } catch {}
+              return { status, ok: true, hasData, preview: text.slice(0, 200) };
+            } catch (e) { return { ok: false, error: e.message }; }
+          }
+        `;
+        const resp = await page.evaluate(js);
+        result.statusCode = resp?.status;
+        result.success = resp?.ok && resp?.hasData;
+        result.hasData = resp?.hasData;
+        result.responsePreview = resp?.preview;
+        break;
+      }
+
+      case Strategy.COOKIE: {
+        // Fetch with credentials: 'include' (uses browser cookies)
+        const js = `
+          async () => {
+            try {
+              const resp = await fetch(${JSON.stringify(url)}, { credentials: 'include' });
+              const status = resp.status;
+              if (!resp.ok) return { status, ok: false };
+              const text = await resp.text();
+              let hasData = false;
+              try {
+                const json = JSON.parse(text);
+                hasData = !!json && (Array.isArray(json) ? json.length > 0 :
+                  typeof json === 'object' && Object.keys(json).length > 0);
+                // Check for API-level error codes (common in Chinese sites)
+                if (json.code !== undefined && json.code !== 0) hasData = false;
+              } catch {}
+              return { status, ok: true, hasData, preview: text.slice(0, 200) };
+            } catch (e) { return { ok: false, error: e.message }; }
+          }
+        `;
+        const resp = await page.evaluate(js);
+        result.statusCode = resp?.status;
+        result.success = resp?.ok && resp?.hasData;
+        result.hasData = resp?.hasData;
+        result.responsePreview = resp?.preview;
+        break;
+      }
+
+      case Strategy.HEADER: {
+        // Fetch with credentials + try to extract common auth headers
+        const js = `
+          async () => {
+            try {
+              // Try to extract CSRF tokens from cookies
+              const cookies = document.cookie.split(';').map(c => c.trim());
+              const csrf = cookies.find(c => c.startsWith('ct0=') || c.startsWith('csrf_token=') || c.startsWith('_csrf='))?.split('=').slice(1).join('=');
+              
+              const headers = {};
+              if (csrf) {
+                headers['X-Csrf-Token'] = csrf;
+                headers['X-XSRF-Token'] = csrf;
+              }
+              
+              const resp = await fetch(${JSON.stringify(url)}, {
+                credentials: 'include',
+                headers
+              });
+              const status = resp.status;
+              if (!resp.ok) return { status, ok: false };
+              const text = await resp.text();
+              let hasData = false;
+              try {
+                const json = JSON.parse(text);
+                hasData = !!json && (Array.isArray(json) ? json.length > 0 :
+                  typeof json === 'object' && Object.keys(json).length > 0);
+                if (json.code !== undefined && json.code !== 0) hasData = false;
+              } catch {}
+              return { status, ok: true, hasData, preview: text.slice(0, 200) };
+            } catch (e) { return { ok: false, error: e.message }; }
+          }
+        `;
+        const resp = await page.evaluate(js);
+        result.statusCode = resp?.status;
+        result.success = resp?.ok && resp?.hasData;
+        result.hasData = resp?.hasData;
+        result.responsePreview = resp?.preview;
+        break;
+      }
+
+      case Strategy.INTERCEPT:
+      case Strategy.UI:
+        // These require specific implementation per-site
+        // Mark as needing manual implementation
+        result.success = false;
+        result.error = `Strategy ${strategy} requires site-specific implementation`;
+        break;
+    }
+  } catch (err: any) {
+    result.success = false;
+    result.error = err.message ?? String(err);
+  }
+
+  return result;
+}
+
+/**
+ * Run the cascade: try each strategy in order until one works.
+ * Returns the simplest working strategy.
+ */
+export async function cascadeProbe(
+  page: IPage,
+  url: string,
+  opts: { maxStrategy?: Strategy; timeout?: number } = {},
+): Promise<CascadeResult> {
+  const maxIdx = opts.maxStrategy
+    ? CASCADE_ORDER.indexOf(opts.maxStrategy)
+    : CASCADE_ORDER.indexOf(Strategy.HEADER); // Don't auto-try INTERCEPT/UI
+
+  const probes: ProbeResult[] = [];
+
+  for (let i = 0; i <= Math.min(maxIdx, CASCADE_ORDER.length - 1); i++) {
+    const strategy = CASCADE_ORDER[i];
+    const probe = await probeEndpoint(page, url, strategy, opts);
+    probes.push(probe);
+
+    if (probe.success) {
+      return {
+        bestStrategy: strategy,
+        probes,
+        confidence: 1.0 - (i * 0.1), // Higher confidence for simpler strategies
+      };
+    }
+  }
+
+  // None worked — default to COOKIE (most common for logged-in sites)
+  return {
+    bestStrategy: Strategy.COOKIE,
+    probes,
+    confidence: 0.3,
+  };
+}
+
+/**
+ * Render cascade results for display.
+ */
+export function renderCascadeResult(result: CascadeResult): string {
+  const lines = [
+    `Strategy Cascade: ${result.bestStrategy} (${(result.confidence * 100).toFixed(0)}% confidence)`,
+  ];
+  for (const probe of result.probes) {
+    const icon = probe.success ? '✅' : '❌';
+    const status = probe.statusCode ? ` [${probe.statusCode}]` : '';
+    const err = probe.error ? ` — ${probe.error}` : '';
+    lines.push(`  ${icon} ${probe.strategy}${status}${err}`);
+  }
+  return lines.join('\n');
+}

package/src/clis/bbc/news.ts

@@ -0,0 +1,42 @@
+/**
+ * BBC News headlines — public RSS feed, no browser needed.
+ * Source: bb-sites/bbc/news.js
+ */
+import { cli, Strategy } from '../../registry.js';
+
+cli({
+  site: 'bbc',
+  name: 'news',
+  description: 'BBC News headlines (RSS)',
+  domain: 'www.bbc.com',
+  strategy: Strategy.PUBLIC,
+  args: [
+    { name: 'limit', type: 'int', default: 20, help: 'Number of headlines (max 50)' },
+  ],
+  columns: ['rank', 'title', 'description', 'url'],
+  func: async (page, kwargs) => {
+    const count = Math.min(kwargs.limit || 20, 50);
+    const resp = await fetch('https://feeds.bbci.co.uk/news/rss.xml');
+    if (!resp.ok) return [];
+    const xml = await resp.text();
+    // Simple XML parsing without DOMParser (works in Node)
+    const items: any[] = [];
+    const itemRegex = /<item>([\s\S]*?)<\/item>/g;
+    let match;
+    while ((match = itemRegex.exec(xml)) && items.length < count) {
+      const block = match[1];
+      const title = block.match(/<title><!\[CDATA\[(.*?)\]\]>|<title>(.*?)<\/title>/)?.[1] || block.match(/<title>(.*?)<\/title>/)?.[1] || '';
+      const desc = block.match(/<description><!\[CDATA\[(.*?)\]\]>|<description>(.*?)<\/description>/)?.[1] || block.match(/<description>(.*?)<\/description>/)?.[1] || '';
+      const link = block.match(/<link>(.*?)<\/link>/)?.[1] || block.match(/<guid[^>]*>(.*?)<\/guid>/)?.[1] || '';
+      if (title) {
+        items.push({
+          rank: items.length + 1,
+          title: title.trim(),
+          description: desc.trim().substring(0, 200),
+          url: link.trim(),
+        });
+      }
+    }
+    return items;
+  },
+});

package/src/clis/boss/search.ts

@@ -0,0 +1,47 @@
+/**
+ * BOSS直聘 job search — browser cookie API.
+ * Source: bb-sites/boss/search.js
+ */
+import { cli, Strategy } from '../../registry.js';
+
+cli({
+  site: 'boss',
+  name: 'search',
+  description: 'BOSS直聘搜索职位',
+  domain: 'www.zhipin.com',
+  strategy: Strategy.COOKIE,
+  args: [
+    { name: 'query', required: true, help: 'Search keyword (e.g. AI agent, 前端)' },
+    { name: 'city', default: '101010100', help: 'City code (101010100=北京, 101020100=上海, 101210100=杭州, 101280100=广州)' },
+    { name: 'limit', type: 'int', default: 15, help: 'Number of results' },
+  ],
+  columns: ['name', 'salary', 'company', 'city', 'experience', 'degree', 'boss', 'url'],
+  func: async (page, kwargs) => {
+    await page.goto('https://www.zhipin.com');
+    await page.wait(2);
+    const data = await page.evaluate(`
+      (async () => {
+        const params = new URLSearchParams({
+          scene: '1', query: '${kwargs.query.replace(/'/g, "\\'")}',
+          city: '${kwargs.city || '101010100'}', page: '1', pageSize: '15',
+          experience: '', degree: '', payType: '', partTime: '',
+          industry: '', scale: '', stage: '', position: '',
+          jobType: '', salary: '', multiBusinessDistrict: '', multiSubway: ''
+        });
+        const resp = await fetch('/wapi/zpgeek/search/joblist.json?' + params.toString(), {credentials: 'include'});
+        if (!resp.ok) return {error: 'HTTP ' + resp.status};
+        const d = await resp.json();
+        if (d.code !== 0) return {error: d.message || 'API error'};
+        const zpData = d.zpData || {};
+        return (zpData.jobList || []).map(j => ({
+          name: j.jobName, salary: j.salaryDesc, company: j.brandName,
+          city: j.cityName, experience: j.jobExperience, degree: j.jobDegree,
+          boss: j.bossName + ' · ' + j.bossTitle,
+          url: j.encryptJobId ? 'https://www.zhipin.com/job_detail/' + j.encryptJobId + '.html' : ''
+        }));
+      })()
+    `);
+    if (!Array.isArray(data)) return [];
+    return data.slice(0, kwargs.limit || 15);
+  },
+});

package/src/clis/ctrip/search.ts

@@ -0,0 +1,62 @@
+/**
+ * 携程旅行搜索 — browser cookie, multi-strategy.
+ * Source: bb-sites/ctrip/search.js (simplified to suggestion API)
+ */
+import { cli, Strategy } from '../../registry.js';
+
+cli({
+  site: 'ctrip',
+  name: 'search',
+  description: '携程旅行搜索',
+  domain: 'www.ctrip.com',
+  strategy: Strategy.COOKIE,
+  args: [
+    { name: 'query', required: true, help: 'Search keyword (city or attraction)' },
+    { name: 'limit', type: 'int', default: 15, help: 'Number of results' },
+  ],
+  columns: ['rank', 'name', 'type', 'score', 'price', 'url'],
+  func: async (page, kwargs) => {
+    const limit = kwargs.limit || 15;
+    await page.goto('https://www.ctrip.com');
+    await page.wait(2);
+    const data = await page.evaluate(`
+      (async () => {
+        const query = '${kwargs.query.replace(/'/g, "\\'")}';
+        const limit = ${limit};
+
+        // Strategy 1: Suggestion API
+        try {
+          const suggestUrl = 'https://m.ctrip.com/restapi/h5api/searchapp/search?action=onekeyali&keyword=' + encodeURIComponent(query);
+          const resp = await fetch(suggestUrl, {credentials: 'include'});
+          if (resp.ok) {
+            const d = await resp.json();
+            const raw = d.data || d.result || d;
+            if (raw && typeof raw === 'object') {
+              // Flatten all result categories
+              const items = [];
+              for (const key of Object.keys(raw)) {
+                const list = Array.isArray(raw[key]) ? raw[key] : [];
+                for (const item of list) {
+                  if (items.length >= limit) break;
+                  items.push({
+                    rank: items.length + 1,
+                    name: item.word || item.name || item.title || '',
+                    type: item.type || item.tpName || key,
+                    score: item.score || '',
+                    price: item.price || item.minPrice || '',
+                    url: item.url || item.surl || '',
+                  });
+                }
+              }
+              if (items.length > 0) return items;
+            }
+          }
+        } catch(e) {}
+
+        return {error: 'No results for: ' + query};
+      })()
+    `);
+    if (!Array.isArray(data)) return [];
+    return data;
+  },
+});

package/src/clis/index.ts

@@ -16,4 +16,31 @@ import './bilibili/user-videos.js';
 import './github/search.js';
 
 // zhihu
-import './zhihu/search.js';
+import './zhihu/question.js';
+
+// xiaohongshu
+import './xiaohongshu/search.js';
+
+// bbc
+import './bbc/news.js';
+
+// weibo
+import './weibo/hot.js';
+
+// boss
+import './boss/search.js';
+
+// yahoo-finance
+import './yahoo-finance/quote.js';
+
+// reuters
+import './reuters/search.js';
+
+// smzdm
+import './smzdm/search.js';
+
+// ctrip
+import './ctrip/search.js';
+
+// youtube
+import './youtube/search.js';

package/src/clis/reddit/hot.yaml

@@ -0,0 +1,46 @@
+site: reddit
+name: hot
+description: Reddit 热门帖子
+domain: www.reddit.com
+
+args:
+  subreddit:
+    type: str
+    default: ""
+    description: "Subreddit name (e.g. programming). Empty for frontpage"
+  limit:
+    type: int
+    default: 20
+    description: Number of posts
+
+pipeline:
+  - navigate: https://www.reddit.com
+
+  - evaluate: |
+      (async () => {
+        const sub = '${{ args.subreddit }}';
+        const path = sub ? '/r/' + sub + '/hot.json' : '/hot.json';
+        const res = await fetch(path + '?limit=${{ args.limit }}&raw_json=1', {
+          credentials: 'include'
+        });
+        const d = await res.json();
+        return (d?.data?.children || []).map(c => ({
+          title: c.data.title,
+          subreddit: c.data.subreddit_name_prefixed,
+          score: c.data.score,
+          comments: c.data.num_comments,
+          author: c.data.author,
+          url: 'https://www.reddit.com' + c.data.permalink,
+        }));
+      })()
+
+  - map:
+      rank: ${{ index + 1 }}
+      title: ${{ item.title }}
+      subreddit: ${{ item.subreddit }}
+      score: ${{ item.score }}
+      comments: ${{ item.comments }}
+
+  - limit: ${{ args.limit }}
+
+columns: [rank, title, subreddit, score, comments]

package/src/clis/reuters/search.ts

@@ -0,0 +1,52 @@
+/**
+ * Reuters news search — API with HTML fallback.
+ * Source: bb-sites/reuters/search.js
+ */
+import { cli, Strategy } from '../../registry.js';
+
+cli({
+  site: 'reuters',
+  name: 'search',
+  description: 'Reuters 路透社新闻搜索',
+  domain: 'www.reuters.com',
+  strategy: Strategy.COOKIE,
+  args: [
+    { name: 'query', required: true, help: 'Search query' },
+    { name: 'limit', type: 'int', default: 10, help: 'Number of results (max 40)' },
+  ],
+  columns: ['rank', 'title', 'date', 'section', 'url'],
+  func: async (page, kwargs) => {
+    const count = Math.min(kwargs.limit || 10, 40);
+    await page.goto('https://www.reuters.com');
+    await page.wait(2);
+    const data = await page.evaluate(`
+      (async () => {
+        const count = ${count};
+        const apiQuery = JSON.stringify({
+          keyword: '${kwargs.query.replace(/'/g, "\\'")}',
+          offset: 0, orderby: 'display_date:desc', size: count, website: 'reuters'
+        });
+        const apiUrl = 'https://www.reuters.com/pf/api/v3/content/fetch/articles-by-search-v2?query=' + encodeURIComponent(apiQuery);
+        try {
+          const resp = await fetch(apiUrl, {credentials: 'include'});
+          if (resp.ok) {
+            const data = await resp.json();
+            const articles = data.result?.articles || data.articles || [];
+            if (articles.length > 0) {
+              return articles.slice(0, count).map((a, i) => ({
+                rank: i + 1,
+                title: a.title || a.headlines?.basic || '',
+                date: (a.display_date || a.published_time || '').split('T')[0],
+                section: a.taxonomy?.section?.name || '',
+                url: a.canonical_url ? 'https://www.reuters.com' + a.canonical_url : '',
+              }));
+            }
+          }
+        } catch(e) {}
+        return {error: 'Reuters API unavailable'};
+      })()
+    `);
+    if (!Array.isArray(data)) return [];
+    return data;
+  },
+});

package/src/clis/smzdm/search.ts

@@ -0,0 +1,66 @@
+/**
+ * 什么值得买搜索好价 — browser cookie, HTML parse.
+ * Source: bb-sites/smzdm/search.js
+ */
+import { cli, Strategy } from '../../registry.js';
+
+cli({
+  site: 'smzdm',
+  name: 'search',
+  description: '什么值得买搜索好价',
+  domain: 'www.smzdm.com',
+  strategy: Strategy.COOKIE,
+  args: [
+    { name: 'keyword', required: true, help: 'Search keyword' },
+    { name: 'limit', type: 'int', default: 20, help: 'Number of results' },
+  ],
+  columns: ['rank', 'title', 'price', 'mall', 'comments', 'url'],
+  func: async (page, kwargs) => {
+    const q = encodeURIComponent(kwargs.keyword);
+    const limit = kwargs.limit || 20;
+    await page.goto('https://www.smzdm.com');
+    await page.wait(2);
+    const data = await page.evaluate(`
+      (async () => {
+        const q = '${q}';
+        const limit = ${limit};
+        // Try youhui channel first, then home
+        for (const channel of ['youhui', 'home']) {
+          try {
+            const resp = await fetch('https://search.smzdm.com/ajax/?c=' + channel + '&s=' + q + '&p=1&v=b', {
+              credentials: 'include',
+              headers: {'X-Requested-With': 'XMLHttpRequest'}
+            });
+            if (!resp.ok) continue;
+            const html = await resp.text();
+            if (html.indexOf('feed-row-wide') === -1) continue;
+            const parser = new DOMParser();
+            const doc = parser.parseFromString(html, 'text/html');
+            const items = doc.querySelectorAll('li.feed-row-wide');
+            const results = [];
+            items.forEach((li, i) => {
+              if (results.length >= limit) return;
+              const titleEl = li.querySelector('h5.feed-block-title > a')
+                           || li.querySelector('h5 > a');
+              if (!titleEl) return;
+              const title = (titleEl.getAttribute('title') || titleEl.textContent || '').trim();
+              const url = titleEl.getAttribute('href') || '';
+              const priceEl = li.querySelector('.z-highlight');
+              const price = priceEl ? priceEl.textContent.trim() : '';
+              let mall = '';
+              const extrasSpan = li.querySelector('.z-feed-foot-r .feed-block-extras span');
+              if (extrasSpan) mall = extrasSpan.textContent.trim();
+              const commentEl = li.querySelector('.feed-btn-comment');
+              const comments = commentEl ? parseInt(commentEl.textContent.trim()) || 0 : 0;
+              results.push({rank: results.length + 1, title, price, mall, comments, url});
+            });
+            if (results.length > 0) return results;
+          } catch(e) { continue; }
+        }
+        return {error: 'No results'};
+      })()
+    `);
+    if (!Array.isArray(data)) return [];
+    return data;
+  },
+});

package/src/clis/v2ex/hot.yaml

@@ -2,6 +2,8 @@ site: v2ex
 name: hot
 description: V2EX 热门话题
 domain: www.v2ex.com
+strategy: public
+browser: false
 
 args:
   limit:
@@ -10,20 +12,14 @@ args:
     description: Number of topics
 
 pipeline:
-  - evaluate: |
-      (async () => {
-        const res = await fetch('https://www.v2ex.com/api/topics/hot.json');
-        return await res.json();
-      })()
+  - fetch:
+      url: https://www.v2ex.com/api/topics/hot.json
 
   - map:
       rank: ${{ index + 1 }}
       title: ${{ item.title }}
-      node: ${{ item.node?.title }}
-      author: ${{ item.member?.username }}
       replies: ${{ item.replies }}
-      url: ${{ item.url }}
 
   - limit: ${{ args.limit }}
 
-columns: [rank, title, node, author, replies]
+columns: [rank, title, replies]

package/src/clis/v2ex/latest.yaml

@@ -2,6 +2,8 @@ site: v2ex
 name: latest
 description: V2EX 最新话题
 domain: www.v2ex.com
+strategy: public
+browser: false
 
 args:
   limit:
@@ -10,19 +12,14 @@ args:
     description: Number of topics
 
 pipeline:
-  - evaluate: |
-      (async () => {
-        const res = await fetch('https://www.v2ex.com/api/topics/latest.json');
-        return await res.json();
-      })()
+  - fetch:
+      url: https://www.v2ex.com/api/topics/latest.json
 
   - map:
       rank: ${{ index + 1 }}
       title: ${{ item.title }}
-      node: ${{ item.node?.title }}
-      author: ${{ item.member?.username }}
       replies: ${{ item.replies }}
 
   - limit: ${{ args.limit }}
 
-columns: [rank, title, node, author, replies]
+columns: [rank, title, replies]

package/src/clis/v2ex/topic.yaml

@@ -0,0 +1,27 @@
+site: v2ex
+name: topic
+description: V2EX 主题详情和回复
+domain: www.v2ex.com
+strategy: public
+browser: false
+
+args:
+  id:
+    type: str
+    required: true
+    description: Topic ID
+
+pipeline:
+  - fetch:
+      url: https://www.v2ex.com/api/topics/show.json
+      params:
+        id: ${{ args.id }}
+
+  - map:
+      title: ${{ item.title }}
+      replies: ${{ item.replies }}
+      url: ${{ item.url }}
+
+  - limit: 1
+
+columns: [title, replies, url]