@jackwener/opencli 1.1.0 → 1.2.0

package/src/clis/devto/top.yaml

@@ -0,0 +1,29 @@
+site: devto
+name: top
+description: Top DEV.to articles of the day
+domain: dev.to
+strategy: public
+browser: false
+
+args:
+  limit:
+    type: int
+    default: 20
+    description: Number of articles
+
+pipeline:
+  - fetch:
+      url: https://dev.to/api/articles?top=1&per_page=${{ args.limit }}
+
+  - map:
+      rank: ${{ index + 1 }}
+      title: ${{ item.title }}
+      author: ${{ item.user.username }}
+      reactions: ${{ item.public_reactions_count }}
+      comments: ${{ item.comments_count }}
+      tags: ${{ item.tag_list | join(', ') }}
+      url: ${{ item.url }}
+
+  - limit: ${{ args.limit }}
+
+columns: [rank, title, author, reactions, comments, tags]

package/src/clis/devto/user.yaml

@@ -0,0 +1,33 @@
+site: devto
+name: user
+description: Recent DEV.to articles from a specific user
+domain: dev.to
+strategy: public
+browser: false
+
+args:
+  username:
+    type: str
+    required: true
+    positional: true
+    description: "DEV.to username (e.g. ben, thepracticaldev)"
+  limit:
+    type: int
+    default: 20
+    description: Number of articles
+
+pipeline:
+  - fetch:
+      url: https://dev.to/api/articles?username=${{ args.username }}&per_page=${{ args.limit }}
+
+  - map:
+      rank: ${{ index + 1 }}
+      title: ${{ item.title }}
+      reactions: ${{ item.public_reactions_count }}
+      comments: ${{ item.comments_count }}
+      tags: ${{ item.tag_list | join(', ') }}
+      url: ${{ item.url }}
+
+  - limit: ${{ args.limit }}
+
+columns: [rank, title, reactions, comments, tags]

package/src/clis/douban/book-hot.ts

@@ -0,0 +1,15 @@
+import { cli, Strategy } from '../../registry.js';
+import { loadDoubanBookHot } from './shared.js';
+
+cli({
+  site: 'douban',
+  name: 'book-hot',
+  description: '豆瓣图书热门榜单',
+  domain: 'book.douban.com',
+  strategy: Strategy.COOKIE,
+  args: [
+    { name: 'limit', type: 'int', default: 20, help: '返回的图书数量' },
+  ],
+  columns: ['rank', 'title', 'rating', 'quote', 'author', 'publisher', 'year', 'url'],
+  func: async (page, args) => loadDoubanBookHot(page, Number(args.limit) || 20),
+});

package/src/clis/douban/marks.ts

@@ -0,0 +1,135 @@
+import { cli, Strategy } from '../../registry.js';
+import type { IPage } from '../../types.js';
+import { DoubanMark, getSelfUid } from './utils.js';
+
+cli({
+  site: 'douban',
+  name: 'marks',
+  description: '导出个人观影标记',
+  domain: 'movie.douban.com',
+  strategy: Strategy.COOKIE,
+  args: [
+    { 
+      name: 'status', 
+      default: 'collect',
+      choices: ['collect', 'wish', 'do', 'all'],
+      help: '标记类型: collect(看过), wish(想看), do(在看), all(全部)' 
+    },
+    { name: 'limit', type: 'int', default: 50, help: '导出数量， 0 表示全部' },
+    { name: 'uid', help: '用户ID，不填则使用当前登录账号' },
+  ],
+  columns: ['title', 'year', 'myRating', 'myStatus', 'myDate', 'myComment', 'url'],
+  func: async (page: IPage, kwargs: { status?: string; limit?: number; uid?: string }) => {
+    const { status = 'collect', limit = 50, uid: providedUid } = kwargs;
+    
+    const uid = providedUid || await getSelfUid(page);
+    
+    const statuses = status === 'all' 
+      ? ['collect', 'wish', 'do'] 
+      : [status];
+    
+    const allMarks: DoubanMark[] = [];
+    
+    for (const s of statuses) {
+      const remaining = limit > 0 ? limit - allMarks.length : 0;
+      if (limit > 0 && remaining <= 0) break;
+      
+      const marks = await fetchMarks(page, uid, s, remaining);
+      allMarks.push(...marks);
+    }
+    
+    return allMarks.slice(0, limit > 0 ? limit : undefined);
+  },
+});
+
+async function fetchMarks(
+  page: IPage, 
+  uid: string, 
+  status: string, 
+  limit: number
+): Promise<DoubanMark[]> {
+  const marks: DoubanMark[] = [];
+  let offset = 0;
+  const pageSize = 30;
+  
+  while (true) {
+    const url = `https://movie.douban.com/people/${uid}/${status}?start=${offset}&sort=time&rating=all&filter=all&mode=grid`;
+    
+    await page.goto(url);
+    
+    await page.wait({ time: 2 });
+    
+    const pageMarks = await page.evaluate(`
+      () => {
+        const results = [];
+        
+        const items = document.querySelectorAll('.item');
+        
+        items.forEach(item => {
+          const titleLink = item.querySelector('.info a[href*="/subject/"]');
+          if (!titleLink) return;
+          
+          const titleEl = titleLink.querySelector('em');
+          const titleText = titleEl?.textContent?.trim() || titleLink.textContent?.trim() || '';
+          const title = titleText.split('/')[0].trim();
+          const href = titleLink.href || '';
+          
+          const idMatch = href.match(/subject\\/(\\d+)/);
+          const movieId = idMatch ? idMatch[1] : '';
+          
+          if (!movieId || !title) return;
+          
+          const ratingSpan = item.querySelector('span[class*="rating"]');
+          let myRating = null;
+          if (ratingSpan) {
+            const cls = ratingSpan.className || '';
+            const ratingMatch = cls.match(/rating(\\d)-t/);
+            if (ratingMatch) {
+              myRating = parseInt(ratingMatch[1], 10) * 2;
+            }
+          }
+          
+          const dateSpan = item.querySelector('.date');
+          const myDate = dateSpan?.textContent?.trim() || '';
+          
+          const commentSpan = item.querySelector('.comment');
+          const myComment = commentSpan?.textContent?.trim() || '';
+          
+          const introSpan = item.querySelector('.intro');
+          let year = '';
+          if (introSpan) {
+            const introText = introSpan.textContent || '';
+            const yearMatch = introText.match(/(\\d{4})/);
+            year = yearMatch ? yearMatch[1] : '';
+          }
+          
+          results.push({
+            movieId,
+            title,
+            year,
+            myRating,
+            myStatus: '${status}',
+            myComment,
+            myDate,
+            url: href || 'https://movie.douban.com/subject/' + movieId
+          });
+        });
+        
+        return results;
+      }
+    `) as DoubanMark[];
+    
+    if (!pageMarks || pageMarks.length === 0) break;
+    
+    marks.push(...pageMarks);
+    
+    if (pageMarks.length < pageSize) break;
+    if (limit > 0 && marks.length >= limit) break;
+    
+    offset += pageSize;
+    
+    await new Promise(resolve => setTimeout(resolve, 1000));
+  }
+  
+  return marks;
+}

package/src/clis/douban/movie-hot.ts

@@ -0,0 +1,15 @@
+import { cli, Strategy } from '../../registry.js';
+import { loadDoubanMovieHot } from './shared.js';
+
+cli({
+  site: 'douban',
+  name: 'movie-hot',
+  description: '豆瓣电影热门榜单',
+  domain: 'movie.douban.com',
+  strategy: Strategy.COOKIE,
+  args: [
+    { name: 'limit', type: 'int', default: 20, help: '返回的电影数量' },
+  ],
+  columns: ['rank', 'title', 'rating', 'quote', 'director', 'year', 'region', 'url'],
+  func: async (page, args) => loadDoubanMovieHot(page, Number(args.limit) || 20),
+});

package/src/clis/douban/reviews.ts

@@ -0,0 +1,127 @@
+import { cli, Strategy } from '../../registry.js';
+import type { IPage } from '../../types.js';
+import { getSelfUid, DoubanReview } from './utils.js';
+
+cli({
+  site: 'douban',
+  name: 'reviews',
+  description: '导出个人影评',
+  domain: 'movie.douban.com',
+  strategy: Strategy.COOKIE,
+  args: [
+    { name: 'limit', type: 'int', default: 20, help: '导出数量' },
+    { name: 'uid', help: '用户ID，不填则使用当前登录账号' },
+    { name: 'full', type: 'bool', default: false, help: '获取完整影评内容' },
+  ],
+  columns: ['movieTitle', 'title', 'myRating', 'votes', 'content', 'url'],
+  func: async (page: IPage, kwargs: { limit?: number; uid?: string; full?: boolean }) => {
+    const { limit = 20, uid: providedUid, full = false } = kwargs;
+    
+    const uid = providedUid || await getSelfUid(page);
+    const reviews = await fetchReviews(page, uid, limit, full);
+    
+    return reviews;
+  },
+});
+
+async function fetchReviews(
+  page: IPage,
+  uid: string,
+  limit: number,
+  full: boolean,
+): Promise<DoubanReview[]> {
+  const reviews: DoubanReview[] = [];
+  let start = 0;
+  const pageSize = 20;
+  
+  while (true) {
+    const url = `https://movie.douban.com/people/${uid}/reviews?start=${start}&sort=time`;
+    
+    await page.goto(url);
+    
+    await page.wait({ time: 1 });
+    
+    const data = await page.evaluate(`
+      () => {
+        const reviews = [];
+        
+        document.querySelectorAll('.tlst').forEach(el => {
+          const movieLinkEl = el.querySelector('.ilst a');
+          const reviewTitleEl = el.querySelector('.nlst a[title]');
+          const ratingEl = el.querySelector('.clst span[class*="allstar"]');
+          const contentEl = el.querySelector('.review-short span');
+          const votesEl = el.querySelector('.review-short .pl span');
+          
+          const movieHref = movieLinkEl?.href || '';
+          const movieId = movieHref.match(/subject\\/(\\d+)/)?.[1] || '';
+          const movieTitle = movieLinkEl?.getAttribute('title') || movieLinkEl?.textContent?.trim() || '';
+          
+          const reviewHref = reviewTitleEl?.href || '';
+          const reviewId = reviewHref.match(/reviews\\/(\\d+)/)?.[1] || '';
+          const title = reviewTitleEl?.textContent?.trim() || '';
+          
+          let myRating = 0;
+          if (ratingEl) {
+            const cls = ratingEl.className || '';
+            const ratingMatch = cls.match(/allstar(\\d)0/);
+            if (ratingMatch) {
+              myRating = parseInt(ratingMatch[1], 10) * 2;
+            }
+          }
+          
+          const votesText = votesEl?.textContent || '';
+          const votesMatch = votesText.match(/(\\d+)/);
+          const votes = votesMatch ? parseInt(votesMatch[1], 10) : 0;
+          
+          reviews.push({
+            reviewId,
+            movieId,
+            movieTitle,
+            title,
+            content: contentEl?.textContent?.trim() || '',
+            myRating,
+            createdAt: '',
+            votes,
+            url: reviewHref,
+          });
+        });
+        
+        return reviews;
+      }
+    `) as DoubanReview[];
+    
+    reviews.push(...data);
+    
+    if (data.length < pageSize) break;
+    if (limit > 0 && reviews.length >= limit) break;
+    
+    start += pageSize;
+  }
+  
+  const result = reviews.slice(0, limit > 0 ? limit : undefined);
+  
+  if (full && result.length > 0) {
+    for (const review of result) {
+      if (review.url) {
+        const fullContent = await fetchFullReview(page, review.url);
+        review.content = fullContent;
+      }
+    }
+  }
+  
+  return result;
+}
+
+async function fetchFullReview(page: IPage, reviewUrl: string): Promise<string> {
+  await page.goto(reviewUrl);
+  await page.wait({ time: 1 });
+  
+  const content = await page.evaluate(`
+    () => {
+      const contentEl = document.querySelector('.review-content');
+      return contentEl?.textContent?.trim() || '';
+    }
+  `) as string;
+  
+  return content;
+}

package/src/clis/douban/search.ts

@@ -0,0 +1,17 @@
+import { cli, Strategy } from '../../registry.js';
+import { searchDouban } from './shared.js';
+
+cli({
+  site: 'douban',
+  name: 'search',
+  description: '搜索豆瓣电影、图书或音乐',
+  domain: 'search.douban.com',
+  strategy: Strategy.COOKIE,
+  args: [
+    { name: 'type', default: 'movie', choices: ['movie', 'book', 'music'], help: '搜索类型（movie=电影, book=图书, music=音乐）' },
+    { name: 'keyword', required: true, positional: true, help: '搜索关键词' },
+    { name: 'limit', type: 'int', default: 20, help: '返回结果数量' },
+  ],
+  columns: ['rank', 'title', 'rating', 'abstract', 'url'],
+  func: async (page, args) => searchDouban(page, args.type, args.keyword, Number(args.limit) || 20),
+});

package/src/clis/douban/shared.ts

@@ -0,0 +1,165 @@
+import { CliError } from '../../errors.js';
+import type { IPage } from '../../types.js';
+
+function clampLimit(limit: number): number {
+  return Math.max(1, Math.min(limit || 20, 50));
+}
+
+async function ensureDoubanReady(page: IPage): Promise<void> {
+  const state = await page.evaluate(`
+    (() => {
+      const title = (document.title || '').trim();
+      const href = (location.href || '').trim();
+      const blocked = href.includes('sec.douban.com') || /登录跳转/.test(title) || /异常请求/.test(document.body?.innerText || '');
+      return { blocked, title, href };
+    })()
+  `);
+  if (state?.blocked) {
+    throw new CliError(
+      'AUTH_REQUIRED',
+      'Douban requires a logged-in browser session before these commands can load data.',
+      'Please sign in to douban.com in the browser that opencli reuses, then rerun the command.',
+    );
+  }
+}
+
+export async function loadDoubanBookHot(page: IPage, limit: number): Promise<any[]> {
+  const safeLimit = clampLimit(limit);
+  await page.goto('https://book.douban.com/chart');
+  await page.wait(4);
+  await ensureDoubanReady(page);
+  const data = await page.evaluate(`
+    (() => {
+      const normalize = (value) => (value || '').replace(/\\s+/g, ' ').trim();
+      const books = [];
+      for (const el of Array.from(document.querySelectorAll('.media.clearfix'))) {
+        try {
+          const titleEl = el.querySelector('h2 a[href*="/subject/"]');
+          const title = normalize(titleEl?.textContent);
+          let url = titleEl?.getAttribute('href') || '';
+          if (!title || !url) continue;
+          if (!url.startsWith('http')) url = 'https://book.douban.com' + url;
+
+          const info = normalize(el.querySelector('.subject-abstract, .pl, .pub')?.textContent);
+          const infoParts = info.split('/').map((part) => part.trim()).filter(Boolean);
+          const ratingText = normalize(el.querySelector('.subject-rating .font-small, .rating_nums, .rating')?.textContent);
+          const quote = Array.from(el.querySelectorAll('.subject-tags .tag'))
+            .map((node) => normalize(node.textContent))
+            .filter(Boolean)
+            .join(' / ');
+
+          books.push({
+            rank: parseInt(normalize(el.querySelector('.green-num-box')?.textContent), 10) || books.length + 1,
+            title,
+            rating: parseFloat(ratingText) || 0,
+            quote,
+            author: infoParts[0] || '',
+            publisher: infoParts.find((part) => /出版社|出版公司|Press/i.test(part)) || infoParts[2] || '',
+            year: infoParts.find((part) => /\\d{4}(?:-\\d{1,2})?/.test(part))?.match(/\\d{4}/)?.[0] || '',
+            price: infoParts.find((part) => /元|USD|\\$|￥/.test(part)) || '',
+            url,
+            cover: el.querySelector('img')?.getAttribute('src') || '',
+          });
+        } catch {}
+      }
+      return books.slice(0, ${safeLimit});
+    })()
+  `);
+  return Array.isArray(data) ? data : [];
+}
+
+export async function loadDoubanMovieHot(page: IPage, limit: number): Promise<any[]> {
+  const safeLimit = clampLimit(limit);
+  await page.goto('https://movie.douban.com/chart');
+  await page.wait(4);
+  await ensureDoubanReady(page);
+  const data = await page.evaluate(`
+    (() => {
+      const normalize = (value) => (value || '').replace(/\\s+/g, ' ').trim();
+      const results = [];
+      for (const el of Array.from(document.querySelectorAll('.item'))) {
+        const titleEl = el.querySelector('.pl2 a');
+        const title = normalize(titleEl?.textContent);
+        let url = titleEl?.getAttribute('href') || '';
+        if (!title || !url) continue;
+        if (!url.startsWith('http')) url = 'https://movie.douban.com' + url;
+
+        const info = normalize(el.querySelector('.pl2 p')?.textContent);
+        const infoParts = info.split('/').map((part) => part.trim()).filter(Boolean);
+        const releaseIndex = (() => {
+          for (let i = infoParts.length - 1; i >= 0; i -= 1) {
+            if (/\\d{4}-\\d{2}-\\d{2}|\\d{4}\\/\\d{2}\\/\\d{2}/.test(infoParts[i])) return i;
+          }
+          return -1;
+        })();
+        const directorPart = releaseIndex >= 1 ? infoParts[releaseIndex - 1] : '';
+        const regionPart = releaseIndex >= 2 ? infoParts[releaseIndex - 2] : '';
+        const yearMatch = info.match(/\\b(19|20)\\d{2}\\b/);
+        results.push({
+          rank: results.length + 1,
+          title,
+          rating: parseFloat(normalize(el.querySelector('.rating_nums')?.textContent)) || 0,
+          quote: normalize(el.querySelector('.inq')?.textContent),
+          director: directorPart.replace(/^导演:\\s*/, ''),
+          year: yearMatch?.[0] || '',
+          region: regionPart,
+          url,
+          cover: el.querySelector('img')?.getAttribute('src') || '',
+        });
+        if (results.length >= ${safeLimit}) break;
+      }
+      return results;
+    })()
+  `);
+  return Array.isArray(data) ? data : [];
+}
+
+export async function searchDouban(page: IPage, type: string, keyword: string, limit: number): Promise<any[]> {
+  const safeLimit = clampLimit(limit);
+  await page.goto(`https://search.douban.com/${encodeURIComponent(type)}/subject_search?search_text=${encodeURIComponent(keyword)}`);
+  await page.wait(2);
+  await ensureDoubanReady(page);
+  const data = await page.evaluate(`
+    (async () => {
+      const type = ${JSON.stringify(type)};
+      const normalize = (value) => (value || '').replace(/\\s+/g, ' ').trim();
+      const seen = new Set();
+      const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+
+      for (let i = 0; i < 20; i += 1) {
+        if (document.querySelector('.item-root .title-text, .item-root .title a')) break;
+        await sleep(300);
+      }
+
+      const items = Array.from(document.querySelectorAll('.item-root'));
+
+      const results = [];
+      for (const el of items) {
+        const titleEl = el.querySelector('.title-text, .title a, a[title]');
+        const title = normalize(titleEl?.textContent) || normalize(titleEl?.getAttribute('title'));
+        let url = titleEl?.getAttribute('href') || '';
+        if (!title || !url) continue;
+        if (!url.startsWith('http')) url = 'https://search.douban.com' + url;
+        if (!url.includes('/subject/') || seen.has(url)) continue;
+        seen.add(url);
+        const ratingText = normalize(el.querySelector('.rating_nums')?.textContent);
+        const abstract = normalize(
+          el.querySelector('.meta.abstract, .meta, .abstract, p')?.textContent,
+        );
+        results.push({
+          rank: results.length + 1,
+          id: url.match(/subject\\/(\\d+)/)?.[1] || '',
+          type,
+          title,
+          rating: ratingText.includes('.') ? parseFloat(ratingText) : 0,
+          abstract: abstract.slice(0, 100) + (abstract.length > 100 ? '...' : ''),
+          url,
+          cover: el.querySelector('img')?.getAttribute('src') || '',
+        });
+        if (results.length >= ${safeLimit}) break;
+      }
+      return results;
+    })()
+  `);
+  return Array.isArray(data) ? data : [];
+}

package/src/clis/douban/subject.yaml

@@ -0,0 +1,76 @@
+site: douban
+name: subject
+description: 获取电影详情
+domain: movie.douban.com
+strategy: cookie
+browser: true
+
+args:
+  id:
+    positional: true
+    required: true
+    type: str
+    description: 电影 ID
+
+pipeline:
+  - navigate: https://movie.douban.com/subject/${{ args.id }}
+
+  - evaluate: |
+      (async () => {
+        const id = '${{ args.id }}';
+        
+        // Wait for page to load
+        await new Promise(r => setTimeout(r, 2000));
+        
+        // Extract title
+        const titleEl = document.querySelector('span[property="v:itemreviewed"]');
+        const title = titleEl?.textContent?.trim() || '';
+        
+        // Extract original title
+        const ogTitleEl = document.querySelector('span[property="v:originalTitle"]');
+        const originalTitle = ogTitleEl?.textContent?.trim() || '';
+        
+        // Extract year
+        const yearEl = document.querySelector('.year');
+        const year = yearEl?.textContent?.trim() || '';
+        
+        // Extract rating
+        const ratingEl = document.querySelector('strong[property="v:average"]');
+        const rating = parseFloat(ratingEl?.textContent || '0');
+        
+        // Extract rating count
+        const ratingCountEl = document.querySelector('span[property="v:votes"]');
+        const ratingCount = parseInt(ratingCountEl?.textContent || '0', 10);
+        
+        // Extract genres
+        const genreEls = document.querySelectorAll('span[property="v:genre"]');
+        const genres = Array.from(genreEls).map(el => el.textContent?.trim()).filter(Boolean).join(',');
+        
+        // Extract directors
+        const directorEls = document.querySelectorAll('a[rel="v:directedBy"]');
+        const directors = Array.from(directorEls).map(el => el.textContent?.trim()).filter(Boolean).join(',');
+        
+        // Extract casts
+        const castEls = document.querySelectorAll('a[rel="v:starring"]');
+        const casts = Array.from(castEls).slice(0, 5).map(el => el.textContent?.trim()).filter(Boolean).join(',');
+        
+        // Extract summary
+        const summaryEl = document.querySelector('span[property="v:summary"]');
+        const summary = summaryEl?.textContent?.trim() || '';
+        
+        return [{
+          id,
+          title,
+          originalTitle,
+          year,
+          rating,
+          ratingCount,
+          genres,
+          directors,
+          casts,
+          summary: summary.substring(0, 200),
+          url: `https://movie.douban.com/subject/${id}`
+        }];
+      })()
+
+columns: [id, title, originalTitle, year, rating, ratingCount, genres, directors, casts, summary, url]

package/src/clis/douban/top250.yaml

@@ -0,0 +1,70 @@
+site: douban
+name: top250
+description: 豆瓣电影 Top250
+domain: movie.douban.com
+strategy: cookie
+browser: true
+
+args:
+  limit:
+    type: int
+    default: 250
+    description: 返回结果数量
+
+pipeline:
+  - navigate: https://movie.douban.com/top250
+
+  - evaluate: |
+      async () => {
+        const results = [];
+        const limit = ${{ args.limit }};
+
+        const parsePage = (doc) => {
+          const items = doc.querySelectorAll('.item');
+          for (const item of items) {
+            if (results.length >= limit) break;
+
+            const rankEl = item.querySelector('.pic em');
+            const linkEl = item.querySelector('a');
+            const titleEl = item.querySelector('.title');
+            const ratingEl = item.querySelector('.rating_num');
+
+            const href = linkEl?.href || '';
+            const matchResult = href.match(/subject\/(\d+)/);
+            const id = matchResult ? matchResult[1] : '';
+
+            const title = titleEl?.textContent?.trim() || '';
+            const rank = parseInt(rankEl?.textContent || '0', 10);
+            const rating = ratingEl?.textContent?.trim() || '';
+
+            if (id && title) {
+              results.push({
+                rank: rank || results.length + 1,
+                id,
+                title,
+                rating: rating ? parseFloat(rating) : 0,
+                url: href
+              });
+            }
+          }
+        };
+
+        parsePage(document);
+
+        for (let start = 25; start < 250 && results.length < limit; start += 25) {
+          const resp = await fetch(`https://movie.douban.com/top250?start=${start}`);
+          if (!resp.ok) break;
+          const html = await resp.text();
+          if (!html) break;
+
+          const doc = new DOMParser().parseFromString(html, 'text/html');
+          parsePage(doc);
+          await new Promise(r => setTimeout(r, 150));
+        }
+
+        return results;
+      }
+
+  - limit: ${{ args.limit }}
+
+columns: [rank, id, title, rating, url]