@jackwener/opencli 1.4.1 → 1.5.0

package/src/clis/imdb/top.ts

@@ -0,0 +1,67 @@
+import { CommandExecutionError } from '../../errors.js';
+import { cli, Strategy } from '../../registry.js';
+import { extractJsonLd, forceEnglishUrl, isChallengePage } from './utils.js';
+
+/**
+ * Fetch the IMDb Top 250 Movies list from JSON-LD structured data on the chart page.
+ */
+cli({
+  site: 'imdb',
+  name: 'top',
+  description: 'IMDb Top 250 Movies',
+  domain: 'www.imdb.com',
+  strategy: Strategy.PUBLIC,
+  browser: true,
+  args: [
+    { name: 'limit', type: 'int', default: 20, help: 'Number of results' },
+  ],
+  columns: ['rank', 'title', 'rating', 'votes', 'genre', 'url'],
+  func: async (page, args) => {
+    const url = forceEnglishUrl('https://www.imdb.com/chart/top/');
+
+    await page.goto(url);
+    await page.wait(2);
+
+    if (await isChallengePage(page)) {
+      throw new CommandExecutionError(
+        'IMDb blocked this request',
+        'Try again with a normal browser session or extension mode',
+      );
+    }
+
+    // Extract the ItemList JSON-LD block which contains all chart entries
+    const ld = await extractJsonLd(page, 'ItemList');
+    if (!ld || !Array.isArray(ld.itemListElement)) {
+      throw new CommandExecutionError(
+        'Could not find chart data on page',
+        'IMDb may have changed their page structure',
+      );
+    }
+
+    const limit = Math.max(1, Math.min(Number(args.limit) || 20, 250));
+    const items = (ld.itemListElement as any[]).slice(0, limit);
+
+    return items.map((entry: any, index: number) => {
+      const item = entry.item || {};
+      const rating = item.aggregateRating || {};
+      const genre = Array.isArray(item.genre)
+        ? item.genre.join(', ')
+        : String(item.genre || '');
+
+      // Normalize relative URLs to absolute IMDb URLs
+      let itemUrl: string = item.url || '';
+      if (itemUrl && !/^https?:\/\//.test(itemUrl)) {
+        itemUrl = 'https://www.imdb.com' + itemUrl;
+      }
+
+      return {
+        rank: entry.position || index + 1,
+        title: String(item.name || ''),
+        rating: rating.ratingValue != null ? String(rating.ratingValue) : '',
+        votes: rating.ratingCount != null ? String(rating.ratingCount) : '',
+        genre,
+        url: itemUrl,
+      };
+    });
+  },
+});

package/src/clis/imdb/trending.ts

@@ -0,0 +1,66 @@
+import { CommandExecutionError } from '../../errors.js';
+import { cli, Strategy } from '../../registry.js';
+import { extractJsonLd, forceEnglishUrl, isChallengePage } from './utils.js';
+
+/**
+ * Fetch the IMDb Most Popular Movies (MovieMeter) list from JSON-LD structured data.
+ */
+cli({
+  site: 'imdb',
+  name: 'trending',
+  description: 'IMDb Most Popular Movies',
+  domain: 'www.imdb.com',
+  strategy: Strategy.PUBLIC,
+  browser: true,
+  args: [
+    { name: 'limit', type: 'int', default: 20, help: 'Number of results' },
+  ],
+  columns: ['rank', 'title', 'rating', 'genre', 'url'],
+  func: async (page, args) => {
+    const url = forceEnglishUrl('https://www.imdb.com/chart/moviemeter/');
+
+    await page.goto(url);
+    await page.wait(2);
+
+    if (await isChallengePage(page)) {
+      throw new CommandExecutionError(
+        'IMDb blocked this request',
+        'Try again with a normal browser session or extension mode',
+      );
+    }
+
+    // Extract the ItemList JSON-LD block which contains all chart entries
+    const ld = await extractJsonLd(page, 'ItemList');
+    if (!ld || !Array.isArray(ld.itemListElement)) {
+      throw new CommandExecutionError(
+        'Could not find chart data on page',
+        'IMDb may have changed their page structure',
+      );
+    }
+
+    const limit = Math.max(1, Math.min(Number(args.limit) || 20, 100));
+    const items = (ld.itemListElement as any[]).slice(0, limit);
+
+    return items.map((entry: any, index: number) => {
+      const item = entry.item || {};
+      const rating = item.aggregateRating || {};
+      const genre = Array.isArray(item.genre)
+        ? item.genre.join(', ')
+        : String(item.genre || '');
+
+      // Normalize relative URLs to absolute IMDb URLs
+      let itemUrl: string = item.url || '';
+      if (itemUrl && !/^https?:\/\//.test(itemUrl)) {
+        itemUrl = 'https://www.imdb.com' + itemUrl;
+      }
+
+      return {
+        rank: entry.position || index + 1,
+        title: String(item.name || ''),
+        rating: rating.ratingValue != null ? String(rating.ratingValue) : '',
+        genre,
+        url: itemUrl,
+      };
+    });
+  },
+});

package/src/clis/imdb/utils.test.ts

@@ -0,0 +1,117 @@
+import { describe, expect, it, vi } from 'vitest';
+import type { IPage } from '../../types.js';
+import {
+  extractJsonLd,
+  forceEnglishUrl,
+  formatDuration,
+  getCurrentImdbId,
+  isChallengePage,
+  normalizeImdbTitleType,
+  normalizeImdbId,
+  waitForImdbPath,
+  waitForImdbReviewsReady,
+  waitForImdbSearchReady,
+} from './utils.js';
+
+describe('normalizeImdbId', () => {
+  it('passes through bare ids', () => {
+    expect(normalizeImdbId('tt1375666', 'tt')).toBe('tt1375666');
+    expect(normalizeImdbId('nm0634240', 'nm')).toBe('nm0634240');
+  });
+
+  it('extracts ids from supported urls', () => {
+    expect(normalizeImdbId('https://www.imdb.com/title/tt1375666/', 'tt')).toBe('tt1375666');
+    expect(normalizeImdbId('https://m.imdb.com/title/tt1375666/', 'tt')).toBe('tt1375666');
+    expect(normalizeImdbId('https://www.imdb.com/de/title/tt1375666/?ref_=nv_sr_srsg_0', 'tt')).toBe('tt1375666');
+    expect(normalizeImdbId('https://www.imdb.com/name/nm0634240/', 'nm')).toBe('nm0634240');
+  });
+
+  it('throws on invalid or mismatched ids', () => {
+    expect(() => normalizeImdbId('invalid', 'tt')).toThrow('Invalid IMDb ID');
+    expect(() => normalizeImdbId('tt1', 'tt')).toThrow('Invalid IMDb ID');
+    expect(() => normalizeImdbId('nm0634240', 'tt')).toThrow('Invalid IMDb ID');
+  });
+});
+
+describe('formatDuration', () => {
+  it('converts ISO-8601 durations to a short human format', () => {
+    expect(formatDuration('PT2H28M')).toBe('2h 28m');
+    expect(formatDuration('PT1H')).toBe('1h');
+    expect(formatDuration('PT45M')).toBe('45m');
+    expect(formatDuration('PT2H')).toBe('2h');
+  });
+
+  it('returns an empty string for invalid input', () => {
+    expect(formatDuration('')).toBe('');
+    expect(formatDuration('invalid')).toBe('');
+  });
+});
+
+describe('forceEnglishUrl', () => {
+  it('adds the English language parameter', () => {
+    expect(forceEnglishUrl('https://www.imdb.com/title/tt1375666/')).toContain('language=en-US');
+  });
+
+  it('preserves existing query parameters', () => {
+    const result = forceEnglishUrl('https://www.imdb.com/title/tt1375666/?ref_=nv');
+    expect(result).toContain('language=en-US');
+    expect(result).toContain('ref_=nv');
+  });
+});
+
+describe('normalizeImdbTitleType', () => {
+  it('maps internal imdb ids to readable labels', () => {
+    expect(normalizeImdbTitleType({ id: 'movie', text: '' })).toBe('Movie');
+    expect(normalizeImdbTitleType({ id: 'tvSeries', text: '' })).toBe('TV Series');
+    expect(normalizeImdbTitleType('short')).toBe('Short');
+  });
+
+  it('preserves explicit text labels', () => {
+    expect(normalizeImdbTitleType({ id: 'movie', text: 'Feature Film' })).toBe('Feature Film');
+  });
+});
+
+describe('extractJsonLd', () => {
+  it('returns the evaluated JSON-LD payload', async () => {
+    const page = {
+      evaluate: vi.fn().mockResolvedValue({ '@type': 'Movie', name: 'Inception' }),
+    } as unknown as IPage;
+
+    await expect(extractJsonLd(page, 'Movie')).resolves.toEqual({ '@type': 'Movie', name: 'Inception' });
+    expect(page.evaluate).toHaveBeenCalledTimes(1);
+    expect(page.evaluate).toHaveBeenCalledWith(expect.stringContaining('"Movie"'));
+  });
+});
+
+describe('isChallengePage', () => {
+  it('returns true when the page evaluation matches a challenge', async () => {
+    const page = {
+      evaluate: vi.fn().mockResolvedValue(true),
+    } as unknown as IPage;
+
+    await expect(isChallengePage(page)).resolves.toBe(true);
+    expect(page.evaluate).toHaveBeenCalledTimes(1);
+  });
+});
+
+describe('imdb browser helpers', () => {
+  it('reads the current imdb id from page metadata', async () => {
+    const page = {
+      evaluate: vi.fn().mockResolvedValue('nm0634240'),
+    } as unknown as IPage;
+
+    await expect(getCurrentImdbId(page, 'nm')).resolves.toBe('nm0634240');
+    expect(page.evaluate).toHaveBeenCalledTimes(1);
+  });
+
+  it('wait helpers resolve mocked readiness booleans', async () => {
+    const page = {
+      evaluate: vi.fn().mockResolvedValue(true),
+    } as unknown as IPage;
+
+    await expect(waitForImdbPath(page, '^/find/?$')).resolves.toBe(true);
+    await expect(waitForImdbSearchReady(page)).resolves.toBe(true);
+    await expect(waitForImdbReviewsReady(page)).resolves.toBe(true);
+    expect(page.evaluate).toHaveBeenCalledTimes(3);
+  });
+});

package/src/clis/imdb/utils.ts

@@ -0,0 +1,305 @@
+import { ArgumentError } from '../../errors.js';
+import type { IPage } from '../../types.js';
+
+/**
+ * Normalize an IMDb title or person input to a bare ID.
+ * Accepts bare IDs, desktop URLs, mobile URLs, and URLs with language prefixes or query params.
+ */
+export function normalizeImdbId(input: string, prefix: 'tt' | 'nm'): string {
+  const trimmed = input.trim();
+  const barePattern = new RegExp(`^${prefix}\\d{7,8}$`);
+  if (barePattern.test(trimmed)) {
+    return trimmed;
+  }
+
+  const pathPattern = new RegExp(`/(?:[a-z]{2}/)?(?:title|name)/(${prefix}\\d{7,8})(?:[/?#]|$)`, 'i');
+  const pathMatch = trimmed.match(pathPattern);
+  if (pathMatch) {
+    return pathMatch[1];
+  }
+
+  throw new ArgumentError(
+    `Invalid IMDb ID: "${input}"`,
+    `Expected ${prefix === 'tt' ? 'title' : 'name'} ID like ${prefix === 'tt' ? 'tt1375666' : 'nm0634240'} or an IMDb URL`,
+  );
+}
+
+/**
+ * Convert an ISO 8601 duration string to a short human-readable format for table display.
+ * Example: PT2H28M -> 2h 28m.
+ */
+export function formatDuration(iso: string): string {
+  if (!iso) {
+    return '';
+  }
+
+  const match = iso.match(/^PT(?:(\d+)H)?(?:(\d+)M)?$/);
+  if (!match) {
+    return '';
+  }
+
+  const parts: string[] = [];
+  if (match[1]) {
+    parts.push(`${match[1]}h`);
+  }
+  if (match[2]) {
+    parts.push(`${match[2]}m`);
+  }
+  return parts.join(' ');
+}
+
+/**
+ * Force an IMDb page URL to use the English language parameter,
+ * reducing structural differences across localized pages.
+ */
+export function forceEnglishUrl(url: string): string {
+  const parsed = new URL(url);
+  parsed.searchParams.set('language', 'en-US');
+  return parsed.toString();
+}
+
+/**
+ * Normalize IMDb title-type payloads that may be represented as an object,
+ * a raw string, or an empty text field with only an internal id.
+ */
+export function normalizeImdbTitleType(input: unknown): string {
+  const raw = (() => {
+    if (typeof input === 'string') return input;
+    if (!input || typeof input !== 'object') return '';
+    const value = input as Record<string, unknown>;
+    return typeof value.text === 'string' && value.text.trim()
+      ? value.text
+      : typeof value.id === 'string'
+        ? value.id
+        : '';
+  })().trim();
+
+  if (!raw) return '';
+
+  const known: Record<string, string> = {
+    movie: 'Movie',
+    short: 'Short',
+    video: 'Video',
+    tvEpisode: 'TV Episode',
+    tvMiniSeries: 'TV Mini Series',
+    tvMovie: 'TV Movie',
+    tvSeries: 'TV Series',
+    tvShort: 'TV Short',
+    tvSpecial: 'TV Special',
+    videoGame: 'Video Game',
+  };
+
+  return known[raw] ?? raw;
+}
+
+/**
+ * Extract structured JSON-LD data from the page.
+ * Accepts a single type string or an array of types to match against @type.
+ */
+export async function extractJsonLd(page: IPage, type?: string | string[]): Promise<Record<string, unknown> | null> {
+  const filterTypes = type ? (Array.isArray(type) ? type : [type]) : [];
+  return page.evaluate(`
+    (function() {
+      var scripts = document.querySelectorAll('script[type="application/ld+json"]');
+      var wantedTypes = ${JSON.stringify(filterTypes)};
+
+      function matchesType(data) {
+        if (wantedTypes.length === 0) {
+          return true;
+        }
+        if (!data || typeof data !== 'object') {
+          return false;
+        }
+        if (wantedTypes.indexOf(data['@type']) !== -1) {
+          return true;
+        }
+        if (Array.isArray(data['@type'])) {
+          for (var t = 0; t < data['@type'].length; t++) {
+            if (wantedTypes.indexOf(data['@type'][t]) !== -1) return true;
+          }
+        }
+        return false;
+      }
+
+      function findMatch(data) {
+        if (Array.isArray(data)) {
+          for (var i = 0; i < data.length; i++) {
+            var itemMatch = findMatch(data[i]);
+            if (itemMatch) {
+              return itemMatch;
+            }
+          }
+          return null;
+        }
+
+        if (!data || typeof data !== 'object') {
+          return null;
+        }
+
+        if (matchesType(data)) {
+          return data;
+        }
+
+        if (Array.isArray(data['@graph'])) {
+          return findMatch(data['@graph']);
+        }
+
+        return null;
+      }
+
+      for (var i = 0; i < scripts.length; i++) {
+        try {
+          var parsed = JSON.parse(scripts[i].textContent || 'null');
+          var match = findMatch(parsed);
+          if (match) {
+            return match;
+          }
+        } catch (error) {
+          void error;
+        }
+      }
+
+      return null;
+    })()
+  `);
+}
+
+/**
+ * Poll until the current IMDb page path matches the expected entity/search path.
+ */
+export async function waitForImdbPath(page: IPage, pathPattern: string, timeoutMs: number = 15000): Promise<boolean> {
+  const result = await page.evaluate(`
+    (async function() {
+      var deadline = Date.now() + ${timeoutMs};
+      var pattern = new RegExp(${JSON.stringify(pathPattern)}, 'i');
+      while (Date.now() < deadline) {
+        if (pattern.test(window.location.pathname)) {
+          return true;
+        }
+        await new Promise(function(resolve) { setTimeout(resolve, 250); });
+      }
+      return pattern.test(window.location.pathname);
+    })()
+  `);
+  return Boolean(result);
+}
+
+/**
+ * Wait until IMDb search results (or the search UI state) has rendered.
+ */
+export async function waitForImdbSearchReady(page: IPage, timeoutMs: number = 15000): Promise<boolean> {
+  const result = await page.evaluate(`
+    (async function() {
+      var deadline = Date.now() + ${timeoutMs};
+
+      function hasSearchResults() {
+        var nextDataEl = document.getElementById('__NEXT_DATA__');
+        if (nextDataEl) {
+          try {
+            var nextData = JSON.parse(nextDataEl.textContent || 'null');
+            var pageProps = nextData && nextData.props && nextData.props.pageProps;
+            var titleResults = (pageProps && pageProps.titleResults && pageProps.titleResults.results) || [];
+            var nameResults = (pageProps && pageProps.nameResults && pageProps.nameResults.results) || [];
+            if (titleResults.length > 0 || nameResults.length > 0) {
+              return true;
+            }
+          } catch (error) {
+            void error;
+          }
+        }
+
+        if (document.querySelector('a[href*="/title/"], a[href*="/name/"]')) {
+          return true;
+        }
+
+        var body = document.body ? (document.body.textContent || '') : '';
+        return body.includes('No results found for') || body.includes('No exact matches');
+      }
+
+      while (Date.now() < deadline) {
+        if (hasSearchResults()) {
+          return true;
+        }
+        await new Promise(function(resolve) { setTimeout(resolve, 250); });
+      }
+
+      return hasSearchResults();
+    })()
+  `);
+  return Boolean(result);
+}
+
+/**
+ * Wait until IMDb review cards (or the page review summary) has rendered.
+ */
+export async function waitForImdbReviewsReady(page: IPage, timeoutMs: number = 15000): Promise<boolean> {
+  const result = await page.evaluate(`
+    (async function() {
+      var deadline = Date.now() + ${timeoutMs};
+
+      function hasReviewContent() {
+        if (document.querySelector('article.user-review-item, [data-testid="review-card-parent"], [data-testid="tturv-total-reviews"]')) {
+          return true;
+        }
+        var body = document.body ? (document.body.textContent || '') : '';
+        return body.includes('No user reviews') || body.includes('Review this title');
+      }
+
+      while (Date.now() < deadline) {
+        if (hasReviewContent()) {
+          return true;
+        }
+        await new Promise(function(resolve) { setTimeout(resolve, 250); });
+      }
+
+      return hasReviewContent();
+    })()
+  `);
+  return Boolean(result);
+}
+
+/**
+ * Read the current IMDb entity id from the page URL/canonical metadata.
+ */
+export async function getCurrentImdbId(page: IPage, prefix: 'tt' | 'nm'): Promise<string> {
+  const result = await page.evaluate(`
+    (function() {
+      var pattern = new RegExp('(${prefix}\\\\d{7,8})', 'i');
+      var candidates = [
+        window.location.pathname || '',
+        document.querySelector('link[rel="canonical"]')?.getAttribute('href') || '',
+        document.querySelector('meta[property="og:url"]')?.getAttribute('content') || ''
+      ];
+
+      for (var i = 0; i < candidates.length; i++) {
+        var match = candidates[i].match(pattern);
+        if (match) {
+          return match[1];
+        }
+      }
+      return '';
+    })()
+  `);
+
+  return typeof result === 'string' ? result : '';
+}
+
+/**
+ * Detect whether the current page is an IMDb bot-challenge or verification page.
+ */
+export async function isChallengePage(page: IPage): Promise<boolean> {
+  const result = await page.evaluate(`
+    (function() {
+      var title = document.title || '';
+      var body = document.body ? (document.body.textContent || '') : '';
+      return title.includes('Robot Check') ||
+        title.includes('Are you a robot') ||
+        title.includes('JavaScript is disabled') ||
+        body.includes('captcha') ||
+        body.includes('verify that you are human') ||
+        body.includes('not a robot');
+    })()
+  `);
+
+  return Boolean(result);
+}

package/src/clis/jd/item.test.ts

@@ -1,5 +1,6 @@
 import { describe, expect, it } from 'vitest';
 import { getRegistry } from '../../registry.js';
+import { __test__ } from './item.js';
 import './item.js';
 
 describe('jd item adapter', () => {
@@ -29,7 +30,23 @@ describe('jd item adapter', () => {
 
   it('includes expected columns', () => {
     expect(command!.columns).toEqual(
-      expect.arrayContaining(['title', 'price', 'shop', 'specs', 'mainImages', 'detailImages']),
+      expect.arrayContaining(['title', 'price', 'shop', 'specs', 'avifImages']),
     );
   });
+
+  it('extracts only pcpubliccms avif images and respects the limit', () => {
+    const result = __test__.extractAvifImages([
+      'https://img14.360buyimg.com/n1/jfs/t1/normal.jpg',
+      'https://img10.360buyimg.com/imgzone/jfs/t1/detail.avif',
+      'https://pcpubliccms.jd.com/image1.avif',
+      'https://pcpubliccms.jd.com/image1.avif',
+      'https://pcpubliccms.jd.com/image2.avif?x=1',
+      'https://example.com/not-jd.avif',
+    ], 2);
+
+    expect(result).toEqual([
+      'https://pcpubliccms.jd.com/image1.avif',
+      'https://pcpubliccms.jd.com/image2.avif?x=1',
+    ]);
+  });
 });

package/src/clis/jd/item.ts

@@ -6,10 +6,17 @@
  */
 import { cli, Strategy } from '../../registry.js';
 
+function extractAvifImages(imageUrls: string[], maxImages: number): string[] {
+  const unique = [...new Set(imageUrls.filter(Boolean))];
+  return unique
+    .filter((url) => url.includes('.avif') && url.includes('pcpubliccms'))
+    .slice(0, maxImages);
+}
+
 cli({
   site: 'jd',
   name: 'item',
-  description: '京东商品详情（价格、主图、详情图、规格参数）',
+  description: '京东商品详情（价格、店铺、规格参数、AVIF 图片）',
   domain: 'item.jd.com',
   strategy: Strategy.COOKIE,
   args: [
@@ -23,10 +30,10 @@ cli({
       name: 'images',
       type: 'int',
       default: 10,
-      help: '详情图数量（默认10）',
+      help: 'AVIF 图片数量上限（默认10）',
     },
   ],
-  columns: ['title', 'price', 'shop', 'specs', 'mainImages', 'detailImages'],
+  columns: ['title', 'price', 'shop', 'specs', 'avifImages'],
   func: async (page, kwargs) => {
     const sku = kwargs.sku;
     const maxImages = kwargs.images as number;
@@ -35,7 +42,7 @@ cli({
     await page.goto(url, { waitUntil: 'load' });
     await page.wait(2);
 
-    // 滚动加载详情图
+    // 滚动加载商品详情区域中的延迟图片
     for (let i = 0; i < 6; i++) {
       await page.evaluate(`window.scrollTo(0, ${i * 2500})`);
       await page.wait(1);
@@ -65,17 +72,9 @@ cli({
         // 所有图片
         const allImgs = Array.from(document.querySelectorAll('img[src*="360buyimg.com"]'));
         const srcs = allImgs.map(img => img.src).filter(Boolean);
-        const unique = [...new Set(srcs)];
-
-        // 主图
-        const mainImgs = unique
-          .filter(u => u.includes('/n1/') || u.includes('/n3/') || u.includes('/n4/') || u.includes('/img/'))
-          .slice(0, maxImg);
 
-        // 详情图
-        const detailImgs = unique
-          .filter(u => u.includes('/babel/') || u.includes('/popshop/'))
-          .slice(0, maxImg);
+        // 所有 avif 图片（去重，只保留 pcpubliccms CDN）
+        const avifImages = ${extractAvifImages.toString()}(srcs, maxImg);
 
         // 规格参数：从页面文本提取
         const text = document.body.innerText;
@@ -92,10 +91,14 @@ cli({
           }
         }
 
-        return { title, price, shop, specs, mainImages: mainImgs, detailImages: detailImgs, totalImages: unique.length };
+        return { title, price, shop, specs, avifImages, totalImages: new Set(srcs).size };
       })()
     `);
 
     return [data];
   },
 });
+
+export const __test__ = {
+  extractAvifImages,
+};