@jackwener/opencli 1.5.5 → 1.5.7

package/src/clis/douyin/hashtag.test.ts

@@ -1,8 +1,21 @@
-import { describe, expect, it } from 'vitest';
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+
+const { browserFetchMock } = vi.hoisted(() => ({
+  browserFetchMock: vi.fn(),
+}));
+
+vi.mock('./_shared/browser-fetch.js', () => ({
+  browserFetch: browserFetchMock,
+}));
+
 import { getRegistry } from '../../registry.js';
 import './hashtag.js';
 
-describe('douyin hashtag registration', () => {
+describe('douyin hashtag', () => {
+  beforeEach(() => {
+    browserFetchMock.mockReset();
+  });
+
   it('registers the hashtag command', () => {
     const registry = getRegistry();
     const cmd = [...registry.values()].find(c => c.site === 'douyin' && c.name === 'hashtag');
@@ -25,4 +38,31 @@ describe('douyin hashtag registration', () => {
     const cmd = [...registry.values()].find(c => c.site === 'douyin' && c.name === 'hashtag');
     expect(cmd?.strategy).toBe('cookie');
   });
+
+  it('parses the current hotspot recommendation shape', async () => {
+    const registry = getRegistry();
+    const command = [...registry.values()].find((cmd) => cmd.site === 'douyin' && cmd.name === 'hashtag');
+    expect(command?.func).toBeDefined();
+    if (!command?.func) throw new Error('douyin hashtag command not registered');
+
+    browserFetchMock.mockResolvedValueOnce({
+      all_sentences: [
+        {
+          word: '在公园花海里大晒一场',
+          hot_value: 12141172,
+          sentence_id: '2448416',
+        },
+      ],
+    });
+
+    const rows = await command.func({} as any, { action: 'hot', keyword: '', limit: 5 });
+
+    expect(rows).toEqual([
+      {
+        name: '在公园花海里大晒一场',
+        id: '2448416',
+        view_count: 12141172,
+      },
+    ]);
+  });
 });

package/src/clis/douyin/hashtag.ts

@@ -42,11 +42,19 @@ cli({
       const kw = kwargs.keyword as string;
       const url = `https://creator.douyin.com/aweme/v1/hotspot/recommend/?${kw ? `keyword=${encodeURIComponent(kw)}&` : ''}aid=1128`;
       const res = await browserFetch(page, 'GET', url) as {
-        hotspot_list: Array<{ sentence: string; hot_value: number }>
+        hotspot_list?: Array<{ sentence: string; hot_value: number }>;
+        all_sentences?: Array<{ sentence_id?: string; word?: string; hot_value: number }>;
       };
-      return (res.hotspot_list ?? []).slice(0, kwargs.limit as number).map(h => ({
+      const items = res.hotspot_list
+        ?? res.all_sentences?.map(h => ({
+          sentence: h.word ?? '',
+          hot_value: h.hot_value,
+          sentence_id: h.sentence_id ?? '',
+        }))
+        ?? [];
+      return items.slice(0, kwargs.limit as number).map(h => ({
         name: h.sentence,
-        id: '',
+        id: 'sentence_id' in h ? h.sentence_id : '',
         view_count: h.hot_value,
       }));
     }

package/src/clis/douyin/profile.test.ts

@@ -1,12 +1,54 @@
-import { describe, expect, it } from 'vitest';
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+
+const { browserFetchMock } = vi.hoisted(() => ({
+  browserFetchMock: vi.fn(),
+}));
+
+vi.mock('./_shared/browser-fetch.js', () => ({
+  browserFetch: browserFetchMock,
+}));
+
 import { getRegistry } from '../../registry.js';
 import './profile.js';
 
 describe('douyin profile registration', () => {
+  beforeEach(() => {
+    browserFetchMock.mockReset();
+  });
+
   it('registers the profile command', () => {
     const registry = getRegistry();
     const values = [...registry.values()];
     const cmd = values.find(c => c.site === 'douyin' && c.name === 'profile');
     expect(cmd).toBeDefined();
   });
+
+  it('maps the current user payload shape returned by creator center', async () => {
+    const registry = getRegistry();
+    const cmd = [...registry.values()].find(c => c.site === 'douyin' && c.name === 'profile');
+    expect(cmd?.func).toBeDefined();
+    if (!cmd?.func) throw new Error('douyin profile command not registered');
+
+    browserFetchMock.mockResolvedValueOnce({
+      user: {
+        uid: '100',
+        nickname: 'creator',
+        follower_count: 12,
+        following_count: 3,
+        aweme_count: 7,
+      },
+    });
+
+    const rows = await cmd.func({} as never, {});
+
+    expect(rows).toEqual([
+      {
+        uid: '100',
+        nickname: 'creator',
+        follower_count: 12,
+        following_count: 3,
+        aweme_count: 7,
+      },
+    ]);
+  });
 });

package/src/clis/douyin/profile.ts

@@ -14,7 +14,14 @@ cli({
   func: async (page: IPage, _kwargs) => {
     const url = 'https://creator.douyin.com/web/api/media/user/info/?aid=1128';
     const res = (await browserFetch(page, 'GET', url)) as {
-      user_info: {
+      user_info?: {
+        uid: string;
+        nickname: string;
+        follower_count: number;
+        following_count: number;
+        aweme_count: number;
+      };
+      user?: {
         uid: string;
         nickname: string;
         follower_count: number;
@@ -22,7 +29,7 @@ cli({
         aweme_count: number;
       };
     };
-    const u = res.user_info;
+    const u = res.user_info ?? res.user;
     if (!u) throw new CommandExecutionError('用户信息获取失败，请确认已登录 creator.douyin.com');
     return [
       {

package/src/clis/douyin/user-videos.test.ts

@@ -0,0 +1,122 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+
+const { fetchDouyinUserVideosMock, fetchDouyinCommentsMock } = vi.hoisted(() => ({
+  fetchDouyinUserVideosMock: vi.fn(),
+  fetchDouyinCommentsMock: vi.fn(),
+}));
+
+vi.mock('./_shared/public-api.js', () => ({
+  fetchDouyinUserVideos: fetchDouyinUserVideosMock,
+  fetchDouyinComments: fetchDouyinCommentsMock,
+}));
+
+import { getRegistry } from '../../registry.js';
+import { DEFAULT_COMMENT_LIMIT, MAX_USER_VIDEOS_LIMIT, normalizeCommentLimit, normalizeUserVideosLimit } from './user-videos.js';
+
+describe('douyin user-videos', () => {
+  beforeEach(() => {
+    fetchDouyinUserVideosMock.mockReset();
+    fetchDouyinCommentsMock.mockReset();
+  });
+
+  it('registers the command', () => {
+    const registry = getRegistry();
+    const values = [...registry.values()];
+    const command = values.find((cmd) => cmd.site === 'douyin' && cmd.name === 'user-videos');
+    expect(command).toBeDefined();
+  });
+
+  it('clamps limit to a safe maximum', () => {
+    expect(normalizeUserVideosLimit(100)).toBe(MAX_USER_VIDEOS_LIMIT);
+    expect(normalizeUserVideosLimit(0)).toBe(1);
+    expect(normalizeCommentLimit(99)).toBe(DEFAULT_COMMENT_LIMIT);
+  });
+
+  it('uses shared public-api helpers and applies clamped limits', async () => {
+    const registry = getRegistry();
+    const command = [...registry.values()].find((cmd) => cmd.site === 'douyin' && cmd.name === 'user-videos');
+    expect(command?.func).toBeDefined();
+    if (!command?.func) throw new Error('douyin user-videos command not registered');
+
+    fetchDouyinUserVideosMock.mockResolvedValueOnce([
+      {
+        aweme_id: '1',
+        desc: 'test video',
+        video: { duration: 1234, play_addr: { url_list: ['https://example.com/video.mp4'] } },
+        statistics: { digg_count: 9 },
+      },
+    ]);
+    fetchDouyinCommentsMock.mockResolvedValueOnce([
+      { text: 'nice', digg_count: 3, nickname: 'alice' },
+    ]);
+
+    const page = {
+      goto: vi.fn().mockResolvedValue(undefined),
+      wait: vi.fn().mockResolvedValue(undefined),
+    };
+
+    const rows = await command.func(page as any, {
+      sec_uid: 'MS4w-test',
+      limit: 100,
+      comment_limit: 99,
+      with_comments: true,
+    });
+
+    expect(fetchDouyinUserVideosMock).toHaveBeenCalledWith(page, 'MS4w-test', MAX_USER_VIDEOS_LIMIT);
+    expect(fetchDouyinCommentsMock).toHaveBeenCalledWith(page, '1', DEFAULT_COMMENT_LIMIT);
+    expect(rows).toEqual([
+      {
+        index: 1,
+        aweme_id: '1',
+        title: 'test video',
+        duration: 1,
+        digg_count: 9,
+        play_url: 'https://example.com/video.mp4',
+        top_comments: [
+          { text: 'nice', digg_count: 3, nickname: 'alice' },
+        ],
+      },
+    ]);
+  });
+
+  it('skips comment enrichment when with_comments is false', async () => {
+    const registry = getRegistry();
+    const command = [...registry.values()].find((cmd) => cmd.site === 'douyin' && cmd.name === 'user-videos');
+    expect(command?.func).toBeDefined();
+    if (!command?.func) throw new Error('douyin user-videos command not registered');
+
+    fetchDouyinUserVideosMock.mockResolvedValueOnce([
+      {
+        aweme_id: '2',
+        desc: 'plain video',
+        video: { duration: 2000, play_addr: { url_list: ['https://example.com/plain.mp4'] } },
+        statistics: { digg_count: 1 },
+      },
+    ]);
+
+    const page = {
+      goto: vi.fn().mockResolvedValue(undefined),
+      wait: vi.fn().mockResolvedValue(undefined),
+    };
+
+    const rows = await command.func(page as any, {
+      sec_uid: 'MS4w-test',
+      limit: 3,
+      with_comments: false,
+      comment_limit: 5,
+    });
+
+    expect(fetchDouyinCommentsMock).not.toHaveBeenCalled();
+    expect(rows).toEqual([
+      {
+        index: 1,
+        aweme_id: '2',
+        title: 'plain video',
+        duration: 2,
+        digg_count: 1,
+        play_url: 'https://example.com/plain.mp4',
+        top_comments: [],
+      },
+    ]);
+  });
+});

package/src/clis/douyin/user-videos.ts

@@ -0,0 +1,101 @@
+import { cli, Strategy } from '../../registry.js';
+import type { IPage } from '../../types.js';
+import { fetchDouyinComments, fetchDouyinUserVideos, type DouyinVideo } from './_shared/public-api.js';
+
+export const MAX_USER_VIDEOS_LIMIT = 20;
+export const USER_VIDEO_COMMENT_CONCURRENCY = 4;
+export const DEFAULT_COMMENT_LIMIT = 10;
+
+type EnrichedDouyinVideo = DouyinVideo & {
+  top_comments?: Array<{
+    text: string;
+    digg_count: number;
+    nickname: string;
+  }>;
+};
+
+export function normalizeUserVideosLimit(limit: unknown): number {
+  const numeric = Number(limit);
+  if (!Number.isFinite(numeric)) return MAX_USER_VIDEOS_LIMIT;
+  return Math.min(MAX_USER_VIDEOS_LIMIT, Math.max(1, Math.round(numeric)));
+}
+
+export function normalizeCommentLimit(limit: unknown): number {
+  const numeric = Number(limit);
+  if (!Number.isFinite(numeric)) return DEFAULT_COMMENT_LIMIT;
+  return Math.min(DEFAULT_COMMENT_LIMIT, Math.max(1, Math.round(numeric)));
+}
+
+async function mapInBatches<T, R>(
+  items: T[],
+  concurrency: number,
+  mapper: (item: T) => Promise<R>,
+): Promise<R[]> {
+  const results: R[] = [];
+  for (let index = 0; index < items.length; index += concurrency) {
+    const chunk = items.slice(index, index + concurrency);
+    results.push(...(await Promise.all(chunk.map(mapper))));
+  }
+  return results;
+}
+
+async function fetchTopComments(
+  page: IPage,
+  awemeId: string,
+  count: number,
+): Promise<Array<{ text: string; digg_count: number; nickname: string }>> {
+  try {
+    return await fetchDouyinComments(page, awemeId, count);
+  } catch {
+    return [];
+  }
+}
+
+cli({
+  site: 'douyin',
+  name: 'user-videos',
+  description: '获取指定用户的视频列表（含下载地址和热门评论）',
+  domain: 'www.douyin.com',
+  strategy: Strategy.COOKIE,
+  args: [
+    { name: 'sec_uid', type: 'string', required: true, positional: true, help: '用户 sec_uid（URL 末尾部分）' },
+    { name: 'limit', type: 'int', default: 20, help: '获取数量（最大 20）' },
+    { name: 'with_comments', type: 'bool', default: true, help: '包含热门评论（默认: true）' },
+    { name: 'comment_limit', type: 'int', default: 10, help: '每个视频获取多少条评论（最大 10）' },
+  ],
+  columns: ['index', 'aweme_id', 'title', 'duration', 'digg_count', 'play_url', 'top_comments'],
+  func: async (page: IPage, kwargs) => {
+    const secUid = kwargs.sec_uid as string;
+    const limit = normalizeUserVideosLimit(kwargs.limit);
+    const withComments = kwargs.with_comments !== false;
+    const commentLimit = normalizeCommentLimit(kwargs.comment_limit);
+
+    await page.goto(`https://www.douyin.com/user/${secUid}`);
+    await page.wait(3);
+
+    const awemeList = (await fetchDouyinUserVideos(page, secUid, limit)).slice(0, limit);
+    const videos: EnrichedDouyinVideo[] = withComments
+      ? await mapInBatches(
+          awemeList,
+          USER_VIDEO_COMMENT_CONCURRENCY,
+          async (video) => ({
+            ...video,
+            top_comments: await fetchTopComments(page, video.aweme_id, commentLimit),
+          }),
+        )
+      : awemeList.map((video) => ({ ...video, top_comments: [] }));
+
+    return videos.map((video, index) => {
+      const playUrl = video.video?.play_addr?.url_list?.[0] ?? '';
+      return {
+        index: index + 1,
+        aweme_id: video.aweme_id,
+        title: video.desc ?? '',
+        duration: Math.round((video.video?.duration ?? 0) / 1000),
+        digg_count: video.statistics?.digg_count ?? 0,
+        play_url: playUrl,
+        top_comments: video.top_comments ?? [],
+      };
+    });
+  },
+});

package/src/clis/douyin/videos.test.ts

@@ -1,12 +1,62 @@
-import { describe, expect, it } from 'vitest';
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+
+const { browserFetchMock } = vi.hoisted(() => ({
+  browserFetchMock: vi.fn(),
+}));
+
+vi.mock('./_shared/browser-fetch.js', () => ({
+  browserFetch: browserFetchMock,
+}));
+
 import { getRegistry } from '../../registry.js';
 import './videos.js';
 
-describe('douyin videos registration', () => {
+describe('douyin videos', () => {
+  beforeEach(() => {
+    browserFetchMock.mockReset();
+  });
+
   it('registers the videos command', () => {
     const registry = getRegistry();
     const values = [...registry.values()];
     const cmd = values.find(c => c.site === 'douyin' && c.name === 'videos');
     expect(cmd).toBeDefined();
   });
+
+  it('parses the current creator work_list api shape', async () => {
+    const registry = getRegistry();
+    const command = [...registry.values()].find((cmd) => cmd.site === 'douyin' && cmd.name === 'videos');
+    expect(command?.func).toBeDefined();
+    if (!command?.func) throw new Error('douyin videos command not registered');
+
+    browserFetchMock.mockResolvedValueOnce({
+      aweme_list: [
+        {
+          aweme_id: '7000000000000000001',
+          desc: '测试视频标题',
+          create_time: 1581571130,
+          statistics: {
+            play_count: 0,
+            digg_count: 12,
+          },
+          status: {
+            is_private: true,
+          },
+        },
+      ],
+    });
+
+    const rows = await command.func({} as any, { limit: 5, page: 1, status: 'all' });
+
+    expect(rows).toEqual([
+      {
+        aweme_id: '7000000000000000001',
+        title: '测试视频标题',
+        status: 'private',
+        play_count: 0,
+        digg_count: 12,
+        create_time: new Date(1581571130 * 1000).toLocaleString('zh-CN', { timeZone: 'Asia/Tokyo' }),
+      },
+    ]);
+  });
 });

package/src/clis/douyin/videos.ts

@@ -2,6 +2,48 @@ import { cli, Strategy } from '../../registry.js';
 import { browserFetch } from './_shared/browser-fetch.js';
 import type { IPage } from '../../types.js';
 
+interface LegacyWorkItem {
+  aweme_id: string;
+  desc: string;
+  status: number;
+  public_time: number;
+  create_time: number;
+  statistics: { play_count: number; digg_count: number };
+}
+
+interface CurrentWorkItem {
+  aweme_id: string;
+  desc?: string;
+  status?: {
+    in_reviewing?: boolean;
+    is_private?: boolean;
+    is_delete?: boolean;
+    is_prohibited?: boolean;
+  };
+  public_time?: number;
+  create_time?: number;
+  statistics?: { play_count?: number; digg_count?: number };
+}
+
+function normalizeVideoStatus(
+  status: number | {
+    in_reviewing?: boolean;
+    is_private?: boolean;
+    is_delete?: boolean;
+    is_prohibited?: boolean;
+  } | undefined,
+  publicTime: number | undefined,
+): number | string {
+  if (typeof status === 'number') return status;
+  if (!status) return publicTime && publicTime > Date.now() / 1000 ? 'scheduled' : 'published';
+  if (status.is_delete) return 'deleted';
+  if (status.is_prohibited) return 'prohibited';
+  if (status.in_reviewing) return 'reviewing';
+  if (status.is_private) return 'private';
+  if (publicTime && publicTime > Date.now() / 1000) return 'scheduled';
+  return 'published';
+}
+
 cli({
   site: 'douyin',
   name: 'videos',
@@ -19,31 +61,23 @@ cli({
     const statusNum = statusMap[kwargs.status as string] ?? 0;
     const url = `https://creator.douyin.com/janus/douyin/creator/pc/work_list?page_size=${kwargs.limit}&page_num=${kwargs.page}&status=${statusNum}`;
     const res = (await browserFetch(page, 'GET', url)) as {
-      data: {
-        work_list: Array<{
-          aweme_id: string;
-          desc: string;
-          status: number;
-          public_time: number;
-          create_time: number;
-          statistics: { play_count: number; digg_count: number };
-        }>;
-      };
+      data?: { work_list?: LegacyWorkItem[] };
+      aweme_list?: CurrentWorkItem[];
     };
-    let items = res.data?.work_list ?? [];
+    let items: Array<LegacyWorkItem | CurrentWorkItem> = res.data?.work_list ?? res.aweme_list ?? [];
 
     // The API has a bug with status=16 for scheduled, so filter client-side
     if (kwargs.status === 'scheduled') {
-      items = items.filter((v) => v.public_time > Date.now() / 1000);
+      items = items.filter((v) => (v.public_time ?? 0) > Date.now() / 1000);
     }
 
     return items.map((v) => ({
       aweme_id: v.aweme_id,
-      title: v.desc,
-      status: v.status,
+      title: v.desc ?? '',
+      status: normalizeVideoStatus(v.status, v.public_time),
       play_count: v.statistics?.play_count ?? 0,
       digg_count: v.statistics?.digg_count ?? 0,
-      create_time: new Date(v.create_time * 1000).toLocaleString('zh-CN', { timeZone: 'Asia/Tokyo' }),
+      create_time: new Date((v.create_time ?? v.public_time ?? 0) * 1000).toLocaleString('zh-CN', { timeZone: 'Asia/Tokyo' }),
     }));
   },
 });

package/src/clis/facebook/search.test.ts

@@ -0,0 +1,70 @@
+/**
+ * Regression test for issue #625.
+ * Facebook search must navigate in the pipeline before DOM extraction.
+ */
+
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import yaml from 'js-yaml';
+import { describe, expect, it, vi } from 'vitest';
+import { executePipeline } from '../../pipeline/index.js';
+import type { IPage } from '../../types.js';
+
+interface YamlCommand {
+  pipeline?: Array<Record<string, unknown>>;
+}
+
+/**
+ * Minimal browser mock for pipeline execution tests.
+ * Only methods touched by this adapter path are implemented.
+ */
+function createMockPage(): IPage {
+  return {
+    goto: vi.fn(),
+    evaluate: vi.fn().mockResolvedValue([]),
+    getCookies: vi.fn().mockResolvedValue([]),
+    snapshot: vi.fn().mockResolvedValue(''),
+    click: vi.fn(),
+    typeText: vi.fn(),
+    pressKey: vi.fn(),
+    scrollTo: vi.fn(),
+    getFormState: vi.fn().mockResolvedValue({}),
+    wait: vi.fn(),
+    tabs: vi.fn().mockResolvedValue([]),
+    closeTab: vi.fn(),
+    newTab: vi.fn(),
+    selectTab: vi.fn(),
+    networkRequests: vi.fn().mockResolvedValue([]),
+    consoleMessages: vi.fn().mockResolvedValue(''),
+    scroll: vi.fn(),
+    autoScroll: vi.fn(),
+    installInterceptor: vi.fn(),
+    getInterceptedRequests: vi.fn().mockResolvedValue([]),
+    waitForCapture: vi.fn().mockResolvedValue(undefined),
+    screenshot: vi.fn().mockResolvedValue(''),
+  };
+}
+
+describe('facebook search pipeline', () => {
+  it('navigates to search results before extracting DOM data', async () => {
+    // Load the YAML adapter directly so the regression test covers the shipped command definition.
+    const filePath = path.resolve(path.dirname(fileURLToPath(import.meta.url)), 'search.yaml');
+    const raw = fs.readFileSync(filePath, 'utf8');
+    const command = yaml.load(raw) as YamlCommand;
+    const pipeline = command.pipeline ?? [];
+    const page = createMockPage();
+
+    await executePipeline(page, pipeline, {
+      args: { query: 'AI agent', limit: 3 },
+    });
+
+    expect(page.goto).toHaveBeenNthCalledWith(1, 'https://www.facebook.com');
+    expect(page.goto).toHaveBeenNthCalledWith(2, 'https://www.facebook.com/search/top?q=AI%20agent', {
+      waitUntil: undefined,
+      settleMs: 4000,
+    });
+    expect(page.evaluate).toHaveBeenCalledTimes(1);
+    expect(String((page.evaluate as ReturnType<typeof vi.fn>).mock.calls[0]?.[0] ?? '')).not.toContain('window.location.href');
+  });
+});

package/src/clis/facebook/search.yaml

@@ -17,12 +17,13 @@ args:
 pipeline:
   - navigate: https://www.facebook.com
 
+  - navigate:
+      url: https://www.facebook.com/search/top?q=${{ args.query | urlencode }}
+      settleMs: 4000
+
   - evaluate: |
       (async () => {
-        const query = ${{ args.query | json }};
         const limit = ${{ args.limit }};
-        window.location.href = 'https://www.facebook.com/search/top?q=' + encodeURIComponent(query);
-        await new Promise(r => setTimeout(r, 4000));
         // Search results are typically in role="article" or role="listitem"
         let items = document.querySelectorAll('[role="article"]');
         if (items.length === 0) {