@jackwener/opencli 1.5.5 → 1.5.7

package/src/clis/xiaohongshu/comments.test.ts

@@ -37,25 +37,23 @@ describe('xiaohongshu comments', () => {
     const page = createPageMock({
       loginWall: false,
       results: [
-        { author: 'Alice', text: 'Great note!', likes: 10, time: '2024-01-01' },
-        { author: 'Bob', text: 'Very helpful', likes: 0, time: '2024-01-02' },
+        { author: 'Alice', text: 'Great note!', likes: 10, time: '2024-01-01', is_reply: false, reply_to: '' },
+        { author: 'Bob', text: 'Very helpful', likes: 0, time: '2024-01-02', is_reply: false, reply_to: '' },
       ],
     });
 
     const result = (await command!.func!(page, { 'note-id': '69aadbcb000000002202f131', limit: 5 })) as any[];
 
     expect((page.goto as any).mock.calls[0][0]).toContain('/explore/69aadbcb000000002202f131');
-    expect(result).toEqual([
-      { rank: 1, author: 'Alice', text: 'Great note!', likes: 10, time: '2024-01-01' },
-      { rank: 2, author: 'Bob', text: 'Very helpful', likes: 0, time: '2024-01-02' },
-    ]);
-    expect(result[0]).not.toHaveProperty('loginWall');
+    expect(result).toHaveLength(2);
+    expect(result[0]).toMatchObject({ rank: 1, author: 'Alice', text: 'Great note!', likes: 10 });
+    expect(result[1]).toMatchObject({ rank: 2, author: 'Bob', text: 'Very helpful', likes: 0 });
   });
 
   it('strips /explore/ prefix from full URL input', async () => {
     const page = createPageMock({
       loginWall: false,
-      results: [{ author: 'Alice', text: 'Nice', likes: 1, time: '2024-01-01' }],
+      results: [{ author: 'Alice', text: 'Nice', likes: 1, time: '2024-01-01', is_reply: false, reply_to: '' }],
     });
 
     await command!.func!(page, {
@@ -66,6 +64,20 @@ describe('xiaohongshu comments', () => {
     expect((page.goto as any).mock.calls[0][0]).toContain('/explore/69aadbcb000000002202f131');
   });
 
+  it('preserves full search_result URL with xsec_token for navigation', async () => {
+    const page = createPageMock({
+      loginWall: false,
+      results: [{ author: 'Alice', text: 'Nice', likes: 1, time: '2024-01-01', is_reply: false, reply_to: '' }],
+    });
+
+    const fullUrl =
+      'https://www.xiaohongshu.com/search_result/69aadbcb000000002202f131?xsec_token=abc&xsec_source=pc_search';
+
+    await command!.func!(page, { 'note-id': fullUrl, limit: 5 });
+
+    expect((page.goto as any).mock.calls[0][0]).toBe(fullUrl);
+  });
+
   it('throws AuthRequiredError when login wall is detected', async () => {
     const page = createPageMock({ loginWall: true, results: [] });
 
@@ -80,12 +92,14 @@ describe('xiaohongshu comments', () => {
     await expect(command!.func!(page, { 'note-id': 'abc123', limit: 5 })).resolves.toEqual([]);
   });
 
-  it('respects the limit', async () => {
+  it('respects the limit for top-level comments', async () => {
     const manyComments = Array.from({ length: 10 }, (_, i) => ({
       author: `User${i}`,
       text: `Comment ${i}`,
       likes: i,
       time: '2024-01-01',
+      is_reply: false,
+      reply_to: '',
     }));
     const page = createPageMock({ loginWall: false, results: manyComments });
 
@@ -94,4 +108,66 @@ describe('xiaohongshu comments', () => {
     expect(result[0].rank).toBe(1);
     expect(result[2].rank).toBe(3);
   });
+
+  it('clamps invalid negative limits to a safe minimum', async () => {
+    const page = createPageMock({
+      loginWall: false,
+      results: [
+        { author: 'Alice', text: 'Great note!', likes: 10, time: '2024-01-01', is_reply: false, reply_to: '' },
+        { author: 'Bob', text: 'Very helpful', likes: 0, time: '2024-01-02', is_reply: false, reply_to: '' },
+      ],
+    });
+
+    const result = (await command!.func!(page, { 'note-id': 'abc123', limit: -3 })) as any[];
+
+    expect(result).toHaveLength(1);
+    expect(result[0]).toMatchObject({ rank: 1, author: 'Alice' });
+  });
+
+  describe('--with-replies', () => {
+    it('includes reply rows with is_reply=true and reply_to set', async () => {
+      const page = createPageMock({
+        loginWall: false,
+        results: [
+          { author: 'Alice', text: 'Main comment', likes: 10, time: '03-25', is_reply: false, reply_to: '' },
+          { author: 'Bob', text: 'Reply to Alice', likes: 3, time: '03-25', is_reply: true, reply_to: 'Alice' },
+          { author: 'Carol', text: 'Another top', likes: 5, time: '03-26', is_reply: false, reply_to: '' },
+        ],
+      });
+
+      const result = (await command!.func!(page, {
+        'note-id': 'abc123', limit: 50, 'with-replies': true,
+      })) as any[];
+
+      expect(result).toHaveLength(3);
+      expect(result[0]).toMatchObject({ author: 'Alice', is_reply: false, reply_to: '' });
+      expect(result[1]).toMatchObject({ author: 'Bob', is_reply: true, reply_to: 'Alice' });
+      expect(result[2]).toMatchObject({ author: 'Carol', is_reply: false, reply_to: '' });
+
+      const script = (page.evaluate as any).mock.calls[0][0];
+      expect(script).toContain('共\\d+条回复');
+      expect(script).toContain('el.click()');
+    });
+
+    it('limits by top-level count, keeping attached replies', async () => {
+      const page = createPageMock({
+        loginWall: false,
+        results: [
+          { author: 'A', text: 'Top 1', likes: 0, time: '', is_reply: false, reply_to: '' },
+          { author: 'A1', text: 'Reply 1', likes: 0, time: '', is_reply: true, reply_to: 'A' },
+          { author: 'A2', text: 'Reply 2', likes: 0, time: '', is_reply: true, reply_to: 'A' },
+          { author: 'B', text: 'Top 2', likes: 0, time: '', is_reply: false, reply_to: '' },
+          { author: 'C', text: 'Top 3', likes: 0, time: '', is_reply: false, reply_to: '' },
+        ],
+      });
+
+      // Limit to 2 top-level comments — should include A + 2 replies + B = 4 rows
+      const result = (await command!.func!(page, {
+        'note-id': 'abc123', limit: 2, 'with-replies': true,
+      })) as any[];
+
+      expect(result).toHaveLength(4);
+      expect(result.map((r: any) => r.author)).toEqual(['A', 'A1', 'A2', 'B']);
+    });
+  });
 });

package/src/clis/xiaohongshu/comments.ts

@@ -1,36 +1,46 @@
 /**
  * Xiaohongshu comments — DOM extraction from note detail page.
  * XHS API requires signed requests, so we scrape the rendered DOM instead.
+ *
+ * Supports both top-level comments and nested replies (楼中楼) via
+ * the --with-replies flag.
  */
 
 import { cli, Strategy } from '../../registry.js';
 import { AuthRequiredError, EmptyResultError } from '../../errors.js';
+import { parseNoteId, buildNoteUrl } from './note-helpers.js';
+
+function parseCommentLimit(raw: unknown, fallback = 20): number {
+  const n = Number(raw);
+  if (!Number.isFinite(n)) return fallback;
+  return Math.max(1, Math.min(Math.floor(n), 50));
+}
 
 cli({
   site: 'xiaohongshu',
   name: 'comments',
-  description: '获取小红书笔记评论（仅主评论，不含楼中楼）',
+  description: '获取小红书笔记评论（支持楼中楼子回复）',
   domain: 'www.xiaohongshu.com',
   strategy: Strategy.COOKIE,
   args: [
-    { name: 'note-id', required: true, positional: true, help: 'Note ID or full /explore/<id> URL' },
-    { name: 'limit', type: 'int', default: 20, help: 'Number of comments (max 50)' },
+    { name: 'note-id', required: true, positional: true, help: 'Note ID or full URL (preserves xsec_token for access)' },
+    { name: 'limit', type: 'int', default: 20, help: 'Number of top-level comments (max 50)' },
+    { name: 'with-replies', type: 'boolean', default: false, help: 'Include nested replies (楼中楼)' },
   ],
-  columns: ['rank', 'author', 'text', 'likes', 'time'],
+  columns: ['rank', 'author', 'text', 'likes', 'time', 'is_reply', 'reply_to'],
   func: async (page, kwargs) => {
-    const limit = Math.min(Number(kwargs.limit) || 20, 50);
-    let noteId = String(kwargs['note-id']).trim();
-
-    // Accept full URLs: /explore/<id> or /note/<id>
-    const urlMatch = noteId.match(/\/explore\/([a-f0-9]+)/) || noteId.match(/\/note\/([a-f0-9]+)/);
-    if (urlMatch) noteId = urlMatch[1];
+    const limit = parseCommentLimit(kwargs.limit);
+    const withReplies = Boolean(kwargs['with-replies']);
+    const raw = String(kwargs['note-id']);
+    const noteId = parseNoteId(raw);
 
-    await page.goto(`https://www.xiaohongshu.com/explore/${noteId}`);
+    await page.goto(buildNoteUrl(raw));
     await page.wait(3);
 
     const data = await page.evaluate(`
       (async () => {
         const wait = (ms) => new Promise(r => setTimeout(r, ms))
+        const withReplies = ${withReplies}
 
         // Check login state
         const loginWall = /登录后查看|请登录/.test(document.body.innerText || '')
@@ -45,6 +55,31 @@ cli({
         }
 
         const clean = (el) => (el?.textContent || '').replace(/\\s+/g, ' ').trim()
+        const parseLikes = (el) => {
+          const raw = clean(el)
+          return /^\\d+$/.test(raw) ? Number(raw) : 0
+        }
+        const expandReplyThreads = async (root) => {
+          if (!withReplies || !root) return
+          const clickedTexts = new Set()
+          for (let round = 0; round < 3; round++) {
+            const expanders = Array.from(root.querySelectorAll('button, [role="button"], span, div')).filter(el => {
+              if (!(el instanceof HTMLElement)) return false
+              const text = clean(el)
+              if (!text || text.length > 24) return false
+              if (!/(展开|更多回复|全部回复|查看.*回复|共\\d+条回复)/.test(text)) return false
+              if (clickedTexts.has(text)) return false
+              return true
+            })
+            if (!expanders.length) break
+            for (const el of expanders) {
+              const text = clean(el)
+              el.click()
+              clickedTexts.add(text)
+              await wait(300)
+            }
+          }
+        }
 
         const results = []
         const parents = document.querySelectorAll('.parent-comment')
@@ -54,13 +89,24 @@ cli({
 
           const author = clean(item.querySelector('.author-wrapper .name, .user-name'))
           const text = clean(item.querySelector('.content, .note-text'))
-          // XHS shows text "赞" when likes = 0; only shows a number when > 0
-          const likesRaw = clean(item.querySelector('.count'))
-          const likes = /^\\d+$/.test(likesRaw) ? Number(likesRaw) : 0
+          const likes = parseLikes(item.querySelector('.count'))
           const time = clean(item.querySelector('.date, .time'))
 
           if (!text) continue
-          results.push({ author, text, likes, time })
+          results.push({ author, text, likes, time, is_reply: false, reply_to: '' })
+
+          // Extract nested replies (楼中楼)
+          if (withReplies) {
+            await expandReplyThreads(p)
+            p.querySelectorAll('.reply-container .comment-item-sub, .sub-comment-list .comment-item').forEach(sub => {
+              const sAuthor = clean(sub.querySelector('.name, .user-name'))
+              const sText = clean(sub.querySelector('.content, .note-text'))
+              const sLikes = parseLikes(sub.querySelector('.count'))
+              const sTime = clean(sub.querySelector('.date, .time'))
+              if (!sText) return
+              results.push({ author: sAuthor, text: sText, likes: sLikes, time: sTime, is_reply: true, reply_to: author })
+            })
+          }
         }
 
         return { loginWall, results }
@@ -75,7 +121,20 @@ cli({
       throw new AuthRequiredError('www.xiaohongshu.com', 'Note comments require login');
     }
 
-    const results: any[] = (data as any).results ?? [];
-    return results.slice(0, limit).map((c: any, i: number) => ({ rank: i + 1, ...c }));
+    const all: any[] = (data as any).results ?? [];
+
+    // When limiting, count only top-level comments; their replies are included for free
+    if (withReplies) {
+      const limited: any[] = [];
+      let topCount = 0;
+      for (const c of all) {
+        if (!c.is_reply) topCount++;
+        if (topCount > limit) break;
+        limited.push(c);
+      }
+      return limited.map((c: any, i: number) => ({ rank: i + 1, ...c }));
+    }
+
+    return all.slice(0, limit).map((c: any, i: number) => ({ rank: i + 1, ...c }));
   },
 });

package/src/clis/xiaohongshu/download.test.ts

@@ -0,0 +1,96 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+import type { IPage } from '../../types.js';
+
+const { mockDownloadMedia, mockFormatCookieHeader } = vi.hoisted(() => ({
+  mockDownloadMedia: vi.fn(),
+  mockFormatCookieHeader: vi.fn(() => 'a=b'),
+}));
+
+vi.mock('../../download/media-download.js', () => ({
+  downloadMedia: mockDownloadMedia,
+}));
+
+vi.mock('../../download/index.js', () => ({
+  formatCookieHeader: mockFormatCookieHeader,
+}));
+
+import { getRegistry } from '../../registry.js';
+import './download.js';
+
+function createPageMock(evaluateResult: any): IPage {
+  return {
+    goto: vi.fn().mockResolvedValue(undefined),
+    evaluate: vi.fn().mockResolvedValue(evaluateResult),
+    snapshot: vi.fn().mockResolvedValue(undefined),
+    click: vi.fn().mockResolvedValue(undefined),
+    typeText: vi.fn().mockResolvedValue(undefined),
+    pressKey: vi.fn().mockResolvedValue(undefined),
+    scrollTo: vi.fn().mockResolvedValue(undefined),
+    getFormState: vi.fn().mockResolvedValue({ forms: [], orphanFields: [] }),
+    wait: vi.fn().mockResolvedValue(undefined),
+    tabs: vi.fn().mockResolvedValue([]),
+    closeTab: vi.fn().mockResolvedValue(undefined),
+    newTab: vi.fn().mockResolvedValue(undefined),
+    selectTab: vi.fn().mockResolvedValue(undefined),
+    networkRequests: vi.fn().mockResolvedValue([]),
+    consoleMessages: vi.fn().mockResolvedValue([]),
+    scroll: vi.fn().mockResolvedValue(undefined),
+    autoScroll: vi.fn().mockResolvedValue(undefined),
+    installInterceptor: vi.fn().mockResolvedValue(undefined),
+    getInterceptedRequests: vi.fn().mockResolvedValue([]),
+    getCookies: vi.fn().mockResolvedValue([{ name: 'sid', value: 'secret', domain: '.xiaohongshu.com' }]),
+    screenshot: vi.fn().mockResolvedValue(''),
+    waitForCapture: vi.fn().mockResolvedValue(undefined),
+  };
+}
+
+describe('xiaohongshu download', () => {
+  const command = getRegistry().get('xiaohongshu/download');
+
+  beforeEach(() => {
+    mockDownloadMedia.mockReset();
+    mockFormatCookieHeader.mockClear();
+    mockDownloadMedia.mockResolvedValue([{ index: 1, type: 'video', status: 'success', size: '1 MB' }]);
+  });
+
+  it('preserves short links for navigation but uses canonical note id for output naming', async () => {
+    const page = createPageMock({
+      noteId: '69bc166f000000001a02069a',
+      media: [{ type: 'video', url: 'https://sns-video-hw.xhscdn.com/example.mp4' }],
+    });
+
+    const shortUrl = 'http://xhslink.com/o/4MKEjsZnhCz';
+    await command!.func!(page, { 'note-id': shortUrl, output: './out' });
+
+    expect((page.goto as any).mock.calls[0][0]).toBe(shortUrl);
+    expect(mockDownloadMedia).toHaveBeenCalledWith(
+      [{ type: 'video', url: 'https://sns-video-hw.xhscdn.com/example.mp4' }],
+      expect.objectContaining({
+        output: './out',
+        subdir: '69bc166f000000001a02069a',
+        filenamePrefix: '69bc166f000000001a02069a',
+        cookies: 'a=b',
+      }),
+    );
+  });
+
+  it('preserves full note URL with xsec_token for navigation', async () => {
+    const page = createPageMock({
+      noteId: '69bc166f000000001a02069a',
+      media: [{ type: 'image', url: 'https://ci.xiaohongshu.com/example.jpg' }],
+    });
+
+    const fullUrl =
+      'https://www.xiaohongshu.com/explore/69bc166f000000001a02069a?xsec_token=abc&xsec_source=pc_search';
+    await command!.func!(page, { 'note-id': fullUrl, output: './out' });
+
+    expect((page.goto as any).mock.calls[0][0]).toBe(fullUrl);
+    expect(mockDownloadMedia).toHaveBeenCalledWith(
+      [{ type: 'image', url: 'https://ci.xiaohongshu.com/example.jpg' }],
+      expect.objectContaining({
+        subdir: '69bc166f000000001a02069a',
+        filenamePrefix: '69bc166f000000001a02069a',
+      }),
+    );
+  });
+});

package/src/clis/xiaohongshu/download.ts

@@ -2,12 +2,16 @@
  * Xiaohongshu download — download images and videos from a note.
  *
  * Usage:
- *   opencli xiaohongshu download --note_id abc123 --output ./xhs
+ *   opencli xiaohongshu download <note-id-or-url> --output ./xhs
+ *
+ * Accepts a bare note ID, a full xiaohongshu.com URL (with xsec_token),
+ * or a short link (http://xhslink.com/...).
  */
 
 import { cli, Strategy } from '../../registry.js';
 import { formatCookieHeader } from '../../download/index.js';
 import { downloadMedia } from '../../download/media-download.js';
+import { buildNoteUrl, parseNoteId } from './note-helpers.js';
 
 cli({
   site: 'xiaohongshu',
@@ -16,16 +20,16 @@ cli({
   domain: 'www.xiaohongshu.com',
   strategy: Strategy.COOKIE,
   args: [
-    { name: 'note-id', positional: true, required: true, help: 'Note ID (from URL)' },
+    { name: 'note-id', positional: true, required: true, help: 'Note ID, full URL, or short link' },
     { name: 'output', default: './xiaohongshu-downloads', help: 'Output directory' },
   ],
   columns: ['index', 'type', 'status', 'size'],
   func: async (page, kwargs) => {
-    const noteId = kwargs['note-id'];
+    const rawInput = String(kwargs['note-id']);
     const output = kwargs.output;
+    const noteId = parseNoteId(rawInput);
 
-    // Navigate to note page
-    await page.goto(`https://www.xiaohongshu.com/explore/${noteId}`);
+    await page.goto(buildNoteUrl(rawInput));
 
     // Extract note info and media URLs
     const data = await page.evaluate(`
@@ -36,6 +40,18 @@ cli({
           author: '',
           media: []
         };
+        const seenMedia = new Set();
+        const pushMedia = (type, url) => {
+          if (!url) return;
+          const key = type + ':' + url;
+          if (seenMedia.has(key)) return;
+          seenMedia.add(key);
+          result.media.push({ type, url });
+        };
+        const locationMatch = (location.pathname || '').match(/\\/(?:explore|note|search_result|discovery\\/item)\\/([a-f0-9]+)/i);
+        if (locationMatch) {
+          result.noteId = locationMatch[1];
+        }
 
         // Get title
         const titleEl = document.querySelector('.title, #detail-title, .note-content .title');
@@ -61,7 +77,6 @@ cli({
           document.querySelectorAll(selector).forEach(img => {
             let src = img.src || img.getAttribute('data-src') || '';
             if (src && (src.includes('xhscdn') || src.includes('xiaohongshu'))) {
-              // Convert to high quality URL (remove resize parameters)
               src = src.split('?')[0];
               src = src.replace(/\\/imageView\\d+\\/\\d+\\/w\\/\\d+/, '');
               imageUrls.add(src);
@@ -69,26 +84,69 @@ cli({
           });
         }
 
-        // Get video if exists
-        const videoSelectors = [
-          'video source',
-          'video[src]',
-          '.player video',
-          '.video-player video'
-        ];
+        // Get video — prefer real URL from page state over blob: URLs
 
-        for (const selector of videoSelectors) {
-          document.querySelectorAll(selector).forEach(v => {
-            const src = v.src || v.getAttribute('src') || '';
-            if (src) {
-              result.media.push({ type: 'video', url: src });
+        // Method 1: Extract from __INITIAL_STATE__ (SSR hydration data)
+        try {
+          const state = window.__INITIAL_STATE__;
+          if (state) {
+            const noteData = state.note?.noteDetailMap || state.note?.note || {};
+            for (const key of Object.keys(noteData)) {
+              const note = noteData[key]?.note || noteData[key];
+              const video = note?.video;
+              if (video) {
+                const vUrl = video.url || video.originVideoKey || video.consumer?.originVideoKey;
+                if (vUrl) {
+                  const fullUrl = vUrl.startsWith('http') ? vUrl : 'https://sns-video-bd.xhscdn.com/' + vUrl;
+                  pushMedia('video', fullUrl);
+                }
+                const streams = video.media?.stream?.h264 || [];
+                for (const stream of streams) {
+                  if (stream.masterUrl) pushMedia('video', stream.masterUrl);
+                }
+              }
             }
-          });
+          }
+        } catch(e) {}
+
+        // Method 2: Extract video URLs from inline script JSON
+        if (result.media.filter(m => m.type === 'video').length === 0) {
+          try {
+            const scripts = document.querySelectorAll('script');
+            for (const s of scripts) {
+              const text = s.textContent || '';
+              const videoMatches = text.match(/https?:\\/\\/sns-video[^"'\\s]+\\.mp4[^"'\\s]*/g)
+                || text.match(/https?:\\/\\/[^"'\\s]*xhscdn[^"'\\s]*\\.mp4[^"'\\s]*/g);
+              if (videoMatches) {
+                videoMatches.forEach(url => {
+                  pushMedia('video', url.replace(/\\\\u002F/g, '/'));
+                });
+              }
+            }
+          } catch(e) {}
+        }
+
+        // Method 3: Fallback to DOM video elements, skip blob: URLs
+        if (result.media.filter(m => m.type === 'video').length === 0) {
+          const videoSelectors = [
+            'video source',
+            'video[src]',
+            '.player video',
+            '.video-player video'
+          ];
+          for (const selector of videoSelectors) {
+            document.querySelectorAll(selector).forEach(v => {
+              const src = v.src || v.getAttribute('src') || '';
+              if (src && !src.startsWith('blob:')) {
+                pushMedia('video', src);
+              }
+            });
+          }
         }
 
         // Add images to media
         imageUrls.forEach(url => {
-          result.media.push({ type: 'image', url: url });
+          pushMedia('image', url);
         });
 
         return result;
@@ -101,12 +159,15 @@ cli({
 
     // Extract cookies for authenticated downloads
     const cookies = formatCookieHeader(await page.getCookies({ domain: 'xiaohongshu.com' }));
+    const resolvedNoteId = typeof data.noteId === 'string' && data.noteId.trim()
+      ? data.noteId.trim()
+      : noteId;
 
     return downloadMedia(data.media, {
       output,
-      subdir: noteId,
+      subdir: resolvedNoteId,
       cookies,
-      filenamePrefix: noteId,
+      filenamePrefix: resolvedNoteId,
       timeout: 60000,
     });
   },

package/src/clis/xiaohongshu/note-helpers.ts

@@ -0,0 +1,25 @@
+/** Side-effect-free helpers shared by xiaohongshu note and comments commands. */
+
+/** Extract a bare note ID from a full URL or raw ID string. */
+export function parseNoteId(input: string): string {
+  const trimmed = input.trim();
+  const match = trimmed.match(/\/(?:explore|note|search_result)\/([a-f0-9]+)/);
+  return match ? match[1] : trimmed;
+}
+
+/**
+ * Build the best navigation URL for a note.
+ *
+ * XHS blocks direct `/explore/<id>` access without a valid `xsec_token`.
+ * When the user passes a full URL (from search results), we preserve it
+ * so the browser navigates with the token intact. For bare IDs we fall
+ * back to the `/explore/<id>` path (works when cookies carry enough context).
+ */
+export function buildNoteUrl(input: string): string {
+  const trimmed = input.trim();
+  if (/^https?:\/\//.test(trimmed)) {
+    // Full URL — navigate as-is; the browser will follow any redirects
+    return trimmed;
+  }
+  return `https://www.xiaohongshu.com/explore/${trimmed}`;
+}