@jackwener/opencli 0.9.6 → 1.0.0

package/src/clis/xiaohongshu/download.ts

@@ -0,0 +1,173 @@
+/**
+ * Xiaohongshu download — download images and videos from a note.
+ *
+ * Usage:
+ *   opencli xiaohongshu download --note-id abc123 --output ./xhs
+ */
+
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import { cli, Strategy } from '../../registry.js';
+import {
+  httpDownload,
+  sanitizeFilename,
+  detectContentType,
+} from '../../download/index.js';
+import { DownloadProgressTracker, formatBytes } from '../../download/progress.js';
+
+cli({
+  site: 'xiaohongshu',
+  name: 'download',
+  description: '下载小红书笔记中的图片和视频',
+  domain: 'www.xiaohongshu.com',
+  strategy: Strategy.COOKIE,
+  args: [
+    { name: 'note_id', required: true, help: 'Note ID (from URL)' },
+    { name: 'output', default: './xiaohongshu-downloads', help: 'Output directory' },
+  ],
+  columns: ['index', 'type', 'status', 'size'],
+  func: async (page, kwargs) => {
+    const noteId = kwargs.note_id;
+    const output = kwargs.output;
+
+    // Navigate to note page
+    await page.goto(`https://www.xiaohongshu.com/explore/${noteId}`);
+    await page.wait(3);
+
+    // Extract note info and media URLs
+    const data = await page.evaluate(`
+      (() => {
+        const result = {
+          noteId: '${noteId}',
+          title: '',
+          author: '',
+          media: []
+        };
+
+        // Get title
+        const titleEl = document.querySelector('.title, #detail-title, .note-content .title');
+        result.title = titleEl?.textContent?.trim() || 'untitled';
+
+        // Get author
+        const authorEl = document.querySelector('.username, .author-name, .name');
+        result.author = authorEl?.textContent?.trim() || 'unknown';
+
+        // Get images - try multiple selectors
+        const imageSelectors = [
+          '.swiper-slide img',
+          '.carousel-image img',
+          '.note-slider img',
+          '.note-image img',
+          '.image-wrapper img',
+          '#noteContainer img[src*="xhscdn"]',
+          'img[src*="ci.xiaohongshu.com"]'
+        ];
+
+        const imageUrls = new Set();
+        for (const selector of imageSelectors) {
+          document.querySelectorAll(selector).forEach(img => {
+            let src = img.src || img.getAttribute('data-src') || '';
+            if (src && (src.includes('xhscdn') || src.includes('xiaohongshu'))) {
+              // Convert to high quality URL (remove resize parameters)
+              src = src.split('?')[0];
+              // Try to get original size
+              src = src.replace(/\\/imageView\\d+\\/\\d+\\/w\\/\\d+/, '');
+              imageUrls.add(src);
+            }
+          });
+        }
+
+        // Get video if exists
+        const videoSelectors = [
+          'video source',
+          'video[src]',
+          '.player video',
+          '.video-player video'
+        ];
+
+        for (const selector of videoSelectors) {
+          document.querySelectorAll(selector).forEach(v => {
+            const src = v.src || v.getAttribute('src') || '';
+            if (src) {
+              result.media.push({
+                type: 'video',
+                url: src
+              });
+            }
+          });
+        }
+
+        // Add images to media
+        imageUrls.forEach(url => {
+          result.media.push({
+            type: 'image',
+            url: url
+          });
+        });
+
+        return result;
+      })()
+    `);
+
+    if (!data || !data.media || data.media.length === 0) {
+      return [{ index: 0, type: '-', status: 'failed', size: 'No media found' }];
+    }
+
+    // Extract cookies for authenticated downloads
+    const cookies = await page.evaluate(`(() => document.cookie)()`);
+
+    // Create output directory
+    const outputDir = path.join(output, noteId);
+    fs.mkdirSync(outputDir, { recursive: true });
+
+    // Download all media files
+    const tracker = new DownloadProgressTracker(data.media.length, true);
+    const results: any[] = [];
+
+    for (let i = 0; i < data.media.length; i++) {
+      const media = data.media[i];
+      const ext = media.type === 'video' ? 'mp4' : 'jpg';
+      const filename = `${noteId}_${i + 1}.${ext}`;
+      const destPath = path.join(outputDir, filename);
+
+      const progressBar = tracker.onFileStart(filename, i);
+
+      try {
+        const result = await httpDownload(media.url, destPath, {
+          cookies: typeof cookies === 'string' ? cookies : '',
+          timeout: 60000,
+          onProgress: (received, total) => {
+            if (progressBar) progressBar.update(received, total);
+          },
+        });
+
+        if (progressBar) {
+          progressBar.complete(result.success, result.success ? formatBytes(result.size) : undefined);
+        }
+
+        tracker.onFileComplete(result.success);
+
+        results.push({
+          index: i + 1,
+          type: media.type,
+          status: result.success ? 'success' : 'failed',
+          size: result.success ? formatBytes(result.size) : (result.error || 'unknown error'),
+        });
+      } catch (err: any) {
+        if (progressBar) progressBar.fail(err.message);
+        tracker.onFileComplete(false);
+
+        results.push({
+          index: i + 1,
+          type: media.type,
+          status: 'failed',
+          size: err.message,
+        });
+      }
+    }
+
+    tracker.finish();
+
+    return results;
+  },
+});

package/src/clis/xiaohongshu/search.ts

@@ -43,7 +43,7 @@ cli({
           const linkEl = el.querySelector('a[href*="/explore/"], a[href*="/search_result/"], a[href*="/note/"]');
 
           const href = linkEl?.getAttribute('href') || '';
-          const noteId = href.match(/\\/(?:explore|note)\\/([a-f0-9]+)/)?.[1] || '';
+          const noteId = href.match(/\\/(?:explore|note)\\/([a-zA-Z0-9]+)/)?.[1] || '';
 
           results.push({
             title: (titleEl?.textContent || '').trim(),

package/src/clis/xiaohongshu/user-helpers.test.ts

@@ -0,0 +1,106 @@
+import { describe, expect, it } from 'vitest';
+import {
+  buildXhsNoteUrl,
+  extractXhsUserNotes,
+  flattenXhsNoteGroups,
+  normalizeXhsUserId,
+} from './user-helpers.js';
+
+describe('normalizeXhsUserId', () => {
+  it('extracts the profile id from a full Xiaohongshu URL', () => {
+    expect(
+      normalizeXhsUserId(
+        'https://www.xiaohongshu.com/user/profile/615529370000000002026001?xsec_source=pc_search'
+      )
+    ).toBe('615529370000000002026001');
+  });
+
+  it('keeps a bare profile id unchanged', () => {
+    expect(normalizeXhsUserId('615529370000000002026001')).toBe('615529370000000002026001');
+  });
+});
+
+describe('flattenXhsNoteGroups', () => {
+  it('flattens grouped note arrays and ignores empty groups', () => {
+    expect(flattenXhsNoteGroups([[{ id: 'a' }], [], null, [{ id: 'b' }]])).toEqual([
+      { id: 'a' },
+      { id: 'b' },
+    ]);
+  });
+});
+
+describe('buildXhsNoteUrl', () => {
+  it('includes xsec token when available', () => {
+    expect(buildXhsNoteUrl('user123', 'note456', 'token789')).toBe(
+      'https://www.xiaohongshu.com/user/profile/user123/note456?xsec_token=token789&xsec_source=pc_user'
+    );
+  });
+});
+
+describe('extractXhsUserNotes', () => {
+  it('normalizes grouped note cards into CLI rows', () => {
+    const rows = extractXhsUserNotes(
+      {
+        noteGroups: [
+          [
+            {
+              id: 'note-1',
+              xsecToken: 'abc',
+              noteCard: {
+                noteId: 'note-1',
+                displayTitle: 'First note',
+                type: 'video',
+                interactInfo: { likedCount: '4.6万' },
+                user: { userId: 'user-1' },
+              },
+            },
+            {
+              noteCard: {
+                note_id: 'note-2',
+                display_title: 'Second note',
+                type: 'normal',
+                interact_info: { liked_count: 42 },
+              },
+            },
+          ],
+          [],
+        ],
+      },
+      'fallback-user'
+    );
+
+    expect(rows).toEqual([
+      {
+        id: 'note-1',
+        title: 'First note',
+        type: 'video',
+        likes: '4.6万',
+        url: 'https://www.xiaohongshu.com/user/profile/user-1/note-1?xsec_token=abc&xsec_source=pc_user',
+      },
+      {
+        id: 'note-2',
+        title: 'Second note',
+        type: 'normal',
+        likes: '42',
+        url: 'https://www.xiaohongshu.com/user/profile/fallback-user/note-2',
+      },
+    ]);
+  });
+
+  it('deduplicates repeated notes by note id', () => {
+    const rows = extractXhsUserNotes(
+      {
+        noteGroups: [
+          [
+            { noteCard: { noteId: 'dup-1', displayTitle: 'keep me' } },
+            { noteCard: { noteId: 'dup-1', displayTitle: 'drop me' } },
+          ],
+        ],
+      },
+      'fallback-user'
+    );
+
+    expect(rows).toHaveLength(1);
+    expect(rows[0]?.title).toBe('keep me');
+  });
+});

package/src/clis/xiaohongshu/user-helpers.ts

@@ -0,0 +1,85 @@
+export interface XhsUserPageSnapshot {
+  noteGroups?: unknown;
+  pageData?: unknown;
+}
+
+export interface XhsUserNoteRow {
+  id: string;
+  title: string;
+  type: string;
+  likes: string;
+  url: string;
+}
+
+function toCleanString(value: unknown): string {
+  return typeof value === 'string' ? value.trim() : value == null ? '' : String(value).trim();
+}
+
+export function normalizeXhsUserId(input: string): string {
+  const trimmed = toCleanString(input);
+  const withoutQuery = trimmed.replace(/[?#].*$/, '');
+  const matched = withoutQuery.match(/\/user\/profile\/([a-zA-Z0-9]+)/);
+  if (matched?.[1]) return matched[1];
+  return withoutQuery.replace(/\/+$/, '').split('/').pop() ?? withoutQuery;
+}
+
+export function flattenXhsNoteGroups(noteGroups: unknown): any[] {
+  if (!Array.isArray(noteGroups)) return [];
+
+  const notes: any[] = [];
+  for (const group of noteGroups) {
+    if (!group) continue;
+    if (Array.isArray(group)) {
+      for (const item of group) {
+        if (item) notes.push(item);
+      }
+      continue;
+    }
+    notes.push(group);
+  }
+
+  return notes;
+}
+
+export function buildXhsNoteUrl(userId: string, noteId: string, xsecToken?: string): string {
+  const cleanUserId = toCleanString(userId);
+  const cleanNoteId = toCleanString(noteId);
+  if (!cleanUserId || !cleanNoteId) return '';
+
+  const url = new URL(`https://www.xiaohongshu.com/user/profile/${cleanUserId}/${cleanNoteId}`);
+  const cleanToken = toCleanString(xsecToken);
+  if (cleanToken) {
+    url.searchParams.set('xsec_token', cleanToken);
+    url.searchParams.set('xsec_source', 'pc_user');
+  }
+  return url.toString();
+}
+
+export function extractXhsUserNotes(snapshot: XhsUserPageSnapshot, fallbackUserId: string): XhsUserNoteRow[] {
+  const notes = flattenXhsNoteGroups(snapshot.noteGroups);
+  const rows: XhsUserNoteRow[] = [];
+  const seen = new Set<string>();
+
+  for (const entry of notes) {
+    const noteCard = entry?.noteCard ?? entry?.note_card ?? entry;
+    if (!noteCard || typeof noteCard !== 'object') continue;
+
+    const noteId = toCleanString(noteCard.noteId ?? noteCard.note_id ?? entry?.noteId ?? entry?.note_id ?? entry?.id);
+    if (!noteId || seen.has(noteId)) continue;
+    seen.add(noteId);
+
+    const userId = toCleanString(noteCard.user?.userId ?? noteCard.user?.user_id ?? fallbackUserId);
+    const xsecToken = toCleanString(entry?.xsecToken ?? entry?.xsec_token ?? noteCard.xsecToken ?? noteCard.xsec_token);
+    const likes = toCleanString(noteCard.interactInfo?.likedCount ?? noteCard.interact_info?.liked_count ?? 0) || '0';
+
+    rows.push({
+      id: noteId,
+      title: toCleanString(noteCard.displayTitle ?? noteCard.display_title ?? noteCard.title),
+      type: toCleanString(noteCard.type),
+      likes,
+      url: buildXhsNoteUrl(userId || fallbackUserId, noteId, xsecToken),
+    });
+  }
+
+  return rows;
+}

package/src/clis/xiaohongshu/user.ts

@@ -1,45 +1,65 @@
 import { cli, Strategy } from '../../registry.js';
+import { extractXhsUserNotes, normalizeXhsUserId } from './user-helpers.js';
+
+async function readUserSnapshot(page: any) {
+  return await page.evaluate(`
+    (() => {
+      const safeClone = (value) => {
+        try {
+          return JSON.parse(JSON.stringify(value ?? null));
+        } catch {
+          return null;
+        }
+      };
+
+      const userStore = window.__INITIAL_STATE__?.user || {};
+      return {
+        noteGroups: safeClone(userStore.notes?._value || userStore.notes || []),
+        pageData: safeClone(userStore.userPageData?._value || userStore.userPageData || {}),
+      };
+    })()
+  `);
+}
 
 cli({
   site: 'xiaohongshu',
   name: 'user',
-  description: 'Get user notes from Xiaohongshu',
-  domain: 'xiaohongshu.com',
-  strategy: Strategy.INTERCEPT,
+  description: 'Get public notes from a Xiaohongshu user profile',
+  domain: 'www.xiaohongshu.com',
+  strategy: Strategy.COOKIE,
   browser: true,
   args: [
-    { name: 'id', type: 'string', required: true },
-    { name: 'limit', type: 'int', default: 15 },
+    { name: 'id', type: 'string', required: true, help: 'User id or profile URL' },
+    { name: 'limit', type: 'int', default: 15, help: 'Number of notes to return' },
   ],
   columns: ['id', 'title', 'type', 'likes', 'url'],
   func: async (page, kwargs) => {
-    await page.goto(`https://www.xiaohongshu.com/user/profile/${kwargs.id}`);
-    await page.wait(5);
-
-    await page.installInterceptor('v1/user/posted');
-
-    // Trigger API by scrolling
-    await page.autoScroll({ times: 2, delayMs: 2000 });
-    
-    // Retrieve data
-    const requests = await page.getInterceptedRequests();
-    if (!requests || requests.length === 0) return [];
-
-    let results: any[] = [];
-    for (const req of requests) {
-      if (req.data && req.data.data && req.data.data.notes) {
-         for (const note of req.data.data.notes) {
-           results.push({
-             id: note.note_id || note.id,
-             title: note.display_title || '',
-             type: note.type || '',
-             likes: note.interact_info?.liked_count || '0',
-             url: `https://www.xiaohongshu.com/explore/${note.note_id || note.id}`
-           });
-         }
-      }
+    const userId = normalizeXhsUserId(String(kwargs.id));
+    const limit = Math.max(1, Number(kwargs.limit ?? 15));
+
+    await page.goto(`https://www.xiaohongshu.com/user/profile/${userId}`);
+    await page.wait(3);
+
+    let snapshot = await readUserSnapshot(page);
+    let results = extractXhsUserNotes(snapshot ?? {}, userId);
+    let previousCount = results.length;
+
+    for (let i = 0; results.length < limit && i < 4; i += 1) {
+      await page.autoScroll({ times: 1, delayMs: 1500 });
+      await page.wait(1);
+
+      snapshot = await readUserSnapshot(page);
+      const nextResults = extractXhsUserNotes(snapshot ?? {}, userId);
+      if (nextResults.length <= previousCount) break;
+
+      results = nextResults;
+      previousCount = nextResults.length;
+    }
+
+    if (results.length === 0) {
+      throw new Error('No public notes found for this Xiaohongshu user.');
     }
 
-    return results.slice(0, kwargs.limit);
-  }
+    return results.slice(0, limit);
+  },
 });

package/src/clis/zhihu/download.test.ts

@@ -0,0 +1,12 @@
+import { describe, expect, it } from 'vitest';
+import { htmlToMarkdown } from './download.js';
+
+describe('htmlToMarkdown', () => {
+  it('renders ordered lists with the original list item content', () => {
+    const html = '<ol><li>First item</li><li>Second item</li></ol>';
+
+    expect(htmlToMarkdown(html)).toContain('1. First item');
+    expect(htmlToMarkdown(html)).toContain('2. Second item');
+    expect(htmlToMarkdown(html)).not.toContain('$1');
+  });
+});