@jackwener/opencli 0.9.5 → 0.9.8

package/src/clis/xiaohongshu/user-helpers.ts

@@ -0,0 +1,85 @@
+export interface XhsUserPageSnapshot {
+  noteGroups?: unknown;
+  pageData?: unknown;
+}
+
+export interface XhsUserNoteRow {
+  id: string;
+  title: string;
+  type: string;
+  likes: string;
+  url: string;
+}
+
+function toCleanString(value: unknown): string {
+  return typeof value === 'string' ? value.trim() : value == null ? '' : String(value).trim();
+}
+
+export function normalizeXhsUserId(input: string): string {
+  const trimmed = toCleanString(input);
+  const withoutQuery = trimmed.replace(/[?#].*$/, '');
+  const matched = withoutQuery.match(/\/user\/profile\/([a-zA-Z0-9]+)/);
+  if (matched?.[1]) return matched[1];
+  return withoutQuery.replace(/\/+$/, '').split('/').pop() ?? withoutQuery;
+}
+
+export function flattenXhsNoteGroups(noteGroups: unknown): any[] {
+  if (!Array.isArray(noteGroups)) return [];
+
+  const notes: any[] = [];
+  for (const group of noteGroups) {
+    if (!group) continue;
+    if (Array.isArray(group)) {
+      for (const item of group) {
+        if (item) notes.push(item);
+      }
+      continue;
+    }
+    notes.push(group);
+  }
+
+  return notes;
+}
+
+export function buildXhsNoteUrl(userId: string, noteId: string, xsecToken?: string): string {
+  const cleanUserId = toCleanString(userId);
+  const cleanNoteId = toCleanString(noteId);
+  if (!cleanUserId || !cleanNoteId) return '';
+
+  const url = new URL(`https://www.xiaohongshu.com/user/profile/${cleanUserId}/${cleanNoteId}`);
+  const cleanToken = toCleanString(xsecToken);
+  if (cleanToken) {
+    url.searchParams.set('xsec_token', cleanToken);
+    url.searchParams.set('xsec_source', 'pc_user');
+  }
+  return url.toString();
+}
+
+export function extractXhsUserNotes(snapshot: XhsUserPageSnapshot, fallbackUserId: string): XhsUserNoteRow[] {
+  const notes = flattenXhsNoteGroups(snapshot.noteGroups);
+  const rows: XhsUserNoteRow[] = [];
+  const seen = new Set<string>();
+
+  for (const entry of notes) {
+    const noteCard = entry?.noteCard ?? entry?.note_card ?? entry;
+    if (!noteCard || typeof noteCard !== 'object') continue;
+
+    const noteId = toCleanString(noteCard.noteId ?? noteCard.note_id ?? entry?.noteId ?? entry?.note_id ?? entry?.id);
+    if (!noteId || seen.has(noteId)) continue;
+    seen.add(noteId);
+
+    const userId = toCleanString(noteCard.user?.userId ?? noteCard.user?.user_id ?? fallbackUserId);
+    const xsecToken = toCleanString(entry?.xsecToken ?? entry?.xsec_token ?? noteCard.xsecToken ?? noteCard.xsec_token);
+    const likes = toCleanString(noteCard.interactInfo?.likedCount ?? noteCard.interact_info?.liked_count ?? 0) || '0';
+
+    rows.push({
+      id: noteId,
+      title: toCleanString(noteCard.displayTitle ?? noteCard.display_title ?? noteCard.title),
+      type: toCleanString(noteCard.type),
+      likes,
+      url: buildXhsNoteUrl(userId || fallbackUserId, noteId, xsecToken),
+    });
+  }
+
+  return rows;
+}

package/src/clis/xiaohongshu/user.ts

@@ -1,45 +1,65 @@
 import { cli, Strategy } from '../../registry.js';
+import { extractXhsUserNotes, normalizeXhsUserId } from './user-helpers.js';
+
+async function readUserSnapshot(page: any) {
+  return await page.evaluate(`
+    (() => {
+      const safeClone = (value) => {
+        try {
+          return JSON.parse(JSON.stringify(value ?? null));
+        } catch {
+          return null;
+        }
+      };
+
+      const userStore = window.__INITIAL_STATE__?.user || {};
+      return {
+        noteGroups: safeClone(userStore.notes?._value || userStore.notes || []),
+        pageData: safeClone(userStore.userPageData?._value || userStore.userPageData || {}),
+      };
+    })()
+  `);
+}
 
 cli({
   site: 'xiaohongshu',
   name: 'user',
-  description: 'Get user notes from Xiaohongshu',
-  domain: 'xiaohongshu.com',
-  strategy: Strategy.INTERCEPT,
+  description: 'Get public notes from a Xiaohongshu user profile',
+  domain: 'www.xiaohongshu.com',
+  strategy: Strategy.COOKIE,
   browser: true,
   args: [
-    { name: 'id', type: 'string', required: true },
-    { name: 'limit', type: 'int', default: 15 },
+    { name: 'id', type: 'string', required: true, help: 'User id or profile URL' },
+    { name: 'limit', type: 'int', default: 15, help: 'Number of notes to return' },
   ],
   columns: ['id', 'title', 'type', 'likes', 'url'],
   func: async (page, kwargs) => {
-    await page.goto(`https://www.xiaohongshu.com/user/profile/${kwargs.id}`);
-    await page.wait(5);
-
-    await page.installInterceptor('v1/user/posted');
-
-    // Trigger API by scrolling
-    await page.autoScroll({ times: 2, delayMs: 2000 });
-    
-    // Retrieve data
-    const requests = await page.getInterceptedRequests();
-    if (!requests || requests.length === 0) return [];
-
-    let results: any[] = [];
-    for (const req of requests) {
-      if (req.data && req.data.data && req.data.data.notes) {
-         for (const note of req.data.data.notes) {
-           results.push({
-             id: note.note_id || note.id,
-             title: note.display_title || '',
-             type: note.type || '',
-             likes: note.interact_info?.liked_count || '0',
-             url: `https://www.xiaohongshu.com/explore/${note.note_id || note.id}`
-           });
-         }
-      }
+    const userId = normalizeXhsUserId(String(kwargs.id));
+    const limit = Math.max(1, Number(kwargs.limit ?? 15));
+
+    await page.goto(`https://www.xiaohongshu.com/user/profile/${userId}`);
+    await page.wait(3);
+
+    let snapshot = await readUserSnapshot(page);
+    let results = extractXhsUserNotes(snapshot ?? {}, userId);
+    let previousCount = results.length;
+
+    for (let i = 0; results.length < limit && i < 4; i += 1) {
+      await page.autoScroll({ times: 1, delayMs: 1500 });
+      await page.wait(1);
+
+      snapshot = await readUserSnapshot(page);
+      const nextResults = extractXhsUserNotes(snapshot ?? {}, userId);
+      if (nextResults.length <= previousCount) break;
+
+      results = nextResults;
+      previousCount = nextResults.length;
+    }
+
+    if (results.length === 0) {
+      throw new Error('No public notes found for this Xiaohongshu user.');
     }
 
-    return results.slice(0, kwargs.limit);
-  }
+    return results.slice(0, limit);
+  },
 });

package/src/clis/zhihu/download.test.ts

@@ -0,0 +1,12 @@
+import { describe, expect, it } from 'vitest';
+import { htmlToMarkdown } from './download.js';
+
+describe('htmlToMarkdown', () => {
+  it('renders ordered lists with the original list item content', () => {
+    const html = '<ol><li>First item</li><li>Second item</li></ol>';
+
+    expect(htmlToMarkdown(html)).toContain('1. First item');
+    expect(htmlToMarkdown(html)).toContain('2. Second item');
+    expect(htmlToMarkdown(html)).not.toContain('$1');
+  });
+});

package/src/clis/zhihu/download.ts

@@ -0,0 +1,223 @@
+/**
+ * Zhihu download — export articles to Markdown format.
+ *
+ * Usage:
+ *   opencli zhihu download --url "https://zhuanlan.zhihu.com/p/xxx" --output ./zhihu
+ */
+
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import { cli, Strategy } from '../../registry.js';
+import { sanitizeFilename, httpDownload } from '../../download/index.js';
+import { formatBytes } from '../../download/progress.js';
+
+/**
+ * Convert HTML content to Markdown.
+ * This is a simplified converter for Zhihu article content.
+ */
+export function htmlToMarkdown(html: string): string {
+  let md = html;
+
+  // Remove script and style tags
+  md = md.replace(/<script[^>]*>[\s\S]*?<\/script>/gi, '');
+  md = md.replace(/<style[^>]*>[\s\S]*?<\/style>/gi, '');
+
+  // Convert headers
+  md = md.replace(/<h1[^>]*>(.*?)<\/h1>/gi, '# $1\n\n');
+  md = md.replace(/<h2[^>]*>(.*?)<\/h2>/gi, '## $1\n\n');
+  md = md.replace(/<h3[^>]*>(.*?)<\/h3>/gi, '### $1\n\n');
+  md = md.replace(/<h4[^>]*>(.*?)<\/h4>/gi, '#### $1\n\n');
+
+  // Convert paragraphs
+  md = md.replace(/<p[^>]*>([\s\S]*?)<\/p>/gi, '$1\n\n');
+
+  // Convert links
+  md = md.replace(/<a[^>]*href="([^"]*)"[^>]*>(.*?)<\/a>/gi, '[$2]($1)');
+
+  // Convert images
+  md = md.replace(/<img[^>]*src="([^"]*)"[^>]*alt="([^"]*)"[^>]*\/?>/gi, '![$2]($1)');
+  md = md.replace(/<img[^>]*src="([^"]*)"[^>]*\/?>/gi, '![]($1)');
+
+  // Convert lists
+  md = md.replace(/<ul[^>]*>([\s\S]*?)<\/ul>/gi, (match, content) => {
+    return content.replace(/<li[^>]*>([\s\S]*?)<\/li>/gi, '- $1\n') + '\n';
+  });
+  md = md.replace(/<ol[^>]*>([\s\S]*?)<\/ol>/gi, (match, content) => {
+    let index = 0;
+    return content.replace(
+      /<li[^>]*>([\s\S]*?)<\/li>/gi,
+      (_itemMatch: string, itemContent: string) => `${++index}. ${itemContent}\n`,
+    ) + '\n';
+  });
+
+  // Convert bold and italic
+  md = md.replace(/<strong[^>]*>(.*?)<\/strong>/gi, '**$1**');
+  md = md.replace(/<b[^>]*>(.*?)<\/b>/gi, '**$1**');
+  md = md.replace(/<em[^>]*>(.*?)<\/em>/gi, '*$1*');
+  md = md.replace(/<i[^>]*>(.*?)<\/i>/gi, '*$1*');
+
+  // Convert code blocks
+  md = md.replace(/<pre[^>]*><code[^>]*>([\s\S]*?)<\/code><\/pre>/gi, '```\n$1\n```\n\n');
+  md = md.replace(/<code[^>]*>(.*?)<\/code>/gi, '`$1`');
+
+  // Convert blockquotes
+  md = md.replace(/<blockquote[^>]*>([\s\S]*?)<\/blockquote>/gi, (match, content) => {
+    return content.split('\n').map((line: string) => `> ${line}`).join('\n') + '\n\n';
+  });
+
+  // Convert line breaks
+  md = md.replace(/<br\s*\/?>/gi, '\n');
+
+  // Remove remaining HTML tags
+  md = md.replace(/<[^>]+>/g, '');
+
+  // Decode HTML entities
+  md = md.replace(/&nbsp;/g, ' ');
+  md = md.replace(/&lt;/g, '<');
+  md = md.replace(/&gt;/g, '>');
+  md = md.replace(/&amp;/g, '&');
+  md = md.replace(/&quot;/g, '"');
+
+  // Clean up extra whitespace
+  md = md.replace(/\n{3,}/g, '\n\n');
+  md = md.trim();
+
+  return md;
+}
+
+cli({
+  site: 'zhihu',
+  name: 'download',
+  description: '导出知乎文章为 Markdown 格式',
+  domain: 'zhuanlan.zhihu.com',
+  strategy: Strategy.COOKIE,
+  args: [
+    { name: 'url', required: true, help: 'Article URL (zhuanlan.zhihu.com/p/xxx)' },
+    { name: 'output', default: './zhihu-articles', help: 'Output directory' },
+    { name: 'download-images', type: 'boolean', default: false, help: 'Download images locally' },
+  ],
+  columns: ['title', 'author', 'status', 'size'],
+  func: async (page, kwargs) => {
+    const url = kwargs.url;
+    const output = kwargs.output;
+    const downloadImages = kwargs['download-images'];
+
+    // Navigate to article page
+    await page.goto(url);
+    await page.wait(3);
+
+    // Extract article content
+    const data = await page.evaluate(`
+      (() => {
+        const result = {
+          title: '',
+          author: '',
+          content: '',
+          publishTime: '',
+          images: []
+        };
+
+        // Get title
+        const titleEl = document.querySelector('.Post-Title, h1.ContentItem-title, .ArticleTitle');
+        result.title = titleEl?.textContent?.trim() || 'untitled';
+
+        // Get author
+        const authorEl = document.querySelector('.AuthorInfo-name, .UserLink-link');
+        result.author = authorEl?.textContent?.trim() || 'unknown';
+
+        // Get publish time
+        const timeEl = document.querySelector('.ContentItem-time, .Post-Time');
+        result.publishTime = timeEl?.textContent?.trim() || '';
+
+        // Get content HTML
+        const contentEl = document.querySelector('.Post-RichTextContainer, .RichText, .ArticleContent');
+        if (contentEl) {
+          result.content = contentEl.innerHTML;
+
+          // Extract image URLs
+          contentEl.querySelectorAll('img').forEach(img => {
+            const src = img.getAttribute('data-original') || img.getAttribute('data-actualsrc') || img.src;
+            if (src && !src.includes('data:image')) {
+              result.images.push(src);
+            }
+          });
+        }
+
+        return result;
+      })()
+    `);
+
+    if (!data || !data.content) {
+      return [{
+        title: 'Error',
+        author: '-',
+        status: 'failed',
+        size: 'Could not extract article content',
+      }];
+    }
+
+    // Create output directory
+    fs.mkdirSync(output, { recursive: true });
+
+    // Convert HTML to Markdown
+    let markdown = htmlToMarkdown(data.content);
+
+    // Create frontmatter
+    const frontmatter = [
+      '---',
+      `title: "${data.title.replace(/"/g, '\\"')}"`,
+      `author: "${data.author.replace(/"/g, '\\"')}"`,
+      `source: "${url}"`,
+      data.publishTime ? `date: "${data.publishTime}"` : '',
+      '---',
+      '',
+    ].filter(Boolean).join('\n');
+
+    // Download images if requested
+    if (downloadImages && data.images && data.images.length > 0) {
+      const imagesDir = path.join(output, 'images');
+      fs.mkdirSync(imagesDir, { recursive: true });
+
+      const cookies = await page.evaluate(`(() => document.cookie)()`);
+
+      for (let i = 0; i < data.images.length; i++) {
+        const imgUrl = data.images[i];
+        const ext = imgUrl.match(/\.(jpg|jpeg|png|gif|webp)/i)?.[1] || 'jpg';
+        const imgFilename = `img_${i + 1}.${ext}`;
+        const imgPath = path.join(imagesDir, imgFilename);
+
+        try {
+          await httpDownload(imgUrl, imgPath, {
+            cookies: typeof cookies === 'string' ? cookies : '',
+            timeout: 30000,
+          });
+
+          // Replace image URL in markdown with local path
+          markdown = markdown.replace(
+            new RegExp(imgUrl.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'), 'g'),
+            `./images/${imgFilename}`,
+          );
+        } catch {
+          // Keep original URL if download fails
+        }
+      }
+    }
+
+    // Write markdown file
+    const safeTitle = sanitizeFilename(data.title, 100);
+    const filename = `${safeTitle}.md`;
+    const filePath = path.join(output, filename);
+
+    const fullContent = frontmatter + '\n' + markdown;
+    fs.writeFileSync(filePath, fullContent, 'utf-8');
+
+    const size = Buffer.byteLength(fullContent, 'utf-8');
+
+    return [{
+      title: data.title,
+      author: data.author,
+      status: 'success',
+      size: formatBytes(size),
+    }];
+  },
+});