@jackwener/opencli 0.9.6 → 0.9.8

package/dist/clis/xiaohongshu/user.js

@@ -1,40 +1,57 @@
 import { cli, Strategy } from '../../registry.js';
+import { extractXhsUserNotes, normalizeXhsUserId } from './user-helpers.js';
+async function readUserSnapshot(page) {
+    return await page.evaluate(`
+    (() => {
+      const safeClone = (value) => {
+        try {
+          return JSON.parse(JSON.stringify(value ?? null));
+        } catch {
+          return null;
+        }
+      };
+
+      const userStore = window.__INITIAL_STATE__?.user || {};
+      return {
+        noteGroups: safeClone(userStore.notes?._value || userStore.notes || []),
+        pageData: safeClone(userStore.userPageData?._value || userStore.userPageData || {}),
+      };
+    })()
+  `);
+}
 cli({
     site: 'xiaohongshu',
     name: 'user',
-    description: 'Get user notes from Xiaohongshu',
-    domain: 'xiaohongshu.com',
-    strategy: Strategy.INTERCEPT,
+    description: 'Get public notes from a Xiaohongshu user profile',
+    domain: 'www.xiaohongshu.com',
+    strategy: Strategy.COOKIE,
     browser: true,
     args: [
-        { name: 'id', type: 'string', required: true },
-        { name: 'limit', type: 'int', default: 15 },
+        { name: 'id', type: 'string', required: true, help: 'User id or profile URL' },
+        { name: 'limit', type: 'int', default: 15, help: 'Number of notes to return' },
     ],
     columns: ['id', 'title', 'type', 'likes', 'url'],
     func: async (page, kwargs) => {
-        await page.goto(`https://www.xiaohongshu.com/user/profile/${kwargs.id}`);
-        await page.wait(5);
-        await page.installInterceptor('v1/user/posted');
-        // Trigger API by scrolling
-        await page.autoScroll({ times: 2, delayMs: 2000 });
-        // Retrieve data
-        const requests = await page.getInterceptedRequests();
-        if (!requests || requests.length === 0)
-            return [];
-        let results = [];
-        for (const req of requests) {
-            if (req.data && req.data.data && req.data.data.notes) {
-                for (const note of req.data.data.notes) {
-                    results.push({
-                        id: note.note_id || note.id,
-                        title: note.display_title || '',
-                        type: note.type || '',
-                        likes: note.interact_info?.liked_count || '0',
-                        url: `https://www.xiaohongshu.com/explore/${note.note_id || note.id}`
-                    });
-                }
-            }
+        const userId = normalizeXhsUserId(String(kwargs.id));
+        const limit = Math.max(1, Number(kwargs.limit ?? 15));
+        await page.goto(`https://www.xiaohongshu.com/user/profile/${userId}`);
+        await page.wait(3);
+        let snapshot = await readUserSnapshot(page);
+        let results = extractXhsUserNotes(snapshot ?? {}, userId);
+        let previousCount = results.length;
+        for (let i = 0; results.length < limit && i < 4; i += 1) {
+            await page.autoScroll({ times: 1, delayMs: 1500 });
+            await page.wait(1);
+            snapshot = await readUserSnapshot(page);
+            const nextResults = extractXhsUserNotes(snapshot ?? {}, userId);
+            if (nextResults.length <= previousCount)
+                break;
+            results = nextResults;
+            previousCount = nextResults.length;
+        }
+        if (results.length === 0) {
+            throw new Error('No public notes found for this Xiaohongshu user.');
         }
-        return results.slice(0, kwargs.limit);
-    }
+        return results.slice(0, limit);
+    },
 });

package/dist/clis/zhihu/download.d.ts

@@ -0,0 +1,11 @@
+/**
+ * Zhihu download — export articles to Markdown format.
+ *
+ * Usage:
+ *   opencli zhihu download --url "https://zhuanlan.zhihu.com/p/xxx" --output ./zhihu
+ */
+/**
+ * Convert HTML content to Markdown.
+ * This is a simplified converter for Zhihu article content.
+ */
+export declare function htmlToMarkdown(html: string): string;

package/dist/clis/zhihu/download.js

@@ -0,0 +1,186 @@
+/**
+ * Zhihu download — export articles to Markdown format.
+ *
+ * Usage:
+ *   opencli zhihu download --url "https://zhuanlan.zhihu.com/p/xxx" --output ./zhihu
+ */
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import { cli, Strategy } from '../../registry.js';
+import { sanitizeFilename, httpDownload } from '../../download/index.js';
+import { formatBytes } from '../../download/progress.js';
+/**
+ * Convert HTML content to Markdown.
+ * This is a simplified converter for Zhihu article content.
+ */
+export function htmlToMarkdown(html) {
+    let md = html;
+    // Remove script and style tags
+    md = md.replace(/<script[^>]*>[\s\S]*?<\/script>/gi, '');
+    md = md.replace(/<style[^>]*>[\s\S]*?<\/style>/gi, '');
+    // Convert headers
+    md = md.replace(/<h1[^>]*>(.*?)<\/h1>/gi, '# $1\n\n');
+    md = md.replace(/<h2[^>]*>(.*?)<\/h2>/gi, '## $1\n\n');
+    md = md.replace(/<h3[^>]*>(.*?)<\/h3>/gi, '### $1\n\n');
+    md = md.replace(/<h4[^>]*>(.*?)<\/h4>/gi, '#### $1\n\n');
+    // Convert paragraphs
+    md = md.replace(/<p[^>]*>([\s\S]*?)<\/p>/gi, '$1\n\n');
+    // Convert links
+    md = md.replace(/<a[^>]*href="([^"]*)"[^>]*>(.*?)<\/a>/gi, '[$2]($1)');
+    // Convert images
+    md = md.replace(/<img[^>]*src="([^"]*)"[^>]*alt="([^"]*)"[^>]*\/?>/gi, '![$2]($1)');
+    md = md.replace(/<img[^>]*src="([^"]*)"[^>]*\/?>/gi, '![]($1)');
+    // Convert lists
+    md = md.replace(/<ul[^>]*>([\s\S]*?)<\/ul>/gi, (match, content) => {
+        return content.replace(/<li[^>]*>([\s\S]*?)<\/li>/gi, '- $1\n') + '\n';
+    });
+    md = md.replace(/<ol[^>]*>([\s\S]*?)<\/ol>/gi, (match, content) => {
+        let index = 0;
+        return content.replace(/<li[^>]*>([\s\S]*?)<\/li>/gi, (_itemMatch, itemContent) => `${++index}. ${itemContent}\n`) + '\n';
+    });
+    // Convert bold and italic
+    md = md.replace(/<strong[^>]*>(.*?)<\/strong>/gi, '**$1**');
+    md = md.replace(/<b[^>]*>(.*?)<\/b>/gi, '**$1**');
+    md = md.replace(/<em[^>]*>(.*?)<\/em>/gi, '*$1*');
+    md = md.replace(/<i[^>]*>(.*?)<\/i>/gi, '*$1*');
+    // Convert code blocks
+    md = md.replace(/<pre[^>]*><code[^>]*>([\s\S]*?)<\/code><\/pre>/gi, '```\n$1\n```\n\n');
+    md = md.replace(/<code[^>]*>(.*?)<\/code>/gi, '`$1`');
+    // Convert blockquotes
+    md = md.replace(/<blockquote[^>]*>([\s\S]*?)<\/blockquote>/gi, (match, content) => {
+        return content.split('\n').map((line) => `> ${line}`).join('\n') + '\n\n';
+    });
+    // Convert line breaks
+    md = md.replace(/<br\s*\/?>/gi, '\n');
+    // Remove remaining HTML tags
+    md = md.replace(/<[^>]+>/g, '');
+    // Decode HTML entities
+    md = md.replace(/&nbsp;/g, ' ');
+    md = md.replace(/&lt;/g, '<');
+    md = md.replace(/&gt;/g, '>');
+    md = md.replace(/&amp;/g, '&');
+    md = md.replace(/&quot;/g, '"');
+    // Clean up extra whitespace
+    md = md.replace(/\n{3,}/g, '\n\n');
+    md = md.trim();
+    return md;
+}
+cli({
+    site: 'zhihu',
+    name: 'download',
+    description: '导出知乎文章为 Markdown 格式',
+    domain: 'zhuanlan.zhihu.com',
+    strategy: Strategy.COOKIE,
+    args: [
+        { name: 'url', required: true, help: 'Article URL (zhuanlan.zhihu.com/p/xxx)' },
+        { name: 'output', default: './zhihu-articles', help: 'Output directory' },
+        { name: 'download-images', type: 'boolean', default: false, help: 'Download images locally' },
+    ],
+    columns: ['title', 'author', 'status', 'size'],
+    func: async (page, kwargs) => {
+        const url = kwargs.url;
+        const output = kwargs.output;
+        const downloadImages = kwargs['download-images'];
+        // Navigate to article page
+        await page.goto(url);
+        await page.wait(3);
+        // Extract article content
+        const data = await page.evaluate(`
+      (() => {
+        const result = {
+          title: '',
+          author: '',
+          content: '',
+          publishTime: '',
+          images: []
+        };
+
+        // Get title
+        const titleEl = document.querySelector('.Post-Title, h1.ContentItem-title, .ArticleTitle');
+        result.title = titleEl?.textContent?.trim() || 'untitled';
+
+        // Get author
+        const authorEl = document.querySelector('.AuthorInfo-name, .UserLink-link');
+        result.author = authorEl?.textContent?.trim() || 'unknown';
+
+        // Get publish time
+        const timeEl = document.querySelector('.ContentItem-time, .Post-Time');
+        result.publishTime = timeEl?.textContent?.trim() || '';
+
+        // Get content HTML
+        const contentEl = document.querySelector('.Post-RichTextContainer, .RichText, .ArticleContent');
+        if (contentEl) {
+          result.content = contentEl.innerHTML;
+
+          // Extract image URLs
+          contentEl.querySelectorAll('img').forEach(img => {
+            const src = img.getAttribute('data-original') || img.getAttribute('data-actualsrc') || img.src;
+            if (src && !src.includes('data:image')) {
+              result.images.push(src);
+            }
+          });
+        }
+
+        return result;
+      })()
+    `);
+        if (!data || !data.content) {
+            return [{
+                    title: 'Error',
+                    author: '-',
+                    status: 'failed',
+                    size: 'Could not extract article content',
+                }];
+        }
+        // Create output directory
+        fs.mkdirSync(output, { recursive: true });
+        // Convert HTML to Markdown
+        let markdown = htmlToMarkdown(data.content);
+        // Create frontmatter
+        const frontmatter = [
+            '---',
+            `title: "${data.title.replace(/"/g, '\\"')}"`,
+            `author: "${data.author.replace(/"/g, '\\"')}"`,
+            `source: "${url}"`,
+            data.publishTime ? `date: "${data.publishTime}"` : '',
+            '---',
+            '',
+        ].filter(Boolean).join('\n');
+        // Download images if requested
+        if (downloadImages && data.images && data.images.length > 0) {
+            const imagesDir = path.join(output, 'images');
+            fs.mkdirSync(imagesDir, { recursive: true });
+            const cookies = await page.evaluate(`(() => document.cookie)()`);
+            for (let i = 0; i < data.images.length; i++) {
+                const imgUrl = data.images[i];
+                const ext = imgUrl.match(/\.(jpg|jpeg|png|gif|webp)/i)?.[1] || 'jpg';
+                const imgFilename = `img_${i + 1}.${ext}`;
+                const imgPath = path.join(imagesDir, imgFilename);
+                try {
+                    await httpDownload(imgUrl, imgPath, {
+                        cookies: typeof cookies === 'string' ? cookies : '',
+                        timeout: 30000,
+                    });
+                    // Replace image URL in markdown with local path
+                    markdown = markdown.replace(new RegExp(imgUrl.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'), 'g'), `./images/${imgFilename}`);
+                }
+                catch {
+                    // Keep original URL if download fails
+                }
+            }
+        }
+        // Write markdown file
+        const safeTitle = sanitizeFilename(data.title, 100);
+        const filename = `${safeTitle}.md`;
+        const filePath = path.join(output, filename);
+        const fullContent = frontmatter + '\n' + markdown;
+        fs.writeFileSync(filePath, fullContent, 'utf-8');
+        const size = Buffer.byteLength(fullContent, 'utf-8');
+        return [{
+                title: data.title,
+                author: data.author,
+                status: 'success',
+                size: formatBytes(size),
+            }];
+    },
+});

package/dist/clis/zhihu/download.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/clis/zhihu/download.test.js

@@ -0,0 +1,10 @@
+import { describe, expect, it } from 'vitest';
+import { htmlToMarkdown } from './download.js';
+describe('htmlToMarkdown', () => {
+    it('renders ordered lists with the original list item content', () => {
+        const html = '<ol><li>First item</li><li>Second item</li></ol>';
+        expect(htmlToMarkdown(html)).toContain('1. First item');
+        expect(htmlToMarkdown(html)).toContain('2. Second item');
+        expect(htmlToMarkdown(html)).not.toContain('$1');
+    });
+});

package/dist/download/index.d.ts

@@ -0,0 +1,79 @@
+/**
+ * Download utilities: HTTP downloads, yt-dlp wrapper, format conversion.
+ */
+export interface DownloadOptions {
+    cookies?: string;
+    headers?: Record<string, string>;
+    timeout?: number;
+    onProgress?: (received: number, total: number) => void;
+}
+export interface YtdlpOptions {
+    cookies?: string;
+    cookiesFile?: string;
+    format?: string;
+    extraArgs?: string[];
+    onProgress?: (percent: number) => void;
+}
+/**
+ * Check if yt-dlp is available in PATH.
+ */
+export declare function checkYtdlp(): boolean;
+/**
+ * Check if ffmpeg is available in PATH.
+ */
+export declare function checkFfmpeg(): boolean;
+/**
+ * Detect content type from URL and optional headers.
+ */
+export declare function detectContentType(url: string, contentType?: string): 'image' | 'video' | 'document' | 'binary';
+/**
+ * Check if URL requires yt-dlp for download.
+ */
+export declare function requiresYtdlp(url: string): boolean;
+/**
+ * HTTP download with progress callback.
+ */
+export declare function httpDownload(url: string, destPath: string, options?: DownloadOptions): Promise<{
+    success: boolean;
+    size: number;
+    error?: string;
+}>;
+/**
+ * Export cookies to Netscape format for yt-dlp.
+ */
+export declare function exportCookiesToNetscape(cookies: Array<{
+    name: string;
+    value: string;
+    domain: string;
+    path?: string;
+    secure?: boolean;
+    httpOnly?: boolean;
+}>, filePath: string): void;
+/**
+ * Download video using yt-dlp.
+ */
+export declare function ytdlpDownload(url: string, destPath: string, options?: YtdlpOptions): Promise<{
+    success: boolean;
+    size: number;
+    error?: string;
+}>;
+/**
+ * Save document content to file.
+ */
+export declare function saveDocument(content: string, destPath: string, format?: 'json' | 'markdown' | 'html' | 'text', metadata?: Record<string, any>): Promise<{
+    success: boolean;
+    size: number;
+    error?: string;
+}>;
+/**
+ * Sanitize filename by removing invalid characters.
+ */
+export declare function sanitizeFilename(name: string, maxLength?: number): string;
+/**
+ * Generate filename from URL if not provided.
+ */
+export declare function generateFilename(url: string, index: number, extension?: string): string;
+/**
+ * Get temp directory for cookie files.
+ */
+export declare function getTempDir(): string;