@jackwener/opencli 1.3.0 → 1.3.2

package/src/discovery.ts

@@ -14,6 +14,7 @@ import * as path from 'node:path';
 import { pathToFileURL } from 'node:url';
 import yaml from 'js-yaml';
 import { type CliCommand, type InternalCliCommand, type Arg, Strategy, registerCommand } from './registry.js';
+import { getErrorMessage } from './errors.js';
 import { log } from './logger.js';
 import type { ManifestEntry } from './build-manifest.js';
 
@@ -45,10 +46,6 @@ interface YamlCliDefinition {
   navigateBefore?: boolean | string;
 }
 
-function getErrorMessage(error: unknown): string {
-  return error instanceof Error ? error.message : String(error);
-}
-
 function parseStrategy(rawStrategy: string | undefined, fallback: Strategy = Strategy.COOKIE): Strategy {
   if (!rawStrategy) return fallback;
   const key = rawStrategy.toUpperCase() as keyof typeof Strategy;
@@ -142,29 +139,32 @@ async function discoverClisFromFs(dir: string): Promise<void> {
   const promises: Promise<unknown>[] = [];
   const entries = await fs.promises.readdir(dir, { withFileTypes: true });
   
-  for (const entry of entries) {
-    if (!entry.isDirectory()) continue;
-    const site = entry.name;
-    const siteDir = path.join(dir, site);
-    const files = await fs.promises.readdir(siteDir);
-    for (const file of files) {
-      const filePath = path.join(siteDir, file);
-      if (file.endsWith('.yaml') || file.endsWith('.yml')) {
-        promises.push(registerYamlCli(filePath, site));
-      } else if (
-        (file.endsWith('.js') && !file.endsWith('.d.js')) ||
-        (file.endsWith('.ts') && !file.endsWith('.d.ts') && !file.endsWith('.test.ts'))
-      ) {
-        if (!(await isCliModule(filePath))) continue;
-        promises.push(
-          import(pathToFileURL(filePath).href).catch((err) => {
-            log.warn(`Failed to load module ${filePath}: ${getErrorMessage(err)}`);
-          })
-        );
+  const sitePromises = entries
+    .filter(entry => entry.isDirectory())
+    .map(async (entry) => {
+      const site = entry.name;
+      const siteDir = path.join(dir, site);
+      const files = await fs.promises.readdir(siteDir);
+      const filePromises: Promise<unknown>[] = [];
+      for (const file of files) {
+        const filePath = path.join(siteDir, file);
+        if (file.endsWith('.yaml') || file.endsWith('.yml')) {
+          filePromises.push(registerYamlCli(filePath, site));
+        } else if (
+          (file.endsWith('.js') && !file.endsWith('.d.js')) ||
+          (file.endsWith('.ts') && !file.endsWith('.d.ts') && !file.endsWith('.test.ts'))
+        ) {
+          if (!(await isCliModule(filePath))) continue;
+          filePromises.push(
+            import(pathToFileURL(filePath).href).catch((err) => {
+              log.warn(`Failed to load module ${filePath}: ${getErrorMessage(err)}`);
+            })
+          );
+        }
       }
-    }
-  }
-  await Promise.all(promises);
+      await Promise.all(filePromises);
+    });
+  await Promise.all(sitePromises);
 }
 
 async function registerYamlCli(filePath: string, defaultSite: string): Promise<void> {

package/src/download/article-download.ts

@@ -0,0 +1,272 @@
+/**
+ * Article download helper — shared logic for downloading articles as Markdown.
+ *
+ * Used by: zhihu/download, weixin/download, and future article adapters.
+ *
+ * Flow: ArticleData → TurndownService → image download → frontmatter → .md file
+ */
+
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import TurndownService from 'turndown';
+import { httpDownload, sanitizeFilename } from './index.js';
+import { formatBytes } from './progress.js';
+
+const IMAGE_CONCURRENCY = 5;
+
+// ============================================================
+// Types
+// ============================================================
+
+export interface ArticleData {
+  title: string;
+  author?: string;
+  publishTime?: string;
+  sourceUrl?: string;
+  contentHtml: string;
+  /** Pre-extracted code blocks to restore after Markdown conversion */
+  codeBlocks?: Array<{ lang: string; code: string }>;
+  /** Image URLs found in the article (pre-collected from DOM) */
+  imageUrls?: string[];
+}
+
+export interface FrontmatterLabels {
+  author?: string;
+  publishTime?: string;
+  sourceUrl?: string;
+}
+
+export interface ArticleDownloadOptions {
+  output: string;
+  downloadImages?: boolean;
+  /** Extra headers for image downloads (e.g. { Referer: '...' }) */
+  imageHeaders?: Record<string, string>;
+  maxTitleLength?: number;
+  /** Custom TurndownService configuration callback */
+  configureTurndown?: (td: TurndownService) => void;
+  /** Custom image extension detector (default: infer from URL extension) */
+  detectImageExt?: (url: string) => string;
+  /** Custom frontmatter labels (default: Chinese labels) */
+  frontmatterLabels?: FrontmatterLabels;
+}
+
+export interface ArticleDownloadResult {
+  title: string;
+  author: string;
+  publish_time: string;
+  status: string;
+  size: string;
+}
+
+const DEFAULT_LABELS: Required<FrontmatterLabels> = {
+  author: '作者',
+  publishTime: '发布时间',
+  sourceUrl: '原文链接',
+};
+
+// ============================================================
+// Markdown Conversion
+// ============================================================
+
+function createTurndown(configure?: (td: TurndownService) => void): TurndownService {
+  const td = new TurndownService({
+    headingStyle: 'atx',
+    codeBlockStyle: 'fenced',
+    bulletListMarker: '-',
+  });
+  td.addRule('linebreak', {
+    filter: 'br',
+    replacement: () => '\n',
+  });
+  if (configure) configure(td);
+  return td;
+}
+
+function convertToMarkdown(
+  contentHtml: string,
+  codeBlocks: Array<{ lang: string; code: string }>,
+  configure?: (td: TurndownService) => void,
+): string {
+  const td = createTurndown(configure);
+  let md = td.turndown(contentHtml);
+
+  // Restore code block placeholders
+  codeBlocks.forEach((block, i) => {
+    const placeholder = `CODEBLOCK-PLACEHOLDER-${i}`;
+    const fenced = `\n\`\`\`${block.lang}\n${block.code}\n\`\`\`\n`;
+    md = md.replace(placeholder, fenced);
+  });
+
+  // Clean up
+  md = md.replace(/\u00a0/g, ' ');
+  md = md.replace(/\n{4,}/g, '\n\n\n');
+  md = md.replace(/[ \t]+$/gm, '');
+
+  return md;
+}
+
+function replaceImageUrls(md: string, urlMap: Record<string, string>): string {
+  return md.replace(/!\[([^\]]*)\]\(([^)]+)\)/g, (match, alt, imgUrl) => {
+    const local = urlMap[imgUrl];
+    return local ? `![${alt}](${local})` : match;
+  });
+}
+
+// ============================================================
+// Image Downloading
+// ============================================================
+
+function defaultDetectImageExt(url: string): string {
+  const extMatch = url.match(/\.(\w{3,4})(?:\?|$)/);
+  return extMatch ? extMatch[1] : 'jpg';
+}
+
+async function downloadImages(
+  imgUrls: string[],
+  imgDir: string,
+  headers?: Record<string, string>,
+  detectExt?: (url: string) => string,
+): Promise<Record<string, string>> {
+  const urlMap: Record<string, string> = {};
+  if (imgUrls.length === 0) return urlMap;
+
+  const detect = detectExt || defaultDetectImageExt;
+
+  // Deduplicate image URLs
+  const seen = new Set<string>();
+  const uniqueUrls = imgUrls.filter(url => {
+    if (seen.has(url)) return false;
+    seen.add(url);
+    return true;
+  });
+
+  for (let i = 0; i < uniqueUrls.length; i += IMAGE_CONCURRENCY) {
+    const batch = uniqueUrls.slice(i, i + IMAGE_CONCURRENCY);
+    const results = await Promise.all(
+      batch.map(async (rawUrl, j) => {
+        const index = i + j + 1;
+        let imgUrl = rawUrl;
+        if (imgUrl.startsWith('//')) imgUrl = `https:${imgUrl}`;
+
+        const ext = detect(imgUrl);
+        const filename = `img_${String(index).padStart(3, '0')}.${ext}`;
+        const filepath = path.join(imgDir, filename);
+
+        try {
+          const result = await httpDownload(imgUrl, filepath, {
+            headers,
+            timeout: 15000,
+          });
+          if (result.success) {
+            return { remoteUrl: rawUrl, localPath: `images/${filename}` };
+          }
+        } catch {
+          // Skip failed downloads
+        }
+        return null;
+      }),
+    );
+
+    for (const r of results) {
+      if (r) urlMap[r.remoteUrl] = r.localPath;
+    }
+  }
+  return urlMap;
+}
+
+// ============================================================
+// Main API
+// ============================================================
+
+/**
+ * Download an article to Markdown with optional image localization.
+ *
+ * Handles the full pipeline:
+ * 1. HTML → Markdown (via TurndownService)
+ * 2. Code block placeholder restoration
+ * 3. Batch image downloading with concurrency + deduplication
+ * 4. Image URL replacement in Markdown
+ * 5. Frontmatter generation (customizable labels)
+ * 6. File write
+ */
+export async function downloadArticle(
+  data: ArticleData,
+  options: ArticleDownloadOptions,
+): Promise<ArticleDownloadResult[]> {
+  const {
+    output,
+    downloadImages: shouldDownloadImages = true,
+    imageHeaders,
+    maxTitleLength = 80,
+    configureTurndown,
+    detectImageExt,
+    frontmatterLabels,
+  } = options;
+
+  const labels = { ...DEFAULT_LABELS, ...frontmatterLabels };
+
+  if (!data.title) {
+    return [{
+      title: 'Error',
+      author: '-',
+      publish_time: '-',
+      status: 'failed — no title',
+      size: '-',
+    }];
+  }
+
+  if (!data.contentHtml) {
+    return [{
+      title: data.title,
+      author: data.author || '-',
+      publish_time: data.publishTime || '-',
+      status: 'failed — no content',
+      size: '-',
+    }];
+  }
+
+  // Convert HTML to Markdown
+  let markdown = convertToMarkdown(
+    data.contentHtml,
+    data.codeBlocks || [],
+    configureTurndown,
+  );
+
+  // Prepare output directory
+  const safeTitle = sanitizeFilename(data.title, maxTitleLength);
+  const articleDir = path.join(output, safeTitle);
+  fs.mkdirSync(articleDir, { recursive: true });
+
+  // Download images
+  if (shouldDownloadImages && data.imageUrls && data.imageUrls.length > 0) {
+    const imagesDir = path.join(articleDir, 'images');
+    fs.mkdirSync(imagesDir, { recursive: true });
+
+    const urlMap = await downloadImages(data.imageUrls, imagesDir, imageHeaders, detectImageExt);
+    markdown = replaceImageUrls(markdown, urlMap);
+  }
+
+  // Build frontmatter with customizable labels
+  const headerLines = [`# ${data.title}`, ''];
+  if (data.author) headerLines.push(`> ${labels.author}: ${data.author}`);
+  if (data.publishTime) headerLines.push(`> ${labels.publishTime}: ${data.publishTime}`);
+  if (data.sourceUrl) headerLines.push(`> ${labels.sourceUrl}: ${data.sourceUrl}`);
+  headerLines.push('', '---', '');
+
+  const fullContent = headerLines.join('\n') + markdown;
+
+  // Write file
+  const filename = `${safeTitle}.md`;
+  const filePath = path.join(articleDir, filename);
+  fs.writeFileSync(filePath, fullContent, 'utf-8');
+
+  const size = Buffer.byteLength(fullContent, 'utf-8');
+
+  return [{
+    title: data.title,
+    author: data.author || '-',
+    publish_time: data.publishTime || '-',
+    status: 'success',
+    size: formatBytes(size),
+  }];
+}

package/src/download/media-download.ts

@@ -0,0 +1,178 @@
+/**
+ * Media download helper — shared logic for batch downloading images/videos.
+ *
+ * Used by: xiaohongshu/download, twitter/download, bilibili/download,
+ * and future media adapters.
+ *
+ * Flow: MediaItem[] → DownloadProgressTracker → httpDownload/ytdlpDownload → results
+ */
+
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import {
+  httpDownload,
+  ytdlpDownload,
+  checkYtdlp,
+  getTempDir,
+  exportCookiesToNetscape,
+} from './index.js';
+import type { BrowserCookie } from '../types.js';
+import { DownloadProgressTracker, formatBytes } from './progress.js';
+
+// ============================================================
+// Types
+// ============================================================
+
+export interface MediaItem {
+  type: 'image' | 'video' | 'video-tweet' | 'video-ytdlp';
+  url: string;
+  /** Optional custom filename (without directory) */
+  filename?: string;
+}
+
+export interface MediaDownloadOptions {
+  output: string;
+  /** Subdirectory inside output */
+  subdir?: string;
+  /** Cookie string for HTTP downloads */
+  cookies?: string;
+  /** Raw browser cookies — auto-exported to Netscape for yt-dlp, auto-cleaned up */
+  browserCookies?: BrowserCookie[];
+  /** Timeout in ms (default: 30000 for images, 60000 for videos) */
+  timeout?: number;
+  /** File name prefix (default: 'download') */
+  filenamePrefix?: string;
+  /** Extra yt-dlp args */
+  ytdlpExtraArgs?: string[];
+  /** Whether to show progress (default: true) */
+  verbose?: boolean;
+}
+
+export interface MediaDownloadResult {
+  index: number;
+  type: string;
+  status: string;
+  size: string;
+}
+
+// ============================================================
+// Main API
+// ============================================================
+
+/**
+ * Batch download media files with progress tracking.
+ *
+ * Handles:
+ * - DownloadProgressTracker for terminal UX
+ * - Automatic httpDownload vs ytdlpDownload routing via MediaItem.type
+ * - Cookie export to Netscape format for yt-dlp (auto-cleanup)
+ * - Directory creation
+ * - Error handling with per-file results
+ */
+export async function downloadMedia(
+  items: MediaItem[],
+  options: MediaDownloadOptions,
+): Promise<MediaDownloadResult[]> {
+  const {
+    output,
+    subdir,
+    cookies,
+    browserCookies,
+    timeout,
+    filenamePrefix = 'download',
+    ytdlpExtraArgs = [],
+    verbose = true,
+  } = options;
+
+  if (!items || items.length === 0) {
+    return [{ index: 0, type: '-', status: 'failed', size: 'No media found' }];
+  }
+
+  // Create output directory
+  const outputDir = subdir ? path.join(output, subdir) : output;
+  fs.mkdirSync(outputDir, { recursive: true });
+
+  // Pre-check yt-dlp availability (once, not per-item)
+  const hasYtdlp = checkYtdlp();
+
+  // Auto-export browser cookies to Netscape format for yt-dlp
+  let cookiesFile: string | undefined;
+  const needsYtdlp = items.some(m => m.type === 'video-tweet' || m.type === 'video-ytdlp');
+  if (needsYtdlp && browserCookies && browserCookies.length > 0) {
+    const tempDir = getTempDir();
+    fs.mkdirSync(tempDir, { recursive: true });
+    cookiesFile = path.join(tempDir, `media_cookies_${Date.now()}.txt`);
+    exportCookiesToNetscape(browserCookies, cookiesFile);
+  }
+
+  const tracker = new DownloadProgressTracker(items.length, verbose);
+  const results: MediaDownloadResult[] = [];
+
+  try {
+    for (let i = 0; i < items.length; i++) {
+      const media = items[i];
+      const isVideo = media.type !== 'image';
+      const ext = isVideo ? 'mp4' : 'jpg';
+      const filename = media.filename || `${filenamePrefix}_${i + 1}.${ext}`;
+      const destPath = path.join(outputDir, filename);
+
+      const progressBar = tracker.onFileStart(filename, i);
+
+      try {
+        let result: { success: boolean; size: number; error?: string };
+        const useYtdlp = (media.type === 'video-tweet' || media.type === 'video-ytdlp') && hasYtdlp;
+
+        if (useYtdlp) {
+          result = await ytdlpDownload(media.url, destPath, {
+            cookiesFile,
+            extraArgs: ytdlpExtraArgs,
+            onProgress: (percent) => {
+              if (progressBar) progressBar.update(percent, 100);
+            },
+          });
+        } else {
+          // Direct HTTP download for images and direct video URLs
+          const dlTimeout = timeout || (isVideo ? 60000 : 30000);
+          result = await httpDownload(media.url, destPath, {
+            cookies,
+            timeout: dlTimeout,
+            onProgress: (received, total) => {
+              if (progressBar) progressBar.update(received, total);
+            },
+          });
+        }
+
+        if (progressBar) {
+          progressBar.complete(result.success, result.success ? formatBytes(result.size) : undefined);
+        }
+        tracker.onFileComplete(result.success);
+
+        results.push({
+          index: i + 1,
+          type: media.type === 'video-tweet' || media.type === 'video-ytdlp' ? 'video' : media.type,
+          status: result.success ? 'success' : 'failed',
+          size: result.success ? formatBytes(result.size) : (result.error || 'unknown error'),
+        });
+      } catch (err: any) {
+        if (progressBar) progressBar.fail(err.message);
+        tracker.onFileComplete(false);
+
+        results.push({
+          index: i + 1,
+          type: media.type,
+          status: 'failed',
+          size: err.message,
+        });
+      }
+    }
+  } finally {
+    tracker.finish();
+
+    // Auto-cleanup exported cookies file
+    if (cookiesFile && fs.existsSync(cookiesFile)) {
+      fs.unlinkSync(cookiesFile);
+    }
+  }
+
+  return results;
+}

package/src/errors.test.ts

@@ -0,0 +1,79 @@
+import { describe, it, expect } from 'vitest';
+import {
+  CliError,
+  BrowserConnectError,
+  AdapterLoadError,
+  CommandExecutionError,
+  ConfigError,
+  AuthRequiredError,
+  TimeoutError,
+  ArgumentError,
+  EmptyResultError,
+  SelectorError,
+} from './errors.js';
+
+describe('Error type hierarchy', () => {
+  it('all error types extend CliError', () => {
+    const errors = [
+      new BrowserConnectError('test'),
+      new AdapterLoadError('test'),
+      new CommandExecutionError('test'),
+      new ConfigError('test'),
+      new AuthRequiredError('example.com'),
+      new TimeoutError('test', 30),
+      new ArgumentError('test'),
+      new EmptyResultError('test/cmd'),
+      new SelectorError('.btn'),
+    ];
+
+    for (const err of errors) {
+      expect(err).toBeInstanceOf(CliError);
+      expect(err).toBeInstanceOf(Error);
+    }
+  });
+
+  it('AuthRequiredError has correct code, domain, and auto-generated hint', () => {
+    const err = new AuthRequiredError('bilibili.com');
+    expect(err.code).toBe('AUTH_REQUIRED');
+    expect(err.domain).toBe('bilibili.com');
+    expect(err.message).toBe('Not logged in to bilibili.com');
+    expect(err.hint).toContain('https://bilibili.com');
+  });
+
+  it('AuthRequiredError accepts custom message', () => {
+    const err = new AuthRequiredError('x.com', 'No ct0 cookie found');
+    expect(err.message).toBe('No ct0 cookie found');
+    expect(err.hint).toContain('https://x.com');
+  });
+
+  it('TimeoutError has correct code and hint', () => {
+    const err = new TimeoutError('bilibili/hot', 60);
+    expect(err.code).toBe('TIMEOUT');
+    expect(err.message).toBe('bilibili/hot timed out after 60s');
+    expect(err.hint).toContain('timeout');
+  });
+
+  it('ArgumentError has correct code', () => {
+    const err = new ArgumentError('Argument "limit" must be a valid number');
+    expect(err.code).toBe('ARGUMENT');
+  });
+
+  it('EmptyResultError has default hint', () => {
+    const err = new EmptyResultError('hackernews/top');
+    expect(err.code).toBe('EMPTY_RESULT');
+    expect(err.message).toBe('hackernews/top returned no data');
+    expect(err.hint).toBeTruthy();
+  });
+
+  it('SelectorError has default hint about page changes', () => {
+    const err = new SelectorError('.submit-btn');
+    expect(err.code).toBe('SELECTOR');
+    expect(err.message).toContain('.submit-btn');
+    expect(err.hint).toContain('report');
+  });
+
+  it('BrowserConnectError has correct code', () => {
+    const err = new BrowserConnectError('Cannot connect');
+    expect(err.code).toBe('BROWSER_CONNECT');
+  });
+});