@jackwener/opencli 0.9.6 → 1.0.0

package/src/clis/zhihu/download.ts

@@ -0,0 +1,223 @@
+/**
+ * Zhihu download — export articles to Markdown format.
+ *
+ * Usage:
+ *   opencli zhihu download --url "https://zhuanlan.zhihu.com/p/xxx" --output ./zhihu
+ */
+
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import { cli, Strategy } from '../../registry.js';
+import { sanitizeFilename, httpDownload } from '../../download/index.js';
+import { formatBytes } from '../../download/progress.js';
+
+/**
+ * Convert HTML content to Markdown.
+ * This is a simplified converter for Zhihu article content.
+ */
+export function htmlToMarkdown(html: string): string {
+  let md = html;
+
+  // Remove script and style tags
+  md = md.replace(/<script[^>]*>[\s\S]*?<\/script>/gi, '');
+  md = md.replace(/<style[^>]*>[\s\S]*?<\/style>/gi, '');
+
+  // Convert headers
+  md = md.replace(/<h1[^>]*>(.*?)<\/h1>/gi, '# $1\n\n');
+  md = md.replace(/<h2[^>]*>(.*?)<\/h2>/gi, '## $1\n\n');
+  md = md.replace(/<h3[^>]*>(.*?)<\/h3>/gi, '### $1\n\n');
+  md = md.replace(/<h4[^>]*>(.*?)<\/h4>/gi, '#### $1\n\n');
+
+  // Convert paragraphs
+  md = md.replace(/<p[^>]*>([\s\S]*?)<\/p>/gi, '$1\n\n');
+
+  // Convert links
+  md = md.replace(/<a[^>]*href="([^"]*)"[^>]*>(.*?)<\/a>/gi, '[$2]($1)');
+
+  // Convert images
+  md = md.replace(/<img[^>]*src="([^"]*)"[^>]*alt="([^"]*)"[^>]*\/?>/gi, '![$2]($1)');
+  md = md.replace(/<img[^>]*src="([^"]*)"[^>]*\/?>/gi, '![]($1)');
+
+  // Convert lists
+  md = md.replace(/<ul[^>]*>([\s\S]*?)<\/ul>/gi, (match, content) => {
+    return content.replace(/<li[^>]*>([\s\S]*?)<\/li>/gi, '- $1\n') + '\n';
+  });
+  md = md.replace(/<ol[^>]*>([\s\S]*?)<\/ol>/gi, (match, content) => {
+    let index = 0;
+    return content.replace(
+      /<li[^>]*>([\s\S]*?)<\/li>/gi,
+      (_itemMatch: string, itemContent: string) => `${++index}. ${itemContent}\n`,
+    ) + '\n';
+  });
+
+  // Convert bold and italic
+  md = md.replace(/<strong[^>]*>(.*?)<\/strong>/gi, '**$1**');
+  md = md.replace(/<b[^>]*>(.*?)<\/b>/gi, '**$1**');
+  md = md.replace(/<em[^>]*>(.*?)<\/em>/gi, '*$1*');
+  md = md.replace(/<i[^>]*>(.*?)<\/i>/gi, '*$1*');
+
+  // Convert code blocks
+  md = md.replace(/<pre[^>]*><code[^>]*>([\s\S]*?)<\/code><\/pre>/gi, '```\n$1\n```\n\n');
+  md = md.replace(/<code[^>]*>(.*?)<\/code>/gi, '`$1`');
+
+  // Convert blockquotes
+  md = md.replace(/<blockquote[^>]*>([\s\S]*?)<\/blockquote>/gi, (match, content) => {
+    return content.split('\n').map((line: string) => `> ${line}`).join('\n') + '\n\n';
+  });
+
+  // Convert line breaks
+  md = md.replace(/<br\s*\/?>/gi, '\n');
+
+  // Remove remaining HTML tags
+  md = md.replace(/<[^>]+>/g, '');
+
+  // Decode HTML entities
+  md = md.replace(/&nbsp;/g, ' ');
+  md = md.replace(/&lt;/g, '<');
+  md = md.replace(/&gt;/g, '>');
+  md = md.replace(/&amp;/g, '&');
+  md = md.replace(/&quot;/g, '"');
+
+  // Clean up extra whitespace
+  md = md.replace(/\n{3,}/g, '\n\n');
+  md = md.trim();
+
+  return md;
+}
+
+cli({
+  site: 'zhihu',
+  name: 'download',
+  description: '导出知乎文章为 Markdown 格式',
+  domain: 'zhuanlan.zhihu.com',
+  strategy: Strategy.COOKIE,
+  args: [
+    { name: 'url', required: true, help: 'Article URL (zhuanlan.zhihu.com/p/xxx)' },
+    { name: 'output', default: './zhihu-articles', help: 'Output directory' },
+    { name: 'download-images', type: 'boolean', default: false, help: 'Download images locally' },
+  ],
+  columns: ['title', 'author', 'status', 'size'],
+  func: async (page, kwargs) => {
+    const url = kwargs.url;
+    const output = kwargs.output;
+    const downloadImages = kwargs['download-images'];
+
+    // Navigate to article page
+    await page.goto(url);
+    await page.wait(3);
+
+    // Extract article content
+    const data = await page.evaluate(`
+      (() => {
+        const result = {
+          title: '',
+          author: '',
+          content: '',
+          publishTime: '',
+          images: []
+        };
+
+        // Get title
+        const titleEl = document.querySelector('.Post-Title, h1.ContentItem-title, .ArticleTitle');
+        result.title = titleEl?.textContent?.trim() || 'untitled';
+
+        // Get author
+        const authorEl = document.querySelector('.AuthorInfo-name, .UserLink-link');
+        result.author = authorEl?.textContent?.trim() || 'unknown';
+
+        // Get publish time
+        const timeEl = document.querySelector('.ContentItem-time, .Post-Time');
+        result.publishTime = timeEl?.textContent?.trim() || '';
+
+        // Get content HTML
+        const contentEl = document.querySelector('.Post-RichTextContainer, .RichText, .ArticleContent');
+        if (contentEl) {
+          result.content = contentEl.innerHTML;
+
+          // Extract image URLs
+          contentEl.querySelectorAll('img').forEach(img => {
+            const src = img.getAttribute('data-original') || img.getAttribute('data-actualsrc') || img.src;
+            if (src && !src.includes('data:image')) {
+              result.images.push(src);
+            }
+          });
+        }
+
+        return result;
+      })()
+    `);
+
+    if (!data || !data.content) {
+      return [{
+        title: 'Error',
+        author: '-',
+        status: 'failed',
+        size: 'Could not extract article content',
+      }];
+    }
+
+    // Create output directory
+    fs.mkdirSync(output, { recursive: true });
+
+    // Convert HTML to Markdown
+    let markdown = htmlToMarkdown(data.content);
+
+    // Create frontmatter
+    const frontmatter = [
+      '---',
+      `title: "${data.title.replace(/"/g, '\\"')}"`,
+      `author: "${data.author.replace(/"/g, '\\"')}"`,
+      `source: "${url}"`,
+      data.publishTime ? `date: "${data.publishTime}"` : '',
+      '---',
+      '',
+    ].filter(Boolean).join('\n');
+
+    // Download images if requested
+    if (downloadImages && data.images && data.images.length > 0) {
+      const imagesDir = path.join(output, 'images');
+      fs.mkdirSync(imagesDir, { recursive: true });
+
+      const cookies = await page.evaluate(`(() => document.cookie)()`);
+
+      for (let i = 0; i < data.images.length; i++) {
+        const imgUrl = data.images[i];
+        const ext = imgUrl.match(/\.(jpg|jpeg|png|gif|webp)/i)?.[1] || 'jpg';
+        const imgFilename = `img_${i + 1}.${ext}`;
+        const imgPath = path.join(imagesDir, imgFilename);
+
+        try {
+          await httpDownload(imgUrl, imgPath, {
+            cookies: typeof cookies === 'string' ? cookies : '',
+            timeout: 30000,
+          });
+
+          // Replace image URL in markdown with local path
+          markdown = markdown.replace(
+            new RegExp(imgUrl.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'), 'g'),
+            `./images/${imgFilename}`,
+          );
+        } catch {
+          // Keep original URL if download fails
+        }
+      }
+    }
+
+    // Write markdown file
+    const safeTitle = sanitizeFilename(data.title, 100);
+    const filename = `${safeTitle}.md`;
+    const filePath = path.join(output, filename);
+
+    const fullContent = frontmatter + '\n' + markdown;
+    fs.writeFileSync(filePath, fullContent, 'utf-8');
+
+    const size = Buffer.byteLength(fullContent, 'utf-8');
+
+    return [{
+      title: data.title,
+      author: data.author,
+      status: 'success',
+      size: formatBytes(size),
+    }];
+  },
+});

package/src/daemon.ts

@@ -0,0 +1,217 @@
+/**
+ * opencli micro-daemon — HTTP + WebSocket bridge between CLI and Chrome Extension.
+ *
+ * Architecture:
+ *   CLI → HTTP POST /command → daemon → WebSocket → Extension
+ *   Extension → WebSocket result → daemon → HTTP response → CLI
+ *
+ * Lifecycle:
+ *   - Auto-spawned by opencli on first browser command
+ *   - Auto-exits after 5 minutes of idle
+ *   - Listens on localhost:19825
+ */
+
+import { createServer, type IncomingMessage, type ServerResponse } from 'node:http';
+import { WebSocketServer, WebSocket } from 'ws';
+
+const PORT = parseInt(process.env.OPENCLI_DAEMON_PORT ?? '19825', 10);
+const IDLE_TIMEOUT = 5 * 60 * 1000; // 5 minutes
+
+// ─── State ───────────────────────────────────────────────────────────
+
+let extensionWs: WebSocket | null = null;
+const pending = new Map<string, {
+  resolve: (data: unknown) => void;
+  reject: (error: Error) => void;
+  timer: ReturnType<typeof setTimeout>;
+}>();
+let idleTimer: ReturnType<typeof setTimeout> | null = null;
+
+// Extension log ring buffer
+interface LogEntry { level: string; msg: string; ts: number; }
+const LOG_BUFFER_SIZE = 200;
+const logBuffer: LogEntry[] = [];
+
+function pushLog(entry: LogEntry): void {
+  logBuffer.push(entry);
+  if (logBuffer.length > LOG_BUFFER_SIZE) logBuffer.shift();
+}
+
+// ─── Idle auto-exit ──────────────────────────────────────────────────
+
+function resetIdleTimer(): void {
+  if (idleTimer) clearTimeout(idleTimer);
+  idleTimer = setTimeout(() => {
+    console.error('[daemon] Idle timeout, shutting down');
+    process.exit(0);
+  }, IDLE_TIMEOUT);
+}
+
+// ─── HTTP Server ─────────────────────────────────────────────────────
+
+function readBody(req: IncomingMessage): Promise<string> {
+  return new Promise((resolve, reject) => {
+    const chunks: Buffer[] = [];
+    req.on('data', (c: Buffer) => chunks.push(c));
+    req.on('end', () => resolve(Buffer.concat(chunks).toString('utf-8')));
+    req.on('error', reject);
+  });
+}
+
+function jsonResponse(res: ServerResponse, status: number, data: unknown): void {
+  res.writeHead(status, { 'Content-Type': 'application/json', 'Access-Control-Allow-Origin': '*' });
+  res.end(JSON.stringify(data));
+}
+
+async function handleRequest(req: IncomingMessage, res: ServerResponse): Promise<void> {
+  res.setHeader('Access-Control-Allow-Origin', '*');
+  res.setHeader('Access-Control-Allow-Methods', 'GET, POST, OPTIONS');
+  res.setHeader('Access-Control-Allow-Headers', 'Content-Type');
+  if (req.method === 'OPTIONS') { res.writeHead(204); res.end(); return; }
+
+  const url = req.url ?? '/';
+  const pathname = url.split('?')[0];
+
+  if (req.method === 'GET' && pathname === '/status') {
+    jsonResponse(res, 200, {
+      ok: true,
+      extensionConnected: extensionWs?.readyState === WebSocket.OPEN,
+      pending: pending.size,
+    });
+    return;
+  }
+
+  if (req.method === 'GET' && pathname === '/logs') {
+    const params = new URL(url, `http://localhost:${PORT}`).searchParams;
+    const level = params.get('level');
+    const filtered = level
+      ? logBuffer.filter(e => e.level === level)
+      : logBuffer;
+    jsonResponse(res, 200, { ok: true, logs: filtered });
+    return;
+  }
+
+  if (req.method === 'DELETE' && pathname === '/logs') {
+    logBuffer.length = 0;
+    jsonResponse(res, 200, { ok: true });
+    return;
+  }
+
+  if (req.method === 'POST' && url === '/command') {
+    resetIdleTimer();
+    try {
+      const body = JSON.parse(await readBody(req));
+      if (!body.id) {
+        jsonResponse(res, 400, { ok: false, error: 'Missing command id' });
+        return;
+      }
+
+      if (!extensionWs || extensionWs.readyState !== WebSocket.OPEN) {
+        jsonResponse(res, 503, { id: body.id, ok: false, error: 'Extension not connected. Please install the opencli Browser Bridge extension.' });
+        return;
+      }
+
+      const result = await new Promise<unknown>((resolve, reject) => {
+        const timer = setTimeout(() => {
+          pending.delete(body.id);
+          reject(new Error('Command timeout (30s)'));
+        }, 30000);
+        pending.set(body.id, { resolve, reject, timer });
+        extensionWs!.send(JSON.stringify(body));
+      });
+
+      jsonResponse(res, 200, result);
+    } catch (err) {
+      jsonResponse(res, err instanceof Error && err.message.includes('timeout') ? 408 : 400, {
+        ok: false,
+        error: err instanceof Error ? err.message : 'Invalid request',
+      });
+    }
+    return;
+  }
+
+  jsonResponse(res, 404, { error: 'Not found' });
+}
+
+// ─── WebSocket for Extension ─────────────────────────────────────────
+
+const httpServer = createServer((req, res) => { handleRequest(req, res).catch(() => { res.writeHead(500); res.end(); }); });
+const wss = new WebSocketServer({ server: httpServer, path: '/ext' });
+
+wss.on('connection', (ws) => {
+  console.error('[daemon] Extension connected');
+  extensionWs = ws;
+
+  ws.on('message', (data) => {
+    try {
+      const msg = JSON.parse(data.toString());
+
+      // Handle log messages from extension
+      if (msg.type === 'log') {
+        const prefix = msg.level === 'error' ? '❌' : msg.level === 'warn' ? '⚠️' : '📋';
+        console.error(`${prefix} [ext] ${msg.msg}`);
+        pushLog({ level: msg.level, msg: msg.msg, ts: msg.ts ?? Date.now() });
+        return;
+      }
+
+      // Handle command results
+      const p = pending.get(msg.id);
+      if (p) {
+        clearTimeout(p.timer);
+        pending.delete(msg.id);
+        p.resolve(msg);
+      }
+    } catch {
+      // Ignore malformed messages
+    }
+  });
+
+  ws.on('close', () => {
+    console.error('[daemon] Extension disconnected');
+    if (extensionWs === ws) {
+      extensionWs = null;
+      // Reject all pending requests since the extension is gone
+      for (const [id, p] of pending) {
+        clearTimeout(p.timer);
+        p.reject(new Error('Extension disconnected'));
+      }
+      pending.clear();
+    }
+  });
+
+  ws.on('error', () => {
+    if (extensionWs === ws) extensionWs = null;
+  });
+});
+
+// ─── Start ───────────────────────────────────────────────────────────
+
+httpServer.listen(PORT, '127.0.0.1', () => {
+  console.error(`[daemon] Listening on http://127.0.0.1:${PORT}`);
+  resetIdleTimer();
+});
+
+httpServer.on('error', (err: NodeJS.ErrnoException) => {
+  if (err.code === 'EADDRINUSE') {
+    console.error(`[daemon] Port ${PORT} already in use — another daemon is likely running. Exiting.`);
+    process.exit(0);
+  }
+  console.error('[daemon] Server error:', err.message);
+  process.exit(1);
+});
+
+// Graceful shutdown
+function shutdown(): void {
+  // Reject all pending requests so CLI doesn't hang
+  for (const [, p] of pending) {
+    clearTimeout(p.timer);
+    p.reject(new Error('Daemon shutting down'));
+  }
+  pending.clear();
+  if (extensionWs) extensionWs.close();
+  httpServer.close();
+  process.exit(0);
+}
+
+process.on('SIGTERM', shutdown);
+process.on('SIGINT', shutdown);