@jackwener/opencli 1.6.2 → 1.6.4

package/src/clis/weixin/download.ts

@@ -1,382 +0,0 @@
-/**
- * WeChat article download — export WeChat Official Account articles to Markdown.
- *
- * Ported from jackwener/wechat-article-to-markdown (JS version) to OpenCLI adapter.
- *
- * Usage:
- *   opencli weixin download --url "https://mp.weixin.qq.com/s/xxx" --output ./weixin
- */
-
-import { cli, Strategy } from '../../registry.js';
-import { downloadArticle } from '../../download/article-download.js';
-
-// ============================================================
-// URL Normalization
-// ============================================================
-
-/**
- * Normalize a pasted WeChat article URL.
- */
-export function normalizeWechatUrl(raw: string): string {
-  let s = (raw || '').trim();
-  if (!s) return s;
-
-  // Strip wrapping quotes / angle brackets
-  if ((s.startsWith('"') && s.endsWith('"')) || (s.startsWith("'") && s.endsWith("'"))) {
-    s = s.slice(1, -1).trim();
-  }
-  if (s.startsWith('<') && s.endsWith('>')) {
-    s = s.slice(1, -1).trim();
-  }
-
-  // Remove backslash escapes before URL-significant characters
-  s = s.replace(/\\+([:/&?=#%])/g, '$1');
-
-  // Decode HTML entities
-  s = s.replace(/&amp;/g, '&').replace(/&lt;/g, '<').replace(/&gt;/g, '>').replace(/&quot;/g, '"');
-
-  // Allow bare hostnames
-  if (s.startsWith('mp.weixin.qq.com/') || s.startsWith('//mp.weixin.qq.com/')) {
-    s = 'https://' + s.replace(/^\/+/, '');
-  }
-
-  // Force https for mp.weixin.qq.com
-  try {
-    const parsed = new URL(s);
-    if (['http:', 'https:'].includes(parsed.protocol) && parsed.hostname.toLowerCase() === 'mp.weixin.qq.com') {
-      parsed.protocol = 'https:';
-      s = parsed.toString();
-    }
-  } catch {
-    // Ignore parse errors
-  }
-
-  return s;
-}
-
-/**
- * Format a WeChat article timestamp as a UTC+8 datetime string.
- * Accepts either Unix seconds or milliseconds.
- */
-export function formatWechatTimestamp(rawTimestamp: string): string {
-  const ts = Number.parseInt(rawTimestamp, 10);
-  if (!Number.isFinite(ts) || ts <= 0) return '';
-
-  const timestampMs = rawTimestamp.length === 13 ? ts : ts * 1000;
-  const d = new Date(timestampMs);
-  const pad = (n: number): string => String(n).padStart(2, '0');
-  const utc8 = new Date(d.getTime() + 8 * 3600 * 1000);
-
-  return (
-    `${utc8.getUTCFullYear()}-` +
-    `${pad(utc8.getUTCMonth() + 1)}-` +
-    `${pad(utc8.getUTCDate())} ` +
-    `${pad(utc8.getUTCHours())}:` +
-    `${pad(utc8.getUTCMinutes())}:` +
-    `${pad(utc8.getUTCSeconds())}`
-  );
-}
-
-/**
- * Extract the raw create_time value from supported WeChat inline script formats.
- */
-export function extractWechatCreateTimeValue(htmlStr: string): string {
-  const jsDecodeMatch = htmlStr.match(
-    /create_time\s*:\s*JsDecode\('([^']+)'\)(?=[\s,;}]|$)/,
-  );
-  if (jsDecodeMatch) return jsDecodeMatch[1];
-
-  const directValueMatch = htmlStr.match(
-    /create_time\s*[:=]\s*(?:"([^"]+)"|'([^']+)'|([0-9A-Za-z]+))(?=[\s,;}]|$)/,
-  );
-  if (!directValueMatch) return '';
-
-  return directValueMatch[1] || directValueMatch[2] || directValueMatch[3] || '';
-}
-
-/**
- * Extract the publish time from DOM text first, then fall back to numeric create_time values.
- */
-export function extractWechatPublishTime(
-  publishTimeText: string | null | undefined,
-  htmlStr: string,
-): string {
-  const normalizedPublishTime = (publishTimeText || '').trim();
-  if (normalizedPublishTime) return normalizedPublishTime;
-
-  const rawCreateTime = extractWechatCreateTimeValue(htmlStr);
-  if (!/^\d{10}$|^\d{13}$/.test(rawCreateTime)) return '';
-
-  return formatWechatTimestamp(rawCreateTime);
-}
-
-/**
- * Detect WeChat anti-bot / verification gate pages before we try to parse the article.
- */
-export function detectWechatAccessIssue(
-  pageText: string | null | undefined,
-  htmlStr: string,
-): string {
-  const normalizedText = (pageText || '').replace(/\s+/g, ' ').trim();
-
-  if (
-    /环境异常/.test(normalizedText) &&
-    /(完成验证后即可继续访问|去验证)/.test(normalizedText)
-  ) {
-    return 'environment verification required';
-  }
-
-  if (/secitptpage\/verify\.html/.test(htmlStr) || /id=["']js_verify["']/.test(htmlStr)) {
-    return 'environment verification required';
-  }
-
-  return '';
-}
-
-export function pickFirstWechatMetaText(...candidates: Array<string | null | undefined>): string {
-  for (const candidate of candidates) {
-    const normalized = (candidate || '').replace(/\s+/g, ' ').trim();
-    if (normalized && normalized !== 'Name cleared') return normalized;
-  }
-  return '';
-}
-
-/**
- * Build a self-contained helper for execution inside page.evaluate().
- */
-export function buildExtractWechatPublishTimeJs(): string {
-  return `(${function extractWechatPublishTimeInPage(
-    publishTimeText: string | null | undefined,
-    htmlStr: string,
-  ) {
-    function formatWechatTimestamp(rawTimestamp: string) {
-      const ts = Number.parseInt(rawTimestamp, 10);
-      if (!Number.isFinite(ts) || ts <= 0) return '';
-
-      const timestampMs = rawTimestamp.length === 13 ? ts : ts * 1000;
-      const d = new Date(timestampMs);
-      const pad = (n: number) => String(n).padStart(2, '0');
-      const utc8 = new Date(d.getTime() + 8 * 3600 * 1000);
-
-      return (
-        `${utc8.getUTCFullYear()}-` +
-        `${pad(utc8.getUTCMonth() + 1)}-` +
-        `${pad(utc8.getUTCDate())} ` +
-        `${pad(utc8.getUTCHours())}:` +
-        `${pad(utc8.getUTCMinutes())}:` +
-        `${pad(utc8.getUTCSeconds())}`
-      );
-    }
-
-    function extractWechatCreateTimeValue(html: string) {
-      const jsDecodeMatch = html.match(
-        /create_time\s*:\s*JsDecode\('([^']+)'\)(?=[\s,;}]|$)/,
-      );
-      if (jsDecodeMatch) return jsDecodeMatch[1];
-
-      const directValueMatch = html.match(
-        /create_time\s*[:=]\s*(?:"([^"]+)"|'([^']+)'|([0-9A-Za-z]+))(?=[\s,;}]|$)/,
-      );
-      if (!directValueMatch) return '';
-
-      return directValueMatch[1] || directValueMatch[2] || directValueMatch[3] || '';
-    }
-
-    const normalizedPublishTime = (publishTimeText || '').trim();
-    if (normalizedPublishTime) return normalizedPublishTime;
-
-    const rawCreateTime = extractWechatCreateTimeValue(htmlStr);
-    if (!/^\d{10}$|^\d{13}$/.test(rawCreateTime)) return '';
-
-    return formatWechatTimestamp(rawCreateTime);
-  }.toString()})`;
-}
-
-/**
- * Build a self-contained access-issue detector for execution inside page.evaluate().
- */
-export function buildDetectWechatAccessIssueJs(): string {
-  return `(${function detectWechatAccessIssueInPage(
-    pageText: string | null | undefined,
-    htmlStr: string,
-  ) {
-    const normalizedText = (pageText || '').replace(/\s+/g, ' ').trim();
-
-    if (
-      /环境异常/.test(normalizedText) &&
-      /(完成验证后即可继续访问|去验证)/.test(normalizedText)
-    ) {
-      return 'environment verification required';
-    }
-
-    if (/secitptpage\/verify\.html/.test(htmlStr) || /id=["']js_verify["']/.test(htmlStr)) {
-      return 'environment verification required';
-    }
-
-    return '';
-  }.toString()})`;
-}
-
-// ============================================================
-// CLI Registration
-// ============================================================
-
-cli({
-  site: 'weixin',
-  name: 'download',
-  description: '下载微信公众号文章为 Markdown 格式',
-  domain: 'mp.weixin.qq.com',
-  strategy: Strategy.COOKIE,
-  args: [
-    { name: 'url', required: true, help: 'WeChat article URL (mp.weixin.qq.com/s/xxx)' },
-    { name: 'output', default: './weixin-articles', help: 'Output directory' },
-    { name: 'download-images', type: 'boolean', default: true, help: 'Download images locally' },
-  ],
-  columns: ['title', 'author', 'publish_time', 'status', 'size'],
-  func: async (page, kwargs) => {
-    const rawUrl = kwargs.url;
-    const url = normalizeWechatUrl(rawUrl);
-
-    if (!url.startsWith('https://mp.weixin.qq.com/')) {
-      return [{ title: 'Error', author: '-', publish_time: '-', status: 'invalid URL', size: '-' }];
-    }
-
-    // Navigate and wait for content to load
-    await page.goto(url);
-    await page.wait(5);
-
-    // Extract article data in browser context
-    const data = await page.evaluate(`
-      (() => {
-        const result = {
-          title: '',
-          author: '',
-          publishTime: '',
-          errorHint: '',
-          contentHtml: '',
-          codeBlocks: [],
-          imageUrls: []
-        };
-
-        const pickFirstText = (...selectors) => {
-          for (const selector of selectors) {
-            const text = document.querySelector(selector)?.textContent?.replace(/\\s+/g, ' ').trim() || '';
-            if (text && text !== 'Name cleared') return text;
-          }
-          return '';
-        };
-
-        // WeChat has multiple article templates. Newer pages use #js_text_title.
-        result.title = pickFirstText(
-          '#activity-name',
-          '#js_text_title',
-          '.rich_media_title',
-        );
-
-        result.author = pickFirstText(
-          '#js_name',
-          '.wx_follow_nickname',
-          '#profileBt .profile_nickname',
-          '.rich_media_meta.rich_media_meta_nickname',
-          '.rich_media_meta_nickname',
-        );
-
-        // Publish time: prefer the rendered DOM text, then fall back to numeric create_time values.
-        const publishTimeEl = document.querySelector('#publish_time');
-        const extractWechatPublishTime = ${buildExtractWechatPublishTimeJs()};
-        result.publishTime = extractWechatPublishTime(
-          publishTimeEl ? publishTimeEl.textContent : '',
-          document.documentElement.innerHTML,
-        );
-
-        const detectWechatAccessIssue = ${buildDetectWechatAccessIssueJs()};
-        result.errorHint = detectWechatAccessIssue(
-          document.body ? document.body.innerText : '',
-          document.documentElement.innerHTML,
-        );
-        if (result.errorHint) return result;
-
-        // Content processing
-        const contentEl = document.querySelector('#js_content');
-        if (!contentEl) return result;
-
-        // Fix lazy-loaded images: data-src -> src
-        contentEl.querySelectorAll('img').forEach(img => {
-          const dataSrc = img.getAttribute('data-src');
-          if (dataSrc) img.setAttribute('src', dataSrc);
-        });
-
-        // Extract code blocks with placeholder replacement
-        const codeBlocks = [];
-        contentEl.querySelectorAll('.code-snippet__fix').forEach(el => {
-          el.querySelectorAll('.code-snippet__line-index').forEach(li => li.remove());
-          const pre = el.querySelector('pre[data-lang]');
-          const lang = pre ? (pre.getAttribute('data-lang') || '') : '';
-          const lines = [];
-          el.querySelectorAll('code').forEach(codeTag => {
-            const text = codeTag.textContent;
-            if (/^[ce]?ounter\\(line/.test(text)) return;
-            lines.push(text);
-          });
-          if (lines.length === 0) lines.push(el.textContent);
-          const placeholder = 'CODEBLOCK-PLACEHOLDER-' + codeBlocks.length;
-          codeBlocks.push({ lang, code: lines.join('\\n') });
-          const p = document.createElement('p');
-          p.textContent = placeholder;
-          el.replaceWith(p);
-        });
-        result.codeBlocks = codeBlocks;
-
-        // Remove noise elements
-        ['script', 'style', '.qr_code_pc', '.reward_area'].forEach(sel => {
-          contentEl.querySelectorAll(sel).forEach(tag => tag.remove());
-        });
-
-        // Collect image URLs (deduplicated)
-        const seen = new Set();
-        contentEl.querySelectorAll('img[src]').forEach(img => {
-          const src = img.getAttribute('src');
-          if (src && !seen.has(src)) {
-            seen.add(src);
-            result.imageUrls.push(src);
-          }
-        });
-
-        result.contentHtml = contentEl.innerHTML;
-        return result;
-      })()
-    `);
-
-    if (data?.errorHint === 'environment verification required') {
-      return [{
-        title: 'Error',
-        author: '-',
-        publish_time: '-',
-        status: 'failed — verification required in WeChat browser page',
-        size: '-',
-      }];
-    }
-
-    return downloadArticle(
-      {
-        title: data?.title || '',
-        author: data?.author,
-        publishTime: data?.publishTime,
-        sourceUrl: url,
-        contentHtml: data?.contentHtml || '',
-        codeBlocks: data?.codeBlocks,
-        imageUrls: data?.imageUrls,
-      },
-      {
-        output: kwargs.output,
-        downloadImages: kwargs['download-images'],
-        imageHeaders: { Referer: 'https://mp.weixin.qq.com/' },
-        frontmatterLabels: { author: '公众号' },
-        detectImageExt: (url) => {
-          const m = url.match(/wx_fmt=(\w+)/) || url.match(/\.(\w{3,4})(?:\?|$)/);
-          return m ? m[1] : 'png';
-        },
-      },
-    );
-  },
-});

package/src/clis/weread/book.ts

@@ -1,271 +0,0 @@
-import { cli, Strategy } from '../../registry.js';
-import { CliError } from '../../errors.js';
-import type { IPage } from '../../types.js';
-import {
-  fetchPrivateApi,
-  fetchWebApi,
-  resolveShelfReader,
-  WEREAD_UA,
-  WEREAD_WEB_ORIGIN,
-} from './utils.js';
-
-interface ReaderFallbackResult {
-  title: string;
-  author: string;
-  publisher: string;
-  intro: string;
-  category: string;
-  rating: string;
-  metadataReady: boolean;
-}
-
-interface SearchHtmlEntry {
-  title: string;
-  author: string;
-  url: string;
-}
-
-function decodeHtmlText(value: string): string {
-  return value
-    .replace(/<[^>]+>/g, '')
-    .replace(/&#x([0-9a-fA-F]+);/gi, (_, n) => String.fromCharCode(parseInt(n, 16)))
-    .replace(/&#(\d+);/g, (_, n) => String.fromCharCode(Number(n)))
-    .replace(/&nbsp;/g, ' ')
-    .replace(/&amp;/g, '&')
-    .replace(/&quot;/g, '"')
-    .trim();
-}
-
-function normalizeSearchText(value: string): string {
-  return value.replace(/\s+/g, ' ').trim();
-}
-
-function buildSearchIdentity(title: string, author: string): string {
-  return `${normalizeSearchText(title)}\u0000${normalizeSearchText(author)}`;
-}
-
-function countSearchTitles(entries: Array<{ title: string }>): Map<string, number> {
-  const counts = new Map<string, number>();
-  for (const entry of entries) {
-    const key = normalizeSearchText(entry.title);
-    if (!key) continue;
-    counts.set(key, (counts.get(key) || 0) + 1);
-  }
-  return counts;
-}
-
-function countSearchIdentities(entries: Array<{ title: string; author: string }>): Map<string, number> {
-  const counts = new Map<string, number>();
-  for (const entry of entries) {
-    const key = buildSearchIdentity(entry.title, entry.author);
-    if (!normalizeSearchText(entry.title) || !normalizeSearchText(entry.author)) continue;
-    counts.set(key, (counts.get(key) || 0) + 1);
-  }
-  return counts;
-}
-
-/**
- * Reuse the public search page as a last-resort reader URL source when the
- * cached shelf page cannot provide a trustworthy bookId-to-reader mapping.
- */
-async function resolveSearchReaderUrl(title: string, author: string): Promise<string> {
-  const normalizedTitle = normalizeSearchText(title);
-  const normalizedAuthor = normalizeSearchText(author);
-  if (!normalizedTitle) return '';
-
-  try {
-    const [data, htmlEntries] = await Promise.all([
-      fetchWebApi('/search/global', { keyword: normalizedTitle }),
-      (async (): Promise<SearchHtmlEntry[]> => {
-        const url = new URL('/web/search/books', WEREAD_WEB_ORIGIN);
-        url.searchParams.set('keyword', normalizedTitle);
-
-        const resp = await fetch(url.toString(), {
-          headers: { 'User-Agent': WEREAD_UA },
-        });
-        if (!resp.ok) return [];
-
-        const html = await resp.text();
-        const items = Array.from(
-          html.matchAll(/<li[^>]*class="wr_bookList_item"[^>]*>([\s\S]*?)<\/li>/g),
-        );
-
-        return items.map((match) => {
-          const chunk = match[1];
-          const hrefMatch = chunk.match(/<a[^>]*href="([^"]+)"[^>]*class="wr_bookList_item_link"[^>]*>|<a[^>]*class="wr_bookList_item_link"[^>]*href="([^"]+)"[^>]*>/);
-          const titleMatch = chunk.match(/<p[^>]*class="wr_bookList_item_title"[^>]*>([\s\S]*?)<\/p>/);
-          const authorMatch = chunk.match(/<p[^>]*class="wr_bookList_item_author"[^>]*>([\s\S]*?)<\/p>/);
-          const href = hrefMatch?.[1] || hrefMatch?.[2] || '';
-
-          return {
-            title: decodeHtmlText(titleMatch?.[1] || ''),
-            author: decodeHtmlText(authorMatch?.[1] || ''),
-            url: href ? new URL(href, WEREAD_WEB_ORIGIN).toString() : '',
-          };
-        }).filter((entry) => entry.title && entry.url);
-      })(),
-    ]);
-
-    const books: any[] = Array.isArray(data?.books) ? data.books : [];
-    const apiIdentityCounts = countSearchIdentities(
-      books.map((item: any) => ({
-        title: item.bookInfo?.title ?? '',
-        author: item.bookInfo?.author ?? '',
-      })),
-    );
-    const htmlIdentityCounts = countSearchIdentities(
-      htmlEntries.filter((entry) => entry.author),
-    );
-    const identityKey = buildSearchIdentity(normalizedTitle, normalizedAuthor);
-    if (
-      normalizedAuthor &&
-      (apiIdentityCounts.get(identityKey) || 0) === 1 &&
-      (htmlIdentityCounts.get(identityKey) || 0) === 1
-    ) {
-      const exactMatch = htmlEntries.find((entry) => buildSearchIdentity(entry.title, entry.author) === identityKey);
-      if (exactMatch?.url) return exactMatch.url;
-    }
-
-    const sameTitleHtmlEntries = htmlEntries.filter((entry) => normalizeSearchText(entry.title) === normalizedTitle);
-    if (normalizedAuthor && sameTitleHtmlEntries.some((entry) => normalizeSearchText(entry.author))) {
-      return '';
-    }
-
-    const apiTitleCounts = countSearchTitles(
-      books.map((item: any) => ({ title: item.bookInfo?.title ?? '' })),
-    );
-    const htmlTitleCounts = countSearchTitles(htmlEntries);
-    if ((apiTitleCounts.get(normalizedTitle) || 0) !== 1 || (htmlTitleCounts.get(normalizedTitle) || 0) !== 1) {
-      return '';
-    }
-
-    return htmlEntries.find((entry) => normalizeSearchText(entry.title) === normalizedTitle)?.url || '';
-  } catch {
-    return '';
-  }
-}
-
-/**
- * Read visible book metadata from the web reader cover/flyleaf page.
- * This path is used as a fallback when the private API session has expired.
- */
-async function loadReaderFallbackResult(page: IPage, readerUrl: string): Promise<ReaderFallbackResult> {
-  await page.goto(readerUrl);
-  await page.wait({ selector: '.horizontalReaderCoverPage_content_bookTitle, .wr_flyleaf_page_bookInfo_bookTitle', timeout: 10 });
-
-  const result = await page.evaluate(`
-    (() => {
-      const text = (node) => node?.textContent?.trim() || '';
-      const bodyText = document.body?.innerText?.replace(/\\s+/g, ' ').trim() || '';
-      const titleSelector = '.horizontalReaderCoverPage_content_bookTitle, .wr_flyleaf_page_bookInfo_bookTitle';
-      const authorSelector = '.horizontalReaderCoverPage_content_author, .wr_flyleaf_page_bookInfo_author';
-      const extractRating = () => {
-        const match = bodyText.match(/微信读书推荐值\\s*([0-9.]+%)/);
-        return match ? match[1] : '';
-      };
-      const extractPublisher = () => {
-        const direct = text(document.querySelector('.introDialog_content_pub_line'));
-        return direct.startsWith('出版社') ? direct.replace(/^出版社\\s*/, '').trim() : '';
-      };
-      const extractIntro = () => {
-        const selectors = [
-          '.horizontalReaderCoverPage_content_bookInfo_intro',
-          '.wr_flyleaf_page_bookIntro_content',
-          '.introDialog_content_intro_para',
-        ];
-        for (const selector of selectors) {
-          const value = text(document.querySelector(selector));
-          if (value) return value;
-        }
-        return '';
-      };
-
-      const categorySource = Array.from(document.scripts)
-        .map((script) => script.textContent || '')
-        .find((scriptText) => scriptText.includes('"category"')) || '';
-      const categoryMatch = categorySource.match(/"category"\\s*:\\s*"([^"]+)"/);
-      const title = text(document.querySelector(titleSelector));
-      const author = text(document.querySelector(authorSelector));
-
-      return {
-        title,
-        author,
-        publisher: extractPublisher(),
-        intro: extractIntro(),
-        category: categoryMatch ? categoryMatch[1].trim() : '',
-        rating: extractRating(),
-        metadataReady: Boolean(title || author),
-      };
-    })()
-  `) as Partial<ReaderFallbackResult>;
-
-  return {
-    title: String(result?.title || '').trim(),
-    author: String(result?.author || '').trim(),
-    publisher: String(result?.publisher || '').trim(),
-    intro: String(result?.intro || '').trim(),
-    category: String(result?.category || '').trim(),
-    rating: String(result?.rating || '').trim(),
-    metadataReady: result?.metadataReady === true,
-  };
-}
-
-cli({
-  site: 'weread',
-  name: 'book',
-  description: 'View book details on WeRead',
-  domain: 'weread.qq.com',
-  strategy: Strategy.COOKIE,
-  args: [
-    { name: 'book-id', positional: true, required: true, help: 'Book ID from search or shelf results' },
-  ],
-  columns: ['title', 'author', 'publisher', 'intro', 'category', 'rating'],
-  func: async (page: IPage, args) => {
-    const bookId = String(args['book-id'] || '').trim();
-
-    try {
-      const data = await fetchPrivateApi(page, '/book/info', { bookId });
-      // newRating is 0-1000 scale per community docs; needs runtime verification
-      const rating = data.newRating ? `${(data.newRating / 10).toFixed(1)}%` : '-';
-      return [{
-        title: data.title ?? '',
-        author: data.author ?? '',
-        publisher: data.publisher ?? '',
-        intro: data.intro ?? '',
-        category: data.category ?? '',
-        rating,
-      }];
-    } catch (error) {
-      if (!(error instanceof CliError) || error.code !== 'AUTH_REQUIRED') {
-        throw error;
-      }
-
-      const { readerUrl: resolvedReaderUrl, snapshot } = await resolveShelfReader(page, bookId);
-      let readerUrl = resolvedReaderUrl;
-      if (!readerUrl) {
-        const cachedBook = snapshot.rawBooks.find((book) => String(book?.bookId || '').trim() === bookId);
-        readerUrl = await resolveSearchReaderUrl(
-          String(cachedBook?.title || ''),
-          String(cachedBook?.author || ''),
-        );
-      }
-      if (!readerUrl) {
-        throw error;
-      }
-
-      const data = await loadReaderFallbackResult(page, readerUrl);
-      if (!data.metadataReady || !data.title) {
-        throw error;
-      }
-
-      return [{
-        title: data.title,
-        author: data.author,
-        publisher: data.publisher,
-        intro: data.intro,
-        category: data.category,
-        rating: data.rating,
-      }];
-    }
-  },
-});