@huyuan-ai/cli 1.0.0 → 1.0.2

package/vendor/opencli/dist/clis/weixin/download.js

@@ -0,0 +1,321 @@
+/**
+ * WeChat article download — export WeChat Official Account articles to Markdown.
+ *
+ * Ported from jackwener/wechat-article-to-markdown (JS version) to OpenCLI adapter.
+ *
+ * Usage:
+ *   opencli weixin download --url "https://mp.weixin.qq.com/s/xxx" --output ./weixin
+ */
+import { cli, Strategy } from '../../registry.js';
+import { downloadArticle } from '../../download/article-download.js';
+// ============================================================
+// URL Normalization
+// ============================================================
+/**
+ * Normalize a pasted WeChat article URL.
+ */
+export function normalizeWechatUrl(raw) {
+    let s = (raw || '').trim();
+    if (!s)
+        return s;
+    // Strip wrapping quotes / angle brackets
+    if ((s.startsWith('"') && s.endsWith('"')) || (s.startsWith("'") && s.endsWith("'"))) {
+        s = s.slice(1, -1).trim();
+    }
+    if (s.startsWith('<') && s.endsWith('>')) {
+        s = s.slice(1, -1).trim();
+    }
+    // Remove backslash escapes before URL-significant characters
+    s = s.replace(/\\+([:/&?=#%])/g, '$1');
+    // Decode HTML entities
+    s = s.replace(/&amp;/g, '&').replace(/&lt;/g, '<').replace(/&gt;/g, '>').replace(/&quot;/g, '"');
+    // Allow bare hostnames
+    if (s.startsWith('mp.weixin.qq.com/') || s.startsWith('//mp.weixin.qq.com/')) {
+        s = 'https://' + s.replace(/^\/+/, '');
+    }
+    // Force https for mp.weixin.qq.com
+    try {
+        const parsed = new URL(s);
+        if (['http:', 'https:'].includes(parsed.protocol) && parsed.hostname.toLowerCase() === 'mp.weixin.qq.com') {
+            parsed.protocol = 'https:';
+            s = parsed.toString();
+        }
+    }
+    catch {
+        // Ignore parse errors
+    }
+    return s;
+}
+/**
+ * Format a WeChat article timestamp as a UTC+8 datetime string.
+ * Accepts either Unix seconds or milliseconds.
+ */
+export function formatWechatTimestamp(rawTimestamp) {
+    const ts = Number.parseInt(rawTimestamp, 10);
+    if (!Number.isFinite(ts) || ts <= 0)
+        return '';
+    const timestampMs = rawTimestamp.length === 13 ? ts : ts * 1000;
+    const d = new Date(timestampMs);
+    const pad = (n) => String(n).padStart(2, '0');
+    const utc8 = new Date(d.getTime() + 8 * 3600 * 1000);
+    return (`${utc8.getUTCFullYear()}-` +
+        `${pad(utc8.getUTCMonth() + 1)}-` +
+        `${pad(utc8.getUTCDate())} ` +
+        `${pad(utc8.getUTCHours())}:` +
+        `${pad(utc8.getUTCMinutes())}:` +
+        `${pad(utc8.getUTCSeconds())}`);
+}
+/**
+ * Extract the raw create_time value from supported WeChat inline script formats.
+ */
+export function extractWechatCreateTimeValue(htmlStr) {
+    const jsDecodeMatch = htmlStr.match(/create_time\s*:\s*JsDecode\('([^']+)'\)(?=[\s,;}]|$)/);
+    if (jsDecodeMatch)
+        return jsDecodeMatch[1];
+    const directValueMatch = htmlStr.match(/create_time\s*[:=]\s*(?:"([^"]+)"|'([^']+)'|([0-9A-Za-z]+))(?=[\s,;}]|$)/);
+    if (!directValueMatch)
+        return '';
+    return directValueMatch[1] || directValueMatch[2] || directValueMatch[3] || '';
+}
+/**
+ * Extract the publish time from DOM text first, then fall back to numeric create_time values.
+ */
+export function extractWechatPublishTime(publishTimeText, htmlStr) {
+    const normalizedPublishTime = (publishTimeText || '').trim();
+    if (normalizedPublishTime)
+        return normalizedPublishTime;
+    const rawCreateTime = extractWechatCreateTimeValue(htmlStr);
+    if (!/^\d{10}$|^\d{13}$/.test(rawCreateTime))
+        return '';
+    return formatWechatTimestamp(rawCreateTime);
+}
+/**
+ * Detect WeChat anti-bot / verification gate pages before we try to parse the article.
+ */
+export function detectWechatAccessIssue(pageText, htmlStr) {
+    const normalizedText = (pageText || '').replace(/\s+/g, ' ').trim();
+    if (/环境异常/.test(normalizedText) &&
+        /(完成验证后即可继续访问|去验证)/.test(normalizedText)) {
+        return 'environment verification required';
+    }
+    if (/secitptpage\/verify\.html/.test(htmlStr) || /id=["']js_verify["']/.test(htmlStr)) {
+        return 'environment verification required';
+    }
+    return '';
+}
+export function pickFirstWechatMetaText(...candidates) {
+    for (const candidate of candidates) {
+        const normalized = (candidate || '').replace(/\s+/g, ' ').trim();
+        if (normalized && normalized !== 'Name cleared')
+            return normalized;
+    }
+    return '';
+}
+/**
+ * Build a self-contained helper for execution inside page.evaluate().
+ */
+export function buildExtractWechatPublishTimeJs() {
+    return `(${function extractWechatPublishTimeInPage(publishTimeText, htmlStr) {
+        function formatWechatTimestamp(rawTimestamp) {
+            const ts = Number.parseInt(rawTimestamp, 10);
+            if (!Number.isFinite(ts) || ts <= 0)
+                return '';
+            const timestampMs = rawTimestamp.length === 13 ? ts : ts * 1000;
+            const d = new Date(timestampMs);
+            const pad = (n) => String(n).padStart(2, '0');
+            const utc8 = new Date(d.getTime() + 8 * 3600 * 1000);
+            return (`${utc8.getUTCFullYear()}-` +
+                `${pad(utc8.getUTCMonth() + 1)}-` +
+                `${pad(utc8.getUTCDate())} ` +
+                `${pad(utc8.getUTCHours())}:` +
+                `${pad(utc8.getUTCMinutes())}:` +
+                `${pad(utc8.getUTCSeconds())}`);
+        }
+        function extractWechatCreateTimeValue(html) {
+            const jsDecodeMatch = html.match(/create_time\s*:\s*JsDecode\('([^']+)'\)(?=[\s,;}]|$)/);
+            if (jsDecodeMatch)
+                return jsDecodeMatch[1];
+            const directValueMatch = html.match(/create_time\s*[:=]\s*(?:"([^"]+)"|'([^']+)'|([0-9A-Za-z]+))(?=[\s,;}]|$)/);
+            if (!directValueMatch)
+                return '';
+            return directValueMatch[1] || directValueMatch[2] || directValueMatch[3] || '';
+        }
+        const normalizedPublishTime = (publishTimeText || '').trim();
+        if (normalizedPublishTime)
+            return normalizedPublishTime;
+        const rawCreateTime = extractWechatCreateTimeValue(htmlStr);
+        if (!/^\d{10}$|^\d{13}$/.test(rawCreateTime))
+            return '';
+        return formatWechatTimestamp(rawCreateTime);
+    }.toString()})`;
+}
+/**
+ * Build a self-contained access-issue detector for execution inside page.evaluate().
+ */
+export function buildDetectWechatAccessIssueJs() {
+    return `(${function detectWechatAccessIssueInPage(pageText, htmlStr) {
+        const normalizedText = (pageText || '').replace(/\s+/g, ' ').trim();
+        if (/环境异常/.test(normalizedText) &&
+            /(完成验证后即可继续访问|去验证)/.test(normalizedText)) {
+            return 'environment verification required';
+        }
+        if (/secitptpage\/verify\.html/.test(htmlStr) || /id=["']js_verify["']/.test(htmlStr)) {
+            return 'environment verification required';
+        }
+        return '';
+    }.toString()})`;
+}
+// ============================================================
+// CLI Registration
+// ============================================================
+cli({
+    site: 'weixin',
+    name: 'download',
+    description: '下载微信公众号文章为 Markdown 格式',
+    domain: 'mp.weixin.qq.com',
+    strategy: Strategy.COOKIE,
+    args: [
+        { name: 'url', required: true, help: 'WeChat article URL (mp.weixin.qq.com/s/xxx)' },
+        { name: 'output', default: './weixin-articles', help: 'Output directory' },
+        { name: 'download-images', type: 'boolean', default: true, help: 'Download images locally' },
+    ],
+    columns: ['title', 'author', 'publish_time', 'status', 'size'],
+    func: async (page, kwargs) => {
+        const rawUrl = kwargs.url;
+        const url = normalizeWechatUrl(rawUrl);
+        if (!url.startsWith('https://mp.weixin.qq.com/')) {
+            return [{ title: 'Error', author: '-', publish_time: '-', status: 'invalid URL', size: '-' }];
+        }
+        // Navigate and wait for content to load
+        await page.goto(url);
+        await page.wait(5);
+        // Extract article data in browser context
+        const data = await page.evaluate(`
+      (() => {
+        const result = {
+          title: '',
+          author: '',
+          publishTime: '',
+          errorHint: '',
+          contentHtml: '',
+          codeBlocks: [],
+          imageUrls: []
+        };
+
+        const pickFirstText = (...selectors) => {
+          for (const selector of selectors) {
+            const text = document.querySelector(selector)?.textContent?.replace(/\\s+/g, ' ').trim() || '';
+            if (text && text !== 'Name cleared') return text;
+          }
+          return '';
+        };
+
+        // WeChat has multiple article templates. Newer pages use #js_text_title.
+        result.title = pickFirstText(
+          '#activity-name',
+          '#js_text_title',
+          '.rich_media_title',
+        );
+
+        result.author = pickFirstText(
+          '#js_name',
+          '.wx_follow_nickname',
+          '#profileBt .profile_nickname',
+          '.rich_media_meta.rich_media_meta_nickname',
+          '.rich_media_meta_nickname',
+        );
+
+        // Publish time: prefer the rendered DOM text, then fall back to numeric create_time values.
+        const publishTimeEl = document.querySelector('#publish_time');
+        const extractWechatPublishTime = ${buildExtractWechatPublishTimeJs()};
+        result.publishTime = extractWechatPublishTime(
+          publishTimeEl ? publishTimeEl.textContent : '',
+          document.documentElement.innerHTML,
+        );
+
+        const detectWechatAccessIssue = ${buildDetectWechatAccessIssueJs()};
+        result.errorHint = detectWechatAccessIssue(
+          document.body ? document.body.innerText : '',
+          document.documentElement.innerHTML,
+        );
+        if (result.errorHint) return result;
+
+        // Content processing
+        const contentEl = document.querySelector('#js_content');
+        if (!contentEl) return result;
+
+        // Fix lazy-loaded images: data-src -> src
+        contentEl.querySelectorAll('img').forEach(img => {
+          const dataSrc = img.getAttribute('data-src');
+          if (dataSrc) img.setAttribute('src', dataSrc);
+        });
+
+        // Extract code blocks with placeholder replacement
+        const codeBlocks = [];
+        contentEl.querySelectorAll('.code-snippet__fix').forEach(el => {
+          el.querySelectorAll('.code-snippet__line-index').forEach(li => li.remove());
+          const pre = el.querySelector('pre[data-lang]');
+          const lang = pre ? (pre.getAttribute('data-lang') || '') : '';
+          const lines = [];
+          el.querySelectorAll('code').forEach(codeTag => {
+            const text = codeTag.textContent;
+            if (/^[ce]?ounter\\(line/.test(text)) return;
+            lines.push(text);
+          });
+          if (lines.length === 0) lines.push(el.textContent);
+          const placeholder = 'CODEBLOCK-PLACEHOLDER-' + codeBlocks.length;
+          codeBlocks.push({ lang, code: lines.join('\\n') });
+          const p = document.createElement('p');
+          p.textContent = placeholder;
+          el.replaceWith(p);
+        });
+        result.codeBlocks = codeBlocks;
+
+        // Remove noise elements
+        ['script', 'style', '.qr_code_pc', '.reward_area'].forEach(sel => {
+          contentEl.querySelectorAll(sel).forEach(tag => tag.remove());
+        });
+
+        // Collect image URLs (deduplicated)
+        const seen = new Set();
+        contentEl.querySelectorAll('img[src]').forEach(img => {
+          const src = img.getAttribute('src');
+          if (src && !seen.has(src)) {
+            seen.add(src);
+            result.imageUrls.push(src);
+          }
+        });
+
+        result.contentHtml = contentEl.innerHTML;
+        return result;
+      })()
+    `);
+        if (data?.errorHint === 'environment verification required') {
+            return [{
+                    title: 'Error',
+                    author: '-',
+                    publish_time: '-',
+                    status: 'failed — verification required in WeChat browser page',
+                    size: '-',
+                }];
+        }
+        return downloadArticle({
+            title: data?.title || '',
+            author: data?.author,
+            publishTime: data?.publishTime,
+            sourceUrl: url,
+            contentHtml: data?.contentHtml || '',
+            codeBlocks: data?.codeBlocks,
+            imageUrls: data?.imageUrls,
+        }, {
+            output: kwargs.output,
+            downloadImages: kwargs['download-images'],
+            imageHeaders: { Referer: 'https://mp.weixin.qq.com/' },
+            frontmatterLabels: { author: '公众号' },
+            detectImageExt: (url) => {
+                const m = url.match(/wx_fmt=(\w+)/) || url.match(/\.(\w{3,4})(?:\?|$)/);
+                return m ? m[1] : 'png';
+            },
+        });
+    },
+});

package/vendor/opencli/dist/clis/weread/book.d.ts

@@ -0,0 +1 @@
+export {};

package/vendor/opencli/dist/clis/weread/book.js

@@ -0,0 +1,219 @@
+import { cli, Strategy } from '../../registry.js';
+import { CliError } from '../../errors.js';
+import { fetchPrivateApi, fetchWebApi, resolveShelfReader, WEREAD_UA, WEREAD_WEB_ORIGIN, } from './utils.js';
+function decodeHtmlText(value) {
+    return value
+        .replace(/<[^>]+>/g, '')
+        .replace(/&#x([0-9a-fA-F]+);/gi, (_, n) => String.fromCharCode(parseInt(n, 16)))
+        .replace(/&#(\d+);/g, (_, n) => String.fromCharCode(Number(n)))
+        .replace(/&nbsp;/g, ' ')
+        .replace(/&amp;/g, '&')
+        .replace(/&quot;/g, '"')
+        .trim();
+}
+function normalizeSearchText(value) {
+    return value.replace(/\s+/g, ' ').trim();
+}
+function buildSearchIdentity(title, author) {
+    return `${normalizeSearchText(title)}\u0000${normalizeSearchText(author)}`;
+}
+function countSearchTitles(entries) {
+    const counts = new Map();
+    for (const entry of entries) {
+        const key = normalizeSearchText(entry.title);
+        if (!key)
+            continue;
+        counts.set(key, (counts.get(key) || 0) + 1);
+    }
+    return counts;
+}
+function countSearchIdentities(entries) {
+    const counts = new Map();
+    for (const entry of entries) {
+        const key = buildSearchIdentity(entry.title, entry.author);
+        if (!normalizeSearchText(entry.title) || !normalizeSearchText(entry.author))
+            continue;
+        counts.set(key, (counts.get(key) || 0) + 1);
+    }
+    return counts;
+}
+/**
+ * Reuse the public search page as a last-resort reader URL source when the
+ * cached shelf page cannot provide a trustworthy bookId-to-reader mapping.
+ */
+async function resolveSearchReaderUrl(title, author) {
+    const normalizedTitle = normalizeSearchText(title);
+    const normalizedAuthor = normalizeSearchText(author);
+    if (!normalizedTitle)
+        return '';
+    try {
+        const [data, htmlEntries] = await Promise.all([
+            fetchWebApi('/search/global', { keyword: normalizedTitle }),
+            (async () => {
+                const url = new URL('/web/search/books', WEREAD_WEB_ORIGIN);
+                url.searchParams.set('keyword', normalizedTitle);
+                const resp = await fetch(url.toString(), {
+                    headers: { 'User-Agent': WEREAD_UA },
+                });
+                if (!resp.ok)
+                    return [];
+                const html = await resp.text();
+                const items = Array.from(html.matchAll(/<li[^>]*class="wr_bookList_item"[^>]*>([\s\S]*?)<\/li>/g));
+                return items.map((match) => {
+                    const chunk = match[1];
+                    const hrefMatch = chunk.match(/<a[^>]*href="([^"]+)"[^>]*class="wr_bookList_item_link"[^>]*>|<a[^>]*class="wr_bookList_item_link"[^>]*href="([^"]+)"[^>]*>/);
+                    const titleMatch = chunk.match(/<p[^>]*class="wr_bookList_item_title"[^>]*>([\s\S]*?)<\/p>/);
+                    const authorMatch = chunk.match(/<p[^>]*class="wr_bookList_item_author"[^>]*>([\s\S]*?)<\/p>/);
+                    const href = hrefMatch?.[1] || hrefMatch?.[2] || '';
+                    return {
+                        title: decodeHtmlText(titleMatch?.[1] || ''),
+                        author: decodeHtmlText(authorMatch?.[1] || ''),
+                        url: href ? new URL(href, WEREAD_WEB_ORIGIN).toString() : '',
+                    };
+                }).filter((entry) => entry.title && entry.url);
+            })(),
+        ]);
+        const books = Array.isArray(data?.books) ? data.books : [];
+        const apiIdentityCounts = countSearchIdentities(books.map((item) => ({
+            title: item.bookInfo?.title ?? '',
+            author: item.bookInfo?.author ?? '',
+        })));
+        const htmlIdentityCounts = countSearchIdentities(htmlEntries.filter((entry) => entry.author));
+        const identityKey = buildSearchIdentity(normalizedTitle, normalizedAuthor);
+        if (normalizedAuthor &&
+            (apiIdentityCounts.get(identityKey) || 0) === 1 &&
+            (htmlIdentityCounts.get(identityKey) || 0) === 1) {
+            const exactMatch = htmlEntries.find((entry) => buildSearchIdentity(entry.title, entry.author) === identityKey);
+            if (exactMatch?.url)
+                return exactMatch.url;
+        }
+        const sameTitleHtmlEntries = htmlEntries.filter((entry) => normalizeSearchText(entry.title) === normalizedTitle);
+        if (normalizedAuthor && sameTitleHtmlEntries.some((entry) => normalizeSearchText(entry.author))) {
+            return '';
+        }
+        const apiTitleCounts = countSearchTitles(books.map((item) => ({ title: item.bookInfo?.title ?? '' })));
+        const htmlTitleCounts = countSearchTitles(htmlEntries);
+        if ((apiTitleCounts.get(normalizedTitle) || 0) !== 1 || (htmlTitleCounts.get(normalizedTitle) || 0) !== 1) {
+            return '';
+        }
+        return htmlEntries.find((entry) => normalizeSearchText(entry.title) === normalizedTitle)?.url || '';
+    }
+    catch {
+        return '';
+    }
+}
+/**
+ * Read visible book metadata from the web reader cover/flyleaf page.
+ * This path is used as a fallback when the private API session has expired.
+ */
+async function loadReaderFallbackResult(page, readerUrl) {
+    await page.goto(readerUrl);
+    await page.wait({ selector: '.horizontalReaderCoverPage_content_bookTitle, .wr_flyleaf_page_bookInfo_bookTitle', timeout: 10 });
+    const result = await page.evaluate(`
+    (() => {
+      const text = (node) => node?.textContent?.trim() || '';
+      const bodyText = document.body?.innerText?.replace(/\\s+/g, ' ').trim() || '';
+      const titleSelector = '.horizontalReaderCoverPage_content_bookTitle, .wr_flyleaf_page_bookInfo_bookTitle';
+      const authorSelector = '.horizontalReaderCoverPage_content_author, .wr_flyleaf_page_bookInfo_author';
+      const extractRating = () => {
+        const match = bodyText.match(/微信读书推荐值\\s*([0-9.]+%)/);
+        return match ? match[1] : '';
+      };
+      const extractPublisher = () => {
+        const direct = text(document.querySelector('.introDialog_content_pub_line'));
+        return direct.startsWith('出版社') ? direct.replace(/^出版社\\s*/, '').trim() : '';
+      };
+      const extractIntro = () => {
+        const selectors = [
+          '.horizontalReaderCoverPage_content_bookInfo_intro',
+          '.wr_flyleaf_page_bookIntro_content',
+          '.introDialog_content_intro_para',
+        ];
+        for (const selector of selectors) {
+          const value = text(document.querySelector(selector));
+          if (value) return value;
+        }
+        return '';
+      };
+
+      const categorySource = Array.from(document.scripts)
+        .map((script) => script.textContent || '')
+        .find((scriptText) => scriptText.includes('"category"')) || '';
+      const categoryMatch = categorySource.match(/"category"\\s*:\\s*"([^"]+)"/);
+      const title = text(document.querySelector(titleSelector));
+      const author = text(document.querySelector(authorSelector));
+
+      return {
+        title,
+        author,
+        publisher: extractPublisher(),
+        intro: extractIntro(),
+        category: categoryMatch ? categoryMatch[1].trim() : '',
+        rating: extractRating(),
+        metadataReady: Boolean(title || author),
+      };
+    })()
+  `);
+    return {
+        title: String(result?.title || '').trim(),
+        author: String(result?.author || '').trim(),
+        publisher: String(result?.publisher || '').trim(),
+        intro: String(result?.intro || '').trim(),
+        category: String(result?.category || '').trim(),
+        rating: String(result?.rating || '').trim(),
+        metadataReady: result?.metadataReady === true,
+    };
+}
+cli({
+    site: 'weread',
+    name: 'book',
+    description: 'View book details on WeRead',
+    domain: 'weread.qq.com',
+    strategy: Strategy.COOKIE,
+    args: [
+        { name: 'book-id', positional: true, required: true, help: 'Book ID from search or shelf results' },
+    ],
+    columns: ['title', 'author', 'publisher', 'intro', 'category', 'rating'],
+    func: async (page, args) => {
+        const bookId = String(args['book-id'] || '').trim();
+        try {
+            const data = await fetchPrivateApi(page, '/book/info', { bookId });
+            // newRating is 0-1000 scale per community docs; needs runtime verification
+            const rating = data.newRating ? `${(data.newRating / 10).toFixed(1)}%` : '-';
+            return [{
+                    title: data.title ?? '',
+                    author: data.author ?? '',
+                    publisher: data.publisher ?? '',
+                    intro: data.intro ?? '',
+                    category: data.category ?? '',
+                    rating,
+                }];
+        }
+        catch (error) {
+            if (!(error instanceof CliError) || error.code !== 'AUTH_REQUIRED') {
+                throw error;
+            }
+            const { readerUrl: resolvedReaderUrl, snapshot } = await resolveShelfReader(page, bookId);
+            let readerUrl = resolvedReaderUrl;
+            if (!readerUrl) {
+                const cachedBook = snapshot.rawBooks.find((book) => String(book?.bookId || '').trim() === bookId);
+                readerUrl = await resolveSearchReaderUrl(String(cachedBook?.title || ''), String(cachedBook?.author || ''));
+            }
+            if (!readerUrl) {
+                throw error;
+            }
+            const data = await loadReaderFallbackResult(page, readerUrl);
+            if (!data.metadataReady || !data.title) {
+                throw error;
+            }
+            return [{
+                    title: data.title,
+                    author: data.author,
+                    publisher: data.publisher,
+                    intro: data.intro,
+                    category: data.category,
+                    rating: data.rating,
+                }];
+        }
+    },
+});

package/vendor/opencli/dist/clis/weread/commands.test.d.ts

@@ -0,0 +1,3 @@
+import './book.js';
+import './highlights.js';
+import './notes.js';