@jackwener/opencli 1.5.6 → 1.5.8

package/src/clis/substack/utils.test.ts

@@ -0,0 +1,54 @@
+import { describe, expect, it, vi } from 'vitest';
+import type { IPage } from '../../types.js';
+import { __test__, loadSubstackArchive, loadSubstackFeed } from './utils.js';
+
+function createPageMock(evaluateResult: unknown): IPage {
+  return {
+    goto: vi.fn().mockResolvedValue(undefined),
+    evaluate: vi.fn().mockResolvedValue(evaluateResult),
+    snapshot: vi.fn().mockResolvedValue(undefined),
+    click: vi.fn().mockResolvedValue(undefined),
+    typeText: vi.fn().mockResolvedValue(undefined),
+    pressKey: vi.fn().mockResolvedValue(undefined),
+    scrollTo: vi.fn().mockResolvedValue(undefined),
+    getFormState: vi.fn().mockResolvedValue({}),
+    wait: vi.fn().mockResolvedValue(undefined),
+    tabs: vi.fn().mockResolvedValue([]),
+    closeTab: vi.fn().mockResolvedValue(undefined),
+    newTab: vi.fn().mockResolvedValue(undefined),
+    selectTab: vi.fn().mockResolvedValue(undefined),
+    networkRequests: vi.fn().mockResolvedValue([]),
+    consoleMessages: vi.fn().mockResolvedValue([]),
+    scroll: vi.fn().mockResolvedValue(undefined),
+    autoScroll: vi.fn().mockResolvedValue(undefined),
+    installInterceptor: vi.fn().mockResolvedValue(undefined),
+    getInterceptedRequests: vi.fn().mockResolvedValue([]),
+    getCookies: vi.fn().mockResolvedValue([]),
+    screenshot: vi.fn().mockResolvedValue(''),
+    waitForCapture: vi.fn().mockResolvedValue(undefined),
+  };
+}
+
+describe('substack utils wait selectors', () => {
+  it('waits for both feed link shapes before scraping the feed', async () => {
+    const page = createPageMock([]);
+
+    await loadSubstackFeed(page, 'https://substack.com/', 5);
+
+    expect(page.wait).toHaveBeenCalledWith({
+      selector: __test__.FEED_POST_LINK_SELECTOR,
+      timeout: 5,
+    });
+  });
+
+  it('waits for archive post links before scraping archive pages', async () => {
+    const page = createPageMock([]);
+
+    await loadSubstackArchive(page, 'https://example.substack.com', 5);
+
+    expect(page.wait).toHaveBeenCalledWith({
+      selector: __test__.ARCHIVE_POST_LINK_SELECTOR,
+      timeout: 5,
+    });
+  });
+});

package/src/clis/substack/utils.ts

@@ -1,6 +1,9 @@
 import { CommandExecutionError } from '../../errors.js';
 import type { IPage } from '../../types.js';
 
+const FEED_POST_LINK_SELECTOR = 'a[href*="/home/post/"], a[href*="/p/"]';
+const ARCHIVE_POST_LINK_SELECTOR = 'a[href*="/p/"]';
+
 export function buildSubstackBrowseUrl(category?: string): string {
   if (!category || category === 'all') return 'https://substack.com/';
   const slug = category === 'tech' ? 'technology' : category;
@@ -10,7 +13,7 @@ export function buildSubstackBrowseUrl(category?: string): string {
 export async function loadSubstackFeed(page: IPage, url: string, limit: number): Promise<any[]> {
   if (!page) throw new CommandExecutionError('Browser session required for substack feed');
   await page.goto(url);
-  await page.wait({ selector: 'article', timeout: 5 });
+  await page.wait({ selector: FEED_POST_LINK_SELECTOR, timeout: 5 });
   const data = await page.evaluate(`
     (async () => {
       await new Promise((resolve) => setTimeout(resolve, 3000));
@@ -79,7 +82,7 @@ export async function loadSubstackFeed(page: IPage, url: string, limit: number):
 export async function loadSubstackArchive(page: IPage, baseUrl: string, limit: number): Promise<any[]> {
   if (!page) throw new CommandExecutionError('Browser session required for substack archive');
   await page.goto(`${baseUrl}/archive`);
-  await page.wait({ selector: 'article', timeout: 5 });
+  await page.wait({ selector: ARCHIVE_POST_LINK_SELECTOR, timeout: 5 });
   const data = await page.evaluate(`
     (async () => {
       await new Promise((resolve) => setTimeout(resolve, 3000));
@@ -131,3 +134,8 @@ export async function loadSubstackArchive(page: IPage, baseUrl: string, limit: n
 
   return Array.isArray(data) ? data : [];
 }
+
+export const __test__ = {
+  FEED_POST_LINK_SELECTOR,
+  ARCHIVE_POST_LINK_SELECTOR,
+};

package/src/clis/v2ex/hot.yaml

@@ -16,10 +16,13 @@ pipeline:
       url: https://www.v2ex.com/api/topics/hot.json
 
   - map:
+      id: ${{ item.id }}
       rank: ${{ index + 1 }}
       title: ${{ item.title }}
+      node: ${{ item.node.title }}
       replies: ${{ item.replies }}
+      url: ${{ item.url }}
 
   - limit: ${{ args.limit }}
 
-columns: [rank, title, replies]
+columns: [id, rank, title, node, replies, url]

package/src/clis/v2ex/latest.yaml

@@ -16,10 +16,13 @@ pipeline:
       url: https://www.v2ex.com/api/topics/latest.json
 
   - map:
+      id: ${{ item.id }}
       rank: ${{ index + 1 }}
       title: ${{ item.title }}
+      node: ${{ item.node.title }}
       replies: ${{ item.replies }}
+      url: ${{ item.url }}
 
   - limit: ${{ args.limit }}
 
-columns: [rank, title, replies]
+columns: [id, rank, title, node, replies, url]

package/src/clis/v2ex/topic.yaml

@@ -19,10 +19,15 @@ pipeline:
         id: ${{ args.id }}
 
   - map:
+      id: ${{ item.id }}
       title: ${{ item.title }}
+      content: ${{ item.content }}
+      member: ${{ item.member.username }}
+      created: ${{ item.created }}
+      node: ${{ item.node.title }}
       replies: ${{ item.replies }}
       url: ${{ item.url }}
 
   - limit: 1
 
-columns: [title, replies, url]
+columns: [id, title, content, member, created, node, replies, url]

package/src/clis/weixin/download.ts

@@ -110,6 +110,37 @@ export function extractWechatPublishTime(
   return formatWechatTimestamp(rawCreateTime);
 }
 
+/**
+ * Detect WeChat anti-bot / verification gate pages before we try to parse the article.
+ */
+export function detectWechatAccessIssue(
+  pageText: string | null | undefined,
+  htmlStr: string,
+): string {
+  const normalizedText = (pageText || '').replace(/\s+/g, ' ').trim();
+
+  if (
+    /环境异常/.test(normalizedText) &&
+    /(完成验证后即可继续访问|去验证)/.test(normalizedText)
+  ) {
+    return 'environment verification required';
+  }
+
+  if (/secitptpage\/verify\.html/.test(htmlStr) || /id=["']js_verify["']/.test(htmlStr)) {
+    return 'environment verification required';
+  }
+
+  return '';
+}
+
+export function pickFirstWechatMetaText(...candidates: Array<string | null | undefined>): string {
+  for (const candidate of candidates) {
+    const normalized = (candidate || '').replace(/\s+/g, ' ').trim();
+    if (normalized && normalized !== 'Name cleared') return normalized;
+  }
+  return '';
+}
+
 /**
  * Build a self-contained helper for execution inside page.evaluate().
  */
@@ -161,6 +192,31 @@ export function buildExtractWechatPublishTimeJs(): string {
   }.toString()})`;
 }
 
+/**
+ * Build a self-contained access-issue detector for execution inside page.evaluate().
+ */
+export function buildDetectWechatAccessIssueJs(): string {
+  return `(${function detectWechatAccessIssueInPage(
+    pageText: string | null | undefined,
+    htmlStr: string,
+  ) {
+    const normalizedText = (pageText || '').replace(/\s+/g, ' ').trim();
+
+    if (
+      /环境异常/.test(normalizedText) &&
+      /(完成验证后即可继续访问|去验证)/.test(normalizedText)
+    ) {
+      return 'environment verification required';
+    }
+
+    if (/secitptpage\/verify\.html/.test(htmlStr) || /id=["']js_verify["']/.test(htmlStr)) {
+      return 'environment verification required';
+    }
+
+    return '';
+  }.toString()})`;
+}
+
 // ============================================================
 // CLI Registration
 // ============================================================
@@ -196,18 +252,34 @@ cli({
           title: '',
           author: '',
           publishTime: '',
+          errorHint: '',
           contentHtml: '',
           codeBlocks: [],
           imageUrls: []
         };
 
-        // Title: #activity-name
-        const titleEl = document.querySelector('#activity-name');
-        result.title = titleEl ? titleEl.textContent.trim() : '';
+        const pickFirstText = (...selectors) => {
+          for (const selector of selectors) {
+            const text = document.querySelector(selector)?.textContent?.replace(/\\s+/g, ' ').trim() || '';
+            if (text && text !== 'Name cleared') return text;
+          }
+          return '';
+        };
+
+        // WeChat has multiple article templates. Newer pages use #js_text_title.
+        result.title = pickFirstText(
+          '#activity-name',
+          '#js_text_title',
+          '.rich_media_title',
+        );
 
-        // Author (WeChat Official Account name): #js_name
-        const authorEl = document.querySelector('#js_name');
-        result.author = authorEl ? authorEl.textContent.trim() : '';
+        result.author = pickFirstText(
+          '#js_name',
+          '.wx_follow_nickname',
+          '#profileBt .profile_nickname',
+          '.rich_media_meta.rich_media_meta_nickname',
+          '.rich_media_meta_nickname',
+        );
 
         // Publish time: prefer the rendered DOM text, then fall back to numeric create_time values.
         const publishTimeEl = document.querySelector('#publish_time');
@@ -217,6 +289,13 @@ cli({
           document.documentElement.innerHTML,
         );
 
+        const detectWechatAccessIssue = ${buildDetectWechatAccessIssueJs()};
+        result.errorHint = detectWechatAccessIssue(
+          document.body ? document.body.innerText : '',
+          document.documentElement.innerHTML,
+        );
+        if (result.errorHint) return result;
+
         // Content processing
         const contentEl = document.querySelector('#js_content');
         if (!contentEl) return result;
@@ -268,6 +347,16 @@ cli({
       })()
     `);
 
+    if (data?.errorHint === 'environment verification required') {
+      return [{
+        title: 'Error',
+        author: '-',
+        publish_time: '-',
+        status: 'failed — verification required in WeChat browser page',
+        size: '-',
+      }];
+    }
+
     return downloadArticle(
       {
         title: data?.title || '',

package/src/clis/weread/book.ts

@@ -1,7 +1,13 @@
 import { cli, Strategy } from '../../registry.js';
 import { CliError } from '../../errors.js';
 import type { IPage } from '../../types.js';
-import { fetchPrivateApi, resolveShelfReaderUrl } from './utils.js';
+import {
+  fetchPrivateApi,
+  fetchWebApi,
+  resolveShelfReader,
+  WEREAD_UA,
+  WEREAD_WEB_ORIGIN,
+} from './utils.js';
 
 interface ReaderFallbackResult {
   title: string;
@@ -13,6 +19,132 @@ interface ReaderFallbackResult {
   metadataReady: boolean;
 }
 
+interface SearchHtmlEntry {
+  title: string;
+  author: string;
+  url: string;
+}
+
+function decodeHtmlText(value: string): string {
+  return value
+    .replace(/<[^>]+>/g, '')
+    .replace(/&#x([0-9a-fA-F]+);/gi, (_, n) => String.fromCharCode(parseInt(n, 16)))
+    .replace(/&#(\d+);/g, (_, n) => String.fromCharCode(Number(n)))
+    .replace(/&nbsp;/g, ' ')
+    .replace(/&amp;/g, '&')
+    .replace(/&quot;/g, '"')
+    .trim();
+}
+
+function normalizeSearchText(value: string): string {
+  return value.replace(/\s+/g, ' ').trim();
+}
+
+function buildSearchIdentity(title: string, author: string): string {
+  return `${normalizeSearchText(title)}\u0000${normalizeSearchText(author)}`;
+}
+
+function countSearchTitles(entries: Array<{ title: string }>): Map<string, number> {
+  const counts = new Map<string, number>();
+  for (const entry of entries) {
+    const key = normalizeSearchText(entry.title);
+    if (!key) continue;
+    counts.set(key, (counts.get(key) || 0) + 1);
+  }
+  return counts;
+}
+
+function countSearchIdentities(entries: Array<{ title: string; author: string }>): Map<string, number> {
+  const counts = new Map<string, number>();
+  for (const entry of entries) {
+    const key = buildSearchIdentity(entry.title, entry.author);
+    if (!normalizeSearchText(entry.title) || !normalizeSearchText(entry.author)) continue;
+    counts.set(key, (counts.get(key) || 0) + 1);
+  }
+  return counts;
+}
+
+/**
+ * Reuse the public search page as a last-resort reader URL source when the
+ * cached shelf page cannot provide a trustworthy bookId-to-reader mapping.
+ */
+async function resolveSearchReaderUrl(title: string, author: string): Promise<string> {
+  const normalizedTitle = normalizeSearchText(title);
+  const normalizedAuthor = normalizeSearchText(author);
+  if (!normalizedTitle) return '';
+
+  try {
+    const [data, htmlEntries] = await Promise.all([
+      fetchWebApi('/search/global', { keyword: normalizedTitle }),
+      (async (): Promise<SearchHtmlEntry[]> => {
+        const url = new URL('/web/search/books', WEREAD_WEB_ORIGIN);
+        url.searchParams.set('keyword', normalizedTitle);
+
+        const resp = await fetch(url.toString(), {
+          headers: { 'User-Agent': WEREAD_UA },
+        });
+        if (!resp.ok) return [];
+
+        const html = await resp.text();
+        const items = Array.from(
+          html.matchAll(/<li[^>]*class="wr_bookList_item"[^>]*>([\s\S]*?)<\/li>/g),
+        );
+
+        return items.map((match) => {
+          const chunk = match[1];
+          const hrefMatch = chunk.match(/<a[^>]*href="([^"]+)"[^>]*class="wr_bookList_item_link"[^>]*>|<a[^>]*class="wr_bookList_item_link"[^>]*href="([^"]+)"[^>]*>/);
+          const titleMatch = chunk.match(/<p[^>]*class="wr_bookList_item_title"[^>]*>([\s\S]*?)<\/p>/);
+          const authorMatch = chunk.match(/<p[^>]*class="wr_bookList_item_author"[^>]*>([\s\S]*?)<\/p>/);
+          const href = hrefMatch?.[1] || hrefMatch?.[2] || '';
+
+          return {
+            title: decodeHtmlText(titleMatch?.[1] || ''),
+            author: decodeHtmlText(authorMatch?.[1] || ''),
+            url: href ? new URL(href, WEREAD_WEB_ORIGIN).toString() : '',
+          };
+        }).filter((entry) => entry.title && entry.url);
+      })(),
+    ]);
+
+    const books: any[] = Array.isArray(data?.books) ? data.books : [];
+    const apiIdentityCounts = countSearchIdentities(
+      books.map((item: any) => ({
+        title: item.bookInfo?.title ?? '',
+        author: item.bookInfo?.author ?? '',
+      })),
+    );
+    const htmlIdentityCounts = countSearchIdentities(
+      htmlEntries.filter((entry) => entry.author),
+    );
+    const identityKey = buildSearchIdentity(normalizedTitle, normalizedAuthor);
+    if (
+      normalizedAuthor &&
+      (apiIdentityCounts.get(identityKey) || 0) === 1 &&
+      (htmlIdentityCounts.get(identityKey) || 0) === 1
+    ) {
+      const exactMatch = htmlEntries.find((entry) => buildSearchIdentity(entry.title, entry.author) === identityKey);
+      if (exactMatch?.url) return exactMatch.url;
+    }
+
+    const sameTitleHtmlEntries = htmlEntries.filter((entry) => normalizeSearchText(entry.title) === normalizedTitle);
+    if (normalizedAuthor && sameTitleHtmlEntries.some((entry) => normalizeSearchText(entry.author))) {
+      return '';
+    }
+
+    const apiTitleCounts = countSearchTitles(
+      books.map((item: any) => ({ title: item.bookInfo?.title ?? '' })),
+    );
+    const htmlTitleCounts = countSearchTitles(htmlEntries);
+    if ((apiTitleCounts.get(normalizedTitle) || 0) !== 1 || (htmlTitleCounts.get(normalizedTitle) || 0) !== 1) {
+      return '';
+    }
+
+    return htmlEntries.find((entry) => normalizeSearchText(entry.title) === normalizedTitle)?.url || '';
+  } catch {
+    return '';
+  }
+}
+
 /**
  * Read visible book metadata from the web reader cover/flyleaf page.
  * This path is used as a fallback when the private API session has expired.
@@ -108,7 +240,15 @@ cli({
         throw error;
       }
 
-      const readerUrl = await resolveShelfReaderUrl(page, bookId);
+      const { readerUrl: resolvedReaderUrl, snapshot } = await resolveShelfReader(page, bookId);
+      let readerUrl = resolvedReaderUrl;
+      if (!readerUrl) {
+        const cachedBook = snapshot.rawBooks.find((book) => String(book?.bookId || '').trim() === bookId);
+        readerUrl = await resolveSearchReaderUrl(
+          String(cachedBook?.title || ''),
+          String(cachedBook?.author || ''),
+        );
+      }
       if (!readerUrl) {
         throw error;
       }