@jackwener/opencli 1.5.5 → 1.5.7

package/dist/clis/tieba/hot.js

@@ -0,0 +1,48 @@
+import { EmptyResultError } from '../../errors.js';
+import { cli, Strategy } from '../../registry.js';
+import { normalizeTiebaLimit } from './utils.js';
+cli({
+    site: 'tieba',
+    name: 'hot',
+    description: 'Tieba hot topics',
+    domain: 'tieba.baidu.com',
+    strategy: Strategy.PUBLIC,
+    browser: true,
+    navigateBefore: false,
+    args: [
+        { name: 'limit', type: 'int', default: 20, help: 'Number of items to return' },
+    ],
+    columns: ['rank', 'title', 'discussions', 'description'],
+    func: async (page, kwargs) => {
+        const limit = normalizeTiebaLimit(kwargs.limit);
+        // Use the default browser settle path so we do not scrape the previous page.
+        await page.goto('https://tieba.baidu.com/hottopic/browse/topicList?res_type=1');
+        const raw = await page.evaluate(`(() => {
+      const items = document.querySelectorAll('li.topic-top-item');
+      return Array.from(items).map((item) => {
+        const titleEl = item.querySelector('a.topic-text');
+        const numEl = item.querySelector('span.topic-num');
+        const descEl = item.querySelector('p.topic-top-item-desc');
+        const href = titleEl?.getAttribute('href') || '';
+
+        return {
+          title: titleEl?.textContent?.trim() || '',
+          discussions: numEl?.textContent?.trim() || '',
+          description: descEl?.textContent?.trim() || '',
+          url: href.startsWith('http') ? href : 'https://tieba.baidu.com' + href,
+        };
+      }).filter((item) => item.title).slice(0, ${limit});
+    })()`);
+        const items = Array.isArray(raw) ? raw : [];
+        if (!items.length) {
+            throw new EmptyResultError('tieba hot', 'Tieba may have blocked the hot page, or the DOM structure may have changed');
+        }
+        return items.map((item, index) => ({
+            rank: index + 1,
+            title: item.title || '',
+            discussions: item.discussions || '',
+            description: item.description || '',
+            url: item.url || '',
+        }));
+    },
+});

package/dist/clis/tieba/posts.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/clis/tieba/posts.js

@@ -0,0 +1,85 @@
+import { EmptyResultError } from '../../errors.js';
+import { cli, Strategy } from '../../registry.js';
+import { buildTiebaPostCardsFromPagePc, buildTiebaPostItems, normalizeTiebaLimit, signTiebaPcParams, } from './utils.js';
+function getForumPageNumber(kwargs) {
+    return Math.max(1, Number(kwargs.page || 1));
+}
+function getForumUrl(kwargs) {
+    const forum = String(kwargs.forum || '');
+    return `https://tieba.baidu.com/f?kw=${encodeURIComponent(forum)}&ie=utf-8&pn=${(getForumPageNumber(kwargs) - 1) * 50}`;
+}
+/**
+ * Rebuild the signed page_pc request instead of scraping only the visible thread cards.
+ */
+function buildTiebaPagePcParams(kwargs, limit) {
+    return {
+        kw: encodeURIComponent(String(kwargs.forum || '')),
+        pn: String(getForumPageNumber(kwargs)),
+        sort_type: '-1',
+        is_newfrs: '1',
+        is_newfeed: '1',
+        rn: '30',
+        rn_need: String(Math.min(Math.max(limit + 10, 10), 30)),
+        tbs: '',
+        subapp_type: 'pc',
+        _client_type: '20',
+    };
+}
+/**
+ * Tieba expects the signed forum-list request to be replayed with the browser's cookies.
+ */
+async function fetchTiebaPagePc(page, kwargs, limit) {
+    await page.goto(getForumUrl(kwargs), { waitUntil: 'none' });
+    await page.wait(2);
+    const params = buildTiebaPagePcParams(kwargs, limit);
+    const cookies = await page.getCookies({ domain: 'tieba.baidu.com' });
+    const cookieHeader = cookies.map((item) => `${item.name}=${item.value}`).join('; ');
+    const body = new URLSearchParams({
+        ...params,
+        sign: signTiebaPcParams(params),
+    }).toString();
+    const response = await fetch('https://tieba.baidu.com/c/f/frs/page_pc', {
+        method: 'POST',
+        headers: {
+            'content-type': 'application/x-www-form-urlencoded;charset=UTF-8',
+            cookie: cookieHeader,
+            'x-requested-with': 'XMLHttpRequest',
+            referer: getForumUrl(kwargs),
+            'user-agent': 'Mozilla/5.0',
+        },
+        body,
+    });
+    const text = await response.text();
+    try {
+        return JSON.parse(text);
+    }
+    catch {
+        return {};
+    }
+}
+cli({
+    site: 'tieba',
+    name: 'posts',
+    description: 'Browse posts in a tieba forum',
+    domain: 'tieba.baidu.com',
+    strategy: Strategy.COOKIE,
+    browser: true,
+    navigateBefore: false,
+    args: [
+        { name: 'forum', positional: true, required: true, type: 'string', help: 'Forum name in Chinese' },
+        { name: 'page', type: 'int', default: 1, help: 'Page number' },
+        { name: 'limit', type: 'int', default: 20, help: 'Number of items to return' },
+    ],
+    columns: ['rank', 'title', 'author', 'replies'],
+    func: async (page, kwargs) => {
+        const limit = normalizeTiebaLimit(kwargs.limit);
+        const payload = await fetchTiebaPagePc(page, kwargs, limit);
+        const rawFeeds = Array.isArray(payload.page_data?.feed_list) ? payload.page_data.feed_list : [];
+        const rawCards = buildTiebaPostCardsFromPagePc(rawFeeds);
+        const items = buildTiebaPostItems(rawCards, limit);
+        if (!items.length || payload.error_code) {
+            throw new EmptyResultError('tieba posts', 'Tieba may have blocked the forum page, or the DOM structure may have changed');
+        }
+        return items;
+    },
+});

package/dist/clis/tieba/read.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/clis/tieba/read.js

@@ -0,0 +1,140 @@
+import { EmptyResultError } from '../../errors.js';
+import { cli, Strategy } from '../../registry.js';
+import { buildTiebaReadItems } from './utils.js';
+function getThreadUrl(kwargs) {
+    const threadId = String(kwargs.id || '');
+    const pageNumber = Math.max(1, Number(kwargs.page || 1));
+    return `https://tieba.baidu.com/p/${encodeURIComponent(threadId)}?pn=${pageNumber}`;
+}
+/**
+ * Ensure the browser actually landed on the requested thread page before we trust the DOM.
+ */
+function assertTiebaReadTargetPage(raw, kwargs) {
+    const expectedThreadId = String(kwargs.id || '').trim();
+    const expectedPageNumber = Math.max(1, Number(kwargs.page || 1));
+    const pathname = String(raw.pageMeta?.pathname || '').trim();
+    const actualThreadId = pathname.match(/^\/p\/(\d+)/)?.[1] || '';
+    const actualPn = String(raw.pageMeta?.pn || '').trim();
+    if (!actualThreadId || actualThreadId !== expectedThreadId) {
+        throw new EmptyResultError('tieba read', 'Tieba did not land on the requested thread page');
+    }
+    if (expectedPageNumber > 1 && actualPn !== String(expectedPageNumber)) {
+        throw new EmptyResultError('tieba read', 'Tieba did not land on the requested page');
+    }
+}
+function buildExtractReadEvaluate() {
+    return `
+    (async () => {
+      const wait = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+      const waitFor = async (predicate, timeoutMs = 4000) => {
+        const start = Date.now();
+        while (Date.now() - start < timeoutMs) {
+          if (predicate()) return true;
+          await wait(100);
+        }
+        return false;
+      };
+      const normalizeText = (value) => (value || '').replace(/\\s+/g, ' ').trim();
+      const getVueProps = (element) => {
+        const vue = element && element.__vue__ ? element.__vue__ : null;
+        return vue ? (vue._props || vue.$props || {}) : {};
+      };
+      const extractStructuredText = (content) => {
+        if (!Array.isArray(content)) return '';
+        return content
+          .map((part) => (part && typeof part === 'object' && typeof part.text === 'string') ? part.text : '')
+          .join('')
+          .replace(/\\s+/g, ' ')
+          .trim();
+      };
+      const parseFloor = (text) => {
+        const match = (text || '').match(/第(\\d+)楼/);
+        return match ? parseInt(match[1], 10) : 0;
+      };
+
+      await waitFor(() => {
+        const hasMainTree = document.querySelector('.pb-title-wrap.pc-pb-title') || document.querySelector('.pb-content-wrap');
+        return Boolean(hasMainTree || document.querySelector('.pb-comment-item'));
+      });
+
+      const titleNode = document.querySelector('.pb-title-wrap.pc-pb-title');
+      const titleProps = getVueProps(titleNode);
+      const mainUser = document.querySelector('.head-line.user-info:not(.no-extra-margin)');
+      const mainUserProps = getVueProps(mainUser);
+      const contentWrap = document.querySelector('.pb-content-wrap');
+      const contentProps = getVueProps(contentWrap);
+      const structuredContent = Array.isArray(contentProps.content) ? contentProps.content : [];
+      const visibleContent = normalizeText(
+        contentWrap?.querySelector('.pb-content-item .text')?.textContent
+        || contentWrap?.querySelector('.text')?.textContent
+        || contentWrap?.textContent
+      );
+
+      return {
+        pageMeta: {
+          pathname: window.location.pathname || '',
+          pn: new URLSearchParams(window.location.search).get('pn') || '',
+        },
+        mainPost: {
+          title: typeof titleProps.title === 'string' && titleProps.title.trim()
+            ? titleProps.title.trim()
+            : normalizeText(titleNode?.textContent).replace(/-百度贴吧$/, '').trim(),
+          author: normalizeText(
+            mainUser?.querySelector('.head-name')?.textContent
+            || mainUser?.querySelector('.name-info .head-name')?.textContent
+            || ''
+          ),
+          fallbackAuthor: mainUserProps?.userShowInfo?.[0]?.text?.text || '',
+          contentText: visibleContent,
+          structuredText: extractStructuredText(structuredContent),
+          visibleTime: (() => {
+            const userText = normalizeText(mainUser?.textContent);
+            const match = userText.match(/(刚刚|昨天|前天|\\d+\\s*(?:分钟|小时|天)前|\\d{2}-\\d{2}(?:\\s+\\d{2}:\\d{2})?|\\d{4}-\\d{2}-\\d{2}(?:\\s+\\d{2}:\\d{2})?)/);
+            return match ? match[1].trim() : '';
+          })(),
+          structuredTime: mainUserProps?.descInfo?.time || 0,
+          hasMedia: structuredContent.length > 0 && !extractStructuredText(structuredContent),
+        },
+        replies: Array.from(document.querySelectorAll('.pb-comment-item')).map((item) => {
+          const meta = item.querySelector('.comment-desc-left')?.textContent?.replace(/\\s+/g, ' ').trim() || '';
+          return {
+            floor: parseFloor(meta),
+            author: item.querySelector('.head-name')?.textContent?.trim() || '',
+            content: item.querySelector('.comment-content .pb-content-item .text')?.textContent?.replace(/\\s+/g, ' ').trim() || '',
+            time: meta,
+          };
+        }),
+      };
+    })()
+  `;
+}
+cli({
+    site: 'tieba',
+    name: 'read',
+    description: 'Read a tieba thread',
+    domain: 'tieba.baidu.com',
+    strategy: Strategy.COOKIE,
+    browser: true,
+    navigateBefore: false,
+    args: [
+        { name: 'id', positional: true, required: true, type: 'string', help: 'Thread ID' },
+        { name: 'page', type: 'int', default: 1, help: 'Page number' },
+        { name: 'limit', type: 'int', default: 30, help: 'Number of replies to return' },
+    ],
+    columns: ['floor', 'author', 'content', 'time'],
+    func: async (page, kwargs) => {
+        const pageNumber = Math.max(1, Number(kwargs.page || 1));
+        // Use the browser's normal settle path so we do not scrape stale DOM from the previous tab state.
+        await page.goto(getThreadUrl(kwargs));
+        const raw = (await page.evaluate(buildExtractReadEvaluate()) || {});
+        assertTiebaReadTargetPage(raw, kwargs);
+        const items = buildTiebaReadItems(raw, {
+            limit: kwargs.limit,
+            includeMainPost: pageNumber === 1,
+        });
+        if (!items.length) {
+            throw new EmptyResultError('tieba read', 'Tieba may have blocked the thread page, or the DOM structure may have changed');
+        }
+        return items;
+    },
+});

package/dist/clis/tieba/search.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/clis/tieba/search.js

@@ -0,0 +1,108 @@
+import { ArgumentError, EmptyResultError } from '../../errors.js';
+import { cli, Strategy } from '../../registry.js';
+import { buildTiebaSearchItems, normalizeTiebaLimit } from './utils.js';
+const MAX_SUPPORTED_PAGE = '1';
+/**
+ * Extract search result cards from tieba's current desktop search page.
+ */
+function buildExtractSearchResultsEvaluate(limit) {
+    return `
+    (async () => {
+      const wait = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+      const waitFor = async (predicate, timeoutMs = 5000) => {
+        const start = Date.now();
+        while (Date.now() - start < timeoutMs) {
+          if (predicate()) return true;
+          await wait(100);
+        }
+        return false;
+      };
+      const getVueProps = (element) => {
+        const vue = element && element.__vue__ ? element.__vue__ : null;
+        return vue ? (vue._props || vue.$props || {}) : {};
+      };
+      await waitFor(() => {
+        const bodyText = document.body?.innerText || '';
+        return Boolean(
+          document.querySelector('.threadcardclass.thread-new3.index-feed-cards')
+          || document.querySelector('.search-no-result, .search-nodata, .no-result')
+          || /百度安全验证|安全验证|请完成验证/.test(bodyText)
+        );
+      });
+      const items = document.querySelectorAll('.threadcardclass.thread-new3.index-feed-cards');
+      return Array.from(items).slice(0, ${limit}).map((item) => {
+        const forum = item.querySelector('.forum-name-text, .forum-name')?.textContent?.trim() || '';
+        const meta = item.querySelector('.user-forum-info')?.textContent?.replace(/\\s+/g, ' ').trim() || '';
+        const metaWithoutForum = forum && meta.startsWith(forum)
+          ? meta.slice(forum.length).trim()
+          : meta;
+        const metaMatch = metaWithoutForum.match(/^(.*?)\\s*发布于\\s*(.+)$/);
+        const actionBar = item.querySelector('.action-bar-container.search-action-bar');
+        const businessInfo = getVueProps(actionBar).businessInfo || {};
+        const href = item.querySelector('a[href*="/p/"]')?.href || '';
+        const threadId = String(businessInfo.thread_id || '').trim();
+        const title = item.querySelector('.title-wrap')?.textContent?.trim()
+          || item.querySelector('.title-content-wrap')?.textContent?.trim()
+          || '';
+        const snippet = item.querySelector('.title-content-wrap')?.textContent?.trim()
+          || item.querySelector('.abstract-wrap')?.textContent?.trim()
+          || '';
+
+        return {
+          title,
+          forum,
+          author: metaMatch ? metaMatch[1].trim() : metaWithoutForum,
+          time: metaMatch ? metaMatch[2].trim() : '',
+          snippet: snippet.substring(0, 200),
+          id: threadId,
+          url: href || (threadId ? 'https://tieba.baidu.com/p/' + threadId : ''),
+        };
+      }).filter((item) => item.title);
+    })()
+  `;
+}
+/**
+ * Normalize CLI args into the concrete search page URL.
+ */
+function getSearchUrl(kwargs) {
+    const keyword = String(kwargs.keyword || '');
+    const pageNumber = Number(kwargs.page || 1);
+    return `https://tieba.baidu.com/f/search/res?qw=${encodeURIComponent(keyword)}&ie=utf-8&pn=${pageNumber}`;
+}
+/**
+ * Tieba's current desktop search UI no longer exposes a reliable browser-page transition.
+ */
+function assertSupportedPage(kwargs) {
+    const pageNumber = String(kwargs.page || 1);
+    if (pageNumber === MAX_SUPPORTED_PAGE)
+        return;
+    throw new ArgumentError(`tieba search currently only supports --page ${MAX_SUPPORTED_PAGE}`, `Baidu Tieba search no longer exposes stable browser pagination; omit --page or use --page ${MAX_SUPPORTED_PAGE}`);
+}
+cli({
+    site: 'tieba',
+    name: 'search',
+    description: 'Search posts across tieba',
+    domain: 'tieba.baidu.com',
+    strategy: Strategy.COOKIE,
+    browser: true,
+    navigateBefore: false,
+    args: [
+        { name: 'keyword', positional: true, required: true, type: 'string', help: 'Search keyword' },
+        // Restrict unsupported pages before the browser session starts.
+        { name: 'page', type: 'int', default: 1, choices: ['1'], help: 'Page number (currently only 1 is supported)' },
+        { name: 'limit', type: 'int', default: 20, help: 'Number of items to return' },
+    ],
+    columns: ['rank', 'title', 'forum', 'author', 'time'],
+    func: async (page, kwargs) => {
+        assertSupportedPage(kwargs);
+        const limit = normalizeTiebaLimit(kwargs.limit);
+        // Use the default browser settle path so we do not read a stale page.
+        await page.goto(getSearchUrl(kwargs));
+        const raw = await page.evaluate(buildExtractSearchResultsEvaluate(limit));
+        const items = buildTiebaSearchItems(Array.isArray(raw) ? raw : [], limit);
+        if (!items.length) {
+            throw new EmptyResultError('tieba search', 'Tieba may have blocked the result page, or the DOM structure may have changed');
+        }
+        return items;
+    },
+});

package/dist/clis/tieba/utils.d.ts

@@ -0,0 +1,101 @@
+/**
+ * Shared Tieba parsing helpers used by the browser adapters.
+ */
+export declare const MAX_TIEBA_LIMIT = 20;
+export interface RawTiebaPostCard {
+    title?: string;
+    author?: string;
+    descInfo?: string;
+    actionTexts?: string[];
+    commentCount?: unknown;
+    threadId?: unknown;
+    url?: unknown;
+}
+export interface RawTiebaPagePcFeedEntry {
+    layout?: string;
+    feed?: {
+        schema?: unknown;
+        log_param?: Array<{
+            key?: unknown;
+            value?: unknown;
+        }>;
+        business_info_map?: Record<string, unknown>;
+        components?: Array<Record<string, unknown>>;
+    };
+}
+export interface TiebaPostItem {
+    rank: number;
+    title: string;
+    author: string;
+    replies: number;
+    last_reply: string;
+    id: string;
+    url: string;
+}
+export interface RawTiebaSearchItem {
+    title?: string;
+    forum?: string;
+    author?: string;
+    time?: string;
+    snippet?: string;
+    id?: string;
+    url?: string;
+}
+export interface TiebaSearchItem {
+    rank: number;
+    title: string;
+    forum: string;
+    author: string;
+    time: string;
+    snippet: string;
+    id: string;
+    url: string;
+}
+export interface RawTiebaMainPost {
+    title?: string;
+    author?: string;
+    fallbackAuthor?: string;
+    contentText?: string;
+    structuredText?: string;
+    visibleTime?: string;
+    structuredTime?: unknown;
+    hasMedia?: boolean;
+}
+export interface RawTiebaReply {
+    floor?: unknown;
+    author?: string;
+    content?: string;
+    time?: string;
+}
+export interface RawTiebaReadPayload {
+    mainPost?: RawTiebaMainPost | null;
+    replies?: RawTiebaReply[];
+}
+export interface TiebaReadItem {
+    floor: number;
+    author: string;
+    content: string;
+    time: string;
+}
+export interface TiebaReadBuildOptions {
+    limit?: unknown;
+    includeMainPost?: boolean;
+}
+/**
+ * Keep the public CLI limit contract aligned with the real implementation.
+ */
+export declare function normalizeTiebaLimit(value: unknown, fallback?: number): number;
+export declare function normalizeText(value: unknown): string;
+/**
+ * Match Tieba PC's signed request contract so forum list fetching stays stable.
+ */
+export declare function signTiebaPcParams(params: Record<string, string>): string;
+export declare function parseTiebaCount(text: string): number;
+export declare function parseTiebaLastReply(text: string): string;
+/**
+ * Convert Tieba's signed `page_pc` feed entries into the stable card shape used by the CLI.
+ */
+export declare function buildTiebaPostCardsFromPagePc(rawFeeds: RawTiebaPagePcFeedEntry[]): RawTiebaPostCard[];
+export declare function buildTiebaPostItems(rawCards: RawTiebaPostCard[], requestedLimit: unknown): TiebaPostItem[];
+export declare function buildTiebaSearchItems(rawItems: RawTiebaSearchItem[], requestedLimit: unknown): TiebaSearchItem[];
+export declare function buildTiebaReadItems(payload: RawTiebaReadPayload, options?: TiebaReadBuildOptions): TiebaReadItem[];