@agentrhq/webcmd 0.1.1 → 0.2.0

package/clis/xiaohongshu/creator-notes.js

@@ -1,479 +0,0 @@
-/**
- * Xiaohongshu Creator Note List — per-note metrics from the creator backend.
- *
- * In CDP mode we capture the real creator analytics API response so the list
- * includes stable note ids and detail-page URLs. If that capture is unavailable,
- * we fall back to the older interceptor and DOM parsing paths.
- *
- * Requires: logged into creator.xiaohongshu.com in Chrome.
- */
-import { cli, Strategy } from '@agentrhq/webcmd/registry';
-import { CommandExecutionError, EmptyResultError } from '@agentrhq/webcmd/errors';
-const DATE_LINE_RE = /^发布于 (\d{4}年\d{2}月\d{2}日 \d{2}:\d{2})$/;
-const METRIC_LINE_RE = /^\d+$/;
-const VISIBILITY_LINE_RE = /可见$/;
-const NOTE_ANALYZE_API_PATH = '/api/galaxy/creator/datacenter/note/analyze/list';
-const NOTE_ANALYZE_PAGE_SIZE = 10;
-const CAPTURE_POLL_ATTEMPTS = 20;
-const CAPTURE_POLL_INTERVAL_S = 0.5;
-const NOTE_DETAIL_PAGE_URL = 'https://creator.xiaohongshu.com/statistics/note-detail';
-const NOTE_ID_HTML_RE = /"noteId":"([0-9a-f]{24})"/g;
-function buildNoteDetailUrl(noteId) {
-    return noteId ? `${NOTE_DETAIL_PAGE_URL}?noteId=${encodeURIComponent(noteId)}` : '';
-}
-function formatPostTime(ts) {
-    if (!ts)
-        return '';
-    // XHS API timestamps are Beijing time (UTC+8)
-    const date = new Date(ts + 8 * 3600_000);
-    const pad = (value) => String(value).padStart(2, '0');
-    return `${date.getUTCFullYear()}年${pad(date.getUTCMonth() + 1)}月${pad(date.getUTCDate())}日 ${pad(date.getUTCHours())}:${pad(date.getUTCMinutes())}`;
-}
-export function parseCreatorNotesText(bodyText) {
-    const lines = bodyText
-        .split('\n')
-        .map((line) => line.trim())
-        .filter(Boolean);
-    const results = [];
-    const seen = new Set();
-    for (let i = 0; i < lines.length; i++) {
-        const dateMatch = lines[i].match(DATE_LINE_RE);
-        if (!dateMatch)
-            continue;
-        let titleIndex = i - 1;
-        while (titleIndex >= 0 && VISIBILITY_LINE_RE.test(lines[titleIndex]))
-            titleIndex--;
-        if (titleIndex < 0)
-            continue;
-        const title = lines[titleIndex];
-        const metrics = [];
-        let cursor = i + 1;
-        while (cursor < lines.length && METRIC_LINE_RE.test(lines[cursor]) && metrics.length < 5) {
-            metrics.push(parseInt(lines[cursor], 10));
-            cursor++;
-        }
-        if (metrics.length < 4)
-            continue;
-        const key = `${title}@@${dateMatch[1]}`;
-        if (seen.has(key))
-            continue;
-        seen.add(key);
-        results.push({
-            id: '',
-            title,
-            date: dateMatch[1],
-            views: metrics[0] ?? 0,
-            likes: metrics[2] ?? 0,
-            collects: metrics[3] ?? 0,
-            comments: metrics[1] ?? 0,
-            url: '',
-        });
-        i = cursor - 1;
-    }
-    return results;
-}
-export function parseCreatorNoteIdsFromHtml(bodyHtml) {
-    const ids = [];
-    const seen = new Set();
-    for (const match of bodyHtml.matchAll(NOTE_ID_HTML_RE)) {
-        const id = match[1];
-        if (!id || seen.has(id))
-            continue;
-        seen.add(id);
-        ids.push(id);
-    }
-    return ids;
-}
-function mapDomCards(cards) {
-    return cards.map((card) => ({
-        id: card.id,
-        title: card.title,
-        date: card.date,
-        views: card.metrics[0] ?? 0,
-        likes: card.metrics[2] ?? 0,
-        collects: card.metrics[3] ?? 0,
-        comments: card.metrics[1] ?? 0,
-        url: buildNoteDetailUrl(card.id),
-    }));
-}
-function mapAnalyzeItems(items) {
-    return (items ?? []).map((item) => ({
-        id: item.id ?? '',
-        title: item.title ?? '',
-        date: formatPostTime(item.post_time),
-        views: item.read_count ?? 0,
-        likes: item.like_count ?? 0,
-        collects: item.fav_count ?? 0,
-        comments: item.comment_count ?? 0,
-        url: buildNoteDetailUrl(item.id),
-    }));
-}
-function unwrapEvaluateResult(payload) {
-    if (payload && typeof payload === 'object' && !Array.isArray(payload) && 'session' in payload && 'data' in payload) {
-        return payload.data;
-    }
-    return payload;
-}
-// Capture the dashboard's signed /api/galaxy/* responses on window.__xhsCapture
-// since a direct fetch() from page.evaluate bypasses the x-s signing and gets 406.
-async function installXhsFetchCaptureHook(page) {
-    await page.evaluate(`(() => {
-    window.__xhsCapture = {};
-    if (window.__xhsCaptureInstalled) return;
-    window.__xhsCaptureInstalled = true;
-    const origFetch = window.fetch;
-    window.fetch = async function(...args) {
-      const resp = await origFetch.apply(this, args);
-      try {
-        const url = typeof args[0] === 'string' ? args[0] : (args[0] && args[0].url) || '';
-        if (url.includes('/api/galaxy/')) {
-          resp.clone().text().then((body) => {
-            try { window.__xhsCapture[url] = { status: resp.status, ok: resp.ok, body }; } catch (_) {}
-          }).catch(() => {});
-        }
-      } catch (_) {}
-      return resp;
-    };
-    const OrigXHR = window.XMLHttpRequest;
-    function HookedXHR() {
-      const xhr = new OrigXHR();
-      const origOpen = xhr.open;
-      let capturedUrl = '';
-      xhr.open = function(method, url, ...rest) {
-        capturedUrl = url;
-        return origOpen.call(this, method, url, ...rest);
-      };
-      xhr.addEventListener('load', () => {
-        try {
-          if (capturedUrl.includes('/api/galaxy/')) {
-            window.__xhsCapture[capturedUrl] = { status: xhr.status, ok: xhr.status >= 200 && xhr.status < 300, body: xhr.responseText };
-          }
-        } catch (_) {}
-      });
-      return xhr;
-    }
-    HookedXHR.prototype = OrigXHR.prototype;
-    for (const key of ['UNSENT', 'OPENED', 'HEADERS_RECEIVED', 'LOADING', 'DONE']) {
-      if (key in OrigXHR) HookedXHR[key] = OrigXHR[key];
-    }
-    window.XMLHttpRequest = HookedXHR;
-  })()`);
-}
-function parseCaptureMapPayload(raw) {
-    const payload = unwrapEvaluateResult(raw);
-    if (typeof payload === 'string') {
-        try {
-            return JSON.parse(payload);
-        }
-        catch {
-            return {};
-        }
-    }
-    if (payload && typeof payload === 'object' && !Array.isArray(payload)) {
-        return payload;
-    }
-    return {};
-}
-function getAnalyzeListPageNumber(url) {
-    try {
-        const parsed = new URL(url, 'https://creator.xiaohongshu.com');
-        const pageNum = Number.parseInt(parsed.searchParams.get('page_num') || '', 10);
-        if (Number.isFinite(pageNum) && pageNum > 0)
-            return pageNum;
-    }
-    catch { }
-    const match = String(url || '').match(/[?&]page_num=(\d+)/);
-    const pageNum = Number.parseInt(match?.[1] || '', 10);
-    return Number.isFinite(pageNum) && pageNum > 0 ? pageNum : Number.MAX_SAFE_INTEGER;
-}
-function harvestAnalyzeListCaptures(captureMap) {
-    const items = [];
-    const seen = new Set();
-    let total = 0;
-    const entries = Object.entries(captureMap)
-        .filter(([url]) => url.includes('/note/analyze/list'))
-        .sort(([a], [b]) => getAnalyzeListPageNumber(a) - getAnalyzeListPageNumber(b));
-    for (const [url, capture] of entries) {
-        if (!capture?.ok) continue;
-        try {
-            const json = JSON.parse(capture.body);
-            const data = json?.data ?? {};
-            if (typeof data.total === 'number' && data.total > total) total = data.total;
-            for (const note of data.note_infos ?? []) {
-                if (!note?.id || seen.has(note.id)) continue;
-                seen.add(note.id);
-                items.push(note);
-            }
-        }
-        catch { }
-    }
-    return { items, total };
-}
-function isAnalyzeCaptureComplete(items, total, limit) {
-    if (total <= 0)
-        return true;
-    return items.length >= Math.min(total, limit);
-}
-async function pollCaptureMap(page) {
-    let captureMap = {};
-    for (let i = 0; i < CAPTURE_POLL_ATTEMPTS; i++) {
-        await page.wait(CAPTURE_POLL_INTERVAL_S);
-        const raw = await page.evaluate('JSON.stringify(window.__xhsCapture || {})');
-        captureMap = parseCaptureMapPayload(raw);
-        if (Object.keys(captureMap).some((url) => url.includes('/note/analyze/list'))) break;
-    }
-    return captureMap;
-}
-// Fresh-published notes return title: "" from /note/analyze/list. Scrape the
-// /new/note-manager card DOM (under its "全部笔记" tab, which surfaces every
-// state including 审核中) so the rows the API leaves empty still get the
-// derived title that the note-manager UI shows.
-async function fetchNoteManagerTitleMap(page, neededCount) {
-    const map = new Map();
-    const scrapeCards = async () => {
-        const cards = unwrapEvaluateResult(await page.evaluate(`() => {
-      const noteIdRe = /"noteId":"([0-9a-f]{24})"/;
-      return Array.from(document.querySelectorAll('div.note[data-impression], div.note')).map((card) => {
-        const impression = card.getAttribute('data-impression') || '';
-        const id = impression.match(noteIdRe)?.[1] || '';
-        const title = (card.querySelector('.title, .raw')?.innerText || '').trim();
-        return { id, title };
-      }).filter((entry) => entry.id && entry.title);
-    }`));
-        for (const card of Array.isArray(cards) ? cards : []) {
-            if (!map.has(card.id)) map.set(card.id, card.title);
-        }
-    };
-    // Scroll the first scrollable ancestor of a note card to the bottom so
-    // the list lazy-loads the rest of its rows. Page-level scrollTo does not
-    // work because the cards live inside an inner overflow-auto container.
-    const scrollInnerListToBottom = async () => {
-        return unwrapEvaluateResult(await page.evaluate(`(() => {
-      const firstCard = document.querySelector('div.note[data-impression]');
-      let el = firstCard && firstCard.parentElement;
-      while (el) {
-        const s = window.getComputedStyle(el);
-        if ((s.overflowY === 'auto' || s.overflowY === 'scroll') && el.scrollHeight > el.clientHeight + 10) {
-          el.scrollTop = el.scrollHeight;
-          return true;
-        }
-        el = el.parentElement;
-      }
-      return false;
-    })()`));
-    };
-    try {
-        await page.goto('https://creator.xiaohongshu.com/new/note-manager');
-        // Poll for the initial hydration batch and then scroll the inner list
-        // container to surface the rest of the rows. The all-notes tab is the
-        // default state so no tab click is needed here.
-        for (let i = 0; i < 12; i++) {
-            await page.wait(1);
-            await scrapeCards();
-            if (map.size >= neededCount) return map;
-            await scrollInnerListToBottom();
-        }
-        return map;
-    }
-    catch {
-        return map;
-    }
-}
-async function fetchCreatorNotesByCapture(page, limit) {
-    // Land on dashboard root before installing the hook so the data-analysis
-    // SPA navigation fires page_num=1's signed request UNDER the hook.
-    await page.goto('https://creator.xiaohongshu.com/statistics');
-    await installXhsFetchCaptureHook(page);
-    await page.evaluate(`(() => {
-    history.pushState({}, '', '/statistics/data-analysis?source=official');
-    window.dispatchEvent(new PopStateEvent('popstate'));
-  })()`);
-    let captureMap = await pollCaptureMap(page);
-    let { items, total } = harvestAnalyzeListCaptures(captureMap);
-    if (items.length === 0) return [];
-    const totalPages = total > 0 ? Math.ceil(total / NOTE_ANALYZE_PAGE_SIZE) : 1;
-    const neededPages = Math.min(totalPages, Math.ceil(limit / NOTE_ANALYZE_PAGE_SIZE));
-    for (let pageNum = 2; pageNum <= neededPages && items.length < limit; pageNum++) {
-        const clicked = unwrapEvaluateResult(await page.evaluate(`(() => {
-      const target = String(${pageNum});
-      // .d-pagination-page renders the page number doubled (a visible span +
-      // an accessibility span), so textContent for page 2 reads "22". Match
-      // both the raw digit and the doubled form to tolerate either render.
-      const btns = Array.from(document.querySelectorAll('.d-pagination-page'));
-      const match = btns.find((btn) => {
-        const text = (btn.textContent || '').trim();
-        return text === target || text === target + target;
-      });
-      if (match) { match.click(); return true; }
-      return false;
-    })()`));
-        if (!clicked) break;
-        const before = items.length;
-        let advanced = false;
-        for (let attempt = 0; attempt < CAPTURE_POLL_ATTEMPTS; attempt++) {
-            await page.wait(CAPTURE_POLL_INTERVAL_S);
-            const raw = await page.evaluate('JSON.stringify(window.__xhsCapture || {})');
-            captureMap = parseCaptureMapPayload(raw);
-            const harvested = harvestAnalyzeListCaptures(captureMap);
-            if (harvested.items.length > before) {
-                items = harvested.items;
-                total = Math.max(total, harvested.total);
-                advanced = true;
-                break;
-            }
-        }
-        if (!advanced) break;
-    }
-    if (!isAnalyzeCaptureComplete(items, total, limit)) {
-        throw new CommandExecutionError(`xiaohongshu creator-notes: captured ${items.length} of ${Math.min(total, limit)} expected analyze rows; refusing partial results`);
-    }
-    const notes = mapAnalyzeItems(items).slice(0, limit);
-    const missingTitles = notes.filter((note) => !note.title).length;
-    if (missingTitles > 0) {
-        const titleMap = await fetchNoteManagerTitleMap(page, notes.length);
-        for (const note of notes) {
-            if (!note.title && note.id && titleMap.has(note.id)) {
-                note.title = titleMap.get(note.id);
-            }
-        }
-    }
-    return notes;
-}
-async function fetchCreatorNotesByApi(page, limit) {
-    const pageSize = Math.min(Math.max(limit, 10), 20);
-    const maxPages = Math.max(1, Math.ceil(limit / pageSize));
-    const notes = [];
-    await page.goto(`https://creator.xiaohongshu.com/statistics/data-analysis?type=0&page_size=${pageSize}&page_num=1`);
-    for (let pageNum = 1; pageNum <= maxPages && notes.length < limit; pageNum++) {
-        const apiPath = `${NOTE_ANALYZE_API_PATH}?type=0&page_size=${pageSize}&page_num=${pageNum}`;
-        const fetched = await page.evaluate(`
-      async () => {
-        try {
-          const resp = await fetch(${JSON.stringify(apiPath)}, { credentials: 'include' });
-          if (!resp.ok) return { error: 'HTTP ' + resp.status };
-          return await resp.json();
-        } catch (e) {
-          return { error: e?.message ?? String(e) };
-        }
-      }
-    `);
-        let items = fetched?.data?.note_infos ?? [];
-        if (!items.length) {
-            await page.installInterceptor(NOTE_ANALYZE_API_PATH);
-            await page.evaluate(`
-        async () => {
-          try {
-            await fetch(${JSON.stringify(apiPath)}, { credentials: 'include' });
-          } catch {}
-          return true;
-        }
-      `);
-            await page.wait(1);
-            const intercepted = await page.getInterceptedRequests();
-            const data = intercepted.find((entry) => Array.isArray(entry?.data?.note_infos));
-            items = data?.data?.note_infos ?? [];
-        }
-        if (!items.length)
-            break;
-        notes.push(...mapAnalyzeItems(items));
-        if (items.length < pageSize)
-            break;
-    }
-    return notes.slice(0, limit);
-}
-export async function fetchCreatorNotes(page, limit) {
-    let notes = [];
-    try {
-        notes = await fetchCreatorNotesByCapture(page, limit);
-    }
-    catch (error) {
-        if (error instanceof CommandExecutionError) throw error;
-    }
-    if (notes.length === 0) {
-        notes = await fetchCreatorNotesByApi(page, limit);
-    }
-    if (notes.length === 0) {
-        await page.goto('https://creator.xiaohongshu.com/new/note-manager');
-        const maxPageDowns = Math.max(0, Math.ceil(limit / 10) + 1);
-        for (let i = 0; i <= maxPageDowns; i++) {
-            const domCards = await page.evaluate(`() => {
-        const noteIdRe = /"noteId":"([0-9a-f]{24})"/;
-        return Array.from(document.querySelectorAll('div.note[data-impression], div.note')).map((card) => {
-          const impression = card.getAttribute('data-impression') || '';
-          const id = impression.match(noteIdRe)?.[1] || '';
-          const title = (card.querySelector('.title, .raw')?.innerText || '').trim();
-          const dateText = (card.querySelector('.time_status, .time')?.innerText || '').trim();
-          const date = dateText.replace(/^发布于\\s*/, '');
-          const metrics = Array.from(card.querySelectorAll('.icon_list .icon'))
-            .map((el) => parseInt((el.innerText || '').trim(), 10))
-            .filter((value) => Number.isFinite(value));
-          return { id, title, date, metrics };
-        });
-      }`);
-            const parsedDomNotes = mapDomCards(Array.isArray(domCards) ? domCards : []).filter((note) => note.title && note.date);
-            if (parsedDomNotes.length > 0) {
-                notes = parsedDomNotes;
-            }
-            if (notes.length >= limit || (notes.length > 0 && i === 0))
-                break;
-            const body = await page.evaluate('() => ({ text: document.body.innerText, html: document.body.innerHTML })');
-            const bodyText = typeof body?.text === 'string' ? body.text : '';
-            const bodyHtml = typeof body?.html === 'string' ? body.html : '';
-            const parsedNotes = parseCreatorNotesText(bodyText);
-            const noteIds = parseCreatorNoteIdsFromHtml(bodyHtml);
-            notes = parsedNotes.map((note, index) => {
-                const id = noteIds[index] ?? '';
-                return {
-                    ...note,
-                    id,
-                    url: buildNoteDetailUrl(id),
-                };
-            });
-            if (notes.length >= limit || i === maxPageDowns)
-                break;
-            await page.pressKey('PageDown');
-            await page.wait(1);
-        }
-    }
-    return notes.slice(0, limit);
-}
-cli({
-    site: 'xiaohongshu',
-    name: 'creator-notes',
-    access: 'read',
-    description: '小红书创作者笔记列表 + 每篇数据 (标题/日期/观看/点赞/收藏/评论)',
-    domain: 'creator.xiaohongshu.com',
-    strategy: Strategy.COOKIE,
-    browser: true,
-    navigateBefore: false,
-    args: [
-        { name: 'limit', type: 'int', default: 20, help: 'Number of notes to return' },
-    ],
-    columns: ['rank', 'id', 'title', 'date', 'views', 'likes', 'collects', 'comments', 'url'],
-    func: async (page, kwargs) => {
-        const limit = kwargs.limit || 20;
-        const notes = await fetchCreatorNotes(page, limit);
-        if (!Array.isArray(notes) || notes.length === 0) {
-            throw new EmptyResultError('xiaohongshu creator-notes', 'No notes found. Ensure you are logged into creator.xiaohongshu.com and the account has published notes.');
-        }
-        return notes
-            .slice(0, limit)
-            .map((n, i) => ({
-            rank: i + 1,
-            id: n.id,
-            title: n.title,
-            date: n.date,
-            views: n.views,
-            likes: n.likes,
-            collects: n.collects,
-            comments: n.comments,
-            url: n.url,
-        }));
-    },
-});
-export const __test__ = {
-    harvestAnalyzeListCaptures,
-    isAnalyzeCaptureComplete,
-    parseCaptureMapPayload,
-    unwrapEvaluateResult,
-};