@jackwener/opencli 1.8.0 → 1.8.1

package/clis/xiaohongshu/creator-notes.js

@@ -8,11 +8,14 @@
  * Requires: logged into creator.xiaohongshu.com in Chrome.
  */
 import { cli, Strategy } from '@jackwener/opencli/registry';
-import { EmptyResultError } from '@jackwener/opencli/errors';
+import { CommandExecutionError, EmptyResultError } from '@jackwener/opencli/errors';
 const DATE_LINE_RE = /^发布于 (\d{4}年\d{2}月\d{2}日 \d{2}:\d{2})$/;
 const METRIC_LINE_RE = /^\d+$/;
 const VISIBILITY_LINE_RE = /可见$/;
 const NOTE_ANALYZE_API_PATH = '/api/galaxy/creator/datacenter/note/analyze/list';
+const NOTE_ANALYZE_PAGE_SIZE = 10;
+const CAPTURE_POLL_ATTEMPTS = 20;
+const CAPTURE_POLL_INTERVAL_S = 0.5;
 const NOTE_DETAIL_PAGE_URL = 'https://creator.xiaohongshu.com/statistics/note-detail';
 const NOTE_ID_HTML_RE = /"noteId":"([0-9a-f]{24})"/g;
 function buildNoteDetailUrl(noteId) {
@@ -105,6 +108,237 @@ function mapAnalyzeItems(items) {
         url: buildNoteDetailUrl(item.id),
     }));
 }
+function unwrapEvaluateResult(payload) {
+    if (payload && typeof payload === 'object' && !Array.isArray(payload) && 'session' in payload && 'data' in payload) {
+        return payload.data;
+    }
+    return payload;
+}
+// Capture the dashboard's signed /api/galaxy/* responses on window.__xhsCapture
+// since a direct fetch() from page.evaluate bypasses the x-s signing and gets 406.
+async function installXhsFetchCaptureHook(page) {
+    await page.evaluate(`(() => {
+    window.__xhsCapture = {};
+    if (window.__xhsCaptureInstalled) return;
+    window.__xhsCaptureInstalled = true;
+    const origFetch = window.fetch;
+    window.fetch = async function(...args) {
+      const resp = await origFetch.apply(this, args);
+      try {
+        const url = typeof args[0] === 'string' ? args[0] : (args[0] && args[0].url) || '';
+        if (url.includes('/api/galaxy/')) {
+          resp.clone().text().then((body) => {
+            try { window.__xhsCapture[url] = { status: resp.status, ok: resp.ok, body }; } catch (_) {}
+          }).catch(() => {});
+        }
+      } catch (_) {}
+      return resp;
+    };
+    const OrigXHR = window.XMLHttpRequest;
+    function HookedXHR() {
+      const xhr = new OrigXHR();
+      const origOpen = xhr.open;
+      let capturedUrl = '';
+      xhr.open = function(method, url, ...rest) {
+        capturedUrl = url;
+        return origOpen.call(this, method, url, ...rest);
+      };
+      xhr.addEventListener('load', () => {
+        try {
+          if (capturedUrl.includes('/api/galaxy/')) {
+            window.__xhsCapture[capturedUrl] = { status: xhr.status, ok: xhr.status >= 200 && xhr.status < 300, body: xhr.responseText };
+          }
+        } catch (_) {}
+      });
+      return xhr;
+    }
+    HookedXHR.prototype = OrigXHR.prototype;
+    for (const key of ['UNSENT', 'OPENED', 'HEADERS_RECEIVED', 'LOADING', 'DONE']) {
+      if (key in OrigXHR) HookedXHR[key] = OrigXHR[key];
+    }
+    window.XMLHttpRequest = HookedXHR;
+  })()`);
+}
+function parseCaptureMapPayload(raw) {
+    const payload = unwrapEvaluateResult(raw);
+    if (typeof payload === 'string') {
+        try {
+            return JSON.parse(payload);
+        }
+        catch {
+            return {};
+        }
+    }
+    if (payload && typeof payload === 'object' && !Array.isArray(payload)) {
+        return payload;
+    }
+    return {};
+}
+function getAnalyzeListPageNumber(url) {
+    try {
+        const parsed = new URL(url, 'https://creator.xiaohongshu.com');
+        const pageNum = Number.parseInt(parsed.searchParams.get('page_num') || '', 10);
+        if (Number.isFinite(pageNum) && pageNum > 0)
+            return pageNum;
+    }
+    catch { }
+    const match = String(url || '').match(/[?&]page_num=(\d+)/);
+    const pageNum = Number.parseInt(match?.[1] || '', 10);
+    return Number.isFinite(pageNum) && pageNum > 0 ? pageNum : Number.MAX_SAFE_INTEGER;
+}
+function harvestAnalyzeListCaptures(captureMap) {
+    const items = [];
+    const seen = new Set();
+    let total = 0;
+    const entries = Object.entries(captureMap)
+        .filter(([url]) => url.includes('/note/analyze/list'))
+        .sort(([a], [b]) => getAnalyzeListPageNumber(a) - getAnalyzeListPageNumber(b));
+    for (const [url, capture] of entries) {
+        if (!capture?.ok) continue;
+        try {
+            const json = JSON.parse(capture.body);
+            const data = json?.data ?? {};
+            if (typeof data.total === 'number' && data.total > total) total = data.total;
+            for (const note of data.note_infos ?? []) {
+                if (!note?.id || seen.has(note.id)) continue;
+                seen.add(note.id);
+                items.push(note);
+            }
+        }
+        catch { }
+    }
+    return { items, total };
+}
+function isAnalyzeCaptureComplete(items, total, limit) {
+    if (total <= 0)
+        return true;
+    return items.length >= Math.min(total, limit);
+}
+async function pollCaptureMap(page) {
+    let captureMap = {};
+    for (let i = 0; i < CAPTURE_POLL_ATTEMPTS; i++) {
+        await page.wait(CAPTURE_POLL_INTERVAL_S);
+        const raw = await page.evaluate('JSON.stringify(window.__xhsCapture || {})');
+        captureMap = parseCaptureMapPayload(raw);
+        if (Object.keys(captureMap).some((url) => url.includes('/note/analyze/list'))) break;
+    }
+    return captureMap;
+}
+// Fresh-published notes return title: "" from /note/analyze/list. Scrape the
+// /new/note-manager card DOM (under its "全部笔记" tab, which surfaces every
+// state including 审核中) so the rows the API leaves empty still get the
+// derived title that the note-manager UI shows.
+async function fetchNoteManagerTitleMap(page, neededCount) {
+    const map = new Map();
+    const scrapeCards = async () => {
+        const cards = unwrapEvaluateResult(await page.evaluate(`() => {
+      const noteIdRe = /"noteId":"([0-9a-f]{24})"/;
+      return Array.from(document.querySelectorAll('div.note[data-impression], div.note')).map((card) => {
+        const impression = card.getAttribute('data-impression') || '';
+        const id = impression.match(noteIdRe)?.[1] || '';
+        const title = (card.querySelector('.title, .raw')?.innerText || '').trim();
+        return { id, title };
+      }).filter((entry) => entry.id && entry.title);
+    }`));
+        for (const card of Array.isArray(cards) ? cards : []) {
+            if (!map.has(card.id)) map.set(card.id, card.title);
+        }
+    };
+    // Scroll the first scrollable ancestor of a note card to the bottom so
+    // the list lazy-loads the rest of its rows. Page-level scrollTo does not
+    // work because the cards live inside an inner overflow-auto container.
+    const scrollInnerListToBottom = async () => {
+        return unwrapEvaluateResult(await page.evaluate(`(() => {
+      const firstCard = document.querySelector('div.note[data-impression]');
+      let el = firstCard && firstCard.parentElement;
+      while (el) {
+        const s = window.getComputedStyle(el);
+        if ((s.overflowY === 'auto' || s.overflowY === 'scroll') && el.scrollHeight > el.clientHeight + 10) {
+          el.scrollTop = el.scrollHeight;
+          return true;
+        }
+        el = el.parentElement;
+      }
+      return false;
+    })()`));
+    };
+    try {
+        await page.goto('https://creator.xiaohongshu.com/new/note-manager');
+        // Poll for the initial hydration batch and then scroll the inner list
+        // container to surface the rest of the rows. The all-notes tab is the
+        // default state so no tab click is needed here.
+        for (let i = 0; i < 12; i++) {
+            await page.wait(1);
+            await scrapeCards();
+            if (map.size >= neededCount) return map;
+            await scrollInnerListToBottom();
+        }
+        return map;
+    }
+    catch {
+        return map;
+    }
+}
+async function fetchCreatorNotesByCapture(page, limit) {
+    // Land on dashboard root before installing the hook so the data-analysis
+    // SPA navigation fires page_num=1's signed request UNDER the hook.
+    await page.goto('https://creator.xiaohongshu.com/statistics');
+    await installXhsFetchCaptureHook(page);
+    await page.evaluate(`(() => {
+    history.pushState({}, '', '/statistics/data-analysis?source=official');
+    window.dispatchEvent(new PopStateEvent('popstate'));
+  })()`);
+    let captureMap = await pollCaptureMap(page);
+    let { items, total } = harvestAnalyzeListCaptures(captureMap);
+    if (items.length === 0) return [];
+    const totalPages = total > 0 ? Math.ceil(total / NOTE_ANALYZE_PAGE_SIZE) : 1;
+    const neededPages = Math.min(totalPages, Math.ceil(limit / NOTE_ANALYZE_PAGE_SIZE));
+    for (let pageNum = 2; pageNum <= neededPages && items.length < limit; pageNum++) {
+        const clicked = unwrapEvaluateResult(await page.evaluate(`(() => {
+      const target = String(${pageNum});
+      // .d-pagination-page renders the page number doubled (a visible span +
+      // an accessibility span), so textContent for page 2 reads "22". Match
+      // both the raw digit and the doubled form to tolerate either render.
+      const btns = Array.from(document.querySelectorAll('.d-pagination-page'));
+      const match = btns.find((btn) => {
+        const text = (btn.textContent || '').trim();
+        return text === target || text === target + target;
+      });
+      if (match) { match.click(); return true; }
+      return false;
+    })()`));
+        if (!clicked) break;
+        const before = items.length;
+        let advanced = false;
+        for (let attempt = 0; attempt < CAPTURE_POLL_ATTEMPTS; attempt++) {
+            await page.wait(CAPTURE_POLL_INTERVAL_S);
+            const raw = await page.evaluate('JSON.stringify(window.__xhsCapture || {})');
+            captureMap = parseCaptureMapPayload(raw);
+            const harvested = harvestAnalyzeListCaptures(captureMap);
+            if (harvested.items.length > before) {
+                items = harvested.items;
+                total = Math.max(total, harvested.total);
+                advanced = true;
+                break;
+            }
+        }
+        if (!advanced) break;
+    }
+    if (!isAnalyzeCaptureComplete(items, total, limit)) {
+        throw new CommandExecutionError(`xiaohongshu creator-notes: captured ${items.length} of ${Math.min(total, limit)} expected analyze rows; refusing partial results`);
+    }
+    const notes = mapAnalyzeItems(items).slice(0, limit);
+    const missingTitles = notes.filter((note) => !note.title).length;
+    if (missingTitles > 0) {
+        const titleMap = await fetchNoteManagerTitleMap(page, notes.length);
+        for (const note of notes) {
+            if (!note.title && note.id && titleMap.has(note.id)) {
+                note.title = titleMap.get(note.id);
+            }
+        }
+    }
+    return notes;
+}
 async function fetchCreatorNotesByApi(page, limit) {
     const pageSize = Math.min(Math.max(limit, 10), 20);
     const maxPages = Math.max(1, Math.ceil(limit / pageSize));
@@ -148,7 +382,16 @@ async function fetchCreatorNotesByApi(page, limit) {
     return notes.slice(0, limit);
 }
 export async function fetchCreatorNotes(page, limit) {
-    let notes = await fetchCreatorNotesByApi(page, limit);
+    let notes = [];
+    try {
+        notes = await fetchCreatorNotesByCapture(page, limit);
+    }
+    catch (error) {
+        if (error instanceof CommandExecutionError) throw error;
+    }
+    if (notes.length === 0) {
+        notes = await fetchCreatorNotesByApi(page, limit);
+    }
     if (notes.length === 0) {
         await page.goto('https://creator.xiaohongshu.com/new/note-manager');
         const maxPageDowns = Math.max(0, Math.ceil(limit / 10) + 1);
@@ -228,3 +471,9 @@ cli({
         }));
     },
 });
+export const __test__ = {
+    harvestAnalyzeListCaptures,
+    isAnalyzeCaptureComplete,
+    parseCaptureMapPayload,
+    unwrapEvaluateResult,
+};

package/clis/xiaohongshu/creator-notes.test.js

@@ -1,7 +1,7 @@
 import { describe, expect, it, vi } from 'vitest';
-import { EmptyResultError } from '@jackwener/opencli/errors';
+import { CommandExecutionError, EmptyResultError } from '@jackwener/opencli/errors';
 import { getRegistry } from '@jackwener/opencli/registry';
-import { parseCreatorNoteIdsFromHtml, parseCreatorNotesText } from './creator-notes.js';
+import { __test__, parseCreatorNoteIdsFromHtml, parseCreatorNotesText } from './creator-notes.js';
 import './creator-notes.js';
 function createPageMock(evaluateResult, interceptedRequests = []) {
     const evaluate = Array.isArray(evaluateResult)
@@ -190,6 +190,83 @@ describe('xiaohongshu creator-notes', () => {
             'dddddddddddddddddddddddd',
         ]);
     });
+    it('harvests captured analyze pages in page order and dedupes note ids', () => {
+        const captureMap = {
+            '/api/galaxy/creator/datacenter/note/analyze/list?type=0&page_size=10&page_num=2': {
+                ok: true,
+                body: JSON.stringify({
+                    data: {
+                        total: 3,
+                        note_infos: [
+                            { id: 'bbbbbbbbbbbbbbbbbbbbbbbb', title: 'page 2' },
+                            { id: 'aaaaaaaaaaaaaaaaaaaaaaaa', title: 'duplicate from page 2' },
+                        ],
+                    },
+                }),
+            },
+            '/api/galaxy/creator/datacenter/note/analyze/list?type=0&page_size=10&page_num=1': {
+                ok: true,
+                body: JSON.stringify({
+                    data: {
+                        total: 3,
+                        note_infos: [
+                            { id: 'aaaaaaaaaaaaaaaaaaaaaaaa', title: 'page 1' },
+                        ],
+                    },
+                }),
+            },
+        };
+        expect(__test__.harvestAnalyzeListCaptures(captureMap)).toEqual({
+            total: 3,
+            items: [
+                { id: 'aaaaaaaaaaaaaaaaaaaaaaaa', title: 'page 1' },
+                { id: 'bbbbbbbbbbbbbbbbbbbbbbbb', title: 'page 2' },
+            ],
+        });
+    });
+    it('treats incomplete captured pagination as fallback-needed instead of partial success', () => {
+        const firstPageItems = Array.from({ length: 10 }, (_, index) => ({
+            id: String(index).padStart(24, '0'),
+        }));
+        expect(__test__.isAnalyzeCaptureComplete(firstPageItems, 25, 20)).toBe(false);
+        expect(__test__.isAnalyzeCaptureComplete(firstPageItems, 25, 10)).toBe(true);
+        expect(__test__.isAnalyzeCaptureComplete(firstPageItems, 0, 20)).toBe(true);
+    });
+    it('unwraps browser bridge capture-map envelopes', () => {
+        const captureMap = {
+            '/api/galaxy/creator/datacenter/note/analyze/list?page_num=1': {
+                ok: true,
+                body: '{"data":{"total":0,"note_infos":[]}}',
+            },
+        };
+        expect(__test__.parseCaptureMapPayload({ session: 'site:xiaohongshu', data: JSON.stringify(captureMap) })).toEqual(captureMap);
+        expect(__test__.parseCaptureMapPayload({ session: 'site:xiaohongshu', data: captureMap })).toEqual(captureMap);
+    });
+    it('does not fall back to partial DOM rows when captured total proves pagination is incomplete', async () => {
+        const cmd = getRegistry().get('xiaohongshu/creator-notes');
+        const captureMap = {
+            '/api/galaxy/creator/datacenter/note/analyze/list?type=0&page_size=10&page_num=1': {
+                ok: true,
+                body: JSON.stringify({
+                    data: {
+                        total: 25,
+                        note_infos: Array.from({ length: 10 }, (_, index) => ({
+                            id: String(index).padStart(24, '0'),
+                            title: `note ${index}`,
+                        })),
+                    },
+                }),
+            },
+        };
+        const page = createPageMock(undefined);
+        page.evaluate = vi.fn()
+            .mockResolvedValueOnce(undefined)
+            .mockResolvedValueOnce(undefined)
+            .mockResolvedValueOnce(JSON.stringify(captureMap))
+            .mockResolvedValueOnce(false);
+
+        await expect(cmd.func(page, { limit: 20 })).rejects.toBeInstanceOf(CommandExecutionError);
+    });
     it('throws EmptyResultError when the creator account has no notes', async () => {
         const cmd = getRegistry().get('xiaohongshu/creator-notes');
         const page = createPageMock(undefined);

package/clis/xiaohongshu/download.js

@@ -52,49 +52,108 @@ export function buildDownloadExtractJs(noteId) {
         const authorEl = document.querySelector('.username, .author-name, .name');
         result.author = authorEl?.textContent?.trim() || 'unknown';
 
-        // Get images - try multiple selectors
-        const imageSelectors = [
-          '.swiper-slide img',
-          '.carousel-image img',
-          '.note-slider img',
-          '.note-image img',
-          '.image-wrapper img',
-          '#noteContainer .media-container img[src*="xhscdn"]',
-          'img[src*="ci.xiaohongshu.com"]'
-        ];
+        // Get images: prefer canonical carousel order from __INITIAL_STATE__
+        // so the saved order matches what the user sees on the platform (#1514).
+        // DOM extraction is used only as a fallback because multiple selectors,
+        // hidden / duplicated / preloaded slides, and lazy rendering can reorder
+        // the discovered nodes away from the platform's display order.
 
-        const imageUrls = new Set();
-        for (const selector of imageSelectors) {
-          document.querySelectorAll(selector).forEach(img => {
-            let src = img.src || img.getAttribute('data-src') || '';
-            if (src && (src.includes('xhscdn') || src.includes('xiaohongshu') || src.includes('rednote'))) {
-              src = src.split('?')[0];
-              src = src.replace(/\\/imageView\\d+\\/\\d+\\/w\\/\\d+/, '');
-              imageUrls.add(src);
+        const normalizeImageUrl = (raw) => {
+          if (!raw || typeof raw !== 'string') return '';
+          let src = raw.split('?')[0];
+          src = src.replace(/\\/imageView\\d+\\/\\d+\\/w\\/\\d+/, '');
+          return src;
+        };
+        const orderedImageUrls = [];
+        const seenImageUrls = new Set();
+        const pushImage = (url) => {
+          if (!url || seenImageUrls.has(url)) return;
+          seenImageUrls.add(url);
+          orderedImageUrls.push(url);
+        };
+
+        const getStructuredNotes = () => {
+          const state = window.__INITIAL_STATE__;
+          const noteData = state?.note?.noteDetailMap || state?.note?.note || {};
+          if (!noteData || typeof noteData !== 'object') return [];
+          const currentIds = [...new Set([result.noteId, '${noteId}'].filter(Boolean))];
+          const notes = [];
+          for (const id of currentIds) {
+            const entry = noteData[id];
+            const note = entry?.note || entry;
+            if (note && typeof note === 'object') notes.push(note);
+          }
+          // Compatibility fallback for legacy single-note stores. Do not use this
+          // when keyed detail maps contain multiple notes, or carousel order can
+          // be polluted by preloaded/previous note entries.
+          const keys = Object.keys(noteData);
+          if (notes.length === 0 && keys.length === 1) {
+            const entry = noteData[keys[0]];
+            const note = entry?.note || entry;
+            if (note && typeof note === 'object') notes.push(note);
+          }
+          return notes;
+        };
+
+        // Method 1: walk __INITIAL_STATE__.note.noteDetailMap[id].note.imageList
+        // in array order. Each entry exposes urlDefault as the canonical CDN URL.
+        let imageInitialStateUsed = false;
+        try {
+          for (const note of getStructuredNotes()) {
+            const list = Array.isArray(note?.imageList) ? note.imageList : [];
+            for (const item of list) {
+              const candidate = item?.urlDefault || item?.urlPre || item?.url
+                || item?.infoList?.find(i => i?.imageScene === 'WB_DFT')?.url
+                || item?.infoList?.[0]?.url
+                || '';
+              const src = normalizeImageUrl(candidate);
+              if (src && (src.includes('xhscdn') || src.includes('xiaohongshu') || src.includes('rednote'))) {
+                pushImage(src);
+                imageInitialStateUsed = true;
+              }
             }
-          });
+          }
+        } catch(e) {}
+
+        // Method 2: fallback to DOM scraping when the structured state is missing
+        // (e.g. preview pages without full SSR hydration). Order may differ from
+        // the carousel; surface it anyway rather than returning zero images.
+        if (!imageInitialStateUsed) {
+          const imageSelectors = [
+            '.swiper-slide img',
+            '.carousel-image img',
+            '.note-slider img',
+            '.note-image img',
+            '.image-wrapper img',
+            '#noteContainer .media-container img[src*="xhscdn"]',
+            'img[src*="ci.xiaohongshu.com"]'
+          ];
+          for (const selector of imageSelectors) {
+            document.querySelectorAll(selector).forEach(img => {
+              const raw = img.src || img.getAttribute('data-src') || '';
+              const src = normalizeImageUrl(raw);
+              if (src && (src.includes('xhscdn') || src.includes('xiaohongshu') || src.includes('rednote'))) {
+                pushImage(src);
+              }
+            });
+          }
         }
 
         // Get video — prefer real URL from page state over blob: URLs
 
         // Method 1: Extract from __INITIAL_STATE__ (SSR hydration data)
         try {
-          const state = window.__INITIAL_STATE__;
-          if (state) {
-            const noteData = state.note?.noteDetailMap || state.note?.note || {};
-            for (const key of Object.keys(noteData)) {
-              const note = noteData[key]?.note || noteData[key];
-              const video = note?.video;
-              if (video) {
-                const vUrl = video.url || video.originVideoKey || video.consumer?.originVideoKey;
-                if (vUrl) {
-                  const fullUrl = vUrl.startsWith('http') ? vUrl : 'https://sns-video-bd.xhscdn.com/' + vUrl;
-                  pushMedia('video', fullUrl);
-                }
-                const streams = video.media?.stream?.h264 || [];
-                for (const stream of streams) {
-                  if (stream.masterUrl) pushMedia('video', stream.masterUrl);
-                }
+          for (const note of getStructuredNotes()) {
+            const video = note?.video;
+            if (video) {
+              const vUrl = video.url || video.originVideoKey || video.consumer?.originVideoKey;
+              if (vUrl) {
+                const fullUrl = vUrl.startsWith('http') ? vUrl : 'https://sns-video-bd.xhscdn.com/' + vUrl;
+                pushMedia('video', fullUrl);
+              }
+              const streams = video.media?.stream?.h264 || [];
+              for (const stream of streams) {
+                if (stream.masterUrl) pushMedia('video', stream.masterUrl);
               }
             }
           }
@@ -135,10 +194,9 @@ export function buildDownloadExtractJs(noteId) {
           }
         }
 
-        // Add images to media
-        imageUrls.forEach(url => {
-          pushMedia('image', url);
-        });
+        // Preserve the pre-existing media type order (videos first, then images)
+        // while keeping image carousel order stable within the image batch.
+        orderedImageUrls.forEach(url => pushMedia('image', url));
 
         return result;
       })()