nolo-cli 0.1.40 → 0.1.41

package/integrations/xhs-reader/backends/playwrightProfileCollector.ts

@@ -0,0 +1,624 @@
+/**
+ * Playwright-based XHS profile collector.
+ *
+ * Opens a profile URL in a desktop browser, reads window.__INITIAL_STATE__,
+ * scrolls to trigger pagination via real user_posted responses,
+ * and captures + normalizes all note data.
+ *
+ * Supports dependency injection of a Page-like object for unit testing.
+ */
+
+import { normalizeProfileState, normalizeProfileNote } from "../normalize";
+import { redactXhsSensitiveValue } from "../redaction";
+import type { XhsProfile, XhsNoteSummary, XhsCollectorDiagnostic, XhsFailureCode } from "../types";
+
+// Minimal page interface for dependency injection
+export interface XhsPageLike {
+  goto(url: string, opts?: { waitUntil?: string }): Promise<void>;
+  evaluate<T>(fn: () => T): Promise<T>;
+  waitForResponse(
+    fn: (resp: { url: () => string; ok: () => boolean }) => boolean,
+    opts?: { timeout?: number },
+  ): Promise<{ json: () => Promise<unknown> }>;
+  on(event: string, handler: (...args: unknown[]) => void): void;
+  /** Current page URL. Available on real Playwright pages. */
+  url?(): string;
+  /** Page title. Available on real Playwright pages. */
+  title?(): Promise<string>;
+  close(): Promise<void>;
+}
+
+export interface CollectProfilePageResult {
+  profile: XhsProfile;
+  notes: XhsNoteSummary[];
+  diagnostic?: XhsCollectorDiagnostic;
+}
+
+interface InternalState {
+  noteMap: Map<string, XhsNoteSummary>;
+  profile: XhsProfile | null;
+}
+
+function unwrapReactiveValue(value: unknown): unknown {
+  if (!value || typeof value !== "object") return value;
+  const record = value as Record<string, unknown>;
+  if ("_value" in record || "_rawValue" in record) {
+    return record._value ?? record._rawValue;
+  }
+  return value;
+}
+
+/**
+ * Detect whether the page shows a login wall or captcha.
+ * Returns a diagnostic code + evidence, or null if the page looks normal.
+ */
+function detectPageBlockers(
+  pageUrl: string,
+  pageTitle: string,
+): { code: XhsFailureCode; reason: string } | null {
+  const lowerUrl = pageUrl.toLowerCase();
+  const lowerTitle = pageTitle.toLowerCase();
+
+  // Login redirect detection
+  if (
+    lowerUrl.includes("/login") ||
+    lowerUrl.includes("loginpage") ||
+    lowerUrl.includes("passport.xiaohongshu.com")
+  ) {
+    return { code: "login_required", reason: "page redirected to login URL" };
+  }
+
+  // Title-based login detection
+  if (
+    lowerTitle.includes("登录") ||
+    lowerTitle.includes("login") ||
+    lowerTitle.includes("sign in")
+  ) {
+    return { code: "login_required", reason: "page title indicates login wall" };
+  }
+
+  // Captcha / verification detection
+  if (
+    lowerTitle.includes("验证") ||
+    lowerTitle.includes("captcha") ||
+    lowerTitle.includes("verify") ||
+    lowerTitle.includes("安全验证")
+  ) {
+    return { code: "blocked", reason: "page title indicates captcha/verification wall" };
+  }
+
+  return null;
+}
+
+/**
+ * Probe __INITIAL_STATE__ for note-like data to count how many notes
+ * are visible in the initial page load.
+ */
+function countNotesInState(state: unknown): number {
+  if (!state || typeof state !== "object") return 0;
+  const s = state as Record<string, unknown>;
+  const userContainer = s.user as Record<string, unknown> | undefined;
+
+  // Real XHS shape: state.user.notes[0]
+  const userNotes = unwrapReactiveValue(userContainer?.notes) as unknown[] | undefined;
+  if (Array.isArray(userNotes) && userNotes.length > 0) {
+    const firstPage = userNotes[0];
+    if (Array.isArray(firstPage)) return firstPage.length;
+  }
+
+  // Alternate: state.user.userPageData.notes / noteList / choosedNotes
+  const userPageData = unwrapReactiveValue(userContainer?.userPageData ?? s.userPageData) as
+    | Record<string, unknown>
+    | undefined;
+  if (userPageData) {
+    const lists = ["notes", "noteList", "choosedNotes"] as const;
+    for (const key of lists) {
+      const arr = userPageData[key];
+      if (Array.isArray(arr) && arr.length > 0) return arr.length;
+    }
+  }
+
+  return 0;
+}
+
+/**
+ * Collect profile and notes from a XHS profile page using Playwright.
+ *
+ * @param page - A Playwright Page or compatible object.
+ * @param profileUrl - The full XHS profile URL.
+ * @param maxScrollPages - Max number of scroll/pagination pages to collect.
+ * @returns Normalized profile + notes list + optional diagnostic.
+ */
+export async function collectProfilePage(
+  page: XhsPageLike,
+  profileUrl: string,
+  maxScrollPages: number = 5,
+): Promise<CollectProfilePageResult> {
+  const collectorState: InternalState = {
+    noteMap: new Map(),
+    profile: null,
+  };
+
+  const diagnostic: XhsCollectorDiagnostic = {
+    code: "unknown",
+    message: "",
+  };
+
+  // Set up response listener before navigation to capture user_posted responses
+  const capturedResponses: Array<{ url: string; status: number; json: () => Promise<unknown> }> = [];
+  const responseHandler = (resp: unknown) => {
+    const r = resp as {
+      url: () => string;
+      ok: () => boolean;
+      status: () => number;
+      json: () => Promise<unknown>;
+    };
+    if (
+      r.url().includes("/api/sns/web/v1/user_posted") &&
+      r.ok()
+    ) {
+      capturedResponses.push({
+        url: r.url(),
+        status: r.status(),
+        json: r.json.bind(r),
+      });
+    }
+  };
+
+  // Some Page implementations use page.on, others need different setup
+  try {
+    page.on("response", responseHandler);
+  } catch {
+    // If on() is not available, we rely on __INITIAL_STATE__ only
+  }
+
+  // Navigate to the profile page
+  await page.goto(profileUrl, { waitUntil: "networkidle" });
+
+  // Capture page state for diagnostics
+  const finalUrl = page.url?.() ?? profileUrl;
+  let pageTitle = "";
+  try {
+    pageTitle = page.title ? await page.title() : "";
+  } catch {
+    // title() may not be available in all test mocks
+  }
+
+  diagnostic.finalUrl = redactXhsSensitiveValue(finalUrl) as string;
+  diagnostic.pageTitle = pageTitle;
+
+  // Check for login/captcha blockers
+  const blocker = detectPageBlockers(finalUrl, pageTitle);
+  if (blocker) {
+    // Try to detect login state more precisely via page content
+    const loginHint = await page.evaluate(() => {
+      try {
+        // Check for common XHS login indicators in the DOM
+        const body = document.body?.innerText ?? "";
+        const hasLoginText = /登录|login|sign.?in|手机号|验证码/i.test(body);
+        // Check for login modal / overlay
+        const loginModal = document.querySelector('[class*="login"]') ??
+                           document.querySelector('[class*="Login"]');
+        return { hasLoginText, hasLoginModal: !!loginModal };
+      } catch {
+        return { hasLoginText: false, hasLoginModal: false };
+      }
+    }).catch(() => ({ hasLoginText: false, hasLoginModal: false }));
+
+    diagnostic.loginDetected = loginHint?.hasLoginText || loginHint?.hasLoginModal;
+    diagnostic.redirectedToLogin = finalUrl.toLowerCase().includes("login");
+
+    return {
+      profile: { userId: extractUserId(profileUrl), nickname: "" },
+      notes: [],
+      diagnostic: {
+        ...diagnostic,
+        code: blocker.code,
+        message: blocker.reason +
+          (diagnostic.loginDetected ? " (login UI detected on page)" : ""),
+      },
+    };
+  }
+
+  await waitForInitialProfileState(page, 15_000);
+
+  // Read __INITIAL_STATE__ from the page
+  const initialState = await readSerializableInitialState(page);
+
+  diagnostic.initialStatePresent = !!initialState;
+  diagnostic.initialStateNoteCount = countNotesInState(initialState);
+
+  if (initialState) {
+    const rawState = initialState as Record<string, unknown>;
+
+    // Extract profile info — try real path (state.user.userPageData) first,
+    // then flat path (state.userPageData) for backward compat with test fakes.
+    const userContainer = rawState.user as Record<string, unknown> | undefined;
+    const userPageData = unwrapReactiveValue(
+      userContainer?.userPageData ?? rawState.userPageData,
+    ) as Record<string, unknown> | undefined;
+
+    if (userPageData) {
+      const userId = extractUserId(profileUrl);
+      collectorState.profile = normalizeProfileState(
+        { userPageData: userPageData as any },
+        userId,
+      );
+    }
+
+    // Extract initial notes from __INITIAL_STATE__
+    const noteList = extractNotesFromState(initialState);
+    for (const note of noteList) {
+      collectorState.noteMap.set(note.noteId, note);
+    }
+  }
+
+  // If no profile from state, create a minimal one from URL
+  if (!collectorState.profile) {
+    const userId = extractUserId(profileUrl);
+    collectorState.profile = { userId, nickname: "" };
+  }
+
+  // Scroll to trigger pagination
+  let scrollCount = 0;
+  let consecutiveEmpty = 0;
+
+  while (scrollCount < maxScrollPages && consecutiveEmpty < 2) {
+    const countBefore = collectorState.noteMap.size;
+
+    // Scroll to bottom
+    await page.evaluate(() => {
+      window.scrollTo(0, document.body.scrollHeight);
+    });
+
+    // Wait for a user_posted response or timeout
+    try {
+      const resp = await page.waitForResponse(
+        (r) =>
+          r.url().includes("/api/sns/web/v1/user_posted") && r.ok(),
+        { timeout: 8000 },
+      );
+
+      const body = await resp.json();
+      const pageNotes = extractNotesFromPostedResponse(body);
+      for (const note of pageNotes) {
+        collectorState.noteMap.set(note.noteId, note);
+      }
+    } catch {
+      // Timeout or error - may have no more pages
+    }
+
+    scrollCount++;
+    if (collectorState.noteMap.size === countBefore) {
+      consecutiveEmpty++;
+    } else {
+      consecutiveEmpty = 0;
+    }
+  }
+
+  // Also process any captured responses we haven't handled
+  for (const resp of capturedResponses) {
+    try {
+      const body = await resp.json();
+      const pageNotes = extractNotesFromPostedResponse(body);
+      for (const note of pageNotes) {
+        collectorState.noteMap.set(note.noteId, note);
+      }
+    } catch {
+      // ignore
+    }
+  }
+
+  if (!collectorState.profile?.nickname || collectorState.noteMap.size === 0) {
+    await waitForInitialProfileState(page, 12_000);
+    const lateState = await readSerializableInitialState(page);
+    if (lateState) {
+      const rawState = lateState as Record<string, unknown>;
+      const userContainer = rawState.user as Record<string, unknown> | undefined;
+      const userPageData = unwrapReactiveValue(
+        userContainer?.userPageData ?? rawState.userPageData,
+      ) as Record<string, unknown> | undefined;
+      if (userPageData && !collectorState.profile?.nickname) {
+        const userId = extractUserId(profileUrl);
+        collectorState.profile = normalizeProfileState(
+          { userPageData: userPageData as any },
+          userId,
+        );
+      }
+      const lateNotes = extractNotesFromState(lateState);
+      for (const note of lateNotes) {
+        collectorState.noteMap.set(note.noteId, note);
+      }
+      diagnostic.initialStatePresent = true;
+      diagnostic.initialStateNoteCount = Math.max(
+        diagnostic.initialStateNoteCount ?? 0,
+        countNotesInState(lateState),
+      );
+    }
+  }
+
+  diagnostic.capturedApiResponseCount = capturedResponses.length;
+  if (capturedResponses.length > 0) {
+    diagnostic.firstApiResponseStatus = capturedResponses[0].status;
+  }
+
+  const notes = Array.from(collectorState.noteMap.values());
+
+  // Build diagnostic for empty results
+  if (notes.length === 0) {
+    // Check if the page has login indicators even without URL redirect
+    const loginCheck = await page.evaluate(() => {
+      try {
+        const body = document.body?.innerText ?? "";
+        return {
+          hasLoginText: /登录|login|sign.?in|请先登录|未登录/i.test(body),
+          hasLoginModal: !!(
+            document.querySelector('[class*="login"]') ??
+            document.querySelector('[class*="Login"]')
+          ),
+          bodyLength: body.length,
+        };
+      } catch {
+        return { hasLoginText: false, hasLoginModal: false, bodyLength: 0 };
+      }
+    }).catch(() => ({ hasLoginText: false, hasLoginModal: false, bodyLength: 0 })) as { hasLoginText: boolean; hasLoginModal: boolean; bodyLength: number };
+    const safeLoginCheck = loginCheck ?? { hasLoginText: false, hasLoginModal: false, bodyLength: 0 };
+
+    diagnostic.loginDetected = safeLoginCheck.hasLoginText || safeLoginCheck.hasLoginModal;
+
+    // Determine the appropriate diagnostic code
+    let code: XhsFailureCode = "empty_profile_state";
+    let message = "";
+
+    if (diagnostic.loginDetected) {
+      code = "login_required";
+      message =
+        "页面检测到登录提示，未获取到笔记。" +
+        "请先用 headless=false + keepOpen=true 手动登录，" +
+        "登录后再次运行即可自动复用登录态。";
+    } else if (!diagnostic.initialStatePresent) {
+      code = "empty_profile_state";
+      message =
+        "页面未注入 __INITIAL_STATE__，可能页面未完全加载或被反爬拦截。" +
+        "请尝试增加 timeoutMs 或使用本地持久 profile。";
+    } else if (
+      diagnostic.initialStatePresent &&
+      (diagnostic.initialStateNoteCount ?? 0) === 0 &&
+      diagnostic.capturedApiResponseCount === 0
+    ) {
+      code = "empty_profile_state";
+      message =
+        "__INITIAL_STATE__ 存在但未发现笔记数据，且未捕获到 user_posted API 响应。" +
+        "可能该用户无公开笔记，或页面在 headless 环境下被限流。" +
+        "建议用本地持久 profile 验证。";
+    } else {
+      code = "empty_profile_state";
+      message = "收集完成但未获取到笔记数据。";
+    }
+
+    diagnostic.code = code;
+    diagnostic.message = message;
+  }
+
+  return {
+    profile: collectorState.profile,
+    notes,
+    diagnostic: notes.length === 0 ? diagnostic : undefined,
+  };
+}
+
+async function readSerializableInitialState(page: XhsPageLike): Promise<unknown> {
+  return page.evaluate(() => {
+    const unwrap = (value: any): any =>
+      value && typeof value === "object" && ("_value" in value || "_rawValue" in value)
+        ? (value._value ?? value._rawValue)
+        : value;
+    const state = (window as any).__INITIAL_STATE__;
+    if (!state) return null;
+
+    const userPageData = unwrap(state.user?.userPageData ?? state.userPageData);
+    const notes = unwrap(state.user?.notes);
+    const firstPageNotes = Array.isArray(notes?.[0]) ? notes[0] : undefined;
+
+    return {
+      user: {
+        userPageData,
+        notes: firstPageNotes ? [firstPageNotes] : notes,
+      },
+      userPageData,
+      note: unwrap(state.note),
+      feed: {
+        feeds: unwrap(state.feed?.feeds),
+        items: unwrap(state.feed?.items),
+      },
+    };
+  }).catch(() => null);
+}
+
+async function waitForInitialProfileState(page: XhsPageLike, timeoutMs: number): Promise<"ready" | "mock_skip" | "timeout"> {
+  const started = Date.now();
+  while (Date.now() - started < timeoutMs) {
+    const ready = await page.evaluate(() => {
+      try {
+        const unwrap = (value: any): any =>
+          value && typeof value === "object" && ("_value" in value || "_rawValue" in value)
+            ? (value._value ?? value._rawValue)
+            : value;
+        const state = (window as any).__INITIAL_STATE__;
+        const userPageData = unwrap(state?.user?.userPageData ?? state?.userPageData);
+        const notes = unwrap(state?.user?.notes);
+        const firstPage = Array.isArray(notes?.[0]) ? notes[0] : undefined;
+        const hasProfile = Boolean(userPageData?.basicInfo);
+        const hasNotes = Array.isArray(firstPage) && firstPage.length > 0;
+        return hasProfile || hasNotes;
+      } catch {
+        return false;
+      }
+    }).catch(() => false);
+    if (ready == null) return "mock_skip";
+    if (ready) return "ready";
+    await new Promise((resolve) => setTimeout(resolve, 500));
+  }
+  return "timeout";
+}
+
+/**
+ * Extract notes from __INITIAL_STATE__ object.
+ * Tries multiple known paths:
+ * - Real XHS shape: state.user.notes[0] (first page note array)
+ * - Real XHS shape: state.user.userPageData.notes (alternate)
+ * - Flat test shape: state.userPageData.notes / noteList / choosedNotes
+ */
+function extractNotesFromState(state: unknown): XhsNoteSummary[] {
+  const s = state as Record<string, unknown>;
+  const userContainer = s.user as Record<string, unknown> | undefined;
+
+  // Real XHS shape: state.user.notes[0] is the first-page note array
+  const userNotes = unwrapReactiveValue(userContainer?.notes) as unknown[] | undefined;
+  if (Array.isArray(userNotes) && userNotes.length > 0) {
+    const firstPage = userNotes[0];
+    if (Array.isArray(firstPage)) {
+      return normalizeNotesList(firstPage as Array<Record<string, unknown>>);
+    }
+  }
+
+  // Real XHS shape: state.user.userPageData.notes
+  const userPageData = unwrapReactiveValue(userContainer?.userPageData ?? s.userPageData) as
+    | Record<string, unknown>
+    | undefined;
+  if (userPageData) {
+    const noteList = (userPageData.notes ??
+      userPageData.noteList ??
+      userPageData.choosedNotes ??
+      []) as Array<Record<string, unknown>>;
+    if (noteList.length > 0) {
+      return normalizeNotesList(noteList);
+    }
+  }
+
+  // Additional: try state.note.noteList (some XHS versions)
+  const noteContainer = unwrapReactiveValue(s.note) as Record<string, unknown> | undefined;
+  if (noteContainer) {
+    const noteList = (noteContainer.noteList ?? noteContainer.notes ?? []) as Array<Record<string, unknown>>;
+    if (Array.isArray(noteList) && noteList.length > 0) {
+      return normalizeNotesList(noteList);
+    }
+  }
+
+  // Additional: try state.feed.feeds (some XHS SPA versions)
+  const feedContainer = unwrapReactiveValue(s.feed) as Record<string, unknown> | undefined;
+  if (feedContainer) {
+    const feeds = unwrapReactiveValue(feedContainer.feeds ?? feedContainer.items);
+    if (Array.isArray(feeds) && feeds.length > 0) {
+      return normalizeNotesList(feeds as Array<Record<string, unknown>>);
+    }
+  }
+
+  return [];
+}
+
+/**
+ * Shared normalization for a raw notes list (works for both INITIAL_STATE
+ * and paginated user_posted responses).
+ */
+function normalizeNotesList(rawNotes: Array<Record<string, unknown>>): XhsNoteSummary[] {
+  return rawNotes
+    .map((raw) => normalizeProfileNote(unwrapNoteWrapper(raw) as any))
+    .filter((n) => n.noteId);
+}
+
+function unwrapNoteWrapper(raw: Record<string, unknown>): Record<string, unknown> {
+  const value = unwrapReactiveValue(raw) as Record<string, unknown>;
+  const noteCard = unwrapReactiveValue(
+    value.note_card ?? value.noteCard ?? value.note ?? value,
+  ) as Record<string, unknown>;
+  const inner = unwrapReactiveValue(
+    noteCard.note_card ?? noteCard.noteCard ?? noteCard,
+  ) as Record<string, unknown>;
+  if (inner === value) return value;
+  return {
+    ...inner,
+    id: inner.id ?? value.id,
+    noteId: inner.noteId ?? value.noteId,
+    note_id: inner.note_id ?? value.note_id,
+    xsecToken: inner.xsecToken ?? value.xsecToken,
+    xsec_token: inner.xsec_token ?? value.xsec_token,
+  };
+}
+
+/**
+ * Extract notes from a user_posted API response body.
+ * Supports both camelCase and snake_case shapes directly through normalizeProfileNote.
+ * Also handles nested wrappers like data.items[].note_card.
+ */
+function extractNotesFromPostedResponse(body: unknown): XhsNoteSummary[] {
+  const b = body as Record<string, unknown>;
+  const data = (b.data ?? b) as Record<string, unknown>;
+
+  // Direct data.notes path
+  const notes = data.notes as Array<Record<string, unknown>> | undefined;
+  if (notes && notes.length > 0) {
+    return normalizeNotesList(notes);
+  }
+
+  // data.items path (some API versions wrap notes in items)
+  const items = data.items as Array<Record<string, unknown>> | undefined;
+  if (items && items.length > 0) {
+    // Items may be note objects directly or wrapped in various wrappers:
+    //   { note_card: { ... } }          — snake_case (real API)
+    //   { noteCard:  { ... } }          — camelCase
+    //   { note:      { ... } }          — older shape
+    //   { note_card: { note_card: … }}  — doubly-nested snake_case
+    //   { noteCard:  { noteCard:  … }}  — doubly-nested camelCase
+    return normalizeNotesList(items);
+  }
+
+  return [];
+}
+
+/**
+ * Extract userId from a profile URL.
+ */
+function extractUserId(url: string): string {
+  const match = url.match(/\/profile\/([a-f0-9]{24})/i);
+  return match ? match[1] : "";
+}
+
+/**
+ * Create a Playwright browser context with XHS-compatible desktop viewport
+ * and user agent.
+ */
+export async function createXhsBrowserContext(
+  playwright: {
+    chromium: {
+      launch: (opts: Record<string, unknown>) => Promise<{
+        newContext: (opts: Record<string, unknown>) => Promise<{
+          newPage: () => Promise<XhsPageLike>;
+        }>;
+      }>;
+    };
+  },
+  opts: {
+    headless?: boolean;
+    userDataDir?: string;
+    cookies?: Array<{ name: string; value: string; domain: string; path?: string }>;
+  } = {},
+) {
+  const browser = await playwright.chromium.launch({
+    headless: opts.headless ?? true,
+    args: ["--disable-blink-features=AutomationControlled"],
+  });
+
+  const context = await browser.newContext({
+    viewport: { width: 1440, height: 900 },
+    userAgent:
+      "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
+    locale: "zh-CN",
+  });
+
+  if (opts.cookies && opts.cookies.length > 0) {
+    await (context as any).addCookies(opts.cookies);
+  }
+
+  return context;
+}