nolo-cli 0.1.41 → 0.1.43

package/integrations/xhs-reader/backends/playwrightProfileCollector.ts

@@ -1,624 +0,0 @@
-/**
- * Playwright-based XHS profile collector.
- *
- * Opens a profile URL in a desktop browser, reads window.__INITIAL_STATE__,
- * scrolls to trigger pagination via real user_posted responses,
- * and captures + normalizes all note data.
- *
- * Supports dependency injection of a Page-like object for unit testing.
- */
-
-import { normalizeProfileState, normalizeProfileNote } from "../normalize";
-import { redactXhsSensitiveValue } from "../redaction";
-import type { XhsProfile, XhsNoteSummary, XhsCollectorDiagnostic, XhsFailureCode } from "../types";
-
-// Minimal page interface for dependency injection
-export interface XhsPageLike {
-  goto(url: string, opts?: { waitUntil?: string }): Promise<void>;
-  evaluate<T>(fn: () => T): Promise<T>;
-  waitForResponse(
-    fn: (resp: { url: () => string; ok: () => boolean }) => boolean,
-    opts?: { timeout?: number },
-  ): Promise<{ json: () => Promise<unknown> }>;
-  on(event: string, handler: (...args: unknown[]) => void): void;
-  /** Current page URL. Available on real Playwright pages. */
-  url?(): string;
-  /** Page title. Available on real Playwright pages. */
-  title?(): Promise<string>;
-  close(): Promise<void>;
-}
-
-export interface CollectProfilePageResult {
-  profile: XhsProfile;
-  notes: XhsNoteSummary[];
-  diagnostic?: XhsCollectorDiagnostic;
-}
-
-interface InternalState {
-  noteMap: Map<string, XhsNoteSummary>;
-  profile: XhsProfile | null;
-}
-
-function unwrapReactiveValue(value: unknown): unknown {
-  if (!value || typeof value !== "object") return value;
-  const record = value as Record<string, unknown>;
-  if ("_value" in record || "_rawValue" in record) {
-    return record._value ?? record._rawValue;
-  }
-  return value;
-}
-
-/**
- * Detect whether the page shows a login wall or captcha.
- * Returns a diagnostic code + evidence, or null if the page looks normal.
- */
-function detectPageBlockers(
-  pageUrl: string,
-  pageTitle: string,
-): { code: XhsFailureCode; reason: string } | null {
-  const lowerUrl = pageUrl.toLowerCase();
-  const lowerTitle = pageTitle.toLowerCase();
-
-  // Login redirect detection
-  if (
-    lowerUrl.includes("/login") ||
-    lowerUrl.includes("loginpage") ||
-    lowerUrl.includes("passport.xiaohongshu.com")
-  ) {
-    return { code: "login_required", reason: "page redirected to login URL" };
-  }
-
-  // Title-based login detection
-  if (
-    lowerTitle.includes("登录") ||
-    lowerTitle.includes("login") ||
-    lowerTitle.includes("sign in")
-  ) {
-    return { code: "login_required", reason: "page title indicates login wall" };
-  }
-
-  // Captcha / verification detection
-  if (
-    lowerTitle.includes("验证") ||
-    lowerTitle.includes("captcha") ||
-    lowerTitle.includes("verify") ||
-    lowerTitle.includes("安全验证")
-  ) {
-    return { code: "blocked", reason: "page title indicates captcha/verification wall" };
-  }
-
-  return null;
-}
-
-/**
- * Probe __INITIAL_STATE__ for note-like data to count how many notes
- * are visible in the initial page load.
- */
-function countNotesInState(state: unknown): number {
-  if (!state || typeof state !== "object") return 0;
-  const s = state as Record<string, unknown>;
-  const userContainer = s.user as Record<string, unknown> | undefined;
-
-  // Real XHS shape: state.user.notes[0]
-  const userNotes = unwrapReactiveValue(userContainer?.notes) as unknown[] | undefined;
-  if (Array.isArray(userNotes) && userNotes.length > 0) {
-    const firstPage = userNotes[0];
-    if (Array.isArray(firstPage)) return firstPage.length;
-  }
-
-  // Alternate: state.user.userPageData.notes / noteList / choosedNotes
-  const userPageData = unwrapReactiveValue(userContainer?.userPageData ?? s.userPageData) as
-    | Record<string, unknown>
-    | undefined;
-  if (userPageData) {
-    const lists = ["notes", "noteList", "choosedNotes"] as const;
-    for (const key of lists) {
-      const arr = userPageData[key];
-      if (Array.isArray(arr) && arr.length > 0) return arr.length;
-    }
-  }
-
-  return 0;
-}
-
-/**
- * Collect profile and notes from a XHS profile page using Playwright.
- *
- * @param page - A Playwright Page or compatible object.
- * @param profileUrl - The full XHS profile URL.
- * @param maxScrollPages - Max number of scroll/pagination pages to collect.
- * @returns Normalized profile + notes list + optional diagnostic.
- */
-export async function collectProfilePage(
-  page: XhsPageLike,
-  profileUrl: string,
-  maxScrollPages: number = 5,
-): Promise<CollectProfilePageResult> {
-  const collectorState: InternalState = {
-    noteMap: new Map(),
-    profile: null,
-  };
-
-  const diagnostic: XhsCollectorDiagnostic = {
-    code: "unknown",
-    message: "",
-  };
-
-  // Set up response listener before navigation to capture user_posted responses
-  const capturedResponses: Array<{ url: string; status: number; json: () => Promise<unknown> }> = [];
-  const responseHandler = (resp: unknown) => {
-    const r = resp as {
-      url: () => string;
-      ok: () => boolean;
-      status: () => number;
-      json: () => Promise<unknown>;
-    };
-    if (
-      r.url().includes("/api/sns/web/v1/user_posted") &&
-      r.ok()
-    ) {
-      capturedResponses.push({
-        url: r.url(),
-        status: r.status(),
-        json: r.json.bind(r),
-      });
-    }
-  };
-
-  // Some Page implementations use page.on, others need different setup
-  try {
-    page.on("response", responseHandler);
-  } catch {
-    // If on() is not available, we rely on __INITIAL_STATE__ only
-  }
-
-  // Navigate to the profile page
-  await page.goto(profileUrl, { waitUntil: "networkidle" });
-
-  // Capture page state for diagnostics
-  const finalUrl = page.url?.() ?? profileUrl;
-  let pageTitle = "";
-  try {
-    pageTitle = page.title ? await page.title() : "";
-  } catch {
-    // title() may not be available in all test mocks
-  }
-
-  diagnostic.finalUrl = redactXhsSensitiveValue(finalUrl) as string;
-  diagnostic.pageTitle = pageTitle;
-
-  // Check for login/captcha blockers
-  const blocker = detectPageBlockers(finalUrl, pageTitle);
-  if (blocker) {
-    // Try to detect login state more precisely via page content
-    const loginHint = await page.evaluate(() => {
-      try {
-        // Check for common XHS login indicators in the DOM
-        const body = document.body?.innerText ?? "";
-        const hasLoginText = /登录|login|sign.?in|手机号|验证码/i.test(body);
-        // Check for login modal / overlay
-        const loginModal = document.querySelector('[class*="login"]') ??
-                           document.querySelector('[class*="Login"]');
-        return { hasLoginText, hasLoginModal: !!loginModal };
-      } catch {
-        return { hasLoginText: false, hasLoginModal: false };
-      }
-    }).catch(() => ({ hasLoginText: false, hasLoginModal: false }));
-
-    diagnostic.loginDetected = loginHint?.hasLoginText || loginHint?.hasLoginModal;
-    diagnostic.redirectedToLogin = finalUrl.toLowerCase().includes("login");
-
-    return {
-      profile: { userId: extractUserId(profileUrl), nickname: "" },
-      notes: [],
-      diagnostic: {
-        ...diagnostic,
-        code: blocker.code,
-        message: blocker.reason +
-          (diagnostic.loginDetected ? " (login UI detected on page)" : ""),
-      },
-    };
-  }
-
-  await waitForInitialProfileState(page, 15_000);
-
-  // Read __INITIAL_STATE__ from the page
-  const initialState = await readSerializableInitialState(page);
-
-  diagnostic.initialStatePresent = !!initialState;
-  diagnostic.initialStateNoteCount = countNotesInState(initialState);
-
-  if (initialState) {
-    const rawState = initialState as Record<string, unknown>;
-
-    // Extract profile info — try real path (state.user.userPageData) first,
-    // then flat path (state.userPageData) for backward compat with test fakes.
-    const userContainer = rawState.user as Record<string, unknown> | undefined;
-    const userPageData = unwrapReactiveValue(
-      userContainer?.userPageData ?? rawState.userPageData,
-    ) as Record<string, unknown> | undefined;
-
-    if (userPageData) {
-      const userId = extractUserId(profileUrl);
-      collectorState.profile = normalizeProfileState(
-        { userPageData: userPageData as any },
-        userId,
-      );
-    }
-
-    // Extract initial notes from __INITIAL_STATE__
-    const noteList = extractNotesFromState(initialState);
-    for (const note of noteList) {
-      collectorState.noteMap.set(note.noteId, note);
-    }
-  }
-
-  // If no profile from state, create a minimal one from URL
-  if (!collectorState.profile) {
-    const userId = extractUserId(profileUrl);
-    collectorState.profile = { userId, nickname: "" };
-  }
-
-  // Scroll to trigger pagination
-  let scrollCount = 0;
-  let consecutiveEmpty = 0;
-
-  while (scrollCount < maxScrollPages && consecutiveEmpty < 2) {
-    const countBefore = collectorState.noteMap.size;
-
-    // Scroll to bottom
-    await page.evaluate(() => {
-      window.scrollTo(0, document.body.scrollHeight);
-    });
-
-    // Wait for a user_posted response or timeout
-    try {
-      const resp = await page.waitForResponse(
-        (r) =>
-          r.url().includes("/api/sns/web/v1/user_posted") && r.ok(),
-        { timeout: 8000 },
-      );
-
-      const body = await resp.json();
-      const pageNotes = extractNotesFromPostedResponse(body);
-      for (const note of pageNotes) {
-        collectorState.noteMap.set(note.noteId, note);
-      }
-    } catch {
-      // Timeout or error - may have no more pages
-    }
-
-    scrollCount++;
-    if (collectorState.noteMap.size === countBefore) {
-      consecutiveEmpty++;
-    } else {
-      consecutiveEmpty = 0;
-    }
-  }
-
-  // Also process any captured responses we haven't handled
-  for (const resp of capturedResponses) {
-    try {
-      const body = await resp.json();
-      const pageNotes = extractNotesFromPostedResponse(body);
-      for (const note of pageNotes) {
-        collectorState.noteMap.set(note.noteId, note);
-      }
-    } catch {
-      // ignore
-    }
-  }
-
-  if (!collectorState.profile?.nickname || collectorState.noteMap.size === 0) {
-    await waitForInitialProfileState(page, 12_000);
-    const lateState = await readSerializableInitialState(page);
-    if (lateState) {
-      const rawState = lateState as Record<string, unknown>;
-      const userContainer = rawState.user as Record<string, unknown> | undefined;
-      const userPageData = unwrapReactiveValue(
-        userContainer?.userPageData ?? rawState.userPageData,
-      ) as Record<string, unknown> | undefined;
-      if (userPageData && !collectorState.profile?.nickname) {
-        const userId = extractUserId(profileUrl);
-        collectorState.profile = normalizeProfileState(
-          { userPageData: userPageData as any },
-          userId,
-        );
-      }
-      const lateNotes = extractNotesFromState(lateState);
-      for (const note of lateNotes) {
-        collectorState.noteMap.set(note.noteId, note);
-      }
-      diagnostic.initialStatePresent = true;
-      diagnostic.initialStateNoteCount = Math.max(
-        diagnostic.initialStateNoteCount ?? 0,
-        countNotesInState(lateState),
-      );
-    }
-  }
-
-  diagnostic.capturedApiResponseCount = capturedResponses.length;
-  if (capturedResponses.length > 0) {
-    diagnostic.firstApiResponseStatus = capturedResponses[0].status;
-  }
-
-  const notes = Array.from(collectorState.noteMap.values());
-
-  // Build diagnostic for empty results
-  if (notes.length === 0) {
-    // Check if the page has login indicators even without URL redirect
-    const loginCheck = await page.evaluate(() => {
-      try {
-        const body = document.body?.innerText ?? "";
-        return {
-          hasLoginText: /登录|login|sign.?in|请先登录|未登录/i.test(body),
-          hasLoginModal: !!(
-            document.querySelector('[class*="login"]') ??
-            document.querySelector('[class*="Login"]')
-          ),
-          bodyLength: body.length,
-        };
-      } catch {
-        return { hasLoginText: false, hasLoginModal: false, bodyLength: 0 };
-      }
-    }).catch(() => ({ hasLoginText: false, hasLoginModal: false, bodyLength: 0 })) as { hasLoginText: boolean; hasLoginModal: boolean; bodyLength: number };
-    const safeLoginCheck = loginCheck ?? { hasLoginText: false, hasLoginModal: false, bodyLength: 0 };
-
-    diagnostic.loginDetected = safeLoginCheck.hasLoginText || safeLoginCheck.hasLoginModal;
-
-    // Determine the appropriate diagnostic code
-    let code: XhsFailureCode = "empty_profile_state";
-    let message = "";
-
-    if (diagnostic.loginDetected) {
-      code = "login_required";
-      message =
-        "页面检测到登录提示，未获取到笔记。" +
-        "请先用 headless=false + keepOpen=true 手动登录，" +
-        "登录后再次运行即可自动复用登录态。";
-    } else if (!diagnostic.initialStatePresent) {
-      code = "empty_profile_state";
-      message =
-        "页面未注入 __INITIAL_STATE__，可能页面未完全加载或被反爬拦截。" +
-        "请尝试增加 timeoutMs 或使用本地持久 profile。";
-    } else if (
-      diagnostic.initialStatePresent &&
-      (diagnostic.initialStateNoteCount ?? 0) === 0 &&
-      diagnostic.capturedApiResponseCount === 0
-    ) {
-      code = "empty_profile_state";
-      message =
-        "__INITIAL_STATE__ 存在但未发现笔记数据，且未捕获到 user_posted API 响应。" +
-        "可能该用户无公开笔记，或页面在 headless 环境下被限流。" +
-        "建议用本地持久 profile 验证。";
-    } else {
-      code = "empty_profile_state";
-      message = "收集完成但未获取到笔记数据。";
-    }
-
-    diagnostic.code = code;
-    diagnostic.message = message;
-  }
-
-  return {
-    profile: collectorState.profile,
-    notes,
-    diagnostic: notes.length === 0 ? diagnostic : undefined,
-  };
-}
-
-async function readSerializableInitialState(page: XhsPageLike): Promise<unknown> {
-  return page.evaluate(() => {
-    const unwrap = (value: any): any =>
-      value && typeof value === "object" && ("_value" in value || "_rawValue" in value)
-        ? (value._value ?? value._rawValue)
-        : value;
-    const state = (window as any).__INITIAL_STATE__;
-    if (!state) return null;
-
-    const userPageData = unwrap(state.user?.userPageData ?? state.userPageData);
-    const notes = unwrap(state.user?.notes);
-    const firstPageNotes = Array.isArray(notes?.[0]) ? notes[0] : undefined;
-
-    return {
-      user: {
-        userPageData,
-        notes: firstPageNotes ? [firstPageNotes] : notes,
-      },
-      userPageData,
-      note: unwrap(state.note),
-      feed: {
-        feeds: unwrap(state.feed?.feeds),
-        items: unwrap(state.feed?.items),
-      },
-    };
-  }).catch(() => null);
-}
-
-async function waitForInitialProfileState(page: XhsPageLike, timeoutMs: number): Promise<"ready" | "mock_skip" | "timeout"> {
-  const started = Date.now();
-  while (Date.now() - started < timeoutMs) {
-    const ready = await page.evaluate(() => {
-      try {
-        const unwrap = (value: any): any =>
-          value && typeof value === "object" && ("_value" in value || "_rawValue" in value)
-            ? (value._value ?? value._rawValue)
-            : value;
-        const state = (window as any).__INITIAL_STATE__;
-        const userPageData = unwrap(state?.user?.userPageData ?? state?.userPageData);
-        const notes = unwrap(state?.user?.notes);
-        const firstPage = Array.isArray(notes?.[0]) ? notes[0] : undefined;
-        const hasProfile = Boolean(userPageData?.basicInfo);
-        const hasNotes = Array.isArray(firstPage) && firstPage.length > 0;
-        return hasProfile || hasNotes;
-      } catch {
-        return false;
-      }
-    }).catch(() => false);
-    if (ready == null) return "mock_skip";
-    if (ready) return "ready";
-    await new Promise((resolve) => setTimeout(resolve, 500));
-  }
-  return "timeout";
-}
-
-/**
- * Extract notes from __INITIAL_STATE__ object.
- * Tries multiple known paths:
- * - Real XHS shape: state.user.notes[0] (first page note array)
- * - Real XHS shape: state.user.userPageData.notes (alternate)
- * - Flat test shape: state.userPageData.notes / noteList / choosedNotes
- */
-function extractNotesFromState(state: unknown): XhsNoteSummary[] {
-  const s = state as Record<string, unknown>;
-  const userContainer = s.user as Record<string, unknown> | undefined;
-
-  // Real XHS shape: state.user.notes[0] is the first-page note array
-  const userNotes = unwrapReactiveValue(userContainer?.notes) as unknown[] | undefined;
-  if (Array.isArray(userNotes) && userNotes.length > 0) {
-    const firstPage = userNotes[0];
-    if (Array.isArray(firstPage)) {
-      return normalizeNotesList(firstPage as Array<Record<string, unknown>>);
-    }
-  }
-
-  // Real XHS shape: state.user.userPageData.notes
-  const userPageData = unwrapReactiveValue(userContainer?.userPageData ?? s.userPageData) as
-    | Record<string, unknown>
-    | undefined;
-  if (userPageData) {
-    const noteList = (userPageData.notes ??
-      userPageData.noteList ??
-      userPageData.choosedNotes ??
-      []) as Array<Record<string, unknown>>;
-    if (noteList.length > 0) {
-      return normalizeNotesList(noteList);
-    }
-  }
-
-  // Additional: try state.note.noteList (some XHS versions)
-  const noteContainer = unwrapReactiveValue(s.note) as Record<string, unknown> | undefined;
-  if (noteContainer) {
-    const noteList = (noteContainer.noteList ?? noteContainer.notes ?? []) as Array<Record<string, unknown>>;
-    if (Array.isArray(noteList) && noteList.length > 0) {
-      return normalizeNotesList(noteList);
-    }
-  }
-
-  // Additional: try state.feed.feeds (some XHS SPA versions)
-  const feedContainer = unwrapReactiveValue(s.feed) as Record<string, unknown> | undefined;
-  if (feedContainer) {
-    const feeds = unwrapReactiveValue(feedContainer.feeds ?? feedContainer.items);
-    if (Array.isArray(feeds) && feeds.length > 0) {
-      return normalizeNotesList(feeds as Array<Record<string, unknown>>);
-    }
-  }
-
-  return [];
-}
-
-/**
- * Shared normalization for a raw notes list (works for both INITIAL_STATE
- * and paginated user_posted responses).
- */
-function normalizeNotesList(rawNotes: Array<Record<string, unknown>>): XhsNoteSummary[] {
-  return rawNotes
-    .map((raw) => normalizeProfileNote(unwrapNoteWrapper(raw) as any))
-    .filter((n) => n.noteId);
-}
-
-function unwrapNoteWrapper(raw: Record<string, unknown>): Record<string, unknown> {
-  const value = unwrapReactiveValue(raw) as Record<string, unknown>;
-  const noteCard = unwrapReactiveValue(
-    value.note_card ?? value.noteCard ?? value.note ?? value,
-  ) as Record<string, unknown>;
-  const inner = unwrapReactiveValue(
-    noteCard.note_card ?? noteCard.noteCard ?? noteCard,
-  ) as Record<string, unknown>;
-  if (inner === value) return value;
-  return {
-    ...inner,
-    id: inner.id ?? value.id,
-    noteId: inner.noteId ?? value.noteId,
-    note_id: inner.note_id ?? value.note_id,
-    xsecToken: inner.xsecToken ?? value.xsecToken,
-    xsec_token: inner.xsec_token ?? value.xsec_token,
-  };
-}
-
-/**
- * Extract notes from a user_posted API response body.
- * Supports both camelCase and snake_case shapes directly through normalizeProfileNote.
- * Also handles nested wrappers like data.items[].note_card.
- */
-function extractNotesFromPostedResponse(body: unknown): XhsNoteSummary[] {
-  const b = body as Record<string, unknown>;
-  const data = (b.data ?? b) as Record<string, unknown>;
-
-  // Direct data.notes path
-  const notes = data.notes as Array<Record<string, unknown>> | undefined;
-  if (notes && notes.length > 0) {
-    return normalizeNotesList(notes);
-  }
-
-  // data.items path (some API versions wrap notes in items)
-  const items = data.items as Array<Record<string, unknown>> | undefined;
-  if (items && items.length > 0) {
-    // Items may be note objects directly or wrapped in various wrappers:
-    //   { note_card: { ... } }          — snake_case (real API)
-    //   { noteCard:  { ... } }          — camelCase
-    //   { note:      { ... } }          — older shape
-    //   { note_card: { note_card: … }}  — doubly-nested snake_case
-    //   { noteCard:  { noteCard:  … }}  — doubly-nested camelCase
-    return normalizeNotesList(items);
-  }
-
-  return [];
-}
-
-/**
- * Extract userId from a profile URL.
- */
-function extractUserId(url: string): string {
-  const match = url.match(/\/profile\/([a-f0-9]{24})/i);
-  return match ? match[1] : "";
-}
-
-/**
- * Create a Playwright browser context with XHS-compatible desktop viewport
- * and user agent.
- */
-export async function createXhsBrowserContext(
-  playwright: {
-    chromium: {
-      launch: (opts: Record<string, unknown>) => Promise<{
-        newContext: (opts: Record<string, unknown>) => Promise<{
-          newPage: () => Promise<XhsPageLike>;
-        }>;
-      }>;
-    };
-  },
-  opts: {
-    headless?: boolean;
-    userDataDir?: string;
-    cookies?: Array<{ name: string; value: string; domain: string; path?: string }>;
-  } = {},
-) {
-  const browser = await playwright.chromium.launch({
-    headless: opts.headless ?? true,
-    args: ["--disable-blink-features=AutomationControlled"],
-  });
-
-  const context = await browser.newContext({
-    viewport: { width: 1440, height: 900 },
-    userAgent:
-      "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
-    locale: "zh-CN",
-  });
-
-  if (opts.cookies && opts.cookies.length > 0) {
-    await (context as any).addCookies(opts.cookies);
-  }
-
-  return context;
-}