nolo-cli 0.1.39 → 0.1.41

package/integrations/xhs-reader/bridge/readXhsProfileWithBridge.ts

@@ -0,0 +1,227 @@
+/**
+ * XHS Profile Reader Bridge
+ *
+ * Uses Playwright persistent context to:
+ * - Reuse login state across sessions (via dedicated profile directory)
+ * - Collect profile + notes via the XHS orchestrator
+ * - Extract cookies for API enrichment (detail/comment fetching)
+ *
+ * Does NOT store or output cookies/xsecToken in results.
+ */
+
+import { join, posix, win32 } from "node:path";
+import { homedir, tmpdir } from "node:os";
+import { collectXhsProfile } from "../orchestrator";
+import type { CollectXhsProfileOptions } from "../orchestrator";
+import type { XhsReadResult, XhsProfileCollection } from "../types";
+import { createXhsFailure } from "../types";
+import { redactXhsSensitiveValue } from "../redaction";
+
+export type ReadXhsProfileWithBridgeOptions = {
+  /** Canonical field name. */
+  profileUrl?: string;
+  /** Alias accepted from tool args (tool schema uses `url`). */
+  url?: string;
+  maxScrollPages?: number;
+  enrichDetails?: boolean;
+  includeComments?: boolean;
+  maxCommentPagesPerNote?: number;
+  minLikesForDetail?: number;
+  minCommentsForCollect?: number;
+  extendedCollectionConsent?: boolean;
+  profileDir?: string;
+  headless?: boolean;
+  keepOpen?: boolean;
+  timeoutMs?: number;
+  collectionMode?: "conservative" | "assisted";
+  assistedAction?: "snapshot" | "read_more_notes" | "read_comments";
+  maxAssistedSteps?: number;
+};
+
+/**
+ * Pure helper: resolve the profile URL from options, supporting both
+ * `profileUrl` (canonical) and `url` (tool schema alias).
+ * Returns undefined if neither is provided.
+ */
+export function resolveXhsBridgeProfileUrl(
+  options: Pick<ReadXhsProfileWithBridgeOptions, "profileUrl" | "url">,
+): string | undefined {
+  return options.profileUrl ?? options.url;
+}
+
+const DESKTOP_APP_ID = "chat.nolo.desktop";
+
+export function resolveDefaultXhsProfileDir({
+  env = process.env,
+  platform = process.platform,
+  homeDir = homedir(),
+  tmpDir = tmpdir(),
+}: {
+  env?: NodeJS.ProcessEnv;
+  platform?: NodeJS.Platform;
+  homeDir?: string;
+  tmpDir?: string;
+} = {}): string {
+  const configured = env.NOLO_XHS_READER_PROFILE_DIR?.trim();
+  if (configured) return configured;
+
+  if (env.NOLO_DESKTOP === "1") {
+    const pathApi = platform === "win32" ? win32 : posix;
+    const dataRoot =
+      platform === "win32"
+        ? env.LOCALAPPDATA?.trim() || pathApi.join(homeDir, "AppData", "Local")
+        : platform === "darwin"
+          ? pathApi.join(homeDir, "Library", "Application Support")
+          : env.XDG_DATA_HOME?.trim() || pathApi.join(homeDir, ".local", "share");
+    const channel = env.NOLO_DESKTOP_CHANNEL?.trim() || "stable";
+    return pathApi.join(dataRoot, DESKTOP_APP_ID, channel, "external-readers", "xhs-profile");
+  }
+
+  return join(tmpDir, "nolo-xhs-reader-profile");
+}
+
+function envFlagEnabled(value: string | undefined): boolean | undefined {
+  if (value == null) return undefined;
+  return !["0", "false", "no", "off"].includes(value.trim().toLowerCase());
+}
+
+/**
+ * Read an XHS profile using Playwright persistent context.
+ *
+ * The profile directory stores browser state including XHS login cookies.
+ * First run: user must log in manually (headless=false, keepOpen=true).
+ * Subsequent runs: login state is reused automatically.
+ */
+export async function readXhsProfileWithBridge(
+  options: ReadXhsProfileWithBridgeOptions,
+): Promise<XhsReadResult<XhsProfileCollection>> {
+  const {
+    profileUrl: profileUrlRaw,
+    url: urlAlias,
+    maxScrollPages = 0,
+    enrichDetails = false,
+    includeComments = false,
+    maxCommentPagesPerNote = 1,
+    minLikesForDetail,
+    minCommentsForCollect,
+    extendedCollectionConsent = false,
+    headless = envFlagEnabled(process.env.NOLO_XHS_READER_HEADLESS) ?? true,
+    keepOpen = false,
+    profileDir = resolveDefaultXhsProfileDir(),
+    timeoutMs = 120_000,
+  } = options;
+
+  // Support both `profileUrl` (canonical) and `url` (tool schema alias)
+  const profileUrl = resolveXhsBridgeProfileUrl({ profileUrl: profileUrlRaw, url: urlAlias });
+  if (!profileUrl) {
+    return createXhsFailure({
+      code: "parse_error",
+      message: "readXhsProfileWithBridge requires a profile URL (profileUrl or url).",
+    });
+  }
+
+  let playwright: any;
+  try {
+    playwright = await import("playwright");
+  } catch {
+    return createXhsFailure({
+      code: "unknown",
+      message:
+        "Playwright is not installed. Run `bun add playwright` or `npx playwright install chromium` first.",
+    });
+  }
+
+  // Wrap the entire Playwright + orchestrator operation in a timeout
+  // so that browser launch time is also bounded.
+  const timeoutPromise = new Promise<XhsReadResult<XhsProfileCollection>>(
+    (_, reject) => {
+      const timer = setTimeout(() => {
+        reject(
+          Object.assign(
+            new Error(`read_xhs_profile timed out after ${timeoutMs}ms`),
+            { code: "READ_XHS_PROFILE_TIMEOUT" },
+          ),
+        );
+      }, timeoutMs);
+      if (typeof timer === "object" && "unref" in timer) timer.unref();
+    },
+  );
+
+  const workPromise = (async () => {
+    let context: any = null;
+    try {
+      // Use persistent context for login state reuse
+      context = await playwright.chromium.launchPersistentContext(profileDir, {
+        headless,
+        args: [
+          "--disable-dev-shm-usage",
+        ],
+        viewport: { width: 1440, height: 900 },
+        locale: "zh-CN",
+      });
+
+      const page = await context.newPage();
+
+      // Extract cookies for API enrichment (not exposed in output)
+      let cookie: string | undefined;
+      if (enrichDetails || includeComments) {
+        try {
+          const cookies = await context.cookies("https://www.xiaohongshu.com");
+          if (cookies.length > 0) {
+            cookie = cookies
+              .map((c: { name: string; value: string }) => `${c.name}=${c.value}`)
+              .join("; ");
+          }
+        } catch {
+          // Cookie extraction failed - proceed without API enrichment
+        }
+      }
+
+      const orchestratorOptions: CollectXhsProfileOptions = {
+        profileUrl,
+        maxScrollPages: extendedCollectionConsent ? maxScrollPages : 0,
+        enrichDetails: extendedCollectionConsent && enrichDetails && !!cookie,
+        includeComments: extendedCollectionConsent && includeComments && !!cookie,
+        maxCommentPagesPerNote: extendedCollectionConsent
+          ? maxCommentPagesPerNote
+          : 1,
+        cookie,
+        page,
+        minLikesForDetail: extendedCollectionConsent
+          ? minLikesForDetail
+          : undefined,
+        minCommentsForCollect: extendedCollectionConsent
+          ? minCommentsForCollect
+          : undefined,
+      };
+
+      const result = await collectXhsProfile(orchestratorOptions);
+
+      // Final redaction pass (cookies should already be stripped by orchestrator)
+      return redactXhsSensitiveValue(result) as XhsReadResult<XhsProfileCollection>;
+    } finally {
+      if (!keepOpen && context) {
+        try {
+          await context.close();
+        } catch {
+          // ignore close errors
+        }
+      }
+    }
+  })();
+
+  try {
+    return await Promise.race([workPromise, timeoutPromise]);
+  } catch (err: any) {
+    if (err?.code === "READ_XHS_PROFILE_TIMEOUT") {
+      return createXhsFailure({
+        code: "network_error",
+        message: err.message,
+      });
+    }
+    return createXhsFailure({
+      code: "network_error",
+      message: `XHS profile read failed: ${String(err?.message ?? err)}`,
+    });
+  }
+}

package/integrations/xhs-reader/client/xhsWebApiClient.ts

@@ -0,0 +1,229 @@
+/**
+ * XHS Web API client for note details and top-level comments.
+ *
+ * Uses the existing desktop browser cookies to make authenticated requests.
+ * Supports dependency injection of fetch for testing.
+ *
+ * Endpoints:
+ * - Note detail: POST /api/sns/web/v1/feed
+ * - Comments: GET /api/sns/web/v2/comment/page
+ *
+ * IMPORTANT: This client does NOT implement write endpoints.
+ * No like, collect, comment, post, delete, or upload actions.
+ */
+
+import { redactXhsSensitiveValue } from "../redaction";
+import {
+  normalizeNoteDetail,
+  normalizeCommentPage,
+} from "../normalize";
+import type {
+  XhsNoteDetail,
+  XhsCommentPage,
+  XhsReadResult,
+  XhsFailureCode,
+} from "../types";
+import { createXhsFailure, createXhsSuccess } from "../types";
+
+const XHS_BASE = "https://edith.xiaohongshu.com";
+
+export interface XhsApiClientOptions {
+  /** Cookie string to authenticate requests. Not stored in results. */
+  cookie: string;
+  /** Optional xsecToken for the session. Not stored in results. */
+  xsecToken?: string;
+  /** Custom fetch implementation for testing. */
+  fetchImpl?: typeof fetch;
+  /** Base URL override (for testing). */
+  baseUrl?: string;
+}
+
+/**
+ * Create an XHS Web API client instance.
+ */
+export function createXhsApiClient(opts: XhsApiClientOptions) {
+  const baseUrl = opts.baseUrl ?? XHS_BASE;
+  const fetchFn = opts.fetchImpl ?? fetch;
+
+  const commonHeaders: Record<string, string> = {
+    cookie: opts.cookie,
+    "user-agent":
+      "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
+    origin: "https://www.xiaohongshu.com",
+    referer: "https://www.xiaohongshu.com/",
+    "content-type": "application/json",
+  };
+
+  if (opts.xsecToken) {
+    commonHeaders["x-xsec-token"] = opts.xsecToken;
+  }
+
+  /**
+   * Fetch note detail by noteId.
+   * Requires per-note xsecToken for the signed request to succeed.
+   */
+  async function getNoteDetail(
+    noteId: string,
+    xsecToken?: string,
+    xsecSource?: string,
+  ): Promise<XhsReadResult<XhsNoteDetail>> {
+    try {
+      const url = `${baseUrl}/api/sns/web/v1/feed`;
+      const resp = await fetchFn(url, {
+        method: "POST",
+        headers: commonHeaders,
+        body: JSON.stringify({
+          source_note_id: noteId,
+          image_formats: ["jpg", "webp", "avif"],
+          extra: { need_body_topic: 1 },
+          ...(xsecToken ? { xsec_token: xsecToken } : {}),
+          ...(xsecSource ? { xsec_source: xsecSource } : {}),
+        }),
+      });
+
+      if (!resp.ok) {
+        const code = mapHttpStatus(resp.status);
+        return createXhsFailure({
+          code,
+          message: `Note detail request failed: HTTP ${resp.status}`,
+        });
+      }
+
+      const body = (await resp.json()) as Record<string, unknown>;
+
+      if (body.code === -1 || body.success === false) {
+        return createXhsFailure({
+          code: "unknown",
+          message: `XHS API error: ${JSON.stringify(
+            redactXhsSensitiveValue(body),
+          )}`,
+        });
+      }
+
+      // The items array contains the note data
+      const items = (body.data as Record<string, unknown>)?.items as
+        | Array<Record<string, unknown>>
+        | undefined;
+      if (!items || items.length === 0) {
+        return createXhsFailure({
+          code: "note_not_found",
+          message: "No items returned from feed API",
+        });
+      }
+
+      const noteData = items[0];
+      const normalized = normalizeNoteDetail(
+        noteData as Record<string, unknown>,
+      );
+
+      return createXhsSuccess(normalized);
+    } catch (err) {
+      return createXhsFailure({
+        code: "network_error",
+        message: `Note detail fetch error: ${String(err)}`,
+      });
+    }
+  }
+
+  /**
+   * Fetch a page of top-level comments for a note.
+   * Requires per-note xsecToken for authentication.
+   */
+  async function getCommentPage(
+    noteId: string,
+    cursor?: string,
+    xsecToken?: string,
+    topCommentId?: string,
+  ): Promise<XhsReadResult<XhsCommentPage>> {
+    try {
+      const params = new URLSearchParams({
+        note_id: noteId,
+        cursor: cursor ?? "",
+        top_comment_id: topCommentId ?? "",
+        image_formats: "jpg,webp,avif",
+        ...(xsecToken ? { xsec_token: xsecToken } : {}),
+      });
+      const url = `${baseUrl}/api/sns/web/v2/comment/page?${params}`;
+
+      const resp = await fetchFn(url, {
+        method: "GET",
+        headers: commonHeaders,
+      });
+
+      if (!resp.ok) {
+        const code = mapHttpStatus(resp.status);
+        return createXhsFailure({
+          code,
+          message: `Comment page request failed: HTTP ${resp.status}`,
+        });
+      }
+
+      const body = (await resp.json()) as Record<string, unknown>;
+
+      if (body.code === -1 || body.success === false) {
+        return createXhsFailure({
+          code: "unknown",
+          message: `XHS comment API error: ${JSON.stringify(
+            redactXhsSensitiveValue(body),
+          )}`,
+        });
+      }
+
+      const normalized = normalizeCommentPage(body);
+
+      return createXhsSuccess(normalized);
+    } catch (err) {
+      return createXhsFailure({
+        code: "network_error",
+        message: `Comment page fetch error: ${String(err)}`,
+      });
+    }
+  }
+
+  /**
+   * Fetch all top-level comments for a note up to maxPages.
+   */
+  async function getAllCommentPages(
+    noteId: string,
+    xsecToken?: string,
+    maxPages: number = 3,
+  ): Promise<XhsReadResult<XhsCommentPage>> {
+    const allComments: XhsCommentPage["comments"] = [];
+    let cursor: string | undefined;
+    let hasMore = true;
+    let pages = 0;
+
+    while (hasMore && pages < maxPages) {
+      const result = await getCommentPage(noteId, cursor, xsecToken);
+      if (!result.ok) {
+        return result;
+      }
+
+      allComments.push(...result.data.comments);
+      hasMore = result.data.hasMore;
+      cursor = result.data.cursor;
+      pages++;
+    }
+
+    return createXhsSuccess({
+      comments: allComments,
+      hasMore,
+      cursor,
+    });
+  }
+
+  return {
+    getNoteDetail,
+    getCommentPage,
+    getAllCommentPages,
+  };
+}
+
+export type XhsApiClient = ReturnType<typeof createXhsApiClient>;
+
+function mapHttpStatus(status: number): XhsFailureCode {
+  if (status === 401 || status === 403) return "not_logged_in";
+  if (status === 404) return "note_not_found";
+  if (status === 429) return "rate_limited";
+  return "network_error";
+}