@zenalexa/unicli 0.220.1 → 0.221.0

package/src/adapters/xiaohongshu/download.ts

@@ -7,16 +7,56 @@
 
 import { cli, Strategy } from "../../registry.js";
 import type { IPage } from "../../types.js";
-import { loadCookies, formatCookieHeader } from "../../engine/cookies.js";
+import {
+  loadCookiesWithCDP,
+  formatCookieHeader,
+} from "../../engine/cookies.js";
+import { mkdir, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { USER_AGENT } from "../../constants.js";
 import { parseNoteId, buildNoteUrl } from "./note-helpers.js";
 
 /** Build authenticated headers for Xiaohongshu downloads. */
-function buildCookieHeader(): string {
-  const cookies = loadCookies("xiaohongshu");
+async function buildCookieHeader(): Promise<string> {
+  const cookies = await loadCookiesWithCDP(
+    "xiaohongshu",
+    "www.xiaohongshu.com",
+  );
   if (cookies) return formatCookieHeader(cookies);
   return "";
 }
 
+function extensionForMedia(type: string, url: string): string {
+  try {
+    const pathname = new URL(url).pathname;
+    const match = /\.[a-zA-Z0-9]{2,5}$/.exec(pathname);
+    if (match) return match[0].toLowerCase();
+  } catch {
+    return type === "video" ? ".mp4" : ".jpg";
+  }
+  return type === "video" ? ".mp4" : ".jpg";
+}
+
+async function downloadMedia(
+  url: string,
+  path: string,
+  cookieHeader: string,
+): Promise<void> {
+  const headers: Record<string, string> = {
+    "User-Agent": USER_AGENT,
+    Referer: "https://www.xiaohongshu.com/",
+  };
+  if (cookieHeader) headers.Cookie = cookieHeader;
+  const response = await fetch(url, { headers });
+  if (!response.ok) {
+    throw new Error(
+      `Xiaohongshu media download failed: HTTP ${response.status}`,
+    );
+  }
+  const bytes = Buffer.from(await response.arrayBuffer());
+  await writeFile(path, bytes);
+}
+
 cli({
   site: "xiaohongshu",
   name: "download",
@@ -37,11 +77,12 @@ cli({
       description: "Output directory",
     },
   ],
-  columns: ["index", "type", "url"],
+  columns: ["index", "type", "path", "url"],
   func: async (page, kwargs) => {
     const p = page as IPage;
     const rawInput = String(kwargs["note-id"]);
     const noteId = parseNoteId(rawInput);
+    const outputDir = String(kwargs.output ?? "./xiaohongshu-downloads");
 
     await p.goto(buildNoteUrl(rawInput));
     await p.wait(3);
@@ -158,13 +199,21 @@ cli({
       return [{ index: 0, type: "-", url: "No media found" }];
     }
 
-    // Cookie header available for authenticated downloads when media pipeline is wired
-    void buildCookieHeader();
+    await mkdir(outputDir, { recursive: true });
+    const cookieHeader = await buildCookieHeader();
 
-    return data.media.map((m: { type: string; url: string }, idx: number) => ({
-      index: idx + 1,
-      type: m.type,
-      url: m.url,
-    }));
+    const rows = [];
+    for (const [idx, media] of data.media.entries()) {
+      const ext = extensionForMedia(media.type, media.url);
+      const path = join(outputDir, `${noteId}-${idx + 1}${ext}`);
+      await downloadMedia(media.url, path, cookieHeader);
+      rows.push({
+        index: idx + 1,
+        type: media.type,
+        path,
+        url: media.url,
+      });
+    }
+    return rows;
   },
 });

package/src/adapters/xiaohongshu/search.ts

@@ -7,6 +7,11 @@
 
 import { cli, Strategy } from "../../registry.js";
 import type { IPage } from "../../types.js";
+import {
+  socialAuthError,
+  socialEmptyError,
+} from "../../social/browser-errors.js";
+import { assertXhsReadable } from "./browser-state.js";
 
 /**
  * Wait for search results or login wall using MutationObserver (max 5 s).
@@ -52,6 +57,7 @@ cli({
   domain: "www.xiaohongshu.com",
   strategy: Strategy.COOKIE,
   browser: true,
+  browserSession: "user",
   args: [
     {
       name: "query",
@@ -70,19 +76,19 @@ cli({
   func: async (page, kwargs) => {
     const p = page as IPage;
     const keyword = encodeURIComponent(String(kwargs.query));
+    const limit = Number(kwargs.limit) || 20;
     await p.goto(
       `https://www.xiaohongshu.com/search_result?keyword=${keyword}&source=web_search_result_notes`,
+      { settleMs: 2500 },
     );
 
     const waitResult = await p.evaluate(WAIT_FOR_CONTENT_JS);
+    await assertXhsReadable(p, "search");
 
     if (waitResult === "login_wall") {
-      throw new Error(
-        "Xiaohongshu search results are blocked behind a login wall",
-      );
+      throw socialAuthError("xiaohongshu", "search");
     }
 
-    // Scroll a couple of times to load more results
     await p.autoScroll({ maxScrolls: 2, delay: 1500 });
 
     const payload = await p.evaluate(`
@@ -121,11 +127,10 @@ cli({
       })()
     `);
 
-    const limit = Number(kwargs.limit) || 20;
     const data: Record<string, unknown>[] = Array.isArray(payload)
       ? (payload as Record<string, unknown>[])
       : [];
-    return data
+    const rows = data
       .filter((item) => item.title)
       .slice(0, limit)
       .map((item, i) => ({
@@ -133,5 +138,12 @@ cli({
         ...item,
         published_at: noteIdToDate(String(item.url ?? "")),
       }));
+    if (rows.length > 0) return rows;
+
+    throw socialEmptyError(
+      "xiaohongshu",
+      "search",
+      `Xiaohongshu search loaded no parseable notes for "${String(kwargs.query)}".`,
+    );
   },
 });

package/src/adapters/xiaohongshu/trending.ts

@@ -0,0 +1,112 @@
+/**
+ * @owner   Xiaohongshu trend extraction.
+ * @does    Reads trending searches, falling back to visible hot notes in Chrome.
+ * @needs   A logged-in Xiaohongshu browser session or valid local cookies.
+ * @feeds   Agents needing current XHS trend context without brittle API-only failure.
+ * @breaks  XHS endpoint and DOM drift can reduce rows; command returns only observed public UI data.
+ */
+
+import { cli, Strategy } from "../../registry.js";
+import type { IPage } from "../../types.js";
+import { socialEmptyError } from "../../social/browser-errors.js";
+import { assertXhsReadable, fetchXhsFeedItems } from "./browser-state.js";
+
+interface TrendRow {
+  rank: number;
+  keyword: string;
+  score: string;
+  url: string;
+}
+
+async function fetchHotList(page: IPage, limit: number): Promise<TrendRow[]> {
+  const raw = await page.evaluate(`
+    (async () => {
+      const limit = ${limit};
+      const resp = await fetch('/api/sns/v1/search/hot_list', {
+        credentials: 'include',
+        headers: { 'Content-Type': 'application/json' }
+      });
+      if (!resp.ok) return { ok: false, status: resp.status };
+      const data = await resp.json();
+      const items = data?.data?.items || data?.data || [];
+      return {
+        ok: true,
+        rows: items.slice(0, limit).map((item, i) => {
+          const keyword = item.title || item.name || item.word || '';
+          return {
+            rank: i + 1,
+            keyword,
+            score: String(item.score || item.hot_value || ''),
+            url: 'https://www.xiaohongshu.com/search_result?keyword=' + encodeURIComponent(keyword),
+          };
+        }).filter((item) => item.keyword)
+      };
+    })()
+  `);
+  const result = raw as { ok?: boolean; rows?: TrendRow[] };
+  return result.ok && Array.isArray(result.rows) ? result.rows : [];
+}
+
+async function fetchHotFeed(page: IPage, limit: number): Promise<TrendRow[]> {
+  const items = await fetchXhsFeedItems(page);
+  const rows = items
+    .map((item) => item as Record<string, unknown>)
+    .map((item) => {
+      const note = item.note_card as Record<string, unknown> | undefined;
+      const interact = note?.interact_info as
+        | Record<string, unknown>
+        | undefined;
+      return {
+        title: String(note?.display_title ?? ""),
+        likes: String(interact?.liked_count ?? ""),
+        id: String(item.id ?? ""),
+      };
+    })
+    .filter((item) => item.title && item.id)
+    .sort((a, b) => Number(b.likes || 0) - Number(a.likes || 0))
+    .slice(0, limit);
+  return rows.map((item, i) => ({
+    rank: i + 1,
+    keyword: item.title,
+    score: item.likes,
+    url: `https://www.xiaohongshu.com/explore/${item.id}`,
+  }));
+}
+
+cli({
+  site: "xiaohongshu",
+  name: "trending",
+  description: "Xiaohongshu trending searches and visible hot notes",
+  domain: "www.xiaohongshu.com",
+  strategy: Strategy.COOKIE,
+  browser: true,
+  browserSession: "user",
+  args: [
+    {
+      name: "limit",
+      type: "int",
+      default: 20,
+      description: "Number of trending topics",
+    },
+  ],
+  columns: ["rank", "keyword", "score", "url"],
+  func: async (page, kwargs) => {
+    const p = page as IPage;
+    const limit = Number(kwargs.limit) || 20;
+    await p.goto("https://www.xiaohongshu.com/explore", { settleMs: 2500 });
+    await p.wait(2);
+    await assertXhsReadable(p, "trending");
+
+    const hotList = await fetchHotList(p, limit);
+    if (hotList.length > 0) return hotList;
+
+    const hotFeed = await fetchHotFeed(p, limit);
+    if (hotFeed.length > 0) return hotFeed;
+
+    throw socialEmptyError(
+      "xiaohongshu",
+      "trending",
+      "Xiaohongshu explore loaded no parseable hot-search or hot-note rows.",
+    );
+  },
+});

package/src/adapters/yahoo/search.yaml

@@ -0,0 +1,52 @@
+# Yahoo Search — keyless HTML scrape.
+#
+# verification_status: keyless-best-effort
+#
+# Yahoo Search at https://search.yahoo.com/search returns SSR HTML
+# with a result list. No token, no API key, no JS gate on first paint.
+# Yahoo's anti-bot does throttle rapid loops; this adapter is sized for
+# interactive use, not high-volume crawling.
+site: yahoo
+name: search
+description: Search the public web via Yahoo Search (no API key needed)
+type: web-api
+domain: search.yahoo.com
+strategy: public
+lint_listing_detail: skip
+
+args:
+  query:
+    type: str
+    required: true
+    positional: true
+    description: Free-text web search query
+  limit:
+    type: int
+    default: 10
+    description: Maximum results returned (Yahoo paginates at ~10 per page)
+
+pipeline:
+  - fetch_text:
+      url: "https://search.yahoo.com/search?p=${{ args.query | url_encode }}"
+      headers:
+        User-Agent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36"
+        Accept: "text/html,application/xhtml+xml"
+
+  - extract:
+      selector: "div.algo"
+      fields:
+        title: "h3.title"
+        url: "h3.title a@href"
+        snippet: "div.compText"
+
+  - limit: ${{ args.limit }}
+
+columns: [title, url, snippet]
+
+# schema-v2 metadata
+capabilities: ["http.fetch"]
+minimum_capability: http.fetch
+trust: public
+confidentiality: public
+quarantine: false
+schema_version: v2

package/src/adapters/youtube/comments-microformat.test.ts

@@ -0,0 +1,35 @@
+import { describe, expect, it } from "vitest";
+
+import { extractYouTubeCommentRows } from "./comments.js";
+
+describe("youtube microformat comments fallback", () => {
+  it("extracts public schema comments when thread renderers are absent", () => {
+    const extracted = extractYouTubeCommentRows({
+      microformat: {
+        microformatDataRenderer: {
+          videoDetails: {
+            comments: [
+              {
+                text: "Public comment",
+                dateCreated: "2026-05-14T00:00:00Z",
+                upvoteCount: "42",
+                author: { alternateName: "@alice", name: "Alice" },
+              },
+            ],
+          },
+        },
+      },
+    });
+
+    expect(extracted.rows).toEqual([
+      expect.objectContaining({
+        id: "microformat:1",
+        author: "@alice",
+        text: "Public comment",
+        likes: 42,
+        replies: 0,
+        created: "2026-05-14T00:00:00Z",
+      }),
+    ]);
+  });
+});

package/src/adapters/youtube/comments-tree.test.ts

@@ -0,0 +1,74 @@
+import { describe, expect, it } from "vitest";
+
+import { normalizeCommentRows } from "../../social/comments.js";
+import { extractYouTubeCommentRows } from "./comments.js";
+
+describe("youtube comment tree extraction", () => {
+  it("keeps inline reply rows under their parent comment", () => {
+    const extracted = extractYouTubeCommentRows({
+      onResponseReceivedEndpoints: [
+        {
+          appendContinuationItemsAction: {
+            continuationItems: [
+              {
+                itemSectionRenderer: {
+                  contents: [
+                    {
+                      commentThreadRenderer: {
+                        comment: {
+                          commentRenderer: {
+                            commentId: "root",
+                            authorText: { simpleText: "Alice" },
+                            contentText: { runs: [{ text: "Root" }] },
+                            voteCount: { simpleText: "5" },
+                            replyCount: 1,
+                          },
+                        },
+                        replies: {
+                          commentRepliesRenderer: {
+                            contents: [
+                              {
+                                commentRenderer: {
+                                  commentId: "child",
+                                  authorText: { simpleText: "Bob" },
+                                  contentText: { runs: [{ text: "Child" }] },
+                                  voteCount: { simpleText: "2" },
+                                },
+                              },
+                            ],
+                          },
+                        },
+                      },
+                    },
+                  ],
+                },
+              },
+            ],
+          },
+        },
+      ],
+    });
+
+    const rows = normalizeCommentRows(extracted.rows, {
+      platform: "youtube",
+      contentId: "video",
+    });
+
+    expect(rows).toEqual([
+      expect.objectContaining({
+        comment_id: "root",
+        parent_id: "",
+        depth: 0,
+        path: "0001",
+        text: "Root",
+      }),
+      expect.objectContaining({
+        comment_id: "child",
+        parent_id: "root",
+        depth: 1,
+        path: "0001.0001",
+        text: "Child",
+      }),
+    ]);
+  });
+});

package/src/adapters/youtube/comments.ts

@@ -6,14 +6,28 @@
  */
 
 import { cli, Strategy } from "../../registry.js";
+import { normalizeCommentRows } from "../../social/comments.js";
 import { innertubeFetch } from "./innertube.js";
 
 interface CommentRenderer {
   commentRenderer?: {
+    commentId?: string;
     authorText?: { simpleText?: string };
     contentText?: { runs?: Array<{ text: string }> };
     voteCount?: { simpleText?: string };
     replyCount?: number;
+    publishedTimeText?: { runs?: Array<{ text: string }>; simpleText?: string };
+  };
+}
+
+interface ReplyContinuationItem {
+  commentRenderer?: CommentRenderer["commentRenderer"];
+  continuationItemRenderer?: {
+    continuationEndpoint?: {
+      continuationCommand?: {
+        token?: string;
+      };
+    };
   };
 }
 
@@ -22,6 +36,11 @@ interface CommentSection {
     contents?: Array<{
       commentThreadRenderer?: {
         comment?: CommentRenderer;
+        replies?: {
+          commentRepliesRenderer?: {
+            contents?: ReplyContinuationItem[];
+          };
+        };
       };
     }>;
   };
@@ -38,11 +57,70 @@ interface NextEndpoint {
 
 interface NextResponse {
   onResponseReceivedEndpoints?: NextEndpoint[];
+  microformat?: {
+    microformatDataRenderer?: {
+      videoDetails?: {
+        comments?: Array<{
+          text?: string;
+          dateCreated?: string;
+          upvoteCount?: string | number;
+          author?: {
+            name?: string;
+            alternateName?: string;
+          };
+        }>;
+      };
+    };
+  };
 }
 
-function extractComments(data: NextResponse): Array<Record<string, unknown>> {
+function rowFromComment(
+  c: NonNullable<CommentRenderer["commentRenderer"]>,
+  parentId = "",
+): Record<string, unknown> {
+  return {
+    id: c.commentId ?? "",
+    parent_id: parentId,
+    author: c.authorText?.simpleText ?? "",
+    text: c.contentText?.runs?.map((r) => r.text).join("") ?? "",
+    likes: c.voteCount?.simpleText ?? "0",
+    replies: c.replyCount ?? 0,
+    created:
+      c.publishedTimeText?.simpleText ??
+      c.publishedTimeText?.runs?.map((r) => r.text).join("") ??
+      "",
+  };
+}
+
+function extractReplyRows(
+  items: ReplyContinuationItem[],
+  parentId: string,
+): {
+  rows: Array<Record<string, unknown>>;
+  continuationTokens: string[];
+} {
+  const rows: Array<Record<string, unknown>> = [];
+  const continuationTokens: string[] = [];
+  for (const item of items) {
+    if (item.commentRenderer) {
+      rows.push(rowFromComment(item.commentRenderer, parentId));
+    }
+    const token =
+      item.continuationItemRenderer?.continuationEndpoint?.continuationCommand
+        ?.token;
+    if (token) continuationTokens.push(token);
+  }
+  return { rows, continuationTokens };
+}
+
+export function extractYouTubeCommentRows(data: NextResponse): {
+  rows: Array<Record<string, unknown>>;
+  replyContinuationTokens: Array<{ parentId: string; token: string }>;
+} {
   const endpoints = data.onResponseReceivedEndpoints ?? [];
-  const results: Array<Record<string, unknown>> = [];
+  const rows: Array<Record<string, unknown>> = [];
+  const replyContinuationTokens: Array<{ parentId: string; token: string }> =
+    [];
 
   for (const ep of endpoints) {
     const items =
@@ -53,19 +131,60 @@ function extractComments(data: NextResponse): Array<Record<string, unknown>> {
     for (const section of items) {
       const contents = section.itemSectionRenderer?.contents ?? [];
       for (const thread of contents) {
-        const c = thread.commentThreadRenderer?.comment?.commentRenderer;
+        const renderer = thread.commentThreadRenderer;
+        const c = renderer?.comment?.commentRenderer;
         if (!c) continue;
-        results.push({
-          author: c.authorText?.simpleText ?? "",
-          text: c.contentText?.runs?.map((r) => r.text).join("") ?? "",
-          likes: c.voteCount?.simpleText ?? "0",
-          replies: c.replyCount ?? 0,
-        });
+        const rootRow = rowFromComment(c);
+        const parentId = String(rootRow.id ?? "");
+        rows.push(rootRow);
+        const replies = renderer?.replies?.commentRepliesRenderer?.contents;
+        if (!replies || !parentId) continue;
+        const extracted = extractReplyRows(replies, parentId);
+        rows.push(...extracted.rows);
+        for (const token of extracted.continuationTokens) {
+          replyContinuationTokens.push({ parentId, token });
+        }
       }
     }
   }
 
-  return results;
+  if (rows.length === 0) {
+    const comments =
+      data.microformat?.microformatDataRenderer?.videoDetails?.comments ?? [];
+    for (const [index, comment] of comments.entries()) {
+      rows.push({
+        id: `microformat:${index + 1}`,
+        author: comment.author?.alternateName ?? comment.author?.name ?? "",
+        text: comment.text ?? "",
+        likes:
+          typeof comment.upvoteCount === "number"
+            ? comment.upvoteCount
+            : Number(comment.upvoteCount ?? 0),
+        replies: 0,
+        created: comment.dateCreated ?? "",
+      });
+    }
+  }
+
+  return { rows, replyContinuationTokens };
+}
+
+function extractContinuationReplyRows(
+  data: NextResponse,
+  parentId: string,
+): Array<Record<string, unknown>> {
+  const rows: Array<Record<string, unknown>> = [];
+  for (const ep of data.onResponseReceivedEndpoints ?? []) {
+    const items =
+      ep.reloadContinuationItemsCommand?.continuationItems ??
+      ep.appendContinuationItemsAction?.continuationItems ??
+      [];
+    for (const item of items as unknown[]) {
+      const renderer = (item as ReplyContinuationItem).commentRenderer;
+      if (renderer) rows.push(rowFromComment(renderer, parentId));
+    }
+  }
+  return rows;
 }
 
 cli({
@@ -88,14 +207,49 @@ cli({
       default: 20,
       description: "Max comments to return",
     },
+    {
+      name: "with-replies",
+      type: "bool",
+      default: false,
+      description: "Fetch available nested replies",
+    },
+  ],
+  columns: [
+    "platform",
+    "content_id",
+    "comment_id",
+    "parent_id",
+    "depth",
+    "path",
+    "author",
+    "text",
+    "likes",
+    "replies",
+    "created",
   ],
-  columns: ["author", "text", "likes", "replies"],
   async func(_page, kwargs) {
     const videoId = kwargs.videoId as string;
     const limit = (kwargs.limit as number) ?? 20;
+    const withReplies = Boolean(kwargs["with-replies"]);
 
     const data = (await innertubeFetch("next", { videoId })) as NextResponse;
+    const extracted = extractYouTubeCommentRows(data);
+    const rows = extracted.rows.slice(0, limit);
+
+    if (withReplies) {
+      for (const continuation of extracted.replyContinuationTokens) {
+        const replyData = (await innertubeFetch("next", {
+          continuation: continuation.token,
+        })) as NextResponse;
+        rows.push(
+          ...extractContinuationReplyRows(replyData, continuation.parentId),
+        );
+      }
+    }
 
-    return extractComments(data).slice(0, limit);
+    return normalizeCommentRows(rows, {
+      platform: "youtube",
+      contentId: videoId,
+    });
   },
 });