@zenalexa/unicli 0.220.0 → 0.221.0

package/src/adapters/xiaohongshu/download.ts

@@ -7,16 +7,56 @@
 
 import { cli, Strategy } from "../../registry.js";
 import type { IPage } from "../../types.js";
-import { loadCookies, formatCookieHeader } from "../../engine/cookies.js";
+import {
+  loadCookiesWithCDP,
+  formatCookieHeader,
+} from "../../engine/cookies.js";
+import { mkdir, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { USER_AGENT } from "../../constants.js";
 import { parseNoteId, buildNoteUrl } from "./note-helpers.js";
 
 /** Build authenticated headers for Xiaohongshu downloads. */
-function buildCookieHeader(): string {
-  const cookies = loadCookies("xiaohongshu");
+async function buildCookieHeader(): Promise<string> {
+  const cookies = await loadCookiesWithCDP(
+    "xiaohongshu",
+    "www.xiaohongshu.com",
+  );
   if (cookies) return formatCookieHeader(cookies);
   return "";
 }
 
+function extensionForMedia(type: string, url: string): string {
+  try {
+    const pathname = new URL(url).pathname;
+    const match = /\.[a-zA-Z0-9]{2,5}$/.exec(pathname);
+    if (match) return match[0].toLowerCase();
+  } catch {
+    return type === "video" ? ".mp4" : ".jpg";
+  }
+  return type === "video" ? ".mp4" : ".jpg";
+}
+
+async function downloadMedia(
+  url: string,
+  path: string,
+  cookieHeader: string,
+): Promise<void> {
+  const headers: Record<string, string> = {
+    "User-Agent": USER_AGENT,
+    Referer: "https://www.xiaohongshu.com/",
+  };
+  if (cookieHeader) headers.Cookie = cookieHeader;
+  const response = await fetch(url, { headers });
+  if (!response.ok) {
+    throw new Error(
+      `Xiaohongshu media download failed: HTTP ${response.status}`,
+    );
+  }
+  const bytes = Buffer.from(await response.arrayBuffer());
+  await writeFile(path, bytes);
+}
+
 cli({
   site: "xiaohongshu",
   name: "download",
@@ -37,11 +77,12 @@ cli({
       description: "Output directory",
     },
   ],
-  columns: ["index", "type", "url"],
+  columns: ["index", "type", "path", "url"],
   func: async (page, kwargs) => {
     const p = page as IPage;
     const rawInput = String(kwargs["note-id"]);
     const noteId = parseNoteId(rawInput);
+    const outputDir = String(kwargs.output ?? "./xiaohongshu-downloads");
 
     await p.goto(buildNoteUrl(rawInput));
     await p.wait(3);
@@ -158,13 +199,21 @@ cli({
       return [{ index: 0, type: "-", url: "No media found" }];
     }
 
-    // Cookie header available for authenticated downloads when media pipeline is wired
-    void buildCookieHeader();
+    await mkdir(outputDir, { recursive: true });
+    const cookieHeader = await buildCookieHeader();
 
-    return data.media.map((m: { type: string; url: string }, idx: number) => ({
-      index: idx + 1,
-      type: m.type,
-      url: m.url,
-    }));
+    const rows = [];
+    for (const [idx, media] of data.media.entries()) {
+      const ext = extensionForMedia(media.type, media.url);
+      const path = join(outputDir, `${noteId}-${idx + 1}${ext}`);
+      await downloadMedia(media.url, path, cookieHeader);
+      rows.push({
+        index: idx + 1,
+        type: media.type,
+        path,
+        url: media.url,
+      });
+    }
+    return rows;
   },
 });

package/src/adapters/xiaohongshu/search.ts

@@ -7,6 +7,11 @@
 
 import { cli, Strategy } from "../../registry.js";
 import type { IPage } from "../../types.js";
+import {
+  socialAuthError,
+  socialEmptyError,
+} from "../../social/browser-errors.js";
+import { assertXhsReadable } from "./browser-state.js";
 
 /**
  * Wait for search results or login wall using MutationObserver (max 5 s).
@@ -52,6 +57,7 @@ cli({
   domain: "www.xiaohongshu.com",
   strategy: Strategy.COOKIE,
   browser: true,
+  browserSession: "user",
   args: [
     {
       name: "query",
@@ -70,19 +76,19 @@ cli({
   func: async (page, kwargs) => {
     const p = page as IPage;
     const keyword = encodeURIComponent(String(kwargs.query));
+    const limit = Number(kwargs.limit) || 20;
     await p.goto(
       `https://www.xiaohongshu.com/search_result?keyword=${keyword}&source=web_search_result_notes`,
+      { settleMs: 2500 },
     );
 
     const waitResult = await p.evaluate(WAIT_FOR_CONTENT_JS);
+    await assertXhsReadable(p, "search");
 
     if (waitResult === "login_wall") {
-      throw new Error(
-        "Xiaohongshu search results are blocked behind a login wall",
-      );
+      throw socialAuthError("xiaohongshu", "search");
     }
 
-    // Scroll a couple of times to load more results
     await p.autoScroll({ maxScrolls: 2, delay: 1500 });
 
     const payload = await p.evaluate(`
@@ -121,11 +127,10 @@ cli({
       })()
     `);
 
-    const limit = Number(kwargs.limit) || 20;
     const data: Record<string, unknown>[] = Array.isArray(payload)
       ? (payload as Record<string, unknown>[])
       : [];
-    return data
+    const rows = data
       .filter((item) => item.title)
       .slice(0, limit)
       .map((item, i) => ({
@@ -133,5 +138,12 @@ cli({
         ...item,
         published_at: noteIdToDate(String(item.url ?? "")),
       }));
+    if (rows.length > 0) return rows;
+
+    throw socialEmptyError(
+      "xiaohongshu",
+      "search",
+      `Xiaohongshu search loaded no parseable notes for "${String(kwargs.query)}".`,
+    );
   },
 });

package/src/adapters/xiaohongshu/trending.ts

@@ -0,0 +1,112 @@
+/**
+ * @owner   Xiaohongshu trend extraction.
+ * @does    Reads trending searches, falling back to visible hot notes in Chrome.
+ * @needs   A logged-in Xiaohongshu browser session or valid local cookies.
+ * @feeds   Agents needing current XHS trend context without brittle API-only failure.
+ * @breaks  XHS endpoint and DOM drift can reduce rows; command returns only observed public UI data.
+ */
+
+import { cli, Strategy } from "../../registry.js";
+import type { IPage } from "../../types.js";
+import { socialEmptyError } from "../../social/browser-errors.js";
+import { assertXhsReadable, fetchXhsFeedItems } from "./browser-state.js";
+
+interface TrendRow {
+  rank: number;
+  keyword: string;
+  score: string;
+  url: string;
+}
+
+async function fetchHotList(page: IPage, limit: number): Promise<TrendRow[]> {
+  const raw = await page.evaluate(`
+    (async () => {
+      const limit = ${limit};
+      const resp = await fetch('/api/sns/v1/search/hot_list', {
+        credentials: 'include',
+        headers: { 'Content-Type': 'application/json' }
+      });
+      if (!resp.ok) return { ok: false, status: resp.status };
+      const data = await resp.json();
+      const items = data?.data?.items || data?.data || [];
+      return {
+        ok: true,
+        rows: items.slice(0, limit).map((item, i) => {
+          const keyword = item.title || item.name || item.word || '';
+          return {
+            rank: i + 1,
+            keyword,
+            score: String(item.score || item.hot_value || ''),
+            url: 'https://www.xiaohongshu.com/search_result?keyword=' + encodeURIComponent(keyword),
+          };
+        }).filter((item) => item.keyword)
+      };
+    })()
+  `);
+  const result = raw as { ok?: boolean; rows?: TrendRow[] };
+  return result.ok && Array.isArray(result.rows) ? result.rows : [];
+}
+
+async function fetchHotFeed(page: IPage, limit: number): Promise<TrendRow[]> {
+  const items = await fetchXhsFeedItems(page);
+  const rows = items
+    .map((item) => item as Record<string, unknown>)
+    .map((item) => {
+      const note = item.note_card as Record<string, unknown> | undefined;
+      const interact = note?.interact_info as
+        | Record<string, unknown>
+        | undefined;
+      return {
+        title: String(note?.display_title ?? ""),
+        likes: String(interact?.liked_count ?? ""),
+        id: String(item.id ?? ""),
+      };
+    })
+    .filter((item) => item.title && item.id)
+    .sort((a, b) => Number(b.likes || 0) - Number(a.likes || 0))
+    .slice(0, limit);
+  return rows.map((item, i) => ({
+    rank: i + 1,
+    keyword: item.title,
+    score: item.likes,
+    url: `https://www.xiaohongshu.com/explore/${item.id}`,
+  }));
+}
+
+cli({
+  site: "xiaohongshu",
+  name: "trending",
+  description: "Xiaohongshu trending searches and visible hot notes",
+  domain: "www.xiaohongshu.com",
+  strategy: Strategy.COOKIE,
+  browser: true,
+  browserSession: "user",
+  args: [
+    {
+      name: "limit",
+      type: "int",
+      default: 20,
+      description: "Number of trending topics",
+    },
+  ],
+  columns: ["rank", "keyword", "score", "url"],
+  func: async (page, kwargs) => {
+    const p = page as IPage;
+    const limit = Number(kwargs.limit) || 20;
+    await p.goto("https://www.xiaohongshu.com/explore", { settleMs: 2500 });
+    await p.wait(2);
+    await assertXhsReadable(p, "trending");
+
+    const hotList = await fetchHotList(p, limit);
+    if (hotList.length > 0) return hotList;
+
+    const hotFeed = await fetchHotFeed(p, limit);
+    if (hotFeed.length > 0) return hotFeed;
+
+    throw socialEmptyError(
+      "xiaohongshu",
+      "trending",
+      "Xiaohongshu explore loaded no parseable hot-search or hot-note rows.",
+    );
+  },
+});

package/src/adapters/yahoo/search.yaml

@@ -0,0 +1,52 @@
+# Yahoo Search — keyless HTML scrape.
+#
+# verification_status: keyless-best-effort
+#
+# Yahoo Search at https://search.yahoo.com/search returns SSR HTML
+# with a result list. No token, no API key, no JS gate on first paint.
+# Yahoo's anti-bot does throttle rapid loops; this adapter is sized for
+# interactive use, not high-volume crawling.
+site: yahoo
+name: search
+description: Search the public web via Yahoo Search (no API key needed)
+type: web-api
+domain: search.yahoo.com
+strategy: public
+lint_listing_detail: skip
+
+args:
+  query:
+    type: str
+    required: true
+    positional: true
+    description: Free-text web search query
+  limit:
+    type: int
+    default: 10
+    description: Maximum results returned (Yahoo paginates at ~10 per page)
+
+pipeline:
+  - fetch_text:
+      url: "https://search.yahoo.com/search?p=${{ args.query | url_encode }}"
+      headers:
+        User-Agent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36"
+        Accept: "text/html,application/xhtml+xml"
+
+  - extract:
+      selector: "div.algo"
+      fields:
+        title: "h3.title"
+        url: "h3.title a@href"
+        snippet: "div.compText"
+
+  - limit: ${{ args.limit }}
+
+columns: [title, url, snippet]
+
+# schema-v2 metadata
+capabilities: ["http.fetch"]
+minimum_capability: http.fetch
+trust: public
+confidentiality: public
+quarantine: false
+schema_version: v2

package/src/adapters/yandere/detail.yaml

@@ -0,0 +1,61 @@
+site: yandere
+name: detail
+description: Get yande.re post detail by post id
+domain: yande.re
+type: web-api
+strategy: public
+
+args:
+  id:
+    type: str
+    required: true
+    positional: true
+
+pipeline:
+  - fetch:
+      url: "https://yande.re/post.json"
+      params:
+        tags: "${{ 'id:' + args.id }}"
+        limit: 1
+
+  - map:
+      id: "${{ item.id }}"
+      rating: "${{ item.rating }}"
+      score: "${{ item.score }}"
+      author: "${{ item.author }}"
+      created_at: "${{ item.created_at }}"
+      width: "${{ item.width }}"
+      height: "${{ item.height }}"
+      file_ext: "${{ item.file_ext }}"
+      file_size: "${{ item.file_size }}"
+      tags: "${{ item.tags }}"
+      source: "${{ item.source }}"
+      preview_url: "${{ item.preview_url }}"
+      sample_url: "${{ item.sample_url }}"
+      file_url: "${{ item.file_url }}"
+      url: "${{ 'https://yande.re/post/show/' + item.id }}"
+
+columns:
+  [
+    id,
+    rating,
+    score,
+    author,
+    width,
+    height,
+    file_ext,
+    file_size,
+    tags,
+    source,
+    sample_url,
+    file_url,
+    url,
+  ]
+
+# schema-v2 metadata — injected by `unicli migrate schema-v2`
+capabilities: ["http.fetch"]
+minimum_capability: http.fetch
+trust: public
+confidentiality: public
+quarantine: false
+schema_version: v2

package/src/adapters/yandere/download.yaml

@@ -0,0 +1,56 @@
+site: yandere
+name: download
+description: Download yande.re post image by post id
+domain: yande.re
+type: web-api
+strategy: public
+
+args:
+  id:
+    type: str
+    required: true
+    positional: true
+  output:
+    type: str
+    default: "./downloads/yandere"
+    description: Output directory
+    x-unicli-kind: path
+  quality:
+    type: str
+    default: sample
+    description: sample or original
+
+columns: [id, rating, tags, url, _download]
+
+pipeline:
+  - fetch:
+      url: "https://yande.re/post.json"
+      params:
+        tags: "${{ 'id:' + args.id }}"
+        limit: 1
+
+  - assert:
+      condition: "data.length > 0"
+      message: "No yande.re post found for this id."
+
+  - download:
+      url: "${{ args.quality === 'original' ? item.file_url : item.sample_url }}"
+      dir: "${{ args.output }}"
+      filename: "${{ item.id }}.${{ (args.quality === 'original' ? item.file_url : item.sample_url) | ext | default('jpg') }}"
+      type: image
+      concurrency: 1
+
+  - map:
+      id: "${{ item.id }}"
+      rating: "${{ item.rating }}"
+      tags: "${{ item.tags }}"
+      url: "${{ 'https://yande.re/post/show/' + item.id }}"
+      _download: "${{ item._download }}"
+
+# schema-v2 metadata — injected by `unicli migrate schema-v2`
+capabilities: ["http.fetch", "http.download"]
+minimum_capability: http.fetch
+trust: public
+confidentiality: public
+quarantine: false
+schema_version: v2

package/src/adapters/yandere/search.yaml

@@ -0,0 +1,67 @@
+site: yandere
+name: search
+description: Search yande.re illustration posts by Moebooru tag query
+domain: yande.re
+type: web-api
+strategy: public
+
+args:
+  tags:
+    type: str
+    required: true
+    positional: true
+    description: Tag query, for example "hanabi rating:safe" or "order:score"
+  limit:
+    type: int
+    default: 20
+  page:
+    type: int
+    default: 1
+
+pipeline:
+  - fetch:
+      url: "https://yande.re/post.json"
+      params:
+        tags: "${{ args.tags }}"
+        limit: "${{ args.limit }}"
+        page: "${{ args.page }}"
+
+  - map:
+      rank: "${{ index + 1 }}"
+      id: "${{ item.id }}"
+      rating: "${{ item.rating }}"
+      score: "${{ item.score }}"
+      author: "${{ item.author }}"
+      width: "${{ item.width }}"
+      height: "${{ item.height }}"
+      file_ext: "${{ item.file_ext }}"
+      tags: "${{ item.tags }}"
+      source: "${{ item.source }}"
+      preview_url: "${{ item.preview_url }}"
+      sample_url: "${{ item.sample_url }}"
+      file_url: "${{ item.file_url }}"
+      url: "${{ 'https://yande.re/post/show/' + item.id }}"
+
+columns:
+  [
+    rank,
+    id,
+    rating,
+    score,
+    author,
+    width,
+    height,
+    tags,
+    source,
+    sample_url,
+    file_url,
+    url,
+  ]
+
+# schema-v2 metadata — injected by `unicli migrate schema-v2`
+capabilities: ["http.fetch"]
+minimum_capability: http.fetch
+trust: public
+confidentiality: public
+quarantine: false
+schema_version: v2

package/src/adapters/yandere/tags.yaml

@@ -0,0 +1,41 @@
+site: yandere
+name: tags
+description: Search yande.re tags by name prefix
+domain: yande.re
+type: web-api
+strategy: public
+
+args:
+  query:
+    type: str
+    required: true
+    positional: true
+  limit:
+    type: int
+    default: 20
+
+pipeline:
+  - fetch:
+      url: "https://yande.re/tag.json"
+      params:
+        name: "${{ args.query }}"
+        limit: "${{ args.limit }}"
+
+  - map:
+      rank: "${{ index + 1 }}"
+      id: "${{ item.id }}"
+      name: "${{ item.name }}"
+      count: "${{ item.count }}"
+      type: "${{ item.type }}"
+      ambiguous: "${{ item.ambiguous }}"
+      url: "${{ 'https://yande.re/post?tags=' + item.name }}"
+
+columns: [rank, id, name, count, type, ambiguous, url]
+
+# schema-v2 metadata — injected by `unicli migrate schema-v2`
+capabilities: ["http.fetch"]
+minimum_capability: http.fetch
+trust: public
+confidentiality: public
+quarantine: false
+schema_version: v2

package/src/adapters/youtube/comments-microformat.test.ts

@@ -0,0 +1,35 @@
+import { describe, expect, it } from "vitest";
+
+import { extractYouTubeCommentRows } from "./comments.js";
+
+describe("youtube microformat comments fallback", () => {
+  it("extracts public schema comments when thread renderers are absent", () => {
+    const extracted = extractYouTubeCommentRows({
+      microformat: {
+        microformatDataRenderer: {
+          videoDetails: {
+            comments: [
+              {
+                text: "Public comment",
+                dateCreated: "2026-05-14T00:00:00Z",
+                upvoteCount: "42",
+                author: { alternateName: "@alice", name: "Alice" },
+              },
+            ],
+          },
+        },
+      },
+    });
+
+    expect(extracted.rows).toEqual([
+      expect.objectContaining({
+        id: "microformat:1",
+        author: "@alice",
+        text: "Public comment",
+        likes: 42,
+        replies: 0,
+        created: "2026-05-14T00:00:00Z",
+      }),
+    ]);
+  });
+});

package/src/adapters/youtube/comments-tree.test.ts

@@ -0,0 +1,74 @@
+import { describe, expect, it } from "vitest";
+
+import { normalizeCommentRows } from "../../social/comments.js";
+import { extractYouTubeCommentRows } from "./comments.js";
+
+describe("youtube comment tree extraction", () => {
+  it("keeps inline reply rows under their parent comment", () => {
+    const extracted = extractYouTubeCommentRows({
+      onResponseReceivedEndpoints: [
+        {
+          appendContinuationItemsAction: {
+            continuationItems: [
+              {
+                itemSectionRenderer: {
+                  contents: [
+                    {
+                      commentThreadRenderer: {
+                        comment: {
+                          commentRenderer: {
+                            commentId: "root",
+                            authorText: { simpleText: "Alice" },
+                            contentText: { runs: [{ text: "Root" }] },
+                            voteCount: { simpleText: "5" },
+                            replyCount: 1,
+                          },
+                        },
+                        replies: {
+                          commentRepliesRenderer: {
+                            contents: [
+                              {
+                                commentRenderer: {
+                                  commentId: "child",
+                                  authorText: { simpleText: "Bob" },
+                                  contentText: { runs: [{ text: "Child" }] },
+                                  voteCount: { simpleText: "2" },
+                                },
+                              },
+                            ],
+                          },
+                        },
+                      },
+                    },
+                  ],
+                },
+              },
+            ],
+          },
+        },
+      ],
+    });
+
+    const rows = normalizeCommentRows(extracted.rows, {
+      platform: "youtube",
+      contentId: "video",
+    });
+
+    expect(rows).toEqual([
+      expect.objectContaining({
+        comment_id: "root",
+        parent_id: "",
+        depth: 0,
+        path: "0001",
+        text: "Root",
+      }),
+      expect.objectContaining({
+        comment_id: "child",
+        parent_id: "root",
+        depth: 1,
+        path: "0001.0001",
+        text: "Child",
+      }),
+    ]);
+  });
+});