@zenalexa/unicli 0.220.0 → 0.221.0

package/src/adapters/youtube/comments.ts

@@ -6,14 +6,28 @@
  */
 
 import { cli, Strategy } from "../../registry.js";
+import { normalizeCommentRows } from "../../social/comments.js";
 import { innertubeFetch } from "./innertube.js";
 
 interface CommentRenderer {
   commentRenderer?: {
+    commentId?: string;
     authorText?: { simpleText?: string };
     contentText?: { runs?: Array<{ text: string }> };
     voteCount?: { simpleText?: string };
     replyCount?: number;
+    publishedTimeText?: { runs?: Array<{ text: string }>; simpleText?: string };
+  };
+}
+
+interface ReplyContinuationItem {
+  commentRenderer?: CommentRenderer["commentRenderer"];
+  continuationItemRenderer?: {
+    continuationEndpoint?: {
+      continuationCommand?: {
+        token?: string;
+      };
+    };
   };
 }
 
@@ -22,6 +36,11 @@ interface CommentSection {
     contents?: Array<{
       commentThreadRenderer?: {
         comment?: CommentRenderer;
+        replies?: {
+          commentRepliesRenderer?: {
+            contents?: ReplyContinuationItem[];
+          };
+        };
       };
     }>;
   };
@@ -38,11 +57,70 @@ interface NextEndpoint {
 
 interface NextResponse {
   onResponseReceivedEndpoints?: NextEndpoint[];
+  microformat?: {
+    microformatDataRenderer?: {
+      videoDetails?: {
+        comments?: Array<{
+          text?: string;
+          dateCreated?: string;
+          upvoteCount?: string | number;
+          author?: {
+            name?: string;
+            alternateName?: string;
+          };
+        }>;
+      };
+    };
+  };
 }
 
-function extractComments(data: NextResponse): Array<Record<string, unknown>> {
+function rowFromComment(
+  c: NonNullable<CommentRenderer["commentRenderer"]>,
+  parentId = "",
+): Record<string, unknown> {
+  return {
+    id: c.commentId ?? "",
+    parent_id: parentId,
+    author: c.authorText?.simpleText ?? "",
+    text: c.contentText?.runs?.map((r) => r.text).join("") ?? "",
+    likes: c.voteCount?.simpleText ?? "0",
+    replies: c.replyCount ?? 0,
+    created:
+      c.publishedTimeText?.simpleText ??
+      c.publishedTimeText?.runs?.map((r) => r.text).join("") ??
+      "",
+  };
+}
+
+function extractReplyRows(
+  items: ReplyContinuationItem[],
+  parentId: string,
+): {
+  rows: Array<Record<string, unknown>>;
+  continuationTokens: string[];
+} {
+  const rows: Array<Record<string, unknown>> = [];
+  const continuationTokens: string[] = [];
+  for (const item of items) {
+    if (item.commentRenderer) {
+      rows.push(rowFromComment(item.commentRenderer, parentId));
+    }
+    const token =
+      item.continuationItemRenderer?.continuationEndpoint?.continuationCommand
+        ?.token;
+    if (token) continuationTokens.push(token);
+  }
+  return { rows, continuationTokens };
+}
+
+export function extractYouTubeCommentRows(data: NextResponse): {
+  rows: Array<Record<string, unknown>>;
+  replyContinuationTokens: Array<{ parentId: string; token: string }>;
+} {
   const endpoints = data.onResponseReceivedEndpoints ?? [];
-  const results: Array<Record<string, unknown>> = [];
+  const rows: Array<Record<string, unknown>> = [];
+  const replyContinuationTokens: Array<{ parentId: string; token: string }> =
+    [];
 
   for (const ep of endpoints) {
     const items =
@@ -53,19 +131,60 @@ function extractComments(data: NextResponse): Array<Record<string, unknown>> {
     for (const section of items) {
       const contents = section.itemSectionRenderer?.contents ?? [];
       for (const thread of contents) {
-        const c = thread.commentThreadRenderer?.comment?.commentRenderer;
+        const renderer = thread.commentThreadRenderer;
+        const c = renderer?.comment?.commentRenderer;
         if (!c) continue;
-        results.push({
-          author: c.authorText?.simpleText ?? "",
-          text: c.contentText?.runs?.map((r) => r.text).join("") ?? "",
-          likes: c.voteCount?.simpleText ?? "0",
-          replies: c.replyCount ?? 0,
-        });
+        const rootRow = rowFromComment(c);
+        const parentId = String(rootRow.id ?? "");
+        rows.push(rootRow);
+        const replies = renderer?.replies?.commentRepliesRenderer?.contents;
+        if (!replies || !parentId) continue;
+        const extracted = extractReplyRows(replies, parentId);
+        rows.push(...extracted.rows);
+        for (const token of extracted.continuationTokens) {
+          replyContinuationTokens.push({ parentId, token });
+        }
       }
     }
   }
 
-  return results;
+  if (rows.length === 0) {
+    const comments =
+      data.microformat?.microformatDataRenderer?.videoDetails?.comments ?? [];
+    for (const [index, comment] of comments.entries()) {
+      rows.push({
+        id: `microformat:${index + 1}`,
+        author: comment.author?.alternateName ?? comment.author?.name ?? "",
+        text: comment.text ?? "",
+        likes:
+          typeof comment.upvoteCount === "number"
+            ? comment.upvoteCount
+            : Number(comment.upvoteCount ?? 0),
+        replies: 0,
+        created: comment.dateCreated ?? "",
+      });
+    }
+  }
+
+  return { rows, replyContinuationTokens };
+}
+
+function extractContinuationReplyRows(
+  data: NextResponse,
+  parentId: string,
+): Array<Record<string, unknown>> {
+  const rows: Array<Record<string, unknown>> = [];
+  for (const ep of data.onResponseReceivedEndpoints ?? []) {
+    const items =
+      ep.reloadContinuationItemsCommand?.continuationItems ??
+      ep.appendContinuationItemsAction?.continuationItems ??
+      [];
+    for (const item of items as unknown[]) {
+      const renderer = (item as ReplyContinuationItem).commentRenderer;
+      if (renderer) rows.push(rowFromComment(renderer, parentId));
+    }
+  }
+  return rows;
 }
 
 cli({
@@ -88,14 +207,49 @@ cli({
       default: 20,
       description: "Max comments to return",
     },
+    {
+      name: "with-replies",
+      type: "bool",
+      default: false,
+      description: "Fetch available nested replies",
+    },
+  ],
+  columns: [
+    "platform",
+    "content_id",
+    "comment_id",
+    "parent_id",
+    "depth",
+    "path",
+    "author",
+    "text",
+    "likes",
+    "replies",
+    "created",
   ],
-  columns: ["author", "text", "likes", "replies"],
   async func(_page, kwargs) {
     const videoId = kwargs.videoId as string;
     const limit = (kwargs.limit as number) ?? 20;
+    const withReplies = Boolean(kwargs["with-replies"]);
 
     const data = (await innertubeFetch("next", { videoId })) as NextResponse;
+    const extracted = extractYouTubeCommentRows(data);
+    const rows = extracted.rows.slice(0, limit);
+
+    if (withReplies) {
+      for (const continuation of extracted.replyContinuationTokens) {
+        const replyData = (await innertubeFetch("next", {
+          continuation: continuation.token,
+        })) as NextResponse;
+        rows.push(
+          ...extractContinuationReplyRows(replyData, continuation.parentId),
+        );
+      }
+    }
 
-    return extractComments(data).slice(0, limit);
+    return normalizeCommentRows(rows, {
+      platform: "youtube",
+      contentId: videoId,
+    });
   },
 });

package/src/adapters/youtube/subtitles.ts

@@ -0,0 +1,44 @@
+/**
+ * @owner   YouTube subtitle adapter.
+ * @does    Extracts LLM-readable subtitle text from public YouTube video URLs.
+ * @needs   `yt-dlp` on PATH and an accessible video subtitle track.
+ * @feeds   Search-to-transcript social video analysis workflows.
+ * @breaks  Upstream yt-dlp extractor changes can alter subtitle availability.
+ */
+
+import { cli, Strategy } from "../../registry.js";
+import { runVideoSubtitleExtraction } from "../../social/video-text.js";
+
+cli({
+  site: "youtube",
+  name: "subtitles",
+  description:
+    "Extract LLM-readable subtitles from a YouTube video URL with optional browser-cookie reuse",
+  domain: "www.youtube.com",
+  strategy: Strategy.PUBLIC,
+  args: [
+    {
+      name: "url",
+      type: "str",
+      required: true,
+      positional: true,
+      description: "YouTube video URL",
+      format: "uri",
+    },
+    {
+      name: "languages",
+      type: "str",
+      default: "zh-Hans,zh,en",
+      description: "Comma-separated subtitle languages",
+    },
+    {
+      name: "cookies-from-browser",
+      type: "str",
+      description: "Browser name for yt-dlp --cookies-from-browser",
+    },
+  ],
+  columns: ["language", "text", "path"],
+  socialCapabilities: ["read", "media", "subtitles"],
+  defaultFormat: "json",
+  func: async (_page, kwargs) => runVideoSubtitleExtraction(kwargs),
+});

package/src/adapters/yt-dlp/subtitles.ts

@@ -0,0 +1,43 @@
+/**
+ * @owner   yt-dlp subtitle adapter.
+ * @does    Extracts subtitle files from any yt-dlp-supported video URL.
+ * @needs   Local yt-dlp binary and optional browser cookies for private or rate-limited videos.
+ * @feeds   Unified social video-text workflows across YouTube, Bilibili, TikTok, Douyin, X, Instagram, and other sites.
+ * @breaks  Returns an error when yt-dlp cannot produce subtitle files for the requested URL and languages.
+ */
+
+import { cli, Strategy } from "../../registry.js";
+import { runVideoSubtitleExtraction } from "../../social/video-text.js";
+
+cli({
+  site: "yt-dlp",
+  name: "subtitles",
+  description:
+    "Extract subtitles from any yt-dlp-supported video URL with optional browser-cookie reuse",
+  strategy: Strategy.PUBLIC,
+  args: [
+    {
+      name: "url",
+      type: "str",
+      required: true,
+      positional: true,
+      description: "Video URL",
+      format: "uri",
+    },
+    {
+      name: "languages",
+      type: "str",
+      default: "zh-Hans,zh,en",
+      description: "Comma-separated subtitle languages",
+    },
+    {
+      name: "cookies-from-browser",
+      type: "str",
+      description: "Browser name for yt-dlp --cookies-from-browser",
+    },
+  ],
+  columns: ["language", "text", "path"],
+  socialCapabilities: ["read", "media", "subtitles"],
+  defaultFormat: "json",
+  func: async (_page, kwargs) => runVideoSubtitleExtraction(kwargs),
+});

package/src/adapters/zhihu/answer-detail.test.ts

@@ -0,0 +1,83 @@
+import { describe, expect, it } from "vitest";
+import {
+  extractQuestionIdFromZhihuUrl,
+  mapZhihuAnswerDetail,
+  parseMaxContent,
+  parseZhihuAnswerTarget,
+  stripZhihuAnswerHtml,
+} from "./answer-detail.js";
+
+describe("zhihu answer-detail helpers", () => {
+  it("parses numeric, URL, and typed answer targets", () => {
+    expect(parseZhihuAnswerTarget("1937205528846655537")).toEqual({
+      answerId: "1937205528846655537",
+      questionId: "",
+    });
+    expect(
+      parseZhihuAnswerTarget(
+        "https://www.zhihu.com/question/123456/answer/789012",
+      ),
+    ).toEqual({ questionId: "123456", answerId: "789012" });
+    expect(parseZhihuAnswerTarget("answer:123456:789012")).toEqual({
+      questionId: "123456",
+      answerId: "789012",
+    });
+  });
+
+  it("rejects non-Zhihu answer targets", () => {
+    expect(() =>
+      parseZhihuAnswerTarget("https://example.com/question/1/answer/2"),
+    ).toThrow("Zhihu answer target");
+    expect(() => parseZhihuAnswerTarget("question:1")).toThrow(
+      "Zhihu answer target",
+    );
+  });
+
+  it("strips answer HTML without collapsing paragraphs", () => {
+    expect(
+      stripZhihuAnswerHtml(
+        "<p>Hello&nbsp;&amp;&nbsp;world</p><blockquote>A<br>B</blockquote>",
+      ),
+    ).toBe("Hello & world\n\nA\nB");
+  });
+
+  it("maps full answer payloads with explicit truncation only", () => {
+    const row = mapZhihuAnswerDetail(
+      {
+        content: "<p>abcdef</p>",
+        voteup_count: 7,
+        comment_count: 2,
+        created_time: 1,
+        updated_time: 2,
+        author: { name: "Ada" },
+        question: { id: 123, title: "Question" },
+      },
+      { answerId: "456", questionId: "" },
+      "",
+      parseMaxContent(3),
+    );
+    expect(row).toMatchObject({
+      id: "456",
+      author: "Ada",
+      votes: 7,
+      comments: 2,
+      question_id: "123",
+      question_title: "Question",
+      content: "abc",
+    });
+  });
+
+  it("extracts question ids from supported Zhihu URLs", () => {
+    expect(
+      extractQuestionIdFromZhihuUrl(
+        "https://www.zhihu.com/question/123/answer/456",
+      ),
+    ).toBe("123");
+    expect(
+      extractQuestionIdFromZhihuUrl("https://zhihu.com/question/123"),
+    ).toBe("123");
+    expect(
+      extractQuestionIdFromZhihuUrl("https://example.com/question/123"),
+    ).toBe("");
+  });
+});

package/src/adapters/zhihu/answer-detail.ts

@@ -0,0 +1,275 @@
+/**
+ * @owner   src/adapters/zhihu/answer-detail.ts
+ * @does    Register full-content Zhihu answer detail reader.
+ * @needs   Logged-in Zhihu browser session, answer API payload, exact answer target parsing.
+ * @feeds   surface coverage ledger, long-form answer extraction workflows, reference parity checks.
+ * @breaks  Zhihu answer URL shape or API content field drift can block answer detail extraction.
+ */
+
+import { cli, Strategy } from "../../registry.js";
+import type { IPage } from "../../types.js";
+
+const ANSWER_ID_RE = /^\d+$/;
+const ANSWER_TYPED_RE = /^answer:(\d+):(\d+)$/;
+const ANSWER_PATH_RE = /^\/question\/(\d+)\/answer\/(\d+)\/?$/;
+const BARE_ANSWER_PATH_RE = /^\/answer\/(\d+)\/?$/;
+const QUESTION_PATH_RE = /^\/question\/(\d+)\/?$/;
+const QUESTION_API_PATH_RE = /^\/api\/v4\/questions\/(\d+)\/?$/;
+
+interface ZhihuAnswerTarget {
+  answerId: string;
+  questionId: string;
+}
+
+interface ZhihuAnswerPayload {
+  id?: unknown;
+  content?: unknown;
+  voteup_count?: unknown;
+  comment_count?: unknown;
+  created_time?: unknown;
+  updated_time?: unknown;
+  author?: { name?: unknown };
+  question?: { id?: unknown; title?: unknown; url?: unknown };
+  error?: { message?: unknown };
+  error_msg?: unknown;
+  message?: unknown;
+}
+
+interface BrowserFetchResult {
+  ok: boolean;
+  status: number;
+  text: string;
+}
+
+function stringField(value: unknown): string {
+  return value === undefined || value === null ? "" : String(value);
+}
+
+function countField(value: unknown): number {
+  const n = Number(value);
+  return Number.isInteger(n) && n >= 0 ? n : 0;
+}
+
+function isoFromUnixSeconds(value: unknown): string {
+  const n = Number(value);
+  return Number.isFinite(n) && n > 0 ? new Date(n * 1000).toISOString() : "";
+}
+
+function pageOf(page: unknown): IPage {
+  if (!page) throw new Error("Zhihu answer-detail requires a browser page.");
+  return page as IPage;
+}
+
+export function stripZhihuAnswerHtml(value: unknown): string {
+  return stringField(value)
+    .replace(/<br\s*\/?\s*>/gi, "\n")
+    .replace(/<\/(?:p|div|h[1-6]|li|blockquote)>/gi, "\n\n")
+    .replace(/<[^>]+>/g, "")
+    .replace(/&nbsp;/g, " ")
+    .replace(/&lt;/g, "<")
+    .replace(/&gt;/g, ">")
+    .replace(/&amp;/g, "&")
+    .replace(/&quot;/g, '"')
+    .replace(/&#39;/g, "'")
+    .replace(/\n{3,}/g, "\n\n")
+    .trim();
+}
+
+export function parseZhihuAnswerTarget(input: unknown): ZhihuAnswerTarget {
+  const value = stringField(input).trim();
+  if (!value) {
+    throw new Error(
+      "Zhihu answer target must be a numeric id, answer URL, or answer:<questionId>:<answerId>.",
+    );
+  }
+  if (ANSWER_ID_RE.test(value)) return { answerId: value, questionId: "" };
+  const typed = value.match(ANSWER_TYPED_RE);
+  if (typed) return { questionId: typed[1], answerId: typed[2] };
+  try {
+    const url = new URL(value);
+    if (
+      url.protocol !== "https:" ||
+      url.username ||
+      url.password ||
+      url.port ||
+      (url.hostname !== "www.zhihu.com" && url.hostname !== "zhihu.com")
+    ) {
+      throw new Error("invalid host");
+    }
+    let match = url.pathname.match(ANSWER_PATH_RE);
+    if (match) return { questionId: match[1], answerId: match[2] };
+    match = url.pathname.match(BARE_ANSWER_PATH_RE);
+    if (match) return { questionId: "", answerId: match[1] };
+  } catch {
+    throw new Error(
+      "Zhihu answer target must be a numeric id, answer URL, or answer:<questionId>:<answerId>.",
+    );
+  }
+  throw new Error(
+    "Zhihu answer target must be a numeric id, answer URL, or answer:<questionId>:<answerId>.",
+  );
+}
+
+export function extractQuestionIdFromZhihuUrl(input: unknown): string {
+  const value = stringField(input).trim();
+  if (!value) return "";
+  if (!URL.canParse(value)) return "";
+  const url = new URL(value);
+  if (
+    url.protocol !== "https:" ||
+    (url.hostname !== "www.zhihu.com" && url.hostname !== "zhihu.com")
+  ) {
+    return "";
+  }
+  return (
+    url.pathname.match(ANSWER_PATH_RE)?.[1] ??
+    url.pathname.match(QUESTION_PATH_RE)?.[1] ??
+    url.pathname.match(QUESTION_API_PATH_RE)?.[1] ??
+    ""
+  );
+}
+
+export function parseMaxContent(value: unknown): number {
+  const n = value === undefined || value === null ? 0 : Number(value);
+  if (!Number.isInteger(n) || n < 0) {
+    throw new Error("--max-content must be a non-negative integer.");
+  }
+  return n;
+}
+
+async function browserFetchJson(page: IPage, url: string): Promise<unknown> {
+  const raw = await page.evaluate(`(async () => {
+    const response = await fetch(${JSON.stringify(url)}, {
+      credentials: "include",
+      headers: { Accept: "application/json" }
+    });
+    const text = await response.text();
+    return JSON.stringify({ ok: response.ok, status: response.status, text });
+  })()`);
+  const result = JSON.parse(stringField(raw)) as BrowserFetchResult;
+  if (!result.ok) {
+    if (result.status === 401 || result.status === 403) {
+      throw new Error(
+        `Zhihu answer-detail requires Zhihu cookies (HTTP ${result.status}).`,
+      );
+    }
+    if (result.status === 404) {
+      throw new Error("No Zhihu answer was found for the requested id.");
+    }
+    throw new Error(
+      `Zhihu answer-detail request failed (HTTP ${result.status}).`,
+    );
+  }
+  try {
+    return JSON.parse(result.text);
+  } catch (err) {
+    throw new Error(
+      `Zhihu answer-detail returned malformed JSON: ${
+        err instanceof Error ? err.message : String(err)
+      }`,
+    );
+  }
+}
+
+export function mapZhihuAnswerDetail(
+  payload: ZhihuAnswerPayload,
+  target: ZhihuAnswerTarget,
+  currentQuestionId: string,
+  maxContent: number,
+): Record<string, unknown> {
+  if (payload.error || payload.error_msg || payload.message) {
+    throw new Error(
+      `Zhihu answer-detail returned an error payload: ${stringField(
+        payload.error?.message ?? payload.error_msg ?? payload.message,
+      )}`,
+    );
+  }
+  if (!Object.prototype.hasOwnProperty.call(payload, "content")) {
+    throw new Error("Zhihu answer-detail payload did not include content.");
+  }
+  const question = payload.question ?? {};
+  const questionId =
+    target.questionId ||
+    currentQuestionId ||
+    extractQuestionIdFromZhihuUrl(question.url) ||
+    stringField(question.id);
+  const stripped = stripZhihuAnswerHtml(payload.content);
+  const content =
+    maxContent > 0 && stripped.length > maxContent
+      ? stripped.slice(0, maxContent)
+      : stripped;
+  return {
+    id: target.answerId || stringField(payload.id),
+    author: stringField(payload.author?.name) || "anonymous",
+    votes: countField(payload.voteup_count),
+    comments: countField(payload.comment_count),
+    question_id: questionId,
+    question_title: stringField(question.title),
+    url: questionId
+      ? `https://www.zhihu.com/question/${questionId}/answer/${target.answerId}`
+      : `https://www.zhihu.com/answer/${target.answerId}`,
+    created_at: isoFromUnixSeconds(payload.created_time),
+    updated_at: isoFromUnixSeconds(payload.updated_time),
+    content,
+  };
+}
+
+cli({
+  site: "zhihu",
+  name: "answer-detail",
+  description: "知乎单个回答完整内容（按 answer ID 获取）",
+  domain: "www.zhihu.com",
+  strategy: Strategy.COOKIE,
+  browser: true,
+  args: [
+    {
+      name: "id",
+      type: "str",
+      required: true,
+      positional: true,
+      description: "Answer ID, full Zhihu answer URL, or typed target",
+    },
+    {
+      name: "max-content",
+      type: "int",
+      default: 0,
+      description:
+        "Optional cap on stripped content length; 0 returns full content",
+    },
+  ],
+  columns: [
+    "id",
+    "author",
+    "votes",
+    "comments",
+    "question_id",
+    "question_title",
+    "url",
+    "created_at",
+    "updated_at",
+    "content",
+  ],
+  func: async (page, kwargs) => {
+    const browser = pageOf(page);
+    const target = parseZhihuAnswerTarget(kwargs.id);
+    const maxContent = parseMaxContent(kwargs["max-content"]);
+    await browser.goto(`https://www.zhihu.com/answer/${target.answerId}`);
+    const currentQuestionId = extractQuestionIdFromZhihuUrl(
+      await browser.url(),
+    );
+    const apiUrl = new URL(
+      `https://www.zhihu.com/api/v4/answers/${target.answerId}`,
+    );
+    apiUrl.searchParams.set(
+      "include",
+      "content,voteup_count,comment_count,author,created_time,updated_time,question",
+    );
+    const payload = (await browserFetchJson(
+      browser,
+      apiUrl.toString(),
+    )) as ZhihuAnswerPayload;
+    return [
+      mapZhihuAnswerDetail(payload, target, currentQuestionId, maxContent),
+    ];
+  },
+});