@zenalexa/unicli 0.220.1 → 0.221.0

package/src/adapters/youtube/subtitles.ts

@@ -0,0 +1,44 @@
+/**
+ * @owner   YouTube subtitle adapter.
+ * @does    Extracts LLM-readable subtitle text from public YouTube video URLs.
+ * @needs   `yt-dlp` on PATH and an accessible video subtitle track.
+ * @feeds   Search-to-transcript social video analysis workflows.
+ * @breaks  Upstream yt-dlp extractor changes can alter subtitle availability.
+ */
+
+import { cli, Strategy } from "../../registry.js";
+import { runVideoSubtitleExtraction } from "../../social/video-text.js";
+
+cli({
+  site: "youtube",
+  name: "subtitles",
+  description:
+    "Extract LLM-readable subtitles from a YouTube video URL with optional browser-cookie reuse",
+  domain: "www.youtube.com",
+  strategy: Strategy.PUBLIC,
+  args: [
+    {
+      name: "url",
+      type: "str",
+      required: true,
+      positional: true,
+      description: "YouTube video URL",
+      format: "uri",
+    },
+    {
+      name: "languages",
+      type: "str",
+      default: "zh-Hans,zh,en",
+      description: "Comma-separated subtitle languages",
+    },
+    {
+      name: "cookies-from-browser",
+      type: "str",
+      description: "Browser name for yt-dlp --cookies-from-browser",
+    },
+  ],
+  columns: ["language", "text", "path"],
+  socialCapabilities: ["read", "media", "subtitles"],
+  defaultFormat: "json",
+  func: async (_page, kwargs) => runVideoSubtitleExtraction(kwargs),
+});

package/src/adapters/yt-dlp/subtitles.ts

@@ -0,0 +1,43 @@
+/**
+ * @owner   yt-dlp subtitle adapter.
+ * @does    Extracts subtitle files from any yt-dlp-supported video URL.
+ * @needs   Local yt-dlp binary and optional browser cookies for private or rate-limited videos.
+ * @feeds   Unified social video-text workflows across YouTube, Bilibili, TikTok, Douyin, X, Instagram, and other sites.
+ * @breaks  Returns an error when yt-dlp cannot produce subtitle files for the requested URL and languages.
+ */
+
+import { cli, Strategy } from "../../registry.js";
+import { runVideoSubtitleExtraction } from "../../social/video-text.js";
+
+cli({
+  site: "yt-dlp",
+  name: "subtitles",
+  description:
+    "Extract subtitles from any yt-dlp-supported video URL with optional browser-cookie reuse",
+  strategy: Strategy.PUBLIC,
+  args: [
+    {
+      name: "url",
+      type: "str",
+      required: true,
+      positional: true,
+      description: "Video URL",
+      format: "uri",
+    },
+    {
+      name: "languages",
+      type: "str",
+      default: "zh-Hans,zh,en",
+      description: "Comma-separated subtitle languages",
+    },
+    {
+      name: "cookies-from-browser",
+      type: "str",
+      description: "Browser name for yt-dlp --cookies-from-browser",
+    },
+  ],
+  columns: ["language", "text", "path"],
+  socialCapabilities: ["read", "media", "subtitles"],
+  defaultFormat: "json",
+  func: async (_page, kwargs) => runVideoSubtitleExtraction(kwargs),
+});

package/src/adapters/zhihu/answer-detail.test.ts

@@ -0,0 +1,83 @@
+import { describe, expect, it } from "vitest";
+import {
+  extractQuestionIdFromZhihuUrl,
+  mapZhihuAnswerDetail,
+  parseMaxContent,
+  parseZhihuAnswerTarget,
+  stripZhihuAnswerHtml,
+} from "./answer-detail.js";
+
+describe("zhihu answer-detail helpers", () => {
+  it("parses numeric, URL, and typed answer targets", () => {
+    expect(parseZhihuAnswerTarget("1937205528846655537")).toEqual({
+      answerId: "1937205528846655537",
+      questionId: "",
+    });
+    expect(
+      parseZhihuAnswerTarget(
+        "https://www.zhihu.com/question/123456/answer/789012",
+      ),
+    ).toEqual({ questionId: "123456", answerId: "789012" });
+    expect(parseZhihuAnswerTarget("answer:123456:789012")).toEqual({
+      questionId: "123456",
+      answerId: "789012",
+    });
+  });
+
+  it("rejects non-Zhihu answer targets", () => {
+    expect(() =>
+      parseZhihuAnswerTarget("https://example.com/question/1/answer/2"),
+    ).toThrow("Zhihu answer target");
+    expect(() => parseZhihuAnswerTarget("question:1")).toThrow(
+      "Zhihu answer target",
+    );
+  });
+
+  it("strips answer HTML without collapsing paragraphs", () => {
+    expect(
+      stripZhihuAnswerHtml(
+        "<p>Hello&nbsp;&amp;&nbsp;world</p><blockquote>A<br>B</blockquote>",
+      ),
+    ).toBe("Hello & world\n\nA\nB");
+  });
+
+  it("maps full answer payloads with explicit truncation only", () => {
+    const row = mapZhihuAnswerDetail(
+      {
+        content: "<p>abcdef</p>",
+        voteup_count: 7,
+        comment_count: 2,
+        created_time: 1,
+        updated_time: 2,
+        author: { name: "Ada" },
+        question: { id: 123, title: "Question" },
+      },
+      { answerId: "456", questionId: "" },
+      "",
+      parseMaxContent(3),
+    );
+    expect(row).toMatchObject({
+      id: "456",
+      author: "Ada",
+      votes: 7,
+      comments: 2,
+      question_id: "123",
+      question_title: "Question",
+      content: "abc",
+    });
+  });
+
+  it("extracts question ids from supported Zhihu URLs", () => {
+    expect(
+      extractQuestionIdFromZhihuUrl(
+        "https://www.zhihu.com/question/123/answer/456",
+      ),
+    ).toBe("123");
+    expect(
+      extractQuestionIdFromZhihuUrl("https://zhihu.com/question/123"),
+    ).toBe("123");
+    expect(
+      extractQuestionIdFromZhihuUrl("https://example.com/question/123"),
+    ).toBe("");
+  });
+});

package/src/adapters/zhihu/answer-detail.ts

@@ -0,0 +1,275 @@
+/**
+ * @owner   src/adapters/zhihu/answer-detail.ts
+ * @does    Register full-content Zhihu answer detail reader.
+ * @needs   Logged-in Zhihu browser session, answer API payload, exact answer target parsing.
+ * @feeds   surface coverage ledger, long-form answer extraction workflows, reference parity checks.
+ * @breaks  Zhihu answer URL shape or API content field drift can block answer detail extraction.
+ */
+
+import { cli, Strategy } from "../../registry.js";
+import type { IPage } from "../../types.js";
+
+const ANSWER_ID_RE = /^\d+$/;
+const ANSWER_TYPED_RE = /^answer:(\d+):(\d+)$/;
+const ANSWER_PATH_RE = /^\/question\/(\d+)\/answer\/(\d+)\/?$/;
+const BARE_ANSWER_PATH_RE = /^\/answer\/(\d+)\/?$/;
+const QUESTION_PATH_RE = /^\/question\/(\d+)\/?$/;
+const QUESTION_API_PATH_RE = /^\/api\/v4\/questions\/(\d+)\/?$/;
+
+interface ZhihuAnswerTarget {
+  answerId: string;
+  questionId: string;
+}
+
+interface ZhihuAnswerPayload {
+  id?: unknown;
+  content?: unknown;
+  voteup_count?: unknown;
+  comment_count?: unknown;
+  created_time?: unknown;
+  updated_time?: unknown;
+  author?: { name?: unknown };
+  question?: { id?: unknown; title?: unknown; url?: unknown };
+  error?: { message?: unknown };
+  error_msg?: unknown;
+  message?: unknown;
+}
+
+interface BrowserFetchResult {
+  ok: boolean;
+  status: number;
+  text: string;
+}
+
+function stringField(value: unknown): string {
+  return value === undefined || value === null ? "" : String(value);
+}
+
+function countField(value: unknown): number {
+  const n = Number(value);
+  return Number.isInteger(n) && n >= 0 ? n : 0;
+}
+
+function isoFromUnixSeconds(value: unknown): string {
+  const n = Number(value);
+  return Number.isFinite(n) && n > 0 ? new Date(n * 1000).toISOString() : "";
+}
+
+function pageOf(page: unknown): IPage {
+  if (!page) throw new Error("Zhihu answer-detail requires a browser page.");
+  return page as IPage;
+}
+
+export function stripZhihuAnswerHtml(value: unknown): string {
+  return stringField(value)
+    .replace(/<br\s*\/?\s*>/gi, "\n")
+    .replace(/<\/(?:p|div|h[1-6]|li|blockquote)>/gi, "\n\n")
+    .replace(/<[^>]+>/g, "")
+    .replace(/&nbsp;/g, " ")
+    .replace(/&lt;/g, "<")
+    .replace(/&gt;/g, ">")
+    .replace(/&amp;/g, "&")
+    .replace(/&quot;/g, '"')
+    .replace(/&#39;/g, "'")
+    .replace(/\n{3,}/g, "\n\n")
+    .trim();
+}
+
+export function parseZhihuAnswerTarget(input: unknown): ZhihuAnswerTarget {
+  const value = stringField(input).trim();
+  if (!value) {
+    throw new Error(
+      "Zhihu answer target must be a numeric id, answer URL, or answer:<questionId>:<answerId>.",
+    );
+  }
+  if (ANSWER_ID_RE.test(value)) return { answerId: value, questionId: "" };
+  const typed = value.match(ANSWER_TYPED_RE);
+  if (typed) return { questionId: typed[1], answerId: typed[2] };
+  try {
+    const url = new URL(value);
+    if (
+      url.protocol !== "https:" ||
+      url.username ||
+      url.password ||
+      url.port ||
+      (url.hostname !== "www.zhihu.com" && url.hostname !== "zhihu.com")
+    ) {
+      throw new Error("invalid host");
+    }
+    let match = url.pathname.match(ANSWER_PATH_RE);
+    if (match) return { questionId: match[1], answerId: match[2] };
+    match = url.pathname.match(BARE_ANSWER_PATH_RE);
+    if (match) return { questionId: "", answerId: match[1] };
+  } catch {
+    throw new Error(
+      "Zhihu answer target must be a numeric id, answer URL, or answer:<questionId>:<answerId>.",
+    );
+  }
+  throw new Error(
+    "Zhihu answer target must be a numeric id, answer URL, or answer:<questionId>:<answerId>.",
+  );
+}
+
+export function extractQuestionIdFromZhihuUrl(input: unknown): string {
+  const value = stringField(input).trim();
+  if (!value) return "";
+  if (!URL.canParse(value)) return "";
+  const url = new URL(value);
+  if (
+    url.protocol !== "https:" ||
+    (url.hostname !== "www.zhihu.com" && url.hostname !== "zhihu.com")
+  ) {
+    return "";
+  }
+  return (
+    url.pathname.match(ANSWER_PATH_RE)?.[1] ??
+    url.pathname.match(QUESTION_PATH_RE)?.[1] ??
+    url.pathname.match(QUESTION_API_PATH_RE)?.[1] ??
+    ""
+  );
+}
+
+export function parseMaxContent(value: unknown): number {
+  const n = value === undefined || value === null ? 0 : Number(value);
+  if (!Number.isInteger(n) || n < 0) {
+    throw new Error("--max-content must be a non-negative integer.");
+  }
+  return n;
+}
+
+async function browserFetchJson(page: IPage, url: string): Promise<unknown> {
+  const raw = await page.evaluate(`(async () => {
+    const response = await fetch(${JSON.stringify(url)}, {
+      credentials: "include",
+      headers: { Accept: "application/json" }
+    });
+    const text = await response.text();
+    return JSON.stringify({ ok: response.ok, status: response.status, text });
+  })()`);
+  const result = JSON.parse(stringField(raw)) as BrowserFetchResult;
+  if (!result.ok) {
+    if (result.status === 401 || result.status === 403) {
+      throw new Error(
+        `Zhihu answer-detail requires Zhihu cookies (HTTP ${result.status}).`,
+      );
+    }
+    if (result.status === 404) {
+      throw new Error("No Zhihu answer was found for the requested id.");
+    }
+    throw new Error(
+      `Zhihu answer-detail request failed (HTTP ${result.status}).`,
+    );
+  }
+  try {
+    return JSON.parse(result.text);
+  } catch (err) {
+    throw new Error(
+      `Zhihu answer-detail returned malformed JSON: ${
+        err instanceof Error ? err.message : String(err)
+      }`,
+    );
+  }
+}
+
+export function mapZhihuAnswerDetail(
+  payload: ZhihuAnswerPayload,
+  target: ZhihuAnswerTarget,
+  currentQuestionId: string,
+  maxContent: number,
+): Record<string, unknown> {
+  if (payload.error || payload.error_msg || payload.message) {
+    throw new Error(
+      `Zhihu answer-detail returned an error payload: ${stringField(
+        payload.error?.message ?? payload.error_msg ?? payload.message,
+      )}`,
+    );
+  }
+  if (!Object.prototype.hasOwnProperty.call(payload, "content")) {
+    throw new Error("Zhihu answer-detail payload did not include content.");
+  }
+  const question = payload.question ?? {};
+  const questionId =
+    target.questionId ||
+    currentQuestionId ||
+    extractQuestionIdFromZhihuUrl(question.url) ||
+    stringField(question.id);
+  const stripped = stripZhihuAnswerHtml(payload.content);
+  const content =
+    maxContent > 0 && stripped.length > maxContent
+      ? stripped.slice(0, maxContent)
+      : stripped;
+  return {
+    id: target.answerId || stringField(payload.id),
+    author: stringField(payload.author?.name) || "anonymous",
+    votes: countField(payload.voteup_count),
+    comments: countField(payload.comment_count),
+    question_id: questionId,
+    question_title: stringField(question.title),
+    url: questionId
+      ? `https://www.zhihu.com/question/${questionId}/answer/${target.answerId}`
+      : `https://www.zhihu.com/answer/${target.answerId}`,
+    created_at: isoFromUnixSeconds(payload.created_time),
+    updated_at: isoFromUnixSeconds(payload.updated_time),
+    content,
+  };
+}
+
+cli({
+  site: "zhihu",
+  name: "answer-detail",
+  description: "知乎单个回答完整内容（按 answer ID 获取）",
+  domain: "www.zhihu.com",
+  strategy: Strategy.COOKIE,
+  browser: true,
+  args: [
+    {
+      name: "id",
+      type: "str",
+      required: true,
+      positional: true,
+      description: "Answer ID, full Zhihu answer URL, or typed target",
+    },
+    {
+      name: "max-content",
+      type: "int",
+      default: 0,
+      description:
+        "Optional cap on stripped content length; 0 returns full content",
+    },
+  ],
+  columns: [
+    "id",
+    "author",
+    "votes",
+    "comments",
+    "question_id",
+    "question_title",
+    "url",
+    "created_at",
+    "updated_at",
+    "content",
+  ],
+  func: async (page, kwargs) => {
+    const browser = pageOf(page);
+    const target = parseZhihuAnswerTarget(kwargs.id);
+    const maxContent = parseMaxContent(kwargs["max-content"]);
+    await browser.goto(`https://www.zhihu.com/answer/${target.answerId}`);
+    const currentQuestionId = extractQuestionIdFromZhihuUrl(
+      await browser.url(),
+    );
+    const apiUrl = new URL(
+      `https://www.zhihu.com/api/v4/answers/${target.answerId}`,
+    );
+    apiUrl.searchParams.set(
+      "include",
+      "content,voteup_count,comment_count,author,created_time,updated_time,question",
+    );
+    const payload = (await browserFetchJson(
+      browser,
+      apiUrl.toString(),
+    )) as ZhihuAnswerPayload;
+    return [
+      mapZhihuAnswerDetail(payload, target, currentQuestionId, maxContent),
+    ];
+  },
+});

package/src/adapters/zhihu/comment-tree.test.ts

@@ -0,0 +1,57 @@
+import { describe, expect, it } from "vitest";
+
+import { normalizeCommentRows } from "../../social/comments.js";
+import { extractZhihuCommentRows } from "./comment.js";
+
+describe("zhihu comment tree extraction", () => {
+  it("normalizes root comments and child comments", () => {
+    const rows = normalizeCommentRows(
+      extractZhihuCommentRows({
+        data: [
+          {
+            id: "root",
+            author: { name: "Alice" },
+            content: "<p>Root</p>",
+            like_count: 4,
+            child_comment_count: 1,
+            created_time: 1,
+            child_comments: [
+              {
+                id: "child",
+                author: { name: "Bob" },
+                content: "<p>Child</p>",
+                like_count: 2,
+                created_time: 2,
+                reply_to_author: { name: "Alice" },
+              },
+            ],
+          },
+        ],
+      }),
+      { platform: "zhihu", contentId: "answer:123" },
+    );
+
+    expect(rows).toEqual([
+      expect.objectContaining({
+        comment_id: "root",
+        parent_id: "",
+        depth: 0,
+        path: "0001",
+        author: "Alice",
+        text: "Root",
+        likes: 4,
+        replies: 1,
+      }),
+      expect.objectContaining({
+        comment_id: "child",
+        parent_id: "root",
+        depth: 1,
+        path: "0001.0001",
+        author: "Bob",
+        text: "Child",
+        likes: 2,
+        reply_to: "Alice",
+      }),
+    ]);
+  });
+});

package/src/adapters/zhihu/comment.ts

@@ -0,0 +1,186 @@
+/**
+ * @owner   Zhihu comment adapter.
+ * @does    Reads Zhihu answer/article comments and normalizes nested child comments.
+ * @needs   Zhihu comment_v5 API and z_c0 cookie for authenticated access.
+ * @feeds   Social comment tree analysis for Zhihu answers and articles.
+ * @breaks  Zhihu API shape, auth cookie requirements, or anti-bot policy changes.
+ */
+
+import { USER_AGENT } from "../../constants.js";
+import {
+  loadCookiesWithCDP,
+  formatCookieHeader,
+} from "../../engine/cookies.js";
+import { cli, Strategy } from "../../registry.js";
+import { normalizeCommentRows } from "../../social/comments.js";
+
+interface ZhihuAuthor {
+  name?: string;
+}
+
+interface ZhihuComment {
+  id?: string | number;
+  author?: ZhihuAuthor;
+  content?: string;
+  like_count?: number;
+  child_comment_count?: number;
+  created_time?: number;
+  child_comments?: ZhihuComment[];
+  reply_to_author?: ZhihuAuthor;
+}
+
+interface ZhihuCommentResponse {
+  data?: ZhihuComment[];
+}
+
+function stripHtml(value: string): string {
+  return value
+    .replace(/<[^>]+>/g, "")
+    .replace(/\s+/g, " ")
+    .trim();
+}
+
+function zhihuCreated(value: number | undefined): string {
+  return value ? new Date(value * 1000).toISOString() : "";
+}
+
+function zhihuCommentToRow(
+  item: ZhihuComment,
+  parentId = "",
+): Record<string, unknown> {
+  const id = item.id === undefined ? "" : String(item.id);
+  return {
+    id,
+    parent_id: parentId,
+    author: item.author?.name ?? "",
+    content: stripHtml(item.content ?? ""),
+    text: stripHtml(item.content ?? ""),
+    likes: item.like_count ?? 0,
+    replies: item.child_comment_count ?? item.child_comments?.length ?? 0,
+    created: zhihuCreated(item.created_time),
+    reply_to: item.reply_to_author?.name ?? "",
+  };
+}
+
+export function extractZhihuCommentRows(
+  response: unknown,
+): Array<Record<string, unknown>> {
+  const data = (response as ZhihuCommentResponse).data;
+  if (!Array.isArray(data)) {
+    throw new Error("Unexpected Zhihu comments response");
+  }
+  const rows: Array<Record<string, unknown>> = [];
+  for (const root of data) {
+    const rootRow = zhihuCommentToRow(root);
+    rows.push(rootRow);
+    const rootId = String(rootRow.id ?? "");
+    for (const child of root.child_comments ?? []) {
+      rows.push(zhihuCommentToRow(child, rootId));
+    }
+  }
+  return rows;
+}
+
+async function zhihuHeaders(): Promise<Record<string, string>> {
+  const headers: Record<string, string> = {
+    "User-Agent": USER_AGENT,
+    Accept: "application/json",
+    Referer: "https://www.zhihu.com/",
+  };
+  const cookies = await loadCookiesWithCDP("zhihu", "www.zhihu.com");
+  if (cookies) headers.Cookie = formatCookieHeader(cookies);
+  return headers;
+}
+
+async function fetchZhihuJson(
+  url: string,
+  headers: Record<string, string>,
+): Promise<unknown> {
+  const response = await fetch(url, { headers });
+  if (!response.ok) {
+    throw new Error(`Zhihu comments request failed: ${response.status}`);
+  }
+  return response.json();
+}
+
+cli({
+  site: "zhihu",
+  name: "comment",
+  description: "Get Zhihu comments with normalized nested reply hierarchy",
+  domain: "www.zhihu.com",
+  strategy: Strategy.COOKIE,
+  args: [
+    {
+      name: "type",
+      required: true,
+      positional: true,
+      description: "Content type: answer or article",
+      choices: ["answer", "article"],
+    },
+    {
+      name: "id",
+      required: true,
+      positional: true,
+      description: "Answer or article ID",
+      "x-unicli-kind": "id",
+    },
+    {
+      name: "limit",
+      type: "int",
+      default: 20,
+      description: "Number of root comments",
+    },
+    {
+      name: "with-replies",
+      type: "bool",
+      default: false,
+      description:
+        "Fetch child replies for root comments that omit child_comments",
+    },
+  ],
+  columns: [
+    "platform",
+    "content_id",
+    "comment_id",
+    "parent_id",
+    "depth",
+    "path",
+    "author",
+    "text",
+    "likes",
+    "replies",
+    "created",
+    "content",
+    "reply_to",
+  ],
+  socialCapabilities: ["read", "comments", "comment_replies"],
+  func: async (_page, kwargs) => {
+    const type = String(kwargs.type);
+    const id = String(kwargs.id);
+    const limit = Number(kwargs.limit ?? 20);
+    const withReplies = Boolean(kwargs["with-replies"]);
+    const headers = await zhihuHeaders();
+    const rootUrl = `https://www.zhihu.com/api/v4/comment_v5/${encodeURIComponent(type)}s/${encodeURIComponent(id)}/root_comment?limit=${encodeURIComponent(String(limit))}&offset=0&order_by=score`;
+    const rootJson = await fetchZhihuJson(rootUrl, headers);
+    const rows = extractZhihuCommentRows(rootJson);
+
+    if (withReplies) {
+      const roots = (rootJson as ZhihuCommentResponse).data ?? [];
+      for (const root of roots) {
+        const rootId = root.id === undefined ? "" : String(root.id);
+        if (!rootId || (root.child_comments?.length ?? 0) > 0) continue;
+        if ((root.child_comment_count ?? 0) <= 0) continue;
+        const childUrl = `https://www.zhihu.com/api/v4/comment_v5/comment/${encodeURIComponent(rootId)}/child_comment?limit=20&offset=0&order_by=score`;
+        const childJson = await fetchZhihuJson(childUrl, headers);
+        for (const child of (childJson as ZhihuCommentResponse).data ?? []) {
+          rows.push(zhihuCommentToRow(child, rootId));
+        }
+      }
+    }
+
+    return normalizeCommentRows(rows, {
+      platform: "zhihu",
+      contentId: `${type}:${id}`,
+    });
+  },
+});