@zenalexa/unicli 0.220.0 → 0.221.0

package/src/adapters/zhihu/comment-tree.test.ts

@@ -0,0 +1,57 @@
+import { describe, expect, it } from "vitest";
+
+import { normalizeCommentRows } from "../../social/comments.js";
+import { extractZhihuCommentRows } from "./comment.js";
+
+describe("zhihu comment tree extraction", () => {
+  it("normalizes root comments and child comments", () => {
+    const rows = normalizeCommentRows(
+      extractZhihuCommentRows({
+        data: [
+          {
+            id: "root",
+            author: { name: "Alice" },
+            content: "<p>Root</p>",
+            like_count: 4,
+            child_comment_count: 1,
+            created_time: 1,
+            child_comments: [
+              {
+                id: "child",
+                author: { name: "Bob" },
+                content: "<p>Child</p>",
+                like_count: 2,
+                created_time: 2,
+                reply_to_author: { name: "Alice" },
+              },
+            ],
+          },
+        ],
+      }),
+      { platform: "zhihu", contentId: "answer:123" },
+    );
+
+    expect(rows).toEqual([
+      expect.objectContaining({
+        comment_id: "root",
+        parent_id: "",
+        depth: 0,
+        path: "0001",
+        author: "Alice",
+        text: "Root",
+        likes: 4,
+        replies: 1,
+      }),
+      expect.objectContaining({
+        comment_id: "child",
+        parent_id: "root",
+        depth: 1,
+        path: "0001.0001",
+        author: "Bob",
+        text: "Child",
+        likes: 2,
+        reply_to: "Alice",
+      }),
+    ]);
+  });
+});

package/src/adapters/zhihu/comment.ts

@@ -0,0 +1,186 @@
+/**
+ * @owner   Zhihu comment adapter.
+ * @does    Reads Zhihu answer/article comments and normalizes nested child comments.
+ * @needs   Zhihu comment_v5 API and z_c0 cookie for authenticated access.
+ * @feeds   Social comment tree analysis for Zhihu answers and articles.
+ * @breaks  Zhihu API shape, auth cookie requirements, or anti-bot policy changes.
+ */
+
+import { USER_AGENT } from "../../constants.js";
+import {
+  loadCookiesWithCDP,
+  formatCookieHeader,
+} from "../../engine/cookies.js";
+import { cli, Strategy } from "../../registry.js";
+import { normalizeCommentRows } from "../../social/comments.js";
+
+interface ZhihuAuthor {
+  name?: string;
+}
+
+interface ZhihuComment {
+  id?: string | number;
+  author?: ZhihuAuthor;
+  content?: string;
+  like_count?: number;
+  child_comment_count?: number;
+  created_time?: number;
+  child_comments?: ZhihuComment[];
+  reply_to_author?: ZhihuAuthor;
+}
+
+interface ZhihuCommentResponse {
+  data?: ZhihuComment[];
+}
+
+function stripHtml(value: string): string {
+  return value
+    .replace(/<[^>]+>/g, "")
+    .replace(/\s+/g, " ")
+    .trim();
+}
+
+function zhihuCreated(value: number | undefined): string {
+  return value ? new Date(value * 1000).toISOString() : "";
+}
+
+function zhihuCommentToRow(
+  item: ZhihuComment,
+  parentId = "",
+): Record<string, unknown> {
+  const id = item.id === undefined ? "" : String(item.id);
+  return {
+    id,
+    parent_id: parentId,
+    author: item.author?.name ?? "",
+    content: stripHtml(item.content ?? ""),
+    text: stripHtml(item.content ?? ""),
+    likes: item.like_count ?? 0,
+    replies: item.child_comment_count ?? item.child_comments?.length ?? 0,
+    created: zhihuCreated(item.created_time),
+    reply_to: item.reply_to_author?.name ?? "",
+  };
+}
+
+export function extractZhihuCommentRows(
+  response: unknown,
+): Array<Record<string, unknown>> {
+  const data = (response as ZhihuCommentResponse).data;
+  if (!Array.isArray(data)) {
+    throw new Error("Unexpected Zhihu comments response");
+  }
+  const rows: Array<Record<string, unknown>> = [];
+  for (const root of data) {
+    const rootRow = zhihuCommentToRow(root);
+    rows.push(rootRow);
+    const rootId = String(rootRow.id ?? "");
+    for (const child of root.child_comments ?? []) {
+      rows.push(zhihuCommentToRow(child, rootId));
+    }
+  }
+  return rows;
+}
+
+async function zhihuHeaders(): Promise<Record<string, string>> {
+  const headers: Record<string, string> = {
+    "User-Agent": USER_AGENT,
+    Accept: "application/json",
+    Referer: "https://www.zhihu.com/",
+  };
+  const cookies = await loadCookiesWithCDP("zhihu", "www.zhihu.com");
+  if (cookies) headers.Cookie = formatCookieHeader(cookies);
+  return headers;
+}
+
+async function fetchZhihuJson(
+  url: string,
+  headers: Record<string, string>,
+): Promise<unknown> {
+  const response = await fetch(url, { headers });
+  if (!response.ok) {
+    throw new Error(`Zhihu comments request failed: ${response.status}`);
+  }
+  return response.json();
+}
+
+cli({
+  site: "zhihu",
+  name: "comment",
+  description: "Get Zhihu comments with normalized nested reply hierarchy",
+  domain: "www.zhihu.com",
+  strategy: Strategy.COOKIE,
+  args: [
+    {
+      name: "type",
+      required: true,
+      positional: true,
+      description: "Content type: answer or article",
+      choices: ["answer", "article"],
+    },
+    {
+      name: "id",
+      required: true,
+      positional: true,
+      description: "Answer or article ID",
+      "x-unicli-kind": "id",
+    },
+    {
+      name: "limit",
+      type: "int",
+      default: 20,
+      description: "Number of root comments",
+    },
+    {
+      name: "with-replies",
+      type: "bool",
+      default: false,
+      description:
+        "Fetch child replies for root comments that omit child_comments",
+    },
+  ],
+  columns: [
+    "platform",
+    "content_id",
+    "comment_id",
+    "parent_id",
+    "depth",
+    "path",
+    "author",
+    "text",
+    "likes",
+    "replies",
+    "created",
+    "content",
+    "reply_to",
+  ],
+  socialCapabilities: ["read", "comments", "comment_replies"],
+  func: async (_page, kwargs) => {
+    const type = String(kwargs.type);
+    const id = String(kwargs.id);
+    const limit = Number(kwargs.limit ?? 20);
+    const withReplies = Boolean(kwargs["with-replies"]);
+    const headers = await zhihuHeaders();
+    const rootUrl = `https://www.zhihu.com/api/v4/comment_v5/${encodeURIComponent(type)}s/${encodeURIComponent(id)}/root_comment?limit=${encodeURIComponent(String(limit))}&offset=0&order_by=score`;
+    const rootJson = await fetchZhihuJson(rootUrl, headers);
+    const rows = extractZhihuCommentRows(rootJson);
+
+    if (withReplies) {
+      const roots = (rootJson as ZhihuCommentResponse).data ?? [];
+      for (const root of roots) {
+        const rootId = root.id === undefined ? "" : String(root.id);
+        if (!rootId || (root.child_comments?.length ?? 0) > 0) continue;
+        if ((root.child_comment_count ?? 0) <= 0) continue;
+        const childUrl = `https://www.zhihu.com/api/v4/comment_v5/comment/${encodeURIComponent(rootId)}/child_comment?limit=20&offset=0&order_by=score`;
+        const childJson = await fetchZhihuJson(childUrl, headers);
+        for (const child of (childJson as ZhihuCommentResponse).data ?? []) {
+          rows.push(zhihuCommentToRow(child, rootId));
+        }
+      }
+    }
+
+    return normalizeCommentRows(rows, {
+      platform: "zhihu",
+      contentId: `${type}:${id}`,
+    });
+  },
+});

package/src/adapters/zhihu/recommend.test.ts

@@ -0,0 +1,65 @@
+import { describe, expect, it } from "vitest";
+import {
+  mapZhihuRecommendItem,
+  normalizeZhihuRecommendTitle,
+  normalizeZhihuRecommendUrl,
+  parseZhihuRecommendLimit,
+  zhihuRecommendItemKey,
+} from "./recommend.js";
+
+describe("zhihu recommend helpers", () => {
+  it("validates positive bounded limits", () => {
+    expect(parseZhihuRecommendLimit(undefined)).toBe(20);
+    expect(parseZhihuRecommendLimit("3")).toBe(3);
+    expect(() => parseZhihuRecommendLimit(0)).toThrow("positive integer");
+    expect(() => parseZhihuRecommendLimit(1001)).toThrow(
+      "no greater than 1000",
+    );
+  });
+
+  it("normalizes answer recommendation rows", () => {
+    const item = {
+      id: "feed-1",
+      target: {
+        id: 456,
+        type: "answer",
+        question: { id: 123, title: "Why?" },
+        author: { name: "Ada" },
+        voteup_count: 9,
+      },
+    };
+    expect(zhihuRecommendItemKey(item)).toBe("answer:456");
+    expect(normalizeZhihuRecommendTitle(item)).toBe("Why?");
+    expect(normalizeZhihuRecommendUrl(item)).toBe(
+      "https://www.zhihu.com/question/123/answer/456",
+    );
+    expect(mapZhihuRecommendItem(item, 1)).toEqual({
+      rank: 1,
+      type: "answer",
+      title: "Why?",
+      author: "Ada",
+      votes: 9,
+      url: "https://www.zhihu.com/question/123/answer/456",
+    });
+  });
+
+  it("normalizes article and question URLs", () => {
+    expect(
+      normalizeZhihuRecommendUrl({
+        target: { id: 789, type: "article", title: "Article" },
+      }),
+    ).toBe("https://zhuanlan.zhihu.com/p/789");
+    expect(
+      normalizeZhihuRecommendUrl({
+        target: { id: 123, type: "question", title: "Question" },
+      }),
+    ).toBe("https://www.zhihu.com/question/123");
+  });
+
+  it("uses feed ids as dedupe keys when targets have no id", () => {
+    expect(zhihuRecommendItemKey({ id: "feed-only", type: "feed" })).toBe(
+      "feed:feed-only",
+    );
+    expect(zhihuRecommendItemKey({ type: "feed" })).toBe("");
+  });
+});

package/src/adapters/zhihu/recommend.ts

@@ -0,0 +1,207 @@
+/**
+ * @owner   src/adapters/zhihu/recommend.ts
+ * @does    Register paginated Zhihu home recommendation reader.
+ * @needs   Logged-in Zhihu browser session, topstory recommendation API, bounded page traversal.
+ * @feeds   surface coverage ledger, Zhihu discovery workflows, reference parity checks.
+ * @breaks  Zhihu recommendation API paging or target shape drift can block feed extraction.
+ */
+
+import { cli, Strategy } from "../../registry.js";
+import type { IPage } from "../../types.js";
+
+const MAX_RECOMMEND_LIMIT = 1000;
+
+interface ZhihuRecommendItem {
+  id?: unknown;
+  type?: unknown;
+  target?: {
+    id?: unknown;
+    type?: unknown;
+    title?: unknown;
+    question?: { id?: unknown; title?: unknown };
+    author?: { name?: unknown };
+    voteup_count?: unknown;
+    reaction?: { statistics?: { like_count?: unknown } };
+  };
+}
+
+interface ZhihuRecommendPage {
+  data?: ZhihuRecommendItem[];
+  paging?: {
+    is_end?: unknown;
+    next?: unknown;
+  };
+}
+
+interface BrowserFetchResult {
+  ok: boolean;
+  status: number;
+  text: string;
+}
+
+function stringField(value: unknown): string {
+  return value === undefined || value === null ? "" : String(value);
+}
+
+function countField(value: unknown): number {
+  const n = Number(value);
+  return Number.isInteger(n) && n >= 0 ? n : 0;
+}
+
+function pageOf(page: unknown): IPage {
+  if (!page) throw new Error("Zhihu recommend requires a browser page.");
+  return page as IPage;
+}
+
+export function parseZhihuRecommendLimit(value: unknown): number {
+  const n = value === undefined || value === null ? 20 : Number(value);
+  if (!Number.isInteger(n) || n <= 0 || n > MAX_RECOMMEND_LIMIT) {
+    throw new Error(
+      `Zhihu recommend limit must be a positive integer no greater than ${MAX_RECOMMEND_LIMIT}.`,
+    );
+  }
+  return n;
+}
+
+export function zhihuRecommendItemKey(item: ZhihuRecommendItem): string {
+  const target = item.target ?? {};
+  if (target.id !== undefined && target.id !== null) {
+    return `${stringField(target.type)}:${stringField(target.id)}`;
+  }
+  if (item.id !== undefined && item.id !== null) {
+    return `feed:${stringField(item.id)}`;
+  }
+  return "";
+}
+
+export function normalizeZhihuRecommendTitle(item: ZhihuRecommendItem): string {
+  const target = item.target ?? {};
+  if (target.type === "answer") return stringField(target.question?.title);
+  return stringField(target.title || target.question?.title);
+}
+
+export function normalizeZhihuRecommendUrl(item: ZhihuRecommendItem): string {
+  const target = item.target ?? {};
+  const id = stringField(target.id);
+  if (target.type === "answer") {
+    const questionId = stringField(target.question?.id);
+    return questionId && id
+      ? `https://www.zhihu.com/question/${questionId}/answer/${id}`
+      : "";
+  }
+  if (target.type === "article") {
+    return id ? `https://zhuanlan.zhihu.com/p/${id}` : "";
+  }
+  if (target.type === "question") {
+    return id ? `https://www.zhihu.com/question/${id}` : "";
+  }
+  return "";
+}
+
+export function mapZhihuRecommendItem(
+  item: ZhihuRecommendItem,
+  rank: number,
+): Record<string, unknown> {
+  const target = item.target ?? {};
+  return {
+    rank,
+    type: stringField(target.type || item.type),
+    title: normalizeZhihuRecommendTitle(item),
+    author: stringField(target.author?.name),
+    votes: countField(
+      target.voteup_count ?? target.reaction?.statistics?.like_count,
+    ),
+    url: normalizeZhihuRecommendUrl(item),
+  };
+}
+
+async function browserFetchRecommend(
+  page: IPage,
+  url: string,
+): Promise<ZhihuRecommendPage> {
+  const raw = await page.evaluate(`(async () => {
+    const response = await fetch(${JSON.stringify(url)}, {
+      credentials: "include",
+      headers: { Accept: "application/json" }
+    });
+    const text = await response.text();
+    return JSON.stringify({ ok: response.ok, status: response.status, text });
+  })()`);
+  const result = JSON.parse(stringField(raw)) as BrowserFetchResult;
+  if (!result.ok) {
+    if (result.status === 401 || result.status === 403) {
+      throw new Error(
+        `Zhihu recommend requires Zhihu cookies (HTTP ${result.status}).`,
+      );
+    }
+    throw new Error(`Zhihu recommend request failed (HTTP ${result.status}).`);
+  }
+  try {
+    return JSON.parse(result.text) as ZhihuRecommendPage;
+  } catch (err) {
+    throw new Error(
+      `Zhihu recommend returned malformed JSON: ${
+        err instanceof Error ? err.message : String(err)
+      }`,
+    );
+  }
+}
+
+async function collectZhihuRecommendations(
+  page: IPage,
+  limit: number,
+): Promise<ZhihuRecommendItem[]> {
+  const items: ZhihuRecommendItem[] = [];
+  const seen = new Set<string>();
+  const visited = new Set<string>();
+  let url =
+    "https://www.zhihu.com/api/v3/feed/topstory/recommend?limit=10&desktop=true";
+  while (url && items.length < limit && !visited.has(url)) {
+    visited.add(url);
+    const data = await browserFetchRecommend(page, url);
+    const rows = Array.isArray(data.data) ? data.data : [];
+    for (const item of rows) {
+      const key = zhihuRecommendItemKey(item);
+      if (key) {
+        if (seen.has(key)) continue;
+        seen.add(key);
+      }
+      items.push(item);
+      if (items.length >= limit) break;
+    }
+    if (data.paging?.is_end) break;
+    url = typeof data.paging?.next === "string" ? data.paging.next : "";
+  }
+  if (items.length === 0) {
+    throw new Error("No Zhihu recommendations returned.");
+  }
+  return items;
+}
+
+cli({
+  site: "zhihu",
+  name: "recommend",
+  description: "知乎首页推荐",
+  domain: "www.zhihu.com",
+  strategy: Strategy.COOKIE,
+  browser: true,
+  args: [
+    {
+      name: "limit",
+      type: "int",
+      default: 20,
+      description:
+        "Number of items to return (max 1000; use normal-sized requests)",
+    },
+  ],
+  columns: ["rank", "type", "title", "author", "votes", "url"],
+  func: async (page, kwargs) => {
+    const browser = pageOf(page);
+    const limit = parseZhihuRecommendLimit(kwargs.limit);
+    await browser.goto("https://www.zhihu.com");
+    const items = await collectZhihuRecommendations(browser, limit);
+    return items
+      .slice(0, limit)
+      .map((item, index) => mapZhihuRecommendItem(item, index + 1));
+  },
+});