@zenalexa/unicli 0.220.1 → 0.221.0

package/src/adapters/reddit/comments.ts

@@ -0,0 +1,159 @@
+/**
+ * @owner   Reddit comments adapter.
+ * @does    Fetches a Reddit post comment listing and emits normalized nested comment rows.
+ * @needs   Public reddit JSON listing endpoint and a normal web User-Agent.
+ * @feeds   Social comment tree analysis and Reddit read workflows.
+ * @breaks  Reddit JSON schema or public endpoint access changes can block unauthenticated reads.
+ */
+
+import { USER_AGENT } from "../../constants.js";
+import { cli, Strategy } from "../../registry.js";
+import { normalizeCommentRows } from "../../social/comments.js";
+
+interface RedditListing {
+  data?: {
+    children?: RedditThing[];
+  };
+}
+
+interface RedditThing {
+  kind?: string;
+  data?: RedditCommentData;
+}
+
+interface RedditCommentData {
+  id?: string;
+  name?: string;
+  parent_id?: string;
+  author?: string;
+  body?: string;
+  score?: number;
+  created_utc?: number;
+  replies?: "" | RedditListing;
+}
+
+function normalizePostPath(input: string): string {
+  const raw = input.trim();
+  if (!raw) throw new Error("Reddit post URL or ID is required");
+  if (/^https?:\/\//i.test(raw)) {
+    const url = new URL(raw);
+    return url.pathname.replace(/^\/+/, "").replace(/\.json$/, "");
+  }
+  return raw.replace(/^\/+/, "").replace(/\.json$/, "");
+}
+
+function collectCommentRows(
+  listing: RedditListing,
+  contentId: string,
+  parentId = "",
+): Array<Record<string, unknown>> {
+  const rows: Array<Record<string, unknown>> = [];
+  for (const thing of listing.data?.children ?? []) {
+    if (thing.kind !== "t1") continue;
+    const data = thing.data;
+    if (!data?.id) continue;
+    const commentId = data.name ?? `t1_${data.id}`;
+    rows.push({
+      id: commentId,
+      parent_id:
+        data.parent_id && data.parent_id.startsWith("t1_")
+          ? data.parent_id
+          : parentId,
+      author: data.author ?? "",
+      body: data.body ?? "",
+      text: data.body ?? "",
+      score: data.score ?? 0,
+      replies:
+        data.replies && typeof data.replies === "object"
+          ? (data.replies.data?.children?.filter((child) => child.kind === "t1")
+              .length ?? 0)
+          : 0,
+      created: data.created_utc
+        ? new Date(data.created_utc * 1000).toISOString()
+        : "",
+      content_id: contentId,
+    });
+    if (data.replies && typeof data.replies === "object") {
+      rows.push(...collectCommentRows(data.replies, contentId, commentId));
+    }
+  }
+  return rows;
+}
+
+export function extractRedditCommentRows(
+  response: unknown,
+  contentId: string,
+): Array<Record<string, unknown>> {
+  if (!Array.isArray(response) || response.length < 2) {
+    throw new Error("Unexpected Reddit comments response");
+  }
+  return collectCommentRows(response[1] as RedditListing, contentId);
+}
+
+cli({
+  site: "reddit",
+  name: "comments",
+  description: "Reddit post comments with nested reply hierarchy",
+  domain: "www.reddit.com",
+  strategy: Strategy.PUBLIC,
+  args: [
+    {
+      name: "url",
+      required: true,
+      positional: true,
+      description: "Reddit post URL, path, or comments ID",
+      "x-unicli-accepts": ["url", "id"],
+    },
+    {
+      name: "sort",
+      default: "top",
+      choices: ["best", "top", "new", "controversial", "old", "qa"],
+      description: "Comment sort order",
+    },
+    {
+      name: "limit",
+      type: "int",
+      default: 20,
+      description: "Number of top-level comments",
+    },
+  ],
+  columns: [
+    "platform",
+    "content_id",
+    "comment_id",
+    "parent_id",
+    "depth",
+    "path",
+    "author",
+    "text",
+    "likes",
+    "replies",
+    "created",
+    "body",
+    "score",
+  ],
+  socialCapabilities: ["read", "comments", "comment_replies"],
+  func: async (_page, kwargs) => {
+    const postPath = normalizePostPath(String(kwargs.url ?? ""));
+    const sort = String(kwargs.sort ?? "top");
+    const limit = Number(kwargs.limit ?? 20);
+    const response = await fetch(
+      `https://www.reddit.com/${postPath}.json?sort=${encodeURIComponent(sort)}&limit=${encodeURIComponent(String(limit))}&raw_json=1`,
+      {
+        headers: {
+          "User-Agent": USER_AGENT,
+          Accept: "application/json",
+        },
+      },
+    );
+    if (!response.ok) {
+      throw new Error(`Reddit comments request failed: ${response.status}`);
+    }
+    const json = await response.json();
+    const rows = extractRedditCommentRows(json, postPath);
+    return normalizeCommentRows(rows, {
+      platform: "reddit",
+      contentId: postPath,
+    });
+  },
+});

package/src/adapters/threads/post.test.ts

@@ -0,0 +1,64 @@
+import { describe, expect, it } from "vitest";
+
+import { parseThreadsPostHtml, parseThreadsProfileHtml } from "./post.js";
+
+const postHtml = String.raw`
+  <html>
+    <head>
+      <meta property="og:title" content="Mark Zuckerberg (@zuck) on Threads" />
+      <meta property="og:description" content="Today we&#039;re starting to roll out Incognito Chat &amp; private message requests." />
+      <meta property="og:url" content="https://www.threads.com/@zuck/post/DYSAIo_FL77" />
+      <meta property="og:image" content="https://scontent.cdninstagram.com/v/t51.2885-15/threads-image.jpg" />
+      <meta property="og:image:width" content="1920" />
+      <meta property="og:image:height" content="1440" />
+      <meta property="al:ios:url" content="barcelona://media?shortcode=DYSAIo_FL77" />
+      <link rel="alternate" href="https://threads.net/ap/users/17841401746480004/post/17955322232962801/" type="application/activity+json" />
+    </head>
+  </html>
+`;
+
+const profileHtml = String.raw`
+  <html>
+    <head>
+      <meta property="og:title" content="Mark Zuckerberg (@zuck) • Threads, Say more" />
+      <meta property="og:description" content="5.5M Followers • 145 Threads • Mostly superintelligence and MMA takes. See the latest conversations with @zuck." />
+      <meta property="og:url" content="https://www.threads.com/@zuck" />
+      <meta property="og:image" content="https://scontent.cdninstagram.com/v/t51.2885-19/profile.jpg" />
+    </head>
+  </html>
+`;
+
+describe("Threads public page metadata", () => {
+  it("extracts a public post from Threads metadata", () => {
+    expect(
+      parseThreadsPostHtml(
+        postHtml,
+        "https://www.threads.net/@zuck/post/DYSAIo_FL77",
+      ),
+    ).toEqual({
+      activity_json_url:
+        "https://threads.net/ap/users/17841401746480004/post/17955322232962801/",
+      author: "Mark Zuckerberg",
+      handle: "@zuck",
+      image_height: 1440,
+      image_url:
+        "https://scontent.cdninstagram.com/v/t51.2885-15/threads-image.jpg",
+      image_width: 1920,
+      shortcode: "DYSAIo_FL77",
+      text: "Today we're starting to roll out Incognito Chat & private message requests.",
+      url: "https://www.threads.com/@zuck/post/DYSAIo_FL77",
+    });
+  });
+
+  it("extracts a public profile from Threads metadata", () => {
+    expect(parseThreadsProfileHtml(profileHtml, "zuck")).toEqual({
+      avatar_url: "https://scontent.cdninstagram.com/v/t51.2885-19/profile.jpg",
+      bio: "Mostly superintelligence and MMA takes.",
+      followers: "5.5M",
+      handle: "@zuck",
+      name: "Mark Zuckerberg",
+      threads: "145",
+      url: "https://www.threads.com/@zuck",
+    });
+  });
+});

package/src/adapters/threads/post.ts

@@ -0,0 +1,366 @@
+/**
+ * @owner   Threads public page metadata adapter.
+ * @does    Register public profile, post, and media readers backed by Threads OG metadata.
+ * @needs   Threads public HTML pages exposing Open Graph metadata.
+ * @feeds   Social research workflows and Threads capability coverage.
+ * @breaks  Threads hiding public metadata or changing canonical post/profile markup.
+ */
+
+import { USER_AGENT } from "../../constants.js";
+import { cli, Strategy } from "../../registry.js";
+
+export interface ThreadsPostRow {
+  author: string;
+  handle: string;
+  text: string;
+  url: string;
+  image_url: string;
+  image_width: number;
+  image_height: number;
+  shortcode: string;
+  activity_json_url: string;
+}
+
+export interface ThreadsProfileRow {
+  name: string;
+  handle: string;
+  followers: string;
+  threads: string;
+  bio: string;
+  url: string;
+  avatar_url: string;
+}
+
+interface Identity {
+  name: string;
+  handle: string;
+}
+
+const THREADS_HOST_RE = /(^|\.)threads\.(net|com)$/i;
+
+function decodeHtml(value: string): string {
+  return value
+    .replace(/&#x([0-9a-f]+);/gi, (_, code: string) =>
+      String.fromCodePoint(Number.parseInt(code, 16)),
+    )
+    .replace(/&#(\d+);/g, (_, code: string) =>
+      String.fromCodePoint(Number.parseInt(code, 10)),
+    )
+    .replace(/ /g, " ")
+    .replace(/&/g, "&")
+    .replace(/&lt;/g, "<")
+    .replace(/&gt;/g, ">")
+    .replace(/&quot;/g, '"')
+    .replace(/&apos;/g, "'")
+    .replace(/&#039;/g, "'")
+    .replace(/\s+/g, " ")
+    .trim();
+}
+
+function attributesFor(tag: string): Record<string, string> {
+  const attrs: Record<string, string> = {};
+  const attrRe = /\s([^\s"'=<>`]+)\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s>]+))/g;
+  for (const match of tag.matchAll(attrRe)) {
+    attrs[match[1].toLowerCase()] = decodeHtml(
+      match[2] ?? match[3] ?? match[4] ?? "",
+    );
+  }
+  return attrs;
+}
+
+function metaContent(html: string, key: string): string {
+  for (const match of html.matchAll(/<meta\b[^>]*>/gi)) {
+    const attrs = attributesFor(match[0]);
+    if (attrs.property === key || attrs.name === key)
+      return attrs.content ?? "";
+  }
+  return "";
+}
+
+function linkHref(html: string, rel: string, type: string): string {
+  for (const match of html.matchAll(/<link\b[^>]*>/gi)) {
+    const attrs = attributesFor(match[0]);
+    const rels = new Set(
+      (attrs.rel ?? "").toLowerCase().split(/\s+/).filter(Boolean),
+    );
+    if (rels.has(rel.toLowerCase()) && attrs.type === type)
+      return attrs.href ?? "";
+  }
+  return "";
+}
+
+function firstNonEmpty(...values: string[]): string {
+  return values.find((value) => value.trim())?.trim() ?? "";
+}
+
+function numberFromMeta(value: string): number {
+  const number = Number.parseInt(value, 10);
+  return Number.isFinite(number) ? number : 0;
+}
+
+function cleanHandle(input: string): string {
+  const trimmed = input.trim();
+  if (!trimmed) throw new Error("Threads handle cannot be empty");
+  if (/^https?:\/\//i.test(trimmed)) {
+    const url = new URL(trimmed);
+    const handle = url.pathname.split("/").find((part) => part.startsWith("@"));
+    if (!handle)
+      throw new Error(`Threads URL does not contain a handle: ${input}`);
+    return handle.replace(/^@/, "");
+  }
+  return trimmed.replace(/^@/, "").replace(/^\/+/, "").split("/")[0];
+}
+
+function handleFromUrl(input: string): string {
+  const url = new URL(input);
+  const handle = url.pathname.split("/").find((part) => part.startsWith("@"));
+  return handle ? handle : "";
+}
+
+function identityFromTitle(title: string, fallbackHandle: string): Identity {
+  const match = title.match(/^\s*(.*?)\s+\((@[A-Za-z0-9._]+)\)/);
+  const handle = match?.[2] ?? fallbackHandle;
+  const name = firstNonEmpty(match?.[1] ?? "", handle.replace(/^@/, ""));
+  return { name, handle };
+}
+
+function ensureThreadsUrl(url: URL): void {
+  if (!THREADS_HOST_RE.test(url.hostname)) {
+    throw new Error(
+      `Expected a threads.net or threads.com URL, got ${url.hostname}`,
+    );
+  }
+}
+
+export function normalizeThreadsProfileUrl(input: string): string {
+  const handle = cleanHandle(input);
+  return `https://www.threads.net/@${encodeURIComponent(handle)}`;
+}
+
+export function normalizeThreadsPostUrl(input: string): string {
+  const trimmed = input.trim();
+  if (!trimmed) throw new Error("Threads post URL cannot be empty");
+  const raw =
+    trimmed.startsWith("@") || trimmed.startsWith("/@")
+      ? `https://www.threads.net/${trimmed.replace(/^\/+/, "")}`
+      : trimmed;
+  const url = new URL(raw);
+  ensureThreadsUrl(url);
+  if (!/^\/@[^/]+\/post\/[^/?#]+/.test(url.pathname)) {
+    throw new Error(
+      `Expected Threads post URL path /@handle/post/shortcode, got ${url.pathname}`,
+    );
+  }
+  return url.toString();
+}
+
+function shortcodeFromPost(url: string, iosUrl: string): string {
+  const iosMatch = iosUrl.match(/[?&]shortcode=([^&]+)/);
+  if (iosMatch) return decodeURIComponent(iosMatch[1]);
+  const urlMatch = url.match(/\/post\/([^/?#]+)/);
+  return urlMatch ? decodeURIComponent(urlMatch[1]) : "";
+}
+
+export function parseThreadsPostHtml(
+  html: string,
+  fallbackUrl: string,
+): ThreadsPostRow {
+  const url = firstNonEmpty(metaContent(html, "og:url"), fallbackUrl);
+  const title = metaContent(html, "og:title");
+  const text = firstNonEmpty(
+    metaContent(html, "og:description"),
+    metaContent(html, "twitter:description"),
+  );
+  const imageUrl = metaContent(html, "og:image");
+  const fallbackHandle = handleFromUrl(url);
+  const identity = identityFromTitle(title, fallbackHandle);
+  if (!identity.handle || (!text && !imageUrl)) {
+    throw new Error("Threads post page did not expose usable public metadata");
+  }
+  return {
+    activity_json_url: linkHref(html, "alternate", "application/activity+json"),
+    author: identity.name,
+    handle: identity.handle,
+    image_height: numberFromMeta(metaContent(html, "og:image:height")),
+    image_url: imageUrl,
+    image_width: numberFromMeta(metaContent(html, "og:image:width")),
+    shortcode: shortcodeFromPost(url, metaContent(html, "al:ios:url")),
+    text,
+    url,
+  };
+}
+
+export function parseThreadsProfileHtml(
+  html: string,
+  handleInput: string,
+): ThreadsProfileRow {
+  const title = metaContent(html, "og:title");
+  const description = metaContent(html, "og:description");
+  const url = firstNonEmpty(
+    metaContent(html, "og:url"),
+    normalizeThreadsProfileUrl(handleInput),
+  );
+  const identity = identityFromTitle(title, `@${cleanHandle(handleInput)}`);
+  const parts = description
+    .split("•")
+    .map((part) => part.trim())
+    .filter(Boolean);
+  const followers =
+    parts
+      .find((part) => /\bFollowers\b/i.test(part))
+      ?.replace(/\s*Followers\b/i, "") ?? "";
+  const threads =
+    parts
+      .find((part) => /\bThreads\b/i.test(part))
+      ?.replace(/\s*Threads\b/i, "") ?? "";
+  const bio = firstNonEmpty(
+    ...parts.filter((part) => !/\b(Followers|Threads)\b/i.test(part)),
+  ).replace(/\s*See the latest conversations with @[^.]+\.?$/i, "");
+  if (!identity.handle || !identity.name) {
+    throw new Error(
+      "Threads profile page did not expose usable public metadata",
+    );
+  }
+  return {
+    avatar_url: metaContent(html, "og:image"),
+    bio,
+    followers,
+    handle: identity.handle,
+    name: identity.name,
+    threads,
+    url,
+  };
+}
+
+function mediaRowsFromPost(
+  post: ThreadsPostRow,
+): Array<Record<string, string | number>> {
+  if (!post.image_url) return [];
+  return [
+    {
+      rank: 1,
+      type: "image",
+      url: post.image_url,
+      width: post.image_width,
+      height: post.image_height,
+      post_url: post.url,
+      shortcode: post.shortcode,
+      source: "og:image",
+    },
+  ];
+}
+
+async function fetchThreadsHtml(url: string): Promise<string> {
+  const response = await fetch(url, {
+    headers: {
+      Accept: "text/html,application/xhtml+xml",
+      "User-Agent": USER_AGENT,
+    },
+  });
+  if (!response.ok) {
+    const preview = (await response.text()).slice(0, 200);
+    throw new Error(`Threads HTTP ${response.status} from ${url}: ${preview}`);
+  }
+  return response.text();
+}
+
+cli({
+  site: "threads",
+  name: "profile",
+  description: "Get public Threads profile metadata",
+  domain: "threads.net",
+  strategy: Strategy.PUBLIC,
+  args: [
+    {
+      name: "handle",
+      required: true,
+      positional: true,
+      description: "Threads handle or profile URL, e.g. zuck",
+    },
+  ],
+  columns: [
+    "name",
+    "handle",
+    "followers",
+    "threads",
+    "bio",
+    "url",
+    "avatar_url",
+  ],
+  socialCapabilities: ["read", "author", "relations"],
+  func: async (_page, kwargs) => {
+    const url = normalizeThreadsProfileUrl(String(kwargs.handle ?? ""));
+    const html = await fetchThreadsHtml(url);
+    return [parseThreadsProfileHtml(html, String(kwargs.handle ?? ""))];
+  },
+});
+
+cli({
+  site: "threads",
+  name: "post",
+  description: "Get a public Threads post from its metadata",
+  domain: "threads.net",
+  strategy: Strategy.PUBLIC,
+  args: [
+    {
+      name: "url",
+      required: true,
+      positional: true,
+      description:
+        "Threads post URL, e.g. https://www.threads.net/@zuck/post/DYSAIo_FL77",
+    },
+  ],
+  columns: [
+    "author",
+    "handle",
+    "text",
+    "url",
+    "image_url",
+    "image_width",
+    "image_height",
+    "shortcode",
+    "activity_json_url",
+  ],
+  socialCapabilities: ["read", "author", "media"],
+  func: async (_page, kwargs) => {
+    const url = normalizeThreadsPostUrl(String(kwargs.url ?? ""));
+    const html = await fetchThreadsHtml(url);
+    return [parseThreadsPostHtml(html, url)];
+  },
+});
+
+cli({
+  site: "threads",
+  name: "media",
+  description: "List public media exposed by a Threads post",
+  domain: "threads.net",
+  strategy: Strategy.PUBLIC,
+  args: [
+    {
+      name: "url",
+      required: true,
+      positional: true,
+      description:
+        "Threads post URL, e.g. https://www.threads.net/@zuck/post/DYSAIo_FL77",
+    },
+  ],
+  columns: [
+    "rank",
+    "type",
+    "url",
+    "width",
+    "height",
+    "post_url",
+    "shortcode",
+    "source",
+  ],
+  socialCapabilities: ["read", "media"],
+  func: async (_page, kwargs) => {
+    const url = normalizeThreadsPostUrl(String(kwargs.url ?? ""));
+    const html = await fetchThreadsHtml(url);
+    const rows = mediaRowsFromPost(parseThreadsPostHtml(html, url));
+    if (rows.length === 0)
+      throw new Error("Threads post metadata did not include media");
+    return rows;
+  },
+});

package/src/adapters/threads/user.yaml

@@ -0,0 +1,73 @@
+site: threads
+name: user
+description: Get recent posts from a Threads user profile
+domain: threads.net
+strategy: intercept
+lint_listing_detail: skip
+browser: true
+
+args:
+  handle:
+    type: str
+    required: true
+    positional: true
+    description: Threads handle without leading @
+  limit:
+    type: int
+    default: 20
+    description: Number of posts
+
+columns: [rank, author, text, likes, replies, url]
+
+pipeline:
+  - navigate:
+      url: https://www.threads.net/@${{ args.handle | urlencode }}
+      settleMs: 4000
+
+  - scroll:
+      times: 2
+
+  - evaluate: |
+      (() => {
+        const limit = ${{ args.limit }};
+        const handle = ${{ args.handle | json }};
+        const clean = value => (value || '').replace(/\s+/g, ' ').trim();
+        const absolute = href => href?.startsWith('http') ? href : (href ? `https://www.threads.net${href}` : '');
+        const cleanPostText = value => {
+          const text = clean(value);
+          if (text === handle || text === `@${handle}`) return '';
+          if (/^(back|home|search|create|notifications|profile|more|like|reply|repost|share)$/i.test(text)) return '';
+          if (/返回|首页|搜索|创建|通知|主页|更多/.test(text)) return '';
+          return text;
+        };
+        const cards = Array.from(document.querySelectorAll('[data-pressable-container="true"], article, [class*="FeedItem"]'));
+        const rows = cards.slice(0, limit).map((post, index) => {
+          const authorEl = post.querySelector('a[href*="/@"] span, a[href*="/@"]');
+          const textEl = post.querySelector('[class*="text"], [class*="caption"], [dir="auto"]');
+          const likeEl = post.querySelector('[aria-label*="like" i], [class*="like" i]');
+          const replyEl = post.querySelector('[aria-label*="repl" i], [class*="reply" i]');
+          const linkEl = post.querySelector('a[href*="/post/"]');
+          return {
+            rank: index + 1,
+            author: clean(authorEl?.textContent) || `@${handle}`,
+            text: cleanPostText(textEl?.textContent).slice(0, 500),
+            likes: clean(likeEl?.textContent),
+            replies: clean(replyEl?.textContent),
+            url: absolute(linkEl?.getAttribute('href')),
+          };
+        }).filter(row => row.text || row.url);
+
+        if (rows.length === 0) {
+          throw new Error('threads user: no posts found; login, empty profile, or selector drift may be blocking extraction');
+        }
+        return rows;
+      })()
+
+# schema-v2 metadata — injected by `unicli migrate schema-v2`
+capabilities:
+  ["cdp-browser.evaluate", "cdp-browser.navigate", "cdp-browser.scroll"]
+minimum_capability: cdp-browser.evaluate
+trust: public
+confidentiality: public
+quarantine: false
+schema_version: v2