@zenalexa/unicli 0.220.1 → 0.221.1

package/src/adapters/threads/post.ts

@@ -0,0 +1,366 @@
+/**
+ * @owner   Threads public page metadata adapter.
+ * @does    Register public profile, post, and media readers backed by Threads OG metadata.
+ * @needs   Threads public HTML pages exposing Open Graph metadata.
+ * @feeds   Social research workflows and Threads capability coverage.
+ * @breaks  Threads hiding public metadata or changing canonical post/profile markup.
+ */
+
+import { USER_AGENT } from "../../constants.js";
+import { cli, Strategy } from "../../registry.js";
+
+export interface ThreadsPostRow {
+  author: string;
+  handle: string;
+  text: string;
+  url: string;
+  image_url: string;
+  image_width: number;
+  image_height: number;
+  shortcode: string;
+  activity_json_url: string;
+}
+
+export interface ThreadsProfileRow {
+  name: string;
+  handle: string;
+  followers: string;
+  threads: string;
+  bio: string;
+  url: string;
+  avatar_url: string;
+}
+
+interface Identity {
+  name: string;
+  handle: string;
+}
+
+const THREADS_HOST_RE = /(^|\.)threads\.(net|com)$/i;
+
+function decodeHtml(value: string): string {
+  return value
+    .replace(/&#x([0-9a-f]+);/gi, (_, code: string) =>
+      String.fromCodePoint(Number.parseInt(code, 16)),
+    )
+    .replace(/&#(\d+);/g, (_, code: string) =>
+      String.fromCodePoint(Number.parseInt(code, 10)),
+    )
+    .replace(/ /g, " ")
+    .replace(/&/g, "&")
+    .replace(/&lt;/g, "<")
+    .replace(/&gt;/g, ">")
+    .replace(/&quot;/g, '"')
+    .replace(/&apos;/g, "'")
+    .replace(/&#039;/g, "'")
+    .replace(/\s+/g, " ")
+    .trim();
+}
+
+function attributesFor(tag: string): Record<string, string> {
+  const attrs: Record<string, string> = {};
+  const attrRe = /\s([^\s"'=<>`]+)\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s>]+))/g;
+  for (const match of tag.matchAll(attrRe)) {
+    attrs[match[1].toLowerCase()] = decodeHtml(
+      match[2] ?? match[3] ?? match[4] ?? "",
+    );
+  }
+  return attrs;
+}
+
+function metaContent(html: string, key: string): string {
+  for (const match of html.matchAll(/<meta\b[^>]*>/gi)) {
+    const attrs = attributesFor(match[0]);
+    if (attrs.property === key || attrs.name === key)
+      return attrs.content ?? "";
+  }
+  return "";
+}
+
+function linkHref(html: string, rel: string, type: string): string {
+  for (const match of html.matchAll(/<link\b[^>]*>/gi)) {
+    const attrs = attributesFor(match[0]);
+    const rels = new Set(
+      (attrs.rel ?? "").toLowerCase().split(/\s+/).filter(Boolean),
+    );
+    if (rels.has(rel.toLowerCase()) && attrs.type === type)
+      return attrs.href ?? "";
+  }
+  return "";
+}
+
+function firstNonEmpty(...values: string[]): string {
+  return values.find((value) => value.trim())?.trim() ?? "";
+}
+
+function numberFromMeta(value: string): number {
+  const number = Number.parseInt(value, 10);
+  return Number.isFinite(number) ? number : 0;
+}
+
+function cleanHandle(input: string): string {
+  const trimmed = input.trim();
+  if (!trimmed) throw new Error("Threads handle cannot be empty");
+  if (/^https?:\/\//i.test(trimmed)) {
+    const url = new URL(trimmed);
+    const handle = url.pathname.split("/").find((part) => part.startsWith("@"));
+    if (!handle)
+      throw new Error(`Threads URL does not contain a handle: ${input}`);
+    return handle.replace(/^@/, "");
+  }
+  return trimmed.replace(/^@/, "").replace(/^\/+/, "").split("/")[0];
+}
+
+function handleFromUrl(input: string): string {
+  const url = new URL(input);
+  const handle = url.pathname.split("/").find((part) => part.startsWith("@"));
+  return handle ? handle : "";
+}
+
+function identityFromTitle(title: string, fallbackHandle: string): Identity {
+  const match = title.match(/^\s*(.*?)\s+\((@[A-Za-z0-9._]+)\)/);
+  const handle = match?.[2] ?? fallbackHandle;
+  const name = firstNonEmpty(match?.[1] ?? "", handle.replace(/^@/, ""));
+  return { name, handle };
+}
+
+function ensureThreadsUrl(url: URL): void {
+  if (!THREADS_HOST_RE.test(url.hostname)) {
+    throw new Error(
+      `Expected a threads.net or threads.com URL, got ${url.hostname}`,
+    );
+  }
+}
+
+export function normalizeThreadsProfileUrl(input: string): string {
+  const handle = cleanHandle(input);
+  return `https://www.threads.net/@${encodeURIComponent(handle)}`;
+}
+
+export function normalizeThreadsPostUrl(input: string): string {
+  const trimmed = input.trim();
+  if (!trimmed) throw new Error("Threads post URL cannot be empty");
+  const raw =
+    trimmed.startsWith("@") || trimmed.startsWith("/@")
+      ? `https://www.threads.net/${trimmed.replace(/^\/+/, "")}`
+      : trimmed;
+  const url = new URL(raw);
+  ensureThreadsUrl(url);
+  if (!/^\/@[^/]+\/post\/[^/?#]+/.test(url.pathname)) {
+    throw new Error(
+      `Expected Threads post URL path /@handle/post/shortcode, got ${url.pathname}`,
+    );
+  }
+  return url.toString();
+}
+
+function shortcodeFromPost(url: string, iosUrl: string): string {
+  const iosMatch = iosUrl.match(/[?&]shortcode=([^&]+)/);
+  if (iosMatch) return decodeURIComponent(iosMatch[1]);
+  const urlMatch = url.match(/\/post\/([^/?#]+)/);
+  return urlMatch ? decodeURIComponent(urlMatch[1]) : "";
+}
+
+export function parseThreadsPostHtml(
+  html: string,
+  fallbackUrl: string,
+): ThreadsPostRow {
+  const url = firstNonEmpty(metaContent(html, "og:url"), fallbackUrl);
+  const title = metaContent(html, "og:title");
+  const text = firstNonEmpty(
+    metaContent(html, "og:description"),
+    metaContent(html, "twitter:description"),
+  );
+  const imageUrl = metaContent(html, "og:image");
+  const fallbackHandle = handleFromUrl(url);
+  const identity = identityFromTitle(title, fallbackHandle);
+  if (!identity.handle || (!text && !imageUrl)) {
+    throw new Error("Threads post page did not expose usable public metadata");
+  }
+  return {
+    activity_json_url: linkHref(html, "alternate", "application/activity+json"),
+    author: identity.name,
+    handle: identity.handle,
+    image_height: numberFromMeta(metaContent(html, "og:image:height")),
+    image_url: imageUrl,
+    image_width: numberFromMeta(metaContent(html, "og:image:width")),
+    shortcode: shortcodeFromPost(url, metaContent(html, "al:ios:url")),
+    text,
+    url,
+  };
+}
+
+export function parseThreadsProfileHtml(
+  html: string,
+  handleInput: string,
+): ThreadsProfileRow {
+  const title = metaContent(html, "og:title");
+  const description = metaContent(html, "og:description");
+  const url = firstNonEmpty(
+    metaContent(html, "og:url"),
+    normalizeThreadsProfileUrl(handleInput),
+  );
+  const identity = identityFromTitle(title, `@${cleanHandle(handleInput)}`);
+  const parts = description
+    .split("•")
+    .map((part) => part.trim())
+    .filter(Boolean);
+  const followers =
+    parts
+      .find((part) => /\bFollowers\b/i.test(part))
+      ?.replace(/\s*Followers\b/i, "") ?? "";
+  const threads =
+    parts
+      .find((part) => /\bThreads\b/i.test(part))
+      ?.replace(/\s*Threads\b/i, "") ?? "";
+  const bio = firstNonEmpty(
+    ...parts.filter((part) => !/\b(Followers|Threads)\b/i.test(part)),
+  ).replace(/\s*See the latest conversations with @[^.]+\.?$/i, "");
+  if (!identity.handle || !identity.name) {
+    throw new Error(
+      "Threads profile page did not expose usable public metadata",
+    );
+  }
+  return {
+    avatar_url: metaContent(html, "og:image"),
+    bio,
+    followers,
+    handle: identity.handle,
+    name: identity.name,
+    threads,
+    url,
+  };
+}
+
+function mediaRowsFromPost(
+  post: ThreadsPostRow,
+): Array<Record<string, string | number>> {
+  if (!post.image_url) return [];
+  return [
+    {
+      rank: 1,
+      type: "image",
+      url: post.image_url,
+      width: post.image_width,
+      height: post.image_height,
+      post_url: post.url,
+      shortcode: post.shortcode,
+      source: "og:image",
+    },
+  ];
+}
+
+async function fetchThreadsHtml(url: string): Promise<string> {
+  const response = await fetch(url, {
+    headers: {
+      Accept: "text/html,application/xhtml+xml",
+      "User-Agent": USER_AGENT,
+    },
+  });
+  if (!response.ok) {
+    const preview = (await response.text()).slice(0, 200);
+    throw new Error(`Threads HTTP ${response.status} from ${url}: ${preview}`);
+  }
+  return response.text();
+}
+
+cli({
+  site: "threads",
+  name: "profile",
+  description: "Get public Threads profile metadata",
+  domain: "threads.net",
+  strategy: Strategy.PUBLIC,
+  args: [
+    {
+      name: "handle",
+      required: true,
+      positional: true,
+      description: "Threads handle or profile URL, e.g. zuck",
+    },
+  ],
+  columns: [
+    "name",
+    "handle",
+    "followers",
+    "threads",
+    "bio",
+    "url",
+    "avatar_url",
+  ],
+  socialCapabilities: ["read", "author", "relations"],
+  func: async (_page, kwargs) => {
+    const url = normalizeThreadsProfileUrl(String(kwargs.handle ?? ""));
+    const html = await fetchThreadsHtml(url);
+    return [parseThreadsProfileHtml(html, String(kwargs.handle ?? ""))];
+  },
+});
+
+cli({
+  site: "threads",
+  name: "post",
+  description: "Get a public Threads post from its metadata",
+  domain: "threads.net",
+  strategy: Strategy.PUBLIC,
+  args: [
+    {
+      name: "url",
+      required: true,
+      positional: true,
+      description:
+        "Threads post URL, e.g. https://www.threads.net/@zuck/post/DYSAIo_FL77",
+    },
+  ],
+  columns: [
+    "author",
+    "handle",
+    "text",
+    "url",
+    "image_url",
+    "image_width",
+    "image_height",
+    "shortcode",
+    "activity_json_url",
+  ],
+  socialCapabilities: ["read", "author", "media"],
+  func: async (_page, kwargs) => {
+    const url = normalizeThreadsPostUrl(String(kwargs.url ?? ""));
+    const html = await fetchThreadsHtml(url);
+    return [parseThreadsPostHtml(html, url)];
+  },
+});
+
+cli({
+  site: "threads",
+  name: "media",
+  description: "List public media exposed by a Threads post",
+  domain: "threads.net",
+  strategy: Strategy.PUBLIC,
+  args: [
+    {
+      name: "url",
+      required: true,
+      positional: true,
+      description:
+        "Threads post URL, e.g. https://www.threads.net/@zuck/post/DYSAIo_FL77",
+    },
+  ],
+  columns: [
+    "rank",
+    "type",
+    "url",
+    "width",
+    "height",
+    "post_url",
+    "shortcode",
+    "source",
+  ],
+  socialCapabilities: ["read", "media"],
+  func: async (_page, kwargs) => {
+    const url = normalizeThreadsPostUrl(String(kwargs.url ?? ""));
+    const html = await fetchThreadsHtml(url);
+    const rows = mediaRowsFromPost(parseThreadsPostHtml(html, url));
+    if (rows.length === 0)
+      throw new Error("Threads post metadata did not include media");
+    return rows;
+  },
+});

package/src/adapters/threads/user.yaml

@@ -0,0 +1,73 @@
+site: threads
+name: user
+description: Get recent posts from a Threads user profile
+domain: threads.net
+strategy: intercept
+lint_listing_detail: skip
+browser: true
+
+args:
+  handle:
+    type: str
+    required: true
+    positional: true
+    description: Threads handle without leading @
+  limit:
+    type: int
+    default: 20
+    description: Number of posts
+
+columns: [rank, author, text, likes, replies, url]
+
+pipeline:
+  - navigate:
+      url: https://www.threads.net/@${{ args.handle | urlencode }}
+      settleMs: 4000
+
+  - scroll:
+      times: 2
+
+  - evaluate: |
+      (() => {
+        const limit = ${{ args.limit }};
+        const handle = ${{ args.handle | json }};
+        const clean = value => (value || '').replace(/\s+/g, ' ').trim();
+        const absolute = href => href?.startsWith('http') ? href : (href ? `https://www.threads.net${href}` : '');
+        const cleanPostText = value => {
+          const text = clean(value);
+          if (text === handle || text === `@${handle}`) return '';
+          if (/^(back|home|search|create|notifications|profile|more|like|reply|repost|share)$/i.test(text)) return '';
+          if (/返回|首页|搜索|创建|通知|主页|更多/.test(text)) return '';
+          return text;
+        };
+        const cards = Array.from(document.querySelectorAll('[data-pressable-container="true"], article, [class*="FeedItem"]'));
+        const rows = cards.slice(0, limit).map((post, index) => {
+          const authorEl = post.querySelector('a[href*="/@"] span, a[href*="/@"]');
+          const textEl = post.querySelector('[class*="text"], [class*="caption"], [dir="auto"]');
+          const likeEl = post.querySelector('[aria-label*="like" i], [class*="like" i]');
+          const replyEl = post.querySelector('[aria-label*="repl" i], [class*="reply" i]');
+          const linkEl = post.querySelector('a[href*="/post/"]');
+          return {
+            rank: index + 1,
+            author: clean(authorEl?.textContent) || `@${handle}`,
+            text: cleanPostText(textEl?.textContent).slice(0, 500),
+            likes: clean(likeEl?.textContent),
+            replies: clean(replyEl?.textContent),
+            url: absolute(linkEl?.getAttribute('href')),
+          };
+        }).filter(row => row.text || row.url);
+
+        if (rows.length === 0) {
+          throw new Error('threads user: no posts found; login, empty profile, or selector drift may be blocking extraction');
+        }
+        return rows;
+      })()
+
+# schema-v2 metadata — injected by `unicli migrate schema-v2`
+capabilities:
+  ["cdp-browser.evaluate", "cdp-browser.navigate", "cdp-browser.scroll"]
+minimum_capability: cdp-browser.evaluate
+trust: public
+confidentiality: public
+quarantine: false
+schema_version: v2

package/src/adapters/tiktok/subtitles.ts

@@ -0,0 +1,44 @@
+/**
+ * @owner   TikTok subtitle adapter.
+ * @does    Extracts LLM-readable subtitle text from public TikTok video URLs.
+ * @needs   `yt-dlp` on PATH and an accessible video subtitle track.
+ * @feeds   Short-video search-to-transcript agent workflows.
+ * @breaks  TikTok or yt-dlp extractor changes can alter subtitle availability.
+ */
+
+import { cli, Strategy } from "../../registry.js";
+import { runVideoSubtitleExtraction } from "../../social/video-text.js";
+
+cli({
+  site: "tiktok",
+  name: "subtitles",
+  description:
+    "Extract LLM-readable subtitles from a TikTok video URL with optional browser-cookie reuse",
+  domain: "www.tiktok.com",
+  strategy: Strategy.PUBLIC,
+  args: [
+    {
+      name: "url",
+      type: "str",
+      required: true,
+      positional: true,
+      description: "TikTok video URL",
+      format: "uri",
+    },
+    {
+      name: "languages",
+      type: "str",
+      default: "zh-Hans,zh,en",
+      description: "Comma-separated subtitle languages",
+    },
+    {
+      name: "cookies-from-browser",
+      type: "str",
+      description: "Browser name for yt-dlp --cookies-from-browser",
+    },
+  ],
+  columns: ["language", "text", "path"],
+  socialCapabilities: ["read", "media", "subtitles"],
+  defaultFormat: "json",
+  func: async (_page, kwargs) => runVideoSubtitleExtraction(kwargs),
+});

package/src/adapters/twitter/accept.ts

@@ -6,7 +6,10 @@
 
 import { cli } from "../../registry.js";
 import { Strategy } from "../../types.js";
-import { loadCookies, formatCookieHeader } from "../../engine/cookies.js";
+import {
+  loadCookiesWithCDP,
+  formatCookieHeader,
+} from "../../engine/cookies.js";
 import { USER_AGENT } from "../../constants.js";
 
 const BEARER_TOKEN =
@@ -30,7 +33,7 @@ cli({
   func: async (_page, kwargs) => {
     const userId = String(kwargs.user_id);
 
-    const cookies = loadCookies("twitter");
+    const cookies = await loadCookiesWithCDP("twitter", "x.com");
     if (!cookies) {
       throw new Error(
         'No cookies found for "twitter". Run: unicli auth setup twitter',

package/src/adapters/twitter/browser-fallback.ts

@@ -0,0 +1,138 @@
+/**
+ * @owner   Twitter browser fallback extraction.
+ * @does    Reads X/Twitter search and trend results from the logged-in web UI.
+ * @needs   A user-owned Chrome session reachable by Uni-CLI's browser layer.
+ * @feeds   twitter.search and twitter.trending.
+ * @breaks  X/Twitter DOM changes can reduce parseable rows.
+ */
+
+import type { IPage } from "../../types.js";
+import { socialEmptyError } from "../../social/browser-errors.js";
+import { assertTwitterReadable, gotoTwitterPage } from "./browser-state.js";
+
+export interface TwitterTweetRow {
+  id: string;
+  author: string;
+  text: string;
+  likes: string;
+  retweets: string;
+  views: string;
+  url: string;
+}
+
+export interface TwitterTrendRow {
+  name: string;
+  tweet_count: string;
+  description: string;
+  url: string;
+}
+
+export async function browserSearchTweets(
+  page: IPage,
+  query: string,
+  limit: number,
+): Promise<TwitterTweetRow[]> {
+  await gotoTwitterPage(
+    page,
+    `https://x.com/search?q=${encodeURIComponent(query)}&src=typed_query&f=live`,
+    "search",
+  );
+  await page.autoScroll({ maxScrolls: 2, delay: 1000 });
+  await assertTwitterReadable(page, "search");
+
+  const raw = await page.evaluate(`
+    (() => {
+      const clean = (value) => (value || '').replace(/\\s+/g, ' ').trim();
+      const rows = [];
+      const seen = new Set();
+      for (const article of document.querySelectorAll('article[data-testid="tweet"]')) {
+        const status = article.querySelector('a[href*="/status/"]');
+        const href = status?.getAttribute('href') || '';
+        const id = (href.match(/\\/status\\/(\\d+)/) || [])[1] || '';
+        if (!id || seen.has(id)) continue;
+        seen.add(id);
+        const text = Array.from(article.querySelectorAll('[data-testid="tweetText"]'))
+          .map((el) => clean(el.textContent || ''))
+          .filter(Boolean)
+          .join('\\n');
+        if (!text) continue;
+        const userName = clean(article.querySelector('[data-testid="User-Name"]')?.textContent || '');
+        const author = userName.split('@')[0] || userName;
+        const metric = (name) => clean(article.querySelector('[data-testid="' + name + '"]')?.textContent || '');
+        rows.push({
+          id,
+          author,
+          text,
+          likes: metric('like'),
+          retweets: metric('retweet'),
+          views: clean(article.querySelector('a[href$="/analytics"]')?.textContent || ''),
+          url: 'https://x.com' + href.split('?')[0],
+        });
+      }
+      return rows;
+    })()
+  `);
+
+  const rows = Array.isArray(raw)
+    ? (raw as TwitterTweetRow[]).slice(0, limit)
+    : [];
+  if (rows.length > 0) return rows;
+  throw socialEmptyError(
+    "twitter",
+    "search",
+    `Twitter/X search loaded no parseable tweets for "${query}".`,
+  );
+}
+
+export async function browserTrendingTopics(
+  page: IPage,
+  limit: number,
+): Promise<TwitterTrendRow[]> {
+  await gotoTwitterPage(
+    page,
+    "https://x.com/explore/tabs/trending",
+    "trending",
+  );
+
+  const raw = await page.evaluate(`
+    (() => {
+      const clean = (value) => (value || '').replace(/\\s+/g, ' ').trim();
+      const rows = [];
+      const seen = new Set();
+      for (const link of document.querySelectorAll('a[href*="/search?q="]')) {
+        const href = link.getAttribute('href') || '';
+        let name = '';
+        try {
+          name = new URL(href, 'https://x.com').searchParams.get('q') || '';
+        } catch {}
+        const lines = clean(link.textContent || '')
+          .split(/(?=Trending|[0-9,.]+\\s+posts)/)
+          .map(clean)
+          .filter(Boolean);
+        if (!name) {
+          name = lines.find((line) => !/^Trending/i.test(line) && !/posts$/i.test(line)) || '';
+        }
+        if (!name || seen.has(name)) continue;
+        seen.add(name);
+        const meta = lines.find((line) => /posts$/i.test(line)) || '';
+        rows.push({
+          name,
+          tweet_count: meta,
+          description: lines.find((line) => /^Trending/i.test(line)) || '',
+          url: 'https://x.com' + href,
+        });
+      }
+      return rows;
+    })()
+  `);
+
+  const rows = Array.isArray(raw)
+    ? (raw as TwitterTrendRow[]).slice(0, limit)
+    : [];
+  if (rows.length > 0) return rows;
+  throw socialEmptyError(
+    "twitter",
+    "trending",
+    "Twitter/X explore loaded no parseable trend rows.",
+  );
+}