@zenalexa/unicli 0.220.0 → 0.221.0

package/src/adapters/twitter/browser-fallback.ts

@@ -0,0 +1,138 @@
+/**
+ * @owner   Twitter browser fallback extraction.
+ * @does    Reads X/Twitter search and trend results from the logged-in web UI.
+ * @needs   A user-owned Chrome session reachable by Uni-CLI's browser layer.
+ * @feeds   twitter.search and twitter.trending.
+ * @breaks  X/Twitter DOM changes can reduce parseable rows.
+ */
+
+import type { IPage } from "../../types.js";
+import { socialEmptyError } from "../../social/browser-errors.js";
+import { assertTwitterReadable, gotoTwitterPage } from "./browser-state.js";
+
+export interface TwitterTweetRow {
+  id: string;
+  author: string;
+  text: string;
+  likes: string;
+  retweets: string;
+  views: string;
+  url: string;
+}
+
+export interface TwitterTrendRow {
+  name: string;
+  tweet_count: string;
+  description: string;
+  url: string;
+}
+
+export async function browserSearchTweets(
+  page: IPage,
+  query: string,
+  limit: number,
+): Promise<TwitterTweetRow[]> {
+  await gotoTwitterPage(
+    page,
+    `https://x.com/search?q=${encodeURIComponent(query)}&src=typed_query&f=live`,
+    "search",
+  );
+  await page.autoScroll({ maxScrolls: 2, delay: 1000 });
+  await assertTwitterReadable(page, "search");
+
+  const raw = await page.evaluate(`
+    (() => {
+      const clean = (value) => (value || '').replace(/\\s+/g, ' ').trim();
+      const rows = [];
+      const seen = new Set();
+      for (const article of document.querySelectorAll('article[data-testid="tweet"]')) {
+        const status = article.querySelector('a[href*="/status/"]');
+        const href = status?.getAttribute('href') || '';
+        const id = (href.match(/\\/status\\/(\\d+)/) || [])[1] || '';
+        if (!id || seen.has(id)) continue;
+        seen.add(id);
+        const text = Array.from(article.querySelectorAll('[data-testid="tweetText"]'))
+          .map((el) => clean(el.textContent || ''))
+          .filter(Boolean)
+          .join('\\n');
+        if (!text) continue;
+        const userName = clean(article.querySelector('[data-testid="User-Name"]')?.textContent || '');
+        const author = userName.split('@')[0] || userName;
+        const metric = (name) => clean(article.querySelector('[data-testid="' + name + '"]')?.textContent || '');
+        rows.push({
+          id,
+          author,
+          text,
+          likes: metric('like'),
+          retweets: metric('retweet'),
+          views: clean(article.querySelector('a[href$="/analytics"]')?.textContent || ''),
+          url: 'https://x.com' + href.split('?')[0],
+        });
+      }
+      return rows;
+    })()
+  `);
+
+  const rows = Array.isArray(raw)
+    ? (raw as TwitterTweetRow[]).slice(0, limit)
+    : [];
+  if (rows.length > 0) return rows;
+  throw socialEmptyError(
+    "twitter",
+    "search",
+    `Twitter/X search loaded no parseable tweets for "${query}".`,
+  );
+}
+
+export async function browserTrendingTopics(
+  page: IPage,
+  limit: number,
+): Promise<TwitterTrendRow[]> {
+  await gotoTwitterPage(
+    page,
+    "https://x.com/explore/tabs/trending",
+    "trending",
+  );
+
+  const raw = await page.evaluate(`
+    (() => {
+      const clean = (value) => (value || '').replace(/\\s+/g, ' ').trim();
+      const rows = [];
+      const seen = new Set();
+      for (const link of document.querySelectorAll('a[href*="/search?q="]')) {
+        const href = link.getAttribute('href') || '';
+        let name = '';
+        try {
+          name = new URL(href, 'https://x.com').searchParams.get('q') || '';
+        } catch {}
+        const lines = clean(link.textContent || '')
+          .split(/(?=Trending|[0-9,.]+\\s+posts)/)
+          .map(clean)
+          .filter(Boolean);
+        if (!name) {
+          name = lines.find((line) => !/^Trending/i.test(line) && !/posts$/i.test(line)) || '';
+        }
+        if (!name || seen.has(name)) continue;
+        seen.add(name);
+        const meta = lines.find((line) => /posts$/i.test(line)) || '';
+        rows.push({
+          name,
+          tweet_count: meta,
+          description: lines.find((line) => /^Trending/i.test(line)) || '',
+          url: 'https://x.com' + href,
+        });
+      }
+      return rows;
+    })()
+  `);
+
+  const rows = Array.isArray(raw)
+    ? (raw as TwitterTrendRow[]).slice(0, limit)
+    : [];
+  if (rows.length > 0) return rows;
+  throw socialEmptyError(
+    "twitter",
+    "trending",
+    "Twitter/X explore loaded no parseable trend rows.",
+  );
+}

package/src/adapters/twitter/browser-state.ts

@@ -0,0 +1,74 @@
+/**
+ * @owner   Twitter browser adapters.
+ * @does    Detects X/Twitter login and challenge pages in browser fallback flows.
+ * @needs   Browser-backed IPage from Uni-CLI runtime.
+ * @feeds   twitter.search and twitter.trending.
+ * @breaks  X/Twitter copy or route changes can require updating page-state detection.
+ */
+
+import type { IPage } from "../../types.js";
+import {
+  socialAuthError,
+  socialChallengeError,
+} from "../../social/browser-errors.js";
+
+interface TwitterPageState {
+  url: string;
+  title: string;
+  text: string;
+}
+
+async function readTwitterPageState(page: IPage): Promise<TwitterPageState> {
+  const raw = await page.evaluate(`
+    (() => ({
+      url: window.location.href,
+      title: document.title || '',
+      text: (document.body?.innerText || '').replace(/\\s+/g, ' ').slice(0, 2000)
+    }))()
+  `);
+  const state = raw as Partial<TwitterPageState>;
+  return {
+    url: String(state.url ?? ""),
+    title: String(state.title ?? ""),
+    text: String(state.text ?? ""),
+  };
+}
+
+export async function gotoTwitterPage(
+  page: IPage,
+  url: string,
+  command: string,
+): Promise<void> {
+  try {
+    await page.goto(url, { settleMs: 2500 });
+  } catch (err) {
+    const message = err instanceof Error ? err.message : String(err);
+    if (!/net::ERR_ABORTED/i.test(message)) throw err;
+  }
+  await page.wait(2);
+  await assertTwitterReadable(page, command);
+}
+
+export async function assertTwitterReadable(
+  page: IPage,
+  command: string,
+): Promise<void> {
+  const state = await readTwitterPageState(page);
+  const haystack = `${state.url}\n${state.title}\n${state.text}`;
+  if (
+    /captcha|cloudflare|challenge|verify you are human|unusual traffic/i.test(
+      haystack,
+    )
+  ) {
+    throw socialChallengeError(
+      "twitter",
+      command,
+      `Twitter/X is showing a challenge page: ${state.title || state.url}`,
+    );
+  }
+  if (
+    /\/i\/flow\/login|Sign in to X|Log in to X|登录 X|登录后/i.test(haystack)
+  ) {
+    throw socialAuthError("twitter", command);
+  }
+}

package/src/adapters/twitter/client.ts

@@ -5,7 +5,10 @@
  * Requires ct0 (CSRF token) and auth_token cookies in ~/.unicli/cookies/twitter.json
  */
 
-import { loadCookies, formatCookieHeader } from "../../engine/cookies.js";
+import {
+  loadCookiesWithCDP,
+  formatCookieHeader,
+} from "../../engine/cookies.js";
 import { USER_AGENT } from "../../constants.js";
 
 // Public bearer token — same for all Twitter web clients, not a secret
@@ -14,6 +17,45 @@ const BEARER_TOKEN =
 
 const GRAPHQL_BASE = "https://x.com/i/api/graphql";
 
+function throwTwitterApiError(
+  label: string,
+  status: number,
+  preview: string,
+): never {
+  const err = new Error(
+    `Twitter API error: HTTP ${status} on ${label}\n${preview.slice(0, 200)}`,
+  ) as Error & {
+    code?: string;
+    suggestion?: string;
+    retryable?: boolean;
+    alternatives?: string[];
+  };
+  err.code =
+    status === 401 || status === 403
+      ? "auth_required"
+      : status === 404
+        ? "upstream_error"
+        : status === 429
+          ? "rate_limited"
+          : status >= 500
+            ? "upstream_error"
+            : "api_error";
+  err.retryable = status === 429 || status >= 500;
+  err.suggestion =
+    status === 404
+      ? "Twitter/X changed or removed this web API operation. Run `unicli repair twitter <command>` or use a browser-backed command while the GraphQL operation is refreshed."
+      : status === 401 || status === 403
+        ? "Refresh X login state with `unicli --auth-retry twitter <command> --args-file <path.json>`, or open https://x.com in Chrome and sign in."
+        : status === 429
+          ? "Twitter/X rate-limited the request. Wait, reduce limit, then retry."
+          : "Twitter/X API returned an upstream error. Retry once; if it persists, run `unicli repair twitter <command>`.";
+  err.alternatives = [
+    "unicli auth import twitter --domain x.com",
+    "unicli browser open https://x.com",
+  ];
+  throw err;
+}
+
 /** Standard Twitter GraphQL feature flags */
 export const FEATURES: Record<string, boolean> = {
   rweb_tipjar_consumption_enabled: true,
@@ -56,7 +98,7 @@ export async function twitterFetch(
   variables: Record<string, unknown>,
   features: Record<string, boolean> = FEATURES,
 ): Promise<unknown> {
-  const cookies = loadCookies("twitter");
+  const cookies = await loadCookiesWithCDP("twitter", "x.com");
   if (!cookies) {
     throw new Error(
       'No cookies found for "twitter". Run: unicli auth setup twitter',
@@ -92,10 +134,7 @@ export async function twitterFetch(
 
   if (!resp.ok) {
     const preview = await resp.text().catch(() => "");
-    throw new Error(
-      `Twitter API error: HTTP ${resp.status} on ${endpoint}\n` +
-        `${preview.slice(0, 200)}`,
-    );
+    throwTwitterApiError(endpoint, resp.status, preview);
   }
 
   return resp.json();
@@ -110,7 +149,7 @@ export async function twitterPostFetch(
   variables: Record<string, unknown>,
   features: Record<string, boolean> = FEATURES,
 ): Promise<unknown> {
-  const cookies = loadCookies("twitter");
+  const cookies = await loadCookiesWithCDP("twitter", "x.com");
   if (!cookies) {
     throw new Error(
       'No cookies found for "twitter". Run: unicli auth setup twitter',
@@ -143,10 +182,7 @@ export async function twitterPostFetch(
 
   if (!resp.ok) {
     const preview = await resp.text().catch(() => "");
-    throw new Error(
-      `Twitter API error: HTTP ${resp.status} on ${endpoint}\n` +
-        `${preview.slice(0, 200)}`,
-    );
+    throwTwitterApiError(endpoint, resp.status, preview);
   }
 
   return resp.json();
@@ -163,7 +199,7 @@ export async function twitterRestFetch(
   path: string,
   params: Record<string, string> = {},
 ): Promise<unknown> {
-  const cookies = loadCookies("twitter");
+  const cookies = await loadCookiesWithCDP("twitter", "x.com");
   if (!cookies) {
     throw new Error(
       'No cookies found for "twitter". Run: unicli auth setup twitter',
@@ -195,10 +231,7 @@ export async function twitterRestFetch(
 
   if (!resp.ok) {
     const preview = await resp.text().catch(() => "");
-    throw new Error(
-      `Twitter REST API error: HTTP ${resp.status} on ${path}\n` +
-        `${preview.slice(0, 200)}`,
-    );
+    throwTwitterApiError(path, resp.status, preview);
   }
 
   return resp.json();
@@ -215,7 +248,7 @@ export async function twitterGuideFetch(
   url: string,
   params: Record<string, string> = {},
 ): Promise<unknown> {
-  const cookies = loadCookies("twitter");
+  const cookies = await loadCookiesWithCDP("twitter", "x.com");
   if (!cookies) {
     throw new Error(
       'No cookies found for "twitter". Run: unicli auth setup twitter',
@@ -247,10 +280,7 @@ export async function twitterGuideFetch(
 
   if (!resp.ok) {
     const preview = await resp.text().catch(() => "");
-    throw new Error(
-      `Twitter Guide API error: HTTP ${resp.status}\n` +
-        `${preview.slice(0, 200)}`,
-    );
+    throwTwitterApiError("guide", resp.status, preview);
   }
 
   return resp.json();

package/src/adapters/twitter/reply-dm.ts

@@ -6,7 +6,10 @@
 
 import { cli } from "../../registry.js";
 import { Strategy } from "../../types.js";
-import { loadCookies, formatCookieHeader } from "../../engine/cookies.js";
+import {
+  loadCookiesWithCDP,
+  formatCookieHeader,
+} from "../../engine/cookies.js";
 import { USER_AGENT } from "../../constants.js";
 
 const BEARER_TOKEN =
@@ -36,7 +39,7 @@ cli({
     const conversationId = String(kwargs.conversation_id);
     const text = kwargs.text as string;
 
-    const cookies = loadCookies("twitter");
+    const cookies = await loadCookiesWithCDP("twitter", "x.com");
     if (!cookies) {
       throw new Error(
         'No cookies found for "twitter". Run: unicli auth setup twitter',

package/src/adapters/twitter/reply.ts

@@ -15,6 +15,7 @@ cli({
   description: "Reply to a tweet",
   domain: "x.com",
   strategy: Strategy.COOKIE,
+  socialCapabilities: ["write_comment"],
   args: [
     {
       name: "tweet_id",

package/src/adapters/twitter/search.ts

@@ -4,14 +4,8 @@
 
 import { cli } from "../../registry.js";
 import { Strategy } from "../../types.js";
-import {
-  twitterFetch,
-  FEATURES,
-  extractTweetsFromInstructions,
-} from "./client.js";
-
-const QUERY_ID = "nK1dw4oV3k4w5TdtcAdSww";
-const ENDPOINT = "SearchTimeline";
+import { browserSearchTweets } from "./browser-fallback.js";
+import type { IPage } from "../../types.js";
 
 cli({
   site: "twitter",
@@ -19,6 +13,8 @@ cli({
   description: "Search tweets by keyword",
   domain: "x.com",
   strategy: Strategy.COOKIE,
+  browser: true,
+  browserSession: "user",
   args: [
     {
       name: "query",
@@ -26,39 +22,17 @@ cli({
       positional: true,
       description: "Search query",
     },
+    {
+      name: "limit",
+      type: "int",
+      default: 20,
+      description: "Number of tweets",
+    },
   ],
   columns: ["id", "author", "text", "likes", "retweets", "views", "url"],
-  func: async (_page, kwargs) => {
+  func: async (page, kwargs) => {
     const query = kwargs.query as string;
     const count = Math.min((kwargs.limit as number) ?? 20, 50);
-
-    const variables = {
-      rawQuery: query,
-      count,
-      querySource: "typed_query",
-      product: "Latest",
-    };
-
-    const data = (await twitterFetch(
-      ENDPOINT,
-      QUERY_ID,
-      variables,
-      FEATURES,
-    )) as Record<string, unknown>;
-
-    // Navigate: data.search_by_raw_query.search_timeline.timeline.instructions
-    const searchByRawQuery = data.data as Record<string, unknown> | undefined;
-    const searchResult = searchByRawQuery?.search_by_raw_query as
-      | Record<string, unknown>
-      | undefined;
-    const searchTimeline = searchResult?.search_timeline as
-      | Record<string, unknown>
-      | undefined;
-    const timeline = searchTimeline?.timeline as
-      | Record<string, unknown>
-      | undefined;
-    const instructions = (timeline?.instructions as unknown[]) ?? [];
-
-    return extractTweetsFromInstructions(instructions);
+    return browserSearchTweets(page as IPage, query, count);
   },
 });

package/src/adapters/twitter/thread.test.ts

@@ -0,0 +1,43 @@
+import { describe, expect, it } from "vitest";
+
+import { normalizeTwitterThreadRows } from "./thread.js";
+
+describe("normalizeTwitterThreadRows", () => {
+  it("adds normalized comment hierarchy fields to thread rows", () => {
+    const rows = normalizeTwitterThreadRows("root", [
+      {
+        id: "root",
+        author: "Root",
+        text: "Root tweet",
+        likes: 10,
+        retweets: 2,
+        views: "100",
+        url: "https://x.com/i/status/root",
+      },
+      {
+        id: "reply-1",
+        author: "Reply",
+        text: "Reply tweet",
+        likes: 1,
+        retweets: 0,
+        views: "5",
+        url: "https://x.com/i/status/reply-1",
+      },
+    ]);
+
+    expect(rows).toEqual([
+      expect.objectContaining({
+        id: "root",
+        parent_id: "",
+        depth: 0,
+        path: "0001",
+      }),
+      expect.objectContaining({
+        id: "reply-1",
+        parent_id: "root",
+        depth: 1,
+        path: "0001.0001",
+      }),
+    ]);
+  });
+});

package/src/adapters/twitter/thread.ts

@@ -13,12 +13,40 @@ import {
 const QUERY_ID = "B9_KmbkLhXt6jRwGjJrweg";
 const ENDPOINT = "TweetDetail";
 
+interface TweetRow {
+  id: string;
+  author: string;
+  text: string;
+  likes: number;
+  retweets: number;
+  views: string;
+  url: string;
+}
+
+export function normalizeTwitterThreadRows(
+  tweetId: string,
+  tweets: TweetRow[],
+): Array<TweetRow & { parent_id: string; depth: number; path: string }> {
+  let replyRank = 0;
+  return tweets.map((tweet) => {
+    const isRoot = tweet.id === tweetId;
+    if (!isRoot) replyRank += 1;
+    return {
+      ...tweet,
+      parent_id: isRoot ? "" : tweetId,
+      depth: isRoot ? 0 : 1,
+      path: isRoot ? "0001" : `0001.${String(replyRank).padStart(4, "0")}`,
+    };
+  });
+}
+
 cli({
   site: "twitter",
   name: "thread",
   description: "Get a tweet and its conversation thread",
   domain: "x.com",
   strategy: Strategy.COOKIE,
+  socialCapabilities: ["read", "comments", "comment_replies"],
   args: [
     {
       name: "tweet_id",
@@ -27,7 +55,18 @@ cli({
       description: "Tweet ID (numeric)",
     },
   ],
-  columns: ["id", "author", "text", "likes", "retweets", "views", "url"],
+  columns: [
+    "id",
+    "parent_id",
+    "author",
+    "text",
+    "likes",
+    "retweets",
+    "views",
+    "url",
+    "depth",
+    "path",
+  ],
   func: async (_page, kwargs) => {
     const tweetId = String(kwargs.tweet_id);
 
@@ -56,6 +95,9 @@ cli({
       | undefined;
     const instructions = (conversation?.instructions as unknown[]) ?? [];
 
-    return extractTweetsFromInstructions(instructions);
+    return normalizeTwitterThreadRows(
+      tweetId,
+      extractTweetsFromInstructions(instructions),
+    );
   },
 });

package/src/adapters/twitter/trending.ts

@@ -4,9 +4,8 @@
 
 import { cli } from "../../registry.js";
 import { Strategy } from "../../types.js";
-import { twitterGuideFetch } from "./client.js";
-
-const GUIDE_URL = "https://x.com/i/api/2/guide.json";
+import { browserTrendingTopics } from "./browser-fallback.js";
+import type { IPage } from "../../types.js";
 
 cli({
   site: "twitter",
@@ -14,99 +13,19 @@ cli({
   description: "Get trending topics",
   domain: "x.com",
   strategy: Strategy.COOKIE,
+  browser: true,
+  browserSession: "user",
+  args: [
+    {
+      name: "limit",
+      type: "int",
+      default: 20,
+      description: "Number of trends",
+    },
+  ],
   columns: ["name", "tweet_count", "description", "url"],
-  func: async (_page, kwargs) => {
+  func: async (page, kwargs) => {
     const count = (kwargs.limit as number) ?? 20;
-
-    const data = (await twitterGuideFetch(GUIDE_URL, {
-      count: String(count),
-      include_page_configuration: "false",
-    })) as Record<string, unknown>;
-
-    // Navigate: data.timeline.instructions[0].addEntries.entries
-    const timeline = data.timeline as Record<string, unknown> | undefined;
-    const instructions = (timeline?.instructions as unknown[]) ?? [];
-
-    const trends: Array<{
-      name: string;
-      tweet_count: string;
-      description: string;
-      url: string;
-    }> = [];
-
-    for (const instruction of instructions) {
-      const inst = instruction as Record<string, unknown>;
-      const addEntries = inst.addEntries as Record<string, unknown> | undefined;
-      if (!addEntries) continue;
-
-      const entries = (addEntries.entries as unknown[]) ?? [];
-
-      for (const entry of entries) {
-        const e = entry as Record<string, unknown>;
-        const content = e.content as Record<string, unknown> | undefined;
-        if (!content) continue;
-
-        // Trend items are nested in timelineModule or timelineItem
-        const items =
-          (content.items as unknown[]) ??
-          (content.item ? [{ item: content.item }] : []);
-
-        for (const item of items) {
-          const i = item as Record<string, unknown>;
-          const itemObj = (i.item ?? i) as Record<string, unknown>;
-          const clientEventInfo = itemObj.clientEventInfo as
-            | Record<string, unknown>
-            | undefined;
-          const details = clientEventInfo?.details as
-            | Record<string, unknown>
-            | undefined;
-          const guideDetails = details?.guideDetails as
-            | Record<string, unknown>
-            | undefined;
-          const transparentGuideDetails =
-            guideDetails?.transparentGuideDetails as
-              | Record<string, unknown>
-              | undefined;
-          const trendMetadata = transparentGuideDetails?.trendMetadata as
-            | Record<string, unknown>
-            | undefined;
-
-          // Also try direct content path
-          const itemContent = (
-            itemObj as Record<string, Record<string, unknown>>
-          ).content;
-          const trend = itemContent?.trend as
-            | Record<string, unknown>
-            | undefined;
-
-          const name =
-            (trendMetadata?.trendName as string) ??
-            (trend?.name as string) ??
-            "";
-          if (!name) continue;
-
-          const tweetCount =
-            (trendMetadata?.metaDescription as string) ??
-            (trend?.tweetCount as string) ??
-            "";
-          const desc =
-            (trend?.description as string) ??
-            (trendMetadata?.metaDescription as string) ??
-            "";
-          const trendUrl = trend?.url as Record<string, unknown> | undefined;
-
-          trends.push({
-            name,
-            tweet_count: tweetCount,
-            description: desc,
-            url:
-              (trendUrl?.url as string) ??
-              `https://x.com/search?q=${encodeURIComponent(name)}`,
-          });
-        }
-      }
-    }
-
-    return trends.slice(0, count);
+    return browserTrendingTopics(page as IPage, count);
   },
 });