@jackwener/opencli 1.7.12 → 1.7.13

package/clis/tiktok/friends.js

@@ -1,44 +1,146 @@
-import { cli } from '@jackwener/opencli/registry';
-cli({
+// TikTok friend / who-to-follow suggestions via page-context API.
+//
+// Replaces legacy DOM-link scraping (`querySelectorAll('a[href*="/@"]')` plus
+// fragile text filters that mistook UI labels for handles). We pull the
+// suggestion list from `__UNIVERSAL_DATA_FOR_REHYDRATION__` first, then
+// top up via `/api/recommend/user/` if the warm snapshot is short.
+
+import { cli, Strategy } from '@jackwener/opencli/registry';
+import {
+    EmptyResultError,
+} from '@jackwener/opencli/errors';
+import {
+    BROWSER_HELPERS,
+    MAX_PAGES,
+    SERVER_PAGE_MAX,
+    TIKTOK_AID,
+    USER_ITEM_NORMALIZER,
+    requireLimit,
+    throwTikTokPageContextError,
+} from './utils.js';
+
+const DEFAULT_LIMIT = 20;
+const MAX_LIMIT = 100;
+
+function buildFriendsScript(limit) {
+    return `
+(async () => {
+  const limit = ${Number(limit)};
+  const maxPages = ${MAX_PAGES};
+  const SERVER_PAGE_MAX = ${SERVER_PAGE_MAX};
+  const pageSize = limit < SERVER_PAGE_MAX ? limit : SERVER_PAGE_MAX;
+  const aid = ${JSON.stringify(TIKTOK_AID)};
+
+  ${BROWSER_HELPERS}
+  ${USER_ITEM_NORMALIZER}
+
+  function collectUsersFromState(root) {
+    if (!root) return [];
+    const out = [];
+    walkObjects(root, (node) => {
+      if (Array.isArray(node)) return false;
+      if (Array.isArray(node.userList)) {
+        for (const entry of node.userList) {
+          out.push(entry?.user || entry);
+        }
+      }
+      if (Array.isArray(node.suggestList)) {
+        for (const entry of node.suggestList) out.push(entry?.user || entry);
+      }
+      return false;
+    });
+    return out;
+  }
+
+  const dedup = new Map();
+  const universal = findUniversalData();
+  for (const user of collectUsersFromState(universal)) {
+    const row = normalizeUserRow(user, dedup.size + 1);
+    if (row && !dedup.has(row.username)) dedup.set(row.username, row);
+  }
+
+  let apiFailure = null;
+  const msToken = getCookie('msToken');
+  if (dedup.size < limit) {
+    let cursor = 0;
+    for (let page = 0; page < maxPages && dedup.size < limit; page += 1) {
+      const params = new URLSearchParams({
+        aid,
+        count: String(pageSize),
+        cursor: String(cursor),
+        scene: '15',
+      });
+      if (msToken) params.set('msToken', msToken);
+      try {
+        const data = await fetchJson('/api/recommend/user/?' + params.toString());
+        assertTikTokApiSuccess(data, 'recommend-user');
+        const list = Array.isArray(data.userList)
+          ? data.userList
+          : (Array.isArray(data.user_list) ? data.user_list : []);
+        if (list.length === 0) break;
+        for (const entry of list) {
+          const row = normalizeUserRow(entry?.user || entry, dedup.size + 1);
+          if (row && !dedup.has(row.username)) dedup.set(row.username, row);
+        }
+        if (data.hasMore !== true) break;
+        cursor = asNumber(data.cursor) ?? cursor + list.length;
+      } catch (error) {
+        apiFailure = error instanceof Error ? error.message : String(error);
+        break;
+      }
+    }
+  }
+
+  const rows = Array.from(dedup.values())
+    .slice(0, limit)
+    .map((row, index) => ({ ...row, index: index + 1 }));
+
+  if (rows.length === 0) {
+    const suffix = apiFailure ? ' (recommend-user API failed: ' + apiFailure + ')' : '';
+    throw new Error('No friend suggestions returned by TikTok' + suffix);
+  }
+  return rows;
+})()
+`;
+}
+
+async function listFriends(page, args) {
+    const limit = requireLimit(args.limit, { fallback: DEFAULT_LIMIT, max: MAX_LIMIT });
+    await page.goto('https://www.tiktok.com/friends', { waitUntil: 'load', settleMs: 5000 });
+    let rows;
+    try {
+        rows = await page.evaluate(buildFriendsScript(limit));
+    } catch (error) {
+        throwTikTokPageContextError(error, {
+            authMessage: 'TikTok requires browser access to load friend suggestions',
+            emptyPattern: /No friend suggestions/,
+            emptyTarget: 'tiktok friends',
+            failureMessage: 'Failed to load TikTok friend suggestions',
+        });
+    }
+    if (!Array.isArray(rows) || rows.length === 0) {
+        throw new EmptyResultError('tiktok friends', 'TikTok returned no friend suggestions');
+    }
+    return rows;
+}
+
+export const friendsCommand = cli({
     site: 'tiktok',
     name: 'friends',
     access: 'read',
-    description: 'Get TikTok friend suggestions',
+    description: 'Get TikTok friend / who-to-follow suggestions via page-context APIs',
     domain: 'www.tiktok.com',
+    strategy: Strategy.COOKIE,
+    browser: true,
     args: [
-        { name: 'limit', type: 'int', default: 20, help: 'Number of suggestions' },
-    ],
-    columns: ['index', 'username', 'name'],
-    pipeline: [
-        { navigate: { url: 'https://www.tiktok.com/friends', settleMs: 5000 } },
-        { evaluate: `(() => {
-  const limit = \${{ args.limit }};
-  const links = Array.from(document.querySelectorAll('a[href*="/@"]'))
-    .filter(function(a) {
-      const text = a.textContent.trim();
-      return text.length > 1 && text.length < 80 &&
-        !text.includes('Profile') && !text.includes('More') && !text.includes('Upload');
-    });
-
-  const seen = {};
-  const results = [];
-  for (const a of links) {
-    const match = a.href.match(/@([^/]+)/);
-    const username = match ? match[1] : '';
-    if (!username || seen[username]) continue;
-    seen[username] = true;
-    const raw = a.textContent.trim();
-    const hasFollow = raw.includes('Follow');
-    const name = raw.replace('Follow', '').replace(username, '').replace('@', '').trim();
-    results.push({
-      index: results.length + 1,
-      username: username,
-      name: name || username,
-    });
-    if (results.length >= limit) break;
-  }
-  return results;
-})()
-` },
+        { name: 'limit', type: 'int', default: DEFAULT_LIMIT, help: `Number of suggestions (max ${MAX_LIMIT})` },
     ],
+    columns: ['index', 'username', 'name', 'secUid', 'verified', 'followers', 'following', 'url'],
+    func: listFriends,
 });
+
+export const __test__ = {
+    buildFriendsScript,
+    DEFAULT_LIMIT,
+    MAX_LIMIT,
+};

package/clis/tiktok/live.js

@@ -1,48 +1,144 @@
-import { cli } from '@jackwener/opencli/registry';
-cli({
+// Browse TikTok live streams via page-context API.
+//
+// Replaces legacy DOM card scraping (`[data-e2e="live-side-nav-item"]` plus a
+// regex pulling viewer counts from card text). We extract from the live-discover
+// state injected on `/live`, then top up via `/api/live/discover/get/`.
+
+import { cli, Strategy } from '@jackwener/opencli/registry';
+import {
+    EmptyResultError,
+} from '@jackwener/opencli/errors';
+import {
+    BROWSER_HELPERS,
+    LIVE_ITEM_NORMALIZER,
+    SERVER_PAGE_MAX,
+    TIKTOK_AID,
+    requireLimit,
+    throwTikTokPageContextError,
+} from './utils.js';
+
+const DEFAULT_LIMIT = 10;
+const MAX_LIMIT = 60;
+
+function buildLiveScript(limit) {
+    return `
+(async () => {
+  const limit = ${Number(limit)};
+  const SERVER_PAGE_MAX = ${SERVER_PAGE_MAX};
+  const pageSize = limit < SERVER_PAGE_MAX ? limit : SERVER_PAGE_MAX;
+  const aid = ${JSON.stringify(TIKTOK_AID)};
+
+  ${BROWSER_HELPERS}
+  ${LIVE_ITEM_NORMALIZER}
+
+  function collectFromState(root) {
+    if (!root) return [];
+    const out = [];
+    walkObjects(root, (node) => {
+      if (Array.isArray(node)) return false;
+      if (Array.isArray(node.liveList)) {
+        for (const entry of node.liveList) out.push(entry);
+      }
+      if (Array.isArray(node.live_list)) {
+        for (const entry of node.live_list) out.push(entry);
+      }
+      if (Array.isArray(node.discoverList)) {
+        for (const entry of node.discoverList) out.push(entry);
+      }
+      return false;
+    });
+    return out;
+  }
+
+  const dedup = new Map();
+  const universal = findUniversalData();
+  for (const item of collectFromState(universal)) {
+    const row = normalizeLiveItem(item, dedup.size + 1);
+    if (row) {
+      const key = row.streamer || row.url;
+      if (key && !dedup.has(key)) dedup.set(key, row);
+    }
+  }
+
+  let apiFailure = null;
+  const msToken = getCookie('msToken');
+  if (dedup.size < limit) {
+    const params = new URLSearchParams({
+      aid,
+      count: String(pageSize),
+      from_page: 'live_discover',
+    });
+    if (msToken) params.set('msToken', msToken);
+    try {
+      const data = await fetchJson('/api/live/discover/get/?' + params.toString());
+      assertTikTokApiSuccess(data, 'live-discover');
+      const list = Array.isArray(data.data?.list)
+        ? data.data.list
+        : (Array.isArray(data.list)
+          ? data.list
+          : (Array.isArray(data.live_list) ? data.live_list : []));
+      for (const entry of list) {
+        const row = normalizeLiveItem(entry, dedup.size + 1);
+        if (row) {
+          const key = row.streamer || row.url;
+          if (key && !dedup.has(key)) dedup.set(key, row);
+        }
+      }
+    } catch (error) {
+      apiFailure = error instanceof Error ? error.message : String(error);
+    }
+  }
+
+  const rows = Array.from(dedup.values())
+    .slice(0, limit)
+    .map((row, index) => ({ ...row, index: index + 1 }));
+
+  if (rows.length === 0) {
+    const suffix = apiFailure ? ' (live-discover API failed: ' + apiFailure + ')' : '';
+    throw new Error('No live streams returned' + suffix);
+  }
+  return rows;
+})()
+`;
+}
+
+async function listLive(page, args) {
+    const limit = requireLimit(args.limit, { fallback: DEFAULT_LIMIT, max: MAX_LIMIT });
+    await page.goto('https://www.tiktok.com/live', { waitUntil: 'load', settleMs: 5000 });
+    let rows;
+    try {
+        rows = await page.evaluate(buildLiveScript(limit));
+    } catch (error) {
+        throwTikTokPageContextError(error, {
+            authMessage: 'TikTok requires browser access to load live streams',
+            emptyPattern: /No live streams returned/,
+            emptyTarget: 'tiktok live',
+            failureMessage: 'Failed to load TikTok live streams',
+        });
+    }
+    if (!Array.isArray(rows) || rows.length === 0) {
+        throw new EmptyResultError('tiktok live', 'TikTok returned no live streams');
+    }
+    return rows;
+}
+
+export const liveCommand = cli({
     site: 'tiktok',
     name: 'live',
     access: 'read',
-    description: 'Browse live streams on TikTok',
+    description: 'Browse TikTok live streams via page-context APIs',
     domain: 'www.tiktok.com',
+    strategy: Strategy.COOKIE,
+    browser: true,
     args: [
-        { name: 'limit', type: 'int', default: 10, help: 'Number of streams' },
-    ],
-    columns: ['index', 'streamer', 'viewers', 'url'],
-    pipeline: [
-        { navigate: { url: 'https://www.tiktok.com/live', settleMs: 5000 } },
-        { evaluate: `(() => {
-  const limit = \${{ args.limit }};
-  // Sidebar live list has structured data
-  const items = document.querySelectorAll('[data-e2e="live-side-nav-item"]');
-  const sidebar = Array.from(items).slice(0, limit).map(function(el, i) {
-    const nameEl = el.querySelector('[data-e2e="live-side-nav-name"]');
-    const countEl = el.querySelector('[data-e2e="person-count"]');
-    const link = el.querySelector('a');
-    return {
-      index: i + 1,
-      streamer: nameEl ? nameEl.textContent.trim() : '',
-      viewers: countEl ? countEl.textContent.trim() : '-',
-      url: link ? link.href : '',
-    };
-  });
-
-  if (sidebar.length > 0) return sidebar;
-
-  // Fallback: main content cards
-  const cards = document.querySelectorAll('[data-e2e="discover-list-live-card"]');
-  return Array.from(cards).slice(0, limit).map(function(card, i) {
-    const text = card.textContent.trim().replace(/\\s+/g, ' ');
-    const link = card.querySelector('a[href*="/live"]');
-    const viewerMatch = text.match(/(\\d[\\d,.]*)\\s*watching/);
-    return {
-      index: i + 1,
-      streamer: text.replace(/LIVE.*$/, '').trim().substring(0, 40),
-      viewers: viewerMatch ? viewerMatch[1] : '-',
-      url: link ? link.href : '',
-    };
-  });
-})()
-` },
+        { name: 'limit', type: 'int', default: DEFAULT_LIMIT, help: `Number of streams (max ${MAX_LIMIT})` },
     ],
+    columns: ['index', 'streamer', 'name', 'title', 'viewers', 'likes', 'secUid', 'url'],
+    func: listLive,
 });
+
+export const __test__ = {
+    buildLiveScript,
+    DEFAULT_LIMIT,
+    MAX_LIMIT,
+};

package/clis/tiktok/notifications.js

@@ -1,50 +1,153 @@
-import { cli } from '@jackwener/opencli/registry';
-cli({
+// Read TikTok inbox notifications via page-context API.
+//
+// Replaces the legacy "click the bell icon then scrape `[data-e2e="inbox-list"]`
+// text content" pipeline. Calls `/api/notice/multi/?notice_type=N` directly
+// from inside the live page so cookies + msToken get forwarded by the browser.
+
+import { cli, Strategy } from '@jackwener/opencli/registry';
+import {
+    EmptyResultError,
+} from '@jackwener/opencli/errors';
+import {
+    BROWSER_HELPERS,
+    MAX_PAGES,
+    NOTIFICATION_NORMALIZER,
+    NOTIFICATION_TYPES,
+    SERVER_PAGE_MAX,
+    TIKTOK_AID,
+    requireLimit,
+    requireNotificationType,
+    throwTikTokPageContextError,
+} from './utils.js';
+
+const DEFAULT_LIMIT = 15;
+const MAX_LIMIT = 100;
+
+function buildNotificationsScript(limit, typeKey) {
+    const typeMeta = NOTIFICATION_TYPES[typeKey];
+    return `
+(async () => {
+  const limit = ${Number(limit)};
+  const maxPages = ${MAX_PAGES};
+  const SERVER_PAGE_MAX = ${SERVER_PAGE_MAX};
+  const pageSize = limit < SERVER_PAGE_MAX ? limit : SERVER_PAGE_MAX;
+  const noticeType = ${Number(typeMeta.code)};
+  const noticeLabel = ${JSON.stringify(typeMeta.label)};
+  const aid = ${JSON.stringify(TIKTOK_AID)};
+
+  ${BROWSER_HELPERS}
+  ${NOTIFICATION_NORMALIZER}
+
+  function collectFromState(root) {
+    if (!root) return [];
+    const out = [];
+    walkObjects(root, (node) => {
+      if (Array.isArray(node)) return false;
+      if (Array.isArray(node.noticeList)) {
+        for (const entry of node.noticeList) out.push(entry);
+      }
+      if (Array.isArray(node.notice_list)) {
+        for (const entry of node.notice_list) out.push(entry);
+      }
+      return false;
+    });
+    return out;
+  }
+
+  const dedup = new Map();
+  const universal = findUniversalData();
+  for (const item of collectFromState(universal)) {
+    const row = normalizeNotification(item, dedup.size + 1);
+    if (row && !dedup.has(row.id)) dedup.set(row.id, row);
+  }
+
+  let apiFailure = null;
+  const msToken = getCookie('msToken');
+  let maxTime = 0;
+  for (let page = 0; page < maxPages && dedup.size < limit; page += 1) {
+    const params = new URLSearchParams({
+      aid,
+      notice_type: String(noticeType),
+      count: String(pageSize),
+      max_time: String(maxTime),
+    });
+    if (msToken) params.set('msToken', msToken);
+    try {
+      const data = await fetchJson('/api/notice/multi/?' + params.toString());
+      assertTikTokApiSuccess(data, 'notice');
+      const list = Array.isArray(data.notice_list_v1)
+        ? data.notice_list_v1
+        : (Array.isArray(data.noticeList) ? data.noticeList : (Array.isArray(data.notice_list) ? data.notice_list : []));
+      if (list.length === 0) break;
+      for (const entry of list) {
+        const row = normalizeNotification(entry, dedup.size + 1);
+        if (row && !dedup.has(row.id)) dedup.set(row.id, row);
+      }
+      if (data.has_more !== true && data.hasMore !== true) break;
+      maxTime = asNumber(data.min_time) ?? asNumber(data.maxTime) ?? maxTime + list.length;
+    } catch (error) {
+      apiFailure = error instanceof Error ? error.message : String(error);
+      break;
+    }
+  }
+
+  const rows = Array.from(dedup.values())
+    .slice(0, limit)
+    .map((row, index) => ({ ...row, index: index + 1 }));
+
+  if (rows.length === 0) {
+    const suffix = apiFailure ? ' (notice API failed: ' + apiFailure + ')' : '';
+    throw new Error('No notifications returned for ' + noticeLabel + suffix);
+  }
+  return rows;
+})()
+`;
+}
+
+async function listNotifications(page, args) {
+    const limit = requireLimit(args.limit, { fallback: DEFAULT_LIMIT, max: MAX_LIMIT });
+    const typeKey = requireNotificationType(args.type);
+    await page.goto('https://www.tiktok.com/foryou', { waitUntil: 'load', settleMs: 5000 });
+    let rows;
+    try {
+        rows = await page.evaluate(buildNotificationsScript(limit, typeKey));
+    } catch (error) {
+        throwTikTokPageContextError(error, {
+            authMessage: 'TikTok requires login to read notifications',
+            emptyPattern: /No notifications returned/,
+            emptyTarget: 'tiktok notifications',
+            failureMessage: 'Failed to load TikTok notifications',
+        });
+    }
+    if (!Array.isArray(rows) || rows.length === 0) {
+        throw new EmptyResultError('tiktok notifications', 'TikTok returned no notifications');
+    }
+    return rows;
+}
+
+export const notificationsCommand = cli({
     site: 'tiktok',
     name: 'notifications',
     access: 'read',
-    description: 'Get TikTok notifications (likes, comments, mentions, followers)',
+    description: 'Read TikTok inbox notifications (likes, comments, mentions, followers) via page-context APIs',
     domain: 'www.tiktok.com',
+    strategy: Strategy.COOKIE,
+    browser: true,
     args: [
-        { name: 'limit', type: 'int', default: 15, help: 'Number of notifications' },
+        { name: 'limit', type: 'int', default: DEFAULT_LIMIT, help: `Number of notifications (max ${MAX_LIMIT})` },
         {
             name: 'type',
             default: 'all',
             help: 'Notification type',
-            choices: ['all', 'likes', 'comments', 'mentions', 'followers'],
+            choices: Object.keys(NOTIFICATION_TYPES),
         },
     ],
-    columns: ['index', 'text'],
-    pipeline: [
-        { navigate: { url: 'https://www.tiktok.com/following', settleMs: 5000 } },
-        { evaluate: `(async () => {
-  const limit = \${{ args.limit }};
-  const type = \${{ args.type | json }};
-  const wait = (ms) => new Promise(r => setTimeout(r, ms));
-
-  // Click inbox icon to open notifications panel
-  const inboxIcon = document.querySelector('[data-e2e="inbox-icon"]');
-  if (inboxIcon) inboxIcon.click();
-  await wait(1500);
-
-  // Click specific tab if needed
-  if (type !== 'all') {
-    const tab = document.querySelector('[data-e2e="' + type + '"]');
-    if (tab) {
-      tab.click();
-      await wait(1500);
-    }
-  }
-
-  const items = document.querySelectorAll('[data-e2e="inbox-list"] > div, [data-e2e="inbox-list"] [role="button"]');
-  return Array.from(items)
-    .filter(el => el.textContent.trim().length > 5)
-    .slice(0, limit)
-    .map((el, i) => ({
-      index: i + 1,
-      text: el.textContent.trim().replace(/\\s+/g, ' ').substring(0, 150),
-    }));
-})()
-` },
-    ],
+    columns: ['index', 'id', 'from', 'text', 'createTime'],
+    func: listNotifications,
 });
+
+export const __test__ = {
+    buildNotificationsScript,
+    DEFAULT_LIMIT,
+    MAX_LIMIT,
+};