@sprid/scout 0.1.0

package/src/discover.js

@@ -0,0 +1,463 @@
+"use strict";
+
+const { acquire, handleChallenge, invoke } = require("./chrome");
+const {
+  TOP,
+  PHOTO,
+  parseCount,
+  loadAccountDetails,
+  measurePosts,
+  decideVerdict,
+  flagOutliers,
+} = require("./tiktok");
+const jobsLib = require("./jobs");
+
+const MAX_ROUNDS = 5;
+const FIRST_ROUND_CANDIDATES_PER_ACCOUNT = 4;
+const MIN_CANDIDATE_POOL = 10;
+const MAX_CANDIDATE_POOL = 500;
+const BOT_THRESHOLD = 3;
+const LIKE_EVERY = 6;
+
+function clamp(value, lo, hi) {
+  return Math.max(lo, Math.min(hi, value));
+}
+
+/**
+ * How big a pool to gather this round. Sized from the pass rate actually
+ * observed, so a strict filter asks for more ground rather than coming back short.
+ */
+function nextPool(missing, passRate) {
+  const needed =
+    passRate > 0 ? Math.ceil(missing / passRate) : missing * FIRST_ROUND_CANDIDATES_PER_ACCOUNT;
+  return clamp(needed, MIN_CANDIDATE_POOL, MAX_CANDIDATE_POOL);
+}
+
+async function likePost(context, username, postId, log) {
+  log("  Liking a post to train the algorithm...");
+  let page = null;
+  try {
+    page = await context.newPage();
+    await page.goto(`https://www.tiktok.com/@${username}/photo/${postId}`, {
+      waitUntil: "domcontentloaded",
+      timeout: 15_000,
+    });
+    await page.waitForTimeout(2000);
+    for (const selector of [
+      '[data-e2e="like-icon"]',
+      '[data-e2e="browse-like-icon"]',
+      'button[aria-label*="ike"]',
+    ]) {
+      try {
+        const button = page.locator(selector).first();
+        if (await button.isVisible({ timeout: 2000 })) {
+          await button.click();
+          await page.waitForTimeout(1000);
+          break;
+        }
+      } catch {}
+    }
+  } catch {
+  } finally {
+    await page?.close().catch(() => {});
+  }
+}
+
+/**
+ * Phase 1. Types each keyword into TikTok's own search and reads the results
+ * the page fetches for itself, Top tab first and Photo behind it when Top runs
+ * thin. Keywords are walked in order and spent only as far as needed.
+ */
+async function searchViaFeed({ keywords, cursor, maxPool, alreadyProcessed, session, log, gate, challenge }) {
+  const { context, newPage } = session;
+  const searchPage = await newPage();
+  const candidates = {};
+  const hashtagFreq = {};
+  const liked = new Set();
+  let consumed = 0;
+
+  async function sweep(keyword, tab) {
+    const before = Object.keys(candidates).length;
+    const slideshowPosts = [];
+    const seenPostIds = new Set();
+    const seenAllIds = new Set();
+
+    const onResponse = async (response) => {
+      if (!response.url().includes("/api/search/")) return;
+      try {
+        const payload = await response.json();
+        for (const item of tab.items(payload)) {
+          const postId = item.id ?? "";
+          if (postId && !seenAllIds.has(postId)) seenAllIds.add(postId);
+          if (!item.imagePost) continue;
+          const author = item.author ?? {};
+          const uid = author.uniqueId ?? "";
+          if (!uid || !postId || seenPostIds.has(postId)) continue;
+          seenPostIds.add(postId);
+          for (const tag of item.textExtra ?? []) {
+            if (tag.hashtagName) hashtagFreq[tag.hashtagName] = (hashtagFreq[tag.hashtagName] ?? 0) + 1;
+          }
+          if (alreadyProcessed.has(uid) || candidates[uid]) continue;
+          slideshowPosts.push({
+            post_id: postId,
+            username: uid,
+            display_name: author.nickname ?? "",
+            bio: author.signature ?? "",
+            followers: tab.followers(item),
+            verified: Boolean(author.verified),
+          });
+        }
+      } catch {}
+    };
+
+    searchPage.on("response", onResponse);
+    try {
+      await searchPage.goto(tab.url(keyword.trim()), {
+        waitUntil: "domcontentloaded",
+        timeout: 20_000,
+      });
+      await searchPage.waitForTimeout(3000);
+      await handleChallenge(searchPage, log, challenge);
+    } catch {
+      searchPage.off("response", onResponse);
+      return { added: 0 };
+    }
+
+    if (tab.confirm) {
+      for (const selector of tab.confirm) {
+        try {
+          const control = searchPage.locator(selector).first();
+          if (await control.isVisible({ timeout: 1500 })) {
+            await control.click();
+            await searchPage.waitForTimeout(1500);
+            break;
+          }
+        } catch {}
+      }
+    }
+
+    let silentScrolls = 0;
+    for (let scroll = 0; scroll < 15; scroll++) {
+      if (!(await gate())) break;
+      const prev = Object.keys(candidates).length;
+      await searchPage.evaluate("window.scrollTo(0, document.body.scrollHeight)");
+      await searchPage.waitForTimeout(2000);
+      await handleChallenge(searchPage, log, challenge);
+
+      for (const post of slideshowPosts) {
+        if (!candidates[post.username] && !alreadyProcessed.has(post.username)) {
+          candidates[post.username] = {
+            username: post.username,
+            display_name: post.display_name,
+            bio: post.bio,
+            followers: post.followers,
+            verified: post.verified,
+            _post_id: post.post_id,
+          };
+        }
+      }
+
+      // Tell the algorithm what this session is into, occasionally.
+      const fresh = slideshowPosts.filter((p) => !liked.has(p.post_id));
+      if (fresh.length && liked.size < Math.ceil(scroll / LIKE_EVERY) + 1) {
+        const pick = fresh[0];
+        liked.add(pick.post_id);
+        await likePost(context, pick.username, pick.post_id, log);
+      }
+
+      if (Object.keys(candidates).length >= maxPool) break;
+      silentScrolls = Object.keys(candidates).length === prev ? silentScrolls + 1 : 0;
+      if (silentScrolls >= 3) break;
+    }
+    searchPage.off("response", onResponse);
+    return { added: Object.keys(candidates).length - before };
+  }
+
+  while (cursor.index < keywords.length && Object.keys(candidates).length < maxPool) {
+    if (!(await gate())) break;
+    const keyword = keywords[cursor.index];
+    log(`\nSearching '${keyword}'...`);
+    const top = await sweep(keyword, TOP);
+    if (top.added < 3) {
+      log(`  Top tab ran thin (${top.added}); trying the Photo tab.`);
+      await sweep(keyword, PHOTO);
+    }
+    cursor.index += 1;
+    consumed += 1;
+    log(`  '${keyword}' done -> ${Object.keys(candidates).length} candidates so far`);
+  }
+
+  await searchPage.close().catch(() => {});
+  return { candidates, hashtagFreq, consumed };
+}
+
+function unmeasuredRecord(username, account, followers) {
+  const { _post_id, ...rest } = account;
+  return {
+    ...rest,
+    username,
+    followers,
+    metrics: { scraped_posts: 0 },
+    verdict: {
+      passed: false,
+      failed: [],
+      noPostsInWindow: false,
+      unmeasured: true,
+      viewsPerFollower: null,
+    },
+  };
+}
+
+/**
+ * Phase 2. Visits each candidate profile and measures it. Every profile opened
+ * is kept, whatever its verdict: the filters sort accounts, they never hide them.
+ */
+async function collectMetrics({ candidates, filters, session, log, gate, onMeasured, onBot }) {
+  const profilePage = session.page;
+  const entries = Object.entries(candidates);
+  let consecutiveEmpty = 0;
+  const challenge = { hold: gate, signal: session.challengeSignal };
+
+  for (const [index, [username, account]] of entries.entries()) {
+    if (!(await gate())) break;
+    log(`\nAnalyzing @${username} (${index + 1}/${entries.length})...`);
+
+    const posts = [];
+    const seen = new Set();
+    let user = {};
+    const onResponse = async (response) => {
+      const url = response.url();
+      if (url.includes("/api/post/item_list/") || url.includes("/api/user/post/")) {
+        try {
+          const data = await response.json();
+          for (const item of data.itemList ?? data.items ?? []) {
+            if (item.id && !seen.has(item.id)) {
+              seen.add(item.id);
+              posts.push(item);
+            }
+          }
+        } catch {}
+      }
+      if (url.includes("/api/user/detail/")) {
+        try {
+          const data = await response.json();
+          if (data.userInfo?.user) user = { ...user, ...data.userInfo.user };
+        } catch {}
+      }
+    };
+    profilePage.on("response", onResponse);
+
+    let domFollowers = 0;
+    try {
+      await profilePage.goto("about:blank", { waitUntil: "domcontentloaded", timeout: 5000 });
+      await profilePage.goto(`https://www.tiktok.com/@${username}`, {
+        waitUntil: "domcontentloaded",
+        timeout: 20_000,
+      });
+      await profilePage.waitForTimeout(3000);
+      await handleChallenge(profilePage, log, challenge);
+
+      try {
+        await profilePage.waitForSelector('[data-e2e="followers-count"]', { timeout: 3000 });
+        const raw = await profilePage.evaluate(
+          invoke(`() => {
+            const el = document.querySelector('[data-e2e="followers-count"]');
+            return el ? (el.textContent || el.innerText || '').trim() : '';
+          }`)
+        );
+        domFollowers = parseCount(raw || "");
+      } catch {
+        try {
+          const raw = await profilePage.evaluate(
+            invoke(`() => {
+              for (const el of document.querySelectorAll('strong[title]')) {
+                const t = (el.title || el.textContent || '').trim();
+                if (t) return t;
+              }
+              return '';
+            }`)
+          );
+          domFollowers = parseCount(raw || "");
+        } catch {}
+      }
+
+      for (let scroll = 0; scroll < 5; scroll++) {
+        if (!(await gate())) break;
+        const before = posts.length;
+        await profilePage.evaluate("window.scrollTo(0, document.body.scrollHeight)");
+        await profilePage.waitForTimeout(2500);
+        await handleChallenge(profilePage, log, challenge);
+        if (posts.length === before && scroll >= 1) break;
+      }
+    } catch {
+      profilePage.off("response", onResponse);
+      if (!(await gate())) break;
+      log("  Profile did not load: recording it as unmeasured.");
+      onMeasured(unmeasuredRecord(username, account, domFollowers || account.followers || 0));
+      continue;
+    }
+    profilePage.off("response", onResponse);
+
+    if (posts.length === 0) {
+      consecutiveEmpty += 1;
+      log(`  No posts loaded: unmeasured (${consecutiveEmpty}/${BOT_THRESHOLD} consecutive)`);
+      onMeasured(unmeasuredRecord(username, account, domFollowers || account.followers || 0));
+      if (consecutiveEmpty >= BOT_THRESHOLD) {
+        onBot(
+          `TikTok stopped serving profile content after ${BOT_THRESHOLD} consecutive empty profiles. Check the Chrome window, then resume.`
+        );
+        consecutiveEmpty = 0;
+      }
+      continue;
+    }
+    consecutiveEmpty = 0;
+
+    const details = await loadAccountDetails(username);
+    const followers =
+      details.followerCount ||
+      domFollowers ||
+      user.followerCount ||
+      posts[0]?.author?.followerCount ||
+      account.followers ||
+      0;
+
+    const measured = measurePosts({
+      posts,
+      followers,
+      avgViewsDays: filters.avgViewsDays,
+      details,
+    });
+    if (!measured) {
+      log("  Post timestamps could not be read: recording it as unmeasured.");
+      onMeasured(unmeasuredRecord(username, account, followers));
+      continue;
+    }
+
+    const verdict = decideVerdict({
+      followers,
+      ratio: measured.ratio,
+      ratioPct: measured.ratioPct,
+      avgViews: measured.avgViews,
+      totalViewsWindow: measured.totalViewsWindow,
+      noPostsInWindow: measured.noPostsInWindow,
+      filters,
+    });
+
+    log(
+      measured.noPostsInWindow
+        ? `  ${posts.length} posts | ${measured.ratioPct}% slideshows | no posts in the last ${filters.avgViewsDays}d`
+        : `  ${posts.length} posts | ${measured.ratioPct}% slideshows | avg ${Math.trunc(
+            measured.avgViews ?? 0
+          ).toLocaleString("en-US")} views`
+    );
+    log(verdict.passed ? "  -> passed" : `  -> rejected: ${verdict.failed.map((f) => f.filter).join(", ") || "no posts in window"}`);
+
+    const { _post_id, ...cleaned } = account;
+    onMeasured({
+      ...cleaned,
+      username,
+      followers,
+      // Kept so a retune can re-window this account without re-visiting it.
+      samples: measured.samples,
+      measuredAt: Date.now() / 1000,
+      display_name: details.nickname || user.nickname || account.display_name || "",
+      bio: details.signature ?? (user.signature || account.bio || ""),
+      verified: details.verified ?? Boolean(user.verified ?? account.verified),
+      metrics: measured.metrics,
+      sample_captions: measured.captions,
+      verdict,
+    });
+  }
+}
+
+async function runDiscovery(job) {
+  const { keywords, filters, maxAccounts, hashtagPivot } = job.meta;
+  const log = (text, alert) => jobsLib.log(job, text, alert);
+  const gate = jobsLib.gate(job);
+  const alreadyProcessed = new Set();
+  const cursor = { index: 0 };
+  const allKeywords = [...keywords];
+  job.state.keywordsTotal = allKeywords.length;
+  job.state.target = maxAccounts;
+
+  const session = await acquire();
+  const challenge = {
+    hold: gate,
+    signal: session.challengeSignal,
+    onDetected: (prompt) => jobsLib.setAlert(job, "captcha", prompt),
+  };
+
+  try {
+    let pivoted = false;
+    for (let round = 0; round < MAX_ROUNDS; round++) {
+      const passed = job.state.measured.filter((a) => a.verdict?.passed).length;
+      const missing = maxAccounts - passed;
+      if (missing <= 0) break;
+      if (cursor.index >= allKeywords.length) {
+        if (!hashtagPivot || pivoted) break;
+        // A second pass on the vocabulary the first pass turned up, which
+        // reaches accounts the original wording never would.
+        const tags = Object.entries(job.state.hashtagFreq ?? {})
+          .sort((a, b) => b[1] - a[1])
+          .slice(0, 8)
+          .map(([tag]) => tag);
+        if (tags.length === 0) break;
+        allKeywords.push(...tags);
+        job.state.keywordsTotal = allKeywords.length;
+        pivoted = true;
+        log(`\nHashtag pivot: searching ${tags.length} hashtags the first pass turned up.`);
+      }
+
+      const measuredSoFar = job.state.measured.length;
+      const passRate = measuredSoFar > 0 ? passed / measuredSoFar : 0;
+      const pool = nextPool(missing, passRate);
+      log(`\nRound ${round + 1}: gathering up to ${pool} candidates.`);
+
+      const search = await searchViaFeed({
+        keywords: allKeywords,
+        cursor,
+        maxPool: pool,
+        alreadyProcessed,
+        session,
+        log,
+        gate,
+        challenge,
+      });
+      job.state.keywordsConsumed = cursor.index;
+      job.state.hashtagFreq = { ...(job.state.hashtagFreq ?? {}), ...search.hashtagFreq };
+      jobsLib.emit("progress", job);
+
+      const found = Object.keys(search.candidates).length;
+      if (found === 0) {
+        log("  No new candidates from that pass.");
+        if (cursor.index >= allKeywords.length) break;
+        continue;
+      }
+      log(`\nMeasuring ${found} candidate profiles.`);
+
+      await collectMetrics({
+        candidates: search.candidates,
+        filters: job.meta.filters,
+        session,
+        log,
+        gate,
+        onMeasured: (account) => {
+          alreadyProcessed.add(account.username);
+          job.state.measured.push(account);
+          flagOutliers(job.state.measured);
+          jobsLib.emit("progress", job);
+        },
+        onBot: (message) => jobsLib.setAlert(job, "bot", message),
+      });
+
+      if (!(await gate())) break;
+    }
+  } finally {
+    await session.release();
+  }
+
+  flagOutliers(job.state.measured);
+  return job.state.measured;
+}
+
+module.exports = { runDiscovery, nextPool };