@sprid/scout 0.1.0

package/src/analyze.js

@@ -0,0 +1,155 @@
+"use strict";
+
+const { acquire, handleChallenge, invoke } = require("./chrome");
+const { parseCount, loadAccountDetails, measurePosts, postSummary } = require("./tiktok");
+const jobsLib = require("./jobs");
+
+const TOP_POSTS = 10;
+
+/**
+ * Scroll a profile and collect the posts the page fetches for itself. Returns
+ * the raw items, so every caller derives its own numbers from the same reading.
+ */
+async function readProfilePosts({ page, username, log, gate, challenge, maxScrolls, until }) {
+  const posts = [];
+  const seen = new Set();
+  let user = {};
+
+  const onResponse = async (response) => {
+    const url = response.url();
+    if (url.includes("/api/post/item_list/") || url.includes("/api/user/post/")) {
+      try {
+        const data = await response.json();
+        for (const item of data.itemList ?? data.items ?? []) {
+          if (item.id && !seen.has(item.id)) {
+            seen.add(item.id);
+            posts.push(item);
+          }
+        }
+      } catch {}
+    }
+    if (url.includes("/api/user/detail/")) {
+      try {
+        const data = await response.json();
+        if (data.userInfo?.user) user = { ...user, ...data.userInfo.user };
+      } catch {}
+    }
+  };
+  page.on("response", onResponse);
+
+  let domFollowers = 0;
+  try {
+    await page.goto("about:blank", { waitUntil: "domcontentloaded", timeout: 5000 });
+    await page.goto(`https://www.tiktok.com/@${username}`, {
+      waitUntil: "domcontentloaded",
+      timeout: 20_000,
+    });
+    await page.waitForTimeout(3000);
+    await handleChallenge(page, log, challenge);
+
+    try {
+      await page.waitForSelector('[data-e2e="followers-count"]', { timeout: 3000 });
+      const raw = await page.evaluate(
+        invoke(`() => {
+          const el = document.querySelector('[data-e2e="followers-count"]');
+          return el ? (el.textContent || el.innerText || '').trim() : '';
+        }`)
+      );
+      domFollowers = parseCount(raw || "");
+    } catch {}
+
+    for (let scroll = 0; scroll < maxScrolls; scroll++) {
+      if (!(await gate())) break;
+      const before = posts.length;
+      await page.evaluate("window.scrollTo(0, document.body.scrollHeight)");
+      await page.waitForTimeout(2500);
+      await handleChallenge(page, log, challenge);
+      log(`  Scroll ${scroll + 1}: ${posts.length} posts loaded`);
+      if (posts.length === before && scroll >= 1) {
+        log("  No new posts: profile fully loaded");
+        break;
+      }
+      // Stop early once the reading is old enough to answer the question asked.
+      if (until && posts.length > 0) {
+        const oldest = Math.min(...posts.map((p) => p.createTime ?? Infinity));
+        if (Number.isFinite(oldest) && oldest < until) {
+          log("  Reached the far edge of the window.");
+          break;
+        }
+      }
+    }
+  } finally {
+    page.off("response", onResponse);
+  }
+
+  return { posts, user, domFollowers };
+}
+
+async function runAnalysis(job) {
+  const { username, days, allVideos } = job.meta;
+  const log = (text, alert) => jobsLib.log(job, text, alert);
+  const gate = jobsLib.gate(job);
+  const session = await acquire();
+  const challenge = {
+    hold: gate,
+    signal: session.challengeSignal,
+    onDetected: (prompt) => jobsLib.setAlert(job, "captcha", prompt),
+  };
+
+  try {
+    log(`Opening @${username} and reading ${allVideos ? "the whole account" : `the last ${days} days`}.`);
+    const until = allVideos ? null : Date.now() / 1000 - days * 86400;
+    const { posts, user, domFollowers } = await readProfilePosts({
+      page: session.page,
+      username,
+      log,
+      gate,
+      challenge,
+      maxScrolls: allVideos ? 60 : 20,
+      until,
+    });
+
+    if (posts.length === 0) {
+      jobsLib.setAlert(
+        job,
+        "unreadable",
+        `Nothing loaded for @${username}. The account is usually private, renamed, or removed.`
+      );
+      throw new Error(`No posts could be read for @${username}.`);
+    }
+
+    const details = await loadAccountDetails(username);
+    const followers =
+      details.followerCount || domFollowers || user.followerCount || posts[0]?.author?.followerCount || 0;
+
+    const windowDays = allVideos ? 3650 : days;
+    const measured = measurePosts({ posts, followers, avgViewsDays: windowDays, details });
+    if (!measured) throw new Error(`Post timestamps could not be read for @${username}.`);
+
+    const cutoff = allVideos ? 0 : Date.now() / 1000 - days * 86400;
+    const inWindow = posts.filter((p) => (p.createTime ?? 0) >= cutoff);
+    const topPosts = [...inWindow]
+      .sort((a, b) => (b.stats?.playCount ?? 0) - (a.stats?.playCount ?? 0))
+      .slice(0, TOP_POSTS)
+      .map(postSummary);
+
+    log(`Read ${posts.length} posts, ${measured.metrics.slideshow_count} of them slideshows.`);
+
+    return {
+      username,
+      display_name: details.nickname || user.nickname || "",
+      bio: details.signature ?? user.signature ?? "",
+      verified: details.verified ?? Boolean(user.verified),
+      followers,
+      window_days: allVideos ? null : days,
+      all_videos: Boolean(allVideos),
+      metrics: measured.metrics,
+      sample_captions: measured.captions,
+      top_posts: topPosts,
+    };
+  } finally {
+    await session.release();
+  }
+}
+
+module.exports = { runAnalysis, readProfilePosts, TOP_POSTS };

package/src/chrome.js

@@ -0,0 +1,396 @@
+"use strict";
+
+const fs = require("node:fs");
+const path = require("node:path");
+const util = require("node:util");
+const { execFile } = require("node:child_process");
+// Plain playwright, driving the user's own Chrome with their own profile. There
+// is deliberately no bot-detection evasion here: this reads pages the person
+// could open themselves, and a dependency whose purpose is hiding that would
+// say something about the product that is not true.
+const { chromium } = require("playwright-core");
+const { chromeBinary, PROFILE_DIR } = require("./config");
+
+const run = util.promisify(execFile);
+
+const TIMEZONE = Intl.DateTimeFormat().resolvedOptions().timeZone || "America/New_York";
+
+const LAUNCH_ARGS = [
+  "--no-first-run",
+  "--no-default-browser-check",
+  "--disable-blink-features=AutomationControlled",
+  "--disable-infobars",
+  "--lang=en-US",
+];
+
+// TikTok renders its server payload into one script tag and then removes it on
+// hydration. Stashing it the moment it appears means a later read still has it.
+const STASH_REHYDRATION = `(() => {
+  const ID = '__UNIVERSAL_DATA_FOR_REHYDRATION__';
+  const stash = () => {
+    const el = document.getElementById(ID);
+    if (el && el.textContent) { window.__SS_REHYDRATION__ = el.textContent; return true; }
+    return false;
+  };
+  if (stash()) return;
+  const start = () => {
+    if (stash()) return;
+    const root = document.documentElement || document;
+    const observer = new MutationObserver(() => { if (stash()) observer.disconnect(); });
+    observer.observe(root, { childList: true, subtree: true });
+    document.addEventListener('DOMContentLoaded', stash);
+  };
+  if (document.documentElement) start();
+  else document.addEventListener('readystatechange', start, { once: true });
+})()`;
+
+function invoke(probe) {
+  return `(${probe})()`;
+}
+
+const CAPTCHA_PROBE = `() => {
+  const visible = (el) => {
+    if (!el) return false;
+    const r = el.getBoundingClientRect();
+    if (r.width < 40 || r.height < 40) return false;
+    const st = getComputedStyle(el);
+    return st.display !== 'none' && st.visibility !== 'hidden' && parseFloat(st.opacity || '1') > 0.05;
+  };
+  const containerSel = [
+    '#captcha-verify-container', '.captcha_verify_container', '.captcha-verify-container',
+    '#captcha_container', '#captcha-verify-image', '[id^="captcha"]',
+    '[class*="captcha_verify"]', '[class*="captcha-verify"]',
+  ];
+  let node = null;
+  for (const s of containerSel) {
+    for (const el of document.querySelectorAll(s)) {
+      if (visible(el)) { node = el; break; }
+    }
+    if (node) break;
+  }
+  if (!node) return null;
+  const promptSel = [
+    '.captcha_verify_bar', '.cap-flex-center', '[class*="verify_message"]',
+    '[class*="Title"]', '[class*="title"]', '.TUXText',
+  ];
+  const root = node.closest('[id*="captcha"],[class*="captcha"]') || node;
+  for (const s of promptSel) {
+    const p = root.querySelector(s) || document.querySelector(s);
+    if (p && p.innerText && p.innerText.trim().length > 3) return p.innerText.trim();
+  }
+  const txt = (root.innerText || '').trim().split('\\n').map(t => t.trim()).filter(Boolean);
+  return txt.length ? txt.slice(0, 2).join(': ') : 'TikTok verification challenge';
+}`;
+
+const LOGIN_PROBE = `() => {
+  const visible = (el) => {
+    if (!el) return false;
+    const r = el.getBoundingClientRect();
+    if (r.width < 20 || r.height < 20) return false;
+    const st = getComputedStyle(el);
+    return st.display !== 'none' && st.visibility !== 'hidden' && parseFloat(st.opacity || '1') > 0.05;
+  };
+  let modal = null;
+  for (const h of document.querySelectorAll('h1, h2, [class*="Title"], [class*="title"]')) {
+    if (visible(h) && /log ?in to tiktok/i.test(h.textContent || '')) {
+      modal = h.closest('[role="dialog"], div[class*="Modal"], div[class*="modal"]') || h.parentElement;
+      break;
+    }
+  }
+  if (!modal) {
+    for (const s of ['#loginContainer', '[id*="login-modal" i]', '[class*="LoginModal"]']) {
+      const el = document.querySelector(s);
+      if (el && visible(el)) { modal = el; break; }
+    }
+  }
+  if (!modal) return false;
+  const closeSel = [
+    '[data-e2e="modal-close-inner-button"]',
+    '[aria-label="Close"]', '[aria-label="close"]',
+    '[class*="CloseWrapper"]', '[class*="closeButton" i]', 'button[class*="Close"]',
+  ];
+  for (const s of closeSel) {
+    const btn = document.querySelector(s);
+    if (btn && visible(btn)) { btn.click(); return true; }
+  }
+  const inner = modal.querySelectorAll('button, [role="button"], div[class*="Close"], svg');
+  for (const b of inner) { if (visible(b)) { b.click(); return true; } }
+  return true;
+}`;
+
+const TEXT_PROBE = `() => (document.body ? (document.body.innerText || '') : '').slice(0, 4000)`;
+
+const CHALLENGE_REQUEST = /\/captcha\/(get|verify)\b/;
+const VERIFY_TEXT =
+  /verify to continue|verification|drag the (slider|puzzle)|slide to|select the .* (shape|object|image)|too many attempts|check you are (a )?human/i;
+
+function watchChallengeRequests(page, signal = { lastRequestAt: 0 }) {
+  page.on("request", (request) => {
+    if (CHALLENGE_REQUEST.test(request.url())) signal.lastRequestAt = Date.now();
+  });
+  return signal;
+}
+
+function requestedRecently(signal) {
+  return Boolean(signal && Date.now() - signal.lastRequestAt < 30_000);
+}
+
+async function detectCaptcha(page) {
+  try {
+    const overlay = await page.evaluate(invoke(CAPTCHA_PROBE));
+    if (overlay) return overlay;
+    const bodyText = await page.evaluate(invoke(TEXT_PROBE));
+    const match = bodyText.match(VERIFY_TEXT);
+    if (match && bodyText.length < 1500) {
+      return bodyText
+        .split("\n")
+        .map((line) => line.trim())
+        .filter(Boolean)
+        .slice(0, 2)
+        .join(": ");
+    }
+    return null;
+  } catch {
+    return null;
+  }
+}
+
+async function detectCaptchaAnywhere(page) {
+  const own = await detectCaptcha(page);
+  if (own) return { prompt: own, page };
+  for (const other of page.context().pages()) {
+    if (other === page || other.isClosed()) continue;
+    const prompt = await detectCaptcha(other);
+    if (prompt) return { prompt, page: other };
+  }
+  return null;
+}
+
+async function dismissLogin(page, log) {
+  let found = false;
+  try {
+    found = await page.evaluate(invoke(LOGIN_PROBE));
+  } catch {
+    return false;
+  }
+  if (!found) return false;
+  log("Login modal detected: closing and reloading");
+  try {
+    await page.waitForTimeout(400);
+    await page.reload({ waitUntil: "domcontentloaded", timeout: 15_000 });
+    await page.waitForTimeout(2000);
+  } catch {}
+  return true;
+}
+
+async function focusChromeWindow() {
+  await run("osascript", ["-e", 'tell application "Google Chrome" to activate']).catch(
+    () => {}
+  );
+}
+
+/**
+ * Three independent signals, because any one of them alone misses cases: the
+ * captcha network calls, the overlay in the DOM, and the words on the page.
+ * Returns "resumed" or "stopped" when a challenge was handled, null otherwise.
+ */
+async function handleChallenge(page, log, hooks = {}) {
+  await dismissLogin(page, log);
+  const found = await detectCaptchaAnywhere(page);
+  const fromNetwork = requestedRecently(hooks.signal);
+  if (!found && !fromNetwork) return null;
+  const prompt = found?.prompt ?? "TikTok verification challenge";
+  log(`CAPTCHA_DETECTED: ${prompt}`, true);
+  log("Solve the verification in the Chrome window, then resume the job.");
+  await (found?.page ?? page).bringToFront().catch(() => {});
+  await focusChromeWindow();
+  hooks.onDetected?.(prompt);
+  const outcome = hooks.hold ? ((await hooks.hold()) ? "resumed" : "stopped") : "resumed";
+  if (hooks.signal) hooks.signal.lastRequestAt = 0;
+  return outcome;
+}
+
+function trackFeedRequests(page, pending) {
+  const isFeed = (url) =>
+    url.includes("/api/post/item_list/") || url.includes("/api/user/post/");
+  page.on("request", (request) => {
+    if (isFeed(request.url())) pending.count += 1;
+  });
+  const done = (request) => {
+    if (isFeed(request.url())) pending.count = Math.max(0, pending.count - 1);
+  };
+  page.on("requestfinished", done);
+  page.on("requestfailed", done);
+}
+
+async function killChrome(profileDir) {
+  const pattern = `user-data-dir=${profileDir}`;
+  await run("pkill", ["-9", "-f", pattern]).catch(() => {});
+  for (let attempt = 0; attempt < 20; attempt++) {
+    const alive = await run("pgrep", ["-f", pattern]).then(
+      () => true,
+      () => false
+    );
+    if (!alive) break;
+    await new Promise((resolve) => setTimeout(resolve, 500));
+  }
+  for (const lock of ["SingletonLock", "SingletonCookie", "SingletonSocket"]) {
+    try {
+      fs.unlinkSync(path.join(profileDir, lock));
+    } catch {}
+  }
+}
+
+let live = null;
+let starting = null;
+const ownedPages = new Set();
+let activeLeases = 0;
+
+async function launch(profileDir) {
+  await killChrome(profileDir);
+  await new Promise((resolve) => setTimeout(resolve, 1000));
+  const context = await chromium.launchPersistentContext(profileDir, {
+    executablePath: chromeBinary(),
+    headless: false,
+    // Let Chrome own its window size, the way a person's browser does. A fixed
+    // viewport is a giveaway and fights the real window.
+    viewport: null,
+    locale: "en-US",
+    timezoneId: TIMEZONE,
+    args: LAUNCH_ARGS,
+  });
+  await context.addInitScript(STASH_REHYDRATION);
+  return { browser: context.browser(), context, profileDir };
+}
+
+async function ensure(profileDir) {
+  if (live && (live.browser?.isConnected() ?? false)) return live;
+  live = null;
+  starting ??= launch(profileDir)
+    .then((started) => {
+      live = started;
+      return started;
+    })
+    .finally(() => {
+      starting = null;
+    });
+  return starting;
+}
+
+/**
+ * One Chrome, many leases. Each caller gets its own pages and closes only
+ * those; the browser goes down when the last lease is released.
+ */
+async function acquire(profileDir = PROFILE_DIR) {
+  activeLeases++;
+  let context;
+  try {
+    ({ context } = await ensure(profileDir));
+  } catch (error) {
+    activeLeases = Math.max(0, activeLeases - 1);
+    throw error;
+  }
+  const owned = [];
+  const challengeSignal = { lastRequestAt: 0 };
+  const open = async () => {
+    const idle = context
+      .pages()
+      .find((p) => !ownedPages.has(p) && (p.url() === "about:blank" || p.url() === ""));
+    const page = idle ?? (await context.newPage());
+    ownedPages.add(page);
+    page.once("close", () => ownedPages.delete(page));
+    watchChallengeRequests(page, challengeSignal);
+    owned.push(page);
+    return page;
+  };
+  const page = await open();
+  let released = false;
+  return {
+    context,
+    page,
+    newPage: open,
+    challengeSignal,
+    async release() {
+      if (released) return;
+      released = true;
+      await Promise.all(owned.map((p) => p.close().catch(() => {})));
+      activeLeases = Math.max(0, activeLeases - 1);
+      if (activeLeases === 0) await shutdown();
+    },
+  };
+}
+
+async function shutdown() {
+  activeLeases = 0;
+  const running = live;
+  live = null;
+  if (!running) return;
+  await running.context.close().catch(() => {});
+  await killChrome(running.profileDir);
+}
+
+function cookieStores(dir = PROFILE_DIR) {
+  return [
+    path.join(dir, "Default", "Cookies"),
+    path.join(dir, "Default", "Network", "Cookies"),
+  ].flatMap((base) => [base, `${base}-journal`, `${base}-wal`, `${base}-shm`]);
+}
+
+/** Read the cookie files as bytes. Cheap, and it needs no browser running. */
+function isLoggedIn() {
+  if (!fs.existsSync(PROFILE_DIR)) return false;
+  return cookieStores().some((file) => {
+    try {
+      const bytes = fs.readFileSync(file);
+      return bytes.includes("tiktok") && (bytes.includes("sessionid") || bytes.includes("sid_tt"));
+    } catch {
+      return false;
+    }
+  });
+}
+
+async function signOut() {
+  await shutdown();
+  await killChrome(PROFILE_DIR);
+  for (const file of cookieStores()) {
+    try {
+      fs.rmSync(file, { force: true });
+    } catch {}
+  }
+}
+
+const LOGIN_URL = "https://www.tiktok.com/login";
+
+/** Opens a real window and waits for the session cookie to appear. */
+async function login(timeoutMs = 180_000) {
+  const session = await acquire();
+  try {
+    await session.page.goto(LOGIN_URL, { waitUntil: "domcontentloaded" });
+    await session.page.bringToFront().catch(() => {});
+    await focusChromeWindow();
+    const deadline = Date.now() + timeoutMs;
+    while (Date.now() < deadline) {
+      const cookies = await session.context.cookies("https://www.tiktok.com");
+      if (cookies.some((c) => c.name === "sessionid" || c.name === "sid_tt")) {
+        await session.page.waitForTimeout(1500);
+        return { loggedIn: true };
+      }
+      await session.page.waitForTimeout(1000);
+    }
+    return { loggedIn: false };
+  } finally {
+    await session.release();
+  }
+}
+
+module.exports = {
+  acquire,
+  shutdown,
+  handleChallenge,
+  trackFeedRequests,
+  invoke,
+  isLoggedIn,
+  signOut,
+  login,
+  focusChromeWindow,
+};

package/src/config.js

@@ -0,0 +1,83 @@
+"use strict";
+
+const os = require("node:os");
+const path = require("node:path");
+const fs = require("node:fs");
+
+/** The one place the product name lives. Rename here and everything follows. */
+const NAME = "scout";
+
+// Research filed under the old name keeps answering. The library is the user's
+// own notebook, so it is read where it already is rather than moved under them.
+const LEGACY_ROOT = path.join(os.homedir(), ".slidescout");
+const CURRENT_ROOT = path.join(os.homedir(), `.${NAME}`);
+const ROOT =
+  !fs.existsSync(CURRENT_ROOT) && fs.existsSync(LEGACY_ROOT) ? LEGACY_ROOT : CURRENT_ROOT;
+const PROFILE_DIR = path.join(ROOT, "chrome-profile");
+const RUNS_DIR = path.join(ROOT, "runs");
+const LIBRARY_FILE = path.join(ROOT, "library.json");
+const DOWNLOADS_DIR = path.join(ROOT, "downloads");
+
+function ensureDirs() {
+  for (const dir of [ROOT, RUNS_DIR, DOWNLOADS_DIR]) {
+    fs.mkdirSync(dir, { recursive: true });
+  }
+}
+
+const CHROME_CANDIDATES = [
+  "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
+  `${os.homedir()}/Applications/Google Chrome.app/Contents/MacOS/Google Chrome`,
+  "/Applications/Google Chrome Beta.app/Contents/MacOS/Google Chrome Beta",
+  "/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary",
+];
+
+class ChromeMissingError extends Error {
+  constructor() {
+    super(
+      "Google Chrome is not installed. This drives your own Chrome so that TikTok sees a real browser with your session: install it from google.com/chrome and try again."
+    );
+    this.name = "ChromeMissingError";
+  }
+}
+
+let resolvedChrome = null;
+function chromeBinary() {
+  if (resolvedChrome) return resolvedChrome;
+  const found = CHROME_CANDIDATES.find((candidate) => {
+    try {
+      fs.accessSync(candidate, fs.constants.X_OK);
+      return true;
+    } catch {
+      return false;
+    }
+  });
+  if (!found) throw new ChromeMissingError();
+  resolvedChrome = found;
+  return found;
+}
+
+function chromeUnavailable() {
+  try {
+    chromeBinary();
+    return null;
+  } catch (error) {
+    return error.message;
+  }
+}
+
+const BROWSER_UA =
+  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/150.0.0.0 Safari/537.36";
+
+module.exports = {
+  NAME,
+  ROOT,
+  PROFILE_DIR,
+  RUNS_DIR,
+  LIBRARY_FILE,
+  DOWNLOADS_DIR,
+  ensureDirs,
+  chromeBinary,
+  chromeUnavailable,
+  ChromeMissingError,
+  BROWSER_UA,
+};