@sprid/scout 0.1.0

package/src/download.js

@@ -0,0 +1,324 @@
+"use strict";
+
+const fs = require("node:fs");
+const path = require("node:path");
+const { acquire, handleChallenge, invoke } = require("./chrome");
+const { extractItem } = require("./tiktok");
+const { sheetFromFrames, ffmpegAvailable, DEFAULT_OFFSETS } = require("./sheet");
+const { capturePlayer, writeFrames } = require("./frames");
+const { readProfilePosts } = require("./analyze");
+const { DOWNLOADS_DIR } = require("./config");
+const jobsLib = require("./jobs");
+
+const DOWNLOAD_DEFAULTS = { count: 20, sort: "views", maxDays: 0 };
+const MAX_DOWNLOAD_COUNT = 200;
+
+const SORTS = {
+  views: (p) => p.stats?.playCount ?? 0,
+  likes: (p) => p.stats?.diggCount ?? 0,
+  comments: (p) => p.stats?.commentCount ?? 0,
+  shares: (p) => p.stats?.shareCount ?? 0,
+  favorites: (p) => p.stats?.collectCount ?? 0,
+  engagement: (p) => {
+    const s = p.stats ?? {};
+    const views = s.playCount ?? 0;
+    if (!views) return 0;
+    return (
+      ((s.diggCount ?? 0) + (s.commentCount ?? 0) + (s.shareCount ?? 0) + (s.collectCount ?? 0)) /
+      views
+    );
+  },
+};
+
+const SORT_VALUES = Object.keys(SORTS);
+
+function slugify(value) {
+  return String(value)
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, "-")
+    .replace(/^-+|-+$/g, "")
+    .slice(0, 60);
+}
+
+function humanAge(seconds) {
+  const days = Math.trunc(seconds / 86400);
+  if (days < 1) return "today";
+  if (days < 30) return `${days}d ago`;
+  if (days < 365) return `${Math.trunc(days / 30)}mo ago`;
+  return `${Math.trunc(days / 365)}y ago`;
+}
+
+/** Read one post. The stashed SSR blob answers first; the XHR is the fallback. */
+async function fetchItem(page, url, log, challenge) {
+  let captured = null;
+  const onResponse = async (response) => {
+    if (captured) return;
+    const responseUrl = response.url();
+    if (!responseUrl.includes("/api/item/detail") && !responseUrl.includes("/aweme/detail")) return;
+    try {
+      const data = await response.json();
+      const item = data.itemInfo?.itemStruct ?? data.aweme_detail;
+      if (item && typeof item === "object") captured = item;
+    } catch {}
+  };
+  page.on("response", onResponse);
+  try {
+    await page.goto(url, { waitUntil: "domcontentloaded", timeout: 30_000 });
+    await page.waitForTimeout(2500);
+    if ((await handleChallenge(page, log, challenge)) === "resumed") {
+      await page.goto(url, { waitUntil: "domcontentloaded", timeout: 30_000 });
+      await page.waitForTimeout(2500);
+    }
+  } catch (error) {
+    log(`  Warning: could not load page: ${String(error).slice(0, 100)}`, true);
+  }
+  const stashed = await page
+    .evaluate(
+      invoke(
+        `() => window.__SS_REHYDRATION__ || (document.getElementById('__UNIVERSAL_DATA_FOR_REHYDRATION__') || {}).textContent || null`
+      )
+    )
+    .catch(() => null);
+  page.off("response", onResponse);
+  return extractItem(stashed) ?? captured;
+}
+
+async function writeSlideshow(post, dir, position, log) {
+  const images = post.imagePost?.images ?? [];
+  if (images.length === 0) {
+    log("  This post is not a slideshow (no images found).", true);
+    return null;
+  }
+  const username = post.author?.uniqueId || "unknown";
+  const folder = `${String(position).padStart(2, "0")}_${slugify(username)}_${slugify(post.id)}`;
+  const postDir = path.join(dir, folder);
+  fs.mkdirSync(postDir, { recursive: true });
+
+  let slides = 0;
+  for (const [index, image] of images.entries()) {
+    const source = image.imageURL?.urlList?.[0];
+    if (!source) continue;
+    try {
+      const response = await fetch(source, { headers: { Referer: "https://www.tiktok.com/" } });
+      if (!response.ok) continue;
+      const extension = response.headers.get("content-type")?.includes("webp") ? "webp" : "jpg";
+      const buffer = Buffer.from(await response.arrayBuffer());
+      fs.writeFileSync(
+        path.join(postDir, `slide_${String(index + 1).padStart(2, "0")}.${extension}`),
+        buffer
+      );
+      slides += 1;
+    } catch (error) {
+      log(`  Warning: slide ${index + 1} failed: ${String(error).slice(0, 80)}`, true);
+    }
+  }
+
+  const stats = post.stats ?? {};
+  const createTime = post.createTime ?? 0;
+  const elapsed = Date.now() / 1000 - createTime;
+  const views = stats.playCount ?? 0;
+  const interactions =
+    (stats.diggCount ?? 0) + (stats.commentCount ?? 0) + (stats.shareCount ?? 0) + (stats.collectCount ?? 0);
+
+  // Everything a ranking this tool does not offer would need, written beside
+  // the images so it can be worked out from what came down.
+  const metadata = {
+    id: post.id,
+    url: `https://www.tiktok.com/@${username}/photo/${post.id}`,
+    description: post.desc ?? "",
+    hashtags: (post.textExtra ?? []).map((t) => t.hashtagName).filter(Boolean),
+    views,
+    likes: stats.diggCount ?? 0,
+    comments: stats.commentCount ?? 0,
+    shares: stats.shareCount ?? 0,
+    favorites: stats.collectCount ?? 0,
+    engagement_rate: views ? Math.round((interactions / views) * 100 * 1e4) / 1e4 : 0,
+    slide_count: slides,
+    published_at: createTime ? new Date(createTime * 1000).toISOString() : null,
+    days_since_published: Math.trunc(elapsed / 86400),
+    age: createTime ? humanAge(elapsed) : null,
+    author: {
+      username,
+      display_name: post.author?.nickname,
+      verified: post.author?.verified ?? false,
+    },
+    music: { title: post.music?.title, author: post.music?.authorName },
+  };
+  fs.writeFileSync(path.join(postDir, "metadata.json"), JSON.stringify(metadata, null, 2));
+  log(`  Saved ${folder} - ${slides} slides`);
+  return { folder, slides, metadata };
+}
+
+/**
+ * A video, read as frames off its own player.
+ *
+ * Not downloaded: the signed playable url answers 403 from node and from inside
+ * the signed-in page alike, because the player runs on Media Source Extensions
+ * and its src is a blob handle with nothing to fetch behind it. Seeking the
+ * player and drawing to a canvas gets the pixels, costs nothing extra on a long
+ * video, and leaves no copy of someone's file on disk - which is the right
+ * shape for evidence anyway.
+ */
+async function writeVideoFrames(post, page, dir, position, log, { offsets } = {}) {
+  const username = post.author?.uniqueId || "unknown";
+  const url = `https://www.tiktok.com/@${username}/video/${post.id}`;
+  const folder = `${String(position).padStart(2, "0")}_${slugify(username)}_${slugify(post.id)}`;
+  const postDir = path.join(dir, folder);
+
+  let captured;
+  try {
+    if (!page.url().includes(post.id)) {
+      await page.goto(url, { waitUntil: "domcontentloaded", timeout: 45_000 });
+      await page.waitForTimeout(2500);
+    }
+    captured = await capturePlayer((source) => page.evaluate(source), {
+      offsets: offsets ?? DEFAULT_OFFSETS,
+    });
+  } catch (error) {
+    log(`  Warning: could not read the player: ${String(error.message).slice(0, 120)}`, true);
+    return null;
+  }
+  if (!captured.shots?.length) {
+    log("  The player gave no frames.", true);
+    return null;
+  }
+
+  const frames = writeFrames(captured.shots, postDir);
+  let sheet = null;
+  if (await ffmpegAvailable()) {
+    try {
+      sheet = await sheetFromFrames(frames, path.join(postDir, "sheet.png"));
+    } catch (error) {
+      log(`  Warning: no sheet: ${String(error.message).slice(0, 120)}`, true);
+    }
+  } else {
+    log("  ffmpeg is not installed, so the frames are there but not tiled into a sheet.", true);
+  }
+
+  const stats = post.stats ?? {};
+  const metadata = {
+    id: post.id,
+    type: "video",
+    url,
+    description: post.desc ?? "",
+    hashtags: (post.textExtra ?? []).map((t) => t.hashtagName).filter(Boolean),
+    duration_seconds: captured.duration ?? post.video?.duration ?? null,
+    frames_at_seconds: frames.map((f) => f.at),
+    views: stats.playCount ?? 0,
+    likes: stats.diggCount ?? 0,
+    comments: stats.commentCount ?? 0,
+    shares: stats.shareCount ?? 0,
+    favorites: stats.collectCount ?? 0,
+    published_at: post.createTime ? new Date(post.createTime * 1000).toISOString() : null,
+    sheet: sheet ? "sheet.png" : null,
+  };
+  fs.writeFileSync(path.join(postDir, "metadata.json"), JSON.stringify(metadata, null, 2));
+  log(`  ${frames.length} frames${sheet ? " and a sheet" : ""}.`);
+  return { folder, slides: 0, metadata, sheet: sheet ? { offsets: sheet.offsets } : null };
+}
+
+async function runDownload(job) {
+  const { accounts, links, count, sort, maxDays, includeVideos, sheetOffsets } = job.meta;
+  const log = (text, alert) => jobsLib.log(job, text, alert);
+  const gate = jobsLib.gate(job);
+  const runDir = path.join(DOWNLOADS_DIR, job.id);
+  fs.mkdirSync(runDir, { recursive: true });
+
+  const session = await acquire();
+  const challenge = {
+    hold: gate,
+    signal: session.challengeSignal,
+    onDetected: (prompt) => jobsLib.setAlert(job, "captcha", prompt),
+  };
+  const saved = [];
+
+  try {
+    if (links?.length) {
+      for (const [index, url] of links.entries()) {
+        if (!(await gate())) break;
+        log(`\n[${index + 1}/${links.length}] ${url}`);
+        const item = await fetchItem(session.page, url, log, challenge);
+        if (!item) {
+          log("  Could not read this post (private, removed, or unreadable).", true);
+          continue;
+        }
+        // A link is whatever it is: a slideshow gets its images, a video gets
+        // its mp4 and a frame sheet, and refusing the second is only useful
+        // when nobody asked for it.
+        const written = item.imagePost
+          ? await writeSlideshow(item, runDir, index + 1, log)
+          : includeVideos
+            ? await writeVideoFrames(item, session.page, runDir, index + 1, log, { offsets: sheetOffsets })
+            : (log("  This post is a video; pass include_videos to pull it.", true), null);
+        if (written) saved.push(written);
+      }
+    } else {
+      let position = 0;
+      for (const account of accounts) {
+        if (!(await gate())) break;
+        const handle = account.username.replace(/^@+/, "");
+        const perCount = Math.min(account.count ?? count, MAX_DOWNLOAD_COUNT);
+        const perSort = account.sort ?? sort;
+        const perDays = account.max_days ?? maxDays;
+        log(`\n@${handle}: taking the top ${perCount} by ${perSort}${perDays ? ` from the last ${perDays} days` : ""}.`);
+
+        const { posts } = await readProfilePosts({
+          page: session.page,
+          username: handle,
+          log,
+          gate,
+          challenge,
+          maxScrolls: 30,
+          until: perDays ? Date.now() / 1000 - perDays * 86400 : null,
+        });
+
+        // Read, then window, then rank, then take. The window is the ground the
+        // ranking happens on rather than a filter applied afterwards.
+        const cutoff = perDays ? Date.now() / 1000 - perDays * 86400 : 0;
+        const wanted = posts
+          .filter((p) => (includeVideos ? true : Boolean(p.imagePost)) && (p.createTime ?? 0) >= cutoff)
+          .sort((a, b) => SORTS[perSort](b) - SORTS[perSort](a))
+          .slice(0, perCount);
+
+        if (wanted.length === 0) {
+          log(`  No ${includeVideos ? "posts" : "slideshows"} matched inside that window.`);
+          continue;
+        }
+        const accountDir = path.join(runDir, slugify(handle));
+        fs.mkdirSync(accountDir, { recursive: true });
+        for (const post of wanted) {
+          if (!(await gate())) break;
+          position += 1;
+          const written = post.imagePost
+            ? await writeSlideshow(post, accountDir, position, log)
+            : await writeVideoFrames(post, session.page, accountDir, position, log, { offsets: sheetOffsets });
+          if (written) saved.push({ ...written, username: handle });
+        }
+      }
+    }
+  } finally {
+    await session.release();
+  }
+
+  log(`\nDone. ${saved.length} post${saved.length === 1 ? "" : "s"} in ${runDir}`);
+  const sheets = saved.filter((entry) => entry.sheet).length;
+  return {
+    directory: runDir,
+    downloaded: saved.length,
+    frame_sheets: sheets,
+    read_the_sheets: sheets
+      ? "Each video folder holds its frames and sheet.png: tiles left to right at the offsets in its metadata.json. Read it to describe the structure - the cover frame, the opening claim, when the first cut lands - never to republish it."
+      : undefined,
+    posts: saved.map((entry) => ({
+      folder: entry.folder,
+      type: entry.metadata.type ?? "slideshow",
+      slides: entry.slides,
+      sheet_offsets_seconds: entry.sheet ? entry.sheet.offsets : undefined,
+      url: entry.metadata.url,
+      views: entry.metadata.views,
+      engagement_rate: entry.metadata.engagement_rate,
+    })),
+  };
+}
+
+module.exports = { runDownload, DOWNLOAD_DEFAULTS, MAX_DOWNLOAD_COUNT, SORT_VALUES };

package/src/frames.js

@@ -0,0 +1,85 @@
+"use strict";
+
+// Frames off the player, rather than a file off a CDN.
+//
+// Measured before it was built this way: the signed playable url answers 403
+// both from node and from inside the signed-in page, because the player runs on
+// Media Source Extensions - its `<video>` src is a `blob:` handle with no bytes
+// behind it to fetch. What does work is seeking that player and drawing each
+// frame to a canvas, which is not tainted, so the pixels come back.
+//
+// It is also the better tool for the job. Nothing is copied to disk as a file,
+// the length of the video costs nothing (a four-minute post is five seeks), and
+// what comes back is what a teardown needs: the cover frame, and the seconds
+// where a viewer decides.
+
+const fs = require("node:fs");
+const path = require("node:path");
+
+/** Frame 0 is the cover the feed shows; 1-3s is where a viewer leaves. */
+const DEFAULT_OFFSETS = [0, 1, 2, 5, 10];
+
+/**
+ * The capture, as source to run inside the page. Returns a JSON string so it
+ * can cross any bridge - playwright's evaluate or an Apple Events round trip.
+ */
+function captureSource(offsets = DEFAULT_OFFSETS, width = 360) {
+  return `(async () => {
+  try {
+    const video = document.querySelector('video');
+    if (!video) return JSON.stringify({ error: 'this page has no video player' });
+    for (let waited = 0; waited < 20 && !video.videoWidth; waited++) {
+      await new Promise((r) => setTimeout(r, 400));
+    }
+    if (!video.videoWidth) return JSON.stringify({ error: 'the player never reported a size' });
+    const canvas = document.createElement('canvas');
+    const w = Math.min(video.videoWidth, ${width});
+    const h = Math.round((video.videoHeight / video.videoWidth) * w);
+    canvas.width = w; canvas.height = h;
+    const ctx = canvas.getContext('2d');
+    video.muted = true;
+    try { await video.play(); } catch (e) {}
+    const shots = [];
+    for (const at of ${JSON.stringify(offsets)}) {
+      if (video.duration && at >= video.duration) continue;
+      await new Promise((done) => {
+        const finish = () => { video.removeEventListener('seeked', finish); done(); };
+        video.addEventListener('seeked', finish);
+        video.currentTime = at;
+        setTimeout(finish, 2500);
+      });
+      await new Promise((r) => setTimeout(r, 350));
+      ctx.drawImage(video, 0, 0, w, h);
+      shots.push({ at, data: canvas.toDataURL('image/jpeg', 0.72) });
+    }
+    try { video.pause(); } catch (e) {}
+    return JSON.stringify({ width: w, height: h, duration: video.duration || null, shots });
+  } catch (e) {
+    return JSON.stringify({ error: String((e && e.message) || e) });
+  }
+})()`;
+}
+
+/**
+ * @param {(source: string) => Promise<string>} evaluate  runs the capture in the
+ *   page and hands back whatever it returned. The two lanes differ only here.
+ */
+async function capturePlayer(evaluate, { offsets = DEFAULT_OFFSETS, width = 360 } = {}) {
+  const raw = await evaluate(captureSource(offsets, width));
+  if (!raw) throw new Error("the page returned nothing");
+  const payload = typeof raw === "string" ? JSON.parse(raw) : raw;
+  if (payload.error) throw new Error(payload.error);
+  return payload;
+}
+
+/** Writes one jpg per frame and returns their paths in offset order. */
+function writeFrames(shots, dir) {
+  fs.mkdirSync(dir, { recursive: true });
+  return shots.map(({ at, data }) => {
+    const file = path.join(dir, `frame_${String(at).replace(".", "_")}s.jpg`);
+    fs.writeFileSync(file, Buffer.from(String(data).split(",")[1] ?? "", "base64"));
+    return { at, file };
+  });
+}
+
+module.exports = { capturePlayer, captureSource, writeFrames, DEFAULT_OFFSETS };