@sprid/scout 0.1.0

package/src/jobs.js

@@ -0,0 +1,370 @@
+"use strict";
+
+const { randomUUID } = require("node:crypto");
+const { EventEmitter } = require("node:events");
+
+// Runs are async by construction: a tool starts one and hands back an id, and
+// the caller follows it with await_job. Nothing blocks a tool call for minutes.
+
+const bus = new EventEmitter();
+bus.setMaxListeners(0);
+
+const jobs = new Map();
+const IN_FLIGHT = new Set(["running", "paused"]);
+
+const AWAIT_DEFAULT_SECONDS = 900;
+const AWAIT_MIN_SECONDS = 10;
+const AWAIT_MAX_SECONDS = 3600;
+const PROGRESS_INTERVAL_MS = 20_000;
+const EVENT_COALESCE_MS = 1500;
+const PROJECTION_MIN_PROFILES = 10;
+const PROJECTION_JUMP_FACTOR = 3;
+
+function emit(type, job, extra) {
+  bus.emit("job", { type, jobId: job.id, job, ...extra });
+}
+
+function createJob(tool, meta = {}) {
+  const job = {
+    id: `${tool}-${randomUUID().slice(0, 8)}`,
+    tool,
+    status: "running",
+    startedAt: Date.now(),
+    finishedAt: null,
+    logs: [],
+    alert: null,
+    error: null,
+    result: null,
+    stopRequested: false,
+    meta,
+    // Discovery keeps a live picture of itself here as it measures.
+    state: { measured: [], keywordsConsumed: 0, keywordsTotal: 0, target: meta.maxAccounts ?? 0 },
+  };
+  jobs.set(job.id, job);
+  return job;
+}
+
+function findJob(id) {
+  return jobs.get(id) ?? null;
+}
+
+function listJobs() {
+  return [...jobs.values()];
+}
+
+function inFlight() {
+  return listJobs().filter((job) => IN_FLIGHT.has(job.status));
+}
+
+function busyWith(tool) {
+  return inFlight().find((job) => job.tool === tool) ?? null;
+}
+
+function log(job, text, isAlert = false) {
+  job.logs.push({ at: Date.now(), text: String(text), alert: isAlert });
+  if (job.logs.length > 400) job.logs.splice(0, job.logs.length - 400);
+  emit("log", job);
+}
+
+function setAlert(job, kind, message) {
+  job.alert = { kind, message, at: Date.now() };
+  job.status = "paused";
+  emit("state", job);
+}
+
+function clearAlert(job) {
+  job.alert = null;
+  emit("state", job);
+}
+
+/**
+ * Every loop calls this between steps. It parks while paused and answers false
+ * when the run has been told to stop, which is how a stop takes effect without
+ * killing the browser mid-page.
+ */
+function gate(job) {
+  return async () => {
+    if (job.stopRequested) return false;
+    while (job.status === "paused" && !job.stopRequested) {
+      await new Promise((resolve) => setTimeout(resolve, 400));
+    }
+    return !job.stopRequested;
+  };
+}
+
+function control(job, action) {
+  if (!IN_FLIGHT.has(job.status)) return { ok: false, reason: `Run is already ${job.status}.` };
+  if (action === "pause") job.status = "paused";
+  if (action === "resume") {
+    job.status = "running";
+    job.alert = null;
+  }
+  if (action === "stop") {
+    job.stopRequested = true;
+    job.status = "running";
+  }
+  emit("state", job);
+  return { ok: true };
+}
+
+function settle(job, status, payload) {
+  job.status = status;
+  job.finishedAt = Date.now();
+  if (status === "error") job.error = payload;
+  else job.result = payload;
+  emit("settled", job);
+}
+
+// ---- vitals ----------------------------------------------------------------
+
+function elapsedSeconds(job) {
+  return Math.round(((job.finishedAt ?? Date.now()) - job.startedAt) / 1000);
+}
+
+function humanElapsed(job) {
+  const seconds = elapsedSeconds(job);
+  if (seconds < 60) return `${seconds}s`;
+  const minutes = Math.floor(seconds / 60);
+  if (minutes < 60) return `${minutes}m ${seconds % 60}s`;
+  return `${Math.floor(minutes / 60)}h ${minutes % 60}m`;
+}
+
+/**
+ * The run's own arithmetic about itself. The projection is computed from the
+ * rate actually measured, so it is a reading for the caller rather than a
+ * promise to hand anyone.
+ */
+function vitals(job) {
+  const measured = job.state.measured;
+  const passed = measured.filter((a) => a.verdict?.passed);
+  const profilesMeasured = measured.length;
+  const passRate = profilesMeasured > 0 ? passed.length / profilesMeasured : 0;
+  const secondsPerProfile = profilesMeasured > 0 ? elapsedSeconds(job) / profilesMeasured : null;
+  const target = job.state.target || 0;
+  const missing = Math.max(0, target - passed.length);
+  let projectedSecondsToTarget = null;
+  if (profilesMeasured > 0 && passRate > 0 && secondsPerProfile !== null && missing > 0) {
+    projectedSecondsToTarget = Math.round((missing / passRate) * secondsPerProfile);
+  }
+  return {
+    profilesMeasured,
+    accountsPassed: passed.length,
+    target,
+    passRate: Math.round(passRate * 1000) / 1000,
+    secondsPerProfile: secondsPerProfile === null ? null : Math.round(secondsPerProfile),
+    keywordsConsumed: job.state.keywordsConsumed,
+    keywordsRemaining: Math.max(0, job.state.keywordsTotal - job.state.keywordsConsumed),
+    projectedSecondsToTarget,
+  };
+}
+
+function vitalsFields(v) {
+  return {
+    profiles_measured: v.profilesMeasured,
+    accounts_passed: v.accountsPassed,
+    target: v.target,
+    pass_rate: v.passRate,
+    seconds_per_profile: v.secondsPerProfile,
+    keywords_consumed: v.keywordsConsumed,
+    keywords_remaining: v.keywordsRemaining,
+    projected_seconds_to_target: v.projectedSecondsToTarget,
+  };
+}
+
+// ---- discovery event diffing ----------------------------------------------
+
+function discoveryBaseline(job) {
+  const measured = job.state.measured;
+  return {
+    passed: new Set(measured.filter((a) => a.verdict?.passed).map((a) => a.username)),
+    marksHigh: new Set(
+      measured.filter((a) => a.verdict?.highViewsPerFollower).map((a) => a.username)
+    ),
+    marksNear: new Set(measured.filter((a) => a.verdict?.nearMiss).map((a) => a.username)),
+    keywordsRemaining: vitals(job).keywordsRemaining,
+    projection: vitals(job).projectedSecondsToTarget,
+  };
+}
+
+function diffDiscovery(job, baseline, summarise) {
+  const measured = job.state.measured;
+  const newPassed = measured.filter((a) => a.verdict?.passed && !baseline.passed.has(a.username));
+  const newHigh = measured
+    .filter((a) => a.verdict?.highViewsPerFollower && !baseline.marksHigh.has(a.username))
+    .map((a) => a.username);
+  const newNear = measured
+    .filter((a) => a.verdict?.nearMiss && !baseline.marksNear.has(a.username))
+    .map((a) => a.username);
+  const v = vitals(job);
+  const fired = [];
+  if (newPassed.length > 0) fired.push("account_passed");
+  if (newHigh.length + newNear.length > 0) fired.push("mark_assigned");
+  if (v.keywordsRemaining === 0 && baseline.keywordsRemaining > 0) fired.push("keywords_exhausted");
+  if (
+    v.profilesMeasured >= PROJECTION_MIN_PROFILES &&
+    baseline.projection !== null &&
+    baseline.projection > 0 &&
+    v.projectedSecondsToTarget !== null &&
+    v.projectedSecondsToTarget > PROJECTION_JUMP_FACTOR * baseline.projection
+  ) {
+    fired.push("projection_jumped");
+  }
+  return {
+    fired,
+    extra: {
+      wake_events: fired,
+      summary: {
+        vitals: vitalsFields(v),
+        new_passed: newPassed.map(summarise),
+        new_marks: { high_views_per_follower: newHigh, near_miss: newNear },
+      },
+    },
+  };
+}
+
+// ---- awaiting --------------------------------------------------------------
+
+const surfacedAlert = new Map();
+
+function baseStatus(job) {
+  return {
+    job_id: job.id,
+    tool: job.tool,
+    status: job.status,
+    elapsed: humanElapsed(job),
+    started_at: new Date(job.startedAt).toISOString(),
+    recent_logs: job.logs.slice(-8).map((entry) => entry.text),
+    alert: job.alert ? { kind: job.alert.kind, message: job.alert.message } : null,
+    error: job.error,
+  };
+}
+
+function nextStep(job) {
+  if (job.status === "done") return "Call job_results for what the run collected.";
+  if (job.status === "error") return `The run failed: ${job.error}. Nothing else will happen.`;
+  if (job.status === "stopped") return "The run was stopped. job_results still reads what it measured before it ended.";
+  if (job.alert?.kind === "captcha") {
+    return "TikTok put a verification in front of the browser. Tell the user to solve it in the Chrome window, then call job_control with resume.";
+  }
+  if (job.alert?.kind === "bot") {
+    return "TikTok has stopped serving this session. Tell the user to check the Chrome window; they may need to wait or log in with another account, then call job_control with resume.";
+  }
+  if (job.alert?.kind === "unreadable") {
+    return "Something came back empty, usually a private or removed post. Call job_control with resume to try once more, or stop.";
+  }
+  if (job.status === "paused") return "The run is paused. Call job_control with resume to continue it.";
+  return "The run is working. Call await_job to be told the moment it finishes or needs a person.";
+}
+
+function awaitResult(job, event, extra = {}) {
+  if (event === "alert" && job.alert) surfacedAlert.set(job.id, job.alert);
+  return { ...baseStatus(job), event, next_step: nextStep(job), ...extra };
+}
+
+/**
+ * Blocks until the run does something worth hearing about. Progress
+ * notifications go out on a heartbeat so a long wait is not mistaken for a
+ * dead call.
+ */
+function waitForJob(job, timeoutSeconds, extra, wakeOn, summarise) {
+  return new Promise((resolve, reject) => {
+    const baselineStatus = job.status;
+    const baselineAlertKind = job.alert?.kind ?? null;
+    const watchEvents = wakeOn === "events" && job.tool === "discover";
+    const eventBaseline = watchEvents ? discoveryBaseline(job) : null;
+    let done = false;
+    let ticks = 0;
+    let coalesceTimer = null;
+
+    const cleanup = () => {
+      bus.off("job", onEvent);
+      clearInterval(heartbeat);
+      clearTimeout(timer);
+      if (coalesceTimer) clearTimeout(coalesceTimer);
+      extra.signal?.removeEventListener("abort", onAbort);
+    };
+    const finish = (event, eventsExtra) => {
+      if (done) return;
+      done = true;
+      cleanup();
+      resolve(awaitResult(job, event, eventsExtra));
+    };
+    const onAbort = () => {
+      if (done) return;
+      done = true;
+      cleanup();
+      reject(extra.signal?.reason ?? new Error("await_job was cancelled"));
+    };
+    const considerEvents = () => {
+      if (!watchEvents || !eventBaseline || coalesceTimer) return;
+      const diff = diffDiscovery(job, eventBaseline, summarise);
+      if (diff.fired.length === 0) return;
+      // Bursts are gathered into one wakeup rather than one per account.
+      coalesceTimer = setTimeout(() => {
+        coalesceTimer = null;
+        const settled = diffDiscovery(job, eventBaseline, summarise);
+        if (settled.fired.length > 0) finish("events", settled.extra);
+      }, EVENT_COALESCE_MS);
+    };
+    const onEvent = (jobEvent) => {
+      if (jobEvent.jobId !== job.id) return;
+      if (jobEvent.type === "settled") return finish("settled");
+      if (jobEvent.type === "state") {
+        const kind = job.alert?.kind ?? null;
+        if (kind !== null && kind !== baselineAlertKind) finish("alert");
+        else if (job.status !== baselineStatus) finish("state");
+        return;
+      }
+      considerEvents();
+    };
+    bus.on("job", onEvent);
+
+    const heartbeat = setInterval(() => {
+      const token = extra._meta?.progressToken;
+      if (token === undefined) return;
+      ticks += 1;
+      const last = job.logs.at(-1)?.text.replace(/\s+/g, " ").trim();
+      const message =
+        last && last.length > 0
+          ? last
+          : `Still waiting on ${job.tool} run ${job.id}, ${humanElapsed(job)} in.`;
+      void extra.sendNotification?.({
+        method: "notifications/progress",
+        params: { progressToken: token, progress: ticks, message },
+      }).catch(() => {});
+    }, PROGRESS_INTERVAL_MS);
+
+    const timer = setTimeout(() => finish("timeout"), timeoutSeconds * 1000);
+    if (extra.signal?.aborted) onAbort();
+    else extra.signal?.addEventListener("abort", onAbort);
+  });
+}
+
+module.exports = {
+  bus,
+  createJob,
+  findJob,
+  listJobs,
+  inFlight,
+  busyWith,
+  log,
+  setAlert,
+  clearAlert,
+  gate,
+  control,
+  settle,
+  emit,
+  vitals,
+  vitalsFields,
+  baseStatus,
+  nextStep,
+  awaitResult,
+  waitForJob,
+  humanElapsed,
+  elapsedSeconds,
+  surfacedAlert,
+  IN_FLIGHT,
+  AWAIT_DEFAULT_SECONDS,
+  AWAIT_MIN_SECONDS,
+  AWAIT_MAX_SECONDS,
+};

package/src/login-cli.js

@@ -0,0 +1,30 @@
+#!/usr/bin/env node
+"use strict";
+
+// Log in without going through an agent. Opens the window, waits, exits.
+
+const chrome = require("./chrome");
+const { chromeUnavailable, ensureDirs } = require("./config");
+
+async function main() {
+  ensureDirs();
+  const missing = chromeUnavailable();
+  if (missing) {
+    console.error(missing);
+    process.exit(1);
+  }
+  if (chrome.isLoggedIn()) {
+    console.log("A TikTok session is already saved. Nothing to do.");
+    return;
+  }
+  console.log("Opening Chrome on TikTok's login page. Log in there; this waits up to 5 minutes.");
+  const { loggedIn } = await chrome.login(300_000);
+  console.log(loggedIn ? "Logged in. The session is saved." : "No session appeared before the timeout.");
+  await chrome.shutdown();
+  process.exit(loggedIn ? 0 : 1);
+}
+
+main().catch((error) => {
+  console.error(error instanceof Error ? error.message : String(error));
+  process.exit(1);
+});

package/src/push-to-sprid.js

@@ -0,0 +1,194 @@
+#!/usr/bin/env node
+"use strict";
+
+// Push a finished download run into Sprid as inspirations.
+//
+// This is the whole integration surface. Scout never runs inside Sprid
+// and Sprid ships no scraping code: the carousels go up through the same
+// public API a phone screenshot upload uses, authenticated with the user's
+// own personal access token. Sprid's position is unchanged, which matters
+// because it is mid-audit with TikTok for the Content Posting API.
+//
+//   SPRID_URL=https://api.sprid.studio SPRID_PAT=sprd_... \
+//     node src/push-to-sprid.js <account-slug> <download-dir>
+//
+// The directory is whatever `job_results` reported for a download run, or any
+// tree of folders each holding slide_*.jpg + metadata.json.
+
+const fs = require("node:fs");
+const path = require("node:path");
+const { LIBRARY_FILE } = require("./config");
+
+const BASE = (process.env.SPRID_URL ?? "https://api.sprid.studio").replace(/\/+$/, "");
+const PAT = process.env.SPRID_PAT;
+
+function die(message) {
+  console.error(message);
+  process.exit(1);
+}
+
+async function api(route, init = {}) {
+  const response = await fetch(`${BASE}/api${route}`, {
+    ...init,
+    headers: {
+      Authorization: `Bearer ${PAT}`,
+      "Content-Type": "application/json",
+      ...(init.headers ?? {}),
+    },
+  });
+  if (!response.ok) {
+    throw new Error(`${response.status} ${route}: ${(await response.text()).slice(0, 300)}`);
+  }
+  return response.json();
+}
+
+/** Every folder holding a metadata.json, at any depth under root. */
+function findSlideshows(root) {
+  const found = [];
+  const walk = (dir) => {
+    let entries;
+    try {
+      entries = fs.readdirSync(dir, { withFileTypes: true });
+    } catch {
+      return;
+    }
+    if (entries.some((e) => e.isFile() && e.name === "metadata.json")) {
+      found.push(dir);
+      return;
+    }
+    for (const entry of entries) {
+      if (entry.isDirectory()) walk(path.join(dir, entry.name));
+    }
+  };
+  walk(root);
+  return found.sort();
+}
+
+function slideFiles(dir) {
+  return fs
+    .readdirSync(dir)
+    .filter((name) => /^slide_\d+\.(jpg|jpeg|webp|png)$/i.test(name))
+    .sort();
+}
+
+function contentTypeFor(filename) {
+  if (/\.webp$/i.test(filename)) return "image/webp";
+  if (/\.png$/i.test(filename)) return "image/png";
+  return "image/jpeg";
+}
+
+/**
+ * The metadata.json scout writes, mapped onto the shape Sprid stores.
+ * Views-per-follower is computed here because scout knows the author's
+ * follower count from the account measurement and the file does not.
+ */
+function toInspiration(meta, followers) {
+  const views = meta.views ?? null;
+  return {
+    source: "tiktok",
+    sourceUrl: meta.url,
+    title: (meta.description ?? "").slice(0, 120) || `@${meta.author?.username ?? "unknown"}`,
+    authorHandle: meta.author?.username ?? null,
+    caption: meta.description ?? "",
+    tags: meta.hashtags ?? [],
+    notes: `Pulled by scout. ${meta.slide_count ?? "?"} slides, published ${meta.published_at ?? "unknown"}.`,
+    authorFollowers: followers ?? undefined,
+    metrics: {
+      views: views ?? undefined,
+      likes: meta.likes ?? undefined,
+      comments: meta.comments ?? undefined,
+      shares: meta.shares ?? undefined,
+      favorites: meta.favorites ?? undefined,
+      engagementRate: meta.engagement_rate ?? undefined,
+      slideCount: meta.slide_count ?? undefined,
+      publishedAt: meta.published_at ?? undefined,
+      viewsPerFollower:
+        views != null && followers ? Math.round((views / followers) * 100) / 100 : undefined,
+    },
+  };
+}
+
+/** Follower counts per handle, read from scout's own library. */
+function followerIndex() {
+  const file = LIBRARY_FILE;
+  try {
+    const library = JSON.parse(fs.readFileSync(file, "utf8"));
+    const index = new Map();
+    for (const account of Object.values(library.accounts ?? {})) {
+      if (account.username && account.followers) {
+        index.set(String(account.username).toLowerCase(), account.followers);
+      }
+    }
+    return index;
+  } catch {
+    return new Map();
+  }
+}
+
+async function main() {
+  const [accountSlug, dir] = process.argv.slice(2);
+  if (!PAT) die("Set SPRID_PAT to a sprd_ token from /settings/tokens.");
+  if (!accountSlug || !dir) {
+    die("Usage: node src/push-to-sprid.js <account-slug> <download-dir>");
+  }
+  if (!fs.existsSync(dir)) die(`No such directory: ${dir}`);
+
+  const folders = findSlideshows(dir);
+  if (folders.length === 0) die(`No slideshows found under ${dir} (looking for metadata.json).`);
+
+  const followers = followerIndex();
+  console.log(`${folders.length} slideshow(s) to push to @${accountSlug}.`);
+  if (followers.size === 0) {
+    console.log("No follower counts in the local library, so these land without a benchmark.");
+  }
+
+  const created = [];
+  for (const [index, folder] of folders.entries()) {
+    const meta = JSON.parse(fs.readFileSync(path.join(folder, "metadata.json"), "utf8"));
+    const files = slideFiles(folder);
+    if (files.length === 0) {
+      console.log(`  [${index + 1}/${folders.length}] ${path.basename(folder)}: no slide images, skipped`);
+      continue;
+    }
+
+    const handle = String(meta.author?.username ?? "").toLowerCase();
+    const body = {
+      accountSlug,
+      ...toInspiration(meta, followers.get(handle) ?? null),
+      slides: files.map((name, position) => ({
+        filename: name,
+        contentType: contentTypeFor(name),
+        // Sprid's vision pass refines these; position is the honest first
+        // guess and it is what the shape signature needs anyway.
+        role: position === 0 ? "hook" : position === files.length - 1 ? "last" : "body",
+      })),
+    };
+
+    const result = await api("/inspirations", { method: "POST", body: JSON.stringify(body) });
+
+    for (const [i, slide] of result.slides.entries()) {
+      if (!slide.uploadUrl) continue;
+      const bytes = fs.readFileSync(path.join(folder, files[i]));
+      const put = await fetch(slide.uploadUrl, {
+        method: "PUT",
+        headers: { "Content-Type": contentTypeFor(files[i]) },
+        body: bytes,
+      });
+      if (!put.ok) throw new Error(`Upload failed for ${files[i]}: ${put.status}`);
+    }
+
+    created.push(result.inspirationId);
+    console.log(
+      `  [${index + 1}/${folders.length}] @${meta.author?.username ?? "?"} · ${files.length} slides -> inspiration ${result.inspirationId}`,
+    );
+  }
+
+  if (created.length === 0) die("Nothing was pushed.");
+  console.log(`\n${created.length} inspiration(s) created.`);
+
+}
+
+main().catch((error) => {
+  console.error(error instanceof Error ? error.message : String(error));
+  process.exit(1);
+});