@sprid/scout 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +14 -0
- package/README.md +115 -0
- package/package.json +32 -0
- package/src/ads/cli.mjs +82 -0
- package/src/ads/extract.mjs +108 -0
- package/src/ads/harvest.mjs +90 -0
- package/src/ads/push.mjs +56 -0
- package/src/ads/run.js +67 -0
- package/src/analyze.js +155 -0
- package/src/chrome.js +396 -0
- package/src/config.js +83 -0
- package/src/discover.js +463 -0
- package/src/download.js +324 -0
- package/src/frames.js +85 -0
- package/src/jobs.js +370 -0
- package/src/login-cli.js +30 -0
- package/src/push-to-sprid.js +194 -0
- package/src/quick.js +318 -0
- package/src/register.js +27 -0
- package/src/server.js +881 -0
- package/src/sheet.js +106 -0
- package/src/store.js +130 -0
- package/src/tiktok.js +391 -0
package/src/sheet.js
ADDED
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
// Frames, tiled into one readable sheet.
|
|
4
|
+
//
|
|
5
|
+
// A model cannot watch thirty seconds of video, and a teardown does not need it
|
|
6
|
+
// to. What decides whether a reel is watched happens in its first seconds, so
|
|
7
|
+
// the default offsets are dense at the start and thin out afterwards: the cover
|
|
8
|
+
// frame, then the moments where a viewer either stays or leaves. The frames
|
|
9
|
+
// themselves come from the player (see frames.js), not from a downloaded file.
|
|
10
|
+
//
|
|
11
|
+
// They are evidence, exactly like the slideshow images beside them: there so a
|
|
12
|
+
// structure can be described and rebuilt, never republished.
|
|
13
|
+
|
|
14
|
+
const fs = require("node:fs");
|
|
15
|
+
const path = require("node:path");
|
|
16
|
+
const util = require("node:util");
|
|
17
|
+
const { execFile } = require("node:child_process");
|
|
18
|
+
|
|
19
|
+
const run = util.promisify(execFile);
|
|
20
|
+
|
|
21
|
+
/** Frame 0 is the cover; 1-3s is the skip window; the rest is shape. */
|
|
22
|
+
const DEFAULT_OFFSETS = [0, 1, 2, 5, 10];
|
|
23
|
+
|
|
24
|
+
class FfmpegMissingError extends Error {
|
|
25
|
+
constructor() {
|
|
26
|
+
super(
|
|
27
|
+
"ffmpeg is not installed, so frames cannot be taken from a video. Install it (brew install ffmpeg) " +
|
|
28
|
+
"or pull the video without a sheet."
|
|
29
|
+
);
|
|
30
|
+
this.name = "FfmpegMissingError";
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
async function ffmpegAvailable() {
|
|
35
|
+
try {
|
|
36
|
+
await run("ffmpeg", ["-version"]);
|
|
37
|
+
return true;
|
|
38
|
+
} catch {
|
|
39
|
+
return false;
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* The same sheet, from frames that were captured rather than extracted: tiles
|
|
45
|
+
* left to right in the order given, a white gutter between them, and the
|
|
46
|
+
* timecodes returned as data because `drawtext` needs a freetype-enabled ffmpeg
|
|
47
|
+
* and ordinary Homebrew builds do not have one.
|
|
48
|
+
*/
|
|
49
|
+
async function sheetFromFrames(frames, outFile) {
|
|
50
|
+
if (frames.length === 0) return null;
|
|
51
|
+
if (!(await ffmpegAvailable())) throw new FfmpegMissingError();
|
|
52
|
+
const inputs = frames.flatMap(({ file }) => ["-i", file]);
|
|
53
|
+
const gutter = frames.map((_, i) => `[${i}:v]pad=iw+6:ih:6:0:color=white[v${i}]`).join(";");
|
|
54
|
+
const chain = frames.map((_, i) => `[v${i}]`).join("");
|
|
55
|
+
await run("ffmpeg", [
|
|
56
|
+
"-v", "error", ...inputs,
|
|
57
|
+
"-filter_complex", `${gutter};${chain}hstack=inputs=${frames.length}[out]`,
|
|
58
|
+
"-map", "[out]", "-y", outFile,
|
|
59
|
+
]);
|
|
60
|
+
return { file: outFile, offsets: frames.map((f) => f.at) };
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/**
|
|
64
|
+
* A wall: the same moment across many posts, four to a row.
|
|
65
|
+
*
|
|
66
|
+
* A sheet reads one post down time. A wall reads one instant across a niche,
|
|
67
|
+
* and it is where a pattern becomes visible - twelve covers side by side say
|
|
68
|
+
* more about what a category's opening frame does than twelve teardowns do.
|
|
69
|
+
*
|
|
70
|
+
* Rows are stacked rather than tiled in one pass, because `xstack` needs every
|
|
71
|
+
* input to agree on size and these come from different posts.
|
|
72
|
+
*/
|
|
73
|
+
async function wallFromFrames(frames, outFile, { cols = 4, cellWidth = 300 } = {}) {
|
|
74
|
+
if (frames.length === 0) return null;
|
|
75
|
+
if (!(await ffmpegAvailable())) throw new FfmpegMissingError();
|
|
76
|
+
const rows = Math.ceil(frames.length / cols);
|
|
77
|
+
const inputs = frames.flatMap(({ file }) => ["-i", file]);
|
|
78
|
+
const parts = [];
|
|
79
|
+
// One scale per cell so a stray resolution cannot break the stack, then a
|
|
80
|
+
// gutter, then the rows, then the column.
|
|
81
|
+
frames.forEach((_, i) => {
|
|
82
|
+
parts.push(`[${i}:v]scale=${cellWidth}:-2,pad=iw+6:ih+6:3:3:color=white[c${i}]`);
|
|
83
|
+
});
|
|
84
|
+
const rowLabels = [];
|
|
85
|
+
for (let r = 0; r < rows; r++) {
|
|
86
|
+
const cells = frames.slice(r * cols, (r + 1) * cols).map((_, i) => `[c${r * cols + i}]`);
|
|
87
|
+
if (cells.length === 0) continue;
|
|
88
|
+
const label = `[r${r}]`;
|
|
89
|
+
parts.push(cells.length === 1 ? `${cells[0]}null${label}` : `${cells.join("")}hstack=inputs=${cells.length}${label}`);
|
|
90
|
+
rowLabels.push(label);
|
|
91
|
+
}
|
|
92
|
+
// A short last row would refuse to stack against the full ones, so it is
|
|
93
|
+
// padded out to the same width instead of being dropped.
|
|
94
|
+
if (rowLabels.length > 1) {
|
|
95
|
+
const full = cols * (cellWidth + 6);
|
|
96
|
+
parts.push(`${rowLabels[rowLabels.length - 1]}pad=${full}:ih:0:0:color=white[rlast]`);
|
|
97
|
+
rowLabels[rowLabels.length - 1] = "[rlast]";
|
|
98
|
+
parts.push(`${rowLabels.join("")}vstack=inputs=${rowLabels.length}[out]`);
|
|
99
|
+
} else {
|
|
100
|
+
parts.push(`${rowLabels[0]}null[out]`);
|
|
101
|
+
}
|
|
102
|
+
await run("ffmpeg", ["-v", "error", ...inputs, "-filter_complex", parts.join(";"), "-map", "[out]", "-y", outFile]);
|
|
103
|
+
return { file: outFile, cells: frames.length, cols };
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
module.exports = { sheetFromFrames, wallFromFrames, ffmpegAvailable, DEFAULT_OFFSETS, FfmpegMissingError };
|
package/src/store.js
ADDED
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
const fs = require("node:fs");
|
|
4
|
+
const path = require("node:path");
|
|
5
|
+
const { RUNS_DIR, LIBRARY_FILE, ensureDirs } = require("./config");
|
|
6
|
+
|
|
7
|
+
// Everything lives in plain JSON on the user's disk. Nothing leaves the
|
|
8
|
+
// machine, and a run's accounts are written the moment they are measured, so a
|
|
9
|
+
// run that dies half way still leaves behind everything it had read.
|
|
10
|
+
|
|
11
|
+
function writeAtomic(file, value) {
|
|
12
|
+
ensureDirs();
|
|
13
|
+
const tmp = `${file}.${process.pid}.tmp`;
|
|
14
|
+
fs.writeFileSync(tmp, JSON.stringify(value, null, 2));
|
|
15
|
+
fs.renameSync(tmp, file);
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
function readJson(file, fallback) {
|
|
19
|
+
try {
|
|
20
|
+
return JSON.parse(fs.readFileSync(file, "utf8"));
|
|
21
|
+
} catch {
|
|
22
|
+
return fallback;
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
function runFile(runId) {
|
|
27
|
+
return path.join(RUNS_DIR, `${runId}.json`);
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
function saveRun(run) {
|
|
31
|
+
writeAtomic(runFile(run.id), run);
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
function loadRun(runId) {
|
|
35
|
+
return readJson(runFile(runId), null);
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
function listRuns() {
|
|
39
|
+
ensureDirs();
|
|
40
|
+
const runs = [];
|
|
41
|
+
for (const entry of fs.readdirSync(RUNS_DIR)) {
|
|
42
|
+
if (!entry.endsWith(".json")) continue;
|
|
43
|
+
const run = readJson(path.join(RUNS_DIR, entry), null);
|
|
44
|
+
if (run) runs.push(run);
|
|
45
|
+
}
|
|
46
|
+
runs.sort((a, b) => (b.started_at ?? "").localeCompare(a.started_at ?? ""));
|
|
47
|
+
return runs;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
// The library is the union of every account a discovery run has ever passed.
|
|
51
|
+
// A later run measuring the same account overwrites the older reading, because
|
|
52
|
+
// the newer numbers are the truer ones.
|
|
53
|
+
|
|
54
|
+
function readLibrary() {
|
|
55
|
+
return readJson(LIBRARY_FILE, { accounts: {} });
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
function fileAccounts(runId, runName, accounts) {
|
|
59
|
+
const library = readLibrary();
|
|
60
|
+
const foundAt = new Date().toISOString();
|
|
61
|
+
for (const account of accounts) {
|
|
62
|
+
if (!account.username) continue;
|
|
63
|
+
library.accounts[account.username.toLowerCase()] = {
|
|
64
|
+
...account,
|
|
65
|
+
run_id: runId,
|
|
66
|
+
run_name: runName,
|
|
67
|
+
found_at: foundAt,
|
|
68
|
+
};
|
|
69
|
+
}
|
|
70
|
+
writeAtomic(LIBRARY_FILE, library);
|
|
71
|
+
return Object.keys(library.accounts).length;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
function getAccount(username) {
|
|
75
|
+
const key = String(username).replace(/^@+/, "").toLowerCase();
|
|
76
|
+
return readLibrary().accounts[key] ?? null;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
const SORTS = {
|
|
80
|
+
found: (a, b) => (b.found_at ?? "").localeCompare(a.found_at ?? ""),
|
|
81
|
+
avg_views: (a, b) => windowedViews(b) - windowedViews(a),
|
|
82
|
+
followers: (a, b) => (b.followers ?? 0) - (a.followers ?? 0),
|
|
83
|
+
slideshow_ratio: (a, b) =>
|
|
84
|
+
(b.metrics?.slideshow_ratio ?? 0) - (a.metrics?.slideshow_ratio ?? 0),
|
|
85
|
+
view_follower_ratio: (a, b) =>
|
|
86
|
+
(b.metrics?.view_follower_ratio ?? 0) - (a.metrics?.view_follower_ratio ?? 0),
|
|
87
|
+
consistency_score: (a, b) =>
|
|
88
|
+
(b.metrics?.consistency_score ?? 0) - (a.metrics?.consistency_score ?? 0),
|
|
89
|
+
last_post_days_ago: (a, b) =>
|
|
90
|
+
(a.metrics?.last_post_days_ago ?? 1e9) - (b.metrics?.last_post_days_ago ?? 1e9),
|
|
91
|
+
};
|
|
92
|
+
|
|
93
|
+
/** The averaged-views field carries its window in its name, so find it by shape. */
|
|
94
|
+
function windowedViews(account) {
|
|
95
|
+
const metrics = account.metrics ?? {};
|
|
96
|
+
const key = Object.keys(metrics).find((k) => k.startsWith("avg_views_last_"));
|
|
97
|
+
return key ? metrics[key] ?? 0 : 0;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
function searchAccounts({ query, runId, sort = "found", limit = 25, offset = 0 }) {
|
|
101
|
+
let rows = Object.values(readLibrary().accounts);
|
|
102
|
+
if (runId) rows = rows.filter((row) => row.run_id === runId);
|
|
103
|
+
if (query) {
|
|
104
|
+
const needle = query.toLowerCase();
|
|
105
|
+
rows = rows.filter((row) => {
|
|
106
|
+
const hay = [
|
|
107
|
+
row.username,
|
|
108
|
+
row.display_name,
|
|
109
|
+
row.bio,
|
|
110
|
+
...(row.metrics?.top_hashtags ?? []),
|
|
111
|
+
]
|
|
112
|
+
.join(" ")
|
|
113
|
+
.toLowerCase();
|
|
114
|
+
return hay.includes(needle);
|
|
115
|
+
});
|
|
116
|
+
}
|
|
117
|
+
rows.sort(SORTS[sort] ?? SORTS.found);
|
|
118
|
+
return { total: rows.length, accounts: rows.slice(offset, offset + limit) };
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
module.exports = {
|
|
122
|
+
saveRun,
|
|
123
|
+
loadRun,
|
|
124
|
+
listRuns,
|
|
125
|
+
fileAccounts,
|
|
126
|
+
getAccount,
|
|
127
|
+
searchAccounts,
|
|
128
|
+
windowedViews,
|
|
129
|
+
SORT_ORDERS: Object.keys(SORTS),
|
|
130
|
+
};
|
package/src/tiktok.js
ADDED
|
@@ -0,0 +1,391 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
const https = require("node:https");
|
|
4
|
+
const { BROWSER_UA } = require("./config");
|
|
5
|
+
|
|
6
|
+
// TikTok is read the way any person reads it: pages are loaded and scrolled,
|
|
7
|
+
// and the JSON the page fetches for itself is read off the wire. Nothing here
|
|
8
|
+
// calls a private API directly.
|
|
9
|
+
|
|
10
|
+
const FEED_ENDPOINTS = ["/api/post/item_list/", "/api/user/post/"];
|
|
11
|
+
const DETAIL_ENDPOINT = "/api/user/detail/";
|
|
12
|
+
const ITEM_ENDPOINTS = ["/api/item/detail", "/aweme/detail"];
|
|
13
|
+
|
|
14
|
+
const TOP = {
|
|
15
|
+
id: "top",
|
|
16
|
+
label: "Top",
|
|
17
|
+
url: (keyword) => `https://www.tiktok.com/search?q=${encodeURIComponent(keyword)}&type=0`,
|
|
18
|
+
items: (payload) => (payload.data ?? []).map((wrapper) => wrapper.item ?? {}),
|
|
19
|
+
followers: (item) => item.author?.followerCount ?? 0,
|
|
20
|
+
confirm: [
|
|
21
|
+
'[data-e2e="search-top-tab"]',
|
|
22
|
+
'a[href*="type=0"]',
|
|
23
|
+
'[role="tab"]:has-text("Top")',
|
|
24
|
+
'button:has-text("Top")',
|
|
25
|
+
'span:has-text("Top")',
|
|
26
|
+
],
|
|
27
|
+
};
|
|
28
|
+
|
|
29
|
+
const PHOTO = {
|
|
30
|
+
id: "photo",
|
|
31
|
+
label: "Photo",
|
|
32
|
+
url: (keyword) => `https://www.tiktok.com/search/photo?q=${encodeURIComponent(keyword)}`,
|
|
33
|
+
items: (payload) => payload.item_list ?? [],
|
|
34
|
+
followers: (item) => item.authorStats?.followerCount ?? item.author?.followerCount ?? 0,
|
|
35
|
+
};
|
|
36
|
+
|
|
37
|
+
function round(value, digits = 0) {
|
|
38
|
+
const factor = 10 ** digits;
|
|
39
|
+
return Math.round(value * factor) / factor;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
function parseCount(text) {
|
|
43
|
+
if (!text) return 0;
|
|
44
|
+
const cleaned = text.trim().toUpperCase().replace(/,/g, "").replace(/\s/g, "");
|
|
45
|
+
const value = parseFloat(cleaned);
|
|
46
|
+
if (Number.isNaN(value)) return 0;
|
|
47
|
+
if (cleaned.endsWith("M")) return Math.trunc(value * 1e6);
|
|
48
|
+
if (cleaned.endsWith("K")) return Math.trunc(value * 1e3);
|
|
49
|
+
return Math.trunc(value);
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
function fetchHtml(url, timeoutMs = 15_000) {
|
|
53
|
+
return new Promise((resolve, reject) => {
|
|
54
|
+
const request = https.get(
|
|
55
|
+
url,
|
|
56
|
+
{ headers: { "User-Agent": BROWSER_UA, "Accept-Language": "en-US,en;q=0.9" } },
|
|
57
|
+
(response) => {
|
|
58
|
+
const location = response.headers.location;
|
|
59
|
+
if (response.statusCode >= 300 && response.statusCode < 400 && location) {
|
|
60
|
+
response.resume();
|
|
61
|
+
fetchHtml(new URL(location, url).toString(), timeoutMs).then(resolve, reject);
|
|
62
|
+
return;
|
|
63
|
+
}
|
|
64
|
+
let body = "";
|
|
65
|
+
response.setEncoding("utf8");
|
|
66
|
+
response.on("data", (chunk) => (body += chunk));
|
|
67
|
+
response.on("end", () => resolve(body));
|
|
68
|
+
}
|
|
69
|
+
);
|
|
70
|
+
request.setTimeout(timeoutMs, () => request.destroy(new Error(`Timed out fetching ${url}`)));
|
|
71
|
+
request.on("error", reject);
|
|
72
|
+
});
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/** The server-rendered blob carries the account totals the feed JSON rounds off. */
|
|
76
|
+
function extractEmbeddedUser(html) {
|
|
77
|
+
const match = html.match(/id="__UNIVERSAL_DATA_FOR_REHYDRATION__"[^>]*>([\s\S]*?)<\/script>/);
|
|
78
|
+
if (!match) return {};
|
|
79
|
+
try {
|
|
80
|
+
const parsed = JSON.parse(match[1]);
|
|
81
|
+
const info = parsed.__DEFAULT_SCOPE__?.["webapp.user-detail"]?.userInfo ?? {};
|
|
82
|
+
const user = info.user ?? {};
|
|
83
|
+
const stats = { ...info.stats, ...info.statsV2 };
|
|
84
|
+
const count = (value) => {
|
|
85
|
+
const n = typeof value === "string" ? parseInt(value, 10) : value;
|
|
86
|
+
if (typeof n !== "number" || !Number.isFinite(n) || n < 0) return undefined;
|
|
87
|
+
return n;
|
|
88
|
+
};
|
|
89
|
+
const text = (value) => (typeof value === "string" ? value : undefined);
|
|
90
|
+
return {
|
|
91
|
+
nickname: text(user.nickname),
|
|
92
|
+
signature: text(user.signature),
|
|
93
|
+
verified: Boolean(user.verified),
|
|
94
|
+
followerCount: count(stats.followerCount),
|
|
95
|
+
// Two names for the like total, and on a payload with no statsV2 the
|
|
96
|
+
// first of them is the one that wraps.
|
|
97
|
+
heartCount: count(stats.heartCount) ?? count(stats.heart),
|
|
98
|
+
videoCount: count(stats.videoCount),
|
|
99
|
+
};
|
|
100
|
+
} catch {
|
|
101
|
+
return {};
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
async function loadAccountDetails(username) {
|
|
106
|
+
try {
|
|
107
|
+
return extractEmbeddedUser(await fetchHtml(`https://www.tiktok.com/@${username}`));
|
|
108
|
+
} catch {
|
|
109
|
+
return {};
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
function extractItem(raw) {
|
|
114
|
+
if (!raw) return null;
|
|
115
|
+
try {
|
|
116
|
+
const scope = JSON.parse(raw).__DEFAULT_SCOPE__;
|
|
117
|
+
if (!scope) return null;
|
|
118
|
+
const detail = scope["webapp.video-detail"] ?? scope["webapp.photo-detail"];
|
|
119
|
+
const item = detail?.itemInfo?.itemStruct;
|
|
120
|
+
return item && typeof item === "object" ? item : null;
|
|
121
|
+
} catch {
|
|
122
|
+
return null;
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
// ---- measurement -----------------------------------------------------------
|
|
127
|
+
|
|
128
|
+
function ratioOf(samples) {
|
|
129
|
+
if (samples.length === 0) return { ratio: 0, ratioPct: 0 };
|
|
130
|
+
const slideshows = samples.filter((s) => s.img).length;
|
|
131
|
+
const ratio = slideshows / samples.length;
|
|
132
|
+
return { ratio, ratioPct: round(ratio * 100, 1) };
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
/**
|
|
136
|
+
* The window is a question about a stretch of time. An account that posted
|
|
137
|
+
* nothing inside it reports that fact, never a zero average.
|
|
138
|
+
*/
|
|
139
|
+
function windowStats(samples, avgViewsDays, anchorSeconds) {
|
|
140
|
+
const cutoff = anchorSeconds - avgViewsDays * 86400;
|
|
141
|
+
const recent = samples.filter((s) => s.t >= cutoff);
|
|
142
|
+
const totalViewsWindow = recent.reduce((sum, s) => sum + s.v, 0);
|
|
143
|
+
const noPostsInWindow = recent.length === 0;
|
|
144
|
+
const avgViews = noPostsInWindow ? null : totalViewsWindow / recent.length;
|
|
145
|
+
return { avgViews, totalViewsWindow, noPostsInWindow };
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
function decideVerdict({
|
|
149
|
+
followers,
|
|
150
|
+
ratio,
|
|
151
|
+
ratioPct,
|
|
152
|
+
avgViews,
|
|
153
|
+
totalViewsWindow,
|
|
154
|
+
noPostsInWindow,
|
|
155
|
+
filters,
|
|
156
|
+
}) {
|
|
157
|
+
const failed = [];
|
|
158
|
+
if (filters.minFollowers > 0 && followers < filters.minFollowers) {
|
|
159
|
+
failed.push({ filter: "followers", measured: followers, threshold: filters.minFollowers });
|
|
160
|
+
}
|
|
161
|
+
if (ratio < filters.minRatio) {
|
|
162
|
+
failed.push({
|
|
163
|
+
filter: "slideshow_ratio",
|
|
164
|
+
measured: ratioPct,
|
|
165
|
+
threshold: round(filters.minRatio * 100, 1),
|
|
166
|
+
});
|
|
167
|
+
}
|
|
168
|
+
if (!noPostsInWindow && avgViews !== null) {
|
|
169
|
+
const avg = Math.trunc(avgViews);
|
|
170
|
+
if (filters.minAvgViews > 0 && avgViews < filters.minAvgViews) {
|
|
171
|
+
failed.push({ filter: "avg_views", measured: avg, threshold: filters.minAvgViews });
|
|
172
|
+
}
|
|
173
|
+
if (filters.minTotalViews > 0 && totalViewsWindow < filters.minTotalViews) {
|
|
174
|
+
failed.push({
|
|
175
|
+
filter: "total_views",
|
|
176
|
+
measured: totalViewsWindow,
|
|
177
|
+
threshold: filters.minTotalViews,
|
|
178
|
+
});
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
const windowFloorRequested = filters.minAvgViews > 0 || filters.minTotalViews > 0;
|
|
182
|
+
const windowBlocked = noPostsInWindow && windowFloorRequested;
|
|
183
|
+
const passed = failed.length === 0 && !windowBlocked;
|
|
184
|
+
const viewsPerFollower =
|
|
185
|
+
avgViews !== null && followers > 0 ? round(avgViews / followers, 2) : null;
|
|
186
|
+
return { passed, failed, noPostsInWindow, unmeasured: false, viewsPerFollower };
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
/** Re-judge a measured account against new filters, with no browsing at all. */
|
|
190
|
+
function reevaluate(account, filters) {
|
|
191
|
+
const samples = account.samples ?? [];
|
|
192
|
+
const { ratio, ratioPct } = ratioOf(samples);
|
|
193
|
+
const w = windowStats(samples, filters.avgViewsDays, account.measuredAt);
|
|
194
|
+
const verdict = decideVerdict({
|
|
195
|
+
followers: account.followers ?? 0,
|
|
196
|
+
ratio,
|
|
197
|
+
ratioPct,
|
|
198
|
+
avgViews: w.avgViews,
|
|
199
|
+
totalViewsWindow: w.totalViewsWindow,
|
|
200
|
+
noPostsInWindow: w.noPostsInWindow,
|
|
201
|
+
filters,
|
|
202
|
+
});
|
|
203
|
+
return { verdict, ...w, ratioPct };
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
const NEAR_MISS_MARGIN = 0.1;
|
|
207
|
+
|
|
208
|
+
/**
|
|
209
|
+
* Marks are arithmetic, never judgment. Whether one matters is the caller's
|
|
210
|
+
* call. The outlier bar is this run's own 90th percentile, so it means
|
|
211
|
+
* "unusual here" rather than "unusual in general".
|
|
212
|
+
*/
|
|
213
|
+
function flagOutliers(accounts) {
|
|
214
|
+
const ratios = accounts
|
|
215
|
+
.map((a) => a.verdict?.viewsPerFollower)
|
|
216
|
+
.filter((v) => typeof v === "number")
|
|
217
|
+
.sort((a, b) => a - b);
|
|
218
|
+
const threshold = ratios.length >= 10 ? ratios[Math.floor(0.9 * (ratios.length - 1))] : null;
|
|
219
|
+
for (const account of accounts) {
|
|
220
|
+
const v = account.verdict;
|
|
221
|
+
if (!v) continue;
|
|
222
|
+
v.highViewsPerFollower =
|
|
223
|
+
threshold !== null && v.viewsPerFollower !== null && v.viewsPerFollower >= threshold;
|
|
224
|
+
v.nearMiss =
|
|
225
|
+
!v.unmeasured &&
|
|
226
|
+
v.failed.length === 1 &&
|
|
227
|
+
v.failed[0].measured >= v.failed[0].threshold * (1 - NEAR_MISS_MARGIN);
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
function breakdownOf(accounts) {
|
|
232
|
+
const b = {
|
|
233
|
+
below_slideshow_ratio: 0,
|
|
234
|
+
below_avg_views: 0,
|
|
235
|
+
below_followers: 0,
|
|
236
|
+
below_total_views: 0,
|
|
237
|
+
no_posts_in_window: 0,
|
|
238
|
+
unmeasured: 0,
|
|
239
|
+
};
|
|
240
|
+
for (const account of accounts) {
|
|
241
|
+
const v = account.verdict;
|
|
242
|
+
if (!v || v.passed) continue;
|
|
243
|
+
if (v.unmeasured) {
|
|
244
|
+
b.unmeasured += 1;
|
|
245
|
+
continue;
|
|
246
|
+
}
|
|
247
|
+
for (const failure of v.failed) {
|
|
248
|
+
if (failure.filter === "slideshow_ratio") b.below_slideshow_ratio += 1;
|
|
249
|
+
else if (failure.filter === "avg_views") b.below_avg_views += 1;
|
|
250
|
+
else if (failure.filter === "followers") b.below_followers += 1;
|
|
251
|
+
else if (failure.filter === "total_views") b.below_total_views += 1;
|
|
252
|
+
}
|
|
253
|
+
if (v.failed.length === 0 && v.noPostsInWindow) b.no_posts_in_window += 1;
|
|
254
|
+
}
|
|
255
|
+
return b;
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
/**
|
|
259
|
+
* Turn a page of raw posts into the measurements the filters are judged on.
|
|
260
|
+
* Everything derived lives here so analysis and discovery agree by construction.
|
|
261
|
+
*/
|
|
262
|
+
function measurePosts({ posts, followers, avgViewsDays, details = {}, now = Date.now() / 1000 }) {
|
|
263
|
+
const timestamps = posts
|
|
264
|
+
.map((p) => p.createTime ?? 0)
|
|
265
|
+
.filter(Boolean)
|
|
266
|
+
.sort((a, b) => b - a);
|
|
267
|
+
if (timestamps.length === 0) return null;
|
|
268
|
+
|
|
269
|
+
const slideshows = posts.filter((p) => p.imagePost);
|
|
270
|
+
const { ratio, ratioPct } = ratioOf(
|
|
271
|
+
posts.map((p) => ({ img: Boolean(p.imagePost), t: p.createTime ?? 0, v: p.stats?.playCount ?? 0 }))
|
|
272
|
+
);
|
|
273
|
+
const samples = posts.map((p) => ({
|
|
274
|
+
t: p.createTime ?? 0,
|
|
275
|
+
v: p.stats?.playCount ?? 0,
|
|
276
|
+
img: Boolean(p.imagePost),
|
|
277
|
+
}));
|
|
278
|
+
const { avgViews, totalViewsWindow, noPostsInWindow } = windowStats(samples, avgViewsDays, now);
|
|
279
|
+
|
|
280
|
+
const totalViews = posts.reduce((sum, p) => sum + (p.stats?.playCount ?? 1), 0) || 1;
|
|
281
|
+
const totalComments = posts.reduce((sum, p) => sum + (p.stats?.commentCount ?? 0), 0);
|
|
282
|
+
const recent90 = posts.filter((p) => (p.createTime ?? 0) >= now - 90 * 86400);
|
|
283
|
+
|
|
284
|
+
let consistency = 0;
|
|
285
|
+
if (timestamps.length > 1) {
|
|
286
|
+
const gaps = timestamps.slice(0, -1).map((t, i) => (t - timestamps[i + 1]) / 86400);
|
|
287
|
+
const avgGap = gaps.reduce((a, b) => a + b, 0) / gaps.length;
|
|
288
|
+
const variance = gaps.reduce((sum, g) => sum + (g - avgGap) ** 2, 0) / gaps.length;
|
|
289
|
+
consistency = round(Math.max(0, 10 - Math.sqrt(variance)), 1);
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
const slideCounts = slideshows.map((p) => p.imagePost?.images?.length ?? 0);
|
|
293
|
+
const avgSlides = slideCounts.length
|
|
294
|
+
? round(slideCounts.reduce((a, b) => a + b, 0) / slideCounts.length, 1)
|
|
295
|
+
: 0;
|
|
296
|
+
|
|
297
|
+
const hashtagFreq = new Map();
|
|
298
|
+
for (const post of posts) {
|
|
299
|
+
for (const tag of post.textExtra ?? []) {
|
|
300
|
+
if (tag.hashtagName) hashtagFreq.set(tag.hashtagName, (hashtagFreq.get(tag.hashtagName) ?? 0) + 1);
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
const topHashtags = [...hashtagFreq.entries()]
|
|
304
|
+
.sort((a, b) => b[1] - a[1])
|
|
305
|
+
.slice(0, 10)
|
|
306
|
+
.map(([tag]) => tag);
|
|
307
|
+
|
|
308
|
+
const cutoff = now - avgViewsDays * 86400;
|
|
309
|
+
const recentSlideshows = posts.filter((p) => (p.createTime ?? 0) >= cutoff && p.imagePost);
|
|
310
|
+
const captions = (recentSlideshows.length ? recentSlideshows : slideshows)
|
|
311
|
+
.slice(0, 15)
|
|
312
|
+
.map((p) => (p.desc ?? "").trim())
|
|
313
|
+
.filter(Boolean);
|
|
314
|
+
|
|
315
|
+
return {
|
|
316
|
+
ratio,
|
|
317
|
+
ratioPct,
|
|
318
|
+
avgViews,
|
|
319
|
+
totalViewsWindow,
|
|
320
|
+
noPostsInWindow,
|
|
321
|
+
samples,
|
|
322
|
+
captions,
|
|
323
|
+
metrics: {
|
|
324
|
+
total_posts_real: details.videoCount ?? posts.length,
|
|
325
|
+
scraped_posts: posts.length,
|
|
326
|
+
slideshow_count: slideshows.length,
|
|
327
|
+
slideshow_ratio: ratioPct,
|
|
328
|
+
// Null, not zero, when the window was empty: a reader must never mistake
|
|
329
|
+
// "did not post" for "was seen zero times".
|
|
330
|
+
[`avg_views_last_${avgViewsDays}d`]: noPostsInWindow ? null : Math.trunc(avgViews ?? 0),
|
|
331
|
+
[`total_views_last_${avgViewsDays}d`]: noPostsInWindow ? null : Math.trunc(totalViewsWindow),
|
|
332
|
+
comment_ratio: round((totalComments / totalViews) * 100, 3),
|
|
333
|
+
view_follower_ratio: avgViews !== null && followers > 0 ? round(avgViews / followers, 2) : null,
|
|
334
|
+
posts_per_week: round(recent90.length / (90 / 7), 2),
|
|
335
|
+
consistency_score: consistency,
|
|
336
|
+
last_post_days_ago: round((now - timestamps[0]) / 86400, 1),
|
|
337
|
+
avg_slides_per_slideshow: avgSlides,
|
|
338
|
+
top_hashtags: topHashtags,
|
|
339
|
+
},
|
|
340
|
+
};
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
function postSummary(post) {
|
|
344
|
+
const stats = post.stats ?? {};
|
|
345
|
+
const views = stats.playCount ?? 0;
|
|
346
|
+
const interactions =
|
|
347
|
+
(stats.diggCount ?? 0) +
|
|
348
|
+
(stats.commentCount ?? 0) +
|
|
349
|
+
(stats.shareCount ?? 0) +
|
|
350
|
+
(stats.collectCount ?? 0);
|
|
351
|
+
const username = post.author?.uniqueId ?? "";
|
|
352
|
+
const isSlideshow = Boolean(post.imagePost);
|
|
353
|
+
return {
|
|
354
|
+
id: post.id,
|
|
355
|
+
type: isSlideshow ? "slideshow" : "video",
|
|
356
|
+
url: `https://www.tiktok.com/@${username}/${isSlideshow ? "photo" : "video"}/${post.id}`,
|
|
357
|
+
description: (post.desc ?? "").slice(0, 300),
|
|
358
|
+
hashtags: (post.textExtra ?? []).map((t) => t.hashtagName).filter(Boolean),
|
|
359
|
+
views,
|
|
360
|
+
likes: stats.diggCount ?? 0,
|
|
361
|
+
comments: stats.commentCount ?? 0,
|
|
362
|
+
shares: stats.shareCount ?? 0,
|
|
363
|
+
favorites: stats.collectCount ?? 0,
|
|
364
|
+
engagement_rate: views ? round((interactions / views) * 100, 4) : 0,
|
|
365
|
+
slide_count: post.imagePost?.images?.length ?? 0,
|
|
366
|
+
published_at: post.createTime ? new Date(post.createTime * 1000).toISOString() : null,
|
|
367
|
+
};
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
module.exports = {
|
|
371
|
+
TOP,
|
|
372
|
+
PHOTO,
|
|
373
|
+
FEED_ENDPOINTS,
|
|
374
|
+
DETAIL_ENDPOINT,
|
|
375
|
+
ITEM_ENDPOINTS,
|
|
376
|
+
round,
|
|
377
|
+
parseCount,
|
|
378
|
+
fetchHtml,
|
|
379
|
+
extractEmbeddedUser,
|
|
380
|
+
extractItem,
|
|
381
|
+
loadAccountDetails,
|
|
382
|
+
ratioOf,
|
|
383
|
+
windowStats,
|
|
384
|
+
decideVerdict,
|
|
385
|
+
reevaluate,
|
|
386
|
+
flagOutliers,
|
|
387
|
+
breakdownOf,
|
|
388
|
+
measurePosts,
|
|
389
|
+
postSummary,
|
|
390
|
+
NEAR_MISS_MARGIN,
|
|
391
|
+
};
|