@xiaohhhh1/canvas-agent 0.4.22 → 0.4.24
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -18,6 +18,19 @@ type CaptureContext = {
|
|
|
18
18
|
shopType?: string;
|
|
19
19
|
periodDays?: number;
|
|
20
20
|
};
|
|
21
|
+
type VideoLearningCaptureContext = {
|
|
22
|
+
market?: string;
|
|
23
|
+
target?: number;
|
|
24
|
+
};
|
|
25
|
+
type VideoLearningDimension = "commerce-video" | "high-sales" | "high-roas" | "high-exposure" | "high-engagement";
|
|
26
|
+
type VisibleVideoItem = {
|
|
27
|
+
text: string;
|
|
28
|
+
links: string[];
|
|
29
|
+
media: string[];
|
|
30
|
+
images: string[];
|
|
31
|
+
headers: string[];
|
|
32
|
+
cells: string[];
|
|
33
|
+
};
|
|
21
34
|
export declare class FastMossIntegration {
|
|
22
35
|
private context;
|
|
23
36
|
private page;
|
|
@@ -67,6 +80,29 @@ export declare class FastMossIntegration {
|
|
|
67
80
|
captured: number;
|
|
68
81
|
lastCapture?: string;
|
|
69
82
|
}>;
|
|
83
|
+
/**
|
|
84
|
+
* Reuse the exact same local, persistent FastMoss browser used by product selection.
|
|
85
|
+
* This reads only visible ranking pages and visible media elements. It never creates
|
|
86
|
+
* private FastMoss API requests and pauses for the operator when a CAPTCHA appears.
|
|
87
|
+
*/
|
|
88
|
+
captureLearningVideos(input?: VideoLearningCaptureContext): Promise<{
|
|
89
|
+
sources: Record<string, unknown>[];
|
|
90
|
+
rejected: {
|
|
91
|
+
dimension: VideoLearningDimension;
|
|
92
|
+
reason: string;
|
|
93
|
+
}[];
|
|
94
|
+
requestedTarget: number;
|
|
95
|
+
complete: boolean;
|
|
96
|
+
phase: FastMossPhase;
|
|
97
|
+
browserOpen: boolean;
|
|
98
|
+
authenticated: boolean;
|
|
99
|
+
verificationRequired: boolean;
|
|
100
|
+
membershipExpired: boolean;
|
|
101
|
+
message: string;
|
|
102
|
+
url: string | null;
|
|
103
|
+
captured: number;
|
|
104
|
+
lastCapture?: string;
|
|
105
|
+
}>;
|
|
70
106
|
close(): Promise<FastMossStatus>;
|
|
71
107
|
observations(): Promise<{
|
|
72
108
|
rows: Record<string, unknown>[];
|
|
@@ -75,11 +111,24 @@ export declare class FastMossIntegration {
|
|
|
75
111
|
private loadRows;
|
|
76
112
|
private useLatestPage;
|
|
77
113
|
private openProductRanking;
|
|
114
|
+
private openVideoLearningPage;
|
|
115
|
+
private readVideoLearningItems;
|
|
116
|
+
private clickVideoLearningTab;
|
|
117
|
+
private clickVideoLearningNextPage;
|
|
118
|
+
private waitForVideoLearningNavigation;
|
|
78
119
|
}
|
|
79
120
|
export declare function fastMossSalesRankUrl(market?: string): string;
|
|
80
121
|
export declare function fastMossRankingUrl(source: FastMossRankingSource, market?: string): string;
|
|
122
|
+
export declare function fastMossVideoLearningRankUrl(market?: string): string;
|
|
81
123
|
export declare function mergeRankingRecord(target: Map<string, Record<string, unknown>>, row: Record<string, unknown>, source: FastMossRankingSource, position: number): void;
|
|
82
124
|
export declare function selectDetailCandidates(records: Array<Record<string, unknown>>, limit?: number): Record<string, unknown>[];
|
|
125
|
+
export declare function isTransientFastMossNavigation(error: unknown): boolean;
|
|
126
|
+
export declare function isAnalyzableVideoUrl(value: string): boolean;
|
|
127
|
+
export declare function videoLearningSourcesFromItems(items: VisibleVideoItem[], context: {
|
|
128
|
+
market: string;
|
|
129
|
+
dimension: VideoLearningDimension;
|
|
130
|
+
observedAt: string;
|
|
131
|
+
}): Record<string, unknown>[];
|
|
83
132
|
export declare function productIdFromUrl(value: string): string;
|
|
84
133
|
/** Extract only the evidence required by the AI short-video selection model. */
|
|
85
134
|
export declare function detailOverviewEvidence(overviewPayload: unknown, basePayload?: unknown): {
|
|
@@ -12,6 +12,10 @@ const PRODUCT_RANK_MAX_PAGES = 50;
|
|
|
12
12
|
const DETAIL_CANDIDATE_LIMIT = 200;
|
|
13
13
|
const DETAIL_TREND_DAYS = 7;
|
|
14
14
|
const DETAIL_FETCH_CONCURRENCY = 3;
|
|
15
|
+
const VIDEO_LEARNING_TARGET = 36;
|
|
16
|
+
const VIDEO_COMMERCE_TARGET = 12;
|
|
17
|
+
const VIDEO_AI_TARGET_PER_DIMENSION = 6;
|
|
18
|
+
const VIDEO_AI_RANKING_PATH = "/zh/creativecenter/top-ai-videos";
|
|
15
19
|
const RANKING_DEFINITIONS = [
|
|
16
20
|
{ source: "new", label: "新品榜", pathname: "/e-commerce/newProducts" },
|
|
17
21
|
{ source: "hot", label: "热推榜", pathname: "/e-commerce/hotlist" },
|
|
@@ -254,6 +258,107 @@ export class FastMossIntegration {
|
|
|
254
258
|
: `四榜单去重后 ${marketRecords.length} 个商品,优先互证 ${detailCandidates.length} 个,其中 ${trendProducts} 个取得最近 ${DETAIL_TREND_DAYS} 天趋势`;
|
|
255
259
|
return { ...this.status(), records: marketRecords, allRows, trendProducts, detailFailures };
|
|
256
260
|
}
|
|
261
|
+
/**
|
|
262
|
+
* Reuse the exact same local, persistent FastMoss browser used by product selection.
|
|
263
|
+
* This reads only visible ranking pages and visible media elements. It never creates
|
|
264
|
+
* private FastMoss API requests and pauses for the operator when a CAPTCHA appears.
|
|
265
|
+
*/
|
|
266
|
+
async captureLearningVideos(input = {}) {
|
|
267
|
+
if (!this.context || !this.page || this.page.isClosed())
|
|
268
|
+
await this.start();
|
|
269
|
+
await this.inspect();
|
|
270
|
+
if (this.verificationRequired)
|
|
271
|
+
throw new Error("请先在 FastMoss 专用浏览器中手动完成验证");
|
|
272
|
+
if (this.membershipExpired)
|
|
273
|
+
throw new Error("FastMoss 会员已过期或当前账号没有视频榜单权限,请先更换账号");
|
|
274
|
+
if (!this.authenticated || !this.page || this.page.isClosed())
|
|
275
|
+
throw new Error("FastMoss 尚未登录;请在恢复的窗口中完成登录后重试");
|
|
276
|
+
// 视频学习只研究可迁移的创意机制,不应被某一个运营市场限制。
|
|
277
|
+
// FastMoss 使用空 region 表示“全部国家/地区”。
|
|
278
|
+
const market = normalizeLearningMarket(input.market);
|
|
279
|
+
const requestedTarget = Math.max(1, Math.min(VIDEO_LEARNING_TARGET, Number(input.target || VIDEO_LEARNING_TARGET)));
|
|
280
|
+
const sources = [];
|
|
281
|
+
const rejected = [];
|
|
282
|
+
const seen = new Set();
|
|
283
|
+
const collect = async (dimension, limit) => {
|
|
284
|
+
if (!this.page)
|
|
285
|
+
return;
|
|
286
|
+
let previousSignature = "";
|
|
287
|
+
for (let pageNumber = 1; pageNumber <= 5 && sources.filter((item) => item.rankingDimension === dimension).length < limit; pageNumber += 1) {
|
|
288
|
+
const snapshot = await this.readVideoLearningItems();
|
|
289
|
+
const signature = JSON.stringify(snapshot.map((item) => [item.text, item.media, item.links]).slice(0, 20));
|
|
290
|
+
if (!snapshot.length || (pageNumber > 1 && signature === previousSignature))
|
|
291
|
+
break;
|
|
292
|
+
previousSignature = signature;
|
|
293
|
+
const normalized = videoLearningSourcesFromItems(snapshot, { market, dimension, observedAt: new Date().toISOString() });
|
|
294
|
+
for (const item of normalized) {
|
|
295
|
+
const key = String(item.platformVideoId || item.sourceUrl || "");
|
|
296
|
+
if (!key || seen.has(key))
|
|
297
|
+
continue;
|
|
298
|
+
// A learning sample must point to an actual visible media/public-video URL.
|
|
299
|
+
// Ranking metadata without readable media is retained only as a rejection,
|
|
300
|
+
// never counted toward the 36 analyzable videos.
|
|
301
|
+
if (!item.sourceUrl || !isAnalyzableVideoUrl(String(item.sourceUrl))) {
|
|
302
|
+
rejected.push({ dimension, reason: "榜单条目没有暴露可读取的视频地址" });
|
|
303
|
+
continue;
|
|
304
|
+
}
|
|
305
|
+
seen.add(key);
|
|
306
|
+
sources.push(item);
|
|
307
|
+
if (sources.filter((source) => source.rankingDimension === dimension).length >= limit)
|
|
308
|
+
break;
|
|
309
|
+
}
|
|
310
|
+
this.captured = sources.length;
|
|
311
|
+
this.message = `正在读取视频学习样本:${dimension} ${sources.filter((item) => item.rankingDimension === dimension).length}/${limit},合计 ${sources.length}/${requestedTarget}`;
|
|
312
|
+
if (sources.filter((item) => item.rankingDimension === dimension).length >= limit)
|
|
313
|
+
break;
|
|
314
|
+
if (!await this.clickVideoLearningNextPage())
|
|
315
|
+
break;
|
|
316
|
+
await this.waitForVideoLearningNavigation();
|
|
317
|
+
await this.inspect();
|
|
318
|
+
if (this.verificationRequired)
|
|
319
|
+
throw new Error("采集视频榜单时出现滑块验证;请手动完成后重新采集,未完成样本不会计数");
|
|
320
|
+
if (this.membershipExpired)
|
|
321
|
+
throw new Error("当前 FastMoss 账号没有继续查看视频榜单的权限");
|
|
322
|
+
}
|
|
323
|
+
};
|
|
324
|
+
await this.openVideoLearningPage(fastMossVideoLearningRankUrl(market), market === "GLOBAL" ? "全球视频商品榜" : `${market} 视频商品榜`);
|
|
325
|
+
await collect("commerce-video", Math.min(VIDEO_COMMERCE_TARGET, requestedTarget));
|
|
326
|
+
const aiDimensions = [
|
|
327
|
+
{ dimension: "high-sales", labels: ["高销量榜单", "High Sales"] },
|
|
328
|
+
{ dimension: "high-roas", labels: ["高ROAS榜单", "High ROAS"] },
|
|
329
|
+
{ dimension: "high-exposure", labels: ["高曝光榜单", "High Exposure"] },
|
|
330
|
+
{ dimension: "high-engagement", labels: ["高互动榜单", "High Engagement"] },
|
|
331
|
+
];
|
|
332
|
+
if (sources.length < requestedTarget) {
|
|
333
|
+
const aiUrl = new URL(VIDEO_AI_RANKING_PATH, FASTMOSS_HOME);
|
|
334
|
+
aiUrl.searchParams.set("region", market === "GLOBAL" ? "" : market);
|
|
335
|
+
await this.openVideoLearningPage(aiUrl.toString(), market === "GLOBAL" ? "全球 AI 带货视频榜" : `${market} AI 带货视频榜`);
|
|
336
|
+
for (const entry of aiDimensions) {
|
|
337
|
+
if (sources.length >= requestedTarget)
|
|
338
|
+
break;
|
|
339
|
+
const clicked = await this.clickVideoLearningTab(entry.labels);
|
|
340
|
+
if (!clicked) {
|
|
341
|
+
rejected.push({ dimension: entry.dimension, reason: `没有找到${entry.labels[0]}标签` });
|
|
342
|
+
continue;
|
|
343
|
+
}
|
|
344
|
+
await this.waitForVideoLearningNavigation();
|
|
345
|
+
await this.inspect();
|
|
346
|
+
if (this.verificationRequired)
|
|
347
|
+
throw new Error("采集 AI 视频榜时出现滑块验证;请手动完成后重新采集");
|
|
348
|
+
const remaining = Math.min(VIDEO_AI_TARGET_PER_DIMENSION, requestedTarget - sources.length);
|
|
349
|
+
await collect(entry.dimension, remaining);
|
|
350
|
+
}
|
|
351
|
+
}
|
|
352
|
+
await mkdir(this.dataDir, { recursive: true });
|
|
353
|
+
const capturedAt = new Date().toISOString();
|
|
354
|
+
await writeFile(path.join(this.dataDir, "video-learning-observations.json"), JSON.stringify({ capturedAt, market, sources, rejected }, null, 2), "utf8");
|
|
355
|
+
this.lastCapture = capturedAt;
|
|
356
|
+
this.captured = sources.length;
|
|
357
|
+
this.message = sources.length >= requestedTarget
|
|
358
|
+
? `已从同一 FastMoss 专用窗口取得 ${sources.length} 条可分析视频样本`
|
|
359
|
+
: `只取得 ${sources.length}/${requestedTarget} 条可分析视频;其余条目未暴露视频地址,不会冒充有效样本`;
|
|
360
|
+
return { ...this.status(), sources, rejected, requestedTarget, complete: sources.length >= requestedTarget };
|
|
361
|
+
}
|
|
257
362
|
async close() {
|
|
258
363
|
const context = this.context;
|
|
259
364
|
this.context = null;
|
|
@@ -330,6 +435,99 @@ export class FastMossIntegration {
|
|
|
330
435
|
}
|
|
331
436
|
throw new Error("FastMoss 商品榜单自动加载超时;请检查网络或在窗口中完成人工验证后重试");
|
|
332
437
|
}
|
|
438
|
+
async openVideoLearningPage(url, label) {
|
|
439
|
+
if (!this.page || this.page.isClosed())
|
|
440
|
+
throw new Error("FastMoss 专用浏览器未打开");
|
|
441
|
+
this.message = `正在自动进入 ${label}`;
|
|
442
|
+
try {
|
|
443
|
+
await this.page.goto(url, { waitUntil: "domcontentloaded", timeout: 60_000 });
|
|
444
|
+
}
|
|
445
|
+
catch (error) {
|
|
446
|
+
// FastMoss 会在进入 AI 榜时自行二次跳转。此时 Playwright 报 ERR_ABORTED,
|
|
447
|
+
// 但浏览器实际仍在加载目标页;继续等待可见榜单,而不是把正常跳转当成失败。
|
|
448
|
+
if (!isTransientFastMossNavigation(error))
|
|
449
|
+
throw error;
|
|
450
|
+
}
|
|
451
|
+
const deadline = Date.now() + PRODUCT_TABLE_TIMEOUT_MS;
|
|
452
|
+
while (Date.now() < deadline) {
|
|
453
|
+
const items = await this.readVideoLearningItems();
|
|
454
|
+
if (items.length)
|
|
455
|
+
return;
|
|
456
|
+
await this.inspect();
|
|
457
|
+
if (this.verificationRequired)
|
|
458
|
+
throw new Error("检测到 FastMoss 滑块验证,请在专用窗口中手动完成后再次采集");
|
|
459
|
+
if (this.membershipExpired)
|
|
460
|
+
throw new Error("当前 FastMoss 账号没有该视频榜单权限");
|
|
461
|
+
if (!this.authenticated)
|
|
462
|
+
throw new Error("FastMoss 登录已失效,请重新登录后再次采集");
|
|
463
|
+
await this.waitForVideoLearningNavigation(750);
|
|
464
|
+
}
|
|
465
|
+
throw new Error(`${label}加载超时;请检查网络或在专用窗口中确认页面可见`);
|
|
466
|
+
}
|
|
467
|
+
async readVideoLearningItems() {
|
|
468
|
+
let lastError;
|
|
469
|
+
for (let attempt = 0; attempt < 4; attempt += 1) {
|
|
470
|
+
this.useLatestPage();
|
|
471
|
+
if (!this.page || this.page.isClosed())
|
|
472
|
+
throw new Error("FastMoss 专用浏览器未打开");
|
|
473
|
+
try {
|
|
474
|
+
return await readVisibleVideoLearningItems(this.page);
|
|
475
|
+
}
|
|
476
|
+
catch (error) {
|
|
477
|
+
lastError = error;
|
|
478
|
+
if (!isTransientFastMossNavigation(error))
|
|
479
|
+
throw error;
|
|
480
|
+
await this.waitForVideoLearningNavigation();
|
|
481
|
+
}
|
|
482
|
+
}
|
|
483
|
+
throw lastError instanceof Error ? lastError : new Error("FastMoss 视频榜页面跳转后没有恢复");
|
|
484
|
+
}
|
|
485
|
+
async clickVideoLearningTab(labels) {
|
|
486
|
+
let lastError;
|
|
487
|
+
for (let attempt = 0; attempt < 3; attempt += 1) {
|
|
488
|
+
this.useLatestPage();
|
|
489
|
+
if (!this.page || this.page.isClosed())
|
|
490
|
+
throw new Error("FastMoss 专用浏览器未打开");
|
|
491
|
+
try {
|
|
492
|
+
return await clickVisibleText(this.page, labels);
|
|
493
|
+
}
|
|
494
|
+
catch (error) {
|
|
495
|
+
lastError = error;
|
|
496
|
+
if (!isTransientFastMossNavigation(error))
|
|
497
|
+
throw error;
|
|
498
|
+
await this.waitForVideoLearningNavigation();
|
|
499
|
+
}
|
|
500
|
+
}
|
|
501
|
+
throw lastError instanceof Error ? lastError : new Error("FastMoss 视频榜标签切换失败");
|
|
502
|
+
}
|
|
503
|
+
async clickVideoLearningNextPage() {
|
|
504
|
+
let lastError;
|
|
505
|
+
for (let attempt = 0; attempt < 3; attempt += 1) {
|
|
506
|
+
this.useLatestPage();
|
|
507
|
+
if (!this.page || this.page.isClosed())
|
|
508
|
+
throw new Error("FastMoss 专用浏览器未打开");
|
|
509
|
+
try {
|
|
510
|
+
return await clickVisibleNextPage(this.page);
|
|
511
|
+
}
|
|
512
|
+
catch (error) {
|
|
513
|
+
lastError = error;
|
|
514
|
+
if (!isTransientFastMossNavigation(error))
|
|
515
|
+
throw error;
|
|
516
|
+
await this.waitForVideoLearningNavigation();
|
|
517
|
+
}
|
|
518
|
+
}
|
|
519
|
+
throw lastError instanceof Error ? lastError : new Error("FastMoss 视频榜翻页失败");
|
|
520
|
+
}
|
|
521
|
+
async waitForVideoLearningNavigation(delay = 900) {
|
|
522
|
+
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
523
|
+
this.useLatestPage();
|
|
524
|
+
if (!this.page || this.page.isClosed())
|
|
525
|
+
return;
|
|
526
|
+
await this.page.waitForLoadState("domcontentloaded", { timeout: 10_000 }).catch((error) => {
|
|
527
|
+
if (!isTransientFastMossNavigation(error))
|
|
528
|
+
throw error;
|
|
529
|
+
});
|
|
530
|
+
}
|
|
333
531
|
}
|
|
334
532
|
export function fastMossSalesRankUrl(market) {
|
|
335
533
|
return fastMossRankingUrl("sales", market);
|
|
@@ -337,11 +535,14 @@ export function fastMossSalesRankUrl(market) {
|
|
|
337
535
|
export function fastMossRankingUrl(source, market) {
|
|
338
536
|
const definition = RANKING_DEFINITIONS.find((item) => item.source === source) || RANKING_DEFINITIONS[0];
|
|
339
537
|
const url = new URL(definition.pathname, FASTMOSS_HOME);
|
|
340
|
-
url.searchParams.set("region", normalizeMarket(market));
|
|
538
|
+
url.searchParams.set("region", market === "GLOBAL" ? "" : normalizeMarket(market));
|
|
341
539
|
url.searchParams.set("page", "1");
|
|
342
540
|
url.searchParams.set("pagesize", String(PRODUCT_RANK_PAGE_SIZE));
|
|
343
541
|
return url.toString();
|
|
344
542
|
}
|
|
543
|
+
export function fastMossVideoLearningRankUrl(market) {
|
|
544
|
+
return fastMossRankingUrl("video", normalizeLearningMarket(market));
|
|
545
|
+
}
|
|
345
546
|
export function mergeRankingRecord(target, row, source, position) {
|
|
346
547
|
const key = String(row.product_id || row.title);
|
|
347
548
|
const existing = target.get(key) || {};
|
|
@@ -369,6 +570,14 @@ function normalizeMarket(value) {
|
|
|
369
570
|
const market = String(value || "MX").trim().toUpperCase();
|
|
370
571
|
return /^[A-Z]{2}$/.test(market) ? market : "MX";
|
|
371
572
|
}
|
|
573
|
+
function normalizeLearningMarket(value) {
|
|
574
|
+
const market = String(value || "GLOBAL").trim().toUpperCase();
|
|
575
|
+
return market === "GLOBAL" || market === "ALL" || market === "全球" || market === "全部" ? "GLOBAL" : normalizeMarket(market);
|
|
576
|
+
}
|
|
577
|
+
export function isTransientFastMossNavigation(error) {
|
|
578
|
+
const message = error instanceof Error ? error.message : String(error || "");
|
|
579
|
+
return /ERR_ABORTED|Execution context was destroyed|navigation interrupted|frame was detached/i.test(message);
|
|
580
|
+
}
|
|
372
581
|
function hasReadableProductRows(snapshot) {
|
|
373
582
|
return snapshot.tables.some((table) => table.rows.some((row) => row.cells.filter(Boolean).length >= 3));
|
|
374
583
|
}
|
|
@@ -430,6 +639,203 @@ async function clickVisibleNextPage(page) {
|
|
|
430
639
|
return true;
|
|
431
640
|
});
|
|
432
641
|
}
|
|
642
|
+
async function clickVisibleText(page, labels) {
|
|
643
|
+
return page.evaluate((expected) => {
|
|
644
|
+
const shown = (element) => Boolean(element.getClientRects().length);
|
|
645
|
+
const normalized = (value) => value.replace(/\s+/g, "").toLowerCase();
|
|
646
|
+
const wanted = expected.map(normalized);
|
|
647
|
+
const candidates = [...document.querySelectorAll("button,a,[role='tab'],[role='button'],label")]
|
|
648
|
+
.filter(shown)
|
|
649
|
+
.filter((element) => {
|
|
650
|
+
const text = normalized(element.innerText || element.textContent || "");
|
|
651
|
+
return wanted.some((label) => text === label || text.includes(label));
|
|
652
|
+
});
|
|
653
|
+
const target = candidates[0];
|
|
654
|
+
if (!target)
|
|
655
|
+
return false;
|
|
656
|
+
target.click();
|
|
657
|
+
return true;
|
|
658
|
+
}, labels);
|
|
659
|
+
}
|
|
660
|
+
async function readVisibleVideoLearningItems(page) {
|
|
661
|
+
return page.evaluate(() => {
|
|
662
|
+
const shown = (element) => {
|
|
663
|
+
const node = element;
|
|
664
|
+
const rect = node.getBoundingClientRect();
|
|
665
|
+
const style = getComputedStyle(node);
|
|
666
|
+
return Boolean(node.getClientRects().length) && rect.width >= 80 && rect.height >= 30 && style.visibility !== "hidden" && style.display !== "none";
|
|
667
|
+
};
|
|
668
|
+
const absoluteUrl = (value) => {
|
|
669
|
+
try {
|
|
670
|
+
return new URL(value, location.href).toString();
|
|
671
|
+
}
|
|
672
|
+
catch {
|
|
673
|
+
return "";
|
|
674
|
+
}
|
|
675
|
+
};
|
|
676
|
+
const mediaUrls = (root) => {
|
|
677
|
+
const direct = [...root.querySelectorAll("video")].flatMap((video) => [
|
|
678
|
+
video.currentSrc,
|
|
679
|
+
video.src,
|
|
680
|
+
video.poster,
|
|
681
|
+
...[...video.querySelectorAll("source[src]")].map((source) => source.src),
|
|
682
|
+
]);
|
|
683
|
+
const data = [...root.querySelectorAll("[data-video],[data-video-url],[data-play-url],[data-src]")].flatMap((element) => [
|
|
684
|
+
element.dataset.video || "",
|
|
685
|
+
element.dataset.videoUrl || "",
|
|
686
|
+
element.dataset.playUrl || "",
|
|
687
|
+
element.dataset.src || "",
|
|
688
|
+
]);
|
|
689
|
+
return [...new Set([...direct, ...data].filter(Boolean).map(absoluteUrl).filter(Boolean))];
|
|
690
|
+
};
|
|
691
|
+
const serialize = (root, headers = [], cells = []) => ({
|
|
692
|
+
text: root.innerText.replace(/\s+/g, " ").trim().slice(0, 8000),
|
|
693
|
+
links: [...new Set([...root.querySelectorAll("a[href]")].map((link) => link.href).filter(Boolean))],
|
|
694
|
+
media: mediaUrls(root),
|
|
695
|
+
images: [...new Set([...root.querySelectorAll("img")].map((image) => image.currentSrc || image.src || image.dataset.src || "").filter(Boolean).map(absoluteUrl).filter(Boolean))],
|
|
696
|
+
headers,
|
|
697
|
+
cells,
|
|
698
|
+
});
|
|
699
|
+
const tableItems = [...document.querySelectorAll("table tbody tr")]
|
|
700
|
+
.filter(shown)
|
|
701
|
+
.map((row) => {
|
|
702
|
+
const table = row.closest("table");
|
|
703
|
+
const headers = [...(table?.querySelectorAll("thead th") || [])].map((cell) => cell.innerText.replace(/\s+/g, " ").trim());
|
|
704
|
+
const cells = [...row.querySelectorAll("td")].map((cell) => cell.innerText.replace(/\s+/g, " ").trim());
|
|
705
|
+
return serialize(row, headers, cells);
|
|
706
|
+
})
|
|
707
|
+
.filter((item) => item.text && (item.links.length || item.media.length || item.images.length));
|
|
708
|
+
const cardCandidates = [...document.querySelectorAll("article,li,[class*='card'],[class*='item']")]
|
|
709
|
+
.filter(shown)
|
|
710
|
+
.filter((element) => {
|
|
711
|
+
const rect = element.getBoundingClientRect();
|
|
712
|
+
const text = element.innerText.replace(/\s+/g, " ").trim();
|
|
713
|
+
return rect.width >= 180 && rect.height >= 160 && text.length >= 8 && Boolean(element.querySelector("video,img"));
|
|
714
|
+
})
|
|
715
|
+
.sort((left, right) => {
|
|
716
|
+
const a = left.getBoundingClientRect();
|
|
717
|
+
const b = right.getBoundingClientRect();
|
|
718
|
+
return a.width * a.height - b.width * b.height;
|
|
719
|
+
});
|
|
720
|
+
const cards = [];
|
|
721
|
+
const accepted = new Set();
|
|
722
|
+
for (const element of cardCandidates) {
|
|
723
|
+
if ([...accepted].some((other) => element.contains(other)))
|
|
724
|
+
continue;
|
|
725
|
+
accepted.add(element);
|
|
726
|
+
cards.push(serialize(element));
|
|
727
|
+
if (cards.length >= 80)
|
|
728
|
+
break;
|
|
729
|
+
}
|
|
730
|
+
const combined = [...tableItems, ...cards];
|
|
731
|
+
const seen = new Set();
|
|
732
|
+
return combined.filter((item) => {
|
|
733
|
+
const key = JSON.stringify([item.text.slice(0, 300), item.media[0], item.links[0], item.images[0]]);
|
|
734
|
+
if (seen.has(key))
|
|
735
|
+
return false;
|
|
736
|
+
seen.add(key);
|
|
737
|
+
return true;
|
|
738
|
+
});
|
|
739
|
+
});
|
|
740
|
+
}
|
|
741
|
+
function firstHttp(values, expected) {
|
|
742
|
+
return values.find((value) => /^https?:\/\//i.test(value) && (!expected || expected.test(value))) || "";
|
|
743
|
+
}
|
|
744
|
+
function videoIdFromValues(values) {
|
|
745
|
+
for (const value of values) {
|
|
746
|
+
const direct = value.match(/(?:video|aweme|item|material)[/=_-](\d{8,})/i)?.[1];
|
|
747
|
+
if (direct)
|
|
748
|
+
return direct;
|
|
749
|
+
try {
|
|
750
|
+
const url = new URL(value);
|
|
751
|
+
const query = url.searchParams.get("aweme_id") || url.searchParams.get("video_id") || url.searchParams.get("item_id");
|
|
752
|
+
if (query && /^\d{8,}$/.test(query))
|
|
753
|
+
return query;
|
|
754
|
+
}
|
|
755
|
+
catch { }
|
|
756
|
+
}
|
|
757
|
+
return "";
|
|
758
|
+
}
|
|
759
|
+
function labeledMetrics(item) {
|
|
760
|
+
const metrics = {};
|
|
761
|
+
item.headers.forEach((header, index) => {
|
|
762
|
+
const value = item.cells[index];
|
|
763
|
+
if (!header || !value)
|
|
764
|
+
return;
|
|
765
|
+
const numeric = numberValue(value);
|
|
766
|
+
const key = /销量|sales|sold/i.test(header) ? "sales"
|
|
767
|
+
: /销售额|成交额|gmv/i.test(header) ? "gmv"
|
|
768
|
+
: /播放|曝光|views?|impressions?/i.test(header) ? "views"
|
|
769
|
+
: /点赞|likes?/i.test(header) ? "likes"
|
|
770
|
+
: /评论|comments?/i.test(header) ? "comments"
|
|
771
|
+
: /roas/i.test(header) ? "roas"
|
|
772
|
+
: "";
|
|
773
|
+
if (key && Number.isFinite(numeric))
|
|
774
|
+
metrics[key] = numeric;
|
|
775
|
+
});
|
|
776
|
+
const patterns = [
|
|
777
|
+
["sales", /(?:总销量|销量|sales)\s*[::]?\s*([\d,.]+(?:万|亿|k|w)?)/i],
|
|
778
|
+
["gmv", /(?:总销售额|销售额|GMV)\s*[::$]?\s*([\d,.]+(?:万|亿|k|w)?)/i],
|
|
779
|
+
["views", /(?:总播放量|视频播放量|播放量|曝光)\s*[::]?\s*([\d,.]+(?:万|亿|k|w)?)/i],
|
|
780
|
+
["likes", /(?:总点赞数|点赞数|点赞)\s*[::]?\s*([\d,.]+(?:万|亿|k|w)?)/i],
|
|
781
|
+
["comments", /(?:总评论数|评论数|评论)\s*[::]?\s*([\d,.]+(?:万|亿|k|w)?)/i],
|
|
782
|
+
["roas", /ROAS\s*[::]?\s*([\d,.]+)/i],
|
|
783
|
+
];
|
|
784
|
+
for (const [key, pattern] of patterns) {
|
|
785
|
+
const match = item.text.match(pattern)?.[1];
|
|
786
|
+
if (match && metrics[key] == null)
|
|
787
|
+
metrics[key] = numberValue(match);
|
|
788
|
+
}
|
|
789
|
+
return metrics;
|
|
790
|
+
}
|
|
791
|
+
export function isAnalyzableVideoUrl(value) {
|
|
792
|
+
try {
|
|
793
|
+
const url = new URL(value);
|
|
794
|
+
if (!/^https?:$/.test(url.protocol))
|
|
795
|
+
return false;
|
|
796
|
+
const pathAndQuery = url.pathname + url.search;
|
|
797
|
+
if (/\.(?:jpe?g|png|webp|gif|avif)(?:$|\?)/i.test(pathAndQuery))
|
|
798
|
+
return false;
|
|
799
|
+
return /\.(?:mp4|mov|m4v|webm)(?:$|\?)/i.test(pathAndQuery)
|
|
800
|
+
|| /tiktok\.com$/i.test(url.hostname) && /\/video\/\d+/i.test(url.pathname)
|
|
801
|
+
|| /(?:tiktokcdn|byteoversea|ibytedtos|muscdn)/i.test(url.hostname) && /(?:video|tos)/i.test(url.pathname);
|
|
802
|
+
}
|
|
803
|
+
catch {
|
|
804
|
+
return false;
|
|
805
|
+
}
|
|
806
|
+
}
|
|
807
|
+
export function videoLearningSourcesFromItems(items, context) {
|
|
808
|
+
const output = [];
|
|
809
|
+
const seen = new Set();
|
|
810
|
+
for (const [index, item] of items.entries()) {
|
|
811
|
+
const mediaUrl = item.media.find((value) => isAnalyzableVideoUrl(value)) || "";
|
|
812
|
+
const publicVideoUrl = firstHttp(item.links, /tiktok\.com\/.*(?:video|v)\//i);
|
|
813
|
+
const traceUrl = firstHttp(item.links, /video|aweme|creative|material/i);
|
|
814
|
+
const sourceUrl = mediaUrl || publicVideoUrl || traceUrl;
|
|
815
|
+
const platformVideoId = videoIdFromValues([sourceUrl, ...item.links, ...item.media]) || stableId(`${context.dimension}|${item.text}|${item.images[0] || ""}`);
|
|
816
|
+
if (seen.has(platformVideoId))
|
|
817
|
+
continue;
|
|
818
|
+
const title = item.cells.find((cell) => cell.length >= 5 && cell.length <= 300) || item.text.split(/(?=销量|销售额|播放|曝光|ROAS)/i)[0]?.trim().slice(0, 500) || "";
|
|
819
|
+
if (!title || (!sourceUrl && !item.images.length))
|
|
820
|
+
continue;
|
|
821
|
+
seen.add(platformVideoId);
|
|
822
|
+
output.push({
|
|
823
|
+
source: "fastmoss-agent",
|
|
824
|
+
platform: "tiktok",
|
|
825
|
+
platformVideoId,
|
|
826
|
+
sourceUrl: sourceUrl || null,
|
|
827
|
+
market: context.market,
|
|
828
|
+
rankingDimension: context.dimension,
|
|
829
|
+
rank: index + 1,
|
|
830
|
+
title,
|
|
831
|
+
productTitle: item.cells[0] || title,
|
|
832
|
+
thumbnailUrl: firstHttp(item.images),
|
|
833
|
+
metrics: labeledMetrics(item),
|
|
834
|
+
observedAt: context.observedAt,
|
|
835
|
+
});
|
|
836
|
+
}
|
|
837
|
+
return output;
|
|
838
|
+
}
|
|
433
839
|
function findChromiumExecutable() {
|
|
434
840
|
const home = os.homedir();
|
|
435
841
|
const candidates = process.platform === "win32" ? [
|
package/dist/server/http.js
CHANGED
|
@@ -137,6 +137,7 @@ export function startHttpServer() {
|
|
|
137
137
|
app.post("/agent/integrations/fastmoss/refresh", route(async (_req, res) => res.json({ ok: true, ...await fastmoss.inspect() })));
|
|
138
138
|
app.post("/agent/integrations/fastmoss/switch-account", route(async (_req, res) => res.json({ ok: true, ...await fastmoss.switchAccount() })));
|
|
139
139
|
app.post("/agent/integrations/fastmoss/capture", route(async (req, res) => res.json({ ok: true, ...await fastmoss.capture(req.body || {}) })));
|
|
140
|
+
app.post("/agent/integrations/fastmoss/video-learning/capture", route(async (req, res) => res.json({ ok: true, ...await fastmoss.captureLearningVideos(req.body || {}) })));
|
|
140
141
|
app.post("/agent/integrations/fastmoss/close", route(async (_req, res) => res.json({ ok: true, ...await fastmoss.close() })));
|
|
141
142
|
app.get("/agent/codex/workspace", (_req, res) => {
|
|
142
143
|
const workspace = ensureSiteWorkspace(config);
|