@xiaohhhh1/canvas-agent 0.4.20 → 0.4.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/integrations/fastmoss.d.ts +68 -0
- package/dist/integrations/fastmoss.js +165 -41
- package/dist/relay-bridge.d.ts +14 -1
- package/dist/relay-bridge.js +29 -7
- package/dist/server/http.js +3 -2
- package/package.json +1 -1
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
export type FastMossRankingSource = "new" | "hot" | "video" | "sales";
|
|
1
2
|
export type FastMossPhase = "idle" | "saved" | "launching" | "login_required" | "ready" | "verification_required" | "membership_expired" | "error";
|
|
2
3
|
export type FastMossStatus = {
|
|
3
4
|
phase: FastMossPhase;
|
|
@@ -37,6 +38,21 @@ export declare class FastMossIntegration {
|
|
|
37
38
|
start(): Promise<FastMossStatus>;
|
|
38
39
|
inspect(): Promise<FastMossStatus>;
|
|
39
40
|
capture(input?: CaptureContext): Promise<{
|
|
41
|
+
records: Record<string, unknown>[];
|
|
42
|
+
allRows: Record<string, unknown>[];
|
|
43
|
+
trendProducts: number;
|
|
44
|
+
detailFailures: number;
|
|
45
|
+
interrupted: boolean;
|
|
46
|
+
phase: FastMossPhase;
|
|
47
|
+
browserOpen: boolean;
|
|
48
|
+
authenticated: boolean;
|
|
49
|
+
verificationRequired: boolean;
|
|
50
|
+
membershipExpired: boolean;
|
|
51
|
+
message: string;
|
|
52
|
+
url: string | null;
|
|
53
|
+
captured: number;
|
|
54
|
+
lastCapture?: string;
|
|
55
|
+
} | {
|
|
40
56
|
records: Record<string, unknown>[];
|
|
41
57
|
allRows: Record<string, unknown>[];
|
|
42
58
|
trendProducts: number;
|
|
@@ -61,7 +77,36 @@ export declare class FastMossIntegration {
|
|
|
61
77
|
private openProductRanking;
|
|
62
78
|
}
|
|
63
79
|
export declare function fastMossSalesRankUrl(market?: string): string;
|
|
80
|
+
export declare function fastMossRankingUrl(source: FastMossRankingSource, market?: string): string;
|
|
81
|
+
export declare function mergeRankingRecord(target: Map<string, Record<string, unknown>>, row: Record<string, unknown>, source: FastMossRankingSource, position: number): void;
|
|
82
|
+
export declare function selectDetailCandidates(records: Array<Record<string, unknown>>, limit?: number): Record<string, unknown>[];
|
|
64
83
|
export declare function productIdFromUrl(value: string): string;
|
|
84
|
+
/** Extract only the evidence required by the AI short-video selection model. */
|
|
85
|
+
export declare function detailOverviewEvidence(overviewPayload: unknown, basePayload?: unknown): {
|
|
86
|
+
recentVideoSales: number;
|
|
87
|
+
recentLiveSales: number;
|
|
88
|
+
recentCreatorCount: number;
|
|
89
|
+
recentVideoCount: number;
|
|
90
|
+
recentLiveCount: number;
|
|
91
|
+
videoTransactionShare: number | null;
|
|
92
|
+
liveTransactionShare: number | null;
|
|
93
|
+
adSalesShare: number | null;
|
|
94
|
+
organicTrafficShare: number | null;
|
|
95
|
+
creatorAdds: number;
|
|
96
|
+
videoAdds: number;
|
|
97
|
+
stock: number;
|
|
98
|
+
rating: number;
|
|
99
|
+
reviewCount: number;
|
|
100
|
+
totalCreatorCount: number;
|
|
101
|
+
totalVideoCount: number;
|
|
102
|
+
totalLiveCount: number;
|
|
103
|
+
viralIndex: number;
|
|
104
|
+
popularityIndex: number;
|
|
105
|
+
countryRank: number;
|
|
106
|
+
categoryRank: number;
|
|
107
|
+
launchTime: number;
|
|
108
|
+
commission: number | null;
|
|
109
|
+
};
|
|
65
110
|
export declare function detailTrendRows(record: Record<string, unknown>, payload: unknown, days?: number): {
|
|
66
111
|
date: string;
|
|
67
112
|
source: string;
|
|
@@ -71,5 +116,28 @@ export declare function detailTrendRows(record: Record<string, unknown>, payload
|
|
|
71
116
|
price: number;
|
|
72
117
|
currency: string;
|
|
73
118
|
period_days: number;
|
|
119
|
+
recentVideoSales: number;
|
|
120
|
+
recentLiveSales: number;
|
|
121
|
+
recentCreatorCount: number;
|
|
122
|
+
recentVideoCount: number;
|
|
123
|
+
recentLiveCount: number;
|
|
124
|
+
videoTransactionShare: number | null;
|
|
125
|
+
liveTransactionShare: number | null;
|
|
126
|
+
adSalesShare: number | null;
|
|
127
|
+
organicTrafficShare: number | null;
|
|
128
|
+
creatorAdds: number;
|
|
129
|
+
videoAdds: number;
|
|
130
|
+
stock: number;
|
|
131
|
+
rating: number;
|
|
132
|
+
reviewCount: number;
|
|
133
|
+
totalCreatorCount: number;
|
|
134
|
+
totalVideoCount: number;
|
|
135
|
+
totalLiveCount: number;
|
|
136
|
+
viralIndex: number;
|
|
137
|
+
popularityIndex: number;
|
|
138
|
+
countryRank: number;
|
|
139
|
+
categoryRank: number;
|
|
140
|
+
launchTime: number;
|
|
141
|
+
commission: number | null;
|
|
74
142
|
}[];
|
|
75
143
|
export {};
|
|
@@ -5,13 +5,19 @@ import path from "node:path";
|
|
|
5
5
|
import { chromium } from "playwright-core";
|
|
6
6
|
import { CONFIG_DIR } from "../config.js";
|
|
7
7
|
const FASTMOSS_HOME = "https://www.fastmoss.com/";
|
|
8
|
-
const FASTMOSS_SALES_RANK = "https://www.fastmoss.com/e-commerce/saleslist";
|
|
9
8
|
const MEMBERSHIP_RECHECK_MS = 5 * 60_000;
|
|
10
9
|
const PRODUCT_TABLE_TIMEOUT_MS = 45_000;
|
|
11
10
|
const PRODUCT_RANK_PAGE_SIZE = 10;
|
|
12
11
|
const PRODUCT_RANK_MAX_PAGES = 50;
|
|
12
|
+
const DETAIL_CANDIDATE_LIMIT = 200;
|
|
13
13
|
const DETAIL_TREND_DAYS = 7;
|
|
14
14
|
const DETAIL_FETCH_CONCURRENCY = 3;
|
|
15
|
+
const RANKING_DEFINITIONS = [
|
|
16
|
+
{ source: "new", label: "新品榜", pathname: "/e-commerce/newProducts" },
|
|
17
|
+
{ source: "hot", label: "热推榜", pathname: "/e-commerce/hotlist" },
|
|
18
|
+
{ source: "video", label: "视频商品榜", pathname: "/e-commerce/hotvideo" },
|
|
19
|
+
{ source: "sales", label: "销量榜", pathname: "/e-commerce/saleslist" },
|
|
20
|
+
];
|
|
15
21
|
export class FastMossIntegration {
|
|
16
22
|
context = null;
|
|
17
23
|
page = null;
|
|
@@ -171,24 +177,28 @@ export class FastMossIntegration {
|
|
|
171
177
|
}
|
|
172
178
|
if (!this.authenticated || !this.page || this.page.isClosed())
|
|
173
179
|
throw new Error("FastMoss 尚未登录;请在恢复的窗口中完成登录后重试");
|
|
174
|
-
await this.openProductRanking(input);
|
|
175
180
|
const collected = new Map();
|
|
176
181
|
const pages = [];
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
182
|
+
for (const ranking of RANKING_DEFINITIONS) {
|
|
183
|
+
await this.openProductRanking(input, ranking);
|
|
184
|
+
let previousSignature = "";
|
|
185
|
+
for (let pageNumber = 1; pageNumber <= PRODUCT_RANK_MAX_PAGES; pageNumber += 1) {
|
|
186
|
+
const snapshot = await readVisibleTableSnapshot(this.page);
|
|
187
|
+
if (!snapshot.tables[0]?.rows.length || snapshot.signature === previousSignature)
|
|
188
|
+
break;
|
|
189
|
+
previousSignature = snapshot.signature;
|
|
190
|
+
pages.push(snapshot.url);
|
|
191
|
+
const pageRecords = extractRows(snapshot.tables, { ...input, date: localDate() }, snapshot.url);
|
|
192
|
+
pageRecords.forEach((row, index) => mergeRankingRecord(collected, row, ranking.source, (pageNumber - 1) * PRODUCT_RANK_PAGE_SIZE + index + 1));
|
|
193
|
+
this.captured = collected.size;
|
|
194
|
+
this.message = `正在读取${ranking.label}第 ${pageNumber} 页,跨榜单去重后 ${this.captured} 个商品`;
|
|
195
|
+
if (!await clickVisibleNextPage(this.page))
|
|
196
|
+
break;
|
|
197
|
+
await this.page.waitForTimeout(800);
|
|
198
|
+
await this.inspect();
|
|
199
|
+
if (this.verificationRequired || this.membershipExpired)
|
|
200
|
+
break;
|
|
201
|
+
}
|
|
192
202
|
if (this.verificationRequired || this.membershipExpired)
|
|
193
203
|
break;
|
|
194
204
|
}
|
|
@@ -197,20 +207,39 @@ export class FastMossIntegration {
|
|
|
197
207
|
throw new Error("已自动进入 FastMoss 商品榜单,但没有读取到商品数据;请检查会员权限或页面是否仍在加载");
|
|
198
208
|
const requestedMarket = normalizeMarket(input.market);
|
|
199
209
|
const marketRecords = records.filter((record) => !record.market || normalizeExtractedMarket(record.market) === requestedMarket);
|
|
210
|
+
const retainPreviousOnInterruption = async (stage, partialTrendRows = [], detailFailures = 0) => {
|
|
211
|
+
await mkdir(this.dataDir, { recursive: true });
|
|
212
|
+
const attemptedAt = new Date().toISOString();
|
|
213
|
+
const previousRows = await this.loadRows();
|
|
214
|
+
await writeFile(path.join(this.dataDir, `capture-${localDate()}-interrupted-${Date.now()}.json`), JSON.stringify({ attemptedAt, interrupted: true, stage, pages, records: marketRecords, trendRows: partialTrendRows, detailFailures }, null, 2), "utf8");
|
|
215
|
+
this.captured = marketRecords.length;
|
|
216
|
+
const previousProducts = new Set(previousRows.map((row) => String(row.product_id))).size;
|
|
217
|
+
this.message = this.verificationRequired
|
|
218
|
+
? `采集在${stage === "ranking" ? "榜单" : "商品详情互证"}阶段遇到人工验证;已保留上次 ${previousProducts} 个商品结果,请完成滑块后重新抓取`
|
|
219
|
+
: `采集在${stage === "ranking" ? "榜单" : "商品详情互证"}阶段发现会员权限异常;已保留上次 ${previousProducts} 个商品结果`;
|
|
220
|
+
return { ...this.status(), records: marketRecords, allRows: previousRows, trendProducts: previousProducts, detailFailures, interrupted: true };
|
|
221
|
+
};
|
|
222
|
+
if (this.verificationRequired || this.membershipExpired)
|
|
223
|
+
return retainPreviousOnInterruption("ranking");
|
|
224
|
+
const detailCandidates = selectDetailCandidates(marketRecords, DETAIL_CANDIDATE_LIMIT);
|
|
225
|
+
const capturePage = this.page;
|
|
200
226
|
const trendRows = [];
|
|
201
227
|
let detailFailures = 0;
|
|
202
|
-
for (let offset = 0; offset <
|
|
203
|
-
const chunk =
|
|
204
|
-
const results = await Promise.all(chunk.map((record) => fetchDetailTrend(
|
|
228
|
+
for (let offset = 0; offset < detailCandidates.length; offset += DETAIL_FETCH_CONCURRENCY) {
|
|
229
|
+
const chunk = detailCandidates.slice(offset, offset + DETAIL_FETCH_CONCURRENCY);
|
|
230
|
+
const results = await Promise.all(chunk.map((record) => fetchDetailTrend(capturePage, record, DETAIL_TREND_DAYS)));
|
|
205
231
|
results.forEach((rows) => rows.length ? trendRows.push(...rows) : detailFailures += 1);
|
|
206
|
-
this.message =
|
|
207
|
-
await
|
|
232
|
+
this.message = `正在互证商品最近 ${DETAIL_TREND_DAYS} 天趋势:${Math.min(offset + chunk.length, detailCandidates.length)}/${detailCandidates.length}`;
|
|
233
|
+
await capturePage.waitForTimeout(150);
|
|
208
234
|
if ((offset / DETAIL_FETCH_CONCURRENCY) % 10 === 0) {
|
|
235
|
+
this.useLatestPage();
|
|
209
236
|
await this.inspect();
|
|
210
237
|
if (this.verificationRequired || this.membershipExpired)
|
|
211
238
|
break;
|
|
212
239
|
}
|
|
213
240
|
}
|
|
241
|
+
if (this.verificationRequired || this.membershipExpired)
|
|
242
|
+
return retainPreviousOnInterruption("detail", trendRows, detailFailures);
|
|
214
243
|
if (!trendRows.length && !this.verificationRequired)
|
|
215
244
|
throw new Error("榜单读取成功,但未能读取商品详情的最近 7 天销量;请检查会员权限后重试");
|
|
216
245
|
await mkdir(this.dataDir, { recursive: true });
|
|
@@ -218,11 +247,11 @@ export class FastMossIntegration {
|
|
|
218
247
|
this.lastCapture = new Date().toISOString();
|
|
219
248
|
this.captured = marketRecords.length;
|
|
220
249
|
await writeFile(path.join(this.dataDir, "observations.json"), JSON.stringify(allRows, null, 2), "utf8");
|
|
221
|
-
await writeFile(path.join(this.dataDir, `capture-${localDate()}.json`), JSON.stringify({ capturedAt: this.lastCapture, pages, records: marketRecords, trendRows, detailFailures }, null, 2), "utf8");
|
|
250
|
+
await writeFile(path.join(this.dataDir, `capture-${localDate()}.json`), JSON.stringify({ capturedAt: this.lastCapture, pages, records: marketRecords, detailCandidates: detailCandidates.map((record) => record.product_id), trendRows, detailFailures }, null, 2), "utf8");
|
|
222
251
|
const trendProducts = new Set(trendRows.map((row) => String(row.product_id))).size;
|
|
223
252
|
this.message = this.verificationRequired
|
|
224
253
|
? `已保存验证前的 ${trendProducts} 个商品趋势;请手动完成滑块后再次抓取`
|
|
225
|
-
:
|
|
254
|
+
: `四榜单去重后 ${marketRecords.length} 个商品,优先互证 ${detailCandidates.length} 个,其中 ${trendProducts} 个取得最近 ${DETAIL_TREND_DAYS} 天趋势`;
|
|
226
255
|
return { ...this.status(), records: marketRecords, allRows, trendProducts, detailFailures };
|
|
227
256
|
}
|
|
228
257
|
async close() {
|
|
@@ -276,18 +305,18 @@ export class FastMossIntegration {
|
|
|
276
305
|
if (pages.length)
|
|
277
306
|
this.page = pages[pages.length - 1];
|
|
278
307
|
}
|
|
279
|
-
async openProductRanking(input) {
|
|
308
|
+
async openProductRanking(input, ranking = RANKING_DEFINITIONS[0]) {
|
|
280
309
|
if (!this.page || this.page.isClosed())
|
|
281
310
|
throw new Error("FastMoss 专用浏览器未打开");
|
|
282
311
|
const market = normalizeMarket(input.market);
|
|
283
|
-
this.message = `正在自动进入 ${market}
|
|
284
|
-
await this.page.goto(
|
|
312
|
+
this.message = `正在自动进入 ${market} ${ranking.label}`;
|
|
313
|
+
await this.page.goto(fastMossRankingUrl(ranking.source, market), { waitUntil: "domcontentloaded", timeout: 60_000 });
|
|
285
314
|
const deadline = Date.now() + PRODUCT_TABLE_TIMEOUT_MS;
|
|
286
315
|
while (Date.now() < deadline) {
|
|
287
316
|
const snapshot = await readVisibleTableSnapshot(this.page);
|
|
288
317
|
if (hasReadableProductRows(snapshot)) {
|
|
289
318
|
this.phase = "ready";
|
|
290
|
-
this.message = `${market}
|
|
319
|
+
this.message = `${market} ${ranking.label}已打开,正在开始采集`;
|
|
291
320
|
return;
|
|
292
321
|
}
|
|
293
322
|
await this.inspect();
|
|
@@ -303,12 +332,39 @@ export class FastMossIntegration {
|
|
|
303
332
|
}
|
|
304
333
|
}
|
|
305
334
|
export function fastMossSalesRankUrl(market) {
|
|
306
|
-
|
|
335
|
+
return fastMossRankingUrl("sales", market);
|
|
336
|
+
}
|
|
337
|
+
export function fastMossRankingUrl(source, market) {
|
|
338
|
+
const definition = RANKING_DEFINITIONS.find((item) => item.source === source) || RANKING_DEFINITIONS[0];
|
|
339
|
+
const url = new URL(definition.pathname, FASTMOSS_HOME);
|
|
307
340
|
url.searchParams.set("region", normalizeMarket(market));
|
|
308
341
|
url.searchParams.set("page", "1");
|
|
309
342
|
url.searchParams.set("pagesize", String(PRODUCT_RANK_PAGE_SIZE));
|
|
310
343
|
return url.toString();
|
|
311
344
|
}
|
|
345
|
+
export function mergeRankingRecord(target, row, source, position) {
|
|
346
|
+
const key = String(row.product_id || row.title);
|
|
347
|
+
const existing = target.get(key) || {};
|
|
348
|
+
const sources = [...new Set([...(Array.isArray(existing.ranking_sources) ? existing.ranking_sources.map(String) : []), source])];
|
|
349
|
+
const positions = existing.ranking_positions && typeof existing.ranking_positions === "object" ? existing.ranking_positions : {};
|
|
350
|
+
target.set(key, {
|
|
351
|
+
...existing,
|
|
352
|
+
...row,
|
|
353
|
+
ranking_sources: sources,
|
|
354
|
+
ranking_positions: { ...positions, [source]: position },
|
|
355
|
+
cross_list_count: sources.filter((item) => item !== "sales").length,
|
|
356
|
+
});
|
|
357
|
+
}
|
|
358
|
+
export function selectDetailCandidates(records, limit = DETAIL_CANDIDATE_LIMIT) {
|
|
359
|
+
const bestPosition = (record) => {
|
|
360
|
+
const positions = record.ranking_positions && typeof record.ranking_positions === "object" ? Object.values(record.ranking_positions) : [];
|
|
361
|
+
return Math.min(...positions.map(Number).filter(Number.isFinite), Number.MAX_SAFE_INTEGER);
|
|
362
|
+
};
|
|
363
|
+
return [...records].sort((left, right) => {
|
|
364
|
+
const crossDifference = Number(right.cross_list_count || 0) - Number(left.cross_list_count || 0);
|
|
365
|
+
return crossDifference || bestPosition(left) - bestPosition(right);
|
|
366
|
+
}).slice(0, Math.max(1, limit));
|
|
367
|
+
}
|
|
312
368
|
function normalizeMarket(value) {
|
|
313
369
|
const market = String(value || "MX").trim().toUpperCase();
|
|
314
370
|
return /^[A-Z]{2}$/.test(market) ? market : "MX";
|
|
@@ -406,6 +462,7 @@ function extractRows(tables, context, pageUrl) {
|
|
|
406
462
|
const storeName = String(pick(/所属店铺|店铺|shop|store/i)).replace(/\s*店铺销量\s*[::].*$/is, "").trim();
|
|
407
463
|
output.push({
|
|
408
464
|
date: context.date,
|
|
465
|
+
capture_date: context.date,
|
|
409
466
|
source: "fastmoss-agent",
|
|
410
467
|
product_id: productId,
|
|
411
468
|
platform_product_id: String(pick(/平台商品|tiktok.*id/i) || productId),
|
|
@@ -444,15 +501,68 @@ export function productIdFromUrl(value) {
|
|
|
444
501
|
return String(value).match(/(?:detail|product|goods|item)[/=_-](\d+)/i)?.[1] || "";
|
|
445
502
|
}
|
|
446
503
|
}
|
|
504
|
+
function percentValue(value) {
|
|
505
|
+
const number = Number(String(value ?? "").replace("%", ""));
|
|
506
|
+
return Number.isFinite(number) ? number : null;
|
|
507
|
+
}
|
|
508
|
+
function distributionShare(value, expected) {
|
|
509
|
+
const container = value && typeof value === "object" ? value : {};
|
|
510
|
+
const list = Array.isArray(container.list) ? container.list : [];
|
|
511
|
+
const item = list.find((entry) => expected.test(String(entry.category ?? entry.source ?? "")));
|
|
512
|
+
return item ? percentValue(item.propotion ?? item.proportion ?? item.percent) : null;
|
|
513
|
+
}
|
|
514
|
+
/** Extract only the evidence required by the AI short-video selection model. */
|
|
515
|
+
export function detailOverviewEvidence(overviewPayload, basePayload) {
|
|
516
|
+
const overviewEnvelope = overviewPayload && typeof overviewPayload === "object" ? overviewPayload : {};
|
|
517
|
+
const overviewData = overviewEnvelope.data && typeof overviewEnvelope.data === "object" ? overviewEnvelope.data : {};
|
|
518
|
+
const overview = overviewData.overview && typeof overviewData.overview === "object" ? overviewData.overview : {};
|
|
519
|
+
const content = overviewData.content_distribution && typeof overviewData.content_distribution === "object" ? overviewData.content_distribution : {};
|
|
520
|
+
const ads = overviewData.ads_distribution && typeof overviewData.ads_distribution === "object" ? overviewData.ads_distribution : {};
|
|
521
|
+
const chart = Array.isArray(overviewData.chart_list) ? overviewData.chart_list : [];
|
|
522
|
+
const baseEnvelope = basePayload && typeof basePayload === "object" ? basePayload : {};
|
|
523
|
+
const baseData = baseEnvelope.data && typeof baseEnvelope.data === "object" ? baseEnvelope.data : {};
|
|
524
|
+
const product = baseData.product && typeof baseData.product === "object" ? baseData.product : {};
|
|
525
|
+
const videoShare = distributionShare(content.units_sold, /^video$/i);
|
|
526
|
+
const liveShare = distributionShare(content.units_sold, /^live$/i);
|
|
527
|
+
const adShare = distributionShare(ads.units_sold, /adtraffic|advert|广告/i);
|
|
528
|
+
const organicShare = distributionShare(ads.units_sold, /othertraffic|organic|自然|其他/i);
|
|
529
|
+
return {
|
|
530
|
+
recentVideoSales: Number(overview.video_sold_count || 0),
|
|
531
|
+
recentLiveSales: Number(overview.live_sold_count || 0),
|
|
532
|
+
recentCreatorCount: Number(overview.author_count || 0),
|
|
533
|
+
recentVideoCount: Number(overview.aweme_count || 0),
|
|
534
|
+
recentLiveCount: Number(overview.live_count || 0),
|
|
535
|
+
videoTransactionShare: videoShare,
|
|
536
|
+
liveTransactionShare: liveShare,
|
|
537
|
+
adSalesShare: adShare,
|
|
538
|
+
organicTrafficShare: organicShare,
|
|
539
|
+
creatorAdds: chart.reduce((sum, point) => sum + Number(point.inc_author_count || 0), 0),
|
|
540
|
+
videoAdds: chart.reduce((sum, point) => sum + Number(point.inc_aweme_count || 0), 0),
|
|
541
|
+
stock: Number(product.stock_count || 0),
|
|
542
|
+
rating: Number(product.product_rating || 0),
|
|
543
|
+
reviewCount: Number(product.review_count || 0),
|
|
544
|
+
totalCreatorCount: Number(product.author_count || 0),
|
|
545
|
+
totalVideoCount: Number(product.aweme_count || 0),
|
|
546
|
+
totalLiveCount: Number(product.live_count || 0),
|
|
547
|
+
viralIndex: Number(product.viral_index || 0),
|
|
548
|
+
popularityIndex: Number(product.popularity_index || 0),
|
|
549
|
+
countryRank: Number(product.country_rank || 0),
|
|
550
|
+
categoryRank: Number(product.category_rank || 0),
|
|
551
|
+
launchTime: Number(product.launch_time || 0),
|
|
552
|
+
commission: percentValue(product.commission_rate),
|
|
553
|
+
};
|
|
554
|
+
}
|
|
447
555
|
export function detailTrendRows(record, payload, days = DETAIL_TREND_DAYS) {
|
|
448
556
|
const envelope = payload && typeof payload === "object" ? payload : {};
|
|
449
557
|
const data = envelope.data && typeof envelope.data === "object" ? envelope.data : {};
|
|
450
558
|
const points = Array.isArray(data.chart_list) ? data.chart_list : [];
|
|
559
|
+
const evidence = detailOverviewEvidence(payload);
|
|
451
560
|
return points.slice(-days).flatMap((point) => {
|
|
452
561
|
const date = String(point.dt || "").slice(0, 10);
|
|
453
562
|
if (!/^\d{4}-\d{2}-\d{2}$/.test(date))
|
|
454
563
|
return [];
|
|
455
564
|
return [{
|
|
565
|
+
...evidence,
|
|
456
566
|
...record,
|
|
457
567
|
date,
|
|
458
568
|
source: "fastmoss-agent-detail-trend",
|
|
@@ -469,18 +579,32 @@ async function fetchDetailTrend(page, record, days) {
|
|
|
469
579
|
const productId = String(record.product_id || productIdFromUrl(String(record.product_url || "")));
|
|
470
580
|
if (!/^\d+$/.test(productId))
|
|
471
581
|
return [];
|
|
472
|
-
const
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
582
|
+
const rawUrl = String(record.product_url || "");
|
|
583
|
+
const detailUrl = /fastmoss\.com\/.*e-commerce\/detail\//i.test(rawUrl)
|
|
584
|
+
? rawUrl
|
|
585
|
+
: new URL(`/zh/e-commerce/detail/${productId}`, FASTMOSS_HOME).toString();
|
|
586
|
+
const detailPage = await page.context().newPage();
|
|
587
|
+
try {
|
|
588
|
+
// Capture only responses emitted by the visible product-detail page.
|
|
589
|
+
// The Agent does not manufacture requests to private endpoints.
|
|
590
|
+
const overviewResponse = detailPage.waitForResponse((response) => {
|
|
591
|
+
const url = response.url();
|
|
592
|
+
return response.ok() && url.includes("/api/goods/v3/overview") && url.includes(productId);
|
|
593
|
+
}, { timeout: PRODUCT_TABLE_TIMEOUT_MS }).then((response) => response.json()).catch(() => null);
|
|
594
|
+
const baseResponse = detailPage.waitForResponse((response) => {
|
|
595
|
+
const url = response.url();
|
|
596
|
+
return response.ok() && url.includes("/api/goods/v3/base") && url.includes(productId);
|
|
597
|
+
}, { timeout: PRODUCT_TABLE_TIMEOUT_MS }).then((response) => response.json()).catch(() => null);
|
|
598
|
+
await detailPage.goto(detailUrl, { waitUntil: "domcontentloaded", timeout: 60_000 });
|
|
599
|
+
const [overview, base] = await Promise.all([overviewResponse, baseResponse]);
|
|
600
|
+
if (!overview)
|
|
601
|
+
return [];
|
|
602
|
+
const evidence = detailOverviewEvidence(overview, base);
|
|
603
|
+
return detailTrendRows({ ...record, ...evidence, product_id: productId, platform_product_id: productId }, overview, days);
|
|
604
|
+
}
|
|
605
|
+
finally {
|
|
606
|
+
await detailPage.close().catch(() => undefined);
|
|
607
|
+
}
|
|
484
608
|
}
|
|
485
609
|
function normalizeExtractedMarket(value) {
|
|
486
610
|
const key = String(value || "").trim().toLowerCase();
|
package/dist/relay-bridge.d.ts
CHANGED
|
@@ -1,7 +1,20 @@
|
|
|
1
1
|
import type { CanvasAgentConfig } from "./config.js";
|
|
2
|
+
export type RelayBridgeStatus = {
|
|
3
|
+
ready: boolean;
|
|
4
|
+
lastReadyAt?: string;
|
|
5
|
+
lastDisconnectAt?: string;
|
|
6
|
+
};
|
|
7
|
+
export type RelayBridgeOptions = {
|
|
8
|
+
relayUrl?: string;
|
|
9
|
+
reconnectDelayMs?: number;
|
|
10
|
+
heartbeatIntervalMs?: number;
|
|
11
|
+
readyTimeoutMs?: number;
|
|
12
|
+
livenessTimeoutMs?: number;
|
|
13
|
+
onStatus?: (status: RelayBridgeStatus) => void;
|
|
14
|
+
};
|
|
2
15
|
/**
|
|
3
16
|
* Keeps an outbound, encrypted connection to the production relay. The canvas
|
|
4
17
|
* browser can then use same-origin requests instead of directly reaching a
|
|
5
18
|
* loopback HTTP address, which Chromium clients can block before CORS runs.
|
|
6
19
|
*/
|
|
7
|
-
export declare function startRelayBridge(config: CanvasAgentConfig): () => void;
|
|
20
|
+
export declare function startRelayBridge(config: CanvasAgentConfig, options?: RelayBridgeOptions): () => void;
|
package/dist/relay-bridge.js
CHANGED
|
@@ -9,8 +9,12 @@ const LIVENESS_TIMEOUT_MS = 45_000;
|
|
|
9
9
|
* browser can then use same-origin requests instead of directly reaching a
|
|
10
10
|
* loopback HTTP address, which Chromium clients can block before CORS runs.
|
|
11
11
|
*/
|
|
12
|
-
export function startRelayBridge(config) {
|
|
13
|
-
const relayUrl = process.env.CANVAS_AGENT_RELAY_URL || DEFAULT_RELAY_URL;
|
|
12
|
+
export function startRelayBridge(config, options = {}) {
|
|
13
|
+
const relayUrl = options.relayUrl || process.env.CANVAS_AGENT_RELAY_URL || DEFAULT_RELAY_URL;
|
|
14
|
+
const reconnectDelayMs = options.reconnectDelayMs ?? RECONNECT_DELAY_MS;
|
|
15
|
+
const heartbeatIntervalMs = options.heartbeatIntervalMs ?? HEARTBEAT_INTERVAL_MS;
|
|
16
|
+
const readyTimeoutMs = options.readyTimeoutMs ?? READY_TIMEOUT_MS;
|
|
17
|
+
const livenessTimeoutMs = options.livenessTimeoutMs ?? LIVENESS_TIMEOUT_MS;
|
|
14
18
|
const subscriptions = new Map();
|
|
15
19
|
let socket = null;
|
|
16
20
|
let stopped = false;
|
|
@@ -19,6 +23,9 @@ export function startRelayBridge(config) {
|
|
|
19
23
|
let reconnectTimer = null;
|
|
20
24
|
let heartbeatTimer = null;
|
|
21
25
|
let readyTimer = null;
|
|
26
|
+
let lastReadyAt;
|
|
27
|
+
let lastDisconnectAt;
|
|
28
|
+
const publishStatus = () => options.onStatus?.({ ready: relayReady, lastReadyAt, lastDisconnectAt });
|
|
22
29
|
const send = (message) => {
|
|
23
30
|
const current = socket;
|
|
24
31
|
if (!relayReady || current?.readyState !== WebSocket.OPEN)
|
|
@@ -52,7 +59,13 @@ export function startRelayBridge(config) {
|
|
|
52
59
|
stopSubscription(clientId);
|
|
53
60
|
const controller = new AbortController();
|
|
54
61
|
subscriptions.set(clientId, controller);
|
|
55
|
-
void pipeEvents(clientId, config, controller.signal, send)
|
|
62
|
+
void pipeEvents(clientId, config, controller.signal, send)
|
|
63
|
+
.catch(() => {
|
|
64
|
+
// A browser may disappear while the relay subscription is being
|
|
65
|
+
// established. That is a recoverable client disconnect, not a
|
|
66
|
+
// process-fatal unhandled rejection.
|
|
67
|
+
})
|
|
68
|
+
.finally(() => {
|
|
56
69
|
if (subscriptions.get(clientId) === controller)
|
|
57
70
|
subscriptions.delete(clientId);
|
|
58
71
|
});
|
|
@@ -99,7 +112,7 @@ export function startRelayBridge(config) {
|
|
|
99
112
|
reconnectTimer = setTimeout(() => {
|
|
100
113
|
reconnectTimer = null;
|
|
101
114
|
connect();
|
|
102
|
-
},
|
|
115
|
+
}, reconnectDelayMs);
|
|
103
116
|
};
|
|
104
117
|
const connect = () => {
|
|
105
118
|
if (stopped || socket)
|
|
@@ -107,6 +120,10 @@ export function startRelayBridge(config) {
|
|
|
107
120
|
try {
|
|
108
121
|
const current = new WebSocket(relayUrl);
|
|
109
122
|
socket = current;
|
|
123
|
+
// Cover DNS/TCP/TLS/WebSocket handshakes as well as the relay hello.
|
|
124
|
+
// Starting this timer only after `open` leaves a CONNECTING socket
|
|
125
|
+
// able to stall forever and prevents every future reconnect.
|
|
126
|
+
readyTimer = setTimeout(() => current.terminate(), readyTimeoutMs);
|
|
110
127
|
let disconnected = false;
|
|
111
128
|
const disconnect = () => {
|
|
112
129
|
if (disconnected)
|
|
@@ -115,6 +132,8 @@ export function startRelayBridge(config) {
|
|
|
115
132
|
if (socket === current)
|
|
116
133
|
socket = null;
|
|
117
134
|
relayReady = false;
|
|
135
|
+
lastDisconnectAt = new Date().toISOString();
|
|
136
|
+
publishStatus();
|
|
118
137
|
clearConnectionTimers();
|
|
119
138
|
abortSubscriptions();
|
|
120
139
|
scheduleReconnect();
|
|
@@ -127,13 +146,14 @@ export function startRelayBridge(config) {
|
|
|
127
146
|
current.terminate();
|
|
128
147
|
return;
|
|
129
148
|
}
|
|
130
|
-
readyTimer = setTimeout(() => current.terminate(), READY_TIMEOUT_MS);
|
|
131
149
|
});
|
|
132
150
|
current.on("message", (raw) => {
|
|
133
151
|
try {
|
|
134
152
|
const message = JSON.parse(raw.toString());
|
|
135
153
|
if (message.type === "ready") {
|
|
136
154
|
relayReady = true;
|
|
155
|
+
lastReadyAt = new Date().toISOString();
|
|
156
|
+
publishStatus();
|
|
137
157
|
lastHeartbeatAck = Date.now();
|
|
138
158
|
if (readyTimer)
|
|
139
159
|
clearTimeout(readyTimer);
|
|
@@ -141,12 +161,12 @@ export function startRelayBridge(config) {
|
|
|
141
161
|
if (heartbeatTimer)
|
|
142
162
|
clearInterval(heartbeatTimer);
|
|
143
163
|
heartbeatTimer = setInterval(() => {
|
|
144
|
-
if (Date.now() - lastHeartbeatAck >
|
|
164
|
+
if (Date.now() - lastHeartbeatAck > livenessTimeoutMs) {
|
|
145
165
|
current.terminate();
|
|
146
166
|
return;
|
|
147
167
|
}
|
|
148
168
|
send({ type: "heartbeat", time: Date.now() });
|
|
149
|
-
},
|
|
169
|
+
}, heartbeatIntervalMs);
|
|
150
170
|
heartbeatTimer.unref();
|
|
151
171
|
return;
|
|
152
172
|
}
|
|
@@ -168,6 +188,8 @@ export function startRelayBridge(config) {
|
|
|
168
188
|
catch {
|
|
169
189
|
socket = null;
|
|
170
190
|
relayReady = false;
|
|
191
|
+
lastDisconnectAt = new Date().toISOString();
|
|
192
|
+
publishStatus();
|
|
171
193
|
clearConnectionTimers();
|
|
172
194
|
scheduleReconnect();
|
|
173
195
|
}
|
package/dist/server/http.js
CHANGED
|
@@ -33,6 +33,7 @@ export function startHttpServer() {
|
|
|
33
33
|
};
|
|
34
34
|
const workflows = new WorkflowManager(config, emit);
|
|
35
35
|
const fastmoss = new FastMossIntegration();
|
|
36
|
+
let relayStatus = { ready: false };
|
|
36
37
|
const app = express();
|
|
37
38
|
app.disable("x-powered-by");
|
|
38
39
|
app.use(express.json({ limit: "30mb" }));
|
|
@@ -56,7 +57,7 @@ export function startHttpServer() {
|
|
|
56
57
|
return void res.json({});
|
|
57
58
|
next();
|
|
58
59
|
});
|
|
59
|
-
app.get("/health", (_req, res) => res.json({ ...session.health(), version: VERSION }));
|
|
60
|
+
app.get("/health", (_req, res) => res.json({ ...session.health(), version: VERSION, relayReady: relayStatus.ready, relayLastReadyAt: relayStatus.lastReadyAt, relayLastDisconnectAt: relayStatus.lastDisconnectAt }));
|
|
60
61
|
app.get("/config", (_req, res) => res.json({ ok: true, url: config.url, hasToken: true }));
|
|
61
62
|
app.use((req, res, next) => {
|
|
62
63
|
if (validToken(req, requestUrl(req, config), config.token))
|
|
@@ -285,7 +286,7 @@ export function startHttpServer() {
|
|
|
285
286
|
console.log("Codex MCP is not installed by this command.");
|
|
286
287
|
console.log("Optional MCP add: codex mcp add infinite-canvas -- npx -y @xiaohhhh1/canvas-agent mcp");
|
|
287
288
|
console.log("Remove manually added MCP: codex mcp remove infinite-canvas");
|
|
288
|
-
startRelayBridge(config);
|
|
289
|
+
startRelayBridge(config, { onStatus: (status) => { relayStatus = status; } });
|
|
289
290
|
if (logger.enabled)
|
|
290
291
|
console.log(`Debug log: ${logger.filePath}`);
|
|
291
292
|
logger.info("Canvas Agent started", { url: config.url, workspace: ensureSiteWorkspace(config).workspacePath, debugLog: logger.filePath });
|