@xiaohhhh1/canvas-agent 0.4.16 → 0.4.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export type FastMossPhase = "idle" | "launching" | "login_required" | "ready" | "verification_required" | "membership_expired" | "error";
|
|
1
|
+
export type FastMossPhase = "idle" | "saved" | "launching" | "login_required" | "ready" | "verification_required" | "membership_expired" | "error";
|
|
2
2
|
export type FastMossStatus = {
|
|
3
3
|
phase: FastMossPhase;
|
|
4
4
|
browserOpen: boolean;
|
|
@@ -13,6 +13,7 @@ export type FastMossStatus = {
|
|
|
13
13
|
type CaptureContext = {
|
|
14
14
|
market?: string;
|
|
15
15
|
category?: string;
|
|
16
|
+
categories?: string[];
|
|
16
17
|
shopType?: string;
|
|
17
18
|
periodDays?: number;
|
|
18
19
|
};
|
|
@@ -28,7 +29,10 @@ export declare class FastMossIntegration {
|
|
|
28
29
|
private lastCapture?;
|
|
29
30
|
private lastMembershipCheckAt;
|
|
30
31
|
private readonly profileDir;
|
|
32
|
+
private readonly authenticatedMarker;
|
|
31
33
|
private readonly dataDir;
|
|
34
|
+
private savedAuthenticated;
|
|
35
|
+
constructor();
|
|
32
36
|
status(): FastMossStatus;
|
|
33
37
|
start(): Promise<FastMossStatus>;
|
|
34
38
|
inspect(): Promise<FastMossStatus>;
|
|
@@ -49,5 +53,7 @@ export declare class FastMossIntegration {
|
|
|
49
53
|
switchAccount(): Promise<FastMossStatus>;
|
|
50
54
|
private loadRows;
|
|
51
55
|
private useLatestPage;
|
|
56
|
+
private openProductRanking;
|
|
52
57
|
}
|
|
58
|
+
export declare function fastMossSalesRankUrl(market?: string): string;
|
|
53
59
|
export {};
|
|
@@ -1,11 +1,15 @@
|
|
|
1
1
|
import { existsSync } from "node:fs";
|
|
2
|
-
import { mkdir, readFile, writeFile } from "node:fs/promises";
|
|
2
|
+
import { mkdir, readFile, rm, writeFile } from "node:fs/promises";
|
|
3
3
|
import os from "node:os";
|
|
4
4
|
import path from "node:path";
|
|
5
5
|
import { chromium } from "playwright-core";
|
|
6
6
|
import { CONFIG_DIR } from "../config.js";
|
|
7
7
|
const FASTMOSS_HOME = "https://www.fastmoss.com/";
|
|
8
|
+
const FASTMOSS_SALES_RANK = "https://www.fastmoss.com/e-commerce/saleslist";
|
|
8
9
|
const MEMBERSHIP_RECHECK_MS = 5 * 60_000;
|
|
10
|
+
const PRODUCT_TABLE_TIMEOUT_MS = 45_000;
|
|
11
|
+
const PRODUCT_RANK_PAGE_SIZE = 10;
|
|
12
|
+
const PRODUCT_RANK_MAX_PAGES = 50;
|
|
9
13
|
export class FastMossIntegration {
|
|
10
14
|
context = null;
|
|
11
15
|
page = null;
|
|
@@ -18,7 +22,16 @@ export class FastMossIntegration {
|
|
|
18
22
|
lastCapture;
|
|
19
23
|
lastMembershipCheckAt = 0;
|
|
20
24
|
profileDir = path.join(CONFIG_DIR, "fastmoss-session");
|
|
25
|
+
authenticatedMarker = path.join(this.profileDir, "authenticated.json");
|
|
21
26
|
dataDir = path.join(CONFIG_DIR, "fastmoss-selection");
|
|
27
|
+
savedAuthenticated = existsSync(this.authenticatedMarker);
|
|
28
|
+
constructor() {
|
|
29
|
+
if (!this.savedAuthenticated)
|
|
30
|
+
return;
|
|
31
|
+
this.phase = "saved";
|
|
32
|
+
this.authenticated = true;
|
|
33
|
+
this.message = "FastMoss 登录已保存在本机；窗口可以关闭，抓取时会自动恢复";
|
|
34
|
+
}
|
|
22
35
|
status() {
|
|
23
36
|
return {
|
|
24
37
|
phase: this.phase,
|
|
@@ -53,8 +66,10 @@ export class FastMossIntegration {
|
|
|
53
66
|
this.context = await chromium.launchPersistentContext(this.profileDir, {
|
|
54
67
|
executablePath,
|
|
55
68
|
headless: false,
|
|
56
|
-
viewport:
|
|
57
|
-
|
|
69
|
+
viewport: { width: 480, height: 900 },
|
|
70
|
+
screen: { width: 515, height: 995 },
|
|
71
|
+
deviceScaleFactor: 1,
|
|
72
|
+
args: ["--window-size=515,995", "--window-position=20,20"],
|
|
58
73
|
});
|
|
59
74
|
}
|
|
60
75
|
catch (error) {
|
|
@@ -69,12 +84,14 @@ export class FastMossIntegration {
|
|
|
69
84
|
this.context.on("close", () => {
|
|
70
85
|
this.context = null;
|
|
71
86
|
this.page = null;
|
|
72
|
-
this.phase = "idle";
|
|
73
|
-
this.authenticated =
|
|
87
|
+
this.phase = this.savedAuthenticated ? "saved" : "idle";
|
|
88
|
+
this.authenticated = this.savedAuthenticated;
|
|
74
89
|
this.verificationRequired = false;
|
|
75
90
|
this.membershipExpired = false;
|
|
76
91
|
this.lastMembershipCheckAt = 0;
|
|
77
|
-
this.message =
|
|
92
|
+
this.message = this.savedAuthenticated
|
|
93
|
+
? "FastMoss 窗口已关闭，登录仍保存在本机；抓取时会自动恢复"
|
|
94
|
+
: "FastMoss 窗口已关闭；请先登录一次";
|
|
78
95
|
});
|
|
79
96
|
if (!this.page.url() || this.page.url() === "about:blank")
|
|
80
97
|
await this.page.goto(FASTMOSS_HOME, { waitUntil: "domcontentloaded", timeout: 60_000 });
|
|
@@ -83,31 +100,45 @@ export class FastMossIntegration {
|
|
|
83
100
|
async inspect() {
|
|
84
101
|
this.useLatestPage();
|
|
85
102
|
if (!this.page || this.page.isClosed()) {
|
|
86
|
-
this.phase = "idle";
|
|
87
|
-
this.authenticated =
|
|
103
|
+
this.phase = this.savedAuthenticated ? "saved" : "idle";
|
|
104
|
+
this.authenticated = this.savedAuthenticated;
|
|
88
105
|
this.verificationRequired = false;
|
|
89
106
|
this.membershipExpired = false;
|
|
90
107
|
this.lastMembershipCheckAt = 0;
|
|
91
|
-
this.message =
|
|
108
|
+
this.message = this.savedAuthenticated
|
|
109
|
+
? "FastMoss 登录已保存在本机；窗口可以关闭，抓取时会自动恢复"
|
|
110
|
+
: "请先启动 FastMoss 专用浏览器并登录";
|
|
92
111
|
return this.status();
|
|
93
112
|
}
|
|
94
113
|
const snapshot = await this.page.evaluate(() => ({
|
|
95
114
|
title: document.title,
|
|
96
115
|
url: location.href,
|
|
97
116
|
text: (document.body?.innerText || "").slice(0, 12_000),
|
|
98
|
-
hasTable:
|
|
117
|
+
hasTable: [...document.querySelectorAll("table tbody tr")].some((row) => {
|
|
118
|
+
const text = (row.textContent || "").replace(/\s+/g, " ").trim();
|
|
119
|
+
return row.querySelectorAll("td").length >= 3 && !/^(?:no data|loading\.?\.?.?|暂无数据|暂无商品|无数据)$/i.test(text);
|
|
120
|
+
}),
|
|
99
121
|
})).catch(() => ({ title: "", url: this.page?.url() || "", text: "", hasTable: false }));
|
|
100
122
|
const haystack = `${snapshot.title}\n${snapshot.url}\n${snapshot.text}`.toLowerCase();
|
|
101
|
-
this.verificationRequired = /captcha|verify|verification|security check|人机验证|安全验证|滑块验证|请完成验证/.test(haystack);
|
|
102
|
-
const loginRequired = /\/login|\/signin|sign in|log in
|
|
123
|
+
this.verificationRequired = /captcha|verify|verification|security check|slide to complete the puzzle|verification failed|人机验证|安全验证|滑块验证|请完成验证/.test(haystack);
|
|
124
|
+
const loginRequired = /\/login|\/signin|sign in|log in|登录账号|密码登录|扫码登录|登录\s*\/\s*注册/.test(haystack) && !snapshot.hasTable;
|
|
103
125
|
const explicitExpiry = /(?:会员|套餐|订阅|专业版|高级版|账号).{0,16}(?:已过期|已到期|到期失效|无法使用)|(?:membership|subscription|plan|account).{0,28}(?:has\s+)?expired|(?:membership|subscription|plan).{0,20}(?:is no longer active|needs renewal)/i.test(haystack);
|
|
104
126
|
const entitlementUnavailable = /(?:当前账号|当前套餐).{0,20}(?:无权|没有权限|不支持|无法)(?:查看|访问|使用)|(?:开通|升级)(?:会员|专业版|套餐).{0,16}(?:后)?(?:才可|方可)(?:查看|访问|使用)|仅限(?:会员|专业版)(?:查看|使用)/i.test(haystack);
|
|
105
127
|
this.membershipExpired = explicitExpiry || (!snapshot.hasTable && entitlementUnavailable);
|
|
106
128
|
this.lastMembershipCheckAt = Date.now();
|
|
107
|
-
this.authenticated = !this.verificationRequired && !loginRequired && /fastmoss\.com/i.test(snapshot.url)
|
|
129
|
+
this.authenticated = !this.verificationRequired && !loginRequired && /fastmoss\.com/i.test(snapshot.url);
|
|
130
|
+
if (this.authenticated) {
|
|
131
|
+
this.savedAuthenticated = true;
|
|
132
|
+
await mkdir(this.profileDir, { recursive: true });
|
|
133
|
+
await writeFile(this.authenticatedMarker, JSON.stringify({ authenticated: true, updatedAt: new Date().toISOString() }), "utf8");
|
|
134
|
+
}
|
|
135
|
+
else if (loginRequired) {
|
|
136
|
+
this.savedAuthenticated = false;
|
|
137
|
+
await rm(this.authenticatedMarker, { force: true });
|
|
138
|
+
}
|
|
108
139
|
if (this.verificationRequired) {
|
|
109
140
|
this.phase = "verification_required";
|
|
110
|
-
this.message = "
|
|
141
|
+
this.message = "检测到 FastMoss 滑块验证，请在移动版登录窗口中手动完成；通过后刷新状态即可继续";
|
|
111
142
|
}
|
|
112
143
|
else if (this.membershipExpired) {
|
|
113
144
|
this.phase = "membership_expired";
|
|
@@ -115,15 +146,17 @@ export class FastMossIntegration {
|
|
|
115
146
|
}
|
|
116
147
|
else if (!this.authenticated) {
|
|
117
148
|
this.phase = "login_required";
|
|
118
|
-
this.message = "
|
|
149
|
+
this.message = "请在移动版窗口中登录 FastMoss；登录成功后可以关闭窗口，登录态会保存在本机";
|
|
119
150
|
}
|
|
120
151
|
else {
|
|
121
152
|
this.phase = "ready";
|
|
122
|
-
this.message = snapshot.hasTable ? "FastMoss
|
|
153
|
+
this.message = snapshot.hasTable ? "FastMoss 已连接，商品榜单可以抓取" : "FastMoss 已登录；窗口可以关闭，采集时会自动进入商品榜单";
|
|
123
154
|
}
|
|
124
155
|
return this.status();
|
|
125
156
|
}
|
|
126
157
|
async capture(input = {}) {
|
|
158
|
+
if (!this.context || !this.page || this.page.isClosed())
|
|
159
|
+
await this.start();
|
|
127
160
|
await this.inspect();
|
|
128
161
|
if (this.verificationRequired)
|
|
129
162
|
throw new Error("请先在 FastMoss 浏览器中手动完成验证");
|
|
@@ -135,40 +168,31 @@ export class FastMossIntegration {
|
|
|
135
168
|
throw new Error("FastMoss 会员已过期或当前账号没有榜单权限，请先更换账号");
|
|
136
169
|
}
|
|
137
170
|
if (!this.authenticated || !this.page || this.page.isClosed())
|
|
138
|
-
throw new Error("FastMoss
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
const hint = `${canvas.className} ${canvas.id} ${canvas.closest("td")?.textContent || ""}`;
|
|
162
|
-
return { src, href, content: canvas.dataset.content || canvas.dataset.url || "", isQr: /qr|qrcode|二维码/i.test(hint) };
|
|
163
|
-
}).filter((canvas) => canvas.src)),
|
|
164
|
-
})).filter((row) => row.cells.some(Boolean)),
|
|
165
|
-
};
|
|
166
|
-
}).sort((left, right) => right.rows.length - left.rows.length);
|
|
167
|
-
return { title: document.title, url: location.href, tables: tables.slice(0, 1) };
|
|
168
|
-
});
|
|
169
|
-
const records = extractRows(snapshot.tables, { ...input, date: localDate() }, snapshot.url);
|
|
171
|
+
throw new Error("FastMoss 尚未登录；请在恢复的窗口中完成登录后重试");
|
|
172
|
+
await this.openProductRanking(input);
|
|
173
|
+
const collected = new Map();
|
|
174
|
+
const pages = [];
|
|
175
|
+
let previousSignature = "";
|
|
176
|
+
for (let pageNumber = 1; pageNumber <= PRODUCT_RANK_MAX_PAGES; pageNumber += 1) {
|
|
177
|
+
const snapshot = await readVisibleTableSnapshot(this.page);
|
|
178
|
+
if (!snapshot.tables[0]?.rows.length || snapshot.signature === previousSignature)
|
|
179
|
+
break;
|
|
180
|
+
previousSignature = snapshot.signature;
|
|
181
|
+
pages.push(snapshot.url);
|
|
182
|
+
const pageRecords = extractRows(snapshot.tables, { ...input, date: localDate() }, snapshot.url);
|
|
183
|
+
pageRecords.forEach((row) => collected.set(String(row.product_id || row.title), row));
|
|
184
|
+
this.captured = collected.size;
|
|
185
|
+
this.message = `正在读取可见榜单第 ${pageNumber} 页，已收集 ${this.captured} 条商品数据`;
|
|
186
|
+
if (!await clickVisibleNextPage(this.page))
|
|
187
|
+
break;
|
|
188
|
+
await this.page.waitForTimeout(800);
|
|
189
|
+
await this.inspect();
|
|
190
|
+
if (this.verificationRequired || this.membershipExpired)
|
|
191
|
+
break;
|
|
192
|
+
}
|
|
193
|
+
const records = [...collected.values()];
|
|
170
194
|
if (!records.length)
|
|
171
|
-
throw new Error("
|
|
195
|
+
throw new Error("已自动进入 FastMoss 商品榜单，但没有读取到商品数据；请检查会员权限或页面是否仍在加载");
|
|
172
196
|
await mkdir(this.dataDir, { recursive: true });
|
|
173
197
|
const stored = await this.loadRows();
|
|
174
198
|
const indexed = new Map(stored.map((row) => [`${row.date}:${row.product_id || row.title}`, row]));
|
|
@@ -177,8 +201,8 @@ export class FastMossIntegration {
|
|
|
177
201
|
this.lastCapture = new Date().toISOString();
|
|
178
202
|
this.captured = records.length;
|
|
179
203
|
await writeFile(path.join(this.dataDir, "observations.json"), JSON.stringify(allRows, null, 2), "utf8");
|
|
180
|
-
await writeFile(path.join(this.dataDir, `capture-${localDate()}.json`), JSON.stringify({ capturedAt: this.lastCapture,
|
|
181
|
-
this.message =
|
|
204
|
+
await writeFile(path.join(this.dataDir, `capture-${localDate()}.json`), JSON.stringify({ capturedAt: this.lastCapture, pages, records }, null, 2), "utf8");
|
|
205
|
+
this.message = this.verificationRequired ? `已保存验证前的 ${records.length} 条数据；请手动完成滑块后再次抓取，程序会接着合并` : `已读取 ${pages.length} 个可见分页，共 ${records.length} 条商品数据`;
|
|
182
206
|
return { ...this.status(), records, allRows };
|
|
183
207
|
}
|
|
184
208
|
async close() {
|
|
@@ -187,12 +211,14 @@ export class FastMossIntegration {
|
|
|
187
211
|
this.page = null;
|
|
188
212
|
if (context)
|
|
189
213
|
await context.close();
|
|
190
|
-
this.phase = "idle";
|
|
191
|
-
this.authenticated =
|
|
214
|
+
this.phase = this.savedAuthenticated ? "saved" : "idle";
|
|
215
|
+
this.authenticated = this.savedAuthenticated;
|
|
192
216
|
this.verificationRequired = false;
|
|
193
217
|
this.membershipExpired = false;
|
|
194
218
|
this.lastMembershipCheckAt = 0;
|
|
195
|
-
this.message =
|
|
219
|
+
this.message = this.savedAuthenticated
|
|
220
|
+
? "FastMoss 窗口已关闭，登录仍保存在本机；抓取时会自动恢复"
|
|
221
|
+
: "FastMoss 窗口已关闭；请先登录一次";
|
|
196
222
|
return this.status();
|
|
197
223
|
}
|
|
198
224
|
async switchAccount() {
|
|
@@ -205,6 +231,8 @@ export class FastMossIntegration {
|
|
|
205
231
|
await this.page.bringToFront();
|
|
206
232
|
this.phase = "login_required";
|
|
207
233
|
this.authenticated = false;
|
|
234
|
+
this.savedAuthenticated = false;
|
|
235
|
+
await rm(this.authenticatedMarker, { force: true });
|
|
208
236
|
this.verificationRequired = false;
|
|
209
237
|
this.membershipExpired = false;
|
|
210
238
|
this.lastMembershipCheckAt = 0;
|
|
@@ -224,6 +252,103 @@ export class FastMossIntegration {
|
|
|
224
252
|
if (pages.length)
|
|
225
253
|
this.page = pages[pages.length - 1];
|
|
226
254
|
}
|
|
255
|
+
async openProductRanking(input) {
|
|
256
|
+
if (!this.page || this.page.isClosed())
|
|
257
|
+
throw new Error("FastMoss 专用浏览器未打开");
|
|
258
|
+
const market = normalizeMarket(input.market);
|
|
259
|
+
this.message = `正在自动进入 ${market} 商品榜单`;
|
|
260
|
+
await this.page.goto(fastMossSalesRankUrl(market), { waitUntil: "domcontentloaded", timeout: 60_000 });
|
|
261
|
+
const deadline = Date.now() + PRODUCT_TABLE_TIMEOUT_MS;
|
|
262
|
+
while (Date.now() < deadline) {
|
|
263
|
+
const snapshot = await readVisibleTableSnapshot(this.page);
|
|
264
|
+
if (hasReadableProductRows(snapshot)) {
|
|
265
|
+
this.phase = "ready";
|
|
266
|
+
this.message = `${market} 商品榜单已打开，正在开始采集`;
|
|
267
|
+
return;
|
|
268
|
+
}
|
|
269
|
+
await this.inspect();
|
|
270
|
+
if (this.verificationRequired)
|
|
271
|
+
throw new Error("检测到 FastMoss 滑块验证，请在专用窗口中手动完成后再次抓取");
|
|
272
|
+
if (this.membershipExpired)
|
|
273
|
+
throw new Error("FastMoss 会员已过期或当前账号没有商品榜单权限，请先更换账号");
|
|
274
|
+
if (!this.authenticated)
|
|
275
|
+
throw new Error("FastMoss 登录已失效，请在专用窗口中重新登录后再次抓取");
|
|
276
|
+
await this.page.waitForTimeout(750);
|
|
277
|
+
}
|
|
278
|
+
throw new Error("FastMoss 商品榜单自动加载超时；请检查网络或在窗口中完成人工验证后重试");
|
|
279
|
+
}
|
|
280
|
+
}
|
|
281
|
+
export function fastMossSalesRankUrl(market) {
|
|
282
|
+
const url = new URL(FASTMOSS_SALES_RANK);
|
|
283
|
+
url.searchParams.set("region", normalizeMarket(market));
|
|
284
|
+
url.searchParams.set("page", "1");
|
|
285
|
+
url.searchParams.set("pagesize", String(PRODUCT_RANK_PAGE_SIZE));
|
|
286
|
+
return url.toString();
|
|
287
|
+
}
|
|
288
|
+
function normalizeMarket(value) {
|
|
289
|
+
const market = String(value || "MX").trim().toUpperCase();
|
|
290
|
+
return /^[A-Z]{2}$/.test(market) ? market : "MX";
|
|
291
|
+
}
|
|
292
|
+
function hasReadableProductRows(snapshot) {
|
|
293
|
+
return snapshot.tables.some((table) => table.rows.some((row) => row.cells.filter(Boolean).length >= 3));
|
|
294
|
+
}
|
|
295
|
+
async function readVisibleTableSnapshot(page) {
|
|
296
|
+
return page.evaluate(() => {
|
|
297
|
+
const shown = (element) => Boolean(element.getClientRects().length);
|
|
298
|
+
const tables = [...document.querySelectorAll("table")].filter(shown).map((table) => {
|
|
299
|
+
const rows = [...table.querySelectorAll("tr")].filter(shown);
|
|
300
|
+
const headers = [...(rows[0]?.querySelectorAll("th,td") || [])].map((cell) => cell.innerText.replace(/\s+/g, " ").trim());
|
|
301
|
+
return {
|
|
302
|
+
headers,
|
|
303
|
+
rows: rows.slice(1).map((row) => ({
|
|
304
|
+
cells: [...row.querySelectorAll("td")].map((cell) => cell.innerText.replace(/\s+/g, " ").trim()),
|
|
305
|
+
links: [...row.querySelectorAll("a[href]")].map((link) => link.href),
|
|
306
|
+
images: [...row.querySelectorAll("img")].map((image) => {
|
|
307
|
+
const src = image.currentSrc || image.src || image.dataset.src || image.dataset.original || "";
|
|
308
|
+
const href = image.closest("a[href]")?.href || "";
|
|
309
|
+
const hint = `${image.alt} ${image.title} ${image.className} ${image.closest("td")?.textContent || ""} ${src}`;
|
|
310
|
+
return { src, href, content: image.dataset.content || image.dataset.url || "", isQr: /qr|qrcode|二维码/i.test(hint) };
|
|
311
|
+
}).filter((image) => image.src).concat([...row.querySelectorAll("canvas")].map((canvas) => {
|
|
312
|
+
let src = "";
|
|
313
|
+
try {
|
|
314
|
+
src = canvas.toDataURL("image/png");
|
|
315
|
+
}
|
|
316
|
+
catch { }
|
|
317
|
+
const href = canvas.closest("a[href]")?.href || "";
|
|
318
|
+
const hint = `${canvas.className} ${canvas.id} ${canvas.closest("td")?.textContent || ""}`;
|
|
319
|
+
return { src, href, content: canvas.dataset.content || canvas.dataset.url || "", isQr: /qr|qrcode|二维码/i.test(hint) };
|
|
320
|
+
}).filter((canvas) => canvas.src)),
|
|
321
|
+
})).filter((row) => {
|
|
322
|
+
const text = row.cells.join(" ").replace(/\s+/g, " ").trim();
|
|
323
|
+
return row.cells.filter(Boolean).length >= 3 && !/^(?:no data|loading\.?\.?.?|暂无数据|暂无商品|无数据)$/i.test(text);
|
|
324
|
+
}),
|
|
325
|
+
};
|
|
326
|
+
}).sort((left, right) => right.rows.length - left.rows.length).slice(0, 1);
|
|
327
|
+
const primary = tables[0];
|
|
328
|
+
const signature = JSON.stringify([location.href, primary?.headers, primary?.rows[0]?.cells, primary?.rows[primary.rows.length - 1]?.cells]);
|
|
329
|
+
return { title: document.title, url: location.href, tables, signature };
|
|
330
|
+
});
|
|
331
|
+
}
|
|
332
|
+
async function clickVisibleNextPage(page) {
|
|
333
|
+
return page.evaluate(() => {
|
|
334
|
+
const shown = (element) => Boolean(element.getClientRects().length);
|
|
335
|
+
const selectors = [
|
|
336
|
+
".ant-pagination-next button", ".ant-pagination-next", ".el-pagination .btn-next",
|
|
337
|
+
"button[aria-label*='next' i]", "[class*='pagination'] [class*='next']", "[class*='pagination'] [aria-label*='下一页']",
|
|
338
|
+
];
|
|
339
|
+
const candidates = [...new Set(selectors.flatMap((selector) => [...document.querySelectorAll(selector)]))];
|
|
340
|
+
const next = candidates.find((element) => {
|
|
341
|
+
if (!shown(element))
|
|
342
|
+
return false;
|
|
343
|
+
const parent = element.closest("li,button,[class*='next']");
|
|
344
|
+
const disabled = element.matches(":disabled") || element.getAttribute("aria-disabled") === "true" || parent?.getAttribute("aria-disabled") === "true" || /disabled/.test(`${element.className} ${parent?.className || ""}`);
|
|
345
|
+
return !disabled;
|
|
346
|
+
});
|
|
347
|
+
if (!next)
|
|
348
|
+
return false;
|
|
349
|
+
next.click();
|
|
350
|
+
return true;
|
|
351
|
+
});
|
|
227
352
|
}
|
|
228
353
|
function findChromiumExecutable() {
|
|
229
354
|
const home = os.homedir();
|
|
@@ -247,11 +372,14 @@ function extractRows(tables, context, pageUrl) {
|
|
|
247
372
|
for (const row of table.rows) {
|
|
248
373
|
const raw = Object.fromEntries(row.cells.map((value, index) => [table.headers[index] || `column_${index + 1}`, value]));
|
|
249
374
|
const pick = (...patterns) => Object.entries(raw).find(([key]) => patterns.some((pattern) => pattern.test(key)))?.[1] || "";
|
|
250
|
-
const
|
|
375
|
+
const productCell = String(pick(/^商品$|商品名|商品标题|product|title/i) || row.cells.find((cell) => cell.length > 5) || "");
|
|
376
|
+
const title = productCell.replace(/\s*(?:售价|价格|price)\s*[：:].*$/is, "").trim();
|
|
251
377
|
const productUrl = row.links.find((link) => /product|goods|item|detail/i.test(link)) || row.links[0] || pageUrl;
|
|
252
|
-
const productId = String(pick(/商品\s*id|product\s*id/i) || productUrl.match(/(?:product|goods|item)[/=_-]([\w-]+)/i)?.[1] || stableId(`${title}:${productUrl}`));
|
|
378
|
+
const productId = String(pick(/商品\s*id|product\s*id/i) || productUrl.match(/(?:detail|product|goods|item)[/=_-]([\w-]+)/i)?.[1] || stableId(`${title}:${productUrl}`));
|
|
253
379
|
const images = row.images.filter((image) => !image.isQr).map((image) => image.src);
|
|
254
380
|
const qr = row.images.find((image) => image.isQr);
|
|
381
|
+
const priceText = String(pick(/^价格$|^售价$|price/i) || productCell.match(/(?:售价|价格|price)\s*[：:]\s*([^\s]+)/i)?.[1] || "");
|
|
382
|
+
const storeName = String(pick(/所属店铺|店铺|shop|store/i)).replace(/\s*店铺销量\s*[：:].*$/is, "").trim();
|
|
255
383
|
output.push({
|
|
256
384
|
date: context.date,
|
|
257
385
|
source: "fastmoss-agent",
|
|
@@ -259,17 +387,17 @@ function extractRows(tables, context, pageUrl) {
|
|
|
259
387
|
platform_product_id: String(pick(/平台商品|tiktok.*id/i) || productId),
|
|
260
388
|
title,
|
|
261
389
|
product_url: productUrl,
|
|
262
|
-
store_name:
|
|
263
|
-
category: String(pick(
|
|
390
|
+
store_name: storeName,
|
|
391
|
+
category: String(pick(/类目|分类|category/i) || context.category || context.categories?.[0] || ""),
|
|
264
392
|
market: String(pick(/国家|市场|market|country/i) || context.market || ""),
|
|
265
393
|
shop_type: String(pick(/店铺类型|shop.*type|seller.*type/i) || context.shopType || ""),
|
|
266
394
|
period_days: context.periodDays || 7,
|
|
267
|
-
price: numberValue(
|
|
395
|
+
price: numberValue(priceText),
|
|
268
396
|
units_sold: numberValue(pick(/销量|sold|sales|orders/i)),
|
|
269
397
|
gmv: numberValue(pick(/gmv|销售额|成交额/i)),
|
|
270
398
|
commission: numberValue(pick(/佣金|commission/i)),
|
|
271
399
|
creators: numberValue(pick(/达人|creator/i)),
|
|
272
|
-
trend_score: numberValue(pick(
|
|
400
|
+
trend_score: numberValue(pick(/趋势|增长|环比|growth|trend/i)),
|
|
273
401
|
image_urls: images,
|
|
274
402
|
image: images[0] || "",
|
|
275
403
|
qr_image_url: qr?.src || "",
|