@xiaohhhh1/canvas-agent 0.4.16 → 0.4.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export type FastMossPhase = "idle" | "launching" | "login_required" | "ready" | "verification_required" | "membership_expired" | "error";
|
|
1
|
+
export type FastMossPhase = "idle" | "saved" | "launching" | "login_required" | "ready" | "verification_required" | "membership_expired" | "error";
|
|
2
2
|
export type FastMossStatus = {
|
|
3
3
|
phase: FastMossPhase;
|
|
4
4
|
browserOpen: boolean;
|
|
@@ -13,6 +13,7 @@ export type FastMossStatus = {
|
|
|
13
13
|
type CaptureContext = {
|
|
14
14
|
market?: string;
|
|
15
15
|
category?: string;
|
|
16
|
+
categories?: string[];
|
|
16
17
|
shopType?: string;
|
|
17
18
|
periodDays?: number;
|
|
18
19
|
};
|
|
@@ -28,7 +29,10 @@ export declare class FastMossIntegration {
|
|
|
28
29
|
private lastCapture?;
|
|
29
30
|
private lastMembershipCheckAt;
|
|
30
31
|
private readonly profileDir;
|
|
32
|
+
private readonly authenticatedMarker;
|
|
31
33
|
private readonly dataDir;
|
|
34
|
+
private savedAuthenticated;
|
|
35
|
+
constructor();
|
|
32
36
|
status(): FastMossStatus;
|
|
33
37
|
start(): Promise<FastMossStatus>;
|
|
34
38
|
inspect(): Promise<FastMossStatus>;
|
|
@@ -49,5 +53,7 @@ export declare class FastMossIntegration {
|
|
|
49
53
|
switchAccount(): Promise<FastMossStatus>;
|
|
50
54
|
private loadRows;
|
|
51
55
|
private useLatestPage;
|
|
56
|
+
private openProductRanking;
|
|
52
57
|
}
|
|
58
|
+
export declare function fastMossSalesRankUrl(market?: string): string;
|
|
53
59
|
export {};
|
|
@@ -1,11 +1,13 @@
|
|
|
1
1
|
import { existsSync } from "node:fs";
|
|
2
|
-
import { mkdir, readFile, writeFile } from "node:fs/promises";
|
|
2
|
+
import { mkdir, readFile, rm, writeFile } from "node:fs/promises";
|
|
3
3
|
import os from "node:os";
|
|
4
4
|
import path from "node:path";
|
|
5
5
|
import { chromium } from "playwright-core";
|
|
6
6
|
import { CONFIG_DIR } from "../config.js";
|
|
7
7
|
const FASTMOSS_HOME = "https://www.fastmoss.com/";
|
|
8
|
+
const FASTMOSS_SALES_RANK = "https://www.fastmoss.com/e-commerce/saleslist";
|
|
8
9
|
const MEMBERSHIP_RECHECK_MS = 5 * 60_000;
|
|
10
|
+
const PRODUCT_TABLE_TIMEOUT_MS = 45_000;
|
|
9
11
|
export class FastMossIntegration {
|
|
10
12
|
context = null;
|
|
11
13
|
page = null;
|
|
@@ -18,7 +20,16 @@ export class FastMossIntegration {
|
|
|
18
20
|
lastCapture;
|
|
19
21
|
lastMembershipCheckAt = 0;
|
|
20
22
|
profileDir = path.join(CONFIG_DIR, "fastmoss-session");
|
|
23
|
+
authenticatedMarker = path.join(this.profileDir, "authenticated.json");
|
|
21
24
|
dataDir = path.join(CONFIG_DIR, "fastmoss-selection");
|
|
25
|
+
savedAuthenticated = existsSync(this.authenticatedMarker);
|
|
26
|
+
constructor() {
|
|
27
|
+
if (!this.savedAuthenticated)
|
|
28
|
+
return;
|
|
29
|
+
this.phase = "saved";
|
|
30
|
+
this.authenticated = true;
|
|
31
|
+
this.message = "FastMoss 登录已保存在本机；窗口可以关闭，抓取时会自动恢复";
|
|
32
|
+
}
|
|
22
33
|
status() {
|
|
23
34
|
return {
|
|
24
35
|
phase: this.phase,
|
|
@@ -53,8 +64,10 @@ export class FastMossIntegration {
|
|
|
53
64
|
this.context = await chromium.launchPersistentContext(this.profileDir, {
|
|
54
65
|
executablePath,
|
|
55
66
|
headless: false,
|
|
56
|
-
viewport:
|
|
57
|
-
|
|
67
|
+
viewport: { width: 480, height: 900 },
|
|
68
|
+
screen: { width: 515, height: 995 },
|
|
69
|
+
deviceScaleFactor: 1,
|
|
70
|
+
args: ["--window-size=515,995", "--window-position=20,20"],
|
|
58
71
|
});
|
|
59
72
|
}
|
|
60
73
|
catch (error) {
|
|
@@ -69,12 +82,14 @@ export class FastMossIntegration {
|
|
|
69
82
|
this.context.on("close", () => {
|
|
70
83
|
this.context = null;
|
|
71
84
|
this.page = null;
|
|
72
|
-
this.phase = "idle";
|
|
73
|
-
this.authenticated =
|
|
85
|
+
this.phase = this.savedAuthenticated ? "saved" : "idle";
|
|
86
|
+
this.authenticated = this.savedAuthenticated;
|
|
74
87
|
this.verificationRequired = false;
|
|
75
88
|
this.membershipExpired = false;
|
|
76
89
|
this.lastMembershipCheckAt = 0;
|
|
77
|
-
this.message =
|
|
90
|
+
this.message = this.savedAuthenticated
|
|
91
|
+
? "FastMoss 窗口已关闭，登录仍保存在本机；抓取时会自动恢复"
|
|
92
|
+
: "FastMoss 窗口已关闭；请先登录一次";
|
|
78
93
|
});
|
|
79
94
|
if (!this.page.url() || this.page.url() === "about:blank")
|
|
80
95
|
await this.page.goto(FASTMOSS_HOME, { waitUntil: "domcontentloaded", timeout: 60_000 });
|
|
@@ -83,31 +98,45 @@ export class FastMossIntegration {
|
|
|
83
98
|
async inspect() {
|
|
84
99
|
this.useLatestPage();
|
|
85
100
|
if (!this.page || this.page.isClosed()) {
|
|
86
|
-
this.phase = "idle";
|
|
87
|
-
this.authenticated =
|
|
101
|
+
this.phase = this.savedAuthenticated ? "saved" : "idle";
|
|
102
|
+
this.authenticated = this.savedAuthenticated;
|
|
88
103
|
this.verificationRequired = false;
|
|
89
104
|
this.membershipExpired = false;
|
|
90
105
|
this.lastMembershipCheckAt = 0;
|
|
91
|
-
this.message =
|
|
106
|
+
this.message = this.savedAuthenticated
|
|
107
|
+
? "FastMoss 登录已保存在本机；窗口可以关闭，抓取时会自动恢复"
|
|
108
|
+
: "请先启动 FastMoss 专用浏览器并登录";
|
|
92
109
|
return this.status();
|
|
93
110
|
}
|
|
94
111
|
const snapshot = await this.page.evaluate(() => ({
|
|
95
112
|
title: document.title,
|
|
96
113
|
url: location.href,
|
|
97
114
|
text: (document.body?.innerText || "").slice(0, 12_000),
|
|
98
|
-
hasTable:
|
|
115
|
+
hasTable: [...document.querySelectorAll("table tbody tr")].some((row) => {
|
|
116
|
+
const text = (row.textContent || "").replace(/\s+/g, " ").trim();
|
|
117
|
+
return row.querySelectorAll("td").length >= 3 && !/^(?:no data|loading\.?\.?.?|暂无数据|暂无商品|无数据)$/i.test(text);
|
|
118
|
+
}),
|
|
99
119
|
})).catch(() => ({ title: "", url: this.page?.url() || "", text: "", hasTable: false }));
|
|
100
120
|
const haystack = `${snapshot.title}\n${snapshot.url}\n${snapshot.text}`.toLowerCase();
|
|
101
|
-
this.verificationRequired = /captcha|verify|verification|security check|人机验证|安全验证|滑块验证|请完成验证/.test(haystack);
|
|
102
|
-
const loginRequired = /\/login|\/signin|sign in|log in
|
|
121
|
+
this.verificationRequired = /captcha|verify|verification|security check|slide to complete the puzzle|verification failed|人机验证|安全验证|滑块验证|请完成验证/.test(haystack);
|
|
122
|
+
const loginRequired = /\/login|\/signin|sign in|log in|登录账号|密码登录|扫码登录|登录\s*\/\s*注册/.test(haystack) && !snapshot.hasTable;
|
|
103
123
|
const explicitExpiry = /(?:会员|套餐|订阅|专业版|高级版|账号).{0,16}(?:已过期|已到期|到期失效|无法使用)|(?:membership|subscription|plan|account).{0,28}(?:has\s+)?expired|(?:membership|subscription|plan).{0,20}(?:is no longer active|needs renewal)/i.test(haystack);
|
|
104
124
|
const entitlementUnavailable = /(?:当前账号|当前套餐).{0,20}(?:无权|没有权限|不支持|无法)(?:查看|访问|使用)|(?:开通|升级)(?:会员|专业版|套餐).{0,16}(?:后)?(?:才可|方可)(?:查看|访问|使用)|仅限(?:会员|专业版)(?:查看|使用)/i.test(haystack);
|
|
105
125
|
this.membershipExpired = explicitExpiry || (!snapshot.hasTable && entitlementUnavailable);
|
|
106
126
|
this.lastMembershipCheckAt = Date.now();
|
|
107
|
-
this.authenticated = !this.verificationRequired && !loginRequired && /fastmoss\.com/i.test(snapshot.url)
|
|
127
|
+
this.authenticated = !this.verificationRequired && !loginRequired && /fastmoss\.com/i.test(snapshot.url);
|
|
128
|
+
if (this.authenticated) {
|
|
129
|
+
this.savedAuthenticated = true;
|
|
130
|
+
await mkdir(this.profileDir, { recursive: true });
|
|
131
|
+
await writeFile(this.authenticatedMarker, JSON.stringify({ authenticated: true, updatedAt: new Date().toISOString() }), "utf8");
|
|
132
|
+
}
|
|
133
|
+
else if (loginRequired) {
|
|
134
|
+
this.savedAuthenticated = false;
|
|
135
|
+
await rm(this.authenticatedMarker, { force: true });
|
|
136
|
+
}
|
|
108
137
|
if (this.verificationRequired) {
|
|
109
138
|
this.phase = "verification_required";
|
|
110
|
-
this.message = "
|
|
139
|
+
this.message = "检测到 FastMoss 滑块验证，请在移动版登录窗口中手动完成；通过后刷新状态即可继续";
|
|
111
140
|
}
|
|
112
141
|
else if (this.membershipExpired) {
|
|
113
142
|
this.phase = "membership_expired";
|
|
@@ -115,15 +144,17 @@ export class FastMossIntegration {
|
|
|
115
144
|
}
|
|
116
145
|
else if (!this.authenticated) {
|
|
117
146
|
this.phase = "login_required";
|
|
118
|
-
this.message = "
|
|
147
|
+
this.message = "请在移动版窗口中登录 FastMoss；登录成功后可以关闭窗口，登录态会保存在本机";
|
|
119
148
|
}
|
|
120
149
|
else {
|
|
121
150
|
this.phase = "ready";
|
|
122
|
-
this.message = snapshot.hasTable ? "FastMoss
|
|
151
|
+
this.message = snapshot.hasTable ? "FastMoss 已连接，商品榜单可以抓取" : "FastMoss 已登录；窗口可以关闭，采集时会自动进入商品榜单";
|
|
123
152
|
}
|
|
124
153
|
return this.status();
|
|
125
154
|
}
|
|
126
155
|
async capture(input = {}) {
|
|
156
|
+
if (!this.context || !this.page || this.page.isClosed())
|
|
157
|
+
await this.start();
|
|
127
158
|
await this.inspect();
|
|
128
159
|
if (this.verificationRequired)
|
|
129
160
|
throw new Error("请先在 FastMoss 浏览器中手动完成验证");
|
|
@@ -135,40 +166,31 @@ export class FastMossIntegration {
|
|
|
135
166
|
throw new Error("FastMoss 会员已过期或当前账号没有榜单权限，请先更换账号");
|
|
136
167
|
}
|
|
137
168
|
if (!this.authenticated || !this.page || this.page.isClosed())
|
|
138
|
-
throw new Error("FastMoss
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
const hint = `${canvas.className} ${canvas.id} ${canvas.closest("td")?.textContent || ""}`;
|
|
162
|
-
return { src, href, content: canvas.dataset.content || canvas.dataset.url || "", isQr: /qr|qrcode|二维码/i.test(hint) };
|
|
163
|
-
}).filter((canvas) => canvas.src)),
|
|
164
|
-
})).filter((row) => row.cells.some(Boolean)),
|
|
165
|
-
};
|
|
166
|
-
}).sort((left, right) => right.rows.length - left.rows.length);
|
|
167
|
-
return { title: document.title, url: location.href, tables: tables.slice(0, 1) };
|
|
168
|
-
});
|
|
169
|
-
const records = extractRows(snapshot.tables, { ...input, date: localDate() }, snapshot.url);
|
|
169
|
+
throw new Error("FastMoss 尚未登录；请在恢复的窗口中完成登录后重试");
|
|
170
|
+
await this.openProductRanking(input);
|
|
171
|
+
const collected = new Map();
|
|
172
|
+
const pages = [];
|
|
173
|
+
let previousSignature = "";
|
|
174
|
+
for (let pageNumber = 1; pageNumber <= 30; pageNumber += 1) {
|
|
175
|
+
const snapshot = await readVisibleTableSnapshot(this.page);
|
|
176
|
+
if (!snapshot.tables[0]?.rows.length || snapshot.signature === previousSignature)
|
|
177
|
+
break;
|
|
178
|
+
previousSignature = snapshot.signature;
|
|
179
|
+
pages.push(snapshot.url);
|
|
180
|
+
const pageRecords = extractRows(snapshot.tables, { ...input, date: localDate() }, snapshot.url);
|
|
181
|
+
pageRecords.forEach((row) => collected.set(String(row.product_id || row.title), row));
|
|
182
|
+
this.captured = collected.size;
|
|
183
|
+
this.message = `正在读取可见榜单第 ${pageNumber} 页，已收集 ${this.captured} 条商品数据`;
|
|
184
|
+
if (!await clickVisibleNextPage(this.page))
|
|
185
|
+
break;
|
|
186
|
+
await this.page.waitForTimeout(800);
|
|
187
|
+
await this.inspect();
|
|
188
|
+
if (this.verificationRequired || this.membershipExpired)
|
|
189
|
+
break;
|
|
190
|
+
}
|
|
191
|
+
const records = [...collected.values()];
|
|
170
192
|
if (!records.length)
|
|
171
|
-
throw new Error("
|
|
193
|
+
throw new Error("已自动进入 FastMoss 商品榜单，但没有读取到商品数据；请检查会员权限或页面是否仍在加载");
|
|
172
194
|
await mkdir(this.dataDir, { recursive: true });
|
|
173
195
|
const stored = await this.loadRows();
|
|
174
196
|
const indexed = new Map(stored.map((row) => [`${row.date}:${row.product_id || row.title}`, row]));
|
|
@@ -177,8 +199,8 @@ export class FastMossIntegration {
|
|
|
177
199
|
this.lastCapture = new Date().toISOString();
|
|
178
200
|
this.captured = records.length;
|
|
179
201
|
await writeFile(path.join(this.dataDir, "observations.json"), JSON.stringify(allRows, null, 2), "utf8");
|
|
180
|
-
await writeFile(path.join(this.dataDir, `capture-${localDate()}.json`), JSON.stringify({ capturedAt: this.lastCapture,
|
|
181
|
-
this.message =
|
|
202
|
+
await writeFile(path.join(this.dataDir, `capture-${localDate()}.json`), JSON.stringify({ capturedAt: this.lastCapture, pages, records }, null, 2), "utf8");
|
|
203
|
+
this.message = this.verificationRequired ? `已保存验证前的 ${records.length} 条数据；请手动完成滑块后再次抓取，程序会接着合并` : `已读取 ${pages.length} 个可见分页，共 ${records.length} 条商品数据`;
|
|
182
204
|
return { ...this.status(), records, allRows };
|
|
183
205
|
}
|
|
184
206
|
async close() {
|
|
@@ -187,12 +209,14 @@ export class FastMossIntegration {
|
|
|
187
209
|
this.page = null;
|
|
188
210
|
if (context)
|
|
189
211
|
await context.close();
|
|
190
|
-
this.phase = "idle";
|
|
191
|
-
this.authenticated =
|
|
212
|
+
this.phase = this.savedAuthenticated ? "saved" : "idle";
|
|
213
|
+
this.authenticated = this.savedAuthenticated;
|
|
192
214
|
this.verificationRequired = false;
|
|
193
215
|
this.membershipExpired = false;
|
|
194
216
|
this.lastMembershipCheckAt = 0;
|
|
195
|
-
this.message =
|
|
217
|
+
this.message = this.savedAuthenticated
|
|
218
|
+
? "FastMoss 窗口已关闭，登录仍保存在本机；抓取时会自动恢复"
|
|
219
|
+
: "FastMoss 窗口已关闭；请先登录一次";
|
|
196
220
|
return this.status();
|
|
197
221
|
}
|
|
198
222
|
async switchAccount() {
|
|
@@ -205,6 +229,8 @@ export class FastMossIntegration {
|
|
|
205
229
|
await this.page.bringToFront();
|
|
206
230
|
this.phase = "login_required";
|
|
207
231
|
this.authenticated = false;
|
|
232
|
+
this.savedAuthenticated = false;
|
|
233
|
+
await rm(this.authenticatedMarker, { force: true });
|
|
208
234
|
this.verificationRequired = false;
|
|
209
235
|
this.membershipExpired = false;
|
|
210
236
|
this.lastMembershipCheckAt = 0;
|
|
@@ -224,6 +250,101 @@ export class FastMossIntegration {
|
|
|
224
250
|
if (pages.length)
|
|
225
251
|
this.page = pages[pages.length - 1];
|
|
226
252
|
}
|
|
253
|
+
async openProductRanking(input) {
|
|
254
|
+
if (!this.page || this.page.isClosed())
|
|
255
|
+
throw new Error("FastMoss 专用浏览器未打开");
|
|
256
|
+
const market = normalizeMarket(input.market);
|
|
257
|
+
this.message = `正在自动进入 ${market} 商品榜单`;
|
|
258
|
+
await this.page.goto(fastMossSalesRankUrl(market), { waitUntil: "domcontentloaded", timeout: 60_000 });
|
|
259
|
+
const deadline = Date.now() + PRODUCT_TABLE_TIMEOUT_MS;
|
|
260
|
+
while (Date.now() < deadline) {
|
|
261
|
+
const snapshot = await readVisibleTableSnapshot(this.page);
|
|
262
|
+
if (hasReadableProductRows(snapshot)) {
|
|
263
|
+
this.phase = "ready";
|
|
264
|
+
this.message = `${market} 商品榜单已打开，正在开始采集`;
|
|
265
|
+
return;
|
|
266
|
+
}
|
|
267
|
+
await this.inspect();
|
|
268
|
+
if (this.verificationRequired)
|
|
269
|
+
throw new Error("检测到 FastMoss 滑块验证，请在专用窗口中手动完成后再次抓取");
|
|
270
|
+
if (this.membershipExpired)
|
|
271
|
+
throw new Error("FastMoss 会员已过期或当前账号没有商品榜单权限，请先更换账号");
|
|
272
|
+
if (!this.authenticated)
|
|
273
|
+
throw new Error("FastMoss 登录已失效，请在专用窗口中重新登录后再次抓取");
|
|
274
|
+
await this.page.waitForTimeout(750);
|
|
275
|
+
}
|
|
276
|
+
throw new Error("FastMoss 商品榜单自动加载超时；请检查网络或在窗口中完成人工验证后重试");
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
export function fastMossSalesRankUrl(market) {
|
|
280
|
+
const url = new URL(FASTMOSS_SALES_RANK);
|
|
281
|
+
url.searchParams.set("region", normalizeMarket(market));
|
|
282
|
+
return url.toString();
|
|
283
|
+
}
|
|
284
|
+
function normalizeMarket(value) {
|
|
285
|
+
const market = String(value || "MX").trim().toUpperCase();
|
|
286
|
+
return /^[A-Z]{2}$/.test(market) ? market : "MX";
|
|
287
|
+
}
|
|
288
|
+
function hasReadableProductRows(snapshot) {
|
|
289
|
+
return snapshot.tables.some((table) => table.rows.some((row) => row.cells.filter(Boolean).length >= 3));
|
|
290
|
+
}
|
|
291
|
+
async function readVisibleTableSnapshot(page) {
|
|
292
|
+
return page.evaluate(() => {
|
|
293
|
+
const shown = (element) => Boolean(element.getClientRects().length);
|
|
294
|
+
const tables = [...document.querySelectorAll("table")].filter(shown).map((table) => {
|
|
295
|
+
const rows = [...table.querySelectorAll("tr")].filter(shown);
|
|
296
|
+
const headers = [...(rows[0]?.querySelectorAll("th,td") || [])].map((cell) => cell.innerText.replace(/\s+/g, " ").trim());
|
|
297
|
+
return {
|
|
298
|
+
headers,
|
|
299
|
+
rows: rows.slice(1).map((row) => ({
|
|
300
|
+
cells: [...row.querySelectorAll("td")].map((cell) => cell.innerText.replace(/\s+/g, " ").trim()),
|
|
301
|
+
links: [...row.querySelectorAll("a[href]")].map((link) => link.href),
|
|
302
|
+
images: [...row.querySelectorAll("img")].map((image) => {
|
|
303
|
+
const src = image.currentSrc || image.src || image.dataset.src || image.dataset.original || "";
|
|
304
|
+
const href = image.closest("a[href]")?.href || "";
|
|
305
|
+
const hint = `${image.alt} ${image.title} ${image.className} ${image.closest("td")?.textContent || ""} ${src}`;
|
|
306
|
+
return { src, href, content: image.dataset.content || image.dataset.url || "", isQr: /qr|qrcode|二维码/i.test(hint) };
|
|
307
|
+
}).filter((image) => image.src).concat([...row.querySelectorAll("canvas")].map((canvas) => {
|
|
308
|
+
let src = "";
|
|
309
|
+
try {
|
|
310
|
+
src = canvas.toDataURL("image/png");
|
|
311
|
+
}
|
|
312
|
+
catch { }
|
|
313
|
+
const href = canvas.closest("a[href]")?.href || "";
|
|
314
|
+
const hint = `${canvas.className} ${canvas.id} ${canvas.closest("td")?.textContent || ""}`;
|
|
315
|
+
return { src, href, content: canvas.dataset.content || canvas.dataset.url || "", isQr: /qr|qrcode|二维码/i.test(hint) };
|
|
316
|
+
}).filter((canvas) => canvas.src)),
|
|
317
|
+
})).filter((row) => {
|
|
318
|
+
const text = row.cells.join(" ").replace(/\s+/g, " ").trim();
|
|
319
|
+
return row.cells.filter(Boolean).length >= 3 && !/^(?:no data|loading\.?\.?.?|暂无数据|暂无商品|无数据)$/i.test(text);
|
|
320
|
+
}),
|
|
321
|
+
};
|
|
322
|
+
}).sort((left, right) => right.rows.length - left.rows.length).slice(0, 1);
|
|
323
|
+
const primary = tables[0];
|
|
324
|
+
const signature = JSON.stringify([location.href, primary?.headers, primary?.rows[0]?.cells, primary?.rows[primary.rows.length - 1]?.cells]);
|
|
325
|
+
return { title: document.title, url: location.href, tables, signature };
|
|
326
|
+
});
|
|
327
|
+
}
|
|
328
|
+
async function clickVisibleNextPage(page) {
|
|
329
|
+
return page.evaluate(() => {
|
|
330
|
+
const shown = (element) => Boolean(element.getClientRects().length);
|
|
331
|
+
const selectors = [
|
|
332
|
+
".ant-pagination-next button", ".ant-pagination-next", ".el-pagination .btn-next",
|
|
333
|
+
"button[aria-label*='next' i]", "[class*='pagination'] [class*='next']", "[class*='pagination'] [aria-label*='下一页']",
|
|
334
|
+
];
|
|
335
|
+
const candidates = [...new Set(selectors.flatMap((selector) => [...document.querySelectorAll(selector)]))];
|
|
336
|
+
const next = candidates.find((element) => {
|
|
337
|
+
if (!shown(element))
|
|
338
|
+
return false;
|
|
339
|
+
const parent = element.closest("li,button,[class*='next']");
|
|
340
|
+
const disabled = element.matches(":disabled") || element.getAttribute("aria-disabled") === "true" || parent?.getAttribute("aria-disabled") === "true" || /disabled/.test(`${element.className} ${parent?.className || ""}`);
|
|
341
|
+
return !disabled;
|
|
342
|
+
});
|
|
343
|
+
if (!next)
|
|
344
|
+
return false;
|
|
345
|
+
next.click();
|
|
346
|
+
return true;
|
|
347
|
+
});
|
|
227
348
|
}
|
|
228
349
|
function findChromiumExecutable() {
|
|
229
350
|
const home = os.homedir();
|
|
@@ -247,11 +368,14 @@ function extractRows(tables, context, pageUrl) {
|
|
|
247
368
|
for (const row of table.rows) {
|
|
248
369
|
const raw = Object.fromEntries(row.cells.map((value, index) => [table.headers[index] || `column_${index + 1}`, value]));
|
|
249
370
|
const pick = (...patterns) => Object.entries(raw).find(([key]) => patterns.some((pattern) => pattern.test(key)))?.[1] || "";
|
|
250
|
-
const
|
|
371
|
+
const productCell = String(pick(/^商品$|商品名|商品标题|product|title/i) || row.cells.find((cell) => cell.length > 5) || "");
|
|
372
|
+
const title = productCell.replace(/\s*(?:售价|价格|price)\s*[：:].*$/is, "").trim();
|
|
251
373
|
const productUrl = row.links.find((link) => /product|goods|item|detail/i.test(link)) || row.links[0] || pageUrl;
|
|
252
|
-
const productId = String(pick(/商品\s*id|product\s*id/i) || productUrl.match(/(?:product|goods|item)[/=_-]([\w-]+)/i)?.[1] || stableId(`${title}:${productUrl}`));
|
|
374
|
+
const productId = String(pick(/商品\s*id|product\s*id/i) || productUrl.match(/(?:detail|product|goods|item)[/=_-]([\w-]+)/i)?.[1] || stableId(`${title}:${productUrl}`));
|
|
253
375
|
const images = row.images.filter((image) => !image.isQr).map((image) => image.src);
|
|
254
376
|
const qr = row.images.find((image) => image.isQr);
|
|
377
|
+
const priceText = String(pick(/^价格$|^售价$|price/i) || productCell.match(/(?:售价|价格|price)\s*[：:]\s*([^\s]+)/i)?.[1] || "");
|
|
378
|
+
const storeName = String(pick(/所属店铺|店铺|shop|store/i)).replace(/\s*店铺销量\s*[：:].*$/is, "").trim();
|
|
255
379
|
output.push({
|
|
256
380
|
date: context.date,
|
|
257
381
|
source: "fastmoss-agent",
|
|
@@ -259,17 +383,17 @@ function extractRows(tables, context, pageUrl) {
|
|
|
259
383
|
platform_product_id: String(pick(/平台商品|tiktok.*id/i) || productId),
|
|
260
384
|
title,
|
|
261
385
|
product_url: productUrl,
|
|
262
|
-
store_name:
|
|
263
|
-
category: String(pick(
|
|
386
|
+
store_name: storeName,
|
|
387
|
+
category: String(pick(/类目|分类|category/i) || context.category || context.categories?.[0] || ""),
|
|
264
388
|
market: String(pick(/国家|市场|market|country/i) || context.market || ""),
|
|
265
389
|
shop_type: String(pick(/店铺类型|shop.*type|seller.*type/i) || context.shopType || ""),
|
|
266
390
|
period_days: context.periodDays || 7,
|
|
267
|
-
price: numberValue(
|
|
391
|
+
price: numberValue(priceText),
|
|
268
392
|
units_sold: numberValue(pick(/销量|sold|sales|orders/i)),
|
|
269
393
|
gmv: numberValue(pick(/gmv|销售额|成交额/i)),
|
|
270
394
|
commission: numberValue(pick(/佣金|commission/i)),
|
|
271
395
|
creators: numberValue(pick(/达人|creator/i)),
|
|
272
|
-
trend_score: numberValue(pick(
|
|
396
|
+
trend_score: numberValue(pick(/趋势|增长|环比|growth|trend/i)),
|
|
273
397
|
image_urls: images,
|
|
274
398
|
image: images[0] || "",
|
|
275
399
|
qr_image_url: qr?.src || "",
|