@xiaohhhh1/canvas-agent 0.4.79 → 0.4.81
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent/codex.d.ts +2 -0
- package/dist/agent/codex.js +10 -4
- package/dist/integrations/fastmoss-opportunity.d.ts +160 -0
- package/dist/integrations/fastmoss-opportunity.js +183 -0
- package/dist/integrations/fastmoss-selection-session.d.ts +112 -0
- package/dist/integrations/fastmoss-selection-session.js +174 -0
- package/dist/integrations/fastmoss-visible-evidence.d.ts +52 -0
- package/dist/integrations/fastmoss-visible-evidence.js +303 -0
- package/dist/integrations/fastmoss.d.ts +45 -0
- package/dist/integrations/fastmoss.js +49 -0
- package/dist/server/http.js +4 -1
- package/dist/video-intelligence/local-analysis.d.ts +5 -0
- package/dist/video-intelligence/local-analysis.js +30 -2
- package/dist/workflow/content-method.d.ts +58 -0
- package/dist/workflow/content-method.js +100 -14
- package/dist/workflow/manager.d.ts +31 -2
- package/dist/workflow/manager.js +317 -13
- package/package.json +2 -2
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
import { mkdir, readFile, rename, writeFile } from "node:fs/promises";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { setTimeout as delay } from "node:timers/promises";
|
|
4
|
+
import { matchesCategory, opportunityCandidate, selectionConfig } from "./fastmoss-opportunity.js";
|
|
5
|
+
const rankings = ["sales", "new", "hot", "video"];
|
|
6
|
+
const safeId = (id) => { if (!/^[a-zA-Z0-9-]{8,80}$/.test(id))
|
|
7
|
+
throw new Error("选品会话ID无效"); return id; };
|
|
8
|
+
const errorText = (error) => error instanceof Error ? error.message : String(error);
|
|
9
|
+
export class FastMossSelectionSessions {
|
|
10
|
+
directory;
|
|
11
|
+
adapter;
|
|
12
|
+
activeId = null;
|
|
13
|
+
task = null;
|
|
14
|
+
serial = Promise.resolve();
|
|
15
|
+
constructor(directory, adapter) {
|
|
16
|
+
this.directory = directory;
|
|
17
|
+
this.adapter = adapter;
|
|
18
|
+
}
|
|
19
|
+
isRunning() { return Boolean(this.activeId); }
|
|
20
|
+
async atomic(file, value) {
|
|
21
|
+
await mkdir(this.directory, { recursive: true });
|
|
22
|
+
const destination = path.join(this.directory, file);
|
|
23
|
+
await writeFile(`${destination}.tmp`, JSON.stringify(value, null, 2), "utf8");
|
|
24
|
+
// Windows may briefly deny replacement while a progress reader holds the
|
|
25
|
+
// old file. Keep the durable old snapshot until replacement succeeds.
|
|
26
|
+
for (let attempt = 0;; attempt++) {
|
|
27
|
+
try {
|
|
28
|
+
await rename(`${destination}.tmp`, destination);
|
|
29
|
+
break;
|
|
30
|
+
}
|
|
31
|
+
catch (error) {
|
|
32
|
+
if (attempt >= 5 || !["EPERM", "EBUSY", "EACCES"].includes(error.code || ""))
|
|
33
|
+
throw error;
|
|
34
|
+
await delay(30 * (attempt + 1));
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
async persist(session) {
|
|
39
|
+
session.updatedAt = new Date().toISOString();
|
|
40
|
+
await this.atomic(`${safeId(session.id)}.json`, session);
|
|
41
|
+
}
|
|
42
|
+
async load(id) {
|
|
43
|
+
try {
|
|
44
|
+
const session = JSON.parse(await readFile(path.join(this.directory, `${safeId(id)}.json`), "utf8"));
|
|
45
|
+
session.config = selectionConfig(session.config);
|
|
46
|
+
return session;
|
|
47
|
+
}
|
|
48
|
+
catch (error) {
|
|
49
|
+
if (error.code === "ENOENT")
|
|
50
|
+
return null;
|
|
51
|
+
throw error;
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
async get(id) {
|
|
55
|
+
if (!id) {
|
|
56
|
+
try {
|
|
57
|
+
id = JSON.parse(await readFile(path.join(this.directory, "latest.json"), "utf8")).id;
|
|
58
|
+
}
|
|
59
|
+
catch (error) {
|
|
60
|
+
if (error.code === "ENOENT")
|
|
61
|
+
return null;
|
|
62
|
+
throw error;
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
const session = id ? await this.load(id) : null;
|
|
66
|
+
if (!session)
|
|
67
|
+
return null;
|
|
68
|
+
session.active = this.activeId === session.id;
|
|
69
|
+
if (!session.active && ["collecting", "reviewing"].includes(session.phase))
|
|
70
|
+
session.phase = "interrupted";
|
|
71
|
+
return { ...session, candidates: session.products.map(p => opportunityCandidate(p, session.config, session.id)) };
|
|
72
|
+
}
|
|
73
|
+
/** Serialize ACK creation; repeat POST with the same id never starts duplicate work. */
|
|
74
|
+
start(id, input, resume = false) {
|
|
75
|
+
const operation = this.serial.then(async () => {
|
|
76
|
+
safeId(id);
|
|
77
|
+
const config = selectionConfig(input);
|
|
78
|
+
let session = await this.load(id);
|
|
79
|
+
if (session && JSON.stringify(session.config) !== JSON.stringify(config))
|
|
80
|
+
throw new Error("该会话的国家或规则不同,请新建一轮选品");
|
|
81
|
+
if (this.activeId) {
|
|
82
|
+
if (this.activeId !== id)
|
|
83
|
+
throw new Error("已有选品正在运行,请先等待完成");
|
|
84
|
+
return this.get(id);
|
|
85
|
+
}
|
|
86
|
+
if (session && !resume)
|
|
87
|
+
return this.get(id);
|
|
88
|
+
if (!session) {
|
|
89
|
+
if (resume)
|
|
90
|
+
throw new Error("没有找到要继续的选品会话");
|
|
91
|
+
session = { id, config, createdAt: new Date().toISOString(), updatedAt: new Date().toISOString(), phase: "collecting", active: true, completedRankings: [], rankingErrors: {}, products: [] };
|
|
92
|
+
}
|
|
93
|
+
session.active = true;
|
|
94
|
+
session.error = undefined;
|
|
95
|
+
session.phase = "collecting";
|
|
96
|
+
await this.persist(session);
|
|
97
|
+
await this.atomic("latest.json", { id });
|
|
98
|
+
this.activeId = id;
|
|
99
|
+
this.task = this.run(session).finally(() => { this.activeId = null; this.task = null; });
|
|
100
|
+
// ACK is returned after durable creation, not after the whole capture.
|
|
101
|
+
return this.get(id);
|
|
102
|
+
});
|
|
103
|
+
this.serial = operation.catch(() => undefined);
|
|
104
|
+
return operation;
|
|
105
|
+
}
|
|
106
|
+
async waitUntilIdle() { await this.serial; await this.task; }
|
|
107
|
+
async run(session) {
|
|
108
|
+
try {
|
|
109
|
+
for (const ranking of rankings) {
|
|
110
|
+
if (session.completedRankings.includes(ranking))
|
|
111
|
+
continue;
|
|
112
|
+
try {
|
|
113
|
+
await this.adapter.discover(ranking, session.config, async (products) => {
|
|
114
|
+
for (const product of products) {
|
|
115
|
+
if (product.market !== session.config.market || !matchesCategory(product.category, session.config))
|
|
116
|
+
continue;
|
|
117
|
+
const existing = session.products.find(p => p.productId === product.productId);
|
|
118
|
+
if (existing)
|
|
119
|
+
existing.rankings = [...new Map([...existing.rankings, ...product.rankings].map(r => [r.source, r])).values()];
|
|
120
|
+
else
|
|
121
|
+
session.products.push(product);
|
|
122
|
+
}
|
|
123
|
+
await this.persist(session);
|
|
124
|
+
});
|
|
125
|
+
session.completedRankings.push(ranking);
|
|
126
|
+
delete session.rankingErrors[ranking];
|
|
127
|
+
}
|
|
128
|
+
catch (error) {
|
|
129
|
+
session.rankingErrors[ranking] = errorText(error);
|
|
130
|
+
}
|
|
131
|
+
await this.persist(session);
|
|
132
|
+
}
|
|
133
|
+
// Round-robin discovery sources: repeated ranking appearances add no score.
|
|
134
|
+
const selected = [];
|
|
135
|
+
const queues = rankings.map(source => session.products.filter(p => p.rankings.some(r => r.source === source) && matchesCategory(p.category, session.config)).sort((a, b) => a.rankings.find(r => r.source === source).position - b.rankings.find(r => r.source === source).position));
|
|
136
|
+
while (queues.some(q => q.length) && selected.length < session.config.candidateLimit)
|
|
137
|
+
for (const queue of queues) {
|
|
138
|
+
const next = queue.shift();
|
|
139
|
+
if (next && !selected.includes(next) && selected.length < session.config.candidateLimit)
|
|
140
|
+
selected.push(next);
|
|
141
|
+
}
|
|
142
|
+
session.phase = "reviewing";
|
|
143
|
+
await this.persist(session);
|
|
144
|
+
for (const product of selected.slice(0, session.config.detailLimit)) {
|
|
145
|
+
if (product.detailState === "complete")
|
|
146
|
+
continue;
|
|
147
|
+
const save = async (next) => { Object.assign(product, next); await this.persist(session); };
|
|
148
|
+
product.detailState = "running";
|
|
149
|
+
delete product.error;
|
|
150
|
+
await save(product);
|
|
151
|
+
try {
|
|
152
|
+
await save({ ...await this.adapter.detail(structuredClone(product), session.config, save), detailState: "complete" });
|
|
153
|
+
}
|
|
154
|
+
catch (error) {
|
|
155
|
+
await save({ ...product, detailState: "failed", error: errorText(error) });
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
session.phase = Object.keys(session.rankingErrors).length || selected.slice(0, session.config.detailLimit).some(p => p.detailState === "failed") ? "interrupted" : "complete";
|
|
159
|
+
}
|
|
160
|
+
catch (error) {
|
|
161
|
+
session.phase = "interrupted";
|
|
162
|
+
session.error = errorText(error);
|
|
163
|
+
}
|
|
164
|
+
finally {
|
|
165
|
+
session.active = false;
|
|
166
|
+
try {
|
|
167
|
+
await this.persist(session);
|
|
168
|
+
}
|
|
169
|
+
catch (error) {
|
|
170
|
+
console.error("FastMoss selection persistence failed", errorText(error));
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
}
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import { type DailySale, type Period, type ProductEvidence, type Ranking, type SelectionConfig, type VideoSale } from "./fastmoss-opportunity.js";
|
|
2
|
+
export type VisibleSection = {
|
|
3
|
+
url: string;
|
|
4
|
+
selector: string;
|
|
5
|
+
observedAt: string;
|
|
6
|
+
text: string;
|
|
7
|
+
blocked: string | null;
|
|
8
|
+
loading: boolean;
|
|
9
|
+
inputs: Array<{
|
|
10
|
+
value: string;
|
|
11
|
+
placeholder: string;
|
|
12
|
+
checked: boolean;
|
|
13
|
+
label: string;
|
|
14
|
+
}>;
|
|
15
|
+
links: Array<{
|
|
16
|
+
href: string;
|
|
17
|
+
text: string;
|
|
18
|
+
}>;
|
|
19
|
+
tables: Array<{
|
|
20
|
+
headers: string[];
|
|
21
|
+
rows: Array<{
|
|
22
|
+
cells: string[];
|
|
23
|
+
links: Array<{
|
|
24
|
+
href: string;
|
|
25
|
+
text: string;
|
|
26
|
+
}>;
|
|
27
|
+
images: string[];
|
|
28
|
+
}>;
|
|
29
|
+
}>;
|
|
30
|
+
};
|
|
31
|
+
/** Serializable DOM-only reader. Never reads app state, network responses or concealed text. */
|
|
32
|
+
export declare function readVisibleSection(selector: string): VisibleSection;
|
|
33
|
+
export declare function sectionPeriod(snapshot: VisibleSection, days?: number): Period | null;
|
|
34
|
+
export declare function rankingProducts(snapshot: VisibleSection, ranking: Ranking, config: SelectionConfig, offset?: number): ProductEvidence[];
|
|
35
|
+
export declare function dailyFromSection(snapshot: VisibleSection, days: number): DailySale[];
|
|
36
|
+
export declare function overviewFromSection(snapshot: VisibleSection): ProductEvidence["overview"];
|
|
37
|
+
export declare function videosFromSection(snapshot: VisibleSection, expected: Period | null): VideoSale[];
|
|
38
|
+
export interface EvidenceBrowser {
|
|
39
|
+
goto(url: string): Promise<void>;
|
|
40
|
+
snapshot(selector: string): Promise<VisibleSection>;
|
|
41
|
+
clickText(selector: string, text: string, occurrence?: number): Promise<void>;
|
|
42
|
+
next(selector: string): Promise<boolean>;
|
|
43
|
+
pause(): Promise<void>;
|
|
44
|
+
}
|
|
45
|
+
export declare class VisibleFastMossCollector {
|
|
46
|
+
private browser;
|
|
47
|
+
constructor(browser: EvidenceBrowser);
|
|
48
|
+
private stable;
|
|
49
|
+
verifyOverview(product: ProductEvidence): Promise<ProductEvidence>;
|
|
50
|
+
discover(ranking: Ranking, config: SelectionConfig, save: (products: ProductEvidence[]) => Promise<void>): Promise<void>;
|
|
51
|
+
detail(product: ProductEvidence, config: SelectionConfig, save: (product: ProductEvidence) => Promise<void>): Promise<ProductEvidence>;
|
|
52
|
+
}
|
|
@@ -0,0 +1,303 @@
|
|
|
1
|
+
import { MARKETS, OPPORTUNITY_VERSION, RANKINGS, isoDate, matchesCategory, visibleNumber } from "./fastmoss-opportunity.js";
|
|
2
|
+
/** Serializable DOM-only reader. Never reads app state, network responses or concealed text. */
|
|
3
|
+
export function readVisibleSection(selector) {
|
|
4
|
+
const root = document.querySelector(selector);
|
|
5
|
+
const readability = [];
|
|
6
|
+
const readable = (element) => {
|
|
7
|
+
const cached = readability.find(([node]) => node === element);
|
|
8
|
+
if (cached)
|
|
9
|
+
return cached[1];
|
|
10
|
+
for (let node = element; node; node = node.parentElement) {
|
|
11
|
+
const style = getComputedStyle(node);
|
|
12
|
+
if (style.display === "none" || style.visibility === "hidden" || Number(style.opacity) === 0 || /blur\((?!0px)/.test(style.filter) || node.getAttribute("aria-hidden") === "true") {
|
|
13
|
+
if (element)
|
|
14
|
+
readability.push([element, false]);
|
|
15
|
+
return false;
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
const value = Boolean(element?.getClientRects().length);
|
|
19
|
+
if (element)
|
|
20
|
+
readability.push([element, value]);
|
|
21
|
+
return value;
|
|
22
|
+
};
|
|
23
|
+
const gateCache = [];
|
|
24
|
+
const gated = (element) => {
|
|
25
|
+
const cached = gateCache.find(([node]) => node === element);
|
|
26
|
+
if (cached)
|
|
27
|
+
return cached[1];
|
|
28
|
+
// Upgrade masks belong to their own section; the global upgrade button is not a gate.
|
|
29
|
+
const value = [...element.querySelectorAll("div,span,p")].some(n => readable(n) && /升级会员解锁|升级.*查看.*数据|仅.*会员.*查看|unlock.*(?:data|access)|sample data|演示数据/i.test([...n.childNodes].filter(c => c.nodeType === 3).map(c => c.textContent || "").join("")));
|
|
30
|
+
gateCache.push([element, value]);
|
|
31
|
+
return value;
|
|
32
|
+
};
|
|
33
|
+
const blocked = !root ? "页面区域尚未出现" : !readable(root) ? "页面区域不可见" : gated(root) && !selector.includes("dashboard-container") ? "该区域需要升级或仅展示样例,未采集" : null;
|
|
34
|
+
const empty = { url: location.href, selector, observedAt: new Date().toISOString(), text: "", blocked, loading: false, inputs: [], links: [], tables: [] };
|
|
35
|
+
if (!root || blocked)
|
|
36
|
+
return empty;
|
|
37
|
+
const inGate = (element) => {
|
|
38
|
+
for (let node = element; node && node !== root; node = node.parentElement)
|
|
39
|
+
if (node.id && gated(node))
|
|
40
|
+
return true;
|
|
41
|
+
return false;
|
|
42
|
+
};
|
|
43
|
+
const allowed = (element) => readable(element) && !inGate(element);
|
|
44
|
+
const text = (element) => {
|
|
45
|
+
if (!allowed(element))
|
|
46
|
+
return "";
|
|
47
|
+
if ([...element.querySelectorAll("*")].some(n => !allowed(n)))
|
|
48
|
+
return [...element.childNodes].map(n => n.nodeType === 3 ? n.textContent || "" : n.nodeType === 1 && "innerText" in n ? text(n) : "").join("\n");
|
|
49
|
+
return element.innerText;
|
|
50
|
+
};
|
|
51
|
+
const links = (element) => [...element.querySelectorAll("a[href]")].filter(allowed).map(a => ({ href: a.href, text: text(a) }));
|
|
52
|
+
return { ...empty, text: text(root).slice(0, 45000), loading: [...root.querySelectorAll(".ant-spin-spinning,[aria-busy=true]")].some(readable),
|
|
53
|
+
inputs: [...root.querySelectorAll("input")].filter(i => readable(i) || (i.type === "radio" && i.closest("label") && allowed(i.closest("label")))).filter(i => !inGate(i)).map(i => ({ value: i.value, placeholder: i.placeholder, checked: i.checked, label: i.closest("label") ? text(i.closest("label")) : "" })),
|
|
54
|
+
links: links(root),
|
|
55
|
+
tables: [...root.querySelectorAll("table")].filter(allowed).map(table => ({ headers: [...table.querySelectorAll("thead th")].map(h => allowed(h) ? text(h).trim() : ""), rows: [...table.querySelectorAll("tbody tr")].filter(row => allowed(row) && !row.className.includes("measure-row")).map(row => {
|
|
56
|
+
const productLink = [...row.querySelectorAll("a[href]")].find(a => /\/e-commerce\/detail\/\d+/.test(a.href));
|
|
57
|
+
return { cells: [...row.querySelectorAll("td")].map(cell => text(cell).trim()), links: links(row), images: [...(productLink?.closest("td") || row).querySelectorAll("img")].filter(allowed).map(i => i.currentSrc || i.src).filter(u => /^https:\/\//.test(u)).slice(0, 5) };
|
|
58
|
+
}) })) };
|
|
59
|
+
}
|
|
60
|
+
export function sectionPeriod(snapshot, days) {
|
|
61
|
+
const start = snapshot.inputs.find(i => i.placeholder === "开始日期")?.value || "";
|
|
62
|
+
const end = snapshot.inputs.find(i => i.placeholder === "结束日期")?.value || "";
|
|
63
|
+
if (!isoDate(start) || !isoDate(end) || start > end || (days && (Date.parse(end) - Date.parse(start)) / 86400000 + 1 !== days))
|
|
64
|
+
return null;
|
|
65
|
+
return { start, end, timeZone: "UTC+08:00" };
|
|
66
|
+
}
|
|
67
|
+
const ref = (snapshot, period) => ({ url: snapshot.url, section: snapshot.selector, observedAt: snapshot.observedAt, period });
|
|
68
|
+
const cell = (headers, cells, pattern) => cells[headers.findIndex(h => pattern.test(h.replace(/\s+/g, "")))] || "";
|
|
69
|
+
export function rankingProducts(snapshot, ranking, config, offset = 0) {
|
|
70
|
+
if (snapshot.blocked || snapshot.loading || new URL(snapshot.url).searchParams.get("region") !== config.market)
|
|
71
|
+
return [];
|
|
72
|
+
const result = [];
|
|
73
|
+
for (const table of snapshot.tables)
|
|
74
|
+
for (const row of table.rows) {
|
|
75
|
+
const link = row.links.find(a => /\/e-commerce\/detail\/\d+/.test(a.href));
|
|
76
|
+
const productId = link?.href.match(/\/e-commerce\/detail\/(\d+)/)?.[1];
|
|
77
|
+
if (!link || !productId)
|
|
78
|
+
continue;
|
|
79
|
+
const country = cell(table.headers, row.cells, /国家|地区|country|market/i);
|
|
80
|
+
if (country && !country.includes(config.market) && !country.includes(MARKETS[config.market]))
|
|
81
|
+
continue;
|
|
82
|
+
const title = (link.text.trim() || cell(table.headers, row.cells, /商品|product/i)).split(/\n\s*(?:售价|价格|佣金)/)[0].trim();
|
|
83
|
+
const category = cell(table.headers, row.cells, /类目|分类|category/i);
|
|
84
|
+
result.push({ version: OPPORTUNITY_VERSION, productId, market: config.market, title, url: link.href, category, storeName: cell(table.headers, row.cells, /所属店铺|店铺名称|shop/i), imageUrls: row.images,
|
|
85
|
+
discoveredAt: snapshot.observedAt, rankings: [{ source: ranking, position: visibleNumber(row.cells[0]) || offset + result.length + 1, ref: ref(snapshot, null) }], identityVerified: false, currency: null, listingDate: null, price: null, stock: null, rating: null, daily: [], videos: [], overview: null, missing: [], detailState: "pending" });
|
|
86
|
+
}
|
|
87
|
+
return [...new Map(result.map(p => [p.productId, p])).values()];
|
|
88
|
+
}
|
|
89
|
+
export function dailyFromSection(snapshot, days) {
|
|
90
|
+
const period = sectionPeriod(snapshot, days);
|
|
91
|
+
if (snapshot.blocked || snapshot.loading || !period)
|
|
92
|
+
return [];
|
|
93
|
+
const table = snapshot.tables.find(t => t.headers.some(h => /^(时间|日期)$/.test(h.trim())) && t.headers.some(h => h.trim() === "销量"));
|
|
94
|
+
if (!table)
|
|
95
|
+
return [];
|
|
96
|
+
return table.rows.flatMap(row => {
|
|
97
|
+
const date = cell(table.headers, row.cells, /^(时间|日期)$/).slice(0, 10);
|
|
98
|
+
if (!isoDate(date) || date < period.start || date > period.end)
|
|
99
|
+
return [];
|
|
100
|
+
return [{ date, units: visibleNumber(cell(table.headers, row.cells, /^销量$/)), ref: ref(snapshot, period) }];
|
|
101
|
+
});
|
|
102
|
+
}
|
|
103
|
+
export function overviewFromSection(snapshot) {
|
|
104
|
+
const period = sectionPeriod(snapshot, 7);
|
|
105
|
+
if (snapshot.blocked || snapshot.loading || !period)
|
|
106
|
+
return null;
|
|
107
|
+
const share = (pattern) => { const value = visibleNumber(snapshot.text.match(pattern)?.[1]); return value !== null && value <= 100 ? value : null; };
|
|
108
|
+
const unitsText = snapshot.inputs.find(i => i.value === "sold_count_show")?.label.trim().split("\n").filter(Boolean)[0] || "";
|
|
109
|
+
const compact = unitsText.match(/\d+(?:\.(\d+))?(万|亿|[kKmMbB])$/);
|
|
110
|
+
const factors = { 万: 1e4, 亿: 1e8, k: 1e3, m: 1e6, b: 1e9 };
|
|
111
|
+
// FastMoss abbreviates by truncation too (26491 appears as 2.64万).
|
|
112
|
+
// Allow one displayed unit of precision, never a blanket percentage.
|
|
113
|
+
const unitsTolerance = compact ? factors[compact[2].toLowerCase()] / 10 ** (compact[1]?.length || 0) : 0;
|
|
114
|
+
return { ref: ref(snapshot, period), units: visibleNumber(unitsText), unitsTolerance, videoShare: share(/(?:^|\n)视频\s*\n\s*([\d.]+%)/), liveShare: share(/(?:^|\n)直播\s*\n\s*([\d.]+%)/), adShare: share(/广告投放[((]视频[))]\s*\n\s*([\d.]+%)/), otherShare: share(/其他流量\s*\n\s*([\d.]+%)/) };
|
|
115
|
+
}
|
|
116
|
+
export function videosFromSection(snapshot, expected) {
|
|
117
|
+
if (snapshot.blocked || snapshot.loading || !expected)
|
|
118
|
+
return [];
|
|
119
|
+
const publishedPeriod = sectionPeriod(snapshot, 7);
|
|
120
|
+
if (!publishedPeriod || publishedPeriod.start !== expected.start || publishedPeriod.end !== expected.end)
|
|
121
|
+
return [];
|
|
122
|
+
// Dates in this section belong to publication filtering. Transaction dates are
|
|
123
|
+
// aligned independently through the selected first period group + column label.
|
|
124
|
+
const firstSelected = snapshot.inputs.find(i => i.checked && /^近\d+天$/.test(i.label.trim()));
|
|
125
|
+
if (firstSelected?.label.trim() !== "近7天")
|
|
126
|
+
return [];
|
|
127
|
+
const table = snapshot.tables.find(t => t.headers.some(h => /^销量\(近7天\)$/.test(h.replace(/\s/g, ""))));
|
|
128
|
+
if (!table)
|
|
129
|
+
return [];
|
|
130
|
+
return table.rows.flatMap(row => {
|
|
131
|
+
const link = row.links.find(a => /\/media-source\/video\/\d+/.test(a.href));
|
|
132
|
+
const id = link?.href.match(/\/media-source\/video\/(\d+)/)?.[1];
|
|
133
|
+
if (!id || !link)
|
|
134
|
+
return [];
|
|
135
|
+
const creator = row.links.find(a => /\/(?:influencer|author)\/detail\/\d+/.test(a.href));
|
|
136
|
+
return [{ id, url: link.href, creatorId: creator?.href.match(/\/detail\/(\d+)/)?.[1] || null, creatorName: row.cells[0]?.split("\n").filter(Boolean).at(-1) || "", published: cell(table.headers, row.cells, /视频发布时间/).slice(0, 10), units: visibleNumber(cell(table.headers, row.cells, /^销量\(近7天\)$/)), ad: /\bAds\b/.test(row.cells[0] || "") ? true : null, ref: ref(snapshot, expected) }];
|
|
137
|
+
});
|
|
138
|
+
}
|
|
139
|
+
export class VisibleFastMossCollector {
|
|
140
|
+
browser;
|
|
141
|
+
constructor(browser) {
|
|
142
|
+
this.browser = browser;
|
|
143
|
+
}
|
|
144
|
+
async stable(selector, accept) {
|
|
145
|
+
let previous = "", stableReads = 0;
|
|
146
|
+
const deadline = Date.now() + 25_000;
|
|
147
|
+
for (let attempt = 0; attempt < 200 && Date.now() < deadline; attempt++) {
|
|
148
|
+
const snapshot = await this.browser.snapshot(selector);
|
|
149
|
+
if (snapshot.blocked && snapshot.blocked !== "页面区域尚未出现")
|
|
150
|
+
throw new Error(snapshot.blocked);
|
|
151
|
+
const signature = JSON.stringify([snapshot.text, snapshot.inputs, snapshot.tables]);
|
|
152
|
+
stableReads = !snapshot.loading && accept(snapshot) && signature === previous ? stableReads + 1 : 0;
|
|
153
|
+
if (stableReads >= 2)
|
|
154
|
+
return snapshot;
|
|
155
|
+
previous = signature;
|
|
156
|
+
await this.browser.pause();
|
|
157
|
+
}
|
|
158
|
+
throw new Error("页面未稳定或时间范围未核实,已保留此前数据");
|
|
159
|
+
}
|
|
160
|
+
async verifyOverview(product) {
|
|
161
|
+
let latest = null;
|
|
162
|
+
let snapshot;
|
|
163
|
+
try {
|
|
164
|
+
snapshot = await this.stable("#overview", s => {
|
|
165
|
+
const overview = overviewFromSection(s);
|
|
166
|
+
if (!overview || overview.units === null || overview.units === undefined)
|
|
167
|
+
return false;
|
|
168
|
+
latest = s;
|
|
169
|
+
const dates = [...new Map(product.daily.filter(d => d.date >= overview.ref.period.start && d.date <= overview.ref.period.end).map(d => [d.date, d])).values()];
|
|
170
|
+
return dates.length === 7 && dates.every(d => d.units !== null) && Math.abs(dates.reduce((sum, d) => sum + d.units, 0) - overview.units) <= (overview.unitsTolerance || 0);
|
|
171
|
+
});
|
|
172
|
+
}
|
|
173
|
+
catch (error) {
|
|
174
|
+
if (!latest)
|
|
175
|
+
throw error;
|
|
176
|
+
// A stable source discrepancy is an observation, not a reason to keep
|
|
177
|
+
// failing the whole product forever. The evaluator blocks recommendation.
|
|
178
|
+
snapshot = latest;
|
|
179
|
+
product.missing.push("7天汇总未与逐日合计对齐,保留可见数值待复核");
|
|
180
|
+
}
|
|
181
|
+
product.overview = overviewFromSection(snapshot);
|
|
182
|
+
product.price = visibleNumber(snapshot.inputs.find(i => i.value === "price")?.label.trim().split("\n").filter(Boolean)[0]);
|
|
183
|
+
product.priceBasis = "7天成交均价";
|
|
184
|
+
return product;
|
|
185
|
+
}
|
|
186
|
+
async discover(ranking, config, save) {
|
|
187
|
+
await this.browser.goto(`https://www.fastmoss.com/zh/e-commerce/${RANKINGS[ranking]}?region=${config.market}&page=1`);
|
|
188
|
+
const seen = new Set();
|
|
189
|
+
const sourceIndex = ["sales", "new", "hot", "video"].indexOf(ranking);
|
|
190
|
+
const target = Math.floor(config.candidateLimit / 4) + (sourceIndex < config.candidateLimit % 4 ? 1 : 0);
|
|
191
|
+
let previous = "";
|
|
192
|
+
// Categories are filtered on observed row text; never claim a hidden site filter was applied.
|
|
193
|
+
for (let page = 0; page < 5 && seen.size < target; page++) {
|
|
194
|
+
const snapshot = await this.stable("#dashboard-container-zh", s => rankingProducts(s, ranking, config).length > 0 && JSON.stringify(s.tables) !== previous);
|
|
195
|
+
const products = rankingProducts(snapshot, ranking, config, page * 10).filter(p => matchesCategory(p.category, config));
|
|
196
|
+
const fresh = products.filter(p => !seen.has(p.productId)).slice(0, target - seen.size);
|
|
197
|
+
for (const product of fresh)
|
|
198
|
+
seen.add(product.productId);
|
|
199
|
+
if (fresh.length)
|
|
200
|
+
await save(fresh);
|
|
201
|
+
previous = JSON.stringify(snapshot.tables);
|
|
202
|
+
if (seen.size >= target || !(await this.browser.next("#dashboard-container-zh")))
|
|
203
|
+
break;
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
async detail(product, config, save) {
|
|
207
|
+
await this.browser.goto(product.url);
|
|
208
|
+
const head = await this.stable("#dashboard-container-zh", s => s.text.includes("预估上架日期") && s.text.includes("数据总览"));
|
|
209
|
+
const text = head.text.split("数据总览")[0];
|
|
210
|
+
product.identityVerified = head.url.includes(`/detail/${product.productId}`) && text.includes(`${MARKETS[config.market]}销量榜`);
|
|
211
|
+
if (!product.identityVerified)
|
|
212
|
+
throw new Error("详情页未核实国家与商品身份");
|
|
213
|
+
product.listingDate = text.match(/预估上架日期[::]?\s*(\d{4}-\d{2}-\d{2})/)?.[1] || null;
|
|
214
|
+
const priceText = text.match(/价格[::]\s*([^\n]+)/)?.[1] || "";
|
|
215
|
+
product.price = visibleNumber(priceText);
|
|
216
|
+
product.currency = priceText.match(/MX\$|R\$|RM|Rp|US\$|[$€£฿¥₱₫]/)?.[0] || null;
|
|
217
|
+
product.stockText = text.match(/(?:^|\n)库存[::]\s*([^\n]+)/)?.[1] || "";
|
|
218
|
+
product.stock = visibleNumber(product.stockText);
|
|
219
|
+
product.rating = visibleNumber(text.match(/(?:^|\n)([\d.]+)\s*\/\s*5\s*评论数/)?.[1]);
|
|
220
|
+
product.reviewCount = visibleNumber(text.match(/评论数[::]?\s*([^\n]+)/)?.[1]);
|
|
221
|
+
if (product.reviewCount === 0)
|
|
222
|
+
product.rating = null;
|
|
223
|
+
product.error = undefined;
|
|
224
|
+
product.missing = [];
|
|
225
|
+
await save(product);
|
|
226
|
+
for (const days of [7, config.opportunity === "emerging" ? 28 : 90]) {
|
|
227
|
+
try {
|
|
228
|
+
await this.browser.clickText("#overview label", `近${days}天`);
|
|
229
|
+
let snapshot = await this.stable("#overview", s => Boolean(sectionPeriod(s, days)) && s.inputs.some(i => i.checked && i.label.trim() === `近${days}天`));
|
|
230
|
+
if (days === 7) {
|
|
231
|
+
product.overview = overviewFromSection(snapshot);
|
|
232
|
+
const averagePrice = snapshot.inputs.find(i => i.value === "price")?.label.trim().split("\n").filter(Boolean)[0];
|
|
233
|
+
product.price = visibleNumber(averagePrice);
|
|
234
|
+
product.priceBasis = "7天成交均价";
|
|
235
|
+
await save(product);
|
|
236
|
+
}
|
|
237
|
+
const period = sectionPeriod(snapshot, days);
|
|
238
|
+
const savedDates = new Set(product.daily.filter(d => d.units !== null && d.ref.period?.start === period.start && d.ref.period?.end === period.end).map(d => d.date));
|
|
239
|
+
if (savedDates.size === days) {
|
|
240
|
+
if (days === 7) {
|
|
241
|
+
await this.verifyOverview(product);
|
|
242
|
+
await save(product);
|
|
243
|
+
}
|
|
244
|
+
continue;
|
|
245
|
+
}
|
|
246
|
+
if (!snapshot.tables.some(t => t.headers.includes("时间")))
|
|
247
|
+
await this.browser.clickText("#overview", "查看表格");
|
|
248
|
+
let previous = "";
|
|
249
|
+
for (let page = 0; page < 24; page++) {
|
|
250
|
+
snapshot = await this.stable("#overview", s => dailyFromSection(s, days).length > 0 && JSON.stringify(s.tables) !== previous);
|
|
251
|
+
const points = dailyFromSection(snapshot, days);
|
|
252
|
+
// Retain 7-day and historical observations separately so conflicting
|
|
253
|
+
// same-day snapshots remain detectable instead of being overwritten.
|
|
254
|
+
product.daily.push(...points);
|
|
255
|
+
await save(product);
|
|
256
|
+
previous = JSON.stringify(snapshot.tables);
|
|
257
|
+
if (!(await this.browser.next("#overview")))
|
|
258
|
+
break;
|
|
259
|
+
}
|
|
260
|
+
if (days === 7) {
|
|
261
|
+
await this.verifyOverview(product);
|
|
262
|
+
await save(product);
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
catch (error) {
|
|
266
|
+
product.missing.push(`${days}天总览:${error instanceof Error ? error.message : String(error)}`);
|
|
267
|
+
await save(product);
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
try {
|
|
271
|
+
// FastMoss mounts lower detail sections only after they enter view.
|
|
272
|
+
await this.browser.clickText("#dashboard-container-zh", "商品关联视频");
|
|
273
|
+
await this.stable("#related_videos", s => s.inputs.some(i => /^近7天$/.test(i.label.trim())));
|
|
274
|
+
await this.browser.clickText("#related_videos label", "近7天", 0);
|
|
275
|
+
await this.browser.clickText("#related_videos label", "近7天", 1);
|
|
276
|
+
const expected = product.overview?.ref.period || null;
|
|
277
|
+
const snapshot = await this.stable("#related_videos", s => (videosFromSection(s, expected).length > 0 || /暂无数据|No data/i.test(s.text)) && s.inputs.filter(i => i.checked && i.label.trim() === "近7天").length === 2);
|
|
278
|
+
product.videos = videosFromSection(snapshot, expected);
|
|
279
|
+
if (!product.videos.length)
|
|
280
|
+
product.missing.push("所选窗口关联视频显示暂无数据,缺少近期新视频成交证据");
|
|
281
|
+
await save(product);
|
|
282
|
+
for (const video of product.videos.slice(0, 5)) {
|
|
283
|
+
try {
|
|
284
|
+
await this.browser.goto(video.url);
|
|
285
|
+
const detail = await this.stable("#dashboard-container-zh", s => s.links.some(l => /\/(?:influencer|author)\/detail\/\d+/.test(l.href)));
|
|
286
|
+
const link = detail.links.find(l => /\/(?:influencer|author)\/detail\/\d+/.test(l.href));
|
|
287
|
+
video.creatorId = link?.href.match(/\/detail\/(\d+)/)?.[1] || null;
|
|
288
|
+
await save(product);
|
|
289
|
+
}
|
|
290
|
+
catch {
|
|
291
|
+
product.missing.push(`视频 ${video.id} 的达人ID未核实`);
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
catch (error) {
|
|
296
|
+
product.missing.push(`近期视频:${error instanceof Error ? error.message : String(error)}`);
|
|
297
|
+
}
|
|
298
|
+
await save(product);
|
|
299
|
+
if (product.missing.some(m => /未稳定|deadline|超时|Timeout|selector/i.test(m)))
|
|
300
|
+
throw new Error("部分详情未完成,已保存可用证据,可继续复核");
|
|
301
|
+
return product;
|
|
302
|
+
}
|
|
303
|
+
}
|
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
import { type Response } from "playwright-core";
|
|
2
|
+
import { FastMossSelectionSessions } from "./fastmoss-selection-session.js";
|
|
3
|
+
import type { SelectionConfig } from "./fastmoss-opportunity.js";
|
|
2
4
|
export type FastMossRankingSource = "new" | "hot" | "video" | "sales";
|
|
3
5
|
export type FastMossPhase = "idle" | "saved" | "launching" | "login_required" | "ready" | "verification_required" | "membership_expired" | "error";
|
|
4
6
|
export type FastMossStatus = {
|
|
@@ -55,11 +57,54 @@ export declare class FastMossIntegration {
|
|
|
55
57
|
private readonly profileDir;
|
|
56
58
|
private readonly authenticatedMarker;
|
|
57
59
|
private readonly dataDir;
|
|
60
|
+
private selectionPage;
|
|
61
|
+
readonly selection: FastMossSelectionSessions;
|
|
58
62
|
private savedAuthenticated;
|
|
59
63
|
private observedVideoRecords;
|
|
60
64
|
private videoResponseTasks;
|
|
61
65
|
private observedVideoPages;
|
|
62
66
|
constructor();
|
|
67
|
+
startSelection(id: string, config: Partial<SelectionConfig>, resume?: boolean): Promise<{
|
|
68
|
+
candidates: {
|
|
69
|
+
source: string;
|
|
70
|
+
sourceProductId: string;
|
|
71
|
+
platformProductId: string;
|
|
72
|
+
title: string;
|
|
73
|
+
market: string;
|
|
74
|
+
category: string;
|
|
75
|
+
storeName: string;
|
|
76
|
+
productUrl: string;
|
|
77
|
+
imageUrls: string[];
|
|
78
|
+
metrics: {
|
|
79
|
+
strategy: "fastmoss-evidence-v2";
|
|
80
|
+
selectionSessionId: string;
|
|
81
|
+
selectionDecision: "test" | "watch" | "skip";
|
|
82
|
+
selectionEligible: boolean;
|
|
83
|
+
portfolioSelected: boolean;
|
|
84
|
+
selectionReasons: string[];
|
|
85
|
+
selectionRisks: string[];
|
|
86
|
+
sales: number | null;
|
|
87
|
+
growthPercent: number | null;
|
|
88
|
+
price: number | null;
|
|
89
|
+
currency: string | null;
|
|
90
|
+
captureDate: string;
|
|
91
|
+
sourceVerified: boolean;
|
|
92
|
+
opportunityAssessment: import("./fastmoss-opportunity.js").Assessment;
|
|
93
|
+
opportunityEvidence: import("./fastmoss-opportunity.js").ProductEvidence;
|
|
94
|
+
};
|
|
95
|
+
}[];
|
|
96
|
+
id: string;
|
|
97
|
+
config: SelectionConfig;
|
|
98
|
+
createdAt: string;
|
|
99
|
+
updatedAt: string;
|
|
100
|
+
phase: "collecting" | "reviewing" | "complete" | "interrupted";
|
|
101
|
+
active: boolean;
|
|
102
|
+
completedRankings: import("./fastmoss-opportunity.js").Ranking[];
|
|
103
|
+
rankingErrors: Partial<Record<import("./fastmoss-opportunity.js").Ranking, string>>;
|
|
104
|
+
products: import("./fastmoss-opportunity.js").ProductEvidence[];
|
|
105
|
+
error?: string;
|
|
106
|
+
} | null>;
|
|
107
|
+
private selectionBrowser;
|
|
63
108
|
status(): FastMossStatus;
|
|
64
109
|
start(): Promise<FastMossStatus>;
|
|
65
110
|
inspect(): Promise<FastMossStatus>;
|