@xiaohhhh1/canvas-agent 0.4.21 → 0.4.22

package/dist/integrations/fastmoss.d.ts

@@ -1,3 +1,4 @@
+export type FastMossRankingSource = "new" | "hot" | "video" | "sales";
 export type FastMossPhase = "idle" | "saved" | "launching" | "login_required" | "ready" | "verification_required" | "membership_expired" | "error";
 export type FastMossStatus = {
     phase: FastMossPhase;
@@ -37,6 +38,21 @@ export declare class FastMossIntegration {
     start(): Promise<FastMossStatus>;
     inspect(): Promise<FastMossStatus>;
     capture(input?: CaptureContext): Promise<{
+        records: Record<string, unknown>[];
+        allRows: Record<string, unknown>[];
+        trendProducts: number;
+        detailFailures: number;
+        interrupted: boolean;
+        phase: FastMossPhase;
+        browserOpen: boolean;
+        authenticated: boolean;
+        verificationRequired: boolean;
+        membershipExpired: boolean;
+        message: string;
+        url: string | null;
+        captured: number;
+        lastCapture?: string;
+    } | {
         records: Record<string, unknown>[];
         allRows: Record<string, unknown>[];
         trendProducts: number;
@@ -61,7 +77,36 @@ export declare class FastMossIntegration {
     private openProductRanking;
 }
 export declare function fastMossSalesRankUrl(market?: string): string;
+export declare function fastMossRankingUrl(source: FastMossRankingSource, market?: string): string;
+export declare function mergeRankingRecord(target: Map<string, Record<string, unknown>>, row: Record<string, unknown>, source: FastMossRankingSource, position: number): void;
+export declare function selectDetailCandidates(records: Array<Record<string, unknown>>, limit?: number): Record<string, unknown>[];
 export declare function productIdFromUrl(value: string): string;
+/** Extract only the evidence required by the AI short-video selection model. */
+export declare function detailOverviewEvidence(overviewPayload: unknown, basePayload?: unknown): {
+    recentVideoSales: number;
+    recentLiveSales: number;
+    recentCreatorCount: number;
+    recentVideoCount: number;
+    recentLiveCount: number;
+    videoTransactionShare: number | null;
+    liveTransactionShare: number | null;
+    adSalesShare: number | null;
+    organicTrafficShare: number | null;
+    creatorAdds: number;
+    videoAdds: number;
+    stock: number;
+    rating: number;
+    reviewCount: number;
+    totalCreatorCount: number;
+    totalVideoCount: number;
+    totalLiveCount: number;
+    viralIndex: number;
+    popularityIndex: number;
+    countryRank: number;
+    categoryRank: number;
+    launchTime: number;
+    commission: number | null;
+};
 export declare function detailTrendRows(record: Record<string, unknown>, payload: unknown, days?: number): {
     date: string;
     source: string;
@@ -71,5 +116,28 @@ export declare function detailTrendRows(record: Record<string, unknown>, payload
     price: number;
     currency: string;
     period_days: number;
+    recentVideoSales: number;
+    recentLiveSales: number;
+    recentCreatorCount: number;
+    recentVideoCount: number;
+    recentLiveCount: number;
+    videoTransactionShare: number | null;
+    liveTransactionShare: number | null;
+    adSalesShare: number | null;
+    organicTrafficShare: number | null;
+    creatorAdds: number;
+    videoAdds: number;
+    stock: number;
+    rating: number;
+    reviewCount: number;
+    totalCreatorCount: number;
+    totalVideoCount: number;
+    totalLiveCount: number;
+    viralIndex: number;
+    popularityIndex: number;
+    countryRank: number;
+    categoryRank: number;
+    launchTime: number;
+    commission: number | null;
 }[];
 export {};

package/dist/integrations/fastmoss.js

@@ -5,13 +5,19 @@ import path from "node:path";
 import { chromium } from "playwright-core";
 import { CONFIG_DIR } from "../config.js";
 const FASTMOSS_HOME = "https://www.fastmoss.com/";
-const FASTMOSS_SALES_RANK = "https://www.fastmoss.com/e-commerce/saleslist";
 const MEMBERSHIP_RECHECK_MS = 5 * 60_000;
 const PRODUCT_TABLE_TIMEOUT_MS = 45_000;
 const PRODUCT_RANK_PAGE_SIZE = 10;
 const PRODUCT_RANK_MAX_PAGES = 50;
+const DETAIL_CANDIDATE_LIMIT = 200;
 const DETAIL_TREND_DAYS = 7;
 const DETAIL_FETCH_CONCURRENCY = 3;
+const RANKING_DEFINITIONS = [
+    { source: "new", label: "新品榜", pathname: "/e-commerce/newProducts" },
+    { source: "hot", label: "热推榜", pathname: "/e-commerce/hotlist" },
+    { source: "video", label: "视频商品榜", pathname: "/e-commerce/hotvideo" },
+    { source: "sales", label: "销量榜", pathname: "/e-commerce/saleslist" },
+];
 export class FastMossIntegration {
     context = null;
     page = null;
@@ -171,24 +177,28 @@ export class FastMossIntegration {
         }
         if (!this.authenticated || !this.page || this.page.isClosed())
             throw new Error("FastMoss 尚未登录；请在恢复的窗口中完成登录后重试");
-        await this.openProductRanking(input);
         const collected = new Map();
         const pages = [];
-        let previousSignature = "";
-        for (let pageNumber = 1; pageNumber <= PRODUCT_RANK_MAX_PAGES; pageNumber += 1) {
-            const snapshot = await readVisibleTableSnapshot(this.page);
-            if (!snapshot.tables[0]?.rows.length || snapshot.signature === previousSignature)
-                break;
-            previousSignature = snapshot.signature;
-            pages.push(snapshot.url);
-            const pageRecords = extractRows(snapshot.tables, { ...input, date: localDate() }, snapshot.url);
-            pageRecords.forEach((row) => collected.set(String(row.product_id || row.title), row));
-            this.captured = collected.size;
-            this.message = `正在读取可见榜单第 ${pageNumber} 页，已收集 ${this.captured} 条商品数据`;
-            if (!await clickVisibleNextPage(this.page))
-                break;
-            await this.page.waitForTimeout(800);
-            await this.inspect();
+        for (const ranking of RANKING_DEFINITIONS) {
+            await this.openProductRanking(input, ranking);
+            let previousSignature = "";
+            for (let pageNumber = 1; pageNumber <= PRODUCT_RANK_MAX_PAGES; pageNumber += 1) {
+                const snapshot = await readVisibleTableSnapshot(this.page);
+                if (!snapshot.tables[0]?.rows.length || snapshot.signature === previousSignature)
+                    break;
+                previousSignature = snapshot.signature;
+                pages.push(snapshot.url);
+                const pageRecords = extractRows(snapshot.tables, { ...input, date: localDate() }, snapshot.url);
+                pageRecords.forEach((row, index) => mergeRankingRecord(collected, row, ranking.source, (pageNumber - 1) * PRODUCT_RANK_PAGE_SIZE + index + 1));
+                this.captured = collected.size;
+                this.message = `正在读取${ranking.label}第 ${pageNumber} 页，跨榜单去重后 ${this.captured} 个商品`;
+                if (!await clickVisibleNextPage(this.page))
+                    break;
+                await this.page.waitForTimeout(800);
+                await this.inspect();
+                if (this.verificationRequired || this.membershipExpired)
+                    break;
+            }
             if (this.verificationRequired || this.membershipExpired)
                 break;
         }
@@ -197,20 +207,39 @@ export class FastMossIntegration {
             throw new Error("已自动进入 FastMoss 商品榜单，但没有读取到商品数据；请检查会员权限或页面是否仍在加载");
         const requestedMarket = normalizeMarket(input.market);
         const marketRecords = records.filter((record) => !record.market || normalizeExtractedMarket(record.market) === requestedMarket);
+        const retainPreviousOnInterruption = async (stage, partialTrendRows = [], detailFailures = 0) => {
+            await mkdir(this.dataDir, { recursive: true });
+            const attemptedAt = new Date().toISOString();
+            const previousRows = await this.loadRows();
+            await writeFile(path.join(this.dataDir, `capture-${localDate()}-interrupted-${Date.now()}.json`), JSON.stringify({ attemptedAt, interrupted: true, stage, pages, records: marketRecords, trendRows: partialTrendRows, detailFailures }, null, 2), "utf8");
+            this.captured = marketRecords.length;
+            const previousProducts = new Set(previousRows.map((row) => String(row.product_id))).size;
+            this.message = this.verificationRequired
+                ? `采集在${stage === "ranking" ? "榜单" : "商品详情互证"}阶段遇到人工验证；已保留上次 ${previousProducts} 个商品结果，请完成滑块后重新抓取`
+                : `采集在${stage === "ranking" ? "榜单" : "商品详情互证"}阶段发现会员权限异常；已保留上次 ${previousProducts} 个商品结果`;
+            return { ...this.status(), records: marketRecords, allRows: previousRows, trendProducts: previousProducts, detailFailures, interrupted: true };
+        };
+        if (this.verificationRequired || this.membershipExpired)
+            return retainPreviousOnInterruption("ranking");
+        const detailCandidates = selectDetailCandidates(marketRecords, DETAIL_CANDIDATE_LIMIT);
+        const capturePage = this.page;
         const trendRows = [];
         let detailFailures = 0;
-        for (let offset = 0; offset < marketRecords.length; offset += DETAIL_FETCH_CONCURRENCY) {
-            const chunk = marketRecords.slice(offset, offset + DETAIL_FETCH_CONCURRENCY);
-            const results = await Promise.all(chunk.map((record) => fetchDetailTrend(this.page, record, DETAIL_TREND_DAYS)));
+        for (let offset = 0; offset < detailCandidates.length; offset += DETAIL_FETCH_CONCURRENCY) {
+            const chunk = detailCandidates.slice(offset, offset + DETAIL_FETCH_CONCURRENCY);
+            const results = await Promise.all(chunk.map((record) => fetchDetailTrend(capturePage, record, DETAIL_TREND_DAYS)));
             results.forEach((rows) => rows.length ? trendRows.push(...rows) : detailFailures += 1);
-            this.message = `正在读取商品最近 ${DETAIL_TREND_DAYS} 天趋势：${Math.min(offset + chunk.length, marketRecords.length)}/${marketRecords.length}`;
-            await this.page.waitForTimeout(150);
+            this.message = `正在互证商品最近 ${DETAIL_TREND_DAYS} 天趋势：${Math.min(offset + chunk.length, detailCandidates.length)}/${detailCandidates.length}`;
+            await capturePage.waitForTimeout(150);
             if ((offset / DETAIL_FETCH_CONCURRENCY) % 10 === 0) {
+                this.useLatestPage();
                 await this.inspect();
                 if (this.verificationRequired || this.membershipExpired)
                     break;
             }
         }
+        if (this.verificationRequired || this.membershipExpired)
+            return retainPreviousOnInterruption("detail", trendRows, detailFailures);
         if (!trendRows.length && !this.verificationRequired)
             throw new Error("榜单读取成功，但未能读取商品详情的最近 7 天销量；请检查会员权限后重试");
         await mkdir(this.dataDir, { recursive: true });
@@ -218,11 +247,11 @@ export class FastMossIntegration {
         this.lastCapture = new Date().toISOString();
         this.captured = marketRecords.length;
         await writeFile(path.join(this.dataDir, "observations.json"), JSON.stringify(allRows, null, 2), "utf8");
-        await writeFile(path.join(this.dataDir, `capture-${localDate()}.json`), JSON.stringify({ capturedAt: this.lastCapture, pages, records: marketRecords, trendRows, detailFailures }, null, 2), "utf8");
+        await writeFile(path.join(this.dataDir, `capture-${localDate()}.json`), JSON.stringify({ capturedAt: this.lastCapture, pages, records: marketRecords, detailCandidates: detailCandidates.map((record) => record.product_id), trendRows, detailFailures }, null, 2), "utf8");
         const trendProducts = new Set(trendRows.map((row) => String(row.product_id))).size;
         this.message = this.verificationRequired
             ? `已保存验证前的 ${trendProducts} 个商品趋势；请手动完成滑块后再次抓取`
-            : `已读取榜单 ${marketRecords.length} 个商品，其中 ${trendProducts} 个取得最近 ${DETAIL_TREND_DAYS} 天趋势`;
+            : `四榜单去重后 ${marketRecords.length} 个商品，优先互证 ${detailCandidates.length} 个，其中 ${trendProducts} 个取得最近 ${DETAIL_TREND_DAYS} 天趋势`;
         return { ...this.status(), records: marketRecords, allRows, trendProducts, detailFailures };
     }
     async close() {
@@ -276,18 +305,18 @@ export class FastMossIntegration {
         if (pages.length)
             this.page = pages[pages.length - 1];
     }
-    async openProductRanking(input) {
+    async openProductRanking(input, ranking = RANKING_DEFINITIONS[0]) {
         if (!this.page || this.page.isClosed())
             throw new Error("FastMoss 专用浏览器未打开");
         const market = normalizeMarket(input.market);
-        this.message = `正在自动进入 ${market} 商品榜单`;
-        await this.page.goto(fastMossSalesRankUrl(market), { waitUntil: "domcontentloaded", timeout: 60_000 });
+        this.message = `正在自动进入 ${market} ${ranking.label}`;
+        await this.page.goto(fastMossRankingUrl(ranking.source, market), { waitUntil: "domcontentloaded", timeout: 60_000 });
         const deadline = Date.now() + PRODUCT_TABLE_TIMEOUT_MS;
         while (Date.now() < deadline) {
             const snapshot = await readVisibleTableSnapshot(this.page);
             if (hasReadableProductRows(snapshot)) {
                 this.phase = "ready";
-                this.message = `${market} 商品榜单已打开，正在开始采集`;
+                this.message = `${market} ${ranking.label}已打开，正在开始采集`;
                 return;
             }
             await this.inspect();
@@ -303,12 +332,39 @@ export class FastMossIntegration {
     }
 }
 export function fastMossSalesRankUrl(market) {
-    const url = new URL(FASTMOSS_SALES_RANK);
+    return fastMossRankingUrl("sales", market);
+}
+export function fastMossRankingUrl(source, market) {
+    const definition = RANKING_DEFINITIONS.find((item) => item.source === source) || RANKING_DEFINITIONS[0];
+    const url = new URL(definition.pathname, FASTMOSS_HOME);
     url.searchParams.set("region", normalizeMarket(market));
     url.searchParams.set("page", "1");
     url.searchParams.set("pagesize", String(PRODUCT_RANK_PAGE_SIZE));
     return url.toString();
 }
+export function mergeRankingRecord(target, row, source, position) {
+    const key = String(row.product_id || row.title);
+    const existing = target.get(key) || {};
+    const sources = [...new Set([...(Array.isArray(existing.ranking_sources) ? existing.ranking_sources.map(String) : []), source])];
+    const positions = existing.ranking_positions && typeof existing.ranking_positions === "object" ? existing.ranking_positions : {};
+    target.set(key, {
+        ...existing,
+        ...row,
+        ranking_sources: sources,
+        ranking_positions: { ...positions, [source]: position },
+        cross_list_count: sources.filter((item) => item !== "sales").length,
+    });
+}
+export function selectDetailCandidates(records, limit = DETAIL_CANDIDATE_LIMIT) {
+    const bestPosition = (record) => {
+        const positions = record.ranking_positions && typeof record.ranking_positions === "object" ? Object.values(record.ranking_positions) : [];
+        return Math.min(...positions.map(Number).filter(Number.isFinite), Number.MAX_SAFE_INTEGER);
+    };
+    return [...records].sort((left, right) => {
+        const crossDifference = Number(right.cross_list_count || 0) - Number(left.cross_list_count || 0);
+        return crossDifference || bestPosition(left) - bestPosition(right);
+    }).slice(0, Math.max(1, limit));
+}
 function normalizeMarket(value) {
     const market = String(value || "MX").trim().toUpperCase();
     return /^[A-Z]{2}$/.test(market) ? market : "MX";
@@ -406,6 +462,7 @@ function extractRows(tables, context, pageUrl) {
             const storeName = String(pick(/所属店铺|店铺|shop|store/i)).replace(/\s*店铺销量\s*[：:].*$/is, "").trim();
             output.push({
                 date: context.date,
+                capture_date: context.date,
                 source: "fastmoss-agent",
                 product_id: productId,
                 platform_product_id: String(pick(/平台商品|tiktok.*id/i) || productId),
@@ -444,15 +501,68 @@ export function productIdFromUrl(value) {
         return String(value).match(/(?:detail|product|goods|item)[/=_-](\d+)/i)?.[1] || "";
     }
 }
+function percentValue(value) {
+    const number = Number(String(value ?? "").replace("%", ""));
+    return Number.isFinite(number) ? number : null;
+}
+function distributionShare(value, expected) {
+    const container = value && typeof value === "object" ? value : {};
+    const list = Array.isArray(container.list) ? container.list : [];
+    const item = list.find((entry) => expected.test(String(entry.category ?? entry.source ?? "")));
+    return item ? percentValue(item.propotion ?? item.proportion ?? item.percent) : null;
+}
+/** Extract only the evidence required by the AI short-video selection model. */
+export function detailOverviewEvidence(overviewPayload, basePayload) {
+    const overviewEnvelope = overviewPayload && typeof overviewPayload === "object" ? overviewPayload : {};
+    const overviewData = overviewEnvelope.data && typeof overviewEnvelope.data === "object" ? overviewEnvelope.data : {};
+    const overview = overviewData.overview && typeof overviewData.overview === "object" ? overviewData.overview : {};
+    const content = overviewData.content_distribution && typeof overviewData.content_distribution === "object" ? overviewData.content_distribution : {};
+    const ads = overviewData.ads_distribution && typeof overviewData.ads_distribution === "object" ? overviewData.ads_distribution : {};
+    const chart = Array.isArray(overviewData.chart_list) ? overviewData.chart_list : [];
+    const baseEnvelope = basePayload && typeof basePayload === "object" ? basePayload : {};
+    const baseData = baseEnvelope.data && typeof baseEnvelope.data === "object" ? baseEnvelope.data : {};
+    const product = baseData.product && typeof baseData.product === "object" ? baseData.product : {};
+    const videoShare = distributionShare(content.units_sold, /^video$/i);
+    const liveShare = distributionShare(content.units_sold, /^live$/i);
+    const adShare = distributionShare(ads.units_sold, /adtraffic|advert|广告/i);
+    const organicShare = distributionShare(ads.units_sold, /othertraffic|organic|自然|其他/i);
+    return {
+        recentVideoSales: Number(overview.video_sold_count || 0),
+        recentLiveSales: Number(overview.live_sold_count || 0),
+        recentCreatorCount: Number(overview.author_count || 0),
+        recentVideoCount: Number(overview.aweme_count || 0),
+        recentLiveCount: Number(overview.live_count || 0),
+        videoTransactionShare: videoShare,
+        liveTransactionShare: liveShare,
+        adSalesShare: adShare,
+        organicTrafficShare: organicShare,
+        creatorAdds: chart.reduce((sum, point) => sum + Number(point.inc_author_count || 0), 0),
+        videoAdds: chart.reduce((sum, point) => sum + Number(point.inc_aweme_count || 0), 0),
+        stock: Number(product.stock_count || 0),
+        rating: Number(product.product_rating || 0),
+        reviewCount: Number(product.review_count || 0),
+        totalCreatorCount: Number(product.author_count || 0),
+        totalVideoCount: Number(product.aweme_count || 0),
+        totalLiveCount: Number(product.live_count || 0),
+        viralIndex: Number(product.viral_index || 0),
+        popularityIndex: Number(product.popularity_index || 0),
+        countryRank: Number(product.country_rank || 0),
+        categoryRank: Number(product.category_rank || 0),
+        launchTime: Number(product.launch_time || 0),
+        commission: percentValue(product.commission_rate),
+    };
+}
 export function detailTrendRows(record, payload, days = DETAIL_TREND_DAYS) {
     const envelope = payload && typeof payload === "object" ? payload : {};
     const data = envelope.data && typeof envelope.data === "object" ? envelope.data : {};
     const points = Array.isArray(data.chart_list) ? data.chart_list : [];
+    const evidence = detailOverviewEvidence(payload);
     return points.slice(-days).flatMap((point) => {
         const date = String(point.dt || "").slice(0, 10);
         if (!/^\d{4}-\d{2}-\d{2}$/.test(date))
             return [];
         return [{
+                ...evidence,
                 ...record,
                 date,
                 source: "fastmoss-agent-detail-trend",
@@ -469,18 +579,32 @@ async function fetchDetailTrend(page, record, days) {
     const productId = String(record.product_id || productIdFromUrl(String(record.product_url || "")));
     if (!/^\d+$/.test(productId))
         return [];
-    const payload = await page.evaluate(async ({ productId, days }) => {
-        const url = new URL("/api/goods/v3/overview", location.origin);
-        url.searchParams.set("product_id", productId);
-        url.searchParams.set("d_type", String(days));
-        url.searchParams.set("_time", String(Math.floor(Date.now() / 1000)));
-        url.searchParams.set("cnonce", String(Math.floor(10_000_000 + Math.random() * 90_000_000)));
-        const response = await fetch(url, { credentials: "include" });
-        if (!response.ok)
-            return null;
-        return response.json().catch(() => null);
-    }, { productId, days }).catch(() => null);
-    return detailTrendRows({ ...record, product_id: productId, platform_product_id: productId }, payload, days);
+    const rawUrl = String(record.product_url || "");
+    const detailUrl = /fastmoss\.com\/.*e-commerce\/detail\//i.test(rawUrl)
+        ? rawUrl
+        : new URL(`/zh/e-commerce/detail/${productId}`, FASTMOSS_HOME).toString();
+    const detailPage = await page.context().newPage();
+    try {
+        // Capture only responses emitted by the visible product-detail page.
+        // The Agent does not manufacture requests to private endpoints.
+        const overviewResponse = detailPage.waitForResponse((response) => {
+            const url = response.url();
+            return response.ok() && url.includes("/api/goods/v3/overview") && url.includes(productId);
+        }, { timeout: PRODUCT_TABLE_TIMEOUT_MS }).then((response) => response.json()).catch(() => null);
+        const baseResponse = detailPage.waitForResponse((response) => {
+            const url = response.url();
+            return response.ok() && url.includes("/api/goods/v3/base") && url.includes(productId);
+        }, { timeout: PRODUCT_TABLE_TIMEOUT_MS }).then((response) => response.json()).catch(() => null);
+        await detailPage.goto(detailUrl, { waitUntil: "domcontentloaded", timeout: 60_000 });
+        const [overview, base] = await Promise.all([overviewResponse, baseResponse]);
+        if (!overview)
+            return [];
+        const evidence = detailOverviewEvidence(overview, base);
+        return detailTrendRows({ ...record, ...evidence, product_id: productId, platform_product_id: productId }, overview, days);
+    }
+    finally {
+        await detailPage.close().catch(() => undefined);
+    }
 }
 function normalizeExtractedMarket(value) {
     const key = String(value || "").trim().toLowerCase();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@xiaohhhh1/canvas-agent",
-  "version": "0.4.21",
+  "version": "0.4.22",
   "type": "module",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",