@xiaohhhh1/canvas-agent 0.4.25 → 0.4.27

package/dist/integrations/fastmoss.d.ts

@@ -1,3 +1,4 @@
+import { type Response } from "playwright-core";
 export type FastMossRankingSource = "new" | "hot" | "video" | "sales";
 export type FastMossPhase = "idle" | "saved" | "launching" | "login_required" | "ready" | "verification_required" | "membership_expired" | "error";
 export type FastMossStatus = {
@@ -30,6 +31,15 @@ type VisibleVideoItem = {
     images: string[];
     headers: string[];
     cells: string[];
+    previewIndex?: number;
+};
+export type ObservedVideoRecord = {
+    videoId: string;
+    authorId?: string;
+    title?: string;
+    sourceUrl?: string;
+    thumbnailUrl?: string;
+    metrics?: Record<string, number>;
 };
 export declare class FastMossIntegration {
     private context;
@@ -46,6 +56,9 @@ export declare class FastMossIntegration {
     private readonly authenticatedMarker;
     private readonly dataDir;
     private savedAuthenticated;
+    private observedVideoRecords;
+    private videoResponseTasks;
+    private observedVideoPages;
     constructor();
     status(): FastMossStatus;
     start(): Promise<FastMossStatus>;
@@ -113,6 +126,15 @@ export declare class FastMossIntegration {
     private openProductRanking;
     private openVideoLearningPage;
     private readVideoLearningItems;
+    /**
+     * AI ranking cards expose their TikTok video id only through the visible
+     * preview action. Open only the cards needed for this batch, capture the
+     * public FastMoss video-detail URL, and close the temporary tab again.
+     */
+    private resolveVideoLearningCardLinks;
+    private resetObservedVideoRecords;
+    private observeVideoRankingResponses;
+    private waitForVideoResponseTasks;
     private clickVideoLearningTab;
     private clickVideoLearningNextPage;
     private waitForVideoLearningNavigation;
@@ -124,7 +146,29 @@ export declare function fastMossAiVideoLearningRankUrl(market?: string): string;
 export declare function mergeRankingRecord(target: Map<string, Record<string, unknown>>, row: Record<string, unknown>, source: FastMossRankingSource, position: number): void;
 export declare function selectDetailCandidates(records: Array<Record<string, unknown>>, limit?: number): Record<string, unknown>[];
 export declare function isTransientFastMossNavigation(error: unknown): boolean;
+export declare function isVideoRankingResponse(response: Response | string): boolean;
+export declare function publicVideoRecordsFromRankPayload(payload: unknown): ObservedVideoRecord[];
+export declare function videoLearningSourcesFromObservedRecords(records: ObservedVideoRecord[], context: {
+    market: string;
+    dimension: VideoLearningDimension;
+    observedAt: string;
+}): {
+    source: string;
+    platform: string;
+    platformVideoId: string;
+    sourceUrl: string;
+    market: string;
+    rankingDimension: VideoLearningDimension;
+    rank: number;
+    title: string;
+    productTitle: string;
+    thumbnailUrl: string | null;
+    metrics: Record<string, number>;
+    observedAt: string;
+}[];
 export declare function isAnalyzableVideoUrl(value: string): boolean;
+export declare function isFastMossVideoDetailUrl(value: string): boolean;
+export declare function publicTikTokVideoUrlFromFastMoss(value: string): string;
 export declare function videoLearningSourcesFromItems(items: VisibleVideoItem[], context: {
     market: string;
     dimension: VideoLearningDimension;

package/dist/integrations/fastmoss.js

@@ -37,6 +37,9 @@ export class FastMossIntegration {
     authenticatedMarker = path.join(this.profileDir, "authenticated.json");
     dataDir = path.join(CONFIG_DIR, "fastmoss-selection");
     savedAuthenticated = existsSync(this.authenticatedMarker);
+    observedVideoRecords = [];
+    videoResponseTasks = new Set();
+    observedVideoPages = new WeakSet();
     constructor() {
         if (!this.savedAuthenticated)
             return;
@@ -90,8 +93,9 @@ export class FastMossIntegration {
             throw error;
         }
         this.page = this.context.pages()[0] || await this.context.newPage();
+        this.observeVideoRankingResponses(this.page);
         this.context.on("page", (page) => {
-            this.page = page;
+            this.observeVideoRankingResponses(page);
         });
         this.context.on("close", () => {
             this.context = null;
@@ -285,12 +289,14 @@ export class FastMossIntegration {
                 return;
             let previousSignature = "";
             for (let pageNumber = 1; pageNumber <= 5 && sources.filter((item) => item.rankingDimension === dimension).length < limit; pageNumber += 1) {
-                const snapshot = await this.readVideoLearningItems();
+                const snapshot = await this.resolveVideoLearningCardLinks(await this.readVideoLearningItems(), limit);
                 const signature = JSON.stringify(snapshot.map((item) => [item.text, item.media, item.links]).slice(0, 20));
                 if (!snapshot.length || (pageNumber > 1 && signature === previousSignature))
                     break;
                 previousSignature = signature;
-                const normalized = videoLearningSourcesFromItems(snapshot, { market, dimension, observedAt: new Date().toISOString() });
+                await this.waitForVideoResponseTasks();
+                const observed = videoLearningSourcesFromObservedRecords(this.observedVideoRecords, { market, dimension, observedAt: new Date().toISOString() });
+                const normalized = [...observed, ...videoLearningSourcesFromItems(snapshot, { market, dimension, observedAt: new Date().toISOString() })];
                 for (const item of normalized) {
                     const key = String(item.platformVideoId || item.sourceUrl || "");
                     if (!key || seen.has(key))
@@ -334,6 +340,7 @@ export class FastMossIntegration {
             for (const entry of aiDimensions) {
                 if (sources.length >= requestedTarget)
                     break;
+                this.resetObservedVideoRecords();
                 const clicked = await this.clickVideoLearningTab(entry.labels);
                 if (!clicked) {
                     rejected.push({ dimension: entry.dimension, reason: `没有找到${entry.labels[0]}标签` });
@@ -405,7 +412,10 @@ export class FastMossIntegration {
     }
     useLatestPage() {
         const pages = this.context?.pages().filter((page) => !page.isClosed()) || [];
-        if (pages.length)
+        const rankingPages = pages.filter((page) => !isFastMossVideoDetailUrl(page.url()));
+        if (rankingPages.length)
+            this.page = rankingPages[rankingPages.length - 1];
+        else if (pages.length)
             this.page = pages[pages.length - 1];
     }
     async openProductRanking(input, ranking = RANKING_DEFINITIONS[0]) {
@@ -436,6 +446,8 @@ export class FastMossIntegration {
     async openVideoLearningPage(url, label) {
         if (!this.page || this.page.isClosed())
             throw new Error("FastMoss 专用浏览器未打开");
+        this.resetObservedVideoRecords();
+        this.observeVideoRankingResponses(this.page);
         this.message = `正在自动进入 ${label}`;
         try {
             await this.page.goto(url, { waitUntil: "domcontentloaded", timeout: 60_000 });
@@ -480,6 +492,77 @@ export class FastMossIntegration {
         }
         throw lastError instanceof Error ? lastError : new Error("FastMoss 视频榜页面跳转后没有恢复");
     }
+    /**
+     * AI ranking cards expose their TikTok video id only through the visible
+     * preview action. Open only the cards needed for this batch, capture the
+     * public FastMoss video-detail URL, and close the temporary tab again.
+     */
+    async resolveVideoLearningCardLinks(items, limit) {
+        if (!this.page || this.page.isClosed())
+            return items;
+        const rankingPage = this.page;
+        const candidates = items
+            .filter((item) => !item.links.some(isFastMossVideoDetailUrl) && Number.isInteger(item.previewIndex))
+            .slice(0, Math.max(0, limit));
+        if (!candidates.length)
+            return items;
+        const covers = rankingPage.locator('div.cursor-pointer > img[class*="aspect-"]');
+        const coverCount = await covers.count();
+        for (const item of candidates) {
+            const index = Number(item.previewIndex);
+            if (!Number.isInteger(index) || index < 0 || index >= coverCount)
+                continue;
+            let popup = null;
+            try {
+                [popup] = await Promise.all([
+                    rankingPage.waitForEvent("popup", { timeout: 6_000 }),
+                    covers.nth(index).click({ timeout: 6_000 }),
+                ]);
+                await popup.waitForURL(/\/media-source\/video\/\d+/i, { timeout: 10_000 });
+                const detailUrl = popup.url();
+                if (isFastMossVideoDetailUrl(detailUrl))
+                    item.links.unshift(detailUrl);
+            }
+            catch {
+                // Keep the item unresolved. The caller records a truthful rejection
+                // instead of counting a cover image or ranking page as a video.
+            }
+            finally {
+                if (popup && !popup.isClosed())
+                    await popup.close().catch(() => undefined);
+                this.page = rankingPage;
+            }
+        }
+        return items;
+    }
+    resetObservedVideoRecords() {
+        this.observedVideoRecords = [];
+        this.videoResponseTasks.clear();
+    }
+    observeVideoRankingResponses(page) {
+        if (this.observedVideoPages.has(page))
+            return;
+        this.observedVideoPages.add(page);
+        page.on("response", (response) => {
+            if (!isVideoRankingResponse(response))
+                return;
+            let task;
+            task = response.json()
+                .then((payload) => {
+                const merged = new Map(this.observedVideoRecords.map((item) => [item.videoId, item]));
+                for (const item of publicVideoRecordsFromRankPayload(payload))
+                    merged.set(item.videoId, { ...merged.get(item.videoId), ...item });
+                this.observedVideoRecords = [...merged.values()];
+            })
+                .catch(() => undefined)
+                .finally(() => this.videoResponseTasks.delete(task));
+            this.videoResponseTasks.add(task);
+        });
+    }
+    async waitForVideoResponseTasks() {
+        if (this.videoResponseTasks.size)
+            await Promise.allSettled([...this.videoResponseTasks]);
+    }
     async clickVideoLearningTab(labels) {
         let lastError;
         for (let attempt = 0; attempt < 3; attempt += 1) {
@@ -692,13 +775,14 @@ async function readVisibleVideoLearningItems(page) {
             ]);
             return [...new Set([...direct, ...data].filter(Boolean).map(absoluteUrl).filter(Boolean))];
         };
-        const serialize = (root, headers = [], cells = []) => ({
+        const serialize = (root, headers = [], cells = [], previewIndex) => ({
             text: root.innerText.replace(/\s+/g, " ").trim().slice(0, 8000),
             links: [...new Set([...root.querySelectorAll("a[href]")].map((link) => link.href).filter(Boolean))],
             media: mediaUrls(root),
             images: [...new Set([...root.querySelectorAll("img")].map((image) => image.currentSrc || image.src || image.dataset.src || "").filter(Boolean).map(absoluteUrl).filter(Boolean))],
             headers,
             cells,
+            ...(previewIndex == null ? {} : { previewIndex }),
         });
         const tableItems = [...document.querySelectorAll("table tbody tr")]
             .filter(shown)
@@ -709,8 +793,15 @@ async function readVisibleVideoLearningItems(page) {
             return serialize(row, headers, cells);
         })
             .filter((item) => item.text && (item.links.length || item.media.length || item.images.length));
+        const previewSelector = 'div.cursor-pointer > img[class*="aspect-"]';
+        const previewImages = [...document.querySelectorAll(previewSelector)];
+        const previewItems = previewImages.map((preview, previewIndex) => {
+            const root = preview.parentElement?.parentElement || preview.parentElement || preview;
+            return serialize(root, [], [], previewIndex);
+        }).filter((item) => item.text && item.images.length);
         const cardCandidates = [...document.querySelectorAll("article,li,[class*='card'],[class*='item']")]
             .filter(shown)
+            .filter((element) => !element.querySelector(previewSelector))
             .filter((element) => {
             const rect = element.getBoundingClientRect();
             const text = element.innerText.replace(/\s+/g, " ").trim();
@@ -731,7 +822,7 @@ async function readVisibleVideoLearningItems(page) {
             if (cards.length >= 80)
                 break;
         }
-        const combined = [...tableItems, ...cards];
+        const combined = [...tableItems, ...previewItems, ...cards];
         const seen = new Set();
         return combined.filter((item) => {
             const key = JSON.stringify([item.text.slice(0, 300), item.media[0], item.links[0], item.images[0]]);
@@ -745,6 +836,87 @@ async function readVisibleVideoLearningItems(page) {
 function firstHttp(values, expected) {
     return values.find((value) => /^https?:\/\//i.test(value) && (!expected || expected.test(value))) || "";
 }
+export function isVideoRankingResponse(response) {
+    const value = typeof response === "string" ? response : response.url();
+    try {
+        const url = new URL(value);
+        return /(?:^|\.)fastmoss\.com$/i.test(url.hostname)
+            && /\/api\/.*(?:video|aweme|media)/i.test(url.pathname);
+    }
+    catch {
+        return false;
+    }
+}
+export function publicVideoRecordsFromRankPayload(payload) {
+    const output = new Map();
+    const visited = new Set();
+    let inspected = 0;
+    const scalar = (record, keys) => {
+        for (const key of keys) {
+            const value = record[key];
+            if (typeof value === "string" || typeof value === "number")
+                return String(value).trim();
+        }
+        return "";
+    };
+    const numeric = (record, keys) => {
+        const raw = scalar(record, keys);
+        const value = numberValue(raw);
+        return Number.isFinite(value) ? value : undefined;
+    };
+    const visit = (value) => {
+        if (!value || typeof value !== "object" || visited.has(value) || inspected >= 20_000)
+            return;
+        visited.add(value);
+        inspected += 1;
+        if (Array.isArray(value)) {
+            value.forEach(visit);
+            return;
+        }
+        const record = value;
+        const videoId = scalar(record, ["video_id", "videoId", "aweme_id", "awemeId", "item_id", "itemId"]);
+        if (/^\d{8,}$/.test(videoId)) {
+            const sourceCandidate = scalar(record, ["video_url", "videoUrl", "play_url", "playUrl", "download_url", "downloadUrl", "share_url", "shareUrl"]);
+            const thumbnailCandidate = scalar(record, ["cover_url", "coverUrl", "cover", "poster", "origin_cover", "originCover"]);
+            const metrics = Object.fromEntries(Object.entries({
+                sales: numeric(record, ["sales", "sold_count", "soldCount", "sale_count", "saleCount"]),
+                gmv: numeric(record, ["gmv", "sale_amount", "saleAmount", "sales_amount", "salesAmount"]),
+                views: numeric(record, ["views", "view_count", "viewCount", "play_count", "playCount"]),
+                likes: numeric(record, ["likes", "like_count", "likeCount", "digg_count", "diggCount"]),
+                comments: numeric(record, ["comments", "comment_count", "commentCount"]),
+                roas: numeric(record, ["roas", "ROAS"]),
+            }).filter((entry) => entry[1] != null));
+            const next = {
+                videoId,
+                authorId: scalar(record, ["author_unique_id", "authorUniqueId", "unique_id", "uniqueId", "author_id", "authorId"]) || undefined,
+                title: scalar(record, ["title", "video_title", "videoTitle", "desc", "description", "content", "text"]) || undefined,
+                sourceUrl: isAnalyzableVideoUrl(sourceCandidate) ? sourceCandidate : undefined,
+                thumbnailUrl: /^https?:\/\//i.test(thumbnailCandidate) ? thumbnailCandidate : undefined,
+                metrics,
+            };
+            output.set(videoId, { ...output.get(videoId), ...next });
+        }
+        Object.values(record).forEach(visit);
+    };
+    visit(payload);
+    return [...output.values()];
+}
+export function videoLearningSourcesFromObservedRecords(records, context) {
+    return records.map((record, index) => ({
+        source: "fastmoss-agent",
+        platform: "tiktok",
+        platformVideoId: record.videoId,
+        sourceUrl: record.sourceUrl || `https://www.tiktok.com/@_/video/${record.videoId}`,
+        market: context.market,
+        rankingDimension: context.dimension,
+        rank: index + 1,
+        title: record.title || `TikTok video ${record.videoId}`,
+        productTitle: record.title || "",
+        thumbnailUrl: record.thumbnailUrl || null,
+        metrics: record.metrics || {},
+        observedAt: context.observedAt,
+    }));
+}
 function videoIdFromValues(values) {
     for (const value of values) {
         const direct = value.match(/(?:video|aweme|item|material)[/=_-](\d{8,})/i)?.[1];
@@ -808,6 +980,21 @@ export function isAnalyzableVideoUrl(value) {
         return false;
     }
 }
+export function isFastMossVideoDetailUrl(value) {
+    try {
+        const url = new URL(value);
+        return /(?:^|\.)fastmoss\.com$/i.test(url.hostname) && /\/media-source\/video\/\d+(?:\/|$)/i.test(url.pathname);
+    }
+    catch {
+        return false;
+    }
+}
+export function publicTikTokVideoUrlFromFastMoss(value) {
+    if (!isFastMossVideoDetailUrl(value))
+        return "";
+    const id = new URL(value).pathname.match(/\/media-source\/video\/(\d+)(?:\/|$)/i)?.[1];
+    return id ? `https://www.tiktok.com/@_/video/${id}` : "";
+}
 export function videoLearningSourcesFromItems(items, context) {
     const output = [];
     const seen = new Set();
@@ -815,7 +1002,8 @@ export function videoLearningSourcesFromItems(items, context) {
         const mediaUrl = item.media.find((value) => isAnalyzableVideoUrl(value)) || "";
         const publicVideoUrl = firstHttp(item.links, /tiktok\.com\/.*(?:video|v)\//i);
         const traceUrl = firstHttp(item.links, /video|aweme|creative|material/i);
-        const sourceUrl = mediaUrl || publicVideoUrl || traceUrl;
+        const publicTraceUrl = publicTikTokVideoUrlFromFastMoss(traceUrl);
+        const sourceUrl = mediaUrl || publicVideoUrl || publicTraceUrl;
         const platformVideoId = videoIdFromValues([sourceUrl, ...item.links, ...item.media]) || stableId(`${context.dimension}|${item.text}|${item.images[0] || ""}`);
         if (seen.has(platformVideoId))
             continue;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@xiaohhhh1/canvas-agent",
-  "version": "0.4.25",
+  "version": "0.4.27",
   "type": "module",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",