@web-auto/webauto 0.1.18 → 0.1.19

package/dist/modules/workflow/blocks/WeiboCollectSearchLinksBlock.js

@@ -1,208 +1,282 @@
 /**
- * Weibo Search Links Collection Block
- * Collects post links from Weibo search results
+ * Workflow Block: WeiboCollectSearchLinksBlock
  *
- * Collection ID format: search:<keyword>
- *
- * Modes:
- * - fresh: Clear existing data and recollect
- * - incremental: Keep existing data and add new posts
+ * 阶段职责：
+ * 1) 仅从搜索结果页采集帖子链接（不做详情内容提取）
+ * 2) 持久化到 phase2-links.jsonl，供后续 WeiboCollectFromLinksBlock 消费
  */
-import { BaseBlock } from '../base';
-import { CollectionDataManager } from '../../collection-manager/index.js';
-import { ProcessRegistry } from '../../process-registry/index.js';
-import { RateLimiter } from '../../rate-limiter/index.js';
-export class WeiboCollectSearchLinksBlock extends BaseBlock {
-    name = 'weibo-collect-search-links';
-    description = 'Collect Weibo post links from search results';
-    async execute(config, context) {
-        const { profile, keyword, target, mode = 'incremental', maxPages = 0 } = config;
-        // Build collection spec
-        const spec = {
-            source: 'search',
-            keyword
-        };
-        // Initialize data manager
-        const dataManager = new CollectionDataManager({
-            platform: 'weibo',
-            env: context.env || 'debug',
-            spec,
-            mode
-        });
-        await dataManager.init();
-        const paths = dataManager.getPaths();
-        // Register process
-        const processRegistry = ProcessRegistry.getInstance();
-        const processId = processRegistry.register({
-            name: `weibo-search-${keyword}`,
-            platform: 'weibo',
-            profile,
-            metadata: { collectionId: dataManager.getCollectionId() }
-        });
-        // Get rate limiter
-        const rateLimiter = RateLimiter.getInstance();
-        const page = context.page;
-        if (!page) {
-            return {
-                success: false,
-                error: 'Page not available',
-                posts: 0,
-                linksFile: paths.linksPath,
-                collectionId: dataManager.getCollectionId(),
-                stats: {
-                    totalPosts: 0,
-                    newPosts: 0,
-                    duplicatesSkipped: 0
-                }
-            };
+import os from 'node:os';
+import path from 'node:path';
+import { promises as fs } from 'node:fs';
+function resolveDownloadRoot() {
+    const custom = process.env.WEBAUTO_DOWNLOAD_ROOT || process.env.WEBAUTO_DOWNLOAD_DIR;
+    if (custom && custom.trim())
+        return custom;
+    const home = process.env.HOME || process.env.USERPROFILE || os.homedir();
+    return path.join(home, '.webauto', 'download');
+}
+function sanitizeFilenamePart(value) {
+    return String(value || '')
+        .trim()
+        .replace(/[\\/:"*?<>|]+/g, '_')
+        .replace(/\s+/g, '_')
+        .slice(0, 80);
+}
+function extractStatusId(url) {
+    const text = String(url || '').trim();
+    if (!text)
+        return '';
+    try {
+        const parsed = new URL(text);
+        const pathname = String(parsed.pathname || '');
+        const statusMatch = pathname.match(/\/status\/([^/?#]+)/i);
+        if (statusMatch?.[1])
+            return statusMatch[1];
+        const parts = pathname.split('/').filter(Boolean);
+        if (parts.length >= 2) {
+            return String(parts[parts.length - 1] || '').trim();
         }
+    }
+    catch {
+        // ignore
+    }
+    return '';
+}
+function extractUserId(authorUrl, safeUrl) {
+    const first = String(authorUrl || '').trim();
+    const second = String(safeUrl || '').trim();
+    const fromUrl = (raw) => {
         try {
-            // Request search quota
-            const searchQuota = await rateLimiter.requestQuota('search', keyword);
-            if (!searchQuota.allowed) {
-                return {
-                    success: false,
-                    error: `Search quota exceeded: ${searchQuota.reason}`,
-                    posts: 0,
-                    linksFile: paths.linksPath,
-                    collectionId: dataManager.getCollectionId(),
-                    stats: {
-                        totalPosts: 0,
-                        newPosts: 0,
-                        duplicatesSkipped: 0
-                    }
-                };
-            }
-            // Navigate to Weibo search
-            const searchUrl = `https://s.weibo.com/weibo?q=${encodeURIComponent(keyword)}`;
-            await page.goto(searchUrl, { waitUntil: 'networkidle' });
-            await page.waitForTimeout(2000);
-            // Handle login page if needed
-            const currentUrl = page.url();
-            if (currentUrl.includes('login') || currentUrl.includes('passport')) {
-                return {
-                    success: false,
-                    error: 'Login required',
-                    posts: 0,
-                    linksFile: paths.linksPath,
-                    collectionId: dataManager.getCollectionId(),
-                    stats: {
-                        totalPosts: 0,
-                        newPosts: 0,
-                        duplicatesSkipped: 0
-                    }
-                };
-            }
-            let collectedPosts = [];
-            let currentPage = 1;
-            let noNewPostsCount = 0;
-            const maxNoNewPosts = 3;
-            while (collectedPosts.length < target && noNewPostsCount < maxNoNewPosts) {
-                // Check heartbeat
-                if (!processRegistry.heartbeat(processId)) {
-                    break;
-                }
-                // Collect visible posts
-                const visiblePosts = await this.collectVisiblePosts(page);
-                let newPostsThisRound = 0;
-                for (const post of visiblePosts) {
-                    if (!dataManager.hasPost(post.id)) {
-                        await dataManager.addPost({
-                            id: post.id,
-                            url: post.url,
-                            collectedAt: new Date().toISOString()
-                        });
-                        collectedPosts.push(post);
-                        newPostsThisRound++;
-                        if (collectedPosts.length >= target)
-                            break;
-                    }
-                }
-                // Check if we found new posts
-                if (newPostsThisRound === 0) {
-                    noNewPostsCount++;
-                }
-                else {
-                    noNewPostsCount = 0;
-                }
-                // Check max pages
-                if (maxPages > 0 && currentPage >= maxPages) {
-                    break;
-                }
-                // Try to go to next page
-                if (collectedPosts.length < target && noNewPostsCount < maxNoNewPosts) {
-                    const hasNextPage = await this.goToNextPage(page, currentPage);
-                    if (!hasNextPage) {
-                        break;
-                    }
-                    currentPage++;
-                    await page.waitForTimeout(2000);
-                }
-            }
-            // Persist data
-            await dataManager.persist();
-            const stats = dataManager.getStats();
-            return {
-                success: true,
-                posts: collectedPosts.length,
-                linksFile: paths.linksPath,
-                collectionId: dataManager.getCollectionId(),
-                stats: {
-                    totalPosts: stats.totalPosts,
-                    newPosts: stats.newPosts,
-                    duplicatesSkipped: stats.duplicatesSkipped
-                }
-            };
+            const parsed = new URL(raw);
+            const m = String(parsed.pathname || '').match(/\/u\/([0-9]+)/i);
+            if (m?.[1])
+                return m[1];
+            const parts = String(parsed.pathname || '').split('/').filter(Boolean);
+            if (parts.length > 0 && /^[0-9]+$/.test(parts[0]))
+                return parts[0];
+        }
+        catch {
+            // ignore
+        }
+        return '';
+    };
+    return fromUrl(first) || fromUrl(second) || '';
+}
+export async function execute(input) {
+    const { sessionId, keyword, env = 'debug', targetCount, maxPages = 10, serviceUrl = 'http://127.0.0.1:7704', } = input;
+    const profile = sessionId;
+    const controllerUrl = `${serviceUrl}/command`;
+    const keywordDir = path.join(resolveDownloadRoot(), 'weibo', env, sanitizeFilenamePart(keyword));
+    const linksPath = path.join(keywordDir, 'phase2-links.jsonl');
+    const maxTarget = Math.max(1, Number(targetCount || 0) || 1);
+    const maxPageCount = Math.max(1, Number(maxPages || 0) || 1);
+    async function controllerAction(action, args = {}) {
+        const res = await fetch(controllerUrl, {
+            method: 'POST',
+            headers: { 'Content-Type': 'application/json' },
+            body: JSON.stringify({ action, args: { profileId: profile, ...args } }),
+            signal: AbortSignal.timeout ? AbortSignal.timeout(30000) : undefined,
+        });
+        const raw = await res.text();
+        if (!res.ok)
+            throw new Error(`HTTP ${res.status}: ${raw}`);
+        try {
+            return raw ? JSON.parse(raw) : {};
         }
-        finally {
-            processRegistry.unregister(processId);
+        catch {
+            return { raw };
         }
     }
-    async collectVisiblePosts(page) {
-        // Search results use .card-wrap elements
-        const posts = await page.evaluate(() => {
-            const cards = document.querySelectorAll('.card-wrap');
-            const results = [];
-            for (const card of cards) {
-                // Find post link
-                const link = card.querySelector('a[href*="/status/"]');
-                if (link) {
-                    const href = link.href;
-                    const match = href.match(/status\/(\d+)/);
-                    if (match) {
-                        results.push({
-                            id: match[1],
-                            url: href
-                        });
-                    }
-                }
+    function unwrapResult(response) {
+        if (response && typeof response === 'object') {
+            if ('result' in response)
+                return response.result;
+            if (response.data && typeof response.data === 'object' && 'result' in response.data) {
+                return response.data.result;
             }
-            return results;
-        });
-        return posts;
+            if ('data' in response)
+                return response.data;
+        }
+        return response;
     }
-    async goToNextPage(page, currentPage) {
-        try {
-            // Find next page link
-            const nextLink = await page.$(`a[href*="page=${currentPage + 1}"]`);
-            if (nextLink) {
-                await nextLink.click();
-                await page.waitForLoadState('networkidle');
-                return true;
+    async function readSearchRows() {
+        const script = `
+      (() => {
+        const clean = (value) => String(value || '').replace(/\\s+/g, ' ').trim();
+        const abs = (value) => {
+          try {
+            return new URL(String(value || ''), location.href).toString();
+          } catch {
+            return '';
+          }
+        };
+        const rows = [];
+        const cards = Array.from(document.querySelectorAll('.card-wrap'));
+        for (const card of cards) {
+          const statusAnchor = card.querySelector('a[href*="/status/"]');
+          const safeUrl = abs(statusAnchor?.getAttribute?.('href') || statusAnchor?.href || '');
+          if (!safeUrl) continue;
+          const authorAnchor =
+            card.querySelector('a[href*="/u/"]') ||
+            card.querySelector('a[title][href*="weibo.com"]') ||
+            card.querySelector('.name a');
+          const authorUrl = abs(authorAnchor?.getAttribute?.('href') || authorAnchor?.href || '');
+          const authorName = clean(
+            authorAnchor?.getAttribute?.('title') ||
+            authorAnchor?.textContent ||
+            '',
+          );
+          const contentEl =
+            card.querySelector('.txt') ||
+            card.querySelector('[node-type="feed_list_content"]') ||
+            card.querySelector('.detail_wbtext') ||
+            card.querySelector('.wbtext');
+          const contentPreview = clean(contentEl?.textContent || '').slice(0, 180);
+          rows.push({
+            safeUrl,
+            authorUrl,
+            authorName,
+            contentPreview,
+          });
+        }
+        return { rows, searchUrl: String(location.href || '') };
+      })()
+    `;
+        const res = await controllerAction('evaluate', { script });
+        const value = unwrapResult(res);
+        return {
+            rows: Array.isArray(value?.rows) ? value.rows : [],
+            searchUrl: String(value?.searchUrl || ''),
+        };
+    }
+    async function findNextPageCenter() {
+        const script = `
+      (() => {
+        const clean = (value) => String(value || '').replace(/\\s+/g, ' ').trim();
+        const isVisible = (node) => {
+          if (!(node instanceof HTMLElement)) return false;
+          const rect = node.getBoundingClientRect();
+          if (rect.width <= 0 || rect.height <= 0) return false;
+          if (rect.bottom <= 0 || rect.top >= window.innerHeight) return false;
+          const style = window.getComputedStyle(node);
+          if (!style) return false;
+          if (style.display === 'none' || style.visibility === 'hidden') return false;
+          if (Number(style.opacity || '1') === 0) return false;
+          return true;
+        };
+        const isDisabled = (node) => {
+          const text = clean(node.className || '');
+          if (text.includes('disable') || text.includes('disabled')) return true;
+          const ariaDisabled = String(node.getAttribute('aria-disabled') || '').toLowerCase();
+          return ariaDisabled === 'true';
+        };
+        const candidates = Array.from(document.querySelectorAll('a, button, span'))
+          .filter((node) => /下一页|下页/.test(clean(node.textContent || '')))
+          .filter((node) => isVisible(node) && !isDisabled(node));
+        const target = candidates[0];
+        if (!target) return { ok: false };
+        const rect = target.getBoundingClientRect();
+        return {
+          ok: true,
+          x: Math.round(rect.left + rect.width / 2),
+          y: Math.round(rect.top + rect.height / 2),
+        };
+      })()
+    `;
+        const res = await controllerAction('evaluate', { script });
+        const value = unwrapResult(res);
+        return {
+            ok: value?.ok === true,
+            x: Number(value?.x || 0),
+            y: Number(value?.y || 0),
+        };
+    }
+    async function writeLinks(rows) {
+        await fs.mkdir(keywordDir, { recursive: true });
+        const jsonl = rows.map((row) => JSON.stringify(row)).join('\n');
+        await fs.writeFile(linksPath, jsonl ? `${jsonl}\n` : '', 'utf8');
+    }
+    const dedup = new Map();
+    let pagesVisited = 0;
+    let noProgressRounds = 0;
+    try {
+        for (let page = 0; page < maxPageCount; page += 1) {
+            pagesVisited += 1;
+            const snapshot = await readSearchRows();
+            let added = 0;
+            for (const row of snapshot.rows) {
+                const safeUrl = String(row?.safeUrl || '').trim();
+                if (!safeUrl)
+                    continue;
+                const statusId = extractStatusId(safeUrl);
+                if (!statusId)
+                    continue;
+                const key = statusId || safeUrl;
+                if (dedup.has(key))
+                    continue;
+                dedup.set(key, {
+                    statusId,
+                    userId: extractUserId(String(row?.authorUrl || ''), safeUrl),
+                    safeUrl,
+                    searchUrl: String(snapshot.searchUrl || ''),
+                    authorName: String(row?.authorName || '').trim() || undefined,
+                    contentPreview: String(row?.contentPreview || '').trim() || undefined,
+                    ts: new Date().toISOString(),
+                });
+                added += 1;
+                if (dedup.size >= maxTarget)
+                    break;
             }
-            // Try clicking next button
-            const nextBtn = await page.$('.next');
-            if (nextBtn) {
-                await nextBtn.click();
-                await page.waitForLoadState('networkidle');
-                return true;
+            if (added === 0)
+                noProgressRounds += 1;
+            else
+                noProgressRounds = 0;
+            if (dedup.size >= maxTarget)
+                break;
+            if (noProgressRounds >= 2)
+                break;
+            let clicked = false;
+            for (let attempt = 0; attempt < 3; attempt += 1) {
+                const next = await findNextPageCenter();
+                if (!next.ok || !Number.isFinite(next.x) || !Number.isFinite(next.y)) {
+                    await controllerAction('mouse:wheel', { deltaX: 0, deltaY: 900 }).catch(() => null);
+                    await new Promise((r) => setTimeout(r, 400));
+                    continue;
+                }
+                await controllerAction('mouse:click', {
+                    x: Math.round(next.x),
+                    y: Math.round(next.y),
+                    button: 'left',
+                    clicks: 1,
+                });
+                clicked = true;
+                await new Promise((r) => setTimeout(r, 1500));
+                break;
             }
-            return false;
-        }
-        catch {
-            return false;
+            if (!clicked)
+                break;
         }
+        const links = Array.from(dedup.values()).slice(0, maxTarget);
+        await writeLinks(links);
+        return {
+            success: true,
+            keywordDir,
+            linksPath,
+            collectedCount: links.length,
+            pagesVisited,
+        };
+    }
+    catch (error) {
+        const links = Array.from(dedup.values()).slice(0, maxTarget);
+        await writeLinks(links).catch(() => null);
+        return {
+            success: false,
+            keywordDir,
+            linksPath,
+            collectedCount: links.length,
+            pagesVisited,
+            error: `WeiboCollectSearchLinks failed: ${error.message}`,
+        };
     }
 }
-export default WeiboCollectSearchLinksBlock;
 //# sourceMappingURL=WeiboCollectSearchLinksBlock.js.map

package/dist/modules/workflow/definitions/weibo-search-workflow-v1.js

@@ -39,6 +39,8 @@ export const weiboSearchWorkflowV1 = {
                 targetCount: '$targetCount',
                 maxComments: '$maxComments',
                 collectComments: '$collectComments',
+                tabCount: '$tabCount',
+                tabOpenDelayMs: '$tabOpenDelayMs',
                 serviceUrl: '$serviceUrl',
             },
         },

package/dist/modules/workflow/src/runner.js

@@ -17,6 +17,7 @@ import * as XiaohongshuCollectLinksBlock from '../blocks/XiaohongshuCollectLinks
 import * as XiaohongshuCollectFromLinksBlock from '../blocks/XiaohongshuCollectFromLinksBlock.js';
 import * as ErrorRecoveryBlock from '../blocks/ErrorRecoveryBlock.js';
 import * as ExecuteWeiboSearchBlock from '../blocks/ExecuteWeiboSearchBlock.js';
+import * as WeiboCollectSearchLinksBlock from '../blocks/WeiboCollectSearchLinksBlock.js';
 import * as WeiboCollectFromLinksBlock from '../blocks/WeiboCollectFromLinksBlock.js';
 import * as WeiboCollectCommentsBlock from '../blocks/WeiboCollectCommentsBlock.js';
 import * as AnchorVerificationBlock from '../blocks/AnchorVerificationBlock.js';
@@ -46,6 +47,7 @@ export function createDefaultWorkflowExecutor() {
     executor.registerBlock('XiaohongshuCollectFromLinksBlock', { execute: XiaohongshuCollectFromLinksBlock.execute });
     executor.registerBlock('OrganizeXhsNotesBlock', { execute: OrganizeXhsNotesBlock.execute });
     executor.registerBlock('ExecuteWeiboSearchBlock', { execute: ExecuteWeiboSearchBlock.execute });
+    executor.registerBlock('WeiboCollectSearchLinksBlock', { execute: WeiboCollectSearchLinksBlock.execute });
     executor.registerBlock('WeiboCollectFromLinksBlock', { execute: WeiboCollectFromLinksBlock.execute });
     executor.registerBlock('WeiboCollectCommentsBlock', { execute: WeiboCollectCommentsBlock.execute });
     return executor;