@web-auto/webauto 0.1.3 → 0.1.6

package/dist/modules/workflow/blocks/WeiboCollectSearchLinksBlock.js

@@ -0,0 +1,208 @@
+/**
+ * Weibo Search Links Collection Block
+ * Collects post links from Weibo search results
+ *
+ * Collection ID format: search:<keyword>
+ *
+ * Modes:
+ * - fresh: Clear existing data and recollect
+ * - incremental: Keep existing data and add new posts
+ */
+import { BaseBlock } from '../base';
+import { CollectionDataManager } from '../../collection-manager/index.js';
+import { ProcessRegistry } from '../../process-registry/index.js';
+import { RateLimiter } from '../../rate-limiter/index.js';
+export class WeiboCollectSearchLinksBlock extends BaseBlock {
+    name = 'weibo-collect-search-links';
+    description = 'Collect Weibo post links from search results';
+    async execute(config, context) {
+        const { profile, keyword, target, mode = 'incremental', maxPages = 0 } = config;
+        // Build collection spec
+        const spec = {
+            source: 'search',
+            keyword
+        };
+        // Initialize data manager
+        const dataManager = new CollectionDataManager({
+            platform: 'weibo',
+            env: context.env || 'debug',
+            spec,
+            mode
+        });
+        await dataManager.init();
+        const paths = dataManager.getPaths();
+        // Register process
+        const processRegistry = ProcessRegistry.getInstance();
+        const processId = processRegistry.register({
+            name: `weibo-search-${keyword}`,
+            platform: 'weibo',
+            profile,
+            metadata: { collectionId: dataManager.getCollectionId() }
+        });
+        // Get rate limiter
+        const rateLimiter = RateLimiter.getInstance();
+        const page = context.page;
+        if (!page) {
+            return {
+                success: false,
+                error: 'Page not available',
+                posts: 0,
+                linksFile: paths.linksPath,
+                collectionId: dataManager.getCollectionId(),
+                stats: {
+                    totalPosts: 0,
+                    newPosts: 0,
+                    duplicatesSkipped: 0
+                }
+            };
+        }
+        try {
+            // Request search quota
+            const searchQuota = await rateLimiter.requestQuota('search', keyword);
+            if (!searchQuota.allowed) {
+                return {
+                    success: false,
+                    error: `Search quota exceeded: ${searchQuota.reason}`,
+                    posts: 0,
+                    linksFile: paths.linksPath,
+                    collectionId: dataManager.getCollectionId(),
+                    stats: {
+                        totalPosts: 0,
+                        newPosts: 0,
+                        duplicatesSkipped: 0
+                    }
+                };
+            }
+            // Navigate to Weibo search
+            const searchUrl = `https://s.weibo.com/weibo?q=${encodeURIComponent(keyword)}`;
+            await page.goto(searchUrl, { waitUntil: 'networkidle' });
+            await page.waitForTimeout(2000);
+            // Handle login page if needed
+            const currentUrl = page.url();
+            if (currentUrl.includes('login') || currentUrl.includes('passport')) {
+                return {
+                    success: false,
+                    error: 'Login required',
+                    posts: 0,
+                    linksFile: paths.linksPath,
+                    collectionId: dataManager.getCollectionId(),
+                    stats: {
+                        totalPosts: 0,
+                        newPosts: 0,
+                        duplicatesSkipped: 0
+                    }
+                };
+            }
+            let collectedPosts = [];
+            let currentPage = 1;
+            let noNewPostsCount = 0;
+            const maxNoNewPosts = 3;
+            while (collectedPosts.length < target && noNewPostsCount < maxNoNewPosts) {
+                // Check heartbeat
+                if (!processRegistry.heartbeat(processId)) {
+                    break;
+                }
+                // Collect visible posts
+                const visiblePosts = await this.collectVisiblePosts(page);
+                let newPostsThisRound = 0;
+                for (const post of visiblePosts) {
+                    if (!dataManager.hasPost(post.id)) {
+                        await dataManager.addPost({
+                            id: post.id,
+                            url: post.url,
+                            collectedAt: new Date().toISOString()
+                        });
+                        collectedPosts.push(post);
+                        newPostsThisRound++;
+                        if (collectedPosts.length >= target)
+                            break;
+                    }
+                }
+                // Check if we found new posts
+                if (newPostsThisRound === 0) {
+                    noNewPostsCount++;
+                }
+                else {
+                    noNewPostsCount = 0;
+                }
+                // Check max pages
+                if (maxPages > 0 && currentPage >= maxPages) {
+                    break;
+                }
+                // Try to go to next page
+                if (collectedPosts.length < target && noNewPostsCount < maxNoNewPosts) {
+                    const hasNextPage = await this.goToNextPage(page, currentPage);
+                    if (!hasNextPage) {
+                        break;
+                    }
+                    currentPage++;
+                    await page.waitForTimeout(2000);
+                }
+            }
+            // Persist data
+            await dataManager.persist();
+            const stats = dataManager.getStats();
+            return {
+                success: true,
+                posts: collectedPosts.length,
+                linksFile: paths.linksPath,
+                collectionId: dataManager.getCollectionId(),
+                stats: {
+                    totalPosts: stats.totalPosts,
+                    newPosts: stats.newPosts,
+                    duplicatesSkipped: stats.duplicatesSkipped
+                }
+            };
+        }
+        finally {
+            processRegistry.unregister(processId);
+        }
+    }
+    async collectVisiblePosts(page) {
+        // Search results use .card-wrap elements
+        const posts = await page.evaluate(() => {
+            const cards = document.querySelectorAll('.card-wrap');
+            const results = [];
+            for (const card of cards) {
+                // Find post link
+                const link = card.querySelector('a[href*="/status/"]');
+                if (link) {
+                    const href = link.href;
+                    const match = href.match(/status\/(\d+)/);
+                    if (match) {
+                        results.push({
+                            id: match[1],
+                            url: href
+                        });
+                    }
+                }
+            }
+            return results;
+        });
+        return posts;
+    }
+    async goToNextPage(page, currentPage) {
+        try {
+            // Find next page link
+            const nextLink = await page.$(`a[href*="page=${currentPage + 1}"]`);
+            if (nextLink) {
+                await nextLink.click();
+                await page.waitForLoadState('networkidle');
+                return true;
+            }
+            // Try clicking next button
+            const nextBtn = await page.$('.next');
+            if (nextBtn) {
+                await nextBtn.click();
+                await page.waitForLoadState('networkidle');
+                return true;
+            }
+            return false;
+        }
+        catch {
+            return false;
+        }
+    }
+}
+export default WeiboCollectSearchLinksBlock;
+//# sourceMappingURL=WeiboCollectSearchLinksBlock.js.map

package/dist/modules/workflow/blocks/WeiboCollectTimelineListBlock.js

@@ -0,0 +1,128 @@
+/**
+ * Weibo Timeline List Collection Block
+ * Collects posts from homepage timeline feed
+ *
+ * Collection ID format: timeline:YYYY-MM-DD
+ */
+import { BaseBlock } from '../base';
+import { CollectionDataManager } from '../../collection-manager';
+import { ProcessRegistry } from '../../process-registry';
+import { RateLimiter } from '../../rate-limiter';
+export class WeiboCollectTimelineListBlock extends BaseBlock {
+    name = 'weibo-collect-timeline-list';
+    description = 'Collect Weibo timeline posts from homepage feed';
+    async execute(config, context) {
+        const { profile, target, mode = 'incremental', date } = config;
+        // Build collection spec
+        const today = date || new Date().toISOString().split('T')[0];
+        const spec = {
+            source: 'timeline',
+            date: today
+        };
+        // Initialize data manager
+        const dataManager = new CollectionDataManager({
+            platform: 'weibo',
+            env: context.env || 'debug',
+            spec,
+            mode
+        });
+        await dataManager.init();
+        const paths = dataManager.getPaths();
+        // Register process
+        const processRegistry = ProcessRegistry.getInstance();
+        const processId = processRegistry.register({
+            name: `weibo-timeline-${today}`,
+            platform: 'weibo',
+            profile,
+            metadata: { collectionId: dataManager.getCollectionId() }
+        });
+        const rateLimiter = RateLimiter.getInstance();
+        const page = context.page;
+        if (!page) {
+            return {
+                success: false,
+                error: 'Page not available',
+                posts: 0,
+                linksFile: paths.linksPath,
+                collectionId: dataManager.getCollectionId()
+            };
+        }
+        try {
+            // Navigate to Weibo homepage
+            await page.goto('https://weibo.com', { waitUntil: 'networkidle' });
+            await page.waitForTimeout(2000);
+            const collectedPosts = [];
+            let noNewPostsCount = 0;
+            const maxNoNewPosts = 3;
+            while (collectedPosts.length < target && noNewPostsCount < maxNoNewPosts) {
+                // Check heartbeat
+                if (!processRegistry.heartbeat(processId)) {
+                    break;
+                }
+                // Collect timeline posts
+                const posts = await this.collectVisiblePosts(page);
+                let newPostsThisRound = 0;
+                for (const post of posts) {
+                    if (!dataManager.hasPost(post.id)) {
+                        await dataManager.addPost({
+                            id: post.id,
+                            url: post.url,
+                            collectedAt: new Date().toISOString()
+                        });
+                        collectedPosts.push(post);
+                        newPostsThisRound++;
+                        if (collectedPosts.length >= target)
+                            break;
+                    }
+                }
+                // Check if we found new posts
+                if (newPostsThisRound === 0) {
+                    noNewPostsCount++;
+                }
+                else {
+                    noNewPostsCount = 0;
+                }
+                // Scroll to load more
+                if (collectedPosts.length < target) {
+                    await page.mouse.wheel(0, 800);
+                    await page.waitForTimeout(1500);
+                }
+            }
+            await dataManager.persist();
+            return {
+                success: true,
+                posts: collectedPosts.length,
+                linksFile: paths.linksPath,
+                collectionId: dataManager.getCollectionId()
+            };
+        }
+        finally {
+            processRegistry.unregister(processId);
+        }
+    }
+    async collectVisiblePosts(page) {
+        // Timeline posts use div[class*='_feed_'] or article
+        const posts = await page.evaluate(() => {
+            const elements = document.querySelectorAll('div[class*="_feed_"], article');
+            const results = [];
+            for (const el of elements) {
+                // Find post link
+                const link = el.querySelector('a[href*="/status/"]');
+                if (link) {
+                    const href = link.href;
+                    const match = href.match(/status\/(\d+)/);
+                    if (match) {
+                        results.push({
+                            id: match[1],
+                            url: href
+                        });
+                    }
+                }
+            }
+            return results;
+        });
+        return posts;
+    }
+}
+export default WeiboCollectTimelineListBlock;
+//# sourceMappingURL=WeiboCollectTimelineListBlock.js.map

package/dist/modules/workflow/blocks/WeiboCollectUserPostsListBlock.js

@@ -0,0 +1,127 @@
+/**
+ * Weibo User Posts List Collection Block
+ * Collects posts from a specific user's profile page
+ *
+ * Collection ID format: user:<userId>:<userName>
+ */
+import { BaseBlock } from '../base';
+import { CollectionDataManager } from '../../collection-manager';
+import { ProcessRegistry } from '../../process-registry';
+export class WeiboCollectUserPostsListBlock extends BaseBlock {
+    name = 'weibo-collect-user-posts-list';
+    description = 'Collect Weibo posts from a specific user profile';
+    async execute(config, context) {
+        const { profile, userId, userName, target, mode = 'incremental' } = config;
+        // Build collection spec
+        const spec = {
+            source: 'user',
+            userId,
+            userName
+        };
+        // Initialize data manager
+        const dataManager = new CollectionDataManager({
+            platform: 'weibo',
+            env: context.env || 'debug',
+            spec,
+            mode
+        });
+        await dataManager.init();
+        const paths = dataManager.getPaths();
+        // Register process
+        const processRegistry = ProcessRegistry.getInstance();
+        const processId = processRegistry.register({
+            name: `weibo-user-${userId}`,
+            platform: 'weibo',
+            profile,
+            metadata: { collectionId: dataManager.getCollectionId() }
+        });
+        const page = context.page;
+        if (!page) {
+            return {
+                success: false,
+                error: 'Page not available',
+                posts: 0,
+                linksFile: paths.linksPath,
+                collectionId: dataManager.getCollectionId()
+            };
+        }
+        try {
+            // Navigate to user profile
+            const userUrl = `https://weibo.com/u/${userId}`;
+            await page.goto(userUrl, { waitUntil: 'networkidle' });
+            await page.waitForTimeout(2000);
+            const collectedPosts = [];
+            let noNewPostsCount = 0;
+            const maxNoNewPosts = 3;
+            while (collectedPosts.length < target && noNewPostsCount < maxNoNewPosts) {
+                // Check heartbeat
+                if (!processRegistry.heartbeat(processId)) {
+                    break;
+                }
+                // Collect user posts
+                const posts = await this.collectVisiblePosts(page);
+                let newPostsThisRound = 0;
+                for (const post of posts) {
+                    if (!dataManager.hasPost(post.id)) {
+                        await dataManager.addPost({
+                            id: post.id,
+                            url: post.url,
+                            collectedAt: new Date().toISOString()
+                        });
+                        collectedPosts.push(post);
+                        newPostsThisRound++;
+                        if (collectedPosts.length >= target)
+                            break;
+                    }
+                }
+                // Check if we found new posts
+                if (newPostsThisRound === 0) {
+                    noNewPostsCount++;
+                }
+                else {
+                    noNewPostsCount = 0;
+                }
+                // Scroll to load more
+                if (collectedPosts.length < target) {
+                    await page.mouse.wheel(0, 800);
+                    await page.waitForTimeout(1500);
+                }
+            }
+            await dataManager.persist();
+            return {
+                success: true,
+                posts: collectedPosts.length,
+                linksFile: paths.linksPath,
+                collectionId: dataManager.getCollectionId()
+            };
+        }
+        finally {
+            processRegistry.unregister(processId);
+        }
+    }
+    async collectVisiblePosts(page) {
+        // User profile posts use div[class*='feed'] or article
+        const posts = await page.evaluate(() => {
+            const elements = document.querySelectorAll('div[class*="feed"], article');
+            const results = [];
+            for (const el of elements) {
+                // Find post link
+                const link = el.querySelector('a[href*="/status/"]');
+                if (link) {
+                    const href = link.href;
+                    const match = href.match(/status\/(\d+)/);
+                    if (match) {
+                        results.push({
+                            id: match[1],
+                            url: href
+                        });
+                    }
+                }
+            }
+            return results;
+        });
+        return posts;
+    }
+}
+export default WeiboCollectUserPostsListBlock;
+//# sourceMappingURL=WeiboCollectUserPostsListBlock.js.map

package/dist/modules/workflow/blocks/helpers/downloadPaths.js

@@ -1,5 +1,6 @@
 import os from 'node:os';
 import path from 'node:path';
+import { existsSync } from 'node:fs';
 export function sanitizeForPath(name) {
     if (!name)
         return '';
@@ -8,6 +9,26 @@ export function sanitizeForPath(name) {
 export function resolveDownloadRoot(custom, homeDir) {
     if (custom && custom.trim())
         return custom;
+    if (process.platform === 'win32') {
+        try {
+            if (existsSync('D:\\'))
+                return 'D:\\webauto';
+        }
+        catch {
+            // ignore
+        }
+        if (homeDir && homeDir.trim())
+            return path.join(homeDir, '.webauto');
+        const envHome = process.env.HOME || process.env.USERPROFILE;
+        if (envHome && envHome.trim())
+            return path.join(envHome, '.webauto');
+        try {
+            return path.join(os.homedir(), '.webauto');
+        }
+        catch {
+            return path.join(process.cwd(), '.webauto');
+        }
+    }
     if (homeDir && homeDir.trim())
         return path.join(homeDir, '.webauto', 'download');
     const envHome = process.env.HOME || process.env.USERPROFILE;

package/dist/modules/workflow/config/workflowRegistry.js

@@ -6,6 +6,7 @@ import { xiaohongshuPhase3CollectWorkflowV1 } from '../definitions/xiaohongshu-p
 import { xiaohongshuPhase1WorkflowV3 } from '../definitions/xiaohongshu-phase1-workflow-v3.js';
 import { xiaohongshuPhase2LinksWorkflowV3 } from '../definitions/xiaohongshu-phase2-links-workflow-v3.js';
 import { xiaohongshuPhase34FromLinksWorkflowV3 } from '../definitions/xiaohongshu-phase34-from-links-workflow-v3.js';
+import { weiboSearchWorkflowV1 } from '../definitions/weibo-search-workflow-v1.js'; // 导入微博搜索工作流
 const registry = new Map();
 function register(def, idOverride) {
     const id = idOverride || def.id;
@@ -23,6 +24,7 @@ register(xiaohongshuPhase3CollectWorkflowV1, 'xiaohongshu-collect-phase3-v1');
 register(xiaohongshuPhase1WorkflowV3, 'xiaohongshu-phase1-v3');
 register(xiaohongshuPhase2LinksWorkflowV3, 'xiaohongshu-phase2-links-v3');
 register(xiaohongshuPhase34FromLinksWorkflowV3, 'xiaohongshu-phase34-from-links-v3');
+register(weiboSearchWorkflowV1, 'weibo-search-v1'); // 注册微博搜索工作流
 export function getWorkflowDefinition(id) {
     return registry.get(id);
 }

package/dist/modules/workflow/definitions/weibo-search-workflow-v1.js

@@ -0,0 +1,47 @@
+/**
+ * 微博搜索工作流 v1
+ *
+ * 流程：
+ * 1) 执行搜索
+ * 2) 采集搜索结果链接（支持翻页）
+ * 3) 从链接采集详情内容和评论
+ */
+export const weiboSearchWorkflowV1 = {
+    id: 'weibo-search-v1',
+    name: '微博搜索采集 v1',
+    steps: [
+        {
+            blockName: 'ExecuteWeiboSearchBlock',
+            input: {
+                sessionId: '$sessionId',
+                keyword: '$keyword',
+                env: '$env',
+                serviceUrl: '$serviceUrl',
+            },
+        },
+        {
+            blockName: 'WeiboCollectSearchLinksBlock',
+            input: {
+                sessionId: '$sessionId',
+                keyword: '$keyword',
+                env: '$env',
+                targetCount: '$targetCount',
+                maxPages: '$maxPages',
+                serviceUrl: '$serviceUrl',
+            },
+        },
+        {
+            blockName: 'WeiboCollectFromLinksBlock',
+            input: {
+                sessionId: '$sessionId',
+                keyword: '$keyword',
+                env: '$env',
+                targetCount: '$targetCount',
+                maxComments: '$maxComments',
+                collectComments: '$collectComments',
+                serviceUrl: '$serviceUrl',
+            },
+        },
+    ],
+};
+//# sourceMappingURL=weibo-search-workflow-v1.js.map

package/dist/modules/workflow/src/runner.js

@@ -16,6 +16,9 @@ import * as XiaohongshuFullCollectBlock from '../blocks/XiaohongshuFullCollectBl
 import * as XiaohongshuCollectLinksBlock from '../blocks/XiaohongshuCollectLinksBlock.js';
 import * as XiaohongshuCollectFromLinksBlock from '../blocks/XiaohongshuCollectFromLinksBlock.js';
 import * as ErrorRecoveryBlock from '../blocks/ErrorRecoveryBlock.js';
+import * as ExecuteWeiboSearchBlock from '../blocks/ExecuteWeiboSearchBlock.js';
+import * as WeiboCollectFromLinksBlock from '../blocks/WeiboCollectFromLinksBlock.js';
+import * as WeiboCollectCommentsBlock from '../blocks/WeiboCollectCommentsBlock.js';
 import * as AnchorVerificationBlock from '../blocks/AnchorVerificationBlock.js';
 import * as CallWorkflowBlock from '../blocks/CallWorkflowBlock.js';
 import * as RecordFixtureBlock from '../blocks/RecordFixtureBlock.js';
@@ -42,6 +45,9 @@ export function createDefaultWorkflowExecutor() {
     executor.registerBlock('XiaohongshuCollectLinksBlock', { execute: XiaohongshuCollectLinksBlock.execute });
     executor.registerBlock('XiaohongshuCollectFromLinksBlock', { execute: XiaohongshuCollectFromLinksBlock.execute });
     executor.registerBlock('OrganizeXhsNotesBlock', { execute: OrganizeXhsNotesBlock.execute });
+    executor.registerBlock('ExecuteWeiboSearchBlock', { execute: ExecuteWeiboSearchBlock.execute });
+    executor.registerBlock('WeiboCollectFromLinksBlock', { execute: WeiboCollectFromLinksBlock.execute });
+    executor.registerBlock('WeiboCollectCommentsBlock', { execute: WeiboCollectCommentsBlock.execute });
     return executor;
 }
 export async function runWorkflowById(workflowId, initialContext) {

package/dist/modules/xiaohongshu/app/src/blocks/Phase34PersistDetailBlock.js

@@ -7,6 +7,7 @@
  * - 生成 README.md（含相对路径引用）
  */
 import { promises as fs } from 'node:fs';
+import { getCurrentTimestamp } from '../../../../collection-manager/date-utils.js';
 import os from 'node:os';
 import path from 'node:path';
 function resolveDownloadRoot() {
@@ -79,6 +80,9 @@ export async function execute(input) {
         lines.push(`- **Note ID**: ${noteId}`);
         lines.push(`- **作者**: ${detail.authorName || '未知'} (${detail.authorId || 'N/A'})`);
         lines.push(`- **发布时间**: ${detail.publishTime || '未知'}`);
+        const ts = getCurrentTimestamp();
+        lines.push(`- **采集时间**: ${ts.collectedAt}`);
+        lines.push(`- **采集时间(本地)**: ${ts.collectedAtLocal}`);
         lines.push(`- **原始链接**: \`https://www.xiaohongshu.com/explore/${noteId}\``);
         lines.push('');
         lines.push(`## 正文`);

package/dist/modules/xiaohongshu/app/src/blocks/Phase3InteractBlock.js

@@ -80,7 +80,7 @@ function parseLikeRuleToken(token) {
     const raw = String(token || '').trim();
     if (!raw)
         return null;
-    const m = raw.match(/^\\{\\s*(.+?)\\s*([+\\-\\uFF0B\\uFF0D])\\s*(.+?)\\s*\\}$/);
+    const m = raw.match(/^\{\s*(.+?)\s*([+\-\uFF0B\uFF0D])\s*(.+?)\s*\}$/);
     if (!m) {
         return { kind: 'contains', include: raw, raw };
     }
@@ -88,7 +88,7 @@ function parseLikeRuleToken(token) {
     const right = normalizeText(m[3]);
     if (!left || !right)
         return null;
-    const op = m[2] === '\\uFF0B' ? '+' : m[2] === '\\uFF0D' ? '-' : m[2];
+    const op = m[2] === '\uFF0B' ? '+' : m[2] === '\uFF0D' ? '-' : m[2];
     if (op === '+') {
         return { kind: 'and', includeA: left, includeB: right, raw: `{${left} + ${right}}` };
     }