@web-auto/webauto 0.1.1 → 0.1.2

package/dist/modules/workflow/blocks/helpers/xhsNoteOrganizer.js

@@ -0,0 +1,140 @@
+import path from 'node:path';
+import { promises as fs } from 'node:fs';
+import { isImageFile, ocrImagesWithMacPlugin, pickMacOcrLanguages } from './macosVisionOcrPlugin.js';
+function readLineValue(md, prefix) {
+    const re = new RegExp(`^${prefix}\\s*(.+)$`, 'm');
+    const m = md.match(re);
+    return m ? String(m[1] ?? '').trim() : null;
+}
+export async function organizeOneNote(options) {
+    const noteDir = options.noteDir;
+    const noteId = options.noteId;
+    const keyword = options.keyword;
+    const contentPath = path.join(noteDir, 'content.md');
+    const commentsPath = path.join(noteDir, 'comments.md');
+    const imagesDir = path.join(noteDir, 'images');
+    const ocrPath = path.join(noteDir, 'ocr.md');
+    const mergedPath = path.join(noteDir, 'merged.md');
+    const content = await fs.readFile(contentPath, 'utf-8');
+    const comments = await fs.readFile(commentsPath, 'utf-8');
+    const link = readLineValue(content, '- 链接:') || undefined;
+    let imageFiles = [];
+    try {
+        const ents = await fs.readdir(imagesDir, { withFileTypes: true });
+        imageFiles = ents
+            .filter((e) => e.isFile())
+            .map((e) => e.name)
+            .filter((n) => isImageFile(n))
+            .sort();
+    }
+    catch {
+        imageFiles = [];
+    }
+    const picked = pickMacOcrLanguages(options.ocrLanguages);
+    const ocrLanguagesUsed = picked.languages;
+    let ocrErrors = 0;
+    let ocrMd = null;
+    const runOcr = options.runOcr !== false;
+    if (runOcr) {
+        const absImages = imageFiles.map((img) => path.join(imagesDir, img));
+        const ocr = imageFiles.length
+            ? await ocrImagesWithMacPlugin(absImages, { languages: ocrLanguagesUsed, timeoutMs: 120_000 })
+            : { languagesUsed: ocrLanguagesUsed, results: [], binPath: '' };
+        const byImage = new Map();
+        for (const r of ocr.results) {
+            byImage.set(String(r.image || '').trim(), { text: r.text, error: r.error });
+        }
+        const lines = [];
+        lines.push(`# OCR（${imageFiles.length}）`);
+        lines.push('');
+        lines.push(`- Note ID: ${noteId}`);
+        lines.push(`- 关键词: ${keyword}`);
+        if (link)
+            lines.push(`- 链接: ${link}`);
+        lines.push(`- OCR 语言: ${ocrLanguagesUsed}`);
+        lines.push('');
+        if (imageFiles.length === 0) {
+            lines.push('（无图片或 images/ 目录不存在）');
+            lines.push('');
+        }
+        else {
+            for (const img of imageFiles) {
+                const imgPath = path.join(imagesDir, img);
+                const rel = `images/${img}`;
+                const res = byImage.get(imgPath) || {};
+                lines.push(`## ${rel}`);
+                lines.push('');
+                if (res.error) {
+                    ocrErrors += 1;
+                    lines.push(`> OCR 失败: ${res.error}`);
+                    lines.push('');
+                }
+                const text = String(res.text || '').trim();
+                if (text) {
+                    lines.push('```');
+                    lines.push(text);
+                    lines.push('```');
+                }
+                else {
+                    lines.push('（无可识别文字）');
+                }
+                lines.push('');
+            }
+        }
+        ocrMd = `${lines.join('\n')}\n`;
+        await fs.writeFile(ocrPath, ocrMd, 'utf-8');
+    }
+    else {
+        try {
+            ocrMd = await fs.readFile(ocrPath, 'utf-8');
+        }
+        catch {
+            if (options.requireExistingOcr) {
+                throw new Error(`missing ocr.md (requireExistingOcr=true): ${ocrPath}`);
+            }
+            const lines = [];
+            lines.push(`# OCR（${imageFiles.length}）`);
+            lines.push('');
+            lines.push(`- Note ID: ${noteId}`);
+            lines.push(`- 关键词: ${keyword}`);
+            if (link)
+                lines.push(`- 链接: ${link}`);
+            lines.push(`- OCR 语言: ${ocrLanguagesUsed}`);
+            lines.push(`- OCR: skipped`);
+            lines.push('');
+            ocrMd = `${lines.join('\n')}\n`;
+            await fs.writeFile(ocrPath, ocrMd, 'utf-8');
+        }
+    }
+    const mergedLines = [];
+    mergedLines.push(`# 合并（${noteId}）`);
+    mergedLines.push('');
+    mergedLines.push(`- Note ID: ${noteId}`);
+    mergedLines.push(`- 关键词: ${keyword}`);
+    if (link)
+        mergedLines.push(`- 链接: ${link}`);
+    mergedLines.push('');
+    mergedLines.push('## 正文');
+    mergedLines.push('');
+    mergedLines.push(content.trim());
+    mergedLines.push('');
+    mergedLines.push('## OCR');
+    mergedLines.push('');
+    mergedLines.push(String(ocrMd || '').trim());
+    mergedLines.push('');
+    mergedLines.push('## 评论');
+    mergedLines.push('');
+    mergedLines.push(comments.trim());
+    mergedLines.push('');
+    await fs.writeFile(mergedPath, `${mergedLines.join('\n')}\n`, 'utf-8');
+    return {
+        noteId,
+        ocrPath,
+        mergedPath,
+        imageCount: imageFiles.length,
+        ocrErrors,
+        ocrLanguagesUsed,
+        ...(link ? { link } : {}),
+    };
+}
+//# sourceMappingURL=xhsNoteOrganizer.js.map

package/dist/modules/workflow/blocks/restore/RestorePhaseBlock.js

@@ -0,0 +1,204 @@
+/**
+ * RestorePhaseBlock
+ *
+ * 基于 URL + 容器匹配恢复到指定阶段（Phase 入口锚点），用于 Phase1–4 的“回归锚点”能力。
+ *
+ * 约定：
+ * - phase=1: 期望在 home/login（交给 Phase1 自己处理）
+ * - phase=2: 期望在搜索入口阶段（home 或 search_result）
+ * - phase=3: 期望在搜索结果阶段（search_result）
+ * - phase=4: 期望在详情页阶段（detail）
+ *
+ * 恢复策略（小红书）：
+ * - 当前在 detail：优先使用 ErrorRecoveryBlock(ESC) 关闭模态框，回到 search_result/home
+ * - 当前在风控/未知页面：优先点击“发现页” sidebar 容器按钮（home/search.discover_button），回到首页
+ * - 当前在 login：交给上层 Phase1 处理，不在此处强行导航
+ */
+import { execute as detectPageState } from '../DetectPageStateBlock.js';
+import { execute as errorRecovery } from '../ErrorRecoveryBlock.js';
+import { logControllerActionError, logControllerActionResult, logControllerActionStart, } from '../helpers/operationLogger.js';
+function expectedStageForPhase(phase) {
+    switch (phase) {
+        case 1:
+            return ['home', 'login', 'unknown'];
+        case 2:
+            return ['home', 'search'];
+        case 3:
+            return ['search'];
+        case 4:
+            return ['detail'];
+        default:
+            return ['unknown'];
+    }
+}
+function choosePhaseFromStage(stage) {
+    if (stage === 'detail')
+        return 4;
+    if (stage === 'search')
+        return 3;
+    if (stage === 'home')
+        return 2;
+    if (stage === 'login')
+        return 1;
+    return 1;
+}
+export async function execute(input) {
+    const { sessionId, phase, serviceUrl = 'http://127.0.0.1:7701' } = input;
+    const controllerUrl = `${serviceUrl}/v1/controller/action`;
+    async function controllerAction(action, payload = {}) {
+        const opId = logControllerActionStart(action, payload, { source: 'RestorePhaseBlock' });
+        try {
+            const res = await fetch(controllerUrl, {
+                method: 'POST',
+                headers: { 'Content-Type': 'application/json' },
+                body: JSON.stringify({ action, payload }),
+            });
+            if (!res.ok) {
+                throw new Error(`HTTP ${res.status}: ${await res.text()}`);
+            }
+            const data = await res.json();
+            const result = data.data || data;
+            logControllerActionResult(opId, action, result, { source: 'RestorePhaseBlock' });
+            return result;
+        }
+        catch (error) {
+            logControllerActionError(opId, action, error, payload, { source: 'RestorePhaseBlock' });
+            throw error;
+        }
+    }
+    async function clickDiscoverButton() {
+        const candidates = [
+            'xiaohongshu_home.discover_button',
+            'xiaohongshu_search.discover_button',
+        ];
+        for (const containerId of candidates) {
+            try {
+                await controllerAction('container:operation', {
+                    containerId,
+                    operationId: 'click',
+                    config: { useSystemMouse: true },
+                    sessionId,
+                });
+                // 给页面一点时间完成导航
+                await new Promise((resolve) => setTimeout(resolve, 1500));
+                return { success: true, method: `discover:${containerId}` };
+            }
+            catch {
+                // 尝试下一个候选
+            }
+        }
+        return { success: false };
+    }
+    try {
+        // 1) 检测当前阶段（入口锚点）
+        const state = await detectPageState({
+            sessionId,
+            platform: 'xiaohongshu',
+            serviceUrl,
+        });
+        const expectedStages = expectedStageForPhase(phase);
+        if (state.success && expectedStages.includes(state.stage)) {
+            // 已经在本 Phase 允许的阶段，不做任何操作
+            return {
+                success: true,
+                restored: false,
+                finalPhase: phase,
+                finalStage: state.stage,
+                url: state.url,
+                method: 'already-at-expected-stage',
+                error: state.error,
+            };
+        }
+        let currentStage = state.stage;
+        let currentUrl = state.url;
+        let method;
+        // 2) 针对不同阶段执行恢复策略
+        if (currentStage === 'detail') {
+            // 使用 ErrorRecoveryBlock 的 ESC 模式关闭详情模态框
+            const rec = await errorRecovery({
+                sessionId,
+                fromStage: 'detail',
+                targetStage: 'search',
+                serviceUrl,
+                recoveryMode: 'esc',
+                maxRetries: 2,
+            });
+            if (!rec.success) {
+                return {
+                    success: false,
+                    restored: false,
+                    finalPhase: phase,
+                    finalStage: currentStage,
+                    url: rec.currentUrl || currentUrl,
+                    method: rec.method,
+                    error: rec.error || 'ErrorRecoveryBlock failed',
+                };
+            }
+            method = `errorRecovery:${rec.method || 'esc'}`;
+            currentUrl = rec.currentUrl || currentUrl;
+            const after = await detectPageState({
+                sessionId,
+                platform: 'xiaohongshu',
+                serviceUrl,
+            });
+            currentStage = after.stage;
+            currentUrl = after.url;
+        }
+        else if (currentStage === 'login') {
+            // 登录页交给 Phase1 处理，不在此处强行导航
+            return {
+                success: false,
+                restored: false,
+                finalPhase: 1,
+                finalStage: 'login',
+                url: state.url,
+                method: 'login-delegate-phase1',
+                error: 'Current stage is login, please run Phase1 to complete login',
+            };
+        }
+        else {
+            // 风控/未知/其他页面：尝试点击发现页 sidebar 容器按钮
+            const clickRes = await clickDiscoverButton();
+            if (!clickRes.success) {
+                return {
+                    success: false,
+                    restored: false,
+                    finalPhase: phase,
+                    finalStage: currentStage,
+                    url: currentUrl,
+                    method: 'discover-button-failed',
+                    error: 'Failed to click discover button to return home',
+                };
+            }
+            method = clickRes.method;
+            const after = await detectPageState({
+                sessionId,
+                platform: 'xiaohongshu',
+                serviceUrl,
+            });
+            currentStage = after.stage;
+            currentUrl = after.url;
+        }
+        const finalPhase = choosePhaseFromStage(currentStage);
+        return {
+            success: true,
+            restored: true,
+            finalPhase,
+            finalStage: currentStage,
+            url: currentUrl,
+            method,
+            error: state.error,
+        };
+    }
+    catch (error) {
+        return {
+            success: false,
+            restored: false,
+            finalPhase: phase,
+            finalStage: 'unknown',
+            url: '',
+            error: error?.message || String(error),
+        };
+    }
+}
+//# sourceMappingURL=RestorePhaseBlock.js.map

package/dist/modules/workflow/config/workflowRegistry.js

@@ -0,0 +1,32 @@
+import { xiaohongshuCollectWorkflowV2 } from '../definitions/xiaohongshu-collect-workflow-v2.js';
+import { xiaohongshuCollectWorkflow } from '../definitions/xiaohongshu-collect-workflow.js';
+import { xiaohongshuNoteCollectWorkflow } from '../definitions/xiaohongshu-note-collect.js';
+import { xiaohongshuFullCollectWorkflowV3 } from '../definitions/xiaohongshu-full-collect-workflow-v3.js';
+import { xiaohongshuPhase3CollectWorkflowV1 } from '../definitions/xiaohongshu-phase3-collect-workflow-v1.js';
+import { xiaohongshuPhase1WorkflowV3 } from '../definitions/xiaohongshu-phase1-workflow-v3.js';
+import { xiaohongshuPhase2LinksWorkflowV3 } from '../definitions/xiaohongshu-phase2-links-workflow-v3.js';
+import { xiaohongshuPhase34FromLinksWorkflowV3 } from '../definitions/xiaohongshu-phase34-from-links-workflow-v3.js';
+const registry = new Map();
+function register(def, idOverride) {
+    const id = idOverride || def.id;
+    if (!id) {
+        throw new Error('Workflow definition missing id');
+    }
+    registry.set(id, { ...def, id });
+}
+// 注册已有的小红书相关 workflow 定义
+register(xiaohongshuCollectWorkflowV2, 'xiaohongshu-collect-v2');
+register(xiaohongshuCollectWorkflow, 'xiaohongshu-collect-v1');
+register(xiaohongshuNoteCollectWorkflow, 'xiaohongshu-note-collect');
+register(xiaohongshuFullCollectWorkflowV3, 'xiaohongshu-collect-full-v3');
+register(xiaohongshuPhase3CollectWorkflowV1, 'xiaohongshu-collect-phase3-v1');
+register(xiaohongshuPhase1WorkflowV3, 'xiaohongshu-phase1-v3');
+register(xiaohongshuPhase2LinksWorkflowV3, 'xiaohongshu-phase2-links-v3');
+register(xiaohongshuPhase34FromLinksWorkflowV3, 'xiaohongshu-phase34-from-links-v3');
+export function getWorkflowDefinition(id) {
+    return registry.get(id);
+}
+export function listWorkflowIds() {
+    return Array.from(registry.keys());
+}
+//# sourceMappingURL=workflowRegistry.js.map

package/dist/modules/workflow/definitions/batch-collect-workflow.js

@@ -0,0 +1,63 @@
+/**
+ * 批量采集 Workflow 定义
+ *
+ * 采集150条微博并生成 Markdown
+ */
+export const batchCollectWorkflow = {
+    id: 'weibo-batch-collect-150',
+    name: '微博批量采集150条',
+    config: {
+        targetCount: 150,
+        scrollDistance: 800,
+        stableWait: 10000,
+        outputPath: 'output/weibo/collect-150.md'
+    },
+    steps: [
+        {
+            blockName: 'StartBrowserService',
+            input: {
+                host: '127.0.0.1',
+                port: 7704
+            }
+        },
+        {
+            blockName: 'EnsureSession',
+            input: {
+                profileId: 'weibo_fresh',
+                url: 'https://weibo.com'
+            }
+        },
+        {
+            blockName: 'InitAutoScroll',
+            input: {
+                sessionId: '$sessionId',
+                scrollDistance: 800
+            }
+        },
+        {
+            blockName: 'CollectBatch',
+            input: {
+                sessionId: '$sessionId',
+                targetCount: 150,
+                containerSelector: '[class*="Feed_retweated"]',
+                scrollDistance: 800,
+                stableWait: 10000
+            }
+        },
+        {
+            blockName: 'RenderMarkdown',
+            input: {
+                posts: '$collectedPosts',
+                template: 'default'
+            }
+        },
+        {
+            blockName: 'SaveFile',
+            input: {
+                content: '$markdown',
+                path: 'output/weibo/collect-150.md'
+            }
+        }
+    ]
+};
+//# sourceMappingURL=batch-collect-workflow.js.map

package/dist/modules/workflow/definitions/scroll-extract-workflow.js

@@ -0,0 +1,74 @@
+/**
+ * 完整滚动+提取测试 Workflow 定义
+ *
+ * 组合 blocks 实现端到端数据采集
+ */
+export const scrollExtractWorkflow = {
+    id: 'weibo-scroll-extract',
+    name: '微博滚动+提取完整流程',
+    steps: [
+        {
+            blockName: 'StartBrowserService',
+            input: {
+                host: '127.0.0.1',
+                port: 7704,
+                wsPort: 8765
+            }
+        },
+        {
+            blockName: 'EnsureSession',
+            input: {
+                profileId: 'weibo_fresh',
+                url: 'https://weibo.com',
+                serviceUrl: 'http://127.0.0.1:7704'
+            }
+        },
+        {
+            blockName: 'InitAutoScroll',
+            input: {
+                sessionId: '$sessionId',
+                scrollStrategy: 'smooth',
+                scrollDistance: 800
+            }
+        },
+        {
+            blockName: 'ScrollNextBatch',
+            input: {
+                sessionId: '$sessionId',
+                distance: 800,
+                behavior: 'smooth'
+            }
+        },
+        {
+            blockName: 'WaitStable',
+            input: {
+                sessionId: '$sessionId',
+                checkInterval: 500,
+                maxWait: 10000
+            }
+        },
+        {
+            blockName: 'MatchContainers',
+            input: {
+                sessionId: '$sessionId',
+                rootSelector: 'body'
+            }
+        },
+        {
+            blockName: 'ExtractPostFields',
+            input: {
+                sessionId: '$sessionId',
+                containerSelector: '[class*="Feed_retweeted"]',
+                index: 0
+            }
+        },
+        {
+            blockName: 'ValidateExtract',
+            input: {
+                fields: '$fields',
+                requiredFields: ['author', 'content']
+            }
+        }
+    ]
+};
+//# sourceMappingURL=scroll-extract-workflow.js.map

package/dist/modules/workflow/definitions/xiaohongshu-collect-workflow-v2.js

@@ -0,0 +1,81 @@
+/**
+ * 小红书关键词采集 Workflow v2（容器驱动版）
+ *
+ * 新增：
+ * - WaitSearchPermitBlock：在执行搜索前先向 SearchGate 申请许可
+ */
+export const xiaohongshuCollectWorkflowV2 = {
+    id: 'xiaohongshu-collect-v2',
+    name: '小红书关键词采集（容器驱动版）',
+    steps: [
+        {
+            blockName: 'EnsureSession',
+            input: {
+                profileId: '$sessionId',
+                url: 'https://www.xiaohongshu.com'
+            }
+        },
+        {
+            blockName: 'EnsureLoginBlock',
+            input: {
+                sessionId: '$sessionId',
+                maxWaitMs: 180000,
+                checkIntervalMs: 5000
+            }
+        },
+        {
+            blockName: 'WaitSearchPermitBlock',
+            input: {
+                sessionId: '$sessionId',
+                keyword: '$keyword',
+            }
+        },
+        {
+            blockName: 'GoToSearchBlock',
+            input: {
+                sessionId: '$sessionId',
+                keyword: '$keyword'
+            }
+        },
+        {
+            blockName: 'CollectSearchListBlock',
+            input: {
+                sessionId: '$sessionId',
+                targetCount: '$targetCount'
+            }
+        },
+        {
+            blockName: 'OpenDetailBlock',
+            input: {
+                sessionId: '$sessionId',
+                containerId: '$firstItemContainerId'
+            }
+        },
+        {
+            blockName: 'ExtractDetailBlock',
+            input: {
+                sessionId: '$sessionId'
+            }
+        },
+        {
+            blockName: 'WarmupCommentsBlock',
+            input: {
+                sessionId: '$sessionId',
+                maxRounds: 8
+            }
+        },
+        {
+            blockName: 'ExpandCommentsBlock',
+            input: {
+                sessionId: '$sessionId'
+            }
+        },
+        {
+            blockName: 'CloseDetailBlock',
+            input: {
+                sessionId: '$sessionId'
+            }
+        }
+    ]
+};
+//# sourceMappingURL=xiaohongshu-collect-workflow-v2.js.map

package/dist/modules/workflow/definitions/xiaohongshu-collect-workflow.js

@@ -0,0 +1,57 @@
+/**
+ * 小红书采集 Workflow 定义
+ */
+export const xiaohongshuCollectWorkflow = {
+    id: 'xiaohongshu-collect',
+    name: '小红书关键词采集（已迁移到 v3 安全链路）',
+    steps: [
+        {
+            blockName: 'StartBrowserService',
+            input: {
+                host: '127.0.0.1',
+                port: 7704
+            }
+        },
+        {
+            blockName: 'EnsureSession',
+            input: {
+                profileId: '$sessionId',
+                url: 'https://www.xiaohongshu.com'
+            }
+        },
+        {
+            blockName: 'EnsureLoginBlock',
+            input: {
+                sessionId: '$sessionId',
+                maxWaitMs: 180000,
+                checkIntervalMs: 5000,
+            },
+        },
+        {
+            blockName: 'WaitSearchPermitBlock',
+            input: {
+                sessionId: '$sessionId',
+                keyword: '$keyword',
+            },
+        },
+        {
+            blockName: 'GoToSearchBlock',
+            input: {
+                sessionId: '$sessionId',
+                keyword: '$keyword',
+            },
+        },
+        {
+            blockName: 'XiaohongshuFullCollectBlock',
+            input: {
+                sessionId: '$sessionId',
+                keyword: '$keyword',
+                env: '$env',
+                targetCount: '$targetCount',
+                maxWarmupRounds: '$maxWarmupRounds',
+                allowClickCommentButton: '$allowClickCommentButton',
+            },
+        },
+    ]
+};
+//# sourceMappingURL=xiaohongshu-collect-workflow.js.map