@web-auto/webauto 0.1.1 → 0.1.2

package/dist/modules/xiaohongshu/app/src/blocks/Phase34ProcessSingleNoteBlock.js

@@ -0,0 +1,114 @@
+/**
+ * Phase 3-4 Block: 处理单条笔记（详情 + 评论）
+ *
+ * 职责：
+ * - 打开详情页（使用 safeUrl）
+ * - 提取详情内容
+ * - 采集评论（支持分批）
+ * - 持久化结果
+ * - 返回搜索页
+ */
+import { execute as openDetail } from './Phase34OpenDetailBlock.js';
+import { execute as extractDetail } from './Phase34ExtractDetailBlock.js';
+import { execute as collectComments } from './Phase34CollectCommentsBlock.js';
+import { execute as persistDetail } from './Phase34PersistDetailBlock.js';
+import { execute as closeDetail } from './Phase34CloseDetailBlock.js';
+function delay(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+async function withTimeout(promise, timeoutMs, errorMessage) {
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), timeoutMs);
+    try {
+        const result = await Promise.race([
+            promise,
+            new Promise((_, reject) => {
+                controller.signal.addEventListener('abort', () => reject(new Error(errorMessage)));
+            })
+        ]);
+        clearTimeout(timer);
+        return result;
+    }
+    catch (err) {
+        clearTimeout(timer);
+        throw err;
+    }
+}
+export async function execute(input) {
+    const { noteId, safeUrl, searchUrl, keyword, env = 'download', profile = 'xiaohongshu_fresh', unifiedApiUrl = 'http://127.0.0.1:7701', maxCommentRounds = 50, commentBatchSize = 50, timeoutMs = 180000, maxRetries = 2, } = input;
+    console.log(`[Phase34ProcessSingleNote] 开始处理: ${noteId}`);
+    let retryCount = 0;
+    let lastError;
+    for (retryCount = 0; retryCount <= maxRetries; retryCount++) {
+        if (retryCount > 0) {
+            console.log(`[Phase34ProcessSingleNote] 重试 ${retryCount}/${maxRetries}: ${noteId}`);
+            await delay(2000);
+        }
+        try {
+            // 1. 打开详情页（带超时）
+            await withTimeout(openDetail({ noteId, safeUrl, profile, unifiedApiUrl }), 30000, '打开详情页超时');
+            // 2. 提取详情内容（带超时）
+            const detailResult = await withTimeout(extractDetail({ noteId, profile, unifiedApiUrl }), 30000, '提取详情超时');
+            if (!detailResult.success) {
+                throw new Error(`提取详情失败: ${detailResult.error || 'unknown'}`);
+            }
+            // 3. 采集评论（带超时）
+            const commentsResult = await withTimeout(collectComments({
+                sessionId: profile,
+                maxRounds: maxCommentRounds,
+                batchSize: commentBatchSize,
+                unifiedApiUrl,
+            }), timeoutMs, '采集评论超时');
+            // 4. 持久化详情（带超时）
+            const persistResult = await withTimeout(persistDetail({
+                noteId,
+                detail: detailResult.detail || {},
+                keyword,
+                env,
+                unifiedApiUrl,
+            }), 60000, '持久化详情超时');
+            // 5. 返回搜索页（无论成功失败都要返回）
+            try {
+                await closeDetail({ profile, unifiedApiUrl });
+                await delay(1000);
+            }
+            catch (err) {
+                console.warn(`[Phase34ProcessSingleNote] 返回搜索页失败: ${err.message}`);
+            }
+            console.log(`[Phase34ProcessSingleNote] ✅ 完成: ${noteId} (重试${retryCount}次)`);
+            return {
+                success: true,
+                noteId,
+                detail: detailResult.detail || null,
+                comments: commentsResult.comments,
+                noteDir: persistResult.noteDir,
+                readmePath: persistResult.readmePath,
+                retryCount,
+            };
+        }
+        catch (err) {
+            lastError = err.message;
+            console.error(`[Phase34ProcessSingleNote] ❌ 尝试${retryCount + 1}失败: ${noteId}`, err.message);
+            // 失败也要返回搜索页
+            try {
+                await closeDetail({ profile, unifiedApiUrl });
+                await delay(1000);
+            }
+            catch (closeErr) {
+                console.warn(`[Phase34ProcessSingleNote] 返回搜索页失败: ${closeErr.message}`);
+            }
+            if (retryCount >= maxRetries) {
+                break;
+            }
+        }
+    }
+    return {
+        success: false,
+        noteId,
+        detail: null,
+        comments: [],
+        error: lastError || '未知错误',
+        retryCount,
+    };
+}
+//# sourceMappingURL=Phase34ProcessSingleNoteBlock.js.map

package/dist/modules/xiaohongshu/app/src/blocks/Phase34ValidateLinksBlock.js

@@ -0,0 +1,90 @@
+/**
+ * Phase 3-4 Block: 链接前置校验
+ *
+ * 职责：
+ * 1. 确认当前在搜索结果页
+ * 2. 读取 phase2-links.jsonl
+ * 3. 过滤有效链接（safeUrl 含 xsec_token）
+ */
+import os from 'node:os';
+import path from 'node:path';
+import { normalizeShard, shardFilterByIndexMod, shardFilterByNoteIdHash, } from './helpers/sharding.js';
+async function controllerAction(action, payload, apiUrl) {
+    const res = await fetch(`${apiUrl}/v1/controller/action`, {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({ action, payload }),
+        signal: AbortSignal.timeout(20000),
+    });
+    const data = await res.json().catch(() => ({}));
+    return data.data || data;
+}
+async function readJsonl(filePath) {
+    const { readFile } = await import('node:fs/promises');
+    try {
+        const content = await readFile(filePath, 'utf8');
+        return content
+            .trim()
+            .split('\n')
+            .filter((line) => line.trim())
+            .map((line) => JSON.parse(line));
+    }
+    catch {
+        return [];
+    }
+}
+function resolveDownloadRoot() {
+    const custom = process.env.WEBAUTO_DOWNLOAD_ROOT || process.env.WEBAUTO_DOWNLOAD_DIR;
+    if (custom && custom.trim())
+        return custom;
+    const home = process.env.HOME || process.env.USERPROFILE || os.homedir();
+    return path.join(home, '.webauto', 'download');
+}
+export async function execute(input) {
+    const { keyword, env = 'debug', linksPath, shardIndex, shardCount, shardBy = 'noteId-hash', maxNotes, profile = 'xiaohongshu_fresh', unifiedApiUrl = 'http://127.0.0.1:7701', } = input;
+    console.log('[Phase34ValidateLinks] 开始校验链接...');
+    // Phase34 只依赖 phase2-links.jsonl，不依赖 .collect-state.json
+    // .collect-state.json 只是 phase2 内部重入状态管理
+    const currentUrl = await controllerAction('browser:execute', {
+        profile,
+        script: 'window.location.href',
+    }, unifiedApiUrl).then((res) => res?.result || res?.data?.result || '');
+    console.log(`[Phase34ValidateLinks] 当前页面: ${currentUrl}`);
+    // 读取 phase2-links.jsonl
+    const defaultPath = path.join(resolveDownloadRoot(), 'xiaohongshu', env, keyword, 'phase2-links.jsonl');
+    const targetPath = linksPath || defaultPath;
+    console.log(`[Phase34ValidateLinks] 读取链接文件: ${targetPath}`);
+    const allLinks = await readJsonl(targetPath);
+    if (allLinks.length === 0) {
+        console.warn('[Phase34ValidateLinks] 链接文件为空或不存在');
+    }
+    console.log(`[Phase34ValidateLinks] 总链接数: ${allLinks.length}`);
+    // 过滤有效链接：safeUrl 必须包含 xsec_token
+    const validLinks = allLinks.filter((item) => {
+        const hasToken = typeof item?.safeUrl === 'string' && item.safeUrl.includes('xsec_token');
+        return hasToken;
+    });
+    const cappedLimit = Number.isFinite(Number(maxNotes)) ? Math.max(1, Math.floor(Number(maxNotes))) : null;
+    const cappedLinks = cappedLimit ? validLinks.slice(0, cappedLimit) : validLinks;
+    const shard = normalizeShard({ index: shardIndex, count: shardCount, by: shardBy });
+    const sharded = shard
+        ? shard.by === 'index-mod'
+            ? shardFilterByIndexMod(cappedLinks, shard)
+            : shardFilterByNoteIdHash(cappedLinks, shard)
+        : cappedLinks;
+    console.log(`[Phase34ValidateLinks] 有效链接数: ${validLinks.length}`);
+    if (cappedLimit) {
+        console.log(`[Phase34ValidateLinks] 全局上限: ${cappedLimit} -> ${cappedLinks.length} 条`);
+    }
+    if (shard) {
+        console.log(`[Phase34ValidateLinks] Shard(${shard.by}): ${shard.index}/${shard.count} -> ${sharded.length} 条`);
+    }
+    return {
+        success: true,
+        links: sharded,
+        totalCount: allLinks.length,
+        validCount: sharded.length,
+        currentUrl,
+    };
+}
+//# sourceMappingURL=Phase34ValidateLinksBlock.js.map