@web-auto/webauto 0.1.4 → 0.1.6

package/dist/modules/rate-limiter/index.js

@@ -0,0 +1,242 @@
+/**
+ * RateLimiter - 统一流控配额系统
+ *
+ * 功能：
+ * - 管理 search/like/comment 等操作的配额
+ * - 支持多维度限制（关键字、profile、全局）
+ * - 可配置规则，支持持久化
+ *
+ * 使用方式：
+ *   const limiter = RateLimiter.getInstance();
+ *   const result = await limiter.acquire('search', { keyword: '春晚' });
+ *   if (!result.granted) {
+ *     console.log(`被拒绝: ${result.reason}, 需等待 ${result.waitMs}ms`);
+ *     return;
+ *   }
+ *   // 执行搜索...
+ *   await limiter.record('search', { keyword: '春晚' });
+ */
+import os from 'node:os';
+import path from 'node:path';
+import { promises as fs } from 'node:fs';
+// 默认配置
+const DEFAULT_CONFIG = {
+    search: {
+        rules: [
+            { scope: 'keyword', windowMs: 600000, max: 3, key: '同一关键字10分钟内最多3次' },
+            { scope: 'global', windowMs: 60000, max: 10, key: '全局搜索1分钟内最多10次' },
+        ],
+    },
+    like: {
+        rules: [
+            { scope: 'profile', windowMs: 60000, max: 6, key: '同一账号1分钟内最多点赞6次' },
+            { scope: 'profile', windowMs: 3600000, max: 100, key: '同一账号1小时内最多点赞100次' },
+        ],
+    },
+    comment: {
+        rules: [
+            { scope: 'profile', windowMs: 60000, max: 1, key: '同一账号1分钟内最多评论1次' },
+            { scope: 'profile', windowMs: 3600000, max: 30, key: '同一账号1小时内最多评论30次' },
+        ],
+    },
+    follow: {
+        rules: [
+            { scope: 'profile', windowMs: 60000, max: 2, key: '同一账号1分钟内最多关注2次' },
+            { scope: 'profile', windowMs: 3600000, max: 20, key: '同一账号1小时内最多关注20次' },
+        ],
+    },
+    repost: {
+        rules: [
+            { scope: 'profile', windowMs: 60000, max: 1, key: '同一账号1分钟内最多转发1次' },
+            { scope: 'profile', windowMs: 3600000, max: 10, key: '同一账号1小时内最多转发10次' },
+        ],
+    },
+};
+function resolveConfigPath() {
+    const home = process.env.HOME || process.env.USERPROFILE || os.homedir();
+    return path.join(home, '.webauto', 'config', 'rate-limits.json');
+}
+function resolveHistoryPath() {
+    const home = process.env.HOME || process.env.USERPROFILE || os.homedir();
+    return path.join(home, '.webauto', 'rate-limiter-history.json');
+}
+export class RateLimiter {
+    static instance = null;
+    config;
+    history = new Map();
+    configPath;
+    historyPath;
+    initialized = false;
+    constructor() {
+        this.config = { ...DEFAULT_CONFIG };
+        this.configPath = resolveConfigPath();
+        this.historyPath = resolveHistoryPath();
+    }
+    static getInstance() {
+        if (!RateLimiter.instance) {
+            RateLimiter.instance = new RateLimiter();
+        }
+        return RateLimiter.instance;
+    }
+    // 初始化：加载配置和历史
+    async init() {
+        if (this.initialized)
+            return;
+        // 加载用户配置（覆盖默认值）
+        try {
+            const content = await fs.readFile(this.configPath, 'utf-8');
+            const userConfig = JSON.parse(content);
+            this.config = { ...DEFAULT_CONFIG, ...userConfig };
+            console.log('[RateLimiter] Loaded user config from', this.configPath);
+        }
+        catch {
+            // 使用默认配置
+        }
+        // 加载历史记录
+        try {
+            const content = await fs.readFile(this.historyPath, 'utf-8');
+            const data = JSON.parse(content);
+            for (const [key, entries] of Object.entries(data)) {
+                this.history.set(key, entries);
+            }
+            // 清理过期历史
+            this.cleanExpiredHistory();
+        }
+        catch {
+            // 无历史记录
+        }
+        this.initialized = true;
+    }
+    // 申请配额
+    async acquire(type, params) {
+        await this.init();
+        const typeConfig = this.config[type];
+        if (!typeConfig) {
+            return { granted: true }; // 无配置的类型默认允许
+        }
+        const now = Date.now();
+        for (const rule of typeConfig.rules) {
+            const key = this.buildKey(type, rule.scope, params);
+            const entries = this.history.get(key) || [];
+            // 统计窗口内的次数
+            const windowStart = now - rule.windowMs;
+            const count = entries.filter(e => e.ts > windowStart).length;
+            if (count >= rule.max) {
+                // 找到最早的过期时间，计算等待时间
+                const oldestInWindow = entries.find(e => e.ts > windowStart);
+                const waitMs = oldestInWindow ? (oldestInWindow.ts + rule.windowMs - now) : rule.windowMs;
+                return {
+                    granted: false,
+                    reason: rule.key || `配额已用尽: ${rule.scope} ${rule.windowMs}ms 内最多 ${rule.max} 次`,
+                    waitMs: Math.max(0, waitMs),
+                    ruleKey: `${type}:${rule.scope}:${rule.windowMs}`,
+                };
+            }
+        }
+        return { granted: true };
+    }
+    // 记录执行
+    async record(type, params) {
+        await this.init();
+        const typeConfig = this.config[type];
+        if (!typeConfig)
+            return;
+        const now = Date.now();
+        // 为所有相关维度记录
+        for (const rule of typeConfig.rules) {
+            const key = this.buildKey(type, rule.scope, params);
+            const entries = this.history.get(key) || [];
+            entries.push({ ts: now, key });
+            this.history.set(key, entries);
+        }
+        // 持久化历史（异步，不阻塞）
+        this.persistHistory().catch(() => { });
+    }
+    // 构建维度键
+    buildKey(type, scope, params) {
+        switch (scope) {
+            case 'keyword':
+                return `${type}:keyword:${params.keyword || '*'}`;
+            case 'profile':
+                return `${type}:profile:${params.profileId || '*'}`;
+            case 'global':
+                return `${type}:global`;
+            default:
+                return `${type}:unknown`;
+        }
+    }
+    // 清理过期历史
+    cleanExpiredHistory() {
+        const now = Date.now();
+        const maxWindow = 3600000; // 1小时
+        for (const [key, entries] of this.history) {
+            const filtered = entries.filter(e => now - e.ts < maxWindow);
+            if (filtered.length === 0) {
+                this.history.delete(key);
+            }
+            else if (filtered.length !== entries.length) {
+                this.history.set(key, filtered);
+            }
+        }
+    }
+    // 持久化历史
+    async persistHistory() {
+        try {
+            const dir = path.dirname(this.historyPath);
+            await fs.mkdir(dir, { recursive: true });
+            const data = {};
+            for (const [key, entries] of this.history) {
+                data[key] = entries;
+            }
+            await fs.writeFile(this.historyPath, JSON.stringify(data), 'utf-8');
+        }
+        catch (err) {
+            console.error('[RateLimiter] Failed to persist history:', err);
+        }
+    }
+    // 获取当前配置
+    getConfig() {
+        return { ...this.config };
+    }
+    // 更新配置
+    async updateConfig(newConfig) {
+        this.config = { ...this.config, ...newConfig };
+        try {
+            const dir = path.dirname(this.configPath);
+            await fs.mkdir(dir, { recursive: true });
+            await fs.writeFile(this.configPath, JSON.stringify(this.config, null, 2), 'utf-8');
+            console.log('[RateLimiter] Config saved to', this.configPath);
+        }
+        catch (err) {
+            console.error('[RateLimiter] Failed to save config:', err);
+        }
+    }
+    // 获取状态（用于调试/UI）
+    getStatus() {
+        const result = [];
+        const now = Date.now();
+        for (const [type, typeConfig] of Object.entries(this.config)) {
+            for (const rule of typeConfig.rules) {
+                const key = `${type}:${rule.scope}`;
+                let totalCount = 0;
+                for (const [k, entries] of this.history) {
+                    if (k.startsWith(key)) {
+                        const windowStart = now - rule.windowMs;
+                        totalCount += entries.filter(e => e.ts > windowStart).length;
+                    }
+                }
+                result.push({
+                    type: type,
+                    scope: rule.scope,
+                    count: totalCount,
+                    max: rule.max,
+                    windowMs: rule.windowMs,
+                });
+            }
+        }
+        return result;
+    }
+}
+// 单例导出
+export const rateLimiter = RateLimiter.getInstance();
+//# sourceMappingURL=index.js.map

package/dist/modules/workflow/blocks/ExecuteWeiboSearchBlock.js

@@ -0,0 +1,128 @@
+/**
+ * Workflow Block: ExecuteWeiboSearchBlock
+ *
+ * 微博搜索执行：直接导航到搜索结果页
+ */
+import os from 'node:os';
+import path from 'node:path';
+import { promises as fs } from 'node:fs';
+function sanitizeFilenamePart(value) {
+    return String(value || '')
+        .trim()
+        .replace(/[\\/:"*?<>|]+/g, '_')
+        .replace(/\s+/g, '_')
+        .slice(0, 80);
+}
+function resolveDownloadRoot() {
+    const custom = process.env.WEBAUTO_DOWNLOAD_ROOT || process.env.WEBAUTO_DOWNLOAD_DIR;
+    if (custom && custom.trim())
+        return custom;
+    const home = process.env.HOME || process.env.USERPROFILE || os.homedir();
+    return path.join(home, '.webauto', 'download');
+}
+function isDebugArtifactsEnabled() {
+    return (process.env.WEBAUTO_DEBUG === '1' ||
+        process.env.WEBAUTO_DEBUG_ARTIFACTS === '1');
+}
+export async function execute(input) {
+    const { sessionId, keyword, env = 'debug', serviceUrl = 'http://127.0.0.1:7704', } = input;
+    const profile = sessionId;
+    const controllerUrl = `${serviceUrl}/command`;
+    const steps = [];
+    const debugEnabled = isDebugArtifactsEnabled();
+    async function controllerAction(action, args = {}) {
+        const res = await fetch(controllerUrl, {
+            method: 'POST',
+            headers: { 'Content-Type': 'application/json' },
+            body: JSON.stringify({ action, args: { profileId: profile, ...args } }),
+            signal: AbortSignal.timeout ? AbortSignal.timeout(30000) : undefined,
+        });
+        const raw = await res.text();
+        if (!res.ok)
+            throw new Error(`HTTP ${res.status}: ${raw}`);
+        try {
+            return JSON.parse(raw);
+        }
+        catch {
+            return { raw };
+        }
+    }
+    async function getCurrentUrl() {
+        const res = await controllerAction('evaluate', { script: 'window.location.href' });
+        return res?.body?.result ?? res?.result ?? '';
+    }
+    async function saveDebugScreenshot(kind, meta = {}) {
+        if (!debugEnabled)
+            return;
+        try {
+            const keywordDir = path.join(resolveDownloadRoot(), 'weibo', env, sanitizeFilenamePart(keyword));
+            const debugDir = path.join(keywordDir, '_debug', 'search');
+            await fs.mkdir(debugDir, { recursive: true });
+            const ts = new Date().toISOString().replace(/[:.]/g, '-');
+            const pngPath = path.join(debugDir, `${ts}-${sanitizeFilenamePart(kind)}.png`);
+            const shot = await controllerAction('screenshot');
+            const b64 = shot?.body?.data ?? shot?.data;
+            if (typeof b64 === 'string' && b64.length > 10) {
+                await fs.writeFile(pngPath, Buffer.from(b64, 'base64'));
+            }
+            console.log(`[ExecuteWeiboSearch][debug] saved ${kind}: ${pngPath}`);
+        }
+        catch (e) {
+            console.warn(`[ExecuteWeiboSearch][debug] save failed (${kind}): ${e?.message || String(e)}`);
+        }
+    }
+    function pushStep(step) {
+        steps.push(step);
+        console.log('[ExecuteWeiboSearch][step]', step.id, step.status, step.error || '');
+    }
+    try {
+        // 直接导航到搜索结果页
+        const currentUrl = await getCurrentUrl();
+        if (!currentUrl.includes('s.weibo.com/weibo')) {
+            pushStep({ id: 'goto_search_page', status: 'running', meta: { from: currentUrl, keyword } });
+            const searchUrl = `https://s.weibo.com/weibo?q=${encodeURIComponent(keyword)}`;
+            console.log('[ExecuteWeiboSearch] navigating to:', searchUrl);
+            await controllerAction('goto', { url: searchUrl });
+            await new Promise(r => setTimeout(r, 3000));
+            pushStep({ id: 'goto_search_page', status: 'success' });
+        }
+        else {
+            pushStep({ id: 'goto_search_page', status: 'skipped', meta: { reason: 'already_on_search_page' } });
+        }
+        // 检查是否成功加载搜索页
+        const finalUrl = await getCurrentUrl();
+        console.log('[ExecuteWeiboSearch] current URL:', finalUrl);
+        // 检查是否有搜索结果卡片
+        const cardsScript = `document.querySelectorAll('.card-wrap').length`;
+        const cardsRes = await controllerAction('evaluate', { script: cardsScript });
+        const cardsCount = cardsRes?.result ?? 0;
+        if (cardsCount > 0) {
+            console.log('[ExecuteWeiboSearch] search results loaded:', cardsCount, 'cards');
+            return {
+                success: true,
+                searchExecuted: true,
+                url: finalUrl,
+                steps,
+            };
+        }
+        // 没有找到搜索结果卡片
+        console.log('[ExecuteWeiboSearch] no search results found');
+        await saveDebugScreenshot('no_search_results', { keyword, url: finalUrl });
+        return {
+            success: true, // 仍然返回成功，因为已经导航到搜索页
+            searchExecuted: true,
+            url: finalUrl,
+            steps,
+        };
+    }
+    catch (error) {
+        return {
+            success: false,
+            searchExecuted: false,
+            url: '',
+            steps,
+            error: `ExecuteWeiboSearch failed: ${error.message}`,
+        };
+    }
+}
+//# sourceMappingURL=ExecuteWeiboSearchBlock.js.map

package/dist/modules/workflow/blocks/PersistXhsNoteBlock.js

@@ -9,6 +9,7 @@
 import os from 'node:os';
 import path from 'node:path';
 import { promises as fs } from 'node:fs';
+import { getCurrentTimestamp } from '../../collection-manager/date-utils.js';
 function sanitizeForPath(name) {
     if (!name)
         return '';
@@ -116,6 +117,7 @@ export async function execute(input) {
         const detailPath = path.join(postDir, 'content.md');
         const commentsPath = path.join(postDir, 'comments.md');
         const commentsDonePath = path.join(postDir, 'comments.done.json');
+        const persistTs = getCurrentTimestamp();
         // 1) 详情（正文/图片）
         if (wantDetail) {
             const existingDetail = await fs.stat(detailPath).catch(() => null);
@@ -160,7 +162,8 @@ export async function execute(input) {
                     lines.push(`- 链接: ${detailUrl}`);
                 if (author)
                     lines.push(`- 作者: ${author}`);
-                lines.push(`- 采集时间: ${new Date().toISOString()}`);
+                lines.push(`- 采集时间: ${persistTs.collectedAt}`);
+                lines.push(`- 采集时间(本地): ${persistTs.collectedAtLocal}`);
                 lines.push('');
                 lines.push('## 正文');
                 lines.push('');
@@ -207,7 +210,8 @@ export async function execute(input) {
                     lines.push(`- Search URL: ${searchUrl}`);
                 if (detailUrl)
                     lines.push(`- 链接: ${detailUrl}`);
-                lines.push(`- 采集时间: ${new Date().toISOString()}`);
+                lines.push(`- 采集时间: ${persistTs.collectedAt}`);
+                lines.push(`- 采集时间(本地): ${persistTs.collectedAtLocal}`);
                 lines.push(`- 评论统计: 抓取=${comments.length}, header=${headerTotal !== null ? headerTotal : '未知'}（reachedEnd=${commentsResult?.reachedEnd ? '是' : '否'}, empty=${commentsResult?.emptyState ? '是' : '否'}${headerPositive
                     ? `, coverage=${Math.round((coverageRatio || 0) * 100)}% (need>=${coverageNeed}, ok=${coverageOk ? '是' : '否'})`
                     : ''}）`);
@@ -246,7 +250,7 @@ export async function execute(input) {
                     stoppedByMaxComments: Boolean(commentsResult?.stoppedByMaxComments),
                     totalComments: comments.length,
                     headerTotal,
-                    ts: new Date().toISOString(),
+                    ts: persistTs.collectedAt,
                 }, null, 2), 'utf-8');
             }
             else {

package/dist/modules/workflow/blocks/RenderMarkdown.js

@@ -20,7 +20,9 @@ export async function execute(input) {
     }
     const lines = [];
     lines.push(`# 微博采集结果`);
-    lines.push(`采集时间: ${new Date().toISOString()}`);
+    const ts = getCurrentTimestamp();
+    lines.push(`采集时间: ${ts.collectedAt}`);
+    lines.push(`采集时间(本地): ${ts.collectedAtLocal}`);
     lines.push(`总计: ${posts.length} 条`);
     lines.push('');
     posts.forEach((post, index) => {
@@ -45,4 +47,5 @@ export async function execute(input) {
         count: posts.length
     };
 }
+import { getCurrentTimestamp } from '../../collection-manager/date-utils.js';
 //# sourceMappingURL=RenderMarkdown.js.map