@web-auto/webauto 0.1.3 → 0.1.6

package/modules/task-scheduler/index.ts

@@ -0,0 +1,293 @@
+/**
+ * Task Scheduler Module
+ * 
+ * Handles:
+ * - Task queuing with priority
+ * - Conflict detection (same profile)
+ * - Task lifecycle management
+ * - Max runs tracking
+ */
+
+import { EventEmitter } from 'events';
+import { execSync } from 'child_process';
+
+export type TaskStatus = 'pending' | 'queued' | 'running' | 'completed' | 'failed' | 'cancelled';
+
+export interface ScheduledTask {
+  id: string;
+  name: string;
+  profile: string;
+  platform: 'xiaohongshu' | 'weibo' | '1688';
+  taskType: 'search' | 'timeline' | 'user-monitor';
+  config: Record<string, any>;
+  schedule: {
+    type: 'interval' | 'daily' | 'once';
+    intervalMinutes?: number;
+    dailyTime?: string; // HH:MM
+  };
+  maxRuns: number | null;
+  currentRuns: number;
+  priority: number;
+  status: TaskStatus;
+  createdAt: string;
+  lastRunAt: string | null;
+  nextRunAt: string | null;
+  pid: number | null;
+}
+
+export interface TaskRunResult {
+  taskId: string;
+  runId: string;
+  success: boolean;
+  error?: string;
+  startedAt: string;
+  finishedAt: string;
+  durationMs: number;
+}
+
+export class TaskScheduler extends EventEmitter {
+  private tasks: Map<string, ScheduledTask> = new Map();
+  private runningProfiles: Map<string, string> = new Map(); // profile -> taskId
+  private queue: string[] = [];
+  private maxRunTimeMs: number;
+  
+  /**
+   * Terminate a process by PID (cross-platform)
+   */
+  private terminateProcess(pid: number): void {
+    if (process.platform === 'win32') {
+      try {
+        execSync(`taskkill /PID ${pid} /T /F`, { stdio: 'ignore' });
+      } catch {
+        // Ignore errors if process already terminated
+      }
+    } else {
+      try {
+        process.kill(pid, 'SIGTERM');
+      } catch {
+        // Ignore errors if process already terminated
+      }
+    }
+  }
+  
+  constructor(options: { maxRunTimeMs?: number } = {}) {
+    super();
+    this.maxRunTimeMs = options.maxRunTimeMs || 30 * 60 * 1000; // 30 min default
+  }
+
+  /**
+   * Add a new scheduled task
+   */
+  addTask(task: Omit<ScheduledTask, 'currentRuns' | 'status' | 'lastRunAt' | 'nextRunAt' | 'pid'>): string {
+    const fullTask: ScheduledTask = {
+      ...task,
+      currentRuns: 0,
+      status: 'pending',
+      lastRunAt: null,
+      nextRunAt: null,
+      pid: null
+    };
+    
+    this.tasks.set(task.id, fullTask);
+    this.emit('task:added', { task: fullTask });
+    
+    return task.id;
+  }
+
+  /**
+   * Check if a profile is available (not running another task)
+   */
+  isProfileAvailable(profile: string): boolean {
+    return !this.runningProfiles.has(profile);
+  }
+
+  /**
+   * Get running task for a profile
+   */
+  getRunningTaskForProfile(profile: string): ScheduledTask | null {
+    const taskId = this.runningProfiles.get(profile);
+    return taskId ? this.tasks.get(taskId) || null : null;
+  }
+
+  /**
+   * Queue a task for execution
+   * Returns false if conflict detected
+   */
+  queueTask(taskId: string): { queued: boolean; reason?: string; conflictingTask?: ScheduledTask } {
+    const task = this.tasks.get(taskId);
+    if (!task) {
+      return { queued: false, reason: 'Task not found' };
+    }
+
+    // Check max runs
+    if (task.maxRuns !== null && task.currentRuns >= task.maxRuns) {
+      return { queued: false, reason: 'Max runs reached' };
+    }
+
+    // Check for profile conflict
+    const runningTask = this.getRunningTaskForProfile(task.profile);
+    if (runningTask) {
+      // Already running - add to queue
+      task.status = 'queued';
+      this.queue.push(taskId);
+      this.queue.sort((a, b) => {
+        const taskA = this.tasks.get(a)!;
+        const taskB = this.tasks.get(b)!;
+        return taskB.priority - taskA.priority;
+      });
+      
+      return { 
+        queued: true, 
+        reason: `Profile ${task.profile} busy with task ${runningTask.id}`,
+        conflictingTask: runningTask
+      };
+    }
+
+    // Profile available - add to queue
+    task.status = 'queued';
+    this.queue.push(taskId);
+    this.emit('task:queued', { task });
+    
+    return { queued: true };
+  }
+
+  /**
+   * Start next task in queue
+   */
+  async startNext(): Promise<ScheduledTask | null> {
+    // Find next task that can run
+    while (this.queue.length > 0) {
+      const taskId = this.queue.shift()!;
+      const task = this.tasks.get(taskId);
+      
+      if (!task) continue;
+      if (task.status !== 'queued') continue;
+      
+      // Check profile availability
+      if (!this.isProfileAvailable(task.profile)) {
+        // Put back at front of queue
+        this.queue.unshift(taskId);
+        return null;
+      }
+
+      // Start task
+      task.status = 'running';
+      task.lastRunAt = new Date().toISOString();
+      task.currentRuns++;
+      this.runningProfiles.set(task.profile, taskId);
+      
+      this.emit('task:started', { task });
+      
+      return task;
+    }
+    
+    return null;
+  }
+
+  /**
+   * Mark task as completed
+   */
+  completeTask(taskId: string, result: TaskRunResult): void {
+    const task = this.tasks.get(taskId);
+    if (!task) return;
+
+    task.status = result.success ? 'completed' : 'failed';
+    task.pid = null;
+    
+    // Release profile
+    this.runningProfiles.delete(task.profile);
+    
+    this.emit('task:completed', { task, result });
+    
+    // Check if task should continue
+    if (task.maxRuns !== null && task.currentRuns >= task.maxRuns) {
+      this.emit('task:max-runs-reached', { task });
+    }
+  }
+
+  /**
+   * Cancel a task
+   */
+  cancelTask(taskId: string): boolean {
+    const task = this.tasks.get(taskId);
+    if (!task) return false;
+
+    // Remove from queue if pending
+    const queueIndex = this.queue.indexOf(taskId);
+    if (queueIndex >= 0) {
+      this.queue.splice(queueIndex, 1);
+    }
+
+    // If running, mark for termination
+    if (task.status === 'running') {
+      task.status = 'cancelled';
+      if (task.pid) {
+        this.terminateProcess(task.pid);
+      }
+      this.runningProfiles.delete(task.profile);
+    }
+
+    this.emit('task:cancelled', { task });
+    return true;
+  }
+
+  /**
+   * Get all tasks
+   */
+  getTasks(): ScheduledTask[] {
+    return Array.from(this.tasks.values());
+  }
+
+  /**
+   * Get tasks by status
+   */
+  getTasksByStatus(status: TaskStatus): ScheduledTask[] {
+    return this.getTasks().filter(t => t.status === status);
+  }
+
+  /**
+   * Get queue length
+   */
+  getQueueLength(): number {
+    return this.queue.length;
+  }
+
+  /**
+   * Simulate task overlap scenario
+   */
+  simulateOverlap(): {
+    scenario: string;
+    queue: string[];
+    running: Array<{ profile: string; taskId: string }>;
+    conflicts: string[];
+  } {
+    const running = Array.from(this.runningProfiles.entries()).map(([profile, taskId]) => ({
+      profile,
+      taskId
+    }));
+    
+    const conflicts: string[] = [];
+    const profileMap = new Map<string, string[]>();
+    
+    for (const task of this.tasks.values()) {
+      if (task.status === 'queued' || task.status === 'running') {
+        const existing = profileMap.get(task.profile) || [];
+        existing.push(task.id);
+        profileMap.set(task.profile, existing);
+        
+        if (existing.length > 1) {
+          conflicts.push(...existing);
+        }
+      }
+    }
+    
+    return {
+      scenario: 'Task overlap simulation',
+      queue: [...this.queue],
+      running,
+      conflicts
+    };
+  }
+}
+
+export default TaskScheduler;

package/modules/workflow/blocks/ExecuteWeiboSearchBlock.ts

@@ -0,0 +1,167 @@
+/**
+ * Workflow Block: ExecuteWeiboSearchBlock
+ *
+ * 微博搜索执行：直接导航到搜索结果页
+ */
+
+import os from 'node:os';
+import path from 'node:path';
+import { promises as fs } from 'node:fs';
+
+export interface ExecuteWeiboSearchInput {
+  sessionId: string;
+  keyword: string;
+  env?: string;
+  serviceUrl?: string;
+}
+
+export interface ExecuteWeiboSearchOutput {
+  success: boolean;
+  searchExecuted: boolean;
+  url: string;
+  steps?: Array<{
+    id: string;
+    status: 'pending' | 'running' | 'success' | 'failed' | 'skipped';
+    error?: string;
+    meta?: Record<string, any>;
+  }>;
+  error?: string;
+}
+
+function sanitizeFilenamePart(value: string): string {
+  return String(value || '')
+    .trim()
+    .replace(/[\\/:"*?<>|]+/g, '_')
+    .replace(/\s+/g, '_')
+    .slice(0, 80);
+}
+
+function resolveDownloadRoot(): string {
+  const custom = process.env.WEBAUTO_DOWNLOAD_ROOT || process.env.WEBAUTO_DOWNLOAD_DIR;
+  if (custom && custom.trim()) return custom;
+  const home = process.env.HOME || process.env.USERPROFILE || os.homedir();
+  return path.join(home, '.webauto', 'download');
+}
+
+function isDebugArtifactsEnabled(): boolean {
+  return (
+    process.env.WEBAUTO_DEBUG === '1' ||
+    process.env.WEBAUTO_DEBUG_ARTIFACTS === '1'
+  );
+}
+
+export async function execute(input: ExecuteWeiboSearchInput): Promise<ExecuteWeiboSearchOutput> {
+  const {
+    sessionId,
+    keyword,
+    env = 'debug',
+    serviceUrl = 'http://127.0.0.1:7704',
+  } = input;
+
+  const profile = sessionId;
+  const controllerUrl = `${serviceUrl}/command`;
+  const steps: ExecuteWeiboSearchOutput['steps'] = [];
+  const debugEnabled = isDebugArtifactsEnabled();
+
+  async function controllerAction(action: string, args: any = {}): Promise<any> {
+    const res = await fetch(controllerUrl, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({ action, args: { profileId: profile, ...args } }),
+      signal: (AbortSignal as any).timeout ? (AbortSignal as any).timeout(30000) : undefined,
+    });
+    const raw = await res.text();
+    if (!res.ok) throw new Error(`HTTP ${res.status}: ${raw}`);
+    try {
+      return JSON.parse(raw);
+    } catch {
+      return { raw };
+    }
+  }
+
+  async function getCurrentUrl(): Promise<string> {
+    const res = await controllerAction('evaluate', { script: 'window.location.href' });
+    return res?.body?.result ?? res?.result ?? '';
+  }
+
+  async function saveDebugScreenshot(kind: string, meta: Record<string, any> = {}): Promise<void> {
+    if (!debugEnabled) return;
+    try {
+      const keywordDir = path.join(resolveDownloadRoot(), 'weibo', env, sanitizeFilenamePart(keyword));
+      const debugDir = path.join(keywordDir, '_debug', 'search');
+      await fs.mkdir(debugDir, { recursive: true });
+      const ts = new Date().toISOString().replace(/[:.]/g, '-');
+      const pngPath = path.join(debugDir, `${ts}-${sanitizeFilenamePart(kind)}.png`);
+
+      const shot = await controllerAction('screenshot');
+      const b64 = shot?.body?.data ?? shot?.data;
+      if (typeof b64 === 'string' && b64.length > 10) {
+        await fs.writeFile(pngPath, Buffer.from(b64, 'base64'));
+      }
+      console.log(`[ExecuteWeiboSearch][debug] saved ${kind}: ${pngPath}`);
+    } catch (e: any) {
+      console.warn(`[ExecuteWeiboSearch][debug] save failed (${kind}): ${e?.message || String(e)}`);
+    }
+  }
+
+  function pushStep(step: NonNullable<ExecuteWeiboSearchOutput['steps']>[number]) {
+    steps.push(step);
+    console.log('[ExecuteWeiboSearch][step]', step.id, step.status, step.error || '');
+  }
+
+  try {
+    // 直接导航到搜索结果页
+    const currentUrl = await getCurrentUrl();
+    
+    if (!currentUrl.includes('s.weibo.com/weibo')) {
+      pushStep({ id: 'goto_search_page', status: 'running', meta: { from: currentUrl, keyword } });
+      
+      const searchUrl = `https://s.weibo.com/weibo?q=${encodeURIComponent(keyword)}`;
+      console.log('[ExecuteWeiboSearch] navigating to:', searchUrl);
+      await controllerAction('goto', { url: searchUrl });
+      await new Promise(r => setTimeout(r, 3000));
+      
+      pushStep({ id: 'goto_search_page', status: 'success' });
+    } else {
+      pushStep({ id: 'goto_search_page', status: 'skipped', meta: { reason: 'already_on_search_page' } });
+    }
+
+    // 检查是否成功加载搜索页
+    const finalUrl = await getCurrentUrl();
+    console.log('[ExecuteWeiboSearch] current URL:', finalUrl);
+    
+    // 检查是否有搜索结果卡片
+    const cardsScript = `document.querySelectorAll('.card-wrap').length`;
+    const cardsRes = await controllerAction('evaluate', { script: cardsScript });
+    const cardsCount = cardsRes?.result ?? 0;
+    
+    if (cardsCount > 0) {
+      console.log('[ExecuteWeiboSearch] search results loaded:', cardsCount, 'cards');
+      return {
+        success: true,
+        searchExecuted: true,
+        url: finalUrl,
+        steps,
+      };
+    }
+    
+    // 没有找到搜索结果卡片
+    console.log('[ExecuteWeiboSearch] no search results found');
+    await saveDebugScreenshot('no_search_results', { keyword, url: finalUrl });
+    
+    return {
+      success: true, // 仍然返回成功，因为已经导航到搜索页
+      searchExecuted: true,
+      url: finalUrl,
+      steps,
+    };
+  } catch (error: any) {
+    return {
+      success: false,
+      searchExecuted: false,
+      url: '',
+      steps,
+      error: `ExecuteWeiboSearch failed: ${error.message}`,
+    };
+  }
+}

package/modules/workflow/blocks/PersistXhsNoteBlock.ts

@@ -10,6 +10,7 @@
 import os from 'node:os';
 import path from 'node:path';
 import { promises as fs } from 'node:fs';
+import { getCurrentTimestamp } from '../../collection-manager/date-utils.js';
 
 export interface PersistXhsNoteInput {
   sessionId: string;
@@ -178,6 +179,7 @@ export async function execute(input: PersistXhsNoteInput): Promise<PersistXhsNot
     const detailPath = path.join(postDir, 'content.md');
     const commentsPath = path.join(postDir, 'comments.md');
     const commentsDonePath = path.join(postDir, 'comments.done.json');
+    const persistTs = getCurrentTimestamp();
 
     // 1) 详情（正文/图片）
     if (wantDetail) {
@@ -223,7 +225,8 @@ export async function execute(input: PersistXhsNoteInput): Promise<PersistXhsNot
         if (searchUrl) lines.push(`- Search URL: ${searchUrl}`);
         if (detailUrl) lines.push(`- 链接: ${detailUrl}`);
         if (author) lines.push(`- 作者: ${author}`);
-        lines.push(`- 采集时间: ${new Date().toISOString()}`);
+        lines.push(`- 采集时间: ${persistTs.collectedAt}`);
+        lines.push(`- 采集时间(本地): ${persistTs.collectedAtLocal}`);
         lines.push('');
         lines.push('## 正文');
         lines.push('');
@@ -276,7 +279,8 @@ export async function execute(input: PersistXhsNoteInput): Promise<PersistXhsNot
         lines.push(`- 关键词: ${keyword || '未知'}`);
         if (searchUrl) lines.push(`- Search URL: ${searchUrl}`);
         if (detailUrl) lines.push(`- 链接: ${detailUrl}`);
-        lines.push(`- 采集时间: ${new Date().toISOString()}`);
+        lines.push(`- 采集时间: ${persistTs.collectedAt}`);
+        lines.push(`- 采集时间(本地): ${persistTs.collectedAtLocal}`);
         lines.push(
           `- 评论统计: 抓取=${comments.length}, header=${
             headerTotal !== null ? headerTotal : '未知'
@@ -329,7 +333,7 @@ export async function execute(input: PersistXhsNoteInput): Promise<PersistXhsNot
               stoppedByMaxComments: Boolean(commentsResult?.stoppedByMaxComments),
               totalComments: comments.length,
               headerTotal,
-              ts: new Date().toISOString(),
+              ts: persistTs.collectedAt,
             },
             null,
             2,

package/modules/workflow/blocks/RenderMarkdown.ts

@@ -39,7 +39,9 @@ export async function execute(input: RenderMarkdownInput): Promise<RenderMarkdow
 
   const lines: string[] = [];
   lines.push(`# 微博采集结果`);
-  lines.push(`采集时间: ${new Date().toISOString()}`);
+  const ts = getCurrentTimestamp();
+  lines.push(`采集时间: ${ts.collectedAt}`);
+  lines.push(`采集时间(本地): ${ts.collectedAtLocal}`);
   lines.push(`总计: ${posts.length} 条`);
   lines.push('');
 
@@ -70,3 +72,4 @@ export async function execute(input: RenderMarkdownInput): Promise<RenderMarkdow
     count: posts.length
   };
 }
+import { getCurrentTimestamp } from '../../collection-manager/date-utils.js';