@web-auto/webauto 0.1.17 → 0.1.19

package/modules/workflow/blocks/WeiboCollectFromLinksBlock.ts

@@ -17,6 +17,8 @@ export interface WeiboCollectFromLinksInput {
   targetCount: number;
   maxComments?: number;
   collectComments?: boolean;
+  tabCount?: number;
+  tabOpenDelayMs?: number;
   serviceUrl?: string;
 }
 
@@ -30,6 +32,7 @@ export interface WeiboCollectFromLinksOutput {
     postsProcessed: number;
     totalComments: number;
     errors: number;
+    tabsUsed: number;
   };
   error?: string;
 }
@@ -140,11 +143,15 @@ export async function execute(input: WeiboCollectFromLinksInput): Promise<WeiboC
     targetCount,
     maxComments = 0,
     collectComments: enableComments = false,  // 默认不采集评论，加快速度
+    tabCount: requestedTabCount = 1,
+    tabOpenDelayMs: requestedTabOpenDelayMs = 800,
     serviceUrl = 'http://127.0.0.1:7704',
   } = input;
 
   const profile = sessionId;
   const controllerUrl = `${serviceUrl}/command`;
+  const tabCount = Math.max(1, Math.min(8, Number(requestedTabCount || 1) || 1));
+  const tabOpenDelayMs = Math.max(0, Number(requestedTabOpenDelayMs || 0) || 0);
   
   const keywordDir = path.join(resolveDownloadRoot(), 'weibo', env, sanitizeFilenamePart(keyword));
   const linksPath = path.join(keywordDir, 'phase2-links.jsonl');
@@ -157,7 +164,7 @@ export async function execute(input: WeiboCollectFromLinksInput): Promise<WeiboC
       linksPath,
       processedCount: 0,
       persistedCount: 0,
-      stats: { postsProcessed: 0, totalComments: 0, errors: 0 },
+      stats: { postsProcessed: 0, totalComments: 0, errors: 0, tabsUsed: 0 },
       error: 'No links found in phase2-links.jsonl',
     };
   }
@@ -197,6 +204,65 @@ export async function execute(input: WeiboCollectFromLinksInput): Promise<WeiboC
     await controllerAction('goto', { url });
     await new Promise(r => setTimeout(r, 500));  // 减少间隔  // 减少等待时间
   }
+
+  async function listPagesDetailed(): Promise<Array<{ index: number; url: string; active: boolean }>> {
+    const res = await controllerAction('browser:page:list', { profileId: profile });
+    const value = unwrapResult(res);
+    const pages = Array.isArray(value?.pages) ? value.pages : (Array.isArray(value) ? value : []);
+    return pages
+      .map((item: any) => ({
+        index: Number(item?.index),
+        url: String(item?.url || ''),
+        active: item?.active === true,
+      }))
+      .filter((item: { index: number }) => Number.isFinite(item.index));
+  }
+
+  async function switchToPage(index: number): Promise<void> {
+    await controllerAction('browser:page:switch', { profileId: profile, index });
+    await new Promise((r) => setTimeout(r, 260));
+  }
+
+  async function openNewTabAndResolveIndex(existingIndexes: Set<number>): Promise<number | null> {
+    await controllerAction('system:shortcut', { app: 'camoufox', shortcut: 'new-tab' });
+    if (tabOpenDelayMs > 0) {
+      await new Promise((r) => setTimeout(r, tabOpenDelayMs));
+    }
+    const after = await listPagesDetailed();
+    const active = after.find((item) => item.active);
+    if (active && !existingIndexes.has(active.index)) return active.index;
+    const added = after.find((item) => !existingIndexes.has(item.index));
+    if (added) return added.index;
+    const fallback = after
+      .map((item) => item.index)
+      .filter((idx) => !existingIndexes.has(idx))
+      .sort((a, b) => a - b);
+    return fallback.length > 0 ? fallback[fallback.length - 1] : null;
+  }
+
+  async function ensureTabPool(count: number): Promise<number[]> {
+    const pages = await listPagesDetailed().catch(() => [] as Array<{ index: number; url: string; active: boolean }>);
+    const active = pages.find((item) => item.active);
+    const pool: number[] = [];
+    if (active && Number.isFinite(active.index)) {
+      pool.push(active.index);
+    } else if (pages.length > 0) {
+      pool.push(pages[0].index);
+    } else {
+      pool.push(0);
+    }
+    while (pool.length < count) {
+      const idx = await openNewTabAndResolveIndex(new Set(pool));
+      if (!Number.isFinite(Number(idx))) break;
+      const next = Number(idx);
+      if (pool.includes(next)) break;
+      pool.push(next);
+    }
+    if (pool.length > 0) {
+      await switchToPage(pool[0]).catch((): null => null);
+    }
+    return pool;
+  }
   
   async function extractPostContent(): Promise<string> {
     const script = `
@@ -272,15 +338,23 @@ export async function execute(input: WeiboCollectFromLinksInput): Promise<WeiboC
   let persistedCount = 0;
   let totalComments = 0;
   let errors = 0;
+  let tabsUsed = 1;
   
   try {
-    const targetLinks = links.slice(0, targetCount);
-    
-    for (const link of targetLinks) {
+    const targetLinks = links.slice(0, Math.max(1, Number(targetCount || 0) || 1));
+    const tabPool = await ensureTabPool(tabCount).catch(() => [0]);
+    const roundRobinTabs = tabPool.length > 0 ? tabPool : [0];
+    tabsUsed = roundRobinTabs.length;
+    console.log(`[WeiboCollectFromLinks] tab pool ready: [${roundRobinTabs.join(', ')}]`);
+
+    for (let idx = 0; idx < targetLinks.length; idx += 1) {
+      const link = targetLinks[idx];
       processedCount++;
-      console.log(`[WeiboCollectFromLinks] Processing: ${link.statusId}`);
+      const tabIndex = roundRobinTabs[idx % roundRobinTabs.length];
+      console.log(`[WeiboCollectFromLinks] Processing: ${link.statusId} (tab=${tabIndex})`);
       
       try {
+        await switchToPage(tabIndex);
         await gotoUrl(link.safeUrl);
         let currentUrl = await getCurrentUrl();
         if (!currentUrl) {
@@ -322,6 +396,7 @@ export async function execute(input: WeiboCollectFromLinksInput): Promise<WeiboC
         postsProcessed: processedCount,
         totalComments,
         errors,
+        tabsUsed,
       },
     };
   } catch (error: any) {
@@ -331,7 +406,7 @@ export async function execute(input: WeiboCollectFromLinksInput): Promise<WeiboC
       linksPath,
       processedCount,
       persistedCount,
-      stats: { postsProcessed: processedCount, totalComments, errors },
+      stats: { postsProcessed: processedCount, totalComments, errors, tabsUsed },
       error: error.message,
     };
   }

package/modules/workflow/blocks/WeiboCollectSearchLinksBlock.ts

@@ -0,0 +1,316 @@
+/**
+ * Workflow Block: WeiboCollectSearchLinksBlock
+ *
+ * 阶段职责：
+ * 1) 仅从搜索结果页采集帖子链接（不做详情内容提取）
+ * 2) 持久化到 phase2-links.jsonl，供后续 WeiboCollectFromLinksBlock 消费
+ */
+
+import os from 'node:os';
+import path from 'node:path';
+import { promises as fs } from 'node:fs';
+
+export interface WeiboCollectSearchLinksInput {
+  sessionId: string;
+  keyword: string;
+  env?: string;
+  targetCount: number;
+  maxPages?: number;
+  serviceUrl?: string;
+}
+
+export interface WeiboCollectSearchLinksOutput {
+  success: boolean;
+  keywordDir: string;
+  linksPath: string;
+  collectedCount: number;
+  pagesVisited: number;
+  error?: string;
+}
+
+interface CollectedLink {
+  statusId: string;
+  userId: string;
+  safeUrl: string;
+  searchUrl: string;
+  authorName?: string;
+  contentPreview?: string;
+  ts: string;
+}
+
+function resolveDownloadRoot(): string {
+  const custom = process.env.WEBAUTO_DOWNLOAD_ROOT || process.env.WEBAUTO_DOWNLOAD_DIR;
+  if (custom && custom.trim()) return custom;
+  const home = process.env.HOME || process.env.USERPROFILE || os.homedir();
+  return path.join(home, '.webauto', 'download');
+}
+
+function sanitizeFilenamePart(value: string): string {
+  return String(value || '')
+    .trim()
+    .replace(/[\\/:"*?<>|]+/g, '_')
+    .replace(/\s+/g, '_')
+    .slice(0, 80);
+}
+
+function extractStatusId(url: string): string {
+  const text = String(url || '').trim();
+  if (!text) return '';
+  try {
+    const parsed = new URL(text);
+    const pathname = String(parsed.pathname || '');
+    const statusMatch = pathname.match(/\/status\/([^/?#]+)/i);
+    if (statusMatch?.[1]) return statusMatch[1];
+    const parts = pathname.split('/').filter(Boolean);
+    if (parts.length >= 2) {
+      return String(parts[parts.length - 1] || '').trim();
+    }
+  } catch {
+    // ignore
+  }
+  return '';
+}
+
+function extractUserId(authorUrl: string, safeUrl: string): string {
+  const first = String(authorUrl || '').trim();
+  const second = String(safeUrl || '').trim();
+  const fromUrl = (raw: string) => {
+    try {
+      const parsed = new URL(raw);
+      const m = String(parsed.pathname || '').match(/\/u\/([0-9]+)/i);
+      if (m?.[1]) return m[1];
+      const parts = String(parsed.pathname || '').split('/').filter(Boolean);
+      if (parts.length > 0 && /^[0-9]+$/.test(parts[0])) return parts[0];
+    } catch {
+      // ignore
+    }
+    return '';
+  };
+  return fromUrl(first) || fromUrl(second) || '';
+}
+
+export async function execute(input: WeiboCollectSearchLinksInput): Promise<WeiboCollectSearchLinksOutput> {
+  const {
+    sessionId,
+    keyword,
+    env = 'debug',
+    targetCount,
+    maxPages = 10,
+    serviceUrl = 'http://127.0.0.1:7704',
+  } = input;
+
+  const profile = sessionId;
+  const controllerUrl = `${serviceUrl}/command`;
+  const keywordDir = path.join(resolveDownloadRoot(), 'weibo', env, sanitizeFilenamePart(keyword));
+  const linksPath = path.join(keywordDir, 'phase2-links.jsonl');
+  const maxTarget = Math.max(1, Number(targetCount || 0) || 1);
+  const maxPageCount = Math.max(1, Number(maxPages || 0) || 1);
+
+  async function controllerAction(action: string, args: any = {}): Promise<any> {
+    const res = await fetch(controllerUrl, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({ action, args: { profileId: profile, ...args } }),
+      signal: (AbortSignal as any).timeout ? (AbortSignal as any).timeout(30000) : undefined,
+    });
+    const raw = await res.text();
+    if (!res.ok) throw new Error(`HTTP ${res.status}: ${raw}`);
+    try {
+      return raw ? JSON.parse(raw) : {};
+    } catch {
+      return { raw };
+    }
+  }
+
+  function unwrapResult(response: any): any {
+    if (response && typeof response === 'object') {
+      if ('result' in response) return response.result;
+      if (response.data && typeof response.data === 'object' && 'result' in response.data) {
+        return response.data.result;
+      }
+      if ('data' in response) return response.data;
+    }
+    return response;
+  }
+
+  async function readSearchRows(): Promise<{ rows: any[]; searchUrl: string }> {
+    const script = `
+      (() => {
+        const clean = (value) => String(value || '').replace(/\\s+/g, ' ').trim();
+        const abs = (value) => {
+          try {
+            return new URL(String(value || ''), location.href).toString();
+          } catch {
+            return '';
+          }
+        };
+        const rows = [];
+        const cards = Array.from(document.querySelectorAll('.card-wrap'));
+        for (const card of cards) {
+          const statusAnchor = card.querySelector('a[href*="/status/"]');
+          const safeUrl = abs(statusAnchor?.getAttribute?.('href') || statusAnchor?.href || '');
+          if (!safeUrl) continue;
+          const authorAnchor =
+            card.querySelector('a[href*="/u/"]') ||
+            card.querySelector('a[title][href*="weibo.com"]') ||
+            card.querySelector('.name a');
+          const authorUrl = abs(authorAnchor?.getAttribute?.('href') || authorAnchor?.href || '');
+          const authorName = clean(
+            authorAnchor?.getAttribute?.('title') ||
+            authorAnchor?.textContent ||
+            '',
+          );
+          const contentEl =
+            card.querySelector('.txt') ||
+            card.querySelector('[node-type="feed_list_content"]') ||
+            card.querySelector('.detail_wbtext') ||
+            card.querySelector('.wbtext');
+          const contentPreview = clean(contentEl?.textContent || '').slice(0, 180);
+          rows.push({
+            safeUrl,
+            authorUrl,
+            authorName,
+            contentPreview,
+          });
+        }
+        return { rows, searchUrl: String(location.href || '') };
+      })()
+    `;
+    const res = await controllerAction('evaluate', { script });
+    const value = unwrapResult(res);
+    return {
+      rows: Array.isArray(value?.rows) ? value.rows : [],
+      searchUrl: String(value?.searchUrl || ''),
+    };
+  }
+
+  async function findNextPageCenter(): Promise<{ ok: boolean; x?: number; y?: number }> {
+    const script = `
+      (() => {
+        const clean = (value) => String(value || '').replace(/\\s+/g, ' ').trim();
+        const isVisible = (node) => {
+          if (!(node instanceof HTMLElement)) return false;
+          const rect = node.getBoundingClientRect();
+          if (rect.width <= 0 || rect.height <= 0) return false;
+          if (rect.bottom <= 0 || rect.top >= window.innerHeight) return false;
+          const style = window.getComputedStyle(node);
+          if (!style) return false;
+          if (style.display === 'none' || style.visibility === 'hidden') return false;
+          if (Number(style.opacity || '1') === 0) return false;
+          return true;
+        };
+        const isDisabled = (node) => {
+          const text = clean(node.className || '');
+          if (text.includes('disable') || text.includes('disabled')) return true;
+          const ariaDisabled = String(node.getAttribute('aria-disabled') || '').toLowerCase();
+          return ariaDisabled === 'true';
+        };
+        const candidates = Array.from(document.querySelectorAll('a, button, span'))
+          .filter((node) => /下一页|下页/.test(clean(node.textContent || '')))
+          .filter((node) => isVisible(node) && !isDisabled(node));
+        const target = candidates[0];
+        if (!target) return { ok: false };
+        const rect = target.getBoundingClientRect();
+        return {
+          ok: true,
+          x: Math.round(rect.left + rect.width / 2),
+          y: Math.round(rect.top + rect.height / 2),
+        };
+      })()
+    `;
+    const res = await controllerAction('evaluate', { script });
+    const value = unwrapResult(res);
+    return {
+      ok: value?.ok === true,
+      x: Number(value?.x || 0),
+      y: Number(value?.y || 0),
+    };
+  }
+
+  async function writeLinks(rows: CollectedLink[]): Promise<void> {
+    await fs.mkdir(keywordDir, { recursive: true });
+    const jsonl = rows.map((row) => JSON.stringify(row)).join('\n');
+    await fs.writeFile(linksPath, jsonl ? `${jsonl}\n` : '', 'utf8');
+  }
+
+  const dedup = new Map<string, CollectedLink>();
+  let pagesVisited = 0;
+  let noProgressRounds = 0;
+
+  try {
+    for (let page = 0; page < maxPageCount; page += 1) {
+      pagesVisited += 1;
+      const snapshot = await readSearchRows();
+      let added = 0;
+
+      for (const row of snapshot.rows) {
+        const safeUrl = String(row?.safeUrl || '').trim();
+        if (!safeUrl) continue;
+        const statusId = extractStatusId(safeUrl);
+        if (!statusId) continue;
+        const key = statusId || safeUrl;
+        if (dedup.has(key)) continue;
+        dedup.set(key, {
+          statusId,
+          userId: extractUserId(String(row?.authorUrl || ''), safeUrl),
+          safeUrl,
+          searchUrl: String(snapshot.searchUrl || ''),
+          authorName: String(row?.authorName || '').trim() || undefined,
+          contentPreview: String(row?.contentPreview || '').trim() || undefined,
+          ts: new Date().toISOString(),
+        });
+        added += 1;
+        if (dedup.size >= maxTarget) break;
+      }
+
+      if (added === 0) noProgressRounds += 1;
+      else noProgressRounds = 0;
+
+      if (dedup.size >= maxTarget) break;
+      if (noProgressRounds >= 2) break;
+
+      let clicked = false;
+      for (let attempt = 0; attempt < 3; attempt += 1) {
+        const next = await findNextPageCenter();
+        if (!next.ok || !Number.isFinite(next.x) || !Number.isFinite(next.y)) {
+          await controllerAction('mouse:wheel', { deltaX: 0, deltaY: 900 }).catch((): null => null);
+          await new Promise((r) => setTimeout(r, 400));
+          continue;
+        }
+        await controllerAction('mouse:click', {
+          x: Math.round(next.x!),
+          y: Math.round(next.y!),
+          button: 'left',
+          clicks: 1,
+        });
+        clicked = true;
+        await new Promise((r) => setTimeout(r, 1500));
+        break;
+      }
+
+      if (!clicked) break;
+    }
+
+    const links = Array.from(dedup.values()).slice(0, maxTarget);
+    await writeLinks(links);
+
+    return {
+      success: true,
+      keywordDir,
+      linksPath,
+      collectedCount: links.length,
+      pagesVisited,
+    };
+  } catch (error: any) {
+    const links = Array.from(dedup.values()).slice(0, maxTarget);
+    await writeLinks(links).catch((): null => null);
+    return {
+      success: false,
+      keywordDir,
+      linksPath,
+      collectedCount: links.length,
+      pagesVisited,
+      error: `WeiboCollectSearchLinks failed: ${error.message}`,
+    };
+  }
+}

package/modules/workflow/definitions/weibo-search-workflow-v1.ts

@@ -40,6 +40,8 @@ export const weiboSearchWorkflowV1 = {
         targetCount: '$targetCount',
         maxComments: '$maxComments',
         collectComments: '$collectComments',
+        tabCount: '$tabCount',
+        tabOpenDelayMs: '$tabOpenDelayMs',
         serviceUrl: '$serviceUrl',
       },
     },

package/modules/workflow/src/runner.ts

@@ -17,6 +17,7 @@ import * as XiaohongshuCollectLinksBlock from '../blocks/XiaohongshuCollectLinks
 import * as XiaohongshuCollectFromLinksBlock from '../blocks/XiaohongshuCollectFromLinksBlock.js';
 import * as ErrorRecoveryBlock from '../blocks/ErrorRecoveryBlock.js';
 import * as ExecuteWeiboSearchBlock from '../blocks/ExecuteWeiboSearchBlock.js';
+import * as WeiboCollectSearchLinksBlock from '../blocks/WeiboCollectSearchLinksBlock.js';
 import * as WeiboCollectFromLinksBlock from '../blocks/WeiboCollectFromLinksBlock.js';
 import * as WeiboCollectCommentsBlock from '../blocks/WeiboCollectCommentsBlock.js';
 import * as AnchorVerificationBlock from '../blocks/AnchorVerificationBlock.js';
@@ -47,6 +48,7 @@ export function createDefaultWorkflowExecutor(): WorkflowExecutor {
   executor.registerBlock('XiaohongshuCollectFromLinksBlock', { execute: XiaohongshuCollectFromLinksBlock.execute });
   executor.registerBlock('OrganizeXhsNotesBlock', { execute: OrganizeXhsNotesBlock.execute });
   executor.registerBlock('ExecuteWeiboSearchBlock', { execute: ExecuteWeiboSearchBlock.execute });
+  executor.registerBlock('WeiboCollectSearchLinksBlock', { execute: WeiboCollectSearchLinksBlock.execute });
   executor.registerBlock('WeiboCollectFromLinksBlock', { execute: WeiboCollectFromLinksBlock.execute });
   executor.registerBlock('WeiboCollectCommentsBlock', { execute: WeiboCollectCommentsBlock.execute });
   return executor;