@web-auto/webauto 0.1.4 → 0.1.6

package/modules/camo-backend/src/internal/SessionManager.ts

@@ -134,7 +134,17 @@ export class SessionManager {
         (process as any).emit(SESSION_CLOSED_EVENT, id);
       }
     };
-    await session.start(options.initialUrl);
+    try {
+      await session.start(options.initialUrl);
+    } catch (err) {
+      this.debugLog('createSession:start_failed', {
+        profileId,
+        error: (err as Error)?.message || String(err),
+      });
+      session.onExit = undefined;
+      await session.close().catch(() => {});
+      throw err;
+    }
     this.sessions.set(profileId, session);
 
     this.debugLog('createSession:started', { profileId });
@@ -156,6 +166,7 @@ export class SessionManager {
       current_url: session.getCurrentUrl(),
       mode: session.modeName,
       owner_pid: this.owners.get(session.id)?.pid || null,
+      recording: session.getRecordingStatus(),
     }));
   }
 

package/modules/camo-backend/src/internal/storage-paths.ts

@@ -36,3 +36,8 @@ export function resolveLocksRoot() {
   return path.join(resolveDataRoot(), 'locks');
 }
 
+export function resolveRecordsRoot() {
+  const envRoot = String(process.env.WEBAUTO_PATHS_RECORDS || '').trim();
+  if (envRoot) return envRoot;
+  return path.join(resolveDataRoot(), 'records');
+}

package/modules/camo-runtime/src/autoscript/action-providers/xhs/comments.mjs

@@ -86,6 +86,43 @@ export function buildCommentsHarvestScript(params = {}) {
       }
       return null;
     };
+    const normalizeInlineText = (value) => String(value || '').replace(/\\s+/g, ' ').trim();
+    const sanitizeAuthorText = (raw, commentText = '') => {
+      const text = normalizeInlineText(raw);
+      if (!text) return '';
+      if (commentText && text === commentText) return '';
+      if (text.length > 40) return '';
+      if (/^(回复|展开|收起|查看更多|评论|赞|分享|发送)$/.test(text)) return '';
+      return text;
+    };
+    const readAuthor = (item, commentText = '') => {
+      const attrNames = ['data-user-name', 'data-username', 'data-user_nickname', 'data-nickname'];
+      for (const attr of attrNames) {
+        const value = sanitizeAuthorText(item.getAttribute?.(attr), commentText);
+        if (value) return value;
+      }
+      const selectors = [
+        '.comment-user .name',
+        '.comment-user .username',
+        '.comment-user .user-name',
+        '.author .name',
+        '.author',
+        '.user-name',
+        '.username',
+        '.name',
+        'a[href*="/user/profile/"]',
+        'a[href*="/user/"]',
+      ];
+      for (const selector of selectors) {
+        const node = item.querySelector(selector);
+        if (!node) continue;
+        const title = sanitizeAuthorText(node.getAttribute?.('title'), commentText);
+        if (title) return title;
+        const text = sanitizeAuthorText(node.textContent, commentText);
+        if (text) return text;
+      }
+      return '';
+    };
 
     const scroller = document.querySelector('.note-scroller')
       || document.querySelector('.comments-el')
@@ -105,9 +142,8 @@ export function buildCommentsHarvestScript(params = {}) {
       const nodes = Array.from(document.querySelectorAll('.comment-item, [class*="comment-item"]'));
       for (const item of nodes) {
         const textNode = item.querySelector('.content, .comment-content, p');
-        const authorNode = item.querySelector('.name, .author, .user-name, [class*="author"], [class*="name"]');
         const text = String((textNode && textNode.textContent) || '').trim();
-        const author = String((authorNode && authorNode.textContent) || '').trim();
+        const author = readAuthor(item, text);
         if (!text) continue;
         const key = author + '::' + text;
         if (commentMap.has(key)) continue;

package/modules/camo-runtime/src/autoscript/action-providers/xhs/interaction.mjs

@@ -74,6 +74,42 @@ function buildCollectLikeTargetsScript() {
       }
       return '';
     };
+    const sanitizeUserName = (raw, commentText = '') => {
+      const text = String(raw || '').replace(/\\s+/g, ' ').trim();
+      if (!text) return '';
+      if (commentText && text === commentText) return '';
+      if (text.length > 40) return '';
+      if (/^(回复|展开|收起|查看更多|评论|赞|分享|发送)$/.test(text)) return '';
+      return text;
+    };
+    const readUserName = (item, commentText = '') => {
+      const attrNames = ['data-user-name', 'data-username', 'data-user_nickname', 'data-nickname'];
+      for (const attr of attrNames) {
+        const value = sanitizeUserName(item.getAttribute?.(attr), commentText);
+        if (value) return value;
+      }
+      const selectors = [
+        '.comment-user .name',
+        '.comment-user .username',
+        '.comment-user .user-name',
+        '.author .name',
+        '.author',
+        '.user-name',
+        '.username',
+        '.name',
+        'a[href*="/user/profile/"]',
+        'a[href*="/user/"]',
+      ];
+      for (const selector of selectors) {
+        const node = item.querySelector(selector);
+        if (!node) continue;
+        const title = sanitizeUserName(node.getAttribute?.('title'), commentText);
+        if (title) return title;
+        const text = sanitizeUserName(node.textContent, commentText);
+        if (text) return text;
+      }
+      return '';
+    };
     const readAttr = (item, attrNames) => {
       for (const attr of attrNames) {
         const value = String(item.getAttribute?.(attr) || '').trim();
@@ -81,6 +117,15 @@ function buildCollectLikeTargetsScript() {
       }
       return '';
     };
+    const readUserId = (item) => {
+      const value = readAttr(item, ['data-user-id', 'data-userid', 'data-user_id']);
+      if (value) return value;
+      const anchor = item.querySelector('a[href*="/user/profile/"], a[href*="/user/"]');
+      const href = String(anchor?.getAttribute?.('href') || '').trim();
+      if (!href) return '';
+      const matched = href.match(/\\/user\\/(?:profile\\/)?([a-zA-Z0-9_-]+)/);
+      return matched && matched[1] ? matched[1] : '';
+    };
 
     const matchedSet = new Set(
       Array.isArray(state.matchedComments)
@@ -92,8 +137,8 @@ function buildCollectLikeTargetsScript() {
       const item = items[index];
       const text = readText(item, ['.content', '.comment-content', 'p']);
       if (!text) continue;
-      const userName = readText(item, ['.name', '.author', '.user-name', '.username', '[class*="author"]', '[class*="name"]']);
-      const userId = readAttr(item, ['data-user-id', 'data-userid', 'data-user_id']);
+      const userName = readUserName(item, text);
+      const userId = readUserId(item);
       const timestamp = readText(item, ['.date', '.time', '.timestamp', '[class*="time"]']);
       const likeControl = findLikeControl(item);
       rows.push({

package/modules/camo-runtime/src/autoscript/action-providers/xhs/search.mjs

@@ -54,6 +54,19 @@ export function buildOpenDetailScript(params = {}) {
   const mode = String(params.mode || 'first').trim().toLowerCase();
   const maxNotes = Math.max(1, Number(params.maxNotes ?? params.limit ?? 20) || 20);
   const keyword = String(params.keyword || '').trim();
+  const resume = params.resume !== false;
+  const incrementalMax = params.incrementalMax !== false;
+  const excludeNoteIds = Array.isArray(params.excludeNoteIds)
+    ? params.excludeNoteIds.map((item) => String(item || '').trim()).filter(Boolean)
+    : [];
+  const seedCollectCount = Math.max(0, Number(params.seedCollectCount || 0) || 0);
+  const seedCollectMaxRounds = Math.max(0, Number(params.seedCollectMaxRounds || 0) || 0);
+  const seedCollectStep = Math.max(120, Number(params.seedCollectStep || 360) || 360);
+  const seedCollectSettleMs = Math.max(100, Number(params.seedCollectSettleMs || 260) || 260);
+  const seedResetToTop = params.seedResetToTop !== false;
+  const nextSeekRounds = Math.max(0, Number(params.nextSeekRounds || 8) || 8);
+  const nextSeekStep = Math.max(0, Number(params.nextSeekStep || 0) || 0);
+  const nextSeekSettleMs = Math.max(120, Number(params.nextSeekSettleMs || 320) || 320);
 
   return `(async () => {
     const STATE_KEY = '__camoXhsState';
@@ -91,36 +104,103 @@ export function buildOpenDetailScript(params = {}) {
     const requestedKeyword = ${JSON.stringify(keyword)};
     const mode = ${JSON.stringify(mode)};
     const previousKeyword = String(state.keyword || '').trim();
+    const keywordChanged = Boolean(requestedKeyword && previousKeyword && requestedKeyword !== previousKeyword);
     if (mode === 'first') {
+      if (!${resume ? 'true' : 'false'} || keywordChanged) {
+        state.visitedNoteIds = [];
+      }
+    } else if (keywordChanged) {
       state.visitedNoteIds = [];
-    } else if (requestedKeyword && previousKeyword && requestedKeyword !== previousKeyword) {
-      state.visitedNoteIds = [];
     }
-    state.maxNotes = Number(${maxNotes});
+    const requestedMaxNotes = Number(${maxNotes});
+    if (mode === 'first') {
+      if (${incrementalMax ? 'true' : 'false'} && ${resume ? 'true' : 'false'} && !keywordChanged) {
+        state.maxNotes = Number(state.visitedNoteIds.length || 0) + requestedMaxNotes;
+      } else {
+        state.maxNotes = requestedMaxNotes;
+      }
+    } else if (!Number.isFinite(Number(state.maxNotes)) || Number(state.maxNotes) <= 0) {
+      state.maxNotes = requestedMaxNotes;
+    }
     if (requestedKeyword) state.keyword = requestedKeyword;
 
     if (mode === 'next' && state.visitedNoteIds.length >= state.maxNotes) {
       throw new Error('AUTOSCRIPT_DONE_MAX_NOTES');
     }
 
-    const nodes = Array.from(document.querySelectorAll('.note-item'))
+    const excludedNoteIds = new Set(${JSON.stringify(excludeNoteIds)});
+    const mapNodes = () => Array.from(document.querySelectorAll('.note-item'))
       .map((item, index) => {
         const cover = item.querySelector('a.cover');
         if (!cover) return null;
         const href = String(cover.getAttribute('href') || '').trim();
-        const noteId = href.split('/').filter(Boolean).pop() || ('idx_' + index);
+        const lastSegment = href.split('/').filter(Boolean).pop() || '';
+        const normalized = lastSegment.split('?')[0].split('#')[0];
+        const noteId = normalized || ('idx_' + index);
         return { cover, href, noteId };
       })
       .filter(Boolean);
+    let nodes = mapNodes();
+    const seedCollectedSet = new Set();
+    const seedCollectEnabled = mode === 'first'
+      && Number(${seedCollectCount}) > 0
+      && Number(${seedCollectMaxRounds}) > 0;
+    if (seedCollectEnabled) {
+      const collectVisible = () => {
+        for (const row of mapNodes()) {
+          if (!row || !row.noteId) continue;
+          seedCollectedSet.add(row.noteId);
+        }
+      };
+      collectVisible();
+      const maxRounds = Number(${seedCollectMaxRounds});
+      const targetCount = Number(${seedCollectCount});
+      for (let round = 0; round < maxRounds && seedCollectedSet.size < targetCount; round += 1) {
+        window.scrollBy({ top: Number(${seedCollectStep}), left: 0, behavior: 'auto' });
+        await new Promise((resolve) => setTimeout(resolve, Number(${seedCollectSettleMs})));
+        collectVisible();
+      }
+      if (${seedResetToTop ? 'true' : 'false'}) {
+        window.scrollTo({ top: 0, behavior: 'auto' });
+        await new Promise((resolve) => setTimeout(resolve, Number(${seedCollectSettleMs})));
+      }
+      nodes = mapNodes();
+    }
     if (nodes.length === 0) throw new Error('NO_SEARCH_RESULT_ITEM');
 
-    let next = null;
-    if (mode === 'next') {
-      next = nodes.find((row) => !state.visitedNoteIds.includes(row.noteId));
-      if (!next) throw new Error('AUTOSCRIPT_DONE_NO_MORE_NOTES');
-    } else {
-      next = nodes.find((row) => !state.visitedNoteIds.includes(row.noteId)) || nodes[0];
+    const isEligible = (row) => (
+      row
+      && row.noteId
+      && !excludedNoteIds.has(row.noteId)
+      && !state.visitedNoteIds.includes(row.noteId)
+    );
+    const resolveSeekStep = () => {
+      const configured = Number(${nextSeekStep});
+      if (Number.isFinite(configured) && configured > 0) return configured;
+      const viewportHeight = Math.max(
+        Number(window.innerHeight || 0) || 0,
+        Number(document.documentElement?.clientHeight || 0) || 0,
+      );
+      return Math.max(240, Math.floor(viewportHeight * 0.9));
+    };
+    const seekStep = resolveSeekStep();
+
+    let next = nodes.find((row) => isEligible(row));
+    if (!next) {
+      let stagnantRounds = 0;
+      for (let round = 0; !next && round < Number(${nextSeekRounds}); round += 1) {
+        const beforeTop = window.scrollY || document.documentElement.scrollTop || document.body.scrollTop || 0;
+        window.scrollBy({ top: seekStep, left: 0, behavior: 'auto' });
+        await new Promise((resolve) => setTimeout(resolve, Number(${nextSeekSettleMs})));
+        nodes = mapNodes();
+        next = nodes.find((row) => isEligible(row));
+        const afterTop = window.scrollY || document.documentElement.scrollTop || document.body.scrollTop || 0;
+        if (Math.abs(afterTop - beforeTop) < 2) stagnantRounds += 1;
+        else stagnantRounds = 0;
+        if (stagnantRounds >= 2) break;
+      }
     }
+    if (!next) throw new Error('AUTOSCRIPT_DONE_NO_MORE_NOTES');
 
     const detailSelectors = [
       '.note-detail-mask',
@@ -176,6 +256,9 @@ export function buildOpenDetailScript(params = {}) {
       openByClick: true,
       beforeUrl,
       afterUrl,
+      excludedCount: excludedNoteIds.size,
+      seedCollectedCount: seedCollectedSet.size,
+      seedCollectedNoteIds: Array.from(seedCollectedSet),
     };
   })()`;
 }

package/modules/camo-runtime/src/autoscript/action-providers/xhs.mjs

@@ -1,3 +1,5 @@
+import path from 'node:path';
+import fsp from 'node:fs/promises';
 import { asErrorPayload } from '../../container/runtime-core/utils.mjs';
 import {
   createEvaluateHandler,
@@ -21,6 +23,210 @@ import {
 } from './xhs/persistence.mjs';
 import { buildOpenDetailScript, buildSubmitSearchScript } from './xhs/search.mjs';
 
+const XHS_OPERATION_LOCKS = new Map();
+
+function toLockKey(text, fallback = '') {
+  const value = String(text || '').trim();
+  return value || fallback;
+}
+
+async function withSerializedLock(lockKey, fn) {
+  const key = toLockKey(lockKey);
+  if (!key) return fn();
+  const previous = XHS_OPERATION_LOCKS.get(key) || Promise.resolve();
+  let release;
+  const gate = new Promise((resolve) => {
+    release = resolve;
+  });
+  XHS_OPERATION_LOCKS.set(key, previous.catch(() => null).then(() => gate));
+  await previous.catch(() => null);
+  try {
+    return await fn();
+  } finally {
+    release();
+    if (XHS_OPERATION_LOCKS.get(key) === gate) XHS_OPERATION_LOCKS.delete(key);
+  }
+}
+
+function normalizeNoteIdList(items) {
+  if (!Array.isArray(items)) return [];
+  const out = [];
+  const seen = new Set();
+  for (const item of items) {
+    const noteId = String(item || '').trim();
+    if (!noteId || seen.has(noteId)) continue;
+    seen.add(noteId);
+    out.push(noteId);
+  }
+  return out;
+}
+
+function resolveSharedClaimPath(params = {}) {
+  const raw = String(params.sharedHarvestPath || params.sharedClaimPath || '').trim();
+  return raw ? path.resolve(raw) : '';
+}
+
+function resolveSearchLockKey(params = {}) {
+  const raw = String(params.searchSerialKey || params.searchLockKey || '').trim();
+  if (raw) return raw;
+  const claimPath = resolveSharedClaimPath(params);
+  return claimPath ? `claim:${claimPath}` : '';
+}
+
+async function loadSharedClaimDoc(filePath) {
+  if (!filePath) {
+    return { noteIds: [], byNoteId: {}, updatedAt: null };
+  }
+  try {
+    const raw = await fsp.readFile(filePath, 'utf8');
+    const parsed = JSON.parse(raw);
+    const noteIds = normalizeNoteIdList(parsed?.noteIds);
+    const byNoteId = parsed?.byNoteId && typeof parsed.byNoteId === 'object' ? parsed.byNoteId : {};
+    return {
+      noteIds,
+      byNoteId,
+      updatedAt: parsed?.updatedAt || null,
+    };
+  } catch {
+    return { noteIds: [], byNoteId: {}, updatedAt: null };
+  }
+}
+
+async function saveSharedClaimDoc(filePath, doc) {
+  if (!filePath) return;
+  const noteIds = normalizeNoteIdList(doc?.noteIds);
+  const payload = {
+    updatedAt: new Date().toISOString(),
+    noteIds,
+    byNoteId: doc?.byNoteId && typeof doc.byNoteId === 'object' ? doc.byNoteId : {},
+  };
+  await fsp.mkdir(path.dirname(filePath), { recursive: true });
+  await fsp.writeFile(filePath, `${JSON.stringify(payload, null, 2)}\n`, 'utf8');
+}
+
+async function executeSubmitSearchOperation({ profileId, params = {} }) {
+  const script = buildSubmitSearchScript(params);
+  const highlight = params.highlight !== false;
+  const lockKey = resolveSearchLockKey(params);
+  return withSerializedLock(lockKey ? `xhs_submit_search:${lockKey}` : '', () => evaluateWithScript({
+    profileId,
+    script,
+    message: 'xhs_submit_search done',
+    highlight,
+  }));
+}
+
+async function executeOpenDetailOperation({ profileId, params = {} }) {
+  const highlight = params.highlight !== false;
+  const claimPath = resolveSharedClaimPath(params);
+  const lockKey = claimPath ? `xhs_open_detail:${claimPath}` : '';
+
+  const mapOpenDetailError = (err, paramsRef = {}) => {
+    const message = String(err?.message || err || '');
+    const mode = String(paramsRef?.mode || '').trim().toLowerCase();
+    if (message.includes('AUTOSCRIPT_DONE_NO_MORE_NOTES')) {
+      return {
+        ok: true,
+        code: 'AUTOSCRIPT_DONE_NO_MORE_NOTES',
+        message: 'no more notes',
+        data: { stopReason: 'no_more_notes' },
+      };
+    }
+    if (message.includes('NO_SEARCH_RESULT_ITEM')) {
+      if (mode === 'first') return null;
+      return {
+        ok: true,
+        code: 'OPERATION_SKIPPED_NO_SEARCH_RESULT_ITEM',
+        message: 'search result item missing',
+        data: { skipped: true },
+      };
+    }
+    return null;
+  };
+
+  const runWithExclude = async (excludeNoteIds) => {
+    const script = buildOpenDetailScript({
+      ...params,
+      excludeNoteIds,
+    });
+    const operationResult = await evaluateWithScript({
+      profileId,
+      script,
+      message: 'xhs_open_detail done',
+      highlight,
+    });
+    const payload = extractEvaluateResultData(operationResult.data) || {};
+    return {
+      operationResult,
+      payload: payload && typeof payload === 'object' ? payload : {},
+    };
+  };
+
+  if (!claimPath) {
+    try {
+      const { operationResult } = await runWithExclude([]);
+      return operationResult;
+    } catch (err) {
+      const mapped = mapOpenDetailError(err, params);
+      if (mapped) return mapped;
+      throw err;
+    }
+  }
+
+  const runLocked = async () => {
+    const claimDoc = await loadSharedClaimDoc(claimPath);
+    const excludeNoteIds = normalizeNoteIdList(claimDoc.noteIds);
+    const { operationResult, payload } = await runWithExclude(excludeNoteIds);
+
+    const claimSet = new Set(excludeNoteIds);
+    const claimAdded = [];
+    const markClaim = (noteId, source = 'open_detail') => {
+      const id = String(noteId || '').trim();
+      if (!id || claimSet.has(id)) return;
+      claimSet.add(id);
+      claimAdded.push(id);
+      claimDoc.byNoteId[id] = {
+        noteId: id,
+        profileId,
+        source,
+        ts: new Date().toISOString(),
+      };
+    };
+
+    const seeded = normalizeNoteIdList(payload.seedCollectedNoteIds);
+    for (const noteId of seeded) markClaim(noteId, 'seed_collect');
+    if (payload.opened === true) markClaim(payload.noteId, 'open_detail');
+    claimDoc.noteIds = Array.from(claimSet);
+    if (claimAdded.length > 0) {
+      await saveSharedClaimDoc(claimPath, claimDoc);
+    }
+
+    const mergedPayload = {
+      ...payload,
+      sharedClaimPath: claimPath,
+      sharedClaimCount: claimDoc.noteIds.length,
+      sharedClaimAdded: claimAdded,
+      dedupExcluded: excludeNoteIds.length,
+    };
+    const mergedData = operationResult.data && typeof operationResult.data === 'object'
+      ? { ...operationResult.data, result: mergedPayload }
+      : { result: mergedPayload };
+
+    return {
+      ...operationResult,
+      data: mergedData,
+    };
+  };
+
+  try {
+    return await withSerializedLock(lockKey, runLocked);
+  } catch (err) {
+    const mapped = mapOpenDetailError(err, params);
+    if (mapped) return mapped;
+    throw err;
+  }
+}
+
 function buildReadStateScript() {
   return `(() => {
     const state = window.__camoXhsState || {};
@@ -108,8 +314,8 @@ async function executeCommentsHarvestOperation({ profileId, params = {} }) {
 
 const XHS_ACTION_HANDLERS = {
   raise_error: handleRaiseError,
-  xhs_submit_search: createEvaluateHandler('xhs_submit_search done', buildSubmitSearchScript),
-  xhs_open_detail: createEvaluateHandler('xhs_open_detail done', buildOpenDetailScript),
+  xhs_submit_search: executeSubmitSearchOperation,
+  xhs_open_detail: executeOpenDetailOperation,
   xhs_detail_harvest: createEvaluateHandler('xhs_detail_harvest done', buildDetailHarvestScript),
   xhs_expand_replies: createEvaluateHandler('xhs_expand_replies done', buildExpandRepliesScript),
   xhs_comments_harvest: executeCommentsHarvestOperation,

package/modules/camo-runtime/src/autoscript/runtime.mjs

@@ -359,6 +359,11 @@ export class AutoscriptRunner {
 
   resolveTimeoutMs(operation) {
     const pacing = this.resolvePacing(operation);
+    const disableTimeout = Boolean(
+      operation?.disableTimeout ?? this.script?.defaults?.disableTimeout,
+    );
+    if (disableTimeout) return 0;
+    if (pacing.timeoutMs === 0) return 0;
     if (Number.isFinite(pacing.timeoutMs) && pacing.timeoutMs > 0) return pacing.timeoutMs;
     return this.getDefaultTimeoutMs(operation);
   }
@@ -778,8 +783,9 @@ export class AutoscriptRunner {
         };
       }
 
+      const failureStatus = operation?.onFailure === 'continue' ? 'skipped' : 'failed';
       this.operationState.set(operation.id, {
-        status: 'failed',
+        status: failureStatus,
         runs: Number(this.operationState.get(operation.id)?.runs || 0) + 1,
         lastError: result.message || 'operation failed',
         updatedAt: nowIso(),