@web-auto/webauto 0.1.4 → 0.1.6

package/dist/modules/xiaohongshu/app/src/xiaohongshu/app/src/blocks/helpers/sharding.js

@@ -0,0 +1,165 @@
+import path from 'node:path';
+import os from 'node:os';
+import fs from 'node:fs/promises';
+function sanitizeForPath(name, fallback = 'unknown') {
+    const text = String(name || '').trim();
+    if (!text)
+        return fallback;
+    const cleaned = text.replace(/[\\/:"*?<>|]+/g, '_').trim();
+    return cleaned || fallback;
+}
+function asStringArray(input) {
+    if (!Array.isArray(input))
+        return [];
+    return input
+        .map((item) => String(item || '').trim())
+        .filter(Boolean);
+}
+function normalizeCollectedUrls(input, completedNoteIds) {
+    if (!Array.isArray(input))
+        return [];
+    const out = [];
+    const seen = new Set();
+    for (const row of input) {
+        if (!row || typeof row !== 'object')
+            continue;
+        const noteId = String(row.noteId || '').trim();
+        if (!noteId || completedNoteIds.has(noteId) || seen.has(noteId))
+            continue;
+        seen.add(noteId);
+        const safeUrl = String(row.safeUrl || '').trim();
+        const searchUrl = String(row.searchUrl || '').trim();
+        const timestampRaw = Number(row.timestamp);
+        const item = {
+            noteId,
+            safeUrl,
+            ...(searchUrl ? { searchUrl } : {}),
+            ...(Number.isFinite(timestampRaw) ? { timestamp: timestampRaw } : {}),
+        };
+        out.push(item);
+    }
+    return out;
+}
+async function resolveStatePath(input) {
+    const root = resolveDownloadRoot(input.downloadRoot);
+    const sanitizedEnv = sanitizeForPath(input.env, 'debug');
+    const sanitizedKeyword = sanitizeForPath(input.keyword, 'unknown');
+    const rawEnv = String(input.env || '').trim();
+    const rawKeyword = String(input.keyword || '').trim();
+    const candidates = [
+        path.join(root, 'xiaohongshu', sanitizedEnv, sanitizedKeyword, '.collect-state.json'),
+    ];
+    const legacyPath = path.join(root, 'xiaohongshu', rawEnv, rawKeyword, '.collect-state.json');
+    if (legacyPath !== candidates[0])
+        candidates.push(legacyPath);
+    for (const filePath of candidates) {
+        try {
+            await fs.access(filePath);
+            return filePath;
+        }
+        catch {
+            // continue
+        }
+    }
+    return null;
+}
+async function getXhsPendingItems(input) {
+    const statePath = await resolveStatePath(input);
+    if (!statePath)
+        return [];
+    try {
+        const content = await fs.readFile(statePath, 'utf8');
+        const state = JSON.parse(content);
+        const completed = new Set(asStringArray(state?.detailCollection?.completedNoteIds));
+        return normalizeCollectedUrls(state?.listCollection?.collectedUrls, completed);
+    }
+    catch (error) {
+        const code = String(error?.code || '');
+        const kind = code === 'ENOENT' ? 'missing_state' : code === 'EACCES' ? 'access_denied' : 'invalid_state';
+        console.warn(`[xhs.sharding] failed to load pending items (${kind}) from ${statePath}: ${error?.message || String(error)}`);
+        return [];
+    }
+}
+export function fnv1a32(input) {
+    // FNV-1a 32-bit
+    let hash = 0x811c9dc5;
+    const str = String(input || '');
+    for (let i = 0; i < str.length; i += 1) {
+        hash ^= str.charCodeAt(i);
+        // hash *= 16777619 (with 32-bit overflow)
+        hash = (hash + ((hash << 1) + (hash << 4) + (hash << 7) + (hash << 8) + (hash << 24))) >>> 0;
+    }
+    return hash >>> 0;
+}
+export function normalizeShard(spec) {
+    if (!spec)
+        return null;
+    const count = Math.max(1, Math.floor(Number(spec.count)));
+    const index = Math.max(0, Math.floor(Number(spec.index)));
+    if (!Number.isFinite(count) || !Number.isFinite(index))
+        return null;
+    if (count <= 1)
+        return null;
+    if (index >= count)
+        return null;
+    return { index, count, by: spec.by || 'noteId-hash' };
+}
+export function shardFilterByNoteIdHash(items, shard) {
+    const list = Array.isArray(items) ? items : [];
+    if (shard.count <= 1)
+        return list;
+    return list.filter((it) => {
+        const id = String(it?.noteId || '').trim();
+        if (!id)
+            return false;
+        return fnv1a32(id) % shard.count === shard.index;
+    });
+}
+export function shardFilterByIndexMod(items, shard) {
+    const list = Array.isArray(items) ? items : [];
+    if (shard.count <= 1)
+        return list;
+    return list.filter((_, idx) => idx % shard.count === shard.index);
+}
+export function resolveDownloadRoot(customRoot) {
+    const fromArg = String(customRoot || '').trim();
+    if (fromArg)
+        return path.resolve(fromArg);
+    const fromEnv = String(process.env.WEBAUTO_DOWNLOAD_ROOT || process.env.WEBAUTO_DOWNLOAD_DIR || '').trim();
+    if (fromEnv)
+        return path.resolve(fromEnv);
+    const home = process.env.HOME || process.env.USERPROFILE || os.homedir();
+    return path.join(home, '.webauto', 'download');
+}
+export async function buildDynamicShardPlan(input) {
+    const { keyword, env, downloadRoot, validProfiles } = input;
+    if (!validProfiles.length)
+        return [];
+    // Load pending items (not completed yet)
+    const pendingItems = await getXhsPendingItems({ keyword, env, downloadRoot });
+    if (pendingItems.length === 0) {
+        return validProfiles.map((profileId) => ({ profileId, assignedNoteIds: [], totalPending: 0 }));
+    }
+    // Calculate how many notes each profile should handle
+    // Using ceil to ensure we cover all pending items
+    const perProfileCount = Math.ceil(pendingItems.length / validProfiles.length);
+    const plans = [];
+    for (let i = 0; i < validProfiles.length; i++) {
+        const profileId = validProfiles[i];
+        const startIdx = i * perProfileCount;
+        const endIdx = Math.min(startIdx + perProfileCount, pendingItems.length);
+        const assignedNoteIds = pendingItems.slice(startIdx, endIdx).map((item) => item.noteId);
+        plans.push({
+            profileId,
+            assignedNoteIds,
+            totalPending: pendingItems.length,
+        });
+    }
+    return plans;
+}
+export async function getPendingItemsByNoteIds(input) {
+    const pendingItems = await getXhsPendingItems({ keyword: input.keyword, env: input.env, downloadRoot: input.downloadRoot });
+    const noteIdSet = new Set(input.noteIds);
+    return pendingItems.filter((item) => noteIdSet.has(item.noteId));
+}
+//# sourceMappingURL=sharding.js.map

package/dist/modules/xiaohongshu/app/src/xiaohongshu/app/src/blocks/helpers/xhsComments.d.ts

@@ -0,0 +1,33 @@
+export type XhsExtractedComment = {
+    user_id?: string;
+    user_name?: string;
+    text?: string;
+    timestamp?: string;
+    like_status?: string;
+};
+export declare function ensureCommentsOpened(sessionId: string, apiUrl: string): Promise<void>;
+export declare function isCommentEnd(sessionId: string, apiUrl: string): Promise<boolean>;
+/**
+ * 往返滚动检测（风控友好）：
+ * - 下滚无变化 -> 上滚一点 -> 再下滚
+ * - 往返 maxRounds 次仍无变化：判定到底（或评论区无法继续加载）
+ */
+export declare function checkBottomWithBackAndForth(sessionId: string, apiUrl: string, maxRounds?: number): Promise<{
+    reachedBottom: boolean;
+    reason: string;
+}>;
+export declare function extractVisibleComments(sessionId: string, apiUrl: string, maxItems: number): Promise<Array<XhsExtractedComment & {
+    domIndex: number;
+}>>;
+export declare function highlightCommentRow(sessionId: string, index: number, apiUrl: string, channel?: string): Promise<any>;
+export declare function scrollComments(sessionId: string, apiUrl: string, distance?: number): Promise<any>;
+export declare function expandAllVisibleReplyButtons(sessionId: string, apiUrl: string, options?: {
+    maxPasses?: number;
+    maxClicksPerPass?: number;
+}): Promise<{
+    clicked: number;
+    passes: number;
+    remaining: number;
+    detected: number;
+}>;
+//# sourceMappingURL=xhsComments.d.ts.map

package/dist/modules/xiaohongshu/app/src/xiaohongshu/app/src/blocks/helpers/xhsComments.js

@@ -0,0 +1,270 @@
+import { controllerAction, delay } from '../../utils/controllerAction.js';
+export async function ensureCommentsOpened(sessionId, apiUrl) {
+    await controllerAction('container:operation', {
+        containerId: 'xiaohongshu_detail.comment_button',
+        operationId: 'highlight',
+        sessionId,
+        timeoutMs: 15000,
+        config: { duration: 1800, channel: 'xhs-comments' },
+    }, apiUrl).catch(() => null);
+    await controllerAction('container:operation', { containerId: 'xiaohongshu_detail.comment_button', operationId: 'click', sessionId, timeoutMs: 15000 }, apiUrl).catch(() => { });
+    await delay(1200);
+}
+export async function isCommentEnd(sessionId, apiUrl) {
+    const end = await controllerAction('container:operation', { containerId: 'xiaohongshu_detail.comment_section.end_marker', operationId: 'extract', sessionId, timeoutMs: 12000 }, apiUrl).catch(() => null);
+    if (end && end?.success === true) {
+        const extracted = Array.isArray(end?.extracted) ? end.extracted : [];
+        if (extracted.length > 0)
+            return true;
+    }
+    // 空评论标记："这是一片荒地"
+    try {
+        const res = await controllerAction('browser:execute', {
+            profile: sessionId,
+            timeoutMs: 12000,
+            script: `(() => {
+          const candidates = [
+            'p.no-comments-text',
+            '.no-comments-text',
+            '[class*="no-comments"]',
+            '[class*="no-comment"]',
+            '[class*="empty-comment"]',
+          ];
+          for (const sel of candidates) {
+            const el = document.querySelector(sel);
+            const t = (el?.textContent || '').replace(/\s+/g, ' ').trim();
+            if (t && (t.includes('这是一片荒地') || t.includes('荒地'))) return true;
+          }
+          return false;
+        })()`,
+        }, apiUrl);
+        if (res?.result === true)
+            return true;
+    }
+    catch {
+        // ignore
+    }
+    return false;
+}
+async function getVisibleCommentSignature(sessionId, apiUrl) {
+    try {
+        const res = await controllerAction('browser:execute', {
+            profile: sessionId,
+            timeoutMs: 12000,
+            script: `(() => {
+          const isVisible = (el) => {
+            const r = el.getBoundingClientRect();
+            return r.width > 0 && r.height > 0 && r.bottom > 0 && r.top < window.innerHeight;
+          };
+          const items = Array.from(document.querySelectorAll('.comment-item')).filter(isVisible);
+          const texts = items
+            .map((el) => {
+              const t = el.querySelector('.content')?.textContent || el.querySelector('p')?.textContent || '';
+              return t.replace(/\s+/g, ' ').trim();
+            })
+            .filter(Boolean)
+            .slice(0, 6);
+          return { count: items.length, head: texts.join('|').slice(0, 200) };
+        })()`,
+        }, apiUrl);
+        return {
+            count: Number(res?.result?.count ?? -1),
+            head: String(res?.result?.head ?? ''),
+        };
+    }
+    catch {
+        return { count: -1, head: '' };
+    }
+}
+/**
+ * 往返滚动检测（风控友好）：
+ * - 下滚无变化 -> 上滚一点 -> 再下滚
+ * - 往返 maxRounds 次仍无变化：判定到底（或评论区无法继续加载）
+ */
+export async function checkBottomWithBackAndForth(sessionId, apiUrl, maxRounds = 3) {
+    let prev = await getVisibleCommentSignature(sessionId, apiUrl);
+    for (let i = 0; i < maxRounds; i += 1) {
+        await scrollComments(sessionId, apiUrl, 420);
+        await delay(800);
+        const afterDown = await getVisibleCommentSignature(sessionId, apiUrl);
+        if (afterDown.count !== prev.count || afterDown.head !== prev.head) {
+            return { reachedBottom: false, reason: 'down_changed' };
+        }
+        // 往回滚动几次再尝试向下，防止卡住
+        await scrollComments(sessionId, apiUrl, -240);
+        await delay(500);
+        await scrollComments(sessionId, apiUrl, 240);
+        await delay(700);
+        const afterBounce = await getVisibleCommentSignature(sessionId, apiUrl);
+        if (afterBounce.count !== prev.count || afterBounce.head !== prev.head) {
+            return { reachedBottom: false, reason: 'bounce_changed' };
+        }
+        prev = afterBounce;
+    }
+    return { reachedBottom: true, reason: 'no_change_after_back_and_forth' };
+}
+export async function extractVisibleComments(sessionId, apiUrl, maxItems) {
+    const res = await controllerAction('container:operation', {
+        containerId: 'xiaohongshu_detail.comment_section.comment_item',
+        operationId: 'extract',
+        sessionId,
+        timeoutMs: 15000,
+        config: { max_items: Math.max(1, Math.min(80, Math.floor(maxItems))), visibleOnly: true },
+    }, apiUrl);
+    if (!res?.success)
+        return [];
+    const extracted = Array.isArray(res?.extracted) ? res.extracted : [];
+    // Map extracted items with their DOM index (position in selectorAll result)
+    const containerRes = await controllerAction('browser:execute', {
+        profile: sessionId,
+        timeoutMs: 12000,
+        script: `(() => {
+        const items = Array.from(document.querySelectorAll('.comment-item'));
+        return items.map((el, idx) => {
+          const rect = el.getBoundingClientRect();
+          return {
+            domIndex: idx,
+            visible: rect.top >= 0 && rect.bottom <= window.innerHeight && rect.height > 0
+          };
+        }).filter(x => x.visible);
+      })()`,
+    }, apiUrl);
+    const visibleIndices = (containerRes?.result || []).map((x) => x.domIndex);
+    return extracted.map((item, idx) => ({ ...item, domIndex: visibleIndices[idx] ?? idx }));
+}
+export async function highlightCommentRow(sessionId, index, apiUrl, channel = 'xhs-comment-row') {
+    return controllerAction('container:operation', {
+        containerId: 'xiaohongshu_detail.comment_section.comment_item',
+        operationId: 'highlight',
+        sessionId,
+        config: {
+            index,
+            target: 'self',
+            style: '6px solid #ff00ff',
+            duration: 8000,
+            channel,
+            visibleOnly: true,
+        },
+    }, apiUrl);
+}
+export async function scrollComments(sessionId, apiUrl, distance = 650) {
+    return controllerAction('container:operation', {
+        containerId: 'xiaohongshu_detail.comment_section',
+        operationId: 'scroll',
+        sessionId,
+        timeoutMs: 15000,
+        config: { direction: 'down', distance: Math.max(60, Math.min(800, Math.floor(distance))) },
+    }, apiUrl);
+}
+export async function expandAllVisibleReplyButtons(sessionId, apiUrl, options = {}) {
+    const maxPasses = Math.max(1, Math.min(12, Number(options.maxPasses || 6)));
+    const maxClicksPerPass = Math.max(1, Math.min(30, Number(options.maxClicksPerPass || 12)));
+    const probeTargets = async () => {
+        const probe = await controllerAction('browser:execute', {
+            profile: sessionId,
+            timeoutMs: 12000,
+            script: `(() => {
+          const root =
+            document.querySelector('.comments-el') ||
+            document.querySelector('.comment-list') ||
+            document.querySelector('.comments-container') ||
+            document.querySelector('[class*="comment-section"]') ||
+            document.body;
+
+          const selector = '.show-more, .reply-expand, [class*="show-more"], [class*="expand"]';
+          const nodes = Array.from(root.querySelectorAll(selector));
+          const viewportH = window.innerHeight || 0;
+          const viewportW = window.innerWidth || 0;
+
+          const targets = [];
+
+          const isVisible = (el) => {
+            const r = el.getBoundingClientRect();
+            return r.width > 0 && r.height > 0 && r.bottom > 0 && r.top < viewportH && r.right > 0 && r.left < viewportW;
+          };
+
+          for (const el of nodes) {
+            if (!(el instanceof HTMLElement)) continue;
+            if (!isVisible(el)) continue;
+
+            const text = (el.textContent || '').replace(/\s+/g, ' ').trim();
+            if (!text || !text.includes('展开')) continue;
+            if (text.includes('收起') || text.includes('折叠')) continue;
+            if (!(text.includes('回复') || text.includes('评论') || text.includes('更多'))) continue;
+
+            const r = el.getBoundingClientRect();
+            const points = [
+              { x: Math.round(r.left + r.width * 0.72), y: Math.round(r.top + r.height * 0.55) },
+              { x: Math.round(r.left + r.width * 0.55), y: Math.round(r.top + r.height * 0.55) },
+              { x: Math.round(r.left + r.width * 0.85), y: Math.round(r.top + r.height * 0.5) },
+            ];
+
+            let picked = null;
+            for (const p of points) {
+              if (!Number.isFinite(p.x) || !Number.isFinite(p.y)) continue;
+              if (p.x < 8 || p.y < 8 || p.x > viewportW - 8 || p.y > viewportH - 8) continue;
+              const hit = document.elementFromPoint(p.x, p.y);
+              if (!hit || !(hit instanceof Element)) continue;
+              if (!(hit === el || el.contains(hit) || hit.contains(el))) continue;
+              if (hit.closest && hit.closest('a[href]')) continue;
+              picked = p;
+              break;
+            }
+
+            if (!picked) continue;
+            targets.push({ x: picked.x, y: picked.y, text });
+          }
+
+          targets.sort((a, b) => (a.y - b.y) || (a.x - b.x));
+          return { targets };
+        })()`,
+        }, apiUrl).catch(() => null);
+        const raw = Array.isArray(probe?.result?.targets)
+            ? probe.result.targets
+            : Array.isArray(probe?.targets)
+                ? probe.targets
+                : [];
+        return raw
+            .map((t) => ({
+            x: Math.round(Number(t?.x)),
+            y: Math.round(Number(t?.y)),
+            text: String(t?.text || ''),
+        }))
+            .filter((t) => Number.isFinite(t.x) && Number.isFinite(t.y));
+    };
+    let totalClicked = 0;
+    let totalDetected = 0;
+    let passes = 0;
+    for (let pass = 0; pass < maxPasses; pass += 1) {
+        const targets = await probeTargets();
+        if (!targets.length)
+            break;
+        totalDetected += targets.length;
+        let clickedThisPass = 0;
+        const toClick = targets.slice(0, maxClicksPerPass);
+        for (const t of toClick) {
+            const clicked = await controllerAction('container:operation', {
+                containerId: 'xiaohongshu_detail.comment_section.show_more_button',
+                operationId: 'click',
+                sessionId,
+                timeoutMs: 15000,
+                config: {
+                    x: t.x,
+                    y: t.y,
+                },
+            }, apiUrl).catch(() => null);
+            if (clicked?.success !== false)
+                clickedThisPass += 1;
+            await delay(220);
+        }
+        passes += 1;
+        totalClicked += clickedThisPass;
+        // No successful click in this pass means target is currently not actionable.
+        if (clickedThisPass <= 0)
+            break;
+        await delay(380);
+    }
+    const remaining = (await probeTargets()).length;
+    return { clicked: totalClicked, passes, remaining, detected: totalDetected };
+}
+//# sourceMappingURL=xhsComments.js.map

package/dist/modules/xiaohongshu/app/src/xiaohongshu/app/src/index.d.ts

@@ -0,0 +1,9 @@
+/**
+ * 小红书 App 模块入口
+ */
+export * as Phase1EnsureServicesBlock from './blocks/Phase1EnsureServicesBlock.js';
+export * as Phase1StartProfileBlock from './blocks/Phase1StartProfileBlock.js';
+export * as Phase1MonitorCookieBlock from './blocks/Phase1MonitorCookieBlock.js';
+export * as Phase2SearchBlock from './blocks/Phase2SearchBlock.js';
+export * as Phase2CollectLinksBlock from './blocks/Phase2CollectLinksBlock.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/modules/xiaohongshu/app/src/xiaohongshu/app/src/index.js

@@ -0,0 +1,9 @@
+/**
+ * 小红书 App 模块入口
+ */
+export * as Phase1EnsureServicesBlock from './blocks/Phase1EnsureServicesBlock.js';
+export * as Phase1StartProfileBlock from './blocks/Phase1StartProfileBlock.js';
+export * as Phase1MonitorCookieBlock from './blocks/Phase1MonitorCookieBlock.js';
+export * as Phase2SearchBlock from './blocks/Phase2SearchBlock.js';
+export * as Phase2CollectLinksBlock from './blocks/Phase2CollectLinksBlock.js';
+//# sourceMappingURL=index.js.map

package/dist/modules/xiaohongshu/app/src/xiaohongshu/app/src/utils/checkpoints.d.ts

@@ -0,0 +1,50 @@
+export type XhsCheckpointId = 'home_ready' | 'search_ready' | 'detail_ready' | 'comments_ready' | 'login_guard' | 'risk_control' | 'offsite' | 'unknown';
+export interface DetectCheckpointInput {
+    sessionId: string;
+    serviceUrl?: string;
+}
+export interface DetectCheckpointOutput {
+    success: boolean;
+    checkpoint: XhsCheckpointId;
+    stage: string;
+    url: string;
+    rootId?: string | null;
+    matchIds?: string[];
+    signals: string[];
+    /** DOM side signals from DetectPageStateBlock (when available) */
+    dom?: {
+        hasDetailMask?: boolean;
+        hasSearchInput?: boolean;
+        readyState?: string;
+        title?: string;
+    };
+    error?: string;
+}
+export declare function detectXhsCheckpoint(input: DetectCheckpointInput): Promise<DetectCheckpointOutput>;
+export interface EnsureCheckpointInput {
+    sessionId: string;
+    target: XhsCheckpointId;
+    serviceUrl?: string;
+    timeoutMs?: number;
+    allowOneLevelUpFallback?: boolean;
+    evidence?: {
+        highlightMs?: number;
+    };
+}
+export interface EnsureCheckpointOutput {
+    success: boolean;
+    from: XhsCheckpointId;
+    to: XhsCheckpointId;
+    reached: XhsCheckpointId;
+    url: string;
+    stage: string;
+    attempts: Array<{
+        action: string;
+        ok: boolean;
+        reason?: string;
+    }>;
+    signals: string[];
+    error?: string;
+}
+export declare function ensureXhsCheckpoint(input: EnsureCheckpointInput): Promise<EnsureCheckpointOutput>;
+//# sourceMappingURL=checkpoints.d.ts.map