@web-auto/webauto 0.1.3 → 0.1.6

package/dist/modules/xiaohongshu/app/src/blocks/helpers/sharding.js

@@ -1,3 +1,85 @@
+import path from 'node:path';
+import os from 'node:os';
+import fs from 'node:fs/promises';
+function sanitizeForPath(name, fallback = 'unknown') {
+    const text = String(name || '').trim();
+    if (!text)
+        return fallback;
+    const cleaned = text.replace(/[\\/:"*?<>|]+/g, '_').trim();
+    return cleaned || fallback;
+}
+function asStringArray(input) {
+    if (!Array.isArray(input))
+        return [];
+    return input
+        .map((item) => String(item || '').trim())
+        .filter(Boolean);
+}
+function normalizeCollectedUrls(input, completedNoteIds) {
+    if (!Array.isArray(input))
+        return [];
+    const out = [];
+    const seen = new Set();
+    for (const row of input) {
+        if (!row || typeof row !== 'object')
+            continue;
+        const noteId = String(row.noteId || '').trim();
+        if (!noteId || completedNoteIds.has(noteId) || seen.has(noteId))
+            continue;
+        seen.add(noteId);
+        const safeUrl = String(row.safeUrl || '').trim();
+        const searchUrl = String(row.searchUrl || '').trim();
+        const timestampRaw = Number(row.timestamp);
+        const item = {
+            noteId,
+            safeUrl,
+            ...(searchUrl ? { searchUrl } : {}),
+            ...(Number.isFinite(timestampRaw) ? { timestamp: timestampRaw } : {}),
+        };
+        out.push(item);
+    }
+    return out;
+}
+async function resolveStatePath(input) {
+    const root = resolveDownloadRoot(input.downloadRoot);
+    const sanitizedEnv = sanitizeForPath(input.env, 'debug');
+    const sanitizedKeyword = sanitizeForPath(input.keyword, 'unknown');
+    const rawEnv = String(input.env || '').trim();
+    const rawKeyword = String(input.keyword || '').trim();
+    const candidates = [
+        path.join(root, 'xiaohongshu', sanitizedEnv, sanitizedKeyword, '.collect-state.json'),
+    ];
+    const legacyPath = path.join(root, 'xiaohongshu', rawEnv, rawKeyword, '.collect-state.json');
+    if (legacyPath !== candidates[0])
+        candidates.push(legacyPath);
+    for (const filePath of candidates) {
+        try {
+            await fs.access(filePath);
+            return filePath;
+        }
+        catch {
+            // continue
+        }
+    }
+    return null;
+}
+async function getXhsPendingItems(input) {
+    const statePath = await resolveStatePath(input);
+    if (!statePath)
+        return [];
+    try {
+        const content = await fs.readFile(statePath, 'utf8');
+        const state = JSON.parse(content);
+        const completed = new Set(asStringArray(state?.detailCollection?.completedNoteIds));
+        return normalizeCollectedUrls(state?.listCollection?.collectedUrls, completed);
+    }
+    catch (error) {
+        const code = String(error?.code || '');
+        const kind = code === 'ENOENT' ? 'missing_state' : code === 'EACCES' ? 'access_denied' : 'invalid_state';
+        console.warn(`[xhs.sharding] failed to load pending items (${kind}) from ${statePath}: ${error?.message || String(error)}`);
+        return [];
+    }
+}
 export function fnv1a32(input) {
     // FNV-1a 32-bit
     let hash = 0x811c9dc5;
@@ -39,4 +121,45 @@ export function shardFilterByIndexMod(items, shard) {
         return list;
     return list.filter((_, idx) => idx % shard.count === shard.index);
 }
+export function resolveDownloadRoot(customRoot) {
+    const fromArg = String(customRoot || '').trim();
+    if (fromArg)
+        return path.resolve(fromArg);
+    const fromEnv = String(process.env.WEBAUTO_DOWNLOAD_ROOT || process.env.WEBAUTO_DOWNLOAD_DIR || '').trim();
+    if (fromEnv)
+        return path.resolve(fromEnv);
+    const home = process.env.HOME || process.env.USERPROFILE || os.homedir();
+    return path.join(home, '.webauto', 'download');
+}
+export async function buildDynamicShardPlan(input) {
+    const { keyword, env, downloadRoot, validProfiles } = input;
+    if (!validProfiles.length)
+        return [];
+    // Load pending items (not completed yet)
+    const pendingItems = await getXhsPendingItems({ keyword, env, downloadRoot });
+    if (pendingItems.length === 0) {
+        return validProfiles.map((profileId) => ({ profileId, assignedNoteIds: [], totalPending: 0 }));
+    }
+    // Calculate how many notes each profile should handle
+    // Using ceil to ensure we cover all pending items
+    const perProfileCount = Math.ceil(pendingItems.length / validProfiles.length);
+    const plans = [];
+    for (let i = 0; i < validProfiles.length; i++) {
+        const profileId = validProfiles[i];
+        const startIdx = i * perProfileCount;
+        const endIdx = Math.min(startIdx + perProfileCount, pendingItems.length);
+        const assignedNoteIds = pendingItems.slice(startIdx, endIdx).map((item) => item.noteId);
+        plans.push({
+            profileId,
+            assignedNoteIds,
+            totalPending: pendingItems.length,
+        });
+    }
+    return plans;
+}
+export async function getPendingItemsByNoteIds(input) {
+    const pendingItems = await getXhsPendingItems({ keyword: input.keyword, env: input.env, downloadRoot: input.downloadRoot });
+    const noteIdSet = new Set(input.noteIds);
+    return pendingItems.filter((item) => noteIdSet.has(item.noteId));
+}
 //# sourceMappingURL=sharding.js.map

package/dist/modules/xiaohongshu/app/src/container-registry/src/index.d.ts

@@ -0,0 +1,37 @@
+export interface SelectorDefinition {
+    css?: string;
+    id?: string;
+    classes?: string[];
+    variant?: string;
+    score?: number;
+}
+export interface ContainerDefinition {
+    id: string;
+    name?: string;
+    type?: string;
+    selectors?: SelectorDefinition[];
+    children?: string[];
+    page_patterns?: string[];
+    pagePatterns?: string[];
+    metadata?: Record<string, any>;
+    [key: string]: any;
+}
+export declare class ContainerRegistry {
+    private indexCache;
+    listSites(): {
+        key: string;
+        website: string;
+        path: string;
+    }[];
+    getContainersForSite(siteKey: string): Record<string, ContainerDefinition>;
+    resolveSiteKey(url: string): string | null;
+    load(): Promise<void>;
+    getContainersForUrl(url: string): Record<string, ContainerDefinition>;
+    private fetchContainersForSite;
+    private ensureIndex;
+    private loadSiteContainers;
+    private walkSite;
+    private loadLegacyFile;
+    private findSiteKey;
+}
+//# sourceMappingURL=index.d.ts.map

package/dist/modules/xiaohongshu/app/src/container-registry/src/index.js

@@ -0,0 +1,184 @@
+import fs from 'fs';
+import path from 'path';
+import os from 'os';
+import { fileURLToPath } from 'url';
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+const PROJECT_ROOT = resolveProjectRoot(__dirname);
+// 新的外置容器树根目录（开发用）：~/.webauto/container-lib
+const PRIMARY_USER_CONTAINER_ROOT = process.env.WEBAUTO_CONTAINER_ROOT || path.join(os.homedir(), '.webauto', 'container-lib');
+const INDEX_PATH = path.join(PROJECT_ROOT, 'apps/webauto/resources/container-library.index.json');
+function isLegacyContainer(definition) {
+    try {
+        return Boolean(definition?.metadata?.legacy_data);
+    }
+    catch {
+        return false;
+    }
+}
+export class ContainerRegistry {
+    indexCache = null;
+    listSites() {
+        const registry = this.ensureIndex();
+        return Object.entries(registry).map(([key, meta]) => ({
+            key,
+            website: meta.website || '',
+            path: meta.path || '',
+        }));
+    }
+    getContainersForSite(siteKey) {
+        if (!siteKey)
+            return {};
+        const registry = this.ensureIndex();
+        const site = registry[siteKey] || { path: path.join('apps/webauto/resources/container-library', siteKey) };
+        return this.fetchContainersForSite(siteKey, site);
+    }
+    resolveSiteKey(url) {
+        const registry = this.ensureIndex();
+        return this.findSiteKey(url, registry);
+    }
+    async load() {
+        // 兼容异步加载，但当前是同步实现
+        this.ensureIndex();
+        return;
+    }
+    getContainersForUrl(url) {
+        const registry = this.ensureIndex();
+        const siteKey = this.findSiteKey(url, registry);
+        if (!siteKey) {
+            return {};
+        }
+        const site = registry[siteKey] || { path: `apps/webauto/resources/container-library/${siteKey}` };
+        return this.fetchContainersForSite(siteKey, site);
+    }
+    fetchContainersForSite(siteKey, site) {
+        // 不做内存级别缓存，确保用户容器定义变更后，每次调用都能读取到最新文件。
+        // 内部会同时加载内置容器与用户容器目录并合并。
+        return this.loadSiteContainers(siteKey, site?.path);
+    }
+    ensureIndex() {
+        if (this.indexCache) {
+            return this.indexCache;
+        }
+        if (fs.existsSync(INDEX_PATH)) {
+            try {
+                this.indexCache = (JSON.parse(fs.readFileSync(INDEX_PATH, 'utf-8')) || {});
+                return this.indexCache;
+            }
+            catch {
+                // fall through
+            }
+        }
+        this.indexCache = {};
+        return this.indexCache;
+    }
+    loadSiteContainers(siteKey, relativePath) {
+        const containers = {};
+        const builtinPath = path.join(PROJECT_ROOT, relativePath || path.join('apps/webauto/resources/container-library', siteKey));
+        if (fs.existsSync(builtinPath)) {
+            this.walkSite(builtinPath, containers);
+            this.loadLegacyFile(builtinPath, containers);
+        }
+        const userPath = path.join(PRIMARY_USER_CONTAINER_ROOT, siteKey);
+        if (fs.existsSync(userPath)) {
+            this.walkSite(userPath, containers);
+            this.loadLegacyFile(userPath, containers);
+        }
+        return containers;
+    }
+    walkSite(sitePath, output) {
+        const stack = [{ dir: sitePath, parts: [] }];
+        while (stack.length) {
+            const { dir, parts } = stack.pop();
+            let hasContainerFile = false;
+            for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
+                if (entry.isFile() && entry.name === 'container.json') {
+                    const relParts = parts.length ? parts : [path.basename(dir)];
+                    const containerId = relParts.join('.');
+                    try {
+                        const raw = JSON.parse(fs.readFileSync(path.join(dir, entry.name), 'utf-8'));
+                        if (raw && typeof raw === 'object') {
+                            if (isLegacyContainer(raw)) {
+                                continue;
+                            }
+                            const id = raw.id || containerId;
+                            output[id] = { id, ...raw };
+                        }
+                    }
+                    catch {
+                        // ignore malformed container
+                    }
+                    hasContainerFile = true;
+                }
+                else if (entry.isDirectory()) {
+                    stack.push({ dir: path.join(dir, entry.name), parts: [...parts, entry.name] });
+                }
+            }
+            if (!hasContainerFile && parts.length === 0) {
+                // root dir may not contain direct containers, continue
+                continue;
+            }
+        }
+    }
+    loadLegacyFile(sitePath, output) {
+        const legacyFile = path.join(sitePath, 'containers.json');
+        if (!fs.existsSync(legacyFile)) {
+            return;
+        }
+        try {
+            const raw = JSON.parse(fs.readFileSync(legacyFile, 'utf-8'));
+            const containers = raw?.containers;
+            if (containers && typeof containers === 'object') {
+                for (const [key, value] of Object.entries(containers)) {
+                    if (!output[key] && value && typeof value === 'object') {
+                        if (isLegacyContainer(value)) {
+                            continue;
+                        }
+                        output[key] = { id: key, ...value };
+                    }
+                }
+            }
+        }
+        catch {
+            // ignore legacy parse error
+        }
+    }
+    findSiteKey(url, registry) {
+        let host = '';
+        try {
+            const parsed = new URL(url);
+            host = (parsed.hostname || '').toLowerCase();
+        }
+        catch {
+            return null;
+        }
+        let bestKey = null;
+        let bestLen = -1;
+        for (const [key, value] of Object.entries(registry)) {
+            const domain = (value.website || '').toLowerCase();
+            if (!domain)
+                continue;
+            if (host === domain || host.endsWith(`.${domain}`)) {
+                if (domain.length > bestLen) {
+                    bestKey = key;
+                    bestLen = domain.length;
+                }
+            }
+        }
+        return bestKey;
+    }
+}
+function resolveProjectRoot(startDir) {
+    let current = startDir;
+    const { root } = path.parse(startDir);
+    while (true) {
+        if (fs.existsSync(path.join(current, 'package.json'))) {
+            return current;
+        }
+        if (current === root) {
+            return startDir;
+        }
+        current = path.resolve(current, '..');
+    }
+}
+//# sourceMappingURL=index.js.map

package/dist/modules/xiaohongshu/app/src/workflow/blocks/AnchorVerificationBlock.d.ts

@@ -0,0 +1,31 @@
+/**
+ * Workflow Block: AnchorVerificationBlock
+ *
+ * 职责：
+ * - 基于容器定义（container-library）验证指定容器锚点是否在视口内且可见
+ * - 支持进入锚点 / 离开锚点两种语义（enter/exit）
+ * - 避免在这里直接调用 containers:match，降低超时对主流程的影响
+ */
+export interface AnchorVerificationInput {
+    sessionId: string;
+    containerId: string;
+    operation: 'enter' | 'exit';
+    expectedVisible?: boolean;
+    timeoutMs?: number;
+    serviceUrl?: string;
+}
+export interface AnchorVerificationOutput {
+    success: boolean;
+    verified: boolean;
+    containerFound: boolean;
+    visible: boolean;
+    rect?: {
+        x: number;
+        y: number;
+        width: number;
+        height: number;
+    };
+    error?: string;
+}
+export declare function execute(input: AnchorVerificationInput): Promise<AnchorVerificationOutput>;
+//# sourceMappingURL=AnchorVerificationBlock.d.ts.map

package/dist/modules/xiaohongshu/app/src/workflow/blocks/AnchorVerificationBlock.js

@@ -0,0 +1,71 @@
+/**
+ * Workflow Block: AnchorVerificationBlock
+ *
+ * 职责：
+ * - 基于容器定义（container-library）验证指定容器锚点是否在视口内且可见
+ * - 支持进入锚点 / 离开锚点两种语义（enter/exit）
+ * - 避免在这里直接调用 containers:match，降低超时对主流程的影响
+ */
+export async function execute(input) {
+    const { sessionId, containerId, operation, expectedVisible = operation === 'enter', 
+    // timeoutMs 目前只用于 browser:execute 的防御性超时
+    timeoutMs = 10000, serviceUrl = 'http://127.0.0.1:7701' } = input;
+    try {
+        console.log(`[AnchorVerification] ${operation} anchor: ${containerId}`);
+        // 通过 container-library 定义获取 selector，并在页面内高亮 + 读取 Rect。
+        // 注意：这里不再调用 containers:match，避免复用阶段性 P0 问题。
+        const { verifyAnchorByContainerId } = await import('./helpers/containerAnchors.js');
+        const anchor = await verifyAnchorByContainerId(containerId, sessionId, serviceUrl, '3px solid #ff4444', 2000);
+        if (!anchor.found) {
+            return {
+                success: false,
+                verified: false,
+                containerFound: false,
+                visible: false,
+                error: anchor.error || `容器未找到: ${containerId}`
+            };
+        }
+        const rect = anchor.rect;
+        if (!rect) {
+            return {
+                success: false,
+                verified: false,
+                containerFound: true,
+                visible: false,
+                error: '未获取到锚点 Rect'
+            };
+        }
+        // 视口内判定：在这里不直接依赖 window.innerHeight，
+        // 仅要求宽高 > 0，由调用方结合 Rect 再做更严格判断。
+        const visible = rect.width > 0 && rect.height > 0;
+        // enter: 期望可见；exit: 期望不可见/不可命中
+        const verified = visible === !!expectedVisible;
+        if (!verified) {
+            return {
+                success: false,
+                verified: false,
+                containerFound: true,
+                visible,
+                rect,
+                error: `锚点验证失败: 期望${expectedVisible ? '可见' : '不可见'}, 实际${visible ? '可见' : '不可见'}`
+            };
+        }
+        return {
+            success: true,
+            verified: true,
+            containerFound: true,
+            visible,
+            rect
+        };
+    }
+    catch (err) {
+        return {
+            success: false,
+            verified: false,
+            containerFound: false,
+            visible: false,
+            error: `锚点验证异常: ${err.message}`
+        };
+    }
+}
+//# sourceMappingURL=AnchorVerificationBlock.js.map

package/dist/modules/xiaohongshu/app/src/workflow/blocks/DetectPageStateBlock.d.ts

@@ -0,0 +1,48 @@
+/**
+ * Workflow Block: DetectPageStateBlock
+ *
+ * 基于 URL + 容器匹配检测当前页面阶段：
+ * - xiaohongshu: login / detail / search / home / unknown
+ * - weibo: login / detail / search / home / unknown
+ *
+ * 用于在进入各 Phase 前做“入口锚点”判定。
+ */
+export interface DetectPageStateOutput {
+    url: string;
+    stage: PageStage;
+    rootId?: string | null;
+    matchIds?: string[];
+    dom?: {
+        hasDetailMask?: boolean;
+        hasSearchInput?: boolean;
+        readyState?: string;
+        title?: string;
+    };
+    error?: string;
+}
+export type PageStage = 'login' | 'detail' | 'search' | 'home' | 'unknown';
+export interface DetectPageStateInput {
+    sessionId: string;
+    platform?: 'xiaohongshu' | 'weibo' | 'auto';
+    serviceUrl?: string;
+}
+export interface DetectPageStateOutput {
+    success: boolean;
+    sessionId: string;
+    platform: 'xiaohongshu' | 'weibo' | 'unknown';
+    url: string;
+    stage: PageStage;
+    pageName?: string;
+    rootId?: string | null;
+    matchIds?: string[];
+    /** DOM side signals (optional, only when available) */
+    dom?: {
+        hasDetailMask?: boolean;
+        hasSearchInput?: boolean;
+        readyState?: string;
+        title?: string;
+    };
+    error?: string;
+}
+export declare function execute(input: DetectPageStateInput): Promise<DetectPageStateOutput>;
+//# sourceMappingURL=DetectPageStateBlock.d.ts.map