@web-auto/webauto 0.1.1 → 0.1.3

package/dist/modules/xiaohongshu/app/src/blocks/XhsDiscoverFallbackBlock.js

@@ -0,0 +1,240 @@
+/**
+ * XHS Discover Fallback Block
+ *
+ * 独立的回退 Block：当出现 shell-page（URL=/explore/<id> 但 DOM=搜索结果）时，
+ * 通过点击「发现」按钮重置状态，然后重新搜索以获取正确的 /search_result URL。
+ *
+ * 约束：全系统级操作（container + keyboard/mouse），禁止 goto/refresh/url 拼接。
+ */
+import os from 'node:os';
+import path from 'node:path';
+import { promises as fs } from 'node:fs';
+import { ContainerRegistry } from '../../../../container-registry/src/index.js';
+import { execute as phase2Search } from './Phase2SearchBlock.js';
+import { detectXhsCheckpoint } from '../utils/checkpoints.js';
+import { controllerAction, delay } from '../utils/controllerAction.js';
+function resolveDownloadRoot() {
+    const custom = process.env.WEBAUTO_DOWNLOAD_ROOT || process.env.WEBAUTO_DOWNLOAD_DIR;
+    if (custom && custom.trim())
+        return custom;
+    const home = process.env.HOME || process.env.USERPROFILE || os.homedir();
+    return path.join(home, '.webauto', 'download');
+}
+function isDebugArtifactsEnabled() {
+    return (process.env.WEBAUTO_DEBUG === '1' ||
+        process.env.WEBAUTO_DEBUG_ARTIFACTS === '1' ||
+        process.env.WEBAUTO_DEBUG_SCREENSHOT === '1');
+}
+function decodeURIComponentSafe(value) {
+    try {
+        return decodeURIComponent(value);
+    }
+    catch {
+        return value;
+    }
+}
+function decodeRepeated(value, maxRounds = 3) {
+    let current = value;
+    for (let i = 0; i < maxRounds; i += 1) {
+        const next = decodeURIComponentSafe(current);
+        if (next === current)
+            break;
+        current = next;
+    }
+    return current;
+}
+function getKeywordFromSearchUrl(searchUrl) {
+    try {
+        const url = new URL(searchUrl);
+        const raw = url.searchParams.get('keyword') || '';
+        if (raw)
+            return decodeRepeated(raw).trim();
+    }
+    catch {
+        // ignore
+    }
+    return null;
+}
+async function saveScreenshot(base64, env, keyword) {
+    if (!isDebugArtifactsEnabled())
+        return null;
+    const dir = path.join(resolveDownloadRoot(), 'xiaohongshu', env, keyword, 'discover-fallback');
+    await fs.mkdir(dir, { recursive: true });
+    const buf = Buffer.from(base64, 'base64');
+    const filePath = path.join(dir, `fallback-${Date.now()}.png`);
+    await fs.writeFile(filePath, buf);
+    return filePath;
+}
+async function saveDomDump(dom, env, keyword) {
+    if (!isDebugArtifactsEnabled())
+        return null;
+    const dir = path.join(resolveDownloadRoot(), 'xiaohongshu', env, keyword, 'discover-fallback');
+    await fs.mkdir(dir, { recursive: true });
+    const filePath = path.join(dir, `fallback-${Date.now()}.json`);
+    await fs.writeFile(filePath, JSON.stringify(dom, null, 2), 'utf8');
+    return filePath;
+}
+export async function execute(input) {
+    const { keyword, profile = 'xiaohongshu_fresh', unifiedApiUrl = 'http://127.0.0.1:7701', env = 'debug', } = input;
+    console.log(`[XhsDiscoverFallback] start keyword=${keyword} profile=${profile}`);
+    const registry = new ContainerRegistry();
+    await registry.load();
+    async function waitCheckpoint(maxWaitMs) {
+        const start = Date.now();
+        let last = await detectXhsCheckpoint({ sessionId: profile, serviceUrl: unifiedApiUrl });
+        while (Date.now() - start < maxWaitMs) {
+            if (last.checkpoint !== 'detail_ready' && last.checkpoint !== 'comments_ready')
+                return last;
+            await delay(500);
+            last = await detectXhsCheckpoint({ sessionId: profile, serviceUrl: unifiedApiUrl });
+        }
+        return last;
+    }
+    // If we are in detail/comments modal, exit first (do NOT click Discover under modal).
+    let checkpoint = await detectXhsCheckpoint({ sessionId: profile, serviceUrl: unifiedApiUrl });
+    if (checkpoint.checkpoint === 'detail_ready' || checkpoint.checkpoint === 'comments_ready') {
+        console.log(`[XhsDiscoverFallback] in ${checkpoint.checkpoint}, exit modal before discover`);
+        try {
+            const r = await controllerAction('container:operation', { containerId: 'xiaohongshu_detail.close_button', operationId: 'click', sessionId: profile, timeoutMs: 15000 }, unifiedApiUrl);
+            console.log(`[XhsDiscoverFallback] close_button click: success=${Boolean(r?.success !== false)}`);
+        }
+        catch {
+            console.log('[XhsDiscoverFallback] close_button click failed (ignored)');
+        }
+        checkpoint = await waitCheckpoint(8000);
+        if (checkpoint.checkpoint === 'detail_ready' || checkpoint.checkpoint === 'comments_ready') {
+            // Fallback: ESC twice
+            for (let i = 0; i < 2; i += 1) {
+                console.log(`[XhsDiscoverFallback] still in ${checkpoint.checkpoint}, press ESC (round=${i + 1})`);
+                await controllerAction('keyboard:press', { profileId: profile, key: 'Escape' }, unifiedApiUrl);
+                checkpoint = await waitCheckpoint(8000);
+                if (checkpoint.checkpoint !== 'detail_ready' && checkpoint.checkpoint !== 'comments_ready')
+                    break;
+            }
+        }
+        // If still in detail/comments, stop (do not proceed to discover).
+        if (checkpoint.checkpoint === 'detail_ready' || checkpoint.checkpoint === 'comments_ready') {
+            let screenshotPath;
+            let domDumpPath;
+            try {
+                const shot = await controllerAction('browser:screenshot', { profileId: profile, fullPage: false }, unifiedApiUrl)
+                    .then((res) => res?.data || res?.result || res?.data?.data || '');
+                if (typeof shot === 'string' && shot) {
+                    const saved = await saveScreenshot(shot, env, keyword);
+                    if (saved)
+                        screenshotPath = saved;
+                }
+            }
+            catch { }
+            try {
+                const dom = await controllerAction('browser:execute', {
+                    profile,
+                    script: '(() => ({ url: window.location.href, title: document.title, readyState: document.readyState, text: document.body ? document.body.innerText.slice(0, 800) : "" }))()'
+                }, unifiedApiUrl).then((res) => res?.result || res?.data?.result || null);
+                if (dom) {
+                    const saved = await saveDomDump(dom, env, keyword);
+                    if (saved)
+                        domDumpPath = saved;
+                }
+            }
+            catch { }
+            return {
+                success: false,
+                finalUrl: checkpoint.url || '',
+                finalCheckpoint: checkpoint.checkpoint,
+                screenshotPath,
+                domDumpPath,
+            };
+        }
+    }
+    const defs = registry.getContainersForUrl('https://www.xiaohongshu.com/explore/');
+    const discoverDef = defs?.['xiaohongshu_detail.discover_button'] || defs?.['xiaohongshu_home.discover_button'];
+    const selectorDefs = Array.isArray(discoverDef?.selectors) ? discoverDef.selectors : [];
+    const primarySelectorDef = selectorDefs.find((s) => s?.variant === 'primary') || selectorDefs[0];
+    const discoverSelector = String(primarySelectorDef?.css || 'a[href="/"]');
+    console.log(`[XhsDiscoverFallback] click discover selector=${discoverSelector}`);
+    let clicked = false;
+    // Prefer container operations (system-level) to avoid selector drift.
+    const candidates = [
+        'xiaohongshu_home.discover_button',
+        'xiaohongshu_search.discover_button',
+    ];
+    for (const id of candidates) {
+        try {
+            const r = await controllerAction('container:operation', { containerId: id, operationId: 'click', sessionId: profile, timeoutMs: 15000 }, unifiedApiUrl);
+            if (r?.success !== false) {
+                clicked = true;
+                console.log(`[XhsDiscoverFallback] container click ok: ${id}`);
+                break;
+            }
+        }
+        catch {
+            // ignore and try next
+        }
+    }
+    // Fallback: raw selector click (still system-level via container:click)
+    if (!clicked) {
+        for (let i = 0; i < 3; i += 1) {
+            const result = await controllerAction('container:click', { profileId: profile, selector: discoverSelector, highlight: true, timeoutMs: 8000 }, unifiedApiUrl).then((res) => Boolean(res?.ok ?? res?.result?.ok ?? res?.data?.ok));
+            if (result) {
+                clicked = true;
+                console.log('[XhsDiscoverFallback] selector click ok');
+                break;
+            }
+            await delay(600);
+        }
+    }
+    console.log(`[XhsDiscoverFallback] click discover done clicked=${clicked}`);
+    // Let XHS settle. Avoid refresh/goto.
+    await delay(2000);
+    // Re-run search.
+    const searchRes = await phase2Search({ keyword, profile, unifiedApiUrl });
+    const finalUrl = searchRes.finalUrl;
+    const finalCheckpoint = await detectXhsCheckpoint({ sessionId: profile, serviceUrl: unifiedApiUrl });
+    const isSearchResultPage = finalUrl.includes('/search_result') && getKeywordFromSearchUrl(finalUrl) === keyword;
+    const isSearchReady = finalCheckpoint.checkpoint === 'search_ready';
+    console.log(`[XhsDiscoverFallback] final checkpoint=${finalCheckpoint.checkpoint} url=${finalUrl}`);
+    if (!isSearchResultPage || !isSearchReady) {
+        let screenshotPath;
+        let domDumpPath;
+        try {
+            const shot = await controllerAction('browser:screenshot', { profileId: profile, fullPage: false }, unifiedApiUrl)
+                .then((res) => res?.data || res?.result || res?.data?.data || '');
+            if (typeof shot === 'string' && shot) {
+                const saved = await saveScreenshot(shot, env, keyword);
+                if (saved)
+                    screenshotPath = saved;
+            }
+        }
+        catch {
+            // ignore
+        }
+        try {
+            const dom = await controllerAction('browser:execute', {
+                profile,
+                script: '(() => ({ url: window.location.href, title: document.title, readyState: document.readyState, text: document.body ? document.body.innerText.slice(0, 800) : "" }))()'
+            }, unifiedApiUrl).then((res) => res?.result || res?.data?.result || null);
+            if (dom) {
+                const saved = await saveDomDump(dom, env, keyword);
+                if (saved)
+                    domDumpPath = saved;
+            }
+        }
+        catch {
+            // ignore
+        }
+        return {
+            success: false,
+            finalUrl,
+            finalCheckpoint: finalCheckpoint.checkpoint,
+            screenshotPath,
+            domDumpPath,
+        };
+    }
+    return {
+        success: true,
+        finalUrl,
+        finalCheckpoint: finalCheckpoint.checkpoint,
+    };
+}
+//# sourceMappingURL=XhsDiscoverFallbackBlock.js.map

package/dist/modules/xiaohongshu/app/src/blocks/helpers/commentMatchDsl.js

@@ -0,0 +1,126 @@
+function normalizeText(s, caseSensitive) {
+    const t = String(s || '').replace(/\s+/g, ' ').trim();
+    return caseSensitive ? t : t.toLowerCase();
+}
+function normalizeTerms(terms, caseSensitive) {
+    const list = Array.isArray(terms) ? terms : [];
+    const out = list
+        .map((x) => normalizeText(String(x || ''), caseSensitive))
+        .filter(Boolean);
+    return Array.from(new Set(out));
+}
+function termHits(text, terms) {
+    const hits = [];
+    for (const t of terms) {
+        if (!t)
+            continue;
+        if (text.includes(t))
+            hits.push(t);
+    }
+    return hits;
+}
+function evalExpr(text, expr, caseSensitive) {
+    if (!expr || typeof expr !== 'object')
+        return { ok: false, hits: [] };
+    switch (expr.op) {
+        case 'any': {
+            const terms = normalizeTerms(expr.terms, caseSensitive);
+            const hits = termHits(text, terms);
+            return { ok: hits.length > 0, hits };
+        }
+        case 'all': {
+            const terms = normalizeTerms(expr.terms, caseSensitive);
+            const hits = termHits(text, terms);
+            return { ok: terms.length > 0 ? hits.length === terms.length : true, hits };
+        }
+        case 'atLeast': {
+            const terms = normalizeTerms(expr.terms, caseSensitive);
+            const min = Math.max(0, Math.floor(Number(expr.min)));
+            const hits = termHits(text, terms);
+            return { ok: min <= 0 ? true : hits.length >= min, hits };
+        }
+        case 'and': {
+            const exprs = Array.isArray(expr.exprs) ? expr.exprs : [];
+            const hitsAll = [];
+            for (const e of exprs) {
+                const r = evalExpr(text, e, caseSensitive);
+                hitsAll.push(...r.hits);
+                if (!r.ok)
+                    return { ok: false, hits: Array.from(new Set(hitsAll)) };
+            }
+            return { ok: true, hits: Array.from(new Set(hitsAll)) };
+        }
+        case 'or': {
+            const exprs = Array.isArray(expr.exprs) ? expr.exprs : [];
+            const hitsAll = [];
+            for (const e of exprs) {
+                const r = evalExpr(text, e, caseSensitive);
+                hitsAll.push(...r.hits);
+                if (r.ok)
+                    return { ok: true, hits: Array.from(new Set(hitsAll)) };
+            }
+            return { ok: false, hits: Array.from(new Set(hitsAll)) };
+        }
+        case 'not': {
+            const r = evalExpr(text, expr.expr, caseSensitive);
+            return { ok: !r.ok, hits: r.hits };
+        }
+        default: {
+            const _exhaustive = expr;
+            return { ok: false, hits: [] };
+        }
+    }
+}
+export function matchCommentTextDsl(textRaw, rule) {
+    const caseSensitive = rule.caseSensitive === true;
+    const text = normalizeText(textRaw, caseSensitive);
+    const exclude = rule.exclude;
+    if (exclude) {
+        const r = evalExpr(text, exclude, caseSensitive);
+        if (r.ok) {
+            const hits = Array.from(new Set(r.hits));
+            return {
+                ok: false,
+                rejectedBy: 'exclude',
+                hits,
+                requireHits: [],
+                excludeHits: hits,
+                preferHits: [],
+                score: 0,
+            };
+        }
+    }
+    let requireHits = [];
+    if (rule.require) {
+        const r = evalExpr(text, rule.require, caseSensitive);
+        requireHits = Array.from(new Set(r.hits));
+        if (!r.ok) {
+            return {
+                ok: false,
+                rejectedBy: 'require',
+                hits: requireHits,
+                requireHits,
+                excludeHits: [],
+                preferHits: [],
+                score: 0,
+            };
+        }
+    }
+    let preferHits = [];
+    if (rule.prefer) {
+        const r = evalExpr(text, rule.prefer, caseSensitive);
+        preferHits = Array.from(new Set(r.hits));
+    }
+    const hits = Array.from(new Set([...requireHits, ...preferHits]));
+    // score: prefer hits first, then require hits
+    const score = preferHits.length * 100 + requireHits.length;
+    return {
+        ok: true,
+        hits,
+        requireHits,
+        excludeHits: [],
+        preferHits,
+        score,
+    };
+}
+//# sourceMappingURL=commentMatchDsl.js.map

package/dist/modules/xiaohongshu/app/src/blocks/helpers/commentMatcher.js

@@ -0,0 +1,99 @@
+function normalizeText(text, caseSensitive) {
+    const s = String(text || '').replace(/\s+/g, ' ').trim();
+    return caseSensitive ? s : s.toLowerCase();
+}
+function normalizeKeywords(list, caseSensitive) {
+    const arr = Array.isArray(list) ? list : [];
+    const out = arr
+        .map((x) => normalizeText(String(x || ''), caseSensitive))
+        .filter(Boolean);
+    // 轻量去重，保持稳定顺序
+    return Array.from(new Set(out));
+}
+function hits(text, keywords) {
+    const found = [];
+    for (const k of keywords) {
+        if (!k)
+            continue;
+        if (text.includes(k))
+            found.push(k);
+    }
+    return found;
+}
+export function matchCommentText(textRaw, rule) {
+    const caseSensitive = rule.caseSensitive === true;
+    const text = normalizeText(textRaw, caseSensitive);
+    const mustNot = normalizeKeywords(rule.mustNot, caseSensitive);
+    const must = normalizeKeywords(rule.must, caseSensitive);
+    const any = normalizeKeywords(rule.any, caseSensitive);
+    const should = normalizeKeywords(rule.should, caseSensitive);
+    const mustNotHits = hits(text, mustNot);
+    if (mustNotHits.length > 0) {
+        return {
+            ok: false,
+            anyHits: [],
+            mustHits: [],
+            shouldHits: [],
+            anyCount: 0,
+            shouldCount: 0,
+            rejectedBy: 'mustNot',
+        };
+    }
+    const mustHits = hits(text, must);
+    if (must.length > 0 && mustHits.length !== must.length) {
+        return {
+            ok: false,
+            anyHits: [],
+            mustHits,
+            shouldHits: [],
+            anyCount: 0,
+            shouldCount: 0,
+            rejectedBy: 'must',
+        };
+    }
+    const anyHits = hits(text, any);
+    const minAny = typeof rule.minAnyMatches === 'number' ? Math.max(0, Math.floor(rule.minAnyMatches)) : (any.length > 0 ? 1 : 0);
+    if (any.length > 0 && anyHits.length < minAny) {
+        return {
+            ok: false,
+            anyHits,
+            mustHits,
+            shouldHits: [],
+            anyCount: anyHits.length,
+            shouldCount: 0,
+            rejectedBy: 'any',
+        };
+    }
+    const shouldHits = hits(text, should);
+    const shouldCount = shouldHits.length;
+    if (typeof rule.minShouldMatches === 'number') {
+        const minShould = Math.max(0, Math.floor(rule.minShouldMatches));
+        if (should.length > 0 && shouldCount < minShould) {
+            return {
+                ok: false,
+                anyHits,
+                mustHits,
+                shouldHits,
+                anyCount: anyHits.length,
+                shouldCount,
+                rejectedBy: 'should',
+            };
+        }
+    }
+    return {
+        ok: true,
+        anyHits,
+        mustHits,
+        shouldHits,
+        anyCount: anyHits.length,
+        shouldCount,
+    };
+}
+export function isLegacyKeywordRule(rule) {
+    if (!rule || typeof rule !== 'object')
+        return false;
+    // legacy: uses any/must/mustNot/should keys
+    const keys = ['any', 'must', 'mustNot', 'should', 'minAnyMatches', 'minShouldMatches', 'caseSensitive'];
+    return keys.some((k) => Object.prototype.hasOwnProperty.call(rule, k));
+}
+//# sourceMappingURL=commentMatcher.js.map

package/dist/modules/xiaohongshu/app/src/blocks/helpers/evidence.js

@@ -0,0 +1,27 @@
+import path from 'node:path';
+import { promises as fs } from 'node:fs';
+import { controllerAction } from '../../utils/controllerAction.js';
+export function resolveHomeDir() {
+    const homeDir = process.env.HOME || process.env.USERPROFILE || '';
+    if (!homeDir)
+        throw new Error('无法获取用户主目录：HOME/USERPROFILE 未设置');
+    return homeDir;
+}
+export function resolveDownloadRoot() {
+    const custom = process.env.WEBAUTO_DOWNLOAD_ROOT || process.env.WEBAUTO_DOWNLOAD_DIR;
+    if (custom && String(custom).trim())
+        return String(custom).trim();
+    return path.join(resolveHomeDir(), '.webauto', 'download');
+}
+export async function takeScreenshotBase64(profileId, unifiedApiUrl) {
+    const shot = await controllerAction('browser:screenshot', { profileId, fullPage: false }, unifiedApiUrl);
+    const base64 = shot?.data || shot?.result || shot?.data?.data;
+    return typeof base64 === 'string' && base64 ? base64 : null;
+}
+export async function savePngBase64(base64, filePath) {
+    const dir = path.dirname(filePath);
+    await fs.mkdir(dir, { recursive: true });
+    await fs.writeFile(filePath, Buffer.from(base64, 'base64'));
+    return filePath;
+}
+//# sourceMappingURL=evidence.js.map

package/dist/modules/xiaohongshu/app/src/blocks/helpers/sharding.js

@@ -0,0 +1,42 @@
+export function fnv1a32(input) {
+    // FNV-1a 32-bit
+    let hash = 0x811c9dc5;
+    const str = String(input || '');
+    for (let i = 0; i < str.length; i += 1) {
+        hash ^= str.charCodeAt(i);
+        // hash *= 16777619 (with 32-bit overflow)
+        hash = (hash + ((hash << 1) + (hash << 4) + (hash << 7) + (hash << 8) + (hash << 24))) >>> 0;
+    }
+    return hash >>> 0;
+}
+export function normalizeShard(spec) {
+    if (!spec)
+        return null;
+    const count = Math.max(1, Math.floor(Number(spec.count)));
+    const index = Math.max(0, Math.floor(Number(spec.index)));
+    if (!Number.isFinite(count) || !Number.isFinite(index))
+        return null;
+    if (count <= 1)
+        return null;
+    if (index >= count)
+        return null;
+    return { index, count, by: spec.by || 'noteId-hash' };
+}
+export function shardFilterByNoteIdHash(items, shard) {
+    const list = Array.isArray(items) ? items : [];
+    if (shard.count <= 1)
+        return list;
+    return list.filter((it) => {
+        const id = String(it?.noteId || '').trim();
+        if (!id)
+            return false;
+        return fnv1a32(id) % shard.count === shard.index;
+    });
+}
+export function shardFilterByIndexMod(items, shard) {
+    const list = Array.isArray(items) ? items : [];
+    if (shard.count <= 1)
+        return list;
+    return list.filter((_, idx) => idx % shard.count === shard.index);
+}
+//# sourceMappingURL=sharding.js.map