@agentrhq/webcmd 0.1.1 → 0.2.0

package/dist/src/pipeline/steps/download.js

@@ -0,0 +1,252 @@
+/**
+ * Pipeline step: download — file download with concurrency and progress.
+ *
+ * Supports:
+ * - Direct HTTP downloads (images, documents)
+ * - yt-dlp integration for video platforms
+ * - Browser cookie forwarding for authenticated downloads
+ * - Filename templating and deduplication
+ */
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import { render } from '../template.js';
+import { getErrorMessage } from '../../errors.js';
+import { httpDownload, ytdlpDownload, saveDocument, detectContentType, requiresYtdlp, sanitizeFilename, generateFilename, exportCookiesToNetscape, getTempDir, formatCookieHeader, } from '../../download/index.js';
+import { DownloadProgressTracker, formatBytes } from '../../download/progress.js';
+import { mapConcurrent } from '../../utils.js';
+/**
+ * Extract cookies from browser page.
+ */
+async function extractBrowserCookies(page, domain) {
+    try {
+        const cookies = await page.getCookies({ domain });
+        return formatCookieHeader(cookies);
+    }
+    catch {
+        return '';
+    }
+}
+/**
+ * Extract cookies as array for yt-dlp Netscape format.
+ */
+async function extractCookiesArray(page, domain) {
+    try {
+        const cookies = await page.getCookies({ domain });
+        return cookies
+            .filter((cookie) => cookie.name)
+            .map((cookie) => ({
+            name: cookie.name,
+            value: cookie.value,
+            domain: cookie.domain,
+            path: cookie.path ?? '/',
+            secure: cookie.secure ?? false,
+            httpOnly: cookie.httpOnly ?? false,
+        }));
+    }
+    catch {
+        return [];
+    }
+}
+function dedupeCookies(cookies) {
+    const deduped = new Map();
+    for (const cookie of cookies) {
+        deduped.set(`${cookie.domain}\t${cookie.path}\t${cookie.name}`, cookie);
+    }
+    return [...deduped.values()];
+}
+export async function stepDownload(page, params, data, args) {
+    // Parse parameters with defaults
+    const p = typeof params === 'object' && params !== null ? params : {};
+    const urlTemplate = typeof params === 'string' ? params : (p.url ?? '');
+    const dirTemplate = p.dir ?? './downloads';
+    const filenameTemplate = p.filename ?? '';
+    const concurrency = typeof p.concurrency === 'number' ? p.concurrency : 3;
+    const skipExisting = p.skip_existing !== false;
+    const timeout = typeof p.timeout === 'number' ? p.timeout * 1000 : 30000;
+    const useYtdlp = p.use_ytdlp ?? false;
+    const ytdlpArgs = Array.isArray(p.ytdlp_args)
+        ? p.ytdlp_args.map((v) => String(v))
+        : [];
+    const contentType = p.type ?? 'auto';
+    const showProgress = p.progress !== false;
+    const contentTemplate = p.content;
+    const metadataTemplate = p.metadata;
+    // Resolve output directory
+    const dir = String(render(dirTemplate, { args, data }));
+    fs.mkdirSync(dir, { recursive: true });
+    // Normalize data to array. Items are row records (string-keyed) produced by
+    // upstream steps; we treat them as Record<string, unknown> and narrow per-use.
+    const items = Array.isArray(data) ? data
+        : data ? [data]
+            : [];
+    if (items.length === 0) {
+        return [];
+    }
+    // Create progress tracker
+    const tracker = new DownloadProgressTracker(items.length, showProgress);
+    // Cache cookie lookups per domain so mixed-domain batches stay isolated without repeated browser calls.
+    const cookieHeaderCache = new Map();
+    let cookiesFile;
+    if (page) {
+        // For yt-dlp, we need to export cookies to Netscape format
+        if (useYtdlp || items.some((item, index) => {
+            const url = String(render(urlTemplate, { args, data, item, index }));
+            return requiresYtdlp(url);
+        })) {
+            try {
+                const ytdlpDomains = [...new Set(items.flatMap((item, index) => {
+                        const url = String(render(urlTemplate, { args, data, item, index }));
+                        if (!useYtdlp && !requiresYtdlp(url))
+                            return [];
+                        try {
+                            return [new URL(url).hostname];
+                        }
+                        catch {
+                            return [];
+                        }
+                    }))];
+                const cookiesArray = dedupeCookies((await Promise.all(ytdlpDomains.map((domain) => extractCookiesArray(page, domain)))).flat());
+                if (cookiesArray.length > 0) {
+                    const tempDir = getTempDir();
+                    fs.mkdirSync(tempDir, { recursive: true });
+                    cookiesFile = path.join(tempDir, `cookies_${Date.now()}.txt`);
+                    exportCookiesToNetscape(cookiesArray, cookiesFile);
+                }
+            }
+            catch {
+                // Ignore cookie extraction errors
+            }
+        }
+    }
+    const results = await mapConcurrent(items, concurrency, async (item, index) => {
+        const startTime = Date.now();
+        // Render URL
+        const url = String(render(urlTemplate, { args, data, item, index }));
+        if (!url) {
+            tracker.onFileComplete(false);
+            return {
+                ...item,
+                _download: { status: 'failed', error: 'Empty URL' },
+            };
+        }
+        // Render filename
+        let filename;
+        if (filenameTemplate) {
+            filename = String(render(filenameTemplate, { args, data, item, index }));
+        }
+        else {
+            filename = generateFilename(url, index);
+        }
+        filename = sanitizeFilename(filename);
+        const destPath = path.join(dir, filename);
+        // Check if file exists and skip_existing is true
+        if (skipExisting && fs.existsSync(destPath)) {
+            tracker.onFileComplete(true, true);
+            return {
+                ...item,
+                _download: {
+                    status: 'skipped',
+                    path: destPath,
+                    size: fs.statSync(destPath).size,
+                },
+            };
+        }
+        // Create progress bar for this file
+        const progressBar = tracker.onFileStart(filename, index);
+        // Determine download method
+        const detectedType = contentType === 'auto' ? detectContentType(url) : contentType;
+        const shouldUseYtdlp = useYtdlp || (detectedType === 'video' && requiresYtdlp(url));
+        let result;
+        try {
+            if (detectedType === 'document' && contentTemplate) {
+                // Save extracted content as document
+                const content = String(render(contentTemplate, { args, data, item, index }));
+                const metadata = metadataTemplate
+                    ? Object.fromEntries(Object.entries(metadataTemplate).map(([k, v]) => [k, render(v, { args, data, item, index })]))
+                    : undefined;
+                const ext = path.extname(filename).toLowerCase();
+                const format = ext === '.json' ? 'json' : ext === '.html' ? 'html' : 'markdown';
+                result = await saveDocument(content, destPath, format, metadata);
+                if (progressBar) {
+                    progressBar.complete(result.success, result.success ? formatBytes(result.size) : undefined);
+                }
+            }
+            else if (shouldUseYtdlp) {
+                // Use yt-dlp for video downloads
+                result = await ytdlpDownload(url, destPath, {
+                    cookiesFile,
+                    extraArgs: ytdlpArgs,
+                    onProgress: (percent) => {
+                        if (progressBar) {
+                            progressBar.update(percent, 100);
+                        }
+                    },
+                });
+                if (progressBar) {
+                    progressBar.complete(result.success, result.success ? formatBytes(result.size) : undefined);
+                }
+            }
+            else {
+                // Direct HTTP download
+                let cookies = '';
+                if (page) {
+                    try {
+                        const targetDomain = new URL(url).hostname;
+                        let cookiePromise = cookieHeaderCache.get(targetDomain);
+                        if (!cookiePromise) {
+                            cookiePromise = extractBrowserCookies(page, targetDomain);
+                            cookieHeaderCache.set(targetDomain, cookiePromise);
+                        }
+                        cookies = await cookiePromise;
+                    }
+                    catch {
+                        cookies = '';
+                    }
+                }
+                result = await httpDownload(url, destPath, {
+                    cookies,
+                    timeout,
+                    onProgress: (received, total) => {
+                        if (progressBar) {
+                            progressBar.update(received, total);
+                        }
+                    },
+                });
+                if (progressBar) {
+                    progressBar.complete(result.success, result.success ? formatBytes(result.size) : undefined);
+                }
+            }
+        }
+        catch (err) {
+            const msg = getErrorMessage(err);
+            result = { success: false, size: 0, error: msg };
+            if (progressBar) {
+                progressBar.fail(msg);
+            }
+        }
+        tracker.onFileComplete(result.success);
+        const duration = Date.now() - startTime;
+        return {
+            ...item,
+            _download: {
+                status: result.success ? 'success' : 'failed',
+                path: result.success ? destPath : undefined,
+                size: result.size,
+                error: result.error,
+                duration,
+            },
+        };
+    });
+    // Cleanup temp cookie file
+    if (cookiesFile && fs.existsSync(cookiesFile)) {
+        try {
+            fs.unlinkSync(cookiesFile);
+        }
+        catch {
+            // Ignore cleanup errors
+        }
+    }
+    // Show summary
+    tracker.finish();
+    return results;
+}

package/dist/src/pipeline/steps/download.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/src/pipeline/steps/download.test.js

@@ -0,0 +1,102 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+import * as os from 'node:os';
+import * as path from 'node:path';
+const { mockHttpDownload, mockYtdlpDownload, mockExportCookiesToNetscape } = vi.hoisted(() => ({
+    mockHttpDownload: vi.fn(),
+    mockYtdlpDownload: vi.fn(),
+    mockExportCookiesToNetscape: vi.fn(),
+}));
+vi.mock('../../download/index.js', async () => {
+    const actual = await vi.importActual('../../download/index.js');
+    return {
+        ...actual,
+        httpDownload: mockHttpDownload,
+        ytdlpDownload: mockYtdlpDownload,
+        exportCookiesToNetscape: mockExportCookiesToNetscape,
+    };
+});
+import { stepDownload } from './download.js';
+function createMockPage(getCookies) {
+    return {
+        goto: vi.fn(),
+        evaluate: vi.fn().mockResolvedValue(null),
+        fetchJson: vi.fn().mockResolvedValue(null),
+        getCookies,
+        snapshot: vi.fn().mockResolvedValue(''),
+        click: vi.fn(),
+        typeText: vi.fn(),
+        fillText: vi.fn(),
+        pressKey: vi.fn(),
+        scrollTo: vi.fn(),
+        getFormState: vi.fn().mockResolvedValue({}),
+        wait: vi.fn(),
+        tabs: vi.fn().mockResolvedValue([]),
+        selectTab: vi.fn(),
+        networkRequests: vi.fn().mockResolvedValue([]),
+        consoleMessages: vi.fn().mockResolvedValue([]),
+        scroll: vi.fn(),
+        autoScroll: vi.fn(),
+        installInterceptor: vi.fn(),
+        getInterceptedRequests: vi.fn().mockResolvedValue([]),
+        screenshot: vi.fn().mockResolvedValue(''),
+        waitForCapture: vi.fn().mockResolvedValue(undefined),
+    };
+}
+describe('stepDownload', () => {
+    beforeEach(() => {
+        mockHttpDownload.mockReset();
+        mockHttpDownload.mockResolvedValue({ success: true, size: 2 });
+        mockYtdlpDownload.mockReset();
+        mockYtdlpDownload.mockResolvedValue({ success: true, size: 2 });
+        mockExportCookiesToNetscape.mockReset();
+    });
+    it('scopes browser cookies to each direct-download target domain', async () => {
+        const page = createMockPage(vi.fn().mockImplementation(async (opts) => {
+            const domain = opts?.domain ?? 'unknown';
+            return [{ name: 'sid', value: domain, domain }];
+        }));
+        await stepDownload(page, {
+            url: '${{ item.url }}',
+            dir: path.join(os.tmpdir(), 'webcmd-download-test'),
+            filename: '${{ index }}.txt',
+            progress: false,
+            concurrency: 1,
+        }, [
+            { url: 'https://a.example/file-1.txt' },
+            { url: 'https://b.example/file-2.txt' },
+        ], {});
+        expect(mockHttpDownload).toHaveBeenNthCalledWith(1, 'https://a.example/file-1.txt', path.join(os.tmpdir(), 'webcmd-download-test', '0.txt'), expect.objectContaining({ cookies: 'sid=a.example' }));
+        expect(mockHttpDownload).toHaveBeenNthCalledWith(2, 'https://b.example/file-2.txt', path.join(os.tmpdir(), 'webcmd-download-test', '1.txt'), expect.objectContaining({ cookies: 'sid=b.example' }));
+    });
+    it('builds yt-dlp cookies from all target domains instead of only the first item', async () => {
+        const getCookies = vi.fn().mockImplementation(async (opts) => {
+            const domain = opts?.domain ?? 'unknown';
+            return [{
+                    name: `sid-${domain}`,
+                    value: domain,
+                    domain,
+                    path: '/',
+                    secure: false,
+                    httpOnly: false,
+                }];
+        });
+        const page = createMockPage(getCookies);
+        await stepDownload(page, {
+            url: '${{ item.url }}',
+            dir: '/tmp/webcmd-download-test',
+            filename: '${{ index }}.mp4',
+            progress: false,
+            concurrency: 1,
+        }, [
+            { url: 'https://www.youtube.com/watch?v=one' },
+            { url: 'https://vimeo.com/12345' },
+        ], {});
+        expect(getCookies).toHaveBeenCalledWith({ domain: 'www.youtube.com' });
+        expect(getCookies).toHaveBeenCalledWith({ domain: 'vimeo.com' });
+        expect(mockExportCookiesToNetscape).toHaveBeenCalledWith(expect.arrayContaining([
+            expect.objectContaining({ name: 'sid-www.youtube.com', domain: 'www.youtube.com' }),
+            expect.objectContaining({ name: 'sid-vimeo.com', domain: 'vimeo.com' }),
+        ]), expect.any(String));
+        expect(mockYtdlpDownload).toHaveBeenCalledTimes(2);
+    });
+});

package/dist/src/pipeline/steps/fetch.d.ts

@@ -0,0 +1,5 @@
+/**
+ * Pipeline step: fetch — HTTP API requests.
+ */
+import type { IPage } from '../../types.js';
+export declare function stepFetch(page: IPage | null, params: unknown, data: unknown, args: Record<string, unknown>): Promise<unknown>;

package/dist/src/pipeline/steps/fetch.js

@@ -0,0 +1,122 @@
+/**
+ * Pipeline step: fetch — HTTP API requests.
+ */
+import { CliError, getErrorMessage } from '../../errors.js';
+import { log } from '../../logger.js';
+import { render } from '../template.js';
+import { isRecord, mapConcurrent } from '../../utils.js';
+/** Single URL fetch helper */
+async function fetchSingle(page, url, method, queryParams, headers, args, data) {
+    const renderedParams = {};
+    for (const [k, v] of Object.entries(queryParams))
+        renderedParams[k] = String(render(v, { args, data }));
+    const renderedHeaders = {};
+    for (const [k, v] of Object.entries(headers))
+        renderedHeaders[k] = String(render(v, { args, data }));
+    let finalUrl = url;
+    if (Object.keys(renderedParams).length > 0) {
+        const qs = new URLSearchParams(renderedParams).toString();
+        finalUrl = `${finalUrl}${finalUrl.includes('?') ? '&' : '?'}${qs}`;
+    }
+    if (page === null) {
+        const resp = await fetch(finalUrl, { method: method.toUpperCase(), headers: renderedHeaders });
+        if (!resp.ok) {
+            throw new CliError('FETCH_ERROR', `HTTP ${resp.status} ${resp.statusText} from ${finalUrl}`);
+        }
+        return resp.json();
+    }
+    return page.fetchJson(finalUrl, { method: method.toUpperCase(), headers: renderedHeaders });
+}
+/**
+ * Batch fetch: send all URLs into the browser as a single evaluate() call.
+ * This eliminates N-1 cross-process IPC round trips, performing all fetches
+ * inside the V8 engine and returning results as one JSON array.
+ */
+async function fetchBatchInBrowser(page, urls, method, headers, concurrency) {
+    const headersJs = JSON.stringify(headers);
+    const urlsJs = JSON.stringify(urls);
+    const methodJs = JSON.stringify(method);
+    return (await page.evaluate(`
+    async () => {
+      const urls = ${urlsJs};
+      const method = ${methodJs};
+      const headers = ${headersJs};
+      const concurrency = ${concurrency};
+
+      const results = new Array(urls.length);
+      let idx = 0;
+
+      async function worker() {
+        while (idx < urls.length) {
+          const i = idx++;
+          try {
+            const resp = await fetch(urls[i], { method, headers, credentials: "include" });
+            if (!resp.ok) {
+              throw new Error('HTTP ' + resp.status + ' ' + resp.statusText + ' from ' + urls[i]);
+            }
+            results[i] = await resp.json();
+          } catch (e) {
+            results[i] = { error: e instanceof Error ? e.message : String(e) };
+            // Note: getErrorMessage() is a Node.js utility — can't use it inside evaluate()
+          }
+        }
+      }
+
+      const workers = Array.from({ length: Math.min(concurrency, urls.length) }, () => worker());
+      await Promise.all(workers);
+      return results;
+    }
+  `));
+}
+export async function stepFetch(page, params, data, args) {
+    const paramObject = isRecord(params) ? params : {};
+    const urlOrObj = typeof params === 'string' ? params : (paramObject.url ?? '');
+    const method = typeof paramObject.method === 'string' ? paramObject.method : 'GET';
+    const queryParams = isRecord(paramObject.params) ? paramObject.params : {};
+    const headers = isRecord(paramObject.headers) ? paramObject.headers : {};
+    const urlTemplate = String(urlOrObj);
+    // Per-item fetch when data is array and URL references item
+    if (Array.isArray(data) && urlTemplate.includes('item')) {
+        const concurrency = typeof paramObject.concurrency === 'number' ? paramObject.concurrency : 5;
+        // Render all URLs upfront
+        const renderedHeaders = {};
+        for (const [k, v] of Object.entries(headers))
+            renderedHeaders[k] = String(render(v, { args, data }));
+        const renderedParams = {};
+        for (const [k, v] of Object.entries(queryParams))
+            renderedParams[k] = String(render(v, { args, data }));
+        const urls = data.map((item, index) => {
+            let url = String(render(urlTemplate, { args, data, item, index }));
+            if (Object.keys(renderedParams).length > 0) {
+                const qs = new URLSearchParams(renderedParams).toString();
+                url = `${url}${url.includes('?') ? '&' : '?'}${qs}`;
+            }
+            return url;
+        });
+        // BATCH IPC: if browser is available, batch all fetches into a single evaluate() call
+        if (page !== null) {
+            const results = await fetchBatchInBrowser(page, urls, method.toUpperCase(), renderedHeaders, concurrency);
+            for (let i = 0; i < results.length; i++) {
+                const r = results[i];
+                if (r && typeof r === 'object' && 'error' in r) {
+                    log.warn(`Batch fetch failed for ${urls[i]}: ${r.error}`);
+                }
+            }
+            return results;
+        }
+        // Non-browser: use concurrent pool (already optimized)
+        return mapConcurrent(data, concurrency, async (item, index) => {
+            const itemUrl = String(render(urlTemplate, { args, data, item, index }));
+            try {
+                return await fetchSingle(null, itemUrl, method, queryParams, headers, args, data);
+            }
+            catch (error) {
+                const message = getErrorMessage(error);
+                log.warn(`Batch fetch failed for ${itemUrl}: ${message}`);
+                return { error: message };
+            }
+        });
+    }
+    const url = render(urlOrObj, { args, data });
+    return fetchSingle(page, String(url), method, queryParams, headers, args, data);
+}

package/dist/src/pipeline/steps/fetch.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/src/pipeline/steps/fetch.test.js

@@ -0,0 +1,117 @@
+import { afterEach, describe, expect, it, vi } from 'vitest';
+import { CliError } from '../../errors.js';
+import { stepFetch } from './fetch.js';
+afterEach(() => {
+    vi.restoreAllMocks();
+    vi.unstubAllGlobals();
+});
+describe('stepFetch', () => {
+    // W1 + W4: non-browser single fetch throws CliError with FETCH_ERROR code and full message
+    it('throws CliError with FETCH_ERROR code on non-ok responses without a browser session', async () => {
+        const jsonMock = vi.fn().mockResolvedValue({ error: 'rate limited' });
+        const fetchMock = vi.fn().mockResolvedValue({
+            ok: false,
+            status: 429,
+            statusText: 'Too Many Requests',
+            json: jsonMock,
+        });
+        vi.stubGlobal('fetch', fetchMock);
+        const err = await stepFetch(null, { url: 'https://api.example.com/items' }, null, {}).catch((e) => e);
+        expect(err).toBeInstanceOf(CliError);
+        expect(err.code).toBe('FETCH_ERROR');
+        expect(err.message).toBe('HTTP 429 Too Many Requests from https://api.example.com/items');
+        expect(jsonMock).not.toHaveBeenCalled();
+    });
+    // W1 + W3: browser single fetch delegates to page.fetchJson, which owns browser-context fetch errors
+    it('throws CliError with FETCH_ERROR code on non-ok responses inside the browser session', async () => {
+        const page = {
+            fetchJson: vi.fn().mockRejectedValue(new CliError('FETCH_ERROR', 'HTTP 401 Unauthorized from https://api.example.com/items')),
+        };
+        const err = await stepFetch(page, { url: 'https://api.example.com/items' }, null, {}).catch((e) => e);
+        expect(err).toBeInstanceOf(CliError);
+        expect(err.code).toBe('FETCH_ERROR');
+        expect(err.message).toBe('HTTP 401 Unauthorized from https://api.example.com/items');
+        expect(page.fetchJson).toHaveBeenCalledWith('https://api.example.com/items', {
+            method: 'GET',
+            headers: {},
+        });
+    });
+    it('returns per-item HTTP errors for batch fetches without a browser session', async () => {
+        const jsonMock = vi.fn().mockResolvedValue({ error: 'upstream unavailable' });
+        const fetchMock = vi.fn().mockResolvedValue({
+            ok: false,
+            status: 503,
+            statusText: 'Service Unavailable',
+            json: jsonMock,
+        });
+        vi.stubGlobal('fetch', fetchMock);
+        await expect(stepFetch(null, { url: 'https://api.example.com/items/${{ item.id }}' }, [{ id: 1 }], {})).resolves.toEqual([
+            { error: 'HTTP 503 Service Unavailable from https://api.example.com/items/1' },
+        ]);
+        expect(jsonMock).not.toHaveBeenCalled();
+    });
+    it('returns per-item HTTP errors for batch browser fetches', async () => {
+        const jsonMock = vi.fn().mockResolvedValue({ error: 'upstream unavailable' });
+        const fetchMock = vi.fn().mockResolvedValue({
+            ok: false,
+            status: 503,
+            statusText: 'Service Unavailable',
+            json: jsonMock,
+        });
+        vi.stubGlobal('fetch', fetchMock);
+        const page = {
+            evaluate: vi.fn(async (js) => Function(`return (${js})`)()()),
+        };
+        await expect(stepFetch(page, { url: 'https://api.example.com/items/${{ item.id }}' }, [{ id: 1 }], {})).resolves.toEqual([
+            { error: 'HTTP 503 Service Unavailable from https://api.example.com/items/1' },
+        ]);
+        expect(jsonMock).not.toHaveBeenCalled();
+    });
+    it('stringifies non-Error batch browser failures consistently', async () => {
+        vi.stubGlobal('fetch', vi.fn().mockRejectedValue('socket hang up'));
+        const page = {
+            evaluate: vi.fn(async (js) => Function(`return (${js})`)()()),
+        };
+        await expect(stepFetch(page, { url: 'https://api.example.com/items/${{ item.id }}' }, [{ id: 1 }], {})).resolves.toEqual([
+            { error: 'socket hang up' },
+        ]);
+    });
+    it('stringifies non-Error batch non-browser failures consistently', async () => {
+        vi.stubGlobal('fetch', vi.fn().mockRejectedValue('socket hang up'));
+        await expect(stepFetch(null, { url: 'https://api.example.com/items/${{ item.id }}' }, [{ id: 1 }], {})).resolves.toEqual([
+            { error: 'socket hang up' },
+        ]);
+    });
+    // W2: batch item failures emit a warning log
+    it('logs a warning for each failed batch item in non-browser mode', async () => {
+        const { log } = await import('../../logger.js');
+        const warnSpy = vi.spyOn(log, 'warn');
+        vi.stubGlobal('fetch', vi.fn().mockResolvedValue({
+            ok: false,
+            status: 503,
+            statusText: 'Service Unavailable',
+            json: vi.fn(),
+        }));
+        await stepFetch(null, { url: 'https://api.example.com/items/${{ item.id }}' }, [{ id: 1 }, { id: 2 }], {});
+        expect(warnSpy).toHaveBeenCalledTimes(2);
+        expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('https://api.example.com/items/1'));
+        expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('https://api.example.com/items/2'));
+    });
+    it('logs a warning for each failed batch item in browser mode', async () => {
+        const { log } = await import('../../logger.js');
+        const warnSpy = vi.spyOn(log, 'warn');
+        vi.stubGlobal('fetch', vi.fn().mockResolvedValue({
+            ok: false,
+            status: 502,
+            statusText: 'Bad Gateway',
+            json: vi.fn(),
+        }));
+        const page = {
+            evaluate: vi.fn(async (js) => Function(`return (${js})`)()()),
+        };
+        await stepFetch(page, { url: 'https://api.example.com/items/${{ item.id }}' }, [{ id: 1 }, { id: 2 }], {});
+        expect(warnSpy).toHaveBeenCalledTimes(2);
+        expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('https://api.example.com/items/1'));
+        expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('https://api.example.com/items/2'));
+    });
+});

package/dist/src/pipeline/steps/intercept.d.ts

@@ -0,0 +1,5 @@
+/**
+ * Pipeline step: intercept — declarative XHR interception.
+ */
+import type { IPage } from '../../types.js';
+export declare function stepIntercept(page: IPage | null, params: unknown, data: unknown, args: Record<string, unknown>): Promise<unknown>;

package/dist/src/pipeline/steps/intercept.js

@@ -0,0 +1,50 @@
+/**
+ * Pipeline step: intercept — declarative XHR interception.
+ */
+import { render, normalizeEvaluateSource } from '../template.js';
+export async function stepIntercept(page, params, data, args) {
+    const cfg = typeof params === 'object' && params !== null ? params : {};
+    const trigger = cfg.trigger ?? '';
+    const capturePattern = cfg.capture ?? '';
+    const timeout = cfg.timeout ?? 8;
+    const selectPath = cfg.select ?? null;
+    if (!capturePattern)
+        return data;
+    // Step 1: Inject fetch/XHR interceptor BEFORE trigger
+    await page.installInterceptor(capturePattern);
+    // Step 2: Execute the trigger action
+    if (trigger.startsWith('navigate:')) {
+        const url = render(trigger.slice('navigate:'.length), { args, data });
+        await page.goto(String(url));
+    }
+    else if (trigger.startsWith('evaluate:')) {
+        const js = trigger.slice('evaluate:'.length);
+        await page.evaluate(normalizeEvaluateSource(render(js, { args, data })));
+    }
+    else if (trigger.startsWith('click:')) {
+        const ref = render(trigger.slice('click:'.length), { args, data });
+        await page.click(String(ref).replace(/^@/, ''));
+    }
+    else if (trigger === 'scroll') {
+        await page.scroll('down');
+    }
+    // Step 3: Wait for network capture (event-driven, not fixed sleep)
+    await page.waitForCapture(timeout);
+    // Step 4: Retrieve captured data
+    const matchingResponses = await page.getInterceptedRequests();
+    // Step 5: Select from response if specified
+    let result = matchingResponses.length === 1 ? matchingResponses[0] :
+        matchingResponses.length > 1 ? matchingResponses : data;
+    if (selectPath && result) {
+        let current = result;
+        for (const part of String(selectPath).split('.')) {
+            if (current && typeof current === 'object' && !Array.isArray(current)) {
+                current = current[part];
+            }
+            else
+                break;
+        }
+        result = current ?? result;
+    }
+    return result;
+}

package/dist/src/pipeline/steps/tap.d.ts

@@ -0,0 +1,12 @@
+/**
+ * Pipeline step: tap — declarative Store Action Bridge.
+ *
+ * Generates a self-contained IIFE that:
+ * 1. Injects fetch + XHR dual interception proxy
+ * 2. Finds the Pinia/Vuex store and calls the action
+ * 3. Captures the response matching the URL pattern
+ * 4. Auto-cleans up interception in finally block
+ * 5. Returns the captured data (optionally sub-selected)
+ */
+import type { IPage } from '../../types.js';
+export declare function stepTap(page: IPage | null, params: unknown, data: unknown, args: Record<string, unknown>): Promise<unknown>;