@agentrhq/webcmd 0.1.1 → 0.1.2

package/dist/src/package-paths.js

@@ -0,0 +1,41 @@
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+export function findPackageRoot(startFile, fileExists = fs.existsSync) {
+    let dir = path.dirname(startFile);
+    while (true) {
+        if (fileExists(path.join(dir, 'package.json')))
+            return dir;
+        const parent = path.dirname(dir);
+        if (parent === dir) {
+            throw new Error(`Could not find package.json above ${startFile}`);
+        }
+        dir = parent;
+    }
+}
+export function getBuiltEntryCandidates(packageRoot, readFile = (filePath) => fs.readFileSync(filePath, 'utf-8')) {
+    const candidates = [];
+    try {
+        const pkg = JSON.parse(readFile(path.join(packageRoot, 'package.json')));
+        if (typeof pkg.bin === 'string') {
+            candidates.push(path.join(packageRoot, pkg.bin));
+        }
+        else if (pkg.bin && typeof pkg.bin === 'object' && typeof pkg.bin.webcmd === 'string') {
+            candidates.push(path.join(packageRoot, pkg.bin.webcmd));
+        }
+        if (typeof pkg.main === 'string') {
+            candidates.push(path.join(packageRoot, pkg.main));
+        }
+    }
+    catch {
+        // Fall through to compatibility candidates below.
+    }
+    // Compatibility fallback for partially-built trees or older layouts.
+    candidates.push(path.join(packageRoot, 'dist', 'src', 'main.js'), path.join(packageRoot, 'dist', 'main.js'));
+    return [...new Set(candidates)];
+}
+export function getCliManifestPath(clisDir) {
+    return path.resolve(clisDir, '..', 'cli-manifest.json');
+}
+export function getFetchAdaptersScriptPath(packageRoot) {
+    return path.join(packageRoot, 'scripts', 'fetch-adapters.js');
+}

package/dist/src/pipeline/executor.d.ts

@@ -0,0 +1,11 @@
+/**
+ * Pipeline executor: runs YAML pipeline steps sequentially.
+ */
+import type { IPage } from '../types.js';
+export interface PipelineContext {
+    args?: Record<string, unknown>;
+    debug?: boolean;
+    /** Max retry attempts per step (default: 2 for browser steps, 0 for others) */
+    stepRetries?: number;
+}
+export declare function executePipeline(page: IPage | null, pipeline: unknown[], ctx?: PipelineContext): Promise<unknown>;

package/dist/src/pipeline/executor.js

@@ -0,0 +1,93 @@
+/**
+ * Pipeline executor: runs YAML pipeline steps sequentially.
+ */
+import { getStep } from './registry.js';
+import { log } from '../logger.js';
+import { ConfigError } from '../errors.js';
+import { BROWSER_ONLY_STEPS } from '../capabilityRouting.js';
+import { isTransientBrowserError } from '../browser/errors.js';
+export async function executePipeline(page, pipeline, ctx = {}) {
+    const args = ctx.args ?? {};
+    const debug = ctx.debug ?? false;
+    let data = null;
+    const total = pipeline.length;
+    try {
+        for (let i = 0; i < pipeline.length; i++) {
+            const step = pipeline[i];
+            if (!step || typeof step !== 'object')
+                continue;
+            for (const [op, params] of Object.entries(step)) {
+                if (debug)
+                    debugStepStart(i + 1, total, op, params);
+                const handler = getStep(op);
+                if (handler) {
+                    data = await executeStepWithRetry(handler, page, params, data, args, op, ctx.stepRetries);
+                }
+                else {
+                    throw new ConfigError(`Unknown pipeline step "${op}" at index ${i}.`, 'Check the YAML pipeline step name or register the custom step before execution.');
+                }
+                if (debug)
+                    debugStepResult(op, data);
+            }
+        }
+    }
+    catch (err) {
+        // Attempt cleanup: release automation tab lease on pipeline failure.
+        if (page?.closeWindow) {
+            try {
+                await page.closeWindow();
+            }
+            catch { /* ignore */ }
+        }
+        throw err;
+    }
+    return data;
+}
+async function executeStepWithRetry(handler, page, params, data, args, op, configRetries) {
+    const maxRetries = configRetries ?? (BROWSER_ONLY_STEPS.has(op) ? 2 : 0);
+    for (let attempt = 0; attempt <= maxRetries; attempt++) {
+        try {
+            return await handler(page, params, data, args);
+        }
+        catch (err) {
+            if (attempt >= maxRetries)
+                throw err;
+            // Only retry on transient browser errors
+            if (!isTransientBrowserError(err))
+                throw err;
+            // Brief delay before retry
+            await new Promise(resolve => setTimeout(resolve, 1000));
+        }
+    }
+    // Unreachable
+    throw new Error(`Step "${op}" failed after ${maxRetries} retries`);
+}
+function debugStepStart(stepNum, total, op, params) {
+    let preview = '';
+    if (typeof params === 'string') {
+        preview = params.length <= 80 ? ` → ${params}` : ` → ${params.slice(0, 77)}...`;
+    }
+    else if (params && typeof params === 'object' && !Array.isArray(params)) {
+        preview = ` (${Object.keys(params).join(', ')})`;
+    }
+    log.step(stepNum, total, op, preview);
+}
+function debugStepResult(op, data) {
+    if (data === null || data === undefined) {
+        log.stepResult('(no data)');
+    }
+    else if (Array.isArray(data)) {
+        log.stepResult(`${data.length} items`);
+    }
+    else if (typeof data === 'object') {
+        const keys = Object.keys(data).slice(0, 5);
+        log.stepResult(`dict (${keys.join(', ')}${Object.keys(data).length > 5 ? '...' : ''})`);
+    }
+    else if (typeof data === 'string') {
+        const p = data.slice(0, 60).replace(/\n/g, '\\n');
+        log.stepResult(`"${p}${data.length > 60 ? '...' : ''}"`);
+    }
+    else {
+        log.stepResult(`${typeof data}`);
+    }
+}

package/dist/src/pipeline/executor.test.d.ts

@@ -0,0 +1,4 @@
+/**
+ * Tests for pipeline/executor.ts: pipeline execution with mock page.
+ */
+export {};

package/dist/src/pipeline/executor.test.js

@@ -0,0 +1,182 @@
+/**
+ * Tests for pipeline/executor.ts: pipeline execution with mock page.
+ */
+import { describe, it, expect, vi } from 'vitest';
+import { executePipeline } from './index.js';
+import { ConfigError } from '../errors.js';
+/** Create a minimal mock page for testing */
+function createMockPage(overrides = {}) {
+    return {
+        goto: vi.fn(),
+        evaluate: vi.fn().mockResolvedValue(null),
+        fetchJson: vi.fn().mockResolvedValue(null),
+        getCookies: vi.fn().mockResolvedValue([]),
+        snapshot: vi.fn().mockResolvedValue(''),
+        click: vi.fn(),
+        typeText: vi.fn(),
+        fillText: vi.fn(),
+        pressKey: vi.fn(),
+        getFormState: vi.fn().mockResolvedValue({}),
+        wait: vi.fn(),
+        tabs: vi.fn().mockResolvedValue([]),
+        selectTab: vi.fn(),
+        networkRequests: vi.fn().mockResolvedValue([]),
+        consoleMessages: vi.fn().mockResolvedValue(''),
+        scroll: vi.fn(),
+        scrollTo: vi.fn(),
+        autoScroll: vi.fn(),
+        installInterceptor: vi.fn(),
+        getInterceptedRequests: vi.fn().mockResolvedValue([]),
+        screenshot: vi.fn().mockResolvedValue(''),
+        waitForCapture: vi.fn().mockResolvedValue(undefined),
+        ...overrides,
+    };
+}
+describe('executePipeline', () => {
+    it('returns null for empty pipeline', async () => {
+        const result = await executePipeline(null, []);
+        expect(result).toBeNull();
+    });
+    it('skips null/invalid steps', async () => {
+        const result = await executePipeline(null, [null, undefined, 42]);
+        expect(result).toBeNull();
+    });
+    it('executes navigate step', async () => {
+        const page = createMockPage();
+        await executePipeline(page, [
+            { navigate: 'https://example.com' },
+        ]);
+        expect(page.goto).toHaveBeenCalledWith('https://example.com');
+    });
+    it('executes evaluate + select pipeline', async () => {
+        const page = createMockPage({
+            evaluate: vi.fn().mockResolvedValue({ data: { list: [{ name: 'a' }, { name: 'b' }] } }),
+        });
+        const result = await executePipeline(page, [
+            { evaluate: '() => ({ data: { list: [{name: "a"}, {name: "b"}] } })' },
+            { select: 'data.list' },
+        ]);
+        expect(result).toEqual([{ name: 'a' }, { name: 'b' }]);
+    });
+    it('executes map step to transform items', async () => {
+        const page = createMockPage({
+            evaluate: vi.fn().mockResolvedValue([
+                { title: 'Hello', count: 10 },
+                { title: 'World', count: 20 },
+            ]),
+        });
+        const result = await executePipeline(page, [
+            { evaluate: 'test' },
+            { map: { name: '${{ item.title }}', score: '${{ item.count }}' } },
+        ]);
+        expect(result).toEqual([
+            { name: 'Hello', score: 10 },
+            { name: 'World', score: 20 },
+        ]);
+    });
+    it('runs inline select inside map step', async () => {
+        const page = createMockPage({
+            evaluate: vi.fn().mockResolvedValue({
+                posts: [
+                    { title: 'First', rank: 1 },
+                    { title: 'Second', rank: 2 },
+                ],
+            }),
+        });
+        const result = await executePipeline(page, [
+            { evaluate: 'test' },
+            {
+                map: {
+                    select: 'posts',
+                    title: '${{ item.title }}',
+                    rank: '${{ item.rank }}',
+                },
+            },
+        ]);
+        expect(result).toEqual([
+            { title: 'First', rank: 1 },
+            { title: 'Second', rank: 2 },
+        ]);
+    });
+    it('executes limit step', async () => {
+        const page = createMockPage({
+            evaluate: vi.fn().mockResolvedValue([1, 2, 3, 4, 5]),
+        });
+        const result = await executePipeline(page, [
+            { evaluate: 'test' },
+            { limit: '3' },
+        ]);
+        expect(result).toEqual([1, 2, 3]);
+    });
+    it('executes sort step', async () => {
+        const page = createMockPage({
+            evaluate: vi.fn().mockResolvedValue([{ n: 3 }, { n: 1 }, { n: 2 }]),
+        });
+        const result = await executePipeline(page, [
+            { evaluate: 'test' },
+            { sort: { by: 'n', order: 'asc' } },
+        ]);
+        expect(result).toEqual([{ n: 1 }, { n: 2 }, { n: 3 }]);
+    });
+    it('executes sort step with desc order', async () => {
+        const page = createMockPage({
+            evaluate: vi.fn().mockResolvedValue([{ n: 1 }, { n: 3 }, { n: 2 }]),
+        });
+        const result = await executePipeline(page, [
+            { evaluate: 'test' },
+            { sort: { by: 'n', order: 'desc' } },
+        ]);
+        expect(result).toEqual([{ n: 3 }, { n: 2 }, { n: 1 }]);
+    });
+    it('executes wait step with number', async () => {
+        const page = createMockPage();
+        await executePipeline(page, [
+            { wait: 2 },
+        ]);
+        expect(page.wait).toHaveBeenCalledWith(2);
+    });
+    it('fails fast on unknown steps', async () => {
+        await expect(executePipeline(null, [
+            { unknownStep: 'test' },
+        ], { debug: true })).rejects.toBeInstanceOf(ConfigError);
+        await expect(executePipeline(null, [
+            { unknownStep: 'test' },
+        ], { debug: true })).rejects.toThrow('Unknown pipeline step "unknownStep"');
+    });
+    it('passes args through template rendering', async () => {
+        const page = createMockPage({
+            evaluate: vi.fn().mockResolvedValue([1, 2, 3, 4, 5]),
+        });
+        const result = await executePipeline(page, [
+            { evaluate: 'test' },
+            { limit: '${{ args.count }}' },
+        ], { args: { count: 2 } });
+        expect(result).toEqual([1, 2]);
+    });
+    it('click step calls page.click', async () => {
+        const page = createMockPage();
+        await executePipeline(page, [
+            { click: '@5' },
+        ]);
+        expect(page.click).toHaveBeenCalledWith('5');
+    });
+    it('fill step calls page.fillText with raw rendered text', async () => {
+        const page = createMockPage();
+        await executePipeline(page, [
+            { fill: { ref: '@5', text: 'line1\\n/ / ${{ args.tail }}' } },
+        ], { args: { tail: 'raw' } });
+        expect(page.fillText).toHaveBeenCalledWith('5', 'line1\\n/ / raw');
+    });
+    it('navigate preserves existing data through pipeline', async () => {
+        const page = createMockPage({
+            evaluate: vi.fn().mockResolvedValue([{ a: 1 }]),
+        });
+        const result = await executePipeline(page, [
+            { evaluate: 'test' },
+            { navigate: 'https://example.com' },
+        ]);
+        // navigate should preserve existing data
+        expect(result).toEqual([{ a: 1 }]);
+        expect(page.goto).toHaveBeenCalledWith('https://example.com');
+    });
+});

package/dist/src/pipeline/index.d.ts

@@ -0,0 +1,5 @@
+/**
+ * Pipeline module: re-exports for backward compatibility.
+ */
+export { executePipeline, type PipelineContext } from './executor.js';
+export { render, evalExpr, resolvePath, normalizeEvaluateSource, type RenderContext } from './template.js';

package/dist/src/pipeline/index.js

@@ -0,0 +1,5 @@
+/**
+ * Pipeline module: re-exports for backward compatibility.
+ */
+export { executePipeline } from './executor.js';
+export { render, evalExpr, resolvePath, normalizeEvaluateSource } from './template.js';

package/dist/src/pipeline/registry.d.ts

@@ -0,0 +1,28 @@
+/**
+ * Dynamic registry for pipeline steps.
+ * Allows core and third-party plugins to register custom YAML operations.
+ */
+import type { IPage } from '../types.js';
+/**
+ * Step handler: all pipeline steps conform to this generic interface.
+ * TData is the type of the `data` state flowing into the step.
+ * TResult is the expected return type.
+ */
+export type StepHandler<TData = unknown, TResult = unknown, TParams = unknown> = (page: IPage | null, params: TParams, data: TData, args: Record<string, unknown>) => Promise<TResult>;
+/**
+ * Get a registered step handler by name.
+ */
+export declare function getStep(name: string): StepHandler | undefined;
+/**
+ * List all currently registered step names. Used by `validate.ts` to allowlist
+ * step names without maintaining a parallel hand-coded list.
+ *
+ * Note: this depends on registerStep() side effects below already having run.
+ * Importing this module triggers all core registrations at the bottom of the
+ * file, so the returned array reflects every core + plugin step at call time.
+ */
+export declare function getRegisteredStepNames(): string[];
+/**
+ * Register a new custom step handler for the YAML pipeline.
+ */
+export declare function registerStep(name: string, handler: StepHandler): void;

package/dist/src/pipeline/registry.js

@@ -0,0 +1,55 @@
+/**
+ * Dynamic registry for pipeline steps.
+ * Allows core and third-party plugins to register custom YAML operations.
+ */
+// Import core steps
+import { stepNavigate, stepClick, stepType, stepFill, stepWait, stepPress, stepSnapshot, stepEvaluate } from './steps/browser.js';
+import { stepFetch } from './steps/fetch.js';
+import { stepSelect, stepMap, stepFilter, stepSort, stepLimit } from './steps/transform.js';
+import { stepIntercept } from './steps/intercept.js';
+import { stepTap } from './steps/tap.js';
+import { stepDownload } from './steps/download.js';
+const _stepRegistry = new Map();
+/**
+ * Get a registered step handler by name.
+ */
+export function getStep(name) {
+    return _stepRegistry.get(name);
+}
+/**
+ * List all currently registered step names. Used by `validate.ts` to allowlist
+ * step names without maintaining a parallel hand-coded list.
+ *
+ * Note: this depends on registerStep() side effects below already having run.
+ * Importing this module triggers all core registrations at the bottom of the
+ * file, so the returned array reflects every core + plugin step at call time.
+ */
+export function getRegisteredStepNames() {
+    return [..._stepRegistry.keys()];
+}
+/**
+ * Register a new custom step handler for the YAML pipeline.
+ */
+export function registerStep(name, handler) {
+    _stepRegistry.set(name, handler);
+}
+// -------------------------------------------------------------
+// Auto-Register Core Steps
+// -------------------------------------------------------------
+registerStep('navigate', stepNavigate);
+registerStep('fetch', stepFetch);
+registerStep('select', stepSelect);
+registerStep('evaluate', stepEvaluate);
+registerStep('snapshot', stepSnapshot);
+registerStep('click', stepClick);
+registerStep('type', stepType);
+registerStep('fill', stepFill);
+registerStep('wait', stepWait);
+registerStep('press', stepPress);
+registerStep('map', stepMap);
+registerStep('filter', stepFilter);
+registerStep('sort', stepSort);
+registerStep('limit', stepLimit);
+registerStep('intercept', stepIntercept);
+registerStep('tap', stepTap);
+registerStep('download', stepDownload);

package/dist/src/pipeline/steps/browser.d.ts

@@ -0,0 +1,13 @@
+/**
+ * Pipeline step: navigate, click, type, wait, press, snapshot.
+ * Browser interaction primitives.
+ */
+import type { IPage } from '../../types.js';
+export declare function stepNavigate(page: IPage | null, params: unknown, data: unknown, args: Record<string, unknown>): Promise<unknown>;
+export declare function stepClick(page: IPage | null, params: unknown, data: unknown, args: Record<string, unknown>): Promise<unknown>;
+export declare function stepType(page: IPage | null, params: unknown, data: unknown, args: Record<string, unknown>): Promise<unknown>;
+export declare function stepFill(page: IPage | null, params: unknown, data: unknown, args: Record<string, unknown>): Promise<unknown>;
+export declare function stepWait(page: IPage | null, params: unknown, data: unknown, args: Record<string, unknown>): Promise<unknown>;
+export declare function stepPress(page: IPage | null, params: unknown, data: unknown, args: Record<string, unknown>): Promise<unknown>;
+export declare function stepSnapshot(page: IPage | null, params: unknown, _data: unknown, _args: Record<string, unknown>): Promise<unknown>;
+export declare function stepEvaluate(page: IPage | null, params: unknown, data: unknown, args: Record<string, unknown>): Promise<unknown>;

package/dist/src/pipeline/steps/browser.js

@@ -0,0 +1,86 @@
+/**
+ * Pipeline step: navigate, click, type, wait, press, snapshot.
+ * Browser interaction primitives.
+ */
+import { render } from '../template.js';
+import { isRecord } from '../../utils.js';
+export async function stepNavigate(page, params, data, args) {
+    if (isRecord(params) && 'url' in params) {
+        const url = String(render(params.url, { args, data }));
+        await page.goto(url, { waitUntil: params.waitUntil, settleMs: typeof params.settleMs === 'number' ? params.settleMs : undefined });
+    }
+    else {
+        const url = render(params, { args, data });
+        await page.goto(String(url));
+    }
+    return data;
+}
+export async function stepClick(page, params, data, args) {
+    await page.click(String(render(params, { args, data })).replace(/^@/, ''));
+    return data;
+}
+export async function stepType(page, params, data, args) {
+    if (isRecord(params)) {
+        const ref = String(render(params.ref ?? '', { args, data })).replace(/^@/, '');
+        const text = String(render(params.text ?? '', { args, data }));
+        await page.typeText(ref, text);
+        if (params.submit)
+            await page.pressKey('Enter');
+    }
+    return data;
+}
+export async function stepFill(page, params, data, args) {
+    if (isRecord(params)) {
+        const ref = String(render(params.ref ?? '', { args, data })).replace(/^@/, '');
+        const text = String(render(params.text ?? '', { args, data }));
+        await page.fillText(ref, text);
+        if (params.submit)
+            await page.pressKey('Enter');
+    }
+    return data;
+}
+export async function stepWait(page, params, data, args) {
+    if (typeof params === 'number')
+        await page.wait(params);
+    else if (isRecord(params)) {
+        if ('text' in params) {
+            await page.wait({
+                text: String(render(params.text, { args, data })),
+                timeout: typeof params.timeout === 'number' ? params.timeout : undefined,
+            });
+        }
+        else if ('time' in params)
+            await page.wait(Number(params.time));
+    }
+    else if (typeof params === 'string')
+        await page.wait(Number(render(params, { args, data })));
+    return data;
+}
+export async function stepPress(page, params, data, args) {
+    await page.pressKey(String(render(params, { args, data })));
+    return data;
+}
+export async function stepSnapshot(page, params, _data, _args) {
+    const opts = isRecord(params) ? params : {};
+    return page.snapshot({
+        interactive: typeof opts.interactive === 'boolean' ? opts.interactive : false,
+        compact: typeof opts.compact === 'boolean' ? opts.compact : false,
+        maxDepth: typeof opts.max_depth === 'number' ? opts.max_depth : undefined,
+        raw: typeof opts.raw === 'boolean' ? opts.raw : false,
+    });
+}
+export async function stepEvaluate(page, params, data, args) {
+    const js = String(render(params, { args, data }));
+    let result = await page.evaluate(js);
+    // Browser may return JSON as a string — auto-parse it
+    if (typeof result === 'string') {
+        const trimmed = result.trim();
+        if ((trimmed.startsWith('[') && trimmed.endsWith(']')) || (trimmed.startsWith('{') && trimmed.endsWith('}'))) {
+            try {
+                result = JSON.parse(trimmed);
+            }
+            catch { }
+        }
+    }
+    return result;
+}

package/dist/src/pipeline/steps/download.d.ts

@@ -0,0 +1,18 @@
+/**
+ * Pipeline step: download — file download with concurrency and progress.
+ *
+ * Supports:
+ * - Direct HTTP downloads (images, documents)
+ * - yt-dlp integration for video platforms
+ * - Browser cookie forwarding for authenticated downloads
+ * - Filename templating and deduplication
+ */
+import type { IPage } from '../../types.js';
+export interface DownloadResult {
+    status: 'success' | 'skipped' | 'failed';
+    path?: string;
+    size?: number;
+    error?: string;
+    duration?: number;
+}
+export declare function stepDownload(page: IPage | null, params: unknown, data: unknown, args: Record<string, unknown>): Promise<unknown>;