@nuanu-ai/agentbrowse 0.2.7 → 0.2.8

package/dist/commands/descriptor-validation.js

@@ -0,0 +1,333 @@
+export function normalizePageSignature(url) {
+    try {
+        const parsed = new URL(url);
+        return `${parsed.origin}${parsed.pathname}`;
+    }
+    catch {
+        return url;
+    }
+}
+export const LOCATOR_DOM_SIGNATURE_SCRIPT = String.raw `
+  if (!(element instanceof HTMLElement)) {
+    return null;
+  }
+
+  const inputTypeOf = () => {
+    if (element.tagName.toLowerCase() !== 'input') return '';
+    return (element.getAttribute('type') || 'text').toLowerCase();
+  };
+
+  const isButtonLikeInput = () => {
+    return element.tagName.toLowerCase() === 'input' &&
+      ['button', 'submit', 'reset'].includes(inputTypeOf());
+  };
+
+  const inferRole = () => {
+    const explicitRole = element.getAttribute('role')?.trim();
+    if (explicitRole) return explicitRole;
+
+    const tag = element.tagName.toLowerCase();
+    if (tag === 'button') return 'button';
+    if (tag === 'a' && element.getAttribute('href')) return 'link';
+    if (tag === 'select') return 'combobox';
+    if (tag === 'textarea') return 'textbox';
+    if (tag === 'input') {
+      if (isButtonLikeInput()) return 'button';
+      return 'textbox';
+    }
+    return '';
+  };
+
+  const textOf = (target) => {
+    if (target === element && isButtonLikeInput()) {
+      const value = (element.getAttribute('value') || element.value || '').trim();
+      if (value) {
+        return value.replace(/\s+/g, ' ');
+      }
+    }
+    const value = target?.innerText?.trim() || target?.textContent?.trim() || '';
+    return value.replace(/\s+/g, ' ');
+  };
+
+  const isMeaningfulLabel = (value) => {
+    const normalized = (value || '').replace(/\s+/g, ' ').trim();
+    return Boolean(normalized) && normalized !== '[object Object]';
+  };
+
+  const composedParentElement = (node) => {
+    if (!node) {
+      return null;
+    }
+    if (node.parentElement) {
+      return node.parentElement;
+    }
+    const root = node.getRootNode?.();
+    return root instanceof ShadowRoot && root.host instanceof HTMLElement ? root.host : null;
+  };
+
+  const ariaLabelledbyTextOf = () => {
+    const labelledBy = element.getAttribute('aria-labelledby')?.trim();
+    if (!labelledBy) return undefined;
+
+    const text = labelledBy
+      .split(/\s+/)
+      .map((id) => textOf(document.getElementById(id)))
+      .filter(Boolean)
+      .join(' ')
+      .trim();
+
+    return isMeaningfulLabel(text) ? text : undefined;
+  };
+
+  const nearestFieldLabelOf = () => {
+    const anchors = [
+      composedParentElement(element),
+      composedParentElement(composedParentElement(element)),
+      composedParentElement(composedParentElement(composedParentElement(element))),
+    ].filter(Boolean);
+
+    for (const anchor of anchors) {
+      if (!(anchor instanceof HTMLElement)) continue;
+
+      const explicitLabels = Array.from(
+        anchor.querySelectorAll('label, legend, [class*="label"], [data-testid*="label"]')
+      ).filter((candidate) => {
+        return (
+          candidate instanceof HTMLElement &&
+          candidate !== element &&
+          !candidate.contains(element) &&
+          !element.contains(candidate)
+        );
+      });
+
+      for (const candidate of explicitLabels) {
+        const candidateText = textOf(candidate);
+        if (isMeaningfulLabel(candidateText)) {
+          return candidateText;
+        }
+      }
+
+      for (const child of Array.from(anchor.children).slice(0, 8)) {
+        if (!(child instanceof HTMLElement)) continue;
+        if (child === element || child.contains(element) || element.contains(child)) continue;
+
+        const candidateText = textOf(child);
+        if (isMeaningfulLabel(candidateText)) {
+          return candidateText;
+        }
+      }
+    }
+
+    return undefined;
+  };
+
+  const syntheticLabelOf = () => {
+    const tag = element.tagName.toLowerCase();
+    const explicitRole = element.getAttribute('role')?.trim();
+    if (tag === 'input') {
+      const inputType = inputTypeOf();
+      if (isButtonLikeInput()) return 'Button';
+      if (inputType === 'tel') return 'Phone input';
+      if (inputType === 'email') return 'Email input';
+      if (inputType === 'password') return 'Password input';
+      if (inputType === 'search') return 'Search input';
+      if (inputType === 'date') return 'Date input';
+      return 'Text input';
+    }
+    if (tag === 'textarea') return 'Text area';
+    if (tag === 'select' || explicitRole === 'combobox') return 'Combobox';
+    if (explicitRole === 'textbox') return 'Text input';
+    return undefined;
+  };
+
+  const labelOf = () => {
+    const ariaLabel = element.getAttribute('aria-label')?.trim();
+    if (isMeaningfulLabel(ariaLabel)) return ariaLabel;
+
+    const customLabel = element.getAttribute('label')?.trim();
+    if (isMeaningfulLabel(customLabel)) return customLabel;
+
+    const ariaLabelledbyText = ariaLabelledbyTextOf();
+    if (isMeaningfulLabel(ariaLabelledbyText)) return ariaLabelledbyText;
+
+    if ('labels' in element && Array.isArray(Array.from(element.labels ?? []))) {
+      const labels = element.labels;
+      const firstLabel = labels && labels.length > 0 ? textOf(labels[0]) : '';
+      if (isMeaningfulLabel(firstLabel)) return firstLabel;
+    }
+
+    const text = textOf(element);
+    if (isMeaningfulLabel(text)) return text;
+
+    const nearestFieldLabel = isButtonLikeInput() ? undefined : nearestFieldLabelOf();
+    if (isMeaningfulLabel(nearestFieldLabel)) return nearestFieldLabel;
+
+    return syntheticLabelOf() || '';
+  };
+
+  return element.tagName.toLowerCase() + '|' + inferRole() + '|' + labelOf();
+`;
+function readLocatorDomSignatureInBrowser(element) {
+    return Function('element', LOCATOR_DOM_SIGNATURE_SCRIPT)(element);
+}
+const LOCATOR_BINDING_SNAPSHOT_SCRIPT = String.raw `
+  if (!(element instanceof HTMLElement)) {
+    return null;
+  }
+
+  const read = Function('element', ${JSON.stringify(LOCATOR_DOM_SIGNATURE_SCRIPT)});
+  const explicitRole = element.getAttribute('role')?.trim() || '';
+  const role =
+    explicitRole ||
+    (element.tagName.toLowerCase() === 'select'
+      ? 'combobox'
+      : element.tagName.toLowerCase() === 'textarea'
+        ? 'textbox'
+        : element.tagName.toLowerCase() === 'input'
+          ? 'textbox'
+          : '');
+  const domSignature = read(element);
+  const label = typeof domSignature === 'string' ? domSignature.split('|').slice(2).join('|') : undefined;
+  return {
+    domSignature,
+    kind: element.tagName.toLowerCase(),
+    role,
+    label: label || undefined,
+    placeholder: element.getAttribute('placeholder')?.trim() || undefined,
+    inputName: element.getAttribute('name')?.trim() || undefined,
+    inputType: element.getAttribute('type')?.trim() || undefined,
+    autocomplete: element.getAttribute('autocomplete')?.trim() || undefined,
+  };
+`;
+export async function readLocatorDomSignature(locator) {
+    try {
+        return await locator
+            .first()
+            .evaluate((element, source) => Function('element', source)(element), LOCATOR_DOM_SIGNATURE_SCRIPT);
+    }
+    catch {
+        return null;
+    }
+}
+export async function readLocatorBindingSnapshot(locator) {
+    try {
+        return await locator
+            .first()
+            .evaluate((element, source) => Function('element', source)(element), LOCATOR_BINDING_SNAPSHOT_SCRIPT);
+    }
+    catch {
+        return null;
+    }
+}
+function normalizeComparable(value) {
+    const normalized = value?.replace(/\s+/g, ' ').trim().toLowerCase();
+    return normalized && normalized.length > 0 ? normalized : undefined;
+}
+export function isCompatibleMutableFieldBinding(target, snapshot) {
+    if (!snapshot) {
+        return false;
+    }
+    const isFieldLike = target.controlFamily === 'text-input' ||
+        target.controlFamily === 'select' ||
+        target.controlFamily === 'datepicker' ||
+        target.allowedActions.includes('fill') ||
+        target.allowedActions.includes('type') ||
+        target.allowedActions.includes('select');
+    if (!isFieldLike) {
+        return false;
+    }
+    const comparablePairs = [
+        [normalizeComparable(target.kind), normalizeComparable(snapshot.kind)],
+        [normalizeComparable(target.semantics?.role), normalizeComparable(snapshot.role)],
+        [normalizeComparable(target.inputName), normalizeComparable(snapshot.inputName)],
+        [normalizeComparable(target.inputType), normalizeComparable(snapshot.inputType)],
+        [normalizeComparable(target.autocomplete), normalizeComparable(snapshot.autocomplete)],
+    ];
+    let compared = false;
+    for (const [expected, actual] of comparablePairs) {
+        if (!expected) {
+            continue;
+        }
+        compared = true;
+        if (!actual || actual !== expected) {
+            return false;
+        }
+    }
+    return compared;
+}
+export const __testDescriptorValidation = {
+    readLocatorDomSignatureInBrowser,
+    locatorDomSignatureScript: LOCATOR_DOM_SIGNATURE_SCRIPT,
+    locatorBindingSnapshotScript: LOCATOR_BINDING_SNAPSHOT_SCRIPT,
+    isCompatibleMutableFieldBinding,
+};
+export async function readLocatorOuterHtml(locator) {
+    try {
+        return await locator.first().evaluate((element) => {
+            if (!(element instanceof HTMLElement)) {
+                return null;
+            }
+            const ownerWindow = element.ownerDocument?.defaultView || window;
+            const TextCtor = ownerWindow.Text;
+            const ElementCtor = ownerWindow.Element;
+            const ShadowRootCtor = ownerWindow.ShadowRoot;
+            const voidTags = new Set([
+                'area',
+                'base',
+                'br',
+                'col',
+                'embed',
+                'hr',
+                'img',
+                'input',
+                'link',
+                'meta',
+                'param',
+                'source',
+                'track',
+                'wbr',
+            ]);
+            const escapeText = (value) => value
+                .replace(/&/g, '&')
+                .replace(/</g, '&lt;')
+                .replace(/>/g, '&gt;');
+            const escapeAttribute = (value) => value
+                .replace(/&/g, '&amp;')
+                .replace(/"/g, '&quot;')
+                .replace(/</g, '&lt;');
+            const serializeChildren = (parent) => Array.from(parent.childNodes)
+                .map((node) => serializeNode(node))
+                .join('');
+            const serializeNode = (node) => {
+                if (node instanceof TextCtor) {
+                    return escapeText(node.textContent ?? '');
+                }
+                if (!(node instanceof ElementCtor)) {
+                    return '';
+                }
+                const tag = node.tagName.toLowerCase();
+                if (['script', 'style', 'noscript', 'template'].includes(tag)) {
+                    return '';
+                }
+                const attrs = node
+                    .getAttributeNames()
+                    .sort()
+                    .map((name) => ` ${name}="${escapeAttribute(node.getAttribute(name) ?? '')}"`)
+                    .join('');
+                const shadowRoot = node.shadowRoot && node.shadowRoot instanceof ShadowRootCtor ? node.shadowRoot : null;
+                const shadowHtml = shadowRoot
+                    ? `<div data-agentbrowse-shadow-root="open">${serializeChildren(shadowRoot)}</div>`
+                    : '';
+                const childHtml = serializeChildren(node);
+                if (voidTags.has(tag)) {
+                    return `<${tag}${attrs}>${shadowHtml}`;
+                }
+                return `<${tag}${attrs}>${shadowHtml}${childHtml}</${tag}>`;
+            };
+            return serializeNode(element);
+        });
+    }
+    catch {
+        return null;
+    }
+}

package/dist/commands/extract-scope-resolution.d.ts

@@ -0,0 +1,20 @@
+import type { Page } from 'playwright-core';
+import type { LocatorCandidate } from '../runtime-state.js';
+export type ScopedExtractTarget = {
+    framePath?: string[];
+    locatorCandidates: LocatorCandidate[];
+    domSignature?: string;
+};
+export type ScopedExtractResolution = {
+    page: Page;
+    selector: string;
+    cleanup: () => Promise<void>;
+    degraded: boolean;
+    degradationReason?: 'iframe-scope-mirror';
+};
+export declare function resolveScopedExtractContext(args: {
+    page: Page;
+    scopeTarget: ScopedExtractTarget;
+    validateDomSignature?: boolean;
+}): Promise<ScopedExtractResolution>;
+//# sourceMappingURL=extract-scope-resolution.d.ts.map

package/dist/commands/extract-scope-resolution.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"extract-scope-resolution.d.ts","sourceRoot":"","sources":["../../src/commands/extract-scope-resolution.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAW,IAAI,EAAE,MAAM,iBAAiB,CAAC;AACrD,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AAM5D,MAAM,MAAM,mBAAmB,GAAG;IAChC,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,iBAAiB,EAAE,gBAAgB,EAAE,CAAC;IACtC,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB,CAAC;AAEF,MAAM,MAAM,uBAAuB,GAAG;IACpC,IAAI,EAAE,IAAI,CAAC;IACX,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAC7B,QAAQ,EAAE,OAAO,CAAC;IAClB,iBAAiB,CAAC,EAAE,qBAAqB,CAAC;CAC3C,CAAC;AAwHF,wBAAsB,2BAA2B,CAAC,IAAI,EAAE;IACtD,IAAI,EAAE,IAAI,CAAC;IACX,WAAW,EAAE,mBAAmB,CAAC;IACjC,oBAAoB,CAAC,EAAE,OAAO,CAAC;CAChC,GAAG,OAAO,CAAC,uBAAuB,CAAC,CAmBnC"}

package/dist/commands/extract-scope-resolution.js

@@ -0,0 +1,100 @@
+import { buildLocator, resolveLocatorRoot } from './action-fallbacks.js';
+import { readLocatorDomSignature, readLocatorOuterHtml } from './descriptor-validation.js';
+const LIVE_SCOPE_ATTRIBUTE = 'data-agentbrowse-scope';
+async function materializeScopedExtractPage(page, scopedLocator) {
+    const scopedHtml = await readLocatorOuterHtml(scopedLocator);
+    if (!scopedHtml) {
+        throw new Error('scope_target_unresolvable');
+    }
+    const scratchPage = await page.context().newPage();
+    await scratchPage.setContent(`<!doctype html><html lang="en"><body><div id="agentbrowse-scope">${scopedHtml}</div></body></html>`);
+    return {
+        page: scratchPage,
+        selector: '#agentbrowse-scope',
+        cleanup: async () => {
+            await scratchPage.close().catch(() => undefined);
+        },
+    };
+}
+async function firstVisibleLocator(locator) {
+    if (!locator) {
+        return null;
+    }
+    const first = typeof locator.first === 'function' ? locator.first() : locator;
+    const count = typeof first.count === 'function' ? await first.count().catch(() => 0) : 1;
+    if (count === 0) {
+        return null;
+    }
+    const visible = typeof first.isVisible === 'function' ? await first.isVisible().catch(() => false) : true;
+    return visible ? first : null;
+}
+async function materializeLiveScopedExtract(page, scopedLocator) {
+    const scopeId = `agentbrowse-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
+    await scopedLocator.evaluate((element, payload) => {
+        if (!(element instanceof HTMLElement)) {
+            throw new Error('unsupported_extract_scope_element');
+        }
+        element.setAttribute(payload.attribute, payload.value);
+    }, {
+        attribute: LIVE_SCOPE_ATTRIBUTE,
+        value: scopeId,
+    });
+    return {
+        page,
+        selector: `[${LIVE_SCOPE_ATTRIBUTE}="${scopeId}"]`,
+        cleanup: async () => {
+            await scopedLocator
+                .evaluate((element, attribute) => {
+                if (element instanceof HTMLElement) {
+                    element.removeAttribute(attribute);
+                }
+            }, LIVE_SCOPE_ATTRIBUTE)
+                .catch(() => undefined);
+        },
+    };
+}
+async function resolveScopedLocator(page, scopeTarget, options) {
+    const locatorRoot = resolveLocatorRoot(page, scopeTarget.framePath);
+    let sawDomSignatureMismatch = false;
+    const validateDomSignature = options?.validateDomSignature ?? true;
+    for (const candidate of scopeTarget.locatorCandidates) {
+        const locator = buildLocator(locatorRoot, candidate);
+        const first = await firstVisibleLocator(locator);
+        if (!first) {
+            continue;
+        }
+        if (validateDomSignature && scopeTarget.domSignature) {
+            const liveSignature = await readLocatorDomSignature(first);
+            if (!liveSignature) {
+                continue;
+            }
+            if (liveSignature !== scopeTarget.domSignature) {
+                sawDomSignatureMismatch = true;
+                continue;
+            }
+        }
+        return first;
+    }
+    if (sawDomSignatureMismatch) {
+        throw new Error('stale_scope_target_dom_signature_changed');
+    }
+    throw new Error('scope_target_unresolvable');
+}
+export async function resolveScopedExtractContext(args) {
+    const scopedLocator = await resolveScopedLocator(args.page, args.scopeTarget, {
+        validateDomSignature: args.validateDomSignature,
+    });
+    if (args.scopeTarget.framePath?.length) {
+        const mirrored = await materializeScopedExtractPage(args.page, scopedLocator);
+        return {
+            ...mirrored,
+            degraded: true,
+            degradationReason: 'iframe-scope-mirror',
+        };
+    }
+    const liveScoped = await materializeLiveScopedExtract(args.page, scopedLocator);
+    return {
+        ...liveScoped,
+        degraded: false,
+    };
+}

package/dist/commands/extract-stagehand-executor.d.ts

@@ -0,0 +1,17 @@
+import type { Page } from 'playwright-core';
+import type { z } from 'zod';
+import type { BrowseSession } from '../session.js';
+import { type RuntimeResolution } from '../runtime-resolution.js';
+export type ExtractStagehandExecution = RuntimeResolution & {
+    data: unknown;
+    resolvedBy: 'stagehand-extract';
+};
+export declare function executeStagehandExtract(args: {
+    session: BrowseSession;
+    instruction: string;
+    schema: z.ZodType;
+    page: Page;
+    selector?: string;
+    degradationReason?: string;
+}): Promise<ExtractStagehandExecution>;
+//# sourceMappingURL=extract-stagehand-executor.d.ts.map

package/dist/commands/extract-stagehand-executor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"extract-stagehand-executor.d.ts","sourceRoot":"","sources":["../../src/commands/extract-stagehand-executor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,KAAK,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAC7B,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAEnD,OAAO,EAA8B,KAAK,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAG9F,MAAM,MAAM,yBAAyB,GAAG,iBAAiB,GAAG;IAC1D,IAAI,EAAE,OAAO,CAAC;IACd,UAAU,EAAE,mBAAmB,CAAC;CACjC,CAAC;AAEF,wBAAsB,uBAAuB,CAAC,IAAI,EAAE;IAClD,OAAO,EAAE,aAAa,CAAC;IACvB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC;IAClB,IAAI,EAAE,IAAI,CAAC;IACX,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B,GAAG,OAAO,CAAC,yBAAyB,CAAC,CAerC"}

package/dist/commands/extract-stagehand-executor.js

@@ -0,0 +1,18 @@
+import { incrementMetric } from '../runtime-state.js';
+import { stagehandRuntimeResolution } from '../runtime-resolution.js';
+import { withStagehand } from '../stagehand-runtime.js';
+export async function executeStagehandExtract(args) {
+    const { session, instruction, schema, page, selector, degradationReason } = args;
+    const data = await withStagehand(session, async (stagehand) => {
+        incrementMetric(session, 'stagehandCalls');
+        return stagehand.extract(instruction, schema, {
+            page,
+            selector,
+        });
+    });
+    return {
+        resolvedBy: 'stagehand-extract',
+        ...stagehandRuntimeResolution(degradationReason),
+        data,
+    };
+}

package/dist/commands/extract.d.ts

@@ -1,5 +1,6 @@
 /**
- * browse extract "<instruction>" [--schema '<json>'] — Extract structured data.
+ * browse extract '<schema-json>' [scopeRef] — Extract structured data from the page or a stored scope.
  */
-export declare function extract(cdpUrl: string, instruction: string, schemaJson?: string): Promise<void>;
+import type { BrowseSession } from '../session.js';
+export declare function extract(session: BrowseSession, schemaJson: string, scopeRef?: string): Promise<void>;
 //# sourceMappingURL=extract.d.ts.map

package/dist/commands/extract.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"extract.d.ts","sourceRoot":"","sources":["../../src/commands/extract.ts"],"names":[],"mappings":"AAAA;;GAEG;AA2BH,wBAAsB,OAAO,CAC3B,MAAM,EAAE,MAAM,EACd,WAAW,EAAE,MAAM,EACnB,UAAU,CAAC,EAAE,MAAM,GAClB,OAAO,CAAC,IAAI,CAAC,CA4Bf"}
+{"version":3,"file":"extract.d.ts","sourceRoot":"","sources":["../../src/commands/extract.ts"],"names":[],"mappings":"AAAA;;GAEG;AAGH,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AA2JnD,wBAAsB,OAAO,CAC3B,OAAO,EAAE,aAAa,EACtB,UAAU,EAAE,MAAM,EAClB,QAAQ,CAAC,EAAE,MAAM,GAChB,OAAO,CAAC,IAAI,CAAC,CAgKf"}