npm - @browserbasehq/orca - Versions diffs - 3.4.0-preview.0 → 3.5.0-preview.1 - Mend

@browserbasehq/orca 3.4.0-preview.0 → 3.5.0-preview.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (919) hide show

package/dist/esm/lib/v3/dom/build/selectorRuntime.generated.js.map ADDED Viewed

@@ -0,0 +1 @@

+ {"version":3,"file":"selectorRuntime.generated.js","sourceRoot":"","sources":["../../../../../../lib/v3/dom/build/selectorRuntime.generated.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAAG,0oRAA0oR,CAAC;AACnrR,MAAM,CAAC,MAAM,sBAAsB,GAAG;IACpC,mBAAmB,EAAE,+EAA+E;IACpG,oBAAoB,EAAE,+EAA+E;IACrG,kBAAkB,EAAE,wOAAwO;IAC5P,qBAAqB,EAAE,+EAA+E;IACtG,oBAAoB,EAAE,6LAA6L;IACnN,mBAAmB,EAAE,iTAAiT;IACtU,oBAAoB,EAAE,8xBAA8xB;IACpzB,kBAAkB,EAAE,+SAA+S;IACnU,qBAAqB,EAAE,mVAAmV;CAClW,CAAC;AACX,MAAM,CAAC,MAAM,yBAAyB,GAAG;IACvC,mBAAmB,EAAE,yDAAyD;IAC9E,oBAAoB,EAAE,0DAA0D;IAChF,kBAAkB,EAAE,wDAAwD;IAC5E,qBAAqB,EAAE,2DAA2D;IAClF,oBAAoB,EAAE,0DAA0D;IAChF,mBAAmB,EAAE,yDAAyD;IAC9E,oBAAoB,EAAE,0DAA0D;IAChF,kBAAkB,EAAE,wDAAwD;IAC5E,qBAAqB,EAAE,2DAA2D;CAC1E,CAAC","sourcesContent":["/*\n * AUTO-GENERATED FILE. DO NOT EDIT.\n * Run `pnpm run build-dom-scripts` to regenerate.\n */\nexport const selectorRuntimeBootstrap = \"if (!globalThis.__stagehandSelectorRuntime) { var __stagehandSelectorRuntimeFactory=(()=>{var P=Object.defineProperty;var A=Object.getOwnPropertyDescriptor;var N=Object.getOwnPropertyNames;var D=Object.prototype.hasOwnProperty;var $=(t,e)=>{for(var n in e)P(t,n,{get:e[n],enumerable:!0})},C=(t,e,n,r)=>{if(e&&typeof e==\\\"object\\\"||typeof e==\\\"function\\\")for(let s of N(e))!D.call(t,s)&&s!==n&&P(t,s,{get:()=>e[s],enumerable:!(r=A(e,s))||r.enumerable});return t};var k=t=>C(P({},\\\"__esModule\\\",{value:!0}),t);var j={};$(j,{countCssWithRoots:()=>B,countTextWithRoots:()=>Y,countXPathNative:()=>U,countXPathWithRoots:()=>K,hasOpenShadowRoots:()=>V,queryCssWithRoots:()=>X,queryTextWithRoots:()=>M,queryXPathNative:()=>Z,queryXPathWithRoots:()=>z});function R(t){let e=String(t||\\\"\\\").trim().replace(/^xpath=/i,\\\"\\\");if(!e)return[];let n=[],r=0;for(;r<e.length;){let s=\\\"child\\\";e.startsWith(\\\"//\\\",r)?(s=\\\"desc\\\",r+=2):e[r]===\\\"/\\\"&&(s=\\\"child\\\",r+=1);let o=r,a=0,i=null;for(;r<e.length;){let l=e[r];if(i)l===i&&(i=null);else if(l===\\\"'\\\"||l==='\\\"')i=l;else if(l===\\\"[\\\")a++;else if(l===\\\"]\\\")a--;else if(l===\\\"/\\\"&&a===0)break;r+=1}let u=e.slice(o,r).trim();if(!u)continue;let{tag:c,predicates:f}=W(u);n.push({axis:s,tag:c,predicates:f})}return n}function q(t){let e=[],n=0;for(;n<t.length;){if(t[n]!==\\\"[\\\"){n++;continue}n++;let r=n,s=null;for(;n<t.length;){let o=t[n];if(s)o===s&&(s=null);else if(o===\\\"'\\\"||o==='\\\"')s=o;else if(o===\\\"]\\\")break;n++}e.push(t.slice(r,n).trim()),n++}return e}function W(t){let e=t.indexOf(\\\"[\\\");if(e===-1)return{tag:t===\\\"\\\"?\\\"*\\\":t.toLowerCase(),predicates:[]};let n=t.slice(0,e).trim(),r=n===\\\"\\\"?\\\"*\\\":n.toLowerCase(),s=t.slice(e),o=[];for(let a of q(s)){let i=d(a);i&&o.push(i)}return{tag:r,predicates:o}}function d(t){let e=t.trim();if(!e)return null;let n=y(e,\\\"or\\\");if(n.length>1){let o=n.map(a=>d(a)).filter(Boolean);return o.length!==n.length?null:{type:\\\"or\\\",predicates:o}}let r=y(e,\\\"and\\\");if(r.length>1){let o=r.map(a=>d(a)).filter(Boolean);return o.length!==r.length?null:{type:\\\"and\\\",predicates:o}}let s=O(e,\\\"not\\\");if(s!=null){let o=d(s);return o?{type:\\\"not\\\",predicate:o}:null}return _(e)}function _(t){let e=/^(?:'([^']*)'|\\\"([^\\\"]*)\\\")$/,n=\\\"[a-zA-Z_][\\\\\\\\w.-]*\\\",r=`(?:'([^']*)'|\\\"([^\\\"]*)\\\")`;if(/^\\\\d+$/.test(t))return{type:\\\"index\\\",index:Math.max(1,Number(t))};let s=t.match(new RegExp(`^normalize-space\\\\\\\$\\\\\\\\s*@(${n})\\\\\\\\s*\\\\\\\$\\\\\\\\s*=\\\\\\\\s*${r}$`));if(s)return{type:\\\"attrEquals\\\",name:s[1],value:s[2]??s[3]??\\\"\\\",normalize:!0};let o=t.match(new RegExp(`^normalize-space\\\\\\\$\\\\\\\\s*(?:text\\\\\\\\(\\\\\\\$|\\\\\\\\.)\\\\\\\\s*\\\\\\\\)\\\\\\\\s*=\\\\\\\\s*${r}$`));if(o)return{type:\\\"textEquals\\\",value:o[1]??o[2]??\\\"\\\",normalize:!0};let a=t.match(new RegExp(`^@(${n})\\\\\\\\s*=\\\\\\\\s*${r}$`));if(a)return{type:\\\"attrEquals\\\",name:a[1],value:a[2]??a[3]??\\\"\\\"};let i=t.match(new RegExp(`^@(${n})$`));if(i)return{type:\\\"attrExists\\\",name:i[1]};let u=t.match(new RegExp(`^contains\\\\\\\$\\\\\\\\s*@(${n})\\\\\\\\s*,\\\\\\\\s*${r}\\\\\\\\s*\\\\\\\$$`));if(u)return{type:\\\"attrContains\\\",name:u[1],value:u[2]??u[3]??\\\"\\\"};let c=t.match(new RegExp(`^starts-with\\\\\\\$\\\\\\\\s*@(${n})\\\\\\\\s*,\\\\\\\\s*${r}\\\\\\\\s*\\\\\\\$$`));if(c)return{type:\\\"attrStartsWith\\\",name:c[1],value:c[2]??c[3]??\\\"\\\"};let f=t.match(new RegExp(`^(?:text\\\\\\\$\\\\\\\$|\\\\\\\\.)\\\\\\\\s*=\\\\\\\\s*${r}$`));if(f)return{type:\\\"textEquals\\\",value:f[1]??f[2]??\\\"\\\"};let l=t.match(new RegExp(`^contains\\\\\\\$\\\\\\\\s*(?:text\\\\\\\\(\\\\\\\$|\\\\\\\\.)\\\\\\\\s*,\\\\\\\\s*${r}\\\\\\\\s*\\\\\\\\)$`));return l?{type:\\\"textContains\\\",value:l[1]??l[2]??\\\"\\\"}:(e.test(t),null)}function y(t,e){let n=[],r=0,s=0,o=null,a=0;for(;a<t.length;){let i=t[a];if(o){i===o&&(o=null),a+=1;continue}if(i===\\\"'\\\"||i==='\\\"'){o=i,a+=1;continue}if(i===\\\"(\\\"){s+=1,a+=1;continue}if(i===\\\")\\\"){s=Math.max(0,s-1),a+=1;continue}if(s===0&&L(t,a,e)){n.push(t.slice(r,a).trim()),a+=e.length,r=a;continue}a+=1}return n.push(t.slice(r).trim()),n.filter(i=>i.length>0)}function L(t,e,n){if(!t.startsWith(n,e))return!1;let r=e>0?t[e-1]:\\\" \\\";if(r===\\\"@\\\")return!1;let s=e+n.length<t.length?t[e+n.length]:\\\" \\\";return S(r)&&S(s)}function S(t){return!/[a-zA-Z0-9_.-]/.test(t)}function O(t,e){let n=`${e}(`;if(!t.startsWith(n)||!t.endsWith(\\\")\\\"))return null;let r=t.slice(n.length,-1);return I(r)?r:null}function I(t){let e=0,n=null;for(let r=0;r<t.length;r+=1){let s=t[r];if(n){s===n&&(n=null);continue}if(s===\\\"'\\\"||s==='\\\"'){n=s;continue}if(s===\\\"(\\\"?e+=1:s===\\\")\\\"&&(e-=1),e<0)return!1}return e===0}var F=t=>t.replace(/\\\\s+/g,\\\" \\\").trim();function b(t){return String(t.textContent??\\\"\\\")}function m(t,e){return e?F(t):t}function x(t,e){switch(e.type){case\\\"and\\\":return e.predicates.every(n=>x(t,n));case\\\"or\\\":return e.predicates.some(n=>x(t,n));case\\\"not\\\":return!x(t,e.predicate);case\\\"attrExists\\\":return t.getAttribute(e.name)!==null;case\\\"attrEquals\\\":{let n=t.getAttribute(e.name);return n===null?!1:m(n,e.normalize)===m(e.value,e.normalize)}case\\\"attrContains\\\":{let n=t.getAttribute(e.name);return n===null?!1:m(n,e.normalize).includes(m(e.value,e.normalize))}case\\\"attrStartsWith\\\":{let n=t.getAttribute(e.name);return n===null?!1:m(n,e.normalize).startsWith(m(e.value,e.normalize))}case\\\"textEquals\\\":return m(b(t),e.normalize)===m(e.value,e.normalize);case\\\"textContains\\\":return m(b(t),e.normalize).includes(m(e.value,e.normalize));case\\\"index\\\":return!0;default:return!0}}function v(t,e){let n=t;for(let r of e){if(!n.length)return[];if(r.type===\\\"index\\\"){let s=r.index-1;n=s>=0&&s<n.length?[n[s]]:[];continue}n=n.filter(s=>x(s,r))}return n}function w(t){let e=new Map;for(let n=0;n<t.length;n+=2){let r=t[n],s=t[n+1];r instanceof Element&&s instanceof ShadowRoot&&e.set(r,s)}return e}function E(t,e){if(!t)return[];if(t instanceof Document)return t.documentElement?[t.documentElement]:[];if(t instanceof ShadowRoot||t instanceof DocumentFragment)return Array.from(t.children||[]);if(!(t instanceof Element))return[];let n=Array.from(t.children||[]);t.shadowRoot&&n.push(...Array.from(t.shadowRoot.children||[]));let r=e.get(t);return r&&n.push(...Array.from(r.children||[])),n}function H(t,e){let n=[],r=new Set,s=[...E(t,e)].reverse();for(;s.length;){let o=s.pop();if(!(o instanceof Element)||r.has(o))continue;r.add(o),n.push(o);let a=E(o,e);for(let i=a.length-1;i>=0;i-=1)s.push(a[i])}return n}function X(t,e,...n){let r=w(n),s=[],o=new Set,a=[...E(this,r)].reverse();for(;a.length&&s.length<e;){let i=a.pop();if(!(i instanceof Element)||o.has(i))continue;o.add(i);try{if(i.matches(t)&&(s.push(i),s.length>=e))break}catch{return[]}let u=E(i,r);for(let c=u.length-1;c>=0;c-=1)a.push(u[c])}return s}function B(t,...e){return X.call(this,t,Number.MAX_SAFE_INTEGER,...e).length}function M(t,e,...n){let r=w(n),s=new Set([\\\"SCRIPT\\\",\\\"STYLE\\\",\\\"TEMPLATE\\\",\\\"NOSCRIPT\\\",\\\"HEAD\\\",\\\"TITLE\\\",\\\"LINK\\\",\\\"META\\\",\\\"HTML\\\",\\\"BODY\\\"]),o=String(t||\\\"\\\").toLowerCase();if(!o)return[];let a=l=>{let g=l.tagName?l.tagName.toUpperCase():\\\"\\\";if(s.has(g))return\\\"\\\";try{let p=l.innerText;if(typeof p==\\\"string\\\"&&p.trim())return p.trim()}catch{}let h=l.textContent;return typeof h==\\\"string\\\"?h.trim():\\\"\\\"},i=[],u=new Set,c=[...E(this,r)].reverse();for(;c.length;){let l=c.pop();if(!(l instanceof Element)||u.has(l))continue;u.add(l);let g=a(l);g&&g.toLowerCase().includes(o)&&i.push(l);let h=E(l,r);for(let p=h.length-1;p>=0;p-=1)c.push(h[p])}let f=[];for(let l of i){let g=!1;for(let h of i)if(l!==h)try{if(l.contains(h)){g=!0;break}}catch{}if(!g&&(f.push(l),f.length>=e))break}return f}function Y(t,...e){return M.call(this,t,Number.MAX_SAFE_INTEGER,...e).length}function T(t){let e=String(t||\\\"\\\").trim();return e?e.replace(/^xpath=/i,\\\"\\\").trim():\\\"\\\"}function G(t,e){return e.tag===\\\"*\\\"?!0:t.localName===e.tag}function z(t,e,...n){let r=w(n),s=R(t);if(!s.length)return[];let o=[this];for(let a of s){let i=[],u=new Set;for(let c of o){let l=(a.axis===\\\"child\\\"?E(c,r):H(c,r)).filter(h=>G(h,a)),g=v(l,a.predicates);for(let h of g)if(!u.has(h)&&(u.add(h),i.push(h),i.length>=e&&a===s[s.length-1]))break}if(!i.length)return[];o=i}return o.slice(0,e)}function K(t,...e){return z.call(this,t,Number.MAX_SAFE_INTEGER,...e).length}function Z(t,e){let n=T(t);if(!n||!Number.isFinite(e)||e<=0)return[];try{let r=this.evaluate(n,this,null,XPathResult.ORDERED_NODE_SNAPSHOT_TYPE,null),s=[],o=Math.min(r.snapshotLength,Math.floor(e));for(let a=0;a<o;a+=1){let i=r.snapshotItem(a);i instanceof Element&&s.push(i)}return s}catch{return[]}}function U(t){let e=T(t);if(!e)return 0;try{let n=this.evaluate(e,this,null,XPathResult.ORDERED_NODE_SNAPSHOT_TYPE,null),r=0;for(let s=0;s<n.snapshotLength;s+=1)n.snapshotItem(s)instanceof Element&&(r+=1);return r}catch{return 0}}function V(){try{let t=this.createTreeWalker(this,NodeFilter.SHOW_ELEMENT);for(;t.nextNode();){let e=t.currentNode;if(e instanceof Element&&e.shadowRoot)return!0}}catch{return!1}return!1}return k(j);})();\\n globalThis.__stagehandSelectorRuntime = __stagehandSelectorRuntimeFactory;\\n}\";\nexport const selectorRuntimeSources = {\n \"countCssWithRoots\": \"function O(t,...e){return C.call(this,t,Number.MAX_SAFE_INTEGER,...e).length}\",\n \"countTextWithRoots\": \"function I(t,...e){return k.call(this,t,Number.MAX_SAFE_INTEGER,...e).length}\",\n \"countXPathNative\": \"function B(t){let e=v(t);if(!e)return 0;try{let n=this.evaluate(e,this,null,XPathResult.ORDERED_NODE_SNAPSHOT_TYPE,null),r=0;for(let s=0;s<n.snapshotLength;s+=1)n.snapshotItem(s)instanceof Element&&(r+=1);return r}catch{return 0}}\",\n \"countXPathWithRoots\": \"function F(t,...e){return W.call(this,t,Number.MAX_SAFE_INTEGER,...e).length}\",\n \"hasOpenShadowRoots\": \"function Y(){try{let t=this.createTreeWalker(this,NodeFilter.SHOW_ELEMENT);for(;t.nextNode();){let e=t.currentNode;if(e instanceof Element&&e.shadowRoot)return!0}}catch{return!1}return!1}\",\n \"queryCssWithRoots\": \"function C(t,e,...n){let r=P(n),s=[],o=new Set,a=[...E(this,r)].reverse();for(;a.length&&s.length<e;){let i=a.pop();if(!(i instanceof Element)||o.has(i))continue;o.add(i);try{if(i.matches(t)&&(s.push(i),s.length>=e))break}catch{return[]}let u=E(i,r);for(let c=u.length-1;c>=0;c-=1)a.push(u[c])}return s}\",\n \"queryTextWithRoots\": \"function k(t,e,...n){let r=P(n),s=new Set([\\\"SCRIPT\\\",\\\"STYLE\\\",\\\"TEMPLATE\\\",\\\"NOSCRIPT\\\",\\\"HEAD\\\",\\\"TITLE\\\",\\\"LINK\\\",\\\"META\\\",\\\"HTML\\\",\\\"BODY\\\"]),o=String(t||\\\"\\\").toLowerCase();if(!o)return[];let a=l=>{let g=l.tagName?l.tagName.toUpperCase():\\\"\\\";if(s.has(g))return\\\"\\\";try{let p=l.innerText;if(typeof p==\\\"string\\\"&&p.trim())return p.trim()}catch{}let h=l.textContent;return typeof h==\\\"string\\\"?h.trim():\\\"\\\"},i=[],u=new Set,c=[...E(this,r)].reverse();for(;c.length;){let l=c.pop();if(!(l instanceof Element)||u.has(l))continue;u.add(l);let g=a(l);g&&g.toLowerCase().includes(o)&&i.push(l);let h=E(l,r);for(let p=h.length-1;p>=0;p-=1)c.push(h[p])}let f=[];for(let l of i){let g=!1;for(let h of i)if(l!==h)try{if(l.contains(h)){g=!0;break}}catch{}if(!g&&(f.push(l),f.length>=e))break}return f}\",\n \"queryXPathNative\": \"function H(t,e){let n=v(t);if(!n||!Number.isFinite(e)||e<=0)return[];try{let r=this.evaluate(n,this,null,XPathResult.ORDERED_NODE_SNAPSHOT_TYPE,null),s=[],o=Math.min(r.snapshotLength,Math.floor(e));for(let a=0;a<o;a+=1){let i=r.snapshotItem(a);i instanceof Element&&s.push(i)}return s}catch{return[]}}\",\n \"queryXPathWithRoots\": \"function W(t,e,...n){let r=P(n),s=b(t);if(!s.length)return[];let o=[this];for(let a of s){let i=[],u=new Set;for(let c of o){let l=(a.axis===\\\"child\\\"?E(c,r):$(c,r)).filter(h=>q(h,a)),g=R(l,a.predicates);for(let h of g)if(!u.has(h)&&(u.add(h),i.push(h),i.length>=e&&a===s[s.length-1]))break}if(!i.length)return[];o=i}return o.slice(0,e)}\"\n} as const;\nexport const selectorRuntimeGlobalRefs = {\n \"countCssWithRoots\": \"globalThis.__stagehandSelectorRuntime.countCssWithRoots\",\n \"countTextWithRoots\": \"globalThis.__stagehandSelectorRuntime.countTextWithRoots\",\n \"countXPathNative\": \"globalThis.__stagehandSelectorRuntime.countXPathNative\",\n \"countXPathWithRoots\": \"globalThis.__stagehandSelectorRuntime.countXPathWithRoots\",\n \"hasOpenShadowRoots\": \"globalThis.__stagehandSelectorRuntime.hasOpenShadowRoots\",\n \"queryCssWithRoots\": \"globalThis.__stagehandSelectorRuntime.queryCssWithRoots\",\n \"queryTextWithRoots\": \"globalThis.__stagehandSelectorRuntime.queryTextWithRoots\",\n \"queryXPathNative\": \"globalThis.__stagehandSelectorRuntime.queryXPathNative\",\n \"queryXPathWithRoots\": \"globalThis.__stagehandSelectorRuntime.queryXPathWithRoots\"\n} as const;\nexport type SelectorRuntimeName = keyof typeof selectorRuntimeSources;\n"]}

package/dist/esm/lib/v3/handlers/extractHandler.js CHANGED Viewed

@@ -41,7 +41,7 @@ export class ExtractHandler {
         this.onMetrics = onMetrics;
     }
     async extract(params) {
-        const { instruction, schema, page, selector, ignoreSelectors, timeout, model, } = params;
+        const { instruction, schema, page, selector, ignoreSelectors, timeout, model, screenshot, } = params;
         const llmClient = this.resolveLlmClient(model);
         const ensureTimeRemaining = createTimeoutGuard(timeout, (ms) => new ExtractTimeoutError(ms));
         // No-args → page text (parity with v2)
@@ -62,6 +62,9 @@ export class ExtractHandler {
         if (!instruction && schema) {
             throw new StagehandInvalidArgumentError("extract() requires an instruction when a schema is provided.");
         }
+        if (screenshot && llmClient.type !== "aisdk") {
+            throw new StagehandInvalidArgumentError("extract({ screenshot: true }) is only supported with AI SDK clients.");
+        }
         const focusSelector = selector?.replace(/^xpath=/, "") ?? "";
         // Build the hybrid snapshot (includes combinedTree; combinedUrlMap optional)
         ensureTimeRemaining();
@@ -70,9 +73,22 @@ export class ExtractHandler {
             focusSelector: focusSelector,
             ignoreSelectors,
         });
+        const screenshotBuffer = screenshot
+            ? await (async () => {
+                ensureTimeRemaining();
+                const buffer = await page.screenshot({
+                    fullPage: false,
+                    type: "png",
+                });
+                ensureTimeRemaining();
+                return buffer;
+            })()
+            : undefined;
         v3Logger({
             category: "extraction",
-            message: "Starting extraction using a11y snapshot",
+            message: screenshot
+                ? "Starting extraction using a11y snapshot and viewport screenshot"
+                : "Starting extraction using a11y snapshot",
             level: 1,
             auxiliary: instruction
                 ? { instruction: { value: instruction, type: "string" } }
@@ -100,6 +116,7 @@ export class ExtractHandler {
             userProvidedInstructions: this.systemPrompt,
             logger: v3Logger,
             logInferenceToFile: this.logInferenceToFile,
+            screenshot: screenshotBuffer,
         });
         const { metadata: { completed }, prompt_tokens, completion_tokens, reasoning_tokens = 0, cached_input_tokens = 0, inference_time_ms, ...rest } = extractionResponse;
         let output = rest;

package/dist/esm/lib/v3/handlers/extractHandler.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"extractHandler.js","sourceRoot":"","sources":["../../../../../lib/v3/handlers/extractHandler.ts"],"names":[],"mappings":"AAAA,oCAAoC;AACpC,OAAO,EAAE,OAAO,IAAI,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAC3D,OAAO,EACL,WAAW,EACX,UAAU,EACV,UAAU,EACV,eAAe,GAChB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAE,QAAQ,EAAE,MAAM,cAAc,CAAC;AACxC,OAAO,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AAC5D,OAAO,EAAE,qBAAqB,EAAE,MAAM,sCAAsC,CAAC;AAK7E,OAAO,EACL,oBAAoB,EACpB,cAAc,GACf,MAAM,4BAA4B,CAAC;AAMpC,OAAO,EACL,6BAA6B,EAC7B,mBAAmB,GACpB,MAAM,8BAA8B,CAAC;AACtC,OAAO,EAAE,kBAAkB,EAAE,MAAM,gCAAgC,CAAC;AAOpE;;;;;;;;;GASG;AACH,MAAM,UAAU,+BAA+B,CAC7C,MAAS;IAET,MAAM,CAAC,WAAW,EAAE,QAAQ,CAAC,GAAG,eAAe,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IAC5D,OAAO,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC;AACjC,CAAC;AAcD,MAAM,OAAO,cAAc;IACR,SAAS,CAAY;IACrB,gBAAgB,CAAiB;IACjC,oBAAoB,CAAgB;IACpC,gBAAgB,CAA4C;IAC5D,YAAY,CAAS;IACrB,kBAAkB,CAAU;IAC5B,YAAY,CAAU;IACtB,SAAS,CAOhB;IAEV,YACE,SAAoB,EACpB,gBAAgC,EAChC,oBAAmC,EACnC,gBAA2D,EAC3D,YAAqB,EACrB,kBAA4B,EAC5B,YAAsB,EACtB,SAOS;QAET,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,gBAAgB,GAAG,gBAAgB,CAAC;QACzC,IAAI,CAAC,oBAAoB,GAAG,oBAAoB,CAAC;QACjD,IAAI,CAAC,gBAAgB,GAAG,gBAAgB,CAAC;QACzC,IAAI,CAAC,YAAY,GAAG,YAAY,IAAI,EAAE,CAAC;QACvC,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,IAAI,KAAK,CAAC;QACtD,IAAI,CAAC,YAAY,GAAG,YAAY,IAAI,KAAK,CAAC;QAC1C,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;IAC7B,CAAC;IAED,KAAK,CAAC,OAAO,CACX,MAA+B;QAE/B,MAAM,EACJ,WAAW,EACX,MAAM,EACN,IAAI,EACJ,QAAQ,EACR,eAAe,EACf,OAAO,EACP,KAAK,GACN,GAAG,MAAM,CAAC;QAEX,MAAM,SAAS,GAAG,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC;QAE/C,MAAM,mBAAmB,GAAG,kBAAkB,CAC5C,OAAO,EACP,CAAC,EAAE,EAAE,EAAE,CAAC,IAAI,mBAAmB,CAAC,EAAE,CAAC,CACpC,CAAC;QAEF,uCAAuC;QACvC,MAAM,MAAM,GAAG,CAAC,WAAW,IAAI,CAAC,MAAM,CAAC;QACvC,IAAI,MAAM,EAAE,CAAC;YACX,MAAM,aAAa,GAAG,QAAQ,EAAE,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,IAAI,EAAE,CAAC;YAC9D,mBAAmB,EAAE,CAAC;YACtB,MAAM,IAAI,GAAG,MAAM,qBAAqB,CAAC,IAAI,EAAE;gBAC7C,YAAY,EAAE,IAAI,CAAC,YAAY;gBAC/B,aAAa,EAAE,aAAa,IAAI,SAAS;gBACzC,eAAe;aAChB,CAAC,CAAC;YACH,mBAAmB,EAAE,CAAC;YAEtB,MAAM,MAAM,GAAG,EAAE,QAAQ,EAAE,IAAI,CAAC,YAAY,EAAE,CAAC;YAC/C,0CAA0C;YAC1C,OAAO,cAAc,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QACtC,CAAC;QAED,IAAI,CAAC,WAAW,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,IAAI,6BAA6B,CACrC,8DAA8D,CAC/D,CAAC;QACJ,CAAC;QAED,MAAM,aAAa,GAAG,QAAQ,EAAE,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,IAAI,EAAE,CAAC;QAE7D,6EAA6E;QAC7E,mBAAmB,EAAE,CAAC;QACtB,MAAM,EAAE,YAAY,EAAE,cAAc,EAAE,GAAG,MAAM,qBAAqB,CAAC,IAAI,EAAE;YACzE,YAAY,EAAE,IAAI,CAAC,YAAY;YAC/B,aAAa,EAAE,aAAa;YAC5B,eAAe;SAChB,CAAC,CAAC;QAEH,QAAQ,CAAC;YACP,QAAQ,EAAE,YAAY;YACtB,OAAO,EAAE,yCAAyC;YAClD,KAAK,EAAE,CAAC;YACR,SAAS,EAAE,WAAW;gBACpB,CAAC,CAAC,EAAE,WAAW,EAAE,EAAE,KAAK,EAAE,WAAW,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE;gBACzD,CAAC,CAAC,SAAS;SACd,CAAC,CAAC;QAEH,qFAAqF;QACrF,MAAM,UAAU,GAAuB,CAAC,MAAM;YAC5C,oBAAoB,CAAuB,CAAC;QAC9C,0EAA0E;QAC1E,MAAM,cAAc,GAAG,UAAU,CAAC,UAAU,CAAC,KAAK,QAAQ,CAAC;QAC3D,MAAM,QAAQ,GAAG,OAAgB,CAAC;QAClC,MAAM,OAAO,GAAG,WAAW,CAAC,UAAU,CAAC,CAAC;QACxC,MAAM,YAAY,GAAuB,cAAc;YACrD,CAAC,CAAE,UAAiC;YACpC,CAAC,CAAE,OAAO,CAAC,MAAM,CAAC;gBACd,CAAC,QAAQ,CAAC,EAAE,UAAwB;aACrC,CAAwB,CAAC;QAE9B,MAAM,CAAC,iBAAiB,EAAE,aAAa,CAAC,GACtC,+BAA+B,CAAC,YAAY,CAAC,CAAC;QAEhD,mBAAmB,EAAE,CAAC;QACtB,MAAM,kBAAkB,GACtB,MAAM,UAAU,CAAqB;YACnC,WAAW;YACX,WAAW,EAAE,YAAY;YACzB,MAAM,EAAE,iBAAuC;YAC/C,SAAS;YACT,wBAAwB,EAAE,IAAI,CAAC,YAAY;YAC3C,MAAM,EAAE,QAAQ;YAChB,kBAAkB,EAAE,IAAI,CAAC,kBAAkB;SAC5C,CAAC,CAAC;QAEL,MAAM,EACJ,QAAQ,EAAE,EAAE,SAAS,EAAE,EACvB,aAAa,EACb,iBAAiB,EACjB,gBAAgB,GAAG,CAAC,EACpB,mBAAmB,GAAG,CAAC,EACvB,iBAAiB,EACjB,GAAG,IAAI,EACR,GAAG,kBAAkB,CAAC;QACvB,IAAI,MAAM,GAAG,IAAgD,CAAC;QAE9D,4CAA4C;QAC5C,IAAI,CAAC,SAAS,EAAE,CACd,cAAc,CAAC,OAAO,EACtB,aAAa,EACb,iBAAiB,EACjB,gBAAgB,EAChB,mBAAmB,EACnB,iBAAiB,CAClB,CAAC;QAEF,2EAA2E;QAC3E,MAAM,OAAO,GAA8B,CAAC,cAAc,IAAI,EAAE,CAG/D,CAAC;QACF,KAAK,MAAM,EAAE,QAAQ,EAAE,IAAI,aAAa,EAAE,CAAC;YACzC,UAAU,CACR,MAAiC,EACjC,QAAQ,EACR,OAA4C,CAC7C,CAAC;QACJ,CAAC;QACD,sDAAsD;QACtD,IAAI,CAAC,cAAc,IAAI,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;YAC5D,MAAM,GAAI,MAAkC,CAAC,QAAQ,CAAC,CAAC;QACzD,CAAC;QAED,MAAM,mBAAmB,GAAG,GAAG,CAAC;QAChC,MAAM,YAAY,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,WAAW,CAAC;QAC3D,MAAM,aAAa,GACjB,YAAY,CAAC,MAAM,GAAG,mBAAmB;YACvC,CAAC,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,mBAAmB,CAAC,GAAG,KAAK;YACpD,CAAC,CAAC,YAAY,CAAC;QAEnB,QAAQ,CAAC;YACP,QAAQ,EAAE,YAAY;YACtB,OAAO,EAAE,SAAS;gBAChB,CAAC,CAAC,mCAAmC;gBACrC,CAAC,CAAC,iDAAiD;YACrD,KAAK,EAAE,CAAC;YACR,SAAS,EAAE;gBACT,aAAa,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,aAAa,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE;gBAC/D,iBAAiB,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,iBAAiB,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE;gBACvE,iBAAiB,EAAE;oBACjB,KAAK,EAAE,MAAM,CAAC,iBAAiB,CAAC;oBAChC,IAAI,EAAE,QAAQ;iBACf;gBACD,MAAM,EAAE,EAAE,KAAK,EAAE,aAAa,EAAE,IAAI,EAAE,QAAQ,EAAE;aACjD;SACF,CAAC,CAAC;QAEH,OAAO,MAAiC,CAAC;IAC3C,CAAC;CACF","sourcesContent":["// lib/v3/handlers/extractHandler.ts\nimport { extract as runExtract } from \"../../inference.js\";\nimport {\n getZFactory,\n getZodType,\n injectUrls,\n transformSchema,\n} from \"../../utils.js\";\nimport { v3Logger } from \"../logger.js\";\nimport { V3FunctionName } from \"../types/public/methods.js\";\nimport { captureHybridSnapshot } from \"../understudy/a11y/snapshot/index.js\";\nimport type { ZodTypeAny } from \"zod\";\nimport { LLMClient } from \"../llm/LLMClient.js\";\nimport { ExtractHandlerParams } from \"../types/private/handlers.js\";\nimport { EncodedId, ZodPathSegments } from \"../types/private/internal.js\";\nimport {\n defaultExtractSchema,\n pageTextSchema,\n} from \"../types/public/methods.js\";\nimport {\n AvailableModel,\n ClientOptions,\n ModelConfiguration,\n} from \"../types/public/model.js\";\nimport {\n StagehandInvalidArgumentError,\n ExtractTimeoutError,\n} from \"../types/public/sdkErrors.js\";\nimport { createTimeoutGuard } from \"./handlerUtils/timeoutGuard.js\";\nimport type {\n InferStagehandSchema,\n StagehandZodObject,\n StagehandZodSchema,\n} from \"../zodCompat.js\";\n\n/*\n Scans the provided Zod schema for any `z.string().url()` fields and\n * replaces them with `z.number()`.\n \n @param schema - The Zod object schema to transform.\n * @returns A tuple containing:\n * 1. The transformed schema (or the original schema if no changes were needed).\n * 2. An array of {@link ZodPathSegments} objects representing all the replaced URL fields,\n * with each path segment showing where in the schema the replacement occurred.\n */\nexport function transformUrlStringsToNumericIds<T extends StagehandZodSchema>(\n schema: T,\n): [StagehandZodSchema, ZodPathSegments[]] {\n const [finalSchema, urlPaths] = transformSchema(schema, []);\n return [finalSchema, urlPaths];\n}\n\ninterface ExtractionResponseBase {\n metadata: { completed: boolean };\n prompt_tokens: number;\n completion_tokens: number;\n reasoning_tokens: number;\n cached_input_tokens?: number;\n inference_time_ms: number;\n}\n\ntype ExtractionResponse<T extends StagehandZodObject> = ExtractionResponseBase &\n InferStagehandSchema<T>;\n\nexport class ExtractHandler {\n private readonly llmClient: LLMClient;\n private readonly defaultModelName: AvailableModel;\n private readonly defaultClientOptions: ClientOptions;\n private readonly resolveLlmClient: (model?: ModelConfiguration) => LLMClient;\n private readonly systemPrompt: string;\n private readonly logInferenceToFile: boolean;\n private readonly experimental: boolean;\n private readonly onMetrics?: (\n functionName: V3FunctionName,\n promptTokens: number,\n completionTokens: number,\n reasoningTokens: number,\n cachedInputTokens: number,\n inferenceTimeMs: number,\n ) => void;\n\n constructor(\n llmClient: LLMClient,\n defaultModelName: AvailableModel,\n defaultClientOptions: ClientOptions,\n resolveLlmClient: (model?: ModelConfiguration) => LLMClient,\n systemPrompt?: string,\n logInferenceToFile?: boolean,\n experimental?: boolean,\n onMetrics?: (\n functionName: V3FunctionName,\n promptTokens: number,\n completionTokens: number,\n reasoningTokens: number,\n cachedInputTokens: number,\n inferenceTimeMs: number,\n ) => void,\n ) {\n this.llmClient = llmClient;\n this.defaultModelName = defaultModelName;\n this.defaultClientOptions = defaultClientOptions;\n this.resolveLlmClient = resolveLlmClient;\n this.systemPrompt = systemPrompt ?? \"\";\n this.logInferenceToFile = logInferenceToFile ?? false;\n this.experimental = experimental ?? false;\n this.onMetrics = onMetrics;\n }\n\n async extract<T extends StagehandZodSchema>(\n params: ExtractHandlerParams<T>,\n ): Promise<InferStagehandSchema<T> \| { pageText: string }> {\n const {\n instruction,\n schema,\n page,\n selector,\n ignoreSelectors,\n timeout,\n model,\n } = params;\n\n const llmClient = this.resolveLlmClient(model);\n\n const ensureTimeRemaining = createTimeoutGuard(\n timeout,\n (ms) => new ExtractTimeoutError(ms),\n );\n\n // No-args → page text (parity with v2)\n const noArgs = !instruction && !schema;\n if (noArgs) {\n const focusSelector = selector?.replace(/^xpath=/i, \"\") ?? \"\";\n ensureTimeRemaining();\n const snap = await captureHybridSnapshot(page, {\n experimental: this.experimental,\n focusSelector: focusSelector \|\| undefined,\n ignoreSelectors,\n });\n ensureTimeRemaining();\n\n const result = { pageText: snap.combinedTree };\n // Validate via the same schema used in v2\n return pageTextSchema.parse(result);\n }\n\n if (!instruction && schema) {\n throw new StagehandInvalidArgumentError(\n \"extract() requires an instruction when a schema is provided.\",\n );\n }\n\n const focusSelector = selector?.replace(/^xpath=/, \"\") ?? \"\";\n\n // Build the hybrid snapshot (includes combinedTree; combinedUrlMap optional)\n ensureTimeRemaining();\n const { combinedTree, combinedUrlMap } = await captureHybridSnapshot(page, {\n experimental: this.experimental,\n focusSelector: focusSelector,\n ignoreSelectors,\n });\n\n v3Logger({\n category: \"extraction\",\n message: \"Starting extraction using a11y snapshot\",\n level: 1,\n auxiliary: instruction\n ? { instruction: { value: instruction, type: \"string\" } }\n : undefined,\n });\n\n // Normalize schema: if instruction provided without schema, use defaultExtractSchema\n const baseSchema: StagehandZodSchema = (schema ??\n defaultExtractSchema) as StagehandZodSchema;\n // Ensure we pass an object schema into inference; wrap non-object schemas\n const isObjectSchema = getZodType(baseSchema) === \"object\";\n const WRAP_KEY = \"value\" as const;\n const factory = getZFactory(baseSchema);\n const objectSchema: StagehandZodObject = isObjectSchema\n ? (baseSchema as StagehandZodObject)\n : (factory.object({\n [WRAP_KEY]: baseSchema as ZodTypeAny,\n }) as StagehandZodObject);\n\n const [transformedSchema, urlFieldPaths] =\n transformUrlStringsToNumericIds(objectSchema);\n\n ensureTimeRemaining();\n const extractionResponse: ExtractionResponse<StagehandZodObject> =\n await runExtract<StagehandZodObject>({\n instruction,\n domElements: combinedTree,\n schema: transformedSchema as StagehandZodObject,\n llmClient,\n userProvidedInstructions: this.systemPrompt,\n logger: v3Logger,\n logInferenceToFile: this.logInferenceToFile,\n });\n\n const {\n metadata: { completed },\n prompt_tokens,\n completion_tokens,\n reasoning_tokens = 0,\n cached_input_tokens = 0,\n inference_time_ms,\n ...rest\n } = extractionResponse;\n let output = rest as InferStagehandSchema<StagehandZodObject>;\n\n // Update EXTRACT metrics from the LLM calls\n this.onMetrics?.(\n V3FunctionName.EXTRACT,\n prompt_tokens,\n completion_tokens,\n reasoning_tokens,\n cached_input_tokens,\n inference_time_ms,\n );\n\n // Re-inject URLs for any url() fields we temporarily converted to number()\n const idToUrl: Record<EncodedId, string> = (combinedUrlMap ?? {}) as Record<\n EncodedId,\n string\n >;\n for (const { segments } of urlFieldPaths) {\n injectUrls(\n output as Record<string, unknown>,\n segments,\n idToUrl as unknown as Record<string, string>,\n );\n }\n // If we wrapped a non-object schema, unwrap the value\n if (!isObjectSchema && output && typeof output === \"object\") {\n output = (output as Record<string, unknown>)[WRAP_KEY];\n }\n\n const resultPreviewLength = 200;\n const resultString = JSON.stringify(output) ?? \"undefined\";\n const resultPreview =\n resultString.length > resultPreviewLength\n ? resultString.slice(0, resultPreviewLength) + \"...\"\n : resultString;\n\n v3Logger({\n category: \"extraction\",\n message: completed\n ? \"Extraction completed successfully\"\n : \"Extraction incomplete after processing all data\",\n level: 1,\n auxiliary: {\n prompt_tokens: { value: String(prompt_tokens), type: \"string\" },\n completion_tokens: { value: String(completion_tokens), type: \"string\" },\n inference_time_ms: {\n value: String(inference_time_ms),\n type: \"string\",\n },\n result: { value: resultPreview, type: \"string\" },\n },\n });\n\n return output as InferStagehandSchema<T>;\n }\n}\n"]}
1	+ {"version":3,"file":"extractHandler.js","sourceRoot":"","sources":["../../../../../lib/v3/handlers/extractHandler.ts"],"names":[],"mappings":"AAAA,oCAAoC;AACpC,OAAO,EAAE,OAAO,IAAI,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAC3D,OAAO,EACL,WAAW,EACX,UAAU,EACV,UAAU,EACV,eAAe,GAChB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAE,QAAQ,EAAE,MAAM,cAAc,CAAC;AACxC,OAAO,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AAC5D,OAAO,EAAE,qBAAqB,EAAE,MAAM,sCAAsC,CAAC;AAK7E,OAAO,EACL,oBAAoB,EACpB,cAAc,GACf,MAAM,4BAA4B,CAAC;AAMpC,OAAO,EACL,6BAA6B,EAC7B,mBAAmB,GACpB,MAAM,8BAA8B,CAAC;AACtC,OAAO,EAAE,kBAAkB,EAAE,MAAM,gCAAgC,CAAC;AAOpE;;;;;;;;;GASG;AACH,MAAM,UAAU,+BAA+B,CAC7C,MAAS;IAET,MAAM,CAAC,WAAW,EAAE,QAAQ,CAAC,GAAG,eAAe,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IAC5D,OAAO,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC;AACjC,CAAC;AAcD,MAAM,OAAO,cAAc;IACR,SAAS,CAAY;IACrB,gBAAgB,CAAiB;IACjC,oBAAoB,CAAgB;IACpC,gBAAgB,CAA4C;IAC5D,YAAY,CAAS;IACrB,kBAAkB,CAAU;IAC5B,YAAY,CAAU;IACtB,SAAS,CAOhB;IAEV,YACE,SAAoB,EACpB,gBAAgC,EAChC,oBAAmC,EACnC,gBAA2D,EAC3D,YAAqB,EACrB,kBAA4B,EAC5B,YAAsB,EACtB,SAOS;QAET,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,gBAAgB,GAAG,gBAAgB,CAAC;QACzC,IAAI,CAAC,oBAAoB,GAAG,oBAAoB,CAAC;QACjD,IAAI,CAAC,gBAAgB,GAAG,gBAAgB,CAAC;QACzC,IAAI,CAAC,YAAY,GAAG,YAAY,IAAI,EAAE,CAAC;QACvC,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,IAAI,KAAK,CAAC;QACtD,IAAI,CAAC,YAAY,GAAG,YAAY,IAAI,KAAK,CAAC;QAC1C,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;IAC7B,CAAC;IAED,KAAK,CAAC,OAAO,CACX,MAA+B;QAE/B,MAAM,EACJ,WAAW,EACX,MAAM,EACN,IAAI,EACJ,QAAQ,EACR,eAAe,EACf,OAAO,EACP,KAAK,EACL,UAAU,GACX,GAAG,MAAM,CAAC;QAEX,MAAM,SAAS,GAAG,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC;QAE/C,MAAM,mBAAmB,GAAG,kBAAkB,CAC5C,OAAO,EACP,CAAC,EAAE,EAAE,EAAE,CAAC,IAAI,mBAAmB,CAAC,EAAE,CAAC,CACpC,CAAC;QAEF,uCAAuC;QACvC,MAAM,MAAM,GAAG,CAAC,WAAW,IAAI,CAAC,MAAM,CAAC;QACvC,IAAI,MAAM,EAAE,CAAC;YACX,MAAM,aAAa,GAAG,QAAQ,EAAE,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,IAAI,EAAE,CAAC;YAC9D,mBAAmB,EAAE,CAAC;YACtB,MAAM,IAAI,GAAG,MAAM,qBAAqB,CAAC,IAAI,EAAE;gBAC7C,YAAY,EAAE,IAAI,CAAC,YAAY;gBAC/B,aAAa,EAAE,aAAa,IAAI,SAAS;gBACzC,eAAe;aAChB,CAAC,CAAC;YACH,mBAAmB,EAAE,CAAC;YAEtB,MAAM,MAAM,GAAG,EAAE,QAAQ,EAAE,IAAI,CAAC,YAAY,EAAE,CAAC;YAC/C,0CAA0C;YAC1C,OAAO,cAAc,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QACtC,CAAC;QAED,IAAI,CAAC,WAAW,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,IAAI,6BAA6B,CACrC,8DAA8D,CAC/D,CAAC;QACJ,CAAC;QAED,IAAI,UAAU,IAAI,SAAS,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;YAC7C,MAAM,IAAI,6BAA6B,CACrC,sEAAsE,CACvE,CAAC;QACJ,CAAC;QAED,MAAM,aAAa,GAAG,QAAQ,EAAE,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,IAAI,EAAE,CAAC;QAE7D,6EAA6E;QAC7E,mBAAmB,EAAE,CAAC;QACtB,MAAM,EAAE,YAAY,EAAE,cAAc,EAAE,GAAG,MAAM,qBAAqB,CAAC,IAAI,EAAE;YACzE,YAAY,EAAE,IAAI,CAAC,YAAY;YAC/B,aAAa,EAAE,aAAa;YAC5B,eAAe;SAChB,CAAC,CAAC;QAEH,MAAM,gBAAgB,GAAG,UAAU;YACjC,CAAC,CAAC,MAAM,CAAC,KAAK,IAAI,EAAE;gBAChB,mBAAmB,EAAE,CAAC;gBACtB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC;oBACnC,QAAQ,EAAE,KAAK;oBACf,IAAI,EAAE,KAAK;iBACZ,CAAC,CAAC;gBACH,mBAAmB,EAAE,CAAC;gBACtB,OAAO,MAAM,CAAC;YAChB,CAAC,CAAC,EAAE;YACN,CAAC,CAAC,SAAS,CAAC;QAEd,QAAQ,CAAC;YACP,QAAQ,EAAE,YAAY;YACtB,OAAO,EAAE,UAAU;gBACjB,CAAC,CAAC,iEAAiE;gBACnE,CAAC,CAAC,yCAAyC;YAC7C,KAAK,EAAE,CAAC;YACR,SAAS,EAAE,WAAW;gBACpB,CAAC,CAAC,EAAE,WAAW,EAAE,EAAE,KAAK,EAAE,WAAW,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE;gBACzD,CAAC,CAAC,SAAS;SACd,CAAC,CAAC;QAEH,qFAAqF;QACrF,MAAM,UAAU,GAAuB,CAAC,MAAM;YAC5C,oBAAoB,CAAuB,CAAC;QAC9C,0EAA0E;QAC1E,MAAM,cAAc,GAAG,UAAU,CAAC,UAAU,CAAC,KAAK,QAAQ,CAAC;QAC3D,MAAM,QAAQ,GAAG,OAAgB,CAAC;QAClC,MAAM,OAAO,GAAG,WAAW,CAAC,UAAU,CAAC,CAAC;QACxC,MAAM,YAAY,GAAuB,cAAc;YACrD,CAAC,CAAE,UAAiC;YACpC,CAAC,CAAE,OAAO,CAAC,MAAM,CAAC;gBACd,CAAC,QAAQ,CAAC,EAAE,UAAwB;aACrC,CAAwB,CAAC;QAE9B,MAAM,CAAC,iBAAiB,EAAE,aAAa,CAAC,GACtC,+BAA+B,CAAC,YAAY,CAAC,CAAC;QAEhD,mBAAmB,EAAE,CAAC;QACtB,MAAM,kBAAkB,GACtB,MAAM,UAAU,CAAqB;YACnC,WAAW;YACX,WAAW,EAAE,YAAY;YACzB,MAAM,EAAE,iBAAuC;YAC/C,SAAS;YACT,wBAAwB,EAAE,IAAI,CAAC,YAAY;YAC3C,MAAM,EAAE,QAAQ;YAChB,kBAAkB,EAAE,IAAI,CAAC,kBAAkB;YAC3C,UAAU,EAAE,gBAAgB;SAC7B,CAAC,CAAC;QAEL,MAAM,EACJ,QAAQ,EAAE,EAAE,SAAS,EAAE,EACvB,aAAa,EACb,iBAAiB,EACjB,gBAAgB,GAAG,CAAC,EACpB,mBAAmB,GAAG,CAAC,EACvB,iBAAiB,EACjB,GAAG,IAAI,EACR,GAAG,kBAAkB,CAAC;QACvB,IAAI,MAAM,GAAG,IAAgD,CAAC;QAE9D,4CAA4C;QAC5C,IAAI,CAAC,SAAS,EAAE,CACd,cAAc,CAAC,OAAO,EACtB,aAAa,EACb,iBAAiB,EACjB,gBAAgB,EAChB,mBAAmB,EACnB,iBAAiB,CAClB,CAAC;QAEF,2EAA2E;QAC3E,MAAM,OAAO,GAA8B,CAAC,cAAc,IAAI,EAAE,CAG/D,CAAC;QACF,KAAK,MAAM,EAAE,QAAQ,EAAE,IAAI,aAAa,EAAE,CAAC;YACzC,UAAU,CACR,MAAiC,EACjC,QAAQ,EACR,OAA4C,CAC7C,CAAC;QACJ,CAAC;QACD,sDAAsD;QACtD,IAAI,CAAC,cAAc,IAAI,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;YAC5D,MAAM,GAAI,MAAkC,CAAC,QAAQ,CAAC,CAAC;QACzD,CAAC;QAED,MAAM,mBAAmB,GAAG,GAAG,CAAC;QAChC,MAAM,YAAY,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,WAAW,CAAC;QAC3D,MAAM,aAAa,GACjB,YAAY,CAAC,MAAM,GAAG,mBAAmB;YACvC,CAAC,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,mBAAmB,CAAC,GAAG,KAAK;YACpD,CAAC,CAAC,YAAY,CAAC;QAEnB,QAAQ,CAAC;YACP,QAAQ,EAAE,YAAY;YACtB,OAAO,EAAE,SAAS;gBAChB,CAAC,CAAC,mCAAmC;gBACrC,CAAC,CAAC,iDAAiD;YACrD,KAAK,EAAE,CAAC;YACR,SAAS,EAAE;gBACT,aAAa,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,aAAa,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE;gBAC/D,iBAAiB,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,iBAAiB,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE;gBACvE,iBAAiB,EAAE;oBACjB,KAAK,EAAE,MAAM,CAAC,iBAAiB,CAAC;oBAChC,IAAI,EAAE,QAAQ;iBACf;gBACD,MAAM,EAAE,EAAE,KAAK,EAAE,aAAa,EAAE,IAAI,EAAE,QAAQ,EAAE;aACjD;SACF,CAAC,CAAC;QAEH,OAAO,MAAiC,CAAC;IAC3C,CAAC;CACF","sourcesContent":["// lib/v3/handlers/extractHandler.ts\nimport { extract as runExtract } from \"../../inference.js\";\nimport {\n getZFactory,\n getZodType,\n injectUrls,\n transformSchema,\n} from \"../../utils.js\";\nimport { v3Logger } from \"../logger.js\";\nimport { V3FunctionName } from \"../types/public/methods.js\";\nimport { captureHybridSnapshot } from \"../understudy/a11y/snapshot/index.js\";\nimport type { ZodTypeAny } from \"zod\";\nimport { LLMClient } from \"../llm/LLMClient.js\";\nimport { ExtractHandlerParams } from \"../types/private/handlers.js\";\nimport { EncodedId, ZodPathSegments } from \"../types/private/internal.js\";\nimport {\n defaultExtractSchema,\n pageTextSchema,\n} from \"../types/public/methods.js\";\nimport {\n AvailableModel,\n ClientOptions,\n ModelConfiguration,\n} from \"../types/public/model.js\";\nimport {\n StagehandInvalidArgumentError,\n ExtractTimeoutError,\n} from \"../types/public/sdkErrors.js\";\nimport { createTimeoutGuard } from \"./handlerUtils/timeoutGuard.js\";\nimport type {\n InferStagehandSchema,\n StagehandZodObject,\n StagehandZodSchema,\n} from \"../zodCompat.js\";\n\n/*\n Scans the provided Zod schema for any `z.string().url()` fields and\n * replaces them with `z.number()`.\n \n @param schema - The Zod object schema to transform.\n * @returns A tuple containing:\n * 1. The transformed schema (or the original schema if no changes were needed).\n * 2. An array of {@link ZodPathSegments} objects representing all the replaced URL fields,\n * with each path segment showing where in the schema the replacement occurred.\n */\nexport function transformUrlStringsToNumericIds<T extends StagehandZodSchema>(\n schema: T,\n): [StagehandZodSchema, ZodPathSegments[]] {\n const [finalSchema, urlPaths] = transformSchema(schema, []);\n return [finalSchema, urlPaths];\n}\n\ninterface ExtractionResponseBase {\n metadata: { completed: boolean };\n prompt_tokens: number;\n completion_tokens: number;\n reasoning_tokens: number;\n cached_input_tokens?: number;\n inference_time_ms: number;\n}\n\ntype ExtractionResponse<T extends StagehandZodObject> = ExtractionResponseBase &\n InferStagehandSchema<T>;\n\nexport class ExtractHandler {\n private readonly llmClient: LLMClient;\n private readonly defaultModelName: AvailableModel;\n private readonly defaultClientOptions: ClientOptions;\n private readonly resolveLlmClient: (model?: ModelConfiguration) => LLMClient;\n private readonly systemPrompt: string;\n private readonly logInferenceToFile: boolean;\n private readonly experimental: boolean;\n private readonly onMetrics?: (\n functionName: V3FunctionName,\n promptTokens: number,\n completionTokens: number,\n reasoningTokens: number,\n cachedInputTokens: number,\n inferenceTimeMs: number,\n ) => void;\n\n constructor(\n llmClient: LLMClient,\n defaultModelName: AvailableModel,\n defaultClientOptions: ClientOptions,\n resolveLlmClient: (model?: ModelConfiguration) => LLMClient,\n systemPrompt?: string,\n logInferenceToFile?: boolean,\n experimental?: boolean,\n onMetrics?: (\n functionName: V3FunctionName,\n promptTokens: number,\n completionTokens: number,\n reasoningTokens: number,\n cachedInputTokens: number,\n inferenceTimeMs: number,\n ) => void,\n ) {\n this.llmClient = llmClient;\n this.defaultModelName = defaultModelName;\n this.defaultClientOptions = defaultClientOptions;\n this.resolveLlmClient = resolveLlmClient;\n this.systemPrompt = systemPrompt ?? \"\";\n this.logInferenceToFile = logInferenceToFile ?? false;\n this.experimental = experimental ?? false;\n this.onMetrics = onMetrics;\n }\n\n async extract<T extends StagehandZodSchema>(\n params: ExtractHandlerParams<T>,\n ): Promise<InferStagehandSchema<T> \| { pageText: string }> {\n const {\n instruction,\n schema,\n page,\n selector,\n ignoreSelectors,\n timeout,\n model,\n screenshot,\n } = params;\n\n const llmClient = this.resolveLlmClient(model);\n\n const ensureTimeRemaining = createTimeoutGuard(\n timeout,\n (ms) => new ExtractTimeoutError(ms),\n );\n\n // No-args → page text (parity with v2)\n const noArgs = !instruction && !schema;\n if (noArgs) {\n const focusSelector = selector?.replace(/^xpath=/i, \"\") ?? \"\";\n ensureTimeRemaining();\n const snap = await captureHybridSnapshot(page, {\n experimental: this.experimental,\n focusSelector: focusSelector \|\| undefined,\n ignoreSelectors,\n });\n ensureTimeRemaining();\n\n const result = { pageText: snap.combinedTree };\n // Validate via the same schema used in v2\n return pageTextSchema.parse(result);\n }\n\n if (!instruction && schema) {\n throw new StagehandInvalidArgumentError(\n \"extract() requires an instruction when a schema is provided.\",\n );\n }\n\n if (screenshot && llmClient.type !== \"aisdk\") {\n throw new StagehandInvalidArgumentError(\n \"extract({ screenshot: true }) is only supported with AI SDK clients.\",\n );\n }\n\n const focusSelector = selector?.replace(/^xpath=/, \"\") ?? \"\";\n\n // Build the hybrid snapshot (includes combinedTree; combinedUrlMap optional)\n ensureTimeRemaining();\n const { combinedTree, combinedUrlMap } = await captureHybridSnapshot(page, {\n experimental: this.experimental,\n focusSelector: focusSelector,\n ignoreSelectors,\n });\n\n const screenshotBuffer = screenshot\n ? await (async () => {\n ensureTimeRemaining();\n const buffer = await page.screenshot({\n fullPage: false,\n type: \"png\",\n });\n ensureTimeRemaining();\n return buffer;\n })()\n : undefined;\n\n v3Logger({\n category: \"extraction\",\n message: screenshot\n ? \"Starting extraction using a11y snapshot and viewport screenshot\"\n : \"Starting extraction using a11y snapshot\",\n level: 1,\n auxiliary: instruction\n ? { instruction: { value: instruction, type: \"string\" } }\n : undefined,\n });\n\n // Normalize schema: if instruction provided without schema, use defaultExtractSchema\n const baseSchema: StagehandZodSchema = (schema ??\n defaultExtractSchema) as StagehandZodSchema;\n // Ensure we pass an object schema into inference; wrap non-object schemas\n const isObjectSchema = getZodType(baseSchema) === \"object\";\n const WRAP_KEY = \"value\" as const;\n const factory = getZFactory(baseSchema);\n const objectSchema: StagehandZodObject = isObjectSchema\n ? (baseSchema as StagehandZodObject)\n : (factory.object({\n [WRAP_KEY]: baseSchema as ZodTypeAny,\n }) as StagehandZodObject);\n\n const [transformedSchema, urlFieldPaths] =\n transformUrlStringsToNumericIds(objectSchema);\n\n ensureTimeRemaining();\n const extractionResponse: ExtractionResponse<StagehandZodObject> =\n await runExtract<StagehandZodObject>({\n instruction,\n domElements: combinedTree,\n schema: transformedSchema as StagehandZodObject,\n llmClient,\n userProvidedInstructions: this.systemPrompt,\n logger: v3Logger,\n logInferenceToFile: this.logInferenceToFile,\n screenshot: screenshotBuffer,\n });\n\n const {\n metadata: { completed },\n prompt_tokens,\n completion_tokens,\n reasoning_tokens = 0,\n cached_input_tokens = 0,\n inference_time_ms,\n ...rest\n } = extractionResponse;\n let output = rest as InferStagehandSchema<StagehandZodObject>;\n\n // Update EXTRACT metrics from the LLM calls\n this.onMetrics?.(\n V3FunctionName.EXTRACT,\n prompt_tokens,\n completion_tokens,\n reasoning_tokens,\n cached_input_tokens,\n inference_time_ms,\n );\n\n // Re-inject URLs for any url() fields we temporarily converted to number()\n const idToUrl: Record<EncodedId, string> = (combinedUrlMap ?? {}) as Record<\n EncodedId,\n string\n >;\n for (const { segments } of urlFieldPaths) {\n injectUrls(\n output as Record<string, unknown>,\n segments,\n idToUrl as unknown as Record<string, string>,\n );\n }\n // If we wrapped a non-object schema, unwrap the value\n if (!isObjectSchema && output && typeof output === \"object\") {\n output = (output as Record<string, unknown>)[WRAP_KEY];\n }\n\n const resultPreviewLength = 200;\n const resultString = JSON.stringify(output) ?? \"undefined\";\n const resultPreview =\n resultString.length > resultPreviewLength\n ? resultString.slice(0, resultPreviewLength) + \"...\"\n : resultString;\n\n v3Logger({\n category: \"extraction\",\n message: completed\n ? \"Extraction completed successfully\"\n : \"Extraction incomplete after processing all data\",\n level: 1,\n auxiliary: {\n prompt_tokens: { value: String(prompt_tokens), type: \"string\" },\n completion_tokens: { value: String(completion_tokens), type: \"string\" },\n inference_time_ms: {\n value: String(inference_time_ms),\n type: \"string\",\n },\n result: { value: resultPreview, type: \"string\" },\n },\n });\n\n return output as InferStagehandSchema<T>;\n }\n}\n"]}

package/dist/esm/lib/v3/handlers/observeHandler.js CHANGED Viewed

@@ -27,7 +27,7 @@ export class ObserveHandler {
         this.onMetrics = onMetrics;
     }
     async observe(params) {
-        const { instruction, page, timeout, selector, model, variables } = params;
+        const { instruction, page, timeout, selector, ignoreSelectors, model, variables, } = params;
         const llmClient = this.resolveLlmClient(model);
         const ensureTimeRemaining = createTimeoutGuard(timeout, (ms) => new ObserveTimeoutError(ms));
         const effectiveInstruction = instruction ??
@@ -49,6 +49,7 @@ export class ObserveHandler {
         const snapshot = await captureHybridSnapshot(page, {
             experimental: this.experimental,
             focusSelector: focusSelector || undefined,
+            ignoreSelectors,
         });
         const combinedTree = snapshot.combinedTree;
         const combinedXpathMap = snapshot.combinedXpathMap ?? {};

package/dist/esm/lib/v3/handlers/observeHandler.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"observeHandler.js","sourceRoot":"","sources":["../../../../../lib/v3/handlers/observeHandler.ts"],"names":[],"mappings":"AAAA,oCAAoC;AACpC,OAAO,EAAE,OAAO,IAAI,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAC3D,OAAO,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAC;AACtD,OAAO,EAAE,QAAQ,EAAE,MAAM,cAAc,CAAC;AACxC,OAAO,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AAC5D,OAAO,EAAE,qBAAqB,EAAE,MAAM,sCAAsC,CAAC;AAE7E,OAAO,EAEL,yBAAyB,GAC1B,MAAM,8BAA8B,CAAC;AAQtC,OAAO,EAAE,mBAAmB,EAAE,MAAM,8BAA8B,CAAC;AACnE,OAAO,EAAE,kBAAkB,EAAE,MAAM,gCAAgC,CAAC;AAEpE,MAAM,OAAO,cAAc;IACR,SAAS,CAAY;IACrB,gBAAgB,CAAiB;IACjC,oBAAoB,CAAgB;IACpC,gBAAgB,CAA4C;IAC5D,YAAY,CAAS;IACrB,kBAAkB,CAAU;IAC5B,YAAY,CAAU;IACtB,SAAS,CAOhB;IAEV,YACE,SAAoB,EACpB,gBAAgC,EAChC,oBAAmC,EACnC,gBAA2D,EAC3D,YAAqB,EACrB,kBAA4B,EAC5B,YAAsB,EACtB,SAOS;QAET,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,gBAAgB,GAAG,gBAAgB,CAAC;QACzC,IAAI,CAAC,oBAAoB,GAAG,oBAAoB,CAAC;QACjD,IAAI,CAAC,gBAAgB,GAAG,gBAAgB,CAAC;QACzC,IAAI,CAAC,YAAY,GAAG,YAAY,IAAI,EAAE,CAAC;QACvC,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,IAAI,KAAK,CAAC;QACtD,IAAI,CAAC,YAAY,GAAG,YAAY,IAAI,KAAK,CAAC;QAC1C,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;IAC7B,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,MAA4B;QACxC,MAAM,EAAE,WAAW,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,MAAM,CAAC;QAE1E,MAAM,SAAS,GAAG,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC;QAE/C,MAAM,mBAAmB,GAAG,kBAAkB,CAC5C,OAAO,EACP,CAAC,EAAE,EAAE,EAAE,CAAC,IAAI,mBAAmB,CAAC,EAAE,CAAC,CACpC,CAAC;QAEF,MAAM,oBAAoB,GACxB,WAAW;YACX,kSAAkS,CAAC;QAErS,QAAQ,CAAC;YACP,QAAQ,EAAE,aAAa;YACvB,OAAO,EAAE,sBAAsB;YAC/B,KAAK,EAAE,CAAC;YACR,SAAS,EAAE;gBACT,WAAW,EAAE;oBACX,KAAK,EAAE,oBAAoB;oBAC3B,IAAI,EAAE,QAAQ;iBACf;aACF;SACF,CAAC,CAAC;QAEH,mEAAmE;QACnE,MAAM,aAAa,GAAG,QAAQ,EAAE,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,IAAI,EAAE,CAAC;QAC9D,mBAAmB,EAAE,CAAC;QACtB,MAAM,QAAQ,GAAG,MAAM,qBAAqB,CAAC,IAAI,EAAE;YACjD,YAAY,EAAE,IAAI,CAAC,YAAY;YAC/B,aAAa,EAAE,aAAa,IAAI,SAAS;SAC1C,CAAC,CAAC;QAEH,MAAM,YAAY,GAAG,QAAQ,CAAC,YAAY,CAAC;QAC3C,MAAM,gBAAgB,GAAG,QAAQ,CAAC,gBAAgB,IAAI,EAAE,CAAC;QAEzD,QAAQ,CAAC;YACP,QAAQ,EAAE,aAAa;YACvB,OAAO,EAAE,6BAA6B;YACtC,KAAK,EAAE,CAAC;SACT,CAAC,CAAC;QAEH,8CAA8C;QAC9C,mBAAmB,EAAE,CAAC;QACtB,MAAM,mBAAmB,GAAG,MAAM,UAAU,CAAC;YAC3C,WAAW,EAAE,oBAAoB;YACjC,WAAW,EAAE,YAAY;YACzB,SAAS;YACT,wBAAwB,EAAE,IAAI,CAAC,YAAY;YAC3C,MAAM,EAAE,QAAQ;YAChB,kBAAkB,EAAE,IAAI,CAAC,kBAAkB;YAC3C,gBAAgB,EAAE,MAAM,CAAC,MAAM,CAAC,yBAAyB,CAAC;YAC1D,SAAS;SACV,CAAC,CAAC;QAEH,MAAM,EACJ,aAAa,GAAG,CAAC,EACjB,iBAAiB,GAAG,CAAC,EACrB,gBAAgB,GAAG,CAAC,EACpB,mBAAmB,GAAG,CAAC,EACvB,iBAAiB,GAAG,CAAC,GACtB,GAAG,mBAAmB,CAAC;QAExB,uDAAuD;QACvD,IAAI,CAAC,SAAS,EAAE,CACd,cAAc,CAAC,OAAO,EACtB,aAAa,EACb,iBAAiB,EACjB,gBAAgB,EAChB,mBAAmB,EACnB,iBAAiB,CAClB,CAAC;QAEF,mDAAmD;QACnD,MAAM,qBAAqB,GAAG,CAC5B,MAAM,OAAO,CAAC,GAAG,CACf,mBAAmB,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE;YACjD,MAAM,EAAE,SAAS,EAAE,GAAG,IAAI,EAAE,GAAG,OAAO,CAAC,CAAC,4CAA4C;YACpF,IAAI,OAAO,SAAS,KAAK,QAAQ,IAAI,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC7D,MAAM,WAAW,GAAG,SAAsB,CAAC;gBAC3C,MAAM,KAAK,GAAG,gBAAgB,CAAC,WAAW,CAAC,CAAC;gBAC5C,MAAM,YAAY,GAAG,oBAAoB,CAAC,KAAK,CAAC,CAAC;gBACjD,IAAI,CAAC,YAAY;oBAAE,OAAO,SAAS,CAAC;gBAEpC,6EAA6E;gBAC7E,IAAI,YAAY,GAAG,IAAI,CAAC,SAAS,CAAC;gBAClC,IACE,IAAI,CAAC,MAAM,KAAK,aAAa;oBAC7B,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC;oBAC7B,IAAI,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,EACzB,CAAC;oBACD,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;oBACpC,4DAA4D;oBAC5D,IACE,OAAO,SAAS,KAAK,QAAQ;wBAC7B,WAAW,CAAC,IAAI,CAAC,SAAS,CAAC,EAC3B,CAAC;wBACD,MAAM,QAAQ,GAAG,gBAAgB,CAAC,SAAsB,CAAC,CAAC;wBAC1D,MAAM,eAAe,GAAG,oBAAoB,CAAC,QAAQ,CAAC,CAAC;wBACvD,IAAI,eAAe,EAAE,CAAC;4BACpB,YAAY,GAAG;gCACb,SAAS,eAAe,EAAE;gCAC1B,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC;6BAC3B,CAAC;wBACJ,CAAC;6BAAM,CAAC;4BACN,uDAAuD;4BACvD,QAAQ,CAAC;gCACP,QAAQ,EAAE,aAAa;gCACvB,OAAO,EAAE,0CAA0C;gCACnD,KAAK,EAAE,CAAC;gCACR,SAAS,EAAE;oCACT,eAAe,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,QAAQ,EAAE;oCACrD,eAAe,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,QAAQ,EAAE;iCACtD;6BACF,CAAC,CAAC;4BACH,OAAO,SAAS,CAAC;wBACnB,CAAC;oBACH,CAAC;yBAAM,CAAC;wBACN,QAAQ,CAAC;4BACP,QAAQ,EAAE,aAAa;4BACvB,OAAO,EAAE,8CAA8C;4BACvD,KAAK,EAAE,CAAC;4BACR,SAAS,EAAE;gCACT,eAAe,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,QAAQ,EAAE;gCACrD,eAAe,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,QAAQ,EAAE;6BACtD;yBACF,CAAC,CAAC;wBACH,OAAO,SAAS,CAAC;oBACnB,CAAC;gBACH,CAAC;gBAED,OAAO;oBACL,GAAG,IAAI;oBACP,SAAS,EAAE,YAAY;oBACvB,QAAQ,EAAE,SAAS,YAAY,EAAE;iBAMlC,CAAC;YACJ,CAAC;YACD,wBAAwB;YACxB,OAAO;gBACL,WAAW,EAAE,gCAAgC;gBAC7C,MAAM,EAAE,eAAe;gBACvB,SAAS,EAAE,EAAE;gBACb,QAAQ,EAAE,eAAe;aAC1B,CAAC;QACJ,CAAC,CAAC,CACH,CACF,CAAC,MAAM,CAAC,CAAI,CAAgB,EAAU,EAAE,CAAC,CAAC,KAAK,SAAS,CAAC,CAAC;QAE3D,QAAQ,CAAC;YACP,QAAQ,EAAE,aAAa;YACvB,OAAO,EAAE,gBAAgB;YACzB,KAAK,EAAE,CAAC;YACR,SAAS,EAAE;gBACT,QAAQ,EAAE;oBACR,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,qBAAqB,CAAC;oBAC5C,IAAI,EAAE,QAAQ;iBACf;aACF;SACF,CAAC,CAAC;QAEH,OAAO,qBAAqB,CAAC;IAC/B,CAAC;CACF","sourcesContent":["// lib/v3/handlers/observeHandler.ts\nimport { observe as runObserve } from \"../../inference.js\";\nimport { trimTrailingTextNode } from \"../../utils.js\";\nimport { v3Logger } from \"../logger.js\";\nimport { V3FunctionName } from \"../types/public/methods.js\";\nimport { captureHybridSnapshot } from \"../understudy/a11y/snapshot/index.js\";\nimport { LLMClient } from \"../llm/LLMClient.js\";\nimport {\n ObserveHandlerParams,\n SupportedUnderstudyAction,\n} from \"../types/private/handlers.js\";\nimport { EncodedId } from \"../types/private/internal.js\";\nimport { Action } from \"../types/public/methods.js\";\nimport {\n AvailableModel,\n ClientOptions,\n ModelConfiguration,\n} from \"../types/public/model.js\";\nimport { ObserveTimeoutError } from \"../types/public/sdkErrors.js\";\nimport { createTimeoutGuard } from \"./handlerUtils/timeoutGuard.js\";\n\nexport class ObserveHandler {\n private readonly llmClient: LLMClient;\n private readonly defaultModelName: AvailableModel;\n private readonly defaultClientOptions: ClientOptions;\n private readonly resolveLlmClient: (model?: ModelConfiguration) => LLMClient;\n private readonly systemPrompt: string;\n private readonly logInferenceToFile: boolean;\n private readonly experimental: boolean;\n private readonly onMetrics?: (\n functionName: V3FunctionName,\n promptTokens: number,\n completionTokens: number,\n reasoningTokens: number,\n cachedInputTokens: number,\n inferenceTimeMs: number,\n ) => void;\n\n constructor(\n llmClient: LLMClient,\n defaultModelName: AvailableModel,\n defaultClientOptions: ClientOptions,\n resolveLlmClient: (model?: ModelConfiguration) => LLMClient,\n systemPrompt?: string,\n logInferenceToFile?: boolean,\n experimental?: boolean,\n onMetrics?: (\n functionName: V3FunctionName,\n promptTokens: number,\n completionTokens: number,\n reasoningTokens: number,\n cachedInputTokens: number,\n inferenceTimeMs: number,\n ) => void,\n ) {\n this.llmClient = llmClient;\n this.defaultModelName = defaultModelName;\n this.defaultClientOptions = defaultClientOptions;\n this.resolveLlmClient = resolveLlmClient;\n this.systemPrompt = systemPrompt ?? \"\";\n this.logInferenceToFile = logInferenceToFile ?? false;\n this.experimental = experimental ?? false;\n this.onMetrics = onMetrics;\n }\n\n async observe(params: ObserveHandlerParams): Promise<Action[]> {\n const { instruction, page, timeout, selector, model, variables } = params;\n\n const llmClient = this.resolveLlmClient(model);\n\n const ensureTimeRemaining = createTimeoutGuard(\n timeout,\n (ms) => new ObserveTimeoutError(ms),\n );\n\n const effectiveInstruction =\n instruction ??\n \"Find elements that can be used for any future actions in the page. These may be navigation links, related pages, section/subsection links, buttons, or other interactive elements. Be comprehensive: if there are multiple elements that may be relevant for future actions, return all of them.\";\n\n v3Logger({\n category: \"observation\",\n message: \"starting observation\",\n level: 1,\n auxiliary: {\n instruction: {\n value: effectiveInstruction,\n type: \"string\",\n },\n },\n });\n\n // Build the hybrid snapshot (a11y-centric text tree + lookup maps)\n const focusSelector = selector?.replace(/^xpath=/i, \"\") ?? \"\";\n ensureTimeRemaining();\n const snapshot = await captureHybridSnapshot(page, {\n experimental: this.experimental,\n focusSelector: focusSelector \|\| undefined,\n });\n\n const combinedTree = snapshot.combinedTree;\n const combinedXpathMap = snapshot.combinedXpathMap ?? {};\n\n v3Logger({\n category: \"observation\",\n message: \"Got accessibility tree data\",\n level: 1,\n });\n\n // Call the LLM to propose actionable elements\n ensureTimeRemaining();\n const observationResponse = await runObserve({\n instruction: effectiveInstruction,\n domElements: combinedTree,\n llmClient,\n userProvidedInstructions: this.systemPrompt,\n logger: v3Logger,\n logInferenceToFile: this.logInferenceToFile,\n supportedActions: Object.values(SupportedUnderstudyAction),\n variables,\n });\n\n const {\n prompt_tokens = 0,\n completion_tokens = 0,\n reasoning_tokens = 0,\n cached_input_tokens = 0,\n inference_time_ms = 0,\n } = observationResponse;\n\n // Update OBSERVE metrics from the LLM observation call\n this.onMetrics?.(\n V3FunctionName.OBSERVE,\n prompt_tokens,\n completion_tokens,\n reasoning_tokens,\n cached_input_tokens,\n inference_time_ms,\n );\n\n // Map elementIds -> selectors via combinedXpathMap\n const elementsWithSelectors = (\n await Promise.all(\n observationResponse.elements.map(async (element) => {\n const { elementId, ...rest } = element; // rest may or may not have method/arguments\n if (typeof elementId === \"string\" && elementId.includes(\"-\")) {\n const lookUpIndex = elementId as EncodedId;\n const xpath = combinedXpathMap[lookUpIndex];\n const trimmedXpath = trimTrailingTextNode(xpath);\n if (!trimmedXpath) return undefined;\n\n // For dragAndDrop, convert element ID in arguments to xpath (target element)\n let resolvedArgs = rest.arguments;\n if (\n rest.method === \"dragAndDrop\" &&\n Array.isArray(rest.arguments) &&\n rest.arguments.length > 0\n ) {\n const targetArg = rest.arguments[0];\n // Check if argument looks like an element ID (e.g., \"1-67\")\n if (\n typeof targetArg === \"string\" &&\n /^\\d+-\\d+$/.test(targetArg)\n ) {\n const argXpath = combinedXpathMap[targetArg as EncodedId];\n const trimmedArgXpath = trimTrailingTextNode(argXpath);\n if (trimmedArgXpath) {\n resolvedArgs = [\n `xpath=${trimmedArgXpath}`,\n ...rest.arguments.slice(1),\n ];\n } else {\n // Target element lookup failed, filter out this action\n v3Logger({\n category: \"observation\",\n message: \"dragAndDrop target element lookup failed\",\n level: 0,\n auxiliary: {\n targetElementId: { value: targetArg, type: \"string\" },\n sourceElementId: { value: elementId, type: \"string\" },\n },\n });\n return undefined;\n }\n } else {\n v3Logger({\n category: \"observation\",\n message: \"dragAndDrop target element invalid ID format\",\n level: 0,\n auxiliary: {\n targetElementId: { value: targetArg, type: \"string\" },\n sourceElementId: { value: elementId, type: \"string\" },\n },\n });\n return undefined;\n }\n }\n\n return {\n ...rest,\n arguments: resolvedArgs,\n selector: `xpath=${trimmedXpath}`,\n } as {\n description: string;\n method?: string;\n arguments?: string[];\n selector: string;\n };\n }\n // shadow-root fallback:\n return {\n description: \"an element inside a shadow DOM\",\n method: \"not-supported\",\n arguments: [],\n selector: \"not-supported\",\n };\n }),\n )\n ).filter(<T>(e: T \| undefined): e is T => e !== undefined);\n\n v3Logger({\n category: \"observation\",\n message: \"found elements\",\n level: 1,\n auxiliary: {\n elements: {\n value: JSON.stringify(elementsWithSelectors),\n type: \"object\",\n },\n },\n });\n\n return elementsWithSelectors;\n }\n}\n"]}
1	+ {"version":3,"file":"observeHandler.js","sourceRoot":"","sources":["../../../../../lib/v3/handlers/observeHandler.ts"],"names":[],"mappings":"AAAA,oCAAoC;AACpC,OAAO,EAAE,OAAO,IAAI,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAC3D,OAAO,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAC;AACtD,OAAO,EAAE,QAAQ,EAAE,MAAM,cAAc,CAAC;AACxC,OAAO,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AAC5D,OAAO,EAAE,qBAAqB,EAAE,MAAM,sCAAsC,CAAC;AAE7E,OAAO,EAEL,yBAAyB,GAC1B,MAAM,8BAA8B,CAAC;AAQtC,OAAO,EAAE,mBAAmB,EAAE,MAAM,8BAA8B,CAAC;AACnE,OAAO,EAAE,kBAAkB,EAAE,MAAM,gCAAgC,CAAC;AAEpE,MAAM,OAAO,cAAc;IACR,SAAS,CAAY;IACrB,gBAAgB,CAAiB;IACjC,oBAAoB,CAAgB;IACpC,gBAAgB,CAA4C;IAC5D,YAAY,CAAS;IACrB,kBAAkB,CAAU;IAC5B,YAAY,CAAU;IACtB,SAAS,CAOhB;IAEV,YACE,SAAoB,EACpB,gBAAgC,EAChC,oBAAmC,EACnC,gBAA2D,EAC3D,YAAqB,EACrB,kBAA4B,EAC5B,YAAsB,EACtB,SAOS;QAET,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,gBAAgB,GAAG,gBAAgB,CAAC;QACzC,IAAI,CAAC,oBAAoB,GAAG,oBAAoB,CAAC;QACjD,IAAI,CAAC,gBAAgB,GAAG,gBAAgB,CAAC;QACzC,IAAI,CAAC,YAAY,GAAG,YAAY,IAAI,EAAE,CAAC;QACvC,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,IAAI,KAAK,CAAC;QACtD,IAAI,CAAC,YAAY,GAAG,YAAY,IAAI,KAAK,CAAC;QAC1C,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;IAC7B,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,MAA4B;QACxC,MAAM,EACJ,WAAW,EACX,IAAI,EACJ,OAAO,EACP,QAAQ,EACR,eAAe,EACf,KAAK,EACL,SAAS,GACV,GAAG,MAAM,CAAC;QAEX,MAAM,SAAS,GAAG,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC;QAE/C,MAAM,mBAAmB,GAAG,kBAAkB,CAC5C,OAAO,EACP,CAAC,EAAE,EAAE,EAAE,CAAC,IAAI,mBAAmB,CAAC,EAAE,CAAC,CACpC,CAAC;QAEF,MAAM,oBAAoB,GACxB,WAAW;YACX,kSAAkS,CAAC;QAErS,QAAQ,CAAC;YACP,QAAQ,EAAE,aAAa;YACvB,OAAO,EAAE,sBAAsB;YAC/B,KAAK,EAAE,CAAC;YACR,SAAS,EAAE;gBACT,WAAW,EAAE;oBACX,KAAK,EAAE,oBAAoB;oBAC3B,IAAI,EAAE,QAAQ;iBACf;aACF;SACF,CAAC,CAAC;QAEH,mEAAmE;QACnE,MAAM,aAAa,GAAG,QAAQ,EAAE,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,IAAI,EAAE,CAAC;QAC9D,mBAAmB,EAAE,CAAC;QACtB,MAAM,QAAQ,GAAG,MAAM,qBAAqB,CAAC,IAAI,EAAE;YACjD,YAAY,EAAE,IAAI,CAAC,YAAY;YAC/B,aAAa,EAAE,aAAa,IAAI,SAAS;YACzC,eAAe;SAChB,CAAC,CAAC;QAEH,MAAM,YAAY,GAAG,QAAQ,CAAC,YAAY,CAAC;QAC3C,MAAM,gBAAgB,GAAG,QAAQ,CAAC,gBAAgB,IAAI,EAAE,CAAC;QAEzD,QAAQ,CAAC;YACP,QAAQ,EAAE,aAAa;YACvB,OAAO,EAAE,6BAA6B;YACtC,KAAK,EAAE,CAAC;SACT,CAAC,CAAC;QAEH,8CAA8C;QAC9C,mBAAmB,EAAE,CAAC;QACtB,MAAM,mBAAmB,GAAG,MAAM,UAAU,CAAC;YAC3C,WAAW,EAAE,oBAAoB;YACjC,WAAW,EAAE,YAAY;YACzB,SAAS;YACT,wBAAwB,EAAE,IAAI,CAAC,YAAY;YAC3C,MAAM,EAAE,QAAQ;YAChB,kBAAkB,EAAE,IAAI,CAAC,kBAAkB;YAC3C,gBAAgB,EAAE,MAAM,CAAC,MAAM,CAAC,yBAAyB,CAAC;YAC1D,SAAS;SACV,CAAC,CAAC;QAEH,MAAM,EACJ,aAAa,GAAG,CAAC,EACjB,iBAAiB,GAAG,CAAC,EACrB,gBAAgB,GAAG,CAAC,EACpB,mBAAmB,GAAG,CAAC,EACvB,iBAAiB,GAAG,CAAC,GACtB,GAAG,mBAAmB,CAAC;QAExB,uDAAuD;QACvD,IAAI,CAAC,SAAS,EAAE,CACd,cAAc,CAAC,OAAO,EACtB,aAAa,EACb,iBAAiB,EACjB,gBAAgB,EAChB,mBAAmB,EACnB,iBAAiB,CAClB,CAAC;QAEF,mDAAmD;QACnD,MAAM,qBAAqB,GAAG,CAC5B,MAAM,OAAO,CAAC,GAAG,CACf,mBAAmB,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE;YACjD,MAAM,EAAE,SAAS,EAAE,GAAG,IAAI,EAAE,GAAG,OAAO,CAAC,CAAC,4CAA4C;YACpF,IAAI,OAAO,SAAS,KAAK,QAAQ,IAAI,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC7D,MAAM,WAAW,GAAG,SAAsB,CAAC;gBAC3C,MAAM,KAAK,GAAG,gBAAgB,CAAC,WAAW,CAAC,CAAC;gBAC5C,MAAM,YAAY,GAAG,oBAAoB,CAAC,KAAK,CAAC,CAAC;gBACjD,IAAI,CAAC,YAAY;oBAAE,OAAO,SAAS,CAAC;gBAEpC,6EAA6E;gBAC7E,IAAI,YAAY,GAAG,IAAI,CAAC,SAAS,CAAC;gBAClC,IACE,IAAI,CAAC,MAAM,KAAK,aAAa;oBAC7B,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC;oBAC7B,IAAI,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,EACzB,CAAC;oBACD,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;oBACpC,4DAA4D;oBAC5D,IACE,OAAO,SAAS,KAAK,QAAQ;wBAC7B,WAAW,CAAC,IAAI,CAAC,SAAS,CAAC,EAC3B,CAAC;wBACD,MAAM,QAAQ,GAAG,gBAAgB,CAAC,SAAsB,CAAC,CAAC;wBAC1D,MAAM,eAAe,GAAG,oBAAoB,CAAC,QAAQ,CAAC,CAAC;wBACvD,IAAI,eAAe,EAAE,CAAC;4BACpB,YAAY,GAAG;gCACb,SAAS,eAAe,EAAE;gCAC1B,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC;6BAC3B,CAAC;wBACJ,CAAC;6BAAM,CAAC;4BACN,uDAAuD;4BACvD,QAAQ,CAAC;gCACP,QAAQ,EAAE,aAAa;gCACvB,OAAO,EAAE,0CAA0C;gCACnD,KAAK,EAAE,CAAC;gCACR,SAAS,EAAE;oCACT,eAAe,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,QAAQ,EAAE;oCACrD,eAAe,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,QAAQ,EAAE;iCACtD;6BACF,CAAC,CAAC;4BACH,OAAO,SAAS,CAAC;wBACnB,CAAC;oBACH,CAAC;yBAAM,CAAC;wBACN,QAAQ,CAAC;4BACP,QAAQ,EAAE,aAAa;4BACvB,OAAO,EAAE,8CAA8C;4BACvD,KAAK,EAAE,CAAC;4BACR,SAAS,EAAE;gCACT,eAAe,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,QAAQ,EAAE;gCACrD,eAAe,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,QAAQ,EAAE;6BACtD;yBACF,CAAC,CAAC;wBACH,OAAO,SAAS,CAAC;oBACnB,CAAC;gBACH,CAAC;gBAED,OAAO;oBACL,GAAG,IAAI;oBACP,SAAS,EAAE,YAAY;oBACvB,QAAQ,EAAE,SAAS,YAAY,EAAE;iBAMlC,CAAC;YACJ,CAAC;YACD,wBAAwB;YACxB,OAAO;gBACL,WAAW,EAAE,gCAAgC;gBAC7C,MAAM,EAAE,eAAe;gBACvB,SAAS,EAAE,EAAE;gBACb,QAAQ,EAAE,eAAe;aAC1B,CAAC;QACJ,CAAC,CAAC,CACH,CACF,CAAC,MAAM,CAAC,CAAI,CAAgB,EAAU,EAAE,CAAC,CAAC,KAAK,SAAS,CAAC,CAAC;QAE3D,QAAQ,CAAC;YACP,QAAQ,EAAE,aAAa;YACvB,OAAO,EAAE,gBAAgB;YACzB,KAAK,EAAE,CAAC;YACR,SAAS,EAAE;gBACT,QAAQ,EAAE;oBACR,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,qBAAqB,CAAC;oBAC5C,IAAI,EAAE,QAAQ;iBACf;aACF;SACF,CAAC,CAAC;QAEH,OAAO,qBAAqB,CAAC;IAC/B,CAAC;CACF","sourcesContent":["// lib/v3/handlers/observeHandler.ts\nimport { observe as runObserve } from \"../../inference.js\";\nimport { trimTrailingTextNode } from \"../../utils.js\";\nimport { v3Logger } from \"../logger.js\";\nimport { V3FunctionName } from \"../types/public/methods.js\";\nimport { captureHybridSnapshot } from \"../understudy/a11y/snapshot/index.js\";\nimport { LLMClient } from \"../llm/LLMClient.js\";\nimport {\n ObserveHandlerParams,\n SupportedUnderstudyAction,\n} from \"../types/private/handlers.js\";\nimport { EncodedId } from \"../types/private/internal.js\";\nimport { Action } from \"../types/public/methods.js\";\nimport {\n AvailableModel,\n ClientOptions,\n ModelConfiguration,\n} from \"../types/public/model.js\";\nimport { ObserveTimeoutError } from \"../types/public/sdkErrors.js\";\nimport { createTimeoutGuard } from \"./handlerUtils/timeoutGuard.js\";\n\nexport class ObserveHandler {\n private readonly llmClient: LLMClient;\n private readonly defaultModelName: AvailableModel;\n private readonly defaultClientOptions: ClientOptions;\n private readonly resolveLlmClient: (model?: ModelConfiguration) => LLMClient;\n private readonly systemPrompt: string;\n private readonly logInferenceToFile: boolean;\n private readonly experimental: boolean;\n private readonly onMetrics?: (\n functionName: V3FunctionName,\n promptTokens: number,\n completionTokens: number,\n reasoningTokens: number,\n cachedInputTokens: number,\n inferenceTimeMs: number,\n ) => void;\n\n constructor(\n llmClient: LLMClient,\n defaultModelName: AvailableModel,\n defaultClientOptions: ClientOptions,\n resolveLlmClient: (model?: ModelConfiguration) => LLMClient,\n systemPrompt?: string,\n logInferenceToFile?: boolean,\n experimental?: boolean,\n onMetrics?: (\n functionName: V3FunctionName,\n promptTokens: number,\n completionTokens: number,\n reasoningTokens: number,\n cachedInputTokens: number,\n inferenceTimeMs: number,\n ) => void,\n ) {\n this.llmClient = llmClient;\n this.defaultModelName = defaultModelName;\n this.defaultClientOptions = defaultClientOptions;\n this.resolveLlmClient = resolveLlmClient;\n this.systemPrompt = systemPrompt ?? \"\";\n this.logInferenceToFile = logInferenceToFile ?? false;\n this.experimental = experimental ?? false;\n this.onMetrics = onMetrics;\n }\n\n async observe(params: ObserveHandlerParams): Promise<Action[]> {\n const {\n instruction,\n page,\n timeout,\n selector,\n ignoreSelectors,\n model,\n variables,\n } = params;\n\n const llmClient = this.resolveLlmClient(model);\n\n const ensureTimeRemaining = createTimeoutGuard(\n timeout,\n (ms) => new ObserveTimeoutError(ms),\n );\n\n const effectiveInstruction =\n instruction ??\n \"Find elements that can be used for any future actions in the page. These may be navigation links, related pages, section/subsection links, buttons, or other interactive elements. Be comprehensive: if there are multiple elements that may be relevant for future actions, return all of them.\";\n\n v3Logger({\n category: \"observation\",\n message: \"starting observation\",\n level: 1,\n auxiliary: {\n instruction: {\n value: effectiveInstruction,\n type: \"string\",\n },\n },\n });\n\n // Build the hybrid snapshot (a11y-centric text tree + lookup maps)\n const focusSelector = selector?.replace(/^xpath=/i, \"\") ?? \"\";\n ensureTimeRemaining();\n const snapshot = await captureHybridSnapshot(page, {\n experimental: this.experimental,\n focusSelector: focusSelector \|\| undefined,\n ignoreSelectors,\n });\n\n const combinedTree = snapshot.combinedTree;\n const combinedXpathMap = snapshot.combinedXpathMap ?? {};\n\n v3Logger({\n category: \"observation\",\n message: \"Got accessibility tree data\",\n level: 1,\n });\n\n // Call the LLM to propose actionable elements\n ensureTimeRemaining();\n const observationResponse = await runObserve({\n instruction: effectiveInstruction,\n domElements: combinedTree,\n llmClient,\n userProvidedInstructions: this.systemPrompt,\n logger: v3Logger,\n logInferenceToFile: this.logInferenceToFile,\n supportedActions: Object.values(SupportedUnderstudyAction),\n variables,\n });\n\n const {\n prompt_tokens = 0,\n completion_tokens = 0,\n reasoning_tokens = 0,\n cached_input_tokens = 0,\n inference_time_ms = 0,\n } = observationResponse;\n\n // Update OBSERVE metrics from the LLM observation call\n this.onMetrics?.(\n V3FunctionName.OBSERVE,\n prompt_tokens,\n completion_tokens,\n reasoning_tokens,\n cached_input_tokens,\n inference_time_ms,\n );\n\n // Map elementIds -> selectors via combinedXpathMap\n const elementsWithSelectors = (\n await Promise.all(\n observationResponse.elements.map(async (element) => {\n const { elementId, ...rest } = element; // rest may or may not have method/arguments\n if (typeof elementId === \"string\" && elementId.includes(\"-\")) {\n const lookUpIndex = elementId as EncodedId;\n const xpath = combinedXpathMap[lookUpIndex];\n const trimmedXpath = trimTrailingTextNode(xpath);\n if (!trimmedXpath) return undefined;\n\n // For dragAndDrop, convert element ID in arguments to xpath (target element)\n let resolvedArgs = rest.arguments;\n if (\n rest.method === \"dragAndDrop\" &&\n Array.isArray(rest.arguments) &&\n rest.arguments.length > 0\n ) {\n const targetArg = rest.arguments[0];\n // Check if argument looks like an element ID (e.g., \"1-67\")\n if (\n typeof targetArg === \"string\" &&\n /^\\d+-\\d+$/.test(targetArg)\n ) {\n const argXpath = combinedXpathMap[targetArg as EncodedId];\n const trimmedArgXpath = trimTrailingTextNode(argXpath);\n if (trimmedArgXpath) {\n resolvedArgs = [\n `xpath=${trimmedArgXpath}`,\n ...rest.arguments.slice(1),\n ];\n } else {\n // Target element lookup failed, filter out this action\n v3Logger({\n category: \"observation\",\n message: \"dragAndDrop target element lookup failed\",\n level: 0,\n auxiliary: {\n targetElementId: { value: targetArg, type: \"string\" },\n sourceElementId: { value: elementId, type: \"string\" },\n },\n });\n return undefined;\n }\n } else {\n v3Logger({\n category: \"observation\",\n message: \"dragAndDrop target element invalid ID format\",\n level: 0,\n auxiliary: {\n targetElementId: { value: targetArg, type: \"string\" },\n sourceElementId: { value: elementId, type: \"string\" },\n },\n });\n return undefined;\n }\n }\n\n return {\n ...rest,\n arguments: resolvedArgs,\n selector: `xpath=${trimmedXpath}`,\n } as {\n description: string;\n method?: string;\n arguments?: string[];\n selector: string;\n };\n }\n // shadow-root fallback:\n return {\n description: \"an element inside a shadow DOM\",\n method: \"not-supported\",\n arguments: [],\n selector: \"not-supported\",\n };\n }),\n )\n ).filter(<T>(e: T \| undefined): e is T => e !== undefined);\n\n v3Logger({\n category: \"observation\",\n message: \"found elements\",\n level: 1,\n auxiliary: {\n elements: {\n value: JSON.stringify(elementsWithSelectors),\n type: \"object\",\n },\n },\n });\n\n return elementsWithSelectors;\n }\n}\n"]}

package/dist/esm/lib/v3/handlers/v3AgentHandler.d.ts CHANGED Viewed

@@ -18,6 +18,7 @@ export declare class V3AgentHandler {
     private createStepHandler;
     execute(instructionOrOptions: string | AgentExecuteOptions): Promise<AgentResult>;
     stream(instructionOrOptions: string | AgentStreamExecuteOptions): Promise<AgentStreamResult>;
+    private emitFinalEvidence;
     private consolidateMetricsAndResult;
     private createTools;
     private handleStop;

package/dist/esm/lib/v3/handlers/v3AgentHandler.js CHANGED Viewed

@@ -8,6 +8,9 @@ import { V3FunctionName } from "../types/public/methods.js";
 import { mapToolResultToActions } from "../agent/utils/actionMapping.js";
 import { MissingLLMConfigurationError, MissingEnvironmentVariableError, StreamingCallbacksInNonStreamingModeError, AgentAbortError, } from "../types/public/sdkErrors.js";
 import { handleDoneToolCall } from "../agent/utils/handleDoneToolCall.js";
+import { captureProbeEvidence, emitPostStepProbeEvidence, } from "../agent/utils/postStepProbeEvidence.js";
+import { wrapEvidenceCallback } from "../agent/utils/wrapEvidenceCallback.js";
+import { inferToolOutput } from "../agent/utils/toolOutputEvidence.js";
 import { CaptchaSolver, CAPTCHA_SOLVED_MSG, CAPTCHA_ERRORED_MSG, } from "../agent/utils/captchaSolver.js";
 function getErrorMessage(error) {
     return error instanceof Error ? error.message : String(error);
@@ -160,13 +163,14 @@ export class V3AgentHandler {
             return options;
         };
     }
-    createStepHandler(state, userCallback) {
+    createStepHandler(state, { userCallback, evidenceCallback, onFinalAnswer }) {
         return async (event) => {
             this.logger({
                 category: "agent",
                 message: `Step finished: ${event.finishReason}`,
                 level: 2,
             });
+            let lastFinalAnswer;
             if (event.toolCalls && event.toolCalls.length > 0) {
                 for (let i = 0; i < event.toolCalls.length; i++) {
                     const toolCall = event.toolCalls[i];
@@ -189,6 +193,12 @@ export class V3AgentHandler {
                                 ? `${allReasoning} ${doneReasoning}`.trim()
                                 : allReasoning || "Task completed successfully";
                         }
+                        lastFinalAnswer = {
+                            message: state.finalMessage,
+                            output: typeof args?.output === "object" && args?.output !== null
+                                ? args.output
+                                : undefined,
+                        };
                     }
                     const mappedActions = mapToolResultToActions({
                         toolCallName: toolCall.toolName,
@@ -201,8 +211,31 @@ export class V3AgentHandler {
                         action.timestamp = Date.now();
                         state.actions.push(action);
                     }
+                    await evidenceCallback?.({
+                        type: "step_finished",
+                        actionName: toolCall.toolName,
+                        actionArgs: typeof args === "object" && args !== null
+                            ? args
+                            : {},
+                        reasoning: event.text ?? "",
+                        toolOutput: inferToolOutput(toolResult),
+                    });
                 }
                 state.currentPageUrl = (await this.v3.context.awaitActivePage()).url();
+                // Harness probe — one screenshot / a11y snapshot per AI SDK step.
+                // The recorder applies the probe to every step_finished received
+                // since the previous probe, so a multi-tool turn shares the same
+                // post-turn observation.
+                await emitPostStepProbeEvidence({
+                    v3: this.v3,
+                    url: state.currentPageUrl,
+                    evidenceCallback,
+                    logger: this.logger,
+                    warningMessage: "Warning: harness probe failed",
+                });
+            }
+            if (lastFinalAnswer) {
+                onFinalAnswer?.(lastFinalAnswer);
             }
             if (userCallback) {
                 await userCallback(event);
@@ -222,6 +255,7 @@ export class V3AgentHandler {
             completed: false,
             currentPageUrl: "",
         };
+        let finalAnswerFromDoneTool;
         let messages = [];
         let captchaSolver;
         try {
@@ -251,6 +285,7 @@ export class V3AgentHandler {
                     throw new StreamingCallbacksInNonStreamingModeError(invalidCallbacks);
                 }
             }
+            const evidenceCallback = wrapEvidenceCallback(callbacks?.onEvidence, this.logger);
             const result = await this.llmClient.generateText({
                 model: wrappedModel,
                 messages: prependSystemMessage(systemPrompt, messages),
@@ -258,7 +293,13 @@ export class V3AgentHandler {
                 stopWhen: (result) => this.handleStop(result, maxSteps),
                 toolChoice: "auto",
                 prepareStep: this.createPrepareStep(callbacks?.prepareStep, captchaSolver),
-                onStepFinish: this.createStepHandler(state, callbacks?.onStepFinish),
+                onStepFinish: this.createStepHandler(state, {
+                    userCallback: callbacks?.onStepFinish,
+                    evidenceCallback,
+                    onFinalAnswer: (answer) => {
+                        finalAnswerFromDoneTool = answer;
+                    },
+                }),
                 abortSignal: preparedOptions.signal,
                 providerOptions: {
                     google: { mediaResolution: "MEDIA_RESOLUTION_HIGH" },
@@ -267,7 +308,12 @@ export class V3AgentHandler {
             });
             const allMessages = [...messages, ...(result.response?.messages || [])];
             const doneResult = await this.ensureDone(state, wrappedModel, allMessages, preparedOptions.instruction, preparedOptions.output, this.logger);
-            return this.consolidateMetricsAndResult(startTime, state, doneResult.messages, result, maxSteps, doneResult.output);
+            const output = doneResult.output ?? finalAnswerFromDoneTool?.output;
+            await this.emitFinalEvidence(state, {
+                message: state.finalMessage,
+                output,
+            }, evidenceCallback);
+            return this.consolidateMetricsAndResult(startTime, state, doneResult.messages, result, maxSteps, output);
         }
         catch (error) {
             // Re-throw validation errors that should propagate to the caller
@@ -303,6 +349,7 @@ export class V3AgentHandler {
         const streamOptions = typeof instructionOrOptions === "object" ? instructionOrOptions : null;
         // Highlight cursor defaults to true for hybrid mode, can be overridden
         const shouldHighlightCursor = streamOptions?.highlightCursor ?? this.mode === "hybrid";
+        let finalAnswerFromDoneTool;
         const { options, maxSteps, systemPrompt, allTools, messages, wrappedModel, initialPageUrl, } = await this.prepareAgent(instructionOrOptions);
         // Enable cursor overlay for hybrid mode (coordinate-based interactions)
         if (shouldHighlightCursor && this.mode === "hybrid") {
@@ -340,6 +387,7 @@ export class V3AgentHandler {
             });
             rejectResult(error);
         };
+        const evidenceCallback = wrapEvidenceCallback(callbacks?.onEvidence, this.logger);
         let streamResult;
         try {
             streamResult = this.llmClient.streamText({
@@ -349,7 +397,13 @@ export class V3AgentHandler {
                 stopWhen: (result) => this.handleStop(result, maxSteps),
                 toolChoice: "auto",
                 prepareStep: this.createPrepareStep(callbacks?.prepareStep, captchaSolver),
-                onStepFinish: this.createStepHandler(state, callbacks?.onStepFinish),
+                onStepFinish: this.createStepHandler(state, {
+                    userCallback: callbacks?.onStepFinish,
+                    evidenceCallback,
+                    onFinalAnswer: (answer) => {
+                        finalAnswerFromDoneTool = answer;
+                    },
+                }),
                 onError: (event) => {
                     captchaSolver?.dispose();
                     if (callbacks?.onError) {
@@ -367,10 +421,17 @@ export class V3AgentHandler {
                         ...messages,
                         ...(event.response?.messages || []),
                     ];
-                    this.ensureDone(state, wrappedModel, allMessages, options.instruction, options.output, this.logger).then((doneResult) => {
-                        const result = this.consolidateMetricsAndResult(startTime, state, doneResult.messages, event, maxSteps, doneResult.output);
+                    this.ensureDone(state, wrappedModel, allMessages, options.instruction, options.output, this.logger)
+                        .then(async (doneResult) => {
+                        const output = doneResult.output ?? finalAnswerFromDoneTool?.output;
+                        await this.emitFinalEvidence(state, {
+                            message: state.finalMessage,
+                            output,
+                        }, evidenceCallback);
+                        const result = this.consolidateMetricsAndResult(startTime, state, doneResult.messages, event, maxSteps, output);
                         resolveResult(result);
-                    });
+                    })
+                        .catch(handleError);
                 },
                 onAbort: (event) => {
                     captchaSolver?.dispose();
@@ -398,6 +459,21 @@ export class V3AgentHandler {
         agentStreamResult.result = resultPromise;
         return agentStreamResult;
     }
+    async emitFinalEvidence(state, finalAnswer, evidenceCallback) {
+        if (!evidenceCallback)
+            return;
+        const observation = await captureProbeEvidence({
+            v3: this.v3,
+            url: state.currentPageUrl,
+            logger: this.logger,
+            warningMessage: "Warning: final harness probe failed",
+        });
+        await evidenceCallback({
+            type: "final_answer",
+            ...finalAnswer,
+            observation,
+        });
+    }
     consolidateMetricsAndResult(startTime, state, inputMessages, result, maxSteps, output) {
         if (!state.finalMessage) {
             const allReasoning = state.collectedReasoning.join(" ").trim();