donobu 2.31.1 → 2.32.0

package/dist/tools/ScrollPageTool.d.ts

@@ -1,29 +1,70 @@
-import { Tool } from './Tool';
+import { Locator } from 'playwright';
 import { ToolCallContext } from '../models/ToolCallContext';
-import { ToolCallResult } from '../models/ToolCallResult';
+import { ReplayableInteraction } from './ReplayableInteraction';
 import { z } from 'zod';
 export declare const ScrollPageCoreSchema: z.ZodObject<{
-    direction: z.ZodEnum<["UP", "DOWN"]>;
+    direction: z.ZodEnum<["UP", "DOWN", "LEFT", "RIGHT"]>;
+    maxScroll: z.ZodOptional<z.ZodBoolean>;
 }, "strip", z.ZodTypeAny, {
-    direction: "UP" | "DOWN";
+    direction: "UP" | "DOWN" | "LEFT" | "RIGHT";
+    maxScroll?: boolean | undefined;
 }, {
-    direction: "UP" | "DOWN";
+    direction: "UP" | "DOWN" | "LEFT" | "RIGHT";
+    maxScroll?: boolean | undefined;
+}>;
+export declare const ScrollPageNonGptSchema: z.ZodObject<{
+    selector: z.ZodObject<{
+        element: z.ZodArray<z.ZodString, "many">;
+        frame: z.ZodNullable<z.ZodString>;
+    }, "strip", z.ZodTypeAny, {
+        frame: string | null;
+        element: string[];
+    }, {
+        frame: string | null;
+        element: string[];
+    }>;
+} & {
+    direction: z.ZodEnum<["UP", "DOWN", "LEFT", "RIGHT"]>;
+    maxScroll: z.ZodOptional<z.ZodBoolean>;
+}, "strip", z.ZodTypeAny, {
+    selector: {
+        frame: string | null;
+        element: string[];
+    };
+    direction: "UP" | "DOWN" | "LEFT" | "RIGHT";
+    maxScroll?: boolean | undefined;
+}, {
+    selector: {
+        frame: string | null;
+        element: string[];
+    };
+    direction: "UP" | "DOWN" | "LEFT" | "RIGHT";
+    maxScroll?: boolean | undefined;
 }>;
 export declare const ScrollPageGptSchema: z.ZodObject<{
     rationale: z.ZodString;
 } & {
-    direction: z.ZodEnum<["UP", "DOWN"]>;
+    whyThisAnnotation: z.ZodString;
+    annotation: z.ZodString;
+} & {
+    direction: z.ZodEnum<["UP", "DOWN", "LEFT", "RIGHT"]>;
+    maxScroll: z.ZodOptional<z.ZodBoolean>;
 }, "strip", z.ZodTypeAny, {
+    annotation: string;
     rationale: string;
-    direction: "UP" | "DOWN";
+    whyThisAnnotation: string;
+    direction: "UP" | "DOWN" | "LEFT" | "RIGHT";
+    maxScroll?: boolean | undefined;
 }, {
+    annotation: string;
     rationale: string;
-    direction: "UP" | "DOWN";
+    whyThisAnnotation: string;
+    direction: "UP" | "DOWN" | "LEFT" | "RIGHT";
+    maxScroll?: boolean | undefined;
 }>;
-export declare class ScrollPageTool extends Tool<typeof ScrollPageCoreSchema, typeof ScrollPageGptSchema> {
+export declare class ScrollPageTool extends ReplayableInteraction<typeof ScrollPageCoreSchema, typeof ScrollPageNonGptSchema, typeof ScrollPageGptSchema> {
     static readonly NAME = "scrollPage";
     constructor();
-    call(context: ToolCallContext, parameters: z.infer<typeof ScrollPageCoreSchema>): Promise<ToolCallResult>;
-    callFromGpt(context: ToolCallContext, parameters: z.infer<typeof ScrollPageGptSchema>): Promise<ToolCallResult>;
+    invoke(_context: ToolCallContext, parameters: z.infer<typeof ScrollPageCoreSchema>, element: Locator): Promise<string>;
 }
 //# sourceMappingURL=ScrollPageTool.d.ts.map

package/dist/tools/ScrollPageTool.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"ScrollPageTool.d.ts","sourceRoot":"","sources":["../../src/tools/ScrollPageTool.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,QAAQ,CAAC;AAC9B,OAAO,EAAE,eAAe,EAAE,MAAM,2BAA2B,CAAC;AAC5D,OAAO,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AAE1D,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,eAAO,MAAM,oBAAoB;;;;;;EAI/B,CAAC;AACH,eAAO,MAAM,mBAAmB;;;;;;;;;;EACe,CAAC;AAEhD,qBAAa,cAAe,SAAQ,IAAI,CACtC,OAAO,oBAAoB,EAC3B,OAAO,mBAAmB,CAC3B;IACC,gBAAuB,IAAI,gBAAgB;;IAYrB,IAAI,CACxB,OAAO,EAAE,eAAe,EACxB,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,oBAAoB,CAAC,GAC/C,OAAO,CAAC,cAAc,CAAC;IA+DJ,WAAW,CAC/B,OAAO,EAAE,eAAe,EACxB,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,mBAAmB,CAAC,GAC9C,OAAO,CAAC,cAAc,CAAC;CAG3B"}
+{"version":3,"file":"ScrollPageTool.d.ts","sourceRoot":"","sources":["../../src/tools/ScrollPageTool.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,EAAE,eAAe,EAAE,MAAM,2BAA2B,CAAC;AAC5D,OAAO,EAEL,qBAAqB,EAEtB,MAAM,yBAAyB,CAAC;AACjC,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,eAAO,MAAM,oBAAoB;;;;;;;;;EAW/B,CAAC;AACH,eAAO,MAAM,sBAAsB;;;;;;;;;;;;;;;;;;;;;;;;;;;;EACc,CAAC;AAElD,eAAO,MAAM,mBAAmB;;;;;;;;;;;;;;;;;;;;EACmB,CAAC;AAEpD,qBAAa,cAAe,SAAQ,qBAAqB,CACvD,OAAO,oBAAoB,EAC3B,OAAO,sBAAsB,EAC7B,OAAO,mBAAmB,CAC3B;IACC,gBAAuB,IAAI,gBAAgB;;IAarC,MAAM,CACV,QAAQ,EAAE,eAAe,EACzB,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,oBAAoB,CAAC,EAChD,OAAO,EAAE,OAAO,GACf,OAAO,CAAC,MAAM,CAAC;CAsDnB"}

package/dist/tools/ScrollPageTool.js

@@ -1,70 +1,76 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.ScrollPageTool = exports.ScrollPageGptSchema = exports.ScrollPageCoreSchema = void 0;
-const Tool_1 = require("./Tool");
-const ToolSchema_1 = require("../models/ToolSchema");
+exports.ScrollPageTool = exports.ScrollPageGptSchema = exports.ScrollPageNonGptSchema = exports.ScrollPageCoreSchema = void 0;
+const ReplayableInteraction_1 = require("./ReplayableInteraction");
 const zod_1 = require("zod");
 exports.ScrollPageCoreSchema = zod_1.z.object({
     direction: zod_1.z
-        .enum(['UP', 'DOWN'])
-        .describe('The direction to scroll - either UP or DOWN'),
+        .enum(['UP', 'DOWN', 'LEFT', 'RIGHT'])
+        .describe('The direction to scroll'),
+    maxScroll: zod_1.z
+        .boolean()
+        .optional()
+        .describe(`Optional. Set to true if the page or element should be scrolled maximally in the given direction.
+If not specified, the page or element will scroll no more than a viewport's worth.`),
 });
-exports.ScrollPageGptSchema = ToolSchema_1.BaseGptArgsSchema.merge(exports.ScrollPageCoreSchema);
-class ScrollPageTool extends Tool_1.Tool {
+exports.ScrollPageNonGptSchema = ReplayableInteraction_1.SelectorBasedSchema.merge(exports.ScrollPageCoreSchema);
+exports.ScrollPageGptSchema = ReplayableInteraction_1.AnnotationBasedSchema.merge(exports.ScrollPageCoreSchema);
+class ScrollPageTool extends ReplayableInteraction_1.ReplayableInteraction {
     constructor() {
-        super(ScrollPageTool.NAME, `Scroll the current page up or down by one viewport. This helps you see more of the 
-current page, of which may contain relevant information below the fold.`, exports.ScrollPageCoreSchema, exports.ScrollPageGptSchema);
+        super(ScrollPageTool.NAME, `Scroll the current page or element. This helps you see more of the
+current page or element, of which may contain relevant information.`, exports.ScrollPageCoreSchema, exports.ScrollPageNonGptSchema, exports.ScrollPageGptSchema);
     }
-    async call(context, parameters) {
-        const page = context.page.current;
-        // compute scroll distance = one viewport height
-        const distance = await page.evaluate(() => window.innerHeight);
-        // 1) Try scrolling the top‐level window
-        const windowScrolled = await page.evaluate((dy) => {
-            const before = window.scrollY;
-            window.scrollBy(0, dy);
-            return window.scrollY !== before;
-        }, parameters.direction === 'DOWN' ? distance : -distance);
-        let didScroll = windowScrolled;
-        // 2) If the window didn’t move, scroll the largest scrollable element
-        if (!windowScrolled) {
-            const fallbackScrolled = await page.evaluate((dy) => {
-                // find all scrollable elements
-                const els = Array.from(document.querySelectorAll('*'))
-                    .filter((el) => {
-                    const style = getComputedStyle(el);
-                    const ovY = style.overflowY;
-                    return ((ovY === 'scroll' || ovY === 'auto') &&
-                        el.scrollHeight > el.clientHeight);
-                })
-                    // sort by visible area descending
-                    .map((el) => ({ el, area: el.clientWidth * el.clientHeight }))
-                    .sort((a, b) => b.area - a.area);
-                if (els.length === 0) {
-                    return false;
+    async invoke(_context, parameters, element) {
+        await element.hover();
+        const didScroll = await element.evaluate((el, parameters) => {
+            const beforeY = el.scrollTop;
+            const beforeX = el.scrollLeft;
+            if (parameters.maxScroll) {
+                /* Jump straight to the edge */
+                switch (parameters.direction) {
+                    case 'UP':
+                        el.scrollTop = 0;
+                        break;
+                    case 'DOWN':
+                        el.scrollTop = el.scrollHeight;
+                        break;
+                    case 'LEFT':
+                        el.scrollLeft = 0;
+                        break;
+                    case 'RIGHT':
+                        el.scrollLeft = el.scrollWidth;
+                        break;
                 }
-                // scroll the largest one
-                const target = els[0].el;
-                const before = target.scrollTop;
-                target.scrollTop += dy;
-                return target.scrollTop !== before;
-            }, parameters.direction === 'DOWN' ? distance : -distance);
-            didScroll = fallbackScrolled;
-        }
-        return didScroll
-            ? {
-                isSuccessful: true,
-                forLlm: `Scrolled ${parameters.direction}`,
-                metadata: null,
             }
-            : {
-                isSuccessful: false,
-                forLlm: `Could not scroll ${parameters.direction} due to nothing to scroll.`,
-                metadata: null,
-            };
-    }
-    async callFromGpt(context, parameters) {
-        return this.call(context, parameters);
+            else {
+                /* Incremental ¾-viewport scroll */
+                const visibleHeight = Math.min(el.clientHeight, window.innerHeight);
+                const visibleWidth = Math.min(el.clientWidth, window.innerWidth);
+                const deltaY = Math.floor(visibleHeight * 0.75);
+                const deltaX = Math.floor(visibleWidth * 0.75);
+                switch (parameters.direction) {
+                    case 'UP':
+                        el.scrollBy(0, -deltaY);
+                        break;
+                    case 'DOWN':
+                        el.scrollBy(0, deltaY);
+                        break;
+                    case 'LEFT':
+                        el.scrollBy(-deltaX, 0);
+                        break;
+                    case 'RIGHT':
+                        el.scrollBy(deltaX, 0);
+                        break;
+                }
+            }
+            const afterY = el.scrollTop;
+            const afterX = el.scrollLeft;
+            /* Return whether any movement occurred */
+            return beforeY !== afterY || beforeX !== afterX;
+        }, parameters);
+        return didScroll
+            ? `Scrolled ${parameters.direction} ${parameters.maxScroll ? ' (max)' : ''}`
+            : `Nothing to scroll ${parameters.direction}`;
     }
 }
 exports.ScrollPageTool = ScrollPageTool;

package/dist/tools/ScrollPageTool.js.map

@@ -1 +1 @@
-{"version":3,"file":"ScrollPageTool.js","sourceRoot":"","sources":["../../src/tools/ScrollPageTool.ts"],"names":[],"mappings":";;;AAAA,iCAA8B;AAG9B,qDAAyD;AACzD,6BAAwB;AAEX,QAAA,oBAAoB,GAAG,OAAC,CAAC,MAAM,CAAC;IAC3C,SAAS,EAAE,OAAC;SACT,IAAI,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;SACpB,QAAQ,CAAC,6CAA6C,CAAC;CAC3D,CAAC,CAAC;AACU,QAAA,mBAAmB,GAC9B,8BAAiB,CAAC,KAAK,CAAC,4BAAoB,CAAC,CAAC;AAEhD,MAAa,cAAe,SAAQ,WAGnC;IAGC;QACE,KAAK,CACH,cAAc,CAAC,IAAI,EACnB;wEACkE,EAClE,4BAAoB,EACpB,2BAAmB,CACpB,CAAC;IACJ,CAAC;IAEe,KAAK,CAAC,IAAI,CACxB,OAAwB,EACxB,UAAgD;QAEhD,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,OAAQ,CAAC;QACnC,gDAAgD;QAChD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;QAE/D,wCAAwC;QACxC,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,QAAQ,CACxC,CAAC,EAAU,EAAE,EAAE;YACb,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC;YAC9B,MAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACvB,OAAO,MAAM,CAAC,OAAO,KAAK,MAAM,CAAC;QACnC,CAAC,EACD,UAAU,CAAC,SAAS,KAAK,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CACvD,CAAC;QAEF,IAAI,SAAS,GAAG,cAAc,CAAC;QAE/B,sEAAsE;QACtE,IAAI,CAAC,cAAc,EAAE,CAAC;YACpB,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,QAAQ,CAC1C,CAAC,EAAU,EAAE,EAAE;gBACb,+BAA+B;gBAC/B,MAAM,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAc,GAAG,CAAC,CAAC;qBAChE,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE;oBACb,MAAM,KAAK,GAAG,gBAAgB,CAAC,EAAE,CAAC,CAAC;oBACnC,MAAM,GAAG,GAAG,KAAK,CAAC,SAAS,CAAC;oBAC5B,OAAO,CACL,CAAC,GAAG,KAAK,QAAQ,IAAI,GAAG,KAAK,MAAM,CAAC;wBACpC,EAAE,CAAC,YAAY,GAAG,EAAE,CAAC,YAAY,CAClC,CAAC;gBACJ,CAAC,CAAC;oBACF,kCAAkC;qBACjC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,CAAC,WAAW,GAAG,EAAE,CAAC,YAAY,EAAE,CAAC,CAAC;qBAC7D,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;gBAEnC,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBACrB,OAAO,KAAK,CAAC;gBACf,CAAC;gBACD,yBAAyB;gBACzB,MAAM,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;gBACzB,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,CAAC;gBAChC,MAAM,CAAC,SAAS,IAAI,EAAE,CAAC;gBACvB,OAAO,MAAM,CAAC,SAAS,KAAK,MAAM,CAAC;YACrC,CAAC,EACD,UAAU,CAAC,SAAS,KAAK,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CACvD,CAAC;YAEF,SAAS,GAAG,gBAAgB,CAAC;QAC/B,CAAC;QAED,OAAO,SAAS;YACd,CAAC,CAAC;gBACE,YAAY,EAAE,IAAI;gBAClB,MAAM,EAAE,YAAY,UAAU,CAAC,SAAS,EAAE;gBAC1C,QAAQ,EAAE,IAAI;aACf;YACH,CAAC,CAAC;gBACE,YAAY,EAAE,KAAK;gBACnB,MAAM,EAAE,oBAAoB,UAAU,CAAC,SAAS,4BAA4B;gBAC5E,QAAQ,EAAE,IAAI;aACf,CAAC;IACR,CAAC;IAEe,KAAK,CAAC,WAAW,CAC/B,OAAwB,EACxB,UAA+C;QAE/C,OAAO,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC;IACxC,CAAC;;AAvFH,wCAwFC;AApFwB,mBAAI,GAAG,YAAY,CAAC"}
+{"version":3,"file":"ScrollPageTool.js","sourceRoot":"","sources":["../../src/tools/ScrollPageTool.ts"],"names":[],"mappings":";;;AAEA,mEAIiC;AACjC,6BAAwB;AAEX,QAAA,oBAAoB,GAAG,OAAC,CAAC,MAAM,CAAC;IAC3C,SAAS,EAAE,OAAC;SACT,IAAI,CAAC,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;SACrC,QAAQ,CAAC,yBAAyB,CAAC;IACtC,SAAS,EAAE,OAAC;SACT,OAAO,EAAE;SACT,QAAQ,EAAE;SACV,QAAQ,CACP;mFAC6E,CAC9E;CACJ,CAAC,CAAC;AACU,QAAA,sBAAsB,GACjC,2CAAmB,CAAC,KAAK,CAAC,4BAAoB,CAAC,CAAC;AAErC,QAAA,mBAAmB,GAC9B,6CAAqB,CAAC,KAAK,CAAC,4BAAoB,CAAC,CAAC;AAEpD,MAAa,cAAe,SAAQ,6CAInC;IAGC;QACE,KAAK,CACH,cAAc,CAAC,IAAI,EACnB;oEAC8D,EAC9D,4BAAoB,EACpB,8BAAsB,EACtB,2BAAmB,CACpB,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,MAAM,CACV,QAAyB,EACzB,UAAgD,EAChD,OAAgB;QAEhB,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;QACtB,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,QAAQ,CAAC,CAAC,EAAE,EAAE,UAAU,EAAE,EAAE;YAC1D,MAAM,OAAO,GAAG,EAAE,CAAC,SAAS,CAAC;YAC7B,MAAM,OAAO,GAAG,EAAE,CAAC,UAAU,CAAC;YAE9B,IAAI,UAAU,CAAC,SAAS,EAAE,CAAC;gBACzB,+BAA+B;gBAC/B,QAAQ,UAAU,CAAC,SAAS,EAAE,CAAC;oBAC7B,KAAK,IAAI;wBACP,EAAE,CAAC,SAAS,GAAG,CAAC,CAAC;wBACjB,MAAM;oBACR,KAAK,MAAM;wBACT,EAAE,CAAC,SAAS,GAAG,EAAE,CAAC,YAAY,CAAC;wBAC/B,MAAM;oBACR,KAAK,MAAM;wBACT,EAAE,CAAC,UAAU,GAAG,CAAC,CAAC;wBAClB,MAAM;oBACR,KAAK,OAAO;wBACV,EAAE,CAAC,UAAU,GAAG,EAAE,CAAC,WAAW,CAAC;wBAC/B,MAAM;gBACV,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,mCAAmC;gBACnC,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,YAAY,EAAE,MAAM,CAAC,WAAW,CAAC,CAAC;gBACpE,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,WAAW,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC;gBACjE,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,aAAa,GAAG,IAAI,CAAC,CAAC;gBAChD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,GAAG,IAAI,CAAC,CAAC;gBAE/C,QAAQ,UAAU,CAAC,SAAS,EAAE,CAAC;oBAC7B,KAAK,IAAI;wBACP,EAAE,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC;wBACxB,MAAM;oBACR,KAAK,MAAM;wBACT,EAAE,CAAC,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;wBACvB,MAAM;oBACR,KAAK,MAAM;wBACT,EAAE,CAAC,QAAQ,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;wBACxB,MAAM;oBACR,KAAK,OAAO;wBACV,EAAE,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;wBACvB,MAAM;gBACV,CAAC;YACH,CAAC;YAED,MAAM,MAAM,GAAG,EAAE,CAAC,SAAS,CAAC;YAC5B,MAAM,MAAM,GAAG,EAAE,CAAC,UAAU,CAAC;YAC7B,0CAA0C;YAC1C,OAAO,OAAO,KAAK,MAAM,IAAI,OAAO,KAAK,MAAM,CAAC;QAClD,CAAC,EAAE,UAAU,CAAC,CAAC;QACf,OAAO,SAAS;YACd,CAAC,CAAC,YAAY,UAAU,CAAC,SAAS,IAAI,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,EAAE;YAC5E,CAAC,CAAC,qBAAqB,UAAU,CAAC,SAAS,EAAE,CAAC;IAClD,CAAC;;AA3EH,wCA4EC;AAvEwB,mBAAI,GAAG,YAAY,CAAC"}