page-agent 0.0.4 → 0.0.6

package/README.md

@@ -4,19 +4,19 @@
 
 [![npm version](https://badge.fury.io/js/page-agent.svg)](https://badge.fury.io/js/page-agent) [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) [![TypeScript](https://img.shields.io/badge/%3C%2F%3E-TypeScript-%230074c1.svg)](http://www.typescriptlang.org/) [![Downloads](https://img.shields.io/npm/dt/page-agent.svg)](https://www.npmjs.com/package/page-agent) [![Bundle Size](https://img.shields.io/bundlephobia/minzip/page-agent)](https://bundlephobia.com/package/page-agent) [![GitHub stars](https://img.shields.io/github/stars/alibaba/page-agent.svg)](https://github.com/alibaba/page-agent)
 
-**Transform your webpage into an AI-powered application with a single script tag.**
-
-An in-page UI agent in javascript. Control web interfaces with natural language.
+The GUI Agent Living in Your Webpage. Control web interfaces with natural language.
 
 🌐 **English** | [中文](./README-zh.md)
 
-👉 [🚀 **Demo**](https://alibaba.github.io/page-agent/) | [📖 **Documentation**](https://alibaba.github.io/page-agent/#/docs/introduction/overview)
+👉 <a href="https://alibaba.github.io/page-agent/" target="_blank"><b>🚀 Demo</b></a> | <a href="https://alibaba.github.io/page-agent/#/docs/introduction/overview" target="_blank"><b>📖 Documentation</b></a> 
+
+<video id="demo-video" src="https://github.com/user-attachments/assets/de8d1964-8bde-494f-a52f-2975469557a5" width="640" crossorigin muted autoplay loop></video>
 
 ---
 
 ## ✨ Features
 
-- **🎯 Easy Integration**
+- **🎯 Easy Integration** - Transform your webpage into an agent with a single script tag.
 - **🔐 Client-Side Processing**
 - **🧠 DOM Extraction**
 - **💬 Natural Language Interface**
@@ -33,7 +33,7 @@ An in-page UI agent in javascript. Control web interfaces with natural language.
 ```html
 <!-- temporary CDN URL. May change in the future -->
 <script
-	src="https://hwcxiuzfylggtcktqgij.supabase.co/storage/v1/object/public/demo-public/v0.0.2/page-agent.js"
+	src="https://hwcxiuzfylggtcktqgij.supabase.co/storage/v1/object/public/demo-public/v0.0.4/page-agent.js"
 	crossorigin="true"
 	type="text/javascript"
 ></script>

package/dist/lib/PageAgent.d.ts

@@ -48,6 +48,13 @@ declare interface AgentConfig {
      * @note when dispose caused by page unload, reason will be 'PAGE_UNLOADING'. this method CANNOT block unloading. async operations may be cut.
      */
     onDispose?: (this: PageAgent, reason?: string) => void;
+    /**
+     * @experimental
+     * Enable the experimental script execution tool that allows executing generated JavaScript code on the page.
+     * @note Can cause unpredictable side effects.
+     * @note May bypass some safe guards and data-masking mechanisms.
+     */
+    experimentalScriptExecutionTool?: boolean;
     /**
      * TODO: @unimplemented
      * hook when action causes a new page to be opened
@@ -167,11 +174,11 @@ declare class EventBus extends EventTarget {
     /**
      * Listen to built-in events
      */
-    on<T extends keyof PageAgentEventMap>(event: T, handler: EventHandler<T & keyof PageAgentEventMap>): void;
+    on<T extends keyof PageAgentEventMap>(event: T, handler: EventHandler<T>): void;
     /**
      * Listen to built-in events (one-time)
      */
-    once<T extends keyof PageAgentEventMap>(event: T, handler: EventHandler<T & keyof PageAgentEventMap>): void;
+    once<T extends keyof PageAgentEventMap>(event: T, handler: EventHandler<T>): void;
     /**
      * Emit built-in events
      */
@@ -492,15 +499,3 @@ declare type TranslationParams = Record<string, string | number>;
 declare type TranslationSchema = DeepStringify<typeof enUS>;
 
 export { }
-
-
-declare module 'react-i18next' {
-    interface CustomTypeOptions {
-        defaultNS: 'common';
-        resources: {
-            common: typeof commonZh;
-            home: typeof homeZh;
-            docs: typeof docsZh;
-        };
-    }
-}

package/dist/lib/page-agent.js

@@ -1189,7 +1189,7 @@ function flatTreeToString(flatTree, include_attributes) {
     }
     return false;
   }, "hasParentWithHighlightIndex");
-  const processNode = /* @__PURE__ */ __name((node, depth, result2) => {
+  const processNode = /* @__PURE__ */ __name((node, depth, result22) => {
     let nextDepth = depth;
     const depthStr = "	".repeat(depth);
     if (node.type === "element") {
@@ -1264,23 +1264,23 @@ function flatTreeToString(flatTree, include_attributes) {
           line += " ";
         }
         line += " />";
-        result2.push(line);
+        result22.push(line);
       }
       for (const child of node.children) {
-        processNode(child, nextDepth, result2);
+        processNode(child, nextDepth, result22);
       }
     } else if (node.type === "text") {
       if (hasParentWithHighlightIndex(node)) {
         return;
       }
       if (node.parent && node.parent.type === "element" && node.parent.isVisible && node.parent.isTopElement) {
-        result2.push(`${depthStr}${node.text ?? ""}`);
+        result22.push(`${depthStr}${node.text ?? ""}`);
       }
     }
   }, "processNode");
-  const result = [];
-  processNode(rootNode, 0, result);
-  return result.join("\n");
+  const result2 = [];
+  processNode(rootNode, 0, result2);
+  return result2.join("\n");
 }
 __name(flatTreeToString, "flatTreeToString");
 const getAllTextTillNextClickableElement = /* @__PURE__ */ __name((node, maxDepth = -1) => {
@@ -1651,6 +1651,8 @@ function lenientParseMacroToolCall(responseData, inputSchema) {
   }
   switch (choice.finish_reason) {
     case "tool_calls":
+    case "function_call":
+    // gemini
     case "stop":
       break;
     case "length":
@@ -1902,8 +1904,8 @@ const _LLM = class _LLM {
   async invoke(messages, tools2, abortSignal) {
     return await withRetry(
       async () => {
-        const result = await this.client.invoke(messages, tools2, abortSignal);
-        return result;
+        const result2 = await this.client.invoke(messages, tools2, abortSignal);
+        return result2;
       },
       // retry settings
       {
@@ -2413,6 +2415,24 @@ tools.set(
     }, "execute")
   })
 );
+tools.set(
+  "execute_javascript",
+  tool({
+    description: "Execute JavaScript code on the current page. Supports async/await syntax. Use with caution!",
+    inputSchema: zod.object({
+      script: zod.string()
+    }),
+    execute: /* @__PURE__ */ __name(async function(input) {
+      try {
+        const asyncFunction = eval(`(async () => { ${input.script} })`);
+        const result = await asyncFunction();
+        return `✅ Executed JavaScript. Result: ${result}` + await getSystemInfo();
+      } catch (error2) {
+        return `❌ Error executing JavaScript: ${error2}` + await getSystemInfo();
+      }
+    }, "execute")
+  })
+);
 async function waitUntil(check, timeout = 60 * 601e3) {
   if (check()) return true;
   return new Promise((resolve, reject) => {
@@ -3372,6 +3392,9 @@ const _PageAgent = class _PageAgent extends EventTarget {
         this.tools.set(name, tool2);
       }
     }
+    if (!this.config.experimentalScriptExecutionTool) {
+      this.tools.delete("execute_javascript");
+    }
     patchReact();
     window.addEventListener("beforeunload", (e) => {
       if (!this.disposed) this.dispose("PAGE_UNLOADING");
@@ -3413,7 +3436,7 @@ const _PageAgent = class _PageAgent extends EventTarget {
           type: "thinking",
           displayText: this.i18n.t("ui.panel.thinking")
         });
-        const result = await __privateGet(this, _llm).invoke(
+        const result2 = await __privateGet(this, _llm).invoke(
           [
             {
               role: "system",
@@ -3427,7 +3450,7 @@ const _PageAgent = class _PageAgent extends EventTarget {
           { AgentOutput: __privateMethod(this, _PageAgent_instances, packMacroTool_fn).call(this) },
           __privateGet(this, _abortController).signal
         );
-        const macroResult = result.toolResult;
+        const macroResult = result2.toolResult;
         const input2 = macroResult.input;
         const output2 = macroResult.output;
         const brain = {
@@ -3444,7 +3467,7 @@ const _PageAgent = class _PageAgent extends EventTarget {
         this.history.push({
           brain,
           action,
-          usage: result.usage
+          usage: result2.usage
         });
         console.log(chalk.green("Step finished:"), actionName);
         console.groupEnd();
@@ -3452,38 +3475,38 @@ const _PageAgent = class _PageAgent extends EventTarget {
         step++;
         if (step > MAX_STEPS) {
           __privateMethod(this, _PageAgent_instances, onDone_fn).call(this, "Step count exceeded maximum limit", false);
-          const result2 = {
+          const result22 = {
             success: false,
             data: "Step count exceeded maximum limit",
             history: this.history
           };
-          await onAfterTask.call(this, result2);
-          return result2;
+          await onAfterTask.call(this, result22);
+          return result22;
         }
         if (actionName === "done") {
           const success = action.input?.success ?? false;
           const text = action.input?.text || "no text provided";
           console.log(chalk.green.bold("Task completed"), success, text);
           __privateMethod(this, _PageAgent_instances, onDone_fn).call(this, text, success);
-          const result2 = {
+          const result22 = {
             success,
             data: text,
             history: this.history
           };
-          await onAfterTask.call(this, result2);
-          return result2;
+          await onAfterTask.call(this, result22);
+          return result22;
         }
       }
     } catch (error2) {
       console.error("Task failed", error2);
       __privateMethod(this, _PageAgent_instances, onDone_fn).call(this, String(error2), false);
-      const result = {
+      const result2 = {
         success: false,
         data: String(error2),
         history: this.history
       };
-      await onAfterTask.call(this, result);
-      return result;
+      await onAfterTask.call(this, result2);
+      return result2;
     }
   }
   dispose(reason) {
@@ -3558,16 +3581,16 @@ packMacroTool_fn = /* @__PURE__ */ __name(function() {
         displayText: getToolExecutingText(toolName, toolInput, this.i18n)
       });
       const startTime = Date.now();
-      let result = await tool2.execute.bind(this)(toolInput);
+      let result2 = await tool2.execute.bind(this)(toolInput);
       const duration = Date.now() - startTime;
-      console.log(chalk.green.bold(`Tool (${toolName}) executed for ${duration}ms`), result);
+      console.log(chalk.green.bold(`Tool (${toolName}) executed for ${duration}ms`), result2);
       if (toolName === "wait") {
         __privateSet(this, _totalWaitTime, __privateGet(this, _totalWaitTime) + Math.round(toolInput.seconds + duration / 1e3));
-        result += `
+        result2 += `
 <sys> You have waited ${__privateGet(this, _totalWaitTime)} seconds accumulatively.`;
         if (__privateGet(this, _totalWaitTime) >= 3)
-          result += "\nDo NOT wait any longer unless you have a good reason.\n";
-        result += "</sys>";
+          result2 += "\nDo NOT wait any longer unless you have a good reason.\n";
+        result2 += "</sys>";
       } else {
         __privateSet(this, _totalWaitTime, 0);
       }
@@ -3577,14 +3600,14 @@ packMacroTool_fn = /* @__PURE__ */ __name(function() {
           type: "tool_executing",
           toolName,
           toolArgs: toolInput,
-          toolResult: result,
+          toolResult: result2,
           displayText: displayResult,
           duration
         });
       await new Promise((resolve) => setTimeout(resolve, 100));
       return {
         input: input2,
-        output: result
+        output: result2
       };
     }, "execute")
   };