@jarvis-agent/core 0.2.6 → 0.3.1

package/dist/index.esm.js

@@ -39669,6 +39669,24 @@ Output:
   "changed": true,
   "changeInfo": "New message received in the group chat. The message content is: 'Hello, how are you?'"
 }`;
+const watch_text_system_prompt = `You are a page content analyzer. Given a page content and a condition description, determine if the condition is currently met on the page.
+Return ONLY a JSON object, no other text.
+- "changed": true means the condition IS met (e.g. the target button exists, the status has changed to the expected value)
+- "changed": false means the condition is NOT yet met
+
+## Example
+Condition: Monitor for a "Retry" button appearing on the page
+### Condition not met
+Output:
+{
+  "changed": false
+}
+### Condition met
+Output:
+{
+  "changed": true,
+  "changeInfo": "The 'Retry' button is present on the page at index 127-128"
+}`;
 class WatchTriggerTool {
     constructor() {
         this.name = TOOL_NAME$a;
@@ -39733,12 +39751,34 @@ class WatchTriggerTool {
                 ],
             };
         }
+        const rlm = new RetryLanguageModel(agentContext.context.config.llms, agentContext.agent.Llms, agentContext.context.config.globalConfig?.streamFirstTimeout, agentContext.context.config.globalConfig?.streamTokenTimeout, agentContext);
+        const useVision = this.isVisionModel(rlm);
+        // Initial condition check (text-based, works with all models)
+        const pageContent = await this.get_page_content(agentContext);
+        const initialCheck = await this.is_condition_met(rlm, pageContent, task_description, agentContext);
+        if (initialCheck.changed) {
+            return {
+                content: [
+                    {
+                        type: "text",
+                        text: initialCheck.changeInfo || "Condition already met on page.",
+                    },
+                ],
+            };
+        }
+        // Enter monitoring loop
         await this.init_eko_observer(agentContext);
-        const image1 = await this.get_screenshot(agentContext);
         const start = new Date().getTime();
         const timeout = (args.timeout || 5) * 60000;
         const frequency = Math.max(500, (args.frequency || 1) * 1000);
-        const rlm = new RetryLanguageModel(agentContext.context.config.llms, agentContext.agent.Llms, agentContext.context.config.globalConfig?.streamFirstTimeout, agentContext.context.config.globalConfig?.streamTokenTimeout, agentContext);
+        let image1;
+        let content1;
+        if (useVision) {
+            image1 = await this.get_screenshot(agentContext);
+        }
+        else {
+            content1 = pageContent;
+        }
         while (new Date().getTime() - start < timeout) {
             await agentContext.context.checkAborted();
             await new Promise((resolve) => setTimeout(resolve, frequency));
@@ -39747,17 +39787,32 @@ class WatchTriggerTool {
                 continue;
             }
             await this.init_eko_observer(agentContext);
-            const image2 = await this.get_screenshot(agentContext);
-            const changeResult = await this.is_dom_change(agentContext, rlm, image1, image2, task_description);
-            if (changeResult.changed) {
-                return {
-                    content: [
-                        {
-                            type: "text",
-                            text: changeResult.changeInfo || "DOM change detected.",
-                        },
-                    ],
-                };
+            if (useVision) {
+                // Vision model: compare screenshots
+                const image2 = await this.get_screenshot(agentContext);
+                const changeResult = await this.is_dom_change(agentContext, rlm, image1, image2, task_description);
+                if (changeResult.changed) {
+                    return {
+                        content: [
+                            { type: "text", text: changeResult.changeInfo || "DOM change detected." },
+                        ],
+                    };
+                }
+            }
+            else {
+                // Text model: compare page content
+                const content2 = await this.get_page_content(agentContext);
+                if (content2 === content1)
+                    continue;
+                const changeResult = await this.is_condition_met(rlm, content2, task_description, agentContext);
+                content1 = content2;
+                if (changeResult.changed) {
+                    return {
+                        content: [
+                            { type: "text", text: changeResult.changeInfo || "Condition met." },
+                        ],
+                    };
+                }
             }
         }
         return {
@@ -39769,6 +39824,65 @@ class WatchTriggerTool {
             ],
         };
     }
+    /** Check if the primary LLM supports vision */
+    isVisionModel(rlm) {
+        const names = rlm.Names;
+        const llms = rlm.Llms;
+        if (!names || names.length === 0)
+            return false;
+        const config = llms[names[0]];
+        if (!config)
+            return false;
+        const provider = String(config.provider || "").toLowerCase();
+        const model = String(config.model || "").toLowerCase();
+        if (provider === "deepseek" || model.includes("deepseek"))
+            return false;
+        if (provider === "anthropic")
+            return true;
+        if (provider === "google")
+            return true;
+        if (model.includes("gpt-4o") || model.includes("gpt-4-vision") || model.includes("gpt-4-turbo"))
+            return true;
+        if (model.includes("claude") || model.includes("gemini"))
+            return true;
+        return false;
+    }
+    /** Get page text content via extract_page_content */
+    async get_page_content(agentContext) {
+        const extract = agentContext.agent["extract_page_content"];
+        if (!extract)
+            return "";
+        const result = await extract.call(agentContext.agent, agentContext);
+        return result?.page_content || "";
+    }
+    /** Check if condition is met using text-based LLM analysis */
+    async is_condition_met(rlm, pageContent, task_description, agentContext) {
+        try {
+            const request = {
+                messages: [
+                    { role: "system", content: watch_text_system_prompt },
+                    {
+                        role: "user",
+                        content: [
+                            {
+                                type: "text",
+                                text: `Condition: ${task_description}\n\nPage content:\n${pageContent.slice(0, 30000)}`,
+                            },
+                        ],
+                    },
+                ],
+                abortSignal: agentContext.context.controller.signal,
+            };
+            const result = await rlm.call(request);
+            let resultText = result.text || "{}";
+            resultText = resultText.substring(resultText.indexOf("{"), resultText.lastIndexOf("}") + 1);
+            return JSON.parse(resultText);
+        }
+        catch (error) {
+            Log.error("Error in is_condition_met:", error);
+        }
+        return { changed: false };
+    }
     async get_screenshot(agentContext) {
         const screenshot = agentContext.agent["screenshot"];
         const imageResult = (await screenshot.call(agentContext.agent, agentContext));
@@ -40142,10 +40256,77 @@ class McpTool {
     }
 }
 
-const TOOL_NAME$6 = "task_result_check";
-class TaskResultCheckTool {
+/**
+ * INPUT: global.skillService for skill metadata and content
+ * OUTPUT: Activated skill instructions + resource listing
+ * POSITION: Chat dialogue tool enabling LLM to load domain-specific skills
+ */
+const TOOL_NAME$6 = "activate_skill";
+/** Works as both DialogueTool (Chat) and Tool (Agent) */
+class ActivateSkillTool {
     constructor() {
         this.name = TOOL_NAME$6;
+        this.noPlan = true;
+    }
+    /** Dynamic description with available skill list */
+    get description() {
+        const skills = global.skillService?.getAllMetadata()?.filter((s) => s.enabled) || [];
+        if (skills.length === 0) {
+            return "Activate a specialized skill. No skills currently available.";
+        }
+        const list = skills.map((s) => `- ${s.name}: ${s.description}`).join("\n");
+        return `Activate a specialized skill that provides domain-specific instructions.\n\nAvailable skills:\n${list}`;
+    }
+    /** Dynamic parameters with enum constraint */
+    get parameters() {
+        const skillNames = (global.skillService?.getAllMetadata()?.filter((s) => s.enabled) || []).map((s) => s.name);
+        return {
+            type: "object",
+            properties: {
+                name: {
+                    type: "string",
+                    ...(skillNames.length > 0 ? { enum: skillNames } : {}),
+                    description: "Name of the skill to activate",
+                },
+            },
+            required: ["name"],
+        };
+    }
+    async execute(args, ..._rest) {
+        const name = args.name;
+        if (!global.skillService) {
+            return {
+                content: [{ type: "text", text: "Skill service not available" }],
+                isError: true,
+            };
+        }
+        const skill = await global.skillService.loadSkill(name);
+        if (!skill) {
+            return {
+                content: [{ type: "text", text: `Skill "${name}" not found` }],
+                isError: true,
+            };
+        }
+        const output = [
+            `<activated_skill name="${skill.metadata.name}">`,
+            `<instructions>`,
+            skill.instructions.trim(),
+            `</instructions>`,
+        ];
+        if (skill.resources.length > 0) {
+            output.push(`<available_resources base="${skill.basePath}">`);
+            skill.resources.forEach((r) => output.push(`  <file>${r}</file>`));
+            output.push(`</available_resources>`);
+        }
+        output.push(`</activated_skill>`);
+        return { content: [{ type: "text", text: output.join("\n") }] };
+    }
+}
+
+const TOOL_NAME$5 = "task_result_check";
+class TaskResultCheckTool {
+    constructor() {
+        this.name = TOOL_NAME$5;
         this.description = `Check the current task execution process and results, evaluate the overall completion status of the current task, and whether the output variables in the nodes are stored.`;
         this.parameters = {
             type: "object",
@@ -40249,10 +40430,10 @@ async function doTaskResultCheck(agentContext, rlm, messages, tools) {
     }
 }
 
-const TOOL_NAME$5 = "todo_list_manager";
+const TOOL_NAME$4 = "todo_list_manager";
 class TodoListManagerTool {
     constructor() {
-        this.name = TOOL_NAME$5;
+        this.name = TOOL_NAME$4;
         this.description =
             "Current task to-do list management, used for managing the to-do list of current tasks. During task execution, the to-do list needs to be updated according to the task execution status: completed, pending. It also detects whether tasks are being executed in repetitive loops during the execution process.";
         this.parameters = {
@@ -40405,6 +40586,14 @@ For repetitive tasks, when executing a forEach node, the \`${TOOL_NAME$b}\` tool
 * watch node
 monitor changes in webpage DOM elements, when executing to the watch node, require the use of the \`${TOOL_NAME$a}\` tool.
 </if>
+<if ${TOOL_NAME$6}Tool>
+* SKILLS
+You can use the \`${TOOL_NAME$6}\` tool to load domain-specific skill instructions when they would help complete the current task.
+</if>
+<if skills>
+Available skills:
+{{skills}}
+</if>
 
 <if mainTask>
 Main task: {{mainTask}}
@@ -40457,6 +40646,14 @@ function getAgentSystemPrompt(agent, agentNode, context, tools, extSysPrompt) {
     for (let i = 0; i < tools.length; i++) {
         toolVars[tools[i].name + "Tool"] = true;
     }
+    // Inject skill list when available
+    let _skills = "";
+    if (global.skillService) {
+        const skills = global.skillService.getAllMetadata().filter((s) => s.enabled);
+        if (skills.length > 0) {
+            _skills = skills.map((s) => `- ${s.name}: ${s.description}`).join("\n");
+        }
+    }
     let mainTask = "";
     let preTaskResult = "";
     if (context.chain.agents.length > 1) {
@@ -40469,6 +40666,7 @@ function getAgentSystemPrompt(agent, agentNode, context, tools, extSysPrompt) {
         agent: agent.Name,
         description: agent.Description,
         extSysPrompt: extSysPrompt?.trim() || "",
+        skills: _skills,
         mainTask: mainTask,
         preTaskResult: preTaskResult.trim(),
         hasWatchNode: agentNode.xml.indexOf("</watch>") > -1,
@@ -40722,6 +40920,13 @@ class Agent {
             callback?.onHumanHelp) {
             tools.push(new HumanInteractTool());
         }
+        // Add skill tool when available
+        if (global.skillService) {
+            const skills = global.skillService.getAllMetadata().filter((s) => s.enabled);
+            if (skills.length > 0) {
+                tools.push(new ActivateSkillTool());
+            }
+        }
         const toolNames = this.tools.map((tool) => tool.name);
         return tools.filter((tool) => toolNames.indexOf(tool.name) == -1);
     }
@@ -44244,7 +44449,7 @@ class ChatContext {
     }
 }
 
-const TOOL_NAME$4 = "webpageQa";
+const TOOL_NAME$3 = "webpageQa";
 const WEBPAGE_QA_PROMPT = `
 You are a helpful assistant that can answer questions based on the provided webpage context.
 
@@ -44265,7 +44470,7 @@ Answer user's question based on the webpage context, the answer should be in the
 `;
 class WebpageQaTool {
     constructor(chatContext, params) {
-        this.name = TOOL_NAME$4;
+        this.name = TOOL_NAME$3;
         this.params = params;
         this.chatContext = chatContext;
         this.description = `This tool is designed only for handling simple web-related tasks, including summarizing webpage content, extracting data from web pages, translating webpage content, and converting webpage information into more easily understandable forms. It does not interact with or operate web pages. For more complex browser tasks, please use deepAction.It does not perform operations on the webpage itself, but only involves reading the page content. Users do not need to provide the web page content, as the tool can automatically extract the content of the web page based on the tabId to respond.`;
@@ -44381,10 +44586,10 @@ class WebpageQaTool {
     }
 }
 
-const TOOL_NAME$3 = "webSearch";
+const TOOL_NAME$2 = "webSearch";
 class WebSearchTool {
     constructor(chatContext, params) {
-        this.name = TOOL_NAME$3;
+        this.name = TOOL_NAME$2;
         this.params = params;
         this.chatContext = chatContext;
         this.description = `Search the web for information using search engine API. This tool can perform web searches to find current information, news, articles, and other web content related to the query. It returns search results with titles, descriptions, URLs, and other relevant metadata, use this tool when users need the latest data/information and have NOT specified a particular platform or website, use the search tool.`;
@@ -44454,12 +44659,12 @@ async function recursiveTextNode(node, callback) {
     }
 }
 
-const TOOL_NAME$2 = "deepAction";
+const TOOL_NAME$1 = "deepAction";
 const deep_action_description = "Delegate tasks to a Javis AI assistant for completion. This assistant can understand natural language instructions and has full control over both networked computers, browser agent, and multiple specialized agents ({agentNames}). The assistant can autonomously decide to use various software tools, browse the internet to query information, write code, and perform direct operations to complete tasks. He can deliver various digitized outputs (text reports, tables, images, music, videos, websites, deepSearch, programs, etc.) and handle design/analysis tasks. and execute operational tasks (such as batch following bloggers of specific topics on certain websites). For operational tasks, the focus is on completing the process actions rather than delivering final outputs, and the assistant can complete these types of tasks well. It should also be noted that users may actively mention deepsearch, which is also one of the capabilities of this tool. If users mention it, please explicitly tell the assistant to use deepsearch. Supports parallel execution of multiple tasks.";
 const deep_action_param_task_description = "Task description, please output the user's original instructions without omitting any information from the user's instructions, and use the same language as the user's question.";
 class DeepActionTool {
     constructor(chatContext, params) {
-        this.name = TOOL_NAME$2;
+        this.name = TOOL_NAME$1;
         this.chatContext = chatContext;
         const agents = this.chatContext.getConfig().agents || [];
         const agentNames = agents.map((agent) => agent.Name).join(", ");
@@ -44616,10 +44821,10 @@ class DeepActionTool {
     }
 }
 
-const TOOL_NAME$1 = "taskVariableStorage";
+const TOOL_NAME = "taskVariableStorage";
 class TaskVariableStorageTool {
     constructor(chatContext, params) {
-        this.name = TOOL_NAME$1;
+        this.name = TOOL_NAME;
         this.params = params;
         this.chatContext = chatContext;
         this.description = `Used for storing, reading, and retrieving variable data, and maintaining input/output variables in task nodes.`;
@@ -44696,71 +44901,6 @@ class TaskVariableStorageTool {
     }
 }
 
-/**
- * INPUT: global.skillService for skill metadata and content
- * OUTPUT: Activated skill instructions + resource listing
- * POSITION: Chat dialogue tool enabling LLM to load domain-specific skills
- */
-const TOOL_NAME = "activate_skill";
-class ActivateSkillTool {
-    constructor() {
-        this.name = TOOL_NAME;
-    }
-    /** Dynamic description with available skill list */
-    get description() {
-        const skills = global.skillService?.getAllMetadata()?.filter((s) => s.enabled) || [];
-        if (skills.length === 0) {
-            return "Activate a specialized skill. No skills currently available.";
-        }
-        const list = skills.map((s) => `- ${s.name}: ${s.description}`).join("\n");
-        return `Activate a specialized skill that provides domain-specific instructions.\n\nAvailable skills:\n${list}`;
-    }
-    /** Dynamic parameters with enum constraint */
-    get parameters() {
-        const skillNames = (global.skillService?.getAllMetadata()?.filter((s) => s.enabled) || []).map((s) => s.name);
-        return {
-            type: "object",
-            properties: {
-                name: {
-                    type: "string",
-                    ...(skillNames.length > 0 ? { enum: skillNames } : {}),
-                    description: "Name of the skill to activate",
-                },
-            },
-            required: ["name"],
-        };
-    }
-    async execute(args) {
-        const name = args.name;
-        if (!global.skillService) {
-            return {
-                content: [{ type: "text", text: "Skill service not available" }],
-                isError: true,
-            };
-        }
-        const skill = await global.skillService.loadSkill(name);
-        if (!skill) {
-            return {
-                content: [{ type: "text", text: `Skill "${name}" not found` }],
-                isError: true,
-            };
-        }
-        const output = [
-            `<activated_skill name="${skill.metadata.name}">`,
-            `<instructions>`,
-            skill.instructions.trim(),
-            `</instructions>`,
-        ];
-        if (skill.resources.length > 0) {
-            output.push(`<available_resources base="${skill.basePath}">`);
-            skill.resources.forEach((r) => output.push(`  <file>${r}</file>`));
-            output.push(`</available_resources>`);
-        }
-        output.push(`</activated_skill>`);
-        return { content: [{ type: "text", text: output.join("\n") }] };
-    }
-}
-
 const CHAT_SYSTEM_TEMPLATE = `
 You are {{name}}, it is an action-oriented assistant in the browser, a general-purpose intelligent agent running in the browser environment.
 
@@ -44771,26 +44911,27 @@ General Principles:
 - Users may switch topics multiple times during ongoing conversations. When calling tools, assistant must focus ONLY on the current user question and ignore previous conversation topics unless they are directly related to the current request. Each question should be treated as independent unless explicitly building on previous context.
 
 For non-chat related tasks issued by users, the following tools need to be called to complete them:
-<if ${TOOL_NAME$2}Tool>
-- ${TOOL_NAME$2}: This tool is used to execute tasks, delegate to Javis AI assistant with full computer control.
-</if>
-<if ${TOOL_NAME$4}Tool>
-- ${TOOL_NAME$4}: When a user's query involves finding content in a webpage within a browser tab, extracting webpage content, summarizing webpage content, translating webpage content, read PDF page content, or converting webpage content into a more understandable format, this tool should be used. If the task requires performing actions based on webpage content, deepAction should be used. only needs to provide the required invocation parameters according to the tool's needs; users do not need to manually provide the content of the browser tab.
+<if ${TOOL_NAME$1}Tool>
+- ${TOOL_NAME$1}: This tool is used to execute tasks, delegate to Javis AI assistant with full computer control.
 </if>
 <if ${TOOL_NAME$3}Tool>
-- ${TOOL_NAME$3}: Search the web for information using search engine API. This tool can perform web searches to find current information, news, articles, and other web content related to the query. It returns search results with titles, descriptions, URLs, and other relevant metadata. Use this tool when you need to find current information from the internet that may not be available in your training data.
+- ${TOOL_NAME$3}: When a user's query involves finding content in a webpage within a browser tab, extracting webpage content, summarizing webpage content, translating webpage content, read PDF page content, or converting webpage content into a more understandable format, this tool should be used. If the task requires performing actions based on webpage content, deepAction should be used. only needs to provide the required invocation parameters according to the tool's needs; users do not need to manually provide the content of the browser tab.
 </if>
-<if ${TOOL_NAME$1}Tool>
-- ${TOOL_NAME$1}: This tool is used to read output variables from task nodes and write input variables to task nodes, mainly used to retrieve variable results after task execution is completed.
+<if ${TOOL_NAME$2}Tool>
+- ${TOOL_NAME$2}: Search the web for information using search engine API. This tool can perform web searches to find current information, news, articles, and other web content related to the query. It returns search results with titles, descriptions, URLs, and other relevant metadata. Use this tool when you need to find current information from the internet that may not be available in your training data.
 </if>
 <if ${TOOL_NAME}Tool>
-- ${TOOL_NAME}: Activate a specialized skill for domain-specific tasks. Use when the user's request matches an available skill.
+- ${TOOL_NAME}: This tool is used to read output variables from task nodes and write input variables to task nodes, mainly used to retrieve variable results after task execution is completed.
+</if>
+<if ${TOOL_NAME$6}Tool>
+- ${TOOL_NAME$6}: Activate a specialized skill for domain-specific tasks. Use when the user's request matches an available skill.
 </if>
 </tool_instructions>
 
 <if skills>
 ## Available Skills
 You have access to specialized skills. Use activate_skill when the user's request matches a skill.
+When you see <use_skill name="..." /> in user messages, you MUST call activate_skill with that skill name immediately before proceeding.
 <available_skills>
 {{skills}}
 </available_skills>