@jarvis-agent/core 0.2.1 → 0.2.3

package/dist/index.esm.js

@@ -39433,12 +39433,13 @@ async function callLLM(rlm, request, streamCallback, errorHandler, finishHandler
                 await errorHandler(request, e, retryNum);
             }
             Log.warn("callLLM abort error: ", e);
-            return streamText
-                ? [
-                    { type: "text", text: streamText },
-                    ...toolParts,
-                ]
-                : toolParts;
+            const parts = [];
+            if (thinkText)
+                parts.push({ type: "reasoning", text: thinkText });
+            if (streamText)
+                parts.push({ type: "text", text: streamText });
+            parts.push(...toolParts);
+            return parts;
         }
         else {
             if (retryNum < config$1.maxRetryNum) {
@@ -39454,30 +39455,34 @@ async function callLLM(rlm, request, streamCallback, errorHandler, finishHandler
     finally {
         reader && reader.releaseLock();
     }
-    return streamText
-        ? [
-            { type: "text", text: streamText },
-            ...toolParts,
-        ]
-        : toolParts;
+    const resultParts = [];
+    if (thinkText)
+        resultParts.push({ type: "reasoning", text: thinkText });
+    if (streamText)
+        resultParts.push({ type: "text", text: streamText });
+    resultParts.push(...toolParts);
+    return resultParts;
 }
 function convertAssistantContent(assistantParts) {
     return assistantParts
-        .filter((part) => part.type == "text" || part.type == "tool-call")
-        .map((part) => part.type === "text"
-        ? {
-            type: "text",
-            text: part.text,
+        .filter((part) => part.type === "text" || part.type === "reasoning" || part.type === "tool-call")
+        .map((part) => {
+        if (part.type === "text") {
+            return { type: "text", text: part.text };
+        }
+        if (part.type === "reasoning") {
+            return { type: "reasoning", text: part.text };
         }
-        : {
+        return {
             type: "tool-call",
             toolCallId: part.toolCallId,
             toolName: part.toolName,
-            input: typeof part.input == "string"
+            input: typeof part.input === "string"
                 ? JSON.parse(part.input || "{}")
                 : part.input || {},
             providerOptions: part.providerOptions,
-        });
+        };
+    });
 }
 
 const global = {
@@ -41599,8 +41604,8 @@ class Agent {
         if (results.length == 0) {
             return null;
         }
-        if (results.every((s) => s.type == "text")) {
-            return results.map((s) => s.text).join("\n\n");
+        if (results.every((s) => s.type === "text" || s.type === "reasoning")) {
+            return results.filter((s) => s.type === "text").map((s) => s.text).join("\n\n");
         }
         const toolCalls = results.filter((s) => s.type == "tool-call");
         if (toolCalls.length > 1 &&
@@ -42522,30 +42527,19 @@ function run_build_dom_tree() {
     /**
      * Get clickable elements on the page
      *
-     * @param {*} markHighlightElements Is mark highlighted
+     * @param {*} doHighlightElements Is highlighted
      * @param {*} includeAttributes [attr_names...]
-     * @returns { element_str, client_rect, selector_map, area_map }
+     * @returns { element_str, selector_map }
      */
-    function get_clickable_elements(markHighlightElements = true, includeAttributes) {
+    function get_clickable_elements(doHighlightElements = true, includeAttributes) {
         window.clickable_elements = {};
         computedStyleCache = new WeakMap();
         document.querySelectorAll("[eko-user-highlight-id]").forEach(ele => ele.removeAttribute("eko-user-highlight-id"));
-        let page_tree = build_dom_tree(markHighlightElements);
+        let page_tree = build_dom_tree(doHighlightElements);
         let element_tree = parse_node(page_tree);
+        let selector_map = create_selector_map(element_tree);
         let element_str = clickable_elements_to_string(element_tree, includeAttributes);
-        let client_rect = {
-            width: window.innerWidth || document.documentElement.clientWidth,
-            height: window.innerHeight || document.documentElement.clientHeight,
-        };
-        if (markHighlightElements) {
-            let selector_map = {};
-            // selector_map = create_selector_map(element_tree);
-            return { element_str, client_rect, selector_map };
-        }
-        else {
-            let area_map = create_area_map(element_tree);
-            return { element_str, client_rect, area_map };
-        }
+        return { element_str, selector_map };
     }
     function get_highlight_element(highlightIndex) {
         let element = document.querySelector(`[eko-user-highlight-id="eko-highlight-${highlightIndex}"]`);
@@ -42644,13 +42638,12 @@ function run_build_dom_tree() {
         process_node(element_tree);
         return formatted_text.join('\n');
     }
-    function create_area_map(element_tree) {
-        let area_map = {};
+    function create_selector_map(element_tree) {
+        let selector_map = {};
         function process_node(node) {
             if (node.tagName) {
                 if (node.highlightIndex != null) {
-                    const element = window.clickable_elements[node.highlightIndex];
-                    area_map[node.highlightIndex] = get_element_real_bounding_rect(element);
+                    selector_map[node.highlightIndex] = node;
                 }
                 for (let i = 0; i < node.children.length; i++) {
                     process_node(node.children[i]);
@@ -42658,38 +42651,7 @@ function run_build_dom_tree() {
             }
         }
         process_node(element_tree);
-        return area_map;
-    }
-    function get_element_real_bounding_rect(element) {
-        if (!element || !(element instanceof Element)) {
-            return { x: 0, y: 0, width: 0, height: 0 };
-        }
-        let rect = element.getBoundingClientRect();
-        let x = rect.left;
-        let y = rect.top;
-        let width = rect.width;
-        let height = rect.height;
-        let win = element.ownerDocument.defaultView;
-        let maxDepth = 10;
-        let depth = 0;
-        while (win && win !== win.parent && depth < maxDepth) {
-            depth++;
-            const frameElement = win.frameElement;
-            if (!frameElement) {
-                break;
-            }
-            const frameRect = frameElement.getBoundingClientRect();
-            x += frameRect.left;
-            y += frameRect.top;
-            // Consider the border and padding of the iframe.
-            const frameStyle = getCachedComputedStyle(frameElement);
-            x += parseFloat(frameStyle.borderLeftWidth) || 0;
-            y += parseFloat(frameStyle.borderTopWidth) || 0;
-            x += parseFloat(frameStyle.paddingLeft) || 0;
-            y += parseFloat(frameStyle.paddingTop) || 0;
-            win = win.parent;
-        }
-        return { x, y, width, height };
+        return selector_map;
     }
     function parse_node(node_data, parent) {
         if (!node_data) {
@@ -42729,9 +42691,8 @@ function run_build_dom_tree() {
         }
         return element_node;
     }
-    function build_dom_tree(markHighlightElements) {
+    function build_dom_tree(doHighlightElements) {
         let highlightIndex = 0; // Reset highlight index
-        let duplicates = new Set();
         function highlightElement(element, index, parentIframe = null) {
             // Create or get highlight container
             let container = document.getElementById('eko-highlight-container');
@@ -42930,36 +42891,13 @@ function run_build_dom_tree() {
                 interactiveRoles.has(ariaRole) ||
                 (tabIndex !== null && tabIndex !== '-1') ||
                 element.getAttribute('data-action') === 'a-dropdown-select' ||
-                element.getAttribute('data-action') === 'a-dropdown-button' ||
-                element.getAttribute('contenteditable') === 'true';
+                element.getAttribute('data-action') === 'a-dropdown-button';
             if (hasInteractiveRole)
                 return true;
-            // const eventTypes = [
-            //   'click',
-            //   'mousedown',
-            //   'mouseup',
-            //   'touchstart',
-            //   'touchend',
-            //   'keydown',
-            //   'keyup',
-            //   'focus',
-            //   'blur',
-            // ];
-            const clickEventTypes = [
-                'click',
-                'mousedown',
-                'mouseup',
-                'touchstart',
-                'touchend',
-            ];
-            // Filter elements that have no real event listeners at all
-            if (window.getEventListeners) {
-                const listeners = window.getEventListeners(element);
-                const hasRealClickListeners = clickEventTypes.some((type) => listeners[type]?.length > 0);
-                if (!hasRealClickListeners) {
-                    return false;
-                }
-            }
+            // Get computed style
+            const style = getCachedComputedStyle(element);
+            // Check if element has click-like styling
+            const hasClickStyling = style.cursor === 'pointer' || element.style.cursor === 'pointer';
             // Check for event listeners
             const hasClickHandler = element.onclick !== null ||
                 element.getAttribute('onclick') !== null ||
@@ -42967,10 +42905,24 @@ function run_build_dom_tree() {
                 element.hasAttribute('@click') ||
                 element.hasAttribute('v-on:click');
             // Helper function to safely get event listeners
-            function getElementEventListeners(el) {
+            function getEventListeners(el) {
+                // if (window.getEventListeners) {
+                //   return window.getEventListeners?.(el) || {};
+                // }
                 // List of common event types to check
                 const listeners = {};
-                for (const type of clickEventTypes) {
+                const eventTypes = [
+                    'click',
+                    'mousedown',
+                    'mouseup',
+                    'touchstart',
+                    'touchend',
+                    'keydown',
+                    'keyup',
+                    'focus',
+                    'blur',
+                ];
+                for (const type of eventTypes) {
                     const handler = el[`on${type}`];
                     if (handler) {
                         listeners[type] = [
@@ -42984,34 +42936,30 @@ function run_build_dom_tree() {
                 return listeners;
             }
             // Check for click-related events on the element itself
-            const listeners = getElementEventListeners(element);
-            const hasClickListeners = clickEventTypes.some((type) => listeners[type]?.length > 0);
+            const listeners = getEventListeners(element);
+            const hasClickListeners = listeners &&
+                (listeners.click?.length > 0 ||
+                    listeners.mousedown?.length > 0 ||
+                    listeners.mouseup?.length > 0 ||
+                    listeners.touchstart?.length > 0 ||
+                    listeners.touchend?.length > 0);
             // Check for ARIA properties that suggest interactivity
             const hasAriaProps = element.hasAttribute('aria-expanded') ||
                 element.hasAttribute('aria-pressed') ||
                 element.hasAttribute('aria-selected') ||
                 element.hasAttribute('aria-checked');
+            // Check for form-related functionality
+            element.form !== undefined ||
+                element.hasAttribute('contenteditable') ||
+                (style && style.userSelect !== 'none');
             // Check if element is draggable
             const isDraggable = element.draggable || element.getAttribute('draggable') === 'true';
-            if (hasAriaProps || hasClickHandler || hasClickListeners || isDraggable) {
-                return true;
-            }
-            // Check if element has click-like styling
-            let hasClickStyling = element.style.cursor === 'pointer' || getCachedComputedStyle(element).cursor === 'pointer';
-            if (hasClickStyling) {
-                let count = 0;
-                let current = element.parentElement;
-                while (current && current !== document.documentElement) {
-                    hasClickStyling = current.style.cursor === 'pointer' || getCachedComputedStyle(current).cursor === 'pointer';
-                    if (hasClickStyling)
-                        return false;
-                    current = current.parentElement;
-                    if (++count > 10)
-                        break;
-                }
-                return true;
-            }
-            return false;
+            return (hasAriaProps ||
+                hasClickStyling ||
+                hasClickHandler ||
+                hasClickListeners ||
+                // isFormRelated ||
+                isDraggable);
         }
         // Helper function to check if element is visible
         function isElementVisible(element) {
@@ -43041,14 +42989,11 @@ function run_build_dom_tree() {
                     if (!topEl)
                         return false;
                     // Check if the element or any of its parents match our target element
-                    let count = 0;
                     let current = topEl;
                     while (current && current !== shadowRoot) {
                         if (current === element)
                             return true;
                         current = current.parentElement;
-                        if (++count > 15)
-                            break;
                     }
                     return false;
                 }
@@ -43063,14 +43008,11 @@ function run_build_dom_tree() {
                 const topEl = document.elementFromPoint(point.x, point.y);
                 if (!topEl)
                     return false;
-                let count = 0;
                 let current = topEl;
                 while (current && current !== document.documentElement) {
                     if (current === element)
                         return true;
                     current = current.parentElement;
-                    if (++count > 15)
-                        break;
                 }
                 return false;
             }
@@ -43094,10 +43036,8 @@ function run_build_dom_tree() {
         }
         // Function to traverse the DOM and create nested JSON
         function buildDomTree(node, parentIframe = null) {
-            if (!node || duplicates.has(node)) {
+            if (!node)
                 return null;
-            }
-            duplicates.add(node);
             // Special case for text nodes
             if (node.nodeType === Node.TEXT_NODE) {
                 const textContent = node.textContent.trim();
@@ -43142,7 +43082,7 @@ function run_build_dom_tree() {
                 if (shouldHighlight) {
                     nodeData.highlightIndex = highlightIndex++;
                     window.clickable_elements[nodeData.highlightIndex] = node;
-                    if (markHighlightElements) {
+                    if (doHighlightElements) {
                         highlightElement(node, nodeData.highlightIndex, parentIframe);
                     }
                 }
@@ -43157,7 +43097,7 @@ function run_build_dom_tree() {
             }
             // Handle shadow DOM
             if (node.shadowRoot) {
-                const shadowChildren = Array.from(node.shadowRoot.children).map((child) => buildDomTree(child, parentIframe)).filter(child => child !== null);
+                const shadowChildren = Array.from(node.shadowRoot.childNodes).map((child) => buildDomTree(child, parentIframe));
                 nodeData.children.push(...shadowChildren);
             }
             // Handle iframes
@@ -43165,7 +43105,7 @@ function run_build_dom_tree() {
                 try {
                     const iframeDoc = node.contentDocument || node.contentWindow.document;
                     if (iframeDoc) {
-                        const iframeChildren = Array.from(iframeDoc.body.children).map((child) => buildDomTree(child, node)).filter(child => child !== null);
+                        const iframeChildren = Array.from(iframeDoc.body.childNodes).map((child) => buildDomTree(child, node));
                         nodeData.children.push(...iframeChildren);
                     }
                 }
@@ -43174,9 +43114,8 @@ function run_build_dom_tree() {
                 }
             }
             else {
-                const style = getCachedComputedStyle(node);
-                if (style && style.display !== 'none') {
-                    const children = Array.from(node.children).map((child) => buildDomTree(child, parentIframe)).filter(child => child !== null);
+                if (nodeData.isVisible != false) {
+                    const children = Array.from(node.childNodes).map((child) => buildDomTree(child, parentIframe));
                     nodeData.children.push(...children);
                 }
             }
@@ -45222,7 +45161,7 @@ async function callChatLLM(chatId, messageId, rlm, messages, tools, toolChoice,
     });
 }
 function convertAssistantToolResults(results) {
-    return results.map((part) => {
+    return results.filter((part) => part.type !== "reasoning").map((part) => {
         if (part.type == "text") {
             return {
                 type: "text",
@@ -45972,10 +45911,13 @@ class ChatAgent {
         }
         for (let i = 0; i < results.length; i++) {
             const result = results[i];
-            if (result.type == "text") {
+            if (result.type === "text") {
                 text = result.text;
                 continue;
             }
+            if (result.type === "reasoning") {
+                continue;
+            }
             let toolResult;
             try {
                 const args = typeof result.input == "string"