@eko-ai/eko 1.0.1

package/dist/extension_content_script.js

@@ -0,0 +1,247 @@
+if (!window.eko) {
+    window.eko = { lastMouseX: 0, lastMouseY: 0 };
+}
+eko.sub = function (event, callback) {
+    if (!eko.subListeners) {
+        eko.subListeners = {};
+    }
+    if (event && callback) {
+        eko.subListeners[event] = callback;
+    }
+};
+document.addEventListener('mousemove', (event) => {
+    eko.lastMouseX = event.clientX;
+    eko.lastMouseY = event.clientY;
+});
+chrome.runtime.onMessage.addListener(function (request, sender, sendResponse) {
+    (async () => {
+        try {
+            switch (request.type) {
+                case 'eko:message': {
+                    let result = null;
+                    if (eko.subListeners && eko.subListeners[request.event]) {
+                        try {
+                            result = await eko.subListeners[request.event](request.params);
+                        }
+                        catch (e) {
+                            console.log(e);
+                        }
+                    }
+                    sendResponse(result);
+                    break;
+                }
+                case 'page:getDetailLinks': {
+                    let result = await eko.getDetailLinks(request.search);
+                    sendResponse(result);
+                    break;
+                }
+                case 'page:getContent': {
+                    let result = await eko.getContent(request.search);
+                    sendResponse(result);
+                    break;
+                }
+                case 'computer:key': {
+                    sendResponse(key(request));
+                    break;
+                }
+                case 'computer:type': {
+                    sendResponse(type(request));
+                    break;
+                }
+                case 'computer:mouse_move': {
+                    sendResponse(mouse_move(request));
+                    break;
+                }
+                case 'computer:left_click': {
+                    sendResponse(simulateMouseEvent(request, ['mousedown', 'mouseup', 'click'], 0));
+                    break;
+                }
+                case 'computer:right_click': {
+                    sendResponse(simulateMouseEvent(request, ['mousedown', 'mouseup', 'contextmenu'], 2));
+                    break;
+                }
+                case 'computer:double_click': {
+                    sendResponse(simulateMouseEvent(request, ['mousedown', 'mouseup', 'click', 'mousedown', 'mouseup', 'click', 'dblclick'], 0));
+                    break;
+                }
+                case 'computer:left_click_drag': {
+                    sendResponse(left_click_drag(request));
+                    break;
+                }
+                case 'computer:scroll_to': {
+                    sendResponse(scroll_to(request));
+                    break;
+                }
+                case 'computer:cursor_position': {
+                    sendResponse({ coordinate: [eko.lastMouseX, eko.lastMouseY] });
+                    break;
+                }
+            }
+        }
+        catch (e) {
+            console.log('onMessage error', e);
+            sendResponse(false);
+        }
+    })();
+    return true;
+});
+function key(request) {
+    const event = new KeyboardEvent(request.keyEventType || 'keydown', {
+        key: request.key,
+        ctrlKey: request.ctrlKey,
+        altKey: request.altKey,
+        shiftKey: request.shiftKey,
+        metaKey: request.metaKey,
+        bubbles: true,
+        cancelable: true,
+    });
+    let coordinate = request.coordinate;
+    let element = (document.activeElement ||
+        document.elementFromPoint(coordinate[0], coordinate[1]));
+    if (element && element.focus) {
+        element.focus();
+    }
+    let result = element === null || element === void 0 ? void 0 : element.dispatchEvent(event);
+    console.log('key', element, request, result);
+    return result;
+}
+function type(request) {
+    let text = request.text;
+    let element;
+    if (request.xpath) {
+        let xpath = request.xpath;
+        let result = document.evaluate(xpath, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null);
+        element = result.singleNodeValue;
+    }
+    else {
+        let coordinate = request.coordinate;
+        element = document.elementFromPoint(coordinate[0], coordinate[1]) || document.activeElement;
+    }
+    if (!element) {
+        return;
+    }
+    let input;
+    if (element.tagName == 'INPUT' ||
+        element.tagName == 'TEXTAREA' ||
+        element.childElementCount == 0) {
+        input = element;
+    }
+    else {
+        input = element.querySelector('input') || element.querySelector('textarea') || element;
+    }
+    input.focus && input.focus();
+    if (!text) {
+        input.value = '';
+    }
+    else {
+        input.value += text;
+    }
+    let result = input.dispatchEvent(new Event('input', { bubbles: true }));
+    console.log('type', input, request, result);
+    return result;
+}
+function mouse_move(request) {
+    let coordinate = request.coordinate;
+    let x = coordinate[0];
+    let y = coordinate[1];
+    const event = new MouseEvent('mousemove', {
+        view: window,
+        bubbles: true,
+        cancelable: true,
+        screenX: x,
+        screenY: y,
+        clientX: x,
+        clientY: y,
+    });
+    let result = document.body.dispatchEvent(event);
+    console.log('mouse_move', document.body, request, result);
+    return result;
+}
+function simulateMouseEvent(request, eventTypes, button) {
+    let element;
+    if (request.xpath) {
+        let xpath = request.xpath;
+        let result = document.evaluate(xpath, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null);
+        element = result.singleNodeValue;
+    }
+    else {
+        let coordinate = request.coordinate;
+        element = document.elementFromPoint(coordinate[0], coordinate[1]) || document.body;
+    }
+    const x = undefined;
+    const y = undefined;
+    let result = false;
+    for (let i = 0; i < eventTypes.length; i++) {
+        const event = new MouseEvent(eventTypes[i], {
+            view: window,
+            bubbles: true,
+            cancelable: true,
+            clientX: x,
+            clientY: y,
+            button, // 0 left; 2 right
+        });
+        result = element.dispatchEvent(event);
+        console.log('simulateMouse', element, { ...request, eventTypes, button }, result);
+    }
+    return result;
+}
+function scroll_to(request) {
+    if (request.xpath) {
+        let xpath = request.xpath;
+        let result = document.evaluate(xpath, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null);
+        let element = result.singleNodeValue;
+        element.scrollIntoView({
+            behavior: 'smooth'
+        });
+    }
+    else {
+        const to_coordinate = request.to_coordinate;
+        window.scrollTo({
+            left: to_coordinate[0],
+            top: to_coordinate[1],
+            behavior: 'smooth',
+        });
+    }
+    console.log('scroll_to', request);
+}
+function left_click_drag(request, steps = 10) {
+    const from_coordinate = request.from_coordinate;
+    const to_coordinate = request.to_coordinate;
+    let startX = from_coordinate[0];
+    let startY = from_coordinate[1];
+    let endX = to_coordinate[0];
+    let endY = to_coordinate[1];
+    let element = document.elementFromPoint(startX, startY) || document.body;
+    const mouseDownEvent = new MouseEvent('mousedown', {
+        bubbles: true,
+        cancelable: true,
+        view: window,
+        clientX: startX,
+        clientY: startY,
+        button: 0,
+    });
+    element.dispatchEvent(mouseDownEvent);
+    for (let i = 1; i <= steps; i++) {
+        const intermediateX = startX + (endX - startX) * (i / steps);
+        const intermediateY = startY + (endY - startY) * (i / steps);
+        const dragEvent = new MouseEvent('mousemove', {
+            bubbles: true,
+            cancelable: true,
+            view: window,
+            clientX: intermediateX,
+            clientY: intermediateY,
+            button: 0,
+        });
+        element.dispatchEvent(dragEvent);
+    }
+    const mouseUpEvent = new MouseEvent('mouseup', {
+        bubbles: true,
+        cancelable: true,
+        view: window,
+        clientX: endX,
+        clientY: endY,
+        button: 0,
+    });
+    console.log('left_click_drag', request);
+    return element.dispatchEvent(mouseUpEvent);
+}

package/dist/fellou/computer.d.ts

@@ -0,0 +1,20 @@
+export declare function can_use_computer(): Promise<boolean>;
+export declare function key(key: string, coordinate?: [number, number]): Promise<boolean>;
+export declare function type(text: string, coordinate?: [number, number]): Promise<boolean>;
+export declare function mouse_move(coordinate: [number, number]): Promise<boolean>;
+export declare function left_click(coordinate?: [number, number]): Promise<boolean>;
+export declare function left_click_drag(coordinate: [number, number]): Promise<boolean>;
+export declare function right_click(coordinate?: [number, number]): Promise<boolean>;
+export declare function double_click(coordinate?: [number, number]): Promise<boolean>;
+export declare function screenshot(windowId?: number): Promise<{
+    image: {
+        type: 'base64';
+        media_type: 'image/png' | 'image/jpeg';
+        data: string;
+    };
+}>;
+export declare function cursor_position(): Promise<{
+    coordinate: [number, number];
+}>;
+export declare function size(): Promise<[number, number]>;
+export declare function scroll(coordinate: [number, number]): Promise<boolean>;

package/dist/fellou/index.d.ts

@@ -0,0 +1,6 @@
+import * as computer from './computer';
+import { ComputerUse } from './tools/computer_use';
+export { ComputerUse };
+export declare const fellou: {
+    computer: typeof computer;
+};

package/dist/fellou/tools/computer_use.d.ts

@@ -0,0 +1,18 @@
+import { ComputerUseParam, ComputerUseResult } from '../../types/tools.types';
+import { Tool, InputSchema, ExecutionContext } from '../../types/action.types';
+/**
+ * Computer Use for fellou
+ */
+export declare class ComputerUse implements Tool<ComputerUseParam, ComputerUseResult> {
+    name: string;
+    description: string;
+    input_schema: InputSchema;
+    constructor();
+    /**
+     * computer
+     *
+     * @param {*} params { action: 'mouse_move', coordinate: [100, 200] }
+     * @returns { success: true, coordinate?: [], image?: { type: 'base64', media_type: 'image/jpeg', data: '/9j...' } }
+     */
+    execute(context: ExecutionContext, params: ComputerUseParam): Promise<ComputerUseResult>;
+}

package/dist/fellou/tools/index.d.ts

@@ -0,0 +1,2 @@
+import { ComputerUse } from './computer_use';
+export { ComputerUse };

package/dist/fellou.cjs.js

@@ -0,0 +1,238 @@
+'use strict';
+
+async function can_use_computer() {
+    try {
+        await size();
+        return true;
+    }
+    catch (e) {
+        return false;
+    }
+}
+async function key(key, coordinate) {
+    if (coordinate) {
+        await mouse_move(coordinate);
+    }
+    let mapping = {
+        space: ' ',
+        return: 'enter',
+        page_up: 'pageup',
+        page_down: 'pagedown',
+        back_space: 'backspace',
+    };
+    let keys = key.replace(/\s+/g, ' ').split(' ');
+    let success = false;
+    for (let i = 0; i < keys.length; i++) {
+        let _key = keys[i];
+        if (_key.indexOf('+') > -1) {
+            let mapped_keys = _key
+                .split('+')
+                .map((k) => mapping[k] || k)
+                .reverse();
+            success = (await runComputeruseCommand('keyTap', mapped_keys)).success;
+        }
+        else {
+            let mapped_key = mapping[_key] || _key;
+            success = (await runComputeruseCommand('keyTap', [mapped_key])).success;
+        }
+        await new Promise((resolve) => setTimeout(() => resolve(), 100));
+    }
+    return success;
+}
+async function type(text, coordinate) {
+    if (coordinate) {
+        await mouse_move(coordinate);
+    }
+    return (await runComputeruseCommand('typeString', [text])).success;
+}
+async function mouse_move(coordinate) {
+    return (await runComputeruseCommand('move', coordinate)).success;
+}
+async function left_click(coordinate) {
+    if (coordinate && coordinate.length > 0) {
+        await mouse_move(coordinate);
+    }
+    return (await runComputeruseCommand('click', ['left'])).success;
+}
+async function left_click_drag(coordinate) {
+    return (await runComputeruseCommand('dragSmooth', coordinate)).success;
+}
+async function right_click(coordinate) {
+    if (coordinate && coordinate.length > 0) {
+        await mouse_move(coordinate);
+    }
+    return (await runComputeruseCommand('click', ['right'])).success;
+}
+async function double_click(coordinate) {
+    if (coordinate && coordinate.length > 0) {
+        await mouse_move(coordinate);
+    }
+    return (await runComputeruseCommand('click', ['left', true])).success;
+}
+async function screenshot(windowId) {
+    let screenshot = (await runComputeruseCommand('captureFullScreen')).data;
+    let dataUrl = screenshot.startsWith('data:') ? screenshot : 'data:image/png;base64,' + screenshot;
+    let data = dataUrl.substring(dataUrl.indexOf('base64,') + 7);
+    return {
+        image: {
+            type: 'base64',
+            media_type: dataUrl.indexOf('image/png') > -1 ? 'image/png' : 'image/jpeg',
+            data: data,
+        },
+    };
+}
+async function cursor_position() {
+    let response = await runComputeruseCommand('mouseLocation');
+    return { coordinate: [response.data.x, response.data.y] };
+}
+async function size() {
+    let response = await runComputeruseCommand('getScreenSize');
+    return [response.data.width, response.data.height];
+}
+async function scroll(coordinate) {
+    return (await runComputeruseCommand('scrollTo', coordinate)).success;
+}
+async function runComputeruseCommand(func, args) {
+    let result = (await window.fellou.ai.computeruse.runCommand({
+        func,
+        args,
+    }));
+    if (result.error) {
+        // error: 'permission-error'
+        throw new Error(result.error);
+    }
+    return result;
+}
+
+var computer = /*#__PURE__*/Object.freeze({
+    __proto__: null,
+    can_use_computer: can_use_computer,
+    cursor_position: cursor_position,
+    double_click: double_click,
+    key: key,
+    left_click: left_click,
+    left_click_drag: left_click_drag,
+    mouse_move: mouse_move,
+    right_click: right_click,
+    screenshot: screenshot,
+    scroll: scroll,
+    size: size,
+    type: type
+});
+
+/**
+ * Computer Use for fellou
+ */
+class ComputerUse {
+    constructor() {
+        this.name = 'computer_use';
+        this.description = `Use a mouse and keyboard to interact with a computer, and take screenshots.
+* This is a browser GUI interface where you do not have access to the address bar or bookmarks. You must operate the browser using inputs like screenshots, mouse, keyboard, etc.
+* Some operations may take time to process, so you may need to wait and take successive screenshots to see the results of your actions. E.g. if you clicked submit button, but it didn't work, try taking another screenshot.
+* Whenever you intend to move the cursor to click on an element, you should consult a screenshot to determine the coordinates of the element before moving the cursor.
+* If you tried clicking on a button or link but it failed to load, even after waiting, try adjusting your cursor position so that the tip of the cursor visually falls on the element that you want to click.
+* Make sure to click any buttons, links, icons, etc with the cursor tip in the center of the element.`;
+        this.input_schema = {
+            type: 'object',
+            properties: {
+                action: {
+                    type: 'string',
+                    description: `The action to perform. The available actions are:
+* \`key\`: Press a key or key-combination on the keyboard.
+- This supports robotgo hotkey syntax.
+- Multiple keys are combined using the "+" symbol.
+- Examples: "a", "enter", "ctrl+s", "command+shift+a", "num0".
+* \`type\`: Type a string of text on the keyboard.
+* \`cursor_position\`: Get the current (x, y) pixel coordinate of the cursor on the screen.
+* \`mouse_move\`: Move the cursor to a specified (x, y) pixel coordinate on the screen.
+* \`left_click\`: Click the left mouse button.
+* \`left_click_drag\`: Click and drag the cursor to a specified (x, y) pixel coordinate on the screen.
+* \`right_click\`: Click the right mouse button.
+* \`double_click\`: Double-click the left mouse button.
+* \`screenshot\`: Take a screenshot of the screen.
+* \`scroll\`: Scroll to the specified (x, y) pixel coordinates on the screen.`,
+                    enum: [
+                        'key',
+                        'type',
+                        'mouse_move',
+                        'left_click',
+                        'left_click_drag',
+                        'right_click',
+                        'double_click',
+                        'screenshot',
+                        'cursor_position',
+                        'scroll',
+                    ],
+                },
+                coordinate: {
+                    type: 'array',
+                    description: '(x, y): The x (pixels from the left edge) and y (pixels from the top edge) coordinates to move the mouse to.',
+                },
+                text: {
+                    type: 'string',
+                    description: 'Required only by `action=type` and `action=key`',
+                },
+            },
+            required: ['action'],
+        };
+    }
+    /**
+     * computer
+     *
+     * @param {*} params { action: 'mouse_move', coordinate: [100, 200] }
+     * @returns { success: true, coordinate?: [], image?: { type: 'base64', media_type: 'image/jpeg', data: '/9j...' } }
+     */
+    async execute(context, params) {
+        if (params === null || !params.action) {
+            throw new Error('Invalid parameters. Expected an object with a "action" property.');
+        }
+        let result;
+        switch (params.action) {
+            case 'key':
+                result = await key(params.text, params.coordinate);
+                break;
+            case 'type':
+                result = await type(params.text, params.coordinate);
+                break;
+            case 'mouse_move':
+                result = await mouse_move(params.coordinate);
+                break;
+            case 'left_click':
+                result = await left_click(params.coordinate);
+                break;
+            case 'left_click_drag':
+                result = await left_click_drag(params.coordinate);
+                break;
+            case 'right_click':
+                result = await right_click(params.coordinate);
+                break;
+            case 'double_click':
+                result = await double_click(params.coordinate);
+                break;
+            case 'screenshot':
+                result = await screenshot();
+                break;
+            case 'cursor_position':
+                result = await cursor_position();
+                break;
+            case 'scroll':
+                result = await scroll(params.coordinate);
+                break;
+            default:
+                throw Error(`Invalid parameters. The "${params.action}" value is not included in the "action" enumeration.`);
+        }
+        if (typeof result == 'boolean') {
+            return { success: result };
+        }
+        else {
+            return { success: true, ...result };
+        }
+    }
+}
+
+const fellou = {
+    computer,
+};
+
+exports.ComputerUse = ComputerUse;
+exports.fellou = fellou;