@godscene/core 1.7.11

package/dist/lib/ai-model/service-caller/request-timeout.js

@@ -0,0 +1,93 @@
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.d = (exports1, definition)=>{
+        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
+            enumerable: true,
+            get: definition[key]
+        });
+    };
+})();
+(()=>{
+    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
+})();
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ("u" > typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+__webpack_require__.d(__webpack_exports__, {
+    AI_CALL_HARD_TIMEOUT_CODE: ()=>AI_CALL_HARD_TIMEOUT_CODE,
+    DEFAULT_AI_CALL_TIMEOUT_MS: ()=>DEFAULT_AI_CALL_TIMEOUT_MS,
+    buildRequestAbortSignal: ()=>buildRequestAbortSignal,
+    isHardTimeoutError: ()=>isHardTimeoutError,
+    resolveEffectiveTimeoutMs: ()=>resolveEffectiveTimeoutMs
+});
+const DEFAULT_AI_CALL_TIMEOUT_MS = 180000;
+const AI_CALL_HARD_TIMEOUT_CODE = 'AI_CALL_HARD_TIMEOUT';
+function resolveEffectiveTimeoutMs(modelConfig) {
+    const { timeout } = modelConfig;
+    if ('number' != typeof timeout) return DEFAULT_AI_CALL_TIMEOUT_MS;
+    if (timeout <= 0) return null;
+    return timeout;
+}
+function isHardTimeoutError(err) {
+    if (!err || 'object' != typeof err) return false;
+    const code = err.code;
+    if (code === AI_CALL_HARD_TIMEOUT_CODE) return true;
+    const cause = err.cause;
+    if (cause && 'object' == typeof cause && cause.code === AI_CALL_HARD_TIMEOUT_CODE) return true;
+    return false;
+}
+function buildRequestAbortSignal(timeoutMs, userSignal) {
+    const controller = new AbortController();
+    if (userSignal?.aborted) {
+        controller.abort(userSignal.reason);
+        return {
+            signal: controller.signal,
+            cleanup: ()=>{}
+        };
+    }
+    let timer;
+    if (null !== timeoutMs) {
+        timer = setTimeout(()=>{
+            const err = new Error(`AI call hard timeout after ${timeoutMs}ms (full request time exceeded)`);
+            err.code = AI_CALL_HARD_TIMEOUT_CODE;
+            controller.abort(err);
+        }, timeoutMs);
+        if ('function' == typeof timer.unref) timer.unref();
+    }
+    const onUserAbort = userSignal ? ()=>controller.abort(userSignal.reason) : void 0;
+    if (userSignal && onUserAbort) userSignal.addEventListener('abort', onUserAbort, {
+        once: true
+    });
+    return {
+        signal: controller.signal,
+        cleanup: ()=>{
+            if (timer) clearTimeout(timer);
+            if (userSignal && onUserAbort) userSignal.removeEventListener('abort', onUserAbort);
+        }
+    };
+}
+exports.AI_CALL_HARD_TIMEOUT_CODE = __webpack_exports__.AI_CALL_HARD_TIMEOUT_CODE;
+exports.DEFAULT_AI_CALL_TIMEOUT_MS = __webpack_exports__.DEFAULT_AI_CALL_TIMEOUT_MS;
+exports.buildRequestAbortSignal = __webpack_exports__.buildRequestAbortSignal;
+exports.isHardTimeoutError = __webpack_exports__.isHardTimeoutError;
+exports.resolveEffectiveTimeoutMs = __webpack_exports__.resolveEffectiveTimeoutMs;
+for(var __rspack_i in __webpack_exports__)if (-1 === [
+    "AI_CALL_HARD_TIMEOUT_CODE",
+    "DEFAULT_AI_CALL_TIMEOUT_MS",
+    "buildRequestAbortSignal",
+    "isHardTimeoutError",
+    "resolveEffectiveTimeoutMs"
+].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});

package/dist/lib/ai-model/ui-tars-planning.js

@@ -0,0 +1,281 @@
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.d = (exports1, definition)=>{
+        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
+            enumerable: true,
+            get: definition[key]
+        });
+    };
+})();
+(()=>{
+    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
+})();
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ("u" > typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+__webpack_require__.d(__webpack_exports__, {
+    uiTarsPlanning: ()=>uiTarsPlanning
+});
+const logger_namespaceObject = require("@godscene/shared/logger");
+const us_keyboard_layout_namespaceObject = require("@godscene/shared/us-keyboard-layout");
+const utils_namespaceObject = require("@godscene/shared/utils");
+const action_parser_namespaceObject = require("@ui-tars/action-parser");
+const ui_tars_planning_js_namespaceObject = require("./prompt/ui-tars-planning.js");
+const index_js_namespaceObject = require("./service-caller/index.js");
+const debug = (0, logger_namespaceObject.getDebug)('ui-tars-planning');
+const warnLog = (0, logger_namespaceObject.getDebug)('ui-tars-planning', {
+    console: true
+});
+const bboxSize = 10;
+const pointToBbox = (point, width, height)=>[
+        Math.round(Math.max(point.x - bboxSize / 2, 0)),
+        Math.round(Math.max(point.y - bboxSize / 2, 0)),
+        Math.round(Math.min(point.x + bboxSize / 2, width)),
+        Math.round(Math.min(point.y + bboxSize / 2, height))
+    ];
+async function uiTarsPlanning(userInstruction, options) {
+    const { conversationHistory, context, modelConfig, actionContext } = options;
+    const { uiTarsModelVersion } = modelConfig;
+    let instruction = userInstruction;
+    if (actionContext) instruction = `<high_priority_knowledge>${actionContext}</high_priority_knowledge>\n<user_instruction>${userInstruction}</user_instruction>`;
+    const systemPrompt = (0, ui_tars_planning_js_namespaceObject.getUiTarsPlanningPrompt)() + instruction;
+    const screenshotBase64 = context.screenshot.base64;
+    conversationHistory.append({
+        role: 'user',
+        content: [
+            {
+                type: 'image_url',
+                image_url: {
+                    url: screenshotBase64
+                }
+            }
+        ]
+    });
+    const res = await (0, index_js_namespaceObject.callAIWithStringResponse)([
+        {
+            role: 'user',
+            content: systemPrompt
+        },
+        ...conversationHistory.snapshot()
+    ], modelConfig, {
+        abortSignal: options.abortSignal
+    });
+    let convertedText;
+    let parsed;
+    try {
+        convertedText = convertBboxToCoordinates(res.content);
+        const { shotSize } = context;
+        const parseResult = (0, action_parser_namespaceObject.actionParser)({
+            prediction: convertedText,
+            factor: [
+                1000,
+                1000
+            ],
+            screenContext: {
+                width: shotSize.width,
+                height: shotSize.height
+            },
+            modelVer: uiTarsModelVersion
+        });
+        parsed = parseResult.parsed;
+    } catch (parseError) {
+        const errorMessage = parseError instanceof Error ? parseError.message : String(parseError);
+        throw new index_js_namespaceObject.AIResponseParseError(`Parse error: ${errorMessage}`, JSON.stringify(res.content, void 0, 2), res.usage);
+    }
+    const { shotSize } = context;
+    debug('ui-tars modelVer', uiTarsModelVersion, ', parsed', JSON.stringify(parsed));
+    const transformActions = [];
+    const unhandledActions = [];
+    let shouldContinue = true;
+    parsed.forEach((action)=>{
+        const actionType = (action.action_type || '').toLowerCase();
+        if ('click' === actionType) {
+            (0, utils_namespaceObject.assert)(action.action_inputs.start_box, 'start_box is required');
+            const point = getPoint(action.action_inputs.start_box, shotSize);
+            const locate = {
+                prompt: action.thought || '',
+                bbox: pointToBbox({
+                    x: point[0],
+                    y: point[1]
+                }, shotSize.width, shotSize.height)
+            };
+            transformActions.push({
+                type: 'Tap',
+                param: {
+                    locate: locate
+                }
+            });
+        } else if ('left_double' === actionType) {
+            (0, utils_namespaceObject.assert)(action.action_inputs.start_box, 'start_box is required');
+            const point = getPoint(action.action_inputs.start_box, shotSize);
+            const locate = {
+                prompt: action.thought || '',
+                bbox: pointToBbox({
+                    x: point[0],
+                    y: point[1]
+                }, shotSize.width, shotSize.height)
+            };
+            transformActions.push({
+                type: 'DoubleClick',
+                param: {
+                    locate: locate
+                },
+                thought: action.thought || ''
+            });
+        } else if ('right_single' === actionType) {
+            (0, utils_namespaceObject.assert)(action.action_inputs.start_box, 'start_box is required');
+            const point = getPoint(action.action_inputs.start_box, shotSize);
+            const locate = {
+                prompt: action.thought || '',
+                bbox: pointToBbox({
+                    x: point[0],
+                    y: point[1]
+                }, shotSize.width, shotSize.height)
+            };
+            transformActions.push({
+                type: 'RightClick',
+                param: {
+                    locate: locate
+                },
+                thought: action.thought || ''
+            });
+        } else if ('drag' === actionType) {
+            (0, utils_namespaceObject.assert)(action.action_inputs.start_box, 'start_box is required');
+            (0, utils_namespaceObject.assert)(action.action_inputs.end_box, 'end_box is required');
+            const startPoint = getPoint(action.action_inputs.start_box, shotSize);
+            const endPoint = getPoint(action.action_inputs.end_box, shotSize);
+            transformActions.push({
+                type: 'DragAndDrop',
+                param: {
+                    from: {
+                        prompt: action.thought || '',
+                        bbox: pointToBbox({
+                            x: startPoint[0],
+                            y: startPoint[1]
+                        }, shotSize.width, shotSize.height)
+                    },
+                    to: {
+                        prompt: action.thought || '',
+                        bbox: pointToBbox({
+                            x: endPoint[0],
+                            y: endPoint[1]
+                        }, shotSize.width, shotSize.height)
+                    }
+                },
+                thought: action.thought || ''
+            });
+        } else if ('type' === actionType) transformActions.push({
+            type: 'Input',
+            param: {
+                value: action.action_inputs.content
+            },
+            thought: action.thought || ''
+        });
+        else if ('scroll' === actionType) transformActions.push({
+            type: 'Scroll',
+            param: {
+                direction: action.action_inputs.direction
+            },
+            thought: action.thought || ''
+        });
+        else if ('finished' === actionType) {
+            shouldContinue = false;
+            transformActions.push({
+                type: 'Finished',
+                param: {},
+                thought: action.thought || ''
+            });
+        } else if ('hotkey' === actionType) if (action.action_inputs.key) {
+            const keys = (0, us_keyboard_layout_namespaceObject.transformHotkeyInput)(action.action_inputs.key);
+            transformActions.push({
+                type: 'KeyboardPress',
+                param: {
+                    keyName: keys.join('+')
+                },
+                thought: action.thought || ''
+            });
+        } else warnLog('No key found in action: hotkey. Will not perform action.');
+        else if ('wait' === actionType) transformActions.push({
+            type: 'Sleep',
+            param: {
+                timeMs: 1000
+            },
+            thought: action.thought || ''
+        });
+        else if (actionType) {
+            unhandledActions.push({
+                type: actionType,
+                thought: action.thought || ''
+            });
+            debug('Unhandled action type:', actionType, 'thought:', action.thought);
+        }
+    });
+    if (0 === transformActions.length) {
+        const errorDetails = [];
+        if (0 === parsed.length) {
+            errorDetails.push('Action parser returned no actions');
+            if (res.content.includes('Thought:') && !res.content.includes('Action:')) errorDetails.push('Response contains "Thought:" but missing "Action:" line');
+            else errorDetails.push('Response may be malformed or empty');
+        }
+        if (unhandledActions.length > 0) {
+            const types = unhandledActions.map((a)=>a.type).join(', ');
+            errorDetails.push(`Unhandled action types: ${types}`);
+        }
+        const errorMessage = [
+            'No actions found in UI-TARS response.',
+            ...errorDetails
+        ].join('\n');
+        throw new index_js_namespaceObject.AIResponseParseError(errorMessage, JSON.stringify(res.content, void 0, 2), res.usage);
+    }
+    debug('transformActions', JSON.stringify(transformActions, null, 2));
+    const log = (0, ui_tars_planning_js_namespaceObject.getSummary)(res.content);
+    conversationHistory.append({
+        role: 'assistant',
+        content: log
+    });
+    return {
+        actions: transformActions,
+        log,
+        usage: res.usage,
+        rawResponse: JSON.stringify(res.content, void 0, 2),
+        shouldContinuePlanning: shouldContinue
+    };
+}
+function convertBboxToCoordinates(text) {
+    const pattern = /<bbox>(\d+)\s+(\d+)\s+(\d+)\s+(\d+)<\/bbox>/g;
+    function replaceMatch(match, x1, y1, x2, y2) {
+        const x1Num = Number.parseInt(x1, 10);
+        const y1Num = Number.parseInt(y1, 10);
+        const x2Num = Number.parseInt(x2, 10);
+        const y2Num = Number.parseInt(y2, 10);
+        const x = Math.floor((x1Num + x2Num) / 2);
+        const y = Math.floor((y1Num + y2Num) / 2);
+        return `(${x},${y})`;
+    }
+    const cleanedText = text.replace(/\[EOS\]/g, '');
+    return cleanedText.replace(pattern, replaceMatch).trim();
+}
+function getPoint(startBox, size) {
+    const [x, y] = JSON.parse(startBox);
+    return [
+        x * size.width,
+        y * size.height
+    ];
+}
+exports.uiTarsPlanning = __webpack_exports__.uiTarsPlanning;
+for(var __rspack_i in __webpack_exports__)if (-1 === [
+    "uiTarsPlanning"
+].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});