@godscene/core 1.7.11

package/dist/lib/ai-model/inspect.js

@@ -0,0 +1,441 @@
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.d = (exports1, definition)=>{
+        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
+            enumerable: true,
+            get: definition[key]
+        });
+    };
+})();
+(()=>{
+    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
+})();
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ("u" > typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+__webpack_require__.d(__webpack_exports__, {
+    AiLocateElement: ()=>AiLocateElement,
+    AiJudgeOrderSensitive: ()=>AiJudgeOrderSensitive,
+    AiExtractElementInfo: ()=>AiExtractElementInfo,
+    AiLocateSection: ()=>AiLocateSection,
+    buildSearchAreaConfig: ()=>buildSearchAreaConfig
+});
+const dom_util_namespaceObject = require("@godscene/shared/extractor/dom-util");
+const img_namespaceObject = require("@godscene/shared/img");
+const logger_namespaceObject = require("@godscene/shared/logger");
+const utils_namespaceObject = require("@godscene/shared/utils");
+const external_common_js_namespaceObject = require("../common.js");
+const parser_js_namespaceObject = require("./auto-glm/parser.js");
+const prompt_js_namespaceObject = require("./auto-glm/prompt.js");
+const util_js_namespaceObject = require("./auto-glm/util.js");
+const extraction_js_namespaceObject = require("./prompt/extraction.js");
+const llm_locator_js_namespaceObject = require("./prompt/llm-locator.js");
+const llm_section_locator_js_namespaceObject = require("./prompt/llm-section-locator.js");
+const order_sensitive_judge_js_namespaceObject = require("./prompt/order-sensitive-judge.js");
+const index_js_namespaceObject = require("./service-caller/index.js");
+const debugInspect = (0, logger_namespaceObject.getDebug)('ai:inspect');
+const debugSection = (0, logger_namespaceObject.getDebug)('ai:section');
+async function buildSearchAreaConfig(options) {
+    const { context, baseRect, modelFamily } = options;
+    const scaleRatio = 2;
+    const sectionRect = (0, external_common_js_namespaceObject.expandSearchArea)(baseRect, context.shotSize);
+    const croppedResult = await (0, img_namespaceObject.cropByRect)(context.screenshot.base64, sectionRect, 'qwen2.5-vl' === modelFamily);
+    const scaledResult = await (0, img_namespaceObject.scaleImage)(croppedResult.imageBase64, scaleRatio);
+    sectionRect.width = scaledResult.width;
+    sectionRect.height = scaledResult.height;
+    return {
+        rect: sectionRect,
+        imageBase64: scaledResult.imageBase64,
+        scale: scaleRatio
+    };
+}
+const extraTextFromUserPrompt = (prompt)=>{
+    if ('string' == typeof prompt) return prompt;
+    return prompt.prompt;
+};
+const promptsToChatParam = async (multimodalPrompt)=>{
+    const msgs = [];
+    if (multimodalPrompt?.images?.length) {
+        msgs.push({
+            role: 'user',
+            content: [
+                {
+                    type: 'text',
+                    text: 'Next, I will provide all the reference images.'
+                }
+            ]
+        });
+        for (const item of multimodalPrompt.images){
+            const base64 = await (0, img_namespaceObject.preProcessImageUrl)(item.url, !!multimodalPrompt.convertHttpImage2Base64);
+            msgs.push({
+                role: 'user',
+                content: [
+                    {
+                        type: 'text',
+                        text: `this is the reference image named '${item.name}':`
+                    }
+                ]
+            });
+            msgs.push({
+                role: 'user',
+                content: [
+                    {
+                        type: 'image_url',
+                        image_url: {
+                            url: base64,
+                            detail: 'high'
+                        }
+                    }
+                ]
+            });
+        }
+    }
+    return msgs;
+};
+async function AiLocateElement(options) {
+    const { context, targetElementDescription, modelConfig } = options;
+    const { modelFamily } = modelConfig;
+    const screenshotBase64 = context.screenshot.base64;
+    (0, utils_namespaceObject.assert)(targetElementDescription, "cannot find the target element description");
+    const targetElementDescriptionText = extraTextFromUserPrompt(targetElementDescription);
+    const userInstructionPrompt = (0, llm_locator_js_namespaceObject.findElementPrompt)(targetElementDescriptionText);
+    const systemPrompt = (0, util_js_namespaceObject.isAutoGLM)(modelFamily) ? (0, prompt_js_namespaceObject.getAutoGLMLocatePrompt)(modelFamily) : (0, llm_locator_js_namespaceObject.systemPromptToLocateElement)(modelFamily);
+    let imagePayload = screenshotBase64;
+    let imageWidth = context.shotSize.width;
+    let imageHeight = context.shotSize.height;
+    let originalImageWidth = imageWidth;
+    let originalImageHeight = imageHeight;
+    if (options.searchConfig) {
+        (0, utils_namespaceObject.assert)(options.searchConfig.rect, 'searchArea is provided but its rect cannot be found. Failed to locate element');
+        (0, utils_namespaceObject.assert)(options.searchConfig.imageBase64, 'searchArea is provided but its imageBase64 cannot be found. Failed to locate element');
+        imagePayload = options.searchConfig.imageBase64;
+        imageWidth = options.searchConfig.rect?.width;
+        imageHeight = options.searchConfig.rect?.height;
+        originalImageWidth = imageWidth;
+        originalImageHeight = imageHeight;
+    } else if ('qwen2.5-vl' === modelFamily) {
+        const paddedResult = await (0, img_namespaceObject.paddingToMatchBlockByBase64)(imagePayload);
+        imageWidth = paddedResult.width;
+        imageHeight = paddedResult.height;
+        imagePayload = paddedResult.imageBase64;
+    }
+    const msgs = [
+        {
+            role: 'system',
+            content: systemPrompt
+        },
+        {
+            role: 'user',
+            content: [
+                {
+                    type: 'image_url',
+                    image_url: {
+                        url: imagePayload,
+                        detail: 'high'
+                    }
+                },
+                {
+                    type: 'text',
+                    text: (0, util_js_namespaceObject.isAutoGLM)(modelFamily) ? `Tap: ${userInstructionPrompt}` : userInstructionPrompt
+                }
+            ]
+        }
+    ];
+    if ('string' != typeof targetElementDescription) {
+        const addOns = await promptsToChatParam({
+            images: targetElementDescription.images,
+            convertHttpImage2Base64: targetElementDescription.convertHttpImage2Base64
+        });
+        msgs.push(...addOns);
+    }
+    if ((0, util_js_namespaceObject.isAutoGLM)(modelFamily)) {
+        const { content: rawResponseContent, usage } = await (0, index_js_namespaceObject.callAIWithStringResponse)(msgs, modelConfig, {
+            abortSignal: options.abortSignal
+        });
+        debugInspect('auto-glm rawResponse:', rawResponseContent);
+        const parsed = (0, parser_js_namespaceObject.parseAutoGLMLocateResponse)(rawResponseContent);
+        debugInspect('auto-glm thinking:', parsed.think);
+        debugInspect('auto-glm coordinates:', parsed.coordinates);
+        let resRect;
+        let matchedElements = [];
+        let errors = [];
+        if (parsed.error || !parsed.coordinates) {
+            errors = [
+                parsed.error || 'Failed to parse auto-glm response'
+            ];
+            debugInspect('auto-glm parse error:', errors[0]);
+        } else {
+            const { x, y } = parsed.coordinates;
+            debugInspect('auto-glm coordinates [0-999]:', {
+                x,
+                y
+            });
+            const pixelX = Math.round(x * imageWidth / 1000);
+            const pixelY = Math.round(y * imageHeight / 1000);
+            debugInspect('auto-glm pixel coordinates:', {
+                pixelX,
+                pixelY
+            });
+            let finalX = pixelX;
+            let finalY = pixelY;
+            if (options.searchConfig?.rect) {
+                finalX += options.searchConfig.rect.left;
+                finalY += options.searchConfig.rect.top;
+            }
+            const element = (0, dom_util_namespaceObject.generateElementByPoint)([
+                finalX,
+                finalY
+            ], targetElementDescriptionText);
+            resRect = element.rect;
+            debugInspect('auto-glm resRect:', resRect);
+            if (element) matchedElements = [
+                element
+            ];
+        }
+        return {
+            rect: resRect,
+            parseResult: {
+                elements: matchedElements,
+                errors
+            },
+            rawResponse: rawResponseContent,
+            usage,
+            reasoning_content: parsed.think
+        };
+    }
+    let res;
+    try {
+        res = await (0, index_js_namespaceObject.callAIWithObjectResponse)(msgs, modelConfig, {
+            abortSignal: options.abortSignal
+        });
+    } catch (callError) {
+        const errorMessage = callError instanceof Error ? callError.message : String(callError);
+        const rawResponse = callError instanceof index_js_namespaceObject.AIResponseParseError ? callError.rawResponse : errorMessage;
+        const usage = callError instanceof index_js_namespaceObject.AIResponseParseError ? callError.usage : void 0;
+        return {
+            rect: void 0,
+            parseResult: {
+                elements: [],
+                errors: [
+                    `AI call error: ${errorMessage}`
+                ]
+            },
+            rawResponse,
+            usage,
+            reasoning_content: void 0
+        };
+    }
+    const rawResponse = JSON.stringify(res.content);
+    let resRect;
+    let matchedElements = [];
+    let errors = 'errors' in res.content ? res.content.errors : [];
+    try {
+        if ('bbox' in res.content && Array.isArray(res.content.bbox) && res.content.bbox.length >= 1) {
+            resRect = (0, external_common_js_namespaceObject.adaptBboxToRect)(res.content.bbox, imageWidth, imageHeight, options.searchConfig?.rect?.left, options.searchConfig?.rect?.top, originalImageWidth, originalImageHeight, modelFamily, options.searchConfig?.scale);
+            debugInspect('resRect', resRect);
+            const element = (0, dom_util_namespaceObject.generateElementByRect)(resRect, targetElementDescriptionText);
+            errors = [];
+            if (element) matchedElements = [
+                element
+            ];
+        }
+    } catch (e) {
+        const msg = e instanceof Error ? `Failed to parse bbox: ${e.message}` : 'unknown error in locate';
+        if (errors && errors?.length !== 0) errors.push(`(${msg})`);
+        else errors = [
+            msg
+        ];
+    }
+    return {
+        rect: resRect,
+        parseResult: {
+            elements: matchedElements,
+            errors: errors
+        },
+        rawResponse,
+        usage: res.usage,
+        reasoning_content: res.reasoning_content
+    };
+}
+async function AiLocateSection(options) {
+    const { context, sectionDescription, modelConfig } = options;
+    const { modelFamily } = modelConfig;
+    const screenshotBase64 = context.screenshot.base64;
+    const systemPrompt = (0, llm_section_locator_js_namespaceObject.systemPromptToLocateSection)(modelFamily);
+    const sectionLocatorInstructionText = (0, llm_section_locator_js_namespaceObject.sectionLocatorInstruction)(extraTextFromUserPrompt(sectionDescription));
+    const msgs = [
+        {
+            role: 'system',
+            content: systemPrompt
+        },
+        {
+            role: 'user',
+            content: [
+                {
+                    type: 'image_url',
+                    image_url: {
+                        url: screenshotBase64,
+                        detail: 'high'
+                    }
+                },
+                {
+                    type: 'text',
+                    text: sectionLocatorInstructionText
+                }
+            ]
+        }
+    ];
+    if ('string' != typeof sectionDescription) {
+        const addOns = await promptsToChatParam({
+            images: sectionDescription.images,
+            convertHttpImage2Base64: sectionDescription.convertHttpImage2Base64
+        });
+        msgs.push(...addOns);
+    }
+    let result;
+    try {
+        result = await (0, index_js_namespaceObject.callAIWithObjectResponse)(msgs, modelConfig, {
+            abortSignal: options.abortSignal
+        });
+    } catch (callError) {
+        const errorMessage = callError instanceof Error ? callError.message : String(callError);
+        const rawResponse = callError instanceof index_js_namespaceObject.AIResponseParseError ? callError.rawResponse : errorMessage;
+        const usage = callError instanceof index_js_namespaceObject.AIResponseParseError ? callError.usage : void 0;
+        return {
+            rect: void 0,
+            imageBase64: void 0,
+            error: `AI call error: ${errorMessage}`,
+            rawResponse,
+            usage
+        };
+    }
+    let searchAreaConfig;
+    const sectionBbox = result.content.bbox;
+    if (sectionBbox) {
+        const targetRect = (0, external_common_js_namespaceObject.adaptBboxToRect)(sectionBbox, context.shotSize.width, context.shotSize.height, 0, 0, context.shotSize.width, context.shotSize.height, modelFamily);
+        debugSection('original targetRect %j', targetRect);
+        const referenceBboxList = result.content.references_bbox || [];
+        debugSection('referenceBboxList %j', referenceBboxList);
+        const referenceRects = referenceBboxList.filter((bbox)=>Array.isArray(bbox)).map((bbox)=>(0, external_common_js_namespaceObject.adaptBboxToRect)(bbox, context.shotSize.width, context.shotSize.height, 0, 0, context.shotSize.width, context.shotSize.height, modelFamily));
+        debugSection('referenceRects %j', referenceRects);
+        const mergedRect = (0, external_common_js_namespaceObject.mergeRects)([
+            targetRect,
+            ...referenceRects
+        ]);
+        debugSection('mergedRect %j', mergedRect);
+        const expandedRect = (0, external_common_js_namespaceObject.expandSearchArea)(mergedRect, context.shotSize);
+        const originalWidth = expandedRect.width;
+        const originalHeight = expandedRect.height;
+        debugSection('expanded sectionRect %j', expandedRect);
+        searchAreaConfig = await buildSearchAreaConfig({
+            context,
+            baseRect: mergedRect,
+            modelFamily
+        });
+        debugSection('scaled sectionRect from %dx%d to %dx%d (scale=%d)', originalWidth, originalHeight, searchAreaConfig.rect.width, searchAreaConfig.rect.height, searchAreaConfig.scale);
+    }
+    return {
+        rect: searchAreaConfig?.rect,
+        imageBase64: searchAreaConfig?.imageBase64,
+        scale: searchAreaConfig?.scale,
+        error: result.content.error,
+        rawResponse: JSON.stringify(result.content),
+        usage: result.usage
+    };
+}
+async function AiExtractElementInfo(options) {
+    const { dataQuery, context, extractOption, multimodalPrompt, modelConfig } = options;
+    const systemPrompt = (0, extraction_js_namespaceObject.systemPromptToExtract)();
+    const screenshotBase64 = context.screenshot.base64;
+    const extractDataPromptText = (0, extraction_js_namespaceObject.extractDataQueryPrompt)(options.pageDescription || '', dataQuery);
+    const userContent = [];
+    if (extractOption?.screenshotIncluded !== false) userContent.push({
+        type: 'image_url',
+        image_url: {
+            url: screenshotBase64,
+            detail: 'high'
+        }
+    });
+    userContent.push({
+        type: 'text',
+        text: extractDataPromptText
+    });
+    const msgs = [
+        {
+            role: 'system',
+            content: systemPrompt
+        },
+        {
+            role: 'user',
+            content: userContent
+        }
+    ];
+    if (multimodalPrompt) {
+        const addOns = await promptsToChatParam({
+            images: multimodalPrompt.images,
+            convertHttpImage2Base64: multimodalPrompt.convertHttpImage2Base64
+        });
+        msgs.push(...addOns);
+    }
+    const { content: rawResponse, usage, reasoning_content } = await (0, index_js_namespaceObject.callAI)(msgs, modelConfig);
+    let parseResult;
+    try {
+        parseResult = (0, extraction_js_namespaceObject.parseXMLExtractionResponse)(rawResponse);
+    } catch (parseError) {
+        const errorMessage = parseError instanceof Error ? parseError.message : String(parseError);
+        throw new index_js_namespaceObject.AIResponseParseError(`XML parse error: ${errorMessage}`, rawResponse, usage);
+    }
+    return {
+        parseResult,
+        rawResponse,
+        usage,
+        reasoning_content
+    };
+}
+async function AiJudgeOrderSensitive(description, callAIFn, modelConfig) {
+    const systemPrompt = (0, order_sensitive_judge_js_namespaceObject.systemPromptToJudgeOrderSensitive)();
+    const userPrompt = (0, order_sensitive_judge_js_namespaceObject.orderSensitiveJudgePrompt)(description);
+    const msgs = [
+        {
+            role: 'system',
+            content: systemPrompt
+        },
+        {
+            role: 'user',
+            content: userPrompt
+        }
+    ];
+    debugInspect("AiJudgeOrderSensitive: deepThink=false, description=%s", description);
+    const result = await callAIFn(msgs, modelConfig, {
+        deepThink: false
+    });
+    return {
+        isOrderSensitive: result.content.isOrderSensitive ?? false,
+        usage: result.usage
+    };
+}
+exports.AiExtractElementInfo = __webpack_exports__.AiExtractElementInfo;
+exports.AiJudgeOrderSensitive = __webpack_exports__.AiJudgeOrderSensitive;
+exports.AiLocateElement = __webpack_exports__.AiLocateElement;
+exports.AiLocateSection = __webpack_exports__.AiLocateSection;
+exports.buildSearchAreaConfig = __webpack_exports__.buildSearchAreaConfig;
+for(var __rspack_i in __webpack_exports__)if (-1 === [
+    "AiExtractElementInfo",
+    "AiJudgeOrderSensitive",
+    "AiLocateElement",
+    "AiLocateSection",
+    "buildSearchAreaConfig"
+].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});