@godscene/core 1.7.11

package/dist/es/ai-model/conversation-history.mjs

@@ -0,0 +1,193 @@
+function _define_property(obj, key, value) {
+    if (key in obj) Object.defineProperty(obj, key, {
+        value: value,
+        enumerable: true,
+        configurable: true,
+        writable: true
+    });
+    else obj[key] = value;
+    return obj;
+}
+var _computedKey;
+_computedKey = Symbol.iterator;
+let _computedKey1 = _computedKey;
+class ConversationHistory {
+    resetPendingFeedbackMessageIfExists() {
+        if (this.pendingFeedbackMessage) this.pendingFeedbackMessage = '';
+    }
+    append(message) {
+        this.messages.push(message);
+    }
+    seed(messages) {
+        this.reset();
+        messages.forEach((message)=>{
+            this.append(message);
+        });
+    }
+    reset() {
+        this.messages.length = 0;
+        this.memories.length = 0;
+        this.subGoals.length = 0;
+        this.historicalLogs.length = 0;
+        this.pendingFeedbackMessage = '';
+    }
+    snapshot(maxImages) {
+        if (void 0 === maxImages) return [
+            ...this.messages
+        ];
+        const clonedMessages = structuredClone(this.messages);
+        let imageCount = 0;
+        for(let i = clonedMessages.length - 1; i >= 0; i--){
+            const message = clonedMessages[i];
+            const content = message.content;
+            if (Array.isArray(content)) for(let j = 0; j < content.length; j++){
+                const item = content[j];
+                if ('image_url' === item.type) {
+                    imageCount++;
+                    if (imageCount > maxImages) content[j] = {
+                        type: 'text',
+                        text: '(image ignored due to size optimization)'
+                    };
+                }
+            }
+        }
+        return clonedMessages;
+    }
+    get length() {
+        return this.messages.length;
+    }
+    [_computedKey1]() {
+        return this.messages[Symbol.iterator]();
+    }
+    toJSON() {
+        return this.snapshot();
+    }
+    setSubGoals(subGoals) {
+        this.subGoals = subGoals.map((goal)=>({
+                ...goal
+            }));
+        this.markFirstPendingAsRunning();
+    }
+    mergeSubGoals(subGoals) {
+        if (0 === this.subGoals.length) return void this.setSubGoals(subGoals);
+        const existingByIndex = new Map(this.subGoals.map((goal)=>[
+                goal.index,
+                goal
+            ]));
+        const mergedSubGoals = subGoals.map((goal)=>{
+            const existingGoal = existingByIndex.get(goal.index);
+            const hasNonEmptyDescription = goal.description.trim().length > 0;
+            if (!existingGoal && !hasNonEmptyDescription) return null;
+            return {
+                ...goal,
+                description: hasNonEmptyDescription || !existingGoal ? goal.description : existingGoal.description
+            };
+        });
+        const validSubGoals = mergedSubGoals.filter((goal)=>null !== goal);
+        if (0 === validSubGoals.length) return;
+        this.setSubGoals(validSubGoals);
+    }
+    updateSubGoal(index, updates) {
+        const goal = this.subGoals.find((g)=>g.index === index);
+        if (!goal) return false;
+        let changed = false;
+        if (void 0 !== updates.status && updates.status !== goal.status) {
+            goal.status = updates.status;
+            changed = true;
+        }
+        if (void 0 !== updates.description && updates.description !== goal.description) {
+            goal.description = updates.description;
+            changed = true;
+        }
+        if (changed) goal.logs = [];
+        return true;
+    }
+    markFirstPendingAsRunning() {
+        const firstPending = this.subGoals.find((g)=>'pending' === g.status);
+        if (firstPending) {
+            firstPending.status = 'running';
+            firstPending.logs = [];
+        }
+    }
+    markSubGoalFinished(index) {
+        const result = this.updateSubGoal(index, {
+            status: 'finished'
+        });
+        if (result) this.markFirstPendingAsRunning();
+        return result;
+    }
+    markAllSubGoalsFinished() {
+        for (const goal of this.subGoals){
+            if ('finished' !== goal.status) goal.logs = [];
+            goal.status = 'finished';
+        }
+    }
+    appendSubGoalLog(log) {
+        if (!log) return;
+        const runningGoal = this.subGoals.find((g)=>'running' === g.status);
+        if (runningGoal) {
+            if (!runningGoal.logs) runningGoal.logs = [];
+            runningGoal.logs.push(log);
+        }
+    }
+    subGoalsToText() {
+        if (0 === this.subGoals.length) return '';
+        const lines = this.subGoals.map((goal)=>`${goal.index}. ${goal.description} (${goal.status})`);
+        const currentGoal = this.subGoals.find((goal)=>'running' === goal.status) || this.subGoals.find((goal)=>'pending' === goal.status);
+        let currentGoalText = '';
+        if (currentGoal) {
+            currentGoalText = `\nCurrent sub-goal is: ${currentGoal.description}`;
+            if (currentGoal.logs && currentGoal.logs.length > 0) {
+                const logLines = currentGoal.logs.map((log)=>`- ${log}`).join('\n');
+                currentGoalText += `\nActions performed for current sub-goal:\n${logLines}`;
+            }
+        }
+        return `Sub-goals:\n${lines.join('\n')}${currentGoalText}`;
+    }
+    appendHistoricalLog(log) {
+        if (log) this.historicalLogs.push(log);
+    }
+    historicalLogsToText() {
+        if (0 === this.historicalLogs.length) return '';
+        const logLines = this.historicalLogs.map((log)=>`- ${log}`).join('\n');
+        return `Here are the steps that have been executed:\n${logLines}`;
+    }
+    appendMemory(memory) {
+        if (memory) this.memories.push(memory);
+    }
+    getMemories() {
+        return [
+            ...this.memories
+        ];
+    }
+    memoriesToText() {
+        if (0 === this.memories.length) return '';
+        return `Memories from previous steps:\n---\n${this.memories.join('\n---\n')}\n`;
+    }
+    clearMemories() {
+        this.memories.length = 0;
+    }
+    compressHistory(threshold, keepCount) {
+        if (this.messages.length <= threshold) return false;
+        const omittedCount = this.messages.length - keepCount;
+        const omittedPlaceholder = {
+            role: 'user',
+            content: `(${omittedCount} previous conversation messages have been omitted)`
+        };
+        const recentMessages = this.messages.slice(-keepCount);
+        this.messages.length = 0;
+        this.messages.push(omittedPlaceholder);
+        for (const msg of recentMessages)this.messages.push(msg);
+        return true;
+    }
+    constructor(options){
+        _define_property(this, "messages", []);
+        _define_property(this, "subGoals", []);
+        _define_property(this, "memories", []);
+        _define_property(this, "historicalLogs", []);
+        _define_property(this, "pendingFeedbackMessage", void 0);
+        if (options?.initialMessages?.length) this.seed(options.initialMessages);
+        this.pendingFeedbackMessage = '';
+    }
+}
+export { ConversationHistory };

package/dist/es/ai-model/index.mjs

@@ -0,0 +1,12 @@
+import { AIResponseParseError, callAI, callAIWithObjectResponse, callAIWithStringResponse } from "./service-caller/index.mjs";
+import { runConnectivityTest } from "./connectivity.mjs";
+import { systemPromptToLocateElement } from "./prompt/llm-locator.mjs";
+import { generatePlaywrightTest, generatePlaywrightTestStream } from "./prompt/playwright-generator.mjs";
+import { generateYamlTest, generateYamlTestStream } from "./prompt/yaml-generator.mjs";
+import { AiExtractElementInfo, AiJudgeOrderSensitive, AiLocateElement, AiLocateSection } from "./inspect.mjs";
+import { plan } from "./llm-planning.mjs";
+import { autoGLMPlanning } from "./auto-glm/planning.mjs";
+import { PointSchema, RectSchema, SizeSchema, TMultimodalPromptSchema, TUserPromptSchema, adaptBboxToRect, dumpActionParam, findAllMidsceneLocatorField, getMidsceneLocationSchema, parseActionParam } from "../common.mjs";
+import { uiTarsPlanning } from "./ui-tars-planning.mjs";
+import { ConversationHistory } from "./conversation-history.mjs";
+export { AIResponseParseError, AiExtractElementInfo, AiJudgeOrderSensitive, AiLocateElement, AiLocateSection, ConversationHistory, PointSchema, RectSchema, SizeSchema, TMultimodalPromptSchema, TUserPromptSchema, adaptBboxToRect, autoGLMPlanning, callAI, callAIWithObjectResponse, callAIWithStringResponse, dumpActionParam, findAllMidsceneLocatorField, generatePlaywrightTest, generatePlaywrightTestStream, generateYamlTest, generateYamlTestStream, getMidsceneLocationSchema, parseActionParam, plan, runConnectivityTest, systemPromptToLocateElement, uiTarsPlanning };

package/dist/es/ai-model/inspect.mjs

@@ -0,0 +1,395 @@
+import { generateElementByPoint, generateElementByRect } from "@godscene/shared/extractor/dom-util";
+import { cropByRect, paddingToMatchBlockByBase64, preProcessImageUrl, scaleImage } from "@godscene/shared/img";
+import { getDebug } from "@godscene/shared/logger";
+import { assert } from "@godscene/shared/utils";
+import { adaptBboxToRect, expandSearchArea, mergeRects } from "../common.mjs";
+import { parseAutoGLMLocateResponse } from "./auto-glm/parser.mjs";
+import { getAutoGLMLocatePrompt } from "./auto-glm/prompt.mjs";
+import { isAutoGLM } from "./auto-glm/util.mjs";
+import { extractDataQueryPrompt, parseXMLExtractionResponse, systemPromptToExtract } from "./prompt/extraction.mjs";
+import { findElementPrompt, systemPromptToLocateElement } from "./prompt/llm-locator.mjs";
+import { sectionLocatorInstruction, systemPromptToLocateSection } from "./prompt/llm-section-locator.mjs";
+import { orderSensitiveJudgePrompt, systemPromptToJudgeOrderSensitive } from "./prompt/order-sensitive-judge.mjs";
+import { AIResponseParseError, callAI, callAIWithObjectResponse, callAIWithStringResponse } from "./service-caller/index.mjs";
+const debugInspect = getDebug('ai:inspect');
+const debugSection = getDebug('ai:section');
+async function buildSearchAreaConfig(options) {
+    const { context, baseRect, modelFamily } = options;
+    const scaleRatio = 2;
+    const sectionRect = expandSearchArea(baseRect, context.shotSize);
+    const croppedResult = await cropByRect(context.screenshot.base64, sectionRect, 'qwen2.5-vl' === modelFamily);
+    const scaledResult = await scaleImage(croppedResult.imageBase64, scaleRatio);
+    sectionRect.width = scaledResult.width;
+    sectionRect.height = scaledResult.height;
+    return {
+        rect: sectionRect,
+        imageBase64: scaledResult.imageBase64,
+        scale: scaleRatio
+    };
+}
+const extraTextFromUserPrompt = (prompt)=>{
+    if ('string' == typeof prompt) return prompt;
+    return prompt.prompt;
+};
+const promptsToChatParam = async (multimodalPrompt)=>{
+    const msgs = [];
+    if (multimodalPrompt?.images?.length) {
+        msgs.push({
+            role: 'user',
+            content: [
+                {
+                    type: 'text',
+                    text: 'Next, I will provide all the reference images.'
+                }
+            ]
+        });
+        for (const item of multimodalPrompt.images){
+            const base64 = await preProcessImageUrl(item.url, !!multimodalPrompt.convertHttpImage2Base64);
+            msgs.push({
+                role: 'user',
+                content: [
+                    {
+                        type: 'text',
+                        text: `this is the reference image named '${item.name}':`
+                    }
+                ]
+            });
+            msgs.push({
+                role: 'user',
+                content: [
+                    {
+                        type: 'image_url',
+                        image_url: {
+                            url: base64,
+                            detail: 'high'
+                        }
+                    }
+                ]
+            });
+        }
+    }
+    return msgs;
+};
+async function AiLocateElement(options) {
+    const { context, targetElementDescription, modelConfig } = options;
+    const { modelFamily } = modelConfig;
+    const screenshotBase64 = context.screenshot.base64;
+    assert(targetElementDescription, "cannot find the target element description");
+    const targetElementDescriptionText = extraTextFromUserPrompt(targetElementDescription);
+    const userInstructionPrompt = findElementPrompt(targetElementDescriptionText);
+    const systemPrompt = isAutoGLM(modelFamily) ? getAutoGLMLocatePrompt(modelFamily) : systemPromptToLocateElement(modelFamily);
+    let imagePayload = screenshotBase64;
+    let imageWidth = context.shotSize.width;
+    let imageHeight = context.shotSize.height;
+    let originalImageWidth = imageWidth;
+    let originalImageHeight = imageHeight;
+    if (options.searchConfig) {
+        assert(options.searchConfig.rect, 'searchArea is provided but its rect cannot be found. Failed to locate element');
+        assert(options.searchConfig.imageBase64, 'searchArea is provided but its imageBase64 cannot be found. Failed to locate element');
+        imagePayload = options.searchConfig.imageBase64;
+        imageWidth = options.searchConfig.rect?.width;
+        imageHeight = options.searchConfig.rect?.height;
+        originalImageWidth = imageWidth;
+        originalImageHeight = imageHeight;
+    } else if ('qwen2.5-vl' === modelFamily) {
+        const paddedResult = await paddingToMatchBlockByBase64(imagePayload);
+        imageWidth = paddedResult.width;
+        imageHeight = paddedResult.height;
+        imagePayload = paddedResult.imageBase64;
+    }
+    const msgs = [
+        {
+            role: 'system',
+            content: systemPrompt
+        },
+        {
+            role: 'user',
+            content: [
+                {
+                    type: 'image_url',
+                    image_url: {
+                        url: imagePayload,
+                        detail: 'high'
+                    }
+                },
+                {
+                    type: 'text',
+                    text: isAutoGLM(modelFamily) ? `Tap: ${userInstructionPrompt}` : userInstructionPrompt
+                }
+            ]
+        }
+    ];
+    if ('string' != typeof targetElementDescription) {
+        const addOns = await promptsToChatParam({
+            images: targetElementDescription.images,
+            convertHttpImage2Base64: targetElementDescription.convertHttpImage2Base64
+        });
+        msgs.push(...addOns);
+    }
+    if (isAutoGLM(modelFamily)) {
+        const { content: rawResponseContent, usage } = await callAIWithStringResponse(msgs, modelConfig, {
+            abortSignal: options.abortSignal
+        });
+        debugInspect('auto-glm rawResponse:', rawResponseContent);
+        const parsed = parseAutoGLMLocateResponse(rawResponseContent);
+        debugInspect('auto-glm thinking:', parsed.think);
+        debugInspect('auto-glm coordinates:', parsed.coordinates);
+        let resRect;
+        let matchedElements = [];
+        let errors = [];
+        if (parsed.error || !parsed.coordinates) {
+            errors = [
+                parsed.error || 'Failed to parse auto-glm response'
+            ];
+            debugInspect('auto-glm parse error:', errors[0]);
+        } else {
+            const { x, y } = parsed.coordinates;
+            debugInspect('auto-glm coordinates [0-999]:', {
+                x,
+                y
+            });
+            const pixelX = Math.round(x * imageWidth / 1000);
+            const pixelY = Math.round(y * imageHeight / 1000);
+            debugInspect('auto-glm pixel coordinates:', {
+                pixelX,
+                pixelY
+            });
+            let finalX = pixelX;
+            let finalY = pixelY;
+            if (options.searchConfig?.rect) {
+                finalX += options.searchConfig.rect.left;
+                finalY += options.searchConfig.rect.top;
+            }
+            const element = generateElementByPoint([
+                finalX,
+                finalY
+            ], targetElementDescriptionText);
+            resRect = element.rect;
+            debugInspect('auto-glm resRect:', resRect);
+            if (element) matchedElements = [
+                element
+            ];
+        }
+        return {
+            rect: resRect,
+            parseResult: {
+                elements: matchedElements,
+                errors
+            },
+            rawResponse: rawResponseContent,
+            usage,
+            reasoning_content: parsed.think
+        };
+    }
+    let res;
+    try {
+        res = await callAIWithObjectResponse(msgs, modelConfig, {
+            abortSignal: options.abortSignal
+        });
+    } catch (callError) {
+        const errorMessage = callError instanceof Error ? callError.message : String(callError);
+        const rawResponse = callError instanceof AIResponseParseError ? callError.rawResponse : errorMessage;
+        const usage = callError instanceof AIResponseParseError ? callError.usage : void 0;
+        return {
+            rect: void 0,
+            parseResult: {
+                elements: [],
+                errors: [
+                    `AI call error: ${errorMessage}`
+                ]
+            },
+            rawResponse,
+            usage,
+            reasoning_content: void 0
+        };
+    }
+    const rawResponse = JSON.stringify(res.content);
+    let resRect;
+    let matchedElements = [];
+    let errors = 'errors' in res.content ? res.content.errors : [];
+    try {
+        if ('bbox' in res.content && Array.isArray(res.content.bbox) && res.content.bbox.length >= 1) {
+            resRect = adaptBboxToRect(res.content.bbox, imageWidth, imageHeight, options.searchConfig?.rect?.left, options.searchConfig?.rect?.top, originalImageWidth, originalImageHeight, modelFamily, options.searchConfig?.scale);
+            debugInspect('resRect', resRect);
+            const element = generateElementByRect(resRect, targetElementDescriptionText);
+            errors = [];
+            if (element) matchedElements = [
+                element
+            ];
+        }
+    } catch (e) {
+        const msg = e instanceof Error ? `Failed to parse bbox: ${e.message}` : 'unknown error in locate';
+        if (errors && errors?.length !== 0) errors.push(`(${msg})`);
+        else errors = [
+            msg
+        ];
+    }
+    return {
+        rect: resRect,
+        parseResult: {
+            elements: matchedElements,
+            errors: errors
+        },
+        rawResponse,
+        usage: res.usage,
+        reasoning_content: res.reasoning_content
+    };
+}
+async function AiLocateSection(options) {
+    const { context, sectionDescription, modelConfig } = options;
+    const { modelFamily } = modelConfig;
+    const screenshotBase64 = context.screenshot.base64;
+    const systemPrompt = systemPromptToLocateSection(modelFamily);
+    const sectionLocatorInstructionText = sectionLocatorInstruction(extraTextFromUserPrompt(sectionDescription));
+    const msgs = [
+        {
+            role: 'system',
+            content: systemPrompt
+        },
+        {
+            role: 'user',
+            content: [
+                {
+                    type: 'image_url',
+                    image_url: {
+                        url: screenshotBase64,
+                        detail: 'high'
+                    }
+                },
+                {
+                    type: 'text',
+                    text: sectionLocatorInstructionText
+                }
+            ]
+        }
+    ];
+    if ('string' != typeof sectionDescription) {
+        const addOns = await promptsToChatParam({
+            images: sectionDescription.images,
+            convertHttpImage2Base64: sectionDescription.convertHttpImage2Base64
+        });
+        msgs.push(...addOns);
+    }
+    let result;
+    try {
+        result = await callAIWithObjectResponse(msgs, modelConfig, {
+            abortSignal: options.abortSignal
+        });
+    } catch (callError) {
+        const errorMessage = callError instanceof Error ? callError.message : String(callError);
+        const rawResponse = callError instanceof AIResponseParseError ? callError.rawResponse : errorMessage;
+        const usage = callError instanceof AIResponseParseError ? callError.usage : void 0;
+        return {
+            rect: void 0,
+            imageBase64: void 0,
+            error: `AI call error: ${errorMessage}`,
+            rawResponse,
+            usage
+        };
+    }
+    let searchAreaConfig;
+    const sectionBbox = result.content.bbox;
+    if (sectionBbox) {
+        const targetRect = adaptBboxToRect(sectionBbox, context.shotSize.width, context.shotSize.height, 0, 0, context.shotSize.width, context.shotSize.height, modelFamily);
+        debugSection('original targetRect %j', targetRect);
+        const referenceBboxList = result.content.references_bbox || [];
+        debugSection('referenceBboxList %j', referenceBboxList);
+        const referenceRects = referenceBboxList.filter((bbox)=>Array.isArray(bbox)).map((bbox)=>adaptBboxToRect(bbox, context.shotSize.width, context.shotSize.height, 0, 0, context.shotSize.width, context.shotSize.height, modelFamily));
+        debugSection('referenceRects %j', referenceRects);
+        const mergedRect = mergeRects([
+            targetRect,
+            ...referenceRects
+        ]);
+        debugSection('mergedRect %j', mergedRect);
+        const expandedRect = expandSearchArea(mergedRect, context.shotSize);
+        const originalWidth = expandedRect.width;
+        const originalHeight = expandedRect.height;
+        debugSection('expanded sectionRect %j', expandedRect);
+        searchAreaConfig = await buildSearchAreaConfig({
+            context,
+            baseRect: mergedRect,
+            modelFamily
+        });
+        debugSection('scaled sectionRect from %dx%d to %dx%d (scale=%d)', originalWidth, originalHeight, searchAreaConfig.rect.width, searchAreaConfig.rect.height, searchAreaConfig.scale);
+    }
+    return {
+        rect: searchAreaConfig?.rect,
+        imageBase64: searchAreaConfig?.imageBase64,
+        scale: searchAreaConfig?.scale,
+        error: result.content.error,
+        rawResponse: JSON.stringify(result.content),
+        usage: result.usage
+    };
+}
+async function AiExtractElementInfo(options) {
+    const { dataQuery, context, extractOption, multimodalPrompt, modelConfig } = options;
+    const systemPrompt = systemPromptToExtract();
+    const screenshotBase64 = context.screenshot.base64;
+    const extractDataPromptText = extractDataQueryPrompt(options.pageDescription || '', dataQuery);
+    const userContent = [];
+    if (extractOption?.screenshotIncluded !== false) userContent.push({
+        type: 'image_url',
+        image_url: {
+            url: screenshotBase64,
+            detail: 'high'
+        }
+    });
+    userContent.push({
+        type: 'text',
+        text: extractDataPromptText
+    });
+    const msgs = [
+        {
+            role: 'system',
+            content: systemPrompt
+        },
+        {
+            role: 'user',
+            content: userContent
+        }
+    ];
+    if (multimodalPrompt) {
+        const addOns = await promptsToChatParam({
+            images: multimodalPrompt.images,
+            convertHttpImage2Base64: multimodalPrompt.convertHttpImage2Base64
+        });
+        msgs.push(...addOns);
+    }
+    const { content: rawResponse, usage, reasoning_content } = await callAI(msgs, modelConfig);
+    let parseResult;
+    try {
+        parseResult = parseXMLExtractionResponse(rawResponse);
+    } catch (parseError) {
+        const errorMessage = parseError instanceof Error ? parseError.message : String(parseError);
+        throw new AIResponseParseError(`XML parse error: ${errorMessage}`, rawResponse, usage);
+    }
+    return {
+        parseResult,
+        rawResponse,
+        usage,
+        reasoning_content
+    };
+}
+async function AiJudgeOrderSensitive(description, callAIFn, modelConfig) {
+    const systemPrompt = systemPromptToJudgeOrderSensitive();
+    const userPrompt = orderSensitiveJudgePrompt(description);
+    const msgs = [
+        {
+            role: 'system',
+            content: systemPrompt
+        },
+        {
+            role: 'user',
+            content: userPrompt
+        }
+    ];
+    debugInspect("AiJudgeOrderSensitive: deepThink=false, description=%s", description);
+    const result = await callAIFn(msgs, modelConfig, {
+        deepThink: false
+    });
+    return {
+        isOrderSensitive: result.content.isOrderSensitive ?? false,
+        usage: result.usage
+    };
+}
+export { AiExtractElementInfo, AiJudgeOrderSensitive, AiLocateElement, AiLocateSection, buildSearchAreaConfig };