@godscene/core 1.7.11

package/dist/lib/ai-model/prompt/llm-section-locator.js

@@ -0,0 +1,79 @@
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.d = (exports1, definition)=>{
+        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
+            enumerable: true,
+            get: definition[key]
+        });
+    };
+})();
+(()=>{
+    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
+})();
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ("u" > typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+__webpack_require__.d(__webpack_exports__, {
+    sectionLocatorInstruction: ()=>sectionLocatorInstruction,
+    systemPromptToLocateSection: ()=>systemPromptToLocateSection
+});
+const env_namespaceObject = require("@godscene/shared/env");
+const external_common_js_namespaceObject = require("./common.js");
+function systemPromptToLocateSection(modelFamily) {
+    const preferredLanguage = (0, env_namespaceObject.getPreferredLanguage)();
+    const bboxFormat = (0, external_common_js_namespaceObject.bboxDescription)(modelFamily);
+    return `
+## Role:
+You are an AI assistant that helps identify UI elements.
+
+## Objective:
+- Find a section containing the target element
+- If the description mentions reference elements, also locate sections containing those references
+
+## Output Format:
+\`\`\`json
+{
+  "bbox": [number, number, number, number],  // ${bboxFormat}
+  "references_bbox"?: [
+    [number, number, number, number],
+    ...
+  ],
+  "error"?: string
+}
+\`\`\`
+
+Fields:
+* \`bbox\` - Bounding box of the section containing the target element
+* \`references_bbox\` - Optional array of bounding boxes for reference elements
+* \`error\` - Optional error message if the section cannot be found. Use ${preferredLanguage}.
+
+Example:
+If the description is "delete button on the second row with title 'Peter'", return:
+\`\`\`json
+{
+  "bbox": [100, 100, 200, 200],
+  "references_bbox": [[100, 100, 200, 200]]
+}
+\`\`\`
+`;
+}
+const sectionLocatorInstruction = (sectionDescription)=>`Find section containing: ${sectionDescription}`;
+exports.sectionLocatorInstruction = __webpack_exports__.sectionLocatorInstruction;
+exports.systemPromptToLocateSection = __webpack_exports__.systemPromptToLocateSection;
+for(var __rspack_i in __webpack_exports__)if (-1 === [
+    "sectionLocatorInstruction",
+    "systemPromptToLocateSection"
+].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});

package/dist/lib/ai-model/prompt/order-sensitive-judge.js

@@ -0,0 +1,70 @@
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.d = (exports1, definition)=>{
+        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
+            enumerable: true,
+            get: definition[key]
+        });
+    };
+})();
+(()=>{
+    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
+})();
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ("u" > typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+__webpack_require__.d(__webpack_exports__, {
+    orderSensitiveJudgePrompt: ()=>orderSensitiveJudgePrompt,
+    systemPromptToJudgeOrderSensitive: ()=>systemPromptToJudgeOrderSensitive
+});
+function systemPromptToJudgeOrderSensitive() {
+    return `
+## Role:
+You are an AI assistant that analyzes UI element descriptions.
+
+## Objective:
+Determine whether a given element description is order-sensitive.
+
+Order-sensitive descriptions contain phrases that specify position or sequence, such as:
+- "the first button"
+- "the second item"
+- "the third row"
+- "the last input"
+- "the 5th element"
+
+Order-insensitive descriptions do not specify position:
+- "login button"
+- "search input"
+- "submit button"
+- "user avatar"
+
+## Output Format:
+\`\`\`json
+{
+  "isOrderSensitive": boolean
+}
+\`\`\`
+
+Return true if the description is order-sensitive, false otherwise.
+`;
+}
+const orderSensitiveJudgePrompt = (description)=>`Analyze this element description: "${description}"`;
+exports.orderSensitiveJudgePrompt = __webpack_exports__.orderSensitiveJudgePrompt;
+exports.systemPromptToJudgeOrderSensitive = __webpack_exports__.systemPromptToJudgeOrderSensitive;
+for(var __rspack_i in __webpack_exports__)if (-1 === [
+    "orderSensitiveJudgePrompt",
+    "systemPromptToJudgeOrderSensitive"
+].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});

package/dist/lib/ai-model/prompt/playwright-generator.js

@@ -0,0 +1,176 @@
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.d = (exports1, definition)=>{
+        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
+            enumerable: true,
+            get: definition[key]
+        });
+    };
+})();
+(()=>{
+    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
+})();
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ("u" > typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+__webpack_require__.d(__webpack_exports__, {
+    createEventCounts: ()=>external_yaml_generator_js_namespaceObject.createEventCounts,
+    filterEventsByType: ()=>external_yaml_generator_js_namespaceObject.filterEventsByType,
+    generatePlaywrightTest: ()=>generatePlaywrightTest,
+    generatePlaywrightTestStream: ()=>generatePlaywrightTestStream,
+    validateEvents: ()=>external_yaml_generator_js_namespaceObject.validateEvents,
+    createMessageContent: ()=>external_yaml_generator_js_namespaceObject.createMessageContent,
+    getScreenshotsForLLM: ()=>external_yaml_generator_js_namespaceObject.getScreenshotsForLLM,
+    extractInputDescriptions: ()=>external_yaml_generator_js_namespaceObject.extractInputDescriptions,
+    prepareEventSummary: ()=>external_yaml_generator_js_namespaceObject.prepareEventSummary,
+    processEventsForLLM: ()=>external_yaml_generator_js_namespaceObject.processEventsForLLM
+});
+const constants_namespaceObject = require("@godscene/shared/constants");
+const external_index_js_namespaceObject = require("../index.js");
+const external_yaml_generator_js_namespaceObject = require("./yaml-generator.js");
+const generatePlaywrightTest = async (events, options, modelConfig)=>{
+    (0, external_yaml_generator_js_namespaceObject.validateEvents)(events);
+    const summary = (0, external_yaml_generator_js_namespaceObject.prepareEventSummary)(events, {
+        testName: options.testName,
+        maxScreenshots: options.maxScreenshots || 3
+    });
+    const playwrightSummary = {
+        ...summary,
+        waitForNetworkIdle: false !== options.waitForNetworkIdle,
+        waitForNetworkIdleTimeout: options.waitForNetworkIdleTimeout || 2000,
+        viewportSize: options.viewportSize || {
+            width: 1280,
+            height: 800
+        }
+    };
+    const screenshots = (0, external_yaml_generator_js_namespaceObject.getScreenshotsForLLM)(events, options.maxScreenshots || 3);
+    const promptText = `Generate a Playwright test using @godscene/web/playwright that reproduces this recorded browser session. The test should be based on the following events and follow the structure of the example provided. Make the test descriptive with appropriate assertions and validations.
+
+Event Summary:
+${JSON.stringify(playwrightSummary, null, 2)}
+
+Generated code should:
+1. Import required dependencies
+2. Set up the test with proper configuration
+3. Include a beforeEach hook to navigate to the starting URL
+4. Implement a test that uses Midscene AI methods (aiTap, aiInput, aiAssert, etc.)
+5. Include appropriate assertions and validations
+6. Follow best practices for Playwright tests
+7. Be ready to execute without further modification
+
+Important: Return ONLY the raw Playwright test code. Do NOT wrap the response in markdown code blocks (no \`\`\`typescript, \`\`\`javascript or \`\`\`). Start directly with the code content.`;
+    const messageContent = (0, external_yaml_generator_js_namespaceObject.createMessageContent)(promptText, screenshots, false !== options.includeScreenshots);
+    const systemPrompt = `You are an expert test automation engineer specializing in Playwright and Midscene. 
+Your task is to generate a complete, executable Playwright test using @godscene/web/playwright that reproduces a recorded browser session.
+
+${constants_namespaceObject.PLAYWRIGHT_EXAMPLE_CODE}`;
+    const prompt = [
+        {
+            role: 'system',
+            content: systemPrompt
+        },
+        {
+            role: 'user',
+            content: messageContent
+        }
+    ];
+    const response = await (0, external_index_js_namespaceObject.callAIWithStringResponse)(prompt, modelConfig);
+    if (response?.content && 'string' == typeof response.content) return response.content;
+    throw new Error('Failed to generate Playwright test code');
+};
+const generatePlaywrightTestStream = async (events, options, modelConfig)=>{
+    (0, external_yaml_generator_js_namespaceObject.validateEvents)(events);
+    const summary = (0, external_yaml_generator_js_namespaceObject.prepareEventSummary)(events, {
+        testName: options.testName,
+        maxScreenshots: options.maxScreenshots || 3
+    });
+    const playwrightSummary = {
+        ...summary,
+        waitForNetworkIdle: false !== options.waitForNetworkIdle,
+        waitForNetworkIdleTimeout: options.waitForNetworkIdleTimeout || 2000,
+        viewportSize: options.viewportSize || {
+            width: 1280,
+            height: 800
+        }
+    };
+    const screenshots = (0, external_yaml_generator_js_namespaceObject.getScreenshotsForLLM)(events, options.maxScreenshots || 3);
+    const promptText = `Generate a Playwright test using @godscene/web/playwright that reproduces this recorded browser session. The test should be based on the following events and follow the structure of the example provided. Make the test descriptive with appropriate assertions and validations.
+
+Event Summary:
+${JSON.stringify(playwrightSummary, null, 2)}
+
+Generated code should:
+1. Import required dependencies
+2. Set up the test with proper configuration
+3. Include a beforeEach hook to navigate to the starting URL
+4. Implement a test that uses Midscene AI methods (aiTap, aiInput, aiAssert, etc.)
+5. Include appropriate assertions and validations
+6. Follow best practices for Playwright tests
+7. Be ready to execute without further modification
+8. can't wrap this test code in markdown code block
+
+Important: Return ONLY the raw Playwright test code. Do NOT wrap the response in markdown code blocks (no \`\`\`typescript, \`\`\`javascript or \`\`\`). Start directly with the code content.`;
+    const messageContent = (0, external_yaml_generator_js_namespaceObject.createMessageContent)(promptText, screenshots, false !== options.includeScreenshots);
+    const systemPrompt = `You are an expert test automation engineer specializing in Playwright and Midscene. 
+Your task is to generate a complete, executable Playwright test using @godscene/web/playwright that reproduces a recorded browser session.
+
+${constants_namespaceObject.PLAYWRIGHT_EXAMPLE_CODE}`;
+    const prompt = [
+        {
+            role: 'system',
+            content: systemPrompt
+        },
+        {
+            role: 'user',
+            content: messageContent
+        }
+    ];
+    if (options.stream && options.onChunk) return await (0, external_index_js_namespaceObject.callAI)(prompt, modelConfig, {
+        stream: true,
+        onChunk: options.onChunk
+    });
+    {
+        const response = await (0, external_index_js_namespaceObject.callAIWithStringResponse)(prompt, modelConfig);
+        if (response?.content && 'string' == typeof response.content) return {
+            content: response.content,
+            usage: response.usage,
+            isStreamed: false
+        };
+        throw new Error('Failed to generate Playwright test code');
+    }
+};
+exports.createEventCounts = __webpack_exports__.createEventCounts;
+exports.createMessageContent = __webpack_exports__.createMessageContent;
+exports.extractInputDescriptions = __webpack_exports__.extractInputDescriptions;
+exports.filterEventsByType = __webpack_exports__.filterEventsByType;
+exports.generatePlaywrightTest = __webpack_exports__.generatePlaywrightTest;
+exports.generatePlaywrightTestStream = __webpack_exports__.generatePlaywrightTestStream;
+exports.getScreenshotsForLLM = __webpack_exports__.getScreenshotsForLLM;
+exports.prepareEventSummary = __webpack_exports__.prepareEventSummary;
+exports.processEventsForLLM = __webpack_exports__.processEventsForLLM;
+exports.validateEvents = __webpack_exports__.validateEvents;
+for(var __rspack_i in __webpack_exports__)if (-1 === [
+    "createEventCounts",
+    "createMessageContent",
+    "extractInputDescriptions",
+    "filterEventsByType",
+    "generatePlaywrightTest",
+    "generatePlaywrightTestStream",
+    "getScreenshotsForLLM",
+    "prepareEventSummary",
+    "processEventsForLLM",
+    "validateEvents"
+].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});

package/dist/lib/ai-model/prompt/ui-tars-planning.js

@@ -0,0 +1,71 @@
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.d = (exports1, definition)=>{
+        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
+            enumerable: true,
+            get: definition[key]
+        });
+    };
+})();
+(()=>{
+    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
+})();
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ("u" > typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+__webpack_require__.d(__webpack_exports__, {
+    getUiTarsPlanningPrompt: ()=>getUiTarsPlanningPrompt,
+    getSummary: ()=>getSummary
+});
+const env_namespaceObject = require("@godscene/shared/env");
+function getUiTarsPlanningPrompt() {
+    const preferredLanguage = (0, env_namespaceObject.getPreferredLanguage)();
+    return `
+You are a GUI agent. You are given a task and your action history, with screenshots. You need to perform the next action to complete the task. 
+
+## Output Format
+\`\`\`
+Thought: ...
+Action: ...
+\`\`\`
+
+## Action Space
+
+click(start_box='[x1, y1, x2, y2]')
+left_double(start_box='[x1, y1, x2, y2]')
+right_single(start_box='[x1, y1, x2, y2]')
+drag(start_box='[x1, y1, x2, y2]', end_box='[x3, y3, x4, y4]')
+hotkey(key='')
+type(content='xxx') # Use escape characters \\', \\\", and \\n in content part to ensure we can parse the content in normal python string format. If you want to submit your input, use \\n at the end of content. 
+scroll(start_box='[x1, y1, x2, y2]', direction='down or up or right or left')
+wait() #Sleep for 5s and take a screenshot to check for any changes.
+finished(content='xxx') # Use escape characters \\', \\", and \\n in content part to ensure we can parse the content in normal python string format.
+
+
+## Note
+- Use ${preferredLanguage} in \`Thought\` part.
+- Write a small plan and finally summarize your next action (with its target element) in one sentence in \`Thought\` part.
+
+## User Instruction
+`;
+}
+const getSummary = (prediction)=>prediction.replace(/Reflection:[\s\S]*?(?=Action_Summary:|Action:|$)/g, '').trim();
+exports.getSummary = __webpack_exports__.getSummary;
+exports.getUiTarsPlanningPrompt = __webpack_exports__.getUiTarsPlanningPrompt;
+for(var __rspack_i in __webpack_exports__)if (-1 === [
+    "getSummary",
+    "getUiTarsPlanningPrompt"
+].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});

package/dist/lib/ai-model/prompt/util.js

@@ -0,0 +1,103 @@
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.d = (exports1, definition)=>{
+        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
+            enumerable: true,
+            get: definition[key]
+        });
+    };
+})();
+(()=>{
+    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
+})();
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ("u" > typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+__webpack_require__.d(__webpack_exports__, {
+    distance: ()=>distance,
+    distanceThreshold: ()=>distanceThreshold,
+    extractXMLTag: ()=>extractXMLTag,
+    parseMarkFinishedIndexes: ()=>parseMarkFinishedIndexes,
+    parseSubGoalsFromXML: ()=>parseSubGoalsFromXML
+});
+function extractXMLTag(xmlString, tagName) {
+    const lowerXmlString = xmlString.toLowerCase();
+    const lowerTagName = tagName.toLowerCase();
+    const closeTag = `</${lowerTagName}>`;
+    const openTag = `<${lowerTagName}>`;
+    const lastCloseIndex = lowerXmlString.lastIndexOf(closeTag);
+    if (-1 === lastCloseIndex) {
+        const lastOpenIndex = lowerXmlString.lastIndexOf(openTag);
+        if (-1 === lastOpenIndex) return;
+        const contentStart = lastOpenIndex + openTag.length;
+        const remaining = xmlString.substring(contentStart);
+        const nextTagIndex = remaining.indexOf('<');
+        const content = -1 === nextTagIndex ? remaining : remaining.substring(0, nextTagIndex);
+        return content.trim();
+    }
+    const searchArea = lowerXmlString.substring(0, lastCloseIndex);
+    const lastOpenIndex = searchArea.lastIndexOf(openTag);
+    if (-1 === lastOpenIndex) return;
+    const contentStart = lastOpenIndex + openTag.length;
+    const contentEnd = lastCloseIndex;
+    const content = xmlString.substring(contentStart, contentEnd);
+    return content.trim();
+}
+function parseSubGoalsFromXML(xmlContent) {
+    const subGoals = [];
+    const regex = /<sub-goal\s+index="(\d+)"\s+status="(pending|finished)"(?:\s*\/>|>([\s\S]*?)<\/sub-goal>)/gi;
+    let match;
+    match = regex.exec(xmlContent);
+    while(null !== match){
+        const index = Number.parseInt(match[1], 10);
+        const status = match[2];
+        const description = match[3]?.trim() || '';
+        subGoals.push({
+            index,
+            status,
+            description
+        });
+        match = regex.exec(xmlContent);
+    }
+    return subGoals;
+}
+function parseMarkFinishedIndexes(xmlContent) {
+    const indexes = [];
+    const regex = /<sub-goal\s+index="(\d+)"\s+status="finished"\s*\/>/gi;
+    let match;
+    match = regex.exec(xmlContent);
+    while(null !== match){
+        indexes.push(Number.parseInt(match[1], 10));
+        match = regex.exec(xmlContent);
+    }
+    return indexes;
+}
+const distanceThreshold = 16;
+function distance(point1, point2) {
+    return Math.sqrt((point1.x - point2.x) ** 2 + (point1.y - point2.y) ** 2);
+}
+exports.distance = __webpack_exports__.distance;
+exports.distanceThreshold = __webpack_exports__.distanceThreshold;
+exports.extractXMLTag = __webpack_exports__.extractXMLTag;
+exports.parseMarkFinishedIndexes = __webpack_exports__.parseMarkFinishedIndexes;
+exports.parseSubGoalsFromXML = __webpack_exports__.parseSubGoalsFromXML;
+for(var __rspack_i in __webpack_exports__)if (-1 === [
+    "distance",
+    "distanceThreshold",
+    "extractXMLTag",
+    "parseMarkFinishedIndexes",
+    "parseSubGoalsFromXML"
+].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});