@godscene/core 1.7.11

package/dist/es/ai-model/prompt/playwright-generator.mjs

@@ -0,0 +1,115 @@
+import { PLAYWRIGHT_EXAMPLE_CODE } from "@godscene/shared/constants";
+import { callAI, callAIWithStringResponse } from "../index.mjs";
+import { createEventCounts, createMessageContent, extractInputDescriptions, filterEventsByType, getScreenshotsForLLM, prepareEventSummary, processEventsForLLM, validateEvents } from "./yaml-generator.mjs";
+const generatePlaywrightTest = async (events, options, modelConfig)=>{
+    validateEvents(events);
+    const summary = prepareEventSummary(events, {
+        testName: options.testName,
+        maxScreenshots: options.maxScreenshots || 3
+    });
+    const playwrightSummary = {
+        ...summary,
+        waitForNetworkIdle: false !== options.waitForNetworkIdle,
+        waitForNetworkIdleTimeout: options.waitForNetworkIdleTimeout || 2000,
+        viewportSize: options.viewportSize || {
+            width: 1280,
+            height: 800
+        }
+    };
+    const screenshots = getScreenshotsForLLM(events, options.maxScreenshots || 3);
+    const promptText = `Generate a Playwright test using @godscene/web/playwright that reproduces this recorded browser session. The test should be based on the following events and follow the structure of the example provided. Make the test descriptive with appropriate assertions and validations.
+
+Event Summary:
+${JSON.stringify(playwrightSummary, null, 2)}
+
+Generated code should:
+1. Import required dependencies
+2. Set up the test with proper configuration
+3. Include a beforeEach hook to navigate to the starting URL
+4. Implement a test that uses Midscene AI methods (aiTap, aiInput, aiAssert, etc.)
+5. Include appropriate assertions and validations
+6. Follow best practices for Playwright tests
+7. Be ready to execute without further modification
+
+Important: Return ONLY the raw Playwright test code. Do NOT wrap the response in markdown code blocks (no \`\`\`typescript, \`\`\`javascript or \`\`\`). Start directly with the code content.`;
+    const messageContent = createMessageContent(promptText, screenshots, false !== options.includeScreenshots);
+    const systemPrompt = `You are an expert test automation engineer specializing in Playwright and Midscene. 
+Your task is to generate a complete, executable Playwright test using @godscene/web/playwright that reproduces a recorded browser session.
+
+${PLAYWRIGHT_EXAMPLE_CODE}`;
+    const prompt = [
+        {
+            role: 'system',
+            content: systemPrompt
+        },
+        {
+            role: 'user',
+            content: messageContent
+        }
+    ];
+    const response = await callAIWithStringResponse(prompt, modelConfig);
+    if (response?.content && 'string' == typeof response.content) return response.content;
+    throw new Error('Failed to generate Playwright test code');
+};
+const generatePlaywrightTestStream = async (events, options, modelConfig)=>{
+    validateEvents(events);
+    const summary = prepareEventSummary(events, {
+        testName: options.testName,
+        maxScreenshots: options.maxScreenshots || 3
+    });
+    const playwrightSummary = {
+        ...summary,
+        waitForNetworkIdle: false !== options.waitForNetworkIdle,
+        waitForNetworkIdleTimeout: options.waitForNetworkIdleTimeout || 2000,
+        viewportSize: options.viewportSize || {
+            width: 1280,
+            height: 800
+        }
+    };
+    const screenshots = getScreenshotsForLLM(events, options.maxScreenshots || 3);
+    const promptText = `Generate a Playwright test using @godscene/web/playwright that reproduces this recorded browser session. The test should be based on the following events and follow the structure of the example provided. Make the test descriptive with appropriate assertions and validations.
+
+Event Summary:
+${JSON.stringify(playwrightSummary, null, 2)}
+
+Generated code should:
+1. Import required dependencies
+2. Set up the test with proper configuration
+3. Include a beforeEach hook to navigate to the starting URL
+4. Implement a test that uses Midscene AI methods (aiTap, aiInput, aiAssert, etc.)
+5. Include appropriate assertions and validations
+6. Follow best practices for Playwright tests
+7. Be ready to execute without further modification
+8. can't wrap this test code in markdown code block
+
+Important: Return ONLY the raw Playwright test code. Do NOT wrap the response in markdown code blocks (no \`\`\`typescript, \`\`\`javascript or \`\`\`). Start directly with the code content.`;
+    const messageContent = createMessageContent(promptText, screenshots, false !== options.includeScreenshots);
+    const systemPrompt = `You are an expert test automation engineer specializing in Playwright and Midscene. 
+Your task is to generate a complete, executable Playwright test using @godscene/web/playwright that reproduces a recorded browser session.
+
+${PLAYWRIGHT_EXAMPLE_CODE}`;
+    const prompt = [
+        {
+            role: 'system',
+            content: systemPrompt
+        },
+        {
+            role: 'user',
+            content: messageContent
+        }
+    ];
+    if (options.stream && options.onChunk) return await callAI(prompt, modelConfig, {
+        stream: true,
+        onChunk: options.onChunk
+    });
+    {
+        const response = await callAIWithStringResponse(prompt, modelConfig);
+        if (response?.content && 'string' == typeof response.content) return {
+            content: response.content,
+            usage: response.usage,
+            isStreamed: false
+        };
+        throw new Error('Failed to generate Playwright test code');
+    }
+};
+export { createEventCounts, createMessageContent, extractInputDescriptions, filterEventsByType, generatePlaywrightTest, generatePlaywrightTestStream, getScreenshotsForLLM, prepareEventSummary, processEventsForLLM, validateEvents };

package/dist/es/ai-model/prompt/ui-tars-planning.mjs

@@ -0,0 +1,34 @@
+import { getPreferredLanguage } from "@godscene/shared/env";
+function getUiTarsPlanningPrompt() {
+    const preferredLanguage = getPreferredLanguage();
+    return `
+You are a GUI agent. You are given a task and your action history, with screenshots. You need to perform the next action to complete the task. 
+
+## Output Format
+\`\`\`
+Thought: ...
+Action: ...
+\`\`\`
+
+## Action Space
+
+click(start_box='[x1, y1, x2, y2]')
+left_double(start_box='[x1, y1, x2, y2]')
+right_single(start_box='[x1, y1, x2, y2]')
+drag(start_box='[x1, y1, x2, y2]', end_box='[x3, y3, x4, y4]')
+hotkey(key='')
+type(content='xxx') # Use escape characters \\', \\\", and \\n in content part to ensure we can parse the content in normal python string format. If you want to submit your input, use \\n at the end of content. 
+scroll(start_box='[x1, y1, x2, y2]', direction='down or up or right or left')
+wait() #Sleep for 5s and take a screenshot to check for any changes.
+finished(content='xxx') # Use escape characters \\', \\", and \\n in content part to ensure we can parse the content in normal python string format.
+
+
+## Note
+- Use ${preferredLanguage} in \`Thought\` part.
+- Write a small plan and finally summarize your next action (with its target element) in one sentence in \`Thought\` part.
+
+## User Instruction
+`;
+}
+const getSummary = (prediction)=>prediction.replace(/Reflection:[\s\S]*?(?=Action_Summary:|Action:|$)/g, '').trim();
+export { getSummary, getUiTarsPlanningPrompt };

package/dist/es/ai-model/prompt/util.mjs

@@ -0,0 +1,57 @@
+function extractXMLTag(xmlString, tagName) {
+    const lowerXmlString = xmlString.toLowerCase();
+    const lowerTagName = tagName.toLowerCase();
+    const closeTag = `</${lowerTagName}>`;
+    const openTag = `<${lowerTagName}>`;
+    const lastCloseIndex = lowerXmlString.lastIndexOf(closeTag);
+    if (-1 === lastCloseIndex) {
+        const lastOpenIndex = lowerXmlString.lastIndexOf(openTag);
+        if (-1 === lastOpenIndex) return;
+        const contentStart = lastOpenIndex + openTag.length;
+        const remaining = xmlString.substring(contentStart);
+        const nextTagIndex = remaining.indexOf('<');
+        const content = -1 === nextTagIndex ? remaining : remaining.substring(0, nextTagIndex);
+        return content.trim();
+    }
+    const searchArea = lowerXmlString.substring(0, lastCloseIndex);
+    const lastOpenIndex = searchArea.lastIndexOf(openTag);
+    if (-1 === lastOpenIndex) return;
+    const contentStart = lastOpenIndex + openTag.length;
+    const contentEnd = lastCloseIndex;
+    const content = xmlString.substring(contentStart, contentEnd);
+    return content.trim();
+}
+function parseSubGoalsFromXML(xmlContent) {
+    const subGoals = [];
+    const regex = /<sub-goal\s+index="(\d+)"\s+status="(pending|finished)"(?:\s*\/>|>([\s\S]*?)<\/sub-goal>)/gi;
+    let match;
+    match = regex.exec(xmlContent);
+    while(null !== match){
+        const index = Number.parseInt(match[1], 10);
+        const status = match[2];
+        const description = match[3]?.trim() || '';
+        subGoals.push({
+            index,
+            status,
+            description
+        });
+        match = regex.exec(xmlContent);
+    }
+    return subGoals;
+}
+function parseMarkFinishedIndexes(xmlContent) {
+    const indexes = [];
+    const regex = /<sub-goal\s+index="(\d+)"\s+status="finished"\s*\/>/gi;
+    let match;
+    match = regex.exec(xmlContent);
+    while(null !== match){
+        indexes.push(Number.parseInt(match[1], 10));
+        match = regex.exec(xmlContent);
+    }
+    return indexes;
+}
+const distanceThreshold = 16;
+function distance(point1, point2) {
+    return Math.sqrt((point1.x - point2.x) ** 2 + (point1.y - point2.y) ** 2);
+}
+export { distance, distanceThreshold, extractXMLTag, parseMarkFinishedIndexes, parseSubGoalsFromXML };

package/dist/es/ai-model/prompt/yaml-generator.mjs

@@ -0,0 +1,201 @@
+import { YAML_EXAMPLE_CODE } from "@godscene/shared/constants";
+import { callAI, callAIWithStringResponse } from "../index.mjs";
+const getScreenshotsForLLM = (events, maxScreenshots = 1)=>{
+    const eventsWithScreenshots = events.filter((event)=>event.screenshotBefore || event.screenshotAfter || event.screenshotWithBox);
+    const sortedEvents = [
+        ...eventsWithScreenshots
+    ].sort((a, b)=>{
+        if ('navigation' === a.type && 'navigation' !== b.type) return -1;
+        if ('navigation' !== a.type && 'navigation' === b.type) return 1;
+        if ('click' === a.type && 'click' !== b.type) return -1;
+        if ('click' !== a.type && 'click' === b.type) return 1;
+        return 0;
+    });
+    const screenshots = [];
+    for (const event of sortedEvents){
+        const screenshot = event.screenshotWithBox || event.screenshotAfter || event.screenshotBefore;
+        if (screenshot && !screenshots.includes(screenshot)) {
+            screenshots.push(screenshot);
+            if (screenshots.length >= maxScreenshots) break;
+        }
+    }
+    return screenshots;
+};
+const filterEventsByType = (events)=>({
+        navigationEvents: events.filter((event)=>'navigation' === event.type),
+        clickEvents: events.filter((event)=>'click' === event.type),
+        inputEvents: events.filter((event)=>'input' === event.type),
+        scrollEvents: events.filter((event)=>'scroll' === event.type)
+    });
+const createEventCounts = (filteredEvents, totalEvents)=>({
+        navigation: filteredEvents.navigationEvents.length,
+        click: filteredEvents.clickEvents.length,
+        input: filteredEvents.inputEvents.length,
+        scroll: filteredEvents.scrollEvents.length,
+        total: totalEvents
+    });
+const extractInputDescriptions = (inputEvents)=>inputEvents.map((event)=>({
+            description: event.elementDescription || '',
+            value: event.value || ''
+        })).filter((item)=>item.description && item.value);
+const processEventsForLLM = (events)=>events.map((event)=>({
+            type: event.type,
+            timestamp: event.timestamp,
+            url: event.url,
+            title: event.title,
+            elementDescription: event.elementDescription,
+            value: event.value,
+            pageInfo: event.pageInfo,
+            elementRect: event.elementRect
+        }));
+const prepareEventSummary = (events, options = {})=>{
+    const filteredEvents = filterEventsByType(events);
+    const eventCounts = createEventCounts(filteredEvents, events.length);
+    const startUrl = filteredEvents.navigationEvents.length > 0 ? filteredEvents.navigationEvents[0].url || '' : '';
+    const clickDescriptions = filteredEvents.clickEvents.map((event)=>event.elementDescription).filter((desc)=>Boolean(desc)).slice(0, 10);
+    const inputDescriptions = extractInputDescriptions(filteredEvents.inputEvents).slice(0, 10);
+    const urls = filteredEvents.navigationEvents.map((e)=>e.url).filter((url)=>Boolean(url)).slice(0, 5);
+    const processedEvents = processEventsForLLM(events);
+    return {
+        testName: options.testName || 'Automated test from recorded events',
+        startUrl,
+        eventCounts,
+        urls,
+        clickDescriptions,
+        inputDescriptions,
+        events: processedEvents
+    };
+};
+const createMessageContent = (promptText, screenshots = [], includeScreenshots = true)=>{
+    const messageContent = [
+        {
+            type: 'text',
+            text: promptText
+        }
+    ];
+    if (includeScreenshots && screenshots.length > 0) {
+        messageContent.unshift({
+            type: 'text',
+            text: 'Here are screenshots from the recording session to help you understand the context:'
+        });
+        screenshots.forEach((screenshot)=>{
+            messageContent.push({
+                type: 'image_url',
+                image_url: {
+                    url: screenshot
+                }
+            });
+        });
+    }
+    return messageContent;
+};
+const getYamlLanguageInstruction = (language)=>{
+    const normalizedLanguage = language?.trim();
+    if (!normalizedLanguage) return '';
+    return `
+Language requirement:
+- Write all human-readable YAML content in ${normalizedLanguage}.
+- Keep YAML keys, field names, and Midscene API names unchanged.`;
+};
+const createYamlPrompt = ({ yamlSummary, screenshots, language })=>{
+    const prompt = [
+        {
+            role: 'system',
+            content: `You are an expert in RPA智能助手 YAML test generation. Generate clean, accurate YAML following these rules: ${YAML_EXAMPLE_CODE}`
+        },
+        {
+            role: 'user',
+            content: `Generate YAML test for RPA智能助手 automation from recorded browser events.
+
+Event Summary:
+${JSON.stringify(yamlSummary, null, 2)}
+
+Convert events:
+- navigation → target.url
+- click → aiTap with element description
+- input → aiInput with value and locate
+- scroll → aiScroll with appropriate direction
+- Add aiAssert for important state changes${getYamlLanguageInstruction(language)}
+
+Important: Return ONLY the raw YAML content. Do NOT wrap the response in markdown code blocks (no \`\`\`yaml or \`\`\`). Start directly with the YAML content.`
+        }
+    ];
+    if (screenshots.length > 0) {
+        prompt.push({
+            role: 'user',
+            content: 'Here are screenshots from the recording session to help you understand the context:'
+        });
+        prompt.push({
+            role: 'user',
+            content: screenshots.map((screenshot)=>({
+                    type: 'image_url',
+                    image_url: {
+                        url: screenshot
+                    }
+                }))
+        });
+    }
+    return prompt;
+};
+const validateEvents = (events)=>{
+    if (!events.length) throw new Error('No events provided for test generation');
+};
+const generateYamlTest = async (events, options, modelConfig)=>{
+    try {
+        validateEvents(events);
+        const summary = prepareEventSummary(events, {
+            testName: options.testName,
+            maxScreenshots: options.maxScreenshots || 3
+        });
+        const yamlSummary = {
+            ...summary,
+            includeTimestamps: options.includeTimestamps || false
+        };
+        const screenshots = getScreenshotsForLLM(events, options.maxScreenshots || 3);
+        const prompt = createYamlPrompt({
+            yamlSummary,
+            screenshots,
+            language: options.language
+        });
+        const response = await callAIWithStringResponse(prompt, modelConfig);
+        if (response?.content && 'string' == typeof response.content) return response.content;
+        throw new Error('Failed to generate YAML test configuration');
+    } catch (error) {
+        throw new Error(`Failed to generate YAML test: ${error}`);
+    }
+};
+const generateYamlTestStream = async (events, options, modelConfig)=>{
+    try {
+        validateEvents(events);
+        const summary = prepareEventSummary(events, {
+            testName: options.testName,
+            maxScreenshots: options.maxScreenshots || 3
+        });
+        const yamlSummary = {
+            ...summary,
+            includeTimestamps: options.includeTimestamps || false
+        };
+        const screenshots = getScreenshotsForLLM(events, options.maxScreenshots || 3);
+        const prompt = createYamlPrompt({
+            yamlSummary,
+            screenshots,
+            language: options.language
+        });
+        if (options.stream && options.onChunk) return await callAI(prompt, modelConfig, {
+            stream: true,
+            onChunk: options.onChunk
+        });
+        {
+            const response = await callAIWithStringResponse(prompt, modelConfig);
+            if (response?.content && 'string' == typeof response.content) return {
+                content: response.content,
+                usage: response.usage,
+                isStreamed: false
+            };
+            throw new Error('Failed to generate YAML test configuration');
+        }
+    } catch (error) {
+        throw new Error(`Failed to generate YAML test: ${error}`);
+    }
+};
+export { createEventCounts, createMessageContent, extractInputDescriptions, filterEventsByType, generateYamlTest, generateYamlTestStream, getScreenshotsForLLM, prepareEventSummary, processEventsForLLM, validateEvents };