@midscene/playground 1.9.8 → 1.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/es/index.mjs +3 -2
- package/dist/es/launcher.mjs +4 -3
- package/dist/es/launcher.mjs.map +1 -1
- package/dist/es/recorder-ai-service.mjs +2 -0
- package/dist/es/recorder-ui-describer-prompt.mjs +56 -0
- package/dist/es/recorder-ui-describer-prompt.mjs.map +1 -0
- package/dist/es/recorder-ui-describer.mjs +392 -0
- package/dist/es/recorder-ui-describer.mjs.map +1 -0
- package/dist/es/server.mjs +32 -8
- package/dist/es/server.mjs.map +1 -1
- package/dist/lib/index.js +19 -6
- package/dist/lib/launcher.js +3 -2
- package/dist/lib/launcher.js.map +1 -1
- package/dist/lib/recorder-ai-service.js +38 -0
- package/dist/lib/recorder-ai-service.js.map +1 -0
- package/dist/lib/recorder-ui-describer-prompt.js +90 -0
- package/dist/lib/recorder-ui-describer-prompt.js.map +1 -0
- package/dist/lib/recorder-ui-describer.js +432 -0
- package/dist/lib/recorder-ui-describer.js.map +1 -0
- package/dist/lib/server.js +40 -7
- package/dist/lib/server.js.map +1 -1
- package/dist/types/index.d.ts +2 -1
- package/dist/types/recorder-ai-service.d.ts +1 -0
- package/dist/types/recorder-ui-describer-prompt.d.ts +1 -0
- package/dist/types/recorder-ui-describer.d.ts +20 -0
- package/dist/types/server.d.ts +4 -0
- package/package.json +9 -3
- package/static/index.html +1 -1
- package/static/static/js/index.9c67d55f.js +951 -0
- package/static/static/js/index.9c67d55f.js.map +1 -0
- package/static/static/js/index.44b83907.js +0 -948
- package/static/static/js/index.44b83907.js.map +0 -1
- /package/static/static/js/{index.44b83907.js.LICENSE.txt → index.9c67d55f.js.LICENSE.txt} +0 -0
|
@@ -0,0 +1,432 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __webpack_require__ = {};
|
|
3
|
+
(()=>{
|
|
4
|
+
__webpack_require__.d = (exports1, definition)=>{
|
|
5
|
+
for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
|
|
6
|
+
enumerable: true,
|
|
7
|
+
get: definition[key]
|
|
8
|
+
});
|
|
9
|
+
};
|
|
10
|
+
})();
|
|
11
|
+
(()=>{
|
|
12
|
+
__webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
|
|
13
|
+
})();
|
|
14
|
+
(()=>{
|
|
15
|
+
__webpack_require__.r = (exports1)=>{
|
|
16
|
+
if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
|
|
17
|
+
value: 'Module'
|
|
18
|
+
});
|
|
19
|
+
Object.defineProperty(exports1, '__esModule', {
|
|
20
|
+
value: true
|
|
21
|
+
});
|
|
22
|
+
};
|
|
23
|
+
})();
|
|
24
|
+
var __webpack_exports__ = {};
|
|
25
|
+
__webpack_require__.r(__webpack_exports__);
|
|
26
|
+
__webpack_require__.d(__webpack_exports__, {
|
|
27
|
+
getRecorderUIEventTargetRect: ()=>getRecorderUIEventTargetRect,
|
|
28
|
+
describeRecorderUIEvent: ()=>describeRecorderUIEvent,
|
|
29
|
+
describeRecorderUIEvents: ()=>describeRecorderUIEvents
|
|
30
|
+
});
|
|
31
|
+
const env_namespaceObject = require("@midscene/shared/env");
|
|
32
|
+
const recorder_namespaceObject = require("@midscene/shared/recorder");
|
|
33
|
+
const external_recorder_ai_service_js_namespaceObject = require("./recorder-ai-service.js");
|
|
34
|
+
const external_recorder_ui_describer_prompt_js_namespaceObject = require("./recorder-ui-describer-prompt.js");
|
|
35
|
+
const RECORDER_UI_DESCRIBER_DEFAULT_RETRIES = 2;
|
|
36
|
+
const RECORDER_UI_DESCRIBER_DEFAULT_RETRY_DELAY_MS = 200;
|
|
37
|
+
const RECORDER_UI_DESCRIBER_DEFAULT_CONCURRENCY = 2;
|
|
38
|
+
function delay(ms) {
|
|
39
|
+
return new Promise((resolve)=>setTimeout(resolve, ms));
|
|
40
|
+
}
|
|
41
|
+
function isResolvedModelConfig(modelConfig) {
|
|
42
|
+
return 'string' == typeof modelConfig.modelName && ('string' == typeof modelConfig.modelDescription || 'string' == typeof modelConfig.intent || 'string' == typeof modelConfig.slot || 'string' == typeof modelConfig.openaiApiKey);
|
|
43
|
+
}
|
|
44
|
+
function resolveRecorderModelConfig(modelConfig) {
|
|
45
|
+
if (isResolvedModelConfig(modelConfig)) return modelConfig;
|
|
46
|
+
const rawModelConfig = Object.entries(modelConfig).reduce((result, [key, value])=>{
|
|
47
|
+
if ('string' == typeof value || 'number' == typeof value) result[key] = value;
|
|
48
|
+
return result;
|
|
49
|
+
}, {});
|
|
50
|
+
return new env_namespaceObject.ModelConfigManager(rawModelConfig).getModelConfig('default');
|
|
51
|
+
}
|
|
52
|
+
function clamp(value, min, max) {
|
|
53
|
+
return Math.min(Math.max(value, min), max);
|
|
54
|
+
}
|
|
55
|
+
function isFiniteNumber(value) {
|
|
56
|
+
return 'number' == typeof value && Number.isFinite(value);
|
|
57
|
+
}
|
|
58
|
+
function isPendingDescription(value) {
|
|
59
|
+
return value?.trim() === 'AI is analyzing element...';
|
|
60
|
+
}
|
|
61
|
+
function getRecorderEventScreenshot(event) {
|
|
62
|
+
return event.screenshotWithBox || event.screenshotBefore || event.screenshotAfter;
|
|
63
|
+
}
|
|
64
|
+
function getRecorderEventAfterScreenshot(event) {
|
|
65
|
+
return event.screenshotAfter || event.screenshotWithBox;
|
|
66
|
+
}
|
|
67
|
+
function normalizeActionType(event) {
|
|
68
|
+
return event.actionType?.trim();
|
|
69
|
+
}
|
|
70
|
+
function getPlatformId(target) {
|
|
71
|
+
return target?.platformId?.toLowerCase();
|
|
72
|
+
}
|
|
73
|
+
function getPlatformSurface(target) {
|
|
74
|
+
switch(getPlatformId(target)){
|
|
75
|
+
case 'web':
|
|
76
|
+
return 'current web page';
|
|
77
|
+
case 'android':
|
|
78
|
+
case 'ios':
|
|
79
|
+
case 'harmony':
|
|
80
|
+
return 'current mobile screen';
|
|
81
|
+
case 'computer':
|
|
82
|
+
return 'current desktop screen';
|
|
83
|
+
default:
|
|
84
|
+
return 'current UI';
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
function getPlatformGuidance(target) {
|
|
88
|
+
switch(getPlatformId(target)){
|
|
89
|
+
case 'web':
|
|
90
|
+
return 'For web targets, use web UI terms such as button, input, link, menu item, tab, dialog, aria-label, placeholder, and form section when visible or inferable.';
|
|
91
|
+
case 'android':
|
|
92
|
+
case 'ios':
|
|
93
|
+
case 'harmony':
|
|
94
|
+
return 'For mobile targets, use mobile UI terms such as tab, list item, text field, icon button, navigation bar, bottom bar, sheet, card, and screen section.';
|
|
95
|
+
case 'computer':
|
|
96
|
+
return 'For desktop/computer targets, use desktop UI terms such as menu item, toolbar button, dialog field, sidebar item, window control, file row, and application region.';
|
|
97
|
+
default:
|
|
98
|
+
return 'Use platform-neutral UI terms such as control, field, item, icon button, list item, region, panel, and page section.';
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
function getPointerActionVerb(event) {
|
|
102
|
+
switch(normalizeActionType(event)){
|
|
103
|
+
case 'Tap':
|
|
104
|
+
return 'Tap';
|
|
105
|
+
case 'DoubleClick':
|
|
106
|
+
return 'Double click';
|
|
107
|
+
case 'LongPress':
|
|
108
|
+
return 'Long press';
|
|
109
|
+
case 'RightClick':
|
|
110
|
+
return 'Right click';
|
|
111
|
+
default:
|
|
112
|
+
return 'Click';
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
function getDragActionVerb(event) {
|
|
116
|
+
switch(normalizeActionType(event)){
|
|
117
|
+
case 'Swipe':
|
|
118
|
+
return 'Swipe';
|
|
119
|
+
case 'DragAndDrop':
|
|
120
|
+
return 'Drag';
|
|
121
|
+
default:
|
|
122
|
+
return 'Drag';
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
function pointToRect(x, y, size, pageInfo) {
|
|
126
|
+
const width = pageInfo.width || size;
|
|
127
|
+
const height = pageInfo.height || size;
|
|
128
|
+
const left = clamp(Math.floor(x - size / 2), 0, Math.max(width - 1, 0));
|
|
129
|
+
const top = clamp(Math.floor(y - size / 2), 0, Math.max(height - 1, 0));
|
|
130
|
+
return {
|
|
131
|
+
left,
|
|
132
|
+
top,
|
|
133
|
+
width: Math.min(size, Math.max(width - left, 1)),
|
|
134
|
+
height: Math.min(size, Math.max(height - top, 1))
|
|
135
|
+
};
|
|
136
|
+
}
|
|
137
|
+
function getPointRectSize(event) {
|
|
138
|
+
switch(event.type){
|
|
139
|
+
case 'scroll':
|
|
140
|
+
return 96;
|
|
141
|
+
case 'drag':
|
|
142
|
+
return 64;
|
|
143
|
+
default:
|
|
144
|
+
return 36;
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
function getRecorderUIEventTargetRect(event) {
|
|
148
|
+
const rect = event.elementRect;
|
|
149
|
+
if (!rect) return null;
|
|
150
|
+
if (isFiniteNumber(rect.width) && rect.width > 0 && isFiniteNumber(rect.height) && rect.height > 0 && (isFiniteNumber(rect.left) || isFiniteNumber(rect.top))) return {
|
|
151
|
+
left: rect.left || 0,
|
|
152
|
+
top: rect.top || 0,
|
|
153
|
+
width: rect.width,
|
|
154
|
+
height: rect.height
|
|
155
|
+
};
|
|
156
|
+
if (isFiniteNumber(rect.x) && isFiniteNumber(rect.y)) return pointToRect(rect.x, rect.y, getPointRectSize(event), event.pageInfo);
|
|
157
|
+
return null;
|
|
158
|
+
}
|
|
159
|
+
function getFallbackDescription(event, target) {
|
|
160
|
+
const pageContext = getPageSemanticContext(event);
|
|
161
|
+
const surface = getPlatformSurface(target);
|
|
162
|
+
switch(event.type){
|
|
163
|
+
case 'navigation':
|
|
164
|
+
return event.url || event.value || event.actionType || 'navigation';
|
|
165
|
+
case 'scroll':
|
|
166
|
+
return pageContext ? `${pageContext} scrollable content` : `scrollable content on the ${surface}`;
|
|
167
|
+
case 'drag':
|
|
168
|
+
return pageContext ? `gesture area in ${pageContext}` : `gesture area on the ${surface}`;
|
|
169
|
+
case 'input':
|
|
170
|
+
return `unresolved input field on the ${surface}`;
|
|
171
|
+
case 'keydown':
|
|
172
|
+
return pageContext ? `focused control in ${pageContext}` : `focused control on the ${surface}`;
|
|
173
|
+
default:
|
|
174
|
+
return pageContext ? `control in ${pageContext}` : `control on the ${surface}`;
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
function buildSemanticAction(event, scrollDestinationDescription) {
|
|
178
|
+
return {
|
|
179
|
+
type: event.type,
|
|
180
|
+
actionType: event.actionType,
|
|
181
|
+
value: event.value,
|
|
182
|
+
url: event.url,
|
|
183
|
+
scrollDestinationDescription
|
|
184
|
+
};
|
|
185
|
+
}
|
|
186
|
+
function getFallbackReplayInstruction(event, elementDescription) {
|
|
187
|
+
return (0, recorder_namespaceObject.buildMidsceneRecorderReplayInstruction)(buildSemanticAction(event), elementDescription);
|
|
188
|
+
}
|
|
189
|
+
function getFallbackActionSummary(event, elementDescription) {
|
|
190
|
+
return (0, recorder_namespaceObject.buildMidsceneRecorderActionSummary)(buildSemanticAction(event), elementDescription);
|
|
191
|
+
}
|
|
192
|
+
function getActionGuidance(event, target) {
|
|
193
|
+
const platformGuidance = getPlatformGuidance(target);
|
|
194
|
+
switch(event.type){
|
|
195
|
+
case 'click':
|
|
196
|
+
return `${platformGuidance} Identify the ${getPointerActionVerb(event).toLowerCase()} target by exact visible text first, then label/placeholder, then role plus stable surrounding context, then icon purpose, then visual position. Never describe it by coordinates, marker location, or as a nearby element.`;
|
|
197
|
+
case 'input':
|
|
198
|
+
return `${platformGuidance} Identify the exact input field at the marker before text entry. Use stable visible label, field role, field name, surrounding section, or sequence intent. Treat hint text that can change by user, time, data, or context as secondary evidence. Preserve the recorded input value only in replayInstruction; never describe the typed value or page title alone as the field.`;
|
|
199
|
+
case 'scroll':
|
|
200
|
+
return `${platformGuidance} Identify the scrollable page/region at the highlighted scroll point and concrete destination content revealed after scrolling. If multiple scrollable regions are visible, preserve the specific region where the scroll happened, such as left/right/top/bottom panel, navigation area, content pane, dialog body, table, list, or menu; do not generalize a panel/list scroll into the whole page. Use newly visible headings, section titles, list/table names, list items, or stable region labels; never say only "more content" or "current page".`;
|
|
201
|
+
case 'drag':
|
|
202
|
+
return `${platformGuidance} Identify the ${getDragActionVerb(event).toLowerCase()} start/end regions or the dragged UI control. Do not describe only the gesture path or coordinates.`;
|
|
203
|
+
case 'keydown':
|
|
204
|
+
return `${platformGuidance} Identify the focused element or keyboard target if visible, and preserve the recorded key in the replay instruction.`;
|
|
205
|
+
default:
|
|
206
|
+
return `${platformGuidance} Identify the UI target involved in this event using the most stable visible text or surrounding context.`;
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
function getEventRawCoordinates(event) {
|
|
210
|
+
const x = event.elementRect?.x;
|
|
211
|
+
const y = event.elementRect?.y;
|
|
212
|
+
if (isFiniteNumber(x) && isFiniteNumber(y)) return {
|
|
213
|
+
x,
|
|
214
|
+
y
|
|
215
|
+
};
|
|
216
|
+
}
|
|
217
|
+
function getPageSemanticContext(event) {
|
|
218
|
+
const candidates = [
|
|
219
|
+
event.title,
|
|
220
|
+
event.url
|
|
221
|
+
].map((item)=>item?.trim()).filter(Boolean);
|
|
222
|
+
return candidates[0];
|
|
223
|
+
}
|
|
224
|
+
function isWeakDescription(value) {
|
|
225
|
+
if (!value) return true;
|
|
226
|
+
if (isPendingDescription(value)) return true;
|
|
227
|
+
const normalized = value.trim().toLowerCase();
|
|
228
|
+
const compact = normalized.replace(/\s+/g, '');
|
|
229
|
+
return 0 === normalized.length || /^\(?\d+(?:\.\d+)?,\s*\d+(?:\.\d+)?\)?$/.test(normalized) || 'target' === normalized || 'element' === normalized || 'target element' === normalized || 'the element' === normalized || 'page element' === normalized || 'input field' === normalized || 'text input' === normalized || 'text field' === normalized || 'search box' === normalized || 'more content' === normalized || 'the page' === normalized || 'current page' === normalized || 'current screen' === normalized || 'the screen' === normalized || 'current ui' === normalized || 'current visible ui' === normalized || 'current visible page' === normalized || 'current visible screen' === normalized || 'main area' === normalized || 'main scrollable area' === normalized || 'scrollable area' === normalized || 'highlighted element' === normalized || 'highlighted item' === normalized || 'marked element' === normalized || 'marked item' === normalized || normalized.includes('ai is analyzing element') || compact.includes('坐标') || compact.includes('附近') || compact.includes('附近的元素') || normalized.includes('coordinate') || normalized.includes('near the coordinate') || normalized.includes('near coordinates') || normalized.includes('nearby element') || normalized.includes('nearby item') || normalized.includes('near the marker') || normalized.includes('near marker') || normalized.includes('near the point') || normalized.includes('near point') || normalized.includes('at the point') || normalized.includes('button near point') || normalized.includes('shown in the screenshot') || normalized.includes('red rectangle') || normalized.includes('red marker') || normalized.includes('red box') || normalized.includes('highlighted element') || normalized.includes('highlighted item') || normalized.includes('highlighted screenshot');
|
|
230
|
+
}
|
|
231
|
+
function isWeakReplayInstruction(value) {
|
|
232
|
+
if (!value) return true;
|
|
233
|
+
if (isPendingDescription(value)) return true;
|
|
234
|
+
const normalized = value.trim().toLowerCase();
|
|
235
|
+
const compact = normalized.replace(/\s+/g, '');
|
|
236
|
+
return compact.includes('坐标') || compact.includes('附近') || normalized.includes('coordinate') || normalized.includes('near the coordinate') || normalized.includes('nearby element') || normalized.includes('nearby item') || normalized.includes('near the marker') || normalized.includes('near marker') || normalized.includes('near the point') || normalized.includes('near point') || normalized.includes('at the point') || normalized.includes('ai is analyzing element') || normalized.includes('more content') || normalized.includes('current page') || normalized.includes('current screen') || normalized.includes('highlighted element') || normalized.includes('highlighted item') || normalized.includes('red marker') || normalized.includes('red box') || normalized.includes('shown in the screenshot') || normalized.includes('highlighted screenshot');
|
|
237
|
+
}
|
|
238
|
+
function normalizeForComparison(value) {
|
|
239
|
+
return value.trim().toLowerCase().replace(/["'`]/g, '').replace(/\s+/g, ' ');
|
|
240
|
+
}
|
|
241
|
+
function isInputValueUsedAsFieldDescription(event, elementDescription) {
|
|
242
|
+
if ('input' !== event.type || !event.value || !elementDescription) return false;
|
|
243
|
+
const typedValue = normalizeForComparison(event.value);
|
|
244
|
+
if (!typedValue) return false;
|
|
245
|
+
const description = normalizeForComparison(elementDescription);
|
|
246
|
+
return description === typedValue || description === `${typedValue} input` || description === `${typedValue} field` || description === `${typedValue} text field` || description === `input ${typedValue}` || description === `field ${typedValue}` || description.includes(`typed value ${typedValue}`) || description.includes(`value ${typedValue}`);
|
|
247
|
+
}
|
|
248
|
+
function hasScrollDestination(replayInstruction, scrollDestinationDescription) {
|
|
249
|
+
if (scrollDestinationDescription && !isWeakDescription(scrollDestinationDescription)) return true;
|
|
250
|
+
const normalized = replayInstruction.toLowerCase();
|
|
251
|
+
return normalized.includes(' until ') || normalized.includes(' visible') || normalized.includes(' reveal') || normalized.includes(' to the ') || normalized.includes(' toward ');
|
|
252
|
+
}
|
|
253
|
+
async function describeWithRetry(event, target, highlightedScreenshot, modelConfig, options) {
|
|
254
|
+
let lastError;
|
|
255
|
+
for(let attempt = 1; attempt <= options.maxRetries; attempt += 1)try {
|
|
256
|
+
const afterScreenshot = getRecorderEventAfterScreenshot(event);
|
|
257
|
+
const pageContext = getPageSemanticContext(event);
|
|
258
|
+
const platformGuidance = getPlatformGuidance(target);
|
|
259
|
+
const userContent = [
|
|
260
|
+
{
|
|
261
|
+
type: 'text',
|
|
262
|
+
text: `Recorder event:
|
|
263
|
+
${JSON.stringify({
|
|
264
|
+
type: event.type,
|
|
265
|
+
actionType: event.actionType,
|
|
266
|
+
value: event.value,
|
|
267
|
+
rawCoordinates: getEventRawCoordinates(event),
|
|
268
|
+
url: event.url,
|
|
269
|
+
title: event.title,
|
|
270
|
+
pageContext,
|
|
271
|
+
pageInfo: event.pageInfo,
|
|
272
|
+
target,
|
|
273
|
+
platformGuidance,
|
|
274
|
+
guidance: getActionGuidance(event, target)
|
|
275
|
+
}, null, 2)}
|
|
276
|
+
|
|
277
|
+
The target or region is highlighted in the screenshot below. Convert this event into semantic replay fields.`
|
|
278
|
+
},
|
|
279
|
+
{
|
|
280
|
+
type: 'image_url',
|
|
281
|
+
image_url: {
|
|
282
|
+
url: highlightedScreenshot,
|
|
283
|
+
detail: 'high'
|
|
284
|
+
}
|
|
285
|
+
}
|
|
286
|
+
];
|
|
287
|
+
if (afterScreenshot) userContent.push({
|
|
288
|
+
type: 'text',
|
|
289
|
+
text: 'Screenshot after the recorded action, for context only:'
|
|
290
|
+
}, {
|
|
291
|
+
type: 'image_url',
|
|
292
|
+
image_url: {
|
|
293
|
+
url: afterScreenshot,
|
|
294
|
+
detail: 'high'
|
|
295
|
+
}
|
|
296
|
+
});
|
|
297
|
+
const response = await (0, external_recorder_ai_service_js_namespaceObject.callAIWithObjectResponse)([
|
|
298
|
+
{
|
|
299
|
+
role: 'system',
|
|
300
|
+
content: external_recorder_ui_describer_prompt_js_namespaceObject.RECORDER_UI_DESCRIBER_SYSTEM_PROMPT
|
|
301
|
+
},
|
|
302
|
+
{
|
|
303
|
+
role: 'user',
|
|
304
|
+
content: userContent
|
|
305
|
+
}
|
|
306
|
+
], modelConfig);
|
|
307
|
+
const content = response.content;
|
|
308
|
+
if (content.error) throw new Error(content.error);
|
|
309
|
+
if (isWeakDescription(content.elementDescription)) throw new Error("AI returned a weak recorder event description.");
|
|
310
|
+
if (isInputValueUsedAsFieldDescription(event, content.elementDescription)) throw new Error("AI used the recorded input value as the field description.");
|
|
311
|
+
const elementDescription = content.elementDescription.trim();
|
|
312
|
+
const scrollDestinationDescription = 'scroll' === event.type ? content.scrollDestinationDescription?.trim() : void 0;
|
|
313
|
+
if ('scroll' === event.type && !hasScrollDestination('', scrollDestinationDescription)) throw new Error("AI returned a scroll description without a destination.");
|
|
314
|
+
const aiReplayInstruction = content.replayInstruction?.trim();
|
|
315
|
+
if (aiReplayInstruction && isWeakReplayInstruction(aiReplayInstruction)) throw new Error('AI returned a weak recorder replay instruction.');
|
|
316
|
+
const semanticAction = buildSemanticAction(event, scrollDestinationDescription);
|
|
317
|
+
const replayInstruction = (0, recorder_namespaceObject.buildMidsceneRecorderReplayInstruction)(semanticAction, elementDescription);
|
|
318
|
+
if (isWeakReplayInstruction(replayInstruction)) throw new Error('AI returned a weak recorder replay instruction.');
|
|
319
|
+
const actionSummary = (0, recorder_namespaceObject.buildMidsceneRecorderActionSummary)(semanticAction, elementDescription);
|
|
320
|
+
return {
|
|
321
|
+
source: 'recorderAI',
|
|
322
|
+
status: 'ready',
|
|
323
|
+
elementDescription,
|
|
324
|
+
replayInstruction,
|
|
325
|
+
actionSummary,
|
|
326
|
+
confidence: content.confidence || 'medium'
|
|
327
|
+
};
|
|
328
|
+
} catch (error) {
|
|
329
|
+
lastError = error;
|
|
330
|
+
if (attempt < options.maxRetries) await delay(options.retryDelayMs);
|
|
331
|
+
}
|
|
332
|
+
throw lastError;
|
|
333
|
+
}
|
|
334
|
+
async function createScreenshotWithBox(event, rect) {
|
|
335
|
+
if (event.screenshotWithBox) return event.screenshotWithBox;
|
|
336
|
+
const screenshot = getRecorderEventScreenshot(event);
|
|
337
|
+
if (!screenshot) return;
|
|
338
|
+
const { compositeElementInfoImg } = await import(/* @vite-ignore */ "@midscene/shared/img");
|
|
339
|
+
return compositeElementInfoImg({
|
|
340
|
+
inputImgBase64: screenshot,
|
|
341
|
+
size: event.pageInfo,
|
|
342
|
+
elementsPositionInfo: [
|
|
343
|
+
{
|
|
344
|
+
rect
|
|
345
|
+
}
|
|
346
|
+
],
|
|
347
|
+
borderThickness: 3,
|
|
348
|
+
annotationPadding: 2
|
|
349
|
+
});
|
|
350
|
+
}
|
|
351
|
+
function createFallbackEvent(event, error, screenshotWithBox, target) {
|
|
352
|
+
const semantic = (0, recorder_namespaceObject.getMidsceneRecorderSemantic)(event);
|
|
353
|
+
const elementDescription = semantic?.elementDescription && !isWeakDescription(semantic.elementDescription) ? semantic.elementDescription : getFallbackDescription(event, target);
|
|
354
|
+
return {
|
|
355
|
+
...event,
|
|
356
|
+
semantic: {
|
|
357
|
+
source: 'heuristic',
|
|
358
|
+
status: 'ready',
|
|
359
|
+
elementDescription,
|
|
360
|
+
replayInstruction: getFallbackReplayInstruction(event, elementDescription),
|
|
361
|
+
actionSummary: getFallbackActionSummary(event, elementDescription),
|
|
362
|
+
confidence: 'low',
|
|
363
|
+
error
|
|
364
|
+
},
|
|
365
|
+
screenshotWithBox: screenshotWithBox || event.screenshotWithBox
|
|
366
|
+
};
|
|
367
|
+
}
|
|
368
|
+
async function describeRecorderUIEvent(input, modelConfig, options = {}) {
|
|
369
|
+
const event = input.event;
|
|
370
|
+
const rect = getRecorderUIEventTargetRect(event);
|
|
371
|
+
const screenshot = getRecorderEventScreenshot(event);
|
|
372
|
+
if (!rect || !screenshot) {
|
|
373
|
+
const error = rect ? 'Recorder event has no screenshot.' : 'Recorder event has no target rectangle.';
|
|
374
|
+
return {
|
|
375
|
+
usedFallback: true,
|
|
376
|
+
event: createFallbackEvent(event, error, void 0, input.target)
|
|
377
|
+
};
|
|
378
|
+
}
|
|
379
|
+
let screenshotWithBox;
|
|
380
|
+
try {
|
|
381
|
+
const resolvedModelConfig = resolveRecorderModelConfig(modelConfig);
|
|
382
|
+
screenshotWithBox = await createScreenshotWithBox(event, rect);
|
|
383
|
+
const semanticFields = await describeWithRetry(event, input.target, screenshotWithBox || screenshot, resolvedModelConfig, {
|
|
384
|
+
maxRetries: options.maxRetries ?? RECORDER_UI_DESCRIBER_DEFAULT_RETRIES,
|
|
385
|
+
retryDelayMs: options.retryDelayMs ?? RECORDER_UI_DESCRIBER_DEFAULT_RETRY_DELAY_MS
|
|
386
|
+
});
|
|
387
|
+
return {
|
|
388
|
+
usedFallback: false,
|
|
389
|
+
event: {
|
|
390
|
+
...event,
|
|
391
|
+
semantic: semanticFields,
|
|
392
|
+
screenshotWithBox: screenshotWithBox || event.screenshotWithBox
|
|
393
|
+
}
|
|
394
|
+
};
|
|
395
|
+
} catch (error) {
|
|
396
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
397
|
+
return {
|
|
398
|
+
usedFallback: true,
|
|
399
|
+
error: message,
|
|
400
|
+
event: createFallbackEvent(event, message, screenshotWithBox, input.target)
|
|
401
|
+
};
|
|
402
|
+
}
|
|
403
|
+
}
|
|
404
|
+
async function describeRecorderUIEvents(inputs, modelConfig, options = {}) {
|
|
405
|
+
const concurrency = Math.max(1, options.concurrency ?? RECORDER_UI_DESCRIBER_DEFAULT_CONCURRENCY);
|
|
406
|
+
const results = new Array(inputs.length);
|
|
407
|
+
let cursor = 0;
|
|
408
|
+
async function worker() {
|
|
409
|
+
while(cursor < inputs.length){
|
|
410
|
+
const index = cursor;
|
|
411
|
+
cursor += 1;
|
|
412
|
+
results[index] = await describeRecorderUIEvent(inputs[index], modelConfig, options);
|
|
413
|
+
}
|
|
414
|
+
}
|
|
415
|
+
await Promise.all(Array.from({
|
|
416
|
+
length: Math.min(concurrency, inputs.length)
|
|
417
|
+
}, ()=>worker()));
|
|
418
|
+
return results;
|
|
419
|
+
}
|
|
420
|
+
exports.describeRecorderUIEvent = __webpack_exports__.describeRecorderUIEvent;
|
|
421
|
+
exports.describeRecorderUIEvents = __webpack_exports__.describeRecorderUIEvents;
|
|
422
|
+
exports.getRecorderUIEventTargetRect = __webpack_exports__.getRecorderUIEventTargetRect;
|
|
423
|
+
for(var __rspack_i in __webpack_exports__)if (-1 === [
|
|
424
|
+
"describeRecorderUIEvent",
|
|
425
|
+
"describeRecorderUIEvents",
|
|
426
|
+
"getRecorderUIEventTargetRect"
|
|
427
|
+
].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
|
|
428
|
+
Object.defineProperty(exports, '__esModule', {
|
|
429
|
+
value: true
|
|
430
|
+
});
|
|
431
|
+
|
|
432
|
+
//# sourceMappingURL=recorder-ui-describer.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"recorder-ui-describer.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../src/recorder-ui-describer.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n }\n }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { Rect } from '@midscene/core';\nimport { type IModelConfig, ModelConfigManager } from '@midscene/shared/env';\nimport type {\n MidsceneRecorderEvent,\n MidsceneRecorderPageInfo,\n MidsceneRecorderSemanticAction,\n MidsceneRecorderTarget,\n} from '@midscene/shared/recorder';\nimport {\n buildMidsceneRecorderActionSummary,\n buildMidsceneRecorderReplayInstruction,\n getMidsceneRecorderSemantic,\n} from '@midscene/shared/recorder';\nimport { callAIWithObjectResponse } from './recorder-ai-service';\nimport { RECORDER_UI_DESCRIBER_SYSTEM_PROMPT } from './recorder-ui-describer-prompt';\n\nexport interface DescribeRecorderUIEventInput {\n event: MidsceneRecorderEvent;\n target?: MidsceneRecorderTarget;\n}\n\nexport interface DescribeRecorderUIEventOptions {\n maxRetries?: number;\n retryDelayMs?: number;\n concurrency?: number;\n}\n\nexport interface DescribeRecorderUIEventResult {\n event: MidsceneRecorderEvent;\n usedFallback: boolean;\n error?: string;\n}\n\ninterface RecorderUIEventAIResponse {\n elementDescription?: string;\n replayInstruction?: string;\n actionSummary?: string;\n scrollDestinationDescription?: string;\n confidence?: 'high' | 'medium' | 'low';\n error?: string;\n}\n\nconst RECORDER_UI_DESCRIBER_DEFAULT_RETRIES = 2;\nconst RECORDER_UI_DESCRIBER_DEFAULT_RETRY_DELAY_MS = 200;\nconst RECORDER_UI_DESCRIBER_DEFAULT_CONCURRENCY = 2;\n\nfunction delay(ms: number) {\n return new Promise((resolve) => setTimeout(resolve, ms));\n}\n\nfunction isResolvedModelConfig(\n modelConfig: IModelConfig | Record<string, unknown>,\n): modelConfig is IModelConfig {\n return (\n typeof modelConfig.modelName === 'string' &&\n (typeof modelConfig.modelDescription === 'string' ||\n typeof modelConfig.intent === 'string' ||\n typeof modelConfig.slot === 'string' ||\n typeof modelConfig.openaiApiKey === 'string')\n );\n}\n\nfunction resolveRecorderModelConfig(\n modelConfig: IModelConfig | Record<string, unknown>,\n): IModelConfig {\n if (isResolvedModelConfig(modelConfig)) {\n return modelConfig;\n }\n const rawModelConfig = Object.entries(modelConfig).reduce<\n Record<string, string | number>\n >((result, [key, value]) => {\n if (typeof value === 'string' || typeof value === 'number') {\n result[key] = value;\n }\n return result;\n }, {});\n return new ModelConfigManager(rawModelConfig).getModelConfig('default');\n}\n\nfunction clamp(value: number, min: number, max: number) {\n return Math.min(Math.max(value, min), max);\n}\n\nfunction isFiniteNumber(value: unknown): value is number {\n return typeof value === 'number' && Number.isFinite(value);\n}\n\nfunction isPendingDescription(value?: string) {\n return value?.trim() === 'AI is analyzing element...';\n}\n\nfunction getRecorderEventScreenshot(event: MidsceneRecorderEvent) {\n return (\n event.screenshotWithBox || event.screenshotBefore || event.screenshotAfter\n );\n}\n\nfunction getRecorderEventAfterScreenshot(event: MidsceneRecorderEvent) {\n return event.screenshotAfter || event.screenshotWithBox;\n}\n\nfunction normalizeActionType(event: MidsceneRecorderEvent) {\n return event.actionType?.trim();\n}\n\nfunction getPlatformId(target?: MidsceneRecorderTarget) {\n return target?.platformId?.toLowerCase();\n}\n\nfunction getPlatformSurface(target?: MidsceneRecorderTarget) {\n switch (getPlatformId(target)) {\n case 'web':\n return 'current web page';\n case 'android':\n case 'ios':\n case 'harmony':\n return 'current mobile screen';\n case 'computer':\n return 'current desktop screen';\n default:\n return 'current UI';\n }\n}\n\nfunction getPlatformGuidance(target?: MidsceneRecorderTarget) {\n switch (getPlatformId(target)) {\n case 'web':\n return 'For web targets, use web UI terms such as button, input, link, menu item, tab, dialog, aria-label, placeholder, and form section when visible or inferable.';\n case 'android':\n case 'ios':\n case 'harmony':\n return 'For mobile targets, use mobile UI terms such as tab, list item, text field, icon button, navigation bar, bottom bar, sheet, card, and screen section.';\n case 'computer':\n return 'For desktop/computer targets, use desktop UI terms such as menu item, toolbar button, dialog field, sidebar item, window control, file row, and application region.';\n default:\n return 'Use platform-neutral UI terms such as control, field, item, icon button, list item, region, panel, and page section.';\n }\n}\n\nfunction getPointerActionVerb(event: MidsceneRecorderEvent) {\n switch (normalizeActionType(event)) {\n case 'Tap':\n return 'Tap';\n case 'DoubleClick':\n return 'Double click';\n case 'LongPress':\n return 'Long press';\n case 'RightClick':\n return 'Right click';\n default:\n return 'Click';\n }\n}\n\nfunction getDragActionVerb(event: MidsceneRecorderEvent) {\n switch (normalizeActionType(event)) {\n case 'Swipe':\n return 'Swipe';\n case 'DragAndDrop':\n return 'Drag';\n default:\n return 'Drag';\n }\n}\n\nfunction pointToRect(\n x: number,\n y: number,\n size: number,\n pageInfo: MidsceneRecorderPageInfo,\n): Rect {\n const width = pageInfo.width || size;\n const height = pageInfo.height || size;\n const left = clamp(Math.floor(x - size / 2), 0, Math.max(width - 1, 0));\n const top = clamp(Math.floor(y - size / 2), 0, Math.max(height - 1, 0));\n return {\n left,\n top,\n width: Math.min(size, Math.max(width - left, 1)),\n height: Math.min(size, Math.max(height - top, 1)),\n };\n}\n\nfunction getPointRectSize(event: MidsceneRecorderEvent) {\n switch (event.type) {\n case 'scroll':\n return 96;\n case 'drag':\n return 64;\n default:\n return 36;\n }\n}\n\nexport function getRecorderUIEventTargetRect(\n event: MidsceneRecorderEvent,\n): Rect | null {\n const rect = event.elementRect;\n if (!rect) {\n return null;\n }\n\n if (\n isFiniteNumber(rect.width) &&\n rect.width > 0 &&\n isFiniteNumber(rect.height) &&\n rect.height > 0 &&\n (isFiniteNumber(rect.left) || isFiniteNumber(rect.top))\n ) {\n return {\n left: rect.left || 0,\n top: rect.top || 0,\n width: rect.width,\n height: rect.height,\n };\n }\n\n if (isFiniteNumber(rect.x) && isFiniteNumber(rect.y)) {\n return pointToRect(rect.x, rect.y, getPointRectSize(event), event.pageInfo);\n }\n\n return null;\n}\n\nfunction getFallbackDescription(\n event: MidsceneRecorderEvent,\n target?: MidsceneRecorderTarget,\n) {\n const pageContext = getPageSemanticContext(event);\n const surface = getPlatformSurface(target);\n\n switch (event.type) {\n case 'navigation':\n return event.url || event.value || event.actionType || 'navigation';\n case 'scroll':\n return pageContext\n ? `${pageContext} scrollable content`\n : `scrollable content on the ${surface}`;\n case 'drag':\n return pageContext\n ? `gesture area in ${pageContext}`\n : `gesture area on the ${surface}`;\n case 'input':\n return `unresolved input field on the ${surface}`;\n case 'keydown':\n return pageContext\n ? `focused control in ${pageContext}`\n : `focused control on the ${surface}`;\n default:\n return pageContext\n ? `control in ${pageContext}`\n : `control on the ${surface}`;\n }\n}\n\nfunction buildSemanticAction(\n event: MidsceneRecorderEvent,\n scrollDestinationDescription?: string,\n): MidsceneRecorderSemanticAction {\n return {\n type: event.type,\n actionType: event.actionType,\n value: event.value,\n url: event.url,\n scrollDestinationDescription,\n };\n}\n\nfunction getFallbackReplayInstruction(\n event: MidsceneRecorderEvent,\n elementDescription: string,\n) {\n return buildMidsceneRecorderReplayInstruction(\n buildSemanticAction(event),\n elementDescription,\n );\n}\n\nfunction getFallbackActionSummary(\n event: MidsceneRecorderEvent,\n elementDescription: string,\n) {\n return buildMidsceneRecorderActionSummary(\n buildSemanticAction(event),\n elementDescription,\n );\n}\n\nfunction getActionGuidance(\n event: MidsceneRecorderEvent,\n target?: MidsceneRecorderTarget,\n) {\n const platformGuidance = getPlatformGuidance(target);\n\n switch (event.type) {\n case 'click':\n return `${platformGuidance} Identify the ${getPointerActionVerb(event).toLowerCase()} target by exact visible text first, then label/placeholder, then role plus stable surrounding context, then icon purpose, then visual position. Never describe it by coordinates, marker location, or as a nearby element.`;\n case 'input':\n return `${platformGuidance} Identify the exact input field at the marker before text entry. Use stable visible label, field role, field name, surrounding section, or sequence intent. Treat hint text that can change by user, time, data, or context as secondary evidence. Preserve the recorded input value only in replayInstruction; never describe the typed value or page title alone as the field.`;\n case 'scroll':\n return `${platformGuidance} Identify the scrollable page/region at the highlighted scroll point and concrete destination content revealed after scrolling. If multiple scrollable regions are visible, preserve the specific region where the scroll happened, such as left/right/top/bottom panel, navigation area, content pane, dialog body, table, list, or menu; do not generalize a panel/list scroll into the whole page. Use newly visible headings, section titles, list/table names, list items, or stable region labels; never say only \"more content\" or \"current page\".`;\n case 'drag':\n return `${platformGuidance} Identify the ${getDragActionVerb(event).toLowerCase()} start/end regions or the dragged UI control. Do not describe only the gesture path or coordinates.`;\n case 'keydown':\n return `${platformGuidance} Identify the focused element or keyboard target if visible, and preserve the recorded key in the replay instruction.`;\n default:\n return `${platformGuidance} Identify the UI target involved in this event using the most stable visible text or surrounding context.`;\n }\n}\n\nfunction getEventRawCoordinates(event: MidsceneRecorderEvent) {\n const x = event.elementRect?.x;\n const y = event.elementRect?.y;\n if (isFiniteNumber(x) && isFiniteNumber(y)) {\n return { x, y };\n }\n return undefined;\n}\n\nfunction getPageSemanticContext(event: MidsceneRecorderEvent) {\n const candidates = [event.title, event.url]\n .map((item) => item?.trim())\n .filter(Boolean) as string[];\n return candidates[0];\n}\n\nfunction isWeakDescription(value?: string) {\n if (!value) {\n return true;\n }\n if (isPendingDescription(value)) {\n return true;\n }\n const normalized = value.trim().toLowerCase();\n const compact = normalized.replace(/\\s+/g, '');\n return (\n normalized.length === 0 ||\n /^\\(?\\d+(?:\\.\\d+)?,\\s*\\d+(?:\\.\\d+)?\\)?$/.test(normalized) ||\n normalized === 'target' ||\n normalized === 'element' ||\n normalized === 'target element' ||\n normalized === 'the element' ||\n normalized === 'page element' ||\n normalized === 'input field' ||\n normalized === 'text input' ||\n normalized === 'text field' ||\n normalized === 'search box' ||\n normalized === 'more content' ||\n normalized === 'the page' ||\n normalized === 'current page' ||\n normalized === 'current screen' ||\n normalized === 'the screen' ||\n normalized === 'current ui' ||\n normalized === 'current visible ui' ||\n normalized === 'current visible page' ||\n normalized === 'current visible screen' ||\n normalized === 'main area' ||\n normalized === 'main scrollable area' ||\n normalized === 'scrollable area' ||\n normalized === 'highlighted element' ||\n normalized === 'highlighted item' ||\n normalized === 'marked element' ||\n normalized === 'marked item' ||\n normalized.includes('ai is analyzing element') ||\n compact.includes('坐标') ||\n compact.includes('附近') ||\n compact.includes('附近的元素') ||\n normalized.includes('coordinate') ||\n normalized.includes('near the coordinate') ||\n normalized.includes('near coordinates') ||\n normalized.includes('nearby element') ||\n normalized.includes('nearby item') ||\n normalized.includes('near the marker') ||\n normalized.includes('near marker') ||\n normalized.includes('near the point') ||\n normalized.includes('near point') ||\n normalized.includes('at the point') ||\n normalized.includes('button near point') ||\n normalized.includes('shown in the screenshot') ||\n normalized.includes('red rectangle') ||\n normalized.includes('red marker') ||\n normalized.includes('red box') ||\n normalized.includes('highlighted element') ||\n normalized.includes('highlighted item') ||\n normalized.includes('highlighted screenshot')\n );\n}\n\nfunction isWeakReplayInstruction(value?: string) {\n if (!value) {\n return true;\n }\n if (isPendingDescription(value)) {\n return true;\n }\n const normalized = value.trim().toLowerCase();\n const compact = normalized.replace(/\\s+/g, '');\n return (\n compact.includes('坐标') ||\n compact.includes('附近') ||\n normalized.includes('coordinate') ||\n normalized.includes('near the coordinate') ||\n normalized.includes('nearby element') ||\n normalized.includes('nearby item') ||\n normalized.includes('near the marker') ||\n normalized.includes('near marker') ||\n normalized.includes('near the point') ||\n normalized.includes('near point') ||\n normalized.includes('at the point') ||\n normalized.includes('ai is analyzing element') ||\n normalized.includes('more content') ||\n normalized.includes('current page') ||\n normalized.includes('current screen') ||\n normalized.includes('highlighted element') ||\n normalized.includes('highlighted item') ||\n normalized.includes('red marker') ||\n normalized.includes('red box') ||\n normalized.includes('shown in the screenshot') ||\n normalized.includes('highlighted screenshot')\n );\n}\n\nfunction normalizeForComparison(value: string) {\n return value.trim().toLowerCase().replace(/[\"'`]/g, '').replace(/\\s+/g, ' ');\n}\n\nfunction isInputValueUsedAsFieldDescription(\n event: MidsceneRecorderEvent,\n elementDescription?: string,\n) {\n if (event.type !== 'input' || !event.value || !elementDescription) {\n return false;\n }\n\n const typedValue = normalizeForComparison(event.value);\n if (!typedValue) {\n return false;\n }\n const description = normalizeForComparison(elementDescription);\n\n return (\n description === typedValue ||\n description === `${typedValue} input` ||\n description === `${typedValue} field` ||\n description === `${typedValue} text field` ||\n description === `input ${typedValue}` ||\n description === `field ${typedValue}` ||\n description.includes(`typed value ${typedValue}`) ||\n description.includes(`value ${typedValue}`)\n );\n}\n\nfunction hasScrollDestination(\n replayInstruction: string,\n scrollDestinationDescription?: string,\n) {\n if (\n scrollDestinationDescription &&\n !isWeakDescription(scrollDestinationDescription)\n ) {\n return true;\n }\n const normalized = replayInstruction.toLowerCase();\n return (\n normalized.includes(' until ') ||\n normalized.includes(' visible') ||\n normalized.includes(' reveal') ||\n normalized.includes(' to the ') ||\n normalized.includes(' toward ')\n );\n}\n\nasync function describeWithRetry(\n event: MidsceneRecorderEvent,\n target: MidsceneRecorderTarget | undefined,\n highlightedScreenshot: string,\n modelConfig: IModelConfig,\n options: Required<\n Pick<DescribeRecorderUIEventOptions, 'maxRetries' | 'retryDelayMs'>\n >,\n) {\n let lastError: unknown;\n for (let attempt = 1; attempt <= options.maxRetries; attempt += 1) {\n try {\n const afterScreenshot = getRecorderEventAfterScreenshot(event);\n const pageContext = getPageSemanticContext(event);\n const platformGuidance = getPlatformGuidance(target);\n const userContent: any[] = [\n {\n type: 'text',\n text: `Recorder event:\n${JSON.stringify(\n {\n type: event.type,\n actionType: event.actionType,\n value: event.value,\n rawCoordinates: getEventRawCoordinates(event),\n url: event.url,\n title: event.title,\n pageContext,\n pageInfo: event.pageInfo,\n target,\n platformGuidance,\n guidance: getActionGuidance(event, target),\n },\n null,\n 2,\n)}\n\nThe target or region is highlighted in the screenshot below. Convert this event into semantic replay fields.`,\n },\n {\n type: 'image_url',\n image_url: {\n url: highlightedScreenshot,\n detail: 'high',\n },\n },\n ];\n if (afterScreenshot) {\n userContent.push(\n {\n type: 'text',\n text: 'Screenshot after the recorded action, for context only:',\n },\n {\n type: 'image_url',\n image_url: {\n url: afterScreenshot,\n detail: 'high',\n },\n },\n );\n }\n const response =\n await callAIWithObjectResponse<RecorderUIEventAIResponse>(\n [\n {\n role: 'system',\n content: RECORDER_UI_DESCRIBER_SYSTEM_PROMPT,\n },\n {\n role: 'user',\n content: userContent,\n },\n ],\n modelConfig,\n );\n\n const content = response.content;\n if (content.error) {\n throw new Error(content.error);\n }\n if (isWeakDescription(content.elementDescription)) {\n throw new Error('AI returned a weak recorder event description.');\n }\n if (\n isInputValueUsedAsFieldDescription(event, content.elementDescription)\n ) {\n throw new Error(\n 'AI used the recorded input value as the field description.',\n );\n }\n const elementDescription = content.elementDescription!.trim();\n const scrollDestinationDescription =\n event.type === 'scroll'\n ? content.scrollDestinationDescription?.trim()\n : undefined;\n if (\n event.type === 'scroll' &&\n !hasScrollDestination('', scrollDestinationDescription)\n ) {\n throw new Error(\n 'AI returned a scroll description without a destination.',\n );\n }\n const aiReplayInstruction = content.replayInstruction?.trim();\n if (aiReplayInstruction && isWeakReplayInstruction(aiReplayInstruction)) {\n throw new Error('AI returned a weak recorder replay instruction.');\n }\n const semanticAction = buildSemanticAction(\n event,\n scrollDestinationDescription,\n );\n const replayInstruction = buildMidsceneRecorderReplayInstruction(\n semanticAction,\n elementDescription,\n );\n if (isWeakReplayInstruction(replayInstruction)) {\n throw new Error('AI returned a weak recorder replay instruction.');\n }\n const actionSummary = buildMidsceneRecorderActionSummary(\n semanticAction,\n elementDescription,\n );\n\n return {\n source: 'recorderAI' as const,\n status: 'ready' as const,\n elementDescription,\n replayInstruction,\n actionSummary,\n confidence: content.confidence || 'medium',\n };\n } catch (error) {\n lastError = error;\n if (attempt < options.maxRetries) {\n await delay(options.retryDelayMs);\n }\n }\n }\n throw lastError;\n}\n\nasync function createScreenshotWithBox(\n event: MidsceneRecorderEvent,\n rect: Rect,\n) {\n if (event.screenshotWithBox) {\n return event.screenshotWithBox;\n }\n const screenshot = getRecorderEventScreenshot(event);\n if (!screenshot) {\n return undefined;\n }\n const { compositeElementInfoImg } = await import(\n /* @vite-ignore */ '@midscene/shared/img'\n );\n return compositeElementInfoImg({\n inputImgBase64: screenshot,\n size: event.pageInfo,\n elementsPositionInfo: [{ rect }],\n borderThickness: 3,\n annotationPadding: 2,\n });\n}\n\nfunction createFallbackEvent(\n event: MidsceneRecorderEvent,\n error: string,\n screenshotWithBox?: string,\n target?: MidsceneRecorderTarget,\n): MidsceneRecorderEvent {\n const semantic = getMidsceneRecorderSemantic(event);\n const elementDescription =\n semantic?.elementDescription &&\n !isWeakDescription(semantic.elementDescription)\n ? semantic.elementDescription\n : getFallbackDescription(event, target);\n return {\n ...event,\n semantic: {\n source: 'heuristic',\n status: 'ready',\n elementDescription,\n replayInstruction: getFallbackReplayInstruction(\n event,\n elementDescription,\n ),\n actionSummary: getFallbackActionSummary(event, elementDescription),\n confidence: 'low',\n error,\n },\n screenshotWithBox: screenshotWithBox || event.screenshotWithBox,\n };\n}\n\nexport async function describeRecorderUIEvent(\n input: DescribeRecorderUIEventInput,\n modelConfig: IModelConfig | Record<string, unknown>,\n options: DescribeRecorderUIEventOptions = {},\n): Promise<DescribeRecorderUIEventResult> {\n const event = input.event;\n const rect = getRecorderUIEventTargetRect(event);\n const screenshot = getRecorderEventScreenshot(event);\n\n if (!rect || !screenshot) {\n const error = !rect\n ? 'Recorder event has no target rectangle.'\n : 'Recorder event has no screenshot.';\n return {\n usedFallback: true,\n event: createFallbackEvent(event, error, undefined, input.target),\n };\n }\n\n let screenshotWithBox: string | undefined;\n try {\n const resolvedModelConfig = resolveRecorderModelConfig(modelConfig);\n screenshotWithBox = await createScreenshotWithBox(event, rect);\n const semanticFields = await describeWithRetry(\n event,\n input.target,\n screenshotWithBox || screenshot,\n resolvedModelConfig,\n {\n maxRetries: options.maxRetries ?? RECORDER_UI_DESCRIBER_DEFAULT_RETRIES,\n retryDelayMs:\n options.retryDelayMs ?? RECORDER_UI_DESCRIBER_DEFAULT_RETRY_DELAY_MS,\n },\n );\n return {\n usedFallback: false,\n event: {\n ...event,\n semantic: semanticFields,\n screenshotWithBox: screenshotWithBox || event.screenshotWithBox,\n },\n };\n } catch (error) {\n const message = error instanceof Error ? error.message : String(error);\n return {\n usedFallback: true,\n error: message,\n event: createFallbackEvent(\n event,\n message,\n screenshotWithBox,\n input.target,\n ),\n };\n }\n}\n\nexport async function describeRecorderUIEvents(\n inputs: DescribeRecorderUIEventInput[],\n modelConfig: IModelConfig | Record<string, unknown>,\n options: DescribeRecorderUIEventOptions = {},\n): Promise<DescribeRecorderUIEventResult[]> {\n const concurrency = Math.max(\n 1,\n options.concurrency ?? RECORDER_UI_DESCRIBER_DEFAULT_CONCURRENCY,\n );\n const results: DescribeRecorderUIEventResult[] = new Array(inputs.length);\n let cursor = 0;\n\n async function worker() {\n while (cursor < inputs.length) {\n const index = cursor;\n cursor += 1;\n results[index] = await describeRecorderUIEvent(\n inputs[index],\n modelConfig,\n options,\n );\n }\n }\n\n await Promise.all(\n Array.from({ length: Math.min(concurrency, inputs.length) }, () =>\n worker(),\n ),\n );\n return results;\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","RECORDER_UI_DESCRIBER_DEFAULT_RETRIES","RECORDER_UI_DESCRIBER_DEFAULT_RETRY_DELAY_MS","RECORDER_UI_DESCRIBER_DEFAULT_CONCURRENCY","delay","ms","Promise","resolve","setTimeout","isResolvedModelConfig","modelConfig","resolveRecorderModelConfig","rawModelConfig","result","value","ModelConfigManager","clamp","min","max","Math","isFiniteNumber","Number","isPendingDescription","getRecorderEventScreenshot","event","getRecorderEventAfterScreenshot","normalizeActionType","getPlatformId","target","getPlatformSurface","getPlatformGuidance","getPointerActionVerb","getDragActionVerb","pointToRect","x","y","size","pageInfo","width","height","left","top","getPointRectSize","getRecorderUIEventTargetRect","rect","getFallbackDescription","pageContext","getPageSemanticContext","surface","buildSemanticAction","scrollDestinationDescription","getFallbackReplayInstruction","elementDescription","buildMidsceneRecorderReplayInstruction","getFallbackActionSummary","buildMidsceneRecorderActionSummary","getActionGuidance","platformGuidance","getEventRawCoordinates","candidates","item","Boolean","isWeakDescription","normalized","compact","isWeakReplayInstruction","normalizeForComparison","isInputValueUsedAsFieldDescription","typedValue","description","hasScrollDestination","replayInstruction","describeWithRetry","highlightedScreenshot","options","lastError","attempt","afterScreenshot","userContent","JSON","response","callAIWithObjectResponse","RECORDER_UI_DESCRIBER_SYSTEM_PROMPT","content","Error","undefined","aiReplayInstruction","semanticAction","actionSummary","error","createScreenshotWithBox","screenshot","compositeElementInfoImg","createFallbackEvent","screenshotWithBox","semantic","getMidsceneRecorderSemantic","describeRecorderUIEvent","input","resolvedModelConfig","semanticFields","message","String","describeRecorderUIEvents","inputs","concurrency","results","Array","cursor","worker","index"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;ACoCA,MAAMI,wCAAwC;AAC9C,MAAMC,+CAA+C;AACrD,MAAMC,4CAA4C;AAElD,SAASC,MAAMC,EAAU;IACvB,OAAO,IAAIC,QAAQ,CAACC,UAAYC,WAAWD,SAASF;AACtD;AAEA,SAASI,sBACPC,WAAmD;IAEnD,OACE,AAAiC,YAAjC,OAAOA,YAAY,SAAS,IAC3B,CAAwC,YAAxC,OAAOA,YAAY,gBAAgB,IAClC,AAA8B,YAA9B,OAAOA,YAAY,MAAM,IACzB,AAA4B,YAA5B,OAAOA,YAAY,IAAI,IACvB,AAAoC,YAApC,OAAOA,YAAY,YAAY,AAAY;AAEjD;AAEA,SAASC,2BACPD,WAAmD;IAEnD,IAAID,sBAAsBC,cACxB,OAAOA;IAET,MAAME,iBAAiBf,OAAO,OAAO,CAACa,aAAa,MAAM,CAEvD,CAACG,QAAQ,CAACjB,KAAKkB,MAAM;QACrB,IAAI,AAAiB,YAAjB,OAAOA,SAAsB,AAAiB,YAAjB,OAAOA,OACtCD,MAAM,CAACjB,IAAI,GAAGkB;QAEhB,OAAOD;IACT,GAAG,CAAC;IACJ,OAAO,IAAIE,oBAAAA,kBAAkBA,CAACH,gBAAgB,cAAc,CAAC;AAC/D;AAEA,SAASI,MAAMF,KAAa,EAAEG,GAAW,EAAEC,GAAW;IACpD,OAAOC,KAAK,GAAG,CAACA,KAAK,GAAG,CAACL,OAAOG,MAAMC;AACxC;AAEA,SAASE,eAAeN,KAAc;IACpC,OAAO,AAAiB,YAAjB,OAAOA,SAAsBO,OAAO,QAAQ,CAACP;AACtD;AAEA,SAASQ,qBAAqBR,KAAc;IAC1C,OAAOA,OAAO,WAAW;AAC3B;AAEA,SAASS,2BAA2BC,KAA4B;IAC9D,OACEA,MAAM,iBAAiB,IAAIA,MAAM,gBAAgB,IAAIA,MAAM,eAAe;AAE9E;AAEA,SAASC,gCAAgCD,KAA4B;IACnE,OAAOA,MAAM,eAAe,IAAIA,MAAM,iBAAiB;AACzD;AAEA,SAASE,oBAAoBF,KAA4B;IACvD,OAAOA,MAAM,UAAU,EAAE;AAC3B;AAEA,SAASG,cAAcC,MAA+B;IACpD,OAAOA,QAAQ,YAAY;AAC7B;AAEA,SAASC,mBAAmBD,MAA+B;IACzD,OAAQD,cAAcC;QACpB,KAAK;YACH,OAAO;QACT,KAAK;QACL,KAAK;QACL,KAAK;YACH,OAAO;QACT,KAAK;YACH,OAAO;QACT;YACE,OAAO;IACX;AACF;AAEA,SAASE,oBAAoBF,MAA+B;IAC1D,OAAQD,cAAcC;QACpB,KAAK;YACH,OAAO;QACT,KAAK;QACL,KAAK;QACL,KAAK;YACH,OAAO;QACT,KAAK;YACH,OAAO;QACT;YACE,OAAO;IACX;AACF;AAEA,SAASG,qBAAqBP,KAA4B;IACxD,OAAQE,oBAAoBF;QAC1B,KAAK;YACH,OAAO;QACT,KAAK;YACH,OAAO;QACT,KAAK;YACH,OAAO;QACT,KAAK;YACH,OAAO;QACT;YACE,OAAO;IACX;AACF;AAEA,SAASQ,kBAAkBR,KAA4B;IACrD,OAAQE,oBAAoBF;QAC1B,KAAK;YACH,OAAO;QACT,KAAK;YACH,OAAO;QACT;YACE,OAAO;IACX;AACF;AAEA,SAASS,YACPC,CAAS,EACTC,CAAS,EACTC,IAAY,EACZC,QAAkC;IAElC,MAAMC,QAAQD,SAAS,KAAK,IAAID;IAChC,MAAMG,SAASF,SAAS,MAAM,IAAID;IAClC,MAAMI,OAAOxB,MAAMG,KAAK,KAAK,CAACe,IAAIE,OAAO,IAAI,GAAGjB,KAAK,GAAG,CAACmB,QAAQ,GAAG;IACpE,MAAMG,MAAMzB,MAAMG,KAAK,KAAK,CAACgB,IAAIC,OAAO,IAAI,GAAGjB,KAAK,GAAG,CAACoB,SAAS,GAAG;IACpE,OAAO;QACLC;QACAC;QACA,OAAOtB,KAAK,GAAG,CAACiB,MAAMjB,KAAK,GAAG,CAACmB,QAAQE,MAAM;QAC7C,QAAQrB,KAAK,GAAG,CAACiB,MAAMjB,KAAK,GAAG,CAACoB,SAASE,KAAK;IAChD;AACF;AAEA,SAASC,iBAAiBlB,KAA4B;IACpD,OAAQA,MAAM,IAAI;QAChB,KAAK;YACH,OAAO;QACT,KAAK;YACH,OAAO;QACT;YACE,OAAO;IACX;AACF;AAEO,SAASmB,6BACdnB,KAA4B;IAE5B,MAAMoB,OAAOpB,MAAM,WAAW;IAC9B,IAAI,CAACoB,MACH,OAAO;IAGT,IACExB,eAAewB,KAAK,KAAK,KACzBA,KAAK,KAAK,GAAG,KACbxB,eAAewB,KAAK,MAAM,KAC1BA,KAAK,MAAM,GAAG,KACbxB,CAAAA,eAAewB,KAAK,IAAI,KAAKxB,eAAewB,KAAK,GAAG,IAErD,OAAO;QACL,MAAMA,KAAK,IAAI,IAAI;QACnB,KAAKA,KAAK,GAAG,IAAI;QACjB,OAAOA,KAAK,KAAK;QACjB,QAAQA,KAAK,MAAM;IACrB;IAGF,IAAIxB,eAAewB,KAAK,CAAC,KAAKxB,eAAewB,KAAK,CAAC,GACjD,OAAOX,YAAYW,KAAK,CAAC,EAAEA,KAAK,CAAC,EAAEF,iBAAiBlB,QAAQA,MAAM,QAAQ;IAG5E,OAAO;AACT;AAEA,SAASqB,uBACPrB,KAA4B,EAC5BI,MAA+B;IAE/B,MAAMkB,cAAcC,uBAAuBvB;IAC3C,MAAMwB,UAAUnB,mBAAmBD;IAEnC,OAAQJ,MAAM,IAAI;QAChB,KAAK;YACH,OAAOA,MAAM,GAAG,IAAIA,MAAM,KAAK,IAAIA,MAAM,UAAU,IAAI;QACzD,KAAK;YACH,OAAOsB,cACH,GAAGA,YAAY,mBAAmB,CAAC,GACnC,CAAC,0BAA0B,EAAEE,SAAS;QAC5C,KAAK;YACH,OAAOF,cACH,CAAC,gBAAgB,EAAEA,aAAa,GAChC,CAAC,oBAAoB,EAAEE,SAAS;QACtC,KAAK;YACH,OAAO,CAAC,8BAA8B,EAAEA,SAAS;QACnD,KAAK;YACH,OAAOF,cACH,CAAC,mBAAmB,EAAEA,aAAa,GACnC,CAAC,uBAAuB,EAAEE,SAAS;QACzC;YACE,OAAOF,cACH,CAAC,WAAW,EAAEA,aAAa,GAC3B,CAAC,eAAe,EAAEE,SAAS;IACnC;AACF;AAEA,SAASC,oBACPzB,KAA4B,EAC5B0B,4BAAqC;IAErC,OAAO;QACL,MAAM1B,MAAM,IAAI;QAChB,YAAYA,MAAM,UAAU;QAC5B,OAAOA,MAAM,KAAK;QAClB,KAAKA,MAAM,GAAG;QACd0B;IACF;AACF;AAEA,SAASC,6BACP3B,KAA4B,EAC5B4B,kBAA0B;IAE1B,OAAOC,AAAAA,IAAAA,yBAAAA,sCAAAA,AAAAA,EACLJ,oBAAoBzB,QACpB4B;AAEJ;AAEA,SAASE,yBACP9B,KAA4B,EAC5B4B,kBAA0B;IAE1B,OAAOG,AAAAA,IAAAA,yBAAAA,kCAAAA,AAAAA,EACLN,oBAAoBzB,QACpB4B;AAEJ;AAEA,SAASI,kBACPhC,KAA4B,EAC5BI,MAA+B;IAE/B,MAAM6B,mBAAmB3B,oBAAoBF;IAE7C,OAAQJ,MAAM,IAAI;QAChB,KAAK;YACH,OAAO,GAAGiC,iBAAiB,cAAc,EAAE1B,qBAAqBP,OAAO,WAAW,GAAG,2NAA2N,CAAC;QACnT,KAAK;YACH,OAAO,GAAGiC,iBAAiB,gXAAgX,CAAC;QAC9Y,KAAK;YACH,OAAO,GAAGA,iBAAiB,yhBAAyhB,CAAC;QACvjB,KAAK;YACH,OAAO,GAAGA,iBAAiB,cAAc,EAAEzB,kBAAkBR,OAAO,WAAW,GAAG,mGAAmG,CAAC;QACxL,KAAK;YACH,OAAO,GAAGiC,iBAAiB,qHAAqH,CAAC;QACnJ;YACE,OAAO,GAAGA,iBAAiB,yGAAyG,CAAC;IACzI;AACF;AAEA,SAASC,uBAAuBlC,KAA4B;IAC1D,MAAMU,IAAIV,MAAM,WAAW,EAAE;IAC7B,MAAMW,IAAIX,MAAM,WAAW,EAAE;IAC7B,IAAIJ,eAAec,MAAMd,eAAee,IACtC,OAAO;QAAED;QAAGC;IAAE;AAGlB;AAEA,SAASY,uBAAuBvB,KAA4B;IAC1D,MAAMmC,aAAa;QAACnC,MAAM,KAAK;QAAEA,MAAM,GAAG;KAAC,CACxC,GAAG,CAAC,CAACoC,OAASA,MAAM,QACpB,MAAM,CAACC;IACV,OAAOF,UAAU,CAAC,EAAE;AACtB;AAEA,SAASG,kBAAkBhD,KAAc;IACvC,IAAI,CAACA,OACH,OAAO;IAET,IAAIQ,qBAAqBR,QACvB,OAAO;IAET,MAAMiD,aAAajD,MAAM,IAAI,GAAG,WAAW;IAC3C,MAAMkD,UAAUD,WAAW,OAAO,CAAC,QAAQ;IAC3C,OACEA,AAAsB,MAAtBA,WAAW,MAAM,IACjB,yCAAyC,IAAI,CAACA,eAC9CA,AAAe,aAAfA,cACAA,AAAe,cAAfA,cACAA,AAAe,qBAAfA,cACAA,AAAe,kBAAfA,cACAA,AAAe,mBAAfA,cACAA,AAAe,kBAAfA,cACAA,AAAe,iBAAfA,cACAA,AAAe,iBAAfA,cACAA,AAAe,iBAAfA,cACAA,AAAe,mBAAfA,cACAA,AAAe,eAAfA,cACAA,AAAe,mBAAfA,cACAA,AAAe,qBAAfA,cACAA,AAAe,iBAAfA,cACAA,AAAe,iBAAfA,cACAA,AAAe,yBAAfA,cACAA,AAAe,2BAAfA,cACAA,AAAe,6BAAfA,cACAA,AAAe,gBAAfA,cACAA,AAAe,2BAAfA,cACAA,AAAe,sBAAfA,cACAA,AAAe,0BAAfA,cACAA,AAAe,uBAAfA,cACAA,AAAe,qBAAfA,cACAA,AAAe,kBAAfA,cACAA,WAAW,QAAQ,CAAC,8BACpBC,QAAQ,QAAQ,CAAC,SACjBA,QAAQ,QAAQ,CAAC,SACjBA,QAAQ,QAAQ,CAAC,YACjBD,WAAW,QAAQ,CAAC,iBACpBA,WAAW,QAAQ,CAAC,0BACpBA,WAAW,QAAQ,CAAC,uBACpBA,WAAW,QAAQ,CAAC,qBACpBA,WAAW,QAAQ,CAAC,kBACpBA,WAAW,QAAQ,CAAC,sBACpBA,WAAW,QAAQ,CAAC,kBACpBA,WAAW,QAAQ,CAAC,qBACpBA,WAAW,QAAQ,CAAC,iBACpBA,WAAW,QAAQ,CAAC,mBACpBA,WAAW,QAAQ,CAAC,wBACpBA,WAAW,QAAQ,CAAC,8BACpBA,WAAW,QAAQ,CAAC,oBACpBA,WAAW,QAAQ,CAAC,iBACpBA,WAAW,QAAQ,CAAC,cACpBA,WAAW,QAAQ,CAAC,0BACpBA,WAAW,QAAQ,CAAC,uBACpBA,WAAW,QAAQ,CAAC;AAExB;AAEA,SAASE,wBAAwBnD,KAAc;IAC7C,IAAI,CAACA,OACH,OAAO;IAET,IAAIQ,qBAAqBR,QACvB,OAAO;IAET,MAAMiD,aAAajD,MAAM,IAAI,GAAG,WAAW;IAC3C,MAAMkD,UAAUD,WAAW,OAAO,CAAC,QAAQ;IAC3C,OACEC,QAAQ,QAAQ,CAAC,SACjBA,QAAQ,QAAQ,CAAC,SACjBD,WAAW,QAAQ,CAAC,iBACpBA,WAAW,QAAQ,CAAC,0BACpBA,WAAW,QAAQ,CAAC,qBACpBA,WAAW,QAAQ,CAAC,kBACpBA,WAAW,QAAQ,CAAC,sBACpBA,WAAW,QAAQ,CAAC,kBACpBA,WAAW,QAAQ,CAAC,qBACpBA,WAAW,QAAQ,CAAC,iBACpBA,WAAW,QAAQ,CAAC,mBACpBA,WAAW,QAAQ,CAAC,8BACpBA,WAAW,QAAQ,CAAC,mBACpBA,WAAW,QAAQ,CAAC,mBACpBA,WAAW,QAAQ,CAAC,qBACpBA,WAAW,QAAQ,CAAC,0BACpBA,WAAW,QAAQ,CAAC,uBACpBA,WAAW,QAAQ,CAAC,iBACpBA,WAAW,QAAQ,CAAC,cACpBA,WAAW,QAAQ,CAAC,8BACpBA,WAAW,QAAQ,CAAC;AAExB;AAEA,SAASG,uBAAuBpD,KAAa;IAC3C,OAAOA,MAAM,IAAI,GAAG,WAAW,GAAG,OAAO,CAAC,UAAU,IAAI,OAAO,CAAC,QAAQ;AAC1E;AAEA,SAASqD,mCACP3C,KAA4B,EAC5B4B,kBAA2B;IAE3B,IAAI5B,AAAe,YAAfA,MAAM,IAAI,IAAgB,CAACA,MAAM,KAAK,IAAI,CAAC4B,oBAC7C,OAAO;IAGT,MAAMgB,aAAaF,uBAAuB1C,MAAM,KAAK;IACrD,IAAI,CAAC4C,YACH,OAAO;IAET,MAAMC,cAAcH,uBAAuBd;IAE3C,OACEiB,gBAAgBD,cAChBC,gBAAgB,GAAGD,WAAW,MAAM,CAAC,IACrCC,gBAAgB,GAAGD,WAAW,MAAM,CAAC,IACrCC,gBAAgB,GAAGD,WAAW,WAAW,CAAC,IAC1CC,gBAAgB,CAAC,MAAM,EAAED,YAAY,IACrCC,gBAAgB,CAAC,MAAM,EAAED,YAAY,IACrCC,YAAY,QAAQ,CAAC,CAAC,YAAY,EAAED,YAAY,KAChDC,YAAY,QAAQ,CAAC,CAAC,MAAM,EAAED,YAAY;AAE9C;AAEA,SAASE,qBACPC,iBAAyB,EACzBrB,4BAAqC;IAErC,IACEA,gCACA,CAACY,kBAAkBZ,+BAEnB,OAAO;IAET,MAAMa,aAAaQ,kBAAkB,WAAW;IAChD,OACER,WAAW,QAAQ,CAAC,cACpBA,WAAW,QAAQ,CAAC,eACpBA,WAAW,QAAQ,CAAC,cACpBA,WAAW,QAAQ,CAAC,eACpBA,WAAW,QAAQ,CAAC;AAExB;AAEA,eAAeS,kBACbhD,KAA4B,EAC5BI,MAA0C,EAC1C6C,qBAA6B,EAC7B/D,WAAyB,EACzBgE,OAEC;IAED,IAAIC;IACJ,IAAK,IAAIC,UAAU,GAAGA,WAAWF,QAAQ,UAAU,EAAEE,WAAW,EAC9D,IAAI;QACF,MAAMC,kBAAkBpD,gCAAgCD;QACxD,MAAMsB,cAAcC,uBAAuBvB;QAC3C,MAAMiC,mBAAmB3B,oBAAoBF;QAC7C,MAAMkD,cAAqB;YACzB;gBACE,MAAM;gBACN,MAAM,CAAC;AACjB,EAAEC,KAAK,SAAS,CACd;oBACE,MAAMvD,MAAM,IAAI;oBAChB,YAAYA,MAAM,UAAU;oBAC5B,OAAOA,MAAM,KAAK;oBAClB,gBAAgBkC,uBAAuBlC;oBACvC,KAAKA,MAAM,GAAG;oBACd,OAAOA,MAAM,KAAK;oBAClBsB;oBACA,UAAUtB,MAAM,QAAQ;oBACxBI;oBACA6B;oBACA,UAAUD,kBAAkBhC,OAAOI;gBACrC,GACA,MACA,GACA;;4GAE0G,CAAC;YACrG;YACA;gBACE,MAAM;gBACN,WAAW;oBACT,KAAK6C;oBACL,QAAQ;gBACV;YACF;SACD;QACD,IAAII,iBACFC,YAAY,IAAI,CACd;YACE,MAAM;YACN,MAAM;QACR,GACA;YACE,MAAM;YACN,WAAW;gBACT,KAAKD;gBACL,QAAQ;YACV;QACF;QAGJ,MAAMG,WACJ,MAAMC,AAAAA,IAAAA,gDAAAA,wBAAAA,AAAAA,EACJ;YACE;gBACE,MAAM;gBACN,SAASC,yDAAAA,mCAAmCA;YAC9C;YACA;gBACE,MAAM;gBACN,SAASJ;YACX;SACD,EACDpE;QAGJ,MAAMyE,UAAUH,SAAS,OAAO;QAChC,IAAIG,QAAQ,KAAK,EACf,MAAM,IAAIC,MAAMD,QAAQ,KAAK;QAE/B,IAAIrB,kBAAkBqB,QAAQ,kBAAkB,GAC9C,MAAM,IAAIC,MAAM;QAElB,IACEjB,mCAAmC3C,OAAO2D,QAAQ,kBAAkB,GAEpE,MAAM,IAAIC,MACR;QAGJ,MAAMhC,qBAAqB+B,QAAQ,kBAAkB,CAAE,IAAI;QAC3D,MAAMjC,+BACJ1B,AAAe,aAAfA,MAAM,IAAI,GACN2D,QAAQ,4BAA4B,EAAE,SACtCE;QACN,IACE7D,AAAe,aAAfA,MAAM,IAAI,IACV,CAAC8C,qBAAqB,IAAIpB,+BAE1B,MAAM,IAAIkC,MACR;QAGJ,MAAME,sBAAsBH,QAAQ,iBAAiB,EAAE;QACvD,IAAIG,uBAAuBrB,wBAAwBqB,sBACjD,MAAM,IAAIF,MAAM;QAElB,MAAMG,iBAAiBtC,oBACrBzB,OACA0B;QAEF,MAAMqB,oBAAoBlB,AAAAA,IAAAA,yBAAAA,sCAAAA,AAAAA,EACxBkC,gBACAnC;QAEF,IAAIa,wBAAwBM,oBAC1B,MAAM,IAAIa,MAAM;QAElB,MAAMI,gBAAgBjC,AAAAA,IAAAA,yBAAAA,kCAAAA,AAAAA,EACpBgC,gBACAnC;QAGF,OAAO;YACL,QAAQ;YACR,QAAQ;YACRA;YACAmB;YACAiB;YACA,YAAYL,QAAQ,UAAU,IAAI;QACpC;IACF,EAAE,OAAOM,OAAO;QACdd,YAAYc;QACZ,IAAIb,UAAUF,QAAQ,UAAU,EAC9B,MAAMtE,MAAMsE,QAAQ,YAAY;IAEpC;IAEF,MAAMC;AACR;AAEA,eAAee,wBACblE,KAA4B,EAC5BoB,IAAU;IAEV,IAAIpB,MAAM,iBAAiB,EACzB,OAAOA,MAAM,iBAAiB;IAEhC,MAAMmE,aAAapE,2BAA2BC;IAC9C,IAAI,CAACmE,YACH;IAEF,MAAM,EAAEC,uBAAuB,EAAE,GAAG,MAAM,MAAM,CAAC,gBAAP;IAG1C,OAAOA,wBAAwB;QAC7B,gBAAgBD;QAChB,MAAMnE,MAAM,QAAQ;QACpB,sBAAsB;YAAC;gBAAEoB;YAAK;SAAE;QAChC,iBAAiB;QACjB,mBAAmB;IACrB;AACF;AAEA,SAASiD,oBACPrE,KAA4B,EAC5BiE,KAAa,EACbK,iBAA0B,EAC1BlE,MAA+B;IAE/B,MAAMmE,WAAWC,AAAAA,IAAAA,yBAAAA,2BAAAA,AAAAA,EAA4BxE;IAC7C,MAAM4B,qBACJ2C,UAAU,sBACV,CAACjC,kBAAkBiC,SAAS,kBAAkB,IAC1CA,SAAS,kBAAkB,GAC3BlD,uBAAuBrB,OAAOI;IACpC,OAAO;QACL,GAAGJ,KAAK;QACR,UAAU;YACR,QAAQ;YACR,QAAQ;YACR4B;YACA,mBAAmBD,6BACjB3B,OACA4B;YAEF,eAAeE,yBAAyB9B,OAAO4B;YAC/C,YAAY;YACZqC;QACF;QACA,mBAAmBK,qBAAqBtE,MAAM,iBAAiB;IACjE;AACF;AAEO,eAAeyE,wBACpBC,KAAmC,EACnCxF,WAAmD,EACnDgE,UAA0C,CAAC,CAAC;IAE5C,MAAMlD,QAAQ0E,MAAM,KAAK;IACzB,MAAMtD,OAAOD,6BAA6BnB;IAC1C,MAAMmE,aAAapE,2BAA2BC;IAE9C,IAAI,CAACoB,QAAQ,CAAC+C,YAAY;QACxB,MAAMF,QAAQ,AAAC7C,OAEX,sCADA;QAEJ,OAAO;YACL,cAAc;YACd,OAAOiD,oBAAoBrE,OAAOiE,OAAOJ,QAAWa,MAAM,MAAM;QAClE;IACF;IAEA,IAAIJ;IACJ,IAAI;QACF,MAAMK,sBAAsBxF,2BAA2BD;QACvDoF,oBAAoB,MAAMJ,wBAAwBlE,OAAOoB;QACzD,MAAMwD,iBAAiB,MAAM5B,kBAC3BhD,OACA0E,MAAM,MAAM,EACZJ,qBAAqBH,YACrBQ,qBACA;YACE,YAAYzB,QAAQ,UAAU,IAAIzE;YAClC,cACEyE,QAAQ,YAAY,IAAIxE;QAC5B;QAEF,OAAO;YACL,cAAc;YACd,OAAO;gBACL,GAAGsB,KAAK;gBACR,UAAU4E;gBACV,mBAAmBN,qBAAqBtE,MAAM,iBAAiB;YACjE;QACF;IACF,EAAE,OAAOiE,OAAO;QACd,MAAMY,UAAUZ,iBAAiBL,QAAQK,MAAM,OAAO,GAAGa,OAAOb;QAChE,OAAO;YACL,cAAc;YACd,OAAOY;YACP,OAAOR,oBACLrE,OACA6E,SACAP,mBACAI,MAAM,MAAM;QAEhB;IACF;AACF;AAEO,eAAeK,yBACpBC,MAAsC,EACtC9F,WAAmD,EACnDgE,UAA0C,CAAC,CAAC;IAE5C,MAAM+B,cAActF,KAAK,GAAG,CAC1B,GACAuD,QAAQ,WAAW,IAAIvE;IAEzB,MAAMuG,UAA2C,IAAIC,MAAMH,OAAO,MAAM;IACxE,IAAII,SAAS;IAEb,eAAeC;QACb,MAAOD,SAASJ,OAAO,MAAM,CAAE;YAC7B,MAAMM,QAAQF;YACdA,UAAU;YACVF,OAAO,CAACI,MAAM,GAAG,MAAMb,wBACrBO,MAAM,CAACM,MAAM,EACbpG,aACAgE;QAEJ;IACF;IAEA,MAAMpE,QAAQ,GAAG,CACfqG,MAAM,IAAI,CAAC;QAAE,QAAQxF,KAAK,GAAG,CAACsF,aAAaD,OAAO,MAAM;IAAE,GAAG,IAC3DK;IAGJ,OAAOH;AACT"}
|