@godscene/core 1.7.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (189) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +9 -0
  3. package/dist/es/agent/agent.mjs +767 -0
  4. package/dist/es/agent/common.mjs +0 -0
  5. package/dist/es/agent/execution-session.mjs +39 -0
  6. package/dist/es/agent/index.mjs +6 -0
  7. package/dist/es/agent/task-builder.mjs +343 -0
  8. package/dist/es/agent/task-cache.mjs +212 -0
  9. package/dist/es/agent/tasks.mjs +428 -0
  10. package/dist/es/agent/ui-utils.mjs +101 -0
  11. package/dist/es/agent/utils.mjs +167 -0
  12. package/dist/es/ai-model/auto-glm/actions.mjs +237 -0
  13. package/dist/es/ai-model/auto-glm/index.mjs +6 -0
  14. package/dist/es/ai-model/auto-glm/parser.mjs +237 -0
  15. package/dist/es/ai-model/auto-glm/planning.mjs +69 -0
  16. package/dist/es/ai-model/auto-glm/prompt.mjs +220 -0
  17. package/dist/es/ai-model/auto-glm/util.mjs +7 -0
  18. package/dist/es/ai-model/connectivity.mjs +136 -0
  19. package/dist/es/ai-model/conversation-history.mjs +193 -0
  20. package/dist/es/ai-model/index.mjs +12 -0
  21. package/dist/es/ai-model/inspect.mjs +395 -0
  22. package/dist/es/ai-model/llm-planning.mjs +231 -0
  23. package/dist/es/ai-model/prompt/common.mjs +5 -0
  24. package/dist/es/ai-model/prompt/describe.mjs +64 -0
  25. package/dist/es/ai-model/prompt/extraction.mjs +129 -0
  26. package/dist/es/ai-model/prompt/llm-locator.mjs +49 -0
  27. package/dist/es/ai-model/prompt/llm-planning.mjs +584 -0
  28. package/dist/es/ai-model/prompt/llm-section-locator.mjs +42 -0
  29. package/dist/es/ai-model/prompt/order-sensitive-judge.mjs +33 -0
  30. package/dist/es/ai-model/prompt/playwright-generator.mjs +115 -0
  31. package/dist/es/ai-model/prompt/ui-tars-planning.mjs +34 -0
  32. package/dist/es/ai-model/prompt/util.mjs +57 -0
  33. package/dist/es/ai-model/prompt/yaml-generator.mjs +201 -0
  34. package/dist/es/ai-model/service-caller/codex-app-server.mjs +573 -0
  35. package/dist/es/ai-model/service-caller/image-detail.mjs +4 -0
  36. package/dist/es/ai-model/service-caller/index.mjs +648 -0
  37. package/dist/es/ai-model/service-caller/request-timeout.mjs +47 -0
  38. package/dist/es/ai-model/ui-tars-planning.mjs +247 -0
  39. package/dist/es/common.mjs +382 -0
  40. package/dist/es/device/device-options.mjs +0 -0
  41. package/dist/es/device/index.mjs +340 -0
  42. package/dist/es/dump/html-utils.mjs +290 -0
  43. package/dist/es/dump/index.mjs +3 -0
  44. package/dist/es/dump/screenshot-restoration.mjs +30 -0
  45. package/dist/es/dump/screenshot-store.mjs +125 -0
  46. package/dist/es/index.mjs +17 -0
  47. package/dist/es/report-cli.mjs +149 -0
  48. package/dist/es/report-generator.mjs +203 -0
  49. package/dist/es/report-markdown.mjs +216 -0
  50. package/dist/es/report.mjs +287 -0
  51. package/dist/es/screenshot-item.mjs +120 -0
  52. package/dist/es/service/index.mjs +272 -0
  53. package/dist/es/service/utils.mjs +13 -0
  54. package/dist/es/skill/index.mjs +35 -0
  55. package/dist/es/task-runner.mjs +261 -0
  56. package/dist/es/task-timing.mjs +10 -0
  57. package/dist/es/tree.mjs +11 -0
  58. package/dist/es/types.mjs +202 -0
  59. package/dist/es/utils.mjs +232 -0
  60. package/dist/es/yaml/builder.mjs +11 -0
  61. package/dist/es/yaml/index.mjs +4 -0
  62. package/dist/es/yaml/player.mjs +425 -0
  63. package/dist/es/yaml/utils.mjs +100 -0
  64. package/dist/es/yaml.mjs +0 -0
  65. package/dist/lib/agent/agent.js +815 -0
  66. package/dist/lib/agent/common.js +5 -0
  67. package/dist/lib/agent/execution-session.js +73 -0
  68. package/dist/lib/agent/index.js +76 -0
  69. package/dist/lib/agent/task-builder.js +380 -0
  70. package/dist/lib/agent/task-cache.js +264 -0
  71. package/dist/lib/agent/tasks.js +471 -0
  72. package/dist/lib/agent/ui-utils.js +153 -0
  73. package/dist/lib/agent/utils.js +238 -0
  74. package/dist/lib/ai-model/auto-glm/actions.js +271 -0
  75. package/dist/lib/ai-model/auto-glm/index.js +64 -0
  76. package/dist/lib/ai-model/auto-glm/parser.js +280 -0
  77. package/dist/lib/ai-model/auto-glm/planning.js +103 -0
  78. package/dist/lib/ai-model/auto-glm/prompt.js +257 -0
  79. package/dist/lib/ai-model/auto-glm/util.js +44 -0
  80. package/dist/lib/ai-model/connectivity.js +180 -0
  81. package/dist/lib/ai-model/conversation-history.js +227 -0
  82. package/dist/lib/ai-model/index.js +127 -0
  83. package/dist/lib/ai-model/inspect.js +441 -0
  84. package/dist/lib/ai-model/llm-planning.js +268 -0
  85. package/dist/lib/ai-model/prompt/common.js +39 -0
  86. package/dist/lib/ai-model/prompt/describe.js +98 -0
  87. package/dist/lib/ai-model/prompt/extraction.js +169 -0
  88. package/dist/lib/ai-model/prompt/llm-locator.js +86 -0
  89. package/dist/lib/ai-model/prompt/llm-planning.js +621 -0
  90. package/dist/lib/ai-model/prompt/llm-section-locator.js +79 -0
  91. package/dist/lib/ai-model/prompt/order-sensitive-judge.js +70 -0
  92. package/dist/lib/ai-model/prompt/playwright-generator.js +176 -0
  93. package/dist/lib/ai-model/prompt/ui-tars-planning.js +71 -0
  94. package/dist/lib/ai-model/prompt/util.js +103 -0
  95. package/dist/lib/ai-model/prompt/yaml-generator.js +262 -0
  96. package/dist/lib/ai-model/service-caller/codex-app-server.js +622 -0
  97. package/dist/lib/ai-model/service-caller/image-detail.js +38 -0
  98. package/dist/lib/ai-model/service-caller/index.js +716 -0
  99. package/dist/lib/ai-model/service-caller/request-timeout.js +93 -0
  100. package/dist/lib/ai-model/ui-tars-planning.js +281 -0
  101. package/dist/lib/common.js +491 -0
  102. package/dist/lib/device/device-options.js +18 -0
  103. package/dist/lib/device/index.js +467 -0
  104. package/dist/lib/dump/html-utils.js +366 -0
  105. package/dist/lib/dump/index.js +58 -0
  106. package/dist/lib/dump/screenshot-restoration.js +64 -0
  107. package/dist/lib/dump/screenshot-store.js +165 -0
  108. package/dist/lib/index.js +184 -0
  109. package/dist/lib/report-cli.js +189 -0
  110. package/dist/lib/report-generator.js +244 -0
  111. package/dist/lib/report-markdown.js +253 -0
  112. package/dist/lib/report.js +333 -0
  113. package/dist/lib/screenshot-item.js +154 -0
  114. package/dist/lib/service/index.js +306 -0
  115. package/dist/lib/service/utils.js +47 -0
  116. package/dist/lib/skill/index.js +69 -0
  117. package/dist/lib/task-runner.js +298 -0
  118. package/dist/lib/task-timing.js +44 -0
  119. package/dist/lib/tree.js +51 -0
  120. package/dist/lib/types.js +298 -0
  121. package/dist/lib/utils.js +314 -0
  122. package/dist/lib/yaml/builder.js +55 -0
  123. package/dist/lib/yaml/index.js +79 -0
  124. package/dist/lib/yaml/player.js +459 -0
  125. package/dist/lib/yaml/utils.js +153 -0
  126. package/dist/lib/yaml.js +18 -0
  127. package/dist/types/agent/agent.d.ts +220 -0
  128. package/dist/types/agent/common.d.ts +0 -0
  129. package/dist/types/agent/execution-session.d.ts +36 -0
  130. package/dist/types/agent/index.d.ts +9 -0
  131. package/dist/types/agent/task-builder.d.ts +34 -0
  132. package/dist/types/agent/task-cache.d.ts +49 -0
  133. package/dist/types/agent/tasks.d.ts +70 -0
  134. package/dist/types/agent/ui-utils.d.ts +14 -0
  135. package/dist/types/agent/utils.d.ts +25 -0
  136. package/dist/types/ai-model/auto-glm/actions.d.ts +78 -0
  137. package/dist/types/ai-model/auto-glm/index.d.ts +6 -0
  138. package/dist/types/ai-model/auto-glm/parser.d.ts +18 -0
  139. package/dist/types/ai-model/auto-glm/planning.d.ts +12 -0
  140. package/dist/types/ai-model/auto-glm/prompt.d.ts +27 -0
  141. package/dist/types/ai-model/auto-glm/util.d.ts +13 -0
  142. package/dist/types/ai-model/connectivity.d.ts +20 -0
  143. package/dist/types/ai-model/conversation-history.d.ts +105 -0
  144. package/dist/types/ai-model/index.d.ts +16 -0
  145. package/dist/types/ai-model/inspect.d.ts +67 -0
  146. package/dist/types/ai-model/llm-planning.d.ts +19 -0
  147. package/dist/types/ai-model/prompt/common.d.ts +2 -0
  148. package/dist/types/ai-model/prompt/describe.d.ts +1 -0
  149. package/dist/types/ai-model/prompt/extraction.d.ts +7 -0
  150. package/dist/types/ai-model/prompt/llm-locator.d.ts +3 -0
  151. package/dist/types/ai-model/prompt/llm-planning.d.ts +10 -0
  152. package/dist/types/ai-model/prompt/llm-section-locator.d.ts +3 -0
  153. package/dist/types/ai-model/prompt/order-sensitive-judge.d.ts +2 -0
  154. package/dist/types/ai-model/prompt/playwright-generator.d.ts +26 -0
  155. package/dist/types/ai-model/prompt/ui-tars-planning.d.ts +2 -0
  156. package/dist/types/ai-model/prompt/util.d.ts +33 -0
  157. package/dist/types/ai-model/prompt/yaml-generator.d.ts +102 -0
  158. package/dist/types/ai-model/service-caller/codex-app-server.d.ts +42 -0
  159. package/dist/types/ai-model/service-caller/image-detail.d.ts +2 -0
  160. package/dist/types/ai-model/service-caller/index.d.ts +60 -0
  161. package/dist/types/ai-model/service-caller/request-timeout.d.ts +32 -0
  162. package/dist/types/ai-model/ui-tars-planning.d.ts +72 -0
  163. package/dist/types/common.d.ts +288 -0
  164. package/dist/types/device/device-options.d.ts +155 -0
  165. package/dist/types/device/index.d.ts +2565 -0
  166. package/dist/types/dump/html-utils.d.ts +75 -0
  167. package/dist/types/dump/index.d.ts +5 -0
  168. package/dist/types/dump/screenshot-restoration.d.ts +8 -0
  169. package/dist/types/dump/screenshot-store.d.ts +49 -0
  170. package/dist/types/index.d.ts +21 -0
  171. package/dist/types/report-cli.d.ts +36 -0
  172. package/dist/types/report-generator.d.ts +88 -0
  173. package/dist/types/report-markdown.d.ts +24 -0
  174. package/dist/types/report.d.ts +52 -0
  175. package/dist/types/screenshot-item.d.ts +67 -0
  176. package/dist/types/service/index.d.ts +24 -0
  177. package/dist/types/service/utils.d.ts +2 -0
  178. package/dist/types/skill/index.d.ts +25 -0
  179. package/dist/types/task-runner.d.ts +50 -0
  180. package/dist/types/task-timing.d.ts +8 -0
  181. package/dist/types/tree.d.ts +4 -0
  182. package/dist/types/types.d.ts +684 -0
  183. package/dist/types/utils.d.ts +45 -0
  184. package/dist/types/yaml/builder.d.ts +2 -0
  185. package/dist/types/yaml/index.d.ts +4 -0
  186. package/dist/types/yaml/player.d.ts +34 -0
  187. package/dist/types/yaml/utils.d.ts +9 -0
  188. package/dist/types/yaml.d.ts +215 -0
  189. package/package.json +130 -0
@@ -0,0 +1,238 @@
1
+ "use strict";
2
+ var __webpack_require__ = {};
3
+ (()=>{
4
+ __webpack_require__.n = (module)=>{
5
+ var getter = module && module.__esModule ? ()=>module['default'] : ()=>module;
6
+ __webpack_require__.d(getter, {
7
+ a: getter
8
+ });
9
+ return getter;
10
+ };
11
+ })();
12
+ (()=>{
13
+ __webpack_require__.d = (exports1, definition)=>{
14
+ for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
15
+ enumerable: true,
16
+ get: definition[key]
17
+ });
18
+ };
19
+ })();
20
+ (()=>{
21
+ __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
22
+ })();
23
+ (()=>{
24
+ __webpack_require__.r = (exports1)=>{
25
+ if ("u" > typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
26
+ value: 'Module'
27
+ });
28
+ Object.defineProperty(exports1, '__esModule', {
29
+ value: true
30
+ });
31
+ };
32
+ })();
33
+ var __webpack_exports__ = {};
34
+ __webpack_require__.r(__webpack_exports__);
35
+ __webpack_require__.d(__webpack_exports__, {
36
+ getMidsceneVersion: ()=>getMidsceneVersion,
37
+ getReportFileName: ()=>getReportFileName,
38
+ matchElementFromPlan: ()=>matchElementFromPlan,
39
+ parsePrompt: ()=>parsePrompt,
40
+ matchElementFromCache: ()=>matchElementFromCache,
41
+ commonContextParser: ()=>commonContextParser,
42
+ transformLogicalElementToScreenshot: ()=>transformLogicalElementToScreenshot,
43
+ ifPlanLocateParamIsBbox: ()=>ifPlanLocateParamIsBbox,
44
+ printReportMsg: ()=>printReportMsg,
45
+ transformLogicalRectToScreenshotRect: ()=>transformLogicalRectToScreenshotRect
46
+ });
47
+ const external_screenshot_item_js_namespaceObject = require("../screenshot-item.js");
48
+ const external_utils_js_namespaceObject = require("../utils.js");
49
+ const env_namespaceObject = require("@godscene/shared/env");
50
+ const extractor_namespaceObject = require("@godscene/shared/extractor");
51
+ const img_namespaceObject = require("@godscene/shared/img");
52
+ const logger_namespaceObject = require("@godscene/shared/logger");
53
+ const utils_namespaceObject = require("@godscene/shared/utils");
54
+ const external_dayjs_namespaceObject = require("dayjs");
55
+ var external_dayjs_default = /*#__PURE__*/ __webpack_require__.n(external_dayjs_namespaceObject);
56
+ const external_task_cache_js_namespaceObject = require("./task-cache.js");
57
+ async function commonContextParser(interfaceInstance, _opt) {
58
+ const debug = (0, logger_namespaceObject.getDebug)('commonContextParser');
59
+ (0, utils_namespaceObject.assert)(interfaceInstance, 'interfaceInstance is required');
60
+ debug("Getting interface description");
61
+ const description = interfaceInstance.describe?.() || '';
62
+ debug("Interface description end");
63
+ debug('Uploading test info to server');
64
+ (0, external_utils_js_namespaceObject.uploadTestInfoToServer)({
65
+ testUrl: description,
66
+ serverUrl: _opt.uploadServerUrl
67
+ });
68
+ debug('UploadTestInfoToServer end');
69
+ debug('will get size');
70
+ const interfaceSize = await interfaceInstance.size();
71
+ const { width: logicalWidth, height: logicalHeight } = interfaceSize;
72
+ if (interfaceSize.dpr) console.warn('Warning: return value of interface.size() include a dpr property, which is not expected and ignored. ');
73
+ if (!Number.isFinite(logicalWidth) || !Number.isFinite(logicalHeight)) throw new Error(`Invalid interface size: width and height must be finite numbers. Received width: ${logicalWidth}, height: ${logicalHeight}`);
74
+ if (logicalWidth <= 0 || logicalHeight <= 0) throw new Error(`Invalid interface size: width and height must be positive numbers. Received width: ${logicalWidth}, height: ${logicalHeight}`);
75
+ debug(`size: ${logicalWidth}x${logicalHeight}`);
76
+ const screenshotBase64 = await interfaceInstance.screenshotBase64();
77
+ const screenshotCapturedAt = Date.now();
78
+ (0, utils_namespaceObject.assert)(screenshotBase64, 'screenshotBase64 is required');
79
+ debug('will get screenshot dimensions');
80
+ const { width: imgWidth, height: imgHeight } = await (0, img_namespaceObject.imageInfoOfBase64)(screenshotBase64);
81
+ if (!Number.isFinite(imgWidth) || !Number.isFinite(imgHeight)) throw new Error(`Invalid screenshot dimensions: width and height must be finite numbers. Received width: ${imgWidth}, height: ${imgHeight}`);
82
+ if (imgWidth <= 0 || imgHeight <= 0) throw new Error(`Invalid screenshot dimensions: width and height must be positive numbers. Received width: ${imgWidth}, height: ${imgHeight}`);
83
+ debug('screenshot dimensions', imgWidth, 'x', imgHeight);
84
+ const logicalIsPortrait = logicalWidth < logicalHeight;
85
+ const screenshotIsPortrait = imgWidth < imgHeight;
86
+ let finalLogicalWidth = logicalWidth;
87
+ if (logicalIsPortrait !== screenshotIsPortrait) {
88
+ debug(`Orientation mismatch detected: logical size ${logicalWidth}x${logicalHeight} (${logicalIsPortrait ? 'portrait' : 'landscape'}) vs screenshot ${imgWidth}x${imgHeight} (${screenshotIsPortrait ? 'portrait' : 'landscape'}). Swapping logical dimensions.`);
89
+ finalLogicalWidth = logicalHeight;
90
+ }
91
+ const userShrinkFactor = _opt.screenshotShrinkFactor ?? 1;
92
+ if (!Number.isFinite(userShrinkFactor) || userShrinkFactor < 1) throw new Error(`Invalid screenshotShrinkFactor: must be a finite number >= 1. Received: ${userShrinkFactor}`);
93
+ const dpr = imgWidth / finalLogicalWidth;
94
+ debug('calculated dpr:', dpr);
95
+ const shrunkShotToLogicalRatio = dpr / userShrinkFactor;
96
+ debug('shrunkShotToLogicalRatio', shrunkShotToLogicalRatio);
97
+ if (1 !== userShrinkFactor) {
98
+ const targetWidth = Math.round(imgWidth / userShrinkFactor);
99
+ const targetHeight = Math.round(imgHeight / userShrinkFactor);
100
+ debug(`Applying screenshot shrink factor: ${userShrinkFactor} (physical: ${imgWidth}x${imgHeight} -> target: ${targetWidth}x${targetHeight})`);
101
+ const resizedBase64 = await (0, img_namespaceObject.resizeImgBase64)(screenshotBase64, {
102
+ width: targetWidth,
103
+ height: targetHeight
104
+ });
105
+ return {
106
+ shotSize: {
107
+ width: targetWidth,
108
+ height: targetHeight
109
+ },
110
+ deprecatedDpr: dpr,
111
+ screenshot: external_screenshot_item_js_namespaceObject.ScreenshotItem.create(resizedBase64, screenshotCapturedAt),
112
+ shrunkShotToLogicalRatio
113
+ };
114
+ }
115
+ return {
116
+ shotSize: {
117
+ width: imgWidth,
118
+ height: imgHeight
119
+ },
120
+ deprecatedDpr: dpr,
121
+ screenshot: external_screenshot_item_js_namespaceObject.ScreenshotItem.create(screenshotBase64, screenshotCapturedAt),
122
+ shrunkShotToLogicalRatio
123
+ };
124
+ }
125
+ function getReportFileName(tag = 'web') {
126
+ const reportTagName = env_namespaceObject.globalConfigManager.getEnvConfigValue(env_namespaceObject.MIDSCENE_REPORT_TAG_NAME);
127
+ const dateTimeInFileName = external_dayjs_default()().format('YYYY-MM-DD_HH-mm-ss');
128
+ const uniqueId = (0, utils_namespaceObject.uuid)().substring(0, 8);
129
+ return `${reportTagName || tag}-${dateTimeInFileName}-${uniqueId}`;
130
+ }
131
+ function printReportMsg(filepath) {
132
+ if (env_namespaceObject.globalConfigManager.getEnvConfigInBoolean(env_namespaceObject.MIDSCENE_REPORT_QUIET)) return;
133
+ (0, utils_namespaceObject.logMsg)(`Midscene - report file updated: ${filepath}`);
134
+ }
135
+ function ifPlanLocateParamIsBbox(planLocateParam) {
136
+ return !!(planLocateParam.bbox && Array.isArray(planLocateParam.bbox) && 4 === planLocateParam.bbox.length);
137
+ }
138
+ function matchElementFromPlan(planLocateParam) {
139
+ if (!planLocateParam) return;
140
+ if (planLocateParam.bbox) {
141
+ const rect = {
142
+ left: planLocateParam.bbox[0],
143
+ top: planLocateParam.bbox[1],
144
+ width: planLocateParam.bbox[2] - planLocateParam.bbox[0] + 1,
145
+ height: planLocateParam.bbox[3] - planLocateParam.bbox[1] + 1
146
+ };
147
+ const element = (0, extractor_namespaceObject.generateElementByRect)(rect, 'string' == typeof planLocateParam.prompt ? planLocateParam.prompt : planLocateParam.prompt?.prompt || '');
148
+ return element;
149
+ }
150
+ }
151
+ async function matchElementFromCache(context, cacheEntry, cachePrompt, cacheable) {
152
+ if (!cacheEntry) return;
153
+ if (false === cacheable) return void (0, external_task_cache_js_namespaceObject.debug)('cache disabled for prompt: %s', cachePrompt);
154
+ if (!context.taskCache?.isCacheResultUsed) return;
155
+ if (!context.interfaceInstance.rectMatchesCacheFeature) return void (0, external_task_cache_js_namespaceObject.debug)('interface does not implement rectMatchesCacheFeature, skip cache');
156
+ try {
157
+ const rect = await context.interfaceInstance.rectMatchesCacheFeature(cacheEntry);
158
+ const element = {
159
+ center: [
160
+ Math.round(rect.left + rect.width / 2),
161
+ Math.round(rect.top + rect.height / 2)
162
+ ],
163
+ rect,
164
+ description: 'string' == typeof cachePrompt ? cachePrompt : cachePrompt.prompt || ''
165
+ };
166
+ (0, external_task_cache_js_namespaceObject.debug)('cache hit, prompt: %s', cachePrompt);
167
+ return element;
168
+ } catch (error) {
169
+ (0, external_task_cache_js_namespaceObject.debug)('rectMatchesCacheFeature error: %s', error);
170
+ return;
171
+ }
172
+ }
173
+ const getMidsceneVersion = ()=>"1.7.10";
174
+ const parsePrompt = (prompt)=>{
175
+ if ('string' == typeof prompt) return {
176
+ textPrompt: prompt,
177
+ multimodalPrompt: void 0
178
+ };
179
+ return {
180
+ textPrompt: prompt.prompt,
181
+ multimodalPrompt: prompt.images ? {
182
+ images: prompt.images,
183
+ convertHttpImage2Base64: !!prompt.convertHttpImage2Base64
184
+ } : void 0
185
+ };
186
+ };
187
+ const transformLogicalElementToScreenshot = (element, shrunkShotToLogicalRatio)=>{
188
+ if (1 === shrunkShotToLogicalRatio) return element;
189
+ return {
190
+ ...element,
191
+ center: [
192
+ Math.round(element.center[0] * shrunkShotToLogicalRatio),
193
+ Math.round(element.center[1] * shrunkShotToLogicalRatio)
194
+ ],
195
+ rect: {
196
+ ...element.rect,
197
+ left: Math.round(element.rect.left * shrunkShotToLogicalRatio),
198
+ top: Math.round(element.rect.top * shrunkShotToLogicalRatio),
199
+ width: Math.round(element.rect.width * shrunkShotToLogicalRatio),
200
+ height: Math.round(element.rect.height * shrunkShotToLogicalRatio)
201
+ }
202
+ };
203
+ };
204
+ const transformLogicalRectToScreenshotRect = (rect, shrunkShotToLogicalRatio)=>{
205
+ if (1 === shrunkShotToLogicalRatio) return rect;
206
+ return {
207
+ ...rect,
208
+ left: Math.round(rect.left * shrunkShotToLogicalRatio),
209
+ top: Math.round(rect.top * shrunkShotToLogicalRatio),
210
+ width: Math.round(rect.width * shrunkShotToLogicalRatio),
211
+ height: Math.round(rect.height * shrunkShotToLogicalRatio)
212
+ };
213
+ };
214
+ exports.commonContextParser = __webpack_exports__.commonContextParser;
215
+ exports.getMidsceneVersion = __webpack_exports__.getMidsceneVersion;
216
+ exports.getReportFileName = __webpack_exports__.getReportFileName;
217
+ exports.ifPlanLocateParamIsBbox = __webpack_exports__.ifPlanLocateParamIsBbox;
218
+ exports.matchElementFromCache = __webpack_exports__.matchElementFromCache;
219
+ exports.matchElementFromPlan = __webpack_exports__.matchElementFromPlan;
220
+ exports.parsePrompt = __webpack_exports__.parsePrompt;
221
+ exports.printReportMsg = __webpack_exports__.printReportMsg;
222
+ exports.transformLogicalElementToScreenshot = __webpack_exports__.transformLogicalElementToScreenshot;
223
+ exports.transformLogicalRectToScreenshotRect = __webpack_exports__.transformLogicalRectToScreenshotRect;
224
+ for(var __rspack_i in __webpack_exports__)if (-1 === [
225
+ "commonContextParser",
226
+ "getMidsceneVersion",
227
+ "getReportFileName",
228
+ "ifPlanLocateParamIsBbox",
229
+ "matchElementFromCache",
230
+ "matchElementFromPlan",
231
+ "parsePrompt",
232
+ "printReportMsg",
233
+ "transformLogicalElementToScreenshot",
234
+ "transformLogicalRectToScreenshotRect"
235
+ ].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
236
+ Object.defineProperty(exports, '__esModule', {
237
+ value: true
238
+ });
@@ -0,0 +1,271 @@
1
+ "use strict";
2
+ var __webpack_require__ = {};
3
+ (()=>{
4
+ __webpack_require__.d = (exports1, definition)=>{
5
+ for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
6
+ enumerable: true,
7
+ get: definition[key]
8
+ });
9
+ };
10
+ })();
11
+ (()=>{
12
+ __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
13
+ })();
14
+ (()=>{
15
+ __webpack_require__.r = (exports1)=>{
16
+ if ("u" > typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
17
+ value: 'Module'
18
+ });
19
+ Object.defineProperty(exports1, '__esModule', {
20
+ value: true
21
+ });
22
+ };
23
+ })();
24
+ var __webpack_exports__ = {};
25
+ __webpack_require__.r(__webpack_exports__);
26
+ __webpack_require__.d(__webpack_exports__, {
27
+ transformAutoGLMAction: ()=>transformAutoGLMAction
28
+ });
29
+ const external_common_js_namespaceObject = require("../../common.js");
30
+ const logger_namespaceObject = require("@godscene/shared/logger");
31
+ const debug = (0, logger_namespaceObject.getDebug)('auto-glm-actions');
32
+ const AUTO_GLM_COORDINATE_MAX = 1000;
33
+ function autoGLMCoordinateToBbox(x, y, width, height) {
34
+ const bbox = (0, external_common_js_namespaceObject.pointToBbox)(x, y, 10);
35
+ return (0, external_common_js_namespaceObject.adaptBbox)(bbox, width, height, 'auto-glm');
36
+ }
37
+ const BACK_BUTTON_NAMES = [
38
+ 'AndroidBackButton',
39
+ 'HarmonyBackButton'
40
+ ];
41
+ const HOME_BUTTON_NAMES = [
42
+ 'AndroidHomeButton',
43
+ 'HarmonyHomeButton'
44
+ ];
45
+ function findActionName(actionSpace, knownNames, defaultName) {
46
+ if (!actionSpace) return defaultName;
47
+ const match = actionSpace.find((a)=>knownNames.includes(a.name));
48
+ return match ? match.name : defaultName;
49
+ }
50
+ function transformAutoGLMAction(action, size, actionSpace) {
51
+ try {
52
+ switch(action._metadata){
53
+ case 'finish':
54
+ {
55
+ const finishAction = action;
56
+ debug('Transform finish action:', finishAction);
57
+ return [
58
+ {
59
+ type: 'Finished',
60
+ param: {},
61
+ thought: finishAction.message
62
+ }
63
+ ];
64
+ }
65
+ case 'do':
66
+ {
67
+ const doAction = action;
68
+ switch(doAction.action){
69
+ case 'Tap':
70
+ {
71
+ const tapAction = doAction;
72
+ debug('Transform Tap action:', tapAction);
73
+ const [x1, y1, x2, y2] = autoGLMCoordinateToBbox(tapAction.element[0], tapAction.element[1], size.width, size.height);
74
+ const locate = {
75
+ prompt: '',
76
+ bbox: [
77
+ x1,
78
+ y1,
79
+ x2,
80
+ y2
81
+ ]
82
+ };
83
+ return [
84
+ {
85
+ type: 'Tap',
86
+ param: {
87
+ locate
88
+ }
89
+ }
90
+ ];
91
+ }
92
+ case 'Double Tap':
93
+ {
94
+ const doubleTapAction = doAction;
95
+ debug('Transform Double Tap action:', doubleTapAction);
96
+ const [x1, y1, x2, y2] = autoGLMCoordinateToBbox(doubleTapAction.element[0], doubleTapAction.element[1], size.width, size.height);
97
+ const locate = {
98
+ prompt: '',
99
+ bbox: [
100
+ x1,
101
+ y1,
102
+ x2,
103
+ y2
104
+ ]
105
+ };
106
+ return [
107
+ {
108
+ type: 'DoubleClick',
109
+ param: {
110
+ locate
111
+ }
112
+ }
113
+ ];
114
+ }
115
+ case 'Type':
116
+ {
117
+ const typeAction = doAction;
118
+ debug('Transform Type action:', typeAction);
119
+ return [
120
+ {
121
+ type: 'Input',
122
+ param: {
123
+ value: typeAction.text
124
+ }
125
+ }
126
+ ];
127
+ }
128
+ case 'Swipe':
129
+ {
130
+ const swipeAction = doAction;
131
+ debug('Transform Swipe action:', swipeAction);
132
+ const [x1, y1, x2, y2] = autoGLMCoordinateToBbox(swipeAction.start[0], swipeAction.start[1], size.width, size.height);
133
+ const locate = {
134
+ prompt: '',
135
+ bbox: [
136
+ x1,
137
+ y1,
138
+ x2,
139
+ y2
140
+ ]
141
+ };
142
+ const deltaX = swipeAction.end[0] - swipeAction.start[0];
143
+ const deltaY = swipeAction.end[1] - swipeAction.start[1];
144
+ let direction;
145
+ let distance;
146
+ const absDeltaX = Math.abs(deltaX);
147
+ const absDeltaY = Math.abs(deltaY);
148
+ if (absDeltaY > absDeltaX) {
149
+ distance = Math.round(absDeltaY * size.height / AUTO_GLM_COORDINATE_MAX);
150
+ direction = deltaY > 0 ? 'up' : 'down';
151
+ } else {
152
+ distance = Math.round(absDeltaX * size.width / AUTO_GLM_COORDINATE_MAX);
153
+ direction = deltaX > 0 ? 'left' : 'right';
154
+ }
155
+ debug(`Calculate swipe direction: ${direction}, distance: ${distance}`);
156
+ return [
157
+ {
158
+ type: 'Scroll',
159
+ param: {
160
+ locate,
161
+ distance,
162
+ direction
163
+ },
164
+ thought: swipeAction.think || ''
165
+ }
166
+ ];
167
+ }
168
+ case 'Long Press':
169
+ {
170
+ const longPressAction = doAction;
171
+ debug('Transform Long Press action:', longPressAction);
172
+ const [x1, y1, x2, y2] = autoGLMCoordinateToBbox(longPressAction.element[0], longPressAction.element[1], size.width, size.height);
173
+ const locate = {
174
+ prompt: '',
175
+ bbox: [
176
+ x1,
177
+ y1,
178
+ x2,
179
+ y2
180
+ ]
181
+ };
182
+ return [
183
+ {
184
+ type: 'LongPress',
185
+ param: {
186
+ locate
187
+ },
188
+ thought: longPressAction.think || ''
189
+ }
190
+ ];
191
+ }
192
+ case 'Back':
193
+ {
194
+ const backAction = doAction;
195
+ debug('Transform Back action:', backAction);
196
+ return [
197
+ {
198
+ type: findActionName(actionSpace, BACK_BUTTON_NAMES, 'AndroidBackButton'),
199
+ param: {},
200
+ thought: backAction.think || ''
201
+ }
202
+ ];
203
+ }
204
+ case 'Home':
205
+ {
206
+ const homeAction = doAction;
207
+ debug('Transform Home action:', homeAction);
208
+ return [
209
+ {
210
+ type: findActionName(actionSpace, HOME_BUTTON_NAMES, 'AndroidHomeButton'),
211
+ param: {},
212
+ thought: homeAction.think || ''
213
+ }
214
+ ];
215
+ }
216
+ case 'Wait':
217
+ {
218
+ const waitAction = doAction;
219
+ debug('Transform Wait action:', waitAction);
220
+ return [
221
+ {
222
+ type: 'Sleep',
223
+ param: {
224
+ timeMs: waitAction.durationMs
225
+ },
226
+ thought: waitAction.think || ''
227
+ }
228
+ ];
229
+ }
230
+ case 'Launch':
231
+ {
232
+ const launchAction = doAction;
233
+ debug('Transform Launch action:', launchAction);
234
+ return [
235
+ {
236
+ type: 'Launch',
237
+ param: {
238
+ uri: launchAction.app
239
+ },
240
+ thought: launchAction.think || ''
241
+ }
242
+ ];
243
+ }
244
+ case 'Interact':
245
+ throw new Error('Action "Interact" from auto-glm is not supported in the current implementation.');
246
+ case 'Call_API':
247
+ throw new Error('Action "Call_API" from auto-glm is not supported in the current implementation.');
248
+ case 'Take_over':
249
+ throw new Error('Action "Take_over" from auto-glm is not supported in the current implementation.');
250
+ case 'Note':
251
+ throw new Error('Action "Note" from auto-glm is not supported in the current implementation.');
252
+ default:
253
+ throw new Error(`Unknown do() action type: ${doAction.action}`);
254
+ }
255
+ }
256
+ default:
257
+ throw new Error(`Unknown action metadata: ${action._metadata}`);
258
+ }
259
+ } catch (error) {
260
+ const errorMessage = error instanceof Error ? error.message : String(error);
261
+ debug('Transform error:', errorMessage);
262
+ throw new Error(`Failed to transform action: ${errorMessage}`);
263
+ }
264
+ }
265
+ exports.transformAutoGLMAction = __webpack_exports__.transformAutoGLMAction;
266
+ for(var __rspack_i in __webpack_exports__)if (-1 === [
267
+ "transformAutoGLMAction"
268
+ ].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
269
+ Object.defineProperty(exports, '__esModule', {
270
+ value: true
271
+ });
@@ -0,0 +1,64 @@
1
+ "use strict";
2
+ var __webpack_require__ = {};
3
+ (()=>{
4
+ __webpack_require__.d = (exports1, definition)=>{
5
+ for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
6
+ enumerable: true,
7
+ get: definition[key]
8
+ });
9
+ };
10
+ })();
11
+ (()=>{
12
+ __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
13
+ })();
14
+ (()=>{
15
+ __webpack_require__.r = (exports1)=>{
16
+ if ("u" > typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
17
+ value: 'Module'
18
+ });
19
+ Object.defineProperty(exports1, '__esModule', {
20
+ value: true
21
+ });
22
+ };
23
+ })();
24
+ var __webpack_exports__ = {};
25
+ __webpack_require__.r(__webpack_exports__);
26
+ __webpack_require__.d(__webpack_exports__, {
27
+ parseAction: ()=>external_parser_js_namespaceObject.parseAction,
28
+ isAutoGLM: ()=>external_util_js_namespaceObject.isAutoGLM,
29
+ isUITars: ()=>external_util_js_namespaceObject.isUITars,
30
+ autoGLMPlanning: ()=>external_planning_js_namespaceObject.autoGLMPlanning,
31
+ getAutoGLMPlanPrompt: ()=>external_prompt_js_namespaceObject.getAutoGLMPlanPrompt,
32
+ getAutoGLMLocatePrompt: ()=>external_prompt_js_namespaceObject.getAutoGLMLocatePrompt,
33
+ transformAutoGLMAction: ()=>external_actions_js_namespaceObject.transformAutoGLMAction,
34
+ parseAutoGLMLocateResponse: ()=>external_parser_js_namespaceObject.parseAutoGLMLocateResponse,
35
+ parseAutoGLMResponse: ()=>external_parser_js_namespaceObject.parseAutoGLMResponse
36
+ });
37
+ const external_prompt_js_namespaceObject = require("./prompt.js");
38
+ const external_parser_js_namespaceObject = require("./parser.js");
39
+ const external_planning_js_namespaceObject = require("./planning.js");
40
+ const external_actions_js_namespaceObject = require("./actions.js");
41
+ const external_util_js_namespaceObject = require("./util.js");
42
+ exports.autoGLMPlanning = __webpack_exports__.autoGLMPlanning;
43
+ exports.getAutoGLMLocatePrompt = __webpack_exports__.getAutoGLMLocatePrompt;
44
+ exports.getAutoGLMPlanPrompt = __webpack_exports__.getAutoGLMPlanPrompt;
45
+ exports.isAutoGLM = __webpack_exports__.isAutoGLM;
46
+ exports.isUITars = __webpack_exports__.isUITars;
47
+ exports.parseAction = __webpack_exports__.parseAction;
48
+ exports.parseAutoGLMLocateResponse = __webpack_exports__.parseAutoGLMLocateResponse;
49
+ exports.parseAutoGLMResponse = __webpack_exports__.parseAutoGLMResponse;
50
+ exports.transformAutoGLMAction = __webpack_exports__.transformAutoGLMAction;
51
+ for(var __rspack_i in __webpack_exports__)if (-1 === [
52
+ "autoGLMPlanning",
53
+ "getAutoGLMLocatePrompt",
54
+ "getAutoGLMPlanPrompt",
55
+ "isAutoGLM",
56
+ "isUITars",
57
+ "parseAction",
58
+ "parseAutoGLMLocateResponse",
59
+ "parseAutoGLMResponse",
60
+ "transformAutoGLMAction"
61
+ ].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
62
+ Object.defineProperty(exports, '__esModule', {
63
+ value: true
64
+ });