@aiscene/core 1.6.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (279) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +9 -0
  3. package/dist/es/agent/agent.mjs +749 -0
  4. package/dist/es/agent/agent.mjs.map +1 -0
  5. package/dist/es/agent/common.mjs +0 -0
  6. package/dist/es/agent/execution-session.mjs +41 -0
  7. package/dist/es/agent/execution-session.mjs.map +1 -0
  8. package/dist/es/agent/index.mjs +6 -0
  9. package/dist/es/agent/task-builder.mjs +332 -0
  10. package/dist/es/agent/task-builder.mjs.map +1 -0
  11. package/dist/es/agent/task-cache.mjs +214 -0
  12. package/dist/es/agent/task-cache.mjs.map +1 -0
  13. package/dist/es/agent/tasks.mjs +424 -0
  14. package/dist/es/agent/tasks.mjs.map +1 -0
  15. package/dist/es/agent/ui-utils.mjs +91 -0
  16. package/dist/es/agent/ui-utils.mjs.map +1 -0
  17. package/dist/es/agent/utils.mjs +198 -0
  18. package/dist/es/agent/utils.mjs.map +1 -0
  19. package/dist/es/ai-model/auto-glm/actions.mjs +237 -0
  20. package/dist/es/ai-model/auto-glm/actions.mjs.map +1 -0
  21. package/dist/es/ai-model/auto-glm/index.mjs +6 -0
  22. package/dist/es/ai-model/auto-glm/parser.mjs +239 -0
  23. package/dist/es/ai-model/auto-glm/parser.mjs.map +1 -0
  24. package/dist/es/ai-model/auto-glm/planning.mjs +71 -0
  25. package/dist/es/ai-model/auto-glm/planning.mjs.map +1 -0
  26. package/dist/es/ai-model/auto-glm/prompt.mjs +222 -0
  27. package/dist/es/ai-model/auto-glm/prompt.mjs.map +1 -0
  28. package/dist/es/ai-model/auto-glm/util.mjs +9 -0
  29. package/dist/es/ai-model/auto-glm/util.mjs.map +1 -0
  30. package/dist/es/ai-model/conversation-history.mjs +195 -0
  31. package/dist/es/ai-model/conversation-history.mjs.map +1 -0
  32. package/dist/es/ai-model/index.mjs +11 -0
  33. package/dist/es/ai-model/inspect.mjs +394 -0
  34. package/dist/es/ai-model/inspect.mjs.map +1 -0
  35. package/dist/es/ai-model/llm-planning.mjs +233 -0
  36. package/dist/es/ai-model/llm-planning.mjs.map +1 -0
  37. package/dist/es/ai-model/prompt/common.mjs +7 -0
  38. package/dist/es/ai-model/prompt/common.mjs.map +1 -0
  39. package/dist/es/ai-model/prompt/describe.mjs +66 -0
  40. package/dist/es/ai-model/prompt/describe.mjs.map +1 -0
  41. package/dist/es/ai-model/prompt/extraction.mjs +169 -0
  42. package/dist/es/ai-model/prompt/extraction.mjs.map +1 -0
  43. package/dist/es/ai-model/prompt/llm-locator.mjs +51 -0
  44. package/dist/es/ai-model/prompt/llm-locator.mjs.map +1 -0
  45. package/dist/es/ai-model/prompt/llm-planning.mjs +568 -0
  46. package/dist/es/ai-model/prompt/llm-planning.mjs.map +1 -0
  47. package/dist/es/ai-model/prompt/llm-section-locator.mjs +44 -0
  48. package/dist/es/ai-model/prompt/llm-section-locator.mjs.map +1 -0
  49. package/dist/es/ai-model/prompt/order-sensitive-judge.mjs +35 -0
  50. package/dist/es/ai-model/prompt/order-sensitive-judge.mjs.map +1 -0
  51. package/dist/es/ai-model/prompt/playwright-generator.mjs +117 -0
  52. package/dist/es/ai-model/prompt/playwright-generator.mjs.map +1 -0
  53. package/dist/es/ai-model/prompt/ui-tars-planning.mjs +36 -0
  54. package/dist/es/ai-model/prompt/ui-tars-planning.mjs.map +1 -0
  55. package/dist/es/ai-model/prompt/util.mjs +59 -0
  56. package/dist/es/ai-model/prompt/util.mjs.map +1 -0
  57. package/dist/es/ai-model/prompt/yaml-generator.mjs +219 -0
  58. package/dist/es/ai-model/prompt/yaml-generator.mjs.map +1 -0
  59. package/dist/es/ai-model/service-caller/codex-app-server.mjs +575 -0
  60. package/dist/es/ai-model/service-caller/codex-app-server.mjs.map +1 -0
  61. package/dist/es/ai-model/service-caller/image-detail.mjs +6 -0
  62. package/dist/es/ai-model/service-caller/image-detail.mjs.map +1 -0
  63. package/dist/es/ai-model/service-caller/index.mjs +473 -0
  64. package/dist/es/ai-model/service-caller/index.mjs.map +1 -0
  65. package/dist/es/ai-model/ui-tars-planning.mjs +249 -0
  66. package/dist/es/ai-model/ui-tars-planning.mjs.map +1 -0
  67. package/dist/es/common.mjs +371 -0
  68. package/dist/es/common.mjs.map +1 -0
  69. package/dist/es/device/device-options.mjs +0 -0
  70. package/dist/es/device/index.mjs +341 -0
  71. package/dist/es/device/index.mjs.map +1 -0
  72. package/dist/es/dump/html-utils.mjs +284 -0
  73. package/dist/es/dump/html-utils.mjs.map +1 -0
  74. package/dist/es/dump/image-restoration.mjs +43 -0
  75. package/dist/es/dump/image-restoration.mjs.map +1 -0
  76. package/dist/es/dump/index.mjs +3 -0
  77. package/dist/es/index.mjs +15 -0
  78. package/dist/es/index.mjs.map +1 -0
  79. package/dist/es/report-generator.mjs +162 -0
  80. package/dist/es/report-generator.mjs.map +1 -0
  81. package/dist/es/report.mjs +137 -0
  82. package/dist/es/report.mjs.map +1 -0
  83. package/dist/es/screenshot-item.mjs +105 -0
  84. package/dist/es/screenshot-item.mjs.map +1 -0
  85. package/dist/es/service/index.mjs +274 -0
  86. package/dist/es/service/index.mjs.map +1 -0
  87. package/dist/es/service/utils.mjs +15 -0
  88. package/dist/es/service/utils.mjs.map +1 -0
  89. package/dist/es/skill/index.mjs +38 -0
  90. package/dist/es/skill/index.mjs.map +1 -0
  91. package/dist/es/task-runner.mjs +263 -0
  92. package/dist/es/task-runner.mjs.map +1 -0
  93. package/dist/es/task-timing.mjs +12 -0
  94. package/dist/es/task-timing.mjs.map +1 -0
  95. package/dist/es/tree.mjs +13 -0
  96. package/dist/es/tree.mjs.map +1 -0
  97. package/dist/es/types.mjs +199 -0
  98. package/dist/es/types.mjs.map +1 -0
  99. package/dist/es/utils.mjs +229 -0
  100. package/dist/es/utils.mjs.map +1 -0
  101. package/dist/es/yaml/builder.mjs +13 -0
  102. package/dist/es/yaml/builder.mjs.map +1 -0
  103. package/dist/es/yaml/index.mjs +4 -0
  104. package/dist/es/yaml/player.mjs +434 -0
  105. package/dist/es/yaml/player.mjs.map +1 -0
  106. package/dist/es/yaml/utils.mjs +102 -0
  107. package/dist/es/yaml/utils.mjs.map +1 -0
  108. package/dist/es/yaml.mjs +0 -0
  109. package/dist/lib/agent/agent.js +797 -0
  110. package/dist/lib/agent/agent.js.map +1 -0
  111. package/dist/lib/agent/common.js +5 -0
  112. package/dist/lib/agent/execution-session.js +75 -0
  113. package/dist/lib/agent/execution-session.js.map +1 -0
  114. package/dist/lib/agent/index.js +81 -0
  115. package/dist/lib/agent/index.js.map +1 -0
  116. package/dist/lib/agent/task-builder.js +369 -0
  117. package/dist/lib/agent/task-builder.js.map +1 -0
  118. package/dist/lib/agent/task-cache.js +266 -0
  119. package/dist/lib/agent/task-cache.js.map +1 -0
  120. package/dist/lib/agent/tasks.js +467 -0
  121. package/dist/lib/agent/tasks.js.map +1 -0
  122. package/dist/lib/agent/ui-utils.js +143 -0
  123. package/dist/lib/agent/ui-utils.js.map +1 -0
  124. package/dist/lib/agent/utils.js +275 -0
  125. package/dist/lib/agent/utils.js.map +1 -0
  126. package/dist/lib/ai-model/auto-glm/actions.js +271 -0
  127. package/dist/lib/ai-model/auto-glm/actions.js.map +1 -0
  128. package/dist/lib/ai-model/auto-glm/index.js +66 -0
  129. package/dist/lib/ai-model/auto-glm/index.js.map +1 -0
  130. package/dist/lib/ai-model/auto-glm/parser.js +282 -0
  131. package/dist/lib/ai-model/auto-glm/parser.js.map +1 -0
  132. package/dist/lib/ai-model/auto-glm/planning.js +105 -0
  133. package/dist/lib/ai-model/auto-glm/planning.js.map +1 -0
  134. package/dist/lib/ai-model/auto-glm/prompt.js +259 -0
  135. package/dist/lib/ai-model/auto-glm/prompt.js.map +1 -0
  136. package/dist/lib/ai-model/auto-glm/util.js +46 -0
  137. package/dist/lib/ai-model/auto-glm/util.js.map +1 -0
  138. package/dist/lib/ai-model/conversation-history.js +229 -0
  139. package/dist/lib/ai-model/conversation-history.js.map +1 -0
  140. package/dist/lib/ai-model/index.js +125 -0
  141. package/dist/lib/ai-model/index.js.map +1 -0
  142. package/dist/lib/ai-model/inspect.js +440 -0
  143. package/dist/lib/ai-model/inspect.js.map +1 -0
  144. package/dist/lib/ai-model/llm-planning.js +270 -0
  145. package/dist/lib/ai-model/llm-planning.js.map +1 -0
  146. package/dist/lib/ai-model/prompt/common.js +41 -0
  147. package/dist/lib/ai-model/prompt/common.js.map +1 -0
  148. package/dist/lib/ai-model/prompt/describe.js +100 -0
  149. package/dist/lib/ai-model/prompt/describe.js.map +1 -0
  150. package/dist/lib/ai-model/prompt/extraction.js +209 -0
  151. package/dist/lib/ai-model/prompt/extraction.js.map +1 -0
  152. package/dist/lib/ai-model/prompt/llm-locator.js +88 -0
  153. package/dist/lib/ai-model/prompt/llm-locator.js.map +1 -0
  154. package/dist/lib/ai-model/prompt/llm-planning.js +605 -0
  155. package/dist/lib/ai-model/prompt/llm-planning.js.map +1 -0
  156. package/dist/lib/ai-model/prompt/llm-section-locator.js +81 -0
  157. package/dist/lib/ai-model/prompt/llm-section-locator.js.map +1 -0
  158. package/dist/lib/ai-model/prompt/order-sensitive-judge.js +72 -0
  159. package/dist/lib/ai-model/prompt/order-sensitive-judge.js.map +1 -0
  160. package/dist/lib/ai-model/prompt/playwright-generator.js +178 -0
  161. package/dist/lib/ai-model/prompt/playwright-generator.js.map +1 -0
  162. package/dist/lib/ai-model/prompt/ui-tars-planning.js +73 -0
  163. package/dist/lib/ai-model/prompt/ui-tars-planning.js.map +1 -0
  164. package/dist/lib/ai-model/prompt/util.js +105 -0
  165. package/dist/lib/ai-model/prompt/util.js.map +1 -0
  166. package/dist/lib/ai-model/prompt/yaml-generator.js +280 -0
  167. package/dist/lib/ai-model/prompt/yaml-generator.js.map +1 -0
  168. package/dist/lib/ai-model/service-caller/codex-app-server.js +624 -0
  169. package/dist/lib/ai-model/service-caller/codex-app-server.js.map +1 -0
  170. package/dist/lib/ai-model/service-caller/image-detail.js +40 -0
  171. package/dist/lib/ai-model/service-caller/image-detail.js.map +1 -0
  172. package/dist/lib/ai-model/service-caller/index.js +538 -0
  173. package/dist/lib/ai-model/service-caller/index.js.map +1 -0
  174. package/dist/lib/ai-model/ui-tars-planning.js +283 -0
  175. package/dist/lib/ai-model/ui-tars-planning.js.map +1 -0
  176. package/dist/lib/common.js +480 -0
  177. package/dist/lib/common.js.map +1 -0
  178. package/dist/lib/device/device-options.js +20 -0
  179. package/dist/lib/device/device-options.js.map +1 -0
  180. package/dist/lib/device/index.js +468 -0
  181. package/dist/lib/device/index.js.map +1 -0
  182. package/dist/lib/dump/html-utils.js +357 -0
  183. package/dist/lib/dump/html-utils.js.map +1 -0
  184. package/dist/lib/dump/image-restoration.js +77 -0
  185. package/dist/lib/dump/image-restoration.js.map +1 -0
  186. package/dist/lib/dump/index.js +60 -0
  187. package/dist/lib/dump/index.js.map +1 -0
  188. package/dist/lib/index.js +146 -0
  189. package/dist/lib/index.js.map +1 -0
  190. package/dist/lib/report-generator.js +200 -0
  191. package/dist/lib/report-generator.js.map +1 -0
  192. package/dist/lib/report.js +171 -0
  193. package/dist/lib/report.js.map +1 -0
  194. package/dist/lib/screenshot-item.js +139 -0
  195. package/dist/lib/screenshot-item.js.map +1 -0
  196. package/dist/lib/service/index.js +308 -0
  197. package/dist/lib/service/index.js.map +1 -0
  198. package/dist/lib/service/utils.js +49 -0
  199. package/dist/lib/service/utils.js.map +1 -0
  200. package/dist/lib/skill/index.js +72 -0
  201. package/dist/lib/skill/index.js.map +1 -0
  202. package/dist/lib/task-runner.js +300 -0
  203. package/dist/lib/task-runner.js.map +1 -0
  204. package/dist/lib/task-timing.js +46 -0
  205. package/dist/lib/task-timing.js.map +1 -0
  206. package/dist/lib/tree.js +53 -0
  207. package/dist/lib/tree.js.map +1 -0
  208. package/dist/lib/types.js +288 -0
  209. package/dist/lib/types.js.map +1 -0
  210. package/dist/lib/utils.js +308 -0
  211. package/dist/lib/utils.js.map +1 -0
  212. package/dist/lib/yaml/builder.js +57 -0
  213. package/dist/lib/yaml/builder.js.map +1 -0
  214. package/dist/lib/yaml/index.js +81 -0
  215. package/dist/lib/yaml/index.js.map +1 -0
  216. package/dist/lib/yaml/player.js +468 -0
  217. package/dist/lib/yaml/player.js.map +1 -0
  218. package/dist/lib/yaml/utils.js +155 -0
  219. package/dist/lib/yaml/utils.js.map +1 -0
  220. package/dist/lib/yaml.js +20 -0
  221. package/dist/lib/yaml.js.map +1 -0
  222. package/dist/types/agent/agent.d.ts +205 -0
  223. package/dist/types/agent/common.d.ts +0 -0
  224. package/dist/types/agent/execution-session.d.ts +36 -0
  225. package/dist/types/agent/index.d.ts +10 -0
  226. package/dist/types/agent/task-builder.d.ts +34 -0
  227. package/dist/types/agent/task-cache.d.ts +49 -0
  228. package/dist/types/agent/tasks.d.ts +69 -0
  229. package/dist/types/agent/ui-utils.d.ts +14 -0
  230. package/dist/types/agent/utils.d.ts +31 -0
  231. package/dist/types/ai-model/auto-glm/actions.d.ts +78 -0
  232. package/dist/types/ai-model/auto-glm/index.d.ts +6 -0
  233. package/dist/types/ai-model/auto-glm/parser.d.ts +18 -0
  234. package/dist/types/ai-model/auto-glm/planning.d.ts +12 -0
  235. package/dist/types/ai-model/auto-glm/prompt.d.ts +27 -0
  236. package/dist/types/ai-model/auto-glm/util.d.ts +13 -0
  237. package/dist/types/ai-model/conversation-history.d.ts +105 -0
  238. package/dist/types/ai-model/index.d.ts +14 -0
  239. package/dist/types/ai-model/inspect.d.ts +67 -0
  240. package/dist/types/ai-model/llm-planning.d.ts +19 -0
  241. package/dist/types/ai-model/prompt/common.d.ts +2 -0
  242. package/dist/types/ai-model/prompt/describe.d.ts +1 -0
  243. package/dist/types/ai-model/prompt/extraction.d.ts +7 -0
  244. package/dist/types/ai-model/prompt/llm-locator.d.ts +3 -0
  245. package/dist/types/ai-model/prompt/llm-planning.d.ts +10 -0
  246. package/dist/types/ai-model/prompt/llm-section-locator.d.ts +3 -0
  247. package/dist/types/ai-model/prompt/order-sensitive-judge.d.ts +2 -0
  248. package/dist/types/ai-model/prompt/playwright-generator.d.ts +26 -0
  249. package/dist/types/ai-model/prompt/ui-tars-planning.d.ts +2 -0
  250. package/dist/types/ai-model/prompt/util.d.ts +33 -0
  251. package/dist/types/ai-model/prompt/yaml-generator.d.ts +100 -0
  252. package/dist/types/ai-model/service-caller/codex-app-server.d.ts +42 -0
  253. package/dist/types/ai-model/service-caller/image-detail.d.ts +2 -0
  254. package/dist/types/ai-model/service-caller/index.d.ts +49 -0
  255. package/dist/types/ai-model/ui-tars-planning.d.ts +72 -0
  256. package/dist/types/common.d.ts +288 -0
  257. package/dist/types/device/device-options.d.ts +142 -0
  258. package/dist/types/device/index.d.ts +2528 -0
  259. package/dist/types/dump/html-utils.d.ts +63 -0
  260. package/dist/types/dump/image-restoration.d.ts +6 -0
  261. package/dist/types/dump/index.d.ts +5 -0
  262. package/dist/types/index.d.ts +17 -0
  263. package/dist/types/report-generator.d.ts +66 -0
  264. package/dist/types/report.d.ts +22 -0
  265. package/dist/types/screenshot-item.d.ts +66 -0
  266. package/dist/types/service/index.d.ts +24 -0
  267. package/dist/types/service/utils.d.ts +2 -0
  268. package/dist/types/skill/index.d.ts +25 -0
  269. package/dist/types/task-runner.d.ts +50 -0
  270. package/dist/types/task-timing.d.ts +8 -0
  271. package/dist/types/tree.d.ts +4 -0
  272. package/dist/types/types.d.ts +669 -0
  273. package/dist/types/utils.d.ts +40 -0
  274. package/dist/types/yaml/builder.d.ts +2 -0
  275. package/dist/types/yaml/index.d.ts +4 -0
  276. package/dist/types/yaml/player.d.ts +34 -0
  277. package/dist/types/yaml/utils.d.ts +9 -0
  278. package/dist/types/yaml.d.ts +215 -0
  279. package/package.json +111 -0
@@ -0,0 +1,139 @@
1
+ "use strict";
2
+ var __webpack_require__ = {};
3
+ (()=>{
4
+ __webpack_require__.d = (exports1, definition)=>{
5
+ for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
6
+ enumerable: true,
7
+ get: definition[key]
8
+ });
9
+ };
10
+ })();
11
+ (()=>{
12
+ __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
13
+ })();
14
+ (()=>{
15
+ __webpack_require__.r = (exports1)=>{
16
+ if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
17
+ value: 'Module'
18
+ });
19
+ Object.defineProperty(exports1, '__esModule', {
20
+ value: true
21
+ });
22
+ };
23
+ })();
24
+ var __webpack_exports__ = {};
25
+ __webpack_require__.r(__webpack_exports__);
26
+ __webpack_require__.d(__webpack_exports__, {
27
+ ScreenshotItem: ()=>ScreenshotItem
28
+ });
29
+ const external_node_fs_namespaceObject = require("node:fs");
30
+ const utils_namespaceObject = require("@midscene/shared/utils");
31
+ const html_utils_js_namespaceObject = require("./dump/html-utils.js");
32
+ function _define_property(obj, key, value) {
33
+ if (key in obj) Object.defineProperty(obj, key, {
34
+ value: value,
35
+ enumerable: true,
36
+ configurable: true,
37
+ writable: true
38
+ });
39
+ else obj[key] = value;
40
+ return obj;
41
+ }
42
+ function detectFormat(base64) {
43
+ if (base64.startsWith('data:image/jpeg')) return 'jpeg';
44
+ if (base64.startsWith('data:image/jpg')) return 'jpeg';
45
+ return 'png';
46
+ }
47
+ class ScreenshotItem {
48
+ static create(base64, capturedAt) {
49
+ return new ScreenshotItem((0, utils_namespaceObject.uuid)(), base64, capturedAt);
50
+ }
51
+ get id() {
52
+ return this._id;
53
+ }
54
+ get format() {
55
+ return this._format;
56
+ }
57
+ get extension() {
58
+ return 'jpeg' === this._format ? 'jpeg' : 'png';
59
+ }
60
+ get capturedAt() {
61
+ return this._capturedAt;
62
+ }
63
+ get base64() {
64
+ if (null !== this._base64) return this._base64;
65
+ if (null !== this._persistedPath) {
66
+ const buffer = (0, external_node_fs_namespaceObject.readFileSync)(this._persistedPath);
67
+ return `data:image/${this._format};base64,${buffer.toString('base64')}`;
68
+ }
69
+ if (null !== this._persistedHtmlPath) {
70
+ const data = (0, html_utils_js_namespaceObject.extractImageByIdSync)(this._persistedHtmlPath, this._id);
71
+ if (data) return data;
72
+ throw new Error(`Screenshot ${this._id}: cannot recover from HTML (id not found in ${this._persistedHtmlPath})`);
73
+ }
74
+ throw new Error(`Screenshot ${this._id}: base64 data released without recovery path`);
75
+ }
76
+ hasBase64() {
77
+ return null !== this._base64;
78
+ }
79
+ markPersistedInline(htmlPath) {
80
+ this._persistedAs = {
81
+ $screenshot: this._id,
82
+ capturedAt: this._capturedAt
83
+ };
84
+ this._persistedHtmlPath = htmlPath;
85
+ this._base64 = null;
86
+ }
87
+ markPersistedToPath(relativePath, absolutePath) {
88
+ this._persistedAs = {
89
+ base64: relativePath,
90
+ capturedAt: this._capturedAt
91
+ };
92
+ this._persistedPath = absolutePath;
93
+ this._base64 = null;
94
+ }
95
+ toSerializable() {
96
+ return this._persistedAs ?? {
97
+ $screenshot: this._id,
98
+ capturedAt: this._capturedAt
99
+ };
100
+ }
101
+ static isSerialized(value) {
102
+ if ('object' != typeof value || null === value) return false;
103
+ const record = value;
104
+ if ('$screenshot' in record && 'string' == typeof record.$screenshot) {
105
+ if (!('capturedAt' in record) || 'number' != typeof record.capturedAt) return false;
106
+ return true;
107
+ }
108
+ if ('base64' in record && 'string' == typeof record.base64) {
109
+ if (!('capturedAt' in record) || 'number' != typeof record.capturedAt) return false;
110
+ return true;
111
+ }
112
+ return false;
113
+ }
114
+ get rawBase64() {
115
+ return this.base64.replace(/^data:image\/(png|jpeg|jpg);base64,/, '');
116
+ }
117
+ constructor(id, base64, capturedAt){
118
+ _define_property(this, "_id", void 0);
119
+ _define_property(this, "_base64", void 0);
120
+ _define_property(this, "_format", void 0);
121
+ _define_property(this, "_capturedAt", void 0);
122
+ _define_property(this, "_persistedAs", null);
123
+ _define_property(this, "_persistedPath", null);
124
+ _define_property(this, "_persistedHtmlPath", null);
125
+ this._id = id;
126
+ this._base64 = base64;
127
+ this._format = detectFormat(base64);
128
+ this._capturedAt = capturedAt;
129
+ }
130
+ }
131
+ exports.ScreenshotItem = __webpack_exports__.ScreenshotItem;
132
+ for(var __rspack_i in __webpack_exports__)if (-1 === [
133
+ "ScreenshotItem"
134
+ ].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
135
+ Object.defineProperty(exports, '__esModule', {
136
+ value: true
137
+ });
138
+
139
+ //# sourceMappingURL=screenshot-item.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"screenshot-item.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../src/screenshot-item.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n }\n }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { readFileSync } from 'node:fs';\nimport { uuid } from '@midscene/shared/utils';\nimport { extractImageByIdSync } from './dump/html-utils';\n\n/**\n * Serialization format for ScreenshotItem\n * - { $screenshot: \"id\" } - inline mode, references imageMap in HTML\n * - { base64: \"path\" } - directory mode, references external file path\n */\nexport type ScreenshotSerializeFormat =\n | { $screenshot: string; capturedAt: number }\n | { base64: string; capturedAt: number };\n\n/**\n * Detect image format from base64 data URI prefix.\n */\nfunction detectFormat(base64: string): 'png' | 'jpeg' {\n if (base64.startsWith('data:image/jpeg')) return 'jpeg';\n if (base64.startsWith('data:image/jpg')) return 'jpeg';\n return 'png';\n}\n\n/**\n * ScreenshotItem encapsulates screenshot data.\n *\n * Supports lazy loading after memory release:\n * - inline mode: reads from HTML file using streaming (extractImageByIdSync)\n * - directory mode: reads from file on disk\n *\n * After persistence, memory is released but the screenshot can be recovered\n * on-demand from disk, making it safe to release memory at any time.\n */\nexport class ScreenshotItem {\n private _id: string;\n private _base64: string | null;\n private _format: 'png' | 'jpeg';\n private _capturedAt: number;\n private _persistedAs: ScreenshotSerializeFormat | null = null;\n private _persistedPath: string | null = null;\n private _persistedHtmlPath: string | null = null;\n\n private constructor(id: string, base64: string, capturedAt: number) {\n this._id = id;\n this._base64 = base64;\n this._format = detectFormat(base64);\n this._capturedAt = capturedAt;\n }\n\n /** Create a new ScreenshotItem from base64 data */\n static create(base64: string, capturedAt: number): ScreenshotItem {\n return new ScreenshotItem(uuid(), base64, capturedAt);\n }\n\n get id(): string {\n return this._id;\n }\n\n /** Get the image format (png or jpeg) */\n get format(): 'png' | 'jpeg' {\n return this._format;\n }\n\n /** Get the file extension for this screenshot */\n get extension(): string {\n return this._format === 'jpeg' ? 'jpeg' : 'png';\n }\n\n /** Get screenshot capture timestamp in milliseconds */\n get capturedAt(): number {\n return this._capturedAt;\n }\n\n get base64(): string {\n // If data is in memory, return it directly\n if (this._base64 !== null) {\n return this._base64;\n }\n\n // Directory mode: recover from file\n if (this._persistedPath !== null) {\n const buffer = readFileSync(this._persistedPath);\n return `data:image/${this._format};base64,${buffer.toString('base64')}`;\n }\n\n // Inline mode: recover from HTML file using streaming\n if (this._persistedHtmlPath !== null) {\n const data = extractImageByIdSync(this._persistedHtmlPath, this._id);\n if (data) {\n return data;\n }\n throw new Error(\n `Screenshot ${this._id}: cannot recover from HTML (id not found in ${this._persistedHtmlPath})`,\n );\n }\n\n throw new Error(\n `Screenshot ${this._id}: base64 data released without recovery path`,\n );\n }\n\n /** Check if base64 data is still available in memory (not yet released) */\n hasBase64(): boolean {\n return this._base64 !== null;\n }\n\n /**\n * Mark as persisted to HTML (inline mode).\n * Releases base64 memory, but keeps HTML path for lazy loading recovery.\n * @param htmlPath - absolute path to the HTML file containing the image\n */\n markPersistedInline(htmlPath: string): void {\n this._persistedAs = {\n $screenshot: this._id,\n capturedAt: this._capturedAt,\n };\n this._persistedHtmlPath = htmlPath;\n this._base64 = null;\n }\n\n /**\n * Mark as persisted to file (directory mode).\n * Releases base64 memory, but keeps file path for lazy loading recovery.\n * @param relativePath - relative path for serialization (e.g., \"./screenshots/id.jpeg\")\n * @param absolutePath - absolute path for lazy loading recovery\n */\n markPersistedToPath(relativePath: string, absolutePath: string): void {\n this._persistedAs = {\n base64: relativePath,\n capturedAt: this._capturedAt,\n };\n this._persistedPath = absolutePath;\n this._base64 = null;\n }\n\n /** Serialize for JSON - format depends on persistence state */\n toSerializable(): ScreenshotSerializeFormat {\n return (\n this._persistedAs ?? {\n $screenshot: this._id,\n capturedAt: this._capturedAt,\n }\n );\n }\n\n /** Check if a value is a serialized ScreenshotItem reference (inline or directory mode) */\n static isSerialized(value: unknown): value is ScreenshotSerializeFormat {\n if (typeof value !== 'object' || value === null) return false;\n const record = value as Record<string, unknown>;\n // Check for inline mode: { $screenshot: string }\n if ('$screenshot' in record && typeof record.$screenshot === 'string') {\n if (!('capturedAt' in record) || typeof record.capturedAt !== 'number') {\n return false;\n }\n return true;\n }\n // Check for directory mode: { base64: string } where base64 is a path\n if ('base64' in record && typeof record.base64 === 'string') {\n if (!('capturedAt' in record) || typeof record.capturedAt !== 'number') {\n return false;\n }\n return true;\n }\n return false;\n }\n\n /**\n * Get base64 data without the data URI prefix.\n * Useful for writing raw binary data to files.\n */\n get rawBase64(): string {\n return this.base64.replace(/^data:image\\/(png|jpeg|jpg);base64,/, '');\n }\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","detectFormat","base64","ScreenshotItem","capturedAt","uuid","buffer","readFileSync","data","extractImageByIdSync","Error","htmlPath","relativePath","absolutePath","value","record","id"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;;ACUA,SAASI,aAAaC,MAAc;IAClC,IAAIA,OAAO,UAAU,CAAC,oBAAoB,OAAO;IACjD,IAAIA,OAAO,UAAU,CAAC,mBAAmB,OAAO;IAChD,OAAO;AACT;AAYO,MAAMC;IAiBX,OAAO,OAAOD,MAAc,EAAEE,UAAkB,EAAkB;QAChE,OAAO,IAAID,eAAeE,AAAAA,IAAAA,sBAAAA,IAAAA,AAAAA,KAAQH,QAAQE;IAC5C;IAEA,IAAI,KAAa;QACf,OAAO,IAAI,CAAC,GAAG;IACjB;IAGA,IAAI,SAAyB;QAC3B,OAAO,IAAI,CAAC,OAAO;IACrB;IAGA,IAAI,YAAoB;QACtB,OAAO,AAAiB,WAAjB,IAAI,CAAC,OAAO,GAAc,SAAS;IAC5C;IAGA,IAAI,aAAqB;QACvB,OAAO,IAAI,CAAC,WAAW;IACzB;IAEA,IAAI,SAAiB;QAEnB,IAAI,AAAiB,SAAjB,IAAI,CAAC,OAAO,EACd,OAAO,IAAI,CAAC,OAAO;QAIrB,IAAI,AAAwB,SAAxB,IAAI,CAAC,cAAc,EAAW;YAChC,MAAME,SAASC,AAAAA,IAAAA,iCAAAA,YAAAA,AAAAA,EAAa,IAAI,CAAC,cAAc;YAC/C,OAAO,CAAC,WAAW,EAAE,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAED,OAAO,QAAQ,CAAC,WAAW;QACzE;QAGA,IAAI,AAA4B,SAA5B,IAAI,CAAC,kBAAkB,EAAW;YACpC,MAAME,OAAOC,AAAAA,IAAAA,8BAAAA,oBAAAA,AAAAA,EAAqB,IAAI,CAAC,kBAAkB,EAAE,IAAI,CAAC,GAAG;YACnE,IAAID,MACF,OAAOA;YAET,MAAM,IAAIE,MACR,CAAC,WAAW,EAAE,IAAI,CAAC,GAAG,CAAC,4CAA4C,EAAE,IAAI,CAAC,kBAAkB,CAAC,CAAC,CAAC;QAEnG;QAEA,MAAM,IAAIA,MACR,CAAC,WAAW,EAAE,IAAI,CAAC,GAAG,CAAC,4CAA4C,CAAC;IAExE;IAGA,YAAqB;QACnB,OAAO,AAAiB,SAAjB,IAAI,CAAC,OAAO;IACrB;IAOA,oBAAoBC,QAAgB,EAAQ;QAC1C,IAAI,CAAC,YAAY,GAAG;YAClB,aAAa,IAAI,CAAC,GAAG;YACrB,YAAY,IAAI,CAAC,WAAW;QAC9B;QACA,IAAI,CAAC,kBAAkB,GAAGA;QAC1B,IAAI,CAAC,OAAO,GAAG;IACjB;IAQA,oBAAoBC,YAAoB,EAAEC,YAAoB,EAAQ;QACpE,IAAI,CAAC,YAAY,GAAG;YAClB,QAAQD;YACR,YAAY,IAAI,CAAC,WAAW;QAC9B;QACA,IAAI,CAAC,cAAc,GAAGC;QACtB,IAAI,CAAC,OAAO,GAAG;IACjB;IAGA,iBAA4C;QAC1C,OACE,IAAI,CAAC,YAAY,IAAI;YACnB,aAAa,IAAI,CAAC,GAAG;YACrB,YAAY,IAAI,CAAC,WAAW;QAC9B;IAEJ;IAGA,OAAO,aAAaC,KAAc,EAAsC;QACtE,IAAI,AAAiB,YAAjB,OAAOA,SAAsBA,AAAU,SAAVA,OAAgB,OAAO;QACxD,MAAMC,SAASD;QAEf,IAAI,iBAAiBC,UAAU,AAA8B,YAA9B,OAAOA,OAAO,WAAW,EAAe;YACrE,IAAI,CAAE,iBAAgBA,MAAK,KAAM,AAA6B,YAA7B,OAAOA,OAAO,UAAU,EACvD,OAAO;YAET,OAAO;QACT;QAEA,IAAI,YAAYA,UAAU,AAAyB,YAAzB,OAAOA,OAAO,MAAM,EAAe;YAC3D,IAAI,CAAE,iBAAgBA,MAAK,KAAM,AAA6B,YAA7B,OAAOA,OAAO,UAAU,EACvD,OAAO;YAET,OAAO;QACT;QACA,OAAO;IACT;IAMA,IAAI,YAAoB;QACtB,OAAO,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,uCAAuC;IACpE;IAlIA,YAAoBC,EAAU,EAAEd,MAAc,EAAEE,UAAkB,CAAE;QARpE,uBAAQ,OAAR;QACA,uBAAQ,WAAR;QACA,uBAAQ,WAAR;QACA,uBAAQ,eAAR;QACA,uBAAQ,gBAAiD;QACzD,uBAAQ,kBAAgC;QACxC,uBAAQ,sBAAoC;QAG1C,IAAI,CAAC,GAAG,GAAGY;QACX,IAAI,CAAC,OAAO,GAAGd;QACf,IAAI,CAAC,OAAO,GAAGD,aAAaC;QAC5B,IAAI,CAAC,WAAW,GAAGE;IACrB;AA8HF"}
@@ -0,0 +1,308 @@
1
+ "use strict";
2
+ var __webpack_require__ = {};
3
+ (()=>{
4
+ __webpack_require__.d = (exports1, definition)=>{
5
+ for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
6
+ enumerable: true,
7
+ get: definition[key]
8
+ });
9
+ };
10
+ })();
11
+ (()=>{
12
+ __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
13
+ })();
14
+ (()=>{
15
+ __webpack_require__.r = (exports1)=>{
16
+ if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
17
+ value: 'Module'
18
+ });
19
+ Object.defineProperty(exports1, '__esModule', {
20
+ value: true
21
+ });
22
+ };
23
+ })();
24
+ var __webpack_exports__ = {};
25
+ __webpack_require__.r(__webpack_exports__);
26
+ __webpack_require__.d(__webpack_exports__, {
27
+ default: ()=>Service
28
+ });
29
+ const util_js_namespaceObject = require("../ai-model/auto-glm/util.js");
30
+ const index_js_namespaceObject = require("../ai-model/index.js");
31
+ const inspect_js_namespaceObject = require("../ai-model/inspect.js");
32
+ const describe_js_namespaceObject = require("../ai-model/prompt/describe.js");
33
+ const external_common_js_namespaceObject = require("../common.js");
34
+ const external_types_js_namespaceObject = require("../types.js");
35
+ const img_namespaceObject = require("@midscene/shared/img");
36
+ const logger_namespaceObject = require("@midscene/shared/logger");
37
+ const utils_namespaceObject = require("@midscene/shared/utils");
38
+ const external_utils_js_namespaceObject = require("./utils.js");
39
+ function _define_property(obj, key, value) {
40
+ if (key in obj) Object.defineProperty(obj, key, {
41
+ value: value,
42
+ enumerable: true,
43
+ configurable: true,
44
+ writable: true
45
+ });
46
+ else obj[key] = value;
47
+ return obj;
48
+ }
49
+ const debug = (0, logger_namespaceObject.getDebug)('ai:service');
50
+ class Service {
51
+ async locate(query, opt, modelConfig, abortSignal) {
52
+ const queryPrompt = 'string' == typeof query ? query : query.prompt;
53
+ (0, utils_namespaceObject.assert)(queryPrompt, 'query is required for locate');
54
+ (0, utils_namespaceObject.assert)('object' == typeof query, 'query should be an object for locate');
55
+ const hasPlanLocatedElement = !!opt?.planLocatedElement?.rect;
56
+ let searchAreaPrompt;
57
+ if (query.deepLocate && !hasPlanLocatedElement) searchAreaPrompt = query.prompt;
58
+ const { modelFamily } = modelConfig;
59
+ if (searchAreaPrompt && !modelFamily) {
60
+ console.warn('The "deepLocate" feature is not supported with multimodal LLM. Please config VL model for Midscene. https://midscenejs.com/model-config');
61
+ searchAreaPrompt = void 0;
62
+ }
63
+ if (searchAreaPrompt && (0, util_js_namespaceObject.isAutoGLM)(modelFamily)) {
64
+ console.warn('The "deepLocate" feature is not supported with AutoGLM.');
65
+ searchAreaPrompt = void 0;
66
+ }
67
+ const context = opt?.context || await this.contextRetrieverFn();
68
+ let searchArea;
69
+ let searchAreaRawResponse;
70
+ let searchAreaUsage;
71
+ let searchAreaResponse;
72
+ if (query.deepLocate && hasPlanLocatedElement) {
73
+ const searchAreaConfig = await (0, inspect_js_namespaceObject.buildSearchAreaConfig)({
74
+ context,
75
+ baseRect: opt.planLocatedElement.rect,
76
+ modelFamily
77
+ });
78
+ searchArea = searchAreaConfig.rect;
79
+ searchAreaRawResponse = JSON.stringify({
80
+ source: 'plan-located-element',
81
+ rect: opt.planLocatedElement.rect
82
+ });
83
+ searchAreaResponse = {
84
+ rect: searchArea,
85
+ imageBase64: searchAreaConfig.imageBase64,
86
+ scale: searchAreaConfig.scale,
87
+ rawResponse: searchAreaRawResponse
88
+ };
89
+ } else if (searchAreaPrompt) {
90
+ searchAreaResponse = await (0, inspect_js_namespaceObject.AiLocateSection)({
91
+ context,
92
+ sectionDescription: searchAreaPrompt,
93
+ modelConfig,
94
+ abortSignal
95
+ });
96
+ (0, utils_namespaceObject.assert)(searchAreaResponse.rect, `cannot find search area for "${searchAreaPrompt}"${searchAreaResponse.error ? `: ${searchAreaResponse.error}` : ''}`);
97
+ searchAreaRawResponse = searchAreaResponse.rawResponse;
98
+ searchAreaUsage = searchAreaResponse.usage;
99
+ searchArea = searchAreaResponse.rect;
100
+ }
101
+ const startTime = Date.now();
102
+ const { parseResult, rect, rawResponse, usage, reasoning_content } = await (0, index_js_namespaceObject.AiLocateElement)({
103
+ context,
104
+ targetElementDescription: queryPrompt,
105
+ searchConfig: searchAreaResponse,
106
+ modelConfig,
107
+ abortSignal
108
+ });
109
+ const timeCost = Date.now() - startTime;
110
+ const taskInfo = {
111
+ ...this.taskInfo ? this.taskInfo : {},
112
+ durationMs: timeCost,
113
+ rawResponse: JSON.stringify(rawResponse),
114
+ formatResponse: JSON.stringify(parseResult),
115
+ usage,
116
+ searchArea,
117
+ searchAreaRawResponse,
118
+ searchAreaUsage,
119
+ reasoning_content
120
+ };
121
+ let errorLog;
122
+ if (parseResult.errors?.length) errorLog = `failed to locate element: \n${parseResult.errors.join('\n')}`;
123
+ const dumpData = {
124
+ type: 'locate',
125
+ userQuery: {
126
+ element: queryPrompt
127
+ },
128
+ matchedElement: [],
129
+ matchedRect: rect,
130
+ data: null,
131
+ taskInfo,
132
+ deepLocate: !!searchArea,
133
+ error: errorLog
134
+ };
135
+ const elements = parseResult.elements || [];
136
+ const dump = (0, external_utils_js_namespaceObject.createServiceDump)({
137
+ ...dumpData,
138
+ matchedElement: elements
139
+ });
140
+ if (errorLog) throw new external_types_js_namespaceObject.ServiceError(errorLog, dump);
141
+ if (elements.length > 1) throw new external_types_js_namespaceObject.ServiceError(`locate: multiple elements found, length = ${elements.length}`, dump);
142
+ if (1 === elements.length) return {
143
+ element: {
144
+ center: elements[0].center,
145
+ rect: elements[0].rect,
146
+ description: elements[0].description
147
+ },
148
+ rect,
149
+ dump
150
+ };
151
+ return {
152
+ element: null,
153
+ rect,
154
+ dump
155
+ };
156
+ }
157
+ async extract(dataDemand, modelConfig, opt, pageDescription, multimodalPrompt, context) {
158
+ (0, utils_namespaceObject.assert)(context, 'context is required for extract');
159
+ (0, utils_namespaceObject.assert)('object' == typeof dataDemand || 'string' == typeof dataDemand, `dataDemand should be object or string, but get ${typeof dataDemand}`);
160
+ const startTime = Date.now();
161
+ let parseResult;
162
+ let rawResponse;
163
+ let usage;
164
+ let reasoning_content;
165
+ try {
166
+ const result = await (0, index_js_namespaceObject.AiExtractElementInfo)({
167
+ context,
168
+ dataQuery: dataDemand,
169
+ multimodalPrompt,
170
+ extractOption: opt,
171
+ modelConfig,
172
+ pageDescription
173
+ });
174
+ parseResult = result.parseResult;
175
+ rawResponse = result.rawResponse;
176
+ usage = result.usage;
177
+ reasoning_content = result.reasoning_content;
178
+ } catch (error) {
179
+ if (error instanceof index_js_namespaceObject.AIResponseParseError) {
180
+ const timeCost = Date.now() - startTime;
181
+ const taskInfo = {
182
+ ...this.taskInfo ? this.taskInfo : {},
183
+ durationMs: timeCost,
184
+ rawResponse: error.rawResponse,
185
+ usage: error.usage
186
+ };
187
+ const dump = (0, external_utils_js_namespaceObject.createServiceDump)({
188
+ type: 'extract',
189
+ userQuery: {
190
+ dataDemand
191
+ },
192
+ matchedElement: [],
193
+ data: null,
194
+ taskInfo,
195
+ error: error.message
196
+ });
197
+ throw new external_types_js_namespaceObject.ServiceError(error.message, dump);
198
+ }
199
+ throw error;
200
+ }
201
+ const timeCost = Date.now() - startTime;
202
+ const taskInfo = {
203
+ ...this.taskInfo ? this.taskInfo : {},
204
+ durationMs: timeCost,
205
+ rawResponse,
206
+ formatResponse: JSON.stringify(parseResult),
207
+ usage,
208
+ reasoning_content
209
+ };
210
+ let errorLog;
211
+ if (parseResult.errors?.length) errorLog = `AI response error: \n${parseResult.errors.join('\n')}`;
212
+ const dumpData = {
213
+ type: 'extract',
214
+ userQuery: {
215
+ dataDemand
216
+ },
217
+ matchedElement: [],
218
+ data: null,
219
+ taskInfo,
220
+ error: errorLog
221
+ };
222
+ const { data, thought } = parseResult || {};
223
+ const dump = (0, external_utils_js_namespaceObject.createServiceDump)({
224
+ ...dumpData,
225
+ data
226
+ });
227
+ if (errorLog && !data) throw new external_types_js_namespaceObject.ServiceError(errorLog, dump);
228
+ return {
229
+ data,
230
+ thought,
231
+ usage,
232
+ reasoning_content,
233
+ dump
234
+ };
235
+ }
236
+ async describe(target, modelConfig, opt) {
237
+ (0, utils_namespaceObject.assert)(target, 'target is required for service.describe');
238
+ const context = await this.contextRetrieverFn();
239
+ const { shotSize } = context;
240
+ const screenshotBase64 = context.screenshot.base64;
241
+ (0, utils_namespaceObject.assert)(screenshotBase64, 'screenshot is required for service.describe');
242
+ const { modelFamily } = modelConfig;
243
+ const systemPrompt = (0, describe_js_namespaceObject.elementDescriberInstruction)();
244
+ const defaultRectSize = 30;
245
+ const targetRect = Array.isArray(target) ? {
246
+ left: Math.floor(target[0] - defaultRectSize / 2),
247
+ top: Math.floor(target[1] - defaultRectSize / 2),
248
+ width: defaultRectSize,
249
+ height: defaultRectSize
250
+ } : target;
251
+ let imagePayload = await (0, img_namespaceObject.compositeElementInfoImg)({
252
+ inputImgBase64: screenshotBase64,
253
+ size: shotSize,
254
+ elementsPositionInfo: [
255
+ {
256
+ rect: targetRect
257
+ }
258
+ ],
259
+ borderThickness: 3
260
+ });
261
+ if (opt?.deepLocate) {
262
+ const searchArea = (0, external_common_js_namespaceObject.expandSearchArea)(targetRect, shotSize);
263
+ debug('describe: cropping to searchArea', searchArea);
264
+ const croppedResult = await (0, img_namespaceObject.cropByRect)(imagePayload, searchArea, 'qwen2.5-vl' === modelFamily);
265
+ imagePayload = croppedResult.imageBase64;
266
+ }
267
+ const msgs = [
268
+ {
269
+ role: 'system',
270
+ content: systemPrompt
271
+ },
272
+ {
273
+ role: 'user',
274
+ content: [
275
+ {
276
+ type: 'image_url',
277
+ image_url: {
278
+ url: imagePayload,
279
+ detail: 'high'
280
+ }
281
+ }
282
+ ]
283
+ }
284
+ ];
285
+ const res = await (0, index_js_namespaceObject.callAIWithObjectResponse)(msgs, modelConfig);
286
+ const { content } = res;
287
+ (0, utils_namespaceObject.assert)(!content.error, `describe failed: ${content.error}`);
288
+ (0, utils_namespaceObject.assert)(content.description, 'failed to describe the element');
289
+ return content;
290
+ }
291
+ constructor(context, opt){
292
+ _define_property(this, "contextRetrieverFn", void 0);
293
+ _define_property(this, "taskInfo", void 0);
294
+ (0, utils_namespaceObject.assert)(context, 'context is required for Service');
295
+ if ('function' == typeof context) this.contextRetrieverFn = context;
296
+ else this.contextRetrieverFn = ()=>Promise.resolve(context);
297
+ if (void 0 !== opt?.taskInfo) this.taskInfo = opt.taskInfo;
298
+ }
299
+ }
300
+ exports["default"] = __webpack_exports__["default"];
301
+ for(var __rspack_i in __webpack_exports__)if (-1 === [
302
+ "default"
303
+ ].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
304
+ Object.defineProperty(exports, '__esModule', {
305
+ value: true
306
+ });
307
+
308
+ //# sourceMappingURL=index.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"service/index.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../src/service/index.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n }\n }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { isAutoGLM, isUITars } from '@/ai-model/auto-glm/util';\nimport {\n AIResponseParseError,\n AiExtractElementInfo,\n AiLocateElement,\n callAIWithObjectResponse,\n} from '@/ai-model/index';\nimport { AiLocateSection, buildSearchAreaConfig } from '@/ai-model/inspect';\nimport { elementDescriberInstruction } from '@/ai-model/prompt/describe';\nimport { type AIArgs, expandSearchArea } from '@/common';\nimport type {\n AIDescribeElementResponse,\n AIUsageInfo,\n DetailedLocateParam,\n LocateResultElement,\n LocateResultWithDump,\n PartialServiceDumpFromSDK,\n PlanningLocateParam,\n Rect,\n ServiceExtractOption,\n ServiceExtractParam,\n ServiceExtractResult,\n ServiceTaskInfo,\n UIContext,\n} from '@/types';\nimport { ServiceError } from '@/types';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport { compositeElementInfoImg, cropByRect } from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { TMultimodalPrompt } from '../common';\nimport { createServiceDump } from './utils';\n\nexport interface LocateOpts {\n context?: UIContext;\n planLocatedElement?: LocateResultElement;\n}\n\nexport type AnyValue<T> = {\n [K in keyof T]: unknown extends T[K] ? any : T[K];\n};\n\ninterface ServiceOptions {\n taskInfo?: Omit<ServiceTaskInfo, 'durationMs'>;\n}\n\nconst debug = getDebug('ai:service');\nexport default class Service {\n contextRetrieverFn: () => Promise<UIContext> | UIContext;\n\n taskInfo?: Omit<ServiceTaskInfo, 'durationMs'>;\n\n constructor(\n context: UIContext | (() => Promise<UIContext> | UIContext),\n opt?: ServiceOptions,\n ) {\n assert(context, 'context is required for Service');\n if (typeof context === 'function') {\n this.contextRetrieverFn = context;\n } else {\n this.contextRetrieverFn = () => Promise.resolve(context);\n }\n\n if (typeof opt?.taskInfo !== 'undefined') {\n this.taskInfo = opt.taskInfo;\n }\n }\n\n async locate(\n query: PlanningLocateParam,\n opt: LocateOpts,\n modelConfig: IModelConfig,\n abortSignal?: AbortSignal,\n ): Promise<LocateResultWithDump> {\n const queryPrompt = typeof query === 'string' ? query : query.prompt;\n assert(queryPrompt, 'query is required for locate');\n\n assert(typeof query === 'object', 'query should be an object for locate');\n\n const hasPlanLocatedElement = !!opt?.planLocatedElement?.rect;\n\n let searchAreaPrompt;\n if (query.deepLocate && !hasPlanLocatedElement) {\n searchAreaPrompt = query.prompt;\n }\n\n const { modelFamily } = modelConfig;\n\n if (searchAreaPrompt && !modelFamily) {\n console.warn(\n 'The \"deepLocate\" feature is not supported with multimodal LLM. Please config VL model for Midscene. https://midscenejs.com/model-config',\n );\n searchAreaPrompt = undefined;\n }\n\n if (searchAreaPrompt && isAutoGLM(modelFamily)) {\n console.warn('The \"deepLocate\" feature is not supported with AutoGLM.');\n searchAreaPrompt = undefined;\n }\n\n const context = opt?.context || (await this.contextRetrieverFn());\n\n let searchArea: Rect | undefined = undefined;\n let searchAreaRawResponse: string | undefined = undefined;\n let searchAreaUsage: AIUsageInfo | undefined = undefined;\n let searchAreaResponse:\n | Awaited<ReturnType<typeof AiLocateSection>>\n | undefined = undefined;\n if (query.deepLocate && hasPlanLocatedElement) {\n const searchAreaConfig = await buildSearchAreaConfig({\n context,\n baseRect: opt.planLocatedElement!.rect,\n modelFamily,\n });\n searchArea = searchAreaConfig.rect;\n\n searchAreaRawResponse = JSON.stringify({\n source: 'plan-located-element',\n rect: opt.planLocatedElement!.rect,\n });\n searchAreaResponse = {\n rect: searchArea,\n imageBase64: searchAreaConfig.imageBase64,\n scale: searchAreaConfig.scale,\n rawResponse: searchAreaRawResponse,\n };\n } else if (searchAreaPrompt) {\n searchAreaResponse = await AiLocateSection({\n context,\n sectionDescription: searchAreaPrompt,\n modelConfig,\n abortSignal,\n });\n assert(\n searchAreaResponse.rect,\n `cannot find search area for \"${searchAreaPrompt}\"${\n searchAreaResponse.error ? `: ${searchAreaResponse.error}` : ''\n }`,\n );\n searchAreaRawResponse = searchAreaResponse.rawResponse;\n searchAreaUsage = searchAreaResponse.usage;\n searchArea = searchAreaResponse.rect;\n }\n\n const startTime = Date.now();\n const { parseResult, rect, rawResponse, usage, reasoning_content } =\n await AiLocateElement({\n context,\n targetElementDescription: queryPrompt,\n searchConfig: searchAreaResponse,\n modelConfig,\n abortSignal,\n });\n\n const timeCost = Date.now() - startTime;\n const taskInfo: ServiceTaskInfo = {\n ...(this.taskInfo ? this.taskInfo : {}),\n durationMs: timeCost,\n rawResponse: JSON.stringify(rawResponse),\n formatResponse: JSON.stringify(parseResult),\n usage,\n searchArea,\n searchAreaRawResponse,\n searchAreaUsage,\n reasoning_content,\n };\n\n let errorLog: string | undefined;\n if (parseResult.errors?.length) {\n errorLog = `failed to locate element: \\n${parseResult.errors.join('\\n')}`;\n }\n\n const dumpData: PartialServiceDumpFromSDK = {\n type: 'locate',\n userQuery: {\n element: queryPrompt,\n },\n matchedElement: [],\n matchedRect: rect,\n data: null,\n taskInfo,\n deepLocate: !!searchArea,\n error: errorLog,\n };\n\n const elements = parseResult.elements || [];\n\n const dump = createServiceDump({\n ...dumpData,\n matchedElement: elements,\n });\n\n if (errorLog) {\n throw new ServiceError(errorLog, dump);\n }\n\n if (elements.length > 1) {\n throw new ServiceError(\n `locate: multiple elements found, length = ${elements.length}`,\n dump,\n );\n }\n\n if (elements.length === 1) {\n return {\n element: {\n center: elements[0]!.center,\n rect: elements[0]!.rect,\n description: elements[0]!.description,\n },\n rect,\n dump,\n };\n }\n\n return {\n element: null,\n rect,\n dump,\n };\n }\n\n async extract<T>(\n dataDemand: ServiceExtractParam,\n modelConfig: IModelConfig,\n opt?: ServiceExtractOption,\n pageDescription?: string,\n multimodalPrompt?: TMultimodalPrompt,\n context?: UIContext,\n ): Promise<ServiceExtractResult<T>> {\n assert(context, 'context is required for extract');\n assert(\n typeof dataDemand === 'object' || typeof dataDemand === 'string',\n `dataDemand should be object or string, but get ${typeof dataDemand}`,\n );\n\n const startTime = Date.now();\n\n let parseResult: Awaited<\n ReturnType<typeof AiExtractElementInfo<T>>\n >['parseResult'];\n let rawResponse: string;\n let usage: Awaited<ReturnType<typeof AiExtractElementInfo<T>>>['usage'];\n let reasoning_content: string | undefined;\n\n try {\n const result = await AiExtractElementInfo<T>({\n context,\n dataQuery: dataDemand,\n multimodalPrompt,\n extractOption: opt,\n modelConfig,\n pageDescription,\n });\n parseResult = result.parseResult;\n rawResponse = result.rawResponse;\n usage = result.usage;\n reasoning_content = result.reasoning_content;\n } catch (error) {\n if (error instanceof AIResponseParseError) {\n // Create dump with usage and rawResponse from the error\n const timeCost = Date.now() - startTime;\n const taskInfo: ServiceTaskInfo = {\n ...(this.taskInfo ? this.taskInfo : {}),\n durationMs: timeCost,\n rawResponse: error.rawResponse,\n usage: error.usage,\n };\n const dump = createServiceDump({\n type: 'extract',\n userQuery: { dataDemand },\n matchedElement: [],\n data: null,\n taskInfo,\n error: error.message,\n });\n throw new ServiceError(error.message, dump);\n }\n throw error;\n }\n\n const timeCost = Date.now() - startTime;\n const taskInfo: ServiceTaskInfo = {\n ...(this.taskInfo ? this.taskInfo : {}),\n durationMs: timeCost,\n rawResponse,\n formatResponse: JSON.stringify(parseResult),\n usage,\n reasoning_content,\n };\n\n let errorLog: string | undefined;\n if (parseResult.errors?.length) {\n errorLog = `AI response error: \\n${parseResult.errors.join('\\n')}`;\n }\n\n const dumpData: PartialServiceDumpFromSDK = {\n type: 'extract',\n userQuery: {\n dataDemand,\n },\n matchedElement: [],\n data: null,\n taskInfo,\n error: errorLog,\n };\n\n const { data, thought } = parseResult || {};\n\n const dump = createServiceDump({\n ...dumpData,\n data,\n });\n\n if (errorLog && !data) {\n throw new ServiceError(errorLog, dump);\n }\n\n return {\n data,\n thought,\n usage,\n reasoning_content,\n dump,\n };\n }\n\n async describe(\n target: Rect | [number, number],\n modelConfig: IModelConfig,\n opt?: {\n deepLocate?: boolean;\n },\n ): Promise<Pick<AIDescribeElementResponse, 'description'>> {\n assert(target, 'target is required for service.describe');\n const context = await this.contextRetrieverFn();\n const { shotSize } = context;\n const screenshotBase64 = context.screenshot.base64;\n assert(screenshotBase64, 'screenshot is required for service.describe');\n // The result of the \"describe\" function will be used for positioning, so essentially it is a form of grounding.\n const { modelFamily } = modelConfig;\n const systemPrompt = elementDescriberInstruction();\n\n // Convert [x,y] center point to Rect if needed\n const defaultRectSize = 30;\n const targetRect: Rect = Array.isArray(target)\n ? {\n left: Math.floor(target[0] - defaultRectSize / 2),\n top: Math.floor(target[1] - defaultRectSize / 2),\n width: defaultRectSize,\n height: defaultRectSize,\n }\n : target;\n\n let imagePayload = await compositeElementInfoImg({\n inputImgBase64: screenshotBase64,\n size: shotSize,\n elementsPositionInfo: [\n {\n rect: targetRect,\n },\n ],\n borderThickness: 3,\n });\n\n if (opt?.deepLocate) {\n const searchArea = expandSearchArea(targetRect, shotSize);\n // Always crop in describe mode. Unlike locate's deepLocate (where\n // cropping too small loses context for finding elements), describe's\n // deepLocate intentionally zooms in so the model produces a more\n // precise description from a focused view. expandSearchArea already\n // guarantees a minimum 400x400 area with surrounding context.\n debug('describe: cropping to searchArea', searchArea);\n const croppedResult = await cropByRect(\n imagePayload,\n searchArea,\n modelFamily === 'qwen2.5-vl',\n );\n imagePayload = croppedResult.imageBase64;\n }\n\n const msgs: AIArgs = [\n { role: 'system', content: systemPrompt },\n {\n role: 'user',\n content: [\n {\n type: 'image_url',\n image_url: {\n url: imagePayload,\n detail: 'high',\n },\n },\n ],\n },\n ];\n\n const res = await callAIWithObjectResponse<AIDescribeElementResponse>(\n msgs,\n modelConfig,\n );\n\n const { content } = res;\n assert(!content.error, `describe failed: ${content.error}`);\n assert(content.description, 'failed to describe the element');\n return content;\n }\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debug","getDebug","Service","query","opt","modelConfig","abortSignal","queryPrompt","assert","hasPlanLocatedElement","searchAreaPrompt","modelFamily","console","undefined","isAutoGLM","context","searchArea","searchAreaRawResponse","searchAreaUsage","searchAreaResponse","searchAreaConfig","buildSearchAreaConfig","JSON","AiLocateSection","startTime","Date","parseResult","rect","rawResponse","usage","reasoning_content","AiLocateElement","timeCost","taskInfo","errorLog","dumpData","elements","dump","createServiceDump","ServiceError","dataDemand","pageDescription","multimodalPrompt","result","AiExtractElementInfo","error","AIResponseParseError","data","thought","target","shotSize","screenshotBase64","systemPrompt","elementDescriberInstruction","defaultRectSize","targetRect","Array","Math","imagePayload","compositeElementInfoImg","expandSearchArea","croppedResult","cropByRect","msgs","res","callAIWithObjectResponse","content","Promise"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;;;;;;;;;ACwCA,MAAMI,QAAQC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AACR,MAAMC;IAqBnB,MAAM,OACJC,KAA0B,EAC1BC,GAAe,EACfC,WAAyB,EACzBC,WAAyB,EACM;QAC/B,MAAMC,cAAc,AAAiB,YAAjB,OAAOJ,QAAqBA,QAAQA,MAAM,MAAM;QACpEK,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOD,aAAa;QAEpBC,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO,AAAiB,YAAjB,OAAOL,OAAoB;QAElC,MAAMM,wBAAwB,CAAC,CAACL,KAAK,oBAAoB;QAEzD,IAAIM;QACJ,IAAIP,MAAM,UAAU,IAAI,CAACM,uBACvBC,mBAAmBP,MAAM,MAAM;QAGjC,MAAM,EAAEQ,WAAW,EAAE,GAAGN;QAExB,IAAIK,oBAAoB,CAACC,aAAa;YACpCC,QAAQ,IAAI,CACV;YAEFF,mBAAmBG;QACrB;QAEA,IAAIH,oBAAoBI,AAAAA,IAAAA,wBAAAA,SAAAA,AAAAA,EAAUH,cAAc;YAC9CC,QAAQ,IAAI,CAAC;YACbF,mBAAmBG;QACrB;QAEA,MAAME,UAAUX,KAAK,WAAY,MAAM,IAAI,CAAC,kBAAkB;QAE9D,IAAIY;QACJ,IAAIC;QACJ,IAAIC;QACJ,IAAIC;QAGJ,IAAIhB,MAAM,UAAU,IAAIM,uBAAuB;YAC7C,MAAMW,mBAAmB,MAAMC,AAAAA,IAAAA,2BAAAA,qBAAAA,AAAAA,EAAsB;gBACnDN;gBACA,UAAUX,IAAI,kBAAkB,CAAE,IAAI;gBACtCO;YACF;YACAK,aAAaI,iBAAiB,IAAI;YAElCH,wBAAwBK,KAAK,SAAS,CAAC;gBACrC,QAAQ;gBACR,MAAMlB,IAAI,kBAAkB,CAAE,IAAI;YACpC;YACAe,qBAAqB;gBACnB,MAAMH;gBACN,aAAaI,iBAAiB,WAAW;gBACzC,OAAOA,iBAAiB,KAAK;gBAC7B,aAAaH;YACf;QACF,OAAO,IAAIP,kBAAkB;YAC3BS,qBAAqB,MAAMI,AAAAA,IAAAA,2BAAAA,eAAAA,AAAAA,EAAgB;gBACzCR;gBACA,oBAAoBL;gBACpBL;gBACAC;YACF;YACAE,IAAAA,sBAAAA,MAAAA,AAAAA,EACEW,mBAAmB,IAAI,EACvB,CAAC,6BAA6B,EAAET,iBAAiB,CAAC,EAChDS,mBAAmB,KAAK,GAAG,CAAC,EAAE,EAAEA,mBAAmB,KAAK,EAAE,GAAG,IAC7D;YAEJF,wBAAwBE,mBAAmB,WAAW;YACtDD,kBAAkBC,mBAAmB,KAAK;YAC1CH,aAAaG,mBAAmB,IAAI;QACtC;QAEA,MAAMK,YAAYC,KAAK,GAAG;QAC1B,MAAM,EAAEC,WAAW,EAAEC,IAAI,EAAEC,WAAW,EAAEC,KAAK,EAAEC,iBAAiB,EAAE,GAChE,MAAMC,AAAAA,IAAAA,yBAAAA,eAAAA,AAAAA,EAAgB;YACpBhB;YACA,0BAA0BR;YAC1B,cAAcY;YACdd;YACAC;QACF;QAEF,MAAM0B,WAAWP,KAAK,GAAG,KAAKD;QAC9B,MAAMS,WAA4B;YAChC,GAAI,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;YACtC,YAAYD;YACZ,aAAaV,KAAK,SAAS,CAACM;YAC5B,gBAAgBN,KAAK,SAAS,CAACI;YAC/BG;YACAb;YACAC;YACAC;YACAY;QACF;QAEA,IAAII;QACJ,IAAIR,YAAY,MAAM,EAAE,QACtBQ,WAAW,CAAC,4BAA4B,EAAER,YAAY,MAAM,CAAC,IAAI,CAAC,OAAO;QAG3E,MAAMS,WAAsC;YAC1C,MAAM;YACN,WAAW;gBACT,SAAS5B;YACX;YACA,gBAAgB,EAAE;YAClB,aAAaoB;YACb,MAAM;YACNM;YACA,YAAY,CAAC,CAACjB;YACd,OAAOkB;QACT;QAEA,MAAME,WAAWV,YAAY,QAAQ,IAAI,EAAE;QAE3C,MAAMW,OAAOC,AAAAA,IAAAA,kCAAAA,iBAAAA,AAAAA,EAAkB;YAC7B,GAAGH,QAAQ;YACX,gBAAgBC;QAClB;QAEA,IAAIF,UACF,MAAM,IAAIK,kCAAAA,YAAYA,CAACL,UAAUG;QAGnC,IAAID,SAAS,MAAM,GAAG,GACpB,MAAM,IAAIG,kCAAAA,YAAYA,CACpB,CAAC,0CAA0C,EAAEH,SAAS,MAAM,EAAE,EAC9DC;QAIJ,IAAID,AAAoB,MAApBA,SAAS,MAAM,EACjB,OAAO;YACL,SAAS;gBACP,QAAQA,QAAQ,CAAC,EAAE,CAAE,MAAM;gBAC3B,MAAMA,QAAQ,CAAC,EAAE,CAAE,IAAI;gBACvB,aAAaA,QAAQ,CAAC,EAAE,CAAE,WAAW;YACvC;YACAT;YACAU;QACF;QAGF,OAAO;YACL,SAAS;YACTV;YACAU;QACF;IACF;IAEA,MAAM,QACJG,UAA+B,EAC/BnC,WAAyB,EACzBD,GAA0B,EAC1BqC,eAAwB,EACxBC,gBAAoC,EACpC3B,OAAmB,EACe;QAClCP,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOO,SAAS;QAChBP,IAAAA,sBAAAA,MAAAA,AAAAA,EACE,AAAsB,YAAtB,OAAOgC,cAA2B,AAAsB,YAAtB,OAAOA,YACzC,CAAC,+CAA+C,EAAE,OAAOA,YAAY;QAGvE,MAAMhB,YAAYC,KAAK,GAAG;QAE1B,IAAIC;QAGJ,IAAIE;QACJ,IAAIC;QACJ,IAAIC;QAEJ,IAAI;YACF,MAAMa,SAAS,MAAMC,AAAAA,IAAAA,yBAAAA,oBAAAA,AAAAA,EAAwB;gBAC3C7B;gBACA,WAAWyB;gBACXE;gBACA,eAAetC;gBACfC;gBACAoC;YACF;YACAf,cAAciB,OAAO,WAAW;YAChCf,cAAce,OAAO,WAAW;YAChCd,QAAQc,OAAO,KAAK;YACpBb,oBAAoBa,OAAO,iBAAiB;QAC9C,EAAE,OAAOE,OAAO;YACd,IAAIA,iBAAiBC,yBAAAA,oBAAoBA,EAAE;gBAEzC,MAAMd,WAAWP,KAAK,GAAG,KAAKD;gBAC9B,MAAMS,WAA4B;oBAChC,GAAI,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;oBACtC,YAAYD;oBACZ,aAAaa,MAAM,WAAW;oBAC9B,OAAOA,MAAM,KAAK;gBACpB;gBACA,MAAMR,OAAOC,AAAAA,IAAAA,kCAAAA,iBAAAA,AAAAA,EAAkB;oBAC7B,MAAM;oBACN,WAAW;wBAAEE;oBAAW;oBACxB,gBAAgB,EAAE;oBAClB,MAAM;oBACNP;oBACA,OAAOY,MAAM,OAAO;gBACtB;gBACA,MAAM,IAAIN,kCAAAA,YAAYA,CAACM,MAAM,OAAO,EAAER;YACxC;YACA,MAAMQ;QACR;QAEA,MAAMb,WAAWP,KAAK,GAAG,KAAKD;QAC9B,MAAMS,WAA4B;YAChC,GAAI,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;YACtC,YAAYD;YACZJ;YACA,gBAAgBN,KAAK,SAAS,CAACI;YAC/BG;YACAC;QACF;QAEA,IAAII;QACJ,IAAIR,YAAY,MAAM,EAAE,QACtBQ,WAAW,CAAC,qBAAqB,EAAER,YAAY,MAAM,CAAC,IAAI,CAAC,OAAO;QAGpE,MAAMS,WAAsC;YAC1C,MAAM;YACN,WAAW;gBACTK;YACF;YACA,gBAAgB,EAAE;YAClB,MAAM;YACNP;YACA,OAAOC;QACT;QAEA,MAAM,EAAEa,IAAI,EAAEC,OAAO,EAAE,GAAGtB,eAAe,CAAC;QAE1C,MAAMW,OAAOC,AAAAA,IAAAA,kCAAAA,iBAAAA,AAAAA,EAAkB;YAC7B,GAAGH,QAAQ;YACXY;QACF;QAEA,IAAIb,YAAY,CAACa,MACf,MAAM,IAAIR,kCAAAA,YAAYA,CAACL,UAAUG;QAGnC,OAAO;YACLU;YACAC;YACAnB;YACAC;YACAO;QACF;IACF;IAEA,MAAM,SACJY,MAA+B,EAC/B5C,WAAyB,EACzBD,GAEC,EACwD;QACzDI,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOyC,QAAQ;QACf,MAAMlC,UAAU,MAAM,IAAI,CAAC,kBAAkB;QAC7C,MAAM,EAAEmC,QAAQ,EAAE,GAAGnC;QACrB,MAAMoC,mBAAmBpC,QAAQ,UAAU,CAAC,MAAM;QAClDP,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO2C,kBAAkB;QAEzB,MAAM,EAAExC,WAAW,EAAE,GAAGN;QACxB,MAAM+C,eAAeC,AAAAA,IAAAA,4BAAAA,2BAAAA,AAAAA;QAGrB,MAAMC,kBAAkB;QACxB,MAAMC,aAAmBC,MAAM,OAAO,CAACP,UACnC;YACE,MAAMQ,KAAK,KAAK,CAACR,MAAM,CAAC,EAAE,GAAGK,kBAAkB;YAC/C,KAAKG,KAAK,KAAK,CAACR,MAAM,CAAC,EAAE,GAAGK,kBAAkB;YAC9C,OAAOA;YACP,QAAQA;QACV,IACAL;QAEJ,IAAIS,eAAe,MAAMC,AAAAA,IAAAA,oBAAAA,uBAAAA,AAAAA,EAAwB;YAC/C,gBAAgBR;YAChB,MAAMD;YACN,sBAAsB;gBACpB;oBACE,MAAMK;gBACR;aACD;YACD,iBAAiB;QACnB;QAEA,IAAInD,KAAK,YAAY;YACnB,MAAMY,aAAa4C,AAAAA,IAAAA,mCAAAA,gBAAAA,AAAAA,EAAiBL,YAAYL;YAMhDlD,MAAM,oCAAoCgB;YAC1C,MAAM6C,gBAAgB,MAAMC,AAAAA,IAAAA,oBAAAA,UAAAA,AAAAA,EAC1BJ,cACA1C,YACAL,AAAgB,iBAAhBA;YAEF+C,eAAeG,cAAc,WAAW;QAC1C;QAEA,MAAME,OAAe;YACnB;gBAAE,MAAM;gBAAU,SAASX;YAAa;YACxC;gBACE,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,WAAW;4BACT,KAAKM;4BACL,QAAQ;wBACV;oBACF;iBACD;YACH;SACD;QAED,MAAMM,MAAM,MAAMC,AAAAA,IAAAA,yBAAAA,wBAAAA,AAAAA,EAChBF,MACA1D;QAGF,MAAM,EAAE6D,OAAO,EAAE,GAAGF;QACpBxD,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO,CAAC0D,QAAQ,KAAK,EAAE,CAAC,iBAAiB,EAAEA,QAAQ,KAAK,EAAE;QAC1D1D,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO0D,QAAQ,WAAW,EAAE;QAC5B,OAAOA;IACT;IAlWA,YACEnD,OAA2D,EAC3DX,GAAoB,CACpB;QAPF;QAEA;QAMEI,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOO,SAAS;QAChB,IAAI,AAAmB,cAAnB,OAAOA,SACT,IAAI,CAAC,kBAAkB,GAAGA;aAE1B,IAAI,CAAC,kBAAkB,GAAG,IAAMoD,QAAQ,OAAO,CAACpD;QAGlD,IAAI,AAAyB,WAAlBX,KAAK,UACd,IAAI,CAAC,QAAQ,GAAGA,IAAI,QAAQ;IAEhC;AAqVF"}
@@ -0,0 +1,49 @@
1
+ "use strict";
2
+ var __webpack_require__ = {};
3
+ (()=>{
4
+ __webpack_require__.d = (exports1, definition)=>{
5
+ for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
6
+ enumerable: true,
7
+ get: definition[key]
8
+ });
9
+ };
10
+ })();
11
+ (()=>{
12
+ __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
13
+ })();
14
+ (()=>{
15
+ __webpack_require__.r = (exports1)=>{
16
+ if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
17
+ value: 'Module'
18
+ });
19
+ Object.defineProperty(exports1, '__esModule', {
20
+ value: true
21
+ });
22
+ };
23
+ })();
24
+ var __webpack_exports__ = {};
25
+ __webpack_require__.r(__webpack_exports__);
26
+ __webpack_require__.d(__webpack_exports__, {
27
+ createServiceDump: ()=>createServiceDump
28
+ });
29
+ const utils_namespaceObject = require("@midscene/shared/utils");
30
+ function createServiceDump(data) {
31
+ const baseData = {
32
+ logTime: Date.now()
33
+ };
34
+ const finalData = {
35
+ logId: (0, utils_namespaceObject.uuid)(),
36
+ ...baseData,
37
+ ...data
38
+ };
39
+ return finalData;
40
+ }
41
+ exports.createServiceDump = __webpack_exports__.createServiceDump;
42
+ for(var __rspack_i in __webpack_exports__)if (-1 === [
43
+ "createServiceDump"
44
+ ].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
45
+ Object.defineProperty(exports, '__esModule', {
46
+ value: true
47
+ });
48
+
49
+ //# sourceMappingURL=utils.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"service/utils.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../src/service/utils.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n }\n }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { DumpMeta, PartialServiceDumpFromSDK, ServiceDump } from '@/types';\nimport { uuid } from '@midscene/shared/utils';\n\nexport function createServiceDump(\n data: PartialServiceDumpFromSDK,\n): ServiceDump {\n const baseData: DumpMeta = {\n logTime: Date.now(),\n };\n const finalData: ServiceDump = {\n logId: uuid(),\n ...baseData,\n ...data,\n };\n\n return finalData;\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","createServiceDump","data","baseData","Date","finalData","uuid"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;ACHO,SAASI,kBACdC,IAA+B;IAE/B,MAAMC,WAAqB;QACzB,SAASC,KAAK,GAAG;IACnB;IACA,MAAMC,YAAyB;QAC7B,OAAOC,AAAAA,IAAAA,sBAAAA,IAAAA,AAAAA;QACP,GAAGH,QAAQ;QACX,GAAGD,IAAI;IACT;IAEA,OAAOG;AACT"}
@@ -0,0 +1,72 @@
1
+ "use strict";
2
+ var __webpack_require__ = {};
3
+ (()=>{
4
+ __webpack_require__.d = (exports1, definition)=>{
5
+ for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
6
+ enumerable: true,
7
+ get: definition[key]
8
+ });
9
+ };
10
+ })();
11
+ (()=>{
12
+ __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
13
+ })();
14
+ (()=>{
15
+ __webpack_require__.r = (exports1)=>{
16
+ if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
17
+ value: 'Module'
18
+ });
19
+ Object.defineProperty(exports1, '__esModule', {
20
+ value: true
21
+ });
22
+ };
23
+ })();
24
+ var __webpack_exports__ = {};
25
+ __webpack_require__.r(__webpack_exports__);
26
+ __webpack_require__.d(__webpack_exports__, {
27
+ runSkillCLI: ()=>runSkillCLI
28
+ });
29
+ const cli_namespaceObject = require("@midscene/shared/cli");
30
+ const mcp_namespaceObject = require("@midscene/shared/mcp");
31
+ const agent_js_namespaceObject = require("../agent/agent.js");
32
+ function _define_property(obj, key, value) {
33
+ if (key in obj) Object.defineProperty(obj, key, {
34
+ value: value,
35
+ enumerable: true,
36
+ configurable: true,
37
+ writable: true
38
+ });
39
+ else obj[key] = value;
40
+ return obj;
41
+ }
42
+ class SkillMidsceneTools extends mcp_namespaceObject.BaseMidsceneTools {
43
+ createTemporaryDevice() {
44
+ return new this.DeviceClass();
45
+ }
46
+ async ensureAgent() {
47
+ if (!this.agent) {
48
+ const device = new this.DeviceClass();
49
+ this.agent = new agent_js_namespaceObject.Agent(device);
50
+ }
51
+ return this.agent;
52
+ }
53
+ constructor(DeviceClass){
54
+ super(), _define_property(this, "DeviceClass", void 0), this.DeviceClass = DeviceClass;
55
+ }
56
+ }
57
+ function runSkillCLI(options) {
58
+ const tools = new SkillMidsceneTools(options.DeviceClass);
59
+ return (0, cli_namespaceObject.runToolsCLI)(tools, options.scriptName).catch((e)=>{
60
+ if (!(e instanceof cli_namespaceObject.CLIError)) console.error(e);
61
+ process.exit(e instanceof cli_namespaceObject.CLIError ? e.exitCode : 1);
62
+ });
63
+ }
64
+ exports.runSkillCLI = __webpack_exports__.runSkillCLI;
65
+ for(var __rspack_i in __webpack_exports__)if (-1 === [
66
+ "runSkillCLI"
67
+ ].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
68
+ Object.defineProperty(exports, '__esModule', {
69
+ value: true
70
+ });
71
+
72
+ //# sourceMappingURL=index.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"skill/index.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../src/skill/index.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n }\n }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { CLIError, runToolsCLI } from '@midscene/shared/cli';\nimport { BaseMidsceneTools } from '@midscene/shared/mcp';\nimport type { BaseAgent, BaseDevice } from '@midscene/shared/mcp';\nimport { Agent } from '../agent/agent';\nimport type { AbstractInterface } from '../device';\n\ntype DeviceClass = new (...args: any[]) => AbstractInterface;\n\n/**\n * Skill tools manager that lazily creates Agent from a Device class.\n * Used by runSkillCLI for CLI / Agent Skills scenarios where no agent exists at startup.\n */\nclass SkillMidsceneTools extends BaseMidsceneTools<BaseAgent> {\n constructor(private DeviceClass: DeviceClass) {\n super();\n }\n\n protected createTemporaryDevice(): BaseDevice {\n return new this.DeviceClass() as unknown as BaseDevice;\n }\n\n protected async ensureAgent(): Promise<BaseAgent> {\n if (!this.agent) {\n const device = new this.DeviceClass();\n this.agent = new Agent(device) as unknown as BaseAgent;\n }\n return this.agent;\n }\n}\n\nexport interface SkillCLIOptions {\n scriptName: string;\n DeviceClass: DeviceClass;\n}\n\n/**\n * Launch a Skill CLI for a custom interface Device class.\n * This enables AI coding assistants (Claude Code, Cline, etc.) to control\n * your custom interface through CLI commands.\n *\n * @example\n * ```typescript\n * #!/usr/bin/env node\n * import { runSkillCLI } from '@midscene/core/skill';\n * import { SampleDevice } from './sample-device';\n *\n * runSkillCLI({\n * DeviceClass: SampleDevice,\n * scriptName: 'my-device',\n * });\n * ```\n */\nexport function runSkillCLI(options: SkillCLIOptions): Promise<void> {\n const tools = new SkillMidsceneTools(options.DeviceClass);\n return runToolsCLI(tools, options.scriptName).catch((e) => {\n if (!(e instanceof CLIError)) console.error(e);\n process.exit(e instanceof CLIError ? e.exitCode : 1);\n });\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","SkillMidsceneTools","BaseMidsceneTools","device","Agent","DeviceClass","runSkillCLI","options","tools","runToolsCLI","e","CLIError","console","process"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;;ACMA,MAAMI,2BAA2BC,oBAAAA,iBAAiBA;IAKtC,wBAAoC;QAC5C,OAAO,IAAI,IAAI,CAAC,WAAW;IAC7B;IAEA,MAAgB,cAAkC;QAChD,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE;YACf,MAAMC,SAAS,IAAI,IAAI,CAAC,WAAW;YACnC,IAAI,CAAC,KAAK,GAAG,IAAIC,yBAAAA,KAAKA,CAACD;QACzB;QACA,OAAO,IAAI,CAAC,KAAK;IACnB;IAdA,YAAoBE,WAAwB,CAAE;QAC5C,KAAK,wDADaA,WAAW,GAAXA;IAEpB;AAaF;AAwBO,SAASC,YAAYC,OAAwB;IAClD,MAAMC,QAAQ,IAAIP,mBAAmBM,QAAQ,WAAW;IACxD,OAAOE,AAAAA,IAAAA,oBAAAA,WAAAA,AAAAA,EAAYD,OAAOD,QAAQ,UAAU,EAAE,KAAK,CAAC,CAACG;QACnD,IAAI,CAAEA,CAAAA,aAAaC,oBAAAA,QAAO,AAAPA,GAAWC,QAAQ,KAAK,CAACF;QAC5CG,QAAQ,IAAI,CAACH,aAAaC,oBAAAA,QAAQA,GAAGD,EAAE,QAAQ,GAAG;IACpD;AACF"}