@midscene/core 1.13.1 → 1.13.2-beta-20260923031905.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (255) hide show
  1. package/dist/es/agent/task-builder.mjs +1 -1
  2. package/dist/es/agent/task-builder.mjs.map +1 -1
  3. package/dist/es/agent/tasks.mjs +4 -4
  4. package/dist/es/agent/tasks.mjs.map +1 -1
  5. package/dist/es/agent/utils.mjs +1 -1
  6. package/dist/es/ai-model/connectivity/run-connectivity-test.mjs +2 -5
  7. package/dist/es/ai-model/connectivity/run-connectivity-test.mjs.map +1 -1
  8. package/dist/es/ai-model/model-adapter/chat-completion.mjs +8 -11
  9. package/dist/es/ai-model/model-adapter/chat-completion.mjs.map +1 -1
  10. package/dist/es/ai-model/model-adapter/resolve.mjs +10 -0
  11. package/dist/es/ai-model/model-adapter/resolve.mjs.map +1 -1
  12. package/dist/es/ai-model/model-adapter/responses.mjs +38 -0
  13. package/dist/es/ai-model/model-adapter/responses.mjs.map +1 -0
  14. package/dist/es/ai-model/model-adapter/temperature-with-semantic-retry.mjs +7 -0
  15. package/dist/es/ai-model/model-adapter/temperature-with-semantic-retry.mjs.map +1 -0
  16. package/dist/es/ai-model/models/gpt.mjs +73 -9
  17. package/dist/es/ai-model/models/gpt.mjs.map +1 -1
  18. package/dist/es/ai-model/models/registry.mjs +2 -2
  19. package/dist/es/ai-model/models/registry.mjs.map +1 -1
  20. package/dist/es/ai-model/service-caller/call-ai.mjs +22 -13
  21. package/dist/es/ai-model/service-caller/call-ai.mjs.map +1 -1
  22. package/dist/es/ai-model/service-caller/codex/call-codex.mjs +10 -8
  23. package/dist/es/ai-model/service-caller/codex/call-codex.mjs.map +1 -1
  24. package/dist/es/ai-model/service-caller/codex/codex-app-server.mjs +29 -28
  25. package/dist/es/ai-model/service-caller/codex/codex-app-server.mjs.map +1 -1
  26. package/dist/es/ai-model/service-caller/index.mjs +1 -2
  27. package/dist/es/ai-model/service-caller/model-call.mjs +4 -4
  28. package/dist/es/ai-model/service-caller/model-call.mjs.map +1 -1
  29. package/dist/es/ai-model/service-caller/openai/call-openai.mjs +13 -8
  30. package/dist/es/ai-model/service-caller/openai/call-openai.mjs.map +1 -1
  31. package/dist/es/ai-model/service-caller/openai/chat-completion/chat-completion.mjs +11 -7
  32. package/dist/es/ai-model/service-caller/openai/chat-completion/chat-completion.mjs.map +1 -1
  33. package/dist/es/ai-model/service-caller/openai/chat-completion/non-stream.mjs +9 -9
  34. package/dist/es/ai-model/service-caller/openai/chat-completion/non-stream.mjs.map +1 -1
  35. package/dist/es/ai-model/service-caller/openai/chat-completion/stream.mjs +4 -8
  36. package/dist/es/ai-model/service-caller/openai/chat-completion/stream.mjs.map +1 -1
  37. package/dist/es/ai-model/service-caller/openai/chat-completion/utils.mjs +28 -1
  38. package/dist/es/ai-model/service-caller/openai/chat-completion/utils.mjs.map +1 -1
  39. package/dist/es/ai-model/service-caller/openai/openai-client.mjs +3 -3
  40. package/dist/es/ai-model/service-caller/openai/openai-client.mjs.map +1 -1
  41. package/dist/es/ai-model/service-caller/openai/responses/non-stream.mjs +21 -0
  42. package/dist/es/ai-model/service-caller/openai/responses/non-stream.mjs.map +1 -0
  43. package/dist/es/ai-model/service-caller/openai/responses/responses.mjs +57 -0
  44. package/dist/es/ai-model/service-caller/openai/responses/responses.mjs.map +1 -0
  45. package/dist/es/ai-model/service-caller/openai/responses/stream.mjs +56 -0
  46. package/dist/es/ai-model/service-caller/openai/responses/stream.mjs.map +1 -0
  47. package/dist/es/ai-model/service-caller/openai/responses/types.mjs +0 -0
  48. package/dist/es/ai-model/service-caller/openai/responses/utils.mjs +72 -0
  49. package/dist/es/ai-model/service-caller/openai/responses/utils.mjs.map +1 -0
  50. package/dist/es/ai-model/service-caller/semantic-retry.mjs.map +1 -1
  51. package/dist/es/ai-model/service-caller/utils.mjs +6 -25
  52. package/dist/es/ai-model/service-caller/utils.mjs.map +1 -1
  53. package/dist/es/ai-model/shared/multimodal-prompt.mjs +5 -8
  54. package/dist/es/ai-model/shared/multimodal-prompt.mjs.map +1 -1
  55. package/dist/es/ai-model/workflows/grounding/locate.mjs +6 -6
  56. package/dist/es/ai-model/workflows/grounding/locate.mjs.map +1 -1
  57. package/dist/es/ai-model/workflows/grounding/planning-action-locate.mjs +5 -5
  58. package/dist/es/ai-model/workflows/grounding/planning-action-locate.mjs.map +1 -1
  59. package/dist/es/ai-model/workflows/grounding/section-locate.mjs +5 -5
  60. package/dist/es/ai-model/workflows/grounding/section-locate.mjs.map +1 -1
  61. package/dist/es/ai-model/workflows/grounding/utils.mjs +4 -7
  62. package/dist/es/ai-model/workflows/grounding/utils.mjs.map +1 -1
  63. package/dist/es/ai-model/workflows/insight/extraction.mjs +9 -15
  64. package/dist/es/ai-model/workflows/insight/extraction.mjs.map +1 -1
  65. package/dist/es/ai-model/workflows/insight/order-sensitive.mjs.map +1 -1
  66. package/dist/es/ai-model/workflows/planning/conversation-history.mjs +17 -9
  67. package/dist/es/ai-model/workflows/planning/conversation-history.mjs.map +1 -1
  68. package/dist/es/ai-model/workflows/planning/custom-planning.mjs +11 -12
  69. package/dist/es/ai-model/workflows/planning/custom-planning.mjs.map +1 -1
  70. package/dist/es/ai-model/workflows/planning/standard-planning.mjs +24 -26
  71. package/dist/es/ai-model/workflows/planning/standard-planning.mjs.map +1 -1
  72. package/dist/es/ai-model/workflows/recorder-generation/common.mjs +2 -4
  73. package/dist/es/ai-model/workflows/recorder-generation/common.mjs.map +1 -1
  74. package/dist/es/ai-model/workflows/recorder-generation/markdown.mjs +3 -5
  75. package/dist/es/ai-model/workflows/recorder-generation/markdown.mjs.map +1 -1
  76. package/dist/es/ai-model/workflows/recorder-generation/metadata.mjs +2 -4
  77. package/dist/es/ai-model/workflows/recorder-generation/metadata.mjs.map +1 -1
  78. package/dist/es/ai-model/workflows/recorder-generation/playwright.mjs.map +1 -1
  79. package/dist/es/ai-model/workflows/recorder-generation/yaml.mjs +2 -4
  80. package/dist/es/ai-model/workflows/recorder-generation/yaml.mjs.map +1 -1
  81. package/dist/es/report-html-template.mjs +1 -1
  82. package/dist/es/service/index.mjs +17 -26
  83. package/dist/es/service/index.mjs.map +1 -1
  84. package/dist/es/types.mjs.map +1 -1
  85. package/dist/es/utils.mjs +1 -1
  86. package/dist/lib/agent/task-builder.js +1 -1
  87. package/dist/lib/agent/task-builder.js.map +1 -1
  88. package/dist/lib/agent/tasks.js +4 -4
  89. package/dist/lib/agent/tasks.js.map +1 -1
  90. package/dist/lib/agent/utils.js +1 -1
  91. package/dist/lib/ai-model/connectivity/run-connectivity-test.js +2 -5
  92. package/dist/lib/ai-model/connectivity/run-connectivity-test.js.map +1 -1
  93. package/dist/lib/ai-model/model-adapter/chat-completion.js +8 -11
  94. package/dist/lib/ai-model/model-adapter/chat-completion.js.map +1 -1
  95. package/dist/lib/ai-model/model-adapter/resolve.js +10 -0
  96. package/dist/lib/ai-model/model-adapter/resolve.js.map +1 -1
  97. package/dist/lib/ai-model/model-adapter/responses.js +75 -0
  98. package/dist/lib/ai-model/model-adapter/responses.js.map +1 -0
  99. package/dist/lib/ai-model/model-adapter/temperature-with-semantic-retry.js +41 -0
  100. package/dist/lib/ai-model/model-adapter/temperature-with-semantic-retry.js.map +1 -0
  101. package/dist/lib/ai-model/models/gpt.js +73 -9
  102. package/dist/lib/ai-model/models/gpt.js.map +1 -1
  103. package/dist/lib/ai-model/models/registry.js +2 -2
  104. package/dist/lib/ai-model/models/registry.js.map +1 -1
  105. package/dist/lib/ai-model/service-caller/call-ai.js +22 -13
  106. package/dist/lib/ai-model/service-caller/call-ai.js.map +1 -1
  107. package/dist/lib/ai-model/service-caller/codex/call-codex.js +10 -8
  108. package/dist/lib/ai-model/service-caller/codex/call-codex.js.map +1 -1
  109. package/dist/lib/ai-model/service-caller/codex/codex-app-server.js +29 -28
  110. package/dist/lib/ai-model/service-caller/codex/codex-app-server.js.map +1 -1
  111. package/dist/lib/ai-model/service-caller/index.js +0 -4
  112. package/dist/lib/ai-model/service-caller/model-call.js +4 -4
  113. package/dist/lib/ai-model/service-caller/model-call.js.map +1 -1
  114. package/dist/lib/ai-model/service-caller/openai/call-openai.js +12 -7
  115. package/dist/lib/ai-model/service-caller/openai/call-openai.js.map +1 -1
  116. package/dist/lib/ai-model/service-caller/openai/chat-completion/chat-completion.js +10 -6
  117. package/dist/lib/ai-model/service-caller/openai/chat-completion/chat-completion.js.map +1 -1
  118. package/dist/lib/ai-model/service-caller/openai/chat-completion/non-stream.js +9 -9
  119. package/dist/lib/ai-model/service-caller/openai/chat-completion/non-stream.js.map +1 -1
  120. package/dist/lib/ai-model/service-caller/openai/chat-completion/stream.js +4 -8
  121. package/dist/lib/ai-model/service-caller/openai/chat-completion/stream.js.map +1 -1
  122. package/dist/lib/ai-model/service-caller/openai/chat-completion/utils.js +32 -2
  123. package/dist/lib/ai-model/service-caller/openai/chat-completion/utils.js.map +1 -1
  124. package/dist/lib/ai-model/service-caller/openai/openai-client.js +5 -5
  125. package/dist/lib/ai-model/service-caller/openai/openai-client.js.map +1 -1
  126. package/dist/lib/ai-model/service-caller/openai/responses/non-stream.js +55 -0
  127. package/dist/lib/ai-model/service-caller/openai/responses/non-stream.js.map +1 -0
  128. package/dist/lib/ai-model/service-caller/openai/responses/responses.js +94 -0
  129. package/dist/lib/ai-model/service-caller/openai/responses/responses.js.map +1 -0
  130. package/dist/lib/ai-model/service-caller/openai/responses/stream.js +90 -0
  131. package/dist/lib/ai-model/service-caller/openai/responses/stream.js.map +1 -0
  132. package/dist/lib/ai-model/service-caller/openai/responses/types.js +20 -0
  133. package/dist/lib/ai-model/service-caller/openai/responses/types.js.map +1 -0
  134. package/dist/lib/ai-model/service-caller/openai/responses/utils.js +112 -0
  135. package/dist/lib/ai-model/service-caller/openai/responses/utils.js.map +1 -0
  136. package/dist/lib/ai-model/service-caller/semantic-retry.js.map +1 -1
  137. package/dist/lib/ai-model/service-caller/utils.js +5 -27
  138. package/dist/lib/ai-model/service-caller/utils.js.map +1 -1
  139. package/dist/lib/ai-model/shared/multimodal-prompt.js +11 -14
  140. package/dist/lib/ai-model/shared/multimodal-prompt.js.map +1 -1
  141. package/dist/lib/ai-model/workflows/grounding/locate.js +6 -6
  142. package/dist/lib/ai-model/workflows/grounding/locate.js.map +1 -1
  143. package/dist/lib/ai-model/workflows/grounding/planning-action-locate.js +5 -5
  144. package/dist/lib/ai-model/workflows/grounding/planning-action-locate.js.map +1 -1
  145. package/dist/lib/ai-model/workflows/grounding/section-locate.js +5 -5
  146. package/dist/lib/ai-model/workflows/grounding/section-locate.js.map +1 -1
  147. package/dist/lib/ai-model/workflows/grounding/utils.js +3 -6
  148. package/dist/lib/ai-model/workflows/grounding/utils.js.map +1 -1
  149. package/dist/lib/ai-model/workflows/insight/extraction.js +8 -14
  150. package/dist/lib/ai-model/workflows/insight/extraction.js.map +1 -1
  151. package/dist/lib/ai-model/workflows/insight/order-sensitive.js.map +1 -1
  152. package/dist/lib/ai-model/workflows/planning/conversation-history.js +17 -9
  153. package/dist/lib/ai-model/workflows/planning/conversation-history.js.map +1 -1
  154. package/dist/lib/ai-model/workflows/planning/custom-planning.js +10 -11
  155. package/dist/lib/ai-model/workflows/planning/custom-planning.js.map +1 -1
  156. package/dist/lib/ai-model/workflows/planning/standard-planning.js +23 -25
  157. package/dist/lib/ai-model/workflows/planning/standard-planning.js.map +1 -1
  158. package/dist/lib/ai-model/workflows/recorder-generation/common.js +2 -4
  159. package/dist/lib/ai-model/workflows/recorder-generation/common.js.map +1 -1
  160. package/dist/lib/ai-model/workflows/recorder-generation/markdown.js +3 -5
  161. package/dist/lib/ai-model/workflows/recorder-generation/markdown.js.map +1 -1
  162. package/dist/lib/ai-model/workflows/recorder-generation/metadata.js +2 -4
  163. package/dist/lib/ai-model/workflows/recorder-generation/metadata.js.map +1 -1
  164. package/dist/lib/ai-model/workflows/recorder-generation/playwright.js.map +1 -1
  165. package/dist/lib/ai-model/workflows/recorder-generation/yaml.js +2 -4
  166. package/dist/lib/ai-model/workflows/recorder-generation/yaml.js.map +1 -1
  167. package/dist/lib/report-html-template.js +1 -1
  168. package/dist/lib/service/index.js +17 -26
  169. package/dist/lib/service/index.js.map +1 -1
  170. package/dist/lib/types.js.map +1 -1
  171. package/dist/lib/utils.js +1 -1
  172. package/dist/types/ai-model/connectivity/run-connectivity-test.d.ts.map +1 -1
  173. package/dist/types/ai-model/index.d.ts +1 -1
  174. package/dist/types/ai-model/index.d.ts.map +1 -1
  175. package/dist/types/ai-model/model-adapter/chat-completion.d.ts.map +1 -1
  176. package/dist/types/ai-model/model-adapter/resolve.d.ts +4 -1
  177. package/dist/types/ai-model/model-adapter/resolve.d.ts.map +1 -1
  178. package/dist/types/ai-model/model-adapter/responses.d.ts +4 -0
  179. package/dist/types/ai-model/model-adapter/responses.d.ts.map +1 -0
  180. package/dist/types/ai-model/model-adapter/temperature-with-semantic-retry.d.ts +6 -0
  181. package/dist/types/ai-model/model-adapter/temperature-with-semantic-retry.d.ts.map +1 -0
  182. package/dist/types/ai-model/model-adapter/types.d.ts +42 -13
  183. package/dist/types/ai-model/model-adapter/types.d.ts.map +1 -1
  184. package/dist/types/ai-model/models/gpt.d.ts +15 -3
  185. package/dist/types/ai-model/models/gpt.d.ts.map +1 -1
  186. package/dist/types/ai-model/models/registry.d.ts +14 -2
  187. package/dist/types/ai-model/models/registry.d.ts.map +1 -1
  188. package/dist/types/ai-model/service-caller/call-ai.d.ts +2 -2
  189. package/dist/types/ai-model/service-caller/call-ai.d.ts.map +1 -1
  190. package/dist/types/ai-model/service-caller/codex/call-codex.d.ts +4 -2
  191. package/dist/types/ai-model/service-caller/codex/call-codex.d.ts.map +1 -1
  192. package/dist/types/ai-model/service-caller/codex/codex-app-server.d.ts +5 -4
  193. package/dist/types/ai-model/service-caller/codex/codex-app-server.d.ts.map +1 -1
  194. package/dist/types/ai-model/service-caller/index.d.ts +0 -1
  195. package/dist/types/ai-model/service-caller/index.d.ts.map +1 -1
  196. package/dist/types/ai-model/service-caller/model-call.d.ts +6 -6
  197. package/dist/types/ai-model/service-caller/model-call.d.ts.map +1 -1
  198. package/dist/types/ai-model/service-caller/openai/call-openai.d.ts +9 -1
  199. package/dist/types/ai-model/service-caller/openai/call-openai.d.ts.map +1 -1
  200. package/dist/types/ai-model/service-caller/openai/chat-completion/chat-completion.d.ts +3 -4
  201. package/dist/types/ai-model/service-caller/openai/chat-completion/chat-completion.d.ts.map +1 -1
  202. package/dist/types/ai-model/service-caller/openai/chat-completion/non-stream.d.ts +1 -1
  203. package/dist/types/ai-model/service-caller/openai/chat-completion/non-stream.d.ts.map +1 -1
  204. package/dist/types/ai-model/service-caller/openai/chat-completion/stream.d.ts +1 -1
  205. package/dist/types/ai-model/service-caller/openai/chat-completion/stream.d.ts.map +1 -1
  206. package/dist/types/ai-model/service-caller/openai/chat-completion/types.d.ts +5 -5
  207. package/dist/types/ai-model/service-caller/openai/chat-completion/types.d.ts.map +1 -1
  208. package/dist/types/ai-model/service-caller/openai/chat-completion/utils.d.ts +5 -0
  209. package/dist/types/ai-model/service-caller/openai/chat-completion/utils.d.ts.map +1 -1
  210. package/dist/types/ai-model/service-caller/openai/openai-client.d.ts +2 -2
  211. package/dist/types/ai-model/service-caller/openai/openai-client.d.ts.map +1 -1
  212. package/dist/types/ai-model/service-caller/openai/responses/non-stream.d.ts +4 -0
  213. package/dist/types/ai-model/service-caller/openai/responses/non-stream.d.ts.map +1 -0
  214. package/dist/types/ai-model/service-caller/openai/responses/responses.d.ts +8 -0
  215. package/dist/types/ai-model/service-caller/openai/responses/responses.d.ts.map +1 -0
  216. package/dist/types/ai-model/service-caller/openai/responses/stream.d.ts +4 -0
  217. package/dist/types/ai-model/service-caller/openai/responses/stream.d.ts.map +1 -0
  218. package/dist/types/ai-model/service-caller/openai/responses/types.d.ts +8 -0
  219. package/dist/types/ai-model/service-caller/openai/responses/types.d.ts.map +1 -0
  220. package/dist/types/ai-model/service-caller/openai/responses/utils.d.ts +8 -0
  221. package/dist/types/ai-model/service-caller/openai/responses/utils.d.ts.map +1 -0
  222. package/dist/types/ai-model/service-caller/semantic-retry.d.ts +2 -2
  223. package/dist/types/ai-model/service-caller/semantic-retry.d.ts.map +1 -1
  224. package/dist/types/ai-model/service-caller/types.d.ts +35 -5
  225. package/dist/types/ai-model/service-caller/types.d.ts.map +1 -1
  226. package/dist/types/ai-model/service-caller/utils.d.ts +5 -9
  227. package/dist/types/ai-model/service-caller/utils.d.ts.map +1 -1
  228. package/dist/types/ai-model/shared/multimodal-prompt.d.ts +3 -3
  229. package/dist/types/ai-model/shared/multimodal-prompt.d.ts.map +1 -1
  230. package/dist/types/ai-model/workflows/grounding/planning-action-locate.d.ts.map +1 -1
  231. package/dist/types/ai-model/workflows/grounding/section-locate.d.ts +2 -1
  232. package/dist/types/ai-model/workflows/grounding/section-locate.d.ts.map +1 -1
  233. package/dist/types/ai-model/workflows/grounding/types.d.ts +3 -2
  234. package/dist/types/ai-model/workflows/grounding/types.d.ts.map +1 -1
  235. package/dist/types/ai-model/workflows/grounding/utils.d.ts +3 -3
  236. package/dist/types/ai-model/workflows/grounding/utils.d.ts.map +1 -1
  237. package/dist/types/ai-model/workflows/insight/extraction.d.ts +1 -1
  238. package/dist/types/ai-model/workflows/insight/extraction.d.ts.map +1 -1
  239. package/dist/types/ai-model/workflows/insight/order-sensitive.d.ts.map +1 -1
  240. package/dist/types/ai-model/workflows/planning/conversation-history.d.ts +10 -8
  241. package/dist/types/ai-model/workflows/planning/conversation-history.d.ts.map +1 -1
  242. package/dist/types/ai-model/workflows/planning/custom-planning.d.ts +2 -2
  243. package/dist/types/ai-model/workflows/planning/custom-planning.d.ts.map +1 -1
  244. package/dist/types/ai-model/workflows/planning/standard-planning.d.ts.map +1 -1
  245. package/dist/types/ai-model/workflows/recorder-generation/common.d.ts +2 -1
  246. package/dist/types/ai-model/workflows/recorder-generation/common.d.ts.map +1 -1
  247. package/dist/types/ai-model/workflows/recorder-generation/markdown.d.ts +2 -2
  248. package/dist/types/ai-model/workflows/recorder-generation/markdown.d.ts.map +1 -1
  249. package/dist/types/ai-model/workflows/recorder-generation/metadata.d.ts.map +1 -1
  250. package/dist/types/ai-model/workflows/recorder-generation/playwright.d.ts.map +1 -1
  251. package/dist/types/ai-model/workflows/recorder-generation/yaml.d.ts.map +1 -1
  252. package/dist/types/service/index.d.ts.map +1 -1
  253. package/dist/types/types.d.ts +11 -2
  254. package/dist/types/types.d.ts.map +1 -1
  255. package/package.json +2 -2
@@ -16,14 +16,21 @@ class ConversationHistory {
16
16
  resetPendingFeedbackMessageIfExists() {
17
17
  if (this.pendingFeedbackMessage) this.pendingFeedbackMessage = '';
18
18
  }
19
- append(message) {
20
- this.messages.push(message);
19
+ appendMessage(message) {
20
+ this.messages.push({
21
+ type: 'input-message',
22
+ message
23
+ });
24
+ }
25
+ appendModelOutput(output) {
26
+ this.messages.push({
27
+ type: 'model-output',
28
+ output
29
+ });
21
30
  }
22
31
  seed(messages) {
23
32
  this.reset();
24
- messages.forEach((message)=>{
25
- this.append(message);
26
- });
33
+ this.messages.push(...messages);
27
34
  }
28
35
  reset() {
29
36
  this.messages.length = 0;
@@ -39,11 +46,12 @@ class ConversationHistory {
39
46
  const clonedMessages = structuredClone(this.messages);
40
47
  let imageCount = 0;
41
48
  for(let i = clonedMessages.length - 1; i >= 0; i--){
42
- const message = clonedMessages[i];
43
- const content = message.content;
49
+ const entry = clonedMessages[i];
50
+ if ('input-message' !== entry.type) continue;
51
+ const content = entry.message.content;
44
52
  if (Array.isArray(content)) for(let j = 0; j < content.length; j++){
45
53
  const item = content[j];
46
- if ('image_url' === item.type) {
54
+ if ('image' === item.type) {
47
55
  imageCount++;
48
56
  if (imageCount > maxImages) content[j] = {
49
57
  type: 'text',
@@ -166,7 +174,7 @@ class ConversationHistory {
166
174
  };
167
175
  const recentMessages = this.messages.slice(-keepCount);
168
176
  this.messages.length = 0;
169
- this.messages.push(omittedPlaceholder);
177
+ this.appendMessage(omittedPlaceholder);
170
178
  for (const msg of recentMessages)this.messages.push(msg);
171
179
  return true;
172
180
  }
@@ -1 +1 @@
1
- {"version":3,"file":"ai-model/workflows/planning/conversation-history.mjs","sources":["../../../../../src/ai-model/workflows/planning/conversation-history.ts"],"sourcesContent":["import type { SubGoal } from '@/types';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport { buildSubGoalsText } from '../../prompt/planning/sub-goals-text';\n\nexport interface ConversationHistoryOptions {\n initialMessages?: ChatCompletionMessageParam[];\n}\n\nexport class ConversationHistory {\n private readonly messages: ChatCompletionMessageParam[] = [];\n private subGoals: SubGoal[] = [];\n private memories: string[] = [];\n private historicalLogs: string[] = [];\n\n public pendingFeedbackMessage: string;\n\n constructor(options?: ConversationHistoryOptions) {\n if (options?.initialMessages?.length) {\n this.seed(options.initialMessages);\n }\n this.pendingFeedbackMessage = '';\n }\n\n resetPendingFeedbackMessageIfExists() {\n if (this.pendingFeedbackMessage) {\n this.pendingFeedbackMessage = '';\n }\n }\n\n append(message: ChatCompletionMessageParam) {\n this.messages.push(message);\n }\n\n seed(messages: ChatCompletionMessageParam[]) {\n this.reset();\n messages.forEach((message) => {\n this.append(message);\n });\n }\n\n reset() {\n this.messages.length = 0;\n this.memories.length = 0;\n this.subGoals.length = 0;\n this.historicalLogs.length = 0;\n this.pendingFeedbackMessage = '';\n }\n\n /**\n * Snapshot the conversation history, and replace the images with text if the number of images exceeds the limit.\n * @param maxImages - The maximum number of images to include in the snapshot. Undefined means no limit.\n * @returns The snapshot of the conversation history.\n */\n snapshot(maxImages?: number): ChatCompletionMessageParam[] {\n if (maxImages === undefined) {\n return [...this.messages];\n }\n\n const clonedMessages = structuredClone(this.messages);\n let imageCount = 0;\n\n // Traverse from the end to the beginning\n for (let i = clonedMessages.length - 1; i >= 0; i--) {\n const message = clonedMessages[i];\n const content = message.content;\n\n // Only process if content is an array\n if (Array.isArray(content)) {\n for (let j = 0; j < content.length; j++) {\n const item = content[j];\n\n // Check if this is an image\n if (item.type === 'image_url') {\n imageCount++;\n\n // If we've exceeded the limit, replace with text\n if (imageCount > maxImages) {\n content[j] = {\n type: 'text',\n text: '(image ignored due to size optimization)',\n };\n }\n }\n }\n }\n }\n\n return clonedMessages;\n }\n\n get length(): number {\n return this.messages.length;\n }\n\n [Symbol.iterator](): IterableIterator<ChatCompletionMessageParam> {\n return this.messages[Symbol.iterator]();\n }\n\n toJSON(): ChatCompletionMessageParam[] {\n return this.snapshot();\n }\n\n // Planning sub-goal management methods\n\n /**\n * Set all sub-goals, replacing any existing ones.\n * Automatically marks the first pending goal as running.\n */\n setSubGoals(subGoals: SubGoal[]): void {\n this.subGoals = subGoals.map((goal) => ({ ...goal }));\n this.markFirstPendingAsRunning();\n }\n\n /**\n * Merge sub-goals from update-plan-content.\n * Preserves existing descriptions when incoming description is empty.\n *\n * This handles compact XML updates like:\n * <sub-goal index=\"1\" status=\"finished\" />\n */\n mergeSubGoals(subGoals: SubGoal[]): void {\n if (this.subGoals.length === 0) {\n this.setSubGoals(subGoals);\n return;\n }\n\n const existingByIndex = new Map(\n this.subGoals.map((goal) => [goal.index, goal] as const),\n );\n\n const mergedSubGoals = subGoals.map((goal) => {\n const existingGoal = existingByIndex.get(goal.index);\n const hasNonEmptyDescription = goal.description.trim().length > 0;\n\n if (!existingGoal && !hasNonEmptyDescription) {\n return null;\n }\n\n return {\n ...goal,\n description:\n hasNonEmptyDescription || !existingGoal\n ? goal.description\n : existingGoal.description,\n };\n });\n\n const validSubGoals = mergedSubGoals.filter((goal) => goal !== null);\n if (validSubGoals.length === 0) {\n return;\n }\n\n this.setSubGoals(validSubGoals);\n }\n\n /**\n * Update a single sub-goal by index.\n * Clears logs if status or description actually changes.\n * @returns true if the sub-goal was found and updated, false otherwise\n */\n updateSubGoal(\n index: number,\n updates: Partial<Omit<SubGoal, 'index'>>,\n ): boolean {\n const goal = this.subGoals.find((g) => g.index === index);\n if (!goal) {\n return false;\n }\n\n let changed = false;\n\n if (updates.status !== undefined && updates.status !== goal.status) {\n goal.status = updates.status;\n changed = true;\n }\n if (\n updates.description !== undefined &&\n updates.description !== goal.description\n ) {\n goal.description = updates.description;\n changed = true;\n }\n\n if (changed) {\n goal.logs = [];\n }\n\n return true;\n }\n\n /**\n * Mark the first pending sub-goal as running.\n * Clears logs since status changes.\n */\n markFirstPendingAsRunning(): void {\n const firstPending = this.subGoals.find((g) => g.status === 'pending');\n if (firstPending) {\n firstPending.status = 'running';\n firstPending.logs = [];\n }\n }\n\n /**\n * Mark a sub-goal as finished.\n * Automatically marks the next pending goal as running.\n * @returns true if the sub-goal was found and updated, false otherwise\n */\n markSubGoalFinished(index: number): boolean {\n const result = this.updateSubGoal(index, { status: 'finished' });\n if (result) {\n this.markFirstPendingAsRunning();\n }\n return result;\n }\n\n /**\n * Mark all sub-goals as finished.\n * Clears logs for any goal whose status actually changes.\n */\n markAllSubGoalsFinished(): void {\n for (const goal of this.subGoals) {\n if (goal.status !== 'finished') {\n goal.logs = [];\n }\n goal.status = 'finished';\n }\n }\n\n /**\n * Append a log entry to the currently running sub-goal.\n * The log describes an action performed while working on the sub-goal.\n */\n appendSubGoalLog(log: string): void {\n if (!log) {\n return;\n }\n const runningGoal = this.subGoals.find((g) => g.status === 'running');\n if (runningGoal) {\n if (!runningGoal.logs) {\n runningGoal.logs = [];\n }\n runningGoal.logs.push(log);\n }\n }\n\n /**\n * Convert sub-goals to text representation.\n * Includes actions performed (logs) for the current sub-goal.\n */\n subGoalsToText(): string {\n return buildSubGoalsText(this.subGoals);\n }\n\n // Historical log management methods (used in non-deepThink mode)\n\n /**\n * Append a log entry to the historical logs list.\n * Used in non-deepThink mode to track executed steps across planning rounds.\n */\n appendHistoricalLog(log: string): void {\n if (log) {\n this.historicalLogs.push(log);\n }\n }\n\n /**\n * Convert historical logs to text representation.\n * Provides context about previously executed steps to the model.\n */\n historicalLogsToText(): string {\n if (this.historicalLogs.length === 0) {\n return '';\n }\n\n const logLines = this.historicalLogs.map((log) => `- ${log}`).join('\\n');\n return `Here are the steps that have been executed:\\n${logLines}`;\n }\n\n // Memory management methods\n\n /**\n * Append a memory to the memories list\n */\n appendMemory(memory: string): void {\n if (memory) {\n this.memories.push(memory);\n }\n }\n\n /**\n * Get all memories\n */\n getMemories(): string[] {\n return [...this.memories];\n }\n\n /**\n * Convert memories to text representation\n */\n memoriesToText(): string {\n if (this.memories.length === 0) {\n return '';\n }\n\n return `Memories from previous steps:\\n---\\n${this.memories.join('\\n---\\n')}\\n`;\n }\n\n /**\n * Clear all memories\n */\n clearMemories(): void {\n this.memories.length = 0;\n }\n\n /**\n * Compress the conversation history if it exceeds the threshold.\n * Removes the oldest messages and replaces them with a single placeholder message.\n * @param threshold - The number of messages that triggers compression.\n * @param keepCount - The number of recent messages to keep after compression.\n * @returns true if compression was performed, false otherwise.\n */\n compressHistory(threshold: number, keepCount: number): boolean {\n if (this.messages.length <= threshold) {\n return false;\n }\n\n const omittedCount = this.messages.length - keepCount;\n const omittedPlaceholder: ChatCompletionMessageParam = {\n role: 'user',\n content: `(${omittedCount} previous conversation messages have been omitted)`,\n };\n\n // Keep only the last `keepCount` messages\n const recentMessages = this.messages.slice(-keepCount);\n\n // Reset and rebuild with placeholder + recent messages\n this.messages.length = 0;\n this.messages.push(omittedPlaceholder);\n for (const msg of recentMessages) {\n this.messages.push(msg);\n }\n\n return true;\n }\n}\n"],"names":["Symbol","ConversationHistory","message","messages","maxImages","undefined","clonedMessages","structuredClone","imageCount","i","content","Array","j","item","subGoals","goal","existingByIndex","Map","mergedSubGoals","existingGoal","hasNonEmptyDescription","validSubGoals","index","updates","g","changed","firstPending","result","log","runningGoal","buildSubGoalsText","logLines","memory","threshold","keepCount","omittedCount","omittedPlaceholder","recentMessages","msg","options"],"mappings":";;;;;;;;;;;;eA8FGA,OAAO,QAAQ;;AAtFX,MAAMC;IAeX,sCAAsC;QACpC,IAAI,IAAI,CAAC,sBAAsB,EAC7B,IAAI,CAAC,sBAAsB,GAAG;IAElC;IAEA,OAAOC,OAAmC,EAAE;QAC1C,IAAI,CAAC,QAAQ,CAAC,IAAI,CAACA;IACrB;IAEA,KAAKC,QAAsC,EAAE;QAC3C,IAAI,CAAC,KAAK;QACVA,SAAS,OAAO,CAAC,CAACD;YAChB,IAAI,CAAC,MAAM,CAACA;QACd;IACF;IAEA,QAAQ;QACN,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG;QACvB,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG;QACvB,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG;QACvB,IAAI,CAAC,cAAc,CAAC,MAAM,GAAG;QAC7B,IAAI,CAAC,sBAAsB,GAAG;IAChC;IAOA,SAASE,SAAkB,EAAgC;QACzD,IAAIA,AAAcC,WAAdD,WACF,OAAO;eAAI,IAAI,CAAC,QAAQ;SAAC;QAG3B,MAAME,iBAAiBC,gBAAgB,IAAI,CAAC,QAAQ;QACpD,IAAIC,aAAa;QAGjB,IAAK,IAAIC,IAAIH,eAAe,MAAM,GAAG,GAAGG,KAAK,GAAGA,IAAK;YACnD,MAAMP,UAAUI,cAAc,CAACG,EAAE;YACjC,MAAMC,UAAUR,QAAQ,OAAO;YAG/B,IAAIS,MAAM,OAAO,CAACD,UAChB,IAAK,IAAIE,IAAI,GAAGA,IAAIF,QAAQ,MAAM,EAAEE,IAAK;gBACvC,MAAMC,OAAOH,OAAO,CAACE,EAAE;gBAGvB,IAAIC,AAAc,gBAAdA,KAAK,IAAI,EAAkB;oBAC7BL;oBAGA,IAAIA,aAAaJ,WACfM,OAAO,CAACE,EAAE,GAAG;wBACX,MAAM;wBACN,MAAM;oBACR;gBAEJ;YACF;QAEJ;QAEA,OAAON;IACT;IAEA,IAAI,SAAiB;QACnB,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM;IAC7B;IAEA,CAAC,cAAD,GAAkE;QAChE,OAAO,IAAI,CAAC,QAAQ,CAACN,OAAO,QAAQ,CAAC;IACvC;IAEA,SAAuC;QACrC,OAAO,IAAI,CAAC,QAAQ;IACtB;IAQA,YAAYc,QAAmB,EAAQ;QACrC,IAAI,CAAC,QAAQ,GAAGA,SAAS,GAAG,CAAC,CAACC,OAAU;gBAAE,GAAGA,IAAI;YAAC;QAClD,IAAI,CAAC,yBAAyB;IAChC;IASA,cAAcD,QAAmB,EAAQ;QACvC,IAAI,AAAyB,MAAzB,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAQ,YAC9B,IAAI,CAAC,WAAW,CAACA;QAInB,MAAME,kBAAkB,IAAIC,IAC1B,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAACF,OAAS;gBAACA,KAAK,KAAK;gBAAEA;aAAK;QAGhD,MAAMG,iBAAiBJ,SAAS,GAAG,CAAC,CAACC;YACnC,MAAMI,eAAeH,gBAAgB,GAAG,CAACD,KAAK,KAAK;YACnD,MAAMK,yBAAyBL,KAAK,WAAW,CAAC,IAAI,GAAG,MAAM,GAAG;YAEhE,IAAI,CAACI,gBAAgB,CAACC,wBACpB,OAAO;YAGT,OAAO;gBACL,GAAGL,IAAI;gBACP,aACEK,0BAA0B,CAACD,eACvBJ,KAAK,WAAW,GAChBI,aAAa,WAAW;YAChC;QACF;QAEA,MAAME,gBAAgBH,eAAe,MAAM,CAAC,CAACH,OAASA,AAAS,SAATA;QACtD,IAAIM,AAAyB,MAAzBA,cAAc,MAAM,EACtB;QAGF,IAAI,CAAC,WAAW,CAACA;IACnB;IAOA,cACEC,KAAa,EACbC,OAAwC,EAC/B;QACT,MAAMR,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAACS,IAAMA,EAAE,KAAK,KAAKF;QACnD,IAAI,CAACP,MACH,OAAO;QAGT,IAAIU,UAAU;QAEd,IAAIF,AAAmBlB,WAAnBkB,QAAQ,MAAM,IAAkBA,QAAQ,MAAM,KAAKR,KAAK,MAAM,EAAE;YAClEA,KAAK,MAAM,GAAGQ,QAAQ,MAAM;YAC5BE,UAAU;QACZ;QACA,IACEF,AAAwBlB,WAAxBkB,QAAQ,WAAW,IACnBA,QAAQ,WAAW,KAAKR,KAAK,WAAW,EACxC;YACAA,KAAK,WAAW,GAAGQ,QAAQ,WAAW;YACtCE,UAAU;QACZ;QAEA,IAAIA,SACFV,KAAK,IAAI,GAAG,EAAE;QAGhB,OAAO;IACT;IAMA,4BAAkC;QAChC,MAAMW,eAAe,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAACF,IAAMA,AAAa,cAAbA,EAAE,MAAM;QACvD,IAAIE,cAAc;YAChBA,aAAa,MAAM,GAAG;YACtBA,aAAa,IAAI,GAAG,EAAE;QACxB;IACF;IAOA,oBAAoBJ,KAAa,EAAW;QAC1C,MAAMK,SAAS,IAAI,CAAC,aAAa,CAACL,OAAO;YAAE,QAAQ;QAAW;QAC9D,IAAIK,QACF,IAAI,CAAC,yBAAyB;QAEhC,OAAOA;IACT;IAMA,0BAAgC;QAC9B,KAAK,MAAMZ,QAAQ,IAAI,CAAC,QAAQ,CAAE;YAChC,IAAIA,AAAgB,eAAhBA,KAAK,MAAM,EACbA,KAAK,IAAI,GAAG,EAAE;YAEhBA,KAAK,MAAM,GAAG;QAChB;IACF;IAMA,iBAAiBa,GAAW,EAAQ;QAClC,IAAI,CAACA,KACH;QAEF,MAAMC,cAAc,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAACL,IAAMA,AAAa,cAAbA,EAAE,MAAM;QACtD,IAAIK,aAAa;YACf,IAAI,CAACA,YAAY,IAAI,EACnBA,YAAY,IAAI,GAAG,EAAE;YAEvBA,YAAY,IAAI,CAAC,IAAI,CAACD;QACxB;IACF;IAMA,iBAAyB;QACvB,OAAOE,kBAAkB,IAAI,CAAC,QAAQ;IACxC;IAQA,oBAAoBF,GAAW,EAAQ;QACrC,IAAIA,KACF,IAAI,CAAC,cAAc,CAAC,IAAI,CAACA;IAE7B;IAMA,uBAA+B;QAC7B,IAAI,AAA+B,MAA/B,IAAI,CAAC,cAAc,CAAC,MAAM,EAC5B,OAAO;QAGT,MAAMG,WAAW,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAACH,MAAQ,CAAC,EAAE,EAAEA,KAAK,EAAE,IAAI,CAAC;QACnE,OAAO,CAAC,6CAA6C,EAAEG,UAAU;IACnE;IAOA,aAAaC,MAAc,EAAQ;QACjC,IAAIA,QACF,IAAI,CAAC,QAAQ,CAAC,IAAI,CAACA;IAEvB;IAKA,cAAwB;QACtB,OAAO;eAAI,IAAI,CAAC,QAAQ;SAAC;IAC3B;IAKA,iBAAyB;QACvB,IAAI,AAAyB,MAAzB,IAAI,CAAC,QAAQ,CAAC,MAAM,EACtB,OAAO;QAGT,OAAO,CAAC,oCAAoC,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;IACjF;IAKA,gBAAsB;QACpB,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG;IACzB;IASA,gBAAgBC,SAAiB,EAAEC,SAAiB,EAAW;QAC7D,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,IAAID,WAC1B,OAAO;QAGT,MAAME,eAAe,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAGD;QAC5C,MAAME,qBAAiD;YACrD,MAAM;YACN,SAAS,CAAC,CAAC,EAAED,aAAa,kDAAkD,CAAC;QAC/E;QAGA,MAAME,iBAAiB,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAACH;QAG5C,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG;QACvB,IAAI,CAAC,QAAQ,CAAC,IAAI,CAACE;QACnB,KAAK,MAAME,OAAOD,eAChB,IAAI,CAAC,QAAQ,CAAC,IAAI,CAACC;QAGrB,OAAO;IACT;IAvUA,YAAYC,OAAoC,CAAE;QAPlD,uBAAiB,YAAyC,EAAE;QAC5D,uBAAQ,YAAsB,EAAE;QAChC,uBAAQ,YAAqB,EAAE;QAC/B,uBAAQ,kBAA2B,EAAE;QAErC,uBAAO,0BAAP;QAGE,IAAIA,SAAS,iBAAiB,QAC5B,IAAI,CAAC,IAAI,CAACA,QAAQ,eAAe;QAEnC,IAAI,CAAC,sBAAsB,GAAG;IAChC;AAmUF"}
1
+ {"version":3,"file":"ai-model/workflows/planning/conversation-history.mjs","sources":["../../../../../src/ai-model/workflows/planning/conversation-history.ts"],"sourcesContent":["import type { ConversationMessage } from '@/ai-model/service-caller/types';\nimport type { RawAssistantOutput, SubGoal } from '@/types';\nimport { buildSubGoalsText } from '../../prompt/planning/sub-goals-text';\nimport type { ConversationEntry } from '../../service-caller/types';\n\nexport interface ConversationHistoryOptions {\n initialMessages?: ConversationEntry[];\n}\n\nexport class ConversationHistory {\n private readonly messages: ConversationEntry[] = [];\n private subGoals: SubGoal[] = [];\n private memories: string[] = [];\n private historicalLogs: string[] = [];\n\n public pendingFeedbackMessage: string;\n\n constructor(options?: ConversationHistoryOptions) {\n if (options?.initialMessages?.length) {\n this.seed(options.initialMessages);\n }\n this.pendingFeedbackMessage = '';\n }\n\n resetPendingFeedbackMessageIfExists() {\n if (this.pendingFeedbackMessage) {\n this.pendingFeedbackMessage = '';\n }\n }\n\n appendMessage(message: ConversationMessage) {\n this.messages.push({ type: 'input-message', message });\n }\n\n appendModelOutput(output: RawAssistantOutput) {\n this.messages.push({ type: 'model-output', output });\n }\n\n seed(messages: ConversationEntry[]) {\n this.reset();\n this.messages.push(...messages);\n }\n\n reset() {\n this.messages.length = 0;\n this.memories.length = 0;\n this.subGoals.length = 0;\n this.historicalLogs.length = 0;\n this.pendingFeedbackMessage = '';\n }\n\n /**\n * Snapshot the conversation history, and replace the images with text if the number of images exceeds the limit.\n * @param maxImages - The maximum number of images to include in the snapshot. Undefined means no limit.\n * @returns The snapshot of the conversation history.\n */\n snapshot(maxImages?: number): ConversationEntry[] {\n if (maxImages === undefined) {\n return [...this.messages];\n }\n\n const clonedMessages = structuredClone(this.messages);\n let imageCount = 0;\n\n // Traverse from the end to the beginning\n for (let i = clonedMessages.length - 1; i >= 0; i--) {\n const entry = clonedMessages[i];\n if (entry.type !== 'input-message') {\n continue;\n }\n const content = entry.message.content;\n\n // Only process if content is an array\n if (Array.isArray(content)) {\n for (let j = 0; j < content.length; j++) {\n const item = content[j];\n\n // Check if this is an image\n if (item.type === 'image') {\n imageCount++;\n\n // If we've exceeded the limit, replace with text\n if (imageCount > maxImages) {\n content[j] = {\n type: 'text',\n text: '(image ignored due to size optimization)',\n };\n }\n }\n }\n }\n }\n\n return clonedMessages;\n }\n\n get length(): number {\n return this.messages.length;\n }\n\n [Symbol.iterator](): IterableIterator<ConversationEntry> {\n return this.messages[Symbol.iterator]();\n }\n\n toJSON(): ConversationEntry[] {\n return this.snapshot();\n }\n\n // Planning sub-goal management methods\n\n /**\n * Set all sub-goals, replacing any existing ones.\n * Automatically marks the first pending goal as running.\n */\n setSubGoals(subGoals: SubGoal[]): void {\n this.subGoals = subGoals.map((goal) => ({ ...goal }));\n this.markFirstPendingAsRunning();\n }\n\n /**\n * Merge sub-goals from update-plan-content.\n * Preserves existing descriptions when incoming description is empty.\n *\n * This handles compact XML updates like:\n * <sub-goal index=\"1\" status=\"finished\" />\n */\n mergeSubGoals(subGoals: SubGoal[]): void {\n if (this.subGoals.length === 0) {\n this.setSubGoals(subGoals);\n return;\n }\n\n const existingByIndex = new Map(\n this.subGoals.map((goal) => [goal.index, goal] as const),\n );\n\n const mergedSubGoals = subGoals.map((goal) => {\n const existingGoal = existingByIndex.get(goal.index);\n const hasNonEmptyDescription = goal.description.trim().length > 0;\n\n if (!existingGoal && !hasNonEmptyDescription) {\n return null;\n }\n\n return {\n ...goal,\n description:\n hasNonEmptyDescription || !existingGoal\n ? goal.description\n : existingGoal.description,\n };\n });\n\n const validSubGoals = mergedSubGoals.filter((goal) => goal !== null);\n if (validSubGoals.length === 0) {\n return;\n }\n\n this.setSubGoals(validSubGoals);\n }\n\n /**\n * Update a single sub-goal by index.\n * Clears logs if status or description actually changes.\n * @returns true if the sub-goal was found and updated, false otherwise\n */\n updateSubGoal(\n index: number,\n updates: Partial<Omit<SubGoal, 'index'>>,\n ): boolean {\n const goal = this.subGoals.find((g) => g.index === index);\n if (!goal) {\n return false;\n }\n\n let changed = false;\n\n if (updates.status !== undefined && updates.status !== goal.status) {\n goal.status = updates.status;\n changed = true;\n }\n if (\n updates.description !== undefined &&\n updates.description !== goal.description\n ) {\n goal.description = updates.description;\n changed = true;\n }\n\n if (changed) {\n goal.logs = [];\n }\n\n return true;\n }\n\n /**\n * Mark the first pending sub-goal as running.\n * Clears logs since status changes.\n */\n markFirstPendingAsRunning(): void {\n const firstPending = this.subGoals.find((g) => g.status === 'pending');\n if (firstPending) {\n firstPending.status = 'running';\n firstPending.logs = [];\n }\n }\n\n /**\n * Mark a sub-goal as finished.\n * Automatically marks the next pending goal as running.\n * @returns true if the sub-goal was found and updated, false otherwise\n */\n markSubGoalFinished(index: number): boolean {\n const result = this.updateSubGoal(index, { status: 'finished' });\n if (result) {\n this.markFirstPendingAsRunning();\n }\n return result;\n }\n\n /**\n * Mark all sub-goals as finished.\n * Clears logs for any goal whose status actually changes.\n */\n markAllSubGoalsFinished(): void {\n for (const goal of this.subGoals) {\n if (goal.status !== 'finished') {\n goal.logs = [];\n }\n goal.status = 'finished';\n }\n }\n\n /**\n * Append a log entry to the currently running sub-goal.\n * The log describes an action performed while working on the sub-goal.\n */\n appendSubGoalLog(log: string): void {\n if (!log) {\n return;\n }\n const runningGoal = this.subGoals.find((g) => g.status === 'running');\n if (runningGoal) {\n if (!runningGoal.logs) {\n runningGoal.logs = [];\n }\n runningGoal.logs.push(log);\n }\n }\n\n /**\n * Convert sub-goals to text representation.\n * Includes actions performed (logs) for the current sub-goal.\n */\n subGoalsToText(): string {\n return buildSubGoalsText(this.subGoals);\n }\n\n // Historical log management methods (used in non-deepThink mode)\n\n /**\n * Append a log entry to the historical logs list.\n * Used in non-deepThink mode to track executed steps across planning rounds.\n */\n appendHistoricalLog(log: string): void {\n if (log) {\n this.historicalLogs.push(log);\n }\n }\n\n /**\n * Convert historical logs to text representation.\n * Provides context about previously executed steps to the model.\n */\n historicalLogsToText(): string {\n if (this.historicalLogs.length === 0) {\n return '';\n }\n\n const logLines = this.historicalLogs.map((log) => `- ${log}`).join('\\n');\n return `Here are the steps that have been executed:\\n${logLines}`;\n }\n\n // Memory management methods\n\n /**\n * Append a memory to the memories list\n */\n appendMemory(memory: string): void {\n if (memory) {\n this.memories.push(memory);\n }\n }\n\n /**\n * Get all memories\n */\n getMemories(): string[] {\n return [...this.memories];\n }\n\n /**\n * Convert memories to text representation\n */\n memoriesToText(): string {\n if (this.memories.length === 0) {\n return '';\n }\n\n return `Memories from previous steps:\\n---\\n${this.memories.join('\\n---\\n')}\\n`;\n }\n\n /**\n * Clear all memories\n */\n clearMemories(): void {\n this.memories.length = 0;\n }\n\n /**\n * Compress the conversation history if it exceeds the threshold.\n * Removes the oldest messages and replaces them with a single placeholder message.\n * @param threshold - The number of messages that triggers compression.\n * @param keepCount - The number of recent messages to keep after compression.\n * @returns true if compression was performed, false otherwise.\n */\n compressHistory(threshold: number, keepCount: number): boolean {\n if (this.messages.length <= threshold) {\n return false;\n }\n\n const omittedCount = this.messages.length - keepCount;\n const omittedPlaceholder: ConversationMessage = {\n role: 'user',\n content: `(${omittedCount} previous conversation messages have been omitted)`,\n };\n\n // Keep only the last `keepCount` messages\n const recentMessages = this.messages.slice(-keepCount);\n\n // Reset and rebuild with placeholder + recent messages\n this.messages.length = 0;\n this.appendMessage(omittedPlaceholder);\n for (const msg of recentMessages) {\n this.messages.push(msg);\n }\n\n return true;\n }\n}\n"],"names":["Symbol","ConversationHistory","message","output","messages","maxImages","undefined","clonedMessages","structuredClone","imageCount","i","entry","content","Array","j","item","subGoals","goal","existingByIndex","Map","mergedSubGoals","existingGoal","hasNonEmptyDescription","validSubGoals","index","updates","g","changed","firstPending","result","log","runningGoal","buildSubGoalsText","logLines","memory","threshold","keepCount","omittedCount","omittedPlaceholder","recentMessages","msg","options"],"mappings":";;;;;;;;;;;;eAoGGA,OAAO,QAAQ;;AA3FX,MAAMC;IAeX,sCAAsC;QACpC,IAAI,IAAI,CAAC,sBAAsB,EAC7B,IAAI,CAAC,sBAAsB,GAAG;IAElC;IAEA,cAAcC,OAA4B,EAAE;QAC1C,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;YAAE,MAAM;YAAiBA;QAAQ;IACtD;IAEA,kBAAkBC,MAA0B,EAAE;QAC5C,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;YAAE,MAAM;YAAgBA;QAAO;IACpD;IAEA,KAAKC,QAA6B,EAAE;QAClC,IAAI,CAAC,KAAK;QACV,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAIA;IACxB;IAEA,QAAQ;QACN,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG;QACvB,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG;QACvB,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG;QACvB,IAAI,CAAC,cAAc,CAAC,MAAM,GAAG;QAC7B,IAAI,CAAC,sBAAsB,GAAG;IAChC;IAOA,SAASC,SAAkB,EAAuB;QAChD,IAAIA,AAAcC,WAAdD,WACF,OAAO;eAAI,IAAI,CAAC,QAAQ;SAAC;QAG3B,MAAME,iBAAiBC,gBAAgB,IAAI,CAAC,QAAQ;QACpD,IAAIC,aAAa;QAGjB,IAAK,IAAIC,IAAIH,eAAe,MAAM,GAAG,GAAGG,KAAK,GAAGA,IAAK;YACnD,MAAMC,QAAQJ,cAAc,CAACG,EAAE;YAC/B,IAAIC,AAAe,oBAAfA,MAAM,IAAI,EACZ;YAEF,MAAMC,UAAUD,MAAM,OAAO,CAAC,OAAO;YAGrC,IAAIE,MAAM,OAAO,CAACD,UAChB,IAAK,IAAIE,IAAI,GAAGA,IAAIF,QAAQ,MAAM,EAAEE,IAAK;gBACvC,MAAMC,OAAOH,OAAO,CAACE,EAAE;gBAGvB,IAAIC,AAAc,YAAdA,KAAK,IAAI,EAAc;oBACzBN;oBAGA,IAAIA,aAAaJ,WACfO,OAAO,CAACE,EAAE,GAAG;wBACX,MAAM;wBACN,MAAM;oBACR;gBAEJ;YACF;QAEJ;QAEA,OAAOP;IACT;IAEA,IAAI,SAAiB;QACnB,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM;IAC7B;IAEA,CAAC,cAAD,GAAyD;QACvD,OAAO,IAAI,CAAC,QAAQ,CAACP,OAAO,QAAQ,CAAC;IACvC;IAEA,SAA8B;QAC5B,OAAO,IAAI,CAAC,QAAQ;IACtB;IAQA,YAAYgB,QAAmB,EAAQ;QACrC,IAAI,CAAC,QAAQ,GAAGA,SAAS,GAAG,CAAC,CAACC,OAAU;gBAAE,GAAGA,IAAI;YAAC;QAClD,IAAI,CAAC,yBAAyB;IAChC;IASA,cAAcD,QAAmB,EAAQ;QACvC,IAAI,AAAyB,MAAzB,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAQ,YAC9B,IAAI,CAAC,WAAW,CAACA;QAInB,MAAME,kBAAkB,IAAIC,IAC1B,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAACF,OAAS;gBAACA,KAAK,KAAK;gBAAEA;aAAK;QAGhD,MAAMG,iBAAiBJ,SAAS,GAAG,CAAC,CAACC;YACnC,MAAMI,eAAeH,gBAAgB,GAAG,CAACD,KAAK,KAAK;YACnD,MAAMK,yBAAyBL,KAAK,WAAW,CAAC,IAAI,GAAG,MAAM,GAAG;YAEhE,IAAI,CAACI,gBAAgB,CAACC,wBACpB,OAAO;YAGT,OAAO;gBACL,GAAGL,IAAI;gBACP,aACEK,0BAA0B,CAACD,eACvBJ,KAAK,WAAW,GAChBI,aAAa,WAAW;YAChC;QACF;QAEA,MAAME,gBAAgBH,eAAe,MAAM,CAAC,CAACH,OAASA,AAAS,SAATA;QACtD,IAAIM,AAAyB,MAAzBA,cAAc,MAAM,EACtB;QAGF,IAAI,CAAC,WAAW,CAACA;IACnB;IAOA,cACEC,KAAa,EACbC,OAAwC,EAC/B;QACT,MAAMR,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAACS,IAAMA,EAAE,KAAK,KAAKF;QACnD,IAAI,CAACP,MACH,OAAO;QAGT,IAAIU,UAAU;QAEd,IAAIF,AAAmBnB,WAAnBmB,QAAQ,MAAM,IAAkBA,QAAQ,MAAM,KAAKR,KAAK,MAAM,EAAE;YAClEA,KAAK,MAAM,GAAGQ,QAAQ,MAAM;YAC5BE,UAAU;QACZ;QACA,IACEF,AAAwBnB,WAAxBmB,QAAQ,WAAW,IACnBA,QAAQ,WAAW,KAAKR,KAAK,WAAW,EACxC;YACAA,KAAK,WAAW,GAAGQ,QAAQ,WAAW;YACtCE,UAAU;QACZ;QAEA,IAAIA,SACFV,KAAK,IAAI,GAAG,EAAE;QAGhB,OAAO;IACT;IAMA,4BAAkC;QAChC,MAAMW,eAAe,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAACF,IAAMA,AAAa,cAAbA,EAAE,MAAM;QACvD,IAAIE,cAAc;YAChBA,aAAa,MAAM,GAAG;YACtBA,aAAa,IAAI,GAAG,EAAE;QACxB;IACF;IAOA,oBAAoBJ,KAAa,EAAW;QAC1C,MAAMK,SAAS,IAAI,CAAC,aAAa,CAACL,OAAO;YAAE,QAAQ;QAAW;QAC9D,IAAIK,QACF,IAAI,CAAC,yBAAyB;QAEhC,OAAOA;IACT;IAMA,0BAAgC;QAC9B,KAAK,MAAMZ,QAAQ,IAAI,CAAC,QAAQ,CAAE;YAChC,IAAIA,AAAgB,eAAhBA,KAAK,MAAM,EACbA,KAAK,IAAI,GAAG,EAAE;YAEhBA,KAAK,MAAM,GAAG;QAChB;IACF;IAMA,iBAAiBa,GAAW,EAAQ;QAClC,IAAI,CAACA,KACH;QAEF,MAAMC,cAAc,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAACL,IAAMA,AAAa,cAAbA,EAAE,MAAM;QACtD,IAAIK,aAAa;YACf,IAAI,CAACA,YAAY,IAAI,EACnBA,YAAY,IAAI,GAAG,EAAE;YAEvBA,YAAY,IAAI,CAAC,IAAI,CAACD;QACxB;IACF;IAMA,iBAAyB;QACvB,OAAOE,kBAAkB,IAAI,CAAC,QAAQ;IACxC;IAQA,oBAAoBF,GAAW,EAAQ;QACrC,IAAIA,KACF,IAAI,CAAC,cAAc,CAAC,IAAI,CAACA;IAE7B;IAMA,uBAA+B;QAC7B,IAAI,AAA+B,MAA/B,IAAI,CAAC,cAAc,CAAC,MAAM,EAC5B,OAAO;QAGT,MAAMG,WAAW,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAACH,MAAQ,CAAC,EAAE,EAAEA,KAAK,EAAE,IAAI,CAAC;QACnE,OAAO,CAAC,6CAA6C,EAAEG,UAAU;IACnE;IAOA,aAAaC,MAAc,EAAQ;QACjC,IAAIA,QACF,IAAI,CAAC,QAAQ,CAAC,IAAI,CAACA;IAEvB;IAKA,cAAwB;QACtB,OAAO;eAAI,IAAI,CAAC,QAAQ;SAAC;IAC3B;IAKA,iBAAyB;QACvB,IAAI,AAAyB,MAAzB,IAAI,CAAC,QAAQ,CAAC,MAAM,EACtB,OAAO;QAGT,OAAO,CAAC,oCAAoC,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;IACjF;IAKA,gBAAsB;QACpB,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG;IACzB;IASA,gBAAgBC,SAAiB,EAAEC,SAAiB,EAAW;QAC7D,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,IAAID,WAC1B,OAAO;QAGT,MAAME,eAAe,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAGD;QAC5C,MAAME,qBAA0C;YAC9C,MAAM;YACN,SAAS,CAAC,CAAC,EAAED,aAAa,kDAAkD,CAAC;QAC/E;QAGA,MAAME,iBAAiB,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAACH;QAG5C,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG;QACvB,IAAI,CAAC,aAAa,CAACE;QACnB,KAAK,MAAME,OAAOD,eAChB,IAAI,CAAC,QAAQ,CAAC,IAAI,CAACC;QAGrB,OAAO;IACT;IA5UA,YAAYC,OAAoC,CAAE;QAPlD,uBAAiB,YAAgC,EAAE;QACnD,uBAAQ,YAAsB,EAAE;QAChC,uBAAQ,YAAqB,EAAE;QAC/B,uBAAQ,kBAA2B,EAAE;QAErC,uBAAO,0BAAP;QAGE,IAAIA,SAAS,iBAAiB,QAC5B,IAAI,CAAC,IAAI,CAACA,QAAQ,eAAe;QAEnC,IAAI,CAAC,sBAAsB,GAAG;IAChC;AAwUF"}
@@ -1,7 +1,7 @@
1
1
  import { ScreenshotItem } from "../../../screenshot-item.mjs";
2
2
  import { prepareModelImage } from "../../model-adapter/image-preprocess.mjs";
3
3
  import { AIResponseParseError, callAIWithStringResponse } from "../../service-caller/index.mjs";
4
- import { preparedReferenceImagesToChatMessages } from "../../shared/multimodal-prompt.mjs";
4
+ import { preparedReferenceImagesToMessages } from "../../shared/multimodal-prompt.mjs";
5
5
  import { normalizePlanningActionLocateFields } from "./locate-normalization.mjs";
6
6
  function appendActionContext(systemPrompt, actionContext) {
7
7
  return systemPrompt + (actionContext ? `${actionContext}\n` : '');
@@ -12,9 +12,9 @@ function buildCustomPlanningMessages(input, config) {
12
12
  const systemPrompt = appendActionContext(config.buildSystemPrompt(), actionContext);
13
13
  const userInstructionText = input.userInstruction.text;
14
14
  const userInstruction = config.buildUserInstruction ? config.buildUserInstruction(userInstructionText) : userInstructionText;
15
- const referenceImageMessages = preparedReferenceImagesToChatMessages(input.userInstruction.referenceImages);
15
+ const referenceImageMessages = preparedReferenceImagesToMessages(input.userInstruction.referenceImages);
16
16
  if (conversationHistory.pendingFeedbackMessage) {
17
- conversationHistory.append({
17
+ conversationHistory.appendMessage({
18
18
  role: 'user',
19
19
  content: [
20
20
  {
@@ -25,14 +25,12 @@ function buildCustomPlanningMessages(input, config) {
25
25
  });
26
26
  conversationHistory.resetPendingFeedbackMessageIfExists();
27
27
  }
28
- conversationHistory.append({
28
+ conversationHistory.appendMessage({
29
29
  role: 'user',
30
30
  content: [
31
31
  {
32
- type: 'image_url',
33
- image_url: {
34
- url: context.screenshot.base64
35
- }
32
+ type: 'image',
33
+ url: context.screenshot.base64
36
34
  }
37
35
  ]
38
36
  });
@@ -84,7 +82,7 @@ async function runCustomPlanning(userInstruction, options, config) {
84
82
  coordinateSystem: config.coordinateSystem
85
83
  };
86
84
  const messages = buildCustomPlanningMessages(input, config.messages);
87
- const { content, usage, rawChoiceMessage } = await callAIWithStringResponse(messages, preparedOptions.modelRuntime, {
85
+ const { content, usage, rawAssistantOutput } = await callAIWithStringResponse(messages, preparedOptions.modelRuntime, {
88
86
  abortSignal: preparedOptions.abortSignal,
89
87
  requiresOriginalImageDetail: preparedOptions.includeLocateInPlanning
90
88
  });
@@ -107,10 +105,11 @@ async function runCustomPlanning(userInstruction, options, config) {
107
105
  shouldContinuePlanning = config.shouldContinuePlanning(parsed, actions);
108
106
  } catch (parseError) {
109
107
  const errorMessage = `Parse error: ${parseError instanceof Error ? parseError.message : String(parseError)}`;
110
- throw new AIResponseParseError(errorMessage, JSON.stringify(content, void 0, 2), usage, rawChoiceMessage);
108
+ throw new AIResponseParseError(errorMessage, JSON.stringify(content, void 0, 2), usage, rawAssistantOutput);
111
109
  }
112
110
  const assistantContent = config.messages.buildAssistantContent?.(parsed, content, input);
113
- if (assistantContent) options.conversationHistory.append({
111
+ if (rawAssistantOutput?.type === 'responses' && options.modelRuntime.adapter.responses.replayRawAssistantOutput) options.conversationHistory.appendModelOutput(rawAssistantOutput);
112
+ else if (assistantContent) options.conversationHistory.appendMessage({
114
113
  role: 'assistant',
115
114
  content: assistantContent
116
115
  });
@@ -120,7 +119,7 @@ async function runCustomPlanning(userInstruction, options, config) {
120
119
  usage,
121
120
  shouldContinuePlanning,
122
121
  rawResponse: JSON.stringify(content, void 0, 2),
123
- rawChoiceMessage
122
+ rawAssistantOutput
124
123
  };
125
124
  }
126
125
  export { buildCustomPlanningMessages, runCustomPlanning };
@@ -1 +1 @@
1
- {"version":3,"file":"ai-model/workflows/planning/custom-planning.mjs","sources":["../../../../../src/ai-model/workflows/planning/custom-planning.ts"],"sourcesContent":["import type { PlanningAIResponse, PlanningAction } from '@/types';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport { ScreenshotItem } from '../../../screenshot-item';\nimport type {\n CustomPlanningInput,\n CustomPlanningMessageConfig,\n ResolvedCustomPlanningDefinition,\n} from '../../model-adapter/custom-planning-types';\nimport { prepareModelImage } from '../../model-adapter/image-preprocess';\nimport {\n AIResponseParseError,\n callAIWithStringResponse,\n} from '../../service-caller/index';\nimport {\n type PreparedUserPrompt,\n preparedReferenceImagesToChatMessages,\n} from '../../shared/multimodal-prompt';\nimport { normalizePlanningActionLocateFields } from './locate-normalization';\nimport type { PlanOptions } from './types';\n\nfunction appendActionContext(\n systemPrompt: string,\n actionContext?: string,\n): string {\n return systemPrompt + (actionContext ? `${actionContext}\\n` : '');\n}\n\nexport function buildCustomPlanningMessages<TParsed>(\n input: CustomPlanningInput,\n config: CustomPlanningMessageConfig<TParsed>,\n): ChatCompletionMessageParam[] {\n const { options } = input;\n const { conversationHistory, context, actionContext } = options;\n const systemPrompt = appendActionContext(\n config.buildSystemPrompt(),\n actionContext,\n );\n const userInstructionText = input.userInstruction.text;\n const userInstruction = config.buildUserInstruction\n ? config.buildUserInstruction(userInstructionText)\n : userInstructionText;\n const referenceImageMessages = preparedReferenceImagesToChatMessages(\n input.userInstruction.referenceImages,\n );\n\n if (conversationHistory.pendingFeedbackMessage) {\n conversationHistory.append({\n role: 'user',\n content: [\n {\n type: 'text',\n text: `${conversationHistory.pendingFeedbackMessage}. The previous action has been executed, here is the latest screenshot. Please continue according to the instruction.`,\n },\n ],\n });\n conversationHistory.resetPendingFeedbackMessageIfExists();\n }\n\n conversationHistory.append({\n role: 'user',\n content: [\n {\n type: 'image_url',\n image_url: { url: context.screenshot.base64 },\n },\n ],\n });\n\n if (config.systemPromptPlacement === 'system-message') {\n return [\n { role: 'system', content: systemPrompt },\n {\n role: 'user',\n content: [{ type: 'text', text: userInstruction }],\n },\n ...referenceImageMessages,\n ...conversationHistory.snapshot(config.historyImageLimit),\n ];\n }\n\n return [\n {\n role: 'user',\n content: `${systemPrompt}${userInstruction}`,\n },\n ...referenceImageMessages,\n ...conversationHistory.snapshot(config.historyImageLimit),\n ];\n}\n\nexport async function runCustomPlanning<TParsed>(\n userInstruction: PreparedUserPrompt,\n options: PlanOptions,\n config: ResolvedCustomPlanningDefinition<TParsed>,\n): Promise<PlanningAIResponse> {\n const { context } = options;\n const preparedImage = await prepareModelImage({\n imageBase64: context.screenshot.base64,\n width: context.shotSize.width,\n height: context.shotSize.height,\n policy: options.modelRuntime.adapter.imagePreprocess,\n });\n const preparedOptions: PlanOptions = {\n ...options,\n context: {\n ...context,\n screenshot: ScreenshotItem.create(\n preparedImage.imageBase64,\n context.screenshot.capturedAt,\n ),\n shotSize: preparedImage.preparedSize,\n },\n };\n const input: CustomPlanningInput = {\n userInstruction,\n options: preparedOptions,\n coordinateSystem: config.coordinateSystem,\n };\n\n const messages = buildCustomPlanningMessages(input, config.messages);\n const { content, usage, rawChoiceMessage } = await callAIWithStringResponse(\n messages,\n preparedOptions.modelRuntime,\n {\n abortSignal: preparedOptions.abortSignal,\n requiresOriginalImageDetail: preparedOptions.includeLocateInPlanning,\n },\n );\n\n let parsed: TParsed;\n let actions: PlanningAction[];\n let shouldContinuePlanning: boolean;\n\n try {\n parsed = config.parseResponse(content, input);\n actions = config.transformActions(parsed, input);\n normalizePlanningActionLocateFields(actions, {\n actionSpace: preparedOptions.actionSpace,\n includeLocateInPlanning: preparedOptions.includeLocateInPlanning,\n locateResultCodec: config.coordinateNormalizer,\n locateResultContext: {\n preparedSize: preparedImage.preparedSize,\n contentSize: preparedImage.contentSize,\n },\n acceptBbox2dAlias: preparedOptions.modelRuntime.adapter.acceptBbox2dAlias,\n });\n shouldContinuePlanning = config.shouldContinuePlanning(parsed, actions);\n } catch (parseError) {\n const errorMessage = `Parse error: ${\n parseError instanceof Error ? parseError.message : String(parseError)\n }`;\n throw new AIResponseParseError(\n errorMessage,\n JSON.stringify(content, undefined, 2),\n usage,\n rawChoiceMessage,\n );\n }\n\n const assistantContent = config.messages.buildAssistantContent?.(\n parsed,\n content,\n input,\n );\n // UI-TARS and Auto-GLM define their own assistant history format.\n if (assistantContent) {\n options.conversationHistory.append({\n role: 'assistant',\n content: assistantContent,\n });\n }\n\n return {\n actions,\n log: config.buildResponseLog(parsed, content),\n usage,\n shouldContinuePlanning,\n rawResponse: JSON.stringify(content, undefined, 2),\n rawChoiceMessage,\n };\n}\n"],"names":["appendActionContext","systemPrompt","actionContext","buildCustomPlanningMessages","input","config","options","conversationHistory","context","userInstructionText","userInstruction","referenceImageMessages","preparedReferenceImagesToChatMessages","runCustomPlanning","preparedImage","prepareModelImage","preparedOptions","ScreenshotItem","messages","content","usage","rawChoiceMessage","callAIWithStringResponse","parsed","actions","shouldContinuePlanning","normalizePlanningActionLocateFields","parseError","errorMessage","Error","String","AIResponseParseError","JSON","undefined","assistantContent"],"mappings":";;;;;AAoBA,SAASA,oBACPC,YAAoB,EACpBC,aAAsB;IAEtB,OAAOD,eAAgBC,CAAAA,gBAAgB,GAAGA,cAAc,EAAE,CAAC,GAAG,EAAC;AACjE;AAEO,SAASC,4BACdC,KAA0B,EAC1BC,MAA4C;IAE5C,MAAM,EAAEC,OAAO,EAAE,GAAGF;IACpB,MAAM,EAAEG,mBAAmB,EAAEC,OAAO,EAAEN,aAAa,EAAE,GAAGI;IACxD,MAAML,eAAeD,oBACnBK,OAAO,iBAAiB,IACxBH;IAEF,MAAMO,sBAAsBL,MAAM,eAAe,CAAC,IAAI;IACtD,MAAMM,kBAAkBL,OAAO,oBAAoB,GAC/CA,OAAO,oBAAoB,CAACI,uBAC5BA;IACJ,MAAME,yBAAyBC,sCAC7BR,MAAM,eAAe,CAAC,eAAe;IAGvC,IAAIG,oBAAoB,sBAAsB,EAAE;QAC9CA,oBAAoB,MAAM,CAAC;YACzB,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM,GAAGA,oBAAoB,sBAAsB,CAAC,qHAAqH,CAAC;gBAC5K;aACD;QACH;QACAA,oBAAoB,mCAAmC;IACzD;IAEAA,oBAAoB,MAAM,CAAC;QACzB,MAAM;QACN,SAAS;YACP;gBACE,MAAM;gBACN,WAAW;oBAAE,KAAKC,QAAQ,UAAU,CAAC,MAAM;gBAAC;YAC9C;SACD;IACH;IAEA,IAAIH,AAAiC,qBAAjCA,OAAO,qBAAqB,EAC9B,OAAO;QACL;YAAE,MAAM;YAAU,SAASJ;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBAAC;oBAAE,MAAM;oBAAQ,MAAMS;gBAAgB;aAAE;QACpD;WACGC;WACAJ,oBAAoB,QAAQ,CAACF,OAAO,iBAAiB;KACzD;IAGH,OAAO;QACL;YACE,MAAM;YACN,SAAS,GAAGJ,eAAeS,iBAAiB;QAC9C;WACGC;WACAJ,oBAAoB,QAAQ,CAACF,OAAO,iBAAiB;KACzD;AACH;AAEO,eAAeQ,kBACpBH,eAAmC,EACnCJ,OAAoB,EACpBD,MAAiD;IAEjD,MAAM,EAAEG,OAAO,EAAE,GAAGF;IACpB,MAAMQ,gBAAgB,MAAMC,kBAAkB;QAC5C,aAAaP,QAAQ,UAAU,CAAC,MAAM;QACtC,OAAOA,QAAQ,QAAQ,CAAC,KAAK;QAC7B,QAAQA,QAAQ,QAAQ,CAAC,MAAM;QAC/B,QAAQF,QAAQ,YAAY,CAAC,OAAO,CAAC,eAAe;IACtD;IACA,MAAMU,kBAA+B;QACnC,GAAGV,OAAO;QACV,SAAS;YACP,GAAGE,OAAO;YACV,YAAYS,eAAe,MAAM,CAC/BH,cAAc,WAAW,EACzBN,QAAQ,UAAU,CAAC,UAAU;YAE/B,UAAUM,cAAc,YAAY;QACtC;IACF;IACA,MAAMV,QAA6B;QACjCM;QACA,SAASM;QACT,kBAAkBX,OAAO,gBAAgB;IAC3C;IAEA,MAAMa,WAAWf,4BAA4BC,OAAOC,OAAO,QAAQ;IACnE,MAAM,EAAEc,OAAO,EAAEC,KAAK,EAAEC,gBAAgB,EAAE,GAAG,MAAMC,yBACjDJ,UACAF,gBAAgB,YAAY,EAC5B;QACE,aAAaA,gBAAgB,WAAW;QACxC,6BAA6BA,gBAAgB,uBAAuB;IACtE;IAGF,IAAIO;IACJ,IAAIC;IACJ,IAAIC;IAEJ,IAAI;QACFF,SAASlB,OAAO,aAAa,CAACc,SAASf;QACvCoB,UAAUnB,OAAO,gBAAgB,CAACkB,QAAQnB;QAC1CsB,oCAAoCF,SAAS;YAC3C,aAAaR,gBAAgB,WAAW;YACxC,yBAAyBA,gBAAgB,uBAAuB;YAChE,mBAAmBX,OAAO,oBAAoB;YAC9C,qBAAqB;gBACnB,cAAcS,cAAc,YAAY;gBACxC,aAAaA,cAAc,WAAW;YACxC;YACA,mBAAmBE,gBAAgB,YAAY,CAAC,OAAO,CAAC,iBAAiB;QAC3E;QACAS,yBAAyBpB,OAAO,sBAAsB,CAACkB,QAAQC;IACjE,EAAE,OAAOG,YAAY;QACnB,MAAMC,eAAe,CAAC,aAAa,EACjCD,sBAAsBE,QAAQF,WAAW,OAAO,GAAGG,OAAOH,aAC1D;QACF,MAAM,IAAII,qBACRH,cACAI,KAAK,SAAS,CAACb,SAASc,QAAW,IACnCb,OACAC;IAEJ;IAEA,MAAMa,mBAAmB7B,OAAO,QAAQ,CAAC,qBAAqB,GAC5DkB,QACAJ,SACAf;IAGF,IAAI8B,kBACF5B,QAAQ,mBAAmB,CAAC,MAAM,CAAC;QACjC,MAAM;QACN,SAAS4B;IACX;IAGF,OAAO;QACLV;QACA,KAAKnB,OAAO,gBAAgB,CAACkB,QAAQJ;QACrCC;QACAK;QACA,aAAaO,KAAK,SAAS,CAACb,SAASc,QAAW;QAChDZ;IACF;AACF"}
1
+ {"version":3,"file":"ai-model/workflows/planning/custom-planning.mjs","sources":["../../../../../src/ai-model/workflows/planning/custom-planning.ts"],"sourcesContent":["import type { PlanningAIResponse, PlanningAction } from '@/types';\nimport { ScreenshotItem } from '../../../screenshot-item';\nimport type {\n CustomPlanningInput,\n CustomPlanningMessageConfig,\n ResolvedCustomPlanningDefinition,\n} from '../../model-adapter/custom-planning-types';\nimport { prepareModelImage } from '../../model-adapter/image-preprocess';\nimport {\n AIResponseParseError,\n callAIWithStringResponse,\n} from '../../service-caller/index';\nimport type { ModelCallMessages } from '../../service-caller/types';\nimport {\n type PreparedUserPrompt,\n preparedReferenceImagesToMessages,\n} from '../../shared/multimodal-prompt';\nimport { normalizePlanningActionLocateFields } from './locate-normalization';\nimport type { PlanOptions } from './types';\n\nfunction appendActionContext(\n systemPrompt: string,\n actionContext?: string,\n): string {\n return systemPrompt + (actionContext ? `${actionContext}\\n` : '');\n}\n\nexport function buildCustomPlanningMessages<TParsed>(\n input: CustomPlanningInput,\n config: CustomPlanningMessageConfig<TParsed>,\n): ModelCallMessages {\n const { options } = input;\n const { conversationHistory, context, actionContext } = options;\n const systemPrompt = appendActionContext(\n config.buildSystemPrompt(),\n actionContext,\n );\n const userInstructionText = input.userInstruction.text;\n const userInstruction = config.buildUserInstruction\n ? config.buildUserInstruction(userInstructionText)\n : userInstructionText;\n const referenceImageMessages = preparedReferenceImagesToMessages(\n input.userInstruction.referenceImages,\n );\n\n if (conversationHistory.pendingFeedbackMessage) {\n conversationHistory.appendMessage({\n role: 'user',\n content: [\n {\n type: 'text',\n text: `${conversationHistory.pendingFeedbackMessage}. The previous action has been executed, here is the latest screenshot. Please continue according to the instruction.`,\n },\n ],\n });\n conversationHistory.resetPendingFeedbackMessageIfExists();\n }\n\n conversationHistory.appendMessage({\n role: 'user',\n content: [\n {\n type: 'image',\n url: context.screenshot.base64,\n },\n ],\n });\n\n if (config.systemPromptPlacement === 'system-message') {\n return [\n { role: 'system', content: systemPrompt },\n {\n role: 'user',\n content: [{ type: 'text', text: userInstruction }],\n },\n ...referenceImageMessages,\n ...conversationHistory.snapshot(config.historyImageLimit),\n ];\n }\n\n return [\n {\n role: 'user',\n content: `${systemPrompt}${userInstruction}`,\n },\n ...referenceImageMessages,\n ...conversationHistory.snapshot(config.historyImageLimit),\n ];\n}\n\nexport async function runCustomPlanning<TParsed>(\n userInstruction: PreparedUserPrompt,\n options: PlanOptions,\n config: ResolvedCustomPlanningDefinition<TParsed>,\n): Promise<PlanningAIResponse> {\n const { context } = options;\n const preparedImage = await prepareModelImage({\n imageBase64: context.screenshot.base64,\n width: context.shotSize.width,\n height: context.shotSize.height,\n policy: options.modelRuntime.adapter.imagePreprocess,\n });\n const preparedOptions: PlanOptions = {\n ...options,\n context: {\n ...context,\n screenshot: ScreenshotItem.create(\n preparedImage.imageBase64,\n context.screenshot.capturedAt,\n ),\n shotSize: preparedImage.preparedSize,\n },\n };\n const input: CustomPlanningInput = {\n userInstruction,\n options: preparedOptions,\n coordinateSystem: config.coordinateSystem,\n };\n\n const messages = buildCustomPlanningMessages(input, config.messages);\n const { content, usage, rawAssistantOutput } = await callAIWithStringResponse(\n messages,\n preparedOptions.modelRuntime,\n {\n abortSignal: preparedOptions.abortSignal,\n requiresOriginalImageDetail: preparedOptions.includeLocateInPlanning,\n },\n );\n\n let parsed: TParsed;\n let actions: PlanningAction[];\n let shouldContinuePlanning: boolean;\n\n try {\n parsed = config.parseResponse(content, input);\n actions = config.transformActions(parsed, input);\n normalizePlanningActionLocateFields(actions, {\n actionSpace: preparedOptions.actionSpace,\n includeLocateInPlanning: preparedOptions.includeLocateInPlanning,\n locateResultCodec: config.coordinateNormalizer,\n locateResultContext: {\n preparedSize: preparedImage.preparedSize,\n contentSize: preparedImage.contentSize,\n },\n acceptBbox2dAlias: preparedOptions.modelRuntime.adapter.acceptBbox2dAlias,\n });\n shouldContinuePlanning = config.shouldContinuePlanning(parsed, actions);\n } catch (parseError) {\n const errorMessage = `Parse error: ${\n parseError instanceof Error ? parseError.message : String(parseError)\n }`;\n throw new AIResponseParseError(\n errorMessage,\n JSON.stringify(content, undefined, 2),\n usage,\n rawAssistantOutput,\n );\n }\n\n const assistantContent = config.messages.buildAssistantContent?.(\n parsed,\n content,\n input,\n );\n // UI-TARS and Auto-GLM define their own assistant history format.\n if (\n rawAssistantOutput?.type === 'responses' &&\n options.modelRuntime.adapter.responses.replayRawAssistantOutput\n ) {\n options.conversationHistory.appendModelOutput(rawAssistantOutput);\n } else if (assistantContent) {\n options.conversationHistory.appendMessage({\n role: 'assistant',\n content: assistantContent,\n });\n }\n\n return {\n actions,\n log: config.buildResponseLog(parsed, content),\n usage,\n shouldContinuePlanning,\n rawResponse: JSON.stringify(content, undefined, 2),\n rawAssistantOutput,\n };\n}\n"],"names":["appendActionContext","systemPrompt","actionContext","buildCustomPlanningMessages","input","config","options","conversationHistory","context","userInstructionText","userInstruction","referenceImageMessages","preparedReferenceImagesToMessages","runCustomPlanning","preparedImage","prepareModelImage","preparedOptions","ScreenshotItem","messages","content","usage","rawAssistantOutput","callAIWithStringResponse","parsed","actions","shouldContinuePlanning","normalizePlanningActionLocateFields","parseError","errorMessage","Error","String","AIResponseParseError","JSON","undefined","assistantContent"],"mappings":";;;;;AAoBA,SAASA,oBACPC,YAAoB,EACpBC,aAAsB;IAEtB,OAAOD,eAAgBC,CAAAA,gBAAgB,GAAGA,cAAc,EAAE,CAAC,GAAG,EAAC;AACjE;AAEO,SAASC,4BACdC,KAA0B,EAC1BC,MAA4C;IAE5C,MAAM,EAAEC,OAAO,EAAE,GAAGF;IACpB,MAAM,EAAEG,mBAAmB,EAAEC,OAAO,EAAEN,aAAa,EAAE,GAAGI;IACxD,MAAML,eAAeD,oBACnBK,OAAO,iBAAiB,IACxBH;IAEF,MAAMO,sBAAsBL,MAAM,eAAe,CAAC,IAAI;IACtD,MAAMM,kBAAkBL,OAAO,oBAAoB,GAC/CA,OAAO,oBAAoB,CAACI,uBAC5BA;IACJ,MAAME,yBAAyBC,kCAC7BR,MAAM,eAAe,CAAC,eAAe;IAGvC,IAAIG,oBAAoB,sBAAsB,EAAE;QAC9CA,oBAAoB,aAAa,CAAC;YAChC,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM,GAAGA,oBAAoB,sBAAsB,CAAC,qHAAqH,CAAC;gBAC5K;aACD;QACH;QACAA,oBAAoB,mCAAmC;IACzD;IAEAA,oBAAoB,aAAa,CAAC;QAChC,MAAM;QACN,SAAS;YACP;gBACE,MAAM;gBACN,KAAKC,QAAQ,UAAU,CAAC,MAAM;YAChC;SACD;IACH;IAEA,IAAIH,AAAiC,qBAAjCA,OAAO,qBAAqB,EAC9B,OAAO;QACL;YAAE,MAAM;YAAU,SAASJ;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBAAC;oBAAE,MAAM;oBAAQ,MAAMS;gBAAgB;aAAE;QACpD;WACGC;WACAJ,oBAAoB,QAAQ,CAACF,OAAO,iBAAiB;KACzD;IAGH,OAAO;QACL;YACE,MAAM;YACN,SAAS,GAAGJ,eAAeS,iBAAiB;QAC9C;WACGC;WACAJ,oBAAoB,QAAQ,CAACF,OAAO,iBAAiB;KACzD;AACH;AAEO,eAAeQ,kBACpBH,eAAmC,EACnCJ,OAAoB,EACpBD,MAAiD;IAEjD,MAAM,EAAEG,OAAO,EAAE,GAAGF;IACpB,MAAMQ,gBAAgB,MAAMC,kBAAkB;QAC5C,aAAaP,QAAQ,UAAU,CAAC,MAAM;QACtC,OAAOA,QAAQ,QAAQ,CAAC,KAAK;QAC7B,QAAQA,QAAQ,QAAQ,CAAC,MAAM;QAC/B,QAAQF,QAAQ,YAAY,CAAC,OAAO,CAAC,eAAe;IACtD;IACA,MAAMU,kBAA+B;QACnC,GAAGV,OAAO;QACV,SAAS;YACP,GAAGE,OAAO;YACV,YAAYS,eAAe,MAAM,CAC/BH,cAAc,WAAW,EACzBN,QAAQ,UAAU,CAAC,UAAU;YAE/B,UAAUM,cAAc,YAAY;QACtC;IACF;IACA,MAAMV,QAA6B;QACjCM;QACA,SAASM;QACT,kBAAkBX,OAAO,gBAAgB;IAC3C;IAEA,MAAMa,WAAWf,4BAA4BC,OAAOC,OAAO,QAAQ;IACnE,MAAM,EAAEc,OAAO,EAAEC,KAAK,EAAEC,kBAAkB,EAAE,GAAG,MAAMC,yBACnDJ,UACAF,gBAAgB,YAAY,EAC5B;QACE,aAAaA,gBAAgB,WAAW;QACxC,6BAA6BA,gBAAgB,uBAAuB;IACtE;IAGF,IAAIO;IACJ,IAAIC;IACJ,IAAIC;IAEJ,IAAI;QACFF,SAASlB,OAAO,aAAa,CAACc,SAASf;QACvCoB,UAAUnB,OAAO,gBAAgB,CAACkB,QAAQnB;QAC1CsB,oCAAoCF,SAAS;YAC3C,aAAaR,gBAAgB,WAAW;YACxC,yBAAyBA,gBAAgB,uBAAuB;YAChE,mBAAmBX,OAAO,oBAAoB;YAC9C,qBAAqB;gBACnB,cAAcS,cAAc,YAAY;gBACxC,aAAaA,cAAc,WAAW;YACxC;YACA,mBAAmBE,gBAAgB,YAAY,CAAC,OAAO,CAAC,iBAAiB;QAC3E;QACAS,yBAAyBpB,OAAO,sBAAsB,CAACkB,QAAQC;IACjE,EAAE,OAAOG,YAAY;QACnB,MAAMC,eAAe,CAAC,aAAa,EACjCD,sBAAsBE,QAAQF,WAAW,OAAO,GAAGG,OAAOH,aAC1D;QACF,MAAM,IAAII,qBACRH,cACAI,KAAK,SAAS,CAACb,SAASc,QAAW,IACnCb,OACAC;IAEJ;IAEA,MAAMa,mBAAmB7B,OAAO,QAAQ,CAAC,qBAAqB,GAC5DkB,QACAJ,SACAf;IAGF,IACEiB,oBAAoB,SAAS,eAC7Bf,QAAQ,YAAY,CAAC,OAAO,CAAC,SAAS,CAAC,wBAAwB,EAE/DA,QAAQ,mBAAmB,CAAC,iBAAiB,CAACe;SACzC,IAAIa,kBACT5B,QAAQ,mBAAmB,CAAC,aAAa,CAAC;QACxC,MAAM;QACN,SAAS4B;IACX;IAGF,OAAO;QACLV;QACA,KAAKnB,OAAO,gBAAgB,CAACkB,QAAQJ;QACrCC;QACAK;QACA,aAAaO,KAAK,SAAS,CAACb,SAASc,QAAW;QAChDZ;IACF;AACF"}
@@ -6,7 +6,7 @@ import { buildStandardPlanningSystemPrompt } from "../../prompt/planning/index.m
6
6
  import { AIResponseParseError, callAI } from "../../service-caller/index.mjs";
7
7
  import { callAiAndParseWithRetry, withSemanticRetryFeedback } from "../../service-caller/semantic-retry.mjs";
8
8
  import { planningModelFamilyRequiredForLocateMessage } from "../../shared/model-locate-result/errors.mjs";
9
- import { preparedReferenceImagesToChatMessages } from "../../shared/multimodal-prompt.mjs";
9
+ import { preparedReferenceImagesToMessages } from "../../shared/multimodal-prompt.mjs";
10
10
  import { parsePlanningActions } from "./parse-planning-actions.mjs";
11
11
  import { parseStandardPlanningResponse } from "./standard-planning-parser.mjs";
12
12
  const debug = getDebug('planning');
@@ -56,7 +56,7 @@ async function callAndParsePlanningResponse(options) {
56
56
  },
57
57
  toParseError: (parseError, response)=>{
58
58
  const errorMessage = parseError instanceof Error ? parseError.message : String(parseError);
59
- return new AIResponseParseError(`XML parse error: ${errorMessage}`, response.content, response.usage, response.rawChoiceMessage, response.reasoning_content);
59
+ return new AIResponseParseError(`XML parse error: ${errorMessage}`, response.content, response.usage, response.rawAssistantOutput, response.reasoning_content);
60
60
  },
61
61
  parseRetryTimes: modelRuntime.config.retryCount,
62
62
  parseRetryInterval: modelRuntime.config.retryInterval,
@@ -101,7 +101,7 @@ async function standardPlan(userInstruction, opts) {
101
101
  });
102
102
  const imagePayload = preparedImage.imageBase64;
103
103
  const actionContext = opts.actionContext ? `${opts.actionContext}\n` : '';
104
- const referenceImageMessages = preparedReferenceImagesToChatMessages(userInstruction.referenceImages);
104
+ const referenceImageMessages = preparedReferenceImagesToMessages(userInstruction.referenceImages);
105
105
  const instruction = [
106
106
  {
107
107
  role: 'user',
@@ -128,11 +128,8 @@ async function standardPlan(userInstruction, opts) {
128
128
  text: `${conversationHistory.pendingFeedbackMessage}. The previous action has been executed, here is the latest screenshot. Please continue according to the instruction.${memoriesSection}${executionProgressSection}`
129
129
  },
130
130
  {
131
- type: 'image_url',
132
- image_url: {
133
- url: imagePayload,
134
- detail: 'high'
135
- }
131
+ type: 'image',
132
+ url: imagePayload
136
133
  }
137
134
  ]
138
135
  };
@@ -145,15 +142,12 @@ async function standardPlan(userInstruction, opts) {
145
142
  text: `This is the current screenshot.${memoriesSection}${executionProgressSection}`
146
143
  },
147
144
  {
148
- type: 'image_url',
149
- image_url: {
150
- url: imagePayload,
151
- detail: 'high'
152
- }
145
+ type: 'image',
146
+ url: imagePayload
153
147
  }
154
148
  ]
155
149
  };
156
- conversationHistory.append(latestFeedbackMessage);
150
+ conversationHistory.appendMessage(latestFeedbackMessage);
157
151
  conversationHistory.compressHistory(50, 20);
158
152
  const historyLog = conversationHistory.snapshot(opts.imagesIncludeCount);
159
153
  const msgs = [
@@ -164,7 +158,7 @@ async function standardPlan(userInstruction, opts) {
164
158
  ...instruction,
165
159
  ...historyLog
166
160
  ];
167
- const { response: { content: rawResponse, usage, reasoning_content, rawChoiceMessage }, planFromAI, actions } = await callAndParsePlanningResponse({
161
+ const { response: { content: rawResponse, usage, reasoning_content, rawAssistantOutput }, planFromAI, actions } = await callAndParsePlanningResponse({
168
162
  messages: msgs,
169
163
  modelRuntime,
170
164
  abortSignal: opts.abortSignal,
@@ -189,7 +183,7 @@ async function standardPlan(userInstruction, opts) {
189
183
  ...planFromAI,
190
184
  actions,
191
185
  rawResponse,
192
- rawChoiceMessage,
186
+ rawAssistantOutput,
193
187
  usage,
194
188
  reasoning_content,
195
189
  yamlFlow,
@@ -202,16 +196,20 @@ async function standardPlan(userInstruction, opts) {
202
196
  if (planFromAI.log) conversationHistory.appendSubGoalLog(planFromAI.log);
203
197
  } else if (planFromAI.log) conversationHistory.appendHistoricalLog(planFromAI.log);
204
198
  if (planFromAI.memory) conversationHistory.appendMemory(planFromAI.memory);
205
- if (modelRuntime.adapter.chatCompletion.replayRawAssistantMessage && rawChoiceMessage) conversationHistory.append(rawChoiceMessage);
206
- else conversationHistory.append({
207
- role: 'assistant',
208
- content: [
209
- {
210
- type: 'text',
211
- text: rawResponse
212
- }
213
- ]
214
- });
199
+ const shouldReplayRawOutput = rawAssistantOutput?.type === 'responses' && modelRuntime.adapter.responses.replayRawAssistantOutput || rawAssistantOutput?.type === 'chat-completion' && modelRuntime.adapter.chatCompletion.replayRawAssistantMessage;
200
+ if (rawAssistantOutput && shouldReplayRawOutput) conversationHistory.appendModelOutput(rawAssistantOutput);
201
+ else {
202
+ if (!rawAssistantOutput) debug('Raw assistant output is unavailable; saving only response text in conversation history.');
203
+ conversationHistory.appendMessage({
204
+ role: 'assistant',
205
+ content: [
206
+ {
207
+ type: 'text',
208
+ text: rawResponse
209
+ }
210
+ ]
211
+ });
212
+ }
215
213
  return returnValue;
216
214
  }
217
215
  export { standardPlan };
@@ -1 +1 @@
1
- {"version":3,"file":"ai-model/workflows/planning/standard-planning.mjs","sources":["../../../../../src/ai-model/workflows/planning/standard-planning.ts"],"sourcesContent":["import type {\n PlanningAIResponse,\n PlanningAction,\n RawResponsePlanningAIResponse,\n} from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport { buildYamlFlowFromPlans } from '../../../common';\nimport { prepareModelImage } from '../../model-adapter/image-preprocess';\nimport { buildStandardPlanningSystemPrompt } from '../../prompt/planning';\nimport { AIResponseParseError, callAI } from '../../service-caller/index';\nimport {\n callAiAndParseWithRetry,\n withSemanticRetryFeedback,\n} from '../../service-caller/semantic-retry';\nimport type {\n LocateResultCodec,\n LocateResultContext,\n} from '../../shared/model-locate-result';\nimport { planningModelFamilyRequiredForLocateMessage } from '../../shared/model-locate-result/errors';\nimport {\n type PreparedUserPrompt,\n preparedReferenceImagesToChatMessages,\n} from '../../shared/multimodal-prompt';\nimport { parsePlanningActions } from './parse-planning-actions';\nimport { parseStandardPlanningResponse } from './standard-planning-parser';\nimport type { PlanOptions } from './types';\n\nconst debug = getDebug('planning');\nconst warnLog = getDebug('planning', { console: true });\n\nconst noPreviousActionsText =\n 'No previous actions have been executed in this aiAct execution yet. If the instruction asks for actions, choose the first action to execute.';\ntype PlanningCallResponse = Awaited<ReturnType<typeof callAI>>;\n\ntype CallAndParsePlanningResponseOptions = {\n messages: ChatCompletionMessageParam[];\n modelRuntime: PlanOptions['modelRuntime'];\n abortSignal?: AbortSignal;\n includeLocateInPlanning: boolean;\n actionSpace: PlanOptions['actionSpace'];\n locateResultCodec?: LocateResultCodec;\n locateResultContext: LocateResultContext;\n includeThought: boolean;\n includeLog: boolean;\n};\n\nasync function callAndParsePlanningResponse(\n options: CallAndParsePlanningResponseOptions,\n): Promise<{\n response: PlanningCallResponse;\n planFromAI: RawResponsePlanningAIResponse;\n actions: PlanningAction[];\n}> {\n const {\n messages,\n modelRuntime,\n abortSignal,\n includeLocateInPlanning,\n actionSpace,\n locateResultCodec,\n locateResultContext,\n includeThought,\n includeLog,\n } = options;\n assert(\n modelRuntime.adapter.planning.kind === 'standard',\n 'callAndParsePlanningResponse requires a standard planning adapter',\n );\n const actionOutputProtocol =\n modelRuntime.adapter.planning.protocol.actionOutputProtocol;\n\n return callAiAndParseWithRetry({\n callAi: (retryAttempt, previousParseError) =>\n callAI(\n withSemanticRetryFeedback(messages, previousParseError),\n modelRuntime,\n {\n abortSignal,\n requiresOriginalImageDetail: includeLocateInPlanning,\n semanticRetryAttempt: retryAttempt,\n },\n ),\n parseResponse: (response) => {\n const planFromAI = parseStandardPlanningResponse(response.content, {\n includeThought,\n actionOutputProtocol,\n actionSpace,\n logSource: includeLog ? 'model' : 'action',\n });\n if (\n !planFromAI.action &&\n planFromAI.finalizeSuccess === undefined &&\n !planFromAI.error\n ) {\n throw new Error(\n 'Incomplete planning response: provide an action, <complete>, or <error>. A <planning> explanation alone is not sufficient.',\n );\n }\n if (planFromAI.action && planFromAI.finalizeSuccess !== undefined) {\n warnLog(\n 'Planning response included both an action and <complete>; ignoring <complete> output.',\n );\n planFromAI.finalizeMessage = undefined;\n planFromAI.finalizeSuccess = undefined;\n }\n\n const actions = planFromAI.action ? [planFromAI.action] : [];\n parsePlanningActions(actions, {\n parseRawLocateParameter: actionOutputProtocol.parseRawLocateParameter,\n actionSpace,\n includeLocateInPlanning,\n locateResultCodec,\n locateResultContext,\n acceptBbox2dAlias: modelRuntime.adapter.acceptBbox2dAlias,\n });\n return { response, planFromAI, actions };\n },\n toParseError: (parseError, response) => {\n const errorMessage =\n parseError instanceof Error ? parseError.message : String(parseError);\n return new AIResponseParseError(\n `XML parse error: ${errorMessage}`,\n response.content,\n response.usage,\n response.rawChoiceMessage,\n response.reasoning_content,\n );\n },\n parseRetryTimes: modelRuntime.config.retryCount,\n parseRetryInterval: modelRuntime.config.retryInterval,\n abortSignal,\n onParseRetry: (parseError) => {\n debug(\n 'retrying plan after response parsing failed: %s',\n parseError instanceof Error ? parseError.message : String(parseError),\n );\n },\n });\n}\n\nexport async function standardPlan(\n userInstruction: PreparedUserPrompt,\n opts: PlanOptions,\n): Promise<PlanningAIResponse> {\n const { context, conversationHistory } = opts;\n const modelRuntime = opts.modelRuntime;\n const { adapter } = modelRuntime;\n const { shotSize } = context;\n const screenshotBase64 = context.screenshot.base64;\n assert(\n adapter.planning.kind === 'standard',\n 'standardPlan requires a standard planning adapter',\n );\n const planningProtocol = adapter.planning.protocol;\n\n if (opts.includeLocateInPlanning && !modelRuntime.config.modelFamily) {\n throw new Error(\n planningModelFamilyRequiredForLocateMessage(modelRuntime.config.slot),\n );\n }\n\n const locateResultCodec = modelRuntime.config.modelFamily\n ? adapter.planning.locateResultCodec\n : undefined;\n\n // Only enable sub-goals when aiAct is in deep-thinking planning mode.\n const includeSubGoals = opts.effort === 'deepThink';\n const includeThought = opts.effort !== 'fast';\n const includeLog = opts.effort !== 'fast';\n\n if (opts.includeLocateInPlanning && !locateResultCodec) {\n throw new Error(\n planningModelFamilyRequiredForLocateMessage(modelRuntime.config.slot),\n );\n }\n\n const systemPrompt = await buildStandardPlanningSystemPrompt({\n actionSpace: opts.actionSpace,\n includeThought,\n includeLog,\n includeSubGoals,\n planningProtocol,\n ...(opts.includeLocateInPlanning && locateResultCodec\n ? {\n includeLocateInPlanning: true,\n locatePromptSpec: locateResultCodec.promptSpec,\n }\n : { includeLocateInPlanning: false }),\n });\n\n const preparedImage = await prepareModelImage({\n imageBase64: screenshotBase64,\n width: shotSize.width,\n height: shotSize.height,\n policy: adapter.imagePreprocess,\n });\n const imagePayload = preparedImage.imageBase64;\n\n const actionContext = opts.actionContext ? `${opts.actionContext}\\n` : '';\n\n const referenceImageMessages = preparedReferenceImagesToChatMessages(\n userInstruction.referenceImages,\n );\n const instruction: ChatCompletionMessageParam[] = [\n {\n role: 'user',\n content: [\n {\n type: 'text',\n text: `${actionContext}<user_instruction>${userInstruction.text}</user_instruction>`,\n },\n ],\n },\n ...referenceImageMessages,\n ];\n\n let latestFeedbackMessage: ChatCompletionMessageParam;\n\n // Build sub-goal status text to include in the message\n // In planning deep-think mode: show full sub-goals with logs\n // Otherwise: show historical execution logs\n const executionProgressText = includeSubGoals\n ? conversationHistory.subGoalsToText()\n : conversationHistory.historicalLogsToText();\n const executionProgressSection = executionProgressText\n ? `\\n\\n${executionProgressText}`\n : conversationHistory.pendingFeedbackMessage\n ? ''\n : `\\n\\n${noPreviousActionsText}`;\n\n // Build memories text to include in the message\n const memoriesText = conversationHistory.memoriesToText();\n const memoriesSection = memoriesText ? `\\n\\n${memoriesText}` : '';\n\n if (conversationHistory.pendingFeedbackMessage) {\n latestFeedbackMessage = {\n role: 'user',\n content: [\n {\n type: 'text',\n text: `${conversationHistory.pendingFeedbackMessage}. The previous action has been executed, here is the latest screenshot. Please continue according to the instruction.${memoriesSection}${executionProgressSection}`,\n },\n {\n type: 'image_url',\n image_url: {\n url: imagePayload,\n detail: 'high',\n },\n },\n ],\n };\n\n conversationHistory.resetPendingFeedbackMessageIfExists();\n } else {\n latestFeedbackMessage = {\n role: 'user',\n content: [\n {\n type: 'text',\n text: `This is the current screenshot.${memoriesSection}${executionProgressSection}`,\n },\n {\n type: 'image_url',\n image_url: {\n url: imagePayload,\n detail: 'high',\n },\n },\n ],\n };\n }\n conversationHistory.append(latestFeedbackMessage);\n\n // Compress history if it exceeds the threshold to avoid context overflow\n conversationHistory.compressHistory(50, 20);\n\n const historyLog = conversationHistory.snapshot(opts.imagesIncludeCount);\n\n const msgs: ChatCompletionMessageParam[] = [\n { role: 'system', content: systemPrompt },\n ...instruction,\n ...historyLog,\n ];\n\n const {\n response: {\n content: rawResponse,\n usage,\n reasoning_content,\n rawChoiceMessage,\n },\n planFromAI,\n actions,\n } = await callAndParsePlanningResponse({\n messages: msgs,\n modelRuntime,\n abortSignal: opts.abortSignal,\n includeLocateInPlanning: opts.includeLocateInPlanning,\n actionSpace: opts.actionSpace,\n locateResultCodec,\n locateResultContext: {\n preparedSize: preparedImage.preparedSize,\n contentSize: preparedImage.contentSize,\n },\n includeThought,\n includeLog,\n });\n\n // YAML is derived from validated actions outside the model-response retry scope.\n // dumpActionParam omits runtime-only locatedPixelResult fields.\n const yamlFlow = buildYamlFlowFromPlans(actions, opts.actionSpace);\n\n let shouldContinuePlanning = true;\n\n // Check if task is completed via <complete> tag\n if (planFromAI.finalizeSuccess !== undefined) {\n debug('task completed via <complete> tag, stop planning');\n shouldContinuePlanning = false;\n // Mark all sub-goals as finished when goal is completed in planning deep-think mode.\n if (includeSubGoals) {\n conversationHistory.markAllSubGoalsFinished();\n }\n }\n\n const returnValue: PlanningAIResponse = {\n ...planFromAI,\n actions,\n rawResponse,\n rawChoiceMessage,\n usage,\n reasoning_content,\n yamlFlow,\n shouldContinuePlanning,\n };\n\n assert(planFromAI, \"can't get plans from AI\");\n\n // TODO: The plan log is recorded before its action has executed, so a failed\n // action may still appear in the next round as an action already performed.\n // Move this write to the successful action execution path in TaskExecutor.action.\n // Update sub-goals in conversation history only in planning deep-think mode.\n if (includeSubGoals) {\n if (planFromAI.updateSubGoals?.length) {\n conversationHistory.mergeSubGoals(planFromAI.updateSubGoals);\n }\n if (planFromAI.markFinishedIndexes?.length) {\n for (const index of planFromAI.markFinishedIndexes) {\n conversationHistory.markSubGoalFinished(index);\n }\n }\n // Append the planning log to the currently running sub-goal\n if (planFromAI.log) {\n conversationHistory.appendSubGoalLog(planFromAI.log);\n }\n } else {\n // Without planning deep-think mode, accumulate logs as historical execution steps.\n if (planFromAI.log) {\n conversationHistory.appendHistoricalLog(planFromAI.log);\n }\n }\n\n // Append memory to conversation history if present\n if (planFromAI.memory) {\n conversationHistory.appendMemory(planFromAI.memory);\n }\n\n // Some model providers require opaque assistant fields to be replayed\n // verbatim in later turns. Keep this opt-in per model adapter so that an\n // unverified provider does not receive non-standard response fields.\n if (\n modelRuntime.adapter.chatCompletion.replayRawAssistantMessage &&\n rawChoiceMessage\n ) {\n conversationHistory.append(rawChoiceMessage as ChatCompletionMessageParam);\n } else {\n conversationHistory.append({\n role: 'assistant',\n content: [\n {\n type: 'text',\n text: rawResponse,\n },\n ],\n });\n }\n\n return returnValue;\n}\n"],"names":["debug","getDebug","warnLog","noPreviousActionsText","callAndParsePlanningResponse","options","messages","modelRuntime","abortSignal","includeLocateInPlanning","actionSpace","locateResultCodec","locateResultContext","includeThought","includeLog","assert","actionOutputProtocol","callAiAndParseWithRetry","retryAttempt","previousParseError","callAI","withSemanticRetryFeedback","response","planFromAI","parseStandardPlanningResponse","undefined","Error","actions","parsePlanningActions","parseError","errorMessage","String","AIResponseParseError","standardPlan","userInstruction","opts","context","conversationHistory","adapter","shotSize","screenshotBase64","planningProtocol","planningModelFamilyRequiredForLocateMessage","includeSubGoals","systemPrompt","buildStandardPlanningSystemPrompt","preparedImage","prepareModelImage","imagePayload","actionContext","referenceImageMessages","preparedReferenceImagesToChatMessages","instruction","latestFeedbackMessage","executionProgressText","executionProgressSection","memoriesText","memoriesSection","historyLog","msgs","rawResponse","usage","reasoning_content","rawChoiceMessage","yamlFlow","buildYamlFlowFromPlans","shouldContinuePlanning","returnValue","index"],"mappings":";;;;;;;;;;;AA6BA,MAAMA,QAAQC,SAAS;AACvB,MAAMC,UAAUD,SAAS,YAAY;IAAE,SAAS;AAAK;AAErD,MAAME,wBACJ;AAeF,eAAeC,6BACbC,OAA4C;IAM5C,MAAM,EACJC,QAAQ,EACRC,YAAY,EACZC,WAAW,EACXC,uBAAuB,EACvBC,WAAW,EACXC,iBAAiB,EACjBC,mBAAmB,EACnBC,cAAc,EACdC,UAAU,EACX,GAAGT;IACJU,OACER,AAAuC,eAAvCA,aAAa,OAAO,CAAC,QAAQ,CAAC,IAAI,EAClC;IAEF,MAAMS,uBACJT,aAAa,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,oBAAoB;IAE7D,OAAOU,wBAAwB;QAC7B,QAAQ,CAACC,cAAcC,qBACrBC,OACEC,0BAA0Bf,UAAUa,qBACpCZ,cACA;gBACEC;gBACA,6BAA6BC;gBAC7B,sBAAsBS;YACxB;QAEJ,eAAe,CAACI;YACd,MAAMC,aAAaC,8BAA8BF,SAAS,OAAO,EAAE;gBACjET;gBACAG;gBACAN;gBACA,WAAWI,aAAa,UAAU;YACpC;YACA,IACE,CAACS,WAAW,MAAM,IAClBA,AAA+BE,WAA/BF,WAAW,eAAe,IAC1B,CAACA,WAAW,KAAK,EAEjB,MAAM,IAAIG,MACR;YAGJ,IAAIH,WAAW,MAAM,IAAIA,AAA+BE,WAA/BF,WAAW,eAAe,EAAgB;gBACjErB,QACE;gBAEFqB,WAAW,eAAe,GAAGE;gBAC7BF,WAAW,eAAe,GAAGE;YAC/B;YAEA,MAAME,UAAUJ,WAAW,MAAM,GAAG;gBAACA,WAAW,MAAM;aAAC,GAAG,EAAE;YAC5DK,qBAAqBD,SAAS;gBAC5B,yBAAyBX,qBAAqB,uBAAuB;gBACrEN;gBACAD;gBACAE;gBACAC;gBACA,mBAAmBL,aAAa,OAAO,CAAC,iBAAiB;YAC3D;YACA,OAAO;gBAAEe;gBAAUC;gBAAYI;YAAQ;QACzC;QACA,cAAc,CAACE,YAAYP;YACzB,MAAMQ,eACJD,sBAAsBH,QAAQG,WAAW,OAAO,GAAGE,OAAOF;YAC5D,OAAO,IAAIG,qBACT,CAAC,iBAAiB,EAAEF,cAAc,EAClCR,SAAS,OAAO,EAChBA,SAAS,KAAK,EACdA,SAAS,gBAAgB,EACzBA,SAAS,iBAAiB;QAE9B;QACA,iBAAiBf,aAAa,MAAM,CAAC,UAAU;QAC/C,oBAAoBA,aAAa,MAAM,CAAC,aAAa;QACrDC;QACA,cAAc,CAACqB;YACb7B,MACE,mDACA6B,sBAAsBH,QAAQG,WAAW,OAAO,GAAGE,OAAOF;QAE9D;IACF;AACF;AAEO,eAAeI,aACpBC,eAAmC,EACnCC,IAAiB;IAEjB,MAAM,EAAEC,OAAO,EAAEC,mBAAmB,EAAE,GAAGF;IACzC,MAAM5B,eAAe4B,KAAK,YAAY;IACtC,MAAM,EAAEG,OAAO,EAAE,GAAG/B;IACpB,MAAM,EAAEgC,QAAQ,EAAE,GAAGH;IACrB,MAAMI,mBAAmBJ,QAAQ,UAAU,CAAC,MAAM;IAClDrB,OACEuB,AAA0B,eAA1BA,QAAQ,QAAQ,CAAC,IAAI,EACrB;IAEF,MAAMG,mBAAmBH,QAAQ,QAAQ,CAAC,QAAQ;IAElD,IAAIH,KAAK,uBAAuB,IAAI,CAAC5B,aAAa,MAAM,CAAC,WAAW,EAClE,MAAM,IAAImB,MACRgB,4CAA4CnC,aAAa,MAAM,CAAC,IAAI;IAIxE,MAAMI,oBAAoBJ,aAAa,MAAM,CAAC,WAAW,GACrD+B,QAAQ,QAAQ,CAAC,iBAAiB,GAClCb;IAGJ,MAAMkB,kBAAkBR,AAAgB,gBAAhBA,KAAK,MAAM;IACnC,MAAMtB,iBAAiBsB,AAAgB,WAAhBA,KAAK,MAAM;IAClC,MAAMrB,aAAaqB,AAAgB,WAAhBA,KAAK,MAAM;IAE9B,IAAIA,KAAK,uBAAuB,IAAI,CAACxB,mBACnC,MAAM,IAAIe,MACRgB,4CAA4CnC,aAAa,MAAM,CAAC,IAAI;IAIxE,MAAMqC,eAAe,MAAMC,kCAAkC;QAC3D,aAAaV,KAAK,WAAW;QAC7BtB;QACAC;QACA6B;QACAF;QACA,GAAIN,KAAK,uBAAuB,IAAIxB,oBAChC;YACE,yBAAyB;YACzB,kBAAkBA,kBAAkB,UAAU;QAChD,IACA;YAAE,yBAAyB;QAAM,CAAC;IACxC;IAEA,MAAMmC,gBAAgB,MAAMC,kBAAkB;QAC5C,aAAaP;QACb,OAAOD,SAAS,KAAK;QACrB,QAAQA,SAAS,MAAM;QACvB,QAAQD,QAAQ,eAAe;IACjC;IACA,MAAMU,eAAeF,cAAc,WAAW;IAE9C,MAAMG,gBAAgBd,KAAK,aAAa,GAAG,GAAGA,KAAK,aAAa,CAAC,EAAE,CAAC,GAAG;IAEvE,MAAMe,yBAAyBC,sCAC7BjB,gBAAgB,eAAe;IAEjC,MAAMkB,cAA4C;QAChD;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM,GAAGH,cAAc,kBAAkB,EAAEf,gBAAgB,IAAI,CAAC,mBAAmB,CAAC;gBACtF;aACD;QACH;WACGgB;KACJ;IAED,IAAIG;IAKJ,MAAMC,wBAAwBX,kBAC1BN,oBAAoB,cAAc,KAClCA,oBAAoB,oBAAoB;IAC5C,MAAMkB,2BAA2BD,wBAC7B,CAAC,IAAI,EAAEA,uBAAuB,GAC9BjB,oBAAoB,sBAAsB,GACxC,KACA,CAAC,IAAI,EAAElC,uBAAuB;IAGpC,MAAMqD,eAAenB,oBAAoB,cAAc;IACvD,MAAMoB,kBAAkBD,eAAe,CAAC,IAAI,EAAEA,cAAc,GAAG;IAE/D,IAAInB,oBAAoB,sBAAsB,EAAE;QAC9CgB,wBAAwB;YACtB,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM,GAAGhB,oBAAoB,sBAAsB,CAAC,qHAAqH,EAAEoB,kBAAkBF,0BAA0B;gBACzN;gBACA;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKP;wBACL,QAAQ;oBACV;gBACF;aACD;QACH;QAEAX,oBAAoB,mCAAmC;IACzD,OACEgB,wBAAwB;QACtB,MAAM;QACN,SAAS;YACP;gBACE,MAAM;gBACN,MAAM,CAAC,+BAA+B,EAAEI,kBAAkBF,0BAA0B;YACtF;YACA;gBACE,MAAM;gBACN,WAAW;oBACT,KAAKP;oBACL,QAAQ;gBACV;YACF;SACD;IACH;IAEFX,oBAAoB,MAAM,CAACgB;IAG3BhB,oBAAoB,eAAe,CAAC,IAAI;IAExC,MAAMqB,aAAarB,oBAAoB,QAAQ,CAACF,KAAK,kBAAkB;IAEvE,MAAMwB,OAAqC;QACzC;YAAE,MAAM;YAAU,SAASf;QAAa;WACrCQ;WACAM;KACJ;IAED,MAAM,EACJ,UAAU,EACR,SAASE,WAAW,EACpBC,KAAK,EACLC,iBAAiB,EACjBC,gBAAgB,EACjB,EACDxC,UAAU,EACVI,OAAO,EACR,GAAG,MAAMvB,6BAA6B;QACrC,UAAUuD;QACVpD;QACA,aAAa4B,KAAK,WAAW;QAC7B,yBAAyBA,KAAK,uBAAuB;QACrD,aAAaA,KAAK,WAAW;QAC7BxB;QACA,qBAAqB;YACnB,cAAcmC,cAAc,YAAY;YACxC,aAAaA,cAAc,WAAW;QACxC;QACAjC;QACAC;IACF;IAIA,MAAMkD,WAAWC,uBAAuBtC,SAASQ,KAAK,WAAW;IAEjE,IAAI+B,yBAAyB;IAG7B,IAAI3C,AAA+BE,WAA/BF,WAAW,eAAe,EAAgB;QAC5CvB,MAAM;QACNkE,yBAAyB;QAEzB,IAAIvB,iBACFN,oBAAoB,uBAAuB;IAE/C;IAEA,MAAM8B,cAAkC;QACtC,GAAG5C,UAAU;QACbI;QACAiC;QACAG;QACAF;QACAC;QACAE;QACAE;IACF;IAEAnD,OAAOQ,YAAY;IAMnB,IAAIoB,iBAAiB;QACnB,IAAIpB,WAAW,cAAc,EAAE,QAC7Bc,oBAAoB,aAAa,CAACd,WAAW,cAAc;QAE7D,IAAIA,WAAW,mBAAmB,EAAE,QAClC,KAAK,MAAM6C,SAAS7C,WAAW,mBAAmB,CAChDc,oBAAoB,mBAAmB,CAAC+B;QAI5C,IAAI7C,WAAW,GAAG,EAChBc,oBAAoB,gBAAgB,CAACd,WAAW,GAAG;IAEvD,OAEE,IAAIA,WAAW,GAAG,EAChBc,oBAAoB,mBAAmB,CAACd,WAAW,GAAG;IAK1D,IAAIA,WAAW,MAAM,EACnBc,oBAAoB,YAAY,CAACd,WAAW,MAAM;IAMpD,IACEhB,aAAa,OAAO,CAAC,cAAc,CAAC,yBAAyB,IAC7DwD,kBAEA1B,oBAAoB,MAAM,CAAC0B;SAE3B1B,oBAAoB,MAAM,CAAC;QACzB,MAAM;QACN,SAAS;YACP;gBACE,MAAM;gBACN,MAAMuB;YACR;SACD;IACH;IAGF,OAAOO;AACT"}
1
+ {"version":3,"file":"ai-model/workflows/planning/standard-planning.mjs","sources":["../../../../../src/ai-model/workflows/planning/standard-planning.ts"],"sourcesContent":["import type { ConversationMessage } from '@/ai-model/service-caller/types';\nimport type {\n PlanningAIResponse,\n PlanningAction,\n RawResponsePlanningAIResponse,\n} from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport { buildYamlFlowFromPlans } from '../../../common';\nimport { prepareModelImage } from '../../model-adapter/image-preprocess';\nimport { buildStandardPlanningSystemPrompt } from '../../prompt/planning';\nimport { AIResponseParseError, callAI } from '../../service-caller/index';\nimport {\n callAiAndParseWithRetry,\n withSemanticRetryFeedback,\n} from '../../service-caller/semantic-retry';\nimport type { ModelCallMessages } from '../../service-caller/types';\nimport type {\n LocateResultCodec,\n LocateResultContext,\n} from '../../shared/model-locate-result';\nimport { planningModelFamilyRequiredForLocateMessage } from '../../shared/model-locate-result/errors';\nimport {\n type PreparedUserPrompt,\n preparedReferenceImagesToMessages,\n} from '../../shared/multimodal-prompt';\nimport { parsePlanningActions } from './parse-planning-actions';\nimport { parseStandardPlanningResponse } from './standard-planning-parser';\nimport type { PlanOptions } from './types';\n\nconst debug = getDebug('planning');\nconst warnLog = getDebug('planning', { console: true });\n\nconst noPreviousActionsText =\n 'No previous actions have been executed in this aiAct execution yet. If the instruction asks for actions, choose the first action to execute.';\ntype PlanningCallResponse = Awaited<ReturnType<typeof callAI>>;\n\ntype CallAndParsePlanningResponseOptions = {\n messages: ModelCallMessages;\n modelRuntime: PlanOptions['modelRuntime'];\n abortSignal?: AbortSignal;\n includeLocateInPlanning: boolean;\n actionSpace: PlanOptions['actionSpace'];\n locateResultCodec?: LocateResultCodec;\n locateResultContext: LocateResultContext;\n includeThought: boolean;\n includeLog: boolean;\n};\n\nasync function callAndParsePlanningResponse(\n options: CallAndParsePlanningResponseOptions,\n): Promise<{\n response: PlanningCallResponse;\n planFromAI: RawResponsePlanningAIResponse;\n actions: PlanningAction[];\n}> {\n const {\n messages,\n modelRuntime,\n abortSignal,\n includeLocateInPlanning,\n actionSpace,\n locateResultCodec,\n locateResultContext,\n includeThought,\n includeLog,\n } = options;\n assert(\n modelRuntime.adapter.planning.kind === 'standard',\n 'callAndParsePlanningResponse requires a standard planning adapter',\n );\n const actionOutputProtocol =\n modelRuntime.adapter.planning.protocol.actionOutputProtocol;\n\n return callAiAndParseWithRetry({\n callAi: (retryAttempt, previousParseError) =>\n callAI(\n withSemanticRetryFeedback(messages, previousParseError),\n modelRuntime,\n {\n abortSignal,\n requiresOriginalImageDetail: includeLocateInPlanning,\n semanticRetryAttempt: retryAttempt,\n },\n ),\n parseResponse: (response) => {\n const planFromAI = parseStandardPlanningResponse(response.content, {\n includeThought,\n actionOutputProtocol,\n actionSpace,\n logSource: includeLog ? 'model' : 'action',\n });\n if (\n !planFromAI.action &&\n planFromAI.finalizeSuccess === undefined &&\n !planFromAI.error\n ) {\n throw new Error(\n 'Incomplete planning response: provide an action, <complete>, or <error>. A <planning> explanation alone is not sufficient.',\n );\n }\n if (planFromAI.action && planFromAI.finalizeSuccess !== undefined) {\n warnLog(\n 'Planning response included both an action and <complete>; ignoring <complete> output.',\n );\n planFromAI.finalizeMessage = undefined;\n planFromAI.finalizeSuccess = undefined;\n }\n\n const actions = planFromAI.action ? [planFromAI.action] : [];\n parsePlanningActions(actions, {\n parseRawLocateParameter: actionOutputProtocol.parseRawLocateParameter,\n actionSpace,\n includeLocateInPlanning,\n locateResultCodec,\n locateResultContext,\n acceptBbox2dAlias: modelRuntime.adapter.acceptBbox2dAlias,\n });\n return { response, planFromAI, actions };\n },\n toParseError: (parseError, response) => {\n const errorMessage =\n parseError instanceof Error ? parseError.message : String(parseError);\n return new AIResponseParseError(\n `XML parse error: ${errorMessage}`,\n response.content,\n response.usage,\n response.rawAssistantOutput,\n response.reasoning_content,\n );\n },\n parseRetryTimes: modelRuntime.config.retryCount,\n parseRetryInterval: modelRuntime.config.retryInterval,\n abortSignal,\n onParseRetry: (parseError) => {\n debug(\n 'retrying plan after response parsing failed: %s',\n parseError instanceof Error ? parseError.message : String(parseError),\n );\n },\n });\n}\n\nexport async function standardPlan(\n userInstruction: PreparedUserPrompt,\n opts: PlanOptions,\n): Promise<PlanningAIResponse> {\n const { context, conversationHistory } = opts;\n const modelRuntime = opts.modelRuntime;\n const { adapter } = modelRuntime;\n const { shotSize } = context;\n const screenshotBase64 = context.screenshot.base64;\n assert(\n adapter.planning.kind === 'standard',\n 'standardPlan requires a standard planning adapter',\n );\n const planningProtocol = adapter.planning.protocol;\n\n if (opts.includeLocateInPlanning && !modelRuntime.config.modelFamily) {\n throw new Error(\n planningModelFamilyRequiredForLocateMessage(modelRuntime.config.slot),\n );\n }\n\n const locateResultCodec = modelRuntime.config.modelFamily\n ? adapter.planning.locateResultCodec\n : undefined;\n\n // Only enable sub-goals when aiAct is in deep-thinking planning mode.\n const includeSubGoals = opts.effort === 'deepThink';\n const includeThought = opts.effort !== 'fast';\n const includeLog = opts.effort !== 'fast';\n\n if (opts.includeLocateInPlanning && !locateResultCodec) {\n throw new Error(\n planningModelFamilyRequiredForLocateMessage(modelRuntime.config.slot),\n );\n }\n\n const systemPrompt = await buildStandardPlanningSystemPrompt({\n actionSpace: opts.actionSpace,\n includeThought,\n includeLog,\n includeSubGoals,\n planningProtocol,\n ...(opts.includeLocateInPlanning && locateResultCodec\n ? {\n includeLocateInPlanning: true,\n locatePromptSpec: locateResultCodec.promptSpec,\n }\n : { includeLocateInPlanning: false }),\n });\n\n const preparedImage = await prepareModelImage({\n imageBase64: screenshotBase64,\n width: shotSize.width,\n height: shotSize.height,\n policy: adapter.imagePreprocess,\n });\n const imagePayload = preparedImage.imageBase64;\n\n const actionContext = opts.actionContext ? `${opts.actionContext}\\n` : '';\n\n const referenceImageMessages = preparedReferenceImagesToMessages(\n userInstruction.referenceImages,\n );\n const instruction: ConversationMessage[] = [\n {\n role: 'user',\n content: [\n {\n type: 'text',\n text: `${actionContext}<user_instruction>${userInstruction.text}</user_instruction>`,\n },\n ],\n },\n ...referenceImageMessages,\n ];\n\n let latestFeedbackMessage: ConversationMessage;\n\n // Build sub-goal status text to include in the message\n // In planning deep-think mode: show full sub-goals with logs\n // Otherwise: show historical execution logs\n const executionProgressText = includeSubGoals\n ? conversationHistory.subGoalsToText()\n : conversationHistory.historicalLogsToText();\n const executionProgressSection = executionProgressText\n ? `\\n\\n${executionProgressText}`\n : conversationHistory.pendingFeedbackMessage\n ? ''\n : `\\n\\n${noPreviousActionsText}`;\n\n // Build memories text to include in the message\n const memoriesText = conversationHistory.memoriesToText();\n const memoriesSection = memoriesText ? `\\n\\n${memoriesText}` : '';\n\n if (conversationHistory.pendingFeedbackMessage) {\n latestFeedbackMessage = {\n role: 'user',\n content: [\n {\n type: 'text',\n text: `${conversationHistory.pendingFeedbackMessage}. The previous action has been executed, here is the latest screenshot. Please continue according to the instruction.${memoriesSection}${executionProgressSection}`,\n },\n {\n type: 'image',\n url: imagePayload,\n },\n ],\n };\n\n conversationHistory.resetPendingFeedbackMessageIfExists();\n } else {\n latestFeedbackMessage = {\n role: 'user',\n content: [\n {\n type: 'text',\n text: `This is the current screenshot.${memoriesSection}${executionProgressSection}`,\n },\n {\n type: 'image',\n url: imagePayload,\n },\n ],\n };\n }\n conversationHistory.appendMessage(latestFeedbackMessage);\n\n // Compress history if it exceeds the threshold to avoid context overflow\n conversationHistory.compressHistory(50, 20);\n\n const historyLog = conversationHistory.snapshot(opts.imagesIncludeCount);\n\n const msgs: ModelCallMessages = [\n { role: 'system', content: systemPrompt },\n ...instruction,\n ...historyLog,\n ];\n\n const {\n response: {\n content: rawResponse,\n usage,\n reasoning_content,\n rawAssistantOutput,\n },\n planFromAI,\n actions,\n } = await callAndParsePlanningResponse({\n messages: msgs,\n modelRuntime,\n abortSignal: opts.abortSignal,\n includeLocateInPlanning: opts.includeLocateInPlanning,\n actionSpace: opts.actionSpace,\n locateResultCodec,\n locateResultContext: {\n preparedSize: preparedImage.preparedSize,\n contentSize: preparedImage.contentSize,\n },\n includeThought,\n includeLog,\n });\n\n // YAML is derived from validated actions outside the model-response retry scope.\n // dumpActionParam omits runtime-only locatedPixelResult fields.\n const yamlFlow = buildYamlFlowFromPlans(actions, opts.actionSpace);\n\n let shouldContinuePlanning = true;\n\n // Check if task is completed via <complete> tag\n if (planFromAI.finalizeSuccess !== undefined) {\n debug('task completed via <complete> tag, stop planning');\n shouldContinuePlanning = false;\n // Mark all sub-goals as finished when goal is completed in planning deep-think mode.\n if (includeSubGoals) {\n conversationHistory.markAllSubGoalsFinished();\n }\n }\n\n const returnValue: PlanningAIResponse = {\n ...planFromAI,\n actions,\n rawResponse,\n rawAssistantOutput,\n usage,\n reasoning_content,\n yamlFlow,\n shouldContinuePlanning,\n };\n\n assert(planFromAI, \"can't get plans from AI\");\n\n // TODO: The plan log is recorded before its action has executed, so a failed\n // action may still appear in the next round as an action already performed.\n // Move this write to the successful action execution path in TaskExecutor.action.\n // Update sub-goals in conversation history only in planning deep-think mode.\n if (includeSubGoals) {\n if (planFromAI.updateSubGoals?.length) {\n conversationHistory.mergeSubGoals(planFromAI.updateSubGoals);\n }\n if (planFromAI.markFinishedIndexes?.length) {\n for (const index of planFromAI.markFinishedIndexes) {\n conversationHistory.markSubGoalFinished(index);\n }\n }\n // Append the planning log to the currently running sub-goal\n if (planFromAI.log) {\n conversationHistory.appendSubGoalLog(planFromAI.log);\n }\n } else {\n // Without planning deep-think mode, accumulate logs as historical execution steps.\n if (planFromAI.log) {\n conversationHistory.appendHistoricalLog(planFromAI.log);\n }\n }\n\n // Append memory to conversation history if present\n if (planFromAI.memory) {\n conversationHistory.appendMemory(planFromAI.memory);\n }\n\n const shouldReplayRawOutput =\n (rawAssistantOutput?.type === 'responses' &&\n modelRuntime.adapter.responses.replayRawAssistantOutput) ||\n (rawAssistantOutput?.type === 'chat-completion' &&\n modelRuntime.adapter.chatCompletion.replayRawAssistantMessage);\n if (rawAssistantOutput && shouldReplayRawOutput) {\n conversationHistory.appendModelOutput(rawAssistantOutput);\n } else {\n if (!rawAssistantOutput) {\n debug(\n 'Raw assistant output is unavailable; saving only response text in conversation history.',\n );\n }\n conversationHistory.appendMessage({\n role: 'assistant',\n content: [\n {\n type: 'text',\n text: rawResponse,\n },\n ],\n });\n }\n\n return returnValue;\n}\n"],"names":["debug","getDebug","warnLog","noPreviousActionsText","callAndParsePlanningResponse","options","messages","modelRuntime","abortSignal","includeLocateInPlanning","actionSpace","locateResultCodec","locateResultContext","includeThought","includeLog","assert","actionOutputProtocol","callAiAndParseWithRetry","retryAttempt","previousParseError","callAI","withSemanticRetryFeedback","response","planFromAI","parseStandardPlanningResponse","undefined","Error","actions","parsePlanningActions","parseError","errorMessage","String","AIResponseParseError","standardPlan","userInstruction","opts","context","conversationHistory","adapter","shotSize","screenshotBase64","planningProtocol","planningModelFamilyRequiredForLocateMessage","includeSubGoals","systemPrompt","buildStandardPlanningSystemPrompt","preparedImage","prepareModelImage","imagePayload","actionContext","referenceImageMessages","preparedReferenceImagesToMessages","instruction","latestFeedbackMessage","executionProgressText","executionProgressSection","memoriesText","memoriesSection","historyLog","msgs","rawResponse","usage","reasoning_content","rawAssistantOutput","yamlFlow","buildYamlFlowFromPlans","shouldContinuePlanning","returnValue","index","shouldReplayRawOutput"],"mappings":";;;;;;;;;;;AA8BA,MAAMA,QAAQC,SAAS;AACvB,MAAMC,UAAUD,SAAS,YAAY;IAAE,SAAS;AAAK;AAErD,MAAME,wBACJ;AAeF,eAAeC,6BACbC,OAA4C;IAM5C,MAAM,EACJC,QAAQ,EACRC,YAAY,EACZC,WAAW,EACXC,uBAAuB,EACvBC,WAAW,EACXC,iBAAiB,EACjBC,mBAAmB,EACnBC,cAAc,EACdC,UAAU,EACX,GAAGT;IACJU,OACER,AAAuC,eAAvCA,aAAa,OAAO,CAAC,QAAQ,CAAC,IAAI,EAClC;IAEF,MAAMS,uBACJT,aAAa,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,oBAAoB;IAE7D,OAAOU,wBAAwB;QAC7B,QAAQ,CAACC,cAAcC,qBACrBC,OACEC,0BAA0Bf,UAAUa,qBACpCZ,cACA;gBACEC;gBACA,6BAA6BC;gBAC7B,sBAAsBS;YACxB;QAEJ,eAAe,CAACI;YACd,MAAMC,aAAaC,8BAA8BF,SAAS,OAAO,EAAE;gBACjET;gBACAG;gBACAN;gBACA,WAAWI,aAAa,UAAU;YACpC;YACA,IACE,CAACS,WAAW,MAAM,IAClBA,AAA+BE,WAA/BF,WAAW,eAAe,IAC1B,CAACA,WAAW,KAAK,EAEjB,MAAM,IAAIG,MACR;YAGJ,IAAIH,WAAW,MAAM,IAAIA,AAA+BE,WAA/BF,WAAW,eAAe,EAAgB;gBACjErB,QACE;gBAEFqB,WAAW,eAAe,GAAGE;gBAC7BF,WAAW,eAAe,GAAGE;YAC/B;YAEA,MAAME,UAAUJ,WAAW,MAAM,GAAG;gBAACA,WAAW,MAAM;aAAC,GAAG,EAAE;YAC5DK,qBAAqBD,SAAS;gBAC5B,yBAAyBX,qBAAqB,uBAAuB;gBACrEN;gBACAD;gBACAE;gBACAC;gBACA,mBAAmBL,aAAa,OAAO,CAAC,iBAAiB;YAC3D;YACA,OAAO;gBAAEe;gBAAUC;gBAAYI;YAAQ;QACzC;QACA,cAAc,CAACE,YAAYP;YACzB,MAAMQ,eACJD,sBAAsBH,QAAQG,WAAW,OAAO,GAAGE,OAAOF;YAC5D,OAAO,IAAIG,qBACT,CAAC,iBAAiB,EAAEF,cAAc,EAClCR,SAAS,OAAO,EAChBA,SAAS,KAAK,EACdA,SAAS,kBAAkB,EAC3BA,SAAS,iBAAiB;QAE9B;QACA,iBAAiBf,aAAa,MAAM,CAAC,UAAU;QAC/C,oBAAoBA,aAAa,MAAM,CAAC,aAAa;QACrDC;QACA,cAAc,CAACqB;YACb7B,MACE,mDACA6B,sBAAsBH,QAAQG,WAAW,OAAO,GAAGE,OAAOF;QAE9D;IACF;AACF;AAEO,eAAeI,aACpBC,eAAmC,EACnCC,IAAiB;IAEjB,MAAM,EAAEC,OAAO,EAAEC,mBAAmB,EAAE,GAAGF;IACzC,MAAM5B,eAAe4B,KAAK,YAAY;IACtC,MAAM,EAAEG,OAAO,EAAE,GAAG/B;IACpB,MAAM,EAAEgC,QAAQ,EAAE,GAAGH;IACrB,MAAMI,mBAAmBJ,QAAQ,UAAU,CAAC,MAAM;IAClDrB,OACEuB,AAA0B,eAA1BA,QAAQ,QAAQ,CAAC,IAAI,EACrB;IAEF,MAAMG,mBAAmBH,QAAQ,QAAQ,CAAC,QAAQ;IAElD,IAAIH,KAAK,uBAAuB,IAAI,CAAC5B,aAAa,MAAM,CAAC,WAAW,EAClE,MAAM,IAAImB,MACRgB,4CAA4CnC,aAAa,MAAM,CAAC,IAAI;IAIxE,MAAMI,oBAAoBJ,aAAa,MAAM,CAAC,WAAW,GACrD+B,QAAQ,QAAQ,CAAC,iBAAiB,GAClCb;IAGJ,MAAMkB,kBAAkBR,AAAgB,gBAAhBA,KAAK,MAAM;IACnC,MAAMtB,iBAAiBsB,AAAgB,WAAhBA,KAAK,MAAM;IAClC,MAAMrB,aAAaqB,AAAgB,WAAhBA,KAAK,MAAM;IAE9B,IAAIA,KAAK,uBAAuB,IAAI,CAACxB,mBACnC,MAAM,IAAIe,MACRgB,4CAA4CnC,aAAa,MAAM,CAAC,IAAI;IAIxE,MAAMqC,eAAe,MAAMC,kCAAkC;QAC3D,aAAaV,KAAK,WAAW;QAC7BtB;QACAC;QACA6B;QACAF;QACA,GAAIN,KAAK,uBAAuB,IAAIxB,oBAChC;YACE,yBAAyB;YACzB,kBAAkBA,kBAAkB,UAAU;QAChD,IACA;YAAE,yBAAyB;QAAM,CAAC;IACxC;IAEA,MAAMmC,gBAAgB,MAAMC,kBAAkB;QAC5C,aAAaP;QACb,OAAOD,SAAS,KAAK;QACrB,QAAQA,SAAS,MAAM;QACvB,QAAQD,QAAQ,eAAe;IACjC;IACA,MAAMU,eAAeF,cAAc,WAAW;IAE9C,MAAMG,gBAAgBd,KAAK,aAAa,GAAG,GAAGA,KAAK,aAAa,CAAC,EAAE,CAAC,GAAG;IAEvE,MAAMe,yBAAyBC,kCAC7BjB,gBAAgB,eAAe;IAEjC,MAAMkB,cAAqC;QACzC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM,GAAGH,cAAc,kBAAkB,EAAEf,gBAAgB,IAAI,CAAC,mBAAmB,CAAC;gBACtF;aACD;QACH;WACGgB;KACJ;IAED,IAAIG;IAKJ,MAAMC,wBAAwBX,kBAC1BN,oBAAoB,cAAc,KAClCA,oBAAoB,oBAAoB;IAC5C,MAAMkB,2BAA2BD,wBAC7B,CAAC,IAAI,EAAEA,uBAAuB,GAC9BjB,oBAAoB,sBAAsB,GACxC,KACA,CAAC,IAAI,EAAElC,uBAAuB;IAGpC,MAAMqD,eAAenB,oBAAoB,cAAc;IACvD,MAAMoB,kBAAkBD,eAAe,CAAC,IAAI,EAAEA,cAAc,GAAG;IAE/D,IAAInB,oBAAoB,sBAAsB,EAAE;QAC9CgB,wBAAwB;YACtB,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM,GAAGhB,oBAAoB,sBAAsB,CAAC,qHAAqH,EAAEoB,kBAAkBF,0BAA0B;gBACzN;gBACA;oBACE,MAAM;oBACN,KAAKP;gBACP;aACD;QACH;QAEAX,oBAAoB,mCAAmC;IACzD,OACEgB,wBAAwB;QACtB,MAAM;QACN,SAAS;YACP;gBACE,MAAM;gBACN,MAAM,CAAC,+BAA+B,EAAEI,kBAAkBF,0BAA0B;YACtF;YACA;gBACE,MAAM;gBACN,KAAKP;YACP;SACD;IACH;IAEFX,oBAAoB,aAAa,CAACgB;IAGlChB,oBAAoB,eAAe,CAAC,IAAI;IAExC,MAAMqB,aAAarB,oBAAoB,QAAQ,CAACF,KAAK,kBAAkB;IAEvE,MAAMwB,OAA0B;QAC9B;YAAE,MAAM;YAAU,SAASf;QAAa;WACrCQ;WACAM;KACJ;IAED,MAAM,EACJ,UAAU,EACR,SAASE,WAAW,EACpBC,KAAK,EACLC,iBAAiB,EACjBC,kBAAkB,EACnB,EACDxC,UAAU,EACVI,OAAO,EACR,GAAG,MAAMvB,6BAA6B;QACrC,UAAUuD;QACVpD;QACA,aAAa4B,KAAK,WAAW;QAC7B,yBAAyBA,KAAK,uBAAuB;QACrD,aAAaA,KAAK,WAAW;QAC7BxB;QACA,qBAAqB;YACnB,cAAcmC,cAAc,YAAY;YACxC,aAAaA,cAAc,WAAW;QACxC;QACAjC;QACAC;IACF;IAIA,MAAMkD,WAAWC,uBAAuBtC,SAASQ,KAAK,WAAW;IAEjE,IAAI+B,yBAAyB;IAG7B,IAAI3C,AAA+BE,WAA/BF,WAAW,eAAe,EAAgB;QAC5CvB,MAAM;QACNkE,yBAAyB;QAEzB,IAAIvB,iBACFN,oBAAoB,uBAAuB;IAE/C;IAEA,MAAM8B,cAAkC;QACtC,GAAG5C,UAAU;QACbI;QACAiC;QACAG;QACAF;QACAC;QACAE;QACAE;IACF;IAEAnD,OAAOQ,YAAY;IAMnB,IAAIoB,iBAAiB;QACnB,IAAIpB,WAAW,cAAc,EAAE,QAC7Bc,oBAAoB,aAAa,CAACd,WAAW,cAAc;QAE7D,IAAIA,WAAW,mBAAmB,EAAE,QAClC,KAAK,MAAM6C,SAAS7C,WAAW,mBAAmB,CAChDc,oBAAoB,mBAAmB,CAAC+B;QAI5C,IAAI7C,WAAW,GAAG,EAChBc,oBAAoB,gBAAgB,CAACd,WAAW,GAAG;IAEvD,OAEE,IAAIA,WAAW,GAAG,EAChBc,oBAAoB,mBAAmB,CAACd,WAAW,GAAG;IAK1D,IAAIA,WAAW,MAAM,EACnBc,oBAAoB,YAAY,CAACd,WAAW,MAAM;IAGpD,MAAM8C,wBACHN,oBAAoB,SAAS,eAC5BxD,aAAa,OAAO,CAAC,SAAS,CAAC,wBAAwB,IACxDwD,oBAAoB,SAAS,qBAC5BxD,aAAa,OAAO,CAAC,cAAc,CAAC,yBAAyB;IACjE,IAAIwD,sBAAsBM,uBACxBhC,oBAAoB,iBAAiB,CAAC0B;SACjC;QACL,IAAI,CAACA,oBACH/D,MACE;QAGJqC,oBAAoB,aAAa,CAAC;YAChC,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAMuB;gBACR;aACD;QACH;IACF;IAEA,OAAOO;AACT"}
@@ -157,10 +157,8 @@ const createMessageContent = (promptText, screenshots = [], includeScreenshots =
157
157
  });
158
158
  screenshots.forEach((screenshot)=>{
159
159
  messageContent.push({
160
- type: 'image_url',
161
- image_url: {
162
- url: screenshot
163
- }
160
+ type: 'image',
161
+ url: screenshot
164
162
  });
165
163
  });
166
164
  }
@@ -1 +1 @@
1
- {"version":3,"file":"ai-model/workflows/recorder-generation/common.mjs","sources":["../../../../../src/ai-model/workflows/recorder-generation/common.ts"],"sourcesContent":["import {\n DEFAULT_MIDSCENE_RECORDER_MARKDOWN_MAX_SCREENSHOTS,\n type MidsceneRecorderEvent,\n type MidsceneRecorderMarkdownScreenshotAsset,\n type MidsceneRecorderSemantic,\n type MidsceneRecorderTarget,\n createMidsceneRecorderMarkdownScreenshotAssets,\n getMidsceneRecorderEventDescription,\n getMidsceneRecorderSemantic,\n} from '@midscene/shared/recorder';\n\nexport interface EventCounts {\n navigation: number;\n click: number;\n input: number;\n scroll: number;\n total: number;\n}\n\nexport interface InputDescription {\n description: string;\n value: string;\n}\n\nexport interface ProcessedEvent {\n hashId: string;\n type: string;\n timestamp: number;\n source?: string;\n actionType?: string;\n url?: string;\n title?: string;\n semantic?: MidsceneRecorderSemantic;\n description?: string;\n value?: string;\n typedText?: string;\n inputIndex?: number;\n isSequentialInput?: boolean;\n hasNeighborInput?: boolean;\n previousInputDescription?: string;\n previousActionDescription?: string;\n nextActionDescription?: string;\n neighborInputValues?: string[];\n pageInfo?: any;\n elementRect?: any;\n screenshotPath?: string;\n}\n\nexport interface EventSummary {\n testName: string;\n startUrl: string;\n eventCounts: EventCounts;\n urls: string[];\n clickDescriptions: string[];\n inputDescriptions: InputDescription[];\n events: ProcessedEvent[];\n}\n\nexport interface RecorderGenerationContext {\n summary: EventSummary;\n screenshotAssets: MidsceneRecorderMarkdownScreenshotAsset[];\n}\n\nexport type ChromeRecordedEvent = MidsceneRecorderEvent;\n\nconst MAX_RECORDER_GENERATION_SEMANTIC_TEXT_LENGTH = 1200;\nconst MAX_RECORDER_GENERATION_SEMANTIC_ERROR_LENGTH = 400;\n\nexport interface RecorderGenerationOptions {\n testName?: string;\n includeTimestamps?: boolean;\n maxScreenshots?: number;\n description?: string;\n /** Language for human-readable generated content (e.g. 'English', 'Chinese'). Keys and API names are kept as-is. */\n language?: string;\n navigationInfo?: {\n urls?: string[];\n titles?: string[];\n initialViewport?: {\n width?: number;\n height?: number;\n };\n };\n}\n\nexport interface RecorderGenerationInput extends RecorderGenerationOptions {\n target: MidsceneRecorderTarget;\n events: MidsceneRecorderEvent[];\n}\n\nexport interface FilteredEvents {\n navigationEvents: ChromeRecordedEvent[];\n clickEvents: ChromeRecordedEvent[];\n inputEvents: ChromeRecordedEvent[];\n scrollEvents: ChromeRecordedEvent[];\n}\n\nfunction cleanRecorderSemanticField(value?: string) {\n return value?.trim() === 'AI is analyzing element...' ? undefined : value;\n}\n\nfunction truncateRecorderGenerationText(value: string, maxLength: number) {\n if (value.length <= maxLength) {\n return value;\n }\n return `${value.slice(0, maxLength)}... [truncated ${value.length - maxLength} chars]`;\n}\n\nfunction compactRecorderSemanticText(value?: string) {\n const cleaned = cleanRecorderSemanticField(value);\n return cleaned\n ? truncateRecorderGenerationText(\n cleaned,\n MAX_RECORDER_GENERATION_SEMANTIC_TEXT_LENGTH,\n )\n : undefined;\n}\n\nfunction compactRecorderSemanticError(value?: string) {\n return value\n ? truncateRecorderGenerationText(\n value,\n MAX_RECORDER_GENERATION_SEMANTIC_ERROR_LENGTH,\n )\n : undefined;\n}\n\nexport function compactRecorderSemanticForGeneration(\n semantic?: MidsceneRecorderSemantic,\n): MidsceneRecorderSemantic | undefined {\n if (!semantic) {\n return undefined;\n }\n\n return {\n source: semantic.source,\n status: semantic.status,\n confidence: semantic.confidence,\n elementDescription: compactRecorderSemanticText(\n semantic.elementDescription,\n ),\n replayInstruction: compactRecorderSemanticText(semantic.replayInstruction),\n actionSummary: compactRecorderSemanticText(semantic.actionSummary),\n error: compactRecorderSemanticError(semantic.error),\n ...(semantic.aiDescribe\n ? {\n aiDescribe: {\n verifyPrompt: semantic.aiDescribe.verifyPrompt,\n verifyPassed: semantic.aiDescribe.verifyPassed,\n deepLocate: semantic.aiDescribe.deepLocate,\n centerDistance: semantic.aiDescribe.centerDistance,\n expectedCenter: semantic.aiDescribe.expectedCenter,\n actualCenter: semantic.aiDescribe.actualCenter,\n },\n }\n : {}),\n };\n}\n\nexport const validateEvents = (events: ChromeRecordedEvent[]): void => {\n if (!events.length) {\n throw new Error('No events provided for test generation');\n }\n};\n\nexport const getScreenshotsForLLM = (\n events: ChromeRecordedEvent[],\n maxScreenshots = 1,\n): string[] => {\n return createMidsceneRecorderMarkdownScreenshotAssets(events, {\n baseDir: './screenshots',\n maxScreenshots,\n }).map((asset) => asset.dataUrl);\n};\n\nexport const filterEventsByType = (\n events: ChromeRecordedEvent[],\n): FilteredEvents => {\n return {\n navigationEvents: events.filter((event) => event.type === 'navigation'),\n clickEvents: events.filter((event) => event.type === 'click'),\n inputEvents: events.filter((event) => event.type === 'input'),\n scrollEvents: events.filter((event) => event.type === 'scroll'),\n };\n};\n\nexport const createEventCounts = (\n filteredEvents: FilteredEvents,\n totalEvents: number,\n): EventCounts => {\n return {\n navigation: filteredEvents.navigationEvents.length,\n click: filteredEvents.clickEvents.length,\n input: filteredEvents.inputEvents.length,\n scroll: filteredEvents.scrollEvents.length,\n total: totalEvents,\n };\n};\n\nexport const extractInputDescriptions = (\n inputEvents: ChromeRecordedEvent[],\n): InputDescription[] => {\n return inputEvents\n .map((event) => {\n const semantic = getMidsceneRecorderSemantic(event);\n return {\n description:\n cleanRecorderSemanticField(semantic?.elementDescription) || '',\n value: event.value || '',\n };\n })\n .filter((item) => item.description && item.value);\n};\n\nexport const processEventsForLLM = (\n events: ChromeRecordedEvent[],\n screenshotPathByEventHash: Map<string, string> = new Map(),\n): ProcessedEvent[] => {\n let inputIndex = 0;\n return events.map((event, index) => {\n const previousEvent = events[index - 1];\n const nextEvent = events[index + 1];\n const previousInput = events\n .slice(0, index)\n .reverse()\n .find((candidate) => candidate.type === 'input');\n const nextInput = events\n .slice(index + 1)\n .find((candidate) => candidate.type === 'input');\n const isInput = event.type === 'input';\n const inputSequenceIndex = isInput ? ++inputIndex : undefined;\n const hasNeighborInput = Boolean(previousInput || nextInput);\n const neighborInputValues = isInput\n ? [previousInput?.value, nextInput?.value].filter(\n (value): value is string => Boolean(value),\n )\n : undefined;\n const semantic = compactRecorderSemanticForGeneration(\n getMidsceneRecorderSemantic(event),\n );\n\n return {\n hashId: event.hashId,\n type: event.type,\n timestamp: event.timestamp,\n source: event.source,\n actionType: event.actionType,\n url: event.url,\n title: event.title,\n semantic,\n description: getMidsceneRecorderEventDescription(event),\n value: event.value,\n previousActionDescription: previousEvent\n ? getMidsceneRecorderEventDescription(previousEvent)\n : undefined,\n nextActionDescription: nextEvent\n ? getMidsceneRecorderEventDescription(nextEvent)\n : undefined,\n ...(isInput\n ? {\n typedText: event.value || '',\n inputIndex: inputSequenceIndex,\n isSequentialInput:\n previousEvent?.type === 'input' || nextEvent?.type === 'input',\n hasNeighborInput,\n previousInputDescription: previousInput\n ? getMidsceneRecorderEventDescription(previousInput)\n : undefined,\n neighborInputValues:\n neighborInputValues && neighborInputValues.length > 0\n ? neighborInputValues\n : undefined,\n }\n : {}),\n pageInfo: event.pageInfo,\n elementRect: event.elementRect,\n screenshotPath: screenshotPathByEventHash.get(event.hashId),\n };\n });\n};\n\nexport const prepareEventSummary = (\n events: ChromeRecordedEvent[],\n options: {\n testName?: string;\n maxScreenshots?: number;\n screenshotPathByEventHash?: Map<string, string>;\n } = {},\n): EventSummary => {\n const filteredEvents = filterEventsByType(events);\n const eventCounts = createEventCounts(filteredEvents, events.length);\n\n const startUrl =\n filteredEvents.navigationEvents.length > 0\n ? filteredEvents.navigationEvents[0].url || ''\n : '';\n\n const clickDescriptions = filteredEvents.clickEvents\n .map((event) => getMidsceneRecorderSemantic(event)?.elementDescription)\n .filter((desc): desc is string => Boolean(desc))\n .slice(0, 10);\n\n const inputDescriptions = extractInputDescriptions(\n filteredEvents.inputEvents,\n ).slice(0, 10);\n\n const urls = filteredEvents.navigationEvents\n .map((e) => e.url)\n .filter((url): url is string => Boolean(url))\n .slice(0, 5);\n\n const processedEvents = processEventsForLLM(\n events,\n options.screenshotPathByEventHash,\n );\n\n return {\n testName: options.testName || 'Automated test from recorded events',\n startUrl,\n eventCounts,\n urls,\n clickDescriptions,\n inputDescriptions,\n events: processedEvents,\n };\n};\n\nexport function prepareRecorderGenerationContext(\n input: RecorderGenerationInput,\n): RecorderGenerationContext {\n validateEvents(input.events);\n\n const maxScreenshots =\n input.maxScreenshots ?? DEFAULT_MIDSCENE_RECORDER_MARKDOWN_MAX_SCREENSHOTS;\n const screenshotAssets = createMidsceneRecorderMarkdownScreenshotAssets(\n input.events,\n {\n baseDir: './screenshots',\n maxScreenshots,\n },\n );\n const screenshotPathByEventHash = new Map(\n screenshotAssets.map((asset) => [asset.eventHashId, asset.relativePath]),\n );\n\n return {\n summary: prepareEventSummary(input.events, {\n testName: input.testName,\n screenshotPathByEventHash,\n }),\n screenshotAssets,\n };\n}\n\nexport const createMessageContent = (\n promptText: string,\n screenshots: string[] = [],\n includeScreenshots = true,\n) => {\n const messageContent: any[] = [\n {\n type: 'text',\n text: promptText,\n },\n ];\n\n if (includeScreenshots && screenshots.length > 0) {\n messageContent.unshift({\n type: 'text',\n text: 'Here are screenshots from the recording session to help you understand the context:',\n });\n\n screenshots.forEach((screenshot) => {\n messageContent.push({\n type: 'image_url',\n image_url: {\n url: screenshot,\n },\n });\n });\n }\n\n return messageContent;\n};\n"],"names":["MAX_RECORDER_GENERATION_SEMANTIC_TEXT_LENGTH","MAX_RECORDER_GENERATION_SEMANTIC_ERROR_LENGTH","cleanRecorderSemanticField","value","undefined","truncateRecorderGenerationText","maxLength","compactRecorderSemanticText","cleaned","compactRecorderSemanticError","compactRecorderSemanticForGeneration","semantic","validateEvents","events","Error","getScreenshotsForLLM","maxScreenshots","createMidsceneRecorderMarkdownScreenshotAssets","asset","filterEventsByType","event","createEventCounts","filteredEvents","totalEvents","extractInputDescriptions","inputEvents","getMidsceneRecorderSemantic","item","processEventsForLLM","screenshotPathByEventHash","Map","inputIndex","index","previousEvent","nextEvent","previousInput","candidate","nextInput","isInput","inputSequenceIndex","hasNeighborInput","Boolean","neighborInputValues","getMidsceneRecorderEventDescription","prepareEventSummary","options","eventCounts","startUrl","clickDescriptions","desc","inputDescriptions","urls","e","url","processedEvents","prepareRecorderGenerationContext","input","DEFAULT_MIDSCENE_RECORDER_MARKDOWN_MAX_SCREENSHOTS","screenshotAssets","createMessageContent","promptText","screenshots","includeScreenshots","messageContent","screenshot"],"mappings":";AAiEA,MAAMA,+CAA+C;AACrD,MAAMC,gDAAgD;AA+BtD,SAASC,2BAA2BC,KAAc;IAChD,OAAOA,OAAO,WAAW,+BAA+BC,SAAYD;AACtE;AAEA,SAASE,+BAA+BF,KAAa,EAAEG,SAAiB;IACtE,IAAIH,MAAM,MAAM,IAAIG,WAClB,OAAOH;IAET,OAAO,GAAGA,MAAM,KAAK,CAAC,GAAGG,WAAW,eAAe,EAAEH,MAAM,MAAM,GAAGG,UAAU,OAAO,CAAC;AACxF;AAEA,SAASC,4BAA4BJ,KAAc;IACjD,MAAMK,UAAUN,2BAA2BC;IAC3C,OAAOK,UACHH,+BACEG,SACAR,gDAEFI;AACN;AAEA,SAASK,6BAA6BN,KAAc;IAClD,OAAOA,QACHE,+BACEF,OACAF,iDAEFG;AACN;AAEO,SAASM,qCACdC,QAAmC;IAEnC,IAAI,CAACA,UACH;IAGF,OAAO;QACL,QAAQA,SAAS,MAAM;QACvB,QAAQA,SAAS,MAAM;QACvB,YAAYA,SAAS,UAAU;QAC/B,oBAAoBJ,4BAClBI,SAAS,kBAAkB;QAE7B,mBAAmBJ,4BAA4BI,SAAS,iBAAiB;QACzE,eAAeJ,4BAA4BI,SAAS,aAAa;QACjE,OAAOF,6BAA6BE,SAAS,KAAK;QAClD,GAAIA,SAAS,UAAU,GACnB;YACE,YAAY;gBACV,cAAcA,SAAS,UAAU,CAAC,YAAY;gBAC9C,cAAcA,SAAS,UAAU,CAAC,YAAY;gBAC9C,YAAYA,SAAS,UAAU,CAAC,UAAU;gBAC1C,gBAAgBA,SAAS,UAAU,CAAC,cAAc;gBAClD,gBAAgBA,SAAS,UAAU,CAAC,cAAc;gBAClD,cAAcA,SAAS,UAAU,CAAC,YAAY;YAChD;QACF,IACA,CAAC,CAAC;IACR;AACF;AAEO,MAAMC,iBAAiB,CAACC;IAC7B,IAAI,CAACA,OAAO,MAAM,EAChB,MAAM,IAAIC,MAAM;AAEpB;AAEO,MAAMC,uBAAuB,CAClCF,QACAG,iBAAiB,CAAC,GAEXC,+CAA+CJ,QAAQ;QAC5D,SAAS;QACTG;IACF,GAAG,GAAG,CAAC,CAACE,QAAUA,MAAM,OAAO;AAG1B,MAAMC,qBAAqB,CAChCN,SAEO;QACL,kBAAkBA,OAAO,MAAM,CAAC,CAACO,QAAUA,AAAe,iBAAfA,MAAM,IAAI;QACrD,aAAaP,OAAO,MAAM,CAAC,CAACO,QAAUA,AAAe,YAAfA,MAAM,IAAI;QAChD,aAAaP,OAAO,MAAM,CAAC,CAACO,QAAUA,AAAe,YAAfA,MAAM,IAAI;QAChD,cAAcP,OAAO,MAAM,CAAC,CAACO,QAAUA,AAAe,aAAfA,MAAM,IAAI;IACnD;AAGK,MAAMC,oBAAoB,CAC/BC,gBACAC,cAEO;QACL,YAAYD,eAAe,gBAAgB,CAAC,MAAM;QAClD,OAAOA,eAAe,WAAW,CAAC,MAAM;QACxC,OAAOA,eAAe,WAAW,CAAC,MAAM;QACxC,QAAQA,eAAe,YAAY,CAAC,MAAM;QAC1C,OAAOC;IACT;AAGK,MAAMC,2BAA2B,CACtCC,cAEOA,YACJ,GAAG,CAAC,CAACL;QACJ,MAAMT,WAAWe,4BAA4BN;QAC7C,OAAO;YACL,aACElB,2BAA2BS,UAAU,uBAAuB;YAC9D,OAAOS,MAAM,KAAK,IAAI;QACxB;IACF,GACC,MAAM,CAAC,CAACO,OAASA,KAAK,WAAW,IAAIA,KAAK,KAAK;AAG7C,MAAMC,sBAAsB,CACjCf,QACAgB,4BAAiD,IAAIC,KAAK;IAE1D,IAAIC,aAAa;IACjB,OAAOlB,OAAO,GAAG,CAAC,CAACO,OAAOY;QACxB,MAAMC,gBAAgBpB,MAAM,CAACmB,QAAQ,EAAE;QACvC,MAAME,YAAYrB,MAAM,CAACmB,QAAQ,EAAE;QACnC,MAAMG,gBAAgBtB,OACnB,KAAK,CAAC,GAAGmB,OACT,OAAO,GACP,IAAI,CAAC,CAACI,YAAcA,AAAmB,YAAnBA,UAAU,IAAI;QACrC,MAAMC,YAAYxB,OACf,KAAK,CAACmB,QAAQ,GACd,IAAI,CAAC,CAACI,YAAcA,AAAmB,YAAnBA,UAAU,IAAI;QACrC,MAAME,UAAUlB,AAAe,YAAfA,MAAM,IAAI;QAC1B,MAAMmB,qBAAqBD,UAAU,EAAEP,aAAa3B;QACpD,MAAMoC,mBAAmBC,QAAQN,iBAAiBE;QAClD,MAAMK,sBAAsBJ,UACxB;YAACH,eAAe;YAAOE,WAAW;SAAM,CAAC,MAAM,CAC7C,CAAClC,QAA2BsC,QAAQtC,UAEtCC;QACJ,MAAMO,WAAWD,qCACfgB,4BAA4BN;QAG9B,OAAO;YACL,QAAQA,MAAM,MAAM;YACpB,MAAMA,MAAM,IAAI;YAChB,WAAWA,MAAM,SAAS;YAC1B,QAAQA,MAAM,MAAM;YACpB,YAAYA,MAAM,UAAU;YAC5B,KAAKA,MAAM,GAAG;YACd,OAAOA,MAAM,KAAK;YAClBT;YACA,aAAagC,oCAAoCvB;YACjD,OAAOA,MAAM,KAAK;YAClB,2BAA2Ba,gBACvBU,oCAAoCV,iBACpC7B;YACJ,uBAAuB8B,YACnBS,oCAAoCT,aACpC9B;YACJ,GAAIkC,UACA;gBACE,WAAWlB,MAAM,KAAK,IAAI;gBAC1B,YAAYmB;gBACZ,mBACEN,eAAe,SAAS,WAAWC,WAAW,SAAS;gBACzDM;gBACA,0BAA0BL,gBACtBQ,oCAAoCR,iBACpC/B;gBACJ,qBACEsC,uBAAuBA,oBAAoB,MAAM,GAAG,IAChDA,sBACAtC;YACR,IACA,CAAC,CAAC;YACN,UAAUgB,MAAM,QAAQ;YACxB,aAAaA,MAAM,WAAW;YAC9B,gBAAgBS,0BAA0B,GAAG,CAACT,MAAM,MAAM;QAC5D;IACF;AACF;AAEO,MAAMwB,sBAAsB,CACjC/B,QACAgC,UAII,CAAC,CAAC;IAEN,MAAMvB,iBAAiBH,mBAAmBN;IAC1C,MAAMiC,cAAczB,kBAAkBC,gBAAgBT,OAAO,MAAM;IAEnE,MAAMkC,WACJzB,eAAe,gBAAgB,CAAC,MAAM,GAAG,IACrCA,eAAe,gBAAgB,CAAC,EAAE,CAAC,GAAG,IAAI,KAC1C;IAEN,MAAM0B,oBAAoB1B,eAAe,WAAW,CACjD,GAAG,CAAC,CAACF,QAAUM,4BAA4BN,QAAQ,oBACnD,MAAM,CAAC,CAAC6B,OAAyBR,QAAQQ,OACzC,KAAK,CAAC,GAAG;IAEZ,MAAMC,oBAAoB1B,yBACxBF,eAAe,WAAW,EAC1B,KAAK,CAAC,GAAG;IAEX,MAAM6B,OAAO7B,eAAe,gBAAgB,CACzC,GAAG,CAAC,CAAC8B,IAAMA,EAAE,GAAG,EAChB,MAAM,CAAC,CAACC,MAAuBZ,QAAQY,MACvC,KAAK,CAAC,GAAG;IAEZ,MAAMC,kBAAkB1B,oBACtBf,QACAgC,QAAQ,yBAAyB;IAGnC,OAAO;QACL,UAAUA,QAAQ,QAAQ,IAAI;QAC9BE;QACAD;QACAK;QACAH;QACAE;QACA,QAAQI;IACV;AACF;AAEO,SAASC,iCACdC,KAA8B;IAE9B5C,eAAe4C,MAAM,MAAM;IAE3B,MAAMxC,iBACJwC,MAAM,cAAc,IAAIC;IAC1B,MAAMC,mBAAmBzC,+CACvBuC,MAAM,MAAM,EACZ;QACE,SAAS;QACTxC;IACF;IAEF,MAAMa,4BAA4B,IAAIC,IACpC4B,iBAAiB,GAAG,CAAC,CAACxC,QAAU;YAACA,MAAM,WAAW;YAAEA,MAAM,YAAY;SAAC;IAGzE,OAAO;QACL,SAAS0B,oBAAoBY,MAAM,MAAM,EAAE;YACzC,UAAUA,MAAM,QAAQ;YACxB3B;QACF;QACA6B;IACF;AACF;AAEO,MAAMC,uBAAuB,CAClCC,YACAC,cAAwB,EAAE,EAC1BC,qBAAqB,IAAI;IAEzB,MAAMC,iBAAwB;QAC5B;YACE,MAAM;YACN,MAAMH;QACR;KACD;IAED,IAAIE,sBAAsBD,YAAY,MAAM,GAAG,GAAG;QAChDE,eAAe,OAAO,CAAC;YACrB,MAAM;YACN,MAAM;QACR;QAEAF,YAAY,OAAO,CAAC,CAACG;YACnBD,eAAe,IAAI,CAAC;gBAClB,MAAM;gBACN,WAAW;oBACT,KAAKC;gBACP;YACF;QACF;IACF;IAEA,OAAOD;AACT"}
1
+ {"version":3,"file":"ai-model/workflows/recorder-generation/common.mjs","sources":["../../../../../src/ai-model/workflows/recorder-generation/common.ts"],"sourcesContent":["import {\n DEFAULT_MIDSCENE_RECORDER_MARKDOWN_MAX_SCREENSHOTS,\n type MidsceneRecorderEvent,\n type MidsceneRecorderMarkdownScreenshotAsset,\n type MidsceneRecorderSemantic,\n type MidsceneRecorderTarget,\n createMidsceneRecorderMarkdownScreenshotAssets,\n getMidsceneRecorderEventDescription,\n getMidsceneRecorderSemantic,\n} from '@midscene/shared/recorder';\nimport type { MessageContent } from '../../service-caller/types';\n\nexport interface EventCounts {\n navigation: number;\n click: number;\n input: number;\n scroll: number;\n total: number;\n}\n\nexport interface InputDescription {\n description: string;\n value: string;\n}\n\nexport interface ProcessedEvent {\n hashId: string;\n type: string;\n timestamp: number;\n source?: string;\n actionType?: string;\n url?: string;\n title?: string;\n semantic?: MidsceneRecorderSemantic;\n description?: string;\n value?: string;\n typedText?: string;\n inputIndex?: number;\n isSequentialInput?: boolean;\n hasNeighborInput?: boolean;\n previousInputDescription?: string;\n previousActionDescription?: string;\n nextActionDescription?: string;\n neighborInputValues?: string[];\n pageInfo?: any;\n elementRect?: any;\n screenshotPath?: string;\n}\n\nexport interface EventSummary {\n testName: string;\n startUrl: string;\n eventCounts: EventCounts;\n urls: string[];\n clickDescriptions: string[];\n inputDescriptions: InputDescription[];\n events: ProcessedEvent[];\n}\n\nexport interface RecorderGenerationContext {\n summary: EventSummary;\n screenshotAssets: MidsceneRecorderMarkdownScreenshotAsset[];\n}\n\nexport type ChromeRecordedEvent = MidsceneRecorderEvent;\n\nconst MAX_RECORDER_GENERATION_SEMANTIC_TEXT_LENGTH = 1200;\nconst MAX_RECORDER_GENERATION_SEMANTIC_ERROR_LENGTH = 400;\n\nexport interface RecorderGenerationOptions {\n testName?: string;\n includeTimestamps?: boolean;\n maxScreenshots?: number;\n description?: string;\n /** Language for human-readable generated content (e.g. 'English', 'Chinese'). Keys and API names are kept as-is. */\n language?: string;\n navigationInfo?: {\n urls?: string[];\n titles?: string[];\n initialViewport?: {\n width?: number;\n height?: number;\n };\n };\n}\n\nexport interface RecorderGenerationInput extends RecorderGenerationOptions {\n target: MidsceneRecorderTarget;\n events: MidsceneRecorderEvent[];\n}\n\nexport interface FilteredEvents {\n navigationEvents: ChromeRecordedEvent[];\n clickEvents: ChromeRecordedEvent[];\n inputEvents: ChromeRecordedEvent[];\n scrollEvents: ChromeRecordedEvent[];\n}\n\nfunction cleanRecorderSemanticField(value?: string) {\n return value?.trim() === 'AI is analyzing element...' ? undefined : value;\n}\n\nfunction truncateRecorderGenerationText(value: string, maxLength: number) {\n if (value.length <= maxLength) {\n return value;\n }\n return `${value.slice(0, maxLength)}... [truncated ${value.length - maxLength} chars]`;\n}\n\nfunction compactRecorderSemanticText(value?: string) {\n const cleaned = cleanRecorderSemanticField(value);\n return cleaned\n ? truncateRecorderGenerationText(\n cleaned,\n MAX_RECORDER_GENERATION_SEMANTIC_TEXT_LENGTH,\n )\n : undefined;\n}\n\nfunction compactRecorderSemanticError(value?: string) {\n return value\n ? truncateRecorderGenerationText(\n value,\n MAX_RECORDER_GENERATION_SEMANTIC_ERROR_LENGTH,\n )\n : undefined;\n}\n\nexport function compactRecorderSemanticForGeneration(\n semantic?: MidsceneRecorderSemantic,\n): MidsceneRecorderSemantic | undefined {\n if (!semantic) {\n return undefined;\n }\n\n return {\n source: semantic.source,\n status: semantic.status,\n confidence: semantic.confidence,\n elementDescription: compactRecorderSemanticText(\n semantic.elementDescription,\n ),\n replayInstruction: compactRecorderSemanticText(semantic.replayInstruction),\n actionSummary: compactRecorderSemanticText(semantic.actionSummary),\n error: compactRecorderSemanticError(semantic.error),\n ...(semantic.aiDescribe\n ? {\n aiDescribe: {\n verifyPrompt: semantic.aiDescribe.verifyPrompt,\n verifyPassed: semantic.aiDescribe.verifyPassed,\n deepLocate: semantic.aiDescribe.deepLocate,\n centerDistance: semantic.aiDescribe.centerDistance,\n expectedCenter: semantic.aiDescribe.expectedCenter,\n actualCenter: semantic.aiDescribe.actualCenter,\n },\n }\n : {}),\n };\n}\n\nexport const validateEvents = (events: ChromeRecordedEvent[]): void => {\n if (!events.length) {\n throw new Error('No events provided for test generation');\n }\n};\n\nexport const getScreenshotsForLLM = (\n events: ChromeRecordedEvent[],\n maxScreenshots = 1,\n): string[] => {\n return createMidsceneRecorderMarkdownScreenshotAssets(events, {\n baseDir: './screenshots',\n maxScreenshots,\n }).map((asset) => asset.dataUrl);\n};\n\nexport const filterEventsByType = (\n events: ChromeRecordedEvent[],\n): FilteredEvents => {\n return {\n navigationEvents: events.filter((event) => event.type === 'navigation'),\n clickEvents: events.filter((event) => event.type === 'click'),\n inputEvents: events.filter((event) => event.type === 'input'),\n scrollEvents: events.filter((event) => event.type === 'scroll'),\n };\n};\n\nexport const createEventCounts = (\n filteredEvents: FilteredEvents,\n totalEvents: number,\n): EventCounts => {\n return {\n navigation: filteredEvents.navigationEvents.length,\n click: filteredEvents.clickEvents.length,\n input: filteredEvents.inputEvents.length,\n scroll: filteredEvents.scrollEvents.length,\n total: totalEvents,\n };\n};\n\nexport const extractInputDescriptions = (\n inputEvents: ChromeRecordedEvent[],\n): InputDescription[] => {\n return inputEvents\n .map((event) => {\n const semantic = getMidsceneRecorderSemantic(event);\n return {\n description:\n cleanRecorderSemanticField(semantic?.elementDescription) || '',\n value: event.value || '',\n };\n })\n .filter((item) => item.description && item.value);\n};\n\nexport const processEventsForLLM = (\n events: ChromeRecordedEvent[],\n screenshotPathByEventHash: Map<string, string> = new Map(),\n): ProcessedEvent[] => {\n let inputIndex = 0;\n return events.map((event, index) => {\n const previousEvent = events[index - 1];\n const nextEvent = events[index + 1];\n const previousInput = events\n .slice(0, index)\n .reverse()\n .find((candidate) => candidate.type === 'input');\n const nextInput = events\n .slice(index + 1)\n .find((candidate) => candidate.type === 'input');\n const isInput = event.type === 'input';\n const inputSequenceIndex = isInput ? ++inputIndex : undefined;\n const hasNeighborInput = Boolean(previousInput || nextInput);\n const neighborInputValues = isInput\n ? [previousInput?.value, nextInput?.value].filter(\n (value): value is string => Boolean(value),\n )\n : undefined;\n const semantic = compactRecorderSemanticForGeneration(\n getMidsceneRecorderSemantic(event),\n );\n\n return {\n hashId: event.hashId,\n type: event.type,\n timestamp: event.timestamp,\n source: event.source,\n actionType: event.actionType,\n url: event.url,\n title: event.title,\n semantic,\n description: getMidsceneRecorderEventDescription(event),\n value: event.value,\n previousActionDescription: previousEvent\n ? getMidsceneRecorderEventDescription(previousEvent)\n : undefined,\n nextActionDescription: nextEvent\n ? getMidsceneRecorderEventDescription(nextEvent)\n : undefined,\n ...(isInput\n ? {\n typedText: event.value || '',\n inputIndex: inputSequenceIndex,\n isSequentialInput:\n previousEvent?.type === 'input' || nextEvent?.type === 'input',\n hasNeighborInput,\n previousInputDescription: previousInput\n ? getMidsceneRecorderEventDescription(previousInput)\n : undefined,\n neighborInputValues:\n neighborInputValues && neighborInputValues.length > 0\n ? neighborInputValues\n : undefined,\n }\n : {}),\n pageInfo: event.pageInfo,\n elementRect: event.elementRect,\n screenshotPath: screenshotPathByEventHash.get(event.hashId),\n };\n });\n};\n\nexport const prepareEventSummary = (\n events: ChromeRecordedEvent[],\n options: {\n testName?: string;\n maxScreenshots?: number;\n screenshotPathByEventHash?: Map<string, string>;\n } = {},\n): EventSummary => {\n const filteredEvents = filterEventsByType(events);\n const eventCounts = createEventCounts(filteredEvents, events.length);\n\n const startUrl =\n filteredEvents.navigationEvents.length > 0\n ? filteredEvents.navigationEvents[0].url || ''\n : '';\n\n const clickDescriptions = filteredEvents.clickEvents\n .map((event) => getMidsceneRecorderSemantic(event)?.elementDescription)\n .filter((desc): desc is string => Boolean(desc))\n .slice(0, 10);\n\n const inputDescriptions = extractInputDescriptions(\n filteredEvents.inputEvents,\n ).slice(0, 10);\n\n const urls = filteredEvents.navigationEvents\n .map((e) => e.url)\n .filter((url): url is string => Boolean(url))\n .slice(0, 5);\n\n const processedEvents = processEventsForLLM(\n events,\n options.screenshotPathByEventHash,\n );\n\n return {\n testName: options.testName || 'Automated test from recorded events',\n startUrl,\n eventCounts,\n urls,\n clickDescriptions,\n inputDescriptions,\n events: processedEvents,\n };\n};\n\nexport function prepareRecorderGenerationContext(\n input: RecorderGenerationInput,\n): RecorderGenerationContext {\n validateEvents(input.events);\n\n const maxScreenshots =\n input.maxScreenshots ?? DEFAULT_MIDSCENE_RECORDER_MARKDOWN_MAX_SCREENSHOTS;\n const screenshotAssets = createMidsceneRecorderMarkdownScreenshotAssets(\n input.events,\n {\n baseDir: './screenshots',\n maxScreenshots,\n },\n );\n const screenshotPathByEventHash = new Map(\n screenshotAssets.map((asset) => [asset.eventHashId, asset.relativePath]),\n );\n\n return {\n summary: prepareEventSummary(input.events, {\n testName: input.testName,\n screenshotPathByEventHash,\n }),\n screenshotAssets,\n };\n}\n\nexport const createMessageContent = (\n promptText: string,\n screenshots: string[] = [],\n includeScreenshots = true,\n) => {\n const messageContent: MessageContent[] = [\n {\n type: 'text',\n text: promptText,\n },\n ];\n\n if (includeScreenshots && screenshots.length > 0) {\n messageContent.unshift({\n type: 'text',\n text: 'Here are screenshots from the recording session to help you understand the context:',\n });\n\n screenshots.forEach((screenshot) => {\n messageContent.push({\n type: 'image',\n url: screenshot,\n });\n });\n }\n\n return messageContent;\n};\n"],"names":["MAX_RECORDER_GENERATION_SEMANTIC_TEXT_LENGTH","MAX_RECORDER_GENERATION_SEMANTIC_ERROR_LENGTH","cleanRecorderSemanticField","value","undefined","truncateRecorderGenerationText","maxLength","compactRecorderSemanticText","cleaned","compactRecorderSemanticError","compactRecorderSemanticForGeneration","semantic","validateEvents","events","Error","getScreenshotsForLLM","maxScreenshots","createMidsceneRecorderMarkdownScreenshotAssets","asset","filterEventsByType","event","createEventCounts","filteredEvents","totalEvents","extractInputDescriptions","inputEvents","getMidsceneRecorderSemantic","item","processEventsForLLM","screenshotPathByEventHash","Map","inputIndex","index","previousEvent","nextEvent","previousInput","candidate","nextInput","isInput","inputSequenceIndex","hasNeighborInput","Boolean","neighborInputValues","getMidsceneRecorderEventDescription","prepareEventSummary","options","eventCounts","startUrl","clickDescriptions","desc","inputDescriptions","urls","e","url","processedEvents","prepareRecorderGenerationContext","input","DEFAULT_MIDSCENE_RECORDER_MARKDOWN_MAX_SCREENSHOTS","screenshotAssets","createMessageContent","promptText","screenshots","includeScreenshots","messageContent","screenshot"],"mappings":";AAkEA,MAAMA,+CAA+C;AACrD,MAAMC,gDAAgD;AA+BtD,SAASC,2BAA2BC,KAAc;IAChD,OAAOA,OAAO,WAAW,+BAA+BC,SAAYD;AACtE;AAEA,SAASE,+BAA+BF,KAAa,EAAEG,SAAiB;IACtE,IAAIH,MAAM,MAAM,IAAIG,WAClB,OAAOH;IAET,OAAO,GAAGA,MAAM,KAAK,CAAC,GAAGG,WAAW,eAAe,EAAEH,MAAM,MAAM,GAAGG,UAAU,OAAO,CAAC;AACxF;AAEA,SAASC,4BAA4BJ,KAAc;IACjD,MAAMK,UAAUN,2BAA2BC;IAC3C,OAAOK,UACHH,+BACEG,SACAR,gDAEFI;AACN;AAEA,SAASK,6BAA6BN,KAAc;IAClD,OAAOA,QACHE,+BACEF,OACAF,iDAEFG;AACN;AAEO,SAASM,qCACdC,QAAmC;IAEnC,IAAI,CAACA,UACH;IAGF,OAAO;QACL,QAAQA,SAAS,MAAM;QACvB,QAAQA,SAAS,MAAM;QACvB,YAAYA,SAAS,UAAU;QAC/B,oBAAoBJ,4BAClBI,SAAS,kBAAkB;QAE7B,mBAAmBJ,4BAA4BI,SAAS,iBAAiB;QACzE,eAAeJ,4BAA4BI,SAAS,aAAa;QACjE,OAAOF,6BAA6BE,SAAS,KAAK;QAClD,GAAIA,SAAS,UAAU,GACnB;YACE,YAAY;gBACV,cAAcA,SAAS,UAAU,CAAC,YAAY;gBAC9C,cAAcA,SAAS,UAAU,CAAC,YAAY;gBAC9C,YAAYA,SAAS,UAAU,CAAC,UAAU;gBAC1C,gBAAgBA,SAAS,UAAU,CAAC,cAAc;gBAClD,gBAAgBA,SAAS,UAAU,CAAC,cAAc;gBAClD,cAAcA,SAAS,UAAU,CAAC,YAAY;YAChD;QACF,IACA,CAAC,CAAC;IACR;AACF;AAEO,MAAMC,iBAAiB,CAACC;IAC7B,IAAI,CAACA,OAAO,MAAM,EAChB,MAAM,IAAIC,MAAM;AAEpB;AAEO,MAAMC,uBAAuB,CAClCF,QACAG,iBAAiB,CAAC,GAEXC,+CAA+CJ,QAAQ;QAC5D,SAAS;QACTG;IACF,GAAG,GAAG,CAAC,CAACE,QAAUA,MAAM,OAAO;AAG1B,MAAMC,qBAAqB,CAChCN,SAEO;QACL,kBAAkBA,OAAO,MAAM,CAAC,CAACO,QAAUA,AAAe,iBAAfA,MAAM,IAAI;QACrD,aAAaP,OAAO,MAAM,CAAC,CAACO,QAAUA,AAAe,YAAfA,MAAM,IAAI;QAChD,aAAaP,OAAO,MAAM,CAAC,CAACO,QAAUA,AAAe,YAAfA,MAAM,IAAI;QAChD,cAAcP,OAAO,MAAM,CAAC,CAACO,QAAUA,AAAe,aAAfA,MAAM,IAAI;IACnD;AAGK,MAAMC,oBAAoB,CAC/BC,gBACAC,cAEO;QACL,YAAYD,eAAe,gBAAgB,CAAC,MAAM;QAClD,OAAOA,eAAe,WAAW,CAAC,MAAM;QACxC,OAAOA,eAAe,WAAW,CAAC,MAAM;QACxC,QAAQA,eAAe,YAAY,CAAC,MAAM;QAC1C,OAAOC;IACT;AAGK,MAAMC,2BAA2B,CACtCC,cAEOA,YACJ,GAAG,CAAC,CAACL;QACJ,MAAMT,WAAWe,4BAA4BN;QAC7C,OAAO;YACL,aACElB,2BAA2BS,UAAU,uBAAuB;YAC9D,OAAOS,MAAM,KAAK,IAAI;QACxB;IACF,GACC,MAAM,CAAC,CAACO,OAASA,KAAK,WAAW,IAAIA,KAAK,KAAK;AAG7C,MAAMC,sBAAsB,CACjCf,QACAgB,4BAAiD,IAAIC,KAAK;IAE1D,IAAIC,aAAa;IACjB,OAAOlB,OAAO,GAAG,CAAC,CAACO,OAAOY;QACxB,MAAMC,gBAAgBpB,MAAM,CAACmB,QAAQ,EAAE;QACvC,MAAME,YAAYrB,MAAM,CAACmB,QAAQ,EAAE;QACnC,MAAMG,gBAAgBtB,OACnB,KAAK,CAAC,GAAGmB,OACT,OAAO,GACP,IAAI,CAAC,CAACI,YAAcA,AAAmB,YAAnBA,UAAU,IAAI;QACrC,MAAMC,YAAYxB,OACf,KAAK,CAACmB,QAAQ,GACd,IAAI,CAAC,CAACI,YAAcA,AAAmB,YAAnBA,UAAU,IAAI;QACrC,MAAME,UAAUlB,AAAe,YAAfA,MAAM,IAAI;QAC1B,MAAMmB,qBAAqBD,UAAU,EAAEP,aAAa3B;QACpD,MAAMoC,mBAAmBC,QAAQN,iBAAiBE;QAClD,MAAMK,sBAAsBJ,UACxB;YAACH,eAAe;YAAOE,WAAW;SAAM,CAAC,MAAM,CAC7C,CAAClC,QAA2BsC,QAAQtC,UAEtCC;QACJ,MAAMO,WAAWD,qCACfgB,4BAA4BN;QAG9B,OAAO;YACL,QAAQA,MAAM,MAAM;YACpB,MAAMA,MAAM,IAAI;YAChB,WAAWA,MAAM,SAAS;YAC1B,QAAQA,MAAM,MAAM;YACpB,YAAYA,MAAM,UAAU;YAC5B,KAAKA,MAAM,GAAG;YACd,OAAOA,MAAM,KAAK;YAClBT;YACA,aAAagC,oCAAoCvB;YACjD,OAAOA,MAAM,KAAK;YAClB,2BAA2Ba,gBACvBU,oCAAoCV,iBACpC7B;YACJ,uBAAuB8B,YACnBS,oCAAoCT,aACpC9B;YACJ,GAAIkC,UACA;gBACE,WAAWlB,MAAM,KAAK,IAAI;gBAC1B,YAAYmB;gBACZ,mBACEN,eAAe,SAAS,WAAWC,WAAW,SAAS;gBACzDM;gBACA,0BAA0BL,gBACtBQ,oCAAoCR,iBACpC/B;gBACJ,qBACEsC,uBAAuBA,oBAAoB,MAAM,GAAG,IAChDA,sBACAtC;YACR,IACA,CAAC,CAAC;YACN,UAAUgB,MAAM,QAAQ;YACxB,aAAaA,MAAM,WAAW;YAC9B,gBAAgBS,0BAA0B,GAAG,CAACT,MAAM,MAAM;QAC5D;IACF;AACF;AAEO,MAAMwB,sBAAsB,CACjC/B,QACAgC,UAII,CAAC,CAAC;IAEN,MAAMvB,iBAAiBH,mBAAmBN;IAC1C,MAAMiC,cAAczB,kBAAkBC,gBAAgBT,OAAO,MAAM;IAEnE,MAAMkC,WACJzB,eAAe,gBAAgB,CAAC,MAAM,GAAG,IACrCA,eAAe,gBAAgB,CAAC,EAAE,CAAC,GAAG,IAAI,KAC1C;IAEN,MAAM0B,oBAAoB1B,eAAe,WAAW,CACjD,GAAG,CAAC,CAACF,QAAUM,4BAA4BN,QAAQ,oBACnD,MAAM,CAAC,CAAC6B,OAAyBR,QAAQQ,OACzC,KAAK,CAAC,GAAG;IAEZ,MAAMC,oBAAoB1B,yBACxBF,eAAe,WAAW,EAC1B,KAAK,CAAC,GAAG;IAEX,MAAM6B,OAAO7B,eAAe,gBAAgB,CACzC,GAAG,CAAC,CAAC8B,IAAMA,EAAE,GAAG,EAChB,MAAM,CAAC,CAACC,MAAuBZ,QAAQY,MACvC,KAAK,CAAC,GAAG;IAEZ,MAAMC,kBAAkB1B,oBACtBf,QACAgC,QAAQ,yBAAyB;IAGnC,OAAO;QACL,UAAUA,QAAQ,QAAQ,IAAI;QAC9BE;QACAD;QACAK;QACAH;QACAE;QACA,QAAQI;IACV;AACF;AAEO,SAASC,iCACdC,KAA8B;IAE9B5C,eAAe4C,MAAM,MAAM;IAE3B,MAAMxC,iBACJwC,MAAM,cAAc,IAAIC;IAC1B,MAAMC,mBAAmBzC,+CACvBuC,MAAM,MAAM,EACZ;QACE,SAAS;QACTxC;IACF;IAEF,MAAMa,4BAA4B,IAAIC,IACpC4B,iBAAiB,GAAG,CAAC,CAACxC,QAAU;YAACA,MAAM,WAAW;YAAEA,MAAM,YAAY;SAAC;IAGzE,OAAO;QACL,SAAS0B,oBAAoBY,MAAM,MAAM,EAAE;YACzC,UAAUA,MAAM,QAAQ;YACxB3B;QACF;QACA6B;IACF;AACF;AAEO,MAAMC,uBAAuB,CAClCC,YACAC,cAAwB,EAAE,EAC1BC,qBAAqB,IAAI;IAEzB,MAAMC,iBAAmC;QACvC;YACE,MAAM;YACN,MAAMH;QACR;KACD;IAED,IAAIE,sBAAsBD,YAAY,MAAM,GAAG,GAAG;QAChDE,eAAe,OAAO,CAAC;YACrB,MAAM;YACN,MAAM;QACR;QAEAF,YAAY,OAAO,CAAC,CAACG;YACnBD,eAAe,IAAI,CAAC;gBAClB,MAAM;gBACN,KAAKC;YACP;QACF;IACF;IAEA,OAAOD;AACT"}
@@ -75,7 +75,7 @@ function getPromptShape(prompt) {
75
75
  if (Array.isArray(content)) {
76
76
  for (const part of content)if ('object' == typeof part && part && 'type' in part) {
77
77
  if ('text' === part.type && 'text' in part) textChars += String(part.text).length;
78
- if ('image_url' === part.type) imageCount += 1;
78
+ if ('image' === part.type) imageCount += 1;
79
79
  }
80
80
  }
81
81
  }
@@ -218,10 +218,8 @@ Important: Return ONLY raw Markdown. Do NOT wrap the response in markdown code b
218
218
  text: `${screenshotRef} for event #${asset.eventIndex + 1}`
219
219
  });
220
220
  content.push({
221
- type: 'image_url',
222
- image_url: {
223
- url: asset.dataUrl
224
- }
221
+ type: 'image',
222
+ url: asset.dataUrl
225
223
  });
226
224
  }
227
225
  return [