@godscene/core 1.7.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (189) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +9 -0
  3. package/dist/es/agent/agent.mjs +767 -0
  4. package/dist/es/agent/common.mjs +0 -0
  5. package/dist/es/agent/execution-session.mjs +39 -0
  6. package/dist/es/agent/index.mjs +6 -0
  7. package/dist/es/agent/task-builder.mjs +343 -0
  8. package/dist/es/agent/task-cache.mjs +212 -0
  9. package/dist/es/agent/tasks.mjs +428 -0
  10. package/dist/es/agent/ui-utils.mjs +101 -0
  11. package/dist/es/agent/utils.mjs +167 -0
  12. package/dist/es/ai-model/auto-glm/actions.mjs +237 -0
  13. package/dist/es/ai-model/auto-glm/index.mjs +6 -0
  14. package/dist/es/ai-model/auto-glm/parser.mjs +237 -0
  15. package/dist/es/ai-model/auto-glm/planning.mjs +69 -0
  16. package/dist/es/ai-model/auto-glm/prompt.mjs +220 -0
  17. package/dist/es/ai-model/auto-glm/util.mjs +7 -0
  18. package/dist/es/ai-model/connectivity.mjs +136 -0
  19. package/dist/es/ai-model/conversation-history.mjs +193 -0
  20. package/dist/es/ai-model/index.mjs +12 -0
  21. package/dist/es/ai-model/inspect.mjs +395 -0
  22. package/dist/es/ai-model/llm-planning.mjs +231 -0
  23. package/dist/es/ai-model/prompt/common.mjs +5 -0
  24. package/dist/es/ai-model/prompt/describe.mjs +64 -0
  25. package/dist/es/ai-model/prompt/extraction.mjs +129 -0
  26. package/dist/es/ai-model/prompt/llm-locator.mjs +49 -0
  27. package/dist/es/ai-model/prompt/llm-planning.mjs +584 -0
  28. package/dist/es/ai-model/prompt/llm-section-locator.mjs +42 -0
  29. package/dist/es/ai-model/prompt/order-sensitive-judge.mjs +33 -0
  30. package/dist/es/ai-model/prompt/playwright-generator.mjs +115 -0
  31. package/dist/es/ai-model/prompt/ui-tars-planning.mjs +34 -0
  32. package/dist/es/ai-model/prompt/util.mjs +57 -0
  33. package/dist/es/ai-model/prompt/yaml-generator.mjs +201 -0
  34. package/dist/es/ai-model/service-caller/codex-app-server.mjs +573 -0
  35. package/dist/es/ai-model/service-caller/image-detail.mjs +4 -0
  36. package/dist/es/ai-model/service-caller/index.mjs +648 -0
  37. package/dist/es/ai-model/service-caller/request-timeout.mjs +47 -0
  38. package/dist/es/ai-model/ui-tars-planning.mjs +247 -0
  39. package/dist/es/common.mjs +382 -0
  40. package/dist/es/device/device-options.mjs +0 -0
  41. package/dist/es/device/index.mjs +340 -0
  42. package/dist/es/dump/html-utils.mjs +290 -0
  43. package/dist/es/dump/index.mjs +3 -0
  44. package/dist/es/dump/screenshot-restoration.mjs +30 -0
  45. package/dist/es/dump/screenshot-store.mjs +125 -0
  46. package/dist/es/index.mjs +17 -0
  47. package/dist/es/report-cli.mjs +149 -0
  48. package/dist/es/report-generator.mjs +203 -0
  49. package/dist/es/report-markdown.mjs +216 -0
  50. package/dist/es/report.mjs +287 -0
  51. package/dist/es/screenshot-item.mjs +120 -0
  52. package/dist/es/service/index.mjs +272 -0
  53. package/dist/es/service/utils.mjs +13 -0
  54. package/dist/es/skill/index.mjs +35 -0
  55. package/dist/es/task-runner.mjs +261 -0
  56. package/dist/es/task-timing.mjs +10 -0
  57. package/dist/es/tree.mjs +11 -0
  58. package/dist/es/types.mjs +202 -0
  59. package/dist/es/utils.mjs +232 -0
  60. package/dist/es/yaml/builder.mjs +11 -0
  61. package/dist/es/yaml/index.mjs +4 -0
  62. package/dist/es/yaml/player.mjs +425 -0
  63. package/dist/es/yaml/utils.mjs +100 -0
  64. package/dist/es/yaml.mjs +0 -0
  65. package/dist/lib/agent/agent.js +815 -0
  66. package/dist/lib/agent/common.js +5 -0
  67. package/dist/lib/agent/execution-session.js +73 -0
  68. package/dist/lib/agent/index.js +76 -0
  69. package/dist/lib/agent/task-builder.js +380 -0
  70. package/dist/lib/agent/task-cache.js +264 -0
  71. package/dist/lib/agent/tasks.js +471 -0
  72. package/dist/lib/agent/ui-utils.js +153 -0
  73. package/dist/lib/agent/utils.js +238 -0
  74. package/dist/lib/ai-model/auto-glm/actions.js +271 -0
  75. package/dist/lib/ai-model/auto-glm/index.js +64 -0
  76. package/dist/lib/ai-model/auto-glm/parser.js +280 -0
  77. package/dist/lib/ai-model/auto-glm/planning.js +103 -0
  78. package/dist/lib/ai-model/auto-glm/prompt.js +257 -0
  79. package/dist/lib/ai-model/auto-glm/util.js +44 -0
  80. package/dist/lib/ai-model/connectivity.js +180 -0
  81. package/dist/lib/ai-model/conversation-history.js +227 -0
  82. package/dist/lib/ai-model/index.js +127 -0
  83. package/dist/lib/ai-model/inspect.js +441 -0
  84. package/dist/lib/ai-model/llm-planning.js +268 -0
  85. package/dist/lib/ai-model/prompt/common.js +39 -0
  86. package/dist/lib/ai-model/prompt/describe.js +98 -0
  87. package/dist/lib/ai-model/prompt/extraction.js +169 -0
  88. package/dist/lib/ai-model/prompt/llm-locator.js +86 -0
  89. package/dist/lib/ai-model/prompt/llm-planning.js +621 -0
  90. package/dist/lib/ai-model/prompt/llm-section-locator.js +79 -0
  91. package/dist/lib/ai-model/prompt/order-sensitive-judge.js +70 -0
  92. package/dist/lib/ai-model/prompt/playwright-generator.js +176 -0
  93. package/dist/lib/ai-model/prompt/ui-tars-planning.js +71 -0
  94. package/dist/lib/ai-model/prompt/util.js +103 -0
  95. package/dist/lib/ai-model/prompt/yaml-generator.js +262 -0
  96. package/dist/lib/ai-model/service-caller/codex-app-server.js +622 -0
  97. package/dist/lib/ai-model/service-caller/image-detail.js +38 -0
  98. package/dist/lib/ai-model/service-caller/index.js +716 -0
  99. package/dist/lib/ai-model/service-caller/request-timeout.js +93 -0
  100. package/dist/lib/ai-model/ui-tars-planning.js +281 -0
  101. package/dist/lib/common.js +491 -0
  102. package/dist/lib/device/device-options.js +18 -0
  103. package/dist/lib/device/index.js +467 -0
  104. package/dist/lib/dump/html-utils.js +366 -0
  105. package/dist/lib/dump/index.js +58 -0
  106. package/dist/lib/dump/screenshot-restoration.js +64 -0
  107. package/dist/lib/dump/screenshot-store.js +165 -0
  108. package/dist/lib/index.js +184 -0
  109. package/dist/lib/report-cli.js +189 -0
  110. package/dist/lib/report-generator.js +244 -0
  111. package/dist/lib/report-markdown.js +253 -0
  112. package/dist/lib/report.js +333 -0
  113. package/dist/lib/screenshot-item.js +154 -0
  114. package/dist/lib/service/index.js +306 -0
  115. package/dist/lib/service/utils.js +47 -0
  116. package/dist/lib/skill/index.js +69 -0
  117. package/dist/lib/task-runner.js +298 -0
  118. package/dist/lib/task-timing.js +44 -0
  119. package/dist/lib/tree.js +51 -0
  120. package/dist/lib/types.js +298 -0
  121. package/dist/lib/utils.js +314 -0
  122. package/dist/lib/yaml/builder.js +55 -0
  123. package/dist/lib/yaml/index.js +79 -0
  124. package/dist/lib/yaml/player.js +459 -0
  125. package/dist/lib/yaml/utils.js +153 -0
  126. package/dist/lib/yaml.js +18 -0
  127. package/dist/types/agent/agent.d.ts +220 -0
  128. package/dist/types/agent/common.d.ts +0 -0
  129. package/dist/types/agent/execution-session.d.ts +36 -0
  130. package/dist/types/agent/index.d.ts +9 -0
  131. package/dist/types/agent/task-builder.d.ts +34 -0
  132. package/dist/types/agent/task-cache.d.ts +49 -0
  133. package/dist/types/agent/tasks.d.ts +70 -0
  134. package/dist/types/agent/ui-utils.d.ts +14 -0
  135. package/dist/types/agent/utils.d.ts +25 -0
  136. package/dist/types/ai-model/auto-glm/actions.d.ts +78 -0
  137. package/dist/types/ai-model/auto-glm/index.d.ts +6 -0
  138. package/dist/types/ai-model/auto-glm/parser.d.ts +18 -0
  139. package/dist/types/ai-model/auto-glm/planning.d.ts +12 -0
  140. package/dist/types/ai-model/auto-glm/prompt.d.ts +27 -0
  141. package/dist/types/ai-model/auto-glm/util.d.ts +13 -0
  142. package/dist/types/ai-model/connectivity.d.ts +20 -0
  143. package/dist/types/ai-model/conversation-history.d.ts +105 -0
  144. package/dist/types/ai-model/index.d.ts +16 -0
  145. package/dist/types/ai-model/inspect.d.ts +67 -0
  146. package/dist/types/ai-model/llm-planning.d.ts +19 -0
  147. package/dist/types/ai-model/prompt/common.d.ts +2 -0
  148. package/dist/types/ai-model/prompt/describe.d.ts +1 -0
  149. package/dist/types/ai-model/prompt/extraction.d.ts +7 -0
  150. package/dist/types/ai-model/prompt/llm-locator.d.ts +3 -0
  151. package/dist/types/ai-model/prompt/llm-planning.d.ts +10 -0
  152. package/dist/types/ai-model/prompt/llm-section-locator.d.ts +3 -0
  153. package/dist/types/ai-model/prompt/order-sensitive-judge.d.ts +2 -0
  154. package/dist/types/ai-model/prompt/playwright-generator.d.ts +26 -0
  155. package/dist/types/ai-model/prompt/ui-tars-planning.d.ts +2 -0
  156. package/dist/types/ai-model/prompt/util.d.ts +33 -0
  157. package/dist/types/ai-model/prompt/yaml-generator.d.ts +102 -0
  158. package/dist/types/ai-model/service-caller/codex-app-server.d.ts +42 -0
  159. package/dist/types/ai-model/service-caller/image-detail.d.ts +2 -0
  160. package/dist/types/ai-model/service-caller/index.d.ts +60 -0
  161. package/dist/types/ai-model/service-caller/request-timeout.d.ts +32 -0
  162. package/dist/types/ai-model/ui-tars-planning.d.ts +72 -0
  163. package/dist/types/common.d.ts +288 -0
  164. package/dist/types/device/device-options.d.ts +155 -0
  165. package/dist/types/device/index.d.ts +2565 -0
  166. package/dist/types/dump/html-utils.d.ts +75 -0
  167. package/dist/types/dump/index.d.ts +5 -0
  168. package/dist/types/dump/screenshot-restoration.d.ts +8 -0
  169. package/dist/types/dump/screenshot-store.d.ts +49 -0
  170. package/dist/types/index.d.ts +21 -0
  171. package/dist/types/report-cli.d.ts +36 -0
  172. package/dist/types/report-generator.d.ts +88 -0
  173. package/dist/types/report-markdown.d.ts +24 -0
  174. package/dist/types/report.d.ts +52 -0
  175. package/dist/types/screenshot-item.d.ts +67 -0
  176. package/dist/types/service/index.d.ts +24 -0
  177. package/dist/types/service/utils.d.ts +2 -0
  178. package/dist/types/skill/index.d.ts +25 -0
  179. package/dist/types/task-runner.d.ts +50 -0
  180. package/dist/types/task-timing.d.ts +8 -0
  181. package/dist/types/tree.d.ts +4 -0
  182. package/dist/types/types.d.ts +684 -0
  183. package/dist/types/utils.d.ts +45 -0
  184. package/dist/types/yaml/builder.d.ts +2 -0
  185. package/dist/types/yaml/index.d.ts +4 -0
  186. package/dist/types/yaml/player.d.ts +34 -0
  187. package/dist/types/yaml/utils.d.ts +9 -0
  188. package/dist/types/yaml.d.ts +215 -0
  189. package/package.json +130 -0
@@ -0,0 +1,425 @@
1
+ import { existsSync, mkdirSync, writeFileSync } from "node:fs";
2
+ import { basename, dirname, join, resolve as external_node_path_resolve } from "node:path";
3
+ import { assert, ifInBrowser, ifInWorker } from "@godscene/shared/utils";
4
+ import { z } from "zod";
5
+ import { getMidsceneRunSubDir } from "@godscene/shared/common";
6
+ import { getDebug } from "@godscene/shared/logger";
7
+ import { buildDetailedLocateParam, buildDetailedLocateParamAndRestParams } from "./utils.mjs";
8
+ function _define_property(obj, key, value) {
9
+ if (key in obj) Object.defineProperty(obj, key, {
10
+ value: value,
11
+ enumerable: true,
12
+ configurable: true,
13
+ writable: true
14
+ });
15
+ else obj[key] = value;
16
+ return obj;
17
+ }
18
+ const debug = getDebug('yaml-player');
19
+ const aiTaskHandlerMap = {
20
+ aiQuery: 'aiQuery',
21
+ aiNumber: 'aiNumber',
22
+ aiString: 'aiString',
23
+ aiBoolean: 'aiBoolean',
24
+ aiAsk: 'aiAsk',
25
+ aiLocate: 'aiLocate'
26
+ };
27
+ const isStringParamSchema = (schema)=>{
28
+ if (!schema) return false;
29
+ const schemaDef = schema?._def;
30
+ if (!schemaDef?.typeName) return false;
31
+ switch(schemaDef.typeName){
32
+ case z.ZodFirstPartyTypeKind.ZodString:
33
+ case z.ZodFirstPartyTypeKind.ZodEnum:
34
+ case z.ZodFirstPartyTypeKind.ZodNativeEnum:
35
+ return true;
36
+ case z.ZodFirstPartyTypeKind.ZodLiteral:
37
+ return 'string' == typeof schemaDef.value;
38
+ case z.ZodFirstPartyTypeKind.ZodOptional:
39
+ case z.ZodFirstPartyTypeKind.ZodNullable:
40
+ case z.ZodFirstPartyTypeKind.ZodDefault:
41
+ return isStringParamSchema(schemaDef.innerType);
42
+ case z.ZodFirstPartyTypeKind.ZodEffects:
43
+ return isStringParamSchema(schemaDef.schema);
44
+ case z.ZodFirstPartyTypeKind.ZodPipeline:
45
+ return isStringParamSchema(schemaDef.out);
46
+ case z.ZodFirstPartyTypeKind.ZodUnion:
47
+ {
48
+ const options = schemaDef.options;
49
+ return Array.isArray(options) ? options.every((option)=>isStringParamSchema(option)) : false;
50
+ }
51
+ default:
52
+ return false;
53
+ }
54
+ };
55
+ const buildShortcutActionParam = (actionName, interfaceAlias, value)=>{
56
+ if ('Launch' === actionName || 'launch' === interfaceAlias) return {
57
+ uri: value
58
+ };
59
+ if ('Terminate' === actionName || 'terminate' === interfaceAlias) return {
60
+ uri: value
61
+ };
62
+ if ('RunAdbShell' === actionName || 'runAdbShell' === interfaceAlias) return {
63
+ command: value
64
+ };
65
+ };
66
+ class ScriptPlayer {
67
+ setResult(key, value) {
68
+ const keyToUse = key || this.unnamedResultIndex++;
69
+ if (this.result[keyToUse]) console.warn(`result key ${keyToUse} already exists, will overwrite`);
70
+ this.result[keyToUse] = value;
71
+ return this.flushResult();
72
+ }
73
+ setPlayerStatus(status, error) {
74
+ this.status = status;
75
+ this.errorInSetup = error;
76
+ }
77
+ notifyCurrentTaskStatusChange(taskIndex) {
78
+ const taskIndexToNotify = 'number' == typeof taskIndex ? taskIndex : this.currentTaskIndex;
79
+ if ('number' != typeof taskIndexToNotify) return;
80
+ const taskStatus = this.taskStatusList[taskIndexToNotify];
81
+ if (this.onTaskStatusChange) this.onTaskStatusChange(taskStatus);
82
+ }
83
+ async setTaskStatus(index, statusValue, error) {
84
+ this.taskStatusList[index].status = statusValue;
85
+ if (error) this.taskStatusList[index].error = error;
86
+ this.notifyCurrentTaskStatusChange(index);
87
+ }
88
+ setTaskIndex(taskIndex) {
89
+ this.currentTaskIndex = taskIndex;
90
+ }
91
+ flushResult() {
92
+ if (this.output) {
93
+ const output = external_node_path_resolve(process.cwd(), this.output);
94
+ const outputDir = dirname(output);
95
+ if (!existsSync(outputDir)) mkdirSync(outputDir, {
96
+ recursive: true
97
+ });
98
+ writeFileSync(output, JSON.stringify(this.result || {}, void 0, 2));
99
+ }
100
+ }
101
+ flushUnstableLogContent() {
102
+ if (this.unstableLogContent) {
103
+ const content = this.interfaceAgent?._unstableLogContent();
104
+ const filePath = external_node_path_resolve(process.cwd(), this.unstableLogContent);
105
+ const outputDir = dirname(filePath);
106
+ if (!existsSync(outputDir)) mkdirSync(outputDir, {
107
+ recursive: true
108
+ });
109
+ writeFileSync(filePath, JSON.stringify(content, null, 2));
110
+ }
111
+ }
112
+ async playTask(taskStatus, agent) {
113
+ const { flow } = taskStatus;
114
+ assert(flow, 'missing flow in task');
115
+ for(const flowItemIndex in flow){
116
+ const currentStep = Number.parseInt(flowItemIndex, 10);
117
+ taskStatus.currentStep = currentStep;
118
+ const flowItem = flow[flowItemIndex];
119
+ if ('Finalize' in flowItem) continue;
120
+ debug(`playing step ${flowItemIndex}, flowItem=${JSON.stringify(flowItem)}`);
121
+ const simpleAIKey = Object.keys(aiTaskHandlerMap).find((key)=>Object.prototype.hasOwnProperty.call(flowItem, key));
122
+ if ('aiAct' in flowItem || 'aiAction' in flowItem || 'ai' in flowItem) {
123
+ const actionTask = flowItem;
124
+ const { aiAct, aiAction, ai, ...actionOptions } = actionTask;
125
+ const prompt = aiAct || aiAction || ai;
126
+ assert(prompt, 'missing prompt for ai (aiAct)');
127
+ await agent.aiAct(prompt, actionOptions);
128
+ } else if ('aiAssert' in flowItem) {
129
+ const assertTask = flowItem;
130
+ const { aiAssert: prompt, errorMessage: msg, name, ...restOpts } = assertTask;
131
+ assert(prompt, 'missing prompt for aiAssert');
132
+ const { pass, thought, message } = await agent.aiAssert(prompt, msg, {
133
+ ...restOpts,
134
+ keepRawResponse: true
135
+ }) || {};
136
+ this.setResult(name, {
137
+ pass,
138
+ thought,
139
+ message
140
+ });
141
+ if (!pass) throw new Error(message);
142
+ } else if (simpleAIKey) {
143
+ const { [simpleAIKey]: prompt, name, ...options } = flowItem;
144
+ assert(prompt, `missing prompt for ${simpleAIKey}`);
145
+ const agentMethod = agent[aiTaskHandlerMap[simpleAIKey]];
146
+ assert('function' == typeof agentMethod, `missing agent method for ${simpleAIKey}`);
147
+ const aiResult = await agentMethod.call(agent, prompt, options);
148
+ this.setResult(name, aiResult);
149
+ } else if ('aiWaitFor' in flowItem) {
150
+ const waitForTask = flowItem;
151
+ const { aiWaitFor, timeout, ...restWaitForOpts } = waitForTask;
152
+ const prompt = aiWaitFor;
153
+ assert(prompt, 'missing prompt for aiWaitFor');
154
+ const waitForOptions = {
155
+ ...restWaitForOpts,
156
+ ...void 0 !== timeout ? {
157
+ timeout,
158
+ timeoutMs: timeout
159
+ } : {}
160
+ };
161
+ await agent.aiWaitFor(prompt, waitForOptions);
162
+ } else if ('sleep' in flowItem) {
163
+ const sleepTask = flowItem;
164
+ const ms = sleepTask.sleep;
165
+ let msNumber = ms;
166
+ if ('string' == typeof ms) msNumber = Number.parseInt(ms, 10);
167
+ assert(msNumber && msNumber > 0, `ms for sleep must be greater than 0, but got ${ms}`);
168
+ await new Promise((resolve)=>setTimeout(resolve, msNumber));
169
+ } else if ("javascript" in flowItem) {
170
+ const evaluateJavaScriptTask = flowItem;
171
+ const result = await agent.evaluateJavaScript(evaluateJavaScriptTask.javascript);
172
+ this.setResult(evaluateJavaScriptTask.name, result);
173
+ } else if ('logScreenshot' in flowItem || 'recordToReport' in flowItem) {
174
+ const recordTask = flowItem;
175
+ const title = recordTask.recordToReport ?? recordTask.logScreenshot ?? 'untitled';
176
+ const content = recordTask.content || '';
177
+ await agent.recordToReport(title, {
178
+ content
179
+ });
180
+ } else if ('aiInput' in flowItem) {
181
+ const { aiInput, value: rawValue, ...inputTask } = flowItem;
182
+ let locatePrompt;
183
+ let value;
184
+ if (inputTask.locate) {
185
+ value = aiInput || rawValue;
186
+ locatePrompt = inputTask.locate;
187
+ } else {
188
+ locatePrompt = aiInput || '';
189
+ value = rawValue;
190
+ }
191
+ await agent.callActionInActionSpace('Input', {
192
+ ...inputTask,
193
+ ...void 0 !== value ? {
194
+ value: String(value)
195
+ } : {},
196
+ ...locatePrompt ? {
197
+ locate: buildDetailedLocateParam(locatePrompt, inputTask)
198
+ } : {}
199
+ });
200
+ } else if ('aiKeyboardPress' in flowItem) {
201
+ const { aiKeyboardPress, ...keyboardPressTask } = flowItem;
202
+ let locatePrompt;
203
+ let keyName;
204
+ if (keyboardPressTask.locate) {
205
+ keyName = aiKeyboardPress;
206
+ locatePrompt = keyboardPressTask.locate;
207
+ } else if (keyboardPressTask.keyName) {
208
+ keyName = keyboardPressTask.keyName;
209
+ locatePrompt = aiKeyboardPress;
210
+ } else keyName = aiKeyboardPress;
211
+ await agent.callActionInActionSpace('KeyboardPress', {
212
+ ...keyboardPressTask,
213
+ ...keyName ? {
214
+ keyName
215
+ } : {},
216
+ ...locatePrompt ? {
217
+ locate: buildDetailedLocateParam(locatePrompt, keyboardPressTask)
218
+ } : {}
219
+ });
220
+ } else if ('aiScroll' in flowItem) {
221
+ const { aiScroll, ...scrollTask } = flowItem;
222
+ const { locate, ...scrollOptions } = scrollTask;
223
+ const locatePrompt = locate ?? aiScroll ?? void 0;
224
+ await agent.aiScroll(locatePrompt, scrollOptions);
225
+ } else if ('aiTap' in flowItem) {
226
+ const { aiTap, prompt, locate, ...tapOptions } = flowItem;
227
+ let locatePrompt;
228
+ let opts = tapOptions;
229
+ const locateObj = locate ?? ('object' == typeof aiTap && null !== aiTap ? aiTap.locate : void 0);
230
+ if ('string' == typeof aiTap && aiTap) locatePrompt = aiTap;
231
+ else if ('object' == typeof locateObj && locateObj?.prompt) {
232
+ const { prompt: lp, ...locateOpts } = locateObj;
233
+ locatePrompt = lp;
234
+ opts = {
235
+ ...locateOpts,
236
+ ...tapOptions
237
+ };
238
+ } else locatePrompt = aiTap?.prompt || prompt || locateObj;
239
+ assert(locatePrompt, 'missing prompt for aiTap');
240
+ await agent.aiTap(locatePrompt, opts);
241
+ } else {
242
+ const actionSpace = this.actionSpace;
243
+ let locatePromptShortcut;
244
+ let actionParamForMatchedAction;
245
+ const matchedAction = actionSpace.find((action)=>{
246
+ const actionInterfaceAlias = action.interfaceAlias;
247
+ if (actionInterfaceAlias && Object.prototype.hasOwnProperty.call(flowItem, actionInterfaceAlias)) {
248
+ actionParamForMatchedAction = flowItem[actionInterfaceAlias];
249
+ if ('string' == typeof actionParamForMatchedAction) locatePromptShortcut = actionParamForMatchedAction;
250
+ return true;
251
+ }
252
+ const keyOfActionInActionSpace = action.name;
253
+ if (Object.prototype.hasOwnProperty.call(flowItem, keyOfActionInActionSpace)) {
254
+ actionParamForMatchedAction = flowItem[keyOfActionInActionSpace];
255
+ if ('string' == typeof actionParamForMatchedAction) locatePromptShortcut = actionParamForMatchedAction;
256
+ return true;
257
+ }
258
+ return false;
259
+ });
260
+ assert(matchedAction, `unknown flowItem in yaml: ${JSON.stringify(flowItem)}`);
261
+ const schemaIsStringParam = isStringParamSchema(matchedAction.paramSchema);
262
+ let stringParamToCall;
263
+ const specialActionParamToCall = 'string' == typeof actionParamForMatchedAction ? buildShortcutActionParam(matchedAction.name, matchedAction.interfaceAlias, actionParamForMatchedAction) : void 0;
264
+ if (specialActionParamToCall) {
265
+ debug(`matchedAction: ${matchedAction.name}`, `flowParams: ${JSON.stringify(specialActionParamToCall)}`);
266
+ const result = await agent.callActionInActionSpace(matchedAction.name, specialActionParamToCall);
267
+ const resultName = flowItem.name;
268
+ if (void 0 !== result) this.setResult(resultName, result);
269
+ } else if ('string' == typeof actionParamForMatchedAction && schemaIsStringParam) {
270
+ if (matchedAction.paramSchema) {
271
+ const parseResult = matchedAction.paramSchema.safeParse(actionParamForMatchedAction);
272
+ if (parseResult.success && 'string' == typeof parseResult.data) stringParamToCall = parseResult.data;
273
+ else if (!parseResult.success) {
274
+ debug(`parse failed for action ${matchedAction.name} with string param`, parseResult.error);
275
+ stringParamToCall = actionParamForMatchedAction;
276
+ }
277
+ } else stringParamToCall = actionParamForMatchedAction;
278
+ if (void 0 !== stringParamToCall) {
279
+ debug(`matchedAction: ${matchedAction.name}`, `flowParams: ${JSON.stringify(stringParamToCall)}`);
280
+ const result = await agent.callActionInActionSpace(matchedAction.name, stringParamToCall);
281
+ const resultName = flowItem.name;
282
+ if (void 0 !== result) this.setResult(resultName, result);
283
+ }
284
+ } else {
285
+ const sourceForParams = locatePromptShortcut && 'string' == typeof actionParamForMatchedAction ? {
286
+ ...flowItem,
287
+ prompt: locatePromptShortcut
288
+ } : 'object' == typeof actionParamForMatchedAction && null !== actionParamForMatchedAction ? actionParamForMatchedAction : flowItem;
289
+ const { locateParam, restParams } = buildDetailedLocateParamAndRestParams(locatePromptShortcut || '', sourceForParams, [
290
+ matchedAction.name,
291
+ matchedAction.interfaceAlias || '_never_mind_'
292
+ ]);
293
+ const flowParams = {
294
+ ...restParams,
295
+ locate: locateParam
296
+ };
297
+ debug(`matchedAction: ${matchedAction.name}`, `flowParams: ${JSON.stringify(flowParams, null, 2)}`);
298
+ const result = await agent.callActionInActionSpace(matchedAction.name, flowParams);
299
+ const resultName = flowItem.name;
300
+ if (void 0 !== result) this.setResult(resultName, result);
301
+ }
302
+ }
303
+ }
304
+ this.reportFile = agent.reportFile;
305
+ await this.flushUnstableLogContent();
306
+ }
307
+ async run() {
308
+ const { target, web, android, ios, computer, tasks } = this.script;
309
+ const webEnv = web || target;
310
+ const androidEnv = android;
311
+ const iosEnv = ios;
312
+ const computerEnv = computer;
313
+ const platform = webEnv || androidEnv || iosEnv || computerEnv;
314
+ this.setPlayerStatus('running');
315
+ let agent = null;
316
+ let freeFn = [];
317
+ try {
318
+ const { agent: newAgent, freeFn: newFreeFn } = await this.setupAgent(platform);
319
+ this.actionSpace = await newAgent.getActionSpace();
320
+ agent = newAgent;
321
+ const originalOnTaskStartTip = agent.onTaskStartTip;
322
+ agent.onTaskStartTip = (tip)=>{
323
+ if ('running' === this.status) this.agentStatusTip = tip;
324
+ originalOnTaskStartTip?.(tip);
325
+ };
326
+ freeFn = [
327
+ ...newFreeFn || [],
328
+ {
329
+ name: 'restore-agent-onTaskStartTip',
330
+ fn: ()=>{
331
+ if (agent) agent.onTaskStartTip = originalOnTaskStartTip;
332
+ }
333
+ }
334
+ ];
335
+ } catch (e) {
336
+ this.setPlayerStatus('error', e);
337
+ return;
338
+ }
339
+ this.interfaceAgent = agent;
340
+ let taskIndex = 0;
341
+ this.setPlayerStatus('running');
342
+ let errorFlag = false;
343
+ while(taskIndex < tasks.length){
344
+ const taskStatus = this.taskStatusList[taskIndex];
345
+ this.setTaskStatus(taskIndex, 'running');
346
+ this.setTaskIndex(taskIndex);
347
+ try {
348
+ await this.playTask(taskStatus, this.interfaceAgent);
349
+ this.setTaskStatus(taskIndex, 'done');
350
+ } catch (e) {
351
+ this.setTaskStatus(taskIndex, 'error', e);
352
+ if (taskStatus.continueOnError) ;
353
+ else {
354
+ this.reportFile = agent.reportFile;
355
+ errorFlag = true;
356
+ break;
357
+ }
358
+ }
359
+ this.reportFile = agent?.reportFile;
360
+ taskIndex++;
361
+ }
362
+ if (errorFlag) this.setPlayerStatus('error');
363
+ else this.setPlayerStatus('done');
364
+ this.agentStatusTip = '';
365
+ for (const fn of freeFn)try {
366
+ await fn.fn();
367
+ } catch (e) {}
368
+ }
369
+ constructor(script, setupAgent, onTaskStatusChange, scriptPath){
370
+ _define_property(this, "script", void 0);
371
+ _define_property(this, "setupAgent", void 0);
372
+ _define_property(this, "onTaskStatusChange", void 0);
373
+ _define_property(this, "currentTaskIndex", void 0);
374
+ _define_property(this, "taskStatusList", void 0);
375
+ _define_property(this, "status", void 0);
376
+ _define_property(this, "reportFile", void 0);
377
+ _define_property(this, "result", void 0);
378
+ _define_property(this, "unnamedResultIndex", void 0);
379
+ _define_property(this, "output", void 0);
380
+ _define_property(this, "unstableLogContent", void 0);
381
+ _define_property(this, "errorInSetup", void 0);
382
+ _define_property(this, "interfaceAgent", void 0);
383
+ _define_property(this, "agentStatusTip", void 0);
384
+ _define_property(this, "target", void 0);
385
+ _define_property(this, "actionSpace", void 0);
386
+ _define_property(this, "scriptPath", void 0);
387
+ this.script = script;
388
+ this.setupAgent = setupAgent;
389
+ this.onTaskStatusChange = onTaskStatusChange;
390
+ this.taskStatusList = [];
391
+ this.status = 'init';
392
+ this.unnamedResultIndex = 0;
393
+ this.interfaceAgent = null;
394
+ this.actionSpace = [];
395
+ this.scriptPath = scriptPath;
396
+ this.result = {};
397
+ const resolvedAiActContext = script.agent?.aiActContext ?? script.agent?.aiActionContext;
398
+ if (void 0 !== resolvedAiActContext && script.agent) {
399
+ if (void 0 === script.agent.aiActContext && void 0 !== script.agent.aiActionContext) console.warn('agent.aiActionContext is deprecated, please use agent.aiActContext instead. The legacy name is still accepted for backward compatibility.');
400
+ script.agent.aiActContext = resolvedAiActContext;
401
+ }
402
+ this.target = script.target || script.web || script.android || script.ios || script.computer || script.config;
403
+ if (ifInBrowser || ifInWorker) {
404
+ this.output = void 0;
405
+ debug('output is undefined in browser or worker');
406
+ } else if (this.target?.output) {
407
+ this.output = external_node_path_resolve(process.cwd(), this.target.output);
408
+ debug('setting output by config.output', this.output);
409
+ } else {
410
+ const scriptName = this.scriptPath ? basename(this.scriptPath, '.yaml').replace(/\.(ya?ml)$/i, '') : "script";
411
+ this.output = join(getMidsceneRunSubDir('output'), `${scriptName}-${Date.now()}.json`);
412
+ debug("setting output by script path", this.output);
413
+ }
414
+ if (ifInBrowser || ifInWorker) this.unstableLogContent = void 0;
415
+ else if ('string' == typeof this.target?.unstableLogContent) this.unstableLogContent = external_node_path_resolve(process.cwd(), this.target.unstableLogContent);
416
+ else if (this.target?.unstableLogContent === true) this.unstableLogContent = join(getMidsceneRunSubDir('output'), 'unstableLogContent.json');
417
+ this.taskStatusList = (script.tasks || []).map((task, taskIndex)=>({
418
+ ...task,
419
+ index: taskIndex,
420
+ status: 'init',
421
+ totalSteps: task.flow?.length || 0
422
+ }));
423
+ }
424
+ }
425
+ export { ScriptPlayer };
@@ -0,0 +1,100 @@
1
+ import { getDebug } from "@godscene/shared/logger";
2
+ import { assert } from "@godscene/shared/utils";
3
+ import js_yaml from "js-yaml";
4
+ const debugUtils = getDebug('yaml:utils');
5
+ const multimodalLocateOptionFieldMap = {
6
+ images: true,
7
+ convertHttpImage2Base64: true
8
+ };
9
+ const multimodalLocateOptionKeys = Object.keys(multimodalLocateOptionFieldMap);
10
+ function extractMultimodalPrompt(opt) {
11
+ if ('object' != typeof opt || null === opt) return;
12
+ const entries = multimodalLocateOptionKeys.map((key)=>[
13
+ key,
14
+ opt[key]
15
+ ]).filter(([, value])=>void 0 !== value);
16
+ return entries.length ? Object.fromEntries(entries) : void 0;
17
+ }
18
+ function interpolateEnvVars(content) {
19
+ const lines = content.split('\n');
20
+ const processedLines = lines.map((line)=>{
21
+ const trimmedLine = line.trimStart();
22
+ if (trimmedLine.startsWith('#')) return line;
23
+ return line.replace(/\$\{([^}]+)\}/g, (_, envVar)=>{
24
+ const value = process.env[envVar.trim()];
25
+ if (void 0 === value) throw new Error(`Environment variable "${envVar.trim()}" is not defined`);
26
+ return value;
27
+ });
28
+ });
29
+ return processedLines.join('\n');
30
+ }
31
+ function parseYamlScript(content, filePath) {
32
+ let processedContent = content;
33
+ if (-1 !== content.indexOf('android') && content.match(/deviceId:\s*(\d+)/)) {
34
+ let matchedDeviceId;
35
+ processedContent = content.replace(/deviceId:\s*(\d+)/g, (match, deviceId)=>{
36
+ matchedDeviceId = deviceId;
37
+ return `deviceId: '${deviceId}'`;
38
+ });
39
+ console.warn(`please use string-style deviceId in yaml script, for example: deviceId: "${matchedDeviceId}"`);
40
+ }
41
+ const interpolatedContent = interpolateEnvVars(processedContent);
42
+ const obj = js_yaml.load(interpolatedContent, {
43
+ schema: js_yaml.JSON_SCHEMA
44
+ });
45
+ const pathTip = filePath ? `, failed to load ${filePath}` : '';
46
+ assert(obj.tasks, `property "tasks" is required in yaml script ${pathTip}`);
47
+ assert(Array.isArray(obj.tasks), `property "tasks" must be an array in yaml script, but got ${obj.tasks}`);
48
+ return obj;
49
+ }
50
+ function buildDetailedLocateParam(locatePrompt, opt) {
51
+ debugUtils('will call buildDetailedLocateParam', locatePrompt, opt);
52
+ let normalizedLocatePrompt = locatePrompt;
53
+ if ('object' == typeof locatePrompt && null !== locatePrompt && 'prompt' in locatePrompt) {
54
+ const { prompt: innerPrompt, ...rest } = locatePrompt;
55
+ const hasMultimodalFields = Object.keys(rest).length > 0;
56
+ normalizedLocatePrompt = hasMultimodalFields ? locatePrompt : innerPrompt;
57
+ }
58
+ let prompt = normalizedLocatePrompt || opt?.prompt || opt?.locate;
59
+ let deepLocate = false;
60
+ let cacheable = true;
61
+ let xpath;
62
+ if ('object' == typeof opt && null !== opt) {
63
+ deepLocate = opt.deepLocate ?? opt.deepThink ?? false;
64
+ cacheable = opt.cacheable ?? true;
65
+ xpath = opt.xpath;
66
+ if (locatePrompt && opt.prompt && locatePrompt !== opt.prompt) console.warn('conflict prompt for item', locatePrompt, opt, 'maybe you put the prompt in the wrong place');
67
+ prompt = prompt || opt.prompt;
68
+ }
69
+ if (!prompt) return void debugUtils('no prompt, will return undefined in buildDetailedLocateParam', opt);
70
+ const multimodalPrompt = extractMultimodalPrompt(opt);
71
+ if (multimodalPrompt) prompt = 'string' == typeof prompt ? {
72
+ prompt,
73
+ ...multimodalPrompt
74
+ } : {
75
+ ...prompt,
76
+ ...multimodalPrompt
77
+ };
78
+ return {
79
+ prompt,
80
+ deepLocate,
81
+ cacheable,
82
+ xpath
83
+ };
84
+ }
85
+ function buildDetailedLocateParamAndRestParams(locatePrompt, opt, excludeKeys = []) {
86
+ const multimodalPrompt = extractMultimodalPrompt(opt);
87
+ const locateParam = buildDetailedLocateParam(locatePrompt, opt);
88
+ const restParams = {};
89
+ if ('object' == typeof opt && null !== opt) {
90
+ const allKeys = Object.keys(opt);
91
+ const locateParamKeys = Object.keys(locateParam || {});
92
+ const multimodalPromptKeys = 'object' == typeof locateParam?.prompt && locateParam?.prompt !== null ? Object.keys(multimodalPrompt || {}) : [];
93
+ for (const key of allKeys)if (!locateParamKeys.includes(key) && !multimodalPromptKeys.includes(key) && !excludeKeys.includes(key) && 'locate' !== key) restParams[key] = opt[key];
94
+ }
95
+ return {
96
+ locateParam,
97
+ restParams
98
+ };
99
+ }
100
+ export { buildDetailedLocateParam, buildDetailedLocateParamAndRestParams, interpolateEnvVars, parseYamlScript };
File without changes