@loadmill/droid-cua 2.2.2 → 2.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (32) hide show
  1. package/README.md +69 -0
  2. package/build/index.js +177 -24
  3. package/build/src/cli/headless-debug.js +55 -0
  4. package/build/src/cli/headless-execution-config.js +203 -0
  5. package/build/src/cli/ink-shell.js +8 -2
  6. package/build/src/commands/help.js +13 -1
  7. package/build/src/commands/run.js +30 -1
  8. package/build/src/core/app-context.js +57 -0
  9. package/build/src/core/execution-engine.js +151 -20
  10. package/build/src/core/prompts.js +3 -247
  11. package/build/src/device/android/actions.js +2 -2
  12. package/build/src/device/assertions.js +4 -23
  13. package/build/src/device/cloud/browserstack/adapter.js +1 -0
  14. package/build/src/device/cloud/lambdatest/adapter.js +402 -0
  15. package/build/src/device/cloud/registry.js +2 -1
  16. package/build/src/device/interface.js +1 -1
  17. package/build/src/device/ios/actions.js +8 -2
  18. package/build/src/device/loadmill.js +4 -3
  19. package/build/src/device/openai.js +32 -26
  20. package/build/src/integrations/loadmill/interpreter.js +3 -56
  21. package/build/src/modes/design-mode-ink.js +12 -17
  22. package/build/src/modes/design-mode.js +12 -17
  23. package/build/src/modes/execution-mode.js +32 -22
  24. package/build/src/prompts/base.js +139 -0
  25. package/build/src/prompts/design.js +115 -0
  26. package/build/src/prompts/editor.js +19 -0
  27. package/build/src/prompts/execution.js +182 -0
  28. package/build/src/prompts/loadmill.js +60 -0
  29. package/build/src/utils/console-output.js +35 -0
  30. package/build/src/utils/run-screenshot-recorder.js +98 -0
  31. package/build/src/utils/structured-debug-log-manager.js +325 -0
  32. package/package.json +2 -1
@@ -0,0 +1,325 @@
1
+ import path from "node:path";
2
+ import { constants } from "node:fs";
3
+ import { access, appendFile, mkdir, writeFile } from "node:fs/promises";
4
+ export const WORKSPACE_DEBUG_BRIDGE_KEY = "__DROID_DESKTOP_DEBUG_LOG_EVENT";
5
+ export const DEVICE_DEBUG_LOG_NAME = "device-events.jsonl";
6
+ const MAX_STRING_LENGTH = 1600;
7
+ function isPromptContentPath(pathParts = []) {
8
+ if (pathParts.length < 2)
9
+ return false;
10
+ const last = pathParts[pathParts.length - 1];
11
+ if (last !== "content")
12
+ return false;
13
+ return pathParts.includes("messages");
14
+ }
15
+ function nowIso() {
16
+ return new Date().toISOString();
17
+ }
18
+ function compactTimestamp(iso) {
19
+ return iso.replace(/[:.]/g, "-");
20
+ }
21
+ function sanitizeValue(value, pathParts = [], eventName = "") {
22
+ if (typeof value === "string") {
23
+ if (eventName === "cua.request.full" || eventName === "cua.response.full" || eventName === "app_context.briefing.full") {
24
+ return value;
25
+ }
26
+ if (isPromptContentPath(pathParts)) {
27
+ return value;
28
+ }
29
+ if (value.length <= MAX_STRING_LENGTH)
30
+ return value;
31
+ return `${value.slice(0, MAX_STRING_LENGTH)}...<truncated:${value.length - MAX_STRING_LENGTH}>`;
32
+ }
33
+ if (Array.isArray(value)) {
34
+ return value.map((item, index) => sanitizeValue(item, [...pathParts, index], eventName));
35
+ }
36
+ if (value && typeof value === "object") {
37
+ const output = {};
38
+ for (const [key, item] of Object.entries(value)) {
39
+ output[key] = sanitizeValue(item, [...pathParts, key], eventName);
40
+ }
41
+ return output;
42
+ }
43
+ return value;
44
+ }
45
+ function serializeLine(event) {
46
+ const payload = {
47
+ ts: nowIso(),
48
+ event: event.event,
49
+ scope: event.scope,
50
+ ids: sanitizeValue(event.ids ?? {}, [], event.event),
51
+ data: sanitizeValue(event.data ?? {}, [], event.event)
52
+ };
53
+ return JSON.stringify(payload) + "\n";
54
+ }
55
+ function buildArtifactsDirPath(filePath) {
56
+ const ext = path.extname(filePath);
57
+ const stem = path.basename(filePath, ext);
58
+ return path.join(path.dirname(filePath), stem);
59
+ }
60
+ async function fileExists(filePath) {
61
+ try {
62
+ await access(filePath, constants.F_OK);
63
+ return true;
64
+ }
65
+ catch {
66
+ return false;
67
+ }
68
+ }
69
+ /**
70
+ * @param {{ enabled?: boolean, logsDirPath?: string | null }} [options]
71
+ */
72
+ export function createStructuredDebugLogManager({ enabled = false, logsDirPath = null } = {}) {
73
+ const executionSessions = new Map();
74
+ const designSessions = new Map();
75
+ let state = {
76
+ enabled,
77
+ logsDirPath,
78
+ deviceLogPath: null,
79
+ currentLogFilePath: null
80
+ };
81
+ function setLogsDirPath(nextLogsDirPath) {
82
+ if (state.logsDirPath === nextLogsDirPath) {
83
+ return;
84
+ }
85
+ state.logsDirPath = nextLogsDirPath;
86
+ state.deviceLogPath = null;
87
+ state.currentLogFilePath = null;
88
+ }
89
+ function getActiveSessionFilePaths() {
90
+ const all = [...executionSessions.values(), ...designSessions.values()];
91
+ return all.map((record) => record.filePath);
92
+ }
93
+ function resolveSessionFile(scope, id) {
94
+ if (scope === "execution") {
95
+ return executionSessions.get(id)?.filePath ?? null;
96
+ }
97
+ return designSessions.get(id)?.filePath ?? null;
98
+ }
99
+ function pickCurrentFile() {
100
+ const activeExecution = [...executionSessions.values()].at(-1)?.filePath ?? null;
101
+ if (activeExecution)
102
+ return activeExecution;
103
+ const activeDesign = [...designSessions.values()].at(-1)?.filePath ?? null;
104
+ if (activeDesign)
105
+ return activeDesign;
106
+ return state.deviceLogPath;
107
+ }
108
+ async function ensureLogsDir() {
109
+ if (!state.logsDirPath) {
110
+ throw new Error("Structured debug logs directory is not configured.");
111
+ }
112
+ await mkdir(state.logsDirPath, { recursive: true });
113
+ return state.logsDirPath;
114
+ }
115
+ async function ensureDeviceLogFile() {
116
+ if (state.deviceLogPath)
117
+ return state.deviceLogPath;
118
+ const dir = await ensureLogsDir();
119
+ state.deviceLogPath = path.join(dir, DEVICE_DEBUG_LOG_NAME);
120
+ return state.deviceLogPath;
121
+ }
122
+ async function appendEvent(filePath, event) {
123
+ await appendFile(filePath, serializeLine(event), "utf-8");
124
+ }
125
+ async function appendDeviceEvent(event) {
126
+ const logFile = await ensureDeviceLogFile();
127
+ await appendEvent(logFile, event);
128
+ for (const sessionFilePath of getActiveSessionFilePaths()) {
129
+ if (sessionFilePath === logFile)
130
+ continue;
131
+ await appendEvent(sessionFilePath, event);
132
+ }
133
+ state.currentLogFilePath = getActiveSessionFilePaths().at(-1) ?? logFile;
134
+ }
135
+ return {
136
+ /**
137
+ * @param {{ enabled?: boolean, logsDirPath?: string | null }} [options]
138
+ */
139
+ async configure({ enabled: nextEnabled = state.enabled, logsDirPath: nextLogsDirPath = state.logsDirPath } = {}) {
140
+ state.enabled = Boolean(nextEnabled);
141
+ setLogsDirPath(nextLogsDirPath ?? null);
142
+ if (state.enabled) {
143
+ await ensureLogsDir();
144
+ }
145
+ return state.enabled;
146
+ },
147
+ isEnabled() {
148
+ return state.enabled;
149
+ },
150
+ installWorkspaceDebugBridge(target = globalThis) {
151
+ if (!target || typeof target !== "object") {
152
+ return;
153
+ }
154
+ target[WORKSPACE_DEBUG_BRIDGE_KEY] = (event) => {
155
+ void this.logWorkspaceEvent(event);
156
+ };
157
+ },
158
+ async startExecutionSession(runId, data = {}) {
159
+ if (!state.enabled)
160
+ return;
161
+ const dir = await ensureLogsDir();
162
+ const startedAt = nowIso();
163
+ const filename = `execution-${runId}-${compactTimestamp(startedAt)}.jsonl`;
164
+ const filePath = path.join(dir, filename);
165
+ const artifactsDirPath = buildArtifactsDirPath(filePath);
166
+ await writeFile(filePath, "", "utf-8");
167
+ await mkdir(artifactsDirPath, { recursive: true });
168
+ executionSessions.set(runId, {
169
+ id: runId,
170
+ scope: "execution",
171
+ filePath,
172
+ artifactsDirPath,
173
+ startedAt
174
+ });
175
+ state.currentLogFilePath = filePath;
176
+ await appendEvent(filePath, {
177
+ event: "session.started",
178
+ scope: "execution",
179
+ ids: { runId },
180
+ data: {
181
+ startedAt,
182
+ artifactsDirPath,
183
+ ...data
184
+ }
185
+ });
186
+ },
187
+ async endExecutionSession(runId, data = {}) {
188
+ if (!state.enabled)
189
+ return;
190
+ const session = executionSessions.get(runId);
191
+ if (!session)
192
+ return;
193
+ await appendEvent(session.filePath, {
194
+ event: "session.ended",
195
+ scope: "execution",
196
+ ids: { runId },
197
+ data: {
198
+ startedAt: session.startedAt,
199
+ endedAt: nowIso(),
200
+ ...data
201
+ }
202
+ });
203
+ executionSessions.delete(runId);
204
+ state.currentLogFilePath = pickCurrentFile();
205
+ },
206
+ async startDesignSession(sessionId, data = {}) {
207
+ if (!state.enabled)
208
+ return;
209
+ const dir = await ensureLogsDir();
210
+ const startedAt = nowIso();
211
+ const filename = `design-${sessionId}-${compactTimestamp(startedAt)}.jsonl`;
212
+ const filePath = path.join(dir, filename);
213
+ const artifactsDirPath = buildArtifactsDirPath(filePath);
214
+ await writeFile(filePath, "", "utf-8");
215
+ await mkdir(artifactsDirPath, { recursive: true });
216
+ designSessions.set(sessionId, {
217
+ id: sessionId,
218
+ scope: "design",
219
+ filePath,
220
+ artifactsDirPath,
221
+ startedAt
222
+ });
223
+ state.currentLogFilePath = filePath;
224
+ await appendEvent(filePath, {
225
+ event: "session.started",
226
+ scope: "design",
227
+ ids: { sessionId },
228
+ data: {
229
+ startedAt,
230
+ artifactsDirPath,
231
+ ...data
232
+ }
233
+ });
234
+ },
235
+ async endDesignSession(sessionId, data = {}) {
236
+ if (!state.enabled)
237
+ return;
238
+ const session = designSessions.get(sessionId);
239
+ if (!session)
240
+ return;
241
+ await appendEvent(session.filePath, {
242
+ event: "session.ended",
243
+ scope: "design",
244
+ ids: { sessionId },
245
+ data: {
246
+ startedAt: session.startedAt,
247
+ endedAt: nowIso(),
248
+ ...data
249
+ }
250
+ });
251
+ designSessions.delete(sessionId);
252
+ state.currentLogFilePath = pickCurrentFile();
253
+ },
254
+ async logDeviceEvent(event) {
255
+ if (!state.enabled)
256
+ return;
257
+ await appendDeviceEvent({
258
+ event: event.event,
259
+ scope: "device",
260
+ ids: event.ids,
261
+ data: event.data
262
+ });
263
+ },
264
+ async logSessionEvent(scope, id, event) {
265
+ if (!state.enabled)
266
+ return;
267
+ const sessionFile = resolveSessionFile(scope, id);
268
+ if (!sessionFile)
269
+ return;
270
+ await appendEvent(sessionFile, {
271
+ event: event.event,
272
+ scope,
273
+ ids: event.ids,
274
+ data: event.data
275
+ });
276
+ },
277
+ async logWorkspaceEvent(rawEvent) {
278
+ if (!state.enabled)
279
+ return;
280
+ const event = typeof rawEvent?.event === "string" ? rawEvent.event : "";
281
+ const scope = rawEvent?.scope === "execution" || rawEvent?.scope === "design" || rawEvent?.scope === "device"
282
+ ? rawEvent.scope
283
+ : null;
284
+ const ids = rawEvent?.ids && typeof rawEvent.ids === "object" ? rawEvent.ids : {};
285
+ const data = rawEvent?.data && typeof rawEvent.data === "object" ? rawEvent.data : {};
286
+ if (!event || !scope)
287
+ return;
288
+ if (scope === "device") {
289
+ await this.logDeviceEvent({ event, ids, data });
290
+ return;
291
+ }
292
+ const idKey = scope === "execution" ? "runId" : "sessionId";
293
+ const idValue = typeof ids[idKey] === "string" ? ids[idKey] : "";
294
+ if (idValue) {
295
+ await this.logSessionEvent(scope, idValue, { event, ids, data });
296
+ return;
297
+ }
298
+ const sessionFilePath = scope === "execution"
299
+ ? [...executionSessions.values()].at(-1)?.filePath ?? null
300
+ : [...designSessions.values()].at(-1)?.filePath ?? null;
301
+ if (!sessionFilePath)
302
+ return;
303
+ await appendEvent(sessionFilePath, { event, scope, ids, data });
304
+ },
305
+ async getCurrentLogFilePath() {
306
+ const current = state.currentLogFilePath ?? pickCurrentFile();
307
+ if (current && (await fileExists(current))) {
308
+ return current;
309
+ }
310
+ if (state.deviceLogPath && (await fileExists(state.deviceLogPath))) {
311
+ return state.deviceLogPath;
312
+ }
313
+ return null;
314
+ },
315
+ getExecutionSessionArtifactsDir(runId) {
316
+ return executionSessions.get(runId)?.artifactsDirPath ?? null;
317
+ },
318
+ getDesignSessionArtifactsDir(sessionId) {
319
+ return designSessions.get(sessionId)?.artifactsDirPath ?? null;
320
+ },
321
+ async getLogsDirPath() {
322
+ return ensureLogsDir();
323
+ }
324
+ };
325
+ }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@loadmill/droid-cua",
3
- "version": "2.2.2",
3
+ "version": "2.4.0",
4
4
  "description": "AI-powered Android testing agent using OpenAI's computer-use model and ADB",
5
5
  "main": "build/index.js",
6
6
  "type": "module",
@@ -15,6 +15,7 @@
15
15
  ],
16
16
  "scripts": {
17
17
  "start": "tsx index.js",
18
+ "test": "node --test unit-tests/*.test.js",
18
19
  "build": "tsc",
19
20
  "clean": "rm -rf build",
20
21
  "prerelease": "npm run build",