agentlane 0.0.0 → 0.4.0-rc.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (146) hide show
  1. package/CHANGELOG.md +118 -0
  2. package/LICENSE +27 -0
  3. package/README.md +167 -2
  4. package/dist/agents/claude-plugin.js +79 -0
  5. package/dist/agents/cli.js +258 -0
  6. package/dist/agents/detect.js +88 -0
  7. package/dist/agents/init.js +251 -0
  8. package/dist/agents/launcher.js +32 -0
  9. package/dist/agents/package.js +119 -0
  10. package/dist/agents/skills.js +59 -0
  11. package/dist/agents/toml.js +325 -0
  12. package/dist/agents/wiring-check.js +117 -0
  13. package/dist/agents/wiring.js +374 -0
  14. package/dist/bench/host.js +104 -0
  15. package/dist/bench/live.js +511 -0
  16. package/dist/bench/mcp-client.js +91 -0
  17. package/dist/bench/model.js +15 -0
  18. package/dist/bench/report.js +162 -0
  19. package/dist/bench/run.js +121 -0
  20. package/dist/bench/schedule.js +140 -0
  21. package/dist/bench/stats.js +59 -0
  22. package/dist/bench-cli.js +167 -0
  23. package/dist/browser/aria.js +184 -0
  24. package/dist/browser/control.js +203 -0
  25. package/dist/browser/env.js +118 -0
  26. package/dist/browser/evidence.js +44 -0
  27. package/dist/browser/install.js +361 -0
  28. package/dist/browser/kinds.js +10 -0
  29. package/dist/browser/lane-browser.js +224 -0
  30. package/dist/browser/libraries.js +56 -0
  31. package/dist/browser/paths.js +29 -0
  32. package/dist/browser/release.js +121 -0
  33. package/dist/browser/screen.js +19 -0
  34. package/dist/browser/signin.js +154 -0
  35. package/dist/browser/stop.js +163 -0
  36. package/dist/browser/tools.js +25 -0
  37. package/dist/cache/build.js +154 -0
  38. package/dist/cache/config.js +186 -0
  39. package/dist/cache/index.js +4 -0
  40. package/dist/cache/key.js +196 -0
  41. package/dist/cache/store.js +303 -0
  42. package/dist/cli.js +178 -0
  43. package/dist/clock.js +4 -0
  44. package/dist/devices/adb.js +160 -0
  45. package/dist/devices/android.js +214 -0
  46. package/dist/devices/backend.js +7 -0
  47. package/dist/devices/bundler.js +46 -0
  48. package/dist/devices/common.js +51 -0
  49. package/dist/devices/controls.js +65 -0
  50. package/dist/devices/dial.js +56 -0
  51. package/dist/devices/emulator.js +396 -0
  52. package/dist/devices/hygiene.js +202 -0
  53. package/dist/devices/index.js +5 -0
  54. package/dist/devices/launch.js +76 -0
  55. package/dist/devices/simctl.js +119 -0
  56. package/dist/devices/tools.js +1 -0
  57. package/dist/devices/variant.js +99 -0
  58. package/dist/devices/xcrun.js +265 -0
  59. package/dist/driver/absence.js +54 -0
  60. package/dist/driver/caller.js +51 -0
  61. package/dist/driver/client.js +312 -0
  62. package/dist/driver/diff.js +180 -0
  63. package/dist/driver/elements.js +70 -0
  64. package/dist/driver/engine-main.js +13 -0
  65. package/dist/driver/engine.js +499 -0
  66. package/dist/driver/errors.js +11 -0
  67. package/dist/driver/exec.js +66 -0
  68. package/dist/driver/failure.js +156 -0
  69. package/dist/driver/flow.js +94 -0
  70. package/dist/driver/idle.js +100 -0
  71. package/dist/driver/images.js +68 -0
  72. package/dist/driver/install.js +272 -0
  73. package/dist/driver/lock.js +101 -0
  74. package/dist/driver/look.js +223 -0
  75. package/dist/driver/maestro-child.js +194 -0
  76. package/dist/driver/maestro-runs.js +46 -0
  77. package/dist/driver/paths.js +105 -0
  78. package/dist/driver/pixels.js +144 -0
  79. package/dist/driver/plan.js +108 -0
  80. package/dist/driver/release.js +7 -0
  81. package/dist/driver/runner.js +42 -0
  82. package/dist/driver/selector.js +91 -0
  83. package/dist/driver/session.js +474 -0
  84. package/dist/driver/suggest.js +86 -0
  85. package/dist/driver/supervisor.js +251 -0
  86. package/dist/driver/tree.js +426 -0
  87. package/dist/driver-cli.js +865 -0
  88. package/dist/evidence/device.js +177 -0
  89. package/dist/evidence/evidence.js +304 -0
  90. package/dist/evidence/files.js +92 -0
  91. package/dist/hooks/env.js +81 -0
  92. package/dist/hooks/run.js +101 -0
  93. package/dist/mcp/server.js +69 -0
  94. package/dist/mcp/tools.js +246 -0
  95. package/dist/name.js +22 -0
  96. package/dist/plugin/aims.js +465 -0
  97. package/dist/plugin/conventions.js +73 -0
  98. package/dist/plugin/guard.js +163 -0
  99. package/dist/plugin/hook-cli.js +216 -0
  100. package/dist/plugin/session-name.js +28 -0
  101. package/dist/plugin/shell.js +254 -0
  102. package/dist/processes.js +77 -0
  103. package/dist/project/config.js +316 -0
  104. package/dist/project/schema.js +335 -0
  105. package/dist/project/validate.js +224 -0
  106. package/dist/registry/config.js +142 -0
  107. package/dist/registry/format.js +22 -0
  108. package/dist/registry/owner.js +265 -0
  109. package/dist/registry/paths.js +50 -0
  110. package/dist/registry/ports.js +26 -0
  111. package/dist/registry/prune.js +160 -0
  112. package/dist/registry/refusal.js +53 -0
  113. package/dist/registry/registry.js +1027 -0
  114. package/dist/registry/resources.js +117 -0
  115. package/dist/registry/rows.js +115 -0
  116. package/dist/registry/status.js +85 -0
  117. package/dist/registry/store.js +274 -0
  118. package/dist/registry/types.js +1 -0
  119. package/dist/registry-cli.js +544 -0
  120. package/dist/services/services.js +268 -0
  121. package/dist/services/system.js +189 -0
  122. package/dist/up/deps.js +1 -0
  123. package/dist/up/doctor.js +424 -0
  124. package/dist/up/exec.js +133 -0
  125. package/dist/up/lane-files.js +75 -0
  126. package/dist/up/lane-services.js +38 -0
  127. package/dist/up/maestro-env.js +172 -0
  128. package/dist/up/preflight.js +232 -0
  129. package/dist/up/release.js +148 -0
  130. package/dist/up/resources.js +178 -0
  131. package/dist/up/restart.js +87 -0
  132. package/dist/up/setup.js +28 -0
  133. package/dist/up/up.js +610 -0
  134. package/dist/up-cli.js +258 -0
  135. package/dist/usage.js +16 -0
  136. package/package.json +58 -3
  137. package/schema/agentlane.schema.json +675 -0
  138. package/skills/create-verification-skill/LICENSE +21 -0
  139. package/skills/create-verification-skill/SKILL.md +88 -0
  140. package/skills/dev-loop/LICENSE +21 -0
  141. package/skills/dev-loop/SKILL.md +61 -0
  142. package/skills/device/SKILL.md +8 -0
  143. package/skills/drive/SKILL.md +64 -0
  144. package/skills/maintain-verification-skill/LICENSE +21 -0
  145. package/skills/maintain-verification-skill/SKILL.md +42 -0
  146. package/skills/setup/SKILL.md +80 -0
@@ -0,0 +1,511 @@
1
+ import { spawn, spawnSync } from 'node:child_process';
2
+ import path from 'node:path';
3
+ import { PLAYWRIGHT_RELEASE } from "../browser/release.js";
4
+ import { adbTools } from "../devices/adb.js";
5
+ import { MAESTRO_DRIVER_PACKAGE } from "../devices/emulator.js";
6
+ import { devClientLink } from "../devices/launch.js";
7
+ import { resolveCallerLane } from "../driver/caller.js";
8
+ import { engineStatus, stopEngine } from "../driver/client.js";
9
+ import { enginePaths } from "../driver/paths.js";
10
+ import { OWNER_PID_ENV } from "../registry/owner.js";
11
+ import { McpClient } from "./mcp-client.js";
12
+ export const CLI_ENTRY = path.join(import.meta.dirname, '..', `cli${path.extname(import.meta.filename)}`);
13
+ /** The example app the bench drives (examples/expo-app): its bundle id, home-screen anchor, tap and flow. */
14
+ export const TARGET_APP = {
15
+ bundleId: 'com.example.expoapp',
16
+ scheme: 'expoexample',
17
+ anchor: 'home-increment',
18
+ tap: '- tapOn:\n id: home-increment\n',
19
+ flow: [
20
+ '- tapOn:',
21
+ ' id: home-open-details',
22
+ '- assertVisible:',
23
+ ' id: details-title',
24
+ '- tapOn:',
25
+ ' id: details-back',
26
+ '- assertVisible:',
27
+ ' id: home-increment',
28
+ '',
29
+ ].join('\n'),
30
+ };
31
+ /**
32
+ * The example's web app on a browser lane: the page `up` opens signed in, the toolkit's `look` shows its Shortest
33
+ * first button by name, a click toggles it, and the flow adds a place through the API to the lane's database.
34
+ */
35
+ export const TARGET_WEB = {
36
+ anchor: '"Shortest first"',
37
+ click: "getByTestId('sort-shortest')",
38
+ flow: (name) => [
39
+ 'async page => {',
40
+ ` await page.getByTestId('add-name').fill(${JSON.stringify(name)});`,
41
+ " await page.getByTestId('add-submit').click();",
42
+ ` await page.getByTestId('places').getByText(${JSON.stringify(name)}, { exact: true }).waitFor();`,
43
+ '}',
44
+ ].join('\n'),
45
+ };
46
+ /** The driver CLI call an agent makes for a click or a flow in the lane's browser: through `exec`, as one Bash call. */
47
+ export function browserCommand(op, flowName) {
48
+ return op === 'click'
49
+ ? ['exec', '--', 'playwright-cli', 'click', TARGET_WEB.click]
50
+ : ['exec', '--', 'playwright-cli', 'run-code', TARGET_WEB.flow(flowName)];
51
+ }
52
+ /**
53
+ * The example app restarted onto its own Metro and API. On iOS the dev client reads --initialUrl from its launch
54
+ * arguments; on Android launchApp's arguments become intent extras, and the dev client takes its bundler from the
55
+ * development-client link that follows, which expo-dev-launcher ignores until its launcher screen is up.
56
+ */
57
+ export function relaunchYaml(metroPort, apiPort, platform = 'ios') {
58
+ const api = apiPort === null ? [] : [` apiBaseUrl: http://localhost:${String(apiPort)}`];
59
+ if (platform === 'android') {
60
+ return [
61
+ '- launchApp:',
62
+ ' stopApp: true',
63
+ ...(api.length > 0 ? [' arguments:', ...api] : []),
64
+ '- extendedWaitUntil:',
65
+ ' visible: DEVELOPMENT SERVERS',
66
+ ' timeout: 15000',
67
+ `- openLink: ${devClientLink({ scheme: TARGET_APP.scheme, port: metroPort })}`,
68
+ '',
69
+ ].join('\n');
70
+ }
71
+ return [
72
+ '- launchApp:',
73
+ ' stopApp: true',
74
+ ' arguments:',
75
+ ` '--initialUrl': http://localhost:${String(metroPort)}`,
76
+ ...api,
77
+ '',
78
+ ].join('\n');
79
+ }
80
+ const GATE_TIMEOUT_MS = 90_000;
81
+ const ADMISSION_TIMEOUT_MS = 20_000;
82
+ function runCli(args, cwd, env) {
83
+ const started = process.hrtime.bigint();
84
+ return new Promise(resolve => {
85
+ const child = spawn(process.execPath, [CLI_ENTRY, ...args], { cwd, env, stdio: ['ignore', 'pipe', 'pipe'] });
86
+ let stdout = '';
87
+ let stderr = '';
88
+ child.stdout.setEncoding('utf8').on('data', (chunk) => (stdout += chunk));
89
+ child.stderr.setEncoding('utf8').on('data', (chunk) => (stderr = (stderr + chunk).slice(-8000)));
90
+ child.on('close', code => {
91
+ resolve({ code, stdout, stderr, ms: Number(process.hrtime.bigint() - started) / 1e6 });
92
+ });
93
+ });
94
+ }
95
+ /** The environment every child of the bench runs in: the bench process owns its lanes, so they end with it. */
96
+ export function benchEnv(env) {
97
+ return { ...env, [OWNER_PID_ENV]: String(process.pid) };
98
+ }
99
+ function parseUp(stdout) {
100
+ try {
101
+ return JSON.parse(stdout);
102
+ }
103
+ catch {
104
+ return null;
105
+ }
106
+ }
107
+ async function up(folder, env, platform, setup) {
108
+ const ran = await runCli(['up', '--json', '--platform', platform, ...(setup ? [] : ['--no-setup'])], folder, env);
109
+ const parsed = parseUp(ran.stdout);
110
+ const failed = parsed?.doctor?.checks?.filter(c => !c.ok) ?? [];
111
+ if (ran.code !== 0 || !parsed || parsed.doctor?.ok !== true) {
112
+ const why = failed.length > 0
113
+ ? `doctor: ${failed.map(c => `${c.name}: ${c.detail}`).join('; ')}`
114
+ : ran.stderr.trim().split('\n').slice(-4).join(' / ');
115
+ throw new Error(`up exited ${String(ran.code)}: ${why}`);
116
+ }
117
+ const steps = {};
118
+ for (const s of parsed.steps ?? [])
119
+ steps[`up ${s.step}`] = s.ms;
120
+ return { ms: ran.ms, steps };
121
+ }
122
+ export async function openTarget(label, folder, deps, setup) {
123
+ const env = benchEnv(deps.env);
124
+ deps.log(`${label}: up in ${folder}`);
125
+ const brought = await up(folder, env, deps.platform, setup);
126
+ deps.log(`${label}: up took ${(brought.ms / 1000).toFixed(1)} s`);
127
+ const port = (service) => lane.ports.find(p => p.service === service)?.port ?? null;
128
+ const { lane, stateDir } = resolveCallerLane({
129
+ env,
130
+ cwd: folder,
131
+ processes: deps.processes,
132
+ need: deps.platform === 'web' ? 'browser' : 'device',
133
+ });
134
+ if (deps.platform === 'web') {
135
+ const browser = lane.browser;
136
+ if (!browser)
137
+ throw new Error(`lane ${lane.id} holds no browser`);
138
+ const mcp = new McpClient(process.execPath, [CLI_ENTRY, 'mcp'], folder, env);
139
+ await mcp.initialize();
140
+ return {
141
+ folder,
142
+ lane,
143
+ stateDir,
144
+ mcp,
145
+ env,
146
+ flows: 0,
147
+ info: {
148
+ sim: label,
149
+ folder: lane.worktree,
150
+ lane: lane.id,
151
+ platform: 'web',
152
+ udid: browser.session,
153
+ deviceName: (await browserVersion(browser.debugPort)) ?? 'unknown browser',
154
+ deviceType: `${browser.driver} ${PLAYWRIGHT_RELEASE.version}`,
155
+ runtime: `debugging port ${String(browser.debugPort)}`,
156
+ metroPort: null,
157
+ apiPort: port('api'),
158
+ webPort: port('web'),
159
+ },
160
+ };
161
+ }
162
+ const device = lane.device;
163
+ if (!device)
164
+ throw new Error(`lane ${lane.id} holds no device`);
165
+ if (device.platform !== deps.platform)
166
+ throw new Error(`lane ${lane.id} holds ${device.name}, not a ${deps.platform} device`);
167
+ const mcp = new McpClient(process.execPath, [CLI_ENTRY, 'mcp'], folder, env);
168
+ await mcp.initialize();
169
+ return {
170
+ folder,
171
+ lane,
172
+ stateDir,
173
+ mcp,
174
+ env,
175
+ flows: 0,
176
+ info: {
177
+ sim: label,
178
+ folder: lane.worktree,
179
+ lane: lane.id,
180
+ platform: device.platform,
181
+ udid: device.platform === 'android' ? (device.serial ?? device.id) : device.id,
182
+ deviceName: device.name,
183
+ deviceType: device.deviceTypeName,
184
+ runtime: device.runtimeName,
185
+ metroPort: port('metro'),
186
+ apiPort: port('api'),
187
+ },
188
+ };
189
+ }
190
+ /** Releases the bench's lane for a folder; the release stops its services and its driver. */
191
+ export async function releaseFolder(folder, deps) {
192
+ const ran = await runCli(['release'], folder, benchEnv(deps.env));
193
+ return ran.code === 0 ? ran.stdout.trim() || 'released' : `release exited ${String(ran.code)}: ${ran.stderr.trim()}`;
194
+ }
195
+ export async function closeTarget(target, deps) {
196
+ const notes = [];
197
+ await target.mcp.close();
198
+ notes.push(`${target.info.sim}: lane ${target.lane.id}: ${await releaseFolder(target.folder, deps)}`);
199
+ const deadline = Date.now() + ADMISSION_TIMEOUT_MS;
200
+ let engine = true;
201
+ while (Date.now() < deadline) {
202
+ engine = (await engineStatus(target.stateDir, target.lane).catch(() => undefined)) !== undefined;
203
+ if (!engine)
204
+ break;
205
+ await deps.sim.sleep(250);
206
+ }
207
+ const state = await deviceState(target, deps);
208
+ notes.push(`${target.info.sim}: after release the driver is ${engine ? 'STILL RUNNING' : 'stopped'}; ${target.info.deviceName} is ${state}`);
209
+ return notes;
210
+ }
211
+ /** What the browser on the lane's debugging port says it is, from Chrome's /json/version; null when nothing answers. */
212
+ async function browserVersion(port) {
213
+ try {
214
+ const response = await fetch(`http://127.0.0.1:${String(port)}/json/version`, { signal: AbortSignal.timeout(2000) });
215
+ const body = (await response.json());
216
+ return typeof body.Browser === 'string' ? body.Browser : null;
217
+ }
218
+ catch {
219
+ return null;
220
+ }
221
+ }
222
+ async function deviceState(target, deps) {
223
+ if (target.info.platform === 'web') {
224
+ const port = target.lane.browser?.debugPort;
225
+ return port !== undefined && (await browserVersion(port)) !== null ? 'still answering' : 'closed';
226
+ }
227
+ if (target.info.platform !== 'android')
228
+ return (await deps.sim.device(target.info.udid))?.state ?? 'unknown';
229
+ try {
230
+ return (await adbTools(deps, target.info.udid).bootCompleted()) ? 'Booted' : 'not booted';
231
+ }
232
+ catch {
233
+ return 'not running';
234
+ }
235
+ }
236
+ async function browserCall(target, op) {
237
+ target.flows += op === 'flow' ? 1 : 0;
238
+ const ran = await runCli(browserCommand(op, `Bench place ${target.lane.id}-${String(target.flows)}`), target.folder, target.env);
239
+ const text = `${ran.stdout}\n${ran.stderr}`;
240
+ return { ms: ran.ms, isError: ran.code !== 0, text, images: 0, bytes: Buffer.byteLength(ran.stdout) };
241
+ }
242
+ function call(target, op) {
243
+ if (target.info.platform === 'web') {
244
+ if (op === 'read')
245
+ return target.mcp.call('look', {});
246
+ if (op === 'screenshot')
247
+ return target.mcp.call('screenshot', {});
248
+ if (op === 'click' || op === 'flow')
249
+ return browserCall(target, op);
250
+ throw new Error(`${op} is a device op; a browser lane takes read, screenshot, click and flow`);
251
+ }
252
+ switch (op) {
253
+ case 'read':
254
+ return target.mcp.call('look', {});
255
+ case 'screenshot':
256
+ return target.mcp.call('screenshot', {});
257
+ case 'quick-tap':
258
+ return target.mcp.call('run', { yaml: TARGET_APP.tap, appId: TARGET_APP.bundleId, quick: true });
259
+ case 'tap':
260
+ return target.mcp.call('run', { yaml: TARGET_APP.tap, appId: TARGET_APP.bundleId });
261
+ case 'flow':
262
+ return target.mcp.call('run', { yaml: TARGET_APP.flow, appId: TARGET_APP.bundleId });
263
+ case 'click':
264
+ throw new Error('click is a browser op; a device takes quick-tap and tap');
265
+ }
266
+ }
267
+ function anchorOf(target) {
268
+ return target.info.platform === 'web' ? TARGET_WEB.anchor : TARGET_APP.anchor;
269
+ }
270
+ function shows(target, answer) {
271
+ return !answer.isError && answer.text.includes(anchorOf(target));
272
+ }
273
+ function brief(text) {
274
+ return text.replace(/\s+/g, ' ').trim().slice(0, 300);
275
+ }
276
+ const elapsed = (from) => Number(process.hrtime.bigint() - from) / 1e6;
277
+ /**
278
+ * One op on the app's home screen. With gate, the app may still be starting: a read is repeated until it shows the
279
+ * anchor, and a screenshot waits for such a read first (taps and flows wait on their own, through Maestro's find). The
280
+ * repeated reads are inside the timed span and counted.
281
+ */
282
+ async function timedOp(target, op, gate) {
283
+ const started = process.hrtime.bigint();
284
+ let gateReads = 0;
285
+ if (op === 'read' || (op === 'screenshot' && gate)) {
286
+ for (;;) {
287
+ const answer = await target.mcp.call('look', {});
288
+ if (shows(target, answer)) {
289
+ if (op === 'read')
290
+ return { ok: true, phases: { op: elapsed(started) }, gateReads, responseBytes: answer.bytes };
291
+ break;
292
+ }
293
+ if (!gate || elapsed(started) > GATE_TIMEOUT_MS) {
294
+ return {
295
+ ok: false,
296
+ phases: {},
297
+ gateReads,
298
+ responseBytes: answer.bytes,
299
+ error: answer.isError ? brief(answer.text) : `the read doesn't show ${anchorOf(target)}: ${brief(answer.text)}`,
300
+ };
301
+ }
302
+ gateReads += 1;
303
+ }
304
+ }
305
+ const gateMs = elapsed(started);
306
+ const answer = await call(target, op);
307
+ if (answer.isError)
308
+ return { ok: false, phases: {}, gateReads, responseBytes: answer.bytes, error: brief(answer.text) };
309
+ if (op === 'screenshot' && answer.images !== 1)
310
+ return { ok: false, phases: {}, gateReads, responseBytes: answer.bytes, error: 'no image in the answer' };
311
+ return {
312
+ ok: true,
313
+ phases: { ...(gate && (gateReads > 0 || op === 'screenshot') ? { gate: gateMs } : {}), op: answer.ms },
314
+ gateReads,
315
+ responseBytes: answer.bytes,
316
+ };
317
+ }
318
+ function psLines() {
319
+ const result = spawnSync('ps', ['-A', '-ww', '-o', 'pid=,command='], { encoding: 'utf8', timeout: 10_000 });
320
+ return result.stdout.split('\n');
321
+ }
322
+ /**
323
+ * What still runs for this lane: its engine, a JVM working in its folder, and XCTest or xcodebuild on its simulator
324
+ * or Maestro's instrumentation on its emulator.
325
+ */
326
+ async function leftovers(target, deps) {
327
+ const found = [];
328
+ if ((await engineStatus(target.stateDir, target.lane).catch(() => undefined)) !== undefined)
329
+ found.push('the driver engine');
330
+ const work = enginePaths(target.stateDir, target.lane.id).work;
331
+ const udid = target.info.udid;
332
+ for (const line of psLines()) {
333
+ if (line.includes(`${work}/`) || line.endsWith(work) || line.includes(`${work} `))
334
+ found.push(`a process in the lane's work dir: ${brief(line).slice(0, 120)}`);
335
+ else if (line.includes(udid) && (line.includes('xcodebuild') || line.includes('maestro-driver-iosUITests-Runner')))
336
+ found.push(`XCTest on the simulator: ${brief(line).slice(0, 120)}`);
337
+ }
338
+ if (target.info.platform === 'android' && (await adbTools(deps, udid).appRunning(MAESTRO_DRIVER_PACKAGE)))
339
+ found.push(`Maestro's instrumentation (${MAESTRO_DRIVER_PACKAGE}) on the emulator`);
340
+ return found;
341
+ }
342
+ async function admit(target, deps, simState) {
343
+ const started = Date.now();
344
+ let problems = [];
345
+ while (Date.now() - started < ADMISSION_TIMEOUT_MS) {
346
+ problems = await leftovers(target, deps);
347
+ const state = await deviceState(target, deps);
348
+ if (state !== simState)
349
+ problems.push(`${target.info.deviceName} is ${state}, not ${simState}`);
350
+ if (problems.length === 0)
351
+ return Date.now() - started;
352
+ await deps.sim.sleep(250);
353
+ }
354
+ throw new Error(`admission: ${problems.join('; ')}`);
355
+ }
356
+ /** Puts the app on its home screen under a running driver: a read, and a relaunch when the anchor isn't showing. */
357
+ async function ensureHome(target) {
358
+ const first = await target.mcp.call('look', {});
359
+ if (shows(target, first))
360
+ return;
361
+ if (target.info.platform === 'web') {
362
+ const url = `http://localhost:${String(target.info.webPort)}/`;
363
+ const opened = await runCli(['exec', '--', 'playwright-cli', 'goto', url], target.folder, target.env);
364
+ if (opened.code !== 0)
365
+ throw new Error(`opening ${url} failed: ${brief(opened.stderr)}`);
366
+ const timed = await timedOp(target, 'read', true);
367
+ if (!timed.ok)
368
+ throw new Error(`the signed-in page never showed: ${timed.error ?? ''}`);
369
+ return;
370
+ }
371
+ const { metroPort, apiPort } = target.info;
372
+ if (metroPort === null)
373
+ throw new Error(`lane ${target.lane.id} has no metro port`);
374
+ const relaunched = await target.mcp.call('run', {
375
+ yaml: relaunchYaml(metroPort, apiPort, target.info.platform),
376
+ appId: TARGET_APP.bundleId,
377
+ });
378
+ if (relaunched.isError)
379
+ throw new Error(`relaunching the app failed: ${brief(relaunched.text)}`);
380
+ const timed = await timedOp(target, 'read', true);
381
+ if (!timed.ok)
382
+ throw new Error(`the app's home screen never showed: ${timed.error ?? ''}`);
383
+ }
384
+ function sampleOf(started, ms, timed, untimed = {}, extra = {}) {
385
+ const phases = { ...extra, ...timed.phases };
386
+ for (const [key, value] of Object.entries(untimed))
387
+ phases[`untimed ${key}`] = value;
388
+ return {
389
+ ms: timed.ok ? ms : 0,
390
+ ok: timed.ok,
391
+ startedAt: started.toISOString(),
392
+ phases,
393
+ gateReads: timed.gateReads,
394
+ responseBytes: timed.responseBytes,
395
+ ...(timed.error === undefined ? {} : { error: timed.error }),
396
+ };
397
+ }
398
+ function deviceOf(target) {
399
+ const platform = target.info.platform ?? 'ios';
400
+ if (platform === 'web')
401
+ throw new Error('a browser lane is measured warm only');
402
+ return platform;
403
+ }
404
+ export function planFor(cell, op, targets, deps) {
405
+ if (deps.platform === 'web' && cell !== 'warm')
406
+ throw new Error(`a browser lane is measured warm only, not ${cell}`);
407
+ const first = targets[0];
408
+ if (!first)
409
+ throw new Error('no targets');
410
+ const bySim = new Map(targets.map(t => [t.info.sim, t]));
411
+ const target = (sim) => {
412
+ const found = bySim.get(sim);
413
+ if (!found)
414
+ throw new Error(`no target ${sim}`);
415
+ return found;
416
+ };
417
+ const env = benchEnv(deps.env);
418
+ switch (cell) {
419
+ case 'warm':
420
+ return {
421
+ cell,
422
+ op,
423
+ sims: targets.map(t => t.info.sim),
424
+ prepare: async () => {
425
+ for (const t of targets)
426
+ await ensureHome(t);
427
+ },
428
+ sample: async (sim) => {
429
+ const started = new Date();
430
+ const timed = await timedOp(target(sim), op, false);
431
+ return sampleOf(started, timed.phases.op ?? 0, timed);
432
+ },
433
+ };
434
+ case 'relaunch':
435
+ return {
436
+ cell,
437
+ op,
438
+ sims: [first.info.sim],
439
+ prepare: () => ensureHome(first),
440
+ sample: async () => {
441
+ const { metroPort, apiPort } = first.info;
442
+ if (metroPort === null)
443
+ throw new Error(`lane ${first.lane.id} has no metro port`);
444
+ const started = new Date();
445
+ const t0 = process.hrtime.bigint();
446
+ const relaunched = await first.mcp.call('run', {
447
+ yaml: relaunchYaml(metroPort, apiPort, deviceOf(first)),
448
+ appId: TARGET_APP.bundleId,
449
+ });
450
+ if (relaunched.isError)
451
+ throw new Error(`relaunch failed: ${brief(relaunched.text)}`);
452
+ const timed = await timedOp(first, op, true);
453
+ return sampleOf(started, elapsed(t0), timed, {}, { relaunch: relaunched.ms });
454
+ },
455
+ };
456
+ case 'cold':
457
+ return {
458
+ cell,
459
+ op,
460
+ sims: [first.info.sim],
461
+ prepare: () => ensureHome(first),
462
+ sample: async () => {
463
+ const p0 = process.hrtime.bigint();
464
+ await ensureHome(first);
465
+ await stopEngine(first.stateDir, first.lane);
466
+ const admitted = await admit(first, deps, 'Booted');
467
+ const prep = elapsed(p0);
468
+ const started = new Date();
469
+ const t0 = process.hrtime.bigint();
470
+ const timed = await timedOp(first, op, false);
471
+ return sampleOf(started, elapsed(t0), timed, { prepare: prep, admission: admitted });
472
+ },
473
+ };
474
+ case 'first-interaction':
475
+ return {
476
+ cell,
477
+ op,
478
+ sims: [first.info.sim],
479
+ prepare: () => Promise.resolve(),
480
+ sample: async () => {
481
+ await stopEngine(first.stateDir, first.lane);
482
+ await admit(first, deps, 'Booted');
483
+ const brought = await up(first.folder, env, deps.platform, false);
484
+ const started = new Date();
485
+ const t0 = process.hrtime.bigint();
486
+ const timed = await timedOp(first, op, true);
487
+ return sampleOf(started, elapsed(t0), timed, { up: brought.ms });
488
+ },
489
+ };
490
+ case 'cold-cold':
491
+ return {
492
+ cell,
493
+ op,
494
+ sims: [first.info.sim],
495
+ selfLoading: true,
496
+ prepare: () => Promise.resolve(),
497
+ sample: async () => {
498
+ const p0 = process.hrtime.bigint();
499
+ await stopEngine(first.stateDir, first.lane);
500
+ await deps.sim.shutdown(first.info.udid);
501
+ const admitted = await admit(first, deps, 'Shutdown');
502
+ const prep = elapsed(p0);
503
+ const started = new Date();
504
+ const t0 = process.hrtime.bigint();
505
+ const brought = await up(first.folder, env, deps.platform, false);
506
+ const timed = await timedOp(first, op, true);
507
+ return sampleOf(started, elapsed(t0), timed, { prepare: prep, admission: admitted }, { up: brought.ms, ...brought.steps });
508
+ },
509
+ };
510
+ }
511
+ }
@@ -0,0 +1,91 @@
1
+ import { spawn } from 'node:child_process';
2
+ import { createInterface } from 'node:readline';
3
+ /** A stdio MCP client for one `mcp` server process, timing each tools/call from request write to response line. */
4
+ export class McpClient {
5
+ child;
6
+ next = 0;
7
+ waiting = new Map();
8
+ exited;
9
+ stderrTail = '';
10
+ constructor(command, args, cwd, env) {
11
+ this.child = spawn(command, args, { cwd, env, stdio: ['pipe', 'pipe', 'pipe'] });
12
+ this.child.stderr?.setEncoding('utf8');
13
+ this.child.stderr?.on('data', (chunk) => {
14
+ this.stderrTail = (this.stderrTail + chunk).slice(-2000);
15
+ });
16
+ this.child.on('exit', (code, signal) => {
17
+ this.exited = signal ?? `code ${String(code)}`;
18
+ for (const resolve of this.waiting.values())
19
+ resolve({ error: { message: `the MCP server exited (${this.exited})` } });
20
+ this.waiting.clear();
21
+ });
22
+ if (this.child.stdout) {
23
+ createInterface({ input: this.child.stdout }).on('line', line => {
24
+ let message;
25
+ try {
26
+ message = JSON.parse(line);
27
+ }
28
+ catch {
29
+ return;
30
+ }
31
+ if (message.id === undefined)
32
+ return;
33
+ this.waiting.get(message.id)?.(message);
34
+ this.waiting.delete(message.id);
35
+ });
36
+ }
37
+ }
38
+ get pid() {
39
+ return this.child.pid;
40
+ }
41
+ request(method, params, timeoutMs) {
42
+ if (this.exited)
43
+ return Promise.resolve({ error: { message: `the MCP server exited (${this.exited}): ${this.stderrTail}` } });
44
+ const id = ++this.next;
45
+ return new Promise(resolve => {
46
+ const timer = setTimeout(() => {
47
+ this.waiting.delete(id);
48
+ resolve({ error: { message: `${method} got no answer within ${String(timeoutMs / 1000)} s` } });
49
+ }, timeoutMs);
50
+ this.waiting.set(id, message => {
51
+ clearTimeout(timer);
52
+ resolve(message);
53
+ });
54
+ this.child.stdin?.write(`${JSON.stringify({ jsonrpc: '2.0', id, method, params })}\n`);
55
+ });
56
+ }
57
+ async initialize() {
58
+ const answer = await this.request('initialize', { protocolVersion: '2025-06-18', capabilities: {}, clientInfo: { name: 'bench', version: '1' } }, 30_000);
59
+ if (answer.error)
60
+ throw new Error(`MCP initialize failed: ${answer.error.message ?? ''} ${this.stderrTail}`);
61
+ this.child.stdin?.write(`${JSON.stringify({ jsonrpc: '2.0', method: 'notifications/initialized' })}\n`);
62
+ }
63
+ async call(name, args, timeoutMs = 180_000) {
64
+ const started = process.hrtime.bigint();
65
+ const answer = await this.request('tools/call', { name, arguments: args }, timeoutMs);
66
+ const ms = Number(process.hrtime.bigint() - started) / 1e6;
67
+ if (answer.error)
68
+ return { ms, isError: true, text: answer.error.message ?? 'error', images: 0, bytes: 0 };
69
+ const content = answer.result?.content ?? [];
70
+ const text = content
71
+ .filter(c => c.type === 'text')
72
+ .map(c => c.text ?? '')
73
+ .join('\n');
74
+ const bytes = content.reduce((sum, c) => sum + (c.text?.length ?? 0) + Math.round(((c.data?.length ?? 0) * 3) / 4), 0);
75
+ return { ms, isError: answer.result?.isError === true, text, images: content.filter(c => c.type === 'image').length, bytes };
76
+ }
77
+ close() {
78
+ if (this.exited)
79
+ return Promise.resolve();
80
+ return new Promise(resolve => {
81
+ const timer = setTimeout(() => {
82
+ this.child.kill('SIGTERM');
83
+ }, 5000);
84
+ this.child.once('exit', () => {
85
+ clearTimeout(timer);
86
+ resolve();
87
+ });
88
+ this.child.stdin?.end();
89
+ });
90
+ }
91
+ }
@@ -0,0 +1,15 @@
1
+ export const CELLS = ['warm', 'relaunch', 'cold', 'first-interaction', 'cold-cold'];
2
+ export const OPS = ['read', 'screenshot', 'quick-tap', 'tap', 'click', 'flow'];
3
+ /** The ops each platform has, and the cells: a browser lane is measured warm. */
4
+ export const PLATFORM_OPS = {
5
+ ios: ['read', 'screenshot', 'quick-tap', 'tap', 'flow'],
6
+ android: ['read', 'screenshot', 'quick-tap', 'tap', 'flow'],
7
+ web: ['read', 'screenshot', 'click', 'flow'],
8
+ };
9
+ export const PLATFORM_CELLS = {
10
+ ios: CELLS,
11
+ android: CELLS.filter(cell => cell !== 'cold-cold'),
12
+ web: ['warm'],
13
+ };
14
+ export const SCHEMA = 'agentlane-bench/1';
15
+ export const MIN_PUBLISHED_SAMPLES = 16;