lynkeus 0.3.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (89) hide show
  1. package/lib/cases/coverage.d.ts +31 -0
  2. package/lib/cases/coverage.js +82 -0
  3. package/lib/cases/draft.d.ts +21 -0
  4. package/lib/cases/draft.js +53 -0
  5. package/lib/cases/fixtures.d.ts +11 -0
  6. package/lib/cases/fixtures.js +59 -8
  7. package/lib/cases/frames.d.ts +6 -0
  8. package/lib/cases/frames.js +28 -0
  9. package/lib/cases/nativeLogs.d.ts +14 -0
  10. package/lib/cases/nativeLogs.js +111 -0
  11. package/lib/cases/runDirs.d.ts +13 -0
  12. package/lib/cases/runDirs.js +48 -0
  13. package/lib/cases/runfile.d.ts +5 -1
  14. package/lib/cases/runfile.js +67 -4
  15. package/lib/cases/runner.d.ts +58 -1
  16. package/lib/cases/runner.js +322 -39
  17. package/lib/cases/templates.d.ts +2 -2
  18. package/lib/cases/templates.js +20 -2
  19. package/lib/cases/traces.js +4 -1
  20. package/lib/cli.js +27 -4
  21. package/lib/commands/cases.d.ts +8 -0
  22. package/lib/commands/cases.js +154 -25
  23. package/lib/commands/ci.d.ts +13 -0
  24. package/lib/commands/ci.js +184 -0
  25. package/lib/commands/coverage.d.ts +1 -0
  26. package/lib/commands/coverage.js +151 -0
  27. package/lib/commands/device.js +32 -2
  28. package/lib/commands/doctor.js +8 -2
  29. package/lib/commands/drive.js +3 -1
  30. package/lib/commands/fixtures.d.ts +7 -0
  31. package/lib/commands/fixtures.js +94 -0
  32. package/lib/commands/headless.d.ts +11 -0
  33. package/lib/commands/headless.js +189 -9
  34. package/lib/commands/init.d.ts +23 -0
  35. package/lib/commands/init.js +231 -26
  36. package/lib/commands/map.d.ts +2 -0
  37. package/lib/commands/map.js +18 -14
  38. package/lib/commands/os.js +43 -10
  39. package/lib/commands/report.d.ts +4 -0
  40. package/lib/commands/report.js +156 -20
  41. package/lib/config.d.ts +41 -0
  42. package/lib/config.js +30 -0
  43. package/lib/device/agentVersion.d.ts +7 -0
  44. package/lib/device/agentVersion.js +14 -0
  45. package/lib/device/devices.js +6 -0
  46. package/lib/device/driver.js +2 -2
  47. package/lib/knowledge/analyticsTraffic.d.ts +10 -0
  48. package/lib/knowledge/analyticsTraffic.js +109 -0
  49. package/lib/knowledge/errors.js +6 -0
  50. package/lib/knowledge/events.d.ts +2 -0
  51. package/lib/knowledge/why.d.ts +2 -0
  52. package/lib/knowledge/why.js +1 -1
  53. package/lib/mcp/server.js +45 -4
  54. package/lib/runtime.d.ts +9 -1
  55. package/lib/runtime.js +25 -3
  56. package/package.json +7 -4
  57. package/src/cases/coverage.ts +99 -0
  58. package/src/cases/draft.ts +75 -0
  59. package/src/cases/fixtures.ts +57 -8
  60. package/src/cases/frames.ts +30 -0
  61. package/src/cases/nativeLogs.ts +112 -0
  62. package/src/cases/runDirs.ts +48 -0
  63. package/src/cases/runfile.ts +64 -4
  64. package/src/cases/runner.ts +372 -42
  65. package/src/cases/templates.ts +20 -2
  66. package/src/cases/traces.ts +3 -1
  67. package/src/cli.ts +27 -4
  68. package/src/commands/cases.ts +165 -23
  69. package/src/commands/ci.ts +189 -0
  70. package/src/commands/coverage.ts +147 -0
  71. package/src/commands/device.ts +27 -2
  72. package/src/commands/doctor.ts +9 -2
  73. package/src/commands/drive.ts +3 -1
  74. package/src/commands/fixtures.ts +102 -0
  75. package/src/commands/headless.ts +197 -9
  76. package/src/commands/init.ts +270 -26
  77. package/src/commands/map.ts +20 -15
  78. package/src/commands/os.ts +38 -10
  79. package/src/commands/report.ts +162 -18
  80. package/src/config.ts +48 -0
  81. package/src/device/agentVersion.ts +17 -0
  82. package/src/device/devices.ts +5 -0
  83. package/src/device/driver.ts +2 -2
  84. package/src/knowledge/analyticsTraffic.ts +142 -0
  85. package/src/knowledge/errors.ts +6 -0
  86. package/src/knowledge/events.ts +2 -1
  87. package/src/knowledge/why.ts +3 -1
  88. package/src/mcp/server.ts +45 -4
  89. package/src/runtime.ts +24 -3
@@ -193,7 +193,7 @@ export const decodeJson = (body: unknown): Span[] => {
193
193
 
194
194
  // What a report keeps of a span's attributes: what it did, not every label an instrumentation adds.
195
195
  const KEEP =
196
- /^(http\.(request\.)?method|http\.(response\.)?status_code|http\.route|http\.target|url\.(path|full)|server\.address|net\.peer\.name|db\.(system|statement|operation|sql\.table)|messaging\.|code\.(namespace|function)|rpc\.|exception\.message|enduser\.id)/;
196
+ /^(http\.(request\.)?method|http\.(response\.)?status_code|http\.route|http\.target|url\.(path|full)|server\.address|net\.peer\.name|messaging\.|code\.(namespace|function)|rpc\.|exception\.message|enduser\.id)/;
197
197
 
198
198
  /**
199
199
  * Listens for OTLP traces while a run goes on: a backend with OTEL_EXPORTER_OTLP_ENDPOINT pointed here sends
@@ -272,6 +272,8 @@ export class TraceSink {
272
272
  return undefined;
273
273
  };
274
274
  for (const span of [...spans].sort((a, b) => (a.startNs < b.startNs ? -1 : 1))) {
275
+ // A database query never leaves the machine: its statement names the backend's tables and columns.
276
+ if (span.attributes['db.system'] !== undefined) continue;
275
277
  const where = place(span);
276
278
  if (!where) continue;
277
279
  const startedMs = [...steps.values()].find((s) => s.index === where.step)!.startedMs;
package/src/cli.ts CHANGED
@@ -11,16 +11,30 @@ import { driveCommands } from './commands/drive.js';
11
11
  import { headlessCommands } from './commands/headless.js';
12
12
  import { mapCommands } from './commands/map.js';
13
13
  import { osCommands } from './commands/os.js';
14
+ import { ciCommands } from './commands/ci.js';
15
+ import { coverageCommands } from './commands/coverage.js';
16
+ import { fixturesCommands } from './commands/fixtures.js';
14
17
  import { reportCommands } from './commands/report.js';
15
18
  import { runsCommands } from './commands/runs.js';
16
- import { graphFile, loadConfig, loadGraph, portFromEnv } from './config.js';
17
- import { agentMismatch } from './device/agentVersion.js';
19
+ import { graphFile, loadConfig, loadGraph, portFromEnv, readToken } from './config.js';
20
+ import { agentMismatch, appSwitched, helloIdentity } from './device/agentVersion.js';
18
21
  import { HEADLESS_HOST, pickAgentDevice, pickAndroidDevice } from './device/devices.js';
19
22
  import { Device } from './device/driver.js';
20
23
  import { type Base, commands, find, helpFor, inSession, referenceMarkdown, register, resolve, usage } from './registry.js';
21
24
  import { execute } from './runtime.js';
22
25
 
23
- register([...driveCommands, ...osCommands, ...mapCommands, ...runsCommands, ...caseCommands, ...reportCommands, ...headlessCommands]);
26
+ register([
27
+ ...driveCommands,
28
+ ...osCommands,
29
+ ...mapCommands,
30
+ ...runsCommands,
31
+ ...caseCommands,
32
+ ...coverageCommands,
33
+ ...ciCommands,
34
+ ...fixturesCommands,
35
+ ...reportCommands,
36
+ ...headlessCommands,
37
+ ]);
24
38
 
25
39
  /** The global flags, read leniently from the whole argv before any command parses its own. */
26
40
  const globals = (argv: string[]) => {
@@ -42,6 +56,8 @@ export type Session = { ctx: Base; stop: () => Promise<void> };
42
56
 
43
57
  // Once per process: a suite reattaches for every case.
44
58
  let mismatchTold = false;
59
+ let switchTold = false;
60
+ let lastApp: string | undefined;
45
61
 
46
62
  // `ownHost`: the port belongs to a host this run started for a case; whatever dials in is the device.
47
63
  const contextFor = (argv: string[], attached = false, port?: number, ownHost = false): Session => {
@@ -56,8 +72,15 @@ const contextFor = (argv: string[], attached = false, port?: number, ownHost = f
56
72
  const appId = (g.app as string | undefined) ?? process.env.LYNKEUS_APP_ID ?? config.app;
57
73
  const started = performance.now();
58
74
  const attachedIn = (d: Device) => err(`attached in ${Math.round(performance.now() - started)}ms (${d.info.name})`);
59
- const server = new AgentServer({ token: process.env.LYNKEUS_TOKEN, port: port ?? portFromEnv(config) });
75
+ const server = new AgentServer({ token: readToken(root), port: port ?? portFromEnv(config) });
60
76
  server.on('hello', (hello: HelloEvent) => {
77
+ const identity = helloIdentity(hello);
78
+ const switched = switchTold ? undefined : appSwitched(lastApp, identity, server.port);
79
+ lastApp = identity;
80
+ if (switched) {
81
+ switchTold = true;
82
+ err(switched);
83
+ }
61
84
  const told = mismatchTold ? undefined : agentMismatch(hello);
62
85
  if (!told) return;
63
86
  mismatchTold = true;
@@ -2,6 +2,7 @@ import { type ChildProcess, spawn } from 'node:child_process';
2
2
  import fs from 'node:fs';
3
3
  import os from 'node:os';
4
4
  import path from 'node:path';
5
+ import { stripVTControlCharacters } from 'node:util';
5
6
 
6
7
  import { Fixtures } from '../cases/fixtures.js';
7
8
  import { readCases } from '../cases/format.js';
@@ -18,6 +19,8 @@ import { eventsFile } from '../knowledge/events.js';
18
19
  import { type Base, define } from '../registry.js';
19
20
  import { describeChanges, pushRun, service } from './report.js';
20
21
  import { TraceSink } from '../cases/traces.js';
22
+ import { megabytes, sizeOf } from '../cases/runDirs.js';
23
+ import { type NativeLogs, startNativeLogs } from '../cases/nativeLogs.js';
21
24
 
22
25
  const startHost = async (root: string, err: (line: string) => void): Promise<ChildProcess> => {
23
26
  // Its own process group: the host starts a test runner of its own, and stopping one without the other leaves an app dialing in.
@@ -27,8 +30,27 @@ const startHost = async (root: string, err: (line: string) => void): Promise<Chi
27
30
  stdio: ['ignore', 'pipe', 'pipe'],
28
31
  detached: true,
29
32
  });
33
+ // Everything the host prints, for the render errors and warnings a failed case points at; the console keeps the run's own lines.
34
+ const logFile = path.join(os.tmpdir(), `lynkeus-headless-${process.pid}.log`);
35
+ const logStream = fs.createWriteStream(logFile);
36
+ logStream.on('error', () => {});
37
+ host.once('exit', () => logStream.end());
30
38
  let output = '';
31
39
  const tail = (n: number) => output.trim().split('\n').slice(-n).join(' · ');
40
+ // `headless start` ends a crash with what it usually means; without that, the error line jest printed.
41
+ const why = () => {
42
+ const plain = stripVTControlCharacters(output);
43
+ const explained = plain.split('lynkeus: what this looks like')[1];
44
+ if (explained)
45
+ return explained
46
+ .trim()
47
+ .split('\n')
48
+ .map((line) => line.trim())
49
+ .join(' · ');
50
+ // Jest prints a suite that throws while loading as "● Test suite failed to run", then the thrown message.
51
+ const thrown = plain.match(/Test suite failed to run\s*\n\s*\n?\s*(\S.*)$/m)?.[1]?.trim();
52
+ return thrown ?? plain.match(/^\s*(\w*Error\b.*|Cannot find module.*)$/m)?.[1]?.trim() ?? tail(3);
53
+ };
32
54
  await new Promise<void>((resolve, reject) => {
33
55
  const settle = (error?: Error) => {
34
56
  clearTimeout(timer);
@@ -37,15 +59,17 @@ const startHost = async (root: string, err: (line: string) => void): Promise<Chi
37
59
  };
38
60
  const timer = setTimeout(() => settle(new Error(`the host did not come up within 180 s: ${tail(3)}`)), 180_000);
39
61
  const read = (chunk: Buffer) => {
62
+ logStream.write(chunk);
40
63
  output = (output + chunk.toString()).slice(-4000);
41
64
  if (/waiting for a driver/.test(output)) settle();
42
- else if (/Test suite failed/.test(output)) settle(new Error(`the host failed to start: ${tail(5)}`));
43
65
  };
44
66
  host.stdout?.on('data', read);
45
67
  host.stderr?.on('data', read);
46
- host.once('exit', (code) => settle(new Error(`the host exited (${code}): ${tail(3)}`)));
68
+ // A suite that fails to load exits right after printing why, so its explanation is read in full here.
69
+ host.once('exit', (code) => settle(new Error(`${/Test suite failed/.test(output) ? 'the host failed to start' : `the host exited (${code})`}: ${why()}`)));
47
70
  });
48
- err('host: up');
71
+ // The listeners above stay: the log takes all the host prints until it stops.
72
+ err(`host: up (its log: ${logFile})`);
49
73
  return host;
50
74
  };
51
75
 
@@ -102,19 +126,27 @@ const idleShare = async (): Promise<number> => {
102
126
  return after.all > before.all ? (after.idle - before.idle) / (after.all - before.all) : 1;
103
127
  };
104
128
 
105
- /** The case files named, or every case under the cases directory; a sentence when there are none. */
106
- const caseFiles = (args: string[], home: string, casesDir: string | undefined): string[] | string => {
129
+ /** Every case under a directory and its subdirectories, in path order, so a suite can group its cases in folders. */
130
+ const casesIn = (dir: string): string[] =>
131
+ fs
132
+ .readdirSync(dir, { withFileTypes: true })
133
+ .filter((e) => !e.name.startsWith('.') && e.name !== 'node_modules')
134
+ .sort((a, b) => a.name.localeCompare(b.name))
135
+ .flatMap((e) => {
136
+ const full = path.join(dir, e.name);
137
+ if (e.isDirectory()) return casesIn(full);
138
+ return e.name.endsWith('.md') && !/^(README|FORMAT|CATALOG)\.md$/i.test(e.name) ? [full] : [];
139
+ });
140
+
141
+ /** The case files named (a directory stands for every case under it), or every case under the cases directory; a sentence when there are none. */
142
+ export const caseFiles = (args: string[], home: string, casesDir: string | undefined): string[] | string => {
107
143
  const dir = path.resolve(home, casesDir ?? 'cases');
108
- const files = args.length
109
- ? args.map((f) => path.resolve(f))
110
- : fs.existsSync(dir)
111
- ? fs
112
- .readdirSync(dir)
113
- .filter((f) => f.endsWith('.md') && !/^(README|FORMAT|CATALOG)\.md$/i.test(f) && !f.startsWith('.'))
114
- .sort()
115
- .map((f) => path.join(dir, f))
116
- : [];
117
- return files.length > 0 ? files : `no cases under ${path.relative(home, dir) || '.'}; \`lynkeus case init\` writes one`;
144
+ const named = args.map((f) => path.resolve(f));
145
+ const missing = named.find((f) => !fs.existsSync(f));
146
+ if (missing) return `no such case file: ${path.relative(process.cwd(), missing) || missing}`;
147
+ const files = named.length ? named.flatMap((f) => (fs.statSync(f).isDirectory() ? casesIn(f) : [f])) : fs.existsSync(dir) ? casesIn(dir) : [];
148
+ const where = named.length === 1 ? path.relative(home, named[0]!) : path.relative(home, dir);
149
+ return files.length > 0 ? files : `no cases under ${where || '.'}; \`lynkeus case init\` writes one`;
118
150
  };
119
151
 
120
152
  type PrepareOptions = { base: Base; config: CasesConfig; store: Store; open: () => Fixtures; limit: number; idle: number; jobs: number };
@@ -169,15 +201,48 @@ const spawnHost = (command: string, port: number, job: number, cwd: string): Chi
169
201
  });
170
202
 
171
203
  /** A run that could not be uploaded still ran: the failure is said, not raised. */
172
- const pushed = async (dir: string, to: Exclude<ReturnType<typeof service>, string>, project: string) => {
204
+ // A run that kept evidence but none of a kind says so when it ends, not only as a gap in the report.
205
+ export const missingEvidence = (reports: CaseReport[], asked: { video: boolean }): string[] => {
206
+ if (reports.length === 0) return [];
207
+ const steps = reports.flatMap((r) => r.steps);
208
+ const fromApp = steps.some((s) => s.sent?.some((e) => e.via !== 'server')) || reports.some((r) => (r.events?.length ?? 0) > 0);
209
+ return [
210
+ ...(fromApp ? [] : ['⚠ no case kept an analytics event from the app (the agent keeps the calls that reach Mixpanel, AppsFlyer and Firebase Analytics).']),
211
+ ...(asked.video && !reports.some((r) => r.video)
212
+ ? ['⚠ --video kept no film: this host does not film (fastsim does; a simulator, an emulator or headless keep frames only).']
213
+ : []),
214
+ ];
215
+ };
216
+
217
+ const pushed = async (
218
+ dir: string,
219
+ to: Exclude<ReturnType<typeof service>, string>,
220
+ project: string,
221
+ err: (line: string) => void,
222
+ afterPush: 'keep' | 'delete' = 'keep',
223
+ ) => {
173
224
  try {
174
- const run = await pushRun(dir, { ...to, project });
175
- return [`report: ${run.url}${run.refused.length ? ` (not kept on this plan: ${run.refused.join(', ')})` : ''}`, ...describeChanges(run.changes)].join('\n');
225
+ const run = await pushRun(dir, { ...to, project, onReady: (url) => err(`report: ${url} (its frames are on their way)`), onProgress: err });
226
+ return [
227
+ `report: ${run.url}${run.refused.length ? ` (not kept on this plan: ${run.refused.join(', ')})` : ''}`,
228
+ ...describeChanges(run.changes),
229
+ keptOrDeleted(dir, afterPush, run.refused.length === 0),
230
+ ].join('\n');
176
231
  } catch (error) {
177
232
  return `the report was not uploaded: ${(error as Error).message}`;
178
233
  }
179
234
  };
180
235
 
236
+ // After a push: the folder deleted when the config asks and the service took every file, else how much it holds.
237
+ export const keptOrDeleted = (dir: string, afterPush: 'keep' | 'delete', complete: boolean): string => {
238
+ const bytes = sizeOf(dir);
239
+ if (afterPush === 'delete' && complete) {
240
+ fs.rmSync(dir, { recursive: true, force: true });
241
+ return `run folder deleted after the push (${megabytes(bytes)})`;
242
+ }
243
+ return `run folder: ${megabytes(bytes)} on disk${afterPush === 'delete' ? ', kept because some files were not taken' : ''} · \`lynkeus report prune ${path.dirname(dir)} --older 7d\` frees old ones`;
244
+ };
245
+
181
246
  export const caseCommands = [
182
247
  define({
183
248
  name: 'case run',
@@ -194,7 +259,32 @@ export const caseCommands = [
194
259
  report: { type: 'string', help: 'append one JSON line per case', value: 'file' },
195
260
  events: { type: 'string', help: 'keep what the app told its analytics, per case, as <dir>/<case id>/events.jsonl', value: 'dir' },
196
261
  out: { type: 'string', help: 'write the run as a directory a report can read: run.json and cases.jsonl, a line per attempt', value: 'dir' },
262
+ 'no-app-events': {
263
+ type: 'boolean',
264
+ help: "with --out: leave the app's own event list unread. Reading it can turn on the app's tracking debug, which may draw on the screen; frames two runs compare should not have it",
265
+ },
197
266
  screenshots: { type: 'boolean', help: 'with --out: a frame of the screen after every step of the app' },
267
+ settle: {
268
+ type: 'number',
269
+ help: 'with --screenshots: how long the screen must stay still before a frame, in ms; 700 for frames two runs can be compared by',
270
+ default: 150,
271
+ value: 'ms',
272
+ },
273
+ 'frames-on-change': {
274
+ type: 'boolean',
275
+ help: 'with --screenshots: a step that only looks (app.see, app.wait) on an unchanged screen shares the last frame',
276
+ },
277
+ still: {
278
+ type: 'number',
279
+ help: 'with --screenshots, on a simulator or emulator: take each frame again until two in a row match, this many ms apart (a native transition the agent cannot see)',
280
+ value: 'ms',
281
+ },
282
+ frames: {
283
+ type: 'string',
284
+ help: "with --screenshots: avif, the default and a few percent of the bytes (a simulator's or emulator's PNG is encoded again), or png, exact, only for frames two runs are compared by pixel for pixel",
285
+ default: 'avif',
286
+ value: 'avif|png',
287
+ },
198
288
  video: { type: 'boolean', help: 'with --out: film each case, where the host films (the step says at what second it began)' },
199
289
  traces: {
200
290
  type: 'boolean',
@@ -204,6 +294,7 @@ export const caseCommands = [
204
294
  push: { type: 'boolean', help: 'with --out: upload the run to the reports service when it ends and print the report’s address' },
205
295
  project: { type: 'string', help: 'with --push: the project the run is filed under (default: the app’s folder name)', value: 'name' },
206
296
  'video-fps': { type: 'number', help: 'frames per second of the film', default: 30, value: 'n' },
297
+ 'video-scale': { type: 'number', help: 'pixels per point of the film, where the host takes one (fastsim: 1 to 3, 2 unless told)', value: 'n' },
207
298
  flakes: { type: 'string', help: 'the ledger of how each case did across runs (default .lynkeus/flakes.json)', value: 'file' },
208
299
  prepared: { type: 'string', help: 'where `case prepare` leaves setups made ahead; a case found there skips its setup', value: 'dir' },
209
300
  jobs: { type: 'number', help: 'cases at once, each on its own host (needs --ports)', value: 'n', default: 1 },
@@ -220,7 +311,33 @@ export const caseCommands = [
220
311
  },
221
312
  needs: 'nothing',
222
313
  session: false,
223
- mcp: false,
314
+ mcp: 'core',
315
+ mcpOmit: [
316
+ 'seen',
317
+ 'report',
318
+ 'events',
319
+ 'settle',
320
+ 'still',
321
+ 'frames-on-change',
322
+ 'frames',
323
+ 'traces',
324
+ 'traces-port',
325
+ 'video-fps',
326
+ 'video-scale',
327
+ 'flakes',
328
+ 'prepared',
329
+ 'jobs',
330
+ 'ports',
331
+ 'host-command',
332
+ 'in-order',
333
+ 'ahead',
334
+ 'idle',
335
+ 'continue',
336
+ 'video',
337
+ 'project',
338
+ ],
339
+ mcpDetails:
340
+ 'Runs case files (default: all in cases.dir) against the attached app, or with `host` in Node. A case that fails comes back with its screen and reason. `out` + `screenshots` + `push` uploads the run and prints its address.',
224
341
  run: async (ctx) => {
225
342
  const config = ctx.config.cases ?? {};
226
343
  const home = config.base ?? ctx.root;
@@ -241,12 +358,18 @@ export const caseCommands = [
241
358
  ? {
242
359
  dir: out.evidenceDir(id, attempt),
243
360
  screenshots: !!ctx.flags.screenshots,
244
- ...(ctx.flags.video ? { video: { fps: Number(ctx.flags['video-fps'] ?? 30) } } : {}),
361
+ settleMs: Number(ctx.flags.settle ?? 150),
362
+ ...(ctx.flags.still ? { stillMs: Number(ctx.flags.still) } : {}),
363
+ ...(ctx.flags['frames-on-change'] ? { onChange: true } : {}),
364
+ frames: ctx.flags.frames === 'png' ? ('png' as const) : ('avif' as const),
365
+ ...(ctx.flags.video
366
+ ? { video: { fps: Number(ctx.flags['video-fps'] ?? 30), ...(ctx.flags['video-scale'] ? { scale: Number(ctx.flags['video-scale']) } : {}) } }
367
+ : {}),
245
368
  }
246
369
  : undefined;
247
370
  const ledger = dry ? undefined : new Flakes(path.resolve(ctx.root, typeof ctx.flags.flakes === 'string' ? ctx.flags.flakes : '.lynkeus/flakes.json'));
248
371
  // A run file files each event under its step, so it needs them collected.
249
- const events = typeof ctx.flags.events === 'string' || out !== undefined;
372
+ const events = typeof ctx.flags.events === 'string' || (out !== undefined && !ctx.flags['no-app-events']);
250
373
  const claimed = dry
251
374
  ? undefined
252
375
  : claimPorts(
@@ -297,6 +420,15 @@ export const caseCommands = [
297
420
  const hostCommand = typeof ctx.flags['host-command'] === 'string' && !dry ? ctx.flags['host-command'] : undefined;
298
421
  if ((jobs > 1 || hostCommand) && (ports.length < jobs || ports.some((p) => !Number.isInteger(p)) || !ctx.forPort))
299
422
  return { text: `--jobs and --host-command need a driver port per job: --ports with ${jobs}`, code: 2 };
423
+ // One simulator or emulator, one log: with several jobs a line could not be told to its case.
424
+ const native: NativeLogs | undefined =
425
+ out && jobs === 1 && !hostCommand && !ctx.flags.host && ctx.config.app
426
+ ? await ctx
427
+ .device()
428
+ .then((d) => d.resolved())
429
+ .then((info) => startNativeLogs(info, ctx.config.app as string))
430
+ .catch(() => undefined)
431
+ : undefined;
300
432
  // Several jobs finish together only if the longest cases start first: one left for the end runs alone
301
433
  // while the other jobs wait. A case the ledger has not timed goes first too, as it may be long.
302
434
  if (jobs > 1 && ledger && !ctx.flags['in-order']) {
@@ -359,6 +491,7 @@ export const caseCommands = [
359
491
  continue: !!ctx.flags.continue,
360
492
  routes: seenDir ? (host ? 'fresh' : true) : false,
361
493
  traces,
494
+ nativeLogs: native,
362
495
  });
363
496
  if (seenDir && report.result === 'passed' && report.routes?.length) {
364
497
  fs.mkdirSync(seenDir, { recursive: true });
@@ -407,6 +540,7 @@ export const caseCommands = [
407
540
  await Promise.all(Array.from({ length: jobs }, (_, i) => job(i)));
408
541
  await preparing;
409
542
  if (out) out.finish({ platform: seen?.platform, native: seen?.native });
543
+ native?.stop();
410
544
  } finally {
411
545
  stop();
412
546
  }
@@ -414,9 +548,15 @@ export const caseCommands = [
414
548
  const unsupported = count('unsupported');
415
549
  const summary = [
416
550
  ...reports.map(
417
- (r) => `${{ passed: '✅', failed: '❌', unsupported: '🚫' }[r.result]} ${r.id}${r.retried && r.result === 'passed' ? ' ⚠ passed on retry' : ''}`,
551
+ (r) =>
552
+ `${{ passed: '✅', failed: '❌', unsupported: '🚫' }[r.result]} ${r.id}${r.retried && r.result === 'passed' ? ' ⚠ passed on retry' : ''}` +
553
+ // In a session or over MCP the caller reads this answer and nothing else: a case that did not pass brings its steps.
554
+ (ctx.attached && r.result !== 'passed' ? `\n${describe(r)}` : ''),
418
555
  ),
419
556
  `${count('passed')}/${reports.length} passed${unsupported ? `, ${unsupported} cannot run on this host` : ''}`,
557
+ // Its full path: a relative --out resolves against where the CLI was started, which over MCP is the app's folder.
558
+ ...(out ? [`run kept in ${path.resolve(ctx.flags.out as string)}`] : []),
559
+ ...(out ? missingEvidence(reports, { video: !!ctx.flags.video }) : []),
420
560
  ...(ledger ? ledger.flaky(reports.map((r) => r.id)) : []),
421
561
  ...(reportsTo && out
422
562
  ? [
@@ -424,6 +564,8 @@ export const caseCommands = [
424
564
  path.resolve(ctx.flags.out as string),
425
565
  reportsTo,
426
566
  typeof ctx.flags.project === 'string' ? ctx.flags.project : path.basename(ctx.root),
567
+ ctx.err,
568
+ ctx.config.report?.afterPush === 'delete' ? 'delete' : 'keep',
427
569
  ),
428
570
  ]
429
571
  : []),
@@ -481,7 +623,7 @@ export const caseCommands = [
481
623
  flags: { force: { type: 'boolean', help: 'overwrite what is there' } },
482
624
  needs: 'nothing',
483
625
  session: false,
484
- mcp: false,
626
+ mcp: 'core',
485
627
  run: async (ctx) => {
486
628
  const dir = path.resolve(ctx.root, ctx.config.cases?.dir ?? 'cases');
487
629
  fs.mkdirSync(dir, { recursive: true });
@@ -0,0 +1,189 @@
1
+ import { type ChildProcess, spawn, spawnSync } from 'node:child_process';
2
+ import fs from 'node:fs';
3
+ import path from 'node:path';
4
+
5
+ import { coverageOf, impactOf, readSeen } from '../cases/coverage.js';
6
+ import { readCases } from '../cases/format.js';
7
+ import { screensAffectedBy } from '../knowledge/query.js';
8
+ import { define } from '../registry.js';
9
+ import { caseFiles } from './cases.js';
10
+ import { changedSources } from './map.js';
11
+
12
+ const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
13
+
14
+ /** The case results of a run folder, by each case's last attempt. */
15
+ export const resultsOf = (dir: string): { id: string; result: string; title?: string }[] => {
16
+ const file = path.join(dir, 'cases.jsonl');
17
+ if (!fs.existsSync(file)) return [];
18
+ const last = new Map<string, { id: string; result: string; title?: string }>();
19
+ for (const line of fs.readFileSync(file, 'utf8').split('\n')) {
20
+ if (!line.trim()) continue;
21
+ const c = JSON.parse(line) as { id: string; result: string; title?: string };
22
+ last.set(c.id, c);
23
+ }
24
+ return [...last.values()];
25
+ };
26
+
27
+ /** The summary a CI job shows: how many passed, which failed, and the report. */
28
+ export const summaryOf = (results: { id: string; result: string; title?: string }[], report: string | undefined, picked: string): string => {
29
+ const failed = results.filter((r) => r.result === 'failed');
30
+ const passed = results.filter((r) => r.result === 'passed').length;
31
+ return [
32
+ `### lynkeus: ${passed}/${results.length} cases passed${failed.length ? `, ${failed.length} failed` : ''}`,
33
+ '',
34
+ picked,
35
+ ...(report ? ['', `[Open the report](${report})`] : []),
36
+ ...(failed.length ? ['', '| Failed | |', '|---|---|', ...failed.map((f) => `| \`${f.id}\` | ${f.title ?? ''} |`)] : []),
37
+ '',
38
+ ].join('\n');
39
+ };
40
+
41
+ const waitFor = async (ok: () => boolean | Promise<boolean>, timeoutMs: number, every = 3000): Promise<boolean> => {
42
+ const until = Date.now() + timeoutMs;
43
+ for (;;) {
44
+ if (await ok()) return true;
45
+ if (Date.now() > until) return false;
46
+ await sleep(every);
47
+ }
48
+ };
49
+
50
+ export const ciCommands = [
51
+ define({
52
+ name: 'ci',
53
+ group: 'runs',
54
+ summary: 'Run the suite the way a CI job does: bring the backend and N hosts up, run the cases across them, report, tear down',
55
+ details:
56
+ 'Reads `ci` in lynkeus.config.json: `backend.up` / `backend.down` (the project’s commands; lynkeus knows no backend) and `backend.ready` (a URL to wait for), `hosts.count` and `hosts.command` (default: a headless host; `{port}` is where it dials), `push`, and extra `run` flags. --affected runs only the cases a branch’s changes reach (as `case impact` finds them). Writes the job summary to $GITHUB_STEP_SUMMARY when it is set. The exit code is the suite’s.',
57
+ flags: {
58
+ affected: { type: 'boolean', help: 'only the cases the changes against --base reach' },
59
+ base: { type: 'string', help: 'git ref --affected diffs against', default: 'origin/main', value: 'ref' },
60
+ jobs: { type: 'number', help: 'how many hosts and jobs (default ci.hosts.count, else 2)', value: 'n' },
61
+ out: { type: 'string', help: 'where the run goes', default: 'lynkeus-run', value: 'dir' },
62
+ 'keep-up': { type: 'boolean', help: 'leave the backend and the hosts up afterwards' },
63
+ 'dry-run': { type: 'boolean', help: 'say what it would do and run nothing' },
64
+ },
65
+ needs: 'nothing',
66
+ session: false,
67
+ mcp: false,
68
+ run: async (ctx) => {
69
+ const ci = ctx.config.ci ?? {};
70
+ const config = ctx.config.cases ?? {};
71
+ const home = config.base ?? ctx.root;
72
+ const self = process.argv[1]!;
73
+ const say = (line: string) => ctx.err(`▸ ${line}`);
74
+ const jobs = Math.max(1, Number(ctx.flags.jobs ?? ci.hosts?.count ?? 2));
75
+ const out = path.resolve(ctx.root, String(ctx.flags.out));
76
+
77
+ // Which cases: all, or the ones the branch's changes reach.
78
+ const all = caseFiles([], home, config.dir);
79
+ if (typeof all === 'string') return { text: all, code: 1 };
80
+ let files = all;
81
+ let picked = `All ${all.length} case files.`;
82
+ if (ctx.flags.affected) {
83
+ const graphFile = path.join(ctx.root, '.lynkeus', 'graph.json');
84
+ if (!fs.existsSync(graphFile)) spawnSync(process.execPath, [self, 'map', 'build'], { cwd: ctx.root, stdio: 'inherit' });
85
+ const cases = all.flatMap(readCases);
86
+ const graph = ctx.graph();
87
+ const coverage = coverageOf(cases, Object.keys(graph.screens).sort(), readSeen(path.join(ctx.root, '.lynkeus', 'seen')));
88
+ const changed = changedSources(ctx.root, String(ctx.flags.base), ctx.err);
89
+ const impact = impactOf(screensAffectedBy(graph, changed), coverage);
90
+ files = [...new Set(impact.cases.map((c) => c.file))];
91
+ picked = `${files.length} of ${all.length} case files: the ones the ${changed.length} changed source file(s) against ${ctx.flags.base} reach.`;
92
+ if (files.length === 0) {
93
+ if (process.env.GITHUB_STEP_SUMMARY) fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, `### lynkeus: no case to run\n\n${picked}\n`);
94
+ return { text: `no case to run: ${picked}`, json: { files } };
95
+ }
96
+ }
97
+ const ports = Array.from({ length: jobs }, (_, i) => 8123 + i);
98
+ const hostCommand = ci.hosts?.command ?? `LYNKEUS_PORT={port} "${process.execPath}" "${self}" headless start`;
99
+ const runArgs = [
100
+ 'case',
101
+ 'run',
102
+ ...files,
103
+ '--jobs',
104
+ String(jobs),
105
+ '--ports',
106
+ ports.join(','),
107
+ '--out',
108
+ out,
109
+ '--screenshots',
110
+ ...(ci.push ? ['--push'] : []),
111
+ ...(ci.run ?? []),
112
+ ];
113
+ if (ctx.flags['dry-run']) {
114
+ const plan = [
115
+ picked,
116
+ ci.backend ? `backend: \`${ci.backend.up}\`${ci.backend.ready ? `, ready at ${ci.backend.ready}` : ''}` : 'backend: none declared (ci.backend)',
117
+ `hosts: ${jobs} × \`${hostCommand}\` on ${ports.join(', ')}`,
118
+ `run: lynkeus ${runArgs.slice(0, 2).join(' ')} … ${runArgs.slice(2 + files.length).join(' ')}`,
119
+ ];
120
+ return { text: plan.join('\n'), json: { files, ports, hostCommand, runArgs } };
121
+ }
122
+
123
+ const hosts: ChildProcess[] = [];
124
+ const down = () => {
125
+ if (ctx.flags['keep-up']) return say('left up (--keep-up)');
126
+ for (const h of hosts)
127
+ if (h.pid)
128
+ try {
129
+ process.kill(-h.pid, 'SIGTERM');
130
+ } catch {}
131
+ if (ci.backend?.down) {
132
+ say('backend down');
133
+ spawnSync('/bin/sh', ['-c', ci.backend.down], { cwd: ctx.root, stdio: 'inherit' });
134
+ }
135
+ };
136
+ try {
137
+ if (ci.backend) {
138
+ say(`backend up: ${ci.backend.up}`);
139
+ const up = spawnSync('/bin/sh', ['-c', ci.backend.up], { cwd: ctx.root, stdio: 'inherit' });
140
+ if (up.status !== 0) return { text: `the backend did not come up (\`${ci.backend.up}\` exited ${up.status})`, code: 1 };
141
+ if (ci.backend.ready) {
142
+ const url = ci.backend.ready;
143
+ say(`waiting for ${url}`);
144
+ const ready = await waitFor(
145
+ () =>
146
+ fetch(url).then(
147
+ (r) => r.ok,
148
+ () => false,
149
+ ),
150
+ ci.backend.timeoutMs ?? 300_000,
151
+ );
152
+ if (!ready) return { text: `${url} never answered`, code: 1 };
153
+ }
154
+ }
155
+ say(`${jobs} host${jobs === 1 ? '' : 's'}: ${hostCommand}`);
156
+ for (const port of ports)
157
+ hosts.push(
158
+ spawn('/bin/sh', ['-c', hostCommand.replaceAll('{port}', String(port))], { cwd: ctx.root, stdio: ['pipe', 'ignore', 'ignore'], detached: true }),
159
+ );
160
+ // A host compiles the app before it answers: the slow part.
161
+ for (const port of ports) {
162
+ const answered = () =>
163
+ spawnSync(process.execPath, [self, 'screen'], {
164
+ cwd: ctx.root,
165
+ env: { ...process.env, LYNKEUS_PORT: String(port) },
166
+ stdio: 'ignore',
167
+ timeout: 20_000,
168
+ }).status === 0;
169
+ if (!(await waitFor(answered, ci.hosts?.timeoutMs ?? 600_000, 5000))) return { text: `the host on ${port} never attached`, code: 1 };
170
+ say(`host on ${port} ready`);
171
+ }
172
+ say(`running ${files.length} case file(s) on ${jobs} host(s)`);
173
+ const run = spawnSync(process.execPath, [self, ...runArgs], {
174
+ cwd: ctx.root,
175
+ encoding: 'utf8',
176
+ stdio: ['ignore', 'pipe', 'inherit'],
177
+ maxBuffer: 64 * 1024 * 1024,
178
+ });
179
+ process.stdout.write(run.stdout ?? '');
180
+ const report = /report: (https?:\/\/\S+)/.exec(run.stdout ?? '')?.[1];
181
+ const summary = summaryOf(resultsOf(out), report, picked);
182
+ if (process.env.GITHUB_STEP_SUMMARY) fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, summary);
183
+ return { text: summary, code: run.status ?? 1 };
184
+ } finally {
185
+ down();
186
+ }
187
+ },
188
+ }),
189
+ ];