lynkeus 0.3.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/cases/coverage.d.ts +31 -0
- package/lib/cases/coverage.js +82 -0
- package/lib/cases/draft.d.ts +21 -0
- package/lib/cases/draft.js +53 -0
- package/lib/cases/fixtures.d.ts +11 -0
- package/lib/cases/fixtures.js +59 -8
- package/lib/cases/frames.d.ts +6 -0
- package/lib/cases/frames.js +28 -0
- package/lib/cases/nativeLogs.d.ts +14 -0
- package/lib/cases/nativeLogs.js +111 -0
- package/lib/cases/runDirs.d.ts +13 -0
- package/lib/cases/runDirs.js +48 -0
- package/lib/cases/runfile.d.ts +5 -1
- package/lib/cases/runfile.js +67 -4
- package/lib/cases/runner.d.ts +58 -1
- package/lib/cases/runner.js +322 -39
- package/lib/cases/templates.d.ts +2 -2
- package/lib/cases/templates.js +20 -2
- package/lib/cases/traces.js +4 -1
- package/lib/cli.js +27 -4
- package/lib/commands/cases.d.ts +8 -0
- package/lib/commands/cases.js +154 -25
- package/lib/commands/ci.d.ts +13 -0
- package/lib/commands/ci.js +184 -0
- package/lib/commands/coverage.d.ts +1 -0
- package/lib/commands/coverage.js +151 -0
- package/lib/commands/device.js +32 -2
- package/lib/commands/doctor.js +8 -2
- package/lib/commands/drive.js +3 -1
- package/lib/commands/fixtures.d.ts +7 -0
- package/lib/commands/fixtures.js +94 -0
- package/lib/commands/headless.d.ts +11 -0
- package/lib/commands/headless.js +189 -9
- package/lib/commands/init.d.ts +23 -0
- package/lib/commands/init.js +231 -26
- package/lib/commands/map.d.ts +2 -0
- package/lib/commands/map.js +18 -14
- package/lib/commands/os.js +43 -10
- package/lib/commands/report.d.ts +4 -0
- package/lib/commands/report.js +156 -20
- package/lib/config.d.ts +41 -0
- package/lib/config.js +30 -0
- package/lib/device/agentVersion.d.ts +7 -0
- package/lib/device/agentVersion.js +14 -0
- package/lib/device/devices.js +6 -0
- package/lib/device/driver.js +2 -2
- package/lib/knowledge/analyticsTraffic.d.ts +10 -0
- package/lib/knowledge/analyticsTraffic.js +109 -0
- package/lib/knowledge/errors.js +6 -0
- package/lib/knowledge/events.d.ts +2 -0
- package/lib/knowledge/why.d.ts +2 -0
- package/lib/knowledge/why.js +1 -1
- package/lib/mcp/server.js +45 -4
- package/lib/runtime.d.ts +9 -1
- package/lib/runtime.js +25 -3
- package/package.json +7 -4
- package/src/cases/coverage.ts +99 -0
- package/src/cases/draft.ts +75 -0
- package/src/cases/fixtures.ts +57 -8
- package/src/cases/frames.ts +30 -0
- package/src/cases/nativeLogs.ts +112 -0
- package/src/cases/runDirs.ts +48 -0
- package/src/cases/runfile.ts +64 -4
- package/src/cases/runner.ts +372 -42
- package/src/cases/templates.ts +20 -2
- package/src/cases/traces.ts +3 -1
- package/src/cli.ts +27 -4
- package/src/commands/cases.ts +165 -23
- package/src/commands/ci.ts +189 -0
- package/src/commands/coverage.ts +147 -0
- package/src/commands/device.ts +27 -2
- package/src/commands/doctor.ts +9 -2
- package/src/commands/drive.ts +3 -1
- package/src/commands/fixtures.ts +102 -0
- package/src/commands/headless.ts +197 -9
- package/src/commands/init.ts +270 -26
- package/src/commands/map.ts +20 -15
- package/src/commands/os.ts +38 -10
- package/src/commands/report.ts +162 -18
- package/src/config.ts +48 -0
- package/src/device/agentVersion.ts +17 -0
- package/src/device/devices.ts +5 -0
- package/src/device/driver.ts +2 -2
- package/src/knowledge/analyticsTraffic.ts +142 -0
- package/src/knowledge/errors.ts +6 -0
- package/src/knowledge/events.ts +2 -1
- package/src/knowledge/why.ts +3 -1
- package/src/mcp/server.ts +45 -4
- package/src/runtime.ts +24 -3
package/lib/cli.js
CHANGED
|
@@ -8,15 +8,29 @@ import { driveCommands } from './commands/drive.js';
|
|
|
8
8
|
import { headlessCommands } from './commands/headless.js';
|
|
9
9
|
import { mapCommands } from './commands/map.js';
|
|
10
10
|
import { osCommands } from './commands/os.js';
|
|
11
|
+
import { ciCommands } from './commands/ci.js';
|
|
12
|
+
import { coverageCommands } from './commands/coverage.js';
|
|
13
|
+
import { fixturesCommands } from './commands/fixtures.js';
|
|
11
14
|
import { reportCommands } from './commands/report.js';
|
|
12
15
|
import { runsCommands } from './commands/runs.js';
|
|
13
|
-
import { graphFile, loadConfig, loadGraph, portFromEnv } from './config.js';
|
|
14
|
-
import { agentMismatch } from './device/agentVersion.js';
|
|
16
|
+
import { graphFile, loadConfig, loadGraph, portFromEnv, readToken } from './config.js';
|
|
17
|
+
import { agentMismatch, appSwitched, helloIdentity } from './device/agentVersion.js';
|
|
15
18
|
import { HEADLESS_HOST, pickAgentDevice, pickAndroidDevice } from './device/devices.js';
|
|
16
19
|
import { Device } from './device/driver.js';
|
|
17
20
|
import { commands, find, helpFor, inSession, referenceMarkdown, register, resolve, usage } from './registry.js';
|
|
18
21
|
import { execute } from './runtime.js';
|
|
19
|
-
register([
|
|
22
|
+
register([
|
|
23
|
+
...driveCommands,
|
|
24
|
+
...osCommands,
|
|
25
|
+
...mapCommands,
|
|
26
|
+
...runsCommands,
|
|
27
|
+
...caseCommands,
|
|
28
|
+
...coverageCommands,
|
|
29
|
+
...ciCommands,
|
|
30
|
+
...fixturesCommands,
|
|
31
|
+
...reportCommands,
|
|
32
|
+
...headlessCommands,
|
|
33
|
+
]);
|
|
20
34
|
/** The global flags, read leniently from the whole argv before any command parses its own. */
|
|
21
35
|
const globals = (argv) => {
|
|
22
36
|
const out = {};
|
|
@@ -39,6 +53,8 @@ const globals = (argv) => {
|
|
|
39
53
|
};
|
|
40
54
|
// Once per process: a suite reattaches for every case.
|
|
41
55
|
let mismatchTold = false;
|
|
56
|
+
let switchTold = false;
|
|
57
|
+
let lastApp;
|
|
42
58
|
// `ownHost`: the port belongs to a host this run started for a case; whatever dials in is the device.
|
|
43
59
|
const contextFor = (argv, attached = false, port, ownHost = false) => {
|
|
44
60
|
const g = globals(argv);
|
|
@@ -51,8 +67,15 @@ const contextFor = (argv, attached = false, port, ownHost = false) => {
|
|
|
51
67
|
const appId = g.app ?? process.env.LYNKEUS_APP_ID ?? config.app;
|
|
52
68
|
const started = performance.now();
|
|
53
69
|
const attachedIn = (d) => err(`attached in ${Math.round(performance.now() - started)}ms (${d.info.name})`);
|
|
54
|
-
const server = new AgentServer({ token:
|
|
70
|
+
const server = new AgentServer({ token: readToken(root), port: port ?? portFromEnv(config) });
|
|
55
71
|
server.on('hello', (hello) => {
|
|
72
|
+
const identity = helloIdentity(hello);
|
|
73
|
+
const switched = switchTold ? undefined : appSwitched(lastApp, identity, server.port);
|
|
74
|
+
lastApp = identity;
|
|
75
|
+
if (switched) {
|
|
76
|
+
switchTold = true;
|
|
77
|
+
err(switched);
|
|
78
|
+
}
|
|
56
79
|
const told = mismatchTold ? undefined : agentMismatch(hello);
|
|
57
80
|
if (!told)
|
|
58
81
|
return;
|
package/lib/commands/cases.d.ts
CHANGED
|
@@ -1 +1,9 @@
|
|
|
1
|
+
import { type CaseReport } from '../cases/runner.js';
|
|
2
|
+
/** The case files named (a directory stands for every case under it), or every case under the cases directory; a sentence when there are none. */
|
|
3
|
+
export declare const caseFiles: (args: string[], home: string, casesDir: string | undefined) => string[] | string;
|
|
4
|
+
/** A run that could not be uploaded still ran: the failure is said, not raised. */
|
|
5
|
+
export declare const missingEvidence: (reports: CaseReport[], asked: {
|
|
6
|
+
video: boolean;
|
|
7
|
+
}) => string[];
|
|
8
|
+
export declare const keptOrDeleted: (dir: string, afterPush: "keep" | "delete", complete: boolean) => string;
|
|
1
9
|
export declare const caseCommands: import("../registry.js").Command[];
|
package/lib/commands/cases.js
CHANGED
|
@@ -2,6 +2,7 @@ import { spawn } from 'node:child_process';
|
|
|
2
2
|
import fs from 'node:fs';
|
|
3
3
|
import os from 'node:os';
|
|
4
4
|
import path from 'node:path';
|
|
5
|
+
import { stripVTControlCharacters } from 'node:util';
|
|
5
6
|
import { Fixtures } from '../cases/fixtures.js';
|
|
6
7
|
import { readCases } from '../cases/format.js';
|
|
7
8
|
import { missingFixtures, preparable, runCase } from '../cases/runner.js';
|
|
@@ -16,6 +17,8 @@ import { eventsFile } from '../knowledge/events.js';
|
|
|
16
17
|
import { define } from '../registry.js';
|
|
17
18
|
import { describeChanges, pushRun, service } from './report.js';
|
|
18
19
|
import { TraceSink } from '../cases/traces.js';
|
|
20
|
+
import { megabytes, sizeOf } from '../cases/runDirs.js';
|
|
21
|
+
import { startNativeLogs } from '../cases/nativeLogs.js';
|
|
19
22
|
const startHost = async (root, err) => {
|
|
20
23
|
// Its own process group: the host starts a test runner of its own, and stopping one without the other leaves an app dialing in.
|
|
21
24
|
const host = spawn(process.execPath, [process.argv[1], 'headless', 'start'], {
|
|
@@ -24,8 +27,27 @@ const startHost = async (root, err) => {
|
|
|
24
27
|
stdio: ['ignore', 'pipe', 'pipe'],
|
|
25
28
|
detached: true,
|
|
26
29
|
});
|
|
30
|
+
// Everything the host prints, for the render errors and warnings a failed case points at; the console keeps the run's own lines.
|
|
31
|
+
const logFile = path.join(os.tmpdir(), `lynkeus-headless-${process.pid}.log`);
|
|
32
|
+
const logStream = fs.createWriteStream(logFile);
|
|
33
|
+
logStream.on('error', () => { });
|
|
34
|
+
host.once('exit', () => logStream.end());
|
|
27
35
|
let output = '';
|
|
28
36
|
const tail = (n) => output.trim().split('\n').slice(-n).join(' · ');
|
|
37
|
+
// `headless start` ends a crash with what it usually means; without that, the error line jest printed.
|
|
38
|
+
const why = () => {
|
|
39
|
+
const plain = stripVTControlCharacters(output);
|
|
40
|
+
const explained = plain.split('lynkeus: what this looks like')[1];
|
|
41
|
+
if (explained)
|
|
42
|
+
return explained
|
|
43
|
+
.trim()
|
|
44
|
+
.split('\n')
|
|
45
|
+
.map((line) => line.trim())
|
|
46
|
+
.join(' · ');
|
|
47
|
+
// Jest prints a suite that throws while loading as "● Test suite failed to run", then the thrown message.
|
|
48
|
+
const thrown = plain.match(/Test suite failed to run\s*\n\s*\n?\s*(\S.*)$/m)?.[1]?.trim();
|
|
49
|
+
return thrown ?? plain.match(/^\s*(\w*Error\b.*|Cannot find module.*)$/m)?.[1]?.trim() ?? tail(3);
|
|
50
|
+
};
|
|
29
51
|
await new Promise((resolve, reject) => {
|
|
30
52
|
const settle = (error) => {
|
|
31
53
|
clearTimeout(timer);
|
|
@@ -36,17 +58,18 @@ const startHost = async (root, err) => {
|
|
|
36
58
|
};
|
|
37
59
|
const timer = setTimeout(() => settle(new Error(`the host did not come up within 180 s: ${tail(3)}`)), 180_000);
|
|
38
60
|
const read = (chunk) => {
|
|
61
|
+
logStream.write(chunk);
|
|
39
62
|
output = (output + chunk.toString()).slice(-4000);
|
|
40
63
|
if (/waiting for a driver/.test(output))
|
|
41
64
|
settle();
|
|
42
|
-
else if (/Test suite failed/.test(output))
|
|
43
|
-
settle(new Error(`the host failed to start: ${tail(5)}`));
|
|
44
65
|
};
|
|
45
66
|
host.stdout?.on('data', read);
|
|
46
67
|
host.stderr?.on('data', read);
|
|
47
|
-
|
|
68
|
+
// A suite that fails to load exits right after printing why, so its explanation is read in full here.
|
|
69
|
+
host.once('exit', (code) => settle(new Error(`${/Test suite failed/.test(output) ? 'the host failed to start' : `the host exited (${code})`}: ${why()}`)));
|
|
48
70
|
});
|
|
49
|
-
|
|
71
|
+
// The listeners above stay: the log takes all the host prints until it stops.
|
|
72
|
+
err(`host: up (its log: ${logFile})`);
|
|
50
73
|
return host;
|
|
51
74
|
};
|
|
52
75
|
const stopGroup = (pid) => {
|
|
@@ -101,19 +124,27 @@ const idleShare = async () => {
|
|
|
101
124
|
const after = total();
|
|
102
125
|
return after.all > before.all ? (after.idle - before.idle) / (after.all - before.all) : 1;
|
|
103
126
|
};
|
|
104
|
-
/**
|
|
105
|
-
const
|
|
127
|
+
/** Every case under a directory and its subdirectories, in path order, so a suite can group its cases in folders. */
|
|
128
|
+
const casesIn = (dir) => fs
|
|
129
|
+
.readdirSync(dir, { withFileTypes: true })
|
|
130
|
+
.filter((e) => !e.name.startsWith('.') && e.name !== 'node_modules')
|
|
131
|
+
.sort((a, b) => a.name.localeCompare(b.name))
|
|
132
|
+
.flatMap((e) => {
|
|
133
|
+
const full = path.join(dir, e.name);
|
|
134
|
+
if (e.isDirectory())
|
|
135
|
+
return casesIn(full);
|
|
136
|
+
return e.name.endsWith('.md') && !/^(README|FORMAT|CATALOG)\.md$/i.test(e.name) ? [full] : [];
|
|
137
|
+
});
|
|
138
|
+
/** The case files named (a directory stands for every case under it), or every case under the cases directory; a sentence when there are none. */
|
|
139
|
+
export const caseFiles = (args, home, casesDir) => {
|
|
106
140
|
const dir = path.resolve(home, casesDir ?? 'cases');
|
|
107
|
-
const
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
.map((f) => path.join(dir, f))
|
|
115
|
-
: [];
|
|
116
|
-
return files.length > 0 ? files : `no cases under ${path.relative(home, dir) || '.'}; \`lynkeus case init\` writes one`;
|
|
141
|
+
const named = args.map((f) => path.resolve(f));
|
|
142
|
+
const missing = named.find((f) => !fs.existsSync(f));
|
|
143
|
+
if (missing)
|
|
144
|
+
return `no such case file: ${path.relative(process.cwd(), missing) || missing}`;
|
|
145
|
+
const files = named.length ? named.flatMap((f) => (fs.statSync(f).isDirectory() ? casesIn(f) : [f])) : fs.existsSync(dir) ? casesIn(dir) : [];
|
|
146
|
+
const where = named.length === 1 ? path.relative(home, named[0]) : path.relative(home, dir);
|
|
147
|
+
return files.length > 0 ? files : `no cases under ${where || '.'}; \`lynkeus case init\` writes one`;
|
|
117
148
|
};
|
|
118
149
|
/**
|
|
119
150
|
* Runs the setups of `cases` ahead of them, in order, `jobs` at a time: never more than `limit` waiting
|
|
@@ -165,15 +196,41 @@ const spawnHost = (command, port, job, cwd) => spawn('/bin/sh', ['-c', command.r
|
|
|
165
196
|
detached: true,
|
|
166
197
|
});
|
|
167
198
|
/** A run that could not be uploaded still ran: the failure is said, not raised. */
|
|
168
|
-
|
|
199
|
+
// A run that kept evidence but none of a kind says so when it ends, not only as a gap in the report.
|
|
200
|
+
export const missingEvidence = (reports, asked) => {
|
|
201
|
+
if (reports.length === 0)
|
|
202
|
+
return [];
|
|
203
|
+
const steps = reports.flatMap((r) => r.steps);
|
|
204
|
+
const fromApp = steps.some((s) => s.sent?.some((e) => e.via !== 'server')) || reports.some((r) => (r.events?.length ?? 0) > 0);
|
|
205
|
+
return [
|
|
206
|
+
...(fromApp ? [] : ['⚠ no case kept an analytics event from the app (the agent keeps the calls that reach Mixpanel, AppsFlyer and Firebase Analytics).']),
|
|
207
|
+
...(asked.video && !reports.some((r) => r.video)
|
|
208
|
+
? ['⚠ --video kept no film: this host does not film (fastsim does; a simulator, an emulator or headless keep frames only).']
|
|
209
|
+
: []),
|
|
210
|
+
];
|
|
211
|
+
};
|
|
212
|
+
const pushed = async (dir, to, project, err, afterPush = 'keep') => {
|
|
169
213
|
try {
|
|
170
|
-
const run = await pushRun(dir, { ...to, project });
|
|
171
|
-
return [
|
|
214
|
+
const run = await pushRun(dir, { ...to, project, onReady: (url) => err(`report: ${url} (its frames are on their way)`), onProgress: err });
|
|
215
|
+
return [
|
|
216
|
+
`report: ${run.url}${run.refused.length ? ` (not kept on this plan: ${run.refused.join(', ')})` : ''}`,
|
|
217
|
+
...describeChanges(run.changes),
|
|
218
|
+
keptOrDeleted(dir, afterPush, run.refused.length === 0),
|
|
219
|
+
].join('\n');
|
|
172
220
|
}
|
|
173
221
|
catch (error) {
|
|
174
222
|
return `the report was not uploaded: ${error.message}`;
|
|
175
223
|
}
|
|
176
224
|
};
|
|
225
|
+
// After a push: the folder deleted when the config asks and the service took every file, else how much it holds.
|
|
226
|
+
export const keptOrDeleted = (dir, afterPush, complete) => {
|
|
227
|
+
const bytes = sizeOf(dir);
|
|
228
|
+
if (afterPush === 'delete' && complete) {
|
|
229
|
+
fs.rmSync(dir, { recursive: true, force: true });
|
|
230
|
+
return `run folder deleted after the push (${megabytes(bytes)})`;
|
|
231
|
+
}
|
|
232
|
+
return `run folder: ${megabytes(bytes)} on disk${afterPush === 'delete' ? ', kept because some files were not taken' : ''} · \`lynkeus report prune ${path.dirname(dir)} --older 7d\` frees old ones`;
|
|
233
|
+
};
|
|
177
234
|
export const caseCommands = [
|
|
178
235
|
define({
|
|
179
236
|
name: 'case run',
|
|
@@ -189,7 +246,32 @@ export const caseCommands = [
|
|
|
189
246
|
report: { type: 'string', help: 'append one JSON line per case', value: 'file' },
|
|
190
247
|
events: { type: 'string', help: 'keep what the app told its analytics, per case, as <dir>/<case id>/events.jsonl', value: 'dir' },
|
|
191
248
|
out: { type: 'string', help: 'write the run as a directory a report can read: run.json and cases.jsonl, a line per attempt', value: 'dir' },
|
|
249
|
+
'no-app-events': {
|
|
250
|
+
type: 'boolean',
|
|
251
|
+
help: "with --out: leave the app's own event list unread. Reading it can turn on the app's tracking debug, which may draw on the screen; frames two runs compare should not have it",
|
|
252
|
+
},
|
|
192
253
|
screenshots: { type: 'boolean', help: 'with --out: a frame of the screen after every step of the app' },
|
|
254
|
+
settle: {
|
|
255
|
+
type: 'number',
|
|
256
|
+
help: 'with --screenshots: how long the screen must stay still before a frame, in ms; 700 for frames two runs can be compared by',
|
|
257
|
+
default: 150,
|
|
258
|
+
value: 'ms',
|
|
259
|
+
},
|
|
260
|
+
'frames-on-change': {
|
|
261
|
+
type: 'boolean',
|
|
262
|
+
help: 'with --screenshots: a step that only looks (app.see, app.wait) on an unchanged screen shares the last frame',
|
|
263
|
+
},
|
|
264
|
+
still: {
|
|
265
|
+
type: 'number',
|
|
266
|
+
help: 'with --screenshots, on a simulator or emulator: take each frame again until two in a row match, this many ms apart (a native transition the agent cannot see)',
|
|
267
|
+
value: 'ms',
|
|
268
|
+
},
|
|
269
|
+
frames: {
|
|
270
|
+
type: 'string',
|
|
271
|
+
help: "with --screenshots: avif, the default and a few percent of the bytes (a simulator's or emulator's PNG is encoded again), or png, exact, only for frames two runs are compared by pixel for pixel",
|
|
272
|
+
default: 'avif',
|
|
273
|
+
value: 'avif|png',
|
|
274
|
+
},
|
|
193
275
|
video: { type: 'boolean', help: 'with --out: film each case, where the host films (the step says at what second it began)' },
|
|
194
276
|
traces: {
|
|
195
277
|
type: 'boolean',
|
|
@@ -199,6 +281,7 @@ export const caseCommands = [
|
|
|
199
281
|
push: { type: 'boolean', help: 'with --out: upload the run to the reports service when it ends and print the report’s address' },
|
|
200
282
|
project: { type: 'string', help: 'with --push: the project the run is filed under (default: the app’s folder name)', value: 'name' },
|
|
201
283
|
'video-fps': { type: 'number', help: 'frames per second of the film', default: 30, value: 'n' },
|
|
284
|
+
'video-scale': { type: 'number', help: 'pixels per point of the film, where the host takes one (fastsim: 1 to 3, 2 unless told)', value: 'n' },
|
|
202
285
|
flakes: { type: 'string', help: 'the ledger of how each case did across runs (default .lynkeus/flakes.json)', value: 'file' },
|
|
203
286
|
prepared: { type: 'string', help: 'where `case prepare` leaves setups made ahead; a case found there skips its setup', value: 'dir' },
|
|
204
287
|
jobs: { type: 'number', help: 'cases at once, each on its own host (needs --ports)', value: 'n', default: 1 },
|
|
@@ -215,7 +298,32 @@ export const caseCommands = [
|
|
|
215
298
|
},
|
|
216
299
|
needs: 'nothing',
|
|
217
300
|
session: false,
|
|
218
|
-
mcp:
|
|
301
|
+
mcp: 'core',
|
|
302
|
+
mcpOmit: [
|
|
303
|
+
'seen',
|
|
304
|
+
'report',
|
|
305
|
+
'events',
|
|
306
|
+
'settle',
|
|
307
|
+
'still',
|
|
308
|
+
'frames-on-change',
|
|
309
|
+
'frames',
|
|
310
|
+
'traces',
|
|
311
|
+
'traces-port',
|
|
312
|
+
'video-fps',
|
|
313
|
+
'video-scale',
|
|
314
|
+
'flakes',
|
|
315
|
+
'prepared',
|
|
316
|
+
'jobs',
|
|
317
|
+
'ports',
|
|
318
|
+
'host-command',
|
|
319
|
+
'in-order',
|
|
320
|
+
'ahead',
|
|
321
|
+
'idle',
|
|
322
|
+
'continue',
|
|
323
|
+
'video',
|
|
324
|
+
'project',
|
|
325
|
+
],
|
|
326
|
+
mcpDetails: 'Runs case files (default: all in cases.dir) against the attached app, or with `host` in Node. A case that fails comes back with its screen and reason. `out` + `screenshots` + `push` uploads the run and prints its address.',
|
|
219
327
|
run: async (ctx) => {
|
|
220
328
|
const config = ctx.config.cases ?? {};
|
|
221
329
|
const home = config.base ?? ctx.root;
|
|
@@ -238,12 +346,18 @@ export const caseCommands = [
|
|
|
238
346
|
? {
|
|
239
347
|
dir: out.evidenceDir(id, attempt),
|
|
240
348
|
screenshots: !!ctx.flags.screenshots,
|
|
241
|
-
|
|
349
|
+
settleMs: Number(ctx.flags.settle ?? 150),
|
|
350
|
+
...(ctx.flags.still ? { stillMs: Number(ctx.flags.still) } : {}),
|
|
351
|
+
...(ctx.flags['frames-on-change'] ? { onChange: true } : {}),
|
|
352
|
+
frames: ctx.flags.frames === 'png' ? 'png' : 'avif',
|
|
353
|
+
...(ctx.flags.video
|
|
354
|
+
? { video: { fps: Number(ctx.flags['video-fps'] ?? 30), ...(ctx.flags['video-scale'] ? { scale: Number(ctx.flags['video-scale']) } : {}) } }
|
|
355
|
+
: {}),
|
|
242
356
|
}
|
|
243
357
|
: undefined;
|
|
244
358
|
const ledger = dry ? undefined : new Flakes(path.resolve(ctx.root, typeof ctx.flags.flakes === 'string' ? ctx.flags.flakes : '.lynkeus/flakes.json'));
|
|
245
359
|
// A run file files each event under its step, so it needs them collected.
|
|
246
|
-
const events = typeof ctx.flags.events === 'string' || out !== undefined;
|
|
360
|
+
const events = typeof ctx.flags.events === 'string' || (out !== undefined && !ctx.flags['no-app-events']);
|
|
247
361
|
const claimed = dry
|
|
248
362
|
? undefined
|
|
249
363
|
: claimPorts(typeof ctx.flags.ports === 'string'
|
|
@@ -297,6 +411,14 @@ export const caseCommands = [
|
|
|
297
411
|
const hostCommand = typeof ctx.flags['host-command'] === 'string' && !dry ? ctx.flags['host-command'] : undefined;
|
|
298
412
|
if ((jobs > 1 || hostCommand) && (ports.length < jobs || ports.some((p) => !Number.isInteger(p)) || !ctx.forPort))
|
|
299
413
|
return { text: `--jobs and --host-command need a driver port per job: --ports with ${jobs}`, code: 2 };
|
|
414
|
+
// One simulator or emulator, one log: with several jobs a line could not be told to its case.
|
|
415
|
+
const native = out && jobs === 1 && !hostCommand && !ctx.flags.host && ctx.config.app
|
|
416
|
+
? await ctx
|
|
417
|
+
.device()
|
|
418
|
+
.then((d) => d.resolved())
|
|
419
|
+
.then((info) => startNativeLogs(info, ctx.config.app))
|
|
420
|
+
.catch(() => undefined)
|
|
421
|
+
: undefined;
|
|
300
422
|
// Several jobs finish together only if the longest cases start first: one left for the end runs alone
|
|
301
423
|
// while the other jobs wait. A case the ledger has not timed goes first too, as it may be long.
|
|
302
424
|
if (jobs > 1 && ledger && !ctx.flags['in-order']) {
|
|
@@ -354,6 +476,7 @@ export const caseCommands = [
|
|
|
354
476
|
continue: !!ctx.flags.continue,
|
|
355
477
|
routes: seenDir ? (host ? 'fresh' : true) : false,
|
|
356
478
|
traces,
|
|
479
|
+
nativeLogs: native,
|
|
357
480
|
});
|
|
358
481
|
if (seenDir && report.result === 'passed' && report.routes?.length) {
|
|
359
482
|
fs.mkdirSync(seenDir, { recursive: true });
|
|
@@ -402,6 +525,7 @@ export const caseCommands = [
|
|
|
402
525
|
await preparing;
|
|
403
526
|
if (out)
|
|
404
527
|
out.finish({ platform: seen?.platform, native: seen?.native });
|
|
528
|
+
native?.stop();
|
|
405
529
|
}
|
|
406
530
|
finally {
|
|
407
531
|
stop();
|
|
@@ -409,12 +533,17 @@ export const caseCommands = [
|
|
|
409
533
|
const count = (result) => reports.filter((r) => r.result === result).length;
|
|
410
534
|
const unsupported = count('unsupported');
|
|
411
535
|
const summary = [
|
|
412
|
-
...reports.map((r) => `${{ passed: '✅', failed: '❌', unsupported: '🚫' }[r.result]} ${r.id}${r.retried && r.result === 'passed' ? ' ⚠ passed on retry' : ''}`
|
|
536
|
+
...reports.map((r) => `${{ passed: '✅', failed: '❌', unsupported: '🚫' }[r.result]} ${r.id}${r.retried && r.result === 'passed' ? ' ⚠ passed on retry' : ''}` +
|
|
537
|
+
// In a session or over MCP the caller reads this answer and nothing else: a case that did not pass brings its steps.
|
|
538
|
+
(ctx.attached && r.result !== 'passed' ? `\n${describe(r)}` : '')),
|
|
413
539
|
`${count('passed')}/${reports.length} passed${unsupported ? `, ${unsupported} cannot run on this host` : ''}`,
|
|
540
|
+
// Its full path: a relative --out resolves against where the CLI was started, which over MCP is the app's folder.
|
|
541
|
+
...(out ? [`run kept in ${path.resolve(ctx.flags.out)}`] : []),
|
|
542
|
+
...(out ? missingEvidence(reports, { video: !!ctx.flags.video }) : []),
|
|
414
543
|
...(ledger ? ledger.flaky(reports.map((r) => r.id)) : []),
|
|
415
544
|
...(reportsTo && out
|
|
416
545
|
? [
|
|
417
|
-
await pushed(path.resolve(ctx.flags.out), reportsTo, typeof ctx.flags.project === 'string' ? ctx.flags.project : path.basename(ctx.root)),
|
|
546
|
+
await pushed(path.resolve(ctx.flags.out), reportsTo, typeof ctx.flags.project === 'string' ? ctx.flags.project : path.basename(ctx.root), ctx.err, ctx.config.report?.afterPush === 'delete' ? 'delete' : 'keep'),
|
|
418
547
|
]
|
|
419
548
|
: []),
|
|
420
549
|
].join('\n');
|
|
@@ -473,7 +602,7 @@ export const caseCommands = [
|
|
|
473
602
|
flags: { force: { type: 'boolean', help: 'overwrite what is there' } },
|
|
474
603
|
needs: 'nothing',
|
|
475
604
|
session: false,
|
|
476
|
-
mcp:
|
|
605
|
+
mcp: 'core',
|
|
477
606
|
run: async (ctx) => {
|
|
478
607
|
const dir = path.resolve(ctx.root, ctx.config.cases?.dir ?? 'cases');
|
|
479
608
|
fs.mkdirSync(dir, { recursive: true });
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/** The case results of a run folder, by each case's last attempt. */
|
|
2
|
+
export declare const resultsOf: (dir: string) => {
|
|
3
|
+
id: string;
|
|
4
|
+
result: string;
|
|
5
|
+
title?: string;
|
|
6
|
+
}[];
|
|
7
|
+
/** The summary a CI job shows: how many passed, which failed, and the report. */
|
|
8
|
+
export declare const summaryOf: (results: {
|
|
9
|
+
id: string;
|
|
10
|
+
result: string;
|
|
11
|
+
title?: string;
|
|
12
|
+
}[], report: string | undefined, picked: string) => string;
|
|
13
|
+
export declare const ciCommands: import("../registry.js").Command[];
|
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
import { spawn, spawnSync } from 'node:child_process';
|
|
2
|
+
import fs from 'node:fs';
|
|
3
|
+
import path from 'node:path';
|
|
4
|
+
import { coverageOf, impactOf, readSeen } from '../cases/coverage.js';
|
|
5
|
+
import { readCases } from '../cases/format.js';
|
|
6
|
+
import { screensAffectedBy } from '../knowledge/query.js';
|
|
7
|
+
import { define } from '../registry.js';
|
|
8
|
+
import { caseFiles } from './cases.js';
|
|
9
|
+
import { changedSources } from './map.js';
|
|
10
|
+
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
11
|
+
/** The case results of a run folder, by each case's last attempt. */
|
|
12
|
+
export const resultsOf = (dir) => {
|
|
13
|
+
const file = path.join(dir, 'cases.jsonl');
|
|
14
|
+
if (!fs.existsSync(file))
|
|
15
|
+
return [];
|
|
16
|
+
const last = new Map();
|
|
17
|
+
for (const line of fs.readFileSync(file, 'utf8').split('\n')) {
|
|
18
|
+
if (!line.trim())
|
|
19
|
+
continue;
|
|
20
|
+
const c = JSON.parse(line);
|
|
21
|
+
last.set(c.id, c);
|
|
22
|
+
}
|
|
23
|
+
return [...last.values()];
|
|
24
|
+
};
|
|
25
|
+
/** The summary a CI job shows: how many passed, which failed, and the report. */
|
|
26
|
+
export const summaryOf = (results, report, picked) => {
|
|
27
|
+
const failed = results.filter((r) => r.result === 'failed');
|
|
28
|
+
const passed = results.filter((r) => r.result === 'passed').length;
|
|
29
|
+
return [
|
|
30
|
+
`### lynkeus: ${passed}/${results.length} cases passed${failed.length ? `, ${failed.length} failed` : ''}`,
|
|
31
|
+
'',
|
|
32
|
+
picked,
|
|
33
|
+
...(report ? ['', `[Open the report](${report})`] : []),
|
|
34
|
+
...(failed.length ? ['', '| Failed | |', '|---|---|', ...failed.map((f) => `| \`${f.id}\` | ${f.title ?? ''} |`)] : []),
|
|
35
|
+
'',
|
|
36
|
+
].join('\n');
|
|
37
|
+
};
|
|
38
|
+
const waitFor = async (ok, timeoutMs, every = 3000) => {
|
|
39
|
+
const until = Date.now() + timeoutMs;
|
|
40
|
+
for (;;) {
|
|
41
|
+
if (await ok())
|
|
42
|
+
return true;
|
|
43
|
+
if (Date.now() > until)
|
|
44
|
+
return false;
|
|
45
|
+
await sleep(every);
|
|
46
|
+
}
|
|
47
|
+
};
|
|
48
|
+
export const ciCommands = [
|
|
49
|
+
define({
|
|
50
|
+
name: 'ci',
|
|
51
|
+
group: 'runs',
|
|
52
|
+
summary: 'Run the suite the way a CI job does: bring the backend and N hosts up, run the cases across them, report, tear down',
|
|
53
|
+
details: 'Reads `ci` in lynkeus.config.json: `backend.up` / `backend.down` (the project’s commands; lynkeus knows no backend) and `backend.ready` (a URL to wait for), `hosts.count` and `hosts.command` (default: a headless host; `{port}` is where it dials), `push`, and extra `run` flags. --affected runs only the cases a branch’s changes reach (as `case impact` finds them). Writes the job summary to $GITHUB_STEP_SUMMARY when it is set. The exit code is the suite’s.',
|
|
54
|
+
flags: {
|
|
55
|
+
affected: { type: 'boolean', help: 'only the cases the changes against --base reach' },
|
|
56
|
+
base: { type: 'string', help: 'git ref --affected diffs against', default: 'origin/main', value: 'ref' },
|
|
57
|
+
jobs: { type: 'number', help: 'how many hosts and jobs (default ci.hosts.count, else 2)', value: 'n' },
|
|
58
|
+
out: { type: 'string', help: 'where the run goes', default: 'lynkeus-run', value: 'dir' },
|
|
59
|
+
'keep-up': { type: 'boolean', help: 'leave the backend and the hosts up afterwards' },
|
|
60
|
+
'dry-run': { type: 'boolean', help: 'say what it would do and run nothing' },
|
|
61
|
+
},
|
|
62
|
+
needs: 'nothing',
|
|
63
|
+
session: false,
|
|
64
|
+
mcp: false,
|
|
65
|
+
run: async (ctx) => {
|
|
66
|
+
const ci = ctx.config.ci ?? {};
|
|
67
|
+
const config = ctx.config.cases ?? {};
|
|
68
|
+
const home = config.base ?? ctx.root;
|
|
69
|
+
const self = process.argv[1];
|
|
70
|
+
const say = (line) => ctx.err(`▸ ${line}`);
|
|
71
|
+
const jobs = Math.max(1, Number(ctx.flags.jobs ?? ci.hosts?.count ?? 2));
|
|
72
|
+
const out = path.resolve(ctx.root, String(ctx.flags.out));
|
|
73
|
+
// Which cases: all, or the ones the branch's changes reach.
|
|
74
|
+
const all = caseFiles([], home, config.dir);
|
|
75
|
+
if (typeof all === 'string')
|
|
76
|
+
return { text: all, code: 1 };
|
|
77
|
+
let files = all;
|
|
78
|
+
let picked = `All ${all.length} case files.`;
|
|
79
|
+
if (ctx.flags.affected) {
|
|
80
|
+
const graphFile = path.join(ctx.root, '.lynkeus', 'graph.json');
|
|
81
|
+
if (!fs.existsSync(graphFile))
|
|
82
|
+
spawnSync(process.execPath, [self, 'map', 'build'], { cwd: ctx.root, stdio: 'inherit' });
|
|
83
|
+
const cases = all.flatMap(readCases);
|
|
84
|
+
const graph = ctx.graph();
|
|
85
|
+
const coverage = coverageOf(cases, Object.keys(graph.screens).sort(), readSeen(path.join(ctx.root, '.lynkeus', 'seen')));
|
|
86
|
+
const changed = changedSources(ctx.root, String(ctx.flags.base), ctx.err);
|
|
87
|
+
const impact = impactOf(screensAffectedBy(graph, changed), coverage);
|
|
88
|
+
files = [...new Set(impact.cases.map((c) => c.file))];
|
|
89
|
+
picked = `${files.length} of ${all.length} case files: the ones the ${changed.length} changed source file(s) against ${ctx.flags.base} reach.`;
|
|
90
|
+
if (files.length === 0) {
|
|
91
|
+
if (process.env.GITHUB_STEP_SUMMARY)
|
|
92
|
+
fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, `### lynkeus: no case to run\n\n${picked}\n`);
|
|
93
|
+
return { text: `no case to run: ${picked}`, json: { files } };
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
const ports = Array.from({ length: jobs }, (_, i) => 8123 + i);
|
|
97
|
+
const hostCommand = ci.hosts?.command ?? `LYNKEUS_PORT={port} "${process.execPath}" "${self}" headless start`;
|
|
98
|
+
const runArgs = [
|
|
99
|
+
'case',
|
|
100
|
+
'run',
|
|
101
|
+
...files,
|
|
102
|
+
'--jobs',
|
|
103
|
+
String(jobs),
|
|
104
|
+
'--ports',
|
|
105
|
+
ports.join(','),
|
|
106
|
+
'--out',
|
|
107
|
+
out,
|
|
108
|
+
'--screenshots',
|
|
109
|
+
...(ci.push ? ['--push'] : []),
|
|
110
|
+
...(ci.run ?? []),
|
|
111
|
+
];
|
|
112
|
+
if (ctx.flags['dry-run']) {
|
|
113
|
+
const plan = [
|
|
114
|
+
picked,
|
|
115
|
+
ci.backend ? `backend: \`${ci.backend.up}\`${ci.backend.ready ? `, ready at ${ci.backend.ready}` : ''}` : 'backend: none declared (ci.backend)',
|
|
116
|
+
`hosts: ${jobs} × \`${hostCommand}\` on ${ports.join(', ')}`,
|
|
117
|
+
`run: lynkeus ${runArgs.slice(0, 2).join(' ')} … ${runArgs.slice(2 + files.length).join(' ')}`,
|
|
118
|
+
];
|
|
119
|
+
return { text: plan.join('\n'), json: { files, ports, hostCommand, runArgs } };
|
|
120
|
+
}
|
|
121
|
+
const hosts = [];
|
|
122
|
+
const down = () => {
|
|
123
|
+
if (ctx.flags['keep-up'])
|
|
124
|
+
return say('left up (--keep-up)');
|
|
125
|
+
for (const h of hosts)
|
|
126
|
+
if (h.pid)
|
|
127
|
+
try {
|
|
128
|
+
process.kill(-h.pid, 'SIGTERM');
|
|
129
|
+
}
|
|
130
|
+
catch { }
|
|
131
|
+
if (ci.backend?.down) {
|
|
132
|
+
say('backend down');
|
|
133
|
+
spawnSync('/bin/sh', ['-c', ci.backend.down], { cwd: ctx.root, stdio: 'inherit' });
|
|
134
|
+
}
|
|
135
|
+
};
|
|
136
|
+
try {
|
|
137
|
+
if (ci.backend) {
|
|
138
|
+
say(`backend up: ${ci.backend.up}`);
|
|
139
|
+
const up = spawnSync('/bin/sh', ['-c', ci.backend.up], { cwd: ctx.root, stdio: 'inherit' });
|
|
140
|
+
if (up.status !== 0)
|
|
141
|
+
return { text: `the backend did not come up (\`${ci.backend.up}\` exited ${up.status})`, code: 1 };
|
|
142
|
+
if (ci.backend.ready) {
|
|
143
|
+
const url = ci.backend.ready;
|
|
144
|
+
say(`waiting for ${url}`);
|
|
145
|
+
const ready = await waitFor(() => fetch(url).then((r) => r.ok, () => false), ci.backend.timeoutMs ?? 300_000);
|
|
146
|
+
if (!ready)
|
|
147
|
+
return { text: `${url} never answered`, code: 1 };
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
say(`${jobs} host${jobs === 1 ? '' : 's'}: ${hostCommand}`);
|
|
151
|
+
for (const port of ports)
|
|
152
|
+
hosts.push(spawn('/bin/sh', ['-c', hostCommand.replaceAll('{port}', String(port))], { cwd: ctx.root, stdio: ['pipe', 'ignore', 'ignore'], detached: true }));
|
|
153
|
+
// A host compiles the app before it answers: the slow part.
|
|
154
|
+
for (const port of ports) {
|
|
155
|
+
const answered = () => spawnSync(process.execPath, [self, 'screen'], {
|
|
156
|
+
cwd: ctx.root,
|
|
157
|
+
env: { ...process.env, LYNKEUS_PORT: String(port) },
|
|
158
|
+
stdio: 'ignore',
|
|
159
|
+
timeout: 20_000,
|
|
160
|
+
}).status === 0;
|
|
161
|
+
if (!(await waitFor(answered, ci.hosts?.timeoutMs ?? 600_000, 5000)))
|
|
162
|
+
return { text: `the host on ${port} never attached`, code: 1 };
|
|
163
|
+
say(`host on ${port} ready`);
|
|
164
|
+
}
|
|
165
|
+
say(`running ${files.length} case file(s) on ${jobs} host(s)`);
|
|
166
|
+
const run = spawnSync(process.execPath, [self, ...runArgs], {
|
|
167
|
+
cwd: ctx.root,
|
|
168
|
+
encoding: 'utf8',
|
|
169
|
+
stdio: ['ignore', 'pipe', 'inherit'],
|
|
170
|
+
maxBuffer: 64 * 1024 * 1024,
|
|
171
|
+
});
|
|
172
|
+
process.stdout.write(run.stdout ?? '');
|
|
173
|
+
const report = /report: (https?:\/\/\S+)/.exec(run.stdout ?? '')?.[1];
|
|
174
|
+
const summary = summaryOf(resultsOf(out), report, picked);
|
|
175
|
+
if (process.env.GITHUB_STEP_SUMMARY)
|
|
176
|
+
fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, summary);
|
|
177
|
+
return { text: summary, code: run.status ?? 1 };
|
|
178
|
+
}
|
|
179
|
+
finally {
|
|
180
|
+
down();
|
|
181
|
+
}
|
|
182
|
+
},
|
|
183
|
+
}),
|
|
184
|
+
];
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare const coverageCommands: import("../registry.js").Command[];
|