lynkeus 0.3.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/cases/coverage.d.ts +31 -0
- package/lib/cases/coverage.js +82 -0
- package/lib/cases/draft.d.ts +21 -0
- package/lib/cases/draft.js +53 -0
- package/lib/cases/fixtures.d.ts +11 -0
- package/lib/cases/fixtures.js +59 -8
- package/lib/cases/frames.d.ts +6 -0
- package/lib/cases/frames.js +28 -0
- package/lib/cases/nativeLogs.d.ts +14 -0
- package/lib/cases/nativeLogs.js +111 -0
- package/lib/cases/runDirs.d.ts +13 -0
- package/lib/cases/runDirs.js +48 -0
- package/lib/cases/runfile.d.ts +5 -1
- package/lib/cases/runfile.js +67 -4
- package/lib/cases/runner.d.ts +58 -1
- package/lib/cases/runner.js +322 -39
- package/lib/cases/templates.d.ts +2 -2
- package/lib/cases/templates.js +20 -2
- package/lib/cases/traces.js +4 -1
- package/lib/cli.js +27 -4
- package/lib/commands/cases.d.ts +8 -0
- package/lib/commands/cases.js +154 -25
- package/lib/commands/ci.d.ts +13 -0
- package/lib/commands/ci.js +184 -0
- package/lib/commands/coverage.d.ts +1 -0
- package/lib/commands/coverage.js +151 -0
- package/lib/commands/device.js +32 -2
- package/lib/commands/doctor.js +8 -2
- package/lib/commands/drive.js +3 -1
- package/lib/commands/fixtures.d.ts +7 -0
- package/lib/commands/fixtures.js +94 -0
- package/lib/commands/headless.d.ts +11 -0
- package/lib/commands/headless.js +189 -9
- package/lib/commands/init.d.ts +23 -0
- package/lib/commands/init.js +231 -26
- package/lib/commands/map.d.ts +2 -0
- package/lib/commands/map.js +18 -14
- package/lib/commands/os.js +43 -10
- package/lib/commands/report.d.ts +4 -0
- package/lib/commands/report.js +156 -20
- package/lib/config.d.ts +41 -0
- package/lib/config.js +30 -0
- package/lib/device/agentVersion.d.ts +7 -0
- package/lib/device/agentVersion.js +14 -0
- package/lib/device/devices.js +6 -0
- package/lib/device/driver.js +2 -2
- package/lib/knowledge/analyticsTraffic.d.ts +10 -0
- package/lib/knowledge/analyticsTraffic.js +109 -0
- package/lib/knowledge/errors.js +6 -0
- package/lib/knowledge/events.d.ts +2 -0
- package/lib/knowledge/why.d.ts +2 -0
- package/lib/knowledge/why.js +1 -1
- package/lib/mcp/server.js +45 -4
- package/lib/runtime.d.ts +9 -1
- package/lib/runtime.js +25 -3
- package/package.json +7 -4
- package/src/cases/coverage.ts +99 -0
- package/src/cases/draft.ts +75 -0
- package/src/cases/fixtures.ts +57 -8
- package/src/cases/frames.ts +30 -0
- package/src/cases/nativeLogs.ts +112 -0
- package/src/cases/runDirs.ts +48 -0
- package/src/cases/runfile.ts +64 -4
- package/src/cases/runner.ts +372 -42
- package/src/cases/templates.ts +20 -2
- package/src/cases/traces.ts +3 -1
- package/src/cli.ts +27 -4
- package/src/commands/cases.ts +165 -23
- package/src/commands/ci.ts +189 -0
- package/src/commands/coverage.ts +147 -0
- package/src/commands/device.ts +27 -2
- package/src/commands/doctor.ts +9 -2
- package/src/commands/drive.ts +3 -1
- package/src/commands/fixtures.ts +102 -0
- package/src/commands/headless.ts +197 -9
- package/src/commands/init.ts +270 -26
- package/src/commands/map.ts +20 -15
- package/src/commands/os.ts +38 -10
- package/src/commands/report.ts +162 -18
- package/src/config.ts +48 -0
- package/src/device/agentVersion.ts +17 -0
- package/src/device/devices.ts +5 -0
- package/src/device/driver.ts +2 -2
- package/src/knowledge/analyticsTraffic.ts +142 -0
- package/src/knowledge/errors.ts +6 -0
- package/src/knowledge/events.ts +2 -1
- package/src/knowledge/why.ts +3 -1
- package/src/mcp/server.ts +45 -4
- package/src/runtime.ts +24 -3
package/src/cases/traces.ts
CHANGED
|
@@ -193,7 +193,7 @@ export const decodeJson = (body: unknown): Span[] => {
|
|
|
193
193
|
|
|
194
194
|
// What a report keeps of a span's attributes: what it did, not every label an instrumentation adds.
|
|
195
195
|
const KEEP =
|
|
196
|
-
/^(http\.(request\.)?method|http\.(response\.)?status_code|http\.route|http\.target|url\.(path|full)|server\.address|net\.peer\.name|
|
|
196
|
+
/^(http\.(request\.)?method|http\.(response\.)?status_code|http\.route|http\.target|url\.(path|full)|server\.address|net\.peer\.name|messaging\.|code\.(namespace|function)|rpc\.|exception\.message|enduser\.id)/;
|
|
197
197
|
|
|
198
198
|
/**
|
|
199
199
|
* Listens for OTLP traces while a run goes on: a backend with OTEL_EXPORTER_OTLP_ENDPOINT pointed here sends
|
|
@@ -272,6 +272,8 @@ export class TraceSink {
|
|
|
272
272
|
return undefined;
|
|
273
273
|
};
|
|
274
274
|
for (const span of [...spans].sort((a, b) => (a.startNs < b.startNs ? -1 : 1))) {
|
|
275
|
+
// A database query never leaves the machine: its statement names the backend's tables and columns.
|
|
276
|
+
if (span.attributes['db.system'] !== undefined) continue;
|
|
275
277
|
const where = place(span);
|
|
276
278
|
if (!where) continue;
|
|
277
279
|
const startedMs = [...steps.values()].find((s) => s.index === where.step)!.startedMs;
|
package/src/cli.ts
CHANGED
|
@@ -11,16 +11,30 @@ import { driveCommands } from './commands/drive.js';
|
|
|
11
11
|
import { headlessCommands } from './commands/headless.js';
|
|
12
12
|
import { mapCommands } from './commands/map.js';
|
|
13
13
|
import { osCommands } from './commands/os.js';
|
|
14
|
+
import { ciCommands } from './commands/ci.js';
|
|
15
|
+
import { coverageCommands } from './commands/coverage.js';
|
|
16
|
+
import { fixturesCommands } from './commands/fixtures.js';
|
|
14
17
|
import { reportCommands } from './commands/report.js';
|
|
15
18
|
import { runsCommands } from './commands/runs.js';
|
|
16
|
-
import { graphFile, loadConfig, loadGraph, portFromEnv } from './config.js';
|
|
17
|
-
import { agentMismatch } from './device/agentVersion.js';
|
|
19
|
+
import { graphFile, loadConfig, loadGraph, portFromEnv, readToken } from './config.js';
|
|
20
|
+
import { agentMismatch, appSwitched, helloIdentity } from './device/agentVersion.js';
|
|
18
21
|
import { HEADLESS_HOST, pickAgentDevice, pickAndroidDevice } from './device/devices.js';
|
|
19
22
|
import { Device } from './device/driver.js';
|
|
20
23
|
import { type Base, commands, find, helpFor, inSession, referenceMarkdown, register, resolve, usage } from './registry.js';
|
|
21
24
|
import { execute } from './runtime.js';
|
|
22
25
|
|
|
23
|
-
register([
|
|
26
|
+
register([
|
|
27
|
+
...driveCommands,
|
|
28
|
+
...osCommands,
|
|
29
|
+
...mapCommands,
|
|
30
|
+
...runsCommands,
|
|
31
|
+
...caseCommands,
|
|
32
|
+
...coverageCommands,
|
|
33
|
+
...ciCommands,
|
|
34
|
+
...fixturesCommands,
|
|
35
|
+
...reportCommands,
|
|
36
|
+
...headlessCommands,
|
|
37
|
+
]);
|
|
24
38
|
|
|
25
39
|
/** The global flags, read leniently from the whole argv before any command parses its own. */
|
|
26
40
|
const globals = (argv: string[]) => {
|
|
@@ -42,6 +56,8 @@ export type Session = { ctx: Base; stop: () => Promise<void> };
|
|
|
42
56
|
|
|
43
57
|
// Once per process: a suite reattaches for every case.
|
|
44
58
|
let mismatchTold = false;
|
|
59
|
+
let switchTold = false;
|
|
60
|
+
let lastApp: string | undefined;
|
|
45
61
|
|
|
46
62
|
// `ownHost`: the port belongs to a host this run started for a case; whatever dials in is the device.
|
|
47
63
|
const contextFor = (argv: string[], attached = false, port?: number, ownHost = false): Session => {
|
|
@@ -56,8 +72,15 @@ const contextFor = (argv: string[], attached = false, port?: number, ownHost = f
|
|
|
56
72
|
const appId = (g.app as string | undefined) ?? process.env.LYNKEUS_APP_ID ?? config.app;
|
|
57
73
|
const started = performance.now();
|
|
58
74
|
const attachedIn = (d: Device) => err(`attached in ${Math.round(performance.now() - started)}ms (${d.info.name})`);
|
|
59
|
-
const server = new AgentServer({ token:
|
|
75
|
+
const server = new AgentServer({ token: readToken(root), port: port ?? portFromEnv(config) });
|
|
60
76
|
server.on('hello', (hello: HelloEvent) => {
|
|
77
|
+
const identity = helloIdentity(hello);
|
|
78
|
+
const switched = switchTold ? undefined : appSwitched(lastApp, identity, server.port);
|
|
79
|
+
lastApp = identity;
|
|
80
|
+
if (switched) {
|
|
81
|
+
switchTold = true;
|
|
82
|
+
err(switched);
|
|
83
|
+
}
|
|
61
84
|
const told = mismatchTold ? undefined : agentMismatch(hello);
|
|
62
85
|
if (!told) return;
|
|
63
86
|
mismatchTold = true;
|
package/src/commands/cases.ts
CHANGED
|
@@ -2,6 +2,7 @@ import { type ChildProcess, spawn } from 'node:child_process';
|
|
|
2
2
|
import fs from 'node:fs';
|
|
3
3
|
import os from 'node:os';
|
|
4
4
|
import path from 'node:path';
|
|
5
|
+
import { stripVTControlCharacters } from 'node:util';
|
|
5
6
|
|
|
6
7
|
import { Fixtures } from '../cases/fixtures.js';
|
|
7
8
|
import { readCases } from '../cases/format.js';
|
|
@@ -18,6 +19,8 @@ import { eventsFile } from '../knowledge/events.js';
|
|
|
18
19
|
import { type Base, define } from '../registry.js';
|
|
19
20
|
import { describeChanges, pushRun, service } from './report.js';
|
|
20
21
|
import { TraceSink } from '../cases/traces.js';
|
|
22
|
+
import { megabytes, sizeOf } from '../cases/runDirs.js';
|
|
23
|
+
import { type NativeLogs, startNativeLogs } from '../cases/nativeLogs.js';
|
|
21
24
|
|
|
22
25
|
const startHost = async (root: string, err: (line: string) => void): Promise<ChildProcess> => {
|
|
23
26
|
// Its own process group: the host starts a test runner of its own, and stopping one without the other leaves an app dialing in.
|
|
@@ -27,8 +30,27 @@ const startHost = async (root: string, err: (line: string) => void): Promise<Chi
|
|
|
27
30
|
stdio: ['ignore', 'pipe', 'pipe'],
|
|
28
31
|
detached: true,
|
|
29
32
|
});
|
|
33
|
+
// Everything the host prints, for the render errors and warnings a failed case points at; the console keeps the run's own lines.
|
|
34
|
+
const logFile = path.join(os.tmpdir(), `lynkeus-headless-${process.pid}.log`);
|
|
35
|
+
const logStream = fs.createWriteStream(logFile);
|
|
36
|
+
logStream.on('error', () => {});
|
|
37
|
+
host.once('exit', () => logStream.end());
|
|
30
38
|
let output = '';
|
|
31
39
|
const tail = (n: number) => output.trim().split('\n').slice(-n).join(' · ');
|
|
40
|
+
// `headless start` ends a crash with what it usually means; without that, the error line jest printed.
|
|
41
|
+
const why = () => {
|
|
42
|
+
const plain = stripVTControlCharacters(output);
|
|
43
|
+
const explained = plain.split('lynkeus: what this looks like')[1];
|
|
44
|
+
if (explained)
|
|
45
|
+
return explained
|
|
46
|
+
.trim()
|
|
47
|
+
.split('\n')
|
|
48
|
+
.map((line) => line.trim())
|
|
49
|
+
.join(' · ');
|
|
50
|
+
// Jest prints a suite that throws while loading as "● Test suite failed to run", then the thrown message.
|
|
51
|
+
const thrown = plain.match(/Test suite failed to run\s*\n\s*\n?\s*(\S.*)$/m)?.[1]?.trim();
|
|
52
|
+
return thrown ?? plain.match(/^\s*(\w*Error\b.*|Cannot find module.*)$/m)?.[1]?.trim() ?? tail(3);
|
|
53
|
+
};
|
|
32
54
|
await new Promise<void>((resolve, reject) => {
|
|
33
55
|
const settle = (error?: Error) => {
|
|
34
56
|
clearTimeout(timer);
|
|
@@ -37,15 +59,17 @@ const startHost = async (root: string, err: (line: string) => void): Promise<Chi
|
|
|
37
59
|
};
|
|
38
60
|
const timer = setTimeout(() => settle(new Error(`the host did not come up within 180 s: ${tail(3)}`)), 180_000);
|
|
39
61
|
const read = (chunk: Buffer) => {
|
|
62
|
+
logStream.write(chunk);
|
|
40
63
|
output = (output + chunk.toString()).slice(-4000);
|
|
41
64
|
if (/waiting for a driver/.test(output)) settle();
|
|
42
|
-
else if (/Test suite failed/.test(output)) settle(new Error(`the host failed to start: ${tail(5)}`));
|
|
43
65
|
};
|
|
44
66
|
host.stdout?.on('data', read);
|
|
45
67
|
host.stderr?.on('data', read);
|
|
46
|
-
|
|
68
|
+
// A suite that fails to load exits right after printing why, so its explanation is read in full here.
|
|
69
|
+
host.once('exit', (code) => settle(new Error(`${/Test suite failed/.test(output) ? 'the host failed to start' : `the host exited (${code})`}: ${why()}`)));
|
|
47
70
|
});
|
|
48
|
-
|
|
71
|
+
// The listeners above stay: the log takes all the host prints until it stops.
|
|
72
|
+
err(`host: up (its log: ${logFile})`);
|
|
49
73
|
return host;
|
|
50
74
|
};
|
|
51
75
|
|
|
@@ -102,19 +126,27 @@ const idleShare = async (): Promise<number> => {
|
|
|
102
126
|
return after.all > before.all ? (after.idle - before.idle) / (after.all - before.all) : 1;
|
|
103
127
|
};
|
|
104
128
|
|
|
105
|
-
/**
|
|
106
|
-
const
|
|
129
|
+
/** Every case under a directory and its subdirectories, in path order, so a suite can group its cases in folders. */
|
|
130
|
+
const casesIn = (dir: string): string[] =>
|
|
131
|
+
fs
|
|
132
|
+
.readdirSync(dir, { withFileTypes: true })
|
|
133
|
+
.filter((e) => !e.name.startsWith('.') && e.name !== 'node_modules')
|
|
134
|
+
.sort((a, b) => a.name.localeCompare(b.name))
|
|
135
|
+
.flatMap((e) => {
|
|
136
|
+
const full = path.join(dir, e.name);
|
|
137
|
+
if (e.isDirectory()) return casesIn(full);
|
|
138
|
+
return e.name.endsWith('.md') && !/^(README|FORMAT|CATALOG)\.md$/i.test(e.name) ? [full] : [];
|
|
139
|
+
});
|
|
140
|
+
|
|
141
|
+
/** The case files named (a directory stands for every case under it), or every case under the cases directory; a sentence when there are none. */
|
|
142
|
+
export const caseFiles = (args: string[], home: string, casesDir: string | undefined): string[] | string => {
|
|
107
143
|
const dir = path.resolve(home, casesDir ?? 'cases');
|
|
108
|
-
const
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
.sort()
|
|
115
|
-
.map((f) => path.join(dir, f))
|
|
116
|
-
: [];
|
|
117
|
-
return files.length > 0 ? files : `no cases under ${path.relative(home, dir) || '.'}; \`lynkeus case init\` writes one`;
|
|
144
|
+
const named = args.map((f) => path.resolve(f));
|
|
145
|
+
const missing = named.find((f) => !fs.existsSync(f));
|
|
146
|
+
if (missing) return `no such case file: ${path.relative(process.cwd(), missing) || missing}`;
|
|
147
|
+
const files = named.length ? named.flatMap((f) => (fs.statSync(f).isDirectory() ? casesIn(f) : [f])) : fs.existsSync(dir) ? casesIn(dir) : [];
|
|
148
|
+
const where = named.length === 1 ? path.relative(home, named[0]!) : path.relative(home, dir);
|
|
149
|
+
return files.length > 0 ? files : `no cases under ${where || '.'}; \`lynkeus case init\` writes one`;
|
|
118
150
|
};
|
|
119
151
|
|
|
120
152
|
type PrepareOptions = { base: Base; config: CasesConfig; store: Store; open: () => Fixtures; limit: number; idle: number; jobs: number };
|
|
@@ -169,15 +201,48 @@ const spawnHost = (command: string, port: number, job: number, cwd: string): Chi
|
|
|
169
201
|
});
|
|
170
202
|
|
|
171
203
|
/** A run that could not be uploaded still ran: the failure is said, not raised. */
|
|
172
|
-
|
|
204
|
+
// A run that kept evidence but none of a kind says so when it ends, not only as a gap in the report.
|
|
205
|
+
export const missingEvidence = (reports: CaseReport[], asked: { video: boolean }): string[] => {
|
|
206
|
+
if (reports.length === 0) return [];
|
|
207
|
+
const steps = reports.flatMap((r) => r.steps);
|
|
208
|
+
const fromApp = steps.some((s) => s.sent?.some((e) => e.via !== 'server')) || reports.some((r) => (r.events?.length ?? 0) > 0);
|
|
209
|
+
return [
|
|
210
|
+
...(fromApp ? [] : ['⚠ no case kept an analytics event from the app (the agent keeps the calls that reach Mixpanel, AppsFlyer and Firebase Analytics).']),
|
|
211
|
+
...(asked.video && !reports.some((r) => r.video)
|
|
212
|
+
? ['⚠ --video kept no film: this host does not film (fastsim does; a simulator, an emulator or headless keep frames only).']
|
|
213
|
+
: []),
|
|
214
|
+
];
|
|
215
|
+
};
|
|
216
|
+
|
|
217
|
+
const pushed = async (
|
|
218
|
+
dir: string,
|
|
219
|
+
to: Exclude<ReturnType<typeof service>, string>,
|
|
220
|
+
project: string,
|
|
221
|
+
err: (line: string) => void,
|
|
222
|
+
afterPush: 'keep' | 'delete' = 'keep',
|
|
223
|
+
) => {
|
|
173
224
|
try {
|
|
174
|
-
const run = await pushRun(dir, { ...to, project });
|
|
175
|
-
return [
|
|
225
|
+
const run = await pushRun(dir, { ...to, project, onReady: (url) => err(`report: ${url} (its frames are on their way)`), onProgress: err });
|
|
226
|
+
return [
|
|
227
|
+
`report: ${run.url}${run.refused.length ? ` (not kept on this plan: ${run.refused.join(', ')})` : ''}`,
|
|
228
|
+
...describeChanges(run.changes),
|
|
229
|
+
keptOrDeleted(dir, afterPush, run.refused.length === 0),
|
|
230
|
+
].join('\n');
|
|
176
231
|
} catch (error) {
|
|
177
232
|
return `the report was not uploaded: ${(error as Error).message}`;
|
|
178
233
|
}
|
|
179
234
|
};
|
|
180
235
|
|
|
236
|
+
// After a push: the folder deleted when the config asks and the service took every file, else how much it holds.
|
|
237
|
+
export const keptOrDeleted = (dir: string, afterPush: 'keep' | 'delete', complete: boolean): string => {
|
|
238
|
+
const bytes = sizeOf(dir);
|
|
239
|
+
if (afterPush === 'delete' && complete) {
|
|
240
|
+
fs.rmSync(dir, { recursive: true, force: true });
|
|
241
|
+
return `run folder deleted after the push (${megabytes(bytes)})`;
|
|
242
|
+
}
|
|
243
|
+
return `run folder: ${megabytes(bytes)} on disk${afterPush === 'delete' ? ', kept because some files were not taken' : ''} · \`lynkeus report prune ${path.dirname(dir)} --older 7d\` frees old ones`;
|
|
244
|
+
};
|
|
245
|
+
|
|
181
246
|
export const caseCommands = [
|
|
182
247
|
define({
|
|
183
248
|
name: 'case run',
|
|
@@ -194,7 +259,32 @@ export const caseCommands = [
|
|
|
194
259
|
report: { type: 'string', help: 'append one JSON line per case', value: 'file' },
|
|
195
260
|
events: { type: 'string', help: 'keep what the app told its analytics, per case, as <dir>/<case id>/events.jsonl', value: 'dir' },
|
|
196
261
|
out: { type: 'string', help: 'write the run as a directory a report can read: run.json and cases.jsonl, a line per attempt', value: 'dir' },
|
|
262
|
+
'no-app-events': {
|
|
263
|
+
type: 'boolean',
|
|
264
|
+
help: "with --out: leave the app's own event list unread. Reading it can turn on the app's tracking debug, which may draw on the screen; frames two runs compare should not have it",
|
|
265
|
+
},
|
|
197
266
|
screenshots: { type: 'boolean', help: 'with --out: a frame of the screen after every step of the app' },
|
|
267
|
+
settle: {
|
|
268
|
+
type: 'number',
|
|
269
|
+
help: 'with --screenshots: how long the screen must stay still before a frame, in ms; 700 for frames two runs can be compared by',
|
|
270
|
+
default: 150,
|
|
271
|
+
value: 'ms',
|
|
272
|
+
},
|
|
273
|
+
'frames-on-change': {
|
|
274
|
+
type: 'boolean',
|
|
275
|
+
help: 'with --screenshots: a step that only looks (app.see, app.wait) on an unchanged screen shares the last frame',
|
|
276
|
+
},
|
|
277
|
+
still: {
|
|
278
|
+
type: 'number',
|
|
279
|
+
help: 'with --screenshots, on a simulator or emulator: take each frame again until two in a row match, this many ms apart (a native transition the agent cannot see)',
|
|
280
|
+
value: 'ms',
|
|
281
|
+
},
|
|
282
|
+
frames: {
|
|
283
|
+
type: 'string',
|
|
284
|
+
help: "with --screenshots: avif, the default and a few percent of the bytes (a simulator's or emulator's PNG is encoded again), or png, exact, only for frames two runs are compared by pixel for pixel",
|
|
285
|
+
default: 'avif',
|
|
286
|
+
value: 'avif|png',
|
|
287
|
+
},
|
|
198
288
|
video: { type: 'boolean', help: 'with --out: film each case, where the host films (the step says at what second it began)' },
|
|
199
289
|
traces: {
|
|
200
290
|
type: 'boolean',
|
|
@@ -204,6 +294,7 @@ export const caseCommands = [
|
|
|
204
294
|
push: { type: 'boolean', help: 'with --out: upload the run to the reports service when it ends and print the report’s address' },
|
|
205
295
|
project: { type: 'string', help: 'with --push: the project the run is filed under (default: the app’s folder name)', value: 'name' },
|
|
206
296
|
'video-fps': { type: 'number', help: 'frames per second of the film', default: 30, value: 'n' },
|
|
297
|
+
'video-scale': { type: 'number', help: 'pixels per point of the film, where the host takes one (fastsim: 1 to 3, 2 unless told)', value: 'n' },
|
|
207
298
|
flakes: { type: 'string', help: 'the ledger of how each case did across runs (default .lynkeus/flakes.json)', value: 'file' },
|
|
208
299
|
prepared: { type: 'string', help: 'where `case prepare` leaves setups made ahead; a case found there skips its setup', value: 'dir' },
|
|
209
300
|
jobs: { type: 'number', help: 'cases at once, each on its own host (needs --ports)', value: 'n', default: 1 },
|
|
@@ -220,7 +311,33 @@ export const caseCommands = [
|
|
|
220
311
|
},
|
|
221
312
|
needs: 'nothing',
|
|
222
313
|
session: false,
|
|
223
|
-
mcp:
|
|
314
|
+
mcp: 'core',
|
|
315
|
+
mcpOmit: [
|
|
316
|
+
'seen',
|
|
317
|
+
'report',
|
|
318
|
+
'events',
|
|
319
|
+
'settle',
|
|
320
|
+
'still',
|
|
321
|
+
'frames-on-change',
|
|
322
|
+
'frames',
|
|
323
|
+
'traces',
|
|
324
|
+
'traces-port',
|
|
325
|
+
'video-fps',
|
|
326
|
+
'video-scale',
|
|
327
|
+
'flakes',
|
|
328
|
+
'prepared',
|
|
329
|
+
'jobs',
|
|
330
|
+
'ports',
|
|
331
|
+
'host-command',
|
|
332
|
+
'in-order',
|
|
333
|
+
'ahead',
|
|
334
|
+
'idle',
|
|
335
|
+
'continue',
|
|
336
|
+
'video',
|
|
337
|
+
'project',
|
|
338
|
+
],
|
|
339
|
+
mcpDetails:
|
|
340
|
+
'Runs case files (default: all in cases.dir) against the attached app, or with `host` in Node. A case that fails comes back with its screen and reason. `out` + `screenshots` + `push` uploads the run and prints its address.',
|
|
224
341
|
run: async (ctx) => {
|
|
225
342
|
const config = ctx.config.cases ?? {};
|
|
226
343
|
const home = config.base ?? ctx.root;
|
|
@@ -241,12 +358,18 @@ export const caseCommands = [
|
|
|
241
358
|
? {
|
|
242
359
|
dir: out.evidenceDir(id, attempt),
|
|
243
360
|
screenshots: !!ctx.flags.screenshots,
|
|
244
|
-
|
|
361
|
+
settleMs: Number(ctx.flags.settle ?? 150),
|
|
362
|
+
...(ctx.flags.still ? { stillMs: Number(ctx.flags.still) } : {}),
|
|
363
|
+
...(ctx.flags['frames-on-change'] ? { onChange: true } : {}),
|
|
364
|
+
frames: ctx.flags.frames === 'png' ? ('png' as const) : ('avif' as const),
|
|
365
|
+
...(ctx.flags.video
|
|
366
|
+
? { video: { fps: Number(ctx.flags['video-fps'] ?? 30), ...(ctx.flags['video-scale'] ? { scale: Number(ctx.flags['video-scale']) } : {}) } }
|
|
367
|
+
: {}),
|
|
245
368
|
}
|
|
246
369
|
: undefined;
|
|
247
370
|
const ledger = dry ? undefined : new Flakes(path.resolve(ctx.root, typeof ctx.flags.flakes === 'string' ? ctx.flags.flakes : '.lynkeus/flakes.json'));
|
|
248
371
|
// A run file files each event under its step, so it needs them collected.
|
|
249
|
-
const events = typeof ctx.flags.events === 'string' || out !== undefined;
|
|
372
|
+
const events = typeof ctx.flags.events === 'string' || (out !== undefined && !ctx.flags['no-app-events']);
|
|
250
373
|
const claimed = dry
|
|
251
374
|
? undefined
|
|
252
375
|
: claimPorts(
|
|
@@ -297,6 +420,15 @@ export const caseCommands = [
|
|
|
297
420
|
const hostCommand = typeof ctx.flags['host-command'] === 'string' && !dry ? ctx.flags['host-command'] : undefined;
|
|
298
421
|
if ((jobs > 1 || hostCommand) && (ports.length < jobs || ports.some((p) => !Number.isInteger(p)) || !ctx.forPort))
|
|
299
422
|
return { text: `--jobs and --host-command need a driver port per job: --ports with ${jobs}`, code: 2 };
|
|
423
|
+
// One simulator or emulator, one log: with several jobs a line could not be told to its case.
|
|
424
|
+
const native: NativeLogs | undefined =
|
|
425
|
+
out && jobs === 1 && !hostCommand && !ctx.flags.host && ctx.config.app
|
|
426
|
+
? await ctx
|
|
427
|
+
.device()
|
|
428
|
+
.then((d) => d.resolved())
|
|
429
|
+
.then((info) => startNativeLogs(info, ctx.config.app as string))
|
|
430
|
+
.catch(() => undefined)
|
|
431
|
+
: undefined;
|
|
300
432
|
// Several jobs finish together only if the longest cases start first: one left for the end runs alone
|
|
301
433
|
// while the other jobs wait. A case the ledger has not timed goes first too, as it may be long.
|
|
302
434
|
if (jobs > 1 && ledger && !ctx.flags['in-order']) {
|
|
@@ -359,6 +491,7 @@ export const caseCommands = [
|
|
|
359
491
|
continue: !!ctx.flags.continue,
|
|
360
492
|
routes: seenDir ? (host ? 'fresh' : true) : false,
|
|
361
493
|
traces,
|
|
494
|
+
nativeLogs: native,
|
|
362
495
|
});
|
|
363
496
|
if (seenDir && report.result === 'passed' && report.routes?.length) {
|
|
364
497
|
fs.mkdirSync(seenDir, { recursive: true });
|
|
@@ -407,6 +540,7 @@ export const caseCommands = [
|
|
|
407
540
|
await Promise.all(Array.from({ length: jobs }, (_, i) => job(i)));
|
|
408
541
|
await preparing;
|
|
409
542
|
if (out) out.finish({ platform: seen?.platform, native: seen?.native });
|
|
543
|
+
native?.stop();
|
|
410
544
|
} finally {
|
|
411
545
|
stop();
|
|
412
546
|
}
|
|
@@ -414,9 +548,15 @@ export const caseCommands = [
|
|
|
414
548
|
const unsupported = count('unsupported');
|
|
415
549
|
const summary = [
|
|
416
550
|
...reports.map(
|
|
417
|
-
(r) =>
|
|
551
|
+
(r) =>
|
|
552
|
+
`${{ passed: '✅', failed: '❌', unsupported: '🚫' }[r.result]} ${r.id}${r.retried && r.result === 'passed' ? ' ⚠ passed on retry' : ''}` +
|
|
553
|
+
// In a session or over MCP the caller reads this answer and nothing else: a case that did not pass brings its steps.
|
|
554
|
+
(ctx.attached && r.result !== 'passed' ? `\n${describe(r)}` : ''),
|
|
418
555
|
),
|
|
419
556
|
`${count('passed')}/${reports.length} passed${unsupported ? `, ${unsupported} cannot run on this host` : ''}`,
|
|
557
|
+
// Its full path: a relative --out resolves against where the CLI was started, which over MCP is the app's folder.
|
|
558
|
+
...(out ? [`run kept in ${path.resolve(ctx.flags.out as string)}`] : []),
|
|
559
|
+
...(out ? missingEvidence(reports, { video: !!ctx.flags.video }) : []),
|
|
420
560
|
...(ledger ? ledger.flaky(reports.map((r) => r.id)) : []),
|
|
421
561
|
...(reportsTo && out
|
|
422
562
|
? [
|
|
@@ -424,6 +564,8 @@ export const caseCommands = [
|
|
|
424
564
|
path.resolve(ctx.flags.out as string),
|
|
425
565
|
reportsTo,
|
|
426
566
|
typeof ctx.flags.project === 'string' ? ctx.flags.project : path.basename(ctx.root),
|
|
567
|
+
ctx.err,
|
|
568
|
+
ctx.config.report?.afterPush === 'delete' ? 'delete' : 'keep',
|
|
427
569
|
),
|
|
428
570
|
]
|
|
429
571
|
: []),
|
|
@@ -481,7 +623,7 @@ export const caseCommands = [
|
|
|
481
623
|
flags: { force: { type: 'boolean', help: 'overwrite what is there' } },
|
|
482
624
|
needs: 'nothing',
|
|
483
625
|
session: false,
|
|
484
|
-
mcp:
|
|
626
|
+
mcp: 'core',
|
|
485
627
|
run: async (ctx) => {
|
|
486
628
|
const dir = path.resolve(ctx.root, ctx.config.cases?.dir ?? 'cases');
|
|
487
629
|
fs.mkdirSync(dir, { recursive: true });
|
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
import { type ChildProcess, spawn, spawnSync } from 'node:child_process';
|
|
2
|
+
import fs from 'node:fs';
|
|
3
|
+
import path from 'node:path';
|
|
4
|
+
|
|
5
|
+
import { coverageOf, impactOf, readSeen } from '../cases/coverage.js';
|
|
6
|
+
import { readCases } from '../cases/format.js';
|
|
7
|
+
import { screensAffectedBy } from '../knowledge/query.js';
|
|
8
|
+
import { define } from '../registry.js';
|
|
9
|
+
import { caseFiles } from './cases.js';
|
|
10
|
+
import { changedSources } from './map.js';
|
|
11
|
+
|
|
12
|
+
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
|
|
13
|
+
|
|
14
|
+
/** The case results of a run folder, by each case's last attempt. */
|
|
15
|
+
export const resultsOf = (dir: string): { id: string; result: string; title?: string }[] => {
|
|
16
|
+
const file = path.join(dir, 'cases.jsonl');
|
|
17
|
+
if (!fs.existsSync(file)) return [];
|
|
18
|
+
const last = new Map<string, { id: string; result: string; title?: string }>();
|
|
19
|
+
for (const line of fs.readFileSync(file, 'utf8').split('\n')) {
|
|
20
|
+
if (!line.trim()) continue;
|
|
21
|
+
const c = JSON.parse(line) as { id: string; result: string; title?: string };
|
|
22
|
+
last.set(c.id, c);
|
|
23
|
+
}
|
|
24
|
+
return [...last.values()];
|
|
25
|
+
};
|
|
26
|
+
|
|
27
|
+
/** The summary a CI job shows: how many passed, which failed, and the report. */
|
|
28
|
+
export const summaryOf = (results: { id: string; result: string; title?: string }[], report: string | undefined, picked: string): string => {
|
|
29
|
+
const failed = results.filter((r) => r.result === 'failed');
|
|
30
|
+
const passed = results.filter((r) => r.result === 'passed').length;
|
|
31
|
+
return [
|
|
32
|
+
`### lynkeus: ${passed}/${results.length} cases passed${failed.length ? `, ${failed.length} failed` : ''}`,
|
|
33
|
+
'',
|
|
34
|
+
picked,
|
|
35
|
+
...(report ? ['', `[Open the report](${report})`] : []),
|
|
36
|
+
...(failed.length ? ['', '| Failed | |', '|---|---|', ...failed.map((f) => `| \`${f.id}\` | ${f.title ?? ''} |`)] : []),
|
|
37
|
+
'',
|
|
38
|
+
].join('\n');
|
|
39
|
+
};
|
|
40
|
+
|
|
41
|
+
const waitFor = async (ok: () => boolean | Promise<boolean>, timeoutMs: number, every = 3000): Promise<boolean> => {
|
|
42
|
+
const until = Date.now() + timeoutMs;
|
|
43
|
+
for (;;) {
|
|
44
|
+
if (await ok()) return true;
|
|
45
|
+
if (Date.now() > until) return false;
|
|
46
|
+
await sleep(every);
|
|
47
|
+
}
|
|
48
|
+
};
|
|
49
|
+
|
|
50
|
+
export const ciCommands = [
|
|
51
|
+
define({
|
|
52
|
+
name: 'ci',
|
|
53
|
+
group: 'runs',
|
|
54
|
+
summary: 'Run the suite the way a CI job does: bring the backend and N hosts up, run the cases across them, report, tear down',
|
|
55
|
+
details:
|
|
56
|
+
'Reads `ci` in lynkeus.config.json: `backend.up` / `backend.down` (the project’s commands; lynkeus knows no backend) and `backend.ready` (a URL to wait for), `hosts.count` and `hosts.command` (default: a headless host; `{port}` is where it dials), `push`, and extra `run` flags. --affected runs only the cases a branch’s changes reach (as `case impact` finds them). Writes the job summary to $GITHUB_STEP_SUMMARY when it is set. The exit code is the suite’s.',
|
|
57
|
+
flags: {
|
|
58
|
+
affected: { type: 'boolean', help: 'only the cases the changes against --base reach' },
|
|
59
|
+
base: { type: 'string', help: 'git ref --affected diffs against', default: 'origin/main', value: 'ref' },
|
|
60
|
+
jobs: { type: 'number', help: 'how many hosts and jobs (default ci.hosts.count, else 2)', value: 'n' },
|
|
61
|
+
out: { type: 'string', help: 'where the run goes', default: 'lynkeus-run', value: 'dir' },
|
|
62
|
+
'keep-up': { type: 'boolean', help: 'leave the backend and the hosts up afterwards' },
|
|
63
|
+
'dry-run': { type: 'boolean', help: 'say what it would do and run nothing' },
|
|
64
|
+
},
|
|
65
|
+
needs: 'nothing',
|
|
66
|
+
session: false,
|
|
67
|
+
mcp: false,
|
|
68
|
+
run: async (ctx) => {
|
|
69
|
+
const ci = ctx.config.ci ?? {};
|
|
70
|
+
const config = ctx.config.cases ?? {};
|
|
71
|
+
const home = config.base ?? ctx.root;
|
|
72
|
+
const self = process.argv[1]!;
|
|
73
|
+
const say = (line: string) => ctx.err(`▸ ${line}`);
|
|
74
|
+
const jobs = Math.max(1, Number(ctx.flags.jobs ?? ci.hosts?.count ?? 2));
|
|
75
|
+
const out = path.resolve(ctx.root, String(ctx.flags.out));
|
|
76
|
+
|
|
77
|
+
// Which cases: all, or the ones the branch's changes reach.
|
|
78
|
+
const all = caseFiles([], home, config.dir);
|
|
79
|
+
if (typeof all === 'string') return { text: all, code: 1 };
|
|
80
|
+
let files = all;
|
|
81
|
+
let picked = `All ${all.length} case files.`;
|
|
82
|
+
if (ctx.flags.affected) {
|
|
83
|
+
const graphFile = path.join(ctx.root, '.lynkeus', 'graph.json');
|
|
84
|
+
if (!fs.existsSync(graphFile)) spawnSync(process.execPath, [self, 'map', 'build'], { cwd: ctx.root, stdio: 'inherit' });
|
|
85
|
+
const cases = all.flatMap(readCases);
|
|
86
|
+
const graph = ctx.graph();
|
|
87
|
+
const coverage = coverageOf(cases, Object.keys(graph.screens).sort(), readSeen(path.join(ctx.root, '.lynkeus', 'seen')));
|
|
88
|
+
const changed = changedSources(ctx.root, String(ctx.flags.base), ctx.err);
|
|
89
|
+
const impact = impactOf(screensAffectedBy(graph, changed), coverage);
|
|
90
|
+
files = [...new Set(impact.cases.map((c) => c.file))];
|
|
91
|
+
picked = `${files.length} of ${all.length} case files: the ones the ${changed.length} changed source file(s) against ${ctx.flags.base} reach.`;
|
|
92
|
+
if (files.length === 0) {
|
|
93
|
+
if (process.env.GITHUB_STEP_SUMMARY) fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, `### lynkeus: no case to run\n\n${picked}\n`);
|
|
94
|
+
return { text: `no case to run: ${picked}`, json: { files } };
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
const ports = Array.from({ length: jobs }, (_, i) => 8123 + i);
|
|
98
|
+
const hostCommand = ci.hosts?.command ?? `LYNKEUS_PORT={port} "${process.execPath}" "${self}" headless start`;
|
|
99
|
+
const runArgs = [
|
|
100
|
+
'case',
|
|
101
|
+
'run',
|
|
102
|
+
...files,
|
|
103
|
+
'--jobs',
|
|
104
|
+
String(jobs),
|
|
105
|
+
'--ports',
|
|
106
|
+
ports.join(','),
|
|
107
|
+
'--out',
|
|
108
|
+
out,
|
|
109
|
+
'--screenshots',
|
|
110
|
+
...(ci.push ? ['--push'] : []),
|
|
111
|
+
...(ci.run ?? []),
|
|
112
|
+
];
|
|
113
|
+
if (ctx.flags['dry-run']) {
|
|
114
|
+
const plan = [
|
|
115
|
+
picked,
|
|
116
|
+
ci.backend ? `backend: \`${ci.backend.up}\`${ci.backend.ready ? `, ready at ${ci.backend.ready}` : ''}` : 'backend: none declared (ci.backend)',
|
|
117
|
+
`hosts: ${jobs} × \`${hostCommand}\` on ${ports.join(', ')}`,
|
|
118
|
+
`run: lynkeus ${runArgs.slice(0, 2).join(' ')} … ${runArgs.slice(2 + files.length).join(' ')}`,
|
|
119
|
+
];
|
|
120
|
+
return { text: plan.join('\n'), json: { files, ports, hostCommand, runArgs } };
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
const hosts: ChildProcess[] = [];
|
|
124
|
+
const down = () => {
|
|
125
|
+
if (ctx.flags['keep-up']) return say('left up (--keep-up)');
|
|
126
|
+
for (const h of hosts)
|
|
127
|
+
if (h.pid)
|
|
128
|
+
try {
|
|
129
|
+
process.kill(-h.pid, 'SIGTERM');
|
|
130
|
+
} catch {}
|
|
131
|
+
if (ci.backend?.down) {
|
|
132
|
+
say('backend down');
|
|
133
|
+
spawnSync('/bin/sh', ['-c', ci.backend.down], { cwd: ctx.root, stdio: 'inherit' });
|
|
134
|
+
}
|
|
135
|
+
};
|
|
136
|
+
try {
|
|
137
|
+
if (ci.backend) {
|
|
138
|
+
say(`backend up: ${ci.backend.up}`);
|
|
139
|
+
const up = spawnSync('/bin/sh', ['-c', ci.backend.up], { cwd: ctx.root, stdio: 'inherit' });
|
|
140
|
+
if (up.status !== 0) return { text: `the backend did not come up (\`${ci.backend.up}\` exited ${up.status})`, code: 1 };
|
|
141
|
+
if (ci.backend.ready) {
|
|
142
|
+
const url = ci.backend.ready;
|
|
143
|
+
say(`waiting for ${url}`);
|
|
144
|
+
const ready = await waitFor(
|
|
145
|
+
() =>
|
|
146
|
+
fetch(url).then(
|
|
147
|
+
(r) => r.ok,
|
|
148
|
+
() => false,
|
|
149
|
+
),
|
|
150
|
+
ci.backend.timeoutMs ?? 300_000,
|
|
151
|
+
);
|
|
152
|
+
if (!ready) return { text: `${url} never answered`, code: 1 };
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
say(`${jobs} host${jobs === 1 ? '' : 's'}: ${hostCommand}`);
|
|
156
|
+
for (const port of ports)
|
|
157
|
+
hosts.push(
|
|
158
|
+
spawn('/bin/sh', ['-c', hostCommand.replaceAll('{port}', String(port))], { cwd: ctx.root, stdio: ['pipe', 'ignore', 'ignore'], detached: true }),
|
|
159
|
+
);
|
|
160
|
+
// A host compiles the app before it answers: the slow part.
|
|
161
|
+
for (const port of ports) {
|
|
162
|
+
const answered = () =>
|
|
163
|
+
spawnSync(process.execPath, [self, 'screen'], {
|
|
164
|
+
cwd: ctx.root,
|
|
165
|
+
env: { ...process.env, LYNKEUS_PORT: String(port) },
|
|
166
|
+
stdio: 'ignore',
|
|
167
|
+
timeout: 20_000,
|
|
168
|
+
}).status === 0;
|
|
169
|
+
if (!(await waitFor(answered, ci.hosts?.timeoutMs ?? 600_000, 5000))) return { text: `the host on ${port} never attached`, code: 1 };
|
|
170
|
+
say(`host on ${port} ready`);
|
|
171
|
+
}
|
|
172
|
+
say(`running ${files.length} case file(s) on ${jobs} host(s)`);
|
|
173
|
+
const run = spawnSync(process.execPath, [self, ...runArgs], {
|
|
174
|
+
cwd: ctx.root,
|
|
175
|
+
encoding: 'utf8',
|
|
176
|
+
stdio: ['ignore', 'pipe', 'inherit'],
|
|
177
|
+
maxBuffer: 64 * 1024 * 1024,
|
|
178
|
+
});
|
|
179
|
+
process.stdout.write(run.stdout ?? '');
|
|
180
|
+
const report = /report: (https?:\/\/\S+)/.exec(run.stdout ?? '')?.[1];
|
|
181
|
+
const summary = summaryOf(resultsOf(out), report, picked);
|
|
182
|
+
if (process.env.GITHUB_STEP_SUMMARY) fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, summary);
|
|
183
|
+
return { text: summary, code: run.status ?? 1 };
|
|
184
|
+
} finally {
|
|
185
|
+
down();
|
|
186
|
+
}
|
|
187
|
+
},
|
|
188
|
+
}),
|
|
189
|
+
];
|