lynkeus 0.3.0 → 0.5.0

package/src/commands/report.ts

@@ -2,6 +2,8 @@ import fs from 'node:fs';
 import path from 'node:path';
 
 import { define } from '../registry.js';
+import { ageOf, megabytes, pruneRuns } from '../cases/runDirs.js';
+import { keptOrDeleted } from './cases.js';
 
 type Fetch = typeof fetch;
 
@@ -22,11 +24,54 @@ const answer = async (response: Response, doing: string) => {
   throw new Error(`${doing}: ${body.error ?? `the service answered ${response.status}`}`);
 };
 
+const UPLOADS_AT_ONCE = 4;
+const CASES_PART = 256 * 1024;
+const UPLOAD_ATTEMPTS = 3;
+
+// A ustar header holds a name of up to 100 bytes and a folder of up to 155 before it.
+const fitsTar = (file: string) => {
+  const cut = file.lastIndexOf('/');
+  return Buffer.byteLength(file.slice(cut + 1)) <= 100 && Buffer.byteLength(cut > 0 ? file.slice(0, cut) : '') <= 155;
+};
+
+/** An uncompressed tar: frames are PNG or AVIF, already compressed, and a tar is read as it arrives. */
+export const tarOf = (entries: [string, Uint8Array][]): Uint8Array => {
+  const blocks: Uint8Array[] = [];
+  for (const [file, data] of entries) {
+    const header = Buffer.alloc(512);
+    const cut = file.lastIndexOf('/');
+    header.write(file.slice(cut + 1), 0, 'utf8');
+    header.write('0000644\0', 100);
+    header.write('0000000\0', 108);
+    header.write('0000000\0', 116);
+    header.write(`${data.length.toString(8).padStart(11, '0')}\0`, 124);
+    header.write(
+      `${Math.floor(Date.now() / 1000)
+        .toString(8)
+        .padStart(11, '0')}\0`,
+      136,
+    );
+    header.write('0', 156);
+    header.write('ustar\0', 257);
+    header.write('00', 263);
+    if (cut > 0) header.write(file.slice(0, cut), 345, 'utf8');
+    header.fill(' ', 148, 156);
+    const sum = header.reduce((n, b) => n + b, 0);
+    header.write(`${sum.toString(8).padStart(6, '0')}\0 `, 148);
+    blocks.push(header, data, Buffer.alloc((512 - (data.length % 512)) % 512));
+  }
+  blocks.push(Buffer.alloc(1024));
+  return Buffer.concat(blocks);
+};
+
 /**
  * Uploads a run directory (`case run --out`) to the reports service: the run, its cases, then every frame and
  * film the cases name. A file the organization's plan does not keep (a film on Free) is left out and listed.
  */
-export const pushRun = async (dir: string, o: { url: string; key: string; project: string; fetch?: Fetch }): Promise<Pushed> => {
+export const pushRun = async (
+  dir: string,
+  o: { url: string; key: string; project: string; fetch?: Fetch; onReady?: (url: string) => void; onProgress?: (line: string) => void },
+): Promise<Pushed> => {
   const call = o.fetch ?? fetch;
   const auth = { authorization: `Bearer ${o.key}` };
   const base = o.url.replace(/\/+$/, '');
@@ -42,31 +87,98 @@ export const pushRun = async (dir: string, o: { url: string; key: string; projec
     'creating the run',
   );
   const id = String(created.id);
-  await answer(await call(`${base}/v1/runs/${id}/cases`, { method: 'PUT', headers: auth, body: cases }), 'sending its cases');
+  // A dropped connection, a 5xx or a 429 is tried again, waiting longer each time.
+  const sendWithRetries = async (target: string, body: Uint8Array, doing: string): Promise<Response> => {
+    for (let attempt = 1; ; attempt++) {
+      const response = await call(target, { method: 'PUT', headers: auth, body }).catch((error: unknown) => {
+        if (attempt >= UPLOAD_ATTEMPTS) throw new Error(`${doing}: ${error instanceof Error ? error.message : String(error)}`);
+        return undefined;
+      });
+      if (response && ((response.status < 500 && response.status !== 429) || attempt >= UPLOAD_ATTEMPTS)) return response;
+      await new Promise((resolve) => setTimeout(resolve, 500 * 2 ** (attempt - 1)));
+    }
+  };
+  // The run file in parts of about 256 KB: with every request's headers and bodies it runs to megabytes, more
+  // than the service reads in one request. A service from before parts answers without `part`, and then
+  // takes the whole file in one.
+  const parts: string[] = [];
+  for (const line of cases.split('\n').filter((l) => l.trim())) {
+    const last = parts.length - 1;
+    if (last >= 0 && Buffer.byteLength(parts[last]!) + Buffer.byteLength(line) < CASES_PART) parts[last] += `\n${line}`;
+    else parts.push(line);
+  }
+  for (const [n, part] of parts.entries()) {
+    const sent = await answer(await sendWithRetries(`${base}/v1/runs/${id}/cases?part=${n}`, Buffer.from(part), 'sending its cases'), 'sending its cases');
+    if (sent.part === undefined) {
+      await answer(await call(`${base}/v1/runs/${id}/cases`, { method: 'PUT', headers: auth, body: cases }), 'sending its cases');
+      break;
+    }
+  }
+  // The report opens as soon as its cases are in: their steps, results and requests show at once, and each
+  // frame as it arrives.
+  const finished = await answer(await call(`${base}/v1/runs/${id}/finish`, { method: 'POST', headers: auth }), 'finishing the run');
+  o.onReady?.(String(finished.url));
 
-  const named = new Set<string>();
+  // Per case: its frames, sent as one tar, and its film, sent on its own (a film can be bigger than a request).
+  const work: { frames: string[]; film?: string }[] = [];
   for (const line of cases.split('\n')) {
     if (!line.trim()) continue;
     const c = JSON.parse(line) as { video?: string; steps?: { screenshot?: string }[] };
-    for (const p of [c.video, ...(c.steps ?? []).map((s) => s.screenshot)]) if (p) named.add(p);
+    const frames = (c.steps ?? []).map((s) => s.screenshot).filter((p): p is string => !!p && fs.existsSync(path.join(dir, p)));
+    const film = c.video && fs.existsSync(path.join(dir, c.video)) ? c.video : undefined;
+    work.push({ frames: [...new Set(frames)], ...(film ? { film } : {}) });
   }
   let files = 0;
   const refused: string[] = [];
-  for (const file of named) {
-    const local = path.join(dir, file);
-    if (!fs.existsSync(local)) continue;
-    const response = await call(`${base}/v1/runs/${id}/files/${file.split('/').map(encodeURIComponent).join('/')}`, {
-      method: 'PUT',
-      headers: auth,
-      body: fs.readFileSync(local),
-    });
+  const sendFile = async (file: string) => {
+    const target = `${base}/v1/runs/${id}/files/${file.split('/').map(encodeURIComponent).join('/')}`;
+    const response = await sendWithRetries(target, fs.readFileSync(path.join(dir, file)), `sending ${file}`);
     if (response.status === 402) refused.push(file);
     else {
       await answer(response, `sending ${file}`);
       files += 1;
     }
-  }
-  const finished = await answer(await call(`${base}/v1/runs/${id}/finish`, { method: 'POST', headers: auth }), 'finishing the run');
+  };
+  // A suite with screenshots is thousands of frames: a request per frame took one over 20 minutes to send. A
+  // service from before bundles answers 404, and the frames then go one by one, as they used to.
+  let bundles = true;
+  const sendFrames = async (frames: string[]) => {
+    const fits = frames.filter(fitsTar);
+    if (bundles && fits.length > 0) {
+      const response = await sendWithRetries(
+        `${base}/v1/runs/${id}/bundle`,
+        tarOf(fits.map((f) => [f, fs.readFileSync(path.join(dir, f))])),
+        `sending ${fits.length} frames`,
+      );
+      if (response.status === 404 || response.status === 405) bundles = false;
+      else {
+        await answer(response, `sending ${fits.length} frames`);
+        files += fits.length;
+        for (const f of frames.filter((f) => !fitsTar(f))) await sendFile(f);
+        return;
+      }
+    }
+    for (const f of frames) await sendFile(f);
+  };
+  const queue = [...work];
+  const total = work.length;
+  let sent = 0;
+  let bytes = 0;
+  const started = Date.now();
+  const sizeOf = (c: { frames: string[]; film?: string }) =>
+    [...c.frames, ...(c.film ? [c.film] : [])].reduce((n, f) => n + fs.statSync(path.join(dir, f)).size, 0);
+  await Promise.all(
+    Array.from({ length: Math.min(UPLOADS_AT_ONCE, queue.length) }, async () => {
+      for (let c = queue.shift(); c !== undefined; c = queue.shift()) {
+        await sendFrames(c.frames);
+        if (c.film) await sendFile(c.film);
+        sent += 1;
+        bytes += sizeOf(c);
+        if (sent === total || sent % Math.max(1, Math.ceil(total / 10)) === 0)
+          o.onProgress?.(`frames: ${sent}/${total} cases, ${(bytes / 1024 / 1024).toFixed(1)} MB in ${((Date.now() - started) / 1000).toFixed(0)} s`);
+      }
+    }),
+  );
   const changes = finished.changes as Pushed['changes'] | undefined;
   return { url: String(finished.url), files, refused, ...(changes ? { changes } : {}) };
 };
@@ -118,12 +230,40 @@ export const plan = async (o: Service & { go?: 'upgrade' | 'manage' }) => {
 };
 
 export const service = (): Service | string => {
-  const url = process.env.LYNKEUS_REPORTS_URL;
+  const url = process.env.LYNKEUS_REPORTS_URL || 'https://lynkeus.io';
   const key = process.env.LYNKEUS_REPORTS_KEY;
-  return url && key ? { url, key } : 'set LYNKEUS_REPORTS_URL and LYNKEUS_REPORTS_KEY (the organization’s API key)';
+  return key ? { url, key } : 'set LYNKEUS_REPORTS_KEY to an API key of your organization (lynkeus.io, API keys); LYNKEUS_REPORTS_URL names another service';
 };
 
 export const reportCommands = [
+  define({
+    name: 'report prune',
+    group: 'runs',
+    summary: 'Delete the runs `case run --out` left in a folder that ended more than a while ago',
+    details:
+      'Looks at the folder given, or each folder right under it, and deletes the ones that hold a run (`run.json` and `cases.jsonl`) older than --older. Nothing else is touched. Pushed reports stay on the service; this only frees the disk.',
+    positionals: [{ name: 'dir', help: 'a run folder, or the folder that holds them' }],
+    flags: {
+      older: { type: 'string', help: 'how old a run must be: 7d, 12h, 30m', value: 'age', default: '7d' },
+      'dry-run': { type: 'boolean', help: 'list what would go, delete nothing' },
+    },
+    needs: 'nothing',
+    session: false,
+    mcp: false,
+    run: async (ctx) => {
+      const root = path.resolve(ctx.args[0] ?? '.');
+      if (!fs.existsSync(root)) return { text: `${ctx.args[0]}: no such folder`, code: 2 };
+      const older = ageOf(String(ctx.flags.older ?? '7d'));
+      if (older === undefined) return { text: `--older ${ctx.flags.older}: give it as 7d, 12h or 30m`, code: 2 };
+      const dry = !!ctx.flags['dry-run'];
+      const { removed, bytes } = pruneRuns(root, older, dry);
+      if (removed.length === 0) return { text: 'no run old enough to delete', json: { removed, bytes } };
+      return {
+        text: [...removed.map((d) => `${dry ? 'would delete' : 'deleted'} ${d}`), `${dry ? 'would free' : 'freed'} ${megabytes(bytes)}`].join('\n'),
+        json: { removed, bytes },
+      };
+    },
+  }),
   define({
     name: 'report push',
     group: 'runs',
@@ -145,8 +285,12 @@ export const reportCommands = [
       if (typeof to === 'string') return { text: to, code: 2 };
       const project = typeof ctx.flags.project === 'string' ? ctx.flags.project : path.basename(ctx.root);
       try {
-        const pushed = await pushRun(dir, { ...to, project });
-        const notes = [...describeChanges(pushed.changes), ...(pushed.refused.length ? [`not kept on this plan: ${pushed.refused.join(', ')}`] : [])];
+        const pushed = await pushRun(dir, { ...to, project, onReady: (url) => ctx.err(`report: ${url} (its frames are on their way)`), onProgress: ctx.err });
+        const notes = [
+          ...describeChanges(pushed.changes),
+          ...(pushed.refused.length ? [`not kept on this plan: ${pushed.refused.join(', ')}`] : []),
+          keptOrDeleted(dir, ctx.config.report?.afterPush === 'delete' ? 'delete' : 'keep', pushed.refused.length === 0),
+        ];
         return { text: [pushed.url, ...notes].join('\n'), json: pushed };
       } catch (error) {
         return { text: (error as Error).message, code: 1 };

package/src/config.ts

@@ -1,3 +1,4 @@
+import crypto from 'node:crypto';
 import fs from 'node:fs';
 import path from 'node:path';
 
@@ -10,6 +11,33 @@ export const graphFile = (root?: string) => path.join(lynkeusDir(root), 'graph.j
 export const edgesFile = (root?: string) => path.join(lynkeusDir(root), 'edges.jsonl');
 export const frontierFile = (root?: string) => path.join(lynkeusDir(root), 'frontier.jsonl');
 export const runsDir = (root?: string) => path.join(lynkeusDir(root), 'runs');
+export const tokenFile = (root?: string) => path.join(lynkeusDir(root), 'token');
+
+/**
+ * The secret the app and its driver share. LYNKEUS_TOKEN wins; otherwise it is the one `lynkeus init` wrote to
+ * `.lynkeus/token`, which the app's bundle reads too, so nobody has to set a variable in two terminals.
+ */
+export const readToken = (root?: string): string | undefined => {
+  if (process.env.LYNKEUS_TOKEN) return process.env.LYNKEUS_TOKEN;
+  try {
+    return fs.readFileSync(tokenFile(root), 'utf8').trim() || undefined;
+  } catch {
+    return undefined;
+  }
+};
+
+/** Makes the project's token if it has none, and keeps it out of git: `.lynkeus` also holds what a team commits. */
+export const ensureToken = (root?: string): { token: string; made: boolean } => {
+  const had = readToken(root);
+  if (had) return { token: had, made: false };
+  const token = crypto.randomBytes(24).toString('hex');
+  fs.mkdirSync(lynkeusDir(root), { recursive: true });
+  fs.writeFileSync(tokenFile(root), `${token}\n`, { mode: 0o600 });
+  const ignore = path.join(lynkeusDir(root), '.gitignore');
+  const ignored = fs.existsSync(ignore) ? fs.readFileSync(ignore, 'utf8') : '';
+  if (!ignored.split('\n').includes('token')) fs.writeFileSync(ignore, `${ignored}${ignored && !ignored.endsWith('\n') ? '\n' : ''}token\n`);
+  return { token, made: true };
+};
 
 export const loadGraph = (root?: string): ScreenGraph | null => {
   const file = graphFile(root);
@@ -23,16 +51,25 @@ export const loadGraph = (root?: string): ScreenGraph | null => {
  */
 export type LynkeusConfig = {
   cases?: CasesConfig;
+  /** What `lynkeus ci` brings up around a suite: the project's backend and the app's hosts. */
+  ci?: CiConfig;
   /** The headless entry (default `qa/headless.tsx`). */
   entry?: string;
   /** The environment variable the app reads its API base from; the headless entry refuses one that is not local. */
   apiEnv?: string;
+  /**
+   * The local backend `headless start` hands the app under `apiEnv` when the environment sets none: an agent
+   * starting the host over MCP cannot export a variable.
+   */
+  apiUrl?: string;
   /** The driver port the app dials. */
   port?: number;
   /** Simulator udid or name. */
   device?: string;
   /** Bundle id or application id. */
   app?: string;
+  /** What a pushed run's local folder becomes: kept (the default), or deleted once the service has every file. */
+  report?: { afterPush?: 'keep' | 'delete' };
   /**
    * The language this app speaks, passed on and never interpreted: `os
    * language` defaults to it, the headless entry mocks this locale, and an MCP
@@ -93,6 +130,17 @@ export const loadConfig = (root?: string): LynkeusConfig => {
   return { ...own, ...extra };
 };
 
+export type CiConfig = {
+  /** Shell commands that bring the project's backend up and down, and a URL that answers once it can serve. */
+  backend?: { up: string; down?: string; ready?: string; timeoutMs?: number };
+  /** How many hosts of the app, and the command that starts one (`{port}` is where it dials). Default: headless. */
+  hosts?: { count?: number; command?: string; timeoutMs?: number };
+  /** Upload the run (needs LYNKEUS_REPORTS_KEY). */
+  push?: boolean;
+  /** Extra `case run` flags, as on the command line. */
+  run?: string[];
+};
+
 export const portFromEnv = (config: LynkeusConfig = {}): number | undefined => (process.env.LYNKEUS_PORT ? Number(process.env.LYNKEUS_PORT) : config.port);
 
 /** A list of regular expressions from the configuration, as one that matches any of them. */

package/src/device/agentVersion.ts

@@ -13,3 +13,20 @@ export const agentMismatch = (hello: Pick<HelloEvent, 'agentVersion'>, cli = CLI
   const agent = hello.agentVersion ? `lynkeus-agent ${hello.agentVersion}` : 'a lynkeus-agent too old to say its version';
   return `⚠️  The app runs ${agent}, this CLI is lynkeus ${cli}: they can misread each other without an error. Install lynkeus-agent@${cli} in the app and rebuild it (or run the lynkeus CLI of the agent's version).`;
 };
+
+/** Who said hello: the app and where it runs, enough to tell two apps dialing the same port apart. */
+export const helloIdentity = (hello: Pick<HelloEvent, 'platform' | 'native' | 'app'>): string => {
+  const app = hello.app ?? {};
+  const name = typeof app.name === 'string' ? app.name : 'an app';
+  const runtime = typeof app.runtime === 'string' ? app.runtime : hello.native ? 'native' : 'hosted';
+  return `${name} (${hello.platform}, ${runtime})`;
+};
+
+/**
+ * The driver keeps the app that dialed in last. Two apps or hosts dialing the same port take turns
+ * without an error, each step going to whichever came last: say so the first time a different one comes in.
+ */
+export const appSwitched = (before: string | undefined, now: string, port: number): string | undefined =>
+  before && before !== now
+    ? `⚠️  Another app attached on port ${port}: ${before} gave way to ${now}. Two apps or hosts are dialing this port and steps go to whichever dialed last; stop one, or give each its own LYNKEUS_PORT.`
+    : undefined;

package/src/device/devices.ts

@@ -63,6 +63,11 @@ export const pickAgentDevice = async (wanted?: string): Promise<DeviceInfo> => {
   const devices = await bootedSimulators();
   const id = wanted ?? process.env.LYNKEUS_DEVICE;
   const device = id ? devices.find((d) => d.id === id || d.name.includes(id)) : devices[0];
+  if (!device && id) {
+    // An adb serial names an Android device; which platform the app is on is only known once it dials in.
+    const android = (await androidDevices()).find((d) => d.id === id);
+    if (android) return android;
+  }
   if (!device) {
     // A headless host dials the driver itself: nothing to boot or attach to.
     if (!id && devices.length === 0) return HEADLESS_HOST;

package/src/device/driver.ts

@@ -6,7 +6,7 @@
 import { AgentServer } from 'lynkeus-client';
 import type { CoreMethods, Screen, TraceEvent } from 'lynkeus-protocol';
 
-import { portFromEnv } from '../config.js';
+import { portFromEnv, readToken } from '../config.js';
 import { withoutHost } from '../knowledge/problems.js';
 import type { DeviceInfo } from './devices.js';
 import { lifecycleFor, type PlatformLifecycle } from './platform.js';
@@ -23,7 +23,7 @@ export class Device {
     this.info = info;
     this.#resolving = options.resolving;
     // The app serves commands only to a driver that presents its token; without one, anything on the loopback could drive a dev build.
-    this.server = options.server ?? new AgentServer({ token: process.env.LYNKEUS_TOKEN, port: portFromEnv() });
+    this.server = options.server ?? new AgentServer({ token: readToken(), port: portFromEnv() });
     this.#lifecycle = lifecycleFor(info.platform);
     this.#appId = options.appId ?? null;
   }

package/src/knowledge/analyticsTraffic.ts

@@ -0,0 +1,142 @@
+// The analytics, crash and attribution services an app talks to over HTTP, known by where it sends: a JS
+// SDK (Segment, PostHog, Amplitude's browser SDK) never touches a native module, so its events show up only
+// as requests. Each request becomes the events its body carries, or one event named after the call.
+import type { RequestRecord } from 'lynkeus-protocol';
+
+export type TrafficEvent = { provider: string; name: string; props?: Record<string, unknown> };
+
+type Plain = Record<string, unknown>;
+// `provider` is the service the request went to: a reader of one shape serves every service that copies it.
+type Reader = (body: unknown, path: string, provider: string) => TrafficEvent[] | undefined;
+
+const isPlain = (v: unknown): v is Plain => !!v && typeof v === 'object' && !Array.isArray(v);
+const plain = (v: unknown): Plain | undefined => (isPlain(v) ? v : undefined);
+const list = (v: unknown): unknown[] => (Array.isArray(v) ? v : v === undefined ? [] : [v]);
+
+const named = (provider: string, items: unknown[], name: (item: Plain) => unknown, props: (item: Plain) => unknown): TrafficEvent[] =>
+  items.filter(isPlain).flatMap((item) => {
+    const n = name(item);
+    if (typeof n !== 'string' || !n) return [];
+    const p = plain(props(item));
+    return [{ provider, name: n, ...(p ? { props: p } : {}) }];
+  });
+
+const segment: Reader = (body, _path, provider) => {
+  const b = plain(body);
+  return named(
+    provider,
+    list(b?.batch ?? b),
+    (i) => i.event ?? (i.type === 'screen' || i.type === 'page' ? `${i.type}: ${i.name}` : i.type),
+    (i) => i.properties ?? i.traits,
+  );
+};
+const amplitude: Reader = (body) =>
+  named(
+    'amplitude',
+    list(plain(body)?.events),
+    (i) => i.event_type,
+    (i) => i.event_properties,
+  );
+const posthog: Reader = (body) => {
+  const b = plain(body);
+  return named(
+    'posthog',
+    list(b?.batch ?? (Array.isArray(body) ? body : b)),
+    (i) => i.event,
+    (i) => i.properties,
+  );
+};
+const mixpanel: Reader = (body) =>
+  named(
+    'mixpanel',
+    list(body),
+    (i) => i.event ?? (i.$set ? '$set' : undefined),
+    (i) => i.properties ?? i.$set,
+  );
+
+/** Host pattern → the service, and how to read its body when there is a known shape. */
+const SERVICES: { host: RegExp; provider: string; read?: Reader }[] = [
+  { host: /(^|\.)segment\.(io|com)$/, provider: 'segment', read: segment },
+  { host: /(^|\.)amplitude\.com$/, provider: 'amplitude', read: amplitude },
+  { host: /(^|\.)posthog\.com$/, provider: 'posthog', read: posthog },
+  { host: /(^|\.)mixpanel\.com$/, provider: 'mixpanel', read: mixpanel },
+  { host: /(^|\.)sentry\.io$|^sentry\./, provider: 'sentry' },
+  { host: /(^|\.)(datadoghq\.(com|eu)|ddog-gov\.com)$/, provider: 'datadog' },
+  { host: /(^|\.)app-measurement\.com$|^firebaselogging/, provider: 'firebase' },
+  { host: /(^|\.)appsflyer(sdk)?\.com$/, provider: 'appsflyer' },
+  { host: /(^|\.)branch\.io$/, provider: 'branch' },
+  { host: /(^|\.)braze\.(com|eu)$/, provider: 'braze' },
+  { host: /(^|\.)onesignal\.com$/, provider: 'onesignal' },
+  { host: /(^|\.)intercom\.(io|com)$/, provider: 'intercom' },
+  { host: /(^|\.)bugsnag\.com$/, provider: 'bugsnag' },
+  { host: /(^|\.)adjust\.(com|net|world)$/, provider: 'adjust' },
+  { host: /(^|\.)clevertap-prod\.com$|(^|\.)wzrkt\.com$/, provider: 'clevertap' },
+  { host: /(^|\.)heapanalytics\.com$/, provider: 'heap' },
+  { host: /(^|\.)statsig(api)?\.(com|net)$/, provider: 'statsig' },
+  { host: /(^|\.)launchdarkly\.com$/, provider: 'launchdarkly' },
+  // Segment-compatible: the same batch shape.
+  { host: /(^|\.)rudderstack\.com$|(^|\.)rudderlabs\.com$/, provider: 'rudderstack', read: segment },
+  { host: /^(track|cdp|track-eu|cdp-eu)\.customer\.io$/, provider: 'customerio', read: segment },
+  { host: /(^|\.)google-analytics\.com$|(^|\.)analytics\.google\.com$/, provider: 'googleanalytics' },
+  { host: /(^|\.)googletagmanager\.com$/, provider: 'googletagmanager' },
+  { host: /^graph\.facebook\.com$/, provider: 'meta' },
+  { host: /(^|\.)tiktok(v)?\.com$|(^|\.)tiktokapis\.com$/, provider: 'tiktok' },
+  { host: /(^|\.)iterable\.com$/, provider: 'iterable' },
+  { host: /(^|\.)moengage\.com$/, provider: 'moengage' },
+  { host: /(^|\.)kochava\.com$/, provider: 'kochava' },
+  { host: /(^|\.)singular\.net$/, provider: 'singular' },
+  { host: /(^|\.)emb-api\.com$|(^|\.)embrace\.io$/, provider: 'embrace' },
+  { host: /(^|\.)instabug\.com$|(^|\.)luciq\.ai$/, provider: 'instabug' },
+  { host: /(^|\.)smartlook\.(cloud|com)$/, provider: 'smartlook' },
+  { host: /(^|\.)urbanairship\.com$|(^|\.)airship\.(com|eu)$|(^|\.)asnapieu\.com$/, provider: 'airship' },
+  { host: /(^|\.)snplow\.net$|(^|\.)snowplowanalytics\.com$/, provider: 'snowplow' },
+  { host: /(^|\.)fullstory\.com$/, provider: 'fullstory' },
+  { host: /(^|\.)mparticle\.com$/, provider: 'mparticle' },
+  { host: /(^|\.)pendo\.io$/, provider: 'pendo' },
+  { host: /(^|\.)lr-ingest\.(io|com)$|(^|\.)logrocket\.(io|com)$/, provider: 'logrocket' },
+  { host: /(^|\.)newrelic\.com$|(^|\.)nr-data\.net$/, provider: 'newrelic' },
+  { host: /(^|\.)hotjar\.(com|io)$/, provider: 'hotjar' },
+  { host: /(^|\.)count\.ly$/, provider: 'countly' },
+];
+
+const parse = (text: string | undefined): unknown => {
+  if (!text) return undefined;
+  try {
+    return JSON.parse(text);
+  } catch {
+    // Mixpanel's classic endpoint sends `data=<base64 JSON>` as a form.
+    const data = /(?:^|&)data=([^&]+)/.exec(text)?.[1];
+    if (!data) return undefined;
+    try {
+      const raw = decodeURIComponent(data);
+      return JSON.parse(raw.startsWith('[') || raw.startsWith('{') ? raw : Buffer.from(raw, 'base64').toString('utf8'));
+    } catch {
+      return undefined;
+    }
+  }
+};
+
+/** The service a request went to, when it is one of the known ones. */
+export const serviceOf = (url: string): string | undefined => {
+  try {
+    const host = new URL(url).hostname.toLowerCase();
+    return SERVICES.find((s) => s.host.test(host))?.provider;
+  } catch {
+    return undefined;
+  }
+};
+
+/** What a step's requests told analytics services: each body's events, or one per call to a service it cannot read. */
+export const trafficEvents = (requests: RequestRecord[]): TrafficEvent[] =>
+  requests.flatMap((r) => {
+    let url: URL;
+    try {
+      url = new URL(r.url);
+    } catch {
+      return [];
+    }
+    const service = SERVICES.find((s) => s.host.test(url.hostname.toLowerCase()));
+    if (!service) return [];
+    const events = service.read?.(parse(r.request?.body), url.pathname, service.provider);
+    return events?.length ? events : [{ provider: service.provider, name: `${r.method} ${url.pathname}` }];
+  });

package/src/knowledge/errors.ts

@@ -47,6 +47,12 @@ const RULES: Rule[] = [
     cause: (m) => `nothing is listening on ${m[1]}:${m[2]}`,
     fix: () => `start the backend (or the lynkeus driver, if this is the agent's socket) and check the port the entry dials.`,
   },
+  {
+    // Babel loads the project's config with require(); a plugin shipped as ESM only loads that way from Node 22.12.
+    re: /ERR_REQUIRE_ESM(?:[^\n]*?require\(\) of ES Module (\S+))?|require\(\) of ES Module (\S+)/,
+    cause: (m) => `Node ${process.versions.node} cannot require() an ES module${m[1] || m[2] ? ` (${m[1] || m[2]})` : ''}, which the babel config loads`,
+    fix: () => `run with Node 22.12 or newer (lynkeus-headless needs >=22), where require() loads ES modules.`,
+  },
   {
     re: /Test environment jest-environment-\w+ cannot be found|Unexpected token 'export'|SyntaxError: Cannot use import statement/,
     cause: () => `jest is not transforming a dependency (ESM in node_modules)`,

package/src/knowledge/events.ts

@@ -4,7 +4,8 @@ import path from 'node:path';
 import { readJsonl } from '../files.js';
 
 export type AppEvent = { name: string; props?: Record<string, unknown> };
-export type RunEvent = AppEvent & { step: number; route?: string };
+/** `provider`: the service the app's own list feeds, when the project says (`cases.appEvents`). */
+export type RunEvent = AppEvent & { step: number; route?: string; provider?: string };
 
 type Drainable = { command<T = unknown>(name: string, params?: unknown): Promise<T> };
 

package/src/knowledge/why.ts

@@ -14,6 +14,8 @@ export type Evidence = {
   trace?: TraceEvent[];
   /** What the failed command was looking for. */
   target?: { testId?: string; text?: string; route?: string };
+  /** Whether the app made any request before `requests` begins: `requests` may be only those of the failed command. */
+  fetchedBefore?: boolean;
 };
 
 export type Finding = { rule: string; message: string };
@@ -159,7 +161,7 @@ export const explain = (e: Evidence): Finding[] => {
       message: `${e.inFlight} request(s) still in flight when the command gave up: the screen had not finished loading. A longer timeout, or wait for the request it needs.`,
     });
 
-  if (done.length === 0 && e.inFlight === 0 && e.target?.route)
+  if (done.length === 0 && e.inFlight === 0 && e.target?.route && !e.fetchedBefore)
     out.push({
       rule: 'no-requests',
       message: 'no request left the app while this ran: nothing was fetched. Is the user logged in, and is the app talking to the backend it should?',

package/src/mcp/server.ts

@@ -60,9 +60,25 @@ const argvFor = (c: Command, input: Record<string, unknown>): string[] => {
   return av;
 };
 
+// What a model is told once, when it connects: the order things are done in, so "set lynkeus up" and "write me
+// a test for this" need nothing more from the person asking.
+const INSTRUCTIONS = `lynkeus drives a React Native app (bare or Expo) by what is on its screen, and runs cases written as readable steps.
+
+In a project you do not know, call status first: it lists what is set up and, for what is not, the one thing that fixes it.
+
+To set lynkeus up: call init (it installs lynkeus-agent, mounts it from the app's entry and makes the project's token; nobody sets a variable). The agent has native code, so the app must then be built again (npx react-native run-ios or run-android; npx expo run:ios or run:android, never Expo Go) with Metro restarted with --reset-cache. Do that with your shell if you have one, or ask the person to. Then status until an app is attached.
+
+To write a test: call case_init once if the project has no cases, and read the README.md it writes for the format. Go to the screen with press, type and wait, calling screen to see what is there (press with settle: true answers with the screen once the app has settled; a screen read right after a press can still be the one it left); prefer testIDs (#id) over text. Write the case as a markdown file in the cases folder, then call case_run with that file. A failed step comes back with the screen it failed on: fix the case or report the app's bug, do not weaken the check.
+
+To test without a simulator: call headless_init (it installs what the Node host needs and writes its entry; an Expo Router app works too), then headless_doctor for the native modules still to mock, then case_run with host. The backend the app talks to must be local: set apiUrl in lynkeus.config.json.
+
+To share a run: case_run with out, screenshots and push uploads it and prints the report's address. That needs LYNKEUS_REPORTS_KEY in this server's environment.
+
+For a run with everything (frames, film, requests, analytics, backend): case_run with out, screenshots, video (film needs fastsim), traces and push. When the run ends it warns about evidence it could not keep: pass that on.`;
+
 export const serve = async (session: Session, options: { full?: boolean } = {}) => {
   const { version } = createRequire(import.meta.url)('../../package.json') as { version: string };
-  const server = new Server({ name: 'lynkeus', version }, { capabilities: { tools: {} } });
+  const server = new Server({ name: 'lynkeus', version }, { capabilities: { tools: {} }, instructions: INSTRUCTIONS });
   const served = commands().filter((c) => overMcp(c, !!options.full));
   const byName = new Map(served.map((c) => [identifier(c.name), c]));
   // So a model reading the app's screens knows their language before it starts guessing.
@@ -101,9 +117,34 @@ export const serve = async (session: Session, options: { full?: boolean } = {})
     if (request.params.name === 'app_locale' && locale) return text(locale);
     const c = byName.get(request.params.name);
     if (!c) return { ...text(`✗ no tool ${request.params.name}`), isError: true };
-    const r = await execute(argvFor(c, (request.params.arguments ?? {}) as Record<string, unknown>), session.ctx);
+    const input = (request.params.arguments ?? {}) as Record<string, unknown>;
+    // An argument the tool does not take is dropped by argvFor; a run would then go on without it (every case
+    // instead of the one named), so it is refused, with what the tool does take.
+    const known = Object.keys(schemaFor(c).properties);
+    const unknown = Object.keys(input).filter((key) => !known.includes(key));
+    if (unknown.length)
+      return { ...text(`✗ ${request.params.name} takes no ${unknown.join(', ')}; it takes ${known.join(', ') || 'no arguments'}`), isError: true };
+    const r = await execute(argvFor(c, input), session.ctx);
     if (r.error) return { ...text(`✗ ${r.error}${r.why ? `\n\nwhy:\n${r.why}` : ''}`), isError: true };
-    return text(r.text || (r.json !== undefined ? JSON.stringify(r.json, null, 2) : 'ok'));
+    const body = text(r.text || (r.json !== undefined ? JSON.stringify(r.json, null, 2) : 'ok'));
+    // A command that ended with a non-zero code (a failed case, a server that does not keep the contract) is
+    // reported as such, with everything it said, so a model does not read it as success.
+    return r.code === 0 ? body : { ...body, isError: true };
   });
-  await server.connect(new StdioServerTransport());
+  // The client going away ends the server: the session holds the driver's port open, and a server left behind
+  // keeps the next one (or a case run) from binding it.
+  const transport = new StdioServerTransport();
+  let stopping = false;
+  const stop = () => {
+    if (stopping) return;
+    stopping = true;
+    void session
+      .stop()
+      .catch(() => undefined)
+      .finally(() => process.exit(0));
+  };
+  process.stdin.once('end', stop);
+  process.stdin.once('close', stop);
+  transport.onclose = stop;
+  await server.connect(transport);
 };