lynkeus 0.2.0 → 0.4.0

package/src/cases/templates.ts

@@ -36,20 +36,20 @@ report carries the screen, the requests and the reason.
 
 | Step | Value | |
 |---|---|---|
-| \`app.press\` | target, or \`{ target, within?, nth?, optional?, hold? }\` | Waits for the control to be there and enabled. \`optional\` presses only if it is on screen. |
-| \`app.type\` | \`{ text, target?, submit?, clear? }\` | \`target\` is tapped first; \`submit\` ends with the return key. |
+| \`app.press\` | target, or \`{ target, within?, nth?, optional?, hold?, timeoutMs? }\` | Waits for the control to be there and enabled. \`optional\` presses only if it is on screen. |
+| \`app.type\` | \`{ text, target?, submit?, clear?, timeoutMs? }\` | \`target\` is tapped first, once it is there (\`timeoutMs\` is how long that may take); \`submit\` ends with the return key. |
 | \`app.swipe\` | \`{ direction, target?, distance? }\` or \`{ from, to }\` | |
 | \`app.wait\` | route, target, ms, \`{ target, scroll? }\` or \`{ request, status? }\` | \`scroll\` swipes until the target is in the window. A \`request\` is \`METHOD /path\`. |
 | \`app.see\` | target, or \`{ testId?, text?, route?, request?, event?, layer?, absent?, timeoutMs? }\` | An assertion on what the app shows or did. Everything named must hold. |
 | \`app.nav\` | route, or \`{ route, params }\` | A teleport, for getting to where the case starts. |
 | \`app.back\` | | |
-| \`app.requests\` | \`mark\` | From here on, \`request\` only counts what the app asks for. |
+| \`app.requests\` | \`mark\` | A \`request\` counts only what the app asked for after the step before it began; this marks further back, until a step has looked for a request. |
 | \`app.call\` | \`{ command, params?, expect?, as? }\` | A command the app registered with \`qa.register\`. |
 | \`app.run\` | \`{ flow, vars? }\` | A lynkeus flow, from the flows folder. |
 | \`app.events\` / \`app.event\` | \`clear\` / \`{ name, props?, absent? }\` | What the app told its analytics. |
 | \`app.<command>\` | its arguments by name | Any other lynkeus command: \`app.mock\`, \`app.network\`, \`app.appstate\`, \`app.deeplink\`… |
 
-A target is \`#testId\`, \`"text"\`, an index or \`x,y\`.
+A target is \`#testId\`, a text, an index or \`x,y\`. Where a step takes \`target\` it also takes \`testId\`. A text inside quotes of its own (\`'"Next"'\`) is always a text, where \`app.wait: Next\` would wait for a route called \`Next\`.
 
 A case that runs over several inputs declares \`matrix: { locale: [en, fr] }\`
 in its frontmatter and runs once per combination; \`{{matrix.locale}}\` is that
@@ -74,6 +74,11 @@ Any other step name is a **fixture**: lynkeus sends it to the command in
 ← {"ok": true, "user_id": "u_1", "name": "Ana"}
 \`\`\`
 
+Before the first case lynkeus sends \`{"command": "lynkeus.describe"}\`. A
+server that answers \`{"commands": ["create_user", …]}\` gets its cases checked
+first: one that names a fixture missing from the list is refused before
+anything runs. A server that answers \`"ok": false\` is not checked.
+
 \`as: u\` keeps the answer; \`{{u.user_id}}\` reads it later. \`assert\` calls
 the \`inspect\` fixture and checks one path of its answer with \`equals\`,
 \`includes\`, \`gte\`, \`lte\`, \`near\`, \`matches\`, \`present\` or \`absent\`.
@@ -139,6 +144,11 @@ const fixtures = {
 for await (const line of readline.createInterface({ input: process.stdin })) {
   if (!line.trim()) continue;
   const { command, params } = JSON.parse(line);
+  // lynkeus asks once what this server answers, and refuses a case that names anything else.
+  if (command === 'lynkeus.describe') {
+    console.log(JSON.stringify({ commands: Object.keys(fixtures) }));
+    continue;
+  }
   try {
     const fixture = fixtures[command];
     if (!fixture) throw new Error(\`no fixture \${command}\`);

package/src/cases/traces.ts

@@ -0,0 +1,298 @@
+import { randomBytes } from 'node:crypto';
+import http from 'node:http';
+
+/** A span as a backend exported it over OTLP, reduced to what a report shows. */
+export type Span = {
+  traceId: string;
+  spanId: string;
+  parentSpanId?: string;
+  name: string;
+  kind: number;
+  startNs: bigint;
+  endNs: bigint;
+  attributes: Record<string, string | number | boolean>;
+  error?: string;
+  service?: string;
+};
+
+/** What a step did on the backend, as the run file keeps it: one line per span, children indented by depth. */
+export type BackendSpan = {
+  name: string;
+  /** server, client, internal, producer or consumer */
+  kind: string;
+  /** ms after the step began */
+  at: number;
+  ms: number;
+  depth: number;
+  service?: string;
+  error?: string;
+  attributes?: Record<string, string | number | boolean>;
+};
+
+const KINDS = ['unspecified', 'internal', 'server', 'client', 'producer', 'consumer'];
+
+export const newTraceId = () => randomBytes(16).toString('hex');
+export const newSpanId = () => randomBytes(8).toString('hex');
+export const traceparent = (traceId: string, spanId: string) => `00-${traceId}-${spanId}-01`;
+
+// OTLP over HTTP sends protobuf by default (Ruby's exporter sends nothing else). This reads the few
+// messages of ExportTraceServiceRequest a report needs, straight from the wire format.
+type Field = { no: number; wire: number; value: bigint | Uint8Array };
+
+const fields = (buf: Uint8Array): Field[] => {
+  const out: Field[] = [];
+  let i = 0;
+  const varint = () => {
+    let result = 0n;
+    let shift = 0n;
+    for (;;) {
+      const b = buf[i++]!;
+      result |= BigInt(b & 0x7f) << shift;
+      if (!(b & 0x80)) return result;
+      shift += 7n;
+    }
+  };
+  while (i < buf.length) {
+    const tag = Number(varint());
+    const no = tag >> 3;
+    const wire = tag & 7;
+    if (wire === 0) out.push({ no, wire, value: varint() });
+    else if (wire === 1) {
+      out.push({ no, wire, value: buf.slice(i, i + 8) });
+      i += 8;
+    } else if (wire === 2) {
+      const length = Number(varint());
+      out.push({ no, wire, value: buf.subarray(i, i + length) });
+      i += length;
+    } else if (wire === 5) {
+      out.push({ no, wire, value: buf.slice(i, i + 4) });
+      i += 4;
+    } else throw new Error(`unknown protobuf wire type ${wire}`);
+  }
+  return out;
+};
+
+const text = (v: bigint | Uint8Array) => (v instanceof Uint8Array ? Buffer.from(v).toString('utf8') : '');
+const hex = (v: bigint | Uint8Array) => (v instanceof Uint8Array ? Buffer.from(v).toString('hex') : '');
+const fixed64 = (v: bigint | Uint8Array) => (v instanceof Uint8Array ? Buffer.from(v).readBigUInt64LE(0) : 0n);
+
+const anyValue = (buf: Uint8Array): string | number | boolean => {
+  for (const f of fields(buf)) {
+    if (f.no === 1) return text(f.value);
+    if (f.no === 2) return f.value === 1n;
+    if (f.no === 3) return Number(BigInt.asIntN(64, f.value as bigint));
+    if (f.no === 4 && f.value instanceof Uint8Array) return Buffer.from(f.value).readDoubleLE(0);
+  }
+  return '';
+};
+
+const keyValues = (list: Field[], no: number) => {
+  const out: Record<string, string | number | boolean> = {};
+  for (const f of list) {
+    if (f.no !== no || !(f.value instanceof Uint8Array)) continue;
+    let key = '';
+    let value: string | number | boolean = '';
+    for (const kv of fields(f.value)) {
+      if (kv.no === 1) key = text(kv.value);
+      if (kv.no === 2 && kv.value instanceof Uint8Array) value = anyValue(kv.value);
+    }
+    if (key) out[key] = value;
+  }
+  return out;
+};
+
+export const decodeProtobuf = (buf: Uint8Array): Span[] => {
+  const spans: Span[] = [];
+  for (const rs of fields(buf)) {
+    if (rs.no !== 1 || !(rs.value instanceof Uint8Array)) continue;
+    const parts = fields(rs.value);
+    const resource = parts.find((p) => p.no === 1);
+    const service = resource?.value instanceof Uint8Array ? keyValues(fields(resource.value), 1)['service.name'] : undefined;
+    for (const ss of parts) {
+      if (ss.no !== 2 || !(ss.value instanceof Uint8Array)) continue;
+      for (const sp of fields(ss.value)) {
+        if (sp.no !== 2 || !(sp.value instanceof Uint8Array)) continue;
+        const s = fields(sp.value);
+        const get = (no: number) => s.find((f) => f.no === no)?.value;
+        const status = get(15);
+        let error: string | undefined;
+        if (status instanceof Uint8Array) {
+          const st = fields(status);
+          const code = st.find((f) => f.no === 3)?.value;
+          if (code === 2n) error = text(st.find((f) => f.no === 2)?.value ?? new Uint8Array()) || 'error';
+        }
+        const parent = get(4);
+        spans.push({
+          traceId: hex(get(1) ?? new Uint8Array()),
+          spanId: hex(get(2) ?? new Uint8Array()),
+          parentSpanId: parent instanceof Uint8Array && parent.length ? hex(parent) : undefined,
+          name: text(get(5) ?? new Uint8Array()),
+          kind: Number(get(6) ?? 0n),
+          startNs: fixed64(get(7) ?? new Uint8Array(8)),
+          endNs: fixed64(get(8) ?? new Uint8Array(8)),
+          attributes: keyValues(s, 9),
+          ...(error ? { error } : {}),
+          ...(typeof service === 'string' ? { service } : {}),
+        });
+      }
+    }
+  }
+  return spans;
+};
+
+type JsonValue = { stringValue?: string; boolValue?: boolean; intValue?: string | number; doubleValue?: number };
+type JsonAttrs = { key: string; value: JsonValue }[] | undefined;
+
+const jsonAttrs = (list: JsonAttrs) =>
+  Object.fromEntries(
+    (list ?? []).map(({ key, value }) => [
+      key,
+      value.stringValue ?? value.boolValue ?? (value.intValue !== undefined ? Number(value.intValue) : (value.doubleValue ?? '')),
+    ]),
+  );
+
+export const decodeJson = (body: unknown): Span[] => {
+  const spans: Span[] = [];
+  const request = body as {
+    resourceSpans?: {
+      resource?: { attributes?: JsonAttrs };
+      scopeSpans?: {
+        spans?: {
+          traceId: string;
+          spanId: string;
+          parentSpanId?: string;
+          name: string;
+          kind?: number;
+          startTimeUnixNano: string;
+          endTimeUnixNano: string;
+          attributes?: JsonAttrs;
+          status?: { code?: number; message?: string };
+        }[];
+      }[];
+    }[];
+  };
+  for (const rs of request.resourceSpans ?? []) {
+    const service = jsonAttrs(rs.resource?.attributes)['service.name'];
+    for (const ss of rs.scopeSpans ?? [])
+      for (const s of ss.spans ?? [])
+        spans.push({
+          traceId: s.traceId.toLowerCase(),
+          spanId: s.spanId.toLowerCase(),
+          parentSpanId: s.parentSpanId ? s.parentSpanId.toLowerCase() : undefined,
+          name: s.name,
+          kind: s.kind ?? 0,
+          startNs: BigInt(s.startTimeUnixNano),
+          endNs: BigInt(s.endTimeUnixNano),
+          attributes: jsonAttrs(s.attributes),
+          ...(s.status?.code === 2 ? { error: s.status.message || 'error' } : {}),
+          ...(typeof service === 'string' ? { service } : {}),
+        });
+  }
+  return spans;
+};
+
+// What a report keeps of a span's attributes: what it did, not every label an instrumentation adds.
+const KEEP =
+  /^(http\.(request\.)?method|http\.(response\.)?status_code|http\.route|http\.target|url\.(path|full)|server\.address|net\.peer\.name|db\.(system|statement|operation|sql\.table)|messaging\.|code\.(namespace|function)|rpc\.|exception\.message|enduser\.id)/;
+
+/**
+ * Listens for OTLP traces while a run goes on: a backend with OTEL_EXPORTER_OTLP_ENDPOINT pointed here sends
+ * what each request did, and the steps that sent those requests (their ids travel in `traceparent`) get it.
+ */
+export class TraceSink {
+  private readonly spans = new Map<string, Span[]>();
+  private server?: http.Server;
+  private readonly arrived = new Map<string, number>();
+
+  constructor(readonly port: number) {}
+
+  async start(): Promise<void> {
+    this.server = http.createServer((req, res) => {
+      if (req.method !== 'POST' || !req.url?.startsWith('/v1/traces')) {
+        res.writeHead(404).end();
+        return;
+      }
+      const chunks: Buffer[] = [];
+      req.on('data', (c: Buffer) => chunks.push(c));
+      req.on('end', () => {
+        try {
+          const body = Buffer.concat(chunks);
+          const spans = (req.headers['content-type'] ?? '').includes('json') ? decodeJson(JSON.parse(body.toString('utf8'))) : decodeProtobuf(body);
+          this.add(spans);
+          res.writeHead(200, { 'content-type': req.headers['content-type'] ?? 'application/x-protobuf' }).end();
+        } catch (error) {
+          res.writeHead(400).end(String(error));
+        }
+      });
+    });
+    await new Promise<void>((resolve, reject) => {
+      this.server!.once('error', reject);
+      this.server!.listen(this.port, '127.0.0.1', () => resolve());
+    });
+  }
+
+  add(spans: Span[]) {
+    for (const span of spans) {
+      this.arrived.set(span.traceId, Date.now());
+      const list = this.spans.get(span.traceId) ?? [];
+      list.push(span);
+      this.spans.set(span.traceId, list);
+    }
+  }
+
+  /**
+   * Backends export in batches, a moment after the requests they served: wait until the trace has had
+   * nothing new for `quietMs`, or `maxMs` has passed. Other traces (a scheduler polling) do not count.
+   */
+  async settle(traceId: string, quietMs = 300, maxMs = 3000): Promise<void> {
+    const started = Date.now();
+    while (Date.now() - started < maxMs) {
+      if (Date.now() - Math.max(this.arrived.get(traceId) ?? 0, started) >= quietMs) return;
+      await new Promise((r) => setTimeout(r, 50));
+    }
+  }
+
+  /**
+   * The spans of one trace, each under the step whose id its chain of parents reaches: `steps` maps a step's
+   * span id to its index and the epoch ms it began. Spans of the trace that reach no step are left out.
+   */
+  take(traceId: string, steps: Map<string, { index: number; startedMs: number }>): Map<number, BackendSpan[]> {
+    const spans = this.spans.get(traceId) ?? [];
+    this.spans.delete(traceId);
+    this.arrived.delete(traceId);
+    const byId = new Map(spans.map((s) => [s.spanId, s]));
+    const out = new Map<number, BackendSpan[]>();
+    const place = (span: Span): { step: number; depth: number } | undefined => {
+      let depth = 0;
+      for (let at: Span | undefined = span; at; depth++) {
+        const step = at.parentSpanId ? steps.get(at.parentSpanId) : undefined;
+        if (step) return { step: step.index, depth };
+        at = at.parentSpanId ? byId.get(at.parentSpanId) : undefined;
+      }
+      return undefined;
+    };
+    for (const span of [...spans].sort((a, b) => (a.startNs < b.startNs ? -1 : 1))) {
+      const where = place(span);
+      if (!where) continue;
+      const startedMs = [...steps.values()].find((s) => s.index === where.step)!.startedMs;
+      const attributes = Object.fromEntries(Object.entries(span.attributes).filter(([k]) => KEEP.test(k)));
+      const list = out.get(where.step) ?? [];
+      list.push({
+        name: span.name,
+        kind: KINDS[span.kind] ?? 'internal',
+        at: Math.max(0, Math.round(Number(span.startNs / 1_000_000n) - startedMs)),
+        ms: Math.round(Number(span.endNs - span.startNs) / 1e5) / 10,
+        depth: where.depth,
+        ...(span.service ? { service: span.service } : {}),
+        ...(span.error ? { error: span.error } : {}),
+        ...(Object.keys(attributes).length ? { attributes } : {}),
+      });
+      out.set(where.step, list);
+    }
+    return out;
+  }
+
+  close() {
+    this.server?.close();
+  }
+}

package/src/cli.ts

@@ -3,6 +3,7 @@ import path from 'node:path';
 import readline from 'node:readline';
 
 import { AgentServer } from 'lynkeus-client';
+import type { HelloEvent } from 'lynkeus-protocol';
 
 import { GLOBAL_FLAGS } from './args.js';
 import { caseCommands } from './commands/cases.js';
@@ -10,14 +11,16 @@ import { driveCommands } from './commands/drive.js';
 import { headlessCommands } from './commands/headless.js';
 import { mapCommands } from './commands/map.js';
 import { osCommands } from './commands/os.js';
+import { reportCommands } from './commands/report.js';
 import { runsCommands } from './commands/runs.js';
-import { graphFile, loadConfig, loadGraph, portFromEnv } from './config.js';
-import { HEADLESS_HOST, pickAgentDevice } from './device/devices.js';
+import { graphFile, loadConfig, loadGraph, portFromEnv, readToken } from './config.js';
+import { agentMismatch } from './device/agentVersion.js';
+import { HEADLESS_HOST, pickAgentDevice, pickAndroidDevice } from './device/devices.js';
 import { Device } from './device/driver.js';
 import { type Base, commands, find, helpFor, inSession, referenceMarkdown, register, resolve, usage } from './registry.js';
 import { execute } from './runtime.js';
 
-register([...driveCommands, ...osCommands, ...mapCommands, ...runsCommands, ...caseCommands, ...headlessCommands]);
+register([...driveCommands, ...osCommands, ...mapCommands, ...runsCommands, ...caseCommands, ...reportCommands, ...headlessCommands]);
 
 /** The global flags, read leniently from the whole argv before any command parses its own. */
 const globals = (argv: string[]) => {
@@ -37,7 +40,11 @@ const globals = (argv: string[]) => {
 
 export type Session = { ctx: Base; stop: () => Promise<void> };
 
-const contextFor = (argv: string[], attached = false): Session => {
+// Once per process: a suite reattaches for every case.
+let mismatchTold = false;
+
+// `ownHost`: the port belongs to a host this run started for a case; whatever dials in is the device.
+const contextFor = (argv: string[], attached = false, port?: number, ownHost = false): Session => {
   const g = globals(argv);
   const root = path.resolve((g.root as string | undefined) ?? process.cwd());
   const config = loadConfig(root);
@@ -49,8 +56,17 @@ const contextFor = (argv: string[], attached = false): Session => {
     const appId = (g.app as string | undefined) ?? process.env.LYNKEUS_APP_ID ?? config.app;
     const started = performance.now();
     const attachedIn = (d: Device) => err(`attached in ${Math.round(performance.now() - started)}ms (${d.info.name})`);
-    const server = new AgentServer({ token: process.env.LYNKEUS_TOKEN, port: portFromEnv(config) });
-    const picking = pickAgentDevice((g.device as string | undefined) ?? process.env.LYNKEUS_DEVICE ?? config.device);
+    const server = new AgentServer({ token: readToken(root), port: port ?? portFromEnv(config) });
+    server.on('hello', (hello: HelloEvent) => {
+      const told = mismatchTold ? undefined : agentMismatch(hello);
+      if (!told) return;
+      mismatchTold = true;
+      err(told);
+    });
+    // A host started for the case is the device: looking for the configured simulator would only replace a
+    // late app's error with one about a simulator nobody asked for.
+    const wantedDevice = (g.device as string | undefined) ?? process.env.LYNKEUS_DEVICE ?? config.device;
+    const picking = ownHost ? Promise.resolve(HEADLESS_HOST) : pickAgentDevice(wantedDevice);
     if (g.launch && appId) {
       const d = new Device(await picking, { appId, server });
       await d.launchApp(appId, { fresh: true });
@@ -73,7 +89,9 @@ const contextFor = (argv: string[], attached = false): Session => {
       name: headless ? HEADLESS_HOST.name : String(hello.app?.name ?? hello.platform),
       platform: hello.platform === 'android' ? ('android' as const) : ('ios' as const),
     };
-    const d = new Device(info, { appId: appId ?? null, server, resolving: headless ? undefined : picking });
+    // The app says which platform it runs on; only then is it known whether simctl or adb names its device.
+    const resolving = headless || ownHost ? undefined : info.platform === 'android' ? pickAndroidDevice(wantedDevice) : picking;
+    const d = new Device(info, { appId: appId ?? null, server, resolving });
     attachedIn(d);
     return d;
   };
@@ -92,6 +110,10 @@ const contextFor = (argv: string[], attached = false): Session => {
       device ??= connect();
       return device;
     },
+    forPort: (other) => {
+      const scoped = contextFor(argv, attached, other, true);
+      return { base: scoped.ctx, stop: scoped.stop };
+    },
   };
   return {
     ctx,