cut-at-k 0.1.0

package/src/bytes.js

@@ -0,0 +1,131 @@
+/**
+ * Cut inside a frame, not just between events.
+ *
+ * Severing between events tests the state machine: does the consumer handle a
+ * run that stopped after three events rather than nine. Severing inside a frame
+ * tests the parser underneath it, which is a different and usually softer
+ * target — a cut in the middle of a UTF-8 sequence, between `data:` and its
+ * newline, or halfway through a JSON payload.
+ *
+ * The interesting offsets are not evenly spread. Most bytes of a frame are
+ * unremarkable; the ones that break things sit at structural seams. So this
+ * offers both: every offset when you can afford it, and the seams when you
+ * cannot.
+ */
+
+const encoder = new TextEncoder();
+
+/** Serialise events as an SSE body, one event per frame. */
+export function toSSE(events, { event: eventName } = {}) {
+  return events
+    .map((e) => {
+      const data = typeof e === 'string' ? e : JSON.stringify(e);
+      const name = eventName ? `event: ${eventName}\n` : '';
+      return `${name}data: ${data}\n\n`;
+    })
+    .join('');
+}
+
+/**
+ * Offsets worth cutting at, for a body you do not want to cut 40,000 times.
+ *
+ * @param {string} body
+ * @returns {{offset: number, why: string}[]}
+ */
+export function seams(body) {
+  const bytes = encoder.encode(body);
+  /** @type {Map<number, string>} */
+  const found = new Map();
+  const note = (offset, why) => {
+    if (offset > 0 && offset < bytes.length && !found.has(offset)) found.set(offset, why);
+  };
+
+  // Structural positions in the SSE framing itself.
+  for (let i = 0; i < bytes.length; i++) {
+    const b = bytes[i];
+
+    // Inside a multi-byte UTF-8 sequence. A continuation byte is 10xxxxxx, so
+    // cutting immediately before one splits a character in half.
+    if ((b & 0b1100_0000) === 0b1000_0000) note(i, 'mid-UTF-8 sequence');
+
+    // Between a field name and its value: `data:` and what follows.
+    if (b === 0x3a /* : */) note(i + 1, 'after a field colon');
+
+    // Between the two newlines that end a frame.
+    if (b === 0x0a /* \n */ && bytes[i + 1] === 0x0a) note(i + 1, 'between frame newlines');
+
+    // Inside the JSON payload, at its own structural marks.
+    if (b === 0x7b /* { */) note(i + 1, 'just inside an object');
+    if (b === 0x22 /* " */) note(i + 1, 'just inside a string');
+    if (b === 0x2c /* , */) note(i + 1, 'after a comma');
+  }
+
+  return [...found.entries()]
+    .map(([offset, why]) => ({ offset, why }))
+    .sort((a, b) => a.offset - b.offset);
+}
+
+/**
+ * Cut a serialised body at chosen byte offsets and record what the consumer is
+ * left with.
+ *
+ * @param {object} options
+ * @param {string} options.body                 The whole serialised stream.
+ * @param {(bytes: string, label: string) => Promise<import('./sever.js').Observation>} options.replay
+ * @param {string} [options.label]
+ * @param {'seams' | 'every'} [options.offsets] Which offsets to try. Defaults
+ *   to the structural seams, which is the version you can run in CI.
+ * @param {(a: import('./sever.js').Observation, b: import('./sever.js').Observation) => boolean} [options.same]
+ */
+export async function severAtEveryByte({
+  body,
+  replay,
+  label = 'stream',
+  offsets = 'seams',
+  same,
+}) {
+  if (typeof body !== 'string' || body.length === 0) {
+    throw new Error('severAtEveryByte: body must be a non-empty string');
+  }
+  const indistinguishable = same ?? ((a, b) => a.settled === b.settled);
+  const bytes = encoder.encode(body);
+
+  const points =
+    offsets === 'every'
+      ? Array.from({ length: bytes.length - 1 }, (_, i) => ({ offset: i + 1, why: 'byte' }))
+      : seams(body);
+
+  const whole = await replay(body, `${label}:whole`);
+  const cuts = [];
+
+  for (const { offset, why } of points) {
+    // Slice the bytes, then decode. A cut inside a multi-byte sequence leaves a
+    // partial character, which is the whole point — decoding with a fatal
+    // decoder here would throw away the case being tested.
+    const prefix = new TextDecoder('utf-8').decode(bytes.slice(0, offset));
+
+    let observation;
+    let threw;
+    try {
+      observation = await replay(prefix, `${label}:@${offset}`);
+    } catch (err) {
+      threw = err instanceof Error ? err.message : String(err);
+    }
+
+    if (!observation) {
+      cuts.push({ offset, why, of: bytes.length, threw });
+      continue;
+    }
+
+    cuts.push({
+      offset,
+      why,
+      of: bytes.length,
+      observation,
+      settledAlike: indistinguishable(observation, whole),
+      terminal: (observation.delivered ?? []).at(-1) ?? '(nothing)',
+    });
+  }
+
+  return { label, whole, cuts, points: points.length, bytes: bytes.length };
+}

package/src/index.js

@@ -0,0 +1,3 @@
+export { severAtEveryPoint } from './sever.js';
+export { summarise, format } from './report.js';
+export { toSSE, seams, severAtEveryByte } from './bytes.js';

package/src/report.js

@@ -0,0 +1,117 @@
+const TERMINALS = new Set(['RUN_FINISHED', 'RUN_ERROR']);
+
+/**
+ * Turn a pile of cuts into the few things worth reading.
+ *
+ * A count of failures is not a finding. Three things have to be separated or
+ * the number means nothing:
+ *
+ *   - A cut that lost nothing. Cutting after the last content event and before
+ *     some trailing bookkeeping removes no information, so of course the
+ *     consumer reports the same thing. Not a defect.
+ *   - A stream that is invalid on purpose. Cutting it before its deliberate
+ *     error removes the error, so the whole stream delivers less than the
+ *     prefix and every comparison is noise.
+ *   - A cut that genuinely lost content and was reported as though it had not.
+ *     That is the finding.
+ *
+ * The first version of this file reported the first two as findings. On the
+ * corpus in results/ that means calling 154 cuts a problem when 54 are real —
+ * `node results/run-ag-ui.mjs results/sever-results.json` prints both numbers.
+ * A tool that cries wolf is worse than no tool, so the distinction is now the
+ * whole design.
+ */
+
+/**
+ * @param {import('./sever.js').CutReport[]} reports
+ * @param {object} [options]
+ * @param {(label: string) => boolean} [options.ignore]
+ *   Streams to leave out of the findings — typically the ones a conformance
+ *   corpus ships as deliberately invalid. Their count is reported separately
+ *   rather than silently dropped.
+ * @param {(cut: import('./sever.js').Observation, whole: import('./sever.js').Observation) => boolean} [options.lostContent]
+ *   Did this cut actually lose something the consumer would have surfaced?
+ *   Without it, a cut that changed nothing cannot be told from one that did,
+ *   and the summary says so instead of guessing.
+ */
+export function summarise(reports, { ignore, lostContent } = {}) {
+  const considered = ignore ? reports.filter((r) => !ignore(r.label)) : reports;
+  const excluded = reports.length - considered.length;
+
+  const all = considered.flatMap((r) => r.cuts.map((c) => ({ ...c, label: r.label, whole: r.whole })));
+  const threw = all.filter((c) => c.threw);
+  const usable = all.filter((c) => !c.threw);
+
+  // Reported the same as the whole run. On its own this is not yet a finding.
+  const reportedAlike = usable.filter((c) => c.settledAlike === true);
+
+  // The finding: something was lost, and the consumer reported otherwise.
+  const findings = lostContent
+    ? reportedAlike.filter((c) => lostContent(c.observation, c.whole))
+    : [];
+
+  // Both channels quiet — no terminal event delivered at all — is the worse
+  // shape. Checking only the LAST event is not enough: a stream carrying more
+  // than one run can deliver a terminal for an earlier run and still end mid-
+  // content, and that consumer did get a signal, just not about this part.
+  const noTerminal = findings.filter(
+    (c) => !(c.observation?.delivered ?? []).some((e) => TERMINALS.has(e)),
+  );
+
+  const prefixBroken = usable.filter((c) => c.prefixHeld === false);
+
+  return {
+    streams: considered.length,
+    excluded,
+    cuts: all.length,
+    threw: threw.length,
+    reportedAlike: reportedAlike.length,
+    lostContentChecked: Boolean(lostContent),
+    findings,
+    noTerminal,
+    prefixBroken,
+    byStream: countBy(findings, (c) => c.label),
+    byTerminal: countBy(findings, (c) => c.terminal),
+  };
+}
+
+/** A short human-readable version, for a terminal or a pull request body. */
+export function format(summary) {
+  const lines = [];
+  lines.push(
+    `${summary.streams} streams, ${summary.cuts} cuts` +
+      (summary.excluded ? `, ${summary.excluded} streams excluded as invalid on purpose` : ''),
+  );
+  if (summary.threw) lines.push(`${summary.threw} cuts threw during replay`);
+  lines.push('');
+  lines.push(`reported the same as the whole run    : ${summary.reportedAlike}`);
+
+  if (!summary.lostContentChecked) {
+    lines.push('');
+    lines.push('No lostContent check was supplied, so none of the above can be');
+    lines.push('called a defect: a cut that removed nothing reports the same for');
+    lines.push('a good reason. Supply lostContent to separate them.');
+    return lines.join('\n');
+  }
+
+  lines.push(`  of which something was actually lost : ${summary.findings.length}`);
+  lines.push(`  of those, no terminal event either   : ${summary.noTerminal.length}`);
+  lines.push(`spread across                          : ${Object.keys(summary.byStream).length} streams`);
+
+  if (summary.prefixBroken.length) {
+    lines.push('');
+    lines.push(`prefix property broken                 : ${summary.prefixBroken.length}`);
+    lines.push('  Check these by hand before reporting them. A stream that is');
+    lines.push('  invalid on purpose looks exactly like this and is not a defect.');
+  }
+  return lines.join('\n');
+}
+
+function countBy(list, key) {
+  const out = {};
+  for (const item of list) {
+    const k = key(item);
+    out[k] = (out[k] ?? 0) + 1;
+  }
+  return out;
+}

package/src/sever.js

@@ -0,0 +1,101 @@
+/**
+ * Cut a stream at every point and record what the consumer is left with.
+ *
+ * The idea is small enough to state in a sentence: a stream that stops early is
+ * not a rare case, it is Tuesday — a dropped connection, a proxy timeout, a
+ * producer that died, a person who pressed stop. So run the same stream through
+ * a consumer once whole, then once for every prefix of it, and compare.
+ *
+ * Nothing here knows about any particular protocol. You supply the events and a
+ * function that replays them through whatever client you are testing; this
+ * supplies the loop and the comparison.
+ */
+
+/**
+ * @typedef {object} Observation
+ *   Whatever the consumer ended up with. Only two fields are interpreted here;
+ *   anything else you attach is carried through to the report untouched.
+ * @property {string[]} [delivered] Event types the consumer actually surfaced.
+ * @property {string}   [settled]   How the awaited call reported, as a string.
+ */
+
+/**
+ * @template E
+ * @param {object} options
+ * @param {E[]} options.events            The whole stream, in order.
+ * @param {(prefix: E[], label: string) => Promise<Observation>} options.replay
+ *   Replays a prefix through the consumer and returns what it was left with.
+ *   It is called once per cut, plus once with the whole stream.
+ * @param {string} [options.label]        Names this stream in the report.
+ * @param {(a: Observation, b: Observation) => boolean} [options.same]
+ *   Decides whether two observations are indistinguishable to application code.
+ *   Defaults to comparing `settled`.
+ * @returns {Promise<CutReport>}
+ */
+export async function severAtEveryPoint({ events, replay, label = 'stream', same }) {
+  if (!Array.isArray(events) || events.length === 0) {
+    throw new Error('severAtEveryPoint: events must be a non-empty array');
+  }
+  const indistinguishable = same ?? ((a, b) => a.settled === b.settled);
+
+  const whole = await replay(events, `${label}:whole`);
+  /** @type {Cut[]} */
+  const cuts = [];
+
+  // k = events.length is the whole stream, which is the baseline, not a cut.
+  for (let k = 1; k < events.length; k++) {
+    let observation;
+    let threw;
+    try {
+      observation = await replay(events.slice(0, k), `${label}:k${k}`);
+    } catch (err) {
+      threw = err instanceof Error ? err.message : String(err);
+    }
+
+    if (!observation) {
+      cuts.push({ k, of: events.length, threw });
+      continue;
+    }
+
+    const delivered = observation.delivered ?? [];
+    cuts.push({
+      k,
+      of: events.length,
+      observation,
+      // Truncation may lose the tail. It must never change the head.
+      prefixHeld: isPrefix(delivered, whole.delivered ?? []),
+      // The property that matters: a run that was cut short must not report
+      // what the same run reports when it finishes.
+      settledAlike: indistinguishable(observation, whole),
+      terminal: lastOf(delivered),
+    });
+  }
+
+  return { label, whole, cuts };
+}
+
+/**
+ * @typedef {object} Cut
+ * @property {number} k
+ * @property {number} of
+ * @property {Observation} [observation]
+ * @property {string} [threw]
+ * @property {boolean} [prefixHeld]
+ * @property {boolean} [settledAlike]
+ * @property {string} [terminal]
+ */
+
+/**
+ * @typedef {object} CutReport
+ * @property {string} label
+ * @property {Observation} whole
+ * @property {Cut[]} cuts
+ */
+
+function isPrefix(short, long) {
+  return short.length <= long.length && short.every((v, i) => v === long[i]);
+}
+
+function lastOf(list) {
+  return list.length ? list[list.length - 1] : '(nothing)';
+}