@moqtap/trace 0.3.0 → 0.4.1

package/src/index.ts

@@ -1,9 +1,10 @@
 // Binary .moqtrace format
 
-export type { MoqtraceWriter, ReadOptions } from './binary.js'
+export type { MoqtraceWriter, ReadOptions, RecoveredRegion } from './binary.js'
 export {
   createMoqtraceWriter,
   FORMAT_VERSION,
+  MalformedHeaderError,
   readMoqtrace,
   readMoqtraceHeader,
   readMoqtraceSegments,
@@ -14,9 +15,12 @@ export {
 } from './binary.js'
 // JSON (convenience)
 export { traceToJSON } from './json.js'
-export type { TraceRecorder } from './recorder.js'
+// Incremental reader, for a stream that arrives in pieces
+export type { MoqtraceReader, ReadItem } from './reader.js'
+export { createMoqtraceReader, TruncatedStreamError } from './reader.js'
+export type { ErrorDetails, TraceRecorder } from './recorder.js'
 // Recorder
-export { createRecorder } from './recorder.js'
+export { createRecorder, MAX_ERROR_RAW_BYTES } from './recorder.js'
 
 // Types
 export type {
@@ -25,6 +29,7 @@ export type {
   DerivationKind,
   DetailLevel,
   DropPolicy,
+  ErrorKind,
   ObjectHeaderEvent,
   ObjectPayloadEvent,
   PeerConnectedEvent,
@@ -46,4 +51,4 @@ export type {
   TraceHeader,
   UnknownEvent,
 } from './types.js'
-export { TRACE_ID_LENGTH } from './types.js'
+export { controlMessageFields, TRACE_ID_LENGTH } from './types.js'

package/src/json.ts

@@ -7,6 +7,13 @@ import type { Trace } from './types.js'
  * This is a one-way export for human inspection and debugging.
  * For lossless round-tripping, use the binary .moqtrace format
  * (`writeMoqtrace` / `readMoqtrace`).
+ *
+ * The unrecognised-key stores are part of the rendering rather than filtered
+ * out of it: `extra` appears on the header, on `segment`, on `sampling` and on
+ * each event, nested values and all. A view that showed only the keys this
+ * version names would reintroduce, for anyone reading a trace through it,
+ * exactly the loss the binary reader stopped making — and it is the view
+ * someone reaches for precisely when they suspect a key went missing.
  */
 export function traceToJSON(trace: Trace): string {
   return JSON.stringify(

package/src/reader.ts

@@ -0,0 +1,286 @@
+/**
+ * Incremental reader for the `.moqtrace` binary format.
+ *
+ * The three entry points in `binary.ts` all take the whole file at once, which
+ * a live capture does not have: the bytes arrive in chunks whose boundaries
+ * fall wherever the transport put them, almost never on an item boundary. This
+ * reader takes those chunks and hands back the items that have become whole,
+ * keeping the remainder for the next one.
+ *
+ * It is the read side of {@link createMoqtraceWriter}, which has been
+ * incremental from the start — the asymmetry this closes was one-sided.
+ *
+ * Decoding is not reimplemented here. The header and event decoders are
+ * imported from `binary.ts` so that a trace read in instalments and the same
+ * trace read whole produce the same values. A second decoder would be a second
+ * set of answers, and nothing tests two readers of one format against each
+ * other, so the disagreement would surface as a rendering bug somewhere far
+ * from here.
+ */
+
+import {
+  cborToEvent,
+  cborToHeader,
+  decode,
+  MAGIC,
+  PREAMBLE_SIZE,
+  SUPPORTED_VERSIONS,
+  startsWithMagic,
+} from './binary.js'
+import { cborItemLength } from './cbor-scan.js'
+import type { TraceEvent, TraceHeader } from './types.js'
+
+/**
+ * The stream ended part-way through an item.
+ *
+ * Deliberately not `TruncatedTraceError`, whose `segments` field promises
+ * "everything that decoded before the cut". An incremental reader has already
+ * handed those events to its caller and does not keep them, so it cannot fill
+ * that field — and filling it with an empty array would report a trace that
+ * decoded nothing, which is a wrong answer rather than a missing one. The
+ * caller holds what decoded; this says only where the bytes stopped.
+ */
+export class TruncatedStreamError extends Error {
+  /**
+   * Offset into the whole stream at which the incomplete item begins — not
+   * where the bytes ran out. It matches the Rust reader's
+   * `MoqTraceError::Truncated { offset }`, so the same cut is named the same
+   * number in both languages.
+   */
+  readonly offset: number
+
+  /**
+   * How many events reached the caller before the cut.
+   *
+   * The events themselves are gone — that is what makes this error different
+   * from {@link TruncatedTraceError} — but their count is not, and it costs one
+   * integer to keep. It is the difference between "the stream was cut" and "the
+   * stream was cut after 4,812 events at byte 190,224", and only the second
+   * tells whoever is holding a damaged capture how much of it they still have.
+   */
+  readonly events: number
+
+  constructor(offset: number, events: number) {
+    super(
+      `Trace stream truncated: the item starting at byte ${offset} is incomplete, ` +
+        `after ${events} event${events === 1 ? '' : 's'}`,
+    )
+    this.name = 'TruncatedStreamError'
+    this.offset = offset
+    this.events = events
+  }
+}
+
+/**
+ * One thing the reader has finished reading.
+ *
+ * The two kinds mirror the Rust crate's `ReadItem`, so a consumer that has read
+ * one implementation recognises the other. A `segment` arrives before any event
+ * that belongs to it, including the first one: unlike the Rust reader, which
+ * takes a blocking source and parses the opening header in its constructor,
+ * this one cannot have a header until bytes arrive, so every header reaches the
+ * caller the same way and there is no special case for the first.
+ */
+export type ReadItem =
+  | { readonly kind: 'segment'; readonly header: TraceHeader }
+  | { readonly kind: 'event'; readonly event: TraceEvent }
+
+/** Reads a `.moqtrace` stream in whatever instalments it arrives in. */
+export interface MoqtraceReader {
+  /**
+   * Add the next chunk and take everything it completed.
+   *
+   * Returns an empty array when the chunk did not finish an item, which is
+   * ordinary and not a signal of anything. Bytes left over are kept.
+   *
+   * @throws {MalformedHeaderError} if a segment header carries no usable value
+   *   for a key the format requires.
+   * @throws {MalformedCborError} if the bytes at an item boundary are not a
+   *   well-formed CBOR item.
+   */
+  push(chunk: Uint8Array): ReadItem[]
+
+  /**
+   * Declare the stream complete and take any last item.
+   *
+   * This can return items. An event shorter than the magic cannot be told from
+   * the start of a segment preamble while more bytes might still arrive, so
+   * such an event waits here until `end()` rules the preamble out.
+   *
+   * @throws {TruncatedStreamError} if bytes remain that do not form a whole
+   *   item — the file stops mid-event, mid-preamble or mid-header.
+   */
+  end(): ReadItem[]
+
+  /**
+   * The header of the segment now being read, or `undefined` before the first
+   * one is whole. In a segmented stream this changes at every boundary.
+   */
+  readonly header: TraceHeader | undefined
+
+  /** Bytes held back because they do not yet form a whole item. */
+  readonly pending: number
+}
+
+/**
+ * A reader that takes the stream in chunks.
+ *
+ * ```ts
+ * const reader = createMoqtraceReader()
+ * for await (const chunk of stream) {
+ *   for (const item of reader.push(chunk)) {
+ *     if (item.kind === 'event') render(item.event)
+ *   }
+ * }
+ * for (const item of reader.end()) { ... }
+ * ```
+ *
+ * `ReadOptions.recover` has no counterpart here **on purpose.** Its behaviour
+ * in the whole-buffer readers — drop the damaged segment, return the rest, say
+ * nothing — is under revision, because a caller cannot tell three segments from
+ * four with one unreadable. Guessing an answer here would make an incremental
+ * reader the place that shipped it, and a live view is where a silently
+ * dropped segment does the most harm. Until that is settled this reader stops
+ * at damage and says where.
+ */
+export function createMoqtraceReader(): MoqtraceReader {
+  // One buffer rather than a chunk list. After a consume it is `subarray`d, so
+  // the retained bytes are a view and not a copy, and in the steady state the
+  // remainder is smaller than one item — the concatenation on each push costs
+  // that remainder, not the stream.
+  let buffer = new Uint8Array(0)
+  // Where `buffer[0]` sits in the whole stream, so an error names an offset
+  // into the file rather than into whatever is left of it.
+  let consumed = 0
+  // Events handed to the caller, kept only so a truncation can say how much of
+  // the capture survived it.
+  let delivered = 0
+  let current: TraceHeader | undefined
+  let ended = false
+
+  function take(count: number): Uint8Array {
+    const slice = buffer.subarray(0, count)
+    buffer = buffer.subarray(count)
+    consumed += count
+    return slice
+  }
+
+  /**
+   * Read a preamble and its header if all of it is here.
+   *
+   * Returns `undefined` to mean "not yet", which is why the length checks are
+   * not `validatePreamble`'s: that function is written for a buffer that is the
+   * whole file, so it reads a short one as a truncated file and throws. Here a
+   * short buffer is the normal state between two chunks. What it does share is
+   * the part worth sharing — the version check against
+   * {@link SUPPORTED_VERSIONS} and, once the bytes are all present,
+   * `cborToHeader`.
+   */
+  function tryReadSegment(): TraceHeader | undefined {
+    if (buffer.length < PREAMBLE_SIZE) return undefined
+
+    const view = new DataView(buffer.buffer, buffer.byteOffset, buffer.byteLength)
+    const version = view.getUint32(8, true)
+    if (!SUPPORTED_VERSIONS.includes(version)) {
+      throw new Error(
+        `Unsupported format version: ${version} (supported: ${SUPPORTED_VERSIONS.join(', ')})`,
+      )
+    }
+
+    const headerLength = view.getUint32(12, true)
+    if (buffer.length < PREAMBLE_SIZE + headerLength) return undefined
+
+    take(PREAMBLE_SIZE)
+    const headerBytes = take(headerLength)
+    return cborToHeader(decode(headerBytes) as Record<string, unknown>)
+  }
+
+  function drain(atEnd: boolean): ReadItem[] {
+    const items: ReadItem[] = []
+
+    for (;;) {
+      if (current === undefined) {
+        // Nothing has been read yet, so the stream must open with a preamble.
+        // Saying so here means a caller who points this at some other file
+        // learns that from the magic rather than from a CBOR error deep in it.
+        if (buffer.length >= MAGIC.length && !startsWithMagic(buffer, 0)) {
+          throw new Error('Invalid magic bytes: not a .moqtrace file')
+        }
+        const header = tryReadSegment()
+        if (header === undefined) break
+        current = header
+        items.push({ kind: 'segment', header })
+        continue
+      }
+
+      if (buffer.length === 0) break
+
+      // A segment boundary is looked for only here, at a real item boundary —
+      // the same eight bytes inside a captured payload are not one. Deciding
+      // needs the whole magic: with fewer bytes than that in hand, "does not
+      // start with the magic" is not yet a fact about the stream, only about
+      // how much of it has arrived. Waiting is wrong at the end of the stream
+      // and right everywhere else, which is what `atEnd` distinguishes.
+      if (buffer.length < MAGIC.length && !atEnd) break
+
+      if (startsWithMagic(buffer, 0)) {
+        const header = tryReadSegment()
+        if (header === undefined) break
+        current = header
+        items.push({ kind: 'segment', header })
+        continue
+      }
+
+      const length = cborItemLength(buffer, 0)
+      if (length === null) break
+
+      const item = decode(take(length))
+      items.push({ kind: 'event', event: cborToEvent(item as Record<string, unknown>) })
+      delivered += 1
+    }
+
+    return items
+  }
+
+  return {
+    push(chunk: Uint8Array): ReadItem[] {
+      if (ended) throw new Error('Cannot push to a reader that has ended')
+      if (chunk.length > 0) {
+        const grown = new Uint8Array(buffer.length + chunk.length)
+        grown.set(buffer, 0)
+        grown.set(chunk, buffer.length)
+        buffer = grown
+      }
+      return drain(false)
+    },
+
+    end(): ReadItem[] {
+      if (ended) return []
+      ended = true
+      const items = drain(true)
+      // The one place a truncation is declared. `drain` only ever stops at
+      // bytes it cannot use yet, which is the ordinary state between chunks and
+      // a fault only once no more are coming — so it reports nothing and leaves
+      // the leftovers, and the verdict is taken here, where "no more are
+      // coming" is known. Raising it in both places, as an earlier revision
+      // did, gave one fault two throw sites that could drift apart.
+      //
+      // A reader that never produced a header is the same fault: it saw no
+      // preamble at all, which includes the empty stream. Returning no items
+      // for that would report a trace that recorded nothing, when what happened
+      // is that no trace arrived.
+      if (buffer.length > 0 || current === undefined) {
+        throw new TruncatedStreamError(consumed, delivered)
+      }
+      return items
+    },
+
+    get header(): TraceHeader | undefined {
+      return current
+    },
+
+    get pending(): number {
+      return buffer.length
+    },
+  }
+}

package/src/recorder.ts

@@ -1,5 +1,84 @@
 import type { SessionState } from '@moqtap/codec/session'
-import type { DetailLevel, RecorderOptions, Trace, TraceEvent, TraceHeader } from './types.js'
+import type {
+  DetailLevel,
+  ErrorKind,
+  RecorderOptions,
+  Trace,
+  TraceErrorEvent,
+  TraceEvent,
+  TraceHeader,
+} from './types.js'
+
+/**
+ * The most bytes a recorder may put in an error event's `raw`.
+ *
+ * A MUST in the format, and a fixed number rather than a suggestion, because
+ * an error event is one of the types sampling must not drop: every other
+ * high-volume field is bounded by sampling, so if this one is not bounded here
+ * it is not bounded at all. A peer that opens ten thousand streams and sends
+ * garbage on each is what a fuzzer does to a relay, and the trace of it should
+ * not be larger than the attack.
+ *
+ * It lives here, in the recorder, and appears nowhere in `binary.ts`. The cap
+ * is addressed to the party deciding what to say about traffic it has just
+ * observed — {@link TraceRecorder.recordError}, and nothing else. A serializer
+ * cannot tell a freshly recorded event from one that arrived by being read, so
+ * a cap applied there would either truncate evidence on a rewrite or refuse a
+ * file the reader was required to accept. `writeMoqtrace` therefore writes a
+ * `raw` of any length, and {@link TraceRecorder.record} — which inserts an
+ * event a caller may well have read out of a file — does not apply it either.
+ */
+export const MAX_ERROR_RAW_BYTES = 4096
+
+/**
+ * What a recorder knows about an error beyond its code and reason.
+ *
+ * Every field is optional and each is gated at its own detail level, so one
+ * call site can hand over everything it has and let the recorder keep what its
+ * level allows.
+ */
+export interface ErrorDetails {
+  /**
+   * QUIC stream the error was observed on.
+   *
+   * Omit it when there was no stream or none is known — that is what its
+   * absence says, and a reader must not read it as stream `0`.
+   */
+  readonly streamId?: bigint
+  /** What sort of failure this was. An open vocabulary; any string is legal. */
+  readonly errorKind?: ErrorKind
+  /**
+   * The offending bytes, in full and untruncated.
+   *
+   * Hand over everything held: the recorder applies {@link
+   * MAX_ERROR_RAW_BYTES} and reports the untruncated length in `rawLength`,
+   * so a caller that pre-truncates loses the one signal saying the capture is
+   * partial. Recorded only at `'full'`, and only once per flow.
+   */
+  readonly raw?: Uint8Array
+  /**
+   * The length to report, where the caller already holds less than it saw.
+   *
+   * Defaults to `raw.length`, which is right whenever `raw` is everything
+   * there was. Set it to state a length larger than the bytes on hand — a
+   * caller that hit a cap of its own upstream, or one that knows a message's
+   * declared length without having kept it.
+   *
+   * **Set it to `null` when the true length is unknown**, and the key is
+   * omitted. SPEC.md requires that rather than a guess, because the guess is
+   * not neutral: `rawlen` equal to the bytes present is this format's signal
+   * for *not truncated*, so defaulting it would assert a complete capture
+   * exactly where the caller knows least about whether the input was
+   * complete. Absent, a reader treats a `raw` of exactly
+   * {@link MAX_ERROR_RAW_BYTES} as possibly truncated, which is the truth.
+   *
+   * The caller this exists for is real rather than hypothetical: a recorder
+   * that stops *reading* a stream once it will not parse — the sensible thing
+   * to do with such a stream — never holds more than its own cap and cannot
+   * know what it did not read.
+   */
+  readonly rawLength?: number | null
+}
 
 export interface TraceRecorder {
   /** Wrap a SessionState to auto-record control messages and state changes. */
@@ -7,7 +86,16 @@ export interface TraceRecorder {
     session: SessionState<M, T>,
   ): SessionState<M, T>
 
-  /** Record an arbitrary event manually. */
+  /**
+   * Record an arbitrary event manually.
+   *
+   * A passthrough: no detail-level gate, and no cap on an error event's
+   * `raw`, because an event handed here may have been read out of a file
+   * rather than built from observed bytes, and shortening that one would
+   * destroy evidence to enforce a rule addressed to whoever recorded it.
+   * Callers building an event from bytes they just saw want
+   * {@link recordError}.
+   */
   record(event: TraceEvent): void
 
   /** Record a stream-opened event. Ignored at 'control' detail level. */
@@ -34,8 +122,15 @@ export interface TraceRecorder {
     payload?: Uint8Array,
   ): void
 
-  /** Record a protocol error. */
-  recordError(errorCode: number, reason: string): void
+  /**
+   * Record a protocol error.
+   *
+   * Recorded at every detail level — an error event is non-droppable — but
+   * `details` is gated key by key: `streamId` and `errorKind` at every
+   * level, `rawLength` from `'headers+sizes'`, and `raw` at `'full'`
+   * alone, capped at {@link MAX_ERROR_RAW_BYTES} and written once per flow.
+   */
+  recordError(errorCode: number, reason: string, details?: ErrorDetails): void
 
   /** Record a user-defined annotation. */
   annotate(label: string, data?: unknown): void
@@ -73,14 +168,47 @@ export function createRecorder(options: RecorderOptions): TraceRecorder {
   const messageTypeId = options.messageTypeId ?? (() => 0)
 
   const events: TraceEvent[] = []
+  /**
+   * Index of the oldest event still live in {@link events}.
+   *
+   * Dropping used to be `events.shift()`, which moves the whole backlog on
+   * every call; at a hundred thousand retained events that is O(n) per
+   * recorded event, and it bites hardest exactly when the recorder is already
+   * under pressure. The head advances instead, and the dead prefix is spliced
+   * away in one pass every few thousand drops, which is O(1) amortised.
+   */
+  let head = 0
+  /** Events dropped to stay under `maxEvents`, reported as `SamplingInfo`. */
+  let dropped = 0
+  /**
+   * Flows whose bytes have already been recorded.
+   *
+   * The format allows `"raw"` once per flow — per stream where the error
+   * names one, per peer where it does not, and this recorder writes no peer
+   * identifier, so every error naming no stream shares one flow.
+   *
+   * The latch is on the field and not on the event: later errors on a flow are
+   * still recorded in full, they simply carry no bytes. Suppressing the events
+   * would discard the causal record precisely when a peer is misbehaving
+   * repeatedly, which is when it matters most. What grows without bound is the
+   * bytes, so that is what is bounded.
+   */
+  const rawRecorded = new Set<string>()
   let _recording = true
   let _seq = 0
   const startTime = Date.now()
 
   function addEvent(event: TraceEvent): void {
     if (!_recording) return
-    if (events.length >= maxEvents) {
-      events.shift()
+    if (events.length - head >= maxEvents) {
+      head++
+      dropped++
+      // Compact in one pass rather than per drop. Bounded both ways so a small
+      // `maxEvents` does not accumulate a long dead prefix.
+      if (head >= 4096 || head >= maxEvents) {
+        events.splice(0, head)
+        head = 0
+      }
     }
     events.push(event)
   }
@@ -89,6 +217,18 @@ export function createRecorder(options: RecorderOptions): TraceRecorder {
     return _seq++
   }
 
+  /** The flow an error belongs to, for the one-`raw`-per-flow latch. */
+  function rawFlow(streamId: bigint | undefined): string {
+    return streamId == null ? 'no-stream' : `stream:${streamId}`
+  }
+
+  /** Take a flow's one slot for recording bytes, or answer that it is spent. */
+  function claimRawSlot(flow: string): boolean {
+    if (rawRecorded.has(flow)) return false
+    rawRecorded.add(flow)
+    return true
+  }
+
   function wrapSession<M extends { type: string }, T extends string>(
     session: SessionState<M, T>,
   ): SessionState<M, T> {
@@ -122,7 +262,7 @@ export function createRecorder(options: RecorderOptions): TraceRecorder {
           timestamp: clock(),
           direction: 1, // rx
           messageType: messageTypeId(message.type),
-          message: message as unknown as Record<string, unknown>,
+          message,
         })
 
         if (result.ok && result.phase !== prevPhase) {
@@ -152,7 +292,7 @@ export function createRecorder(options: RecorderOptions): TraceRecorder {
           timestamp: clock(),
           direction: 0, // tx
           messageType: messageTypeId(message.type),
-          message: message as unknown as Record<string, unknown>,
+          message,
         })
 
         if (result.ok && result.phase !== prevPhase) {
@@ -231,14 +371,40 @@ export function createRecorder(options: RecorderOptions): TraceRecorder {
       addEvent(event)
     },
 
-    recordError(errorCode, reason) {
-      addEvent({
+    recordError(errorCode, reason, details) {
+      const streamId = details?.streamId
+      const raw = details?.raw
+      // What the recorder held, before the cap below — the whole point of the
+      // key, since a reader compares it against the bytes it got to learn that
+      // the capture is partial and by how much. An explicit `null` means the
+      // caller does not know, and omits the key rather than claiming the bytes
+      // present are all there were.
+      const rawLength =
+        details?.rawLength === null ? undefined : (details?.rawLength ?? raw?.length)
+      // Claimed only when the bytes are actually going to be written, so a
+      // recording below 'full' does not spend a flow's one slot on an event
+      // that carried nothing.
+      const writeRaw =
+        raw != null && detailRank >= DETAIL_RANK.full && claimRawSlot(rawFlow(streamId))
+
+      const event: TraceErrorEvent = {
         type: 'error',
         seq: nextSeq(),
         timestamp: clock(),
         errorCode,
         reason,
-      })
+        ...(streamId != null ? { streamId } : {}),
+        ...(details?.errorKind != null ? { errorKind: details.errorKind } : {}),
+        // A size, gated with the other sizes and one level below the bytes: an
+        // error may name a data stream, so at 'control' this length would be a
+        // fact about media volume in a trace whose declared level excludes it.
+        // Written whether or not the bytes are, which is most of its value.
+        ...(rawLength != null && detailRank >= DETAIL_RANK['headers+sizes'] ? { rawLength } : {}),
+        ...(writeRaw && raw != null
+          ? { raw: raw.length > MAX_ERROR_RAW_BYTES ? raw.slice(0, MAX_ERROR_RAW_BYTES) : raw }
+          : {}),
+      }
+      addEvent(event)
     },
 
     annotate(label, data) {
@@ -263,8 +429,19 @@ export function createRecorder(options: RecorderOptions): TraceRecorder {
         ...(options.source != null ? { source: options.source } : {}),
         ...(options.endpoint != null ? { endpoint: options.endpoint } : {}),
         ...(options.sessionId != null ? { sessionId: options.sessionId } : {}),
+        // Carried through so a recorder with something to say the format has
+        // no key for can say it, and so a header this package builds has the
+        // same shape as one it reads. `writeMoqtrace` writes the store after
+        // the keys above and drops any entry naming one of them.
+        ...(options.extra != null ? { extra: options.extra } : {}),
+        // Only when something was actually dropped: the key's presence is
+        // what tells a reader this trace is incomplete, so writing it with a
+        // zero would make every complete trace look sampled.
+        ...(dropped > 0
+          ? { sampling: { dropPolicy: 'head' as const, droppedTotal: dropped } }
+          : {}),
       }
-      return { header, events: [...events] }
+      return { header, events: events.slice(head) }
     },
 
     get recording() {