@moqtap/trace 0.2.0 → 0.4.1

package/src/cbor-scan.ts

@@ -0,0 +1,185 @@
+/**
+ * Measure one CBOR item without decoding it.
+ *
+ * A `.moqtrace` file is a CBOR sequence, and a segmented one splices a
+ * preamble between two of its items. Finding those boundaries means knowing
+ * where each item ends — and knowing it *only* at real item boundaries. A
+ * plain byte scan for the magic would also match the same eight bytes sitting
+ * inside a captured payload, and would split a trace that was never segmented.
+ *
+ * Measuring is cheaper than decoding: a byte or text string is skipped by its
+ * declared length rather than read, so the cost follows the structure of the
+ * data and not its size.
+ */
+
+/** The item at the given offset ran past the end of the buffer. */
+export const INCOMPLETE = null
+
+/** Thrown when the bytes are not a well-formed CBOR item at all. */
+export class MalformedCborError extends Error {
+  readonly offset: number
+
+  constructor(message: string, offset: number) {
+    super(`${message} at byte ${offset}`)
+    this.name = 'MalformedCborError'
+    this.offset = offset
+  }
+}
+
+const MAJOR_UNSIGNED = 0
+const MAJOR_NEGATIVE = 1
+const MAJOR_BYTES = 2
+const MAJOR_TEXT = 3
+const MAJOR_ARRAY = 4
+const MAJOR_MAP = 5
+const MAJOR_TAG = 6
+const MAJOR_SIMPLE = 7
+
+const AI_ONE_BYTE = 24
+const AI_TWO_BYTES = 25
+const AI_FOUR_BYTES = 26
+const AI_EIGHT_BYTES = 27
+const AI_INDEFINITE = 31
+const BREAK = 0xff
+
+/** Bytes that follow the initial byte for each additional-information value. */
+function headerExtraBytes(additionalInfo: number, offset: number): number {
+  if (additionalInfo < AI_ONE_BYTE) return 0
+  switch (additionalInfo) {
+    case AI_ONE_BYTE:
+      return 1
+    case AI_TWO_BYTES:
+      return 2
+    case AI_FOUR_BYTES:
+      return 4
+    case AI_EIGHT_BYTES:
+      return 8
+    case AI_INDEFINITE:
+      return 0
+    default:
+      throw new MalformedCborError(`reserved additional information ${additionalInfo}`, offset)
+  }
+}
+
+/** Read the argument encoded in the item header. */
+function readArgument(bytes: Uint8Array, offset: number, additionalInfo: number): number {
+  if (additionalInfo < AI_ONE_BYTE) return additionalInfo
+  const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength)
+  switch (additionalInfo) {
+    case AI_ONE_BYTE:
+      return view.getUint8(offset + 1)
+    case AI_TWO_BYTES:
+      return view.getUint16(offset + 1, false)
+    case AI_FOUR_BYTES:
+      return view.getUint32(offset + 1, false)
+    case AI_EIGHT_BYTES: {
+      const value = view.getBigUint64(offset + 1, false)
+      // A length this large cannot be indexed anyway; treating it as a number
+      // is safe because the caller only ever compares it against the buffer.
+      return Number(value)
+    }
+    default:
+      throw new MalformedCborError(`reserved additional information ${additionalInfo}`, offset)
+  }
+}
+
+/**
+ * Advance past the item at `offset`, returning the offset just after it, or
+ * {@link INCOMPLETE} if the buffer ends part-way through.
+ *
+ * @param allowBreak whether a break code is a legal item here — true only
+ *   directly inside an indefinite-length container.
+ */
+function skipItem(bytes: Uint8Array, offset: number, allowBreak: boolean): number | null {
+  if (offset >= bytes.length) return INCOMPLETE
+
+  const initial = bytes[offset] as number
+  if (initial === BREAK) {
+    if (!allowBreak) {
+      throw new MalformedCborError('break outside an indefinite-length item', offset)
+    }
+    return offset + 1
+  }
+
+  const major = initial >> 5
+  const additionalInfo = initial & 0x1f
+  const extra = headerExtraBytes(additionalInfo, offset)
+  if (offset + 1 + extra > bytes.length) return INCOMPLETE
+
+  const indefinite = additionalInfo === AI_INDEFINITE
+  if (indefinite && (major === MAJOR_UNSIGNED || major === MAJOR_NEGATIVE || major === MAJOR_TAG)) {
+    throw new MalformedCborError(`major type ${major} cannot be indefinite-length`, offset)
+  }
+
+  const afterHeader = offset + 1 + extra
+
+  switch (major) {
+    case MAJOR_UNSIGNED:
+    case MAJOR_NEGATIVE:
+      return afterHeader
+
+    case MAJOR_BYTES:
+    case MAJOR_TEXT: {
+      if (!indefinite) {
+        const length = readArgument(bytes, offset, additionalInfo)
+        const end = afterHeader + length
+        return end > bytes.length ? INCOMPLETE : end
+      }
+      // Indefinite strings are a run of definite-length chunks, then a break.
+      let cursor = afterHeader
+      for (;;) {
+        if (cursor >= bytes.length) return INCOMPLETE
+        if (bytes[cursor] === BREAK) return cursor + 1
+        const next = skipItem(bytes, cursor, false)
+        if (next === INCOMPLETE) return INCOMPLETE
+        cursor = next
+      }
+    }
+
+    case MAJOR_ARRAY:
+    case MAJOR_MAP: {
+      const perEntry = major === MAJOR_MAP ? 2 : 1
+      if (!indefinite) {
+        const count = readArgument(bytes, offset, additionalInfo) * perEntry
+        let cursor = afterHeader
+        for (let i = 0; i < count; i++) {
+          const next = skipItem(bytes, cursor, false)
+          if (next === INCOMPLETE) return INCOMPLETE
+          cursor = next
+        }
+        return cursor
+      }
+      let cursor = afterHeader
+      for (;;) {
+        if (cursor >= bytes.length) return INCOMPLETE
+        if (bytes[cursor] === BREAK) return cursor + 1
+        for (let i = 0; i < perEntry; i++) {
+          const next = skipItem(bytes, cursor, false)
+          if (next === INCOMPLETE) return INCOMPLETE
+          cursor = next
+        }
+      }
+    }
+
+    case MAJOR_TAG:
+      return skipItem(bytes, afterHeader, false)
+
+    case MAJOR_SIMPLE:
+      // Floats and simple values carry no content beyond the header.
+      return afterHeader
+
+    default:
+      throw new MalformedCborError(`unknown major type ${major}`, offset)
+  }
+}
+
+/**
+ * Byte length of the CBOR item beginning at `start`, or {@link INCOMPLETE} if
+ * the buffer ends before the item does.
+ *
+ * @throws {MalformedCborError} if the bytes are not a well-formed item.
+ */
+export function cborItemLength(bytes: Uint8Array, start: number): number | null {
+  const end = skipItem(bytes, start, false)
+  return end === INCOMPLETE ? INCOMPLETE : end - start
+}

package/src/index.ts

@@ -1,32 +1,54 @@
 // Binary .moqtrace format
 
-export type { MoqtraceWriter } from './binary.js'
+export type { MoqtraceWriter, ReadOptions, RecoveredRegion } from './binary.js'
 export {
   createMoqtraceWriter,
+  FORMAT_VERSION,
+  MalformedHeaderError,
   readMoqtrace,
   readMoqtraceHeader,
+  readMoqtraceSegments,
+  SUPPORTED_VERSIONS,
+  TruncatedTraceError,
   writeMoqtrace,
+  writeMoqtraceSegments,
 } from './binary.js'
 // JSON (convenience)
 export { traceToJSON } from './json.js'
-export type { TraceRecorder } from './recorder.js'
+// Incremental reader, for a stream that arrives in pieces
+export type { MoqtraceReader, ReadItem } from './reader.js'
+export { createMoqtraceReader, TruncatedStreamError } from './reader.js'
+export type { ErrorDetails, TraceRecorder } from './recorder.js'
 // Recorder
-export { createRecorder } from './recorder.js'
+export { createRecorder, MAX_ERROR_RAW_BYTES } from './recorder.js'
 
 // Types
 export type {
   AnnotationEvent,
   ControlMessageEvent,
+  DerivationKind,
   DetailLevel,
+  DropPolicy,
+  ErrorKind,
   ObjectHeaderEvent,
   ObjectPayloadEvent,
+  PeerConnectedEvent,
+  PeerDisconnectedEvent,
+  PeerRole,
   Perspective,
   RecorderOptions,
+  SamplingInfo,
+  SegmentInfo,
+  Side,
   StateChangeEvent,
   StreamClosedEvent,
   StreamOpenedEvent,
+  SubscriptionDerivationEvent,
+  SubscriptionRef,
   Trace,
   TraceErrorEvent,
   TraceEvent,
   TraceHeader,
+  UnknownEvent,
 } from './types.js'
+export { controlMessageFields, TRACE_ID_LENGTH } from './types.js'

package/src/json.ts

@@ -7,6 +7,13 @@ import type { Trace } from './types.js'
  * This is a one-way export for human inspection and debugging.
  * For lossless round-tripping, use the binary .moqtrace format
  * (`writeMoqtrace` / `readMoqtrace`).
+ *
+ * The unrecognised-key stores are part of the rendering rather than filtered
+ * out of it: `extra` appears on the header, on `segment`, on `sampling` and on
+ * each event, nested values and all. A view that showed only the keys this
+ * version names would reintroduce, for anyone reading a trace through it,
+ * exactly the loss the binary reader stopped making — and it is the view
+ * someone reaches for precisely when they suspect a key went missing.
  */
 export function traceToJSON(trace: Trace): string {
   return JSON.stringify(

package/src/reader.ts

@@ -0,0 +1,286 @@
+/**
+ * Incremental reader for the `.moqtrace` binary format.
+ *
+ * The three entry points in `binary.ts` all take the whole file at once, which
+ * a live capture does not have: the bytes arrive in chunks whose boundaries
+ * fall wherever the transport put them, almost never on an item boundary. This
+ * reader takes those chunks and hands back the items that have become whole,
+ * keeping the remainder for the next one.
+ *
+ * It is the read side of {@link createMoqtraceWriter}, which has been
+ * incremental from the start — the asymmetry this closes was one-sided.
+ *
+ * Decoding is not reimplemented here. The header and event decoders are
+ * imported from `binary.ts` so that a trace read in instalments and the same
+ * trace read whole produce the same values. A second decoder would be a second
+ * set of answers, and nothing tests two readers of one format against each
+ * other, so the disagreement would surface as a rendering bug somewhere far
+ * from here.
+ */
+
+import {
+  cborToEvent,
+  cborToHeader,
+  decode,
+  MAGIC,
+  PREAMBLE_SIZE,
+  SUPPORTED_VERSIONS,
+  startsWithMagic,
+} from './binary.js'
+import { cborItemLength } from './cbor-scan.js'
+import type { TraceEvent, TraceHeader } from './types.js'
+
+/**
+ * The stream ended part-way through an item.
+ *
+ * Deliberately not `TruncatedTraceError`, whose `segments` field promises
+ * "everything that decoded before the cut". An incremental reader has already
+ * handed those events to its caller and does not keep them, so it cannot fill
+ * that field — and filling it with an empty array would report a trace that
+ * decoded nothing, which is a wrong answer rather than a missing one. The
+ * caller holds what decoded; this says only where the bytes stopped.
+ */
+export class TruncatedStreamError extends Error {
+  /**
+   * Offset into the whole stream at which the incomplete item begins — not
+   * where the bytes ran out. It matches the Rust reader's
+   * `MoqTraceError::Truncated { offset }`, so the same cut is named the same
+   * number in both languages.
+   */
+  readonly offset: number
+
+  /**
+   * How many events reached the caller before the cut.
+   *
+   * The events themselves are gone — that is what makes this error different
+   * from {@link TruncatedTraceError} — but their count is not, and it costs one
+   * integer to keep. It is the difference between "the stream was cut" and "the
+   * stream was cut after 4,812 events at byte 190,224", and only the second
+   * tells whoever is holding a damaged capture how much of it they still have.
+   */
+  readonly events: number
+
+  constructor(offset: number, events: number) {
+    super(
+      `Trace stream truncated: the item starting at byte ${offset} is incomplete, ` +
+        `after ${events} event${events === 1 ? '' : 's'}`,
+    )
+    this.name = 'TruncatedStreamError'
+    this.offset = offset
+    this.events = events
+  }
+}
+
+/**
+ * One thing the reader has finished reading.
+ *
+ * The two kinds mirror the Rust crate's `ReadItem`, so a consumer that has read
+ * one implementation recognises the other. A `segment` arrives before any event
+ * that belongs to it, including the first one: unlike the Rust reader, which
+ * takes a blocking source and parses the opening header in its constructor,
+ * this one cannot have a header until bytes arrive, so every header reaches the
+ * caller the same way and there is no special case for the first.
+ */
+export type ReadItem =
+  | { readonly kind: 'segment'; readonly header: TraceHeader }
+  | { readonly kind: 'event'; readonly event: TraceEvent }
+
+/** Reads a `.moqtrace` stream in whatever instalments it arrives in. */
+export interface MoqtraceReader {
+  /**
+   * Add the next chunk and take everything it completed.
+   *
+   * Returns an empty array when the chunk did not finish an item, which is
+   * ordinary and not a signal of anything. Bytes left over are kept.
+   *
+   * @throws {MalformedHeaderError} if a segment header carries no usable value
+   *   for a key the format requires.
+   * @throws {MalformedCborError} if the bytes at an item boundary are not a
+   *   well-formed CBOR item.
+   */
+  push(chunk: Uint8Array): ReadItem[]
+
+  /**
+   * Declare the stream complete and take any last item.
+   *
+   * This can return items. An event shorter than the magic cannot be told from
+   * the start of a segment preamble while more bytes might still arrive, so
+   * such an event waits here until `end()` rules the preamble out.
+   *
+   * @throws {TruncatedStreamError} if bytes remain that do not form a whole
+   *   item — the file stops mid-event, mid-preamble or mid-header.
+   */
+  end(): ReadItem[]
+
+  /**
+   * The header of the segment now being read, or `undefined` before the first
+   * one is whole. In a segmented stream this changes at every boundary.
+   */
+  readonly header: TraceHeader | undefined
+
+  /** Bytes held back because they do not yet form a whole item. */
+  readonly pending: number
+}
+
+/**
+ * A reader that takes the stream in chunks.
+ *
+ * ```ts
+ * const reader = createMoqtraceReader()
+ * for await (const chunk of stream) {
+ *   for (const item of reader.push(chunk)) {
+ *     if (item.kind === 'event') render(item.event)
+ *   }
+ * }
+ * for (const item of reader.end()) { ... }
+ * ```
+ *
+ * `ReadOptions.recover` has no counterpart here **on purpose.** Its behaviour
+ * in the whole-buffer readers — drop the damaged segment, return the rest, say
+ * nothing — is under revision, because a caller cannot tell three segments from
+ * four with one unreadable. Guessing an answer here would make an incremental
+ * reader the place that shipped it, and a live view is where a silently
+ * dropped segment does the most harm. Until that is settled this reader stops
+ * at damage and says where.
+ */
+export function createMoqtraceReader(): MoqtraceReader {
+  // One buffer rather than a chunk list. After a consume it is `subarray`d, so
+  // the retained bytes are a view and not a copy, and in the steady state the
+  // remainder is smaller than one item — the concatenation on each push costs
+  // that remainder, not the stream.
+  let buffer = new Uint8Array(0)
+  // Where `buffer[0]` sits in the whole stream, so an error names an offset
+  // into the file rather than into whatever is left of it.
+  let consumed = 0
+  // Events handed to the caller, kept only so a truncation can say how much of
+  // the capture survived it.
+  let delivered = 0
+  let current: TraceHeader | undefined
+  let ended = false
+
+  function take(count: number): Uint8Array {
+    const slice = buffer.subarray(0, count)
+    buffer = buffer.subarray(count)
+    consumed += count
+    return slice
+  }
+
+  /**
+   * Read a preamble and its header if all of it is here.
+   *
+   * Returns `undefined` to mean "not yet", which is why the length checks are
+   * not `validatePreamble`'s: that function is written for a buffer that is the
+   * whole file, so it reads a short one as a truncated file and throws. Here a
+   * short buffer is the normal state between two chunks. What it does share is
+   * the part worth sharing — the version check against
+   * {@link SUPPORTED_VERSIONS} and, once the bytes are all present,
+   * `cborToHeader`.
+   */
+  function tryReadSegment(): TraceHeader | undefined {
+    if (buffer.length < PREAMBLE_SIZE) return undefined
+
+    const view = new DataView(buffer.buffer, buffer.byteOffset, buffer.byteLength)
+    const version = view.getUint32(8, true)
+    if (!SUPPORTED_VERSIONS.includes(version)) {
+      throw new Error(
+        `Unsupported format version: ${version} (supported: ${SUPPORTED_VERSIONS.join(', ')})`,
+      )
+    }
+
+    const headerLength = view.getUint32(12, true)
+    if (buffer.length < PREAMBLE_SIZE + headerLength) return undefined
+
+    take(PREAMBLE_SIZE)
+    const headerBytes = take(headerLength)
+    return cborToHeader(decode(headerBytes) as Record<string, unknown>)
+  }
+
+  function drain(atEnd: boolean): ReadItem[] {
+    const items: ReadItem[] = []
+
+    for (;;) {
+      if (current === undefined) {
+        // Nothing has been read yet, so the stream must open with a preamble.
+        // Saying so here means a caller who points this at some other file
+        // learns that from the magic rather than from a CBOR error deep in it.
+        if (buffer.length >= MAGIC.length && !startsWithMagic(buffer, 0)) {
+          throw new Error('Invalid magic bytes: not a .moqtrace file')
+        }
+        const header = tryReadSegment()
+        if (header === undefined) break
+        current = header
+        items.push({ kind: 'segment', header })
+        continue
+      }
+
+      if (buffer.length === 0) break
+
+      // A segment boundary is looked for only here, at a real item boundary —
+      // the same eight bytes inside a captured payload are not one. Deciding
+      // needs the whole magic: with fewer bytes than that in hand, "does not
+      // start with the magic" is not yet a fact about the stream, only about
+      // how much of it has arrived. Waiting is wrong at the end of the stream
+      // and right everywhere else, which is what `atEnd` distinguishes.
+      if (buffer.length < MAGIC.length && !atEnd) break
+
+      if (startsWithMagic(buffer, 0)) {
+        const header = tryReadSegment()
+        if (header === undefined) break
+        current = header
+        items.push({ kind: 'segment', header })
+        continue
+      }
+
+      const length = cborItemLength(buffer, 0)
+      if (length === null) break
+
+      const item = decode(take(length))
+      items.push({ kind: 'event', event: cborToEvent(item as Record<string, unknown>) })
+      delivered += 1
+    }
+
+    return items
+  }
+
+  return {
+    push(chunk: Uint8Array): ReadItem[] {
+      if (ended) throw new Error('Cannot push to a reader that has ended')
+      if (chunk.length > 0) {
+        const grown = new Uint8Array(buffer.length + chunk.length)
+        grown.set(buffer, 0)
+        grown.set(chunk, buffer.length)
+        buffer = grown
+      }
+      return drain(false)
+    },
+
+    end(): ReadItem[] {
+      if (ended) return []
+      ended = true
+      const items = drain(true)
+      // The one place a truncation is declared. `drain` only ever stops at
+      // bytes it cannot use yet, which is the ordinary state between chunks and
+      // a fault only once no more are coming — so it reports nothing and leaves
+      // the leftovers, and the verdict is taken here, where "no more are
+      // coming" is known. Raising it in both places, as an earlier revision
+      // did, gave one fault two throw sites that could drift apart.
+      //
+      // A reader that never produced a header is the same fault: it saw no
+      // preamble at all, which includes the empty stream. Returning no items
+      // for that would report a trace that recorded nothing, when what happened
+      // is that no trace arrived.
+      if (buffer.length > 0 || current === undefined) {
+        throw new TruncatedStreamError(consumed, delivered)
+      }
+      return items
+    },
+
+    get header(): TraceHeader | undefined {
+      return current
+    },
+
+    get pending(): number {
+      return buffer.length
+    },
+  }
+}