@moqtap/trace 0.1.3 → 0.3.0

package/src/cbor-scan.ts

@@ -0,0 +1,185 @@
+/**
+ * Measure one CBOR item without decoding it.
+ *
+ * A `.moqtrace` file is a CBOR sequence, and a segmented one splices a
+ * preamble between two of its items. Finding those boundaries means knowing
+ * where each item ends — and knowing it *only* at real item boundaries. A
+ * plain byte scan for the magic would also match the same eight bytes sitting
+ * inside a captured payload, and would split a trace that was never segmented.
+ *
+ * Measuring is cheaper than decoding: a byte or text string is skipped by its
+ * declared length rather than read, so the cost follows the structure of the
+ * data and not its size.
+ */
+
+/** The item at the given offset ran past the end of the buffer. */
+export const INCOMPLETE = null
+
+/** Thrown when the bytes are not a well-formed CBOR item at all. */
+export class MalformedCborError extends Error {
+  readonly offset: number
+
+  constructor(message: string, offset: number) {
+    super(`${message} at byte ${offset}`)
+    this.name = 'MalformedCborError'
+    this.offset = offset
+  }
+}
+
+const MAJOR_UNSIGNED = 0
+const MAJOR_NEGATIVE = 1
+const MAJOR_BYTES = 2
+const MAJOR_TEXT = 3
+const MAJOR_ARRAY = 4
+const MAJOR_MAP = 5
+const MAJOR_TAG = 6
+const MAJOR_SIMPLE = 7
+
+const AI_ONE_BYTE = 24
+const AI_TWO_BYTES = 25
+const AI_FOUR_BYTES = 26
+const AI_EIGHT_BYTES = 27
+const AI_INDEFINITE = 31
+const BREAK = 0xff
+
+/** Bytes that follow the initial byte for each additional-information value. */
+function headerExtraBytes(additionalInfo: number, offset: number): number {
+  if (additionalInfo < AI_ONE_BYTE) return 0
+  switch (additionalInfo) {
+    case AI_ONE_BYTE:
+      return 1
+    case AI_TWO_BYTES:
+      return 2
+    case AI_FOUR_BYTES:
+      return 4
+    case AI_EIGHT_BYTES:
+      return 8
+    case AI_INDEFINITE:
+      return 0
+    default:
+      throw new MalformedCborError(`reserved additional information ${additionalInfo}`, offset)
+  }
+}
+
+/** Read the argument encoded in the item header. */
+function readArgument(bytes: Uint8Array, offset: number, additionalInfo: number): number {
+  if (additionalInfo < AI_ONE_BYTE) return additionalInfo
+  const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength)
+  switch (additionalInfo) {
+    case AI_ONE_BYTE:
+      return view.getUint8(offset + 1)
+    case AI_TWO_BYTES:
+      return view.getUint16(offset + 1, false)
+    case AI_FOUR_BYTES:
+      return view.getUint32(offset + 1, false)
+    case AI_EIGHT_BYTES: {
+      const value = view.getBigUint64(offset + 1, false)
+      // A length this large cannot be indexed anyway; treating it as a number
+      // is safe because the caller only ever compares it against the buffer.
+      return Number(value)
+    }
+    default:
+      throw new MalformedCborError(`reserved additional information ${additionalInfo}`, offset)
+  }
+}
+
+/**
+ * Advance past the item at `offset`, returning the offset just after it, or
+ * {@link INCOMPLETE} if the buffer ends part-way through.
+ *
+ * @param allowBreak whether a break code is a legal item here — true only
+ *   directly inside an indefinite-length container.
+ */
+function skipItem(bytes: Uint8Array, offset: number, allowBreak: boolean): number | null {
+  if (offset >= bytes.length) return INCOMPLETE
+
+  const initial = bytes[offset] as number
+  if (initial === BREAK) {
+    if (!allowBreak) {
+      throw new MalformedCborError('break outside an indefinite-length item', offset)
+    }
+    return offset + 1
+  }
+
+  const major = initial >> 5
+  const additionalInfo = initial & 0x1f
+  const extra = headerExtraBytes(additionalInfo, offset)
+  if (offset + 1 + extra > bytes.length) return INCOMPLETE
+
+  const indefinite = additionalInfo === AI_INDEFINITE
+  if (indefinite && (major === MAJOR_UNSIGNED || major === MAJOR_NEGATIVE || major === MAJOR_TAG)) {
+    throw new MalformedCborError(`major type ${major} cannot be indefinite-length`, offset)
+  }
+
+  const afterHeader = offset + 1 + extra
+
+  switch (major) {
+    case MAJOR_UNSIGNED:
+    case MAJOR_NEGATIVE:
+      return afterHeader
+
+    case MAJOR_BYTES:
+    case MAJOR_TEXT: {
+      if (!indefinite) {
+        const length = readArgument(bytes, offset, additionalInfo)
+        const end = afterHeader + length
+        return end > bytes.length ? INCOMPLETE : end
+      }
+      // Indefinite strings are a run of definite-length chunks, then a break.
+      let cursor = afterHeader
+      for (;;) {
+        if (cursor >= bytes.length) return INCOMPLETE
+        if (bytes[cursor] === BREAK) return cursor + 1
+        const next = skipItem(bytes, cursor, false)
+        if (next === INCOMPLETE) return INCOMPLETE
+        cursor = next
+      }
+    }
+
+    case MAJOR_ARRAY:
+    case MAJOR_MAP: {
+      const perEntry = major === MAJOR_MAP ? 2 : 1
+      if (!indefinite) {
+        const count = readArgument(bytes, offset, additionalInfo) * perEntry
+        let cursor = afterHeader
+        for (let i = 0; i < count; i++) {
+          const next = skipItem(bytes, cursor, false)
+          if (next === INCOMPLETE) return INCOMPLETE
+          cursor = next
+        }
+        return cursor
+      }
+      let cursor = afterHeader
+      for (;;) {
+        if (cursor >= bytes.length) return INCOMPLETE
+        if (bytes[cursor] === BREAK) return cursor + 1
+        for (let i = 0; i < perEntry; i++) {
+          const next = skipItem(bytes, cursor, false)
+          if (next === INCOMPLETE) return INCOMPLETE
+          cursor = next
+        }
+      }
+    }
+
+    case MAJOR_TAG:
+      return skipItem(bytes, afterHeader, false)
+
+    case MAJOR_SIMPLE:
+      // Floats and simple values carry no content beyond the header.
+      return afterHeader
+
+    default:
+      throw new MalformedCborError(`unknown major type ${major}`, offset)
+  }
+}
+
+/**
+ * Byte length of the CBOR item beginning at `start`, or {@link INCOMPLETE} if
+ * the buffer ends before the item does.
+ *
+ * @throws {MalformedCborError} if the bytes are not a well-formed item.
+ */
+export function cborItemLength(bytes: Uint8Array, start: number): number | null {
+  const end = skipItem(bytes, start, false)
+  return end === INCOMPLETE ? INCOMPLETE : end - start
+}

package/src/index.ts

@@ -1,11 +1,16 @@
 // Binary .moqtrace format
 
-export type { MoqtraceWriter } from './binary.js'
+export type { MoqtraceWriter, ReadOptions } from './binary.js'
 export {
   createMoqtraceWriter,
+  FORMAT_VERSION,
   readMoqtrace,
   readMoqtraceHeader,
+  readMoqtraceSegments,
+  SUPPORTED_VERSIONS,
+  TruncatedTraceError,
   writeMoqtrace,
+  writeMoqtraceSegments,
 } from './binary.js'
 // JSON (convenience)
 export { traceToJSON } from './json.js'
@@ -17,16 +22,28 @@ export { createRecorder } from './recorder.js'
 export type {
   AnnotationEvent,
   ControlMessageEvent,
+  DerivationKind,
   DetailLevel,
+  DropPolicy,
   ObjectHeaderEvent,
   ObjectPayloadEvent,
+  PeerConnectedEvent,
+  PeerDisconnectedEvent,
+  PeerRole,
   Perspective,
   RecorderOptions,
+  SamplingInfo,
+  SegmentInfo,
+  Side,
   StateChangeEvent,
   StreamClosedEvent,
   StreamOpenedEvent,
+  SubscriptionDerivationEvent,
+  SubscriptionRef,
   Trace,
   TraceErrorEvent,
   TraceEvent,
   TraceHeader,
+  UnknownEvent,
 } from './types.js'
+export { TRACE_ID_LENGTH } from './types.js'

package/src/recorder.ts

@@ -58,6 +58,16 @@ const DETAIL_RANK: Record<DetailLevel, number> = {
 export function createRecorder(options: RecorderOptions): TraceRecorder {
   const detail = options.detail
   const detailRank = DETAIL_RANK[detail]
+  // A reader must tolerate a detail level it does not know; a recorder cannot.
+  // Asked for a level this build does not implement, it would either capture
+  // less than the caller believes — writing a header that claims a detail the
+  // events do not have — or capture more, which for the payload-bearing
+  // levels is a privacy failure. Neither is worth a trace, so refuse instead.
+  if (detailRank === undefined) {
+    throw new Error(
+      `Unknown detail level '${detail}': cannot record at a level this build does not implement`,
+    )
+  }
   const maxEvents = options.maxEvents ?? 100_000
   const clock = options.clock ?? (() => Math.round(performance.now() * 1000))
   const messageTypeId = options.messageTypeId ?? (() => 0)

package/src/types.ts

@@ -1,22 +1,110 @@
-export type DetailLevel = 'control' | 'headers' | 'headers+sizes' | 'headers+data' | 'full'
-export type Perspective = 'client' | 'server' | 'observer'
+/**
+ * Any string other than the ones named beside it.
+ *
+ * Unions built with this keep editor completion for the values this version
+ * knows, while still accepting a value it does not. Nothing in the format
+ * rejects an unrecognised string: new perspectives, detail levels and drop
+ * policies may be added without a version bump, and a reader that refused one
+ * would refuse a file it can otherwise read in full.
+ */
+type OtherString = string & Record<never, never>
+
+export type DetailLevel =
+  | 'control'
+  | 'headers'
+  | 'headers+sizes'
+  | 'headers+data'
+  | 'full'
+  | OtherString
+
+export type Perspective = 'client' | 'server' | 'observer' | 'relay-tap' | OtherString
+
+/** Drop strategy a sampled source applied when it could not keep up. */
+export type DropPolicy = 'head' | 'tail' | 'sampled' | OtherString
+
+/** Length of a trace ID, in bytes. Fixed by the format. */
+export const TRACE_ID_LENGTH = 16
+
+/**
+ * Per-segment metadata.
+ *
+ * Present only in segmented traces. A single-shot file must not carry it —
+ * its absence is what tells a reader that sequence numbers and timestamps are
+ * file-global rather than segment-local.
+ */
+export interface SegmentInfo {
+  /** 0-based sequence number of this segment within the stream. */
+  readonly sequence: number
+  /** Nominal segment duration in milliseconds. A hint; the real one may differ. */
+  readonly durationMs?: number
+  /** Opaque identifier shared by every segment of the same logical stream. */
+  readonly streamId?: string
+  /** True if this segment continues a previous one with the same `streamId`. */
+  readonly continues?: boolean
+}
+
+/**
+ * Sampling and filtering metadata. Present only when events were dropped or
+ * filtered at the source; its absence means the trace is complete relative to
+ * its detail level.
+ */
+export interface SamplingInfo {
+  /** Fraction of source events retained, in (0, 1]. */
+  readonly effectiveRate?: number
+  /** Per-segment cap that triggered drops, if rate-limited. */
+  readonly maxEventsPerSec?: number
+  /** Drop strategy applied when the cap was exceeded. */
+  readonly dropPolicy?: DropPolicy
+  /** Cumulative events dropped since `startTime`. */
+  readonly droppedTotal?: number
+  /** Events dropped within this segment only. */
+  readonly droppedSegment?: number
+  /** Source-side filter rule that selected events. */
+  readonly rule?: string
+  /** Language the rule is written in ('prefix', 'glob', 'cel'). */
+  readonly ruleLang?: string
+  /** Event type IDs the drop policy was applied to. */
+  readonly appliesTo?: number[]
+}
 
 export interface TraceHeader {
   readonly protocol: string
   readonly perspective: Perspective
   readonly detail: DetailLevel
+  /**
+   * Recording start (Unix epoch milliseconds). In a segmented trace this is
+   * the segment's start, not the stream's.
+   */
   readonly startTime: number
   readonly endTime?: number
   readonly transport?: string
   readonly source?: string
   readonly endpoint?: string
   readonly sessionId?: string
+  /** Present only when this header begins one segment of a segmented stream. */
+  readonly segment?: SegmentInfo
+  /** Present only when events were dropped or filtered at the source. */
+  readonly sampling?: SamplingInfo
   readonly custom?: Record<string, unknown>
 }
 
 interface BaseEvent {
+  /**
+   * Monotonically increasing sequence number. Segment-local in a segmented
+   * trace, so ordering across segments is `(segment.sequence, seq)`.
+   */
   readonly seq: number
+  /** Microseconds since the containing segment's `startTime`. */
   readonly timestamp: number
+  /**
+   * Which peer this event pertains to.
+   *
+   * Required when the perspective is `'relay-tap'`, where one trace covers
+   * many concurrent sessions; omitted otherwise. The identifier is
+   * source-local: the same string in two traces from different sources does
+   * not name the same peer.
+   */
+  readonly peer?: string
 }
 
 export interface ControlMessageEvent extends BaseEvent {
@@ -25,6 +113,15 @@ export interface ControlMessageEvent extends BaseEvent {
   readonly messageType: number
   readonly message: Record<string, unknown>
   readonly raw?: Uint8Array
+  /**
+   * QUIC stream the message travelled on.
+   *
+   * Optional, and only meaningful for a recorder that knows it. It matters
+   * from draft-17, where the control plane is no longer one stream: each
+   * request gets its own bidirectional stream and responses carry no request
+   * id, so the stream is the only thing tying a response to its request.
+   */
+  readonly streamId?: bigint
 }
 
 export interface StreamOpenedEvent extends BaseEvent {
@@ -76,6 +173,110 @@ export interface AnnotationEvent extends BaseEvent {
   readonly data: unknown
 }
 
+/** Role a peer is acting in at connection time. */
+export type PeerRole = 'publisher' | 'subscriber' | 'both' | OtherString
+
+/**
+ * Which end of a relay a peer sits on.
+ *
+ * This is the direction the connection was made in, not the direction
+ * subscriptions flow. For subscription causality see
+ * {@link SubscriptionDerivationEvent}.
+ */
+export type Side = 'downstream' | 'upstream' | OtherString
+
+/**
+ * A new peer session was established. Emitted only by `'relay-tap'` sources;
+ * the peer it identifies is on the event's `peer` field.
+ */
+export interface PeerConnectedEvent extends BaseEvent {
+  readonly type: 'peer-connected'
+  readonly endpoint?: string
+  readonly transport?: string
+  readonly role?: PeerRole
+  readonly side?: Side
+}
+
+/** A peer session ended. Emitted only by `'relay-tap'` sources. */
+export interface PeerDisconnectedEvent extends BaseEvent {
+  readonly type: 'peer-disconnected'
+  readonly errorCode: number
+  readonly reason?: string
+}
+
+/**
+ * A `(peer, request id)` pair naming one subscription on one peer.
+ *
+ * Request ids are unique only within a peer's session, so neither half
+ * identifies a subscription on its own.
+ */
+export interface SubscriptionRef {
+  readonly peer: string
+  readonly requestId: bigint
+}
+
+/** How an upstream subscription came to serve a downstream one. */
+export type DerivationKind = 'created' | 'shared' | OtherString
+
+/**
+ * An upstream subscription was created or extended in causal response to
+ * downstream ones. Emitted only by `'relay-tap'` sources.
+ *
+ * This is the primitive multi-hop correlation is built from: a collector
+ * reconstructs an end-to-end tree from these links plus the trace ids
+ * propagated along them.
+ *
+ * The four timestamps share the `timestamp` field's timebase — the emitting
+ * source's own clock. Differences between them are meaningful with no
+ * cross-hop clock agreement, which is the point: they measure how long one
+ * hop took. Never subtract one source's timestamp from another's.
+ *
+ * A source emits the event as soon as the downstream SUBSCRIBE arrives,
+ * carrying whichever timestamps it has, and may emit it again for the same
+ * pair as the rest arrive. Treat a later event for the same pair as an update
+ * to the earlier one, not as a second derivation.
+ */
+export interface SubscriptionDerivationEvent extends BaseEvent {
+  readonly type: 'subscription-derivation'
+  readonly upstream: SubscriptionRef
+  /** Every downstream subscription currently served by `upstream`. */
+  readonly downstream: SubscriptionRef[]
+  readonly kind: DerivationKind
+  /**
+   * Trace id propagated along the subscription chain: exactly
+   * {@link TRACE_ID_LENGTH} raw bytes, never hex- or base64-encoded, so two
+   * implementations produce identical values for the same chain.
+   */
+  readonly traceId?: Uint8Array
+  /** Track namespace the subscription targets, one entry per field. */
+  readonly namespace?: Uint8Array[]
+  readonly trackName?: Uint8Array
+  /** When the downstream SUBSCRIBE was received. */
+  readonly tDownstreamReceived?: number
+  /** When the upstream SUBSCRIBE was sent. Absent for a terminal subscription. */
+  readonly tUpstreamSent?: number
+  /** When SUBSCRIBE_OK arrived from upstream. */
+  readonly tUpstreamOkReceived?: number
+  /** When SUBSCRIBE_OK was sent downstream. */
+  readonly tDownstreamOkSent?: number
+}
+
+/**
+ * An event whose type this version of the package does not know.
+ *
+ * New event types may be added without a format version bump, so a reader
+ * that dropped or relabelled them would make one tool's ignorance permanent
+ * for everything downstream of it. The fields are kept verbatim, so an
+ * unknown event survives a read-modify-write round trip intact.
+ */
+export interface UnknownEvent extends BaseEvent {
+  readonly type: 'unknown'
+  /** The `"e"` discriminant that was read. */
+  readonly eventType: number
+  /** Every key from the event map other than `n`, `t`, `p` and `e`. */
+  readonly fields: Record<string, unknown>
+}
+
 export type TraceEvent =
   | ControlMessageEvent
   | StreamOpenedEvent
@@ -85,6 +286,10 @@ export type TraceEvent =
   | StateChangeEvent
   | TraceErrorEvent
   | AnnotationEvent
+  | PeerConnectedEvent
+  | PeerDisconnectedEvent
+  | SubscriptionDerivationEvent
+  | UnknownEvent
 
 export interface Trace {
   readonly header: TraceHeader