@durable-streams/client-conformance-tests 0.1.0

package/src/protocol.ts

@@ -0,0 +1,656 @@
+/**
+ * Protocol types for client conformance testing.
+ *
+ * This module defines the stdin/stdout protocol used for communication
+ * between the test runner and client adapters in any language.
+ *
+ * Communication is line-based JSON over stdin/stdout:
+ * - Test runner writes TestCommand as JSON line to client's stdin
+ * - Client writes TestResult as JSON line to stdout
+ * - Each command expects exactly one result
+ */
+
+// =============================================================================
+// Commands (sent from test runner to client adapter via stdin)
+// =============================================================================
+
+/**
+ * Initialize the client adapter with configuration.
+ * Must be the first command sent.
+ */
+export interface InitCommand {
+  type: `init`
+  /** Base URL of the reference server */
+  serverUrl: string
+  /** Optional timeout in milliseconds for operations */
+  timeoutMs?: number
+}
+
+/**
+ * Create a new stream (PUT request).
+ */
+export interface CreateCommand {
+  type: `create`
+  /** Full URL path for the stream (relative to serverUrl) */
+  path: string
+  /** Content type for the stream */
+  contentType?: string
+  /** Optional TTL in seconds */
+  ttlSeconds?: number
+  /** Optional absolute expiry timestamp (ISO 8601) */
+  expiresAt?: string
+  /** Custom headers to include */
+  headers?: Record<string, string>
+}
+
+/**
+ * Connect to an existing stream without creating it.
+ */
+export interface ConnectCommand {
+  type: `connect`
+  path: string
+  headers?: Record<string, string>
+}
+
+/**
+ * Append data to a stream (POST request).
+ */
+export interface AppendCommand {
+  type: `append`
+  path: string
+  /** Data to append - string for text, base64 for binary */
+  data: string
+  /** Whether data is base64 encoded binary */
+  binary?: boolean
+  /** Optional sequence number for ordering */
+  seq?: number
+  /** Custom headers to include */
+  headers?: Record<string, string>
+}
+
+/**
+ * Read from a stream (GET request).
+ */
+export interface ReadCommand {
+  type: `read`
+  path: string
+  /** Starting offset (opaque string from previous reads) */
+  offset?: string
+  /** Live mode: false for catch-up only, "long-poll" or "sse" for live */
+  live?: false | `long-poll` | `sse`
+  /** Timeout for long-poll in milliseconds */
+  timeoutMs?: number
+  /** Maximum number of chunks to read (for testing) */
+  maxChunks?: number
+  /** Whether to wait until up-to-date before returning */
+  waitForUpToDate?: boolean
+  /** Custom headers to include */
+  headers?: Record<string, string>
+}
+
+/**
+ * Get stream metadata (HEAD request).
+ */
+export interface HeadCommand {
+  type: `head`
+  path: string
+  headers?: Record<string, string>
+}
+
+/**
+ * Delete a stream (DELETE request).
+ */
+export interface DeleteCommand {
+  type: `delete`
+  path: string
+  headers?: Record<string, string>
+}
+
+/**
+ * Shutdown the client adapter gracefully.
+ */
+export interface ShutdownCommand {
+  type: `shutdown`
+}
+
+// =============================================================================
+// Dynamic Headers/Params Commands
+// =============================================================================
+
+/**
+ * Configure a dynamic header that is evaluated per-request.
+ * The adapter should store this and apply it to subsequent operations.
+ *
+ * This tests the client's ability to support header functions for scenarios
+ * like OAuth token refresh, request correlation IDs, etc.
+ */
+export interface SetDynamicHeaderCommand {
+  type: `set-dynamic-header`
+  /** Header name to set */
+  name: string
+  /** Type of dynamic value */
+  valueType: `counter` | `timestamp` | `token`
+  /** Initial value (for token type) */
+  initialValue?: string
+}
+
+/**
+ * Configure a dynamic URL parameter that is evaluated per-request.
+ */
+export interface SetDynamicParamCommand {
+  type: `set-dynamic-param`
+  /** Param name to set */
+  name: string
+  /** Type of dynamic value */
+  valueType: `counter` | `timestamp`
+}
+
+/**
+ * Clear all dynamic headers and params.
+ */
+export interface ClearDynamicCommand {
+  type: `clear-dynamic`
+}
+
+// =============================================================================
+// Benchmark Commands
+// =============================================================================
+
+/**
+ * Execute a timed benchmark operation.
+ * The adapter times the operation internally using high-resolution timing.
+ */
+export interface BenchmarkCommand {
+  type: `benchmark`
+  /** Unique ID for this benchmark iteration */
+  iterationId: string
+  /** The operation to benchmark */
+  operation: BenchmarkOperation
+}
+
+/**
+ * Benchmark operation types - what to measure.
+ */
+export type BenchmarkOperation =
+  | BenchmarkAppendOp
+  | BenchmarkReadOp
+  | BenchmarkRoundtripOp
+  | BenchmarkCreateOp
+  | BenchmarkThroughputAppendOp
+  | BenchmarkThroughputReadOp
+
+export interface BenchmarkAppendOp {
+  op: `append`
+  path: string
+  /** Size in bytes - adapter generates random payload */
+  size: number
+}
+
+export interface BenchmarkReadOp {
+  op: `read`
+  path: string
+  offset?: string
+}
+
+export interface BenchmarkRoundtripOp {
+  op: `roundtrip`
+  path: string
+  /** Size in bytes */
+  size: number
+  /** Live mode for reading */
+  live?: `long-poll` | `sse`
+  /** Content type for SSE compatibility */
+  contentType?: string
+}
+
+export interface BenchmarkCreateOp {
+  op: `create`
+  path: string
+  contentType?: string
+}
+
+export interface BenchmarkThroughputAppendOp {
+  op: `throughput_append`
+  path: string
+  /** Number of messages to send */
+  count: number
+  /** Size per message in bytes */
+  size: number
+  /** Concurrency level */
+  concurrency: number
+}
+
+export interface BenchmarkThroughputReadOp {
+  op: `throughput_read`
+  path: string
+  /** Expected number of JSON messages to read and parse */
+  expectedCount?: number
+}
+
+/**
+ * All possible commands from test runner to client.
+ */
+export type TestCommand =
+  | InitCommand
+  | CreateCommand
+  | ConnectCommand
+  | AppendCommand
+  | ReadCommand
+  | HeadCommand
+  | DeleteCommand
+  | ShutdownCommand
+  | SetDynamicHeaderCommand
+  | SetDynamicParamCommand
+  | ClearDynamicCommand
+  | BenchmarkCommand
+
+// =============================================================================
+// Results (sent from client adapter to test runner via stdout)
+// =============================================================================
+
+/**
+ * Successful initialization result.
+ */
+export interface InitResult {
+  type: `init`
+  success: true
+  /** Client implementation name (e.g., "typescript", "python", "go") */
+  clientName: string
+  /** Client implementation version */
+  clientVersion: string
+  /** Supported features */
+  features?: {
+    /** Supports automatic batching */
+    batching?: boolean
+    /** Supports SSE mode */
+    sse?: boolean
+    /** Supports long-poll mode */
+    longPoll?: boolean
+    /** Supports streaming reads */
+    streaming?: boolean
+    /** Supports dynamic headers/params (functions evaluated per-request) */
+    dynamicHeaders?: boolean
+  }
+}
+
+/**
+ * Successful create result.
+ */
+export interface CreateResult {
+  type: `create`
+  success: true
+  /** HTTP status code received */
+  status: number
+  /** Stream offset after creation */
+  offset?: string
+  /** Response headers of interest */
+  headers?: Record<string, string>
+}
+
+/**
+ * Successful connect result.
+ */
+export interface ConnectResult {
+  type: `connect`
+  success: true
+  status: number
+  offset?: string
+  headers?: Record<string, string>
+}
+
+/**
+ * Successful append result.
+ */
+export interface AppendResult {
+  type: `append`
+  success: true
+  status: number
+  /** New offset after append */
+  offset?: string
+  /** Response headers */
+  headers?: Record<string, string>
+  /** Headers that were sent in the request (for dynamic header testing) */
+  headersSent?: Record<string, string>
+  /** Params that were sent in the request (for dynamic param testing) */
+  paramsSent?: Record<string, string>
+}
+
+/**
+ * A chunk of data read from the stream.
+ */
+export interface ReadChunk {
+  /** Data content - string for text, base64 for binary */
+  data: string
+  /** Whether data is base64 encoded */
+  binary?: boolean
+  /** Offset of this chunk */
+  offset?: string
+}
+
+/**
+ * Successful read result.
+ */
+export interface ReadResult {
+  type: `read`
+  success: true
+  status: number
+  /** Chunks of data read */
+  chunks: Array<ReadChunk>
+  /** Final offset after reading */
+  offset?: string
+  /** Whether stream is up-to-date (caught up to head) */
+  upToDate?: boolean
+  /** Cursor value if provided */
+  cursor?: string
+  /** Response headers */
+  headers?: Record<string, string>
+  /** Headers that were sent in the request (for dynamic header testing) */
+  headersSent?: Record<string, string>
+  /** Params that were sent in the request (for dynamic param testing) */
+  paramsSent?: Record<string, string>
+}
+
+/**
+ * Successful head result.
+ */
+export interface HeadResult {
+  type: `head`
+  success: true
+  status: number
+  /** Current tail offset */
+  offset?: string
+  /** Stream content type */
+  contentType?: string
+  /** TTL remaining in seconds */
+  ttlSeconds?: number
+  /** Absolute expiry (ISO 8601) */
+  expiresAt?: string
+  headers?: Record<string, string>
+}
+
+/**
+ * Successful delete result.
+ */
+export interface DeleteResult {
+  type: `delete`
+  success: true
+  status: number
+  headers?: Record<string, string>
+}
+
+/**
+ * Successful shutdown result.
+ */
+export interface ShutdownResult {
+  type: `shutdown`
+  success: true
+}
+
+/**
+ * Successful set-dynamic-header result.
+ */
+export interface SetDynamicHeaderResult {
+  type: `set-dynamic-header`
+  success: true
+}
+
+/**
+ * Successful set-dynamic-param result.
+ */
+export interface SetDynamicParamResult {
+  type: `set-dynamic-param`
+  success: true
+}
+
+/**
+ * Successful clear-dynamic result.
+ */
+export interface ClearDynamicResult {
+  type: `clear-dynamic`
+  success: true
+}
+
+/**
+ * Successful benchmark result with timing.
+ */
+export interface BenchmarkResult {
+  type: `benchmark`
+  success: true
+  iterationId: string
+  /** Timing in nanoseconds (as string since bigint doesn't JSON serialize) */
+  durationNs: string
+  /** Optional metrics */
+  metrics?: {
+    /** Bytes transferred */
+    bytesTransferred?: number
+    /** Messages processed */
+    messagesProcessed?: number
+    /** Operations per second (for throughput tests) */
+    opsPerSecond?: number
+    /** Bytes per second (for throughput tests) */
+    bytesPerSecond?: number
+  }
+}
+
+/**
+ * Error result for any failed operation.
+ */
+export interface ErrorResult {
+  type: `error`
+  success: false
+  /** Original command type that failed */
+  commandType: TestCommand[`type`]
+  /** HTTP status code if available */
+  status?: number
+  /** Error code (e.g., "NETWORK_ERROR", "TIMEOUT", "CONFLICT") */
+  errorCode: string
+  /** Human-readable error message */
+  message: string
+  /** Additional error details */
+  details?: Record<string, unknown>
+}
+
+/**
+ * All possible results from client to test runner.
+ */
+export type TestResult =
+  | InitResult
+  | CreateResult
+  | ConnectResult
+  | AppendResult
+  | ReadResult
+  | HeadResult
+  | DeleteResult
+  | ShutdownResult
+  | SetDynamicHeaderResult
+  | SetDynamicParamResult
+  | ClearDynamicResult
+  | BenchmarkResult
+  | ErrorResult
+
+// =============================================================================
+// Utilities
+// =============================================================================
+
+/**
+ * Parse a JSON line into a TestCommand.
+ */
+export function parseCommand(line: string): TestCommand {
+  return JSON.parse(line) as TestCommand
+}
+
+/**
+ * Serialize a TestResult to a JSON line.
+ */
+export function serializeResult(result: TestResult): string {
+  return JSON.stringify(result)
+}
+
+/**
+ * Parse a JSON line into a TestResult.
+ */
+export function parseResult(line: string): TestResult {
+  return JSON.parse(line) as TestResult
+}
+
+/**
+ * Serialize a TestCommand to a JSON line.
+ */
+export function serializeCommand(command: TestCommand): string {
+  return JSON.stringify(command)
+}
+
+/**
+ * Encode binary data to base64 for transmission.
+ */
+export function encodeBase64(data: Uint8Array): string {
+  return Buffer.from(data).toString(`base64`)
+}
+
+/**
+ * Decode base64 string back to binary data.
+ */
+export function decodeBase64(encoded: string): Uint8Array {
+  return new Uint8Array(Buffer.from(encoded, `base64`))
+}
+
+/**
+ * Standard error codes for ErrorResult.
+ */
+export const ErrorCodes = {
+  /** Network connection failed */
+  NETWORK_ERROR: `NETWORK_ERROR`,
+  /** Operation timed out */
+  TIMEOUT: `TIMEOUT`,
+  /** Stream already exists (409 Conflict) */
+  CONFLICT: `CONFLICT`,
+  /** Stream not found (404) */
+  NOT_FOUND: `NOT_FOUND`,
+  /** Sequence number conflict (409) */
+  SEQUENCE_CONFLICT: `SEQUENCE_CONFLICT`,
+  /** Invalid offset format */
+  INVALID_OFFSET: `INVALID_OFFSET`,
+  /** Server returned unexpected status */
+  UNEXPECTED_STATUS: `UNEXPECTED_STATUS`,
+  /** Failed to parse response */
+  PARSE_ERROR: `PARSE_ERROR`,
+  /** Client internal error */
+  INTERNAL_ERROR: `INTERNAL_ERROR`,
+  /** Operation not supported by this client */
+  NOT_SUPPORTED: `NOT_SUPPORTED`,
+} as const
+
+export type ErrorCode = (typeof ErrorCodes)[keyof typeof ErrorCodes]
+
+// =============================================================================
+// Benchmark Statistics
+// =============================================================================
+
+/**
+ * Statistical summary of benchmark results.
+ */
+export interface BenchmarkStats {
+  /** Minimum value in milliseconds */
+  min: number
+  /** Maximum value in milliseconds */
+  max: number
+  /** Arithmetic mean in milliseconds */
+  mean: number
+  /** Median (p50) in milliseconds */
+  median: number
+  /** 75th percentile in milliseconds */
+  p75: number
+  /** 95th percentile in milliseconds */
+  p95: number
+  /** 99th percentile in milliseconds */
+  p99: number
+  /** Standard deviation in milliseconds */
+  stdDev: number
+  /** Margin of error (95% confidence) in milliseconds */
+  marginOfError: number
+  /** Number of samples */
+  sampleCount: number
+}
+
+/**
+ * Calculate statistics from an array of durations in nanoseconds.
+ */
+export function calculateStats(durationsNs: Array<bigint>): BenchmarkStats {
+  if (durationsNs.length === 0) {
+    return {
+      min: 0,
+      max: 0,
+      mean: 0,
+      median: 0,
+      p75: 0,
+      p95: 0,
+      p99: 0,
+      stdDev: 0,
+      marginOfError: 0,
+      sampleCount: 0,
+    }
+  }
+
+  // Convert to milliseconds for statistics
+  const samplesMs = durationsNs.map((ns) => Number(ns) / 1_000_000)
+  const sorted = [...samplesMs].sort((a, b) => a - b)
+  const n = sorted.length
+
+  const min = sorted[0]!
+  const max = sorted[n - 1]!
+  const mean = samplesMs.reduce((a, b) => a + b, 0) / n
+
+  // Percentiles (nearest rank method, 0-based indexing)
+  const percentile = (p: number) => {
+    const idx = Math.floor((n - 1) * p)
+    return sorted[idx]!
+  }
+
+  const median = percentile(0.5)
+  const p75 = percentile(0.75)
+  const p95 = percentile(0.95)
+  const p99 = percentile(0.99)
+
+  // Standard deviation
+  const squaredDiffs = samplesMs.map((v) => Math.pow(v - mean, 2))
+  const variance = squaredDiffs.reduce((a, b) => a + b, 0) / n
+  const stdDev = Math.sqrt(variance)
+
+  // Margin of error (95% confidence, z = 1.96)
+  const marginOfError = (1.96 * stdDev) / Math.sqrt(n)
+
+  return {
+    min,
+    max,
+    mean,
+    median,
+    p75,
+    p95,
+    p99,
+    stdDev,
+    marginOfError,
+    sampleCount: n,
+  }
+}
+
+/**
+ * Format a BenchmarkStats object for display.
+ */
+export function formatStats(
+  stats: BenchmarkStats,
+  unit = `ms`
+): Record<string, string> {
+  const fmt = (v: number) => `${v.toFixed(2)} ${unit}`
+  return {
+    Min: fmt(stats.min),
+    Max: fmt(stats.max),
+    Mean: fmt(stats.mean),
+    Median: fmt(stats.median),
+    P75: fmt(stats.p75),
+    P95: fmt(stats.p95),
+    P99: fmt(stats.p99),
+    StdDev: fmt(stats.stdDev),
+    "Margin of Error": fmt(stats.marginOfError),
+    Samples: stats.sampleCount.toString(),
+  }
+}