@durable-streams/client-conformance-tests 0.1.0

package/src/benchmark-scenarios.ts

@@ -0,0 +1,311 @@
+/**
+ * Benchmark scenario definitions.
+ *
+ * Each scenario defines what to measure, how many iterations,
+ * and success criteria for the benchmark.
+ */
+
+import type { BenchmarkOperation } from "./protocol.js"
+
+// =============================================================================
+// Types
+// =============================================================================
+
+export interface BenchmarkScenario {
+  /** Unique scenario ID */
+  id: string
+  /** Human-readable name */
+  name: string
+  /** Description */
+  description: string
+  /** Category for grouping */
+  category: `latency` | `throughput` | `streaming`
+  /** Required client features */
+  requires?: Array<`batching` | `sse` | `longPoll` | `streaming`>
+  /** Scenario configuration */
+  config: BenchmarkScenarioConfig
+  /** Success criteria */
+  criteria?: BenchmarkCriteria
+  /** Factory to create benchmark operations for each iteration */
+  createOperation: (ctx: ScenarioContext) => BenchmarkOperation
+  /** Optional setup before running iterations */
+  setup?: (ctx: ScenarioContext) => Promise<SetupResult>
+  /** Optional cleanup after running iterations */
+  cleanup?: (ctx: ScenarioContext) => Promise<void>
+}
+
+export interface BenchmarkScenarioConfig {
+  /** Number of warmup iterations (not measured) */
+  warmupIterations: number
+  /** Number of measured iterations */
+  measureIterations: number
+  /** Message size in bytes (where applicable) */
+  messageSize: number
+  /** Concurrency level (for throughput tests) */
+  concurrency?: number
+}
+
+export interface BenchmarkCriteria {
+  /** Maximum acceptable p50 latency in ms */
+  maxP50Ms?: number
+  /** Maximum acceptable p99 latency in ms */
+  maxP99Ms?: number
+  /** Minimum throughput in operations/second */
+  minOpsPerSecond?: number
+  /** Minimum throughput in MB/second */
+  minMBPerSecond?: number
+}
+
+export interface ScenarioContext {
+  /** Base path for streams (unique per scenario run) */
+  basePath: string
+  /** Current iteration number (0-based) */
+  iteration: number
+  /** Stored values from setup */
+  setupData: Record<string, unknown>
+}
+
+export interface SetupResult {
+  /** Data to pass to each iteration */
+  data?: Record<string, unknown>
+}
+
+// =============================================================================
+// Latency Scenarios
+// =============================================================================
+
+export const appendLatencyScenario: BenchmarkScenario = {
+  id: `latency-append`,
+  name: `Append Latency`,
+  description: `Measure time to complete a single append operation`,
+  category: `latency`,
+  config: {
+    warmupIterations: 10,
+    measureIterations: 100,
+    messageSize: 100, // 100 bytes
+  },
+  criteria: {
+    maxP50Ms: 20,
+    maxP99Ms: 100,
+  },
+  createOperation: (ctx) => ({
+    op: `append`,
+    path: `${ctx.basePath}/stream`,
+    size: 100,
+  }),
+  setup: (ctx) => {
+    ctx.setupData.streamPath = `${ctx.basePath}/stream`
+    return Promise.resolve({})
+  },
+}
+
+export const readLatencyScenario: BenchmarkScenario = {
+  id: `latency-read`,
+  name: `Read Latency`,
+  description: `Measure time to complete a single read operation`,
+  category: `latency`,
+  config: {
+    warmupIterations: 10,
+    measureIterations: 100,
+    messageSize: 100,
+  },
+  criteria: {
+    maxP50Ms: 20,
+    maxP99Ms: 100,
+  },
+  createOperation: (ctx) => ({
+    op: `read`,
+    path: `${ctx.basePath}/stream`,
+    offset: ctx.setupData.offset as string | undefined,
+  }),
+  setup: (ctx) => {
+    ctx.setupData.streamPath = `${ctx.basePath}/stream`
+    return Promise.resolve({})
+  },
+}
+
+export const roundtripLatencyScenario: BenchmarkScenario = {
+  id: `latency-roundtrip`,
+  name: `Roundtrip Latency`,
+  description: `Measure time to append and immediately read back via long-poll`,
+  category: `latency`,
+  requires: [`longPoll`],
+  config: {
+    warmupIterations: 5,
+    measureIterations: 50,
+    messageSize: 100,
+  },
+  criteria: {
+    maxP50Ms: 50,
+    maxP99Ms: 200,
+  },
+  createOperation: (ctx) => ({
+    op: `roundtrip`,
+    path: `${ctx.basePath}/roundtrip-${ctx.iteration}`,
+    size: 100,
+    live: `long-poll`,
+  }),
+}
+
+export const createLatencyScenario: BenchmarkScenario = {
+  id: `latency-create`,
+  name: `Create Latency`,
+  description: `Measure time to create a new stream`,
+  category: `latency`,
+  config: {
+    warmupIterations: 5,
+    measureIterations: 50,
+    messageSize: 0,
+  },
+  criteria: {
+    maxP50Ms: 30,
+    maxP99Ms: 150,
+  },
+  createOperation: (ctx) => ({
+    op: `create`,
+    path: `${ctx.basePath}/create-${ctx.iteration}`,
+    contentType: `application/octet-stream`,
+  }),
+}
+
+// =============================================================================
+// Throughput Scenarios
+// =============================================================================
+
+export const smallMessageThroughputScenario: BenchmarkScenario = {
+  id: `throughput-small-messages`,
+  name: `Small Message Throughput`,
+  description: `Measure throughput for 100-byte messages at high concurrency`,
+  category: `throughput`,
+  requires: [`batching`],
+  config: {
+    warmupIterations: 2,
+    measureIterations: 10,
+    messageSize: 100,
+    concurrency: 200,
+  },
+  criteria: {
+    minOpsPerSecond: 1000,
+  },
+  createOperation: (ctx) => ({
+    op: `throughput_append`,
+    path: `${ctx.basePath}/throughput-small`,
+    count: 10000,
+    size: 100,
+    concurrency: 200,
+  }),
+}
+
+export const largeMessageThroughputScenario: BenchmarkScenario = {
+  id: `throughput-large-messages`,
+  name: `Large Message Throughput`,
+  description: `Measure throughput for 1MB messages`,
+  category: `throughput`,
+  requires: [`batching`],
+  config: {
+    warmupIterations: 1,
+    measureIterations: 5,
+    messageSize: 1024 * 1024, // 1MB
+    concurrency: 10,
+  },
+  criteria: {
+    minOpsPerSecond: 20,
+  },
+  createOperation: (ctx) => ({
+    op: `throughput_append`,
+    path: `${ctx.basePath}/throughput-large`,
+    count: 50,
+    size: 1024 * 1024,
+    concurrency: 10,
+  }),
+}
+
+export const readThroughputScenario: BenchmarkScenario = {
+  id: `throughput-read`,
+  name: `Read Throughput`,
+  description: `Measure JSON parsing and iteration speed reading back messages`,
+  category: `throughput`,
+  config: {
+    warmupIterations: 1,
+    measureIterations: 5,
+    messageSize: 100, // ~100 bytes per JSON message
+  },
+  criteria: {
+    minMBPerSecond: 3, // Python is slower, so use lower threshold
+  },
+  createOperation: (ctx) => ({
+    op: `throughput_read`,
+    path: `${ctx.basePath}/throughput-read`,
+    expectedCount: ctx.setupData.expectedCount as number | undefined,
+  }),
+  setup: (ctx) => {
+    // Expecting 10000 JSON messages to be pre-populated
+    ctx.setupData.expectedCount = 10000
+    return Promise.resolve({ data: { expectedCount: 10000 } })
+  },
+}
+
+// =============================================================================
+// Streaming Scenarios
+// =============================================================================
+
+export const sseLatencyScenario: BenchmarkScenario = {
+  id: `streaming-sse-latency`,
+  name: `SSE First Event Latency`,
+  description: `Measure time to receive first event via SSE`,
+  category: `streaming`,
+  requires: [`sse`],
+  config: {
+    warmupIterations: 3,
+    measureIterations: 20,
+    messageSize: 100,
+  },
+  criteria: {
+    maxP50Ms: 100,
+    maxP99Ms: 500,
+  },
+  createOperation: (ctx) => ({
+    op: `roundtrip`,
+    path: `${ctx.basePath}/sse-latency-${ctx.iteration}`,
+    size: 100,
+    live: `sse`,
+    contentType: `application/json`, // SSE requires JSON-compatible content type
+  }),
+}
+
+// =============================================================================
+// All Scenarios
+// =============================================================================
+
+export const allScenarios: Array<BenchmarkScenario> = [
+  // Latency
+  appendLatencyScenario,
+  readLatencyScenario,
+  roundtripLatencyScenario,
+  createLatencyScenario,
+  // Throughput
+  smallMessageThroughputScenario,
+  largeMessageThroughputScenario,
+  readThroughputScenario,
+  // Streaming
+  sseLatencyScenario,
+]
+
+export const scenariosByCategory: Record<
+  `latency` | `throughput` | `streaming`,
+  Array<BenchmarkScenario>
+> = {
+  latency: allScenarios.filter((s) => s.category === `latency`),
+  throughput: allScenarios.filter((s) => s.category === `throughput`),
+  streaming: allScenarios.filter((s) => s.category === `streaming`),
+}
+
+export function getScenarioById(id: string): BenchmarkScenario | undefined {
+  return allScenarios.find((s) => s.id === id)
+}
+
+export function getScenariosByCategory(
+  category: `latency` | `throughput` | `streaming`
+): Array<BenchmarkScenario> {
+  return scenariosByCategory[category]
+}

package/src/cli.ts

@@ -0,0 +1,294 @@
+#!/usr/bin/env node
+/**
+ * CLI for running client conformance tests and benchmarks.
+ *
+ * Usage:
+ *   npx @durable-streams/client-conformance-tests --run ts
+ *   npx @durable-streams/client-conformance-tests --run ./my-python-client
+ *   npx @durable-streams/client-conformance-tests --run ./client --suite producer
+ *   npx @durable-streams/client-conformance-tests --bench ts
+ */
+
+import { runConformanceTests } from "./runner.js"
+import { runBenchmarks } from "./benchmark-runner.js"
+import type { RunnerOptions } from "./runner.js"
+import type { BenchmarkRunnerOptions } from "./benchmark-runner.js"
+
+const HELP = `
+Durable Streams Client Conformance Test Suite
+
+Usage:
+  npx @durable-streams/client-conformance-tests --run <adapter> [options]
+  npx @durable-streams/client-conformance-tests --bench <adapter> [options]
+
+Arguments:
+  <adapter>           Path to client adapter executable, or "ts" for built-in TypeScript adapter
+
+Conformance Test Options:
+  --run <adapter>     Run conformance tests with the specified adapter
+  --suite <name>      Run only specific suite(s): producer, consumer, lifecycle
+                      Can be specified multiple times
+  --tag <name>        Run only tests with specific tag(s)
+                      Can be specified multiple times
+  --fail-fast         Stop on first test failure
+  --timeout <ms>      Timeout for each test in milliseconds (default: 30000)
+
+Benchmark Options:
+  --bench <adapter>   Run benchmarks with the specified adapter
+  --scenario <id>     Run only specific scenario(s) by ID
+                      Can be specified multiple times
+  --category <name>   Run only scenarios in category: latency, throughput, streaming
+                      Can be specified multiple times
+  --format <fmt>      Output format: console, json, markdown (default: console)
+
+Common Options:
+  --verbose           Show detailed output for each operation
+  --port <port>       Port for reference server (default: random)
+  --help, -h          Show this help message
+
+Conformance Test Examples:
+  # Test the TypeScript client
+  npx @durable-streams/client-conformance-tests --run ts
+
+  # Test a Python client adapter
+  npx @durable-streams/client-conformance-tests --run ./adapters/python_adapter.py
+
+  # Test only producer functionality
+  npx @durable-streams/client-conformance-tests --run ts --suite producer
+
+  # Test with verbose output and stop on first failure
+  npx @durable-streams/client-conformance-tests --run ts --verbose --fail-fast
+
+Benchmark Examples:
+  # Run all benchmarks with TypeScript client
+  npx @durable-streams/client-conformance-tests --bench ts
+
+  # Run only latency benchmarks
+  npx @durable-streams/client-conformance-tests --bench ts --category latency
+
+  # Run specific scenario
+  npx @durable-streams/client-conformance-tests --bench ts --scenario latency-append
+
+  # Output as JSON for CI
+  npx @durable-streams/client-conformance-tests --bench ts --format json
+
+Implementing a Client Adapter:
+  A client adapter is an executable that communicates via stdin/stdout using
+  JSON-line protocol. See the documentation for the protocol specification
+  and examples in different languages.
+
+  The adapter receives JSON commands on stdin (one per line) and responds
+  with JSON results on stdout (one per line).
+
+  Commands: init, create, connect, append, read, head, delete, shutdown, benchmark
+
+  Example flow:
+    Runner -> Client: {"type":"init","serverUrl":"http://localhost:3000"}
+    Client -> Runner: {"type":"init","success":true,"clientName":"my-client","clientVersion":"1.0.0"}
+    Runner -> Client: {"type":"create","path":"/test-stream"}
+    Client -> Runner: {"type":"create","success":true,"status":201}
+    ...
+`
+
+type ParsedOptions =
+  | { mode: `conformance`; options: RunnerOptions }
+  | { mode: `benchmark`; options: BenchmarkRunnerOptions }
+  | null
+
+function parseArgs(args: Array<string>): ParsedOptions {
+  let mode: `conformance` | `benchmark` | null = null
+  let clientAdapter = ``
+
+  // Conformance-specific options
+  const suites: Array<`producer` | `consumer` | `lifecycle`> = []
+  const tags: Array<string> = []
+  let failFast = false
+  let testTimeout = 30000
+
+  // Benchmark-specific options
+  const scenarios: Array<string> = []
+  const categories: Array<`latency` | `throughput` | `streaming`> = []
+  let format: `console` | `json` | `markdown` = `console`
+
+  // Common options
+  let verbose = false
+  let serverPort = 0
+
+  let i = 0
+  while (i < args.length) {
+    const arg = args[i]!
+
+    if (arg === `--help` || arg === `-h`) {
+      console.log(HELP)
+      process.exit(0)
+    }
+
+    if (arg === `--run`) {
+      mode = `conformance`
+      i++
+      if (i >= args.length) {
+        console.error(`Error: --run requires an adapter path`)
+        return null
+      }
+      clientAdapter = args[i]!
+    } else if (arg === `--bench`) {
+      mode = `benchmark`
+      i++
+      if (i >= args.length) {
+        console.error(`Error: --bench requires an adapter path`)
+        return null
+      }
+      clientAdapter = args[i]!
+    } else if (arg === `--suite`) {
+      i++
+      if (i >= args.length) {
+        console.error(`Error: --suite requires a suite name`)
+        return null
+      }
+      const suite = args[i] as `producer` | `consumer` | `lifecycle`
+      if (![`producer`, `consumer`, `lifecycle`].includes(suite)) {
+        console.error(
+          `Error: Invalid suite "${suite}". Must be: producer, consumer, lifecycle`
+        )
+        return null
+      }
+      suites.push(suite)
+    } else if (arg === `--tag`) {
+      i++
+      if (i >= args.length) {
+        console.error(`Error: --tag requires a tag name`)
+        return null
+      }
+      tags.push(args[i]!)
+    } else if (arg === `--scenario`) {
+      i++
+      if (i >= args.length) {
+        console.error(`Error: --scenario requires a scenario ID`)
+        return null
+      }
+      scenarios.push(args[i]!)
+    } else if (arg === `--category`) {
+      i++
+      if (i >= args.length) {
+        console.error(`Error: --category requires a category name`)
+        return null
+      }
+      const category = args[i] as `latency` | `throughput` | `streaming`
+      if (![`latency`, `throughput`, `streaming`].includes(category)) {
+        console.error(
+          `Error: Invalid category "${category}". Must be: latency, throughput, streaming`
+        )
+        return null
+      }
+      categories.push(category)
+    } else if (arg === `--format`) {
+      i++
+      if (i >= args.length) {
+        console.error(`Error: --format requires a format name`)
+        return null
+      }
+      const fmt = args[i] as `console` | `json` | `markdown`
+      if (![`console`, `json`, `markdown`].includes(fmt)) {
+        console.error(
+          `Error: Invalid format "${fmt}". Must be: console, json, markdown`
+        )
+        return null
+      }
+      format = fmt
+    } else if (arg === `--verbose`) {
+      verbose = true
+    } else if (arg === `--fail-fast`) {
+      failFast = true
+    } else if (arg === `--timeout`) {
+      i++
+      if (i >= args.length) {
+        console.error(`Error: --timeout requires a value in milliseconds`)
+        return null
+      }
+      testTimeout = parseInt(args[i]!, 10)
+      if (isNaN(testTimeout)) {
+        console.error(`Error: --timeout must be a number`)
+        return null
+      }
+    } else if (arg === `--port`) {
+      i++
+      if (i >= args.length) {
+        console.error(`Error: --port requires a port number`)
+        return null
+      }
+      serverPort = parseInt(args[i]!, 10)
+      if (isNaN(serverPort)) {
+        console.error(`Error: --port must be a number`)
+        return null
+      }
+    } else if (arg.startsWith(`-`)) {
+      console.error(`Error: Unknown option "${arg}"`)
+      return null
+    }
+
+    i++
+  }
+
+  // Validate required options
+  if (!mode || !clientAdapter) {
+    console.error(`Error: --run <adapter> or --bench <adapter> is required`)
+    console.log(`\nRun with --help for usage information`)
+    return null
+  }
+
+  if (mode === `conformance`) {
+    const options: RunnerOptions = {
+      clientAdapter,
+      verbose,
+      failFast,
+      testTimeout,
+      serverPort,
+    }
+    if (suites.length > 0) options.suites = suites
+    if (tags.length > 0) options.tags = tags
+    return { mode: `conformance`, options }
+  } else {
+    const options: BenchmarkRunnerOptions = {
+      clientAdapter,
+      verbose,
+      serverPort,
+      format,
+    }
+    if (scenarios.length > 0) options.scenarios = scenarios
+    if (categories.length > 0) options.categories = categories
+    return { mode: `benchmark`, options }
+  }
+}
+
+async function main(): Promise<void> {
+  const args = process.argv.slice(2)
+
+  if (args.length === 0) {
+    console.log(HELP)
+    process.exit(0)
+  }
+
+  const parsed = parseArgs(args)
+  if (!parsed) {
+    process.exit(1)
+  }
+
+  try {
+    if (parsed.mode === `conformance`) {
+      const summary = await runConformanceTests(parsed.options)
+      if (summary.failed > 0) {
+        process.exit(1)
+      }
+    } else {
+      const summary = await runBenchmarks(parsed.options)
+      if (summary.failed > 0) {
+        process.exit(1)
+      }
+    }
+  } catch (err) {
+    console.error(`Error running ${parsed.mode}:`, err)
+    process.exit(1)
+  }
+}
+
+main()

package/src/index.ts

@@ -0,0 +1,50 @@
+/**
+ * Client Conformance Test Suite for Durable Streams
+ *
+ * This package provides a comprehensive test suite to verify that a client
+ * correctly implements the Durable Streams protocol for both producers and consumers,
+ * along with performance benchmarking capabilities.
+ *
+ * @packageDocumentation
+ */
+
+// Conformance testing
+export {
+  runConformanceTests,
+  loadEmbeddedTestSuites,
+  filterByCategory,
+  countTests,
+  type RunnerOptions,
+  type TestRunResult,
+  type RunSummary,
+} from "./runner.js"
+
+export {
+  type TestSuite,
+  type TestCase,
+  type TestOperation,
+  type ClientFeature,
+  loadTestSuites,
+} from "./test-cases.js"
+
+// Benchmarking
+export {
+  runBenchmarks,
+  allScenarios,
+  getScenarioById,
+  type BenchmarkRunnerOptions,
+  type ScenarioResult,
+  type BenchmarkSummary,
+} from "./benchmark-runner.js"
+
+export {
+  type BenchmarkScenario,
+  type BenchmarkScenarioConfig,
+  type BenchmarkCriteria,
+  type ScenarioContext,
+  getScenariosByCategory,
+  scenariosByCategory,
+} from "./benchmark-scenarios.js"
+
+// Re-export protocol types for adapter implementers
+export * from "./protocol.js"