@durable-streams/client-conformance-tests 0.1.8 → 0.1.9

package/src/benchmark-scenarios.ts

@@ -190,7 +190,7 @@ export const smallMessageThroughputScenario: BenchmarkScenario = {
   createOperation: (ctx) => ({
     op: `throughput_append`,
     path: `${ctx.basePath}/throughput-small`,
-    count: 10000,
+    count: 100000,
     size: 100,
     concurrency: 200,
   }),
@@ -239,9 +239,9 @@ export const readThroughputScenario: BenchmarkScenario = {
     expectedCount: ctx.setupData.expectedCount as number | undefined,
   }),
   setup: (ctx) => {
-    // Expecting 10000 JSON messages to be pre-populated
-    ctx.setupData.expectedCount = 10000
-    return Promise.resolve({ data: { expectedCount: 10000 } })
+    // Expecting 100000 JSON messages to be pre-populated
+    ctx.setupData.expectedCount = 100000
+    return Promise.resolve({ data: { expectedCount: 100000 } })
   },
 }
 

package/src/cli.ts

@@ -10,7 +10,7 @@
  */
 
 import { runConformanceTests } from "./runner.js"
-import { runBenchmarks } from "./benchmark-runner.js"
+import { aggregateBenchmarkResults, runBenchmarks } from "./benchmark-runner.js"
 import type { RunnerOptions } from "./runner.js"
 import type { BenchmarkRunnerOptions } from "./benchmark-runner.js"
 
@@ -20,6 +20,7 @@ Durable Streams Client Conformance Test Suite
 Usage:
   npx @durable-streams/client-conformance-tests --run <adapter> [options]
   npx @durable-streams/client-conformance-tests --bench <adapter> [options]
+  npx @durable-streams/client-conformance-tests --report <dir>
 
 Arguments:
   <adapter>           Path to client adapter executable, or "ts" for built-in TypeScript adapter
@@ -41,6 +42,10 @@ Benchmark Options:
                       Can be specified multiple times
   --format <fmt>      Output format: console, json, markdown (default: console)
 
+Report Options:
+  --report <dir>      Aggregate benchmark results from JSON files in directory
+                      Each subdirectory should contain a benchmark-results.json file
+
 Common Options:
   --verbose           Show detailed output for each operation
   --port <port>       Port for reference server (default: random)
@@ -72,6 +77,10 @@ Benchmark Examples:
   # Output as JSON for CI
   npx @durable-streams/client-conformance-tests --bench ts --format json
 
+Report Examples:
+  # Aggregate benchmark results from CI artifacts
+  npx @durable-streams/client-conformance-tests --report ./benchmark-results
+
 Implementing a Client Adapter:
   A client adapter is an executable that communicates via stdin/stdout using
   JSON-line protocol. See the documentation for the protocol specification
@@ -93,11 +102,13 @@ Implementing a Client Adapter:
 type ParsedOptions =
   | { mode: `conformance`; options: RunnerOptions }
   | { mode: `benchmark`; options: BenchmarkRunnerOptions }
+  | { mode: `report`; resultsDir: string }
   | null
 
 function parseArgs(args: Array<string>): ParsedOptions {
-  let mode: `conformance` | `benchmark` | null = null
+  let mode: `conformance` | `benchmark` | `report` | null = null
   let clientAdapter = ``
+  let resultsDir = ``
 
   // Conformance-specific options
   const suites: Array<`producer` | `consumer` | `lifecycle`> = []
@@ -139,6 +150,14 @@ function parseArgs(args: Array<string>): ParsedOptions {
         return null
       }
       clientAdapter = args[i]!
+    } else if (arg === `--report`) {
+      mode = `report`
+      i++
+      if (i >= args.length) {
+        console.error(`Error: --report requires a directory path`)
+        return null
+      }
+      resultsDir = args[i]!
     } else if (arg === `--suite`) {
       i++
       if (i >= args.length) {
@@ -230,8 +249,26 @@ function parseArgs(args: Array<string>): ParsedOptions {
   }
 
   // Validate required options
-  if (!mode || !clientAdapter) {
-    console.error(`Error: --run <adapter> or --bench <adapter> is required`)
+  if (!mode) {
+    console.error(
+      `Error: --run <adapter>, --bench <adapter>, or --report <dir> is required`
+    )
+    console.log(`\nRun with --help for usage information`)
+    return null
+  }
+
+  if (mode === `report`) {
+    if (!resultsDir) {
+      console.error(`Error: --report requires a directory path`)
+      return null
+    }
+    return { mode: `report`, resultsDir }
+  }
+
+  if (!clientAdapter) {
+    console.error(
+      `Error: --run <adapter> or --bench <adapter> requires an adapter path`
+    )
     console.log(`\nRun with --help for usage information`)
     return null
   }
@@ -279,11 +316,15 @@ async function main(): Promise<void> {
       if (summary.failed > 0) {
         process.exit(1)
       }
-    } else {
+    } else if (parsed.mode === `benchmark`) {
       const summary = await runBenchmarks(parsed.options)
       if (summary.failed > 0) {
         process.exit(1)
       }
+    } else {
+      // parsed.mode === `report`
+      const report = await aggregateBenchmarkResults(parsed.resultsDir)
+      console.log(report)
     }
   } catch (err) {
     console.error(`Error running ${parsed.mode}:`, err)

package/src/protocol.ts

@@ -120,8 +120,8 @@ export interface ReadCommand {
   path: string
   /** Starting offset (opaque string from previous reads) */
   offset?: string
-  /** Live mode: false for catch-up only, "long-poll" or "sse" for live */
-  live?: false | `long-poll` | `sse`
+  /** Live mode: false for catch-up only, true for auto-select, "long-poll" or "sse" for explicit */
+  live?: false | true | `long-poll` | `sse`
   /** Timeout for long-poll in milliseconds */
   timeoutMs?: number
   /** Maximum number of chunks to read (for testing) */
@@ -196,6 +196,59 @@ export interface ClearDynamicCommand {
   type: `clear-dynamic`
 }
 
+// =============================================================================
+// Validation Commands
+// =============================================================================
+
+/**
+ * Test client-side input validation.
+ *
+ * This command tests that the client properly validates input parameters
+ * before making any network requests. The adapter should attempt to create
+ * the specified object with the given parameters and report whether
+ * validation passed or failed.
+ */
+export interface ValidateCommand {
+  type: `validate`
+  /** What to validate */
+  target: ValidateTarget
+}
+
+/**
+ * Validation targets - what client-side validation to test.
+ */
+export type ValidateTarget = ValidateRetryOptions | ValidateIdempotentProducer
+
+/**
+ * Validate RetryOptions construction.
+ */
+export interface ValidateRetryOptions {
+  target: `retry-options`
+  /** Max retries (should reject < 0) */
+  maxRetries?: number
+  /** Initial delay in ms (should reject <= 0) */
+  initialDelayMs?: number
+  /** Max delay in ms (should reject < initialDelayMs) */
+  maxDelayMs?: number
+  /** Backoff multiplier (should reject < 1.0) */
+  multiplier?: number
+}
+
+/**
+ * Validate IdempotentProducer construction.
+ */
+export interface ValidateIdempotentProducer {
+  target: `idempotent-producer`
+  /** Producer ID (required, non-empty) */
+  producerId?: string
+  /** Starting epoch (should reject < 0) */
+  epoch?: number
+  /** Max batch bytes (should reject <= 0) */
+  maxBatchBytes?: number
+  /** Max batch items (should reject <= 0) */
+  maxBatchItems?: number
+}
+
 // =============================================================================
 // Benchmark Commands
 // =============================================================================
@@ -289,6 +342,7 @@ export type TestCommand =
   | SetDynamicParamCommand
   | ClearDynamicCommand
   | BenchmarkCommand
+  | ValidateCommand
 
 // =============================================================================
 // Results (sent from client adapter to test runner via stdout)
@@ -312,10 +366,18 @@ export interface InitResult {
     sse?: boolean
     /** Supports long-poll mode */
     longPoll?: boolean
+    /** Supports auto mode (catch-up then auto-select SSE or long-poll) */
+    auto?: boolean
     /** Supports streaming reads */
     streaming?: boolean
     /** Supports dynamic headers/params (functions evaluated per-request) */
     dynamicHeaders?: boolean
+    /** Supports RetryOptions validation (PHP-specific) */
+    retryOptions?: boolean
+    /** Supports maxBatchItems option (PHP-specific) */
+    batchItems?: boolean
+    /** Rejects zero values as invalid (vs treating 0 as "use default" like Go) */
+    strictZeroValidation?: boolean
   }
 }
 
@@ -492,6 +554,14 @@ export interface ClearDynamicResult {
   success: true
 }
 
+/**
+ * Successful validate result (validation passed).
+ */
+export interface ValidateResult {
+  type: `validate`
+  success: true
+}
+
 /**
  * Successful benchmark result with timing.
  */
@@ -549,6 +619,7 @@ export type TestResult =
   | SetDynamicHeaderResult
   | SetDynamicParamResult
   | ClearDynamicResult
+  | ValidateResult
   | BenchmarkResult
   | ErrorResult
 
@@ -622,6 +693,8 @@ export const ErrorCodes = {
   INTERNAL_ERROR: `INTERNAL_ERROR`,
   /** Operation not supported by this client */
   NOT_SUPPORTED: `NOT_SUPPORTED`,
+  /** Invalid argument passed to client API */
+  INVALID_ARGUMENT: `INVALID_ARGUMENT`,
 } as const
 
 export type ErrorCode = (typeof ErrorCodes)[keyof typeof ErrorCodes]

package/src/runner.ts

@@ -77,8 +77,12 @@ interface ClientFeatures {
   batching?: boolean
   sse?: boolean
   longPoll?: boolean
+  auto?: boolean
   streaming?: boolean
   dynamicHeaders?: boolean
+  retryOptions?: boolean
+  batchItems?: boolean
+  strictZeroValidation?: boolean
 }
 
 interface ExecutionContext {
@@ -300,7 +304,13 @@ async function executeOperation(
 
     case `append`: {
       const path = resolveVariables(op.path, variables)
-      const data = op.data ? resolveVariables(op.data, variables) : ``
+      // Handle json property (YAML object) by stringifying, or use data string directly
+      const data =
+        op.json !== undefined
+          ? JSON.stringify(op.json)
+          : op.data
+            ? resolveVariables(op.data, variables)
+            : ``
 
       const result = await client.send(
         {
@@ -729,6 +739,7 @@ async function executeOperation(
             method: op.method,
             corruptBody: op.corruptBody,
             jitterMs: op.jitterMs,
+            injectSseEvent: op.injectSseEvent,
           }),
         })
 
@@ -742,6 +753,8 @@ async function executeOperation(
           faultTypes.push(`truncate=${op.truncateBodyBytes}b`)
         if (op.corruptBody) faultTypes.push(`corrupt`)
         if (op.probability != null) faultTypes.push(`p=${op.probability}`)
+        if (op.injectSseEvent)
+          faultTypes.push(`sse:${op.injectSseEvent.eventType}`)
         const faultDesc = faultTypes.join(`,`) || `unknown`
 
         if (verbose) {
@@ -835,6 +848,25 @@ async function executeOperation(
       return { result }
     }
 
+    case `validate`: {
+      const result = await client.send(
+        {
+          type: `validate`,
+          target: op.target,
+        },
+        commandTimeout
+      )
+
+      if (verbose) {
+        const targetType = op.target.target
+        console.log(
+          `  validate ${targetType}: ${result.success ? `ok` : `failed`}`
+        )
+      }
+
+      return { result }
+    }
+
     default:
       return { error: `Unknown operation: ${(op as TestOperation).action}` }
   }
@@ -879,6 +911,21 @@ function validateExpectation(
     }
   }
 
+  // Check error message contains expected strings
+  if (expect.messageContains !== undefined) {
+    if (result.success) {
+      return `Expected error with message containing ${JSON.stringify(expect.messageContains)}, but operation succeeded`
+    }
+    if (isErrorResult(result)) {
+      const missing = (expect.messageContains as Array<string>).filter(
+        (s) => !result.message.toLowerCase().includes(s.toLowerCase())
+      )
+      if (missing.length > 0) {
+        return `Expected error message to contain [${(expect.messageContains as Array<string>).join(`, `)}], missing: [${missing.join(`, `)}]. Actual message: "${result.message}"`
+      }
+    }
+  }
+
   // Check data (for read results)
   if (expect.data !== undefined && isReadResult(result)) {
     const actualData = result.chunks.map((c) => c.data).join(``)
@@ -1039,6 +1086,23 @@ function validateExpectation(
     }
   }
 
+  // Check valid (for validation operations)
+  if (expect.valid !== undefined) {
+    if (expect.valid === true && !result.success) {
+      return `Expected validation to pass, but it failed`
+    }
+    if (expect.valid === false && result.success) {
+      return `Expected validation to fail, but it passed`
+    }
+  }
+
+  // Check errorContains (for validation operations with error message substring)
+  if (expect.errorContains !== undefined && isErrorResult(result)) {
+    if (!result.message.includes(expect.errorContains as string)) {
+      return `Expected error message to contain "${expect.errorContains}", got "${result.message}"`
+    }
+  }
+
   return null
 }
 
@@ -1051,9 +1115,16 @@ function featureToProperty(feature: string): keyof ClientFeatures | undefined {
     sse: `sse`,
     "long-poll": `longPoll`,
     longPoll: `longPoll`,
+    auto: `auto`,
     streaming: `streaming`,
     dynamicHeaders: `dynamicHeaders`,
     "dynamic-headers": `dynamicHeaders`,
+    retryOptions: `retryOptions`,
+    "retry-options": `retryOptions`,
+    batchItems: `batchItems`,
+    "batch-items": `batchItems`,
+    strictZeroValidation: `strictZeroValidation`,
+    "strict-zero-validation": `strictZeroValidation`,
   }
   return map[feature]
 }

package/src/test-cases.ts

@@ -116,6 +116,8 @@ export interface AppendOperation {
   path: string
   /** Data to append (string) */
   data?: string
+  /** JSON data to append (will be stringified) */
+  json?: unknown
   /** Binary data (base64 encoded) */
   binaryData?: string
   /** Sequence number for ordering (Stream-Seq header) */
@@ -364,6 +366,13 @@ export interface InjectErrorOperation {
   corruptBody?: boolean
   /** Add jitter to delay (random 0-jitterMs added to delayMs) */
   jitterMs?: number
+  /** Inject an SSE event with custom type and data (for testing SSE parsing) */
+  injectSseEvent?: {
+    /** Event type (e.g., "unknown", "control", "data") */
+    eventType: string
+    /** Event data (will be sent as-is) */
+    data: string
+  }
 }
 
 /**
@@ -405,6 +414,49 @@ export interface ClearDynamicOperation {
   action: `clear-dynamic`
 }
 
+/**
+ * Validate client-side input parameters.
+ * Tests that clients properly validate inputs before making network requests.
+ */
+export interface ValidateOperation {
+  action: `validate`
+  /** What to validate */
+  target: ValidateTarget
+  expect?: ValidateExpectation
+}
+
+/**
+ * Validation target types.
+ */
+export type ValidateTarget =
+  | ValidateRetryOptionsTarget
+  | ValidateIdempotentProducerTarget
+
+export interface ValidateRetryOptionsTarget {
+  target: `retry-options`
+  maxRetries?: number
+  initialDelayMs?: number
+  maxDelayMs?: number
+  multiplier?: number
+}
+
+export interface ValidateIdempotentProducerTarget {
+  target: `idempotent-producer`
+  producerId?: string
+  epoch?: number
+  maxBatchBytes?: number
+  maxBatchItems?: number
+}
+
+export interface ValidateExpectation {
+  /** If true, validation should pass */
+  valid?: boolean
+  /** Expected error code if validation fails */
+  errorCode?: string
+  /** Expected error message substring if validation fails */
+  errorContains?: string
+}
+
 /**
  * All possible test operations.
  */
@@ -428,6 +480,7 @@ export type TestOperation =
   | SetDynamicHeaderOperation
   | SetDynamicParamOperation
   | ClearDynamicOperation
+  | ValidateOperation
 
 // =============================================================================
 // Expectations
@@ -441,6 +494,8 @@ interface BaseExpectation {
   status?: number
   /** Expected error code (if operation should fail) */
   errorCode?: string
+  /** Strings that should be present in error message (for context validation) */
+  messageContains?: Array<string>
   /** Store result in variable */
   storeAs?: string
 }

package/test-cases/consumer/error-context.yaml

@@ -0,0 +1,67 @@
+id: consumer-error-context
+name: Consumer Error Context
+description: |
+  Tests that error messages include helpful context (URL, status, etc.).
+  Catching bugs where errors lose context or provide unhelpful messages.
+category: consumer
+tags:
+  - errors
+  - context
+  - messages
+
+tests:
+  - id: not-found-includes-path
+    name: 404 error includes stream path in message
+    description: Error message should include the stream path for debugging
+    operations:
+      - action: read
+        path: /error-context-nonexistent-stream
+        live: false
+        expect:
+          status: 404
+          errorCode: NOT_FOUND
+          messageContains:
+            - "error-context-nonexistent-stream"
+
+  - id: not-found-on-append-includes-path
+    name: 404 on append includes stream path in message
+    description: Append to non-existent stream should report the path
+    operations:
+      - action: append
+        path: /error-context-append-nonexistent
+        data: "test"
+        expect:
+          status: 404
+          errorCode: NOT_FOUND
+          messageContains:
+            - "error-context-append-nonexistent"
+
+  - id: invalid-offset-includes-context
+    name: Invalid offset error includes helpful context
+    description: Error for invalid offset should indicate the nature of the error
+    setup:
+      - action: create
+        as: streamPath
+      - action: append
+        path: ${streamPath}
+        data: "data"
+    operations:
+      - action: read
+        path: ${streamPath}
+        offset: "not-a-valid-offset-format"
+        live: false
+        expect:
+          status: 400
+          errorCode: INVALID_OFFSET
+
+  - id: head-not-found-includes-path
+    name: HEAD 404 error includes stream path
+    description: HEAD request to non-existent stream should include path in error
+    operations:
+      - action: head
+        path: /error-context-head-nonexistent
+        expect:
+          status: 404
+          errorCode: NOT_FOUND
+          messageContains:
+            - "error-context-head-nonexistent"

package/test-cases/consumer/json-parsing-errors.yaml

@@ -0,0 +1,115 @@
+id: consumer-json-parsing-errors
+name: JSON Parsing Error Handling
+description: Tests that clients properly handle malformed JSON responses instead of silently failing
+category: consumer
+tags:
+  - json
+  - error-handling
+  - resilience
+  - fault-injection
+
+tests:
+  - id: truncated-json-response
+    name: Client throws on truncated JSON response
+    description: Client should throw PARSE_ERROR for incomplete JSON, not return empty array
+    requiredFeatures: [json]
+    setup:
+      - action: create
+        as: streamPath
+        contentType: application/json
+      - action: append
+        path: ${streamPath}
+        json: { "key": "some long value that will be truncated" }
+    operations:
+      # Truncate the response mid-JSON (10 bytes is mid-object)
+      - action: inject-error
+        path: ${streamPath}
+        truncateBodyBytes: 10
+        method: GET
+        count: 1
+      - action: read
+        path: ${streamPath}
+        expect:
+          errorCode: PARSE_ERROR
+    cleanup:
+      - action: clear-errors
+
+  - id: corrupted-json-response
+    name: Client throws on corrupted JSON response
+    description: Client should throw PARSE_ERROR for invalid JSON, not return empty array
+    requiredFeatures: [json]
+    setup:
+      - action: create
+        as: streamPath
+        contentType: application/json
+      - action: append
+        path: ${streamPath}
+        json: { "valid": "json data here" }
+    operations:
+      # Corrupt the JSON by flipping bits
+      - action: inject-error
+        path: ${streamPath}
+        corruptBody: true
+        method: GET
+        count: 1
+      - action: read
+        path: ${streamPath}
+        expect:
+          errorCode: PARSE_ERROR
+    cleanup:
+      - action: clear-errors
+
+  - id: truncated-json-array
+    name: Client throws on truncated JSON array
+    description: Client should throw when JSON array is truncated mid-element
+    requiredFeatures: [json]
+    setup:
+      - action: create
+        as: streamPath
+        contentType: application/json
+      - action: append
+        path: ${streamPath}
+        json: [{ "item": 1 }, { "item": 2 }, { "item": 3 }]
+    operations:
+      # Truncate mid-array
+      - action: inject-error
+        path: ${streamPath}
+        truncateBodyBytes: 15
+        method: GET
+        count: 1
+      - action: read
+        path: ${streamPath}
+        expect:
+          errorCode: PARSE_ERROR
+    cleanup:
+      - action: clear-errors
+
+  - id: recovery-after-parse-error
+    name: Client recovers after parse error
+    description: After a parse error, subsequent requests should succeed
+    requiredFeatures: [json]
+    setup:
+      - action: create
+        as: streamPath
+        contentType: application/json
+      - action: append
+        path: ${streamPath}
+        json: { "data": "valid" }
+    operations:
+      # First request fails with truncated body
+      - action: inject-error
+        path: ${streamPath}
+        truncateBodyBytes: 5
+        method: GET
+        count: 1
+      - action: read
+        path: ${streamPath}
+        expect:
+          errorCode: PARSE_ERROR
+      # Second request should succeed (no more faults)
+      - action: read
+        path: ${streamPath}
+        expect:
+          status: 200
+    cleanup:
+      - action: clear-errors