@durable-streams/client-conformance-tests 0.1.5 → 0.1.7

package/src/runner.ts

@@ -366,6 +366,71 @@ async function executeOperation(
       }
     }
 
+    case `idempotent-append`: {
+      const path = resolveVariables(op.path, variables)
+      const data = resolveVariables(op.data, variables)
+
+      const result = await client.send(
+        {
+          type: `idempotent-append`,
+          path,
+          data,
+          producerId: op.producerId,
+          epoch: op.epoch ?? 0,
+          autoClaim: op.autoClaim ?? false,
+          headers: op.headers,
+        },
+        commandTimeout
+      )
+
+      if (verbose) {
+        console.log(
+          `  idempotent-append ${path}: ${result.success ? `ok` : `failed`}`
+        )
+      }
+
+      if (
+        result.success &&
+        result.type === `idempotent-append` &&
+        op.expect?.storeOffsetAs
+      ) {
+        variables.set(op.expect.storeOffsetAs, result.offset ?? ``)
+      }
+
+      return { result }
+    }
+
+    case `idempotent-append-batch`: {
+      const path = resolveVariables(op.path, variables)
+
+      // Send items to client which will batch them internally
+      const items = op.items.map((item) =>
+        resolveVariables(item.data, variables)
+      )
+
+      const result = await client.send(
+        {
+          type: `idempotent-append-batch`,
+          path,
+          items,
+          producerId: op.producerId,
+          epoch: op.epoch ?? 0,
+          autoClaim: op.autoClaim ?? false,
+          maxInFlight: op.maxInFlight,
+          headers: op.headers,
+        },
+        commandTimeout
+      )
+
+      if (verbose) {
+        console.log(
+          `  idempotent-append-batch ${path}: ${result.success ? `ok` : `failed`}`
+        )
+      }
+
+      return { result }
+    }
+
     case `read`: {
       const path = resolveVariables(op.path, variables)
       const offset = op.offset
@@ -538,7 +603,8 @@ async function executeOperation(
 
     case `server-append`: {
       // Direct HTTP append to server, bypassing client adapter
-      // Used for concurrent operations when adapter is blocked on a read
+      // Used for concurrent operations when adapter is blocked on a read,
+      // and for testing protocol-level behavior like idempotent producers
       const path = resolveVariables(op.path, variables)
       const data = resolveVariables(op.data, variables)
 
@@ -550,28 +616,72 @@ async function executeOperation(
         const contentType =
           headResponse.headers.get(`content-type`) ?? `application/octet-stream`
 
+        // Build headers, including producer headers if present
+        const headers: Record<string, string> = {
+          "content-type": contentType,
+          ...op.headers,
+        }
+        if (op.producerId !== undefined) {
+          headers[`Producer-Id`] = op.producerId
+        }
+        if (op.producerEpoch !== undefined) {
+          headers[`Producer-Epoch`] = op.producerEpoch.toString()
+        }
+        if (op.producerSeq !== undefined) {
+          headers[`Producer-Seq`] = op.producerSeq.toString()
+        }
+
         const response = await fetch(`${ctx.serverUrl}${path}`, {
           method: `POST`,
           body: data,
-          headers: {
-            "content-type": contentType,
-            ...op.headers,
-          },
+          headers,
         })
 
+        const status = response.status
+        const offset = response.headers.get(`Stream-Next-Offset`) ?? undefined
+        const duplicate = status === 204
+        const producerEpoch = response.headers.get(`Producer-Epoch`)
+        const producerSeq = response.headers.get(`Producer-Seq`)
+        const producerExpectedSeq = response.headers.get(
+          `Producer-Expected-Seq`
+        )
+        const producerReceivedSeq = response.headers.get(
+          `Producer-Received-Seq`
+        )
+
         if (verbose) {
           console.log(
-            `  server-append ${path}: ${response.ok ? `ok` : `failed (${response.status})`}`
+            `  server-append ${path}: status=${status}${duplicate ? ` (duplicate)` : ``}`
           )
         }
 
-        if (!response.ok) {
-          return {
-            error: `Server append failed with status ${response.status}`,
-          }
+        // Build result for expectation verification
+        // success: true means we got a valid protocol response (even 403/409)
+        // The status field indicates the actual operation result
+        const result: TestResult = {
+          type: `append`,
+          success: true,
+          status,
+          offset,
+          duplicate,
+          producerEpoch: producerEpoch
+            ? parseInt(producerEpoch, 10)
+            : undefined,
+          producerSeq: producerSeq ? parseInt(producerSeq, 10) : undefined,
+          producerExpectedSeq: producerExpectedSeq
+            ? parseInt(producerExpectedSeq, 10)
+            : undefined,
+          producerReceivedSeq: producerReceivedSeq
+            ? parseInt(producerReceivedSeq, 10)
+            : undefined,
         }
 
-        return {}
+        // Store offset if requested
+        if (op.expect?.storeOffsetAs && offset) {
+          variables.set(op.expect.storeOffsetAs, offset)
+        }
+
+        return { result }
       } catch (err) {
         return {
           error: `Server append failed: ${err instanceof Error ? err.message : String(err)}`,
@@ -599,7 +709,7 @@ async function executeOperation(
     }
 
     case `inject-error`: {
-      // Inject an error via the test server's control endpoint
+      // Inject a fault via the test server's control endpoint
       const path = resolveVariables(op.path, variables)
 
       try {
@@ -611,23 +721,43 @@ async function executeOperation(
             status: op.status,
             count: op.count ?? 1,
             retryAfter: op.retryAfter,
+            // New fault injection parameters
+            delayMs: op.delayMs,
+            dropConnection: op.dropConnection,
+            truncateBodyBytes: op.truncateBodyBytes,
+            probability: op.probability,
+            method: op.method,
+            corruptBody: op.corruptBody,
+            jitterMs: op.jitterMs,
           }),
         })
 
+        // Build descriptive log message
+        const faultTypes = []
+        if (op.status != null) faultTypes.push(`status=${op.status}`)
+        if (op.delayMs != null) faultTypes.push(`delay=${op.delayMs}ms`)
+        if (op.jitterMs != null) faultTypes.push(`jitter=${op.jitterMs}ms`)
+        if (op.dropConnection) faultTypes.push(`dropConnection`)
+        if (op.truncateBodyBytes != null)
+          faultTypes.push(`truncate=${op.truncateBodyBytes}b`)
+        if (op.corruptBody) faultTypes.push(`corrupt`)
+        if (op.probability != null) faultTypes.push(`p=${op.probability}`)
+        const faultDesc = faultTypes.join(`,`) || `unknown`
+
         if (verbose) {
           console.log(
-            `  inject-error ${path} ${op.status}x${op.count ?? 1}: ${response.ok ? `ok` : `failed`}`
+            `  inject-error ${path} [${faultDesc}]x${op.count ?? 1}: ${response.ok ? `ok` : `failed`}`
           )
         }
 
         if (!response.ok) {
-          return { error: `Failed to inject error: ${response.status}` }
+          return { error: `Failed to inject fault: ${response.status}` }
         }
 
         return {}
       } catch (err) {
         return {
-          error: `Failed to inject error: ${err instanceof Error ? err.message : String(err)}`,
+          error: `Failed to inject fault: ${err instanceof Error ? err.message : String(err)}`,
         }
       }
     }
@@ -776,6 +906,22 @@ function validateExpectation(
     }
   }
 
+  // Check dataExact - verifies exact messages in order
+  if (expect.dataExact !== undefined && isReadResult(result)) {
+    const expectedMessages = expect.dataExact as Array<string>
+    const actualMessages = result.chunks.map((c) => c.data)
+
+    if (actualMessages.length !== expectedMessages.length) {
+      return `Expected ${expectedMessages.length} messages, got ${actualMessages.length}. Expected: [${expectedMessages.join(`, `)}], got: [${actualMessages.join(`, `)}]`
+    }
+
+    for (let i = 0; i < expectedMessages.length; i++) {
+      if (actualMessages[i] !== expectedMessages[i]) {
+        return `Message ${i} mismatch: expected "${expectedMessages[i]}", got "${actualMessages[i]}"`
+      }
+    }
+  }
+
   // Check upToDate
   if (expect.upToDate !== undefined && isReadResult(result)) {
     if (result.upToDate !== expect.upToDate) {
@@ -858,6 +1004,41 @@ function validateExpectation(
     }
   }
 
+  // Check duplicate (for idempotent producer 204 responses)
+  if (expect.duplicate !== undefined && isAppendResult(result)) {
+    if (result.duplicate !== expect.duplicate) {
+      return `Expected duplicate=${expect.duplicate}, got ${result.duplicate}`
+    }
+  }
+
+  // Check producerEpoch (returned on 200/403)
+  if (expect.producerEpoch !== undefined && isAppendResult(result)) {
+    if (result.producerEpoch !== expect.producerEpoch) {
+      return `Expected producerEpoch=${expect.producerEpoch}, got ${result.producerEpoch}`
+    }
+  }
+
+  // Check producerSeq (returned on 200/204 - highest accepted sequence)
+  if (expect.producerSeq !== undefined && isAppendResult(result)) {
+    if (result.producerSeq !== expect.producerSeq) {
+      return `Expected producerSeq=${expect.producerSeq}, got ${result.producerSeq}`
+    }
+  }
+
+  // Check producerExpectedSeq (returned on 409 sequence gap)
+  if (expect.producerExpectedSeq !== undefined && isAppendResult(result)) {
+    if (result.producerExpectedSeq !== expect.producerExpectedSeq) {
+      return `Expected producerExpectedSeq=${expect.producerExpectedSeq}, got ${result.producerExpectedSeq}`
+    }
+  }
+
+  // Check producerReceivedSeq (returned on 409 sequence gap)
+  if (expect.producerReceivedSeq !== undefined && isAppendResult(result)) {
+    if (result.producerReceivedSeq !== expect.producerReceivedSeq) {
+      return `Expected producerReceivedSeq=${expect.producerReceivedSeq}, got ${result.producerReceivedSeq}`
+    }
+  }
+
   return null
 }
 
@@ -1041,8 +1222,12 @@ export async function runConformanceTests(
   const totalTests = countTests(suites)
   console.log(`\nRunning ${totalTests} client conformance tests...\n`)
 
-  // Start reference server
-  const server = new DurableStreamTestServer({ port: options.serverPort ?? 0 })
+  // Start reference server with short long-poll timeout for testing
+  // Tests use timeoutMs: 1000, so server timeout should be shorter
+  const server = new DurableStreamTestServer({
+    port: options.serverPort ?? 0,
+    longPollTimeout: 500, // 500ms timeout for testing
+  })
   await server.start()
   const serverUrl = server.url
 

package/src/test-cases.ts

@@ -118,10 +118,16 @@ export interface AppendOperation {
   data?: string
   /** Binary data (base64 encoded) */
   binaryData?: string
-  /** Sequence number for ordering */
+  /** Sequence number for ordering (Stream-Seq header) */
   seq?: number
   headers?: Record<string, string>
   expect?: AppendExpectation
+  /** Producer ID for idempotent producers */
+  producerId?: string
+  /** Producer epoch for idempotent producers */
+  producerEpoch?: number
+  /** Producer sequence for idempotent producers */
+  producerSeq?: number
 }
 
 /**
@@ -140,6 +146,64 @@ export interface AppendBatchOperation {
   expect?: AppendBatchExpectation
 }
 
+/**
+ * Append via IdempotentProducer client (tests client-side exactly-once semantics).
+ */
+export interface IdempotentAppendOperation {
+  action: `idempotent-append`
+  path: string
+  /** Producer ID */
+  producerId: string
+  /** Producer epoch */
+  epoch?: number
+  /** Data to append (string or JSON for JSON streams) */
+  data: string
+  /** Auto-claim epoch on 403 */
+  autoClaim?: boolean
+  headers?: Record<string, string>
+  expect?: IdempotentAppendExpectation
+}
+
+/**
+ * Batch append via IdempotentProducer client (tests client-side JSON batching).
+ */
+export interface IdempotentAppendBatchOperation {
+  action: `idempotent-append-batch`
+  path: string
+  /** Producer ID */
+  producerId: string
+  /** Producer epoch */
+  epoch?: number
+  /** Items to append (will be batched by the client) */
+  items: Array<{
+    data: string
+  }>
+  /** Auto-claim epoch on 403 */
+  autoClaim?: boolean
+  /** Max concurrent batches in flight (default 1, set higher to test 409 retry) */
+  maxInFlight?: number
+  headers?: Record<string, string>
+  expect?: IdempotentAppendBatchExpectation
+}
+
+/**
+ * Expectation for idempotent-append operation.
+ */
+export interface IdempotentAppendExpectation extends BaseExpectation {
+  /** Expected duplicate flag */
+  duplicate?: boolean
+  /** Store the returned offset */
+  storeOffsetAs?: string
+}
+
+/**
+ * Expectation for idempotent-append-batch operation.
+ */
+export interface IdempotentAppendBatchExpectation extends BaseExpectation {
+  /** All items should succeed */
+  allSucceed?: boolean
+}
+
 /**
  * Read from a stream.
  */
@@ -223,13 +287,42 @@ export interface AssertOperation {
 
 /**
  * Append to stream via direct server HTTP (bypasses client adapter).
- * Used for concurrent operations when adapter is blocked on a read.
+ * Used for concurrent operations when adapter is blocked on a read,
+ * and for testing protocol-level behavior like idempotent producers.
  */
 export interface ServerAppendOperation {
   action: `server-append`
   path: string
   data: string
   headers?: Record<string, string>
+  /** Producer ID for idempotent producers */
+  producerId?: string
+  /** Producer epoch for idempotent producers */
+  producerEpoch?: number
+  /** Producer sequence for idempotent producers */
+  producerSeq?: number
+  /** Expected result */
+  expect?: ServerAppendExpectation
+}
+
+/**
+ * Expectation for server-append operation.
+ */
+export interface ServerAppendExpectation {
+  /** Expected HTTP status code */
+  status?: number
+  /** Store the returned offset */
+  storeOffsetAs?: string
+  /** Expected duplicate flag (true for 204 idempotent success) */
+  duplicate?: boolean
+  /** Expected producer epoch in response */
+  producerEpoch?: number
+  /** Expected producer seq in response (highest accepted sequence) */
+  producerSeq?: number
+  /** Expected producer expected seq (on 409 sequence gap) */
+  producerExpectedSeq?: number
+  /** Expected producer received seq (on 409 sequence gap) */
+  producerReceivedSeq?: number
 }
 
 /**
@@ -243,19 +336,34 @@ export interface AwaitOperation {
 }
 
 /**
- * Inject an error to be returned on the next N requests to a path.
+ * Inject a fault to be triggered on the next N requests to a path.
  * Used for testing retry/resilience behavior.
+ * Supports various fault types: errors, delays, connection drops, body corruption, etc.
  */
 export interface InjectErrorOperation {
   action: `inject-error`
-  /** Stream path to inject error for */
+  /** Stream path to inject fault for */
   path: string
-  /** HTTP status code to return */
-  status: number
-  /** Number of times to return this error (default: 1) */
+  /** HTTP status code to return (if set, returns error response) */
+  status?: number
+  /** Number of times to trigger this fault (default: 1) */
   count?: number
   /** Optional Retry-After header value (seconds) */
   retryAfter?: number
+  /** Delay in milliseconds before responding */
+  delayMs?: number
+  /** Drop the connection after sending headers (simulates network failure) */
+  dropConnection?: boolean
+  /** Truncate response body to this many bytes */
+  truncateBodyBytes?: number
+  /** Probability of triggering fault (0-1, default 1.0 = always) */
+  probability?: number
+  /** Only match specific HTTP method (GET, POST, PUT, DELETE) */
+  method?: string
+  /** Corrupt the response body by flipping random bits */
+  corruptBody?: boolean
+  /** Add jitter to delay (random 0-jitterMs added to delayMs) */
+  jitterMs?: number
 }
 
 /**
@@ -305,6 +413,8 @@ export type TestOperation =
   | ConnectOperation
   | AppendOperation
   | AppendBatchOperation
+  | IdempotentAppendOperation
+  | IdempotentAppendBatchOperation
   | ReadOperation
   | HeadOperation
   | DeleteOperation
@@ -345,13 +455,23 @@ export interface ConnectExpectation extends BaseExpectation {
 }
 
 export interface AppendExpectation extends BaseExpectation {
-  status?: 200 | 404 | 409 | number
+  status?: 200 | 204 | 400 | 403 | 404 | 409 | number
   /** Store the returned offset */
   storeOffsetAs?: string
   /** Expected headers that were sent (for dynamic header testing) */
   headersSent?: Record<string, string>
   /** Expected params that were sent (for dynamic param testing) */
   paramsSent?: Record<string, string>
+  /** Expected duplicate flag (true for 204 idempotent success) */
+  duplicate?: boolean
+  /** Expected producer epoch in response */
+  producerEpoch?: number
+  /** Expected producer seq in response (highest accepted sequence) */
+  producerSeq?: number
+  /** Expected producer expected seq (on 409 sequence gap) */
+  producerExpectedSeq?: number
+  /** Expected producer received seq (on 409 sequence gap) */
+  producerReceivedSeq?: number
 }
 
 export interface AppendBatchExpectation extends BaseExpectation {
@@ -371,6 +491,8 @@ export interface ReadExpectation extends BaseExpectation {
   dataContains?: string
   /** Expected data to contain all of these substrings */
   dataContainsAll?: Array<string>
+  /** Expected exact messages in order (for JSON streams, verifies each chunk) */
+  dataExact?: Array<string>
   /** Expected number of chunks */
   chunkCount?: number
   /** Minimum number of chunks */

package/test-cases/consumer/error-handling.yaml

@@ -56,6 +56,48 @@ tests:
           status: 400
           errorCode: INVALID_OFFSET
 
+  - id: read-offset-now-nonexistent
+    name: Read with offset=now on non-existent stream
+    description: offset=now should not mask a missing stream - must return 404
+    operations:
+      - action: read
+        path: /nonexistent-offset-now-stream
+        offset: "now"
+        live: false
+        expect:
+          status: 404
+          errorCode: NOT_FOUND
+
+  - id: read-offset-now-nonexistent-longpoll
+    name: Long-poll with offset=now on non-existent stream
+    description: offset=now with long-poll should not mask a missing stream - must return 404
+    requires:
+      - longPoll
+    operations:
+      - action: read
+        path: /nonexistent-offset-now-longpoll
+        offset: "now"
+        live: long-poll
+        timeoutMs: 1000
+        expect:
+          status: 404
+          errorCode: NOT_FOUND
+
+  - id: read-offset-now-nonexistent-sse
+    name: SSE with offset=now on non-existent stream
+    description: offset=now with SSE should not mask a missing stream - must return 404
+    requires:
+      - sse
+    operations:
+      - action: read
+        path: /nonexistent-offset-now-sse
+        offset: "now"
+        live: sse
+        timeoutMs: 1000
+        expect:
+          status: 404
+          errorCode: NOT_FOUND
+
   - id: read-future-offset
     name: Read with future offset
     description: Client should handle offset beyond stream end