orez-sync-cf-host 0.4.52 → 0.4.53

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "orez-sync-cf-host",
-  "version": "0.4.52",
+  "version": "0.4.53",
   "type": "module",
   "exports": {
     ".": "./src/index.ts"
@@ -11,7 +11,7 @@
     "bundle": "bun run build:wasm && wrangler deploy --dry-run --outdir dist",
     "measure": "bun run build:wasm && bun measure.mjs",
     "test": "bun run test:config && bun run test:platform && bun run test:integration && bun run test:ingest && bun run test:restart",
-    "test:config": "bun test config.test.mjs",
+    "test:config": "bun test config.test.mjs write-safeguards.test.mjs",
     "test:integration": "bun run build:wasm && bun integration-test.mjs",
     "test:ingest": "bun run build:wasm && bun ingest-test.mjs",
     "test:restart": "bun restart-test.mjs",

package/src/config.ts

@@ -21,6 +21,9 @@ export function validateSyncHostConfig<Env extends SyncHostEnv>(
   if (config.mutateBinding !== undefined && !hasDelegate) {
     throw new TypeError('sync host config mutateBinding requires mutateUrl')
   }
+  if (config.delegatedPushRetry !== undefined && !hasDelegate) {
+    throw new TypeError('sync host config delegatedPushRetry requires mutateUrl')
+  }
   if (config.mutateBinding !== undefined && !config.mutateBinding) {
     throw new TypeError('sync host config mutateBinding must not be empty')
   }
@@ -42,6 +45,26 @@ export function validateSyncHostConfig<Env extends SyncHostEnv>(
     if (!Number.isSafeInteger(interval) || interval < 1_000) {
       throw new TypeError('upstream.intervalMs must be a safe integer >= 1000')
     }
+    for (const [name, value] of Object.entries({
+      ingestBudgetRows: config.upstream.ingestBudgetRows ?? 250_000,
+      ingestBudgetWindowMs: config.upstream.ingestBudgetWindowMs ?? 300_000,
+      ingestBackoffMs: config.upstream.ingestBackoffMs ?? 1_000,
+      ingestMaxBackoffMs: config.upstream.ingestMaxBackoffMs ?? 60_000,
+    })) {
+      if (!Number.isSafeInteger(value) || value < 1)
+        throw new TypeError(`upstream.${name} must be a positive safe integer`)
+    }
+  }
+  if (config.delegatedPushRetry) {
+    for (const [name, value] of Object.entries({
+      maxAttempts: config.delegatedPushRetry.maxAttempts ?? 3,
+      initialBackoffMs: config.delegatedPushRetry.initialBackoffMs ?? 100,
+      maxBackoffMs: config.delegatedPushRetry.maxBackoffMs ?? 1_000,
+      timeoutMs: config.delegatedPushRetry.timeoutMs ?? 5_000,
+    })) {
+      if (!Number.isSafeInteger(value) || value < 1)
+        throw new TypeError(`delegatedPushRetry.${name} must be a positive safe integer`)
+    }
   }
   return config
 }

package/src/host.ts

@@ -28,6 +28,12 @@ import {
   SqlStorageSyncDb,
 } from './sql-storage-adapter.js'
 import { MutationApplicationError } from './types.js'
+import {
+  IngestBreakerError,
+  IngestCircuitBreaker,
+  retryDelayMs,
+  shouldRetryDelegatedPush,
+} from './write-safeguards.js'
 
 import type {
   DeferredEffect,
@@ -141,6 +147,18 @@ function errorMessage(error: unknown): string {
   return error instanceof Error ? error.message : String(error)
 }
 
+function errorBody(error: unknown): Record<string, unknown> {
+  if (error instanceof IngestBreakerError) {
+    return {
+      error: error.error,
+      windowRows: error.windowRows,
+      budget: error.budget,
+      retryAfterMs: error.retryAfterMs,
+    }
+  }
+  return { error: errorMessage(error) }
+}
+
 function requestError(message: string, status = 400): Error & { status: number } {
   return Object.assign(new Error(message), { status })
 }
@@ -268,6 +286,14 @@ export function createSyncDurableObject<Env extends SyncHostEnv>(
   const wakeCoalesceMs = config.wakeCoalesceMs ?? 25
   const upstreamIntervalMs = config.upstream?.intervalMs ?? 15_000
   const upstreamLimit = config.upstream?.changeLimit ?? 1_000
+  const ingestBudgetRows = config.upstream?.ingestBudgetRows ?? 250_000
+  const ingestBudgetWindowMs = config.upstream?.ingestBudgetWindowMs ?? 5 * 60_000
+  const ingestBackoffMs = config.upstream?.ingestBackoffMs ?? 1_000
+  const ingestMaxBackoffMs = config.upstream?.ingestMaxBackoffMs ?? 60_000
+  const delegateMaxAttempts = config.delegatedPushRetry?.maxAttempts ?? 3
+  const delegateInitialBackoffMs = config.delegatedPushRetry?.initialBackoffMs ?? 100
+  const delegateMaxBackoffMs = config.delegatedPushRetry?.maxBackoffMs ?? 1_000
+  const delegateTimeoutMs = config.delegatedPushRetry?.timeoutMs ?? 5_000
 
   return class SyncDurableObject extends DurableObject<Env> {
     readonly #engineDb: SqlStorageSyncDb
@@ -283,6 +309,13 @@ export function createSyncDurableObject<Env extends SyncHostEnv>(
     #wakeRecipients = new Set<WebSocket>()
     #wakePromise: Promise<void> | null = null
     #ingestPromise: Promise<number> | null = null
+    #ingestBreaker = new IngestCircuitBreaker({
+      budgetRows: ingestBudgetRows,
+      windowMs: ingestBudgetWindowMs,
+      initialBackoffMs: ingestBackoffMs,
+      maxBackoffMs: ingestMaxBackoffMs,
+      now: () => Date.now(),
+    })
 
     constructor(ctx: DurableObjectState, env: Env) {
       super(ctx, env)
@@ -301,6 +334,17 @@ export function createSyncDurableObject<Env extends SyncHostEnv>(
           this.#directSql.exec(
             "INSERT OR IGNORE INTO _zsync_host_control (key, value) VALUES ('writerEnabled', '1')"
           )
+          const ingestBreakerReason = this.#controlGet('ingestBreakerReason')
+          if (
+            ingestBreakerReason === 'ingestBudgetExceeded' ||
+            ingestBreakerReason === 'ingestCursorStalled'
+          ) {
+            this.#ingestBreaker.restore(
+              ingestBreakerReason,
+              Number(this.#controlGet('ingestBreakerRetryAt')),
+              Number(this.#controlGet('ingestBreakerTrips'))
+            )
+          }
           this.#wasm(() => engine_init_schema(this.#engineDb, config.schema))
           if (config.queryAware || config.resolveQuery)
             this.#wasm(() => engine_init_query_schema(this.#engineDb))
@@ -368,6 +412,34 @@ export function createSyncDurableObject<Env extends SyncHostEnv>(
       )
     }
 
+    #controlDelete(...keys: string[]): void {
+      if (keys.length === 0) return
+      this.#directSql.exec(
+        `DELETE FROM _zsync_host_control WHERE key IN (${keys.map(() => '?').join(', ')})`,
+        keys
+      )
+    }
+
+    #persistIngestBreaker(): void {
+      const status = this.#ingestBreaker.status()
+      if (!status.reason || status.retryAt === null) return
+      this.#controlSet('ingestBreakerReason', status.reason)
+      this.#controlSet('ingestBreakerRetryAt', String(status.retryAt))
+      this.#controlSet('ingestBreakerTrips', String(status.consecutiveTrips))
+    }
+
+    #recoverIngestBreaker(): void {
+      const wasTripped = this.#ingestBreaker.status().reason !== null
+      this.#ingestBreaker.recovered()
+      if (wasTripped) {
+        this.#controlDelete(
+          'ingestBreakerReason',
+          'ingestBreakerRetryAt',
+          'ingestBreakerTrips'
+        )
+      }
+    }
+
     #writerEnabled(): boolean {
       return this.#controlGet('writerEnabled') === '1'
     }
@@ -437,9 +509,49 @@ export function createSyncDurableObject<Env extends SyncHostEnv>(
       return this.#controlGet('upstreamPath')
     }
 
+    #tripIngest(
+      reason: 'ingestBudgetExceeded' | 'ingestCursorStalled',
+      fields: Record<string, unknown>
+    ): never {
+      try {
+        return this.#ingestBreaker.trip(reason)
+      } catch (error) {
+        this.#persistIngestBreaker()
+        const status = this.#ingestBreaker.status()
+        console.error(
+          JSON.stringify({
+            event: 'sync_upstream_ingest_breaker_tripped',
+            ...status,
+            reason,
+            ...fields,
+          })
+        )
+        throw error
+      }
+    }
+
+    #recordIngestRows(rows: number, fields: Record<string, unknown>): void {
+      try {
+        this.#ingestBreaker.record(rows)
+      } catch (error) {
+        this.#persistIngestBreaker()
+        const status = this.#ingestBreaker.status()
+        console.error(
+          JSON.stringify({
+            event: 'sync_upstream_ingest_breaker_tripped',
+            ...status,
+            reason: 'ingestBudgetExceeded',
+            ...fields,
+          })
+        )
+        throw error
+      }
+    }
+
     #ingest(upstreamPath?: string | null): Promise<number> {
       if (!config.upstream) return Promise.resolve(0)
       if (this.#ingestPromise) return this.#ingestPromise
+      this.#ingestBreaker.assertReady()
       const path = upstreamPath ?? this.#controlGet('upstreamPath')
       if (!path) return Promise.resolve(0)
       this.#ingestPromise = (async () => {
@@ -468,6 +580,20 @@ export function createSyncDurableObject<Env extends SyncHostEnv>(
               )
             ) as ApplyUpstreamResult
             total += rebuilt.applied
+            this.#recordIngestRows(rebuilt.applied, {
+              phase: 'snapshot',
+              cursor,
+              resultWatermark: rebuilt.watermark,
+            })
+            const nextCursor = this.#engineState()?.upstreamWatermark ?? cursor
+            if (String(nextCursor) === String(cursor)) {
+              this.#tripIngest('ingestCursorStalled', {
+                phase: 'snapshot',
+                cursor,
+                resultWatermark: rebuilt.watermark,
+                applied: rebuilt.applied,
+              })
+            }
             continue
           }
           if (!response.ok) {
@@ -481,11 +607,33 @@ export function createSyncDurableObject<Env extends SyncHostEnv>(
             this.#wasm(() => engine_apply_upstream(this.#engineDb, config.schema, batch))
           ) as ApplyUpstreamResult
           total += result.applied
+          this.#recordIngestRows(result.applied, {
+            phase: 'changes',
+            cursor,
+            batchWatermark: batch.watermark,
+          })
+          const nextCursor = this.#engineState()?.upstreamWatermark ?? cursor
+          if (batch.changes.length > 0 && String(nextCursor) === String(cursor)) {
+            this.#tripIngest('ingestCursorStalled', {
+              phase: 'changes',
+              cursor,
+              batchWatermark: batch.watermark,
+              changeRows: batch.changes.length,
+              applied: result.applied,
+            })
+          }
           if (result.caughtUp) break
           if (result.applied === 0) {
-            throw new Error('upstream change cursor made no progress')
+            this.#tripIngest('ingestCursorStalled', {
+              phase: 'changes',
+              cursor,
+              batchWatermark: batch.watermark,
+              changeRows: batch.changes.length,
+              applied: result.applied,
+            })
           }
         }
+        this.#recoverIngestBreaker()
         if (total > 0) await this.#enqueueWake('__upstream__')
         return total
       })().finally(() => {
@@ -494,6 +642,58 @@ export function createSyncDurableObject<Env extends SyncHostEnv>(
       return this.#ingestPromise
     }
 
+    async #fetchDelegatedPush(
+      endpoint: URL,
+      headers: Headers,
+      body: ArrayBuffer
+    ): Promise<Response> {
+      const binding = this.#serviceBinding(
+        config.mutateBinding ?? config.upstream?.binding
+      )
+      let lastError: unknown = null
+      for (let attempt = 1; attempt <= delegateMaxAttempts; attempt++) {
+        let response: Response | null = null
+        try {
+          response = await binding.fetch(endpoint.toString(), {
+            method: 'POST',
+            headers,
+            body,
+            signal: AbortSignal.timeout(delegateTimeoutMs),
+          })
+        } catch (error) {
+          lastError = error
+        }
+        if (
+          !shouldRetryDelegatedPush(
+            response?.status ?? null,
+            attempt,
+            delegateMaxAttempts
+          )
+        ) {
+          if (response) return response
+          throw lastError
+        }
+        await response?.body?.cancel()
+        const delayMs = retryDelayMs(
+          attempt,
+          delegateInitialBackoffMs,
+          delegateMaxBackoffMs
+        )
+        console.warn(
+          JSON.stringify({
+            event: 'sync_delegated_push_retry',
+            attempt,
+            maxAttempts: delegateMaxAttempts,
+            status: response?.status ?? null,
+            delayMs,
+            error: response ? null : errorMessage(lastError),
+          })
+        )
+        await scheduler.wait(delayMs)
+      }
+      throw lastError ?? new Error('delegated push retry exhausted')
+    }
+
     #log(fields: Record<string, unknown>): void {
       console.log(
         JSON.stringify({
@@ -800,13 +1000,11 @@ export function createSyncDurableObject<Env extends SyncHostEnv>(
           headers.delete(NAMESPACE_HEADER)
           headers.delete(UPSTREAM_PATH_HEADER)
           headers.set('host', endpoint.host)
-          const upstreamResponse = await this.#serviceBinding(
-            config.mutateBinding ?? config.upstream?.binding
-          ).fetch(endpoint.toString(), {
-            method: 'POST',
+          const upstreamResponse = await this.#fetchDelegatedPush(
+            endpoint,
             headers,
-            body: bytes,
-          })
+            bytes
+          )
           if (!upstreamResponse.ok) {
             return new Response(upstreamResponse.body, upstreamResponse)
           }
@@ -860,7 +1058,7 @@ export function createSyncDurableObject<Env extends SyncHostEnv>(
           return json({ pushResponse: upstreamBody.pushResponse ?? upstreamBody })
         } catch (error) {
           const status = statusOf(error)
-          return json({ error: errorMessage(error) }, status)
+          return json(errorBody(error), status)
         }
       }
       try {
@@ -1083,6 +1281,7 @@ export function createSyncDurableObject<Env extends SyncHostEnv>(
           heapLimitBytes: heap?.jsHeapSizeLimit ?? null,
           engine: this.#engineState(),
           counters: this.#counters,
+          ingestBreaker: this.#ingestBreaker.status(),
         })
       }
       if (route === '/admin/sql') {
@@ -1151,6 +1350,17 @@ export function createSyncDurableObject<Env extends SyncHostEnv>(
             return json({ ok: true, writerEnabled: enabled })
           })
       }
+      if (route === '/admin/ingest-breaker') {
+        if (request.method === 'GET') return json(this.#ingestBreaker.status())
+        this.#ingestBreaker.reopen()
+        this.#controlDelete(
+          'ingestBreakerReason',
+          'ingestBreakerRetryAt',
+          'ingestBreakerTrips'
+        )
+        console.log(JSON.stringify({ event: 'sync_upstream_ingest_breaker_reopened' }))
+        return json({ ok: true, ...this.#ingestBreaker.status() })
+      }
       if (route === '/admin/fault') {
         return request
           .json()
@@ -1202,7 +1412,7 @@ export function createSyncDurableObject<Env extends SyncHostEnv>(
           const applied = await this.#ingest(upstreamPath)
           return json({ ok: true, applied })
         } catch (error) {
-          return json({ error: errorMessage(error) }, statusOf(error))
+          return json(errorBody(error), statusOf(error))
         }
       }
 
@@ -1212,7 +1422,10 @@ export function createSyncDurableObject<Env extends SyncHostEnv>(
         try {
           await this.#ingest(upstreamPath)
         } catch (error) {
-          return json({ error: errorMessage(error) }, statusOf(error))
+          // Workerd requires a forwarded request body to be consumed even when
+          // ingest fails before the pull handler parses it.
+          await request.arrayBuffer()
+          return json(errorBody(error), statusOf(error))
         }
         return this.#pull(request, claims, namespace)
       }
@@ -1234,7 +1447,10 @@ export function createSyncDurableObject<Env extends SyncHostEnv>(
           })
         )
       } finally {
-        await this.ctx.storage.setAlarm(Date.now() + upstreamIntervalMs)
+        const retryAfterMs = this.#ingestBreaker.status().retryAfterMs
+        await this.ctx.storage.setAlarm(
+          Date.now() + Math.max(upstreamIntervalMs, retryAfterMs)
+        )
       }
     }
 

package/src/ingest-harness-worker.ts

@@ -27,11 +27,21 @@ interface Env extends SyncHostEnv {
   UPSTREAM_DO: DurableObjectNamespace
 }
 
+const runawayNamespaces = new Set<string>()
+let delegatedFailuresRemaining = 0
+let delegatedAttempts = 0
+
 const config: SyncHostConfig<Env> = {
   hostVersion: 'upstream-ingest-harness',
   schema,
   mutateUrl: '/api/zero/push',
   mutateBinding: 'APP',
+  delegatedPushRetry: {
+    maxAttempts: 3,
+    initialBackoffMs: 10,
+    maxBackoffMs: 20,
+    timeoutMs: 1_000,
+  },
   upstream: {
     binding: 'DATA',
     namespacePath: (namespace) => `/${namespace}`,
@@ -85,12 +95,36 @@ async function upstreamFetch(request: Request, env: Env): Promise<Response> {
 /** Self service-binding target backed by the real ZeroSqlDO. */
 export class DataService extends WorkerEntrypoint<Env> {
   fetch(request: Request): Promise<Response> {
-    const pathname = new URL(request.url).pathname
+    const url = new URL(request.url)
+    const pathname = url.pathname
     if (!pathname.endsWith('/changes') && !pathname.endsWith('/snapshot')) {
       return Promise.resolve(
         new Response('DATA route rejected non-feed request', { status: 418 })
       )
     }
+    const namespace = pathname.split('/')[1] ?? ''
+    if (pathname.endsWith('/changes') && runawayNamespaces.has(namespace)) {
+      return Promise.resolve(
+        Response.json({
+          watermark: 100,
+          changes: [
+            {
+              watermark: 1,
+              tableName: 'item',
+              op: 'INSERT',
+              rowData: {
+                id: 'runaway-replay',
+                label: 'replayed without cursor progress',
+                rank: 1,
+                done: false,
+                meta: null,
+              },
+              oldData: null,
+            },
+          ],
+        })
+      )
+    }
     return upstreamFetch(request, this.env)
   }
 }
@@ -102,6 +136,13 @@ export class AppService extends WorkerEntrypoint<Env> {
         new Response('APP route rejected non-push request', { status: 418 })
       )
     }
+    delegatedAttempts++
+    if (delegatedFailuresRemaining > 0) {
+      delegatedFailuresRemaining--
+      return Promise.resolve(
+        Response.json({ error: 'synthetic delegated push failure' }, { status: 503 })
+      )
+    }
     return upstreamFetch(request, this.env)
   }
 }
@@ -110,6 +151,40 @@ const syncWorker = createSyncWorker(config)
 export default {
   fetch(request: Request, env: Env, ctx: ExecutionContext): Promise<Response> {
     const url = new URL(request.url)
+    if (url.pathname.startsWith('/runaway-control/')) {
+      const namespace = url.pathname.slice('/runaway-control/'.length)
+      return request
+        .json()
+        .catch(() => ({}))
+        .then((body) => {
+          if ((body as { enabled?: unknown }).enabled === true)
+            runawayNamespaces.add(namespace)
+          else runawayNamespaces.delete(namespace)
+          return Response.json({
+            ok: true,
+            namespace,
+            enabled: runawayNamespaces.has(namespace),
+          })
+        })
+    }
+    if (url.pathname === '/delegation-control') {
+      if (request.method === 'GET') {
+        return Promise.resolve(
+          Response.json({ delegatedFailuresRemaining, delegatedAttempts })
+        )
+      }
+      return request
+        .json()
+        .catch(() => ({}))
+        .then((body) => {
+          delegatedFailuresRemaining = Math.max(
+            0,
+            Number((body as { failures?: unknown }).failures) || 0
+          )
+          delegatedAttempts = 0
+          return Response.json({ delegatedFailuresRemaining, delegatedAttempts })
+        })
+    }
     if (url.pathname.startsWith('/upstream/')) {
       url.pathname = url.pathname.slice('/upstream'.length)
       return upstreamFetch(new Request(url, request), env)

package/src/types.ts

@@ -116,6 +116,25 @@ export type UpstreamConfig = {
   changeLimit?: number
   /** Durable Object alarm safety net. Defaults to 15 seconds. */
   intervalMs?: number
+  /** Applied upstream rows allowed per rolling window. Defaults to 250,000. */
+  ingestBudgetRows?: number
+  /** Rolling ingest budget window. Defaults to five minutes. */
+  ingestBudgetWindowMs?: number
+  /** Initial breaker cooldown. Defaults to one second. */
+  ingestBackoffMs?: number
+  /** Maximum breaker cooldown. Defaults to one minute. */
+  ingestMaxBackoffMs?: number
+}
+
+export type DelegatedPushRetryConfig = {
+  /** Total attempts including the first request. Defaults to 3. */
+  maxAttempts?: number
+  /** Initial exponential delay. Defaults to 100ms. */
+  initialBackoffMs?: number
+  /** Delay cap. Defaults to 1,000ms. */
+  maxBackoffMs?: number
+  /** Per-attempt service-binding timeout. Defaults to 5,000ms. */
+  timeoutMs?: number
 }
 
 export type SyncHostConfig<Env extends SyncHostEnv = SyncHostEnv> = {
@@ -126,6 +145,7 @@ export type SyncHostConfig<Env extends SyncHostEnv = SyncHostEnv> = {
   mutateUrl?: string
   /** Env binding for delegated pushes; defaults to upstream.binding. */
   mutateBinding?: string
+  delegatedPushRetry?: DelegatedPushRetryConfig
   /** Required for delegated push; forbidden with local mutators (no dual apply). */
   upstream?: UpstreamConfig
   /** Application DDL and optional seed, called before sync-core schema init. */

package/src/write-safeguards.ts

@@ -0,0 +1,157 @@
+export type IngestBreakerReason = 'ingestBudgetExceeded' | 'ingestCursorStalled'
+
+export type IngestBreakerOptions = {
+  budgetRows: number
+  windowMs: number
+  initialBackoffMs: number
+  maxBackoffMs: number
+  now: () => number
+}
+
+export type IngestBreakerStatus = {
+  windowRows: number
+  budget: number
+  windowMs: number
+  tripped: boolean
+  reason: IngestBreakerReason | null
+  retryAt: number | null
+  retryAfterMs: number
+  consecutiveTrips: number
+}
+
+type Sample = { at: number; rows: number }
+
+export class IngestBreakerError extends Error {
+  readonly status = 429
+
+  constructor(
+    readonly error: IngestBreakerReason,
+    readonly windowRows: number,
+    readonly budget: number,
+    readonly retryAfterMs: number
+  ) {
+    super(`${error}: ${windowRows}/${budget} rows; retry after ${retryAfterMs}ms`)
+    this.name = 'IngestBreakerError'
+  }
+}
+
+export class IngestCircuitBreaker {
+  readonly #options: IngestBreakerOptions
+  #samples: Sample[] = []
+  #windowRows = 0
+  #reason: IngestBreakerReason | null = null
+  #retryAt: number | null = null
+  #consecutiveTrips = 0
+
+  constructor(options: IngestBreakerOptions) {
+    for (const [name, value] of Object.entries(options)) {
+      if (name === 'now') continue
+      if (!Number.isSafeInteger(value) || Number(value) < 1)
+        throw new TypeError(`${name} must be a positive safe integer`)
+    }
+    this.#options = options
+  }
+
+  #prune(now: number): void {
+    const cutoff = now - this.#options.windowMs
+    let remove = 0
+    while (remove < this.#samples.length && this.#samples[remove]!.at <= cutoff) {
+      this.#windowRows -= this.#samples[remove]!.rows
+      remove++
+    }
+    if (remove > 0) this.#samples.splice(0, remove)
+  }
+
+  status(): IngestBreakerStatus {
+    const now = this.#options.now()
+    this.#prune(now)
+    const retryAfterMs = Math.max(0, (this.#retryAt ?? now) - now)
+    return {
+      windowRows: this.#windowRows,
+      budget: this.#options.budgetRows,
+      windowMs: this.#options.windowMs,
+      tripped: this.#reason !== null && retryAfterMs > 0,
+      reason: this.#reason,
+      retryAt: this.#retryAt,
+      retryAfterMs,
+      consecutiveTrips: this.#consecutiveTrips,
+    }
+  }
+
+  assertReady(): void {
+    const status = this.status()
+    if (status.tripped && status.reason) {
+      throw new IngestBreakerError(
+        status.reason,
+        status.windowRows,
+        status.budget,
+        status.retryAfterMs
+      )
+    }
+  }
+
+  record(rows: number): void {
+    this.assertReady()
+    if (!Number.isSafeInteger(rows) || rows <= 0) return
+    const now = this.#options.now()
+    this.#prune(now)
+    const last = this.#samples[this.#samples.length - 1]
+    if (last?.at === now) last.rows += rows
+    else this.#samples.push({ at: now, rows })
+    this.#windowRows += rows
+    if (this.#windowRows > this.#options.budgetRows) this.trip('ingestBudgetExceeded')
+  }
+
+  trip(reason: IngestBreakerReason): never {
+    const now = this.#options.now()
+    this.#consecutiveTrips++
+    const delay = Math.min(
+      this.#options.maxBackoffMs,
+      this.#options.initialBackoffMs * 2 ** (this.#consecutiveTrips - 1)
+    )
+    this.#reason = reason
+    this.#retryAt = now + delay
+    throw new IngestBreakerError(
+      reason,
+      this.#windowRows,
+      this.#options.budgetRows,
+      delay
+    )
+  }
+
+  recovered(): void {
+    this.#reason = null
+    this.#retryAt = null
+    this.#consecutiveTrips = 0
+  }
+
+  restore(reason: IngestBreakerReason, retryAt: number, consecutiveTrips: number): void {
+    if (!Number.isFinite(retryAt) || retryAt <= 0) return
+    this.#reason = reason
+    this.#retryAt = retryAt
+    this.#consecutiveTrips = Math.max(1, Math.floor(consecutiveTrips) || 1)
+  }
+
+  reopen(): void {
+    this.#samples = []
+    this.#windowRows = 0
+    this.recovered()
+  }
+}
+
+export function retryDelayMs(
+  attempt: number,
+  initialBackoffMs: number,
+  maxBackoffMs: number
+): number {
+  return Math.min(maxBackoffMs, initialBackoffMs * 2 ** Math.max(0, attempt - 1))
+}
+
+export function shouldRetryDelegatedPush(
+  responseStatus: number | null,
+  attempt: number,
+  maxAttempts: number
+): boolean {
+  if (attempt >= maxAttempts) return false
+  return responseStatus === null || responseStatus === 429 || responseStatus >= 500
+}