orez-sync-cf-host 0.4.52 → 0.4.54

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "orez-sync-cf-host",
-  "version": "0.4.52",
+  "version": "0.4.54",
   "type": "module",
   "exports": {
     ".": "./src/index.ts"
@@ -11,7 +11,7 @@
     "bundle": "bun run build:wasm && wrangler deploy --dry-run --outdir dist",
     "measure": "bun run build:wasm && bun measure.mjs",
     "test": "bun run test:config && bun run test:platform && bun run test:integration && bun run test:ingest && bun run test:restart",
-    "test:config": "bun test config.test.mjs",
+    "test:config": "bun test config.test.mjs write-safeguards.test.mjs",
     "test:integration": "bun run build:wasm && bun integration-test.mjs",
     "test:ingest": "bun run build:wasm && bun ingest-test.mjs",
     "test:restart": "bun restart-test.mjs",

package/src/config.ts

@@ -21,6 +21,9 @@ export function validateSyncHostConfig<Env extends SyncHostEnv>(
   if (config.mutateBinding !== undefined && !hasDelegate) {
     throw new TypeError('sync host config mutateBinding requires mutateUrl')
   }
+  if (config.delegatedPushRetry !== undefined && !hasDelegate) {
+    throw new TypeError('sync host config delegatedPushRetry requires mutateUrl')
+  }
   if (config.mutateBinding !== undefined && !config.mutateBinding) {
     throw new TypeError('sync host config mutateBinding must not be empty')
   }
@@ -42,6 +45,26 @@ export function validateSyncHostConfig<Env extends SyncHostEnv>(
     if (!Number.isSafeInteger(interval) || interval < 1_000) {
       throw new TypeError('upstream.intervalMs must be a safe integer >= 1000')
     }
+    for (const [name, value] of Object.entries({
+      ingestBudgetRows: config.upstream.ingestBudgetRows ?? 150_000,
+      ingestBudgetWindowMs: config.upstream.ingestBudgetWindowMs ?? 300_000,
+      ingestBackoffMs: config.upstream.ingestBackoffMs ?? 1_000,
+      ingestMaxBackoffMs: config.upstream.ingestMaxBackoffMs ?? 60_000,
+    })) {
+      if (!Number.isSafeInteger(value) || value < 1)
+        throw new TypeError(`upstream.${name} must be a positive safe integer`)
+    }
+  }
+  if (config.delegatedPushRetry) {
+    for (const [name, value] of Object.entries({
+      maxAttempts: config.delegatedPushRetry.maxAttempts ?? 3,
+      initialBackoffMs: config.delegatedPushRetry.initialBackoffMs ?? 100,
+      maxBackoffMs: config.delegatedPushRetry.maxBackoffMs ?? 1_000,
+      timeoutMs: config.delegatedPushRetry.timeoutMs ?? 5_000,
+    })) {
+      if (!Number.isSafeInteger(value) || value < 1)
+        throw new TypeError(`delegatedPushRetry.${name} must be a positive safe integer`)
+    }
   }
   return config
 }

package/src/host.ts

@@ -28,6 +28,12 @@ import {
   SqlStorageSyncDb,
 } from './sql-storage-adapter.js'
 import { MutationApplicationError } from './types.js'
+import {
+  IngestBreakerError,
+  IngestCircuitBreaker,
+  retryDelayMs,
+  shouldRetryDelegatedPush,
+} from './write-safeguards.js'
 
 import type {
   DeferredEffect,
@@ -141,6 +147,18 @@ function errorMessage(error: unknown): string {
   return error instanceof Error ? error.message : String(error)
 }
 
+function errorBody(error: unknown): Record<string, unknown> {
+  if (error instanceof IngestBreakerError) {
+    return {
+      error: error.error,
+      windowRows: error.windowRows,
+      budget: error.budget,
+      retryAfterMs: error.retryAfterMs,
+    }
+  }
+  return { error: errorMessage(error) }
+}
+
 function requestError(message: string, status = 400): Error & { status: number } {
   return Object.assign(new Error(message), { status })
 }
@@ -268,6 +286,14 @@ export function createSyncDurableObject<Env extends SyncHostEnv>(
   const wakeCoalesceMs = config.wakeCoalesceMs ?? 25
   const upstreamIntervalMs = config.upstream?.intervalMs ?? 15_000
   const upstreamLimit = config.upstream?.changeLimit ?? 1_000
+  const ingestBudgetRows = config.upstream?.ingestBudgetRows ?? 150_000
+  const ingestBudgetWindowMs = config.upstream?.ingestBudgetWindowMs ?? 5 * 60_000
+  const ingestBackoffMs = config.upstream?.ingestBackoffMs ?? 1_000
+  const ingestMaxBackoffMs = config.upstream?.ingestMaxBackoffMs ?? 60_000
+  const delegateMaxAttempts = config.delegatedPushRetry?.maxAttempts ?? 3
+  const delegateInitialBackoffMs = config.delegatedPushRetry?.initialBackoffMs ?? 100
+  const delegateMaxBackoffMs = config.delegatedPushRetry?.maxBackoffMs ?? 1_000
+  const delegateTimeoutMs = config.delegatedPushRetry?.timeoutMs ?? 5_000
 
   return class SyncDurableObject extends DurableObject<Env> {
     readonly #engineDb: SqlStorageSyncDb
@@ -283,11 +309,22 @@ export function createSyncDurableObject<Env extends SyncHostEnv>(
     #wakeRecipients = new Set<WebSocket>()
     #wakePromise: Promise<void> | null = null
     #ingestPromise: Promise<number> | null = null
+    #recordingIngestBillable = false
+    #ingestBreaker = new IngestCircuitBreaker({
+      budgetRows: ingestBudgetRows,
+      windowMs: ingestBudgetWindowMs,
+      initialBackoffMs: ingestBackoffMs,
+      maxBackoffMs: ingestMaxBackoffMs,
+      now: () => Date.now(),
+    })
 
     constructor(ctx: DurableObjectState, env: Env) {
       super(ctx, env)
-      this.#engineDb = new SqlStorageSyncDb(ctx.storage.sql)
-      this.#directSql = new SqlStorageDirect(ctx.storage.sql)
+      const recordRowsWritten = (rows: number) => {
+        if (this.#recordingIngestBillable) this.#ingestBreaker.recordBillable(rows)
+      }
+      this.#engineDb = new SqlStorageSyncDb(ctx.storage.sql, recordRowsWritten)
+      this.#directSql = new SqlStorageDirect(ctx.storage.sql, recordRowsWritten)
       this.#mutatorSql = new SqlStorageMutatorTransaction(this.#directSql, (ast) =>
         this.#wasm(() => engine_compile_query(config.schema, ast))
       )
@@ -301,6 +338,17 @@ export function createSyncDurableObject<Env extends SyncHostEnv>(
           this.#directSql.exec(
             "INSERT OR IGNORE INTO _zsync_host_control (key, value) VALUES ('writerEnabled', '1')"
           )
+          const ingestBreakerReason = this.#controlGet('ingestBreakerReason')
+          if (
+            ingestBreakerReason === 'ingestBudgetExceeded' ||
+            ingestBreakerReason === 'ingestCursorStalled'
+          ) {
+            this.#ingestBreaker.restore(
+              ingestBreakerReason,
+              Number(this.#controlGet('ingestBreakerRetryAt')),
+              Number(this.#controlGet('ingestBreakerTrips'))
+            )
+          }
           this.#wasm(() => engine_init_schema(this.#engineDb, config.schema))
           if (config.queryAware || config.resolveQuery)
             this.#wasm(() => engine_init_query_schema(this.#engineDb))
@@ -368,6 +416,34 @@ export function createSyncDurableObject<Env extends SyncHostEnv>(
       )
     }
 
+    #controlDelete(...keys: string[]): void {
+      if (keys.length === 0) return
+      this.#directSql.exec(
+        `DELETE FROM _zsync_host_control WHERE key IN (${keys.map(() => '?').join(', ')})`,
+        keys
+      )
+    }
+
+    #persistIngestBreaker(): void {
+      const status = this.#ingestBreaker.status()
+      if (!status.reason || status.retryAt === null) return
+      this.#controlSet('ingestBreakerReason', status.reason)
+      this.#controlSet('ingestBreakerRetryAt', String(status.retryAt))
+      this.#controlSet('ingestBreakerTrips', String(status.consecutiveTrips))
+    }
+
+    #recoverIngestBreaker(): void {
+      const wasTripped = this.#ingestBreaker.status().reason !== null
+      this.#ingestBreaker.recovered()
+      if (wasTripped) {
+        this.#controlDelete(
+          'ingestBreakerReason',
+          'ingestBreakerRetryAt',
+          'ingestBreakerTrips'
+        )
+      }
+    }
+
     #writerEnabled(): boolean {
       return this.#controlGet('writerEnabled') === '1'
     }
@@ -427,6 +503,26 @@ export function createSyncDurableObject<Env extends SyncHostEnv>(
       }
     }
 
+    async #upstreamWriteBudgetStatus(): Promise<Response> {
+      if (!config.upstream) return json({ error: 'upstream is not configured' }, 404)
+      const path = this.#controlGet('upstreamPath')
+      if (!path) return json({ error: 'upstream path is not known yet' }, 409)
+      const endpoint = new URL(`${path}/_orez/write-budget`, 'https://upstream.invalid')
+      const response = await this.#serviceBinding().fetch(endpoint.toString(), {
+        headers: { host: endpoint.host },
+      })
+      if (!response.ok) {
+        return json(
+          {
+            error: 'upstream write-budget status unavailable',
+            upstreamStatus: response.status,
+          },
+          502
+        )
+      }
+      return json(await response.json())
+    }
+
     #rememberUpstreamPath(request: Request): string | null {
       if (!config.upstream) return null
       const path = request.headers.get(UPSTREAM_PATH_HEADER)
@@ -437,9 +533,59 @@ export function createSyncDurableObject<Env extends SyncHostEnv>(
       return this.#controlGet('upstreamPath')
     }
 
+    #tripIngest(
+      reason: 'ingestBudgetExceeded' | 'ingestCursorStalled',
+      fields: Record<string, unknown>
+    ): never {
+      try {
+        return this.#ingestBreaker.trip(reason)
+      } catch (error) {
+        this.#persistIngestBreaker()
+        const status = this.#ingestBreaker.status()
+        console.error(
+          JSON.stringify({
+            event: 'sync_upstream_ingest_breaker_tripped',
+            ...status,
+            reason,
+            ...fields,
+          })
+        )
+        throw error
+      }
+    }
+
+    #recordIngestLogicalRows(rows: number): void {
+      this.#ingestBreaker.recordLogical(rows)
+    }
+
+    #withIngestBilling<T>(fields: Record<string, unknown>, apply: () => T): T {
+      this.#recordingIngestBillable = true
+      try {
+        return apply()
+      } catch (error) {
+        this.#recordingIngestBillable = false
+        if (error instanceof IngestBreakerError) {
+          this.#persistIngestBreaker()
+          const status = this.#ingestBreaker.status()
+          console.error(
+            JSON.stringify({
+              event: 'sync_upstream_ingest_breaker_tripped',
+              ...status,
+              reason: 'ingestBudgetExceeded',
+              ...fields,
+            })
+          )
+        }
+        throw error
+      } finally {
+        this.#recordingIngestBillable = false
+      }
+    }
+
     #ingest(upstreamPath?: string | null): Promise<number> {
       if (!config.upstream) return Promise.resolve(0)
       if (this.#ingestPromise) return this.#ingestPromise
+      this.#ingestBreaker.assertReady()
       const path = upstreamPath ?? this.#controlGet('upstreamPath')
       if (!path) return Promise.resolve(0)
       this.#ingestPromise = (async () => {
@@ -462,12 +608,30 @@ export function createSyncDurableObject<Env extends SyncHostEnv>(
               throw new Error(`upstream snapshot returned ${snapshotResponse.status}`)
             }
             const snapshot = await snapshotResponse.json()
-            const rebuilt = this.ctx.storage.transactionSync(() =>
-              this.#wasm(() =>
-                engine_apply_upstream_snapshot(this.#engineDb, config.schema, snapshot)
-              )
-            ) as ApplyUpstreamResult
+            const rebuilt = this.#withIngestBilling(
+              { phase: 'snapshot', cursor },
+              () =>
+                this.ctx.storage.transactionSync(() =>
+                  this.#wasm(() =>
+                    engine_apply_upstream_snapshot(
+                      this.#engineDb,
+                      config.schema,
+                      snapshot
+                    )
+                  )
+                ) as ApplyUpstreamResult
+            )
             total += rebuilt.applied
+            this.#recordIngestLogicalRows(rebuilt.applied)
+            const nextCursor = this.#engineState()?.upstreamWatermark ?? cursor
+            if (String(nextCursor) === String(cursor)) {
+              this.#tripIngest('ingestCursorStalled', {
+                phase: 'snapshot',
+                cursor,
+                resultWatermark: rebuilt.watermark,
+                applied: rebuilt.applied,
+              })
+            }
             continue
           }
           if (!response.ok) {
@@ -477,15 +641,44 @@ export function createSyncDurableObject<Env extends SyncHostEnv>(
           if (!Number.isSafeInteger(batch.watermark) || !Array.isArray(batch.changes)) {
             throw new Error('invalid upstream changes response')
           }
-          const result = this.ctx.storage.transactionSync(() =>
-            this.#wasm(() => engine_apply_upstream(this.#engineDb, config.schema, batch))
+          const result = this.#withIngestBilling(
+            {
+              phase: 'changes',
+              cursor,
+              batchWatermark: batch.watermark,
+              changeRows: batch.changes.length,
+            },
+            () =>
+              this.ctx.storage.transactionSync(() =>
+                this.#wasm(() =>
+                  engine_apply_upstream(this.#engineDb, config.schema, batch)
+                )
+              )
           ) as ApplyUpstreamResult
           total += result.applied
+          this.#recordIngestLogicalRows(result.applied)
+          const nextCursor = this.#engineState()?.upstreamWatermark ?? cursor
+          if (batch.changes.length > 0 && String(nextCursor) === String(cursor)) {
+            this.#tripIngest('ingestCursorStalled', {
+              phase: 'changes',
+              cursor,
+              batchWatermark: batch.watermark,
+              changeRows: batch.changes.length,
+              applied: result.applied,
+            })
+          }
           if (result.caughtUp) break
           if (result.applied === 0) {
-            throw new Error('upstream change cursor made no progress')
+            this.#tripIngest('ingestCursorStalled', {
+              phase: 'changes',
+              cursor,
+              batchWatermark: batch.watermark,
+              changeRows: batch.changes.length,
+              applied: result.applied,
+            })
           }
         }
+        this.#recoverIngestBreaker()
         if (total > 0) await this.#enqueueWake('__upstream__')
         return total
       })().finally(() => {
@@ -494,6 +687,58 @@ export function createSyncDurableObject<Env extends SyncHostEnv>(
       return this.#ingestPromise
     }
 
+    async #fetchDelegatedPush(
+      endpoint: URL,
+      headers: Headers,
+      body: ArrayBuffer
+    ): Promise<Response> {
+      const binding = this.#serviceBinding(
+        config.mutateBinding ?? config.upstream?.binding
+      )
+      let lastError: unknown = null
+      for (let attempt = 1; attempt <= delegateMaxAttempts; attempt++) {
+        let response: Response | null = null
+        try {
+          response = await binding.fetch(endpoint.toString(), {
+            method: 'POST',
+            headers,
+            body,
+            signal: AbortSignal.timeout(delegateTimeoutMs),
+          })
+        } catch (error) {
+          lastError = error
+        }
+        if (
+          !shouldRetryDelegatedPush(
+            response?.status ?? null,
+            attempt,
+            delegateMaxAttempts
+          )
+        ) {
+          if (response) return response
+          throw lastError
+        }
+        await response?.body?.cancel()
+        const delayMs = retryDelayMs(
+          attempt,
+          delegateInitialBackoffMs,
+          delegateMaxBackoffMs
+        )
+        console.warn(
+          JSON.stringify({
+            event: 'sync_delegated_push_retry',
+            attempt,
+            maxAttempts: delegateMaxAttempts,
+            status: response?.status ?? null,
+            delayMs,
+            error: response ? null : errorMessage(lastError),
+          })
+        )
+        await scheduler.wait(delayMs)
+      }
+      throw lastError ?? new Error('delegated push retry exhausted')
+    }
+
     #log(fields: Record<string, unknown>): void {
       console.log(
         JSON.stringify({
@@ -800,13 +1045,11 @@ export function createSyncDurableObject<Env extends SyncHostEnv>(
           headers.delete(NAMESPACE_HEADER)
           headers.delete(UPSTREAM_PATH_HEADER)
           headers.set('host', endpoint.host)
-          const upstreamResponse = await this.#serviceBinding(
-            config.mutateBinding ?? config.upstream?.binding
-          ).fetch(endpoint.toString(), {
-            method: 'POST',
+          const upstreamResponse = await this.#fetchDelegatedPush(
+            endpoint,
             headers,
-            body: bytes,
-          })
+            bytes
+          )
           if (!upstreamResponse.ok) {
             return new Response(upstreamResponse.body, upstreamResponse)
           }
@@ -860,7 +1103,7 @@ export function createSyncDurableObject<Env extends SyncHostEnv>(
           return json({ pushResponse: upstreamBody.pushResponse ?? upstreamBody })
         } catch (error) {
           const status = statusOf(error)
-          return json({ error: errorMessage(error) }, status)
+          return json(errorBody(error), status)
         }
       }
       try {
@@ -1060,6 +1303,8 @@ export function createSyncDurableObject<Env extends SyncHostEnv>(
 
     #admin(route: string, request: Request): Promise<Response> | Response {
       if (route === '/admin/health') return json({ ok: true })
+      if (route === '/admin/upstream-write-budget' && request.method === 'GET')
+        return this.#upstreamWriteBudgetStatus()
       if (route === '/admin/status') {
         const heap = (
           performance as Performance & {
@@ -1083,6 +1328,7 @@ export function createSyncDurableObject<Env extends SyncHostEnv>(
           heapLimitBytes: heap?.jsHeapSizeLimit ?? null,
           engine: this.#engineState(),
           counters: this.#counters,
+          ingestBreaker: this.#ingestBreaker.status(),
         })
       }
       if (route === '/admin/sql') {
@@ -1151,6 +1397,17 @@ export function createSyncDurableObject<Env extends SyncHostEnv>(
             return json({ ok: true, writerEnabled: enabled })
           })
       }
+      if (route === '/admin/ingest-breaker') {
+        if (request.method === 'GET') return json(this.#ingestBreaker.status())
+        this.#ingestBreaker.reopen()
+        this.#controlDelete(
+          'ingestBreakerReason',
+          'ingestBreakerRetryAt',
+          'ingestBreakerTrips'
+        )
+        console.log(JSON.stringify({ event: 'sync_upstream_ingest_breaker_reopened' }))
+        return json({ ok: true, ...this.#ingestBreaker.status() })
+      }
       if (route === '/admin/fault') {
         return request
           .json()
@@ -1202,7 +1459,7 @@ export function createSyncDurableObject<Env extends SyncHostEnv>(
           const applied = await this.#ingest(upstreamPath)
           return json({ ok: true, applied })
         } catch (error) {
-          return json({ error: errorMessage(error) }, statusOf(error))
+          return json(errorBody(error), statusOf(error))
         }
       }
 
@@ -1212,7 +1469,10 @@ export function createSyncDurableObject<Env extends SyncHostEnv>(
         try {
           await this.#ingest(upstreamPath)
         } catch (error) {
-          return json({ error: errorMessage(error) }, statusOf(error))
+          // Workerd requires a forwarded request body to be consumed even when
+          // ingest fails before the pull handler parses it.
+          await request.arrayBuffer()
+          return json(errorBody(error), statusOf(error))
         }
         return this.#pull(request, claims, namespace)
       }
@@ -1234,7 +1494,10 @@ export function createSyncDurableObject<Env extends SyncHostEnv>(
           })
         )
       } finally {
-        await this.ctx.storage.setAlarm(Date.now() + upstreamIntervalMs)
+        const retryAfterMs = this.#ingestBreaker.status().retryAfterMs
+        await this.ctx.storage.setAlarm(
+          Date.now() + Math.max(upstreamIntervalMs, retryAfterMs)
+        )
       }
     }
 

package/src/ingest-harness-worker.ts

@@ -27,11 +27,21 @@ interface Env extends SyncHostEnv {
   UPSTREAM_DO: DurableObjectNamespace
 }
 
+const runawayNamespaces = new Set<string>()
+let delegatedFailuresRemaining = 0
+let delegatedAttempts = 0
+
 const config: SyncHostConfig<Env> = {
   hostVersion: 'upstream-ingest-harness',
   schema,
   mutateUrl: '/api/zero/push',
   mutateBinding: 'APP',
+  delegatedPushRetry: {
+    maxAttempts: 3,
+    initialBackoffMs: 10,
+    maxBackoffMs: 20,
+    timeoutMs: 1_000,
+  },
   upstream: {
     binding: 'DATA',
     namespacePath: (namespace) => `/${namespace}`,
@@ -60,6 +70,8 @@ async function upstreamFetch(request: Request, env: Env): Promise<Response> {
   const [, namespace, ...rest] = url.pathname.split('/')
   if (!namespace) return new Response('namespace required', { status: 400 })
   const stub = env.UPSTREAM_DO.get(env.UPSTREAM_DO.idFromName(namespace))
+  url.pathname = `/${rest.join('/')}`
+  if (url.pathname === '/_orez/write-budget') return stub.fetch(new Request(url, request))
   const exec = async (sql: string, params: unknown[] = []) => {
     const response = await stub.fetch('https://upstream.invalid/exec', {
       method: 'POST',
@@ -78,19 +90,46 @@ async function upstreamFetch(request: Request, env: Env): Promise<Response> {
     'INSERT OR IGNORE INTO _zero_schema_tables (name, schema_json) VALUES (?, ?)',
     ['item', JSON.stringify(schema.tables.item)]
   )
-  url.pathname = `/${rest.join('/')}`
   return stub.fetch(new Request(url, request))
 }
 
 /** Self service-binding target backed by the real ZeroSqlDO. */
 export class DataService extends WorkerEntrypoint<Env> {
   fetch(request: Request): Promise<Response> {
-    const pathname = new URL(request.url).pathname
-    if (!pathname.endsWith('/changes') && !pathname.endsWith('/snapshot')) {
+    const url = new URL(request.url)
+    const pathname = url.pathname
+    if (
+      !pathname.endsWith('/changes') &&
+      !pathname.endsWith('/snapshot') &&
+      !pathname.endsWith('/_orez/write-budget')
+    ) {
       return Promise.resolve(
         new Response('DATA route rejected non-feed request', { status: 418 })
       )
     }
+    const namespace = pathname.split('/')[1] ?? ''
+    if (pathname.endsWith('/changes') && runawayNamespaces.has(namespace)) {
+      return Promise.resolve(
+        Response.json({
+          watermark: 100,
+          changes: [
+            {
+              watermark: 1,
+              tableName: 'item',
+              op: 'INSERT',
+              rowData: {
+                id: 'runaway-replay',
+                label: 'replayed without cursor progress',
+                rank: 1,
+                done: false,
+                meta: null,
+              },
+              oldData: null,
+            },
+          ],
+        })
+      )
+    }
     return upstreamFetch(request, this.env)
   }
 }
@@ -102,6 +141,13 @@ export class AppService extends WorkerEntrypoint<Env> {
         new Response('APP route rejected non-push request', { status: 418 })
       )
     }
+    delegatedAttempts++
+    if (delegatedFailuresRemaining > 0) {
+      delegatedFailuresRemaining--
+      return Promise.resolve(
+        Response.json({ error: 'synthetic delegated push failure' }, { status: 503 })
+      )
+    }
     return upstreamFetch(request, this.env)
   }
 }
@@ -110,6 +156,40 @@ const syncWorker = createSyncWorker(config)
 export default {
   fetch(request: Request, env: Env, ctx: ExecutionContext): Promise<Response> {
     const url = new URL(request.url)
+    if (url.pathname.startsWith('/runaway-control/')) {
+      const namespace = url.pathname.slice('/runaway-control/'.length)
+      return request
+        .json()
+        .catch(() => ({}))
+        .then((body) => {
+          if ((body as { enabled?: unknown }).enabled === true)
+            runawayNamespaces.add(namespace)
+          else runawayNamespaces.delete(namespace)
+          return Response.json({
+            ok: true,
+            namespace,
+            enabled: runawayNamespaces.has(namespace),
+          })
+        })
+    }
+    if (url.pathname === '/delegation-control') {
+      if (request.method === 'GET') {
+        return Promise.resolve(
+          Response.json({ delegatedFailuresRemaining, delegatedAttempts })
+        )
+      }
+      return request
+        .json()
+        .catch(() => ({}))
+        .then((body) => {
+          delegatedFailuresRemaining = Math.max(
+            0,
+            Number((body as { failures?: unknown }).failures) || 0
+          )
+          delegatedAttempts = 0
+          return Response.json({ delegatedFailuresRemaining, delegatedAttempts })
+        })
+    }
     if (url.pathname.startsWith('/upstream/')) {
       url.pathname = url.pathname.slice('/upstream'.length)
       return upstreamFetch(new Request(url, request), env)

package/src/sql-storage-adapter.ts

@@ -1,3 +1,5 @@
+import { trackBillableCursorRows } from './write-safeguards.js'
+
 import type { MutatorSql, SyncSql } from './types.js'
 
 type WireValue =
@@ -86,7 +88,10 @@ function encodeResult(value: unknown): WireValue {
 export class SqlStorageSyncDb implements JsSyncDb {
   readonly stats: AdapterStats = { execCalls: 0, queryCalls: 0, sqlMs: 0 }
 
-  constructor(private readonly sql: SqlStorage) {}
+  constructor(
+    private readonly sql: SqlStorage,
+    private readonly recordRowsWritten: (rows: number) => void = () => {}
+  ) {}
 
   resetStats(): void {
     this.stats.execCalls = 0
@@ -97,7 +102,10 @@ export class SqlStorageSyncDb implements JsSyncDb {
   exec(sql: string, params: WireValue[]): void {
     assertHostSql(sql)
     const start = performance.now()
-    this.sql.exec(sql, ...params.map(decodeBinding))
+    trackBillableCursorRows(
+      this.sql.exec(sql, ...params.map(decodeBinding)),
+      this.recordRowsWritten
+    )
     this.stats.execCalls++
     this.stats.sqlMs += performance.now() - start
   }
@@ -105,7 +113,10 @@ export class SqlStorageSyncDb implements JsSyncDb {
   query(sql: string, params: WireValue[]): WireRow[] {
     assertHostSql(sql)
     const start = performance.now()
-    const cursor = this.sql.exec(sql, ...params.map(decodeBinding))
+    const cursor = trackBillableCursorRows(
+      this.sql.exec(sql, ...params.map(decodeBinding)),
+      this.recordRowsWritten
+    )
     const columns = [...cursor.columnNames]
     // A SqlStorage cursor must be fully consumed before any await. Returning a
     // materialized array here makes that property structural.
@@ -121,11 +132,14 @@ export class SqlStorageSyncDb implements JsSyncDb {
 
 /** Direct application SQL surface used for initialization and admin reads. */
 export class SqlStorageDirect implements SyncSql {
-  constructor(private readonly sql: SqlStorage) {}
+  constructor(
+    private readonly sql: SqlStorage,
+    private readonly recordRowsWritten: (rows: number) => void = () => {}
+  ) {}
 
   exec(sql: string, params: readonly unknown[] = []): void {
     assertHostSql(sql)
-    this.sql.exec(sql, ...params)
+    trackBillableCursorRows(this.sql.exec(sql, ...params), this.recordRowsWritten)
   }
 
   query<Row extends Record<string, unknown> = Record<string, unknown>>(
@@ -133,7 +147,10 @@ export class SqlStorageDirect implements SyncSql {
     params: readonly unknown[] = []
   ): Row[] {
     assertHostSql(sql)
-    return this.sql.exec(sql, ...params).toArray() as Row[]
+    return trackBillableCursorRows(
+      this.sql.exec(sql, ...params),
+      this.recordRowsWritten
+    ).toArray() as Row[]
   }
 }
 

package/src/types.ts

@@ -116,6 +116,25 @@ export type UpstreamConfig = {
   changeLimit?: number
   /** Durable Object alarm safety net. Defaults to 15 seconds. */
   intervalMs?: number
+  /** Billable SQLite rows written by ingest per rolling window. Defaults to 150,000. */
+  ingestBudgetRows?: number
+  /** Rolling ingest budget window. Defaults to five minutes. */
+  ingestBudgetWindowMs?: number
+  /** Initial breaker cooldown. Defaults to one second. */
+  ingestBackoffMs?: number
+  /** Maximum breaker cooldown. Defaults to one minute. */
+  ingestMaxBackoffMs?: number
+}
+
+export type DelegatedPushRetryConfig = {
+  /** Total attempts including the first request. Defaults to 3. */
+  maxAttempts?: number
+  /** Initial exponential delay. Defaults to 100ms. */
+  initialBackoffMs?: number
+  /** Delay cap. Defaults to 1,000ms. */
+  maxBackoffMs?: number
+  /** Per-attempt service-binding timeout. Defaults to 5,000ms. */
+  timeoutMs?: number
 }
 
 export type SyncHostConfig<Env extends SyncHostEnv = SyncHostEnv> = {
@@ -126,6 +145,7 @@ export type SyncHostConfig<Env extends SyncHostEnv = SyncHostEnv> = {
   mutateUrl?: string
   /** Env binding for delegated pushes; defaults to upstream.binding. */
   mutateBinding?: string
+  delegatedPushRetry?: DelegatedPushRetryConfig
   /** Required for delegated push; forbidden with local mutators (no dual apply). */
   upstream?: UpstreamConfig
   /** Application DDL and optional seed, called before sync-core schema init. */

package/src/write-safeguards.ts

@@ -0,0 +1,249 @@
+export type IngestBreakerReason = 'ingestBudgetExceeded' | 'ingestCursorStalled'
+
+export type IngestBreakerOptions = {
+  budgetRows: number
+  windowMs: number
+  initialBackoffMs: number
+  maxBackoffMs: number
+  now: () => number
+}
+
+export type IngestBreakerStatus = {
+  windowRows: number
+  billableRows: number
+  logicalRows: number
+  budget: number
+  windowMs: number
+  tripped: boolean
+  reason: IngestBreakerReason | null
+  retryAt: number | null
+  retryAfterMs: number
+  consecutiveTrips: number
+}
+
+type Sample = { at: number; billableRows: number; logicalRows: number }
+
+export class IngestBreakerError extends Error {
+  readonly status = 429
+
+  constructor(
+    readonly error: IngestBreakerReason,
+    readonly windowRows: number,
+    readonly budget: number,
+    readonly retryAfterMs: number
+  ) {
+    super(`${error}: ${windowRows}/${budget} rows; retry after ${retryAfterMs}ms`)
+    this.name = 'IngestBreakerError'
+  }
+}
+
+export class IngestCircuitBreaker {
+  readonly #options: IngestBreakerOptions
+  #samples: Sample[] = []
+  #billableRows = 0
+  #logicalRows = 0
+  #reason: IngestBreakerReason | null = null
+  #retryAt: number | null = null
+  #consecutiveTrips = 0
+
+  constructor(options: IngestBreakerOptions) {
+    for (const [name, value] of Object.entries(options)) {
+      if (name === 'now') continue
+      if (!Number.isSafeInteger(value) || Number(value) < 1)
+        throw new TypeError(`${name} must be a positive safe integer`)
+    }
+    this.#options = options
+  }
+
+  #prune(now: number): void {
+    const cutoff = now - this.#options.windowMs
+    let remove = 0
+    while (remove < this.#samples.length && this.#samples[remove]!.at <= cutoff) {
+      this.#billableRows -= this.#samples[remove]!.billableRows
+      this.#logicalRows -= this.#samples[remove]!.logicalRows
+      remove++
+    }
+    if (remove > 0) this.#samples.splice(0, remove)
+  }
+
+  status(): IngestBreakerStatus {
+    const now = this.#options.now()
+    this.#prune(now)
+    const retryAfterMs = Math.max(0, (this.#retryAt ?? now) - now)
+    return {
+      windowRows: this.#billableRows,
+      billableRows: this.#billableRows,
+      logicalRows: this.#logicalRows,
+      budget: this.#options.budgetRows,
+      windowMs: this.#options.windowMs,
+      tripped: this.#reason !== null && retryAfterMs > 0,
+      reason: this.#reason,
+      retryAt: this.#retryAt,
+      retryAfterMs,
+      consecutiveTrips: this.#consecutiveTrips,
+    }
+  }
+
+  assertReady(): void {
+    const status = this.status()
+    if (status.tripped && status.reason) {
+      throw new IngestBreakerError(
+        status.reason,
+        status.windowRows,
+        status.budget,
+        status.retryAfterMs
+      )
+    }
+  }
+
+  record(rows: number): void {
+    this.recordLogical(rows)
+    this.recordBillable(rows)
+  }
+
+  #sample(now: number): Sample {
+    this.#prune(now)
+    const last = this.#samples[this.#samples.length - 1]
+    if (last?.at === now) return last
+    const sample = { at: now, billableRows: 0, logicalRows: 0 }
+    this.#samples.push(sample)
+    return sample
+  }
+
+  recordLogical(rows: number): void {
+    if (!Number.isSafeInteger(rows) || rows <= 0) return
+    const sample = this.#sample(this.#options.now())
+    sample.logicalRows += rows
+    this.#logicalRows += rows
+  }
+
+  recordBillable(rows: number): void {
+    this.assertReady()
+    if (!Number.isSafeInteger(rows) || rows <= 0) return
+    const now = this.#options.now()
+    const sample = this.#sample(now)
+    sample.billableRows += rows
+    this.#billableRows += rows
+    if (this.#billableRows > this.#options.budgetRows) this.trip('ingestBudgetExceeded')
+  }
+
+  trip(reason: IngestBreakerReason): never {
+    const now = this.#options.now()
+    this.#consecutiveTrips++
+    const delay = Math.min(
+      this.#options.maxBackoffMs,
+      this.#options.initialBackoffMs * 2 ** (this.#consecutiveTrips - 1)
+    )
+    this.#reason = reason
+    this.#retryAt = now + delay
+    throw new IngestBreakerError(
+      reason,
+      this.#billableRows,
+      this.#options.budgetRows,
+      delay
+    )
+  }
+
+  recovered(): void {
+    this.#reason = null
+    this.#retryAt = null
+    this.#consecutiveTrips = 0
+  }
+
+  restore(reason: IngestBreakerReason, retryAt: number, consecutiveTrips: number): void {
+    if (!Number.isFinite(retryAt) || retryAt <= 0) return
+    this.#reason = reason
+    this.#retryAt = retryAt
+    this.#consecutiveTrips = Math.max(1, Math.floor(consecutiveTrips) || 1)
+  }
+
+  reopen(): void {
+    this.#samples = []
+    this.#billableRows = 0
+    this.#logicalRows = 0
+    this.recovered()
+  }
+}
+
+type BillableCursor = {
+  rowsWritten?: number
+  next?: (...args: unknown[]) => unknown
+  one?: (...args: unknown[]) => unknown
+  toArray?: (...args: unknown[]) => unknown
+  raw?: (...args: unknown[]) => unknown
+  [Symbol.iterator]?: () => Iterator<unknown>
+}
+
+export function trackBillableCursorRows<Cursor extends BillableCursor>(
+  cursor: Cursor,
+  record: (rows: number) => void
+): Cursor {
+  if (!cursor || typeof cursor !== 'object') return cursor
+  let accountedRows = 0
+  const account = () => {
+    const current = Number(cursor.rowsWritten ?? 0)
+    if (!Number.isSafeInteger(current) || current <= accountedRows) return
+    const delta = current - accountedRows
+    accountedRows = current
+    record(delta)
+  }
+  const wrapIterator = (iterator: object): object =>
+    new Proxy(iterator, {
+      get(target, property) {
+        const value = Reflect.get(target, property, target)
+        if (property === 'next' && typeof value === 'function') {
+          return (...args: unknown[]) => {
+            try {
+              return Reflect.apply(value, target, args)
+            } finally {
+              account()
+            }
+          }
+        }
+        return typeof value === 'function' ? value.bind(target) : value
+      },
+    })
+  let proxy: Cursor
+  proxy = new Proxy(cursor, {
+    get(target, property) {
+      const value = Reflect.get(target, property, target)
+      if (property === Symbol.iterator) return () => proxy as unknown as Iterator<unknown>
+      if (
+        (property === 'next' || property === 'one' || property === 'toArray') &&
+        typeof value === 'function'
+      ) {
+        return (...args: unknown[]) => {
+          try {
+            return Reflect.apply(value, target, args)
+          } finally {
+            account()
+          }
+        }
+      }
+      if (property === 'raw' && typeof value === 'function') {
+        return (...args: unknown[]) =>
+          wrapIterator(Reflect.apply(value, target, args) as object)
+      }
+      return typeof value === 'function' ? value.bind(target) : value
+    },
+  })
+  account()
+  return proxy
+}
+
+export function retryDelayMs(
+  attempt: number,
+  initialBackoffMs: number,
+  maxBackoffMs: number
+): number {
+  return Math.min(maxBackoffMs, initialBackoffMs * 2 ** Math.max(0, attempt - 1))
+}
+
+export function shouldRetryDelegatedPush(
+  responseStatus: number | null,
+  attempt: number,
+  maxAttempts: number
+): boolean {
+  if (attempt >= maxAttempts) return false
+  return responseStatus === null || responseStatus === 429 || responseStatus >= 500
+}