orez-sync-cf-host 0.4.53 → 0.4.55

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "orez-sync-cf-host",
-  "version": "0.4.53",
+  "version": "0.4.55",
   "type": "module",
   "exports": {
     ".": "./src/index.ts"

package/src/config.ts

@@ -46,7 +46,7 @@ export function validateSyncHostConfig<Env extends SyncHostEnv>(
       throw new TypeError('upstream.intervalMs must be a safe integer >= 1000')
     }
     for (const [name, value] of Object.entries({
-      ingestBudgetRows: config.upstream.ingestBudgetRows ?? 250_000,
+      ingestBudgetRows: config.upstream.ingestBudgetRows ?? 150_000,
       ingestBudgetWindowMs: config.upstream.ingestBudgetWindowMs ?? 300_000,
       ingestBackoffMs: config.upstream.ingestBackoffMs ?? 1_000,
       ingestMaxBackoffMs: config.upstream.ingestMaxBackoffMs ?? 60_000,

package/src/host.ts

@@ -286,7 +286,7 @@ export function createSyncDurableObject<Env extends SyncHostEnv>(
   const wakeCoalesceMs = config.wakeCoalesceMs ?? 25
   const upstreamIntervalMs = config.upstream?.intervalMs ?? 15_000
   const upstreamLimit = config.upstream?.changeLimit ?? 1_000
-  const ingestBudgetRows = config.upstream?.ingestBudgetRows ?? 250_000
+  const ingestBudgetRows = config.upstream?.ingestBudgetRows ?? 150_000
   const ingestBudgetWindowMs = config.upstream?.ingestBudgetWindowMs ?? 5 * 60_000
   const ingestBackoffMs = config.upstream?.ingestBackoffMs ?? 1_000
   const ingestMaxBackoffMs = config.upstream?.ingestMaxBackoffMs ?? 60_000
@@ -309,6 +309,7 @@ export function createSyncDurableObject<Env extends SyncHostEnv>(
     #wakeRecipients = new Set<WebSocket>()
     #wakePromise: Promise<void> | null = null
     #ingestPromise: Promise<number> | null = null
+    #recordingIngestBillable = false
     #ingestBreaker = new IngestCircuitBreaker({
       budgetRows: ingestBudgetRows,
       windowMs: ingestBudgetWindowMs,
@@ -319,8 +320,11 @@ export function createSyncDurableObject<Env extends SyncHostEnv>(
 
     constructor(ctx: DurableObjectState, env: Env) {
       super(ctx, env)
-      this.#engineDb = new SqlStorageSyncDb(ctx.storage.sql)
-      this.#directSql = new SqlStorageDirect(ctx.storage.sql)
+      const recordRowsWritten = (rows: number) => {
+        if (this.#recordingIngestBillable) this.#ingestBreaker.recordBillable(rows)
+      }
+      this.#engineDb = new SqlStorageSyncDb(ctx.storage.sql, recordRowsWritten)
+      this.#directSql = new SqlStorageDirect(ctx.storage.sql, recordRowsWritten)
       this.#mutatorSql = new SqlStorageMutatorTransaction(this.#directSql, (ast) =>
         this.#wasm(() => engine_compile_query(config.schema, ast))
       )
@@ -499,6 +503,26 @@ export function createSyncDurableObject<Env extends SyncHostEnv>(
       }
     }
 
+    async #upstreamWriteBudgetStatus(): Promise<Response> {
+      if (!config.upstream) return json({ error: 'upstream is not configured' }, 404)
+      const path = this.#controlGet('upstreamPath')
+      if (!path) return json({ error: 'upstream path is not known yet' }, 409)
+      const endpoint = new URL(`${path}/_orez/write-budget`, 'https://upstream.invalid')
+      const response = await this.#serviceBinding().fetch(endpoint.toString(), {
+        headers: { host: endpoint.host },
+      })
+      if (!response.ok) {
+        return json(
+          {
+            error: 'upstream write-budget status unavailable',
+            upstreamStatus: response.status,
+          },
+          502
+        )
+      }
+      return json(await response.json())
+    }
+
     #rememberUpstreamPath(request: Request): string | null {
       if (!config.upstream) return null
       const path = request.headers.get(UPSTREAM_PATH_HEADER)
@@ -530,21 +554,31 @@ export function createSyncDurableObject<Env extends SyncHostEnv>(
       }
     }
 
-    #recordIngestRows(rows: number, fields: Record<string, unknown>): void {
+    #recordIngestLogicalRows(rows: number): void {
+      this.#ingestBreaker.recordLogical(rows)
+    }
+
+    #withIngestBilling<T>(fields: Record<string, unknown>, apply: () => T): T {
+      this.#recordingIngestBillable = true
       try {
-        this.#ingestBreaker.record(rows)
+        return apply()
       } catch (error) {
-        this.#persistIngestBreaker()
-        const status = this.#ingestBreaker.status()
-        console.error(
-          JSON.stringify({
-            event: 'sync_upstream_ingest_breaker_tripped',
-            ...status,
-            reason: 'ingestBudgetExceeded',
-            ...fields,
-          })
-        )
+        this.#recordingIngestBillable = false
+        if (error instanceof IngestBreakerError) {
+          this.#persistIngestBreaker()
+          const status = this.#ingestBreaker.status()
+          console.error(
+            JSON.stringify({
+              event: 'sync_upstream_ingest_breaker_tripped',
+              ...status,
+              reason: 'ingestBudgetExceeded',
+              ...fields,
+            })
+          )
+        }
         throw error
+      } finally {
+        this.#recordingIngestBillable = false
       }
     }
 
@@ -574,17 +608,21 @@ export function createSyncDurableObject<Env extends SyncHostEnv>(
               throw new Error(`upstream snapshot returned ${snapshotResponse.status}`)
             }
             const snapshot = await snapshotResponse.json()
-            const rebuilt = this.ctx.storage.transactionSync(() =>
-              this.#wasm(() =>
-                engine_apply_upstream_snapshot(this.#engineDb, config.schema, snapshot)
-              )
-            ) as ApplyUpstreamResult
+            const rebuilt = this.#withIngestBilling(
+              { phase: 'snapshot', cursor },
+              () =>
+                this.ctx.storage.transactionSync(() =>
+                  this.#wasm(() =>
+                    engine_apply_upstream_snapshot(
+                      this.#engineDb,
+                      config.schema,
+                      snapshot
+                    )
+                  )
+                ) as ApplyUpstreamResult
+            )
             total += rebuilt.applied
-            this.#recordIngestRows(rebuilt.applied, {
-              phase: 'snapshot',
-              cursor,
-              resultWatermark: rebuilt.watermark,
-            })
+            this.#recordIngestLogicalRows(rebuilt.applied)
             const nextCursor = this.#engineState()?.upstreamWatermark ?? cursor
             if (String(nextCursor) === String(cursor)) {
               this.#tripIngest('ingestCursorStalled', {
@@ -603,15 +641,22 @@ export function createSyncDurableObject<Env extends SyncHostEnv>(
           if (!Number.isSafeInteger(batch.watermark) || !Array.isArray(batch.changes)) {
             throw new Error('invalid upstream changes response')
           }
-          const result = this.ctx.storage.transactionSync(() =>
-            this.#wasm(() => engine_apply_upstream(this.#engineDb, config.schema, batch))
+          const result = this.#withIngestBilling(
+            {
+              phase: 'changes',
+              cursor,
+              batchWatermark: batch.watermark,
+              changeRows: batch.changes.length,
+            },
+            () =>
+              this.ctx.storage.transactionSync(() =>
+                this.#wasm(() =>
+                  engine_apply_upstream(this.#engineDb, config.schema, batch)
+                )
+              )
           ) as ApplyUpstreamResult
           total += result.applied
-          this.#recordIngestRows(result.applied, {
-            phase: 'changes',
-            cursor,
-            batchWatermark: batch.watermark,
-          })
+          this.#recordIngestLogicalRows(result.applied)
           const nextCursor = this.#engineState()?.upstreamWatermark ?? cursor
           if (batch.changes.length > 0 && String(nextCursor) === String(cursor)) {
             this.#tripIngest('ingestCursorStalled', {
@@ -1258,6 +1303,8 @@ export function createSyncDurableObject<Env extends SyncHostEnv>(
 
     #admin(route: string, request: Request): Promise<Response> | Response {
       if (route === '/admin/health') return json({ ok: true })
+      if (route === '/admin/upstream-write-budget' && request.method === 'GET')
+        return this.#upstreamWriteBudgetStatus()
       if (route === '/admin/status') {
         const heap = (
           performance as Performance & {

package/src/ingest-harness-worker.ts

@@ -70,6 +70,8 @@ async function upstreamFetch(request: Request, env: Env): Promise<Response> {
   const [, namespace, ...rest] = url.pathname.split('/')
   if (!namespace) return new Response('namespace required', { status: 400 })
   const stub = env.UPSTREAM_DO.get(env.UPSTREAM_DO.idFromName(namespace))
+  url.pathname = `/${rest.join('/')}`
+  if (url.pathname === '/_orez/write-budget') return stub.fetch(new Request(url, request))
   const exec = async (sql: string, params: unknown[] = []) => {
     const response = await stub.fetch('https://upstream.invalid/exec', {
       method: 'POST',
@@ -88,7 +90,6 @@ async function upstreamFetch(request: Request, env: Env): Promise<Response> {
     'INSERT OR IGNORE INTO _zero_schema_tables (name, schema_json) VALUES (?, ?)',
     ['item', JSON.stringify(schema.tables.item)]
   )
-  url.pathname = `/${rest.join('/')}`
   return stub.fetch(new Request(url, request))
 }
 
@@ -97,7 +98,11 @@ export class DataService extends WorkerEntrypoint<Env> {
   fetch(request: Request): Promise<Response> {
     const url = new URL(request.url)
     const pathname = url.pathname
-    if (!pathname.endsWith('/changes') && !pathname.endsWith('/snapshot')) {
+    if (
+      !pathname.endsWith('/changes') &&
+      !pathname.endsWith('/snapshot') &&
+      !pathname.endsWith('/_orez/write-budget')
+    ) {
       return Promise.resolve(
         new Response('DATA route rejected non-feed request', { status: 418 })
       )

package/src/sql-storage-adapter.ts

@@ -1,3 +1,5 @@
+import { trackBillableCursorRows } from './write-safeguards.js'
+
 import type { MutatorSql, SyncSql } from './types.js'
 
 type WireValue =
@@ -86,7 +88,10 @@ function encodeResult(value: unknown): WireValue {
 export class SqlStorageSyncDb implements JsSyncDb {
   readonly stats: AdapterStats = { execCalls: 0, queryCalls: 0, sqlMs: 0 }
 
-  constructor(private readonly sql: SqlStorage) {}
+  constructor(
+    private readonly sql: SqlStorage,
+    private readonly recordRowsWritten: (rows: number) => void = () => {}
+  ) {}
 
   resetStats(): void {
     this.stats.execCalls = 0
@@ -97,7 +102,10 @@ export class SqlStorageSyncDb implements JsSyncDb {
   exec(sql: string, params: WireValue[]): void {
     assertHostSql(sql)
     const start = performance.now()
-    this.sql.exec(sql, ...params.map(decodeBinding))
+    trackBillableCursorRows(
+      this.sql.exec(sql, ...params.map(decodeBinding)),
+      this.recordRowsWritten
+    )
     this.stats.execCalls++
     this.stats.sqlMs += performance.now() - start
   }
@@ -105,7 +113,10 @@ export class SqlStorageSyncDb implements JsSyncDb {
   query(sql: string, params: WireValue[]): WireRow[] {
     assertHostSql(sql)
     const start = performance.now()
-    const cursor = this.sql.exec(sql, ...params.map(decodeBinding))
+    const cursor = trackBillableCursorRows(
+      this.sql.exec(sql, ...params.map(decodeBinding)),
+      this.recordRowsWritten
+    )
     const columns = [...cursor.columnNames]
     // A SqlStorage cursor must be fully consumed before any await. Returning a
     // materialized array here makes that property structural.
@@ -121,11 +132,14 @@ export class SqlStorageSyncDb implements JsSyncDb {
 
 /** Direct application SQL surface used for initialization and admin reads. */
 export class SqlStorageDirect implements SyncSql {
-  constructor(private readonly sql: SqlStorage) {}
+  constructor(
+    private readonly sql: SqlStorage,
+    private readonly recordRowsWritten: (rows: number) => void = () => {}
+  ) {}
 
   exec(sql: string, params: readonly unknown[] = []): void {
     assertHostSql(sql)
-    this.sql.exec(sql, ...params)
+    trackBillableCursorRows(this.sql.exec(sql, ...params), this.recordRowsWritten)
   }
 
   query<Row extends Record<string, unknown> = Record<string, unknown>>(
@@ -133,7 +147,10 @@ export class SqlStorageDirect implements SyncSql {
     params: readonly unknown[] = []
   ): Row[] {
     assertHostSql(sql)
-    return this.sql.exec(sql, ...params).toArray() as Row[]
+    return trackBillableCursorRows(
+      this.sql.exec(sql, ...params),
+      this.recordRowsWritten
+    ).toArray() as Row[]
   }
 }
 

package/src/types.ts

@@ -116,7 +116,7 @@ export type UpstreamConfig = {
   changeLimit?: number
   /** Durable Object alarm safety net. Defaults to 15 seconds. */
   intervalMs?: number
-  /** Applied upstream rows allowed per rolling window. Defaults to 250,000. */
+  /** Billable SQLite rows written by ingest per rolling window. Defaults to 150,000. */
   ingestBudgetRows?: number
   /** Rolling ingest budget window. Defaults to five minutes. */
   ingestBudgetWindowMs?: number

package/src/write-safeguards.ts

@@ -10,6 +10,8 @@ export type IngestBreakerOptions = {
 
 export type IngestBreakerStatus = {
   windowRows: number
+  billableRows: number
+  logicalRows: number
   budget: number
   windowMs: number
   tripped: boolean
@@ -19,7 +21,7 @@ export type IngestBreakerStatus = {
   consecutiveTrips: number
 }
 
-type Sample = { at: number; rows: number }
+type Sample = { at: number; billableRows: number; logicalRows: number }
 
 export class IngestBreakerError extends Error {
   readonly status = 429
@@ -38,7 +40,8 @@ export class IngestBreakerError extends Error {
 export class IngestCircuitBreaker {
   readonly #options: IngestBreakerOptions
   #samples: Sample[] = []
-  #windowRows = 0
+  #billableRows = 0
+  #logicalRows = 0
   #reason: IngestBreakerReason | null = null
   #retryAt: number | null = null
   #consecutiveTrips = 0
@@ -56,7 +59,8 @@ export class IngestCircuitBreaker {
     const cutoff = now - this.#options.windowMs
     let remove = 0
     while (remove < this.#samples.length && this.#samples[remove]!.at <= cutoff) {
-      this.#windowRows -= this.#samples[remove]!.rows
+      this.#billableRows -= this.#samples[remove]!.billableRows
+      this.#logicalRows -= this.#samples[remove]!.logicalRows
       remove++
     }
     if (remove > 0) this.#samples.splice(0, remove)
@@ -67,7 +71,9 @@ export class IngestCircuitBreaker {
     this.#prune(now)
     const retryAfterMs = Math.max(0, (this.#retryAt ?? now) - now)
     return {
-      windowRows: this.#windowRows,
+      windowRows: this.#billableRows,
+      billableRows: this.#billableRows,
+      logicalRows: this.#logicalRows,
       budget: this.#options.budgetRows,
       windowMs: this.#options.windowMs,
       tripped: this.#reason !== null && retryAfterMs > 0,
@@ -91,15 +97,34 @@ export class IngestCircuitBreaker {
   }
 
   record(rows: number): void {
+    this.recordLogical(rows)
+    this.recordBillable(rows)
+  }
+
+  #sample(now: number): Sample {
+    this.#prune(now)
+    const last = this.#samples[this.#samples.length - 1]
+    if (last?.at === now) return last
+    const sample = { at: now, billableRows: 0, logicalRows: 0 }
+    this.#samples.push(sample)
+    return sample
+  }
+
+  recordLogical(rows: number): void {
+    if (!Number.isSafeInteger(rows) || rows <= 0) return
+    const sample = this.#sample(this.#options.now())
+    sample.logicalRows += rows
+    this.#logicalRows += rows
+  }
+
+  recordBillable(rows: number): void {
     this.assertReady()
     if (!Number.isSafeInteger(rows) || rows <= 0) return
     const now = this.#options.now()
-    this.#prune(now)
-    const last = this.#samples[this.#samples.length - 1]
-    if (last?.at === now) last.rows += rows
-    else this.#samples.push({ at: now, rows })
-    this.#windowRows += rows
-    if (this.#windowRows > this.#options.budgetRows) this.trip('ingestBudgetExceeded')
+    const sample = this.#sample(now)
+    sample.billableRows += rows
+    this.#billableRows += rows
+    if (this.#billableRows > this.#options.budgetRows) this.trip('ingestBudgetExceeded')
   }
 
   trip(reason: IngestBreakerReason): never {
@@ -113,7 +138,7 @@ export class IngestCircuitBreaker {
     this.#retryAt = now + delay
     throw new IngestBreakerError(
       reason,
-      this.#windowRows,
+      this.#billableRows,
       this.#options.budgetRows,
       delay
     )
@@ -134,11 +159,78 @@ export class IngestCircuitBreaker {
 
   reopen(): void {
     this.#samples = []
-    this.#windowRows = 0
+    this.#billableRows = 0
+    this.#logicalRows = 0
     this.recovered()
   }
 }
 
+type BillableCursor = {
+  rowsWritten?: number
+  next?: (...args: unknown[]) => unknown
+  one?: (...args: unknown[]) => unknown
+  toArray?: (...args: unknown[]) => unknown
+  raw?: (...args: unknown[]) => unknown
+  [Symbol.iterator]?: () => Iterator<unknown>
+}
+
+export function trackBillableCursorRows<Cursor extends BillableCursor>(
+  cursor: Cursor,
+  record: (rows: number) => void
+): Cursor {
+  if (!cursor || typeof cursor !== 'object') return cursor
+  let accountedRows = 0
+  const account = () => {
+    const current = Number(cursor.rowsWritten ?? 0)
+    if (!Number.isSafeInteger(current) || current <= accountedRows) return
+    const delta = current - accountedRows
+    accountedRows = current
+    record(delta)
+  }
+  const wrapIterator = (iterator: object): object =>
+    new Proxy(iterator, {
+      get(target, property) {
+        const value = Reflect.get(target, property, target)
+        if (property === 'next' && typeof value === 'function') {
+          return (...args: unknown[]) => {
+            try {
+              return Reflect.apply(value, target, args)
+            } finally {
+              account()
+            }
+          }
+        }
+        return typeof value === 'function' ? value.bind(target) : value
+      },
+    })
+  let proxy: Cursor
+  proxy = new Proxy(cursor, {
+    get(target, property) {
+      const value = Reflect.get(target, property, target)
+      if (property === Symbol.iterator) return () => proxy as unknown as Iterator<unknown>
+      if (
+        (property === 'next' || property === 'one' || property === 'toArray') &&
+        typeof value === 'function'
+      ) {
+        return (...args: unknown[]) => {
+          try {
+            return Reflect.apply(value, target, args)
+          } finally {
+            account()
+          }
+        }
+      }
+      if (property === 'raw' && typeof value === 'function') {
+        return (...args: unknown[]) =>
+          wrapIterator(Reflect.apply(value, target, args) as object)
+      }
+      return typeof value === 'function' ? value.bind(target) : value
+    },
+  })
+  account()
+  return proxy
+}
+
 export function retryDelayMs(
   attempt: number,
   initialBackoffMs: number,