orez-sync-cf-host 0.5.4 → 0.5.5

package/README.md

@@ -12,9 +12,10 @@ to a per-namespace Durable Object, and never logs tokens, mutation arguments, or
 row contents. Pull runs `engine_handle_pull` inside `transactionSync`; push runs
 Rust `push_validate`/`preflight`/`finalize`/`record_app_error` steps around the
 registered asynchronous TypeScript mutator inside `ctx.storage.transaction`.
-Every SQL cursor is materialized before an await. Application effects are
-collected and run only after their transaction commits; application failures use
-the required second transaction to advance the LMID marker.
+Every SQL cursor is materialized before an await. Mutators may await only their
+`MutatorSql` operations; timers, fetches, and other external work belong in
+`context.defer`, which runs only after commit. Application failures use the
+required second transaction to advance the LMID marker.
 
 ## Wake channel and eviction
 
@@ -54,6 +55,14 @@ The runner accepts only loopback or the `lslcf.workers.dev` test worker and does
 not mutate a production route. These controls are mechanisms, not authorization
 to perform a production cutover.
 
+`POST /admin/resnapshot` is available only when the consumer configured an
+upstream data service. It reads that namespace's authoritative `/snapshot`,
+atomically replaces the derived application tables, then consumes `/changes`
+until caught up. Engine metadata, client last-mutation IDs, operator controls,
+and the authoritative upstream database are preserved. The JSON response
+includes before/after upstream watermarks and the number of snapshot plus
+catch-up rows applied.
+
 ## Counter and HTTP wire representation
 
 Inside the wasm/JavaScript engine boundary, cookies, watermarks, and LMIDs are

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "orez-sync-cf-host",
-  "version": "0.5.4",
+  "version": "0.5.5",
   "type": "module",
   "exports": {
     ".": "./src/index.ts"

package/src/harness-config.ts

@@ -325,7 +325,6 @@ const harnessMutators = registerMutators({
         value.id,
       ])
     )
-    await scheduler.wait(1)
     throw new MutationApplicationError('intentional-rollback')
   },
 })

package/src/host.ts

@@ -609,16 +609,67 @@ export function createSyncDurableObject<Env extends SyncHostEnv>(
       }
     }
 
-    #ingest(upstreamPath?: string | null): Promise<number> {
-      if (!config.upstream) return Promise.resolve(0)
-      if (this.#ingestPromise) return this.#ingestPromise
+    async #applyUpstreamSnapshot(
+      path: string,
+      cursor: string,
+      allowSameCursor: boolean
+    ): Promise<ApplyUpstreamResult> {
+      const endpoint = new URL(`${path}/snapshot`, 'https://upstream.invalid')
+      const response = await this.#serviceBinding().fetch(endpoint.toString(), {
+        headers: { host: endpoint.host },
+      })
+      if (!response.ok) {
+        throw new Error(`upstream snapshot returned ${response.status}`)
+      }
+      const snapshot = await response.json()
+      const rebuilt = this.#withIngestBilling({ phase: 'snapshot', cursor }, () =>
+        this.ctx.storage.transactionSync(() =>
+          this.#wasm(() =>
+            engine_apply_upstream_snapshot(this.#engineDb, config.schema, snapshot)
+          )
+        )
+      ) as ApplyUpstreamResult
+      this.#recordIngestLogicalRows(rebuilt.applied)
+      const nextCursor = this.#engineState()?.upstreamWatermark ?? cursor
+      if (!allowSameCursor && String(nextCursor) === String(cursor)) {
+        this.#tripIngest('ingestCursorStalled', {
+          phase: 'snapshot',
+          cursor,
+          resultWatermark: rebuilt.watermark,
+          applied: rebuilt.applied,
+        })
+      }
+      return rebuilt
+    }
+
+    #ingest(upstreamPath?: string | null, forceSnapshot = false): Promise<number> {
+      if (!config.upstream) {
+        return forceSnapshot
+          ? Promise.reject(requestError('upstream is not configured'))
+          : Promise.resolve(0)
+      }
+      if (this.#ingestPromise) {
+        return forceSnapshot
+          ? this.#ingestPromise.then(() => this.#ingest(upstreamPath, true))
+          : this.#ingestPromise
+      }
       this.#ingestBreaker.assertReady()
       const path = upstreamPath ?? this.#controlGet('upstreamPath')
-      if (!path) return Promise.resolve(0)
+      if (!path) {
+        return forceSnapshot
+          ? Promise.reject(requestError('upstream path is not available'))
+          : Promise.resolve(0)
+      }
       this.#ingestPromise = (async () => {
         let total = 0
         for (;;) {
           const cursor = this.#engineState()?.upstreamWatermark ?? '0'
+          if (forceSnapshot) {
+            forceSnapshot = false
+            const rebuilt = await this.#applyUpstreamSnapshot(path, cursor, true)
+            total += rebuilt.applied
+            continue
+          }
           const endpoint = new URL(`${path}/changes`, 'https://upstream.invalid')
           endpoint.searchParams.set('watermark', cursor)
           endpoint.searchParams.set('limit', String(upstreamLimit))
@@ -626,39 +677,8 @@ export function createSyncDurableObject<Env extends SyncHostEnv>(
             headers: { host: endpoint.host },
           })
           if (response.status === 410) {
-            const snapshotEndpoint = new URL(`${path}/snapshot`, endpoint)
-            const snapshotResponse = await this.#serviceBinding().fetch(
-              snapshotEndpoint.toString(),
-              { headers: { host: snapshotEndpoint.host } }
-            )
-            if (!snapshotResponse.ok) {
-              throw new Error(`upstream snapshot returned ${snapshotResponse.status}`)
-            }
-            const snapshot = await snapshotResponse.json()
-            const rebuilt = this.#withIngestBilling(
-              { phase: 'snapshot', cursor },
-              () =>
-                this.ctx.storage.transactionSync(() =>
-                  this.#wasm(() =>
-                    engine_apply_upstream_snapshot(
-                      this.#engineDb,
-                      config.schema,
-                      snapshot
-                    )
-                  )
-                ) as ApplyUpstreamResult
-            )
+            const rebuilt = await this.#applyUpstreamSnapshot(path, cursor, false)
             total += rebuilt.applied
-            this.#recordIngestLogicalRows(rebuilt.applied)
-            const nextCursor = this.#engineState()?.upstreamWatermark ?? cursor
-            if (String(nextCursor) === String(cursor)) {
-              this.#tripIngest('ingestCursorStalled', {
-                phase: 'snapshot',
-                cursor,
-                resultWatermark: rebuilt.watermark,
-                applied: rebuilt.applied,
-              })
-            }
             continue
           }
           if (!response.ok) {
@@ -1345,7 +1365,11 @@ export function createSyncDurableObject<Env extends SyncHostEnv>(
       return new Response(null, { status: 101, webSocket: client })
     }
 
-    #admin(route: string, request: Request): Promise<Response> | Response {
+    #admin(
+      route: string,
+      request: Request,
+      upstreamPath: string | null
+    ): Promise<Response> | Response {
       if (route === '/admin/health') return json({ ok: true })
       if (route === '/admin/upstream-write-budget' && request.method === 'GET')
         return this.#upstreamWriteBudgetStatus()
@@ -1391,6 +1415,27 @@ export function createSyncDurableObject<Env extends SyncHostEnv>(
         )
         return json({ ok: true, engine: this.#engineState() })
       }
+      if (route === '/admin/resnapshot') {
+        if (request.method !== 'POST') {
+          return json({ error: 'method not allowed' }, 405)
+        }
+        return (async () => {
+          try {
+            const beforeUpstreamWatermark = this.#engineState()?.upstreamWatermark ?? null
+            const applied = await this.#ingest(upstreamPath, true)
+            const engine = this.#engineState()
+            return json({
+              ok: true,
+              applied,
+              beforeUpstreamWatermark,
+              afterUpstreamWatermark: engine?.upstreamWatermark ?? null,
+              engine,
+            })
+          } catch (error) {
+            return json(errorBody(error), statusOf(error))
+          }
+        })()
+      }
       if (route === '/admin/drop-next-push-response') {
         this.#dropNextPushResponse = true
         return json({ ok: true })
@@ -1498,7 +1543,7 @@ export function createSyncDurableObject<Env extends SyncHostEnv>(
       const route = routeAfterNamespace(new URL(request.url).pathname)
       const namespace = request.headers.get(NAMESPACE_HEADER) ?? 'unknown'
       const upstreamPath = this.#rememberUpstreamPath(request)
-      if (route.startsWith('/admin/')) return this.#admin(route, request)
+      if (route.startsWith('/admin/')) return this.#admin(route, request, upstreamPath)
 
       if (route === '/wake' && request.method === 'GET') return this.#wake(request)
       if (route === '/notify' && request.method === 'POST') {

package/src/ingest-harness-worker.ts

@@ -31,6 +31,8 @@ const runawayNamespaces = new Set<string>()
 const numericTextNamespaces = new Set<string>()
 const jsonValueNamespaces = new Set<string>()
 const hydratedNamespaces = new Set<string>()
+const heldSnapshots = new Set<string>()
+const activeSnapshots = new Set<string>()
 let delegatedFailuresRemaining = 0
 let delegatedAttempts = 0
 let delegatedPushFailedRemaining = 0
@@ -208,6 +210,16 @@ export class DataService extends WorkerEntrypoint<Env> {
       )
     }
     const response = await upstreamFetch(request, this.env)
+    if (pathname.endsWith('/snapshot')) {
+      if (heldSnapshots.has(namespace)) {
+        activeSnapshots.add(namespace)
+        try {
+          while (heldSnapshots.has(namespace)) await scheduler.wait(10)
+        } finally {
+          activeSnapshots.delete(namespace)
+        }
+      }
+    }
     if (pathname.endsWith('/changes') && response.ok) hydratedNamespaces.add(namespace)
     return response
   }
@@ -253,6 +265,31 @@ const syncWorker = createSyncWorker(config)
 export default {
   fetch(request: Request, env: Env, ctx: ExecutionContext): Promise<Response> {
     const url = new URL(request.url)
+    if (url.pathname.startsWith('/snapshot-control/')) {
+      const namespace = url.pathname.slice('/snapshot-control/'.length)
+      if (request.method === 'GET') {
+        return Promise.resolve(
+          Response.json({
+            active: activeSnapshots.has(namespace),
+            held: heldSnapshots.has(namespace),
+          })
+        )
+      }
+      return request
+        .json()
+        .catch(() => ({}))
+        .then((body) => {
+          if ((body as { hold?: unknown }).hold === true) {
+            heldSnapshots.add(namespace)
+          } else {
+            heldSnapshots.delete(namespace)
+          }
+          return Response.json({
+            active: activeSnapshots.has(namespace),
+            held: heldSnapshots.has(namespace),
+          })
+        })
+    }
     if (url.pathname.startsWith('/json-values-control/')) {
       const namespace = url.pathname.slice('/json-values-control/'.length)
       return request

package/src/platform-probe-worker.ts

@@ -79,7 +79,7 @@ export class ProbeDurableObject extends DurableObject<Env> {
       const { balance } = this.ctx.storage.sql
         .exec("SELECT balance FROM accounts WHERE id = 'primary'")
         .one() as { balance: number }
-      await scheduler.wait(1)
+      await Promise.resolve()
       this.ctx.storage.sql.exec(
         "UPDATE accounts SET balance = ? WHERE id = 'primary'",
         balance + 10
@@ -98,7 +98,7 @@ export class ProbeDurableObject extends DurableObject<Env> {
       this.ctx.storage.sql.exec(
         "UPDATE accounts SET balance = balance - 5 WHERE id = 'primary'"
       )
-      await scheduler.wait(1)
+      await Promise.resolve()
       this.ctx.storage.sql.exec(
         'INSERT INTO ledger (account_id, amount, note) VALUES (?, ?, ?)',
         'primary',
@@ -121,7 +121,7 @@ export class ProbeDurableObject extends DurableObject<Env> {
       "UPDATE accounts SET balance = ? WHERE id = 'primary'",
       balance + 777
     )
-    await scheduler.wait(1)
+    await Promise.resolve()
     this.ctx.storage.sql.exec(
       'INSERT INTO ledger (account_id, amount, note) VALUES (?, ?, ?)',
       'primary',
@@ -144,9 +144,10 @@ export class ProbeDurableObject extends DurableObject<Env> {
         const expected = push_preflight(this.#db, mutationID)
         wasmMs += performance.now() - wasmStarted
 
-        // This await is deliberately inside the storage transaction and before
-        // the remaining SQL operations. M0 exists to prove this contract.
-        await scheduler.wait(1)
+        // MutatorSql methods resolve through the microtask queue. Cross that
+        // same boundary here without admitting timers or other external work
+        // into the storage transaction.
+        await Promise.resolve()
         await this.#mutate(name, deferred)
 
         wasmStarted = performance.now()
@@ -236,7 +237,7 @@ export class ProbeDurableObject extends DurableObject<Env> {
           this.ctx.storage.sql.exec(
             "UPDATE accounts SET balance = balance + 123 WHERE id = 'primary'"
           )
-          await scheduler.wait(1)
+          await Promise.resolve()
           this.ctx.storage.sql.exec(
             'INSERT INTO outbox (topic, payload) VALUES (?, ?)',
             'must.rollback',