orez-sync-cf-host 0.15.0 → 0.16.0

package/dist/config.js

@@ -66,6 +66,9 @@ export function validateSyncHostConfig(config) {
             ingestBudgetWindowMs: config.upstream.ingestBudgetWindowMs ?? 300_000,
             ingestBackoffMs: config.upstream.ingestBackoffMs ?? 1_000,
             ingestMaxBackoffMs: config.upstream.ingestMaxBackoffMs ?? 60_000,
+            requestTimeoutMs: config.upstream.requestTimeoutMs ?? 30_000,
+            maxResponseBytes: config.upstream.maxResponseBytes ?? 8 * 1024 * 1024,
+            maxRequestBytes: config.upstream.maxRequestBytes ?? 1024 * 1024,
         })) {
             if (!Number.isSafeInteger(value) || value < 1)
                 throw new TypeError(`upstream.${name} must be a positive safe integer`);

package/dist/generated/package.json

@@ -1,7 +1,7 @@
 {
   "name": "sync-wasm",
   "type": "module",
-  "version": "0.1.5",
+  "version": "0.1.6",
   "license": "MIT",
   "files": [
     "sync_wasm_bg.wasm",

package/dist/generated/sync_wasm.js

@@ -507,7 +507,7 @@ function __wbg_get_imports() {
             const ret = Object.entries(arg0);
             return ret;
         },
-        __wbg_exec_b39c16e60dbe8f94: function() { return handleError(function (arg0, arg1, arg2, arg3) {
+        __wbg_exec_47d28d9cda4f7ff5: function() { return handleError(function (arg0, arg1, arg2, arg3) {
             arg0.exec(getStringFromWasm0(arg1, arg2), arg3);
         }, arguments); },
         __wbg_get_507a50627bffa49b: function(arg0, arg1) {
@@ -617,7 +617,7 @@ function __wbg_get_imports() {
         __wbg_prototypesetcall_4770620bbe4688a0: function(arg0, arg1, arg2) {
             Uint8Array.prototype.set.call(getArrayU8FromWasm0(arg0, arg1), arg2);
         },
-        __wbg_query_7af3043ddb76c6e2: function() { return handleError(function (arg0, arg1, arg2, arg3) {
+        __wbg_query_cf7ee856b3b99d70: function() { return handleError(function (arg0, arg1, arg2, arg3) {
             const ret = arg0.query(getStringFromWasm0(arg1, arg2), arg3);
             return ret;
         }, arguments); },

package/dist/generated/sync_wasm_bg.js

@@ -502,7 +502,7 @@ export function __wbg_entries_015dc610cd81ede0(arg0) {
     const ret = Object.entries(arg0);
     return ret;
 }
-export function __wbg_exec_b39c16e60dbe8f94() { return handleError(function (arg0, arg1, arg2, arg3) {
+export function __wbg_exec_47d28d9cda4f7ff5() { return handleError(function (arg0, arg1, arg2, arg3) {
     arg0.exec(getStringFromWasm0(arg1, arg2), arg3);
 }, arguments); }
 export function __wbg_get_507a50627bffa49b(arg0, arg1) {
@@ -612,7 +612,7 @@ export function __wbg_next_71f2aa1cb3d1e37e() { return handleError(function (arg
 export function __wbg_prototypesetcall_4770620bbe4688a0(arg0, arg1, arg2) {
     Uint8Array.prototype.set.call(getArrayU8FromWasm0(arg0, arg1), arg2);
 }
-export function __wbg_query_7af3043ddb76c6e2() { return handleError(function (arg0, arg1, arg2, arg3) {
+export function __wbg_query_cf7ee856b3b99d70() { return handleError(function (arg0, arg1, arg2, arg3) {
     const ret = arg0.query(getStringFromWasm0(arg1, arg2), arg3);
     return ret;
 }, arguments); }

package/dist/host.d.ts

@@ -9,8 +9,11 @@ import type { Schema } from '@rocicorp/zero';
 export declare function createSyncWorker<Env extends SyncHostEnv, S extends Schema = Schema>(config: SyncHostConfig<Env, S>): ExportedHandler<Env> & {
     notify(env: Env, namespace: string): Promise<Response>;
 };
+export interface SyncDurableObjectInstance<Env extends SyncHostEnv> extends DurableObject<Env> {
+    fetch(request: Request): Promise<Response>;
+}
 export interface SyncDurableObjectConstructor<Env extends SyncHostEnv> {
-    new (ctx: DurableObjectState, env: Env): DurableObject<Env>;
+    new (ctx: DurableObjectState, env: Env): SyncDurableObjectInstance<Env>;
 }
 /** Create the namespace Durable Object class for one bundled consumer config. */
 export declare function createSyncDurableObject<Env extends SyncHostEnv, S extends Schema = Schema>(config: SyncHostConfig<Env, S>): SyncDurableObjectConstructor<Env>;

package/dist/host.js

@@ -4,7 +4,9 @@ import { createSocketHost } from 'orez-sync-executor/realtime';
 import { validateSyncHostConfig } from './config.js';
 import { createQueryCompiler } from './query-compiler.js';
 import { resolveQueryPatch } from './query-patch.js';
+import { ServingLagTracker } from './serving-lag.js';
 import { decodeSqlParams, SqlStorageDirect, SqlStorageMutatorTransaction, SqlStorageSyncDb, } from './sql-storage-adapter.js';
+import { DEFAULT_UPSTREAM_MAX_RESPONSE_BYTES, DEFAULT_UPSTREAM_MAX_REQUEST_BYTES, DEFAULT_UPSTREAM_REQUEST_TIMEOUT_MS, fetchBoundedUpstreamJson, readBoundedJsonResponse, readBoundedStream, UpstreamResponseLimitError, } from './upstream-response.js';
 import { engine_authorize_realtime_subscription, engine_apply_snapshot_changes, engine_apply_snapshot_page, engine_apply_upstream, engine_begin_snapshot_generation, engine_finalize, engine_finalize_snapshot_generation, engine_handle_query_pull, engine_init_query_schema, engine_init_schema, engine_invalidate, engine_memory_bytes, engine_preflight, engine_prune, engine_push_validate, engine_read_snapshot_progress, engine_schema_revision, engine_state, engine_version, } from './wasm.js';
 import { IngestBreakerError, IngestCircuitBreaker, retryDelayMs, shouldRetryDelegatedPush, } from './write-safeguards.js';
 const NAMESPACE_HEADER = 'x-orez-sync-namespace';
@@ -126,6 +128,24 @@ async function requestObject(request) {
         throw requestError('request body must be a JSON object');
     return value;
 }
+async function boundedRequestObject(request, maxBytes, timeoutMs) {
+    try {
+        const bytes = await readBoundedStream(request.body, maxBytes, AbortSignal.timeout(timeoutMs));
+        const value = JSON.parse(new TextDecoder().decode(bytes));
+        if (!value || typeof value !== 'object' || Array.isArray(value)) {
+            throw requestError('request body must be a JSON object');
+        }
+        return value;
+    }
+    catch (error) {
+        if (error instanceof UpstreamResponseLimitError) {
+            throw requestError(`request body exceeded ${maxBytes} bytes`, 413);
+        }
+        if (error instanceof SyntaxError)
+            throw requestError('invalid JSON request body');
+        throw error;
+    }
+}
 function routeAfterNamespace(pathname) {
     const [, , ...parts] = pathname.split('/');
     return `/${parts.join('/')}`;
@@ -160,6 +180,7 @@ async function forwardedSyncRequest(request) {
     const headers = new Headers(request.headers);
     return {
         claims: claims,
+        body: body,
         request: jsonBodyRequest(request, headers, body),
     };
 }
@@ -225,6 +246,8 @@ function socketCloseQuietly(socket, code, reason) {
  */
 export function createSyncWorker(config) {
     validateSyncHostConfig(config);
+    const requestTimeoutMs = config.upstream?.requestTimeoutMs ?? DEFAULT_UPSTREAM_REQUEST_TIMEOUT_MS;
+    const maxRequestBytes = config.upstream?.maxRequestBytes ?? DEFAULT_UPSTREAM_MAX_REQUEST_BYTES;
     const allowedOrigins = new Set(config.allowedOrigins ?? []);
     const upstreamPath = (namespace) => {
         if (!config.upstream)
@@ -374,7 +397,7 @@ export function createSyncWorker(config) {
                     }
                     if ((route === '/pull' || route === '/push') && request.method === 'POST') {
                         try {
-                            const body = await requestObject(request);
+                            const body = await boundedRequestObject(request, maxRequestBytes, requestTimeoutMs);
                             forwardedBody = { claims, body };
                         }
                         catch (error) {
@@ -443,6 +466,9 @@ export function createSyncDurableObject(config) {
     const wakeCoalesceMs = config.wakeCoalesceMs ?? 25;
     const upstreamIntervalMs = config.upstream?.intervalMs ?? 15_000;
     const upstreamLimit = config.upstream?.changeLimit ?? 1_000;
+    const upstreamRequestTimeoutMs = config.upstream?.requestTimeoutMs ?? DEFAULT_UPSTREAM_REQUEST_TIMEOUT_MS;
+    const upstreamMaxResponseBytes = config.upstream?.maxResponseBytes ?? DEFAULT_UPSTREAM_MAX_RESPONSE_BYTES;
+    const upstreamMaxRequestBytes = config.upstream?.maxRequestBytes ?? DEFAULT_UPSTREAM_MAX_REQUEST_BYTES;
     const ingestBudgetRows = config.upstream?.ingestBudgetRows ?? 150_000;
     const ingestBudgetWindowMs = config.upstream?.ingestBudgetWindowMs ?? 5 * 60_000;
     const ingestBackoffMs = config.upstream?.ingestBackoffMs ?? 1_000;
@@ -471,6 +497,8 @@ export function createSyncDurableObject(config) {
         #counters = freshCounters();
         #sqlBilling = { rowsRead: 0, rowsWritten: 0 };
         #pulling = new Set();
+        #activePullGroups = new Map();
+        #servingLag = new ServingLagTracker();
         #wakeOrigins = new Set();
         #wakeRecipients = new Set();
         #wakeTables = new Set();
@@ -558,6 +586,15 @@ export function createSyncDurableObject(config) {
                     schema: config.schema,
                 })
                 : null;
+            // Upstream-backed namespaces are initialized lazily so a notify for a
+            // namespace with no wake subscribers remains a true zero-read/write
+            // fast path. Local-execution namespaces have no such notify path, and
+            // preserving eager initialization there matters for consumers that
+            // subclass this Durable Object and handle an application route before
+            // delegating to super.fetch().
+            if (!config.upstream) {
+                ctx.blockConcurrencyWhile(async () => this.#initialize());
+            }
         }
         #initialize() {
             if (this.#initialized)
@@ -703,6 +740,41 @@ export function createSyncDurableObject(config) {
             }
             return value;
         }
+        #activeClientGroupIDs() {
+            const groups = new Set(this.#activePullGroups.keys());
+            for (const socket of this.ctx.getWebSockets(WAKE_SUBSCRIBER_TAG)) {
+                const clientGroupID = socketAttachment(socket)?.identity?.clientGroupID;
+                if (clientGroupID)
+                    groups.add(clientGroupID);
+            }
+            return groups;
+        }
+        #beginActivePull(clientGroupID) {
+            this.#activePullGroups.set(clientGroupID, (this.#activePullGroups.get(clientGroupID) ?? 0) + 1);
+        }
+        #endActivePull(clientGroupID) {
+            const count = this.#activePullGroups.get(clientGroupID) ?? 0;
+            if (count <= 1)
+                this.#activePullGroups.delete(clientGroupID);
+            else
+                this.#activePullGroups.set(clientGroupID, count - 1);
+        }
+        async #fetchUpstreamJson(endpoint) {
+            const result = await fetchBoundedUpstreamJson(this.#serviceBinding(), endpoint.toString(), { headers: { host: endpoint.host } }, {
+                timeoutMs: upstreamRequestTimeoutMs,
+                maxBytes: upstreamMaxResponseBytes,
+            });
+            const sourceTimeMs = result.body &&
+                typeof result.body === 'object' &&
+                !Array.isArray(result.body) &&
+                typeof result.body.sourceTimeMs === 'number'
+                ? result.body.sourceTimeMs
+                : null;
+            if (sourceTimeMs !== null) {
+                this.#servingLag.recordClockSkew(sourceTimeMs, result.sendTimeMs, result.receiveTimeMs);
+            }
+            return result;
+        }
         async #upstreamWriteBudgetStatus() {
             if (!config.upstream)
                 return json({ error: 'upstream is not configured' }, 404);
@@ -710,16 +782,14 @@ export function createSyncDurableObject(config) {
             if (path === null)
                 return json({ error: 'upstream path is not known yet' }, 409);
             const endpoint = new URL(`${path}/_orez/write-budget`, 'https://upstream.invalid');
-            const response = await this.#serviceBinding().fetch(endpoint.toString(), {
-                headers: { host: endpoint.host },
-            });
+            const { response, body } = await this.#fetchUpstreamJson(endpoint);
             if (!response.ok) {
                 return json({
                     error: 'upstream write-budget status unavailable',
                     upstreamStatus: response.status,
                 }, 502);
             }
-            return json(await response.json());
+            return json(body);
         }
         #rememberUpstreamPath(request) {
             if (!config.upstream)
@@ -810,13 +880,11 @@ export function createSyncDurableObject(config) {
             endpoint.searchParams.set('limit', String(limit));
             if (cursor !== null)
                 endpoint.searchParams.set('cursor', cursor);
-            const response = await this.#serviceBinding().fetch(endpoint.toString(), {
-                headers: { host: endpoint.host },
-            });
+            const { response, body } = await this.#fetchUpstreamJson(endpoint);
             if (!response.ok) {
                 throw requestError(`upstream snapshot page returned ${response.status}`, response.status >= 500 ? 502 : response.status);
             }
-            const page = (await response.json());
+            const page = body;
             if (!Number.isSafeInteger(page.watermark) ||
                 Number(page.watermark) < 0 ||
                 !Array.isArray(page.rows) ||
@@ -877,6 +945,7 @@ export function createSyncDurableObject(config) {
                 let pendingPage = null;
                 let snapshotPageLimit = DEFAULT_SNAPSHOT_PAGE_ROWS;
                 let snapshotCompleted = false;
+                let oldestLiveCommitTimeMs;
                 const changedTables = new Set();
                 for (;;) {
                     if (progress?.state === 'paging') {
@@ -921,9 +990,7 @@ export function createSyncDurableObject(config) {
                         const endpoint = new URL(`${path}/changes`, 'https://upstream.invalid');
                         endpoint.searchParams.set('since', cursor);
                         endpoint.searchParams.set('limit', String(upstreamLimit));
-                        const response = await this.#serviceBinding().fetch(endpoint.toString(), {
-                            headers: { host: endpoint.host },
-                        });
+                        const { response, body } = await this.#fetchUpstreamJson(endpoint);
                         if (response.status === 410) {
                             const begun = await this.#beginSnapshotGeneration(path);
                             progress = begun.progress;
@@ -934,7 +1001,7 @@ export function createSyncDurableObject(config) {
                         if (!response.ok) {
                             throw new Error(`upstream snapshot catch-up returned ${response.status}`);
                         }
-                        const batch = (await response.json());
+                        const batch = body;
                         if (!Number.isSafeInteger(batch.watermark) || !Array.isArray(batch.changes)) {
                             throw new Error('invalid upstream changes response');
                         }
@@ -989,9 +1056,7 @@ export function createSyncDurableObject(config) {
                     const endpoint = new URL(`${path}/changes`, 'https://upstream.invalid');
                     endpoint.searchParams.set('watermark', cursor);
                     endpoint.searchParams.set('limit', String(upstreamLimit));
-                    const response = await this.#serviceBinding().fetch(endpoint.toString(), {
-                        headers: { host: endpoint.host },
-                    });
+                    const { response, body } = await this.#fetchUpstreamJson(endpoint);
                     if (response.status === 410) {
                         const begun = await this.#beginSnapshotGeneration(path);
                         progress = begun.progress;
@@ -1002,10 +1067,18 @@ export function createSyncDurableObject(config) {
                     if (!response.ok) {
                         throw new Error(`upstream changes returned ${response.status}`);
                     }
-                    const batch = (await response.json());
+                    const batch = body;
                     if (!Number.isSafeInteger(batch.watermark) || !Array.isArray(batch.changes)) {
                         throw new Error('invalid upstream changes response');
                     }
+                    const batchCommitTimeMs = batch.oldestCommitTimeMs;
+                    if (typeof batchCommitTimeMs === 'number' &&
+                        Number.isFinite(batchCommitTimeMs)) {
+                        oldestLiveCommitTimeMs =
+                            oldestLiveCommitTimeMs === undefined
+                                ? batchCommitTimeMs
+                                : Math.min(oldestLiveCommitTimeMs, batchCommitTimeMs);
+                    }
                     for (const table of upstreamBatchTables(batch))
                         changedTables.add(table);
                     const result = this.#withIngestBilling({
@@ -1044,6 +1117,18 @@ export function createSyncDurableObject(config) {
                 }
                 this.#recoverIngestBreaker();
                 const endingWatermark = this.#engineState().watermark;
+                if (oldestLiveCommitTimeMs !== undefined) {
+                    const activeGroups = this.#activeClientGroupIDs();
+                    if (endingWatermark !== startingWatermark || total > 0) {
+                        this.#servingLag.onVersionReady(endingWatermark, oldestLiveCommitTimeMs, activeGroups);
+                    }
+                    else {
+                        const servedAt = Date.now();
+                        for (const _clientGroupID of activeGroups) {
+                            this.#servingLag.recordNoChange(oldestLiveCommitTimeMs, servedAt);
+                        }
+                    }
+                }
                 if (snapshotCompleted || total > 0 || endingWatermark !== startingWatermark) {
                     await this.#enqueueWake('__upstream__', changedTables);
                 }
@@ -1259,7 +1344,10 @@ export function createSyncDurableObject(config) {
                     wasmBoundaryCalls: this.#counters.wasmBoundaryCalls,
                     sql: this.#engineDb.stats,
                 });
-                return json(response);
+                const result = json(response);
+                const clientGroupID = typeof body.clientGroupID === 'string' ? body.clientGroupID : '';
+                this.#servingLag.onVersionServed(clientGroupID, response.cookie);
+                return result;
             }
             catch (error) {
                 const status = statusOf(error);
@@ -1299,7 +1387,7 @@ export function createSyncDurableObject(config) {
             if (!this.#writerEnabled()) {
                 // Workerd requires the request stream to be consumed before the DO
                 // returns a response. Discard it without parsing or logging payloads.
-                await request.arrayBuffer();
+                await readBoundedStream(request.body, upstreamMaxRequestBytes, AbortSignal.timeout(upstreamRequestTimeoutMs));
                 const state = this.#engineStateBestEffort();
                 this.#log({
                     namespaceHash: namespace,
@@ -1325,7 +1413,8 @@ export function createSyncDurableObject(config) {
             }
             if (config.mutateUrl) {
                 try {
-                    const bytes = await request.arrayBuffer();
+                    const boundedBytes = await readBoundedStream(request.body, upstreamMaxRequestBytes, AbortSignal.timeout(upstreamRequestTimeoutMs));
+                    const bytes = boundedBytes.buffer.slice(boundedBytes.byteOffset, boundedBytes.byteOffset + boundedBytes.byteLength);
                     const body = JSON.parse(new TextDecoder().decode(bytes));
                     const plan = this.#wasm(() => engine_push_validate(body));
                     if (plan.kind === 'respond')
@@ -1337,9 +1426,11 @@ export function createSyncDurableObject(config) {
                     headers.set('host', endpoint.host);
                     const upstreamResponse = await this.#fetchDelegatedPush(endpoint, headers, bytes, this.#engineState().upstreamWatermark === '0');
                     if (!upstreamResponse.ok) {
-                        return new Response(upstreamResponse.body, upstreamResponse);
+                        const upstreamErrorBody = await readBoundedStream(upstreamResponse.body, upstreamMaxResponseBytes, AbortSignal.timeout(upstreamRequestTimeoutMs));
+                        const errorBytes = upstreamErrorBody.buffer.slice(upstreamErrorBody.byteOffset, upstreamErrorBody.byteOffset + upstreamErrorBody.byteLength);
+                        return new Response(errorBytes, upstreamResponse);
                     }
-                    const upstreamBody = (await upstreamResponse.json());
+                    const upstreamBody = (await readBoundedJsonResponse(upstreamResponse, upstreamMaxResponseBytes, AbortSignal.timeout(upstreamRequestTimeoutMs)));
                     const delegatedResponse = upstreamBody.pushResponse ?? upstreamBody;
                     if (isStructuredPushFailed(delegatedResponse)) {
                         // PushFailed is a successful protocol response describing an
@@ -1803,17 +1894,28 @@ export function createSyncDurableObject(config) {
                 catch (error) {
                     return errorResponse(error);
                 }
-                // pull and push both establish the upstream schema and snapshot barrier.
+                const clientGroupID = route === '/pull' && typeof forwarded.body.clientGroupID === 'string'
+                    ? forwarded.body.clientGroupID
+                    : null;
+                if (clientGroupID)
+                    this.#beginActivePull(clientGroupID);
                 try {
-                    await this.#ingest(upstreamPath);
-                }
-                catch (error) {
-                    return errorResponse(error);
+                    // pull and push both establish the upstream schema and snapshot barrier.
+                    try {
+                        await this.#ingest(upstreamPath);
+                    }
+                    catch (error) {
+                        return errorResponse(error);
+                    }
+                    if (route === '/pull') {
+                        return await this.#pull(forwarded.request, forwarded.claims, namespace);
+                    }
+                    return await this.#push(forwarded.request, forwarded.claims, namespace, upstreamPath);
                 }
-                if (route === '/pull') {
-                    return this.#pull(forwarded.request, forwarded.claims, namespace);
+                finally {
+                    if (clientGroupID)
+                        this.#endActivePull(clientGroupID);
                 }
-                return this.#push(forwarded.request, forwarded.claims, namespace, upstreamPath);
             }
             return json({ error: 'not found' }, 404);
         }

package/dist/index.d.ts

@@ -3,6 +3,6 @@ export { createQueryCompiler } from './query-compiler.js';
 export { resolveQueryPatch } from './query-patch.js';
 export { DEFAULT_TRANSACTION_QUERY_BUDGET, TransactionQueryBudgetError, executeTransactionQueryPlan, executeTransactionQueryPlanAsync, } from './transaction-query.js';
 export type { SyncHostConfig, SyncHostEnv, ServiceBinding, SyncSql, UpstreamConfig, } from './types.js';
-export type { SyncDurableObjectConstructor } from './host.js';
+export type { SyncDurableObjectConstructor, SyncDurableObjectInstance } from './host.js';
 export type { CompiledTransactionQueryNode, CompiledTransactionQueryPlan, CompiledTransactionQueryRelationship, TransactionQueryBinding, TransactionQueryBudget, TransactionQueryColumn, TransactionQueryColumnType, TransactionQueryExecutionOptions, TransactionQueryWireValue, } from './transaction-query.js';
 export type { TransactionQueryCompiler } from './query-compiler.js';

package/dist/serving-lag.d.ts

@@ -0,0 +1,27 @@
+type MetricAttributes = Record<string, string | number | boolean>;
+type Histogram = {
+    record(value: number, attributes?: MetricAttributes): void;
+};
+type Counter = {
+    add(value: number, attributes?: MetricAttributes): void;
+};
+export type ServingLagInstruments = {
+    servingLag: Histogram;
+    servingLagClamps: Counter;
+    upstreamClockSkew: Histogram;
+};
+/**
+ * Coalesces upstream commits independently for each active client group. A
+ * pull clears a pending observation even when its patch is empty: being
+ * current through the version is the completion event, not receiving a row.
+ */
+export declare class ServingLagTracker {
+    #private;
+    private readonly metrics;
+    constructor(metrics?: ServingLagInstruments);
+    onVersionReady(watermarkValue: unknown, upstreamCommitTimeMs: number, clientGroupIDs: Iterable<string>): void;
+    onVersionServed(clientGroupID: string, watermarkValue: unknown, nowMs?: number): void;
+    recordNoChange(upstreamCommitTimeMs: number, nowMs?: number): void;
+    recordClockSkew(sourceTimeMs: number, sendTimeMs: number, receiveTimeMs: number): void;
+}
+export {};

package/dist/serving-lag.js

@@ -0,0 +1,83 @@
+import { metrics } from '@opentelemetry/api';
+let defaultInstruments;
+function instruments() {
+    if (defaultInstruments)
+        return defaultInstruments;
+    const meter = metrics.getMeter('orez');
+    defaultInstruments = {
+        servingLag: meter.createHistogram('orez.sync.e2e_serving_lag', {
+            description: 'End-to-end completion lag from an upstream commit until a client group is served through that advancement.',
+            unit: 's',
+        }),
+        servingLagClamps: meter.createCounter('orez.sync.e2e_serving_lag_clamps', {
+            description: 'Negative end-to-end serving-lag observations clamped to zero because the upstream clock was ahead.',
+            unit: '{observation}',
+        }),
+        upstreamClockSkew: meter.createHistogram('orez.replication.upstream_clock_skew', {
+            description: 'Estimated upstream clock offset from the Orez host. Positive means upstream is ahead.',
+            unit: 's',
+        }),
+    };
+    return defaultInstruments;
+}
+function counter(value) {
+    if (typeof value === 'number' && Number.isSafeInteger(value) && value >= 0) {
+        return BigInt(value);
+    }
+    if (typeof value === 'string' && /^(?:0|[1-9]\d*)$/.test(value)) {
+        return BigInt(value);
+    }
+    return null;
+}
+/**
+ * Coalesces upstream commits independently for each active client group. A
+ * pull clears a pending observation even when its patch is empty: being
+ * current through the version is the completion event, not receiving a row.
+ */
+export class ServingLagTracker {
+    metrics;
+    #pending = new Map();
+    constructor(metrics = instruments()) {
+        this.metrics = metrics;
+    }
+    onVersionReady(watermarkValue, upstreamCommitTimeMs, clientGroupIDs) {
+        const watermark = counter(watermarkValue);
+        if (watermark === null || !Number.isFinite(upstreamCommitTimeMs))
+            return;
+        for (const clientGroupID of clientGroupIDs) {
+            if (!clientGroupID)
+                continue;
+            const previous = this.#pending.get(clientGroupID);
+            this.#pending.set(clientGroupID, {
+                watermark,
+                commitTimeMs: previous
+                    ? Math.min(previous.commitTimeMs, upstreamCommitTimeMs)
+                    : upstreamCommitTimeMs,
+            });
+        }
+    }
+    onVersionServed(clientGroupID, watermarkValue, nowMs = Date.now()) {
+        const served = counter(watermarkValue);
+        const pending = this.#pending.get(clientGroupID);
+        if (served === null || !pending || served < pending.watermark)
+            return;
+        this.#pending.delete(clientGroupID);
+        this.#record(nowMs - pending.commitTimeMs, { outcome: 'advanced' });
+    }
+    recordNoChange(upstreamCommitTimeMs, nowMs = Date.now()) {
+        if (!Number.isFinite(upstreamCommitTimeMs))
+            return;
+        this.#record(nowMs - upstreamCommitTimeMs, { outcome: 'no_change' });
+    }
+    recordClockSkew(sourceTimeMs, sendTimeMs, receiveTimeMs) {
+        if (![sourceTimeMs, sendTimeMs, receiveTimeMs].every(Number.isFinite))
+            return;
+        this.metrics.upstreamClockSkew.record((sourceTimeMs - (sendTimeMs + receiveTimeMs) / 2) / 1_000);
+    }
+    #record(rawLagMs, attributes) {
+        const clamped = rawLagMs < 0;
+        this.metrics.servingLag.record(Math.max(0, rawLagMs) / 1_000, attributes);
+        if (clamped)
+            this.metrics.servingLagClamps.add(1, attributes);
+    }
+}

package/dist/types.d.ts

@@ -20,6 +20,12 @@ export type UpstreamConfig = {
     namespacePath: string | ((namespace: string) => string);
     /** Feed page size; the cursor loop continues until the reported head is reached. */
     changeLimit?: number;
+    /** Whole-request deadline, including the successful response body. Defaults to 30s. */
+    requestTimeoutMs?: number;
+    /** Maximum decoded JSON response size. Defaults to 8 MiB. */
+    maxResponseBytes?: number;
+    /** Maximum forwarded pull or push request size. Defaults to 1 MiB. */
+    maxRequestBytes?: number;
     /** Active wake-socket alarm safety net. Defaults to 15 seconds. */
     intervalMs?: number;
     /** Billable SQLite rows written by ingest per rolling window. Defaults to 150,000. */

package/dist/upstream-response.d.ts

@@ -0,0 +1,22 @@
+export declare const DEFAULT_UPSTREAM_REQUEST_TIMEOUT_MS = 30000;
+export declare const DEFAULT_UPSTREAM_MAX_RESPONSE_BYTES: number;
+export declare const DEFAULT_UPSTREAM_MAX_REQUEST_BYTES: number;
+export declare class UpstreamResponseLimitError extends Error {
+    readonly maxBytes: number;
+    readonly status = 502;
+    constructor(maxBytes: number);
+}
+export declare function readBoundedJsonResponse(response: Response, maxBytes: number, signal: AbortSignal): Promise<unknown>;
+export declare function readBoundedStream(stream: ReadableStream<Uint8Array> | null, maxBytes: number, signal: AbortSignal): Promise<Uint8Array>;
+export declare function fetchBoundedUpstreamJson(binding: {
+    fetch(input: string | Request, init?: RequestInit): Promise<Response>;
+}, input: string, init: RequestInit, options: {
+    timeoutMs: number;
+    maxBytes: number;
+    now?: () => number;
+}): Promise<{
+    response: Response;
+    body: unknown | null;
+    sendTimeMs: number;
+    receiveTimeMs: number;
+}>;

package/dist/upstream-response.js

@@ -0,0 +1,101 @@
+export const DEFAULT_UPSTREAM_REQUEST_TIMEOUT_MS = 30_000;
+export const DEFAULT_UPSTREAM_MAX_RESPONSE_BYTES = 8 * 1024 * 1024;
+export const DEFAULT_UPSTREAM_MAX_REQUEST_BYTES = 1024 * 1024;
+export class UpstreamResponseLimitError extends Error {
+    maxBytes;
+    status = 502;
+    constructor(maxBytes) {
+        super(`upstream response exceeded ${maxBytes} bytes`);
+        this.maxBytes = maxBytes;
+        this.name = 'UpstreamResponseLimitError';
+    }
+}
+function abortError(signal) {
+    return signal.reason instanceof Error
+        ? signal.reason
+        : new Error('upstream response was aborted');
+}
+async function readWithAbort(read, signal, cancel) {
+    if (signal.aborted)
+        throw abortError(signal);
+    let remove = () => { };
+    const aborted = new Promise((_, reject) => {
+        const onAbort = () => {
+            void cancel().catch(() => { });
+            reject(abortError(signal));
+        };
+        signal.addEventListener('abort', onAbort, { once: true });
+        remove = () => signal.removeEventListener('abort', onAbort);
+    });
+    try {
+        return await Promise.race([read, aborted]);
+    }
+    finally {
+        remove();
+    }
+}
+export async function readBoundedJsonResponse(response, maxBytes, signal) {
+    const contentLength = Number(response.headers.get('content-length'));
+    if (Number.isFinite(contentLength) && contentLength > maxBytes) {
+        await response.body?.cancel().catch(() => { });
+        throw new UpstreamResponseLimitError(maxBytes);
+    }
+    if (!response.body)
+        throw new Error('upstream response has no body');
+    const body = await readBoundedStream(response.body, maxBytes, signal);
+    return JSON.parse(new TextDecoder().decode(body));
+}
+export async function readBoundedStream(stream, maxBytes, signal) {
+    if (!stream)
+        return new Uint8Array();
+    const reader = stream.getReader();
+    const chunks = [];
+    let bytes = 0;
+    for (;;) {
+        if (signal.aborted) {
+            await reader.cancel().catch(() => { });
+            throw abortError(signal);
+        }
+        const result = await readWithAbort(reader.read(), signal, () => reader.cancel());
+        if (result.done)
+            break;
+        bytes += result.value.byteLength;
+        if (bytes > maxBytes) {
+            await reader.cancel().catch(() => { });
+            throw new UpstreamResponseLimitError(maxBytes);
+        }
+        chunks.push(result.value);
+    }
+    const body = new Uint8Array(bytes);
+    let offset = 0;
+    for (const chunk of chunks) {
+        body.set(chunk, offset);
+        offset += chunk.byteLength;
+    }
+    return body;
+}
+export async function fetchBoundedUpstreamJson(binding, input, init, options) {
+    const now = options.now ?? Date.now;
+    const controller = new AbortController();
+    const timer = setTimeout(() => {
+        controller.abort(new Error(`upstream response missed ${options.timeoutMs}ms deadline`));
+    }, options.timeoutMs);
+    const sendTimeMs = now();
+    try {
+        const response = await binding.fetch(input, { ...init, signal: controller.signal });
+        if (!response.ok) {
+            await response.body?.cancel().catch(() => { });
+            return { response, body: null, sendTimeMs, receiveTimeMs: now() };
+        }
+        const body = await readBoundedJsonResponse(response, options.maxBytes, controller.signal);
+        return { response, body, sendTimeMs, receiveTimeMs: now() };
+    }
+    catch (error) {
+        if (controller.signal.aborted)
+            throw abortError(controller.signal);
+        throw error;
+    }
+    finally {
+        clearTimeout(timer);
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "orez-sync-cf-host",
-  "version": "0.15.0",
+  "version": "0.16.0",
   "description": "Internal orez Cloudflare sync host. Do not import directly: consumers use orez-lite/cloudflare/sync.",
   "type": "module",
   "exports": {
@@ -34,7 +34,8 @@
     },
     "./wasm-module.wasm": {
       "types": "./dist/generated/sync_wasm_bg.wasm.d.ts",
-      "import": "./dist/generated/sync_wasm_bg.wasm"
+      "import": "./dist/generated/sync_wasm_bg.wasm",
+      "default": "./dist/generated/sync_wasm_bg.wasm"
     }
   },
   "scripts": {
@@ -47,7 +48,7 @@
     "bundle": "bun run build:dist && wrangler deploy --dry-run --outdir .wrangler/bundle",
     "measure": "bun run build:dist && bun measure.mjs",
     "test": "bun run test:config && bun run test:query-compiler && bun run test:package && bun run test:platform && bun run test:integration && bun run test:large-data && bun run test:ingest && bun run test:restart",
-    "test:config": "bun test config.test.mjs write-safeguards.test.mjs transaction-query.test.mjs",
+    "test:config": "bun test config.test.mjs write-safeguards.test.mjs transaction-query.test.mjs upstream-response.test.mjs serving-lag.test.mjs",
     "test:integration": "bun run build:dist && bun integration-test.mjs",
     "test:ingest": "bun run build:dist && bun ingest-test.mjs",
     "test:package": "bun run build:dist && node package-runtime-test.mjs",
@@ -58,7 +59,8 @@
     "test:large-data": "bun run build:dist && bun large-data-test.mjs"
   },
   "dependencies": {
-    "orez-sync-executor": "0.15.0"
+    "@opentelemetry/api": "^1.9.0",
+    "orez-sync-executor": "0.16.0"
   },
   "devDependencies": {
     "@cloudflare/workers-types": "4.20260617.1",