@rocicorp/zero 0.7.2024120400 → 0.7.2024120700

package/out/zero-cache/src/services/view-syncer/view-syncer.js

@@ -1,9 +1,14 @@
+import { trace } from '@opentelemetry/api';
 import { Lock } from '@rocicorp/lock';
 import { resolver } from '@rocicorp/resolver';
+import { manualSpan, startAsyncSpan, startSpan, } from '../../../../otel/src/span.js';
+import { version } from '../../../../otel/src/version.js';
 import { assert, unreachable } from '../../../../shared/src/asserts.js';
 import { CustomKeyMap } from '../../../../shared/src/custom-key-map.js';
 import { must } from '../../../../shared/src/must.js';
+import { randInt } from '../../../../shared/src/rand.js';
 import { ErrorKind, } from '../../../../zero-protocol/src/mod.js';
+import { transformAndHashQuery } from '../../auth/read-authorizer.js';
 import { stringify } from '../../types/bigint-json.js';
 import { ErrorForClient } from '../../types/error-for-client.js';
 import { rowIDString } from '../../types/row-key.js';
@@ -15,12 +20,11 @@ import { CVRConfigDrivenUpdater, CVRQueryDrivenUpdater, } from './cvr.js';
 import { PipelineDriver } from './pipeline-driver.js';
 import { cmpVersions, EMPTY_CVR_VERSION, versionFromString, versionString, versionToCookie, } from './schema/types.js';
 import { SchemaChangeError } from './snapshotter.js';
-import { transformAndHashQuery } from '../../auth/read-authorizer.js';
+const tracer = trace.getTracer('view-syncer', version);
 const DEFAULT_KEEPALIVE_MS = 5_000;
-// TODO: make idle timeout more intelligent when browser-level client
-//       management can provide signals of whether a client is likely to
-//       reconnect.
-const DEFAULT_IDLE_TIMEOUT_MS = 0;
+function randomID() {
+    return randInt(1, Number.MAX_SAFE_INTEGER).toString(36);
+}
 export class ViewSyncerService {
     id;
     #shardID;
@@ -29,25 +33,21 @@ export class ViewSyncerService {
     #stateChanges;
     #drainCoordinator;
     #keepaliveMs;
-    #idleTimeoutMs;
     // Note: It is fine to update this variable outside of the lock.
     #lastConnectTime = 0;
+    // Note: It is okay to add/remove clients without acquiring the lock.
+    #clients = new Map();
     // Serialize on this lock for:
     // (1) storage or database-dependent operations
     // (2) updating member variables.
-    // (3) initializing a new client, to ensure that it only gets pokes after
-    //     we have processed its initConnectionMessage.
-    //
-    // Note that it is okay to remove/delete clients without acquiring the lock.
     #lock = new Lock();
-    #clients = new Map();
     #cvrStore;
     #stopped = resolver();
     #permissions;
     #cvr;
     #pipelinesSynced = false;
     #authData;
-    constructor(lc, taskID, clientGroupID, shardID, db, pipelineDriver, versionChanges, drainCoordinator, permissions, keepaliveMs = DEFAULT_KEEPALIVE_MS, idleTimeoutMs = DEFAULT_IDLE_TIMEOUT_MS) {
+    constructor(lc, taskID, clientGroupID, shardID, db, pipelineDriver, versionChanges, drainCoordinator, permissions, keepaliveMs = DEFAULT_KEEPALIVE_MS) {
         this.id = clientGroupID;
         this.#shardID = shardID;
         this.#lc = lc;
@@ -55,7 +55,6 @@ export class ViewSyncerService {
         this.#stateChanges = versionChanges;
         this.#drainCoordinator = drainCoordinator;
         this.#keepaliveMs = keepaliveMs;
-        this.#idleTimeoutMs = idleTimeoutMs;
         this.#cvrStore = new CVRStore(lc, db, taskID, clientGroupID, 
         // On failure, cancel the #stateChanges subscription. The run()
         // loop will then await #cvrStore.flushed() which rejects if necessary.
@@ -64,11 +63,16 @@ export class ViewSyncerService {
     }
     #runInLockWithCVR(fn) {
         return this.#lock.withLock(async () => {
+            const lc = this.#lc.withContext('lock', randomID());
+            if (!this.#stateChanges.active) {
+                return; // view-syncer has been shutdown
+            }
             if (!this.#cvr) {
-                this.#cvr = await this.#cvrStore.load(this.#lastConnectTime);
+                this.#cvr = await this.#cvrStore.load(lc, this.#lastConnectTime);
             }
+            lc.debug?.(`cvr@${versionString(this.#cvr.version)}`);
             try {
-                return await fn(this.#cvr);
+                await fn(lc, this.#cvr);
             }
             catch (e) {
                 // Clear cached state if an error is encountered.
@@ -89,33 +93,33 @@ export class ViewSyncerService {
                     // On the first version-ready signal, connect to the replica.
                     this.#pipelines.init();
                 }
-                await this.#runInLockWithCVR(async (cvr) => {
+                await this.#runInLockWithCVR(async (lc, cvr) => {
                     if ((cvr.replicaVersion !== null ||
                         cvr.version.stateVersion !== '00') &&
                         cvr.replicaVersion !== this.#pipelines.replicaVersion) {
-                        const msg = `Replica Version mismatch: CVR=${cvr.replicaVersion}, DB=${this.#pipelines.replicaVersion}`;
-                        this.#lc.info?.(`resetting CVR: ${msg}`);
-                        throw new ErrorForClient(['error', ErrorKind.ClientNotFound, msg]);
+                        const message = `Replica Version mismatch: CVR=${cvr.replicaVersion}, DB=${this.#pipelines.replicaVersion}`;
+                        lc.info?.(`resetting CVR: ${message}`);
+                        throw new ErrorForClient({ kind: ErrorKind.ClientNotFound, message });
                     }
                     if (this.#pipelinesSynced) {
-                        const result = await this.#advancePipelines(cvr);
+                        const result = await this.#advancePipelines(lc, cvr);
                         if (result === 'success') {
                             return;
                         }
-                        this.#lc.info?.(`resetting for schema change: ${result.message}`);
+                        lc.info?.(`resetting for schema change: ${result.message}`);
                         this.#pipelines.reset();
                     }
                     // Advance the snapshot to the current version.
                     const version = this.#pipelines.advanceWithoutDiff();
                     const cvrVer = versionString(cvr.version);
                     if (version < cvr.version.stateVersion) {
-                        this.#lc.debug?.(`replica@${version} is behind cvr@${cvrVer}`);
+                        lc.debug?.(`replica@${version} is behind cvr@${cvrVer}`);
                         return; // Wait for the next advancement.
                     }
                     // stateVersion is at or beyond CVR version for the first time.
-                    this.#lc.info?.(`init pipelines@${version} (cvr@${cvrVer})`);
-                    this.#hydrateUnchangedQueries(cvr);
-                    await this.#syncQueryPipelineSet(cvr);
+                    lc.info?.(`init pipelines@${version} (cvr@${cvrVer})`);
+                    this.#hydrateUnchangedQueries(lc, cvr);
+                    await this.#syncQueryPipelineSet(lc, cvr);
                     this.#pipelinesSynced = true;
                 });
             }
@@ -127,13 +131,13 @@ export class ViewSyncerService {
             this.#cleanup();
         }
         catch (e) {
-            this.#lc.error?.(e);
+            this.#lc.error?.(`stopping view-syncer: ${String(e)}`, e);
             this.#cleanup(e);
         }
         finally {
             // Always wait for the cvrStore to flush, regardless of how the service
             // was stopped.
-            await this.#cvrStore.flushed().catch(e => this.#lc.error?.(e));
+            await this.#cvrStore.flushed(this.#lc).catch(e => this.#lc.error?.(e));
             this.#lc.info?.('view-syncer stopped');
             this.#stopped.resolve();
         }
@@ -141,31 +145,7 @@ export class ViewSyncerService {
     #totalHydrationTimeMs() {
         return this.#pipelines.totalHydrationTimeMs();
     }
-    // The shutdownToken is an object associated with an shutdown timeout function,
-    // the latter of which checks the token with identity equality before
-    // executing. Setting the #shutdownToken to a new object or to `null`
-    // effectively cancels the previous timeout.
-    #shutdownToken = null;
-    #startShutdownTimer(reason) {
-        if (this.#shutdownToken) {
-            // Previous timeout is canceled for efficiency
-            // (but not necessary for correctness).
-            clearTimeout(this.#shutdownToken?.timeoutID);
-            this.#lc.debug?.(`${reason}. resetting idle timer`);
-        }
-        else {
-            this.#lc.debug?.(`${reason}. starting idle timer`);
-        }
-        const shutdownToken = {};
-        this.#shutdownToken = shutdownToken;
-        shutdownToken.timeoutID = setTimeout(() => {
-            // If #idleToken has changed, this timeout is effectively canceled.
-            if (this.#shutdownToken === shutdownToken) {
-                this.#lc.info?.('shutting down after idle timeout');
-                this.#stateChanges.cancel(); // Note: #versionChanges.active becomes false.
-            }
-        }, reason === 'keepalive' ? this.#keepaliveMs : this.#idleTimeoutMs);
-    }
+    #keepAliveUntil = 0;
     /**
      * Guarantees that the ViewSyncer will remain running for at least
      * its configured `keepaliveMs`. This is called when establishing a
@@ -179,12 +159,25 @@ export class ViewSyncerService {
         if (!this.#stateChanges.active) {
             return false;
         }
-        if (this.#shutdownToken) {
-            // Resets the idle timer for another `#keepaliveMs`.
-            this.#startShutdownTimer('keepalive');
-        }
+        this.#keepAliveUntil = Date.now() + this.#keepaliveMs;
         return true;
     }
+    #shutdownTimer = null;
+    async #tryShutdown() {
+        // Keep the view-syncer alive if there are pending rows being flushed.
+        // It's better to do this before shutting down since it may take a
+        // while, during which new connections may come in.
+        await this.#cvrStore.flushed(this.#lc).catch(e => this.#lc.error?.(e));
+        this.#shutdownTimer = null;
+        if (Date.now() <= this.#keepAliveUntil) {
+            this.#lc.debug?.('not shutting down: keepalive');
+            this.#shutdownTimer = setTimeout(() => this.#tryShutdown(), this.#keepaliveMs);
+        }
+        else if (this.#clients.size === 0) {
+            this.#lc.info?.('shutting down');
+            this.#stateChanges.cancel(); // Note: #stateChanges.active becomes false.
+        }
+    }
     #deleteClient(clientID, client) {
         // Note: It is okay to delete / cleanup clients without acquiring the lock.
         // In fact, it is important to do so in order to guarantee that idle cleanup
@@ -193,30 +186,39 @@ export class ViewSyncerService {
         const c = this.#clients.get(clientID);
         if (c === client) {
             this.#clients.delete(clientID);
-            if (this.#clients.size === 0) {
-                this.#startShutdownTimer('no more clients');
+            if (this.#clients.size === 0 && this.#shutdownTimer === null) {
+                this.#shutdownTimer = setTimeout(() => this.#tryShutdown(), 0);
             }
         }
     }
-    async initConnection(ctx, initConnectionMessage) {
-        this.#lastConnectTime = Date.now();
-        const { clientID, wsID, baseCookie, schemaVersion, tokenData } = ctx;
-        this.#authData = pickToken(this.#authData, tokenData?.decoded);
-        const lc = this.#lc
-            .withContext('clientID', clientID)
-            .withContext('wsID', wsID);
-        // Setup the downstream connection.
-        const downstream = Subscription.create({
-            cleanup: (_, err) => {
-                err
-                    ? lc.error?.(`client closed with error`, err)
-                    : lc.info?.('client closed');
-                this.#deleteClient(clientID, newClient);
-            },
+    initConnection(ctx, initConnectionMessage) {
+        return startSpan(tracer, 'vs.initConnection', () => {
+            this.#lastConnectTime = Date.now();
+            const { clientID, wsID, baseCookie, schemaVersion, tokenData } = ctx;
+            this.#authData = pickToken(this.#authData, tokenData?.decoded);
+            const lc = this.#lc
+                .withContext('clientID', clientID)
+                .withContext('wsID', wsID);
+            // Setup the downstream connection.
+            const downstream = Subscription.create({
+                cleanup: (_, err) => {
+                    err
+                        ? lc.error?.(`client closed with error`, err)
+                        : lc.info?.('client closed');
+                    this.#deleteClient(clientID, newClient);
+                },
+            });
+            const newClient = new ClientHandler(lc, this.id, clientID, wsID, this.#shardID, baseCookie, schemaVersion, downstream);
+            this.#clients.get(clientID)?.close(`replaced by wsID: ${wsID}`);
+            this.#clients.set(clientID, newClient);
+            // Note: initConnection() must be synchronous so that `downstream` is
+            // immediately returned to the caller (connection.ts). This ensures
+            // that if the connection is subsequently closed, the `downstream`
+            // subscription can be properly canceled even if #runInLockForClient()
+            // has not had a chance to run.
+            void this.#runInLockForClient(ctx, initConnectionMessage, this.#patchQueries, newClient).catch(e => newClient.fail(e));
+            return downstream;
         });
-        const newClient = new ClientHandler(lc, this.id, clientID, wsID, this.#shardID, baseCookie, schemaVersion, downstream);
-        await this.#runInLockForClient(ctx, initConnectionMessage, this.#patchQueries, newClient);
-        return downstream;
     }
     async changeDesiredQueries(ctx, msg) {
         await this.#runInLockForClient(ctx, msg, this.#patchQueries);
@@ -225,30 +227,17 @@ export class ViewSyncerService {
      * Runs the given `fn` to process the `msg` from within the `#lock`,
      * optionally adding the `newClient` if supplied.
      */
-    async #runInLockForClient(ctx, msg, fn, newClient) {
+    #runInLockForClient(ctx, msg, fn, newClient) {
         const { clientID, wsID } = ctx;
         const [cmd, body] = msg;
-        const lc = this.#lc
-            .withContext('clientID', clientID)
-            .withContext('wsID', wsID)
-            .withContext('cmd', cmd);
-        // Clear and cancel any shutdown timeout.
-        if (this.#shutdownToken) {
-            clearTimeout(this.#shutdownToken.timeoutID);
-            this.#shutdownToken = null;
-        }
-        let client;
-        try {
-            await this.#runInLockWithCVR(cvr => {
-                lc.debug?.(cmd, body);
-                if (newClient) {
-                    assert(newClient.wsID === wsID);
-                    this.#clients.get(clientID)?.close();
-                    this.#clients.set(clientID, newClient);
-                    client = newClient;
-                    checkClientAndCVRVersions(client.version(), cvr.version);
-                }
-                else {
+        return startAsyncSpan(tracer, `vs.#runInLockForClient(${cmd})`, async () => {
+            let client;
+            try {
+                await this.#runInLockWithCVR((lc, cvr) => {
+                    lc = lc
+                        .withContext('clientID', clientID)
+                        .withContext('wsID', wsID)
+                        .withContext('cmd', cmd);
                     client = this.#clients.get(clientID);
                     if (client?.wsID !== wsID) {
                         // Only respond to messages of the currently connected client.
@@ -256,24 +245,39 @@ export class ViewSyncerService {
                         lc.debug?.(`client no longer connected. dropping ${cmd} message`);
                         return;
                     }
-                }
-                return fn(lc, clientID, body, cvr);
-            });
-        }
-        catch (e) {
-            lc.error?.(`closing connection with error`, e);
-            if (client) {
-                // Ideally, propagate the exception to the client's downstream subscription ...
-                client.fail(e);
+                    if (newClient) {
+                        assert(newClient === client);
+                        checkClientAndCVRVersions(client.version(), cvr.version);
+                    }
+                    lc.debug?.(cmd, body);
+                    return fn(lc, clientID, body, cvr);
+                });
             }
-            else {
-                // unless the exception happened before the client could be looked up.
-                throw e;
+            catch (e) {
+                this.#lc
+                    .withContext('clientID', clientID)
+                    .withContext('wsID', wsID)
+                    .withContext('cmd', cmd)
+                    .error?.(`closing connection with error`, e);
+                if (client) {
+                    // Ideally, propagate the exception to the client's downstream subscription ...
+                    client.fail(e);
+                }
+                else {
+                    // unless the exception happened before the client could be looked up.
+                    throw e;
+                }
             }
-        }
+        });
+    }
+    #getClients(atVersion) {
+        const clients = [...this.#clients.values()];
+        return atVersion
+            ? clients.filter(c => cmpVersions(c.version() ?? EMPTY_CVR_VERSION, atVersion) === 0)
+            : clients;
     }
     // Must be called from within #lock.
-    #patchQueries = async (lc, clientID, { desiredQueriesPatch }, cvr) => {
+    #patchQueries = (lc, clientID, { desiredQueriesPatch }, cvr) => startAsyncSpan(tracer, 'vs.#patchQueries', async () => {
         // Apply requested patches.
         if (desiredQueriesPatch.length) {
             lc.debug?.(`applying ${desiredQueriesPatch.length} query patches`);
@@ -282,7 +286,9 @@ export class ViewSyncerService {
             for (const patch of desiredQueriesPatch) {
                 switch (patch.op) {
                     case 'put':
-                        patches.push(...updater.putDesiredQueries(clientID, { [patch.hash]: patch.ast }));
+                        patches.push(...updater.putDesiredQueries(clientID, {
+                            [patch.hash]: patch.ast,
+                        }));
                         break;
                     case 'del':
                         patches.push(...updater.deleteDesiredQueries(clientID, [patch.hash]));
@@ -298,11 +304,7 @@ export class ViewSyncerService {
                 // (Clients that are behind the cvr.version need to be caught up in
                 //  #syncQueryPipelineSet(), as row data may be needed for catchup)
                 const newCVR = this.#cvr;
-                const pokers = [...this.#clients.values()]
-                    .filter(c => 
-                // i.e. only clients that were caught up with the previous cvr
-                cmpVersions(c.version() ?? EMPTY_CVR_VERSION, cvr.version) === 0)
-                    .map(c => c.startPoke(newCVR.version));
+                const pokers = this.#getClients(cvr.version).map(c => c.startPoke(newCVR.version));
                 for (const patch of patches) {
                     pokers.forEach(poker => poker.addPatch(patch));
                 }
@@ -311,9 +313,9 @@ export class ViewSyncerService {
             cvr = this.#cvr; // For #syncQueryPipelineSet().
         }
         if (this.#pipelinesSynced) {
-            await this.#syncQueryPipelineSet(cvr);
+            await this.#syncQueryPipelineSet(lc, cvr);
         }
-    };
+    });
     /**
      * Adds and hydrates pipelines for queries whose results are already
      * recorded in the CVR. Namely:
@@ -329,12 +331,12 @@ export class ViewSyncerService {
      *
      * This must be called from within the #lock.
      */
-    #hydrateUnchangedQueries(cvr) {
+    #hydrateUnchangedQueries(lc, cvr) {
         assert(this.#pipelines.initialized());
         const dbVersion = this.#pipelines.currentVersion();
         const cvrVersion = cvr.version;
         if (cvrVersion.stateVersion !== dbVersion) {
-            this.#lc.info?.(`CVR (${versionToCookie(cvrVersion)}) is behind db ${dbVersion}`);
+            lc.info?.(`CVR (${versionToCookie(cvrVersion)}) is behind db ${dbVersion}`);
             return; // hydration needs to be run with the CVR updater.
         }
         const gotQueries = Object.entries(cvr.queries).filter(([_, state]) => state.transformationHash !== undefined);
@@ -351,11 +353,16 @@ export class ViewSyncerService {
             }
             const start = Date.now();
             let count = 0;
-            for (const _ of this.#pipelines.addQuery(transformationHash, transformedAst)) {
-                count++;
-            }
+            startSpan(tracer, 'vs.#hydrateUnchangedQueries.addQuery', span => {
+                span.setAttribute('queryHash', hash);
+                span.setAttribute('transformationHash', transformationHash);
+                span.setAttribute('table', ast.table);
+                for (const _ of this.#pipelines.addQuery(transformationHash, transformedAst)) {
+                    count++;
+                }
+            });
             const elapsed = Date.now() - start;
-            this.#lc.debug?.(`hydrated ${count} rows for ${hash} (${elapsed} ms)`);
+            lc.debug?.(`hydrated ${count} rows for ${hash} (${elapsed} ms)`);
         }
     }
     /**
@@ -366,91 +373,100 @@ export class ViewSyncerService {
      *
      * This must be called from within the #lock.
      */
-    async #syncQueryPipelineSet(cvr) {
-        assert(this.#pipelines.initialized());
-        const lc = this.#lc.withContext('cvrVersion', versionString(cvr.version));
-        const hydratedQueries = this.#pipelines.addedQueries();
-        // Convert queries to their transformed ast's and hashes
-        const transformationHashToHash = new Map();
-        const serverQueries = Object.entries(cvr.queries).map(([id, q]) => {
-            const { query, hash: transformationHash } = transformAndHashQuery(q.ast, this.#permissions, this.#authData);
-            assert(query !== undefined && transformationHash !== undefined, 'This query may not be run because the table it reads is not readable. Table: ' +
-                q.ast.table);
-            transformationHashToHash.set(transformationHash, id);
-            return {
-                id,
-                // TODO(mlaw): follow up to handle the case where we statically determine
-                // the query cannot be run and is `undefined`.
-                ast: query,
-                transformationHash,
-                desired: q.internal || Object.keys(q.desiredBy).length > 0,
-            };
+    #syncQueryPipelineSet(lc, cvr) {
+        return startAsyncSpan(tracer, 'vs.#syncQueryPipelineSet', async () => {
+            assert(this.#pipelines.initialized());
+            const hydratedQueries = this.#pipelines.addedQueries();
+            // Convert queries to their transformed ast's and hashes
+            const transformationHashToHash = new Map();
+            const serverQueries = Object.entries(cvr.queries).map(([id, q]) => {
+                const { query, hash: transformationHash } = transformAndHashQuery(q.ast, this.#permissions, this.#authData);
+                assert(query !== undefined && transformationHash !== undefined, 'This query may not be run because the table it reads is not readable. Table: ' +
+                    q.ast.table);
+                transformationHashToHash.set(transformationHash, id);
+                return {
+                    id,
+                    // TODO(mlaw): follow up to handle the case where we statically determine
+                    // the query cannot be run and is `undefined`.
+                    ast: query,
+                    transformationHash,
+                    desired: q.internal || Object.keys(q.desiredBy).length > 0,
+                };
+            });
+            const addQueries = serverQueries.filter(q => q.desired && !hydratedQueries.has(q.transformationHash));
+            const removeQueries = serverQueries.filter(q => !q.desired);
+            const desiredQueries = new Set(serverQueries.filter(q => q.desired).map(q => q.transformationHash));
+            const unhydrateQueries = [...hydratedQueries].filter(transformationHash => !desiredQueries.has(transformationHash));
+            if (addQueries.length > 0 ||
+                removeQueries.length > 0 ||
+                unhydrateQueries.length > 0) {
+                await this.#addAndRemoveQueries(lc, cvr, addQueries, removeQueries, unhydrateQueries, transformationHashToHash);
+            }
+            else {
+                await this.#catchupClients(lc, cvr);
+            }
+            // If CVR was non-empty, then the CVR, database, and all clients
+            // should now be at the same version.
+            if (serverQueries.length > 0) {
+                const cvrVersion = must(this.#cvr).version;
+                const dbVersion = this.#pipelines.currentVersion();
+                assert(cvrVersion.stateVersion === dbVersion, `CVR@${versionString(cvrVersion)}" does not match DB@${dbVersion}`);
+            }
         });
-        const addQueries = serverQueries.filter(q => q.desired && !hydratedQueries.has(q.transformationHash));
-        const removeQueries = serverQueries.filter(q => !q.desired);
-        const desiredQueries = new Set(serverQueries.filter(q => q.desired).map(q => q.transformationHash));
-        const unhydrateQueries = [...hydratedQueries].filter(transformationHash => !desiredQueries.has(transformationHash));
-        if (addQueries.length > 0 ||
-            removeQueries.length > 0 ||
-            unhydrateQueries.length > 0) {
-            await this.#addAndRemoveQueries(lc, cvr, addQueries, removeQueries, unhydrateQueries, transformationHashToHash);
-        }
-        else {
-            await this.#catchupClients(lc, cvr);
-        }
-        // If CVR was non-empty, then the CVR, database, and all clients
-        // should now be at the same version.
-        if (serverQueries.length > 0) {
-            const cvrVersion = must(this.#cvr).version;
-            const dbVersion = this.#pipelines.currentVersion();
-            assert(cvrVersion.stateVersion === dbVersion, `CVR@${versionString(cvrVersion)}" does not match DB@${dbVersion}`);
-        }
     }
     // This must be called from within the #lock.
-    async #addAndRemoveQueries(lc, cvr, addQueries, removeQueries, unhydrateQueries, transformationHashToHash) {
-        assert(addQueries.length > 0 ||
-            removeQueries.length > 0 ||
-            unhydrateQueries.length > 0);
-        const start = Date.now();
-        const stateVersion = this.#pipelines.currentVersion();
-        lc = lc.withContext('stateVersion', stateVersion);
-        lc.info?.(`hydrating ${addQueries.length} queries`);
-        const updater = new CVRQueryDrivenUpdater(this.#cvrStore, cvr, stateVersion, this.#pipelines.replicaVersion);
-        // Note: This kicks off background PG queries for CVR data associated with the
-        // executed and removed queries.
-        const { newVersion, queryPatches } = updater.trackQueries(lc, addQueries, removeQueries);
-        const pokers = [...this.#clients.values()].map(c => c.startPoke(newVersion, this.#pipelines.currentSchemaVersions()));
-        for (const patch of queryPatches) {
-            pokers.forEach(poker => poker.addPatch(patch));
-        }
-        // Removing queries is easy. The pipelines are dropped, and the CVR
-        // updater handles the updates and pokes.
-        for (const q of removeQueries) {
-            this.#pipelines.removeQuery(q.transformationHash);
-        }
-        for (const hash of unhydrateQueries) {
-            this.#pipelines.removeQuery(hash);
-        }
-        const pipelines = this.#pipelines;
-        function* generateRowChanges() {
-            for (const q of addQueries) {
-                lc.debug?.(`adding pipeline for query ${q.id}`, q.ast);
-                yield* pipelines.addQuery(q.transformationHash, q.ast);
+    #addAndRemoveQueries(lc, cvr, addQueries, removeQueries, unhydrateQueries, transformationHashToHash) {
+        return startAsyncSpan(tracer, 'vs.#addAndRemoveQueries', async () => {
+            assert(addQueries.length > 0 ||
+                removeQueries.length > 0 ||
+                unhydrateQueries.length > 0);
+            const start = Date.now();
+            const stateVersion = this.#pipelines.currentVersion();
+            lc = lc.withContext('stateVersion', stateVersion);
+            lc.info?.(`hydrating ${addQueries.length} queries`);
+            const updater = new CVRQueryDrivenUpdater(this.#cvrStore, cvr, stateVersion, this.#pipelines.replicaVersion);
+            // Note: This kicks off background PG queries for CVR data associated with the
+            // executed and removed queries.
+            const { newVersion, queryPatches } = updater.trackQueries(lc, addQueries, removeQueries);
+            const pokers = this.#getClients().map(c => c.startPoke(newVersion, this.#pipelines.currentSchemaVersions()));
+            for (const patch of queryPatches) {
+                pokers.forEach(poker => poker.addPatch(patch));
             }
-        }
-        // #processChanges does batched de-duping of rows. Wrap all pipelines in
-        // a single generator in order to maximize de-duping.
-        await this.#processChanges(lc, generateRowChanges(), updater, pokers, transformationHashToHash);
-        for (const patch of await updater.deleteUnreferencedRows(lc)) {
-            pokers.forEach(poker => poker.addPatch(patch));
-        }
-        // Commit the changes and update the CVR snapshot.
-        this.#cvr = (await updater.flush(lc, this.#lastConnectTime)).cvr;
-        // Before ending the poke, catch up clients that were behind the old CVR.
-        await this.#catchupClients(lc, cvr, addQueries.map(q => q.id), pokers);
-        // Signal clients to commit.
-        pokers.forEach(poker => poker.end());
-        lc.info?.(`finished processing queries (${Date.now() - start} ms)`);
+            // Removing queries is easy. The pipelines are dropped, and the CVR
+            // updater handles the updates and pokes.
+            for (const q of removeQueries) {
+                this.#pipelines.removeQuery(q.transformationHash);
+            }
+            for (const hash of unhydrateQueries) {
+                this.#pipelines.removeQuery(hash);
+            }
+            const pipelines = this.#pipelines;
+            function* generateRowChanges() {
+                for (const q of addQueries) {
+                    lc.debug?.(`adding pipeline for query ${q.id}`, q.ast);
+                    const start = performance.now();
+                    yield* pipelines.addQuery(q.transformationHash, q.ast);
+                    const end = performance.now();
+                    manualSpan(tracer, 'vs.addAndConsumeQuery', end - start, {
+                        hash: q.id,
+                        transformationHash: q.transformationHash,
+                    });
+                }
+            }
+            // #processChanges does batched de-duping of rows. Wrap all pipelines in
+            // a single generator in order to maximize de-duping.
+            await this.#processChanges(lc, generateRowChanges(), updater, pokers, transformationHashToHash);
+            for (const patch of await updater.deleteUnreferencedRows(lc)) {
+                pokers.forEach(poker => poker.addPatch(patch));
+            }
+            // Commit the changes and update the CVR snapshot.
+            this.#cvr = (await updater.flush(lc, this.#lastConnectTime)).cvr;
+            // Before ending the poke, catch up clients that were behind the old CVR.
+            await this.#catchupClients(lc, cvr, addQueries.map(q => q.id), pokers);
+            // Signal clients to commit.
+            pokers.forEach(poker => poker.end());
+            lc.info?.(`finished processing queries (${Date.now() - start} ms)`);
+        });
     }
     /**
      * @param cvr The CVR to which clients should be caught up to. This does
@@ -463,105 +479,116 @@ export class ViewSyncerService {
      *     using the version from the supplied `cvr`.
      */
     // Must be called within #lock
-    async #catchupClients(lc, cvr, excludeQueryHashes = [], usePokers) {
-        const pokers = usePokers ??
-            [...this.#clients.values()].map(c => c.startPoke(cvr.version, this.#pipelines.currentSchemaVersions()));
-        const catchupFrom = [...this.#clients.values()]
-            .map(c => c.version())
-            .reduce((a, b) => (cmpVersions(a, b) < 0 ? a : b), cvr.version);
-        // This is an AsyncGenerator which won't execute until awaited.
-        const rowPatches = this.#cvrStore.catchupRowPatches(lc, catchupFrom, cvr, excludeQueryHashes);
-        // This is a plain async function that kicks off immediately.
-        const configPatches = this.#cvrStore.catchupConfigPatches(lc, catchupFrom, cvr);
-        // await the rowPatches first so that the AsyncGenerator kicks off.
-        let rowPatchCount = 0;
-        for await (const rows of rowPatches) {
-            for (const row of rows) {
-                const { schema, table } = row;
-                const rowKey = row.rowKey;
-                const toVersion = versionFromString(row.patchVersion);
-                const id = { schema, table, rowKey };
-                let patch;
-                if (!row.refCounts) {
-                    patch = { type: 'row', op: 'del', id };
-                }
-                else {
-                    const row = must(this.#pipelines.getRow(table, rowKey), `Missing row ${table}:${stringify(rowKey)}`);
-                    const { contents } = contentsAndVersion(row);
-                    patch = { type: 'row', op: 'put', id, contents };
+    #catchupClients(lc, cvr, excludeQueryHashes = [], usePokers) {
+        return startAsyncSpan(tracer, 'vs.#catchupClients', async (span) => {
+            const clients = this.#getClients();
+            const pokers = usePokers ??
+                clients.map(c => c.startPoke(cvr.version, this.#pipelines.currentSchemaVersions()));
+            span.setAttribute('numPokers', pokers.length);
+            span.setAttribute('numClients', clients.length);
+            const catchupFrom = clients
+                .map(c => c.version())
+                .reduce((a, b) => (cmpVersions(a, b) < 0 ? a : b), cvr.version);
+            // This is an AsyncGenerator which won't execute until awaited.
+            const rowPatches = this.#cvrStore.catchupRowPatches(lc, catchupFrom, cvr, excludeQueryHashes);
+            // This is a plain async function that kicks off immediately.
+            const configPatches = this.#cvrStore.catchupConfigPatches(lc, catchupFrom, cvr);
+            // await the rowPatches first so that the AsyncGenerator kicks off.
+            let rowPatchCount = 0;
+            for await (const rows of rowPatches) {
+                for (const row of rows) {
+                    const { schema, table } = row;
+                    const rowKey = row.rowKey;
+                    const toVersion = versionFromString(row.patchVersion);
+                    const id = { schema, table, rowKey };
+                    let patch;
+                    if (!row.refCounts) {
+                        patch = { type: 'row', op: 'del', id };
+                    }
+                    else {
+                        const row = must(this.#pipelines.getRow(table, rowKey), `Missing row ${table}:${stringify(rowKey)}`);
+                        const { contents } = contentsAndVersion(row);
+                        patch = { type: 'row', op: 'put', id, contents };
+                    }
+                    const patchToVersion = { patch, toVersion };
+                    pokers.forEach(poker => poker.addPatch(patchToVersion));
+                    rowPatchCount++;
                 }
-                const patchToVersion = { patch, toVersion };
-                pokers.forEach(poker => poker.addPatch(patchToVersion));
-                rowPatchCount++;
-            }
-        }
-        if (rowPatchCount) {
-            lc.debug?.(`sent ${rowPatchCount} row patches`);
-        }
-        // Then await the config patches which were fetched in parallel.
-        for (const patch of await configPatches) {
-            pokers.forEach(poker => poker.addPatch(patch));
-        }
-        if (!usePokers) {
-            pokers.forEach(poker => poker.end());
-        }
-    }
-    async #processChanges(lc, changes, updater, pokers, transformationHashToHash) {
-        const start = Date.now();
-        const rows = new CustomKeyMap(rowIDString);
-        let total = 0;
-        const processBatch = async () => {
-            const elapsed = Date.now() - start;
-            total += rows.size;
-            lc.debug?.(`processing ${rows.size} (of ${total}) rows (${elapsed} ms)`);
-            const patches = await updater.received(this.#lc, rows);
-            patches.forEach(patch => pokers.forEach(poker => poker.addPatch(patch)));
-            rows.clear();
-        };
-        for (const change of changes) {
-            const { type, queryHash: transformationHash, table, rowKey, row } = change;
-            const queryHash = must(transformationHashToHash.get(transformationHash), 'could not find the original hash for the transformation hash');
-            const rowID = { schema: '', table, rowKey: rowKey };
-            let parsedRow = rows.get(rowID);
-            let rc;
-            if (!parsedRow) {
-                parsedRow = { refCounts: {} };
-                rows.set(rowID, parsedRow);
-                rc = 0;
             }
-            else {
-                rc = parsedRow.refCounts[queryHash] ?? 0;
+            span.setAttribute('rowPatchCount', rowPatchCount);
+            if (rowPatchCount) {
+                lc.debug?.(`sent ${rowPatchCount} row patches`);
             }
-            const updateVersion = (row) => {
-                if (!parsedRow.version) {
-                    const { version, contents } = contentsAndVersion(row);
-                    parsedRow.version = version;
-                    parsedRow.contents = contents;
-                }
-            };
-            switch (type) {
-                case 'add':
-                    updateVersion(row);
-                    rc++;
-                    break;
-                case 'edit':
-                    updateVersion(row);
-                    // No update to rc.
-                    break;
-                case 'remove':
-                    rc--;
-                    break;
-                default:
-                    unreachable(type);
+            // Then await the config patches which were fetched in parallel.
+            for (const patch of await configPatches) {
+                pokers.forEach(poker => poker.addPatch(patch));
             }
-            parsedRow.refCounts[queryHash] = rc;
-            if (rows.size % CURSOR_PAGE_SIZE === 0) {
-                await processBatch();
+            if (!usePokers) {
+                pokers.forEach(poker => poker.end());
             }
-        }
-        if (rows.size) {
-            await processBatch();
-        }
+        });
+    }
+    #processChanges(lc, changes, updater, pokers, transformationHashToHash) {
+        return startAsyncSpan(tracer, 'vs.#processChanges', async () => {
+            const start = Date.now();
+            const rows = new CustomKeyMap(rowIDString);
+            let total = 0;
+            const processBatch = () => startAsyncSpan(tracer, 'processBatch', async () => {
+                const elapsed = Date.now() - start;
+                total += rows.size;
+                lc.debug?.(`processing ${rows.size} (of ${total}) rows (${elapsed} ms)`);
+                const patches = await updater.received(lc, rows);
+                patches.forEach(patch => pokers.forEach(poker => poker.addPatch(patch)));
+                rows.clear();
+            });
+            await startAsyncSpan(tracer, 'loopingChanges', async (span) => {
+                for (const change of changes) {
+                    const { type, queryHash: transformationHash, table, rowKey, row, } = change;
+                    const queryHash = must(transformationHashToHash.get(transformationHash), 'could not find the original hash for the transformation hash');
+                    const rowID = { schema: '', table, rowKey: rowKey };
+                    let parsedRow = rows.get(rowID);
+                    let rc;
+                    if (!parsedRow) {
+                        parsedRow = { refCounts: {} };
+                        rows.set(rowID, parsedRow);
+                        rc = 0;
+                    }
+                    else {
+                        rc = parsedRow.refCounts[queryHash] ?? 0;
+                    }
+                    const updateVersion = (row) => {
+                        if (!parsedRow.version) {
+                            const { version, contents } = contentsAndVersion(row);
+                            parsedRow.version = version;
+                            parsedRow.contents = contents;
+                        }
+                    };
+                    switch (type) {
+                        case 'add':
+                            updateVersion(row);
+                            rc++;
+                            break;
+                        case 'edit':
+                            updateVersion(row);
+                            // No update to rc.
+                            break;
+                        case 'remove':
+                            rc--;
+                            break;
+                        default:
+                            unreachable(type);
+                    }
+                    parsedRow.refCounts[queryHash] = rc;
+                    if (rows.size % CURSOR_PAGE_SIZE === 0) {
+                        await processBatch();
+                    }
+                }
+                if (rows.size) {
+                    await processBatch();
+                }
+                span.setAttribute('totalRows', total);
+            });
+        });
     }
     /**
      * Advance to the current snapshot of the replica and apply / send
@@ -571,38 +598,43 @@ export class ViewSyncerService {
      *
      * Returns false if the advancement failed due to a schema change.
      */
-    async #advancePipelines(cvr) {
-        assert(this.#pipelines.initialized());
-        const start = Date.now();
-        const { version, numChanges, changes } = this.#pipelines.advance();
-        const lc = this.#lc.withContext('newVersion', version);
-        // Probably need a new updater type. CVRAdvancementUpdater?
-        const updater = new CVRQueryDrivenUpdater(this.#cvrStore, cvr, version, this.#pipelines.replicaVersion);
-        const pokers = [...this.#clients.values()].map(c => c.startPoke(updater.updatedVersion(), this.#pipelines.currentSchemaVersions()));
-        lc.debug?.(`applying ${numChanges} to advance to ${version}`);
-        const transformationHashToHash = new Map();
-        for (const query of Object.values(cvr.queries)) {
-            if (!query.transformationHash) {
-                continue;
+    #advancePipelines(lc, cvr) {
+        return startAsyncSpan(tracer, 'vs.#advancePipelines', async () => {
+            assert(this.#pipelines.initialized());
+            const start = Date.now();
+            const { version, numChanges, changes } = this.#pipelines.advance();
+            lc = lc.withContext('newVersion', version);
+            // Probably need a new updater type. CVRAdvancementUpdater?
+            const updater = new CVRQueryDrivenUpdater(this.#cvrStore, cvr, version, this.#pipelines.replicaVersion);
+            // Only poke clients that are at the cvr.version. New clients that
+            // are behind need to first be caught up when their initConnection
+            // message is processed (and #syncQueryPipelines is called).
+            const pokers = this.#getClients(cvr.version).map(c => c.startPoke(updater.updatedVersion(), this.#pipelines.currentSchemaVersions()));
+            lc.debug?.(`applying ${numChanges} to advance to ${version}`);
+            const transformationHashToHash = new Map();
+            for (const query of Object.values(cvr.queries)) {
+                if (!query.transformationHash) {
+                    continue;
+                }
+                transformationHashToHash.set(query.transformationHash, query.id);
             }
-            transformationHashToHash.set(query.transformationHash, query.id);
-        }
-        try {
-            await this.#processChanges(lc, changes, updater, pokers, transformationHashToHash);
-        }
-        catch (e) {
-            if (e instanceof SchemaChangeError) {
-                pokers.forEach(poker => poker.cancel());
-                return e;
+            try {
+                await this.#processChanges(lc, changes, updater, pokers, transformationHashToHash);
             }
-            throw e;
-        }
-        // Commit the changes and update the CVR snapshot.
-        this.#cvr = (await updater.flush(lc, this.#lastConnectTime)).cvr;
-        // Signal clients to commit.
-        pokers.forEach(poker => poker.end());
-        lc.info?.(`finished processing advancement (${Date.now() - start} ms)`);
-        return 'success';
+            catch (e) {
+                if (e instanceof SchemaChangeError) {
+                    pokers.forEach(poker => poker.cancel());
+                    return e;
+                }
+                throw e;
+            }
+            // Commit the changes and update the CVR snapshot.
+            this.#cvr = (await updater.flush(lc, this.#lastConnectTime)).cvr;
+            // Signal clients to commit.
+            pokers.forEach(poker => poker.end());
+            lc.info?.(`finished processing advancement (${Date.now() - start} ms)`);
+            return 'success';
+        });
     }
     stop() {
         this.#lc.info?.('stopping view syncer');
@@ -616,7 +648,7 @@ export class ViewSyncerService {
                 client.fail(err);
             }
             else {
-                client.close();
+                client.close('shutting down');
             }
         }
     }
@@ -634,19 +666,17 @@ function checkClientAndCVRVersions(client, cvr) {
     if (cmpVersions(cvr, NEW_CVR_VERSION) === 0 &&
         cmpVersions(client, NEW_CVR_VERSION) > 0) {
         // CVR is empty but client is not.
-        throw new ErrorForClient([
-            'error',
-            ErrorKind.ClientNotFound,
-            'Client not found',
-        ]);
+        throw new ErrorForClient({
+            kind: ErrorKind.ClientNotFound,
+            message: 'Client not found',
+        });
     }
     if (cmpVersions(client, cvr) > 0) {
         // Client is ahead of a non-empty CVR.
-        throw new ErrorForClient([
-            'error',
-            ErrorKind.InvalidConnectionRequestBaseCookie,
-            `CVR is at version ${versionString(cvr)}`,
-        ]);
+        throw new ErrorForClient({
+            kind: ErrorKind.InvalidConnectionRequestBaseCookie,
+            message: `CVR is at version ${versionString(cvr)}`,
+        });
     }
 }
 export function pickToken(previousToken, newToken) {
@@ -655,22 +685,20 @@ export function pickToken(previousToken, newToken) {
     }
     if (newToken) {
         if (previousToken.sub !== newToken.sub) {
-            throw new ErrorForClient([
-                'error',
-                ErrorKind.Unauthorized,
-                'The user id in the new token does not match the previous token. Client groups are pinned to a single user.',
-            ]);
+            throw new ErrorForClient({
+                kind: ErrorKind.Unauthorized,
+                message: 'The user id in the new token does not match the previous token. Client groups are pinned to a single user.',
+            });
         }
         if (previousToken.iat === undefined) {
             // No issued at time for the existing token? We take the most recently received token.
             return newToken;
         }
         if (newToken.iat === undefined) {
-            throw new ErrorForClient([
-                'error',
-                ErrorKind.Unauthorized,
-                'The new token does not have an issued at time but the prior token does. Tokens for a client group must either all have issued at times or all not have issued at times',
-            ]);
+            throw new ErrorForClient({
+                kind: ErrorKind.Unauthorized,
+                message: 'The new token does not have an issued at time but the prior token does. Tokens for a client group must either all have issued at times or all not have issued at times',
+            });
         }
         // The new token is newer, so we take it.
         if (previousToken.iat < newToken.iat) {
@@ -680,10 +708,9 @@ export function pickToken(previousToken, newToken) {
         return previousToken;
     }
     // previousToken !== undefined but newToken is undefined
-    throw new ErrorForClient([
-        'error',
-        ErrorKind.Unauthorized,
-        'No token provided. An unauthenticated client cannot connect to an authenticated client group.',
-    ]);
+    throw new ErrorForClient({
+        kind: ErrorKind.Unauthorized,
+        message: 'No token provided. An unauthenticated client cannot connect to an authenticated client group.',
+    });
 }
 //# sourceMappingURL=view-syncer.js.map