@rocicorp/zero 0.22.2025071100 → 0.22.2025071900

package/out/zero-cache/src/services/view-syncer/view-syncer.js

@@ -10,6 +10,7 @@ import { hasOwn } from "../../../../shared/src/has-own.js";
 import { must } from "../../../../shared/src/must.js";
 import { randInt } from "../../../../shared/src/rand.js";
 import { ErrorKind } from "../../../../zero-protocol/src/error-kind.js";
+import { clampTTL, MAX_TTL_MS } from "../../../../zql/src/query/ttl.js";
 import { transformAndHashQuery, } from "../../auth/read-authorizer.js";
 import {} from "../../config/zero-config.js";
 import { CustomQueryTransformer } from "../../custom-queries/transform-query.js";
@@ -21,15 +22,13 @@ import { Subscription } from "../../types/subscription.js";
 import { ZERO_VERSION_COLUMN_NAME } from "../replicator/schema/replication-state.js";
 import { ClientHandler, startPoke, } from "./client-handler.js";
 import { CVRStore } from "./cvr-store.js";
-import { CVRConfigDrivenUpdater, CVRQueryDrivenUpdater, CVRUpdater, getInactiveQueries, nextEvictionTime, } from "./cvr.js";
+import { CVRConfigDrivenUpdater, CVRQueryDrivenUpdater, CVRUpdater, nextEvictionTime, } from "./cvr.js";
 import { PipelineDriver } from "./pipeline-driver.js";
 import { cmpVersions, EMPTY_CVR_VERSION, versionFromString, versionString, versionToCookie, } from "./schema/types.js";
 import { ResetPipelinesSignal } from "./snapshotter.js";
+import { ttlClockAsNumber, ttlClockFromNumber, } from "./ttl-clock.js";
 const tracer = trace.getTracer('view-syncer', version);
 const DEFAULT_KEEPALIVE_MS = 5_000;
-// We have previously said that the goal is to have 20MB on the client.
-// If we assume each row is ~1KB, then we can have 20,000 rows.
-const DEFAULT_MAX_ROW_COUNT = 20_000;
 function randomID() {
     return randInt(1, Number.MAX_SAFE_INTEGER).toString(36);
 }
@@ -38,7 +37,14 @@ function randomID() {
  * some flushes do not write to the CVR and in those cases we
  * use a timer to update the ttlClock every minute.
  */
-const TTL_CLOCK_INTERVAL = 60_000;
+export const TTL_CLOCK_INTERVAL = 60_000;
+/**
+ * This is some extra time we delay the TTL timer to allow for some
+ * slack in the timing of the timer. This is to allow multiple evictions
+ * to happen in a short period of time without having to wait for the
+ * next tick of the timer.
+ */
+export const TTL_TIMER_HYSTERESIS = 50; // ms
 export class ViewSyncerService {
     id;
     #shard;
@@ -48,7 +54,7 @@ export class ViewSyncerService {
     #drainCoordinator;
     #keepaliveMs;
     #slowHydrateThreshold;
-    #pullConfig;
+    #queryConfig;
     // The ViewSyncerService is only started in response to a connection,
     // so #lastConnectTime is always initialized to now(). This is necessary
     // to handle race conditions in which, e.g. the replica is ready and the
@@ -60,7 +66,7 @@ export class ViewSyncerService {
      * The TTL clock is used to determine the time at which queries are considered
      * expired.
      */
-    #ttlClock = Date.now();
+    #ttlClock;
     /**
      * The base time for the TTL clock. This is used to compute the current TTL
      * clock value. The first time a connection is made, this is set to the
@@ -73,14 +79,12 @@ export class ViewSyncerService {
      * time that has passed since the last time we set it.
      */
     #ttlClockBase = Date.now();
-    get ttlClockBase() {
-        return this.#ttlClockBase;
-    }
     /**
      * We update the ttlClock every minute to ensure that it is not too much
      * out of sync with the current time.
      */
     #ttlClockInterval = 0;
+    #ttlClockInitializedPromise;
     // Note: It is okay to add/remove clients without acquiring the lock.
     #clients = new Map();
     // Serialize on this lock for:
@@ -95,25 +99,13 @@ export class ViewSyncerService {
     // auth and cookie headers directly
     #authData;
     #httpCookie;
-    /**
-     * The {@linkcode maxRowCount} is used for the eviction of inactive queries.
-     * An inactive query is a query that is no longer desired but is kept alive
-     * due to its TTL. When the number of rows in the CVR exceeds
-     * {@linkcode maxRowCount} we keep removing inactive queries (even if they are
-     * not expired yet) until the actual row count is below the max row count.
-     *
-     * There is no guarantee that the number of rows in the CVR will be below this
-     * if there are active queries that have a lot of rows.
-     */
-    maxRowCount;
     #expiredQueriesTimer = 0;
-    #nextExpiredQueryTime = 0;
     #setTimeout;
     #customQueryTransformer;
-    constructor(pullConfig, lc, shard, taskID, clientGroupID, db, pipelineDriver, versionChanges, drainCoordinator, slowHydrateThreshold, keepaliveMs = DEFAULT_KEEPALIVE_MS, maxRowCount = DEFAULT_MAX_ROW_COUNT, setTimeoutFn = setTimeout.bind(globalThis)) {
+    constructor(pullConfig, lc, shard, taskID, clientGroupID, db, pipelineDriver, versionChanges, drainCoordinator, slowHydrateThreshold, keepaliveMs = DEFAULT_KEEPALIVE_MS, setTimeoutFn = setTimeout.bind(globalThis)) {
         this.id = clientGroupID;
         this.#shard = shard;
-        this.#pullConfig = pullConfig;
+        this.#queryConfig = pullConfig;
         this.#lc = lc;
         this.#pipelines = pipelineDriver;
         this.#stateChanges = versionChanges;
@@ -124,7 +116,6 @@ export class ViewSyncerService {
         // On failure, cancel the #stateChanges subscription. The run()
         // loop will then await #cvrStore.flushed() which rejects if necessary.
         () => this.#stateChanges.cancel());
-        this.maxRowCount = maxRowCount;
         this.#setTimeout = setTimeoutFn;
         if (pullConfig.url) {
             this.#customQueryTransformer = new CustomQueryTransformer({
@@ -230,17 +221,18 @@ export class ViewSyncerService {
         }
     }
     // must be called from within #lock
-    #removeExpiredQueries = async (lc, cvr) => {
-        const now = Date.now();
-        const ttlClock = this.#getTTLClock(now);
+    #removeExpiredQueries = async (lc, cvr, ttlClock) => {
         if (hasExpiredQueries(cvr, ttlClock)) {
             lc = lc.withContext('method', '#removeExpiredQueries');
-            lc.info?.('Queries have expired');
+            lc.debug?.('Queries have expired');
             // #syncQueryPipelineSet() will remove the expired queries.
             await this.#syncQueryPipelineSet(lc, cvr);
             this.#pipelinesSynced = true;
-            this.#scheduleExpireEviction(lc, cvr);
         }
+        // Even if we have expired queries, we still need to schedule next eviction
+        // since there might be inactivated queries that need to be expired queries
+        // in the future.
+        this.#scheduleExpireEviction(lc, cvr, ttlClock);
     };
     #totalHydrationTimeMs() {
         return this.#pipelines.totalHydrationTimeMs();
@@ -290,7 +282,7 @@ export class ViewSyncerService {
         // If no clients have connected while waiting for the row flush, shutdown.
         return this.#clients.size === 0;
     }
-    #deleteClient(clientID, client) {
+    #deleteClientDueToDisconnect(clientID, client) {
         // Note: It is okay to delete / cleanup clients without acquiring the lock.
         // In fact, it is important to do so in order to guarantee that idle cleanup
         // is performed in a timely manner, regardless of the amount of work
@@ -299,7 +291,11 @@ export class ViewSyncerService {
         if (c === client) {
             this.#clients.delete(clientID);
             if (this.#clients.size === 0) {
-                this.#updateTTLClockInCVRWithoutLock(this.#lc);
+                // It is possible to delete a client before we read the ttl clock from
+                // the CVR.
+                if (this.#hasTTLClock()) {
+                    this.#updateTTLClockInCVRWithoutLock(this.#lc);
+                }
                 this.#stopExpireTimer();
                 this.#scheduleShutdown();
             }
@@ -309,7 +305,6 @@ export class ViewSyncerService {
         this.#lc.debug?.('Stopping expired queries timer');
         clearTimeout(this.#expiredQueriesTimer);
         this.#expiredQueriesTimer = 0;
-        this.#nextExpiredQueryTime = 0;
     }
     initConnection(ctx, initConnectionMessage) {
         this.#lc.debug?.('viewSyncer.initConnection');
@@ -327,7 +322,7 @@ export class ViewSyncerService {
                     err
                         ? lc[getLogLevel(err)]?.(`client closed with error`, err)
                         : lc.info?.('client closed');
-                    this.#deleteClient(clientID, newClient);
+                    this.#deleteClientDueToDisconnect(clientID, newClient);
                 },
             });
             if (this.#clients.size === 0) {
@@ -337,13 +332,8 @@ export class ViewSyncerService {
                 const now = Date.now();
                 this.#ttlClockBase = now;
                 // Get the TTL clock from the CVR store, or initialize it to now.
-                this.#cvrStore
-                    .getTTLClock()
-                    .then(ttlClock => {
-                    this.#ttlClock = ttlClock ?? now;
-                })
-                    .catch(e => {
-                    this.#lc.error?.('failed to get TTL clock', e);
+                this.#readTTLClockFromCVR().catch(e => {
+                    lc.error?.('failed to read TTL clock', e);
                 });
             }
             const newClient = new ClientHandler(lc, this.id, clientID, wsID, this.#shard, baseCookie, schemaVersion, downstream);
@@ -369,10 +359,15 @@ export class ViewSyncerService {
             this.#lc.error?.('deleteClients failed', e);
         }
     }
+    #hasTTLClock() {
+        return this.#ttlClock !== undefined;
+    }
     #getTTLClock(now) {
         // We will update ttlClock with delta from the ttlClockBase to the current time.
         const delta = now - this.#ttlClockBase;
-        const ttlClock = this.#ttlClock + delta;
+        assert(this.#ttlClock !== undefined);
+        const ttlClock = ttlClockFromNumber(ttlClockAsNumber(this.#ttlClock) + delta);
+        assert(ttlClockAsNumber(ttlClock) <= now);
         this.#ttlClock = ttlClock;
         this.#ttlClockBase = now;
         return ttlClock;
@@ -492,6 +487,8 @@ export class ViewSyncerService {
     #handleConfigUpdate = (lc, clientID, { clientSchema, deleted, desiredQueriesPatch, activeClients, }, cvr) => startAsyncSpan(tracer, 'vs.#patchQueries', async () => {
         const deletedClientIDs = [];
         const deletedClientGroupIDs = [];
+        const now = Date.now();
+        const ttlClock = this.#getTTLClock(now);
         cvr = await this.#updateCVRConfig(lc, cvr, clientID, updater => {
             const patches = [];
             if (clientSchema) {
@@ -500,14 +497,13 @@ export class ViewSyncerService {
             // Apply requested patches.
             lc.debug?.(`applying ${desiredQueriesPatch?.length} query patches`);
             if (desiredQueriesPatch?.length) {
-                const now = Date.now();
                 for (const patch of desiredQueriesPatch) {
                     switch (patch.op) {
                         case 'put':
                             patches.push(...updater.putDesiredQueries(clientID, [patch]));
                             break;
                         case 'del':
-                            patches.push(...updater.markDesiredQueriesAsInactive(clientID, [patch.hash], now));
+                            patches.push(...updater.markDesiredQueriesAsInactive(clientID, [patch.hash], ttlClock));
                             break;
                         case 'clear':
                             patches.push(...updater.clearDesiredQueries(clientID));
@@ -533,7 +529,7 @@ export class ViewSyncerService {
                 }
             }
             for (const cid of clientIDsToDelete) {
-                const patchesDueToClient = updater.deleteClient(cid);
+                const patchesDueToClient = updater.deleteClient(cid, ttlClock);
                 patches.push(...patchesDueToClient);
                 deletedClientIDs.push(cid);
             }
@@ -553,36 +549,34 @@ export class ViewSyncerService {
             const clients = this.#getClients();
             await Promise.allSettled(clients.map(client => client.sendDeleteClients(lc, deletedClientIDs, deletedClientGroupIDs)));
         }
-        this.#scheduleExpireEviction(lc, cvr);
-        await this.#evictInactiveQueries(lc, cvr);
+        this.#scheduleExpireEviction(lc, cvr, ttlClock);
     });
-    #scheduleExpireEviction(lc, cvr) {
+    #scheduleExpireEviction(lc, cvr, ttlClock) {
+        this.#stopExpireTimer();
         // first see if there is any inactive query with a ttl.
         const next = nextEvictionTime(cvr);
         if (next === undefined) {
             lc.debug?.('no inactive queries with ttl');
             // no inactive queries with a ttl. Cancel existing timeout if any.
-            this.#stopExpireTimer();
-            return;
-        }
-        if (this.#nextExpiredQueryTime === next) {
-            lc.debug?.('eviction timer already scheduled');
             return;
         }
-        if (this.#expiredQueriesTimer) {
-            clearTimeout(this.#expiredQueriesTimer);
-        }
-        const now = Date.now();
-        this.#nextExpiredQueryTime = next;
-        const ttlClock = this.#getTTLClock(now);
-        lc.debug?.('Scheduling eviction timer to run in ', next - ttlClock, 'ms');
-        this.#expiredQueriesTimer = this.#setTimeout(() => this.#runInLockWithCVR(this.#removeExpiredQueries).catch(e => 
-        // If an error occurs (e.g. ownership change), propagate the error
-        // to the main run() loop via the #stateChanges Subscription.
-        this.#stateChanges.fail(e)), 
-        // If the expire time is too far in the future we will run it in an hour.
-        // At that point in time it will be rescheduled as needed again.
-        Math.min(next - ttlClock, 60 * 60 * 1000));
+        // It is common for many queries to be evicted close to the same time, so
+        // we add a small delay so we can collapse multiple evictions into a
+        // single timer. However, don't add the delay if we're already at the
+        // maximum timer limit, as that's not about collapsing.
+        const delay = Math.max(TTL_TIMER_HYSTERESIS, Math.min(ttlClockAsNumber(next) -
+            ttlClockAsNumber(ttlClock) +
+            TTL_TIMER_HYSTERESIS, MAX_TTL_MS));
+        lc.debug?.('Scheduling eviction timer to run in ', delay, 'ms');
+        this.#expiredQueriesTimer = this.#setTimeout(() => {
+            this.#expiredQueriesTimer = 0;
+            const now = Date.now();
+            const ttlClock = this.#getTTLClock(now);
+            this.#runInLockWithCVR((lc, cvr) => this.#removeExpiredQueries(lc, cvr, ttlClock)).catch(e => 
+            // If an error occurs (e.g. ownership change), propagate the error
+            // to the main run() loop via the #stateChanges Subscription.
+            this.#stateChanges.fail(e));
+        }, delay);
     }
     /**
      * Adds and hydrates pipelines for queries whose results are already
@@ -628,9 +622,9 @@ export class ViewSyncerService {
         }
         if (this.#customQueryTransformer && customQueries.length > 0) {
             const transformedCustomQueries = await this.#customQueryTransformer.transform({
-                apiKey: this.#pullConfig.apiKey,
+                apiKey: this.#queryConfig.apiKey,
                 token: this.#authData?.raw,
-                cookie: this.#pullConfig.forwardCookies
+                cookie: this.#queryConfig.forwardCookies
                     ? this.#httpCookie
                     : undefined,
             }, customQueries);
@@ -681,6 +675,15 @@ export class ViewSyncerService {
             lc.debug?.(`hydrated ${count} rows for ${hash} (${elapsed} ms)`);
         }
     }
+    async #readTTLClockFromCVR() {
+        if (this.#ttlClockInitializedPromise) {
+            return this.#ttlClockInitializedPromise;
+        }
+        this.#ttlClockInitializedPromise = this.#cvrStore
+            .getTTLClock()
+            .then(t => t ?? ttlClockFromNumber(0));
+        return (this.#ttlClock = await this.#ttlClockInitializedPromise);
+    }
     /**
      * Adds and/or removes queries to/from the PipelineDriver to bring it
      * in sync with the set of queries in the CVR (both got and desired).
@@ -696,7 +699,14 @@ export class ViewSyncerService {
             // Convert queries to their transformed ast's and hashes
             const hashToIDs = new Map();
             const now = Date.now();
-            const ttlClock = this.#getTTLClock(now);
+            let ttlClock;
+            if (!this.#hasTTLClock()) {
+                // Fetch it from the CVR or initialize it to now.
+                ttlClock = await this.#readTTLClockFromCVR();
+            }
+            else {
+                ttlClock = this.#getTTLClock(now);
+            }
             // group cvr queries into:
             // 1. custom queries
             // 2. everything else
@@ -733,7 +743,7 @@ export class ViewSyncerService {
             }
             if (this.#customQueryTransformer && customQueries.size > 0) {
                 const transformedCustomQueries = await this.#customQueryTransformer.transform({
-                    apiKey: this.#pullConfig.apiKey,
+                    apiKey: this.#queryConfig.apiKey,
                     token: this.#authData?.raw,
                     cookie: this.#httpCookie,
                 }, customQueries.values());
@@ -795,7 +805,7 @@ export class ViewSyncerService {
             assert(addQueries.length > 0 ||
                 removeQueries.length > 0 ||
                 unhydrateQueries.length > 0);
-            const start = Date.now();
+            const start = performance.now();
             const stateVersion = this.#pipelines.currentVersion();
             lc = lc.withContext('stateVersion', stateVersion);
             lc.info?.(`hydrating ${addQueries.length} queries`);
@@ -850,7 +860,7 @@ export class ViewSyncerService {
             await this.#catchupClients(lc, cvr, finalVersion, addQueries.map(q => q.id), pokers);
             // Signal clients to commit.
             await pokers.end(finalVersion);
-            const wallTime = Date.now() - start;
+            const wallTime = performance.now() - start;
             lc.info?.(`finished processing queries (process: ${totalProcessTime} ms, wall: ${wallTime} ms)`);
         });
     }
@@ -924,11 +934,11 @@ export class ViewSyncerService {
     }
     #processChanges(lc, timer, changes, updater, pokers, hashToIDs) {
         return startAsyncSpan(tracer, 'vs.#processChanges', async () => {
-            const start = Date.now();
+            const start = performance.now();
             const rows = new CustomKeyMap(rowIDString);
             let total = 0;
             const processBatch = () => startAsyncSpan(tracer, 'processBatch', async () => {
-                const wallElapsed = Date.now() - start;
+                const wallElapsed = performance.now() - start;
                 total += rows.size;
                 lc.debug?.(`processing ${rows.size} (of ${total}) rows (${wallElapsed} ms)`);
                 const patches = await updater.received(lc, rows);
@@ -1027,65 +1037,23 @@ export class ViewSyncerService {
             const finalVersion = this.#cvr.version;
             // Signal clients to commit.
             await pokers.end(finalVersion);
-            await this.#evictInactiveQueries(lc, this.#cvr);
             const elapsed = performance.now() - start;
             lc.info?.(`finished processing advancement of ${numChanges} changes (${elapsed} ms)`);
             histograms.transactionAdvanceTime().record(elapsed);
             return 'success';
         });
     }
-    // This must be called from within the #lock.
-    #evictInactiveQueries(lc, cvr) {
-        lc = lc.withContext('method', '#evictInactiveQueries');
-        return startAsyncSpan(tracer, 'vs.#evictInactiveQueries', async () => {
-            const { rowCount: rowCountBeforeEvictions } = this.#cvrStore;
-            if (rowCountBeforeEvictions <= this.maxRowCount) {
-                lc.debug?.(`rowCount: ${rowCountBeforeEvictions} <= maxRowCount: ${this.maxRowCount}`);
-                return;
-            }
-            lc.info?.(`Trying to evict inactive queries, rowCount: ${rowCountBeforeEvictions} > maxRowCount: ${this.maxRowCount}`);
-            const inactiveQueries = getInactiveQueries(cvr);
-            if (!inactiveQueries.length) {
-                lc.info?.('No inactive queries to evict');
-                return;
-            }
-            const hashToIDs = createHashToIDs(cvr);
-            for (const inactiveQuery of inactiveQueries) {
-                const { hash } = inactiveQuery;
-                const q = cvr.queries[hash];
-                assert(q, 'query not found in CVR');
-                assert(q.type !== 'internal', 'internal queries should not be evicted');
-                const rowCountBeforeCurrentEviction = this.#cvrStore.rowCount;
-                await this.#addAndRemoveQueries(lc, cvr, [], [
-                    {
-                        id: hash,
-                        transformationHash: must(q.transformationHash),
-                    },
-                ], [], hashToIDs);
-                lc.debug?.('Evicted', hash, 'Reduced rowCount from', rowCountBeforeCurrentEviction, 'to', this.#cvrStore.rowCount);
-                if (this.#cvrStore.rowCount <= this.maxRowCount) {
-                    lc.info?.('Evicted', hash, 'Reduced rowCount from', rowCountBeforeEvictions, 'to', this.#cvrStore.rowCount);
-                    break;
-                }
-                // We continue with the updated/current state of the CVR.
-                cvr = must(this.#cvr);
-            }
-            const cvrVersion = must(this.#cvr).version;
-            const dbVersion = this.#pipelines.currentVersion();
-            assert(cvrVersion.stateVersion === dbVersion, `CVR@${versionString(cvrVersion)}" does not match DB@${dbVersion}`);
-        });
-    }
     inspect(context, msg) {
         return this.#runInLockForClient(context, msg, this.#handleInspect);
     }
     // eslint-disable-next-line require-await
-    #handleInspect = async (lc, clientID, body, _cvr) => {
+    #handleInspect = async (lc, clientID, body, cvr) => {
         const client = must(this.#clients.get(clientID));
         body.op;
         client.sendInspectResponse(lc, {
             op: 'queries',
             id: body.id,
-            value: await this.#cvrStore.inspectQueries(lc, body.clientID),
+            value: await this.#cvrStore.inspectQueries(lc, cvr.ttlClock, body.clientID),
         });
     };
     stop() {
@@ -1095,9 +1063,7 @@ export class ViewSyncerService {
     }
     #cleanup(err) {
         this.#stopTTLClockInterval();
-        clearTimeout(this.#expiredQueriesTimer);
-        this.#expiredQueriesTimer = 0;
-        this.#nextExpiredQueryTime = 0;
+        this.#stopExpireTimer();
         this.#pipelines.destroy();
         for (const client of this.#clients.values()) {
             if (err) {
@@ -1204,10 +1170,12 @@ function expired(ttlClock, q) {
     for (const clientID in clientState) {
         if (hasOwn(clientState, clientID)) {
             const { ttl, inactivatedAt } = clientState[clientID];
-            if (ttl < 0 || inactivatedAt === undefined) {
+            if (inactivatedAt === undefined) {
                 return false;
             }
-            if (inactivatedAt + ttl > ttlClock) {
+            const clampedTTL = clampTTL(ttl);
+            if (ttlClockAsNumber(inactivatedAt) + clampedTTL >
+                ttlClockAsNumber(ttlClock)) {
                 return false;
             }
         }