@rocicorp/zero 0.22.2025071101 → 0.22.2025072500

package/out/zero-cache/src/services/view-syncer/view-syncer.js

@@ -10,6 +10,7 @@ import { hasOwn } from "../../../../shared/src/has-own.js";
 import { must } from "../../../../shared/src/must.js";
 import { randInt } from "../../../../shared/src/rand.js";
 import { ErrorKind } from "../../../../zero-protocol/src/error-kind.js";
+import { clampTTL, MAX_TTL_MS } from "../../../../zql/src/query/ttl.js";
 import { transformAndHashQuery, } from "../../auth/read-authorizer.js";
 import {} from "../../config/zero-config.js";
 import { CustomQueryTransformer } from "../../custom-queries/transform-query.js";
@@ -21,15 +22,13 @@ import { Subscription } from "../../types/subscription.js";
 import { ZERO_VERSION_COLUMN_NAME } from "../replicator/schema/replication-state.js";
 import { ClientHandler, startPoke, } from "./client-handler.js";
 import { CVRStore } from "./cvr-store.js";
-import { CVRConfigDrivenUpdater, CVRQueryDrivenUpdater, CVRUpdater, getInactiveQueries, nextEvictionTime, } from "./cvr.js";
+import { CVRConfigDrivenUpdater, CVRQueryDrivenUpdater, CVRUpdater, nextEvictionTime, } from "./cvr.js";
 import { PipelineDriver } from "./pipeline-driver.js";
 import { cmpVersions, EMPTY_CVR_VERSION, versionFromString, versionString, versionToCookie, } from "./schema/types.js";
 import { ResetPipelinesSignal } from "./snapshotter.js";
+import { ttlClockAsNumber, ttlClockFromNumber, } from "./ttl-clock.js";
 const tracer = trace.getTracer('view-syncer', version);
 const DEFAULT_KEEPALIVE_MS = 5_000;
-// We have previously said that the goal is to have 20MB on the client.
-// If we assume each row is ~1KB, then we can have 20,000 rows.
-const DEFAULT_MAX_ROW_COUNT = 20_000;
 function randomID() {
     return randInt(1, Number.MAX_SAFE_INTEGER).toString(36);
 }
@@ -38,7 +37,14 @@ function randomID() {
  * some flushes do not write to the CVR and in those cases we
  * use a timer to update the ttlClock every minute.
  */
-const TTL_CLOCK_INTERVAL = 60_000;
+export const TTL_CLOCK_INTERVAL = 60_000;
+/**
+ * This is some extra time we delay the TTL timer to allow for some
+ * slack in the timing of the timer. This is to allow multiple evictions
+ * to happen in a short period of time without having to wait for the
+ * next tick of the timer.
+ */
+export const TTL_TIMER_HYSTERESIS = 50; // ms
 export class ViewSyncerService {
     id;
     #shard;
@@ -48,7 +54,7 @@ export class ViewSyncerService {
     #drainCoordinator;
     #keepaliveMs;
     #slowHydrateThreshold;
-    #pullConfig;
+    #queryConfig;
     // The ViewSyncerService is only started in response to a connection,
     // so #lastConnectTime is always initialized to now(). This is necessary
     // to handle race conditions in which, e.g. the replica is ready and the
@@ -60,7 +66,7 @@ export class ViewSyncerService {
      * The TTL clock is used to determine the time at which queries are considered
      * expired.
      */
-    #ttlClock = Date.now();
+    #ttlClock;
     /**
      * The base time for the TTL clock. This is used to compute the current TTL
      * clock value. The first time a connection is made, this is set to the
@@ -73,9 +79,6 @@ export class ViewSyncerService {
      * time that has passed since the last time we set it.
      */
     #ttlClockBase = Date.now();
-    get ttlClockBase() {
-        return this.#ttlClockBase;
-    }
     /**
      * We update the ttlClock every minute to ensure that it is not too much
      * out of sync with the current time.
@@ -95,25 +98,13 @@ export class ViewSyncerService {
     // auth and cookie headers directly
     #authData;
     #httpCookie;
-    /**
-     * The {@linkcode maxRowCount} is used for the eviction of inactive queries.
-     * An inactive query is a query that is no longer desired but is kept alive
-     * due to its TTL. When the number of rows in the CVR exceeds
-     * {@linkcode maxRowCount} we keep removing inactive queries (even if they are
-     * not expired yet) until the actual row count is below the max row count.
-     *
-     * There is no guarantee that the number of rows in the CVR will be below this
-     * if there are active queries that have a lot of rows.
-     */
-    maxRowCount;
     #expiredQueriesTimer = 0;
-    #nextExpiredQueryTime = 0;
     #setTimeout;
     #customQueryTransformer;
-    constructor(pullConfig, lc, shard, taskID, clientGroupID, db, pipelineDriver, versionChanges, drainCoordinator, slowHydrateThreshold, keepaliveMs = DEFAULT_KEEPALIVE_MS, maxRowCount = DEFAULT_MAX_ROW_COUNT, setTimeoutFn = setTimeout.bind(globalThis)) {
+    constructor(pullConfig, lc, shard, taskID, clientGroupID, db, pipelineDriver, versionChanges, drainCoordinator, slowHydrateThreshold, keepaliveMs = DEFAULT_KEEPALIVE_MS, setTimeoutFn = setTimeout.bind(globalThis)) {
         this.id = clientGroupID;
         this.#shard = shard;
-        this.#pullConfig = pullConfig;
+        this.#queryConfig = pullConfig;
         this.#lc = lc;
         this.#pipelines = pipelineDriver;
         this.#stateChanges = versionChanges;
@@ -124,7 +115,6 @@ export class ViewSyncerService {
         // On failure, cancel the #stateChanges subscription. The run()
         // loop will then await #cvrStore.flushed() which rejects if necessary.
         () => this.#stateChanges.cancel());
-        this.maxRowCount = maxRowCount;
         this.#setTimeout = setTimeoutFn;
         if (pullConfig.url) {
             this.#customQueryTransformer = new CustomQueryTransformer({
@@ -155,6 +145,16 @@ export class ViewSyncerService {
             if (!this.#cvr) {
                 this.#lc.debug?.('loading CVR');
                 this.#cvr = await this.#cvrStore.load(lc, this.#lastConnectTime);
+                this.#ttlClock = this.#cvr.ttlClock;
+                this.#ttlClockBase = Date.now();
+            }
+            else {
+                // Make sure the CVR ttlClock is up to date.
+                const now = Date.now();
+                this.#cvr = {
+                    ...this.#cvr,
+                    ttlClock: this.#getTTLClock(now),
+                };
             }
             try {
                 await fn(lc, this.#cvr);
@@ -231,16 +231,17 @@ export class ViewSyncerService {
     }
     // must be called from within #lock
     #removeExpiredQueries = async (lc, cvr) => {
-        const now = Date.now();
-        const ttlClock = this.#getTTLClock(now);
-        if (hasExpiredQueries(cvr, ttlClock)) {
+        if (hasExpiredQueries(cvr)) {
             lc = lc.withContext('method', '#removeExpiredQueries');
-            lc.info?.('Queries have expired');
+            lc.debug?.('Queries have expired');
             // #syncQueryPipelineSet() will remove the expired queries.
             await this.#syncQueryPipelineSet(lc, cvr);
             this.#pipelinesSynced = true;
-            this.#scheduleExpireEviction(lc, cvr);
         }
+        // Even if we have expired queries, we still need to schedule next eviction
+        // since there might be inactivated queries that need to be expired queries
+        // in the future.
+        this.#scheduleExpireEviction(lc, cvr);
     };
     #totalHydrationTimeMs() {
         return this.#pipelines.totalHydrationTimeMs();
@@ -290,7 +291,7 @@ export class ViewSyncerService {
         // If no clients have connected while waiting for the row flush, shutdown.
         return this.#clients.size === 0;
     }
-    #deleteClient(clientID, client) {
+    #deleteClientDueToDisconnect(clientID, client) {
         // Note: It is okay to delete / cleanup clients without acquiring the lock.
         // In fact, it is important to do so in order to guarantee that idle cleanup
         // is performed in a timely manner, regardless of the amount of work
@@ -299,7 +300,11 @@ export class ViewSyncerService {
         if (c === client) {
             this.#clients.delete(clientID);
             if (this.#clients.size === 0) {
-                this.#updateTTLClockInCVRWithoutLock(this.#lc);
+                // It is possible to delete a client before we read the ttl clock from
+                // the CVR.
+                if (this.#ttlClock !== undefined) {
+                    this.#updateTTLClockInCVRWithoutLock(this.#lc);
+                }
                 this.#stopExpireTimer();
                 this.#scheduleShutdown();
             }
@@ -309,7 +314,6 @@ export class ViewSyncerService {
         this.#lc.debug?.('Stopping expired queries timer');
         clearTimeout(this.#expiredQueriesTimer);
         this.#expiredQueriesTimer = 0;
-        this.#nextExpiredQueryTime = 0;
     }
     initConnection(ctx, initConnectionMessage) {
         this.#lc.debug?.('viewSyncer.initConnection');
@@ -327,7 +331,7 @@ export class ViewSyncerService {
                     err
                         ? lc[getLogLevel(err)]?.(`client closed with error`, err)
                         : lc.info?.('client closed');
-                    this.#deleteClient(clientID, newClient);
+                    this.#deleteClientDueToDisconnect(clientID, newClient);
                 },
             });
             if (this.#clients.size === 0) {
@@ -336,15 +340,6 @@ export class ViewSyncerService {
                 // subscription is returned immediately.
                 const now = Date.now();
                 this.#ttlClockBase = now;
-                // Get the TTL clock from the CVR store, or initialize it to now.
-                this.#cvrStore
-                    .getTTLClock()
-                    .then(ttlClock => {
-                    this.#ttlClock = ttlClock ?? now;
-                })
-                    .catch(e => {
-                    this.#lc.error?.('failed to get TTL clock', e);
-                });
             }
             const newClient = new ClientHandler(lc, this.id, clientID, wsID, this.#shard, baseCookie, schemaVersion, downstream);
             this.#clients.get(clientID)?.close(`replaced by wsID: ${wsID}`);
@@ -372,7 +367,9 @@ export class ViewSyncerService {
     #getTTLClock(now) {
         // We will update ttlClock with delta from the ttlClockBase to the current time.
         const delta = now - this.#ttlClockBase;
-        const ttlClock = this.#ttlClock + delta;
+        assert(this.#ttlClock !== undefined);
+        const ttlClock = ttlClockFromNumber(ttlClockAsNumber(this.#ttlClock) + delta);
+        assert(ttlClockAsNumber(ttlClock) <= now);
         this.#ttlClock = ttlClock;
         this.#ttlClockBase = now;
         return ttlClock;
@@ -493,6 +490,7 @@ export class ViewSyncerService {
         const deletedClientIDs = [];
         const deletedClientGroupIDs = [];
         cvr = await this.#updateCVRConfig(lc, cvr, clientID, updater => {
+            const { ttlClock } = cvr;
             const patches = [];
             if (clientSchema) {
                 updater.setClientSchema(lc, clientSchema);
@@ -500,14 +498,13 @@ export class ViewSyncerService {
             // Apply requested patches.
             lc.debug?.(`applying ${desiredQueriesPatch?.length} query patches`);
             if (desiredQueriesPatch?.length) {
-                const now = Date.now();
                 for (const patch of desiredQueriesPatch) {
                     switch (patch.op) {
                         case 'put':
                             patches.push(...updater.putDesiredQueries(clientID, [patch]));
                             break;
                         case 'del':
-                            patches.push(...updater.markDesiredQueriesAsInactive(clientID, [patch.hash], now));
+                            patches.push(...updater.markDesiredQueriesAsInactive(clientID, [patch.hash], ttlClock));
                             break;
                         case 'clear':
                             patches.push(...updater.clearDesiredQueries(clientID));
@@ -533,7 +530,7 @@ export class ViewSyncerService {
                 }
             }
             for (const cid of clientIDsToDelete) {
-                const patchesDueToClient = updater.deleteClient(cid);
+                const patchesDueToClient = updater.deleteClient(cid, ttlClock);
                 patches.push(...patchesDueToClient);
                 deletedClientIDs.push(cid);
             }
@@ -554,35 +551,32 @@ export class ViewSyncerService {
             await Promise.allSettled(clients.map(client => client.sendDeleteClients(lc, deletedClientIDs, deletedClientGroupIDs)));
         }
         this.#scheduleExpireEviction(lc, cvr);
-        await this.#evictInactiveQueries(lc, cvr);
     });
     #scheduleExpireEviction(lc, cvr) {
+        const { ttlClock } = cvr;
+        this.#stopExpireTimer();
         // first see if there is any inactive query with a ttl.
         const next = nextEvictionTime(cvr);
         if (next === undefined) {
             lc.debug?.('no inactive queries with ttl');
             // no inactive queries with a ttl. Cancel existing timeout if any.
-            this.#stopExpireTimer();
-            return;
-        }
-        if (this.#nextExpiredQueryTime === next) {
-            lc.debug?.('eviction timer already scheduled');
             return;
         }
-        if (this.#expiredQueriesTimer) {
-            clearTimeout(this.#expiredQueriesTimer);
-        }
-        const now = Date.now();
-        this.#nextExpiredQueryTime = next;
-        const ttlClock = this.#getTTLClock(now);
-        lc.debug?.('Scheduling eviction timer to run in ', next - ttlClock, 'ms');
-        this.#expiredQueriesTimer = this.#setTimeout(() => this.#runInLockWithCVR(this.#removeExpiredQueries).catch(e => 
-        // If an error occurs (e.g. ownership change), propagate the error
-        // to the main run() loop via the #stateChanges Subscription.
-        this.#stateChanges.fail(e)), 
-        // If the expire time is too far in the future we will run it in an hour.
-        // At that point in time it will be rescheduled as needed again.
-        Math.min(next - ttlClock, 60 * 60 * 1000));
+        // It is common for many queries to be evicted close to the same time, so
+        // we add a small delay so we can collapse multiple evictions into a
+        // single timer. However, don't add the delay if we're already at the
+        // maximum timer limit, as that's not about collapsing.
+        const delay = Math.max(TTL_TIMER_HYSTERESIS, Math.min(ttlClockAsNumber(next) -
+            ttlClockAsNumber(ttlClock) +
+            TTL_TIMER_HYSTERESIS, MAX_TTL_MS));
+        lc.debug?.('Scheduling eviction timer to run in ', delay, 'ms');
+        this.#expiredQueriesTimer = this.#setTimeout(() => {
+            this.#expiredQueriesTimer = 0;
+            this.#runInLockWithCVR((lc, cvr) => this.#removeExpiredQueries(lc, cvr)).catch(e => 
+            // If an error occurs (e.g. ownership change), propagate the error
+            // to the main run() loop via the #stateChanges Subscription.
+            this.#stateChanges.fail(e));
+        }, delay);
     }
     /**
      * Adds and hydrates pipelines for queries whose results are already
@@ -628,9 +622,9 @@ export class ViewSyncerService {
         }
         if (this.#customQueryTransformer && customQueries.length > 0) {
             const transformedCustomQueries = await this.#customQueryTransformer.transform({
-                apiKey: this.#pullConfig.apiKey,
+                apiKey: this.#queryConfig.apiKey,
                 token: this.#authData?.raw,
-                cookie: this.#pullConfig.forwardCookies
+                cookie: this.#queryConfig.forwardCookies
                     ? this.#httpCookie
                     : undefined,
             }, customQueries);
@@ -695,6 +689,10 @@ export class ViewSyncerService {
             const hydratedQueries = this.#pipelines.addedQueries();
             // Convert queries to their transformed ast's and hashes
             const hashToIDs = new Map();
+            if (this.#ttlClock === undefined) {
+                // Get it from the CVR or initialize it to now.
+                this.#ttlClock = cvr.ttlClock;
+            }
             const now = Date.now();
             const ttlClock = this.#getTTLClock(now);
             // group cvr queries into:
@@ -733,7 +731,7 @@ export class ViewSyncerService {
             }
             if (this.#customQueryTransformer && customQueries.size > 0) {
                 const transformedCustomQueries = await this.#customQueryTransformer.transform({
-                    apiKey: this.#pullConfig.apiKey,
+                    apiKey: this.#queryConfig.apiKey,
                     token: this.#authData?.raw,
                     cookie: this.#httpCookie,
                 }, customQueries.values());
@@ -795,7 +793,7 @@ export class ViewSyncerService {
             assert(addQueries.length > 0 ||
                 removeQueries.length > 0 ||
                 unhydrateQueries.length > 0);
-            const start = Date.now();
+            const start = performance.now();
             const stateVersion = this.#pipelines.currentVersion();
             lc = lc.withContext('stateVersion', stateVersion);
             lc.info?.(`hydrating ${addQueries.length} queries`);
@@ -850,7 +848,7 @@ export class ViewSyncerService {
             await this.#catchupClients(lc, cvr, finalVersion, addQueries.map(q => q.id), pokers);
             // Signal clients to commit.
             await pokers.end(finalVersion);
-            const wallTime = Date.now() - start;
+            const wallTime = performance.now() - start;
             lc.info?.(`finished processing queries (process: ${totalProcessTime} ms, wall: ${wallTime} ms)`);
         });
     }
@@ -924,11 +922,11 @@ export class ViewSyncerService {
     }
     #processChanges(lc, timer, changes, updater, pokers, hashToIDs) {
         return startAsyncSpan(tracer, 'vs.#processChanges', async () => {
-            const start = Date.now();
+            const start = performance.now();
             const rows = new CustomKeyMap(rowIDString);
             let total = 0;
             const processBatch = () => startAsyncSpan(tracer, 'processBatch', async () => {
-                const wallElapsed = Date.now() - start;
+                const wallElapsed = performance.now() - start;
                 total += rows.size;
                 lc.debug?.(`processing ${rows.size} (of ${total}) rows (${wallElapsed} ms)`);
                 const patches = await updater.received(lc, rows);
@@ -1027,65 +1025,23 @@ export class ViewSyncerService {
             const finalVersion = this.#cvr.version;
             // Signal clients to commit.
             await pokers.end(finalVersion);
-            await this.#evictInactiveQueries(lc, this.#cvr);
             const elapsed = performance.now() - start;
             lc.info?.(`finished processing advancement of ${numChanges} changes (${elapsed} ms)`);
             histograms.transactionAdvanceTime().record(elapsed);
             return 'success';
         });
     }
-    // This must be called from within the #lock.
-    #evictInactiveQueries(lc, cvr) {
-        lc = lc.withContext('method', '#evictInactiveQueries');
-        return startAsyncSpan(tracer, 'vs.#evictInactiveQueries', async () => {
-            const { rowCount: rowCountBeforeEvictions } = this.#cvrStore;
-            if (rowCountBeforeEvictions <= this.maxRowCount) {
-                lc.debug?.(`rowCount: ${rowCountBeforeEvictions} <= maxRowCount: ${this.maxRowCount}`);
-                return;
-            }
-            lc.info?.(`Trying to evict inactive queries, rowCount: ${rowCountBeforeEvictions} > maxRowCount: ${this.maxRowCount}`);
-            const inactiveQueries = getInactiveQueries(cvr);
-            if (!inactiveQueries.length) {
-                lc.info?.('No inactive queries to evict');
-                return;
-            }
-            const hashToIDs = createHashToIDs(cvr);
-            for (const inactiveQuery of inactiveQueries) {
-                const { hash } = inactiveQuery;
-                const q = cvr.queries[hash];
-                assert(q, 'query not found in CVR');
-                assert(q.type !== 'internal', 'internal queries should not be evicted');
-                const rowCountBeforeCurrentEviction = this.#cvrStore.rowCount;
-                await this.#addAndRemoveQueries(lc, cvr, [], [
-                    {
-                        id: hash,
-                        transformationHash: must(q.transformationHash),
-                    },
-                ], [], hashToIDs);
-                lc.debug?.('Evicted', hash, 'Reduced rowCount from', rowCountBeforeCurrentEviction, 'to', this.#cvrStore.rowCount);
-                if (this.#cvrStore.rowCount <= this.maxRowCount) {
-                    lc.info?.('Evicted', hash, 'Reduced rowCount from', rowCountBeforeEvictions, 'to', this.#cvrStore.rowCount);
-                    break;
-                }
-                // We continue with the updated/current state of the CVR.
-                cvr = must(this.#cvr);
-            }
-            const cvrVersion = must(this.#cvr).version;
-            const dbVersion = this.#pipelines.currentVersion();
-            assert(cvrVersion.stateVersion === dbVersion, `CVR@${versionString(cvrVersion)}" does not match DB@${dbVersion}`);
-        });
-    }
     inspect(context, msg) {
         return this.#runInLockForClient(context, msg, this.#handleInspect);
     }
     // eslint-disable-next-line require-await
-    #handleInspect = async (lc, clientID, body, _cvr) => {
+    #handleInspect = async (lc, clientID, body, cvr) => {
         const client = must(this.#clients.get(clientID));
         body.op;
         client.sendInspectResponse(lc, {
             op: 'queries',
             id: body.id,
-            value: await this.#cvrStore.inspectQueries(lc, body.clientID),
+            value: await this.#cvrStore.inspectQueries(lc, cvr.ttlClock, body.clientID),
         });
     };
     stop() {
@@ -1095,9 +1051,7 @@ export class ViewSyncerService {
     }
     #cleanup(err) {
         this.#stopTTLClockInterval();
-        clearTimeout(this.#expiredQueriesTimer);
-        this.#expiredQueriesTimer = 0;
-        this.#nextExpiredQueryTime = 0;
+        this.#stopExpireTimer();
         this.#pipelines.destroy();
         for (const client of this.#clients.values()) {
             if (err) {
@@ -1204,17 +1158,20 @@ function expired(ttlClock, q) {
     for (const clientID in clientState) {
         if (hasOwn(clientState, clientID)) {
             const { ttl, inactivatedAt } = clientState[clientID];
-            if (ttl < 0 || inactivatedAt === undefined) {
+            if (inactivatedAt === undefined) {
                 return false;
             }
-            if (inactivatedAt + ttl > ttlClock) {
+            const clampedTTL = clampTTL(ttl);
+            if (ttlClockAsNumber(inactivatedAt) + clampedTTL >
+                ttlClockAsNumber(ttlClock)) {
                 return false;
             }
         }
     }
     return true;
 }
-function hasExpiredQueries(cvr, ttlClock) {
+function hasExpiredQueries(cvr) {
+    const { ttlClock } = cvr;
     for (const q of Object.values(cvr.queries)) {
         if (expired(ttlClock, q)) {
             return true;