@rocicorp/zero 0.25.0-canary.9 → 0.25.1

package/out/zero-cache/src/services/view-syncer/schema/init.js

@@ -147,6 +147,33 @@ async function initViewSyncerSchema(log, db, shard) {
       `;
     }
   };
+  const migratedV15ToV16 = {
+    migrateSchema: async (_, sql) => {
+      await sql`ALTER TABLE ${sql(schema)}.instances ADD COLUMN "profileID" TEXT`;
+      await sql`ALTER TABLE ${sql(schema)}.instances ADD COLUMN "deleted" BOOL DEFAULT FALSE`;
+      await sql`
+        DROP INDEX IF EXISTS ${sql(schema)}.instances_last_active`;
+      await sql`
+        CREATE INDEX instances_last_active ON ${sql(schema)}.instances ("lastActive")
+          WHERE NOT "deleted"`;
+      await sql`
+        CREATE INDEX tombstones_last_active ON ${sql(schema)}.instances ("lastActive")
+          WHERE "deleted"`;
+      await sql`
+        CREATE INDEX profile_ids_last_active ON ${sql(schema)}.instances ("lastActive", "profileID")
+          WHERE "profileID" IS NOT NULL`;
+    },
+    // Backfill profileIDs to the `cg${clientGroupID}`, as is done for
+    // client groups from old zero-clients that don't send a profileID.
+    migrateData: async (lc, sql) => {
+      lc.info?.("Backfilling instance.profileIDs");
+      await sql`
+        UPDATE ${sql(schema)}.instances
+        SET "profileID" = 'cg' || "clientGroupID"
+        WHERE "profileID" IS NULL
+      `;
+    }
+  };
   const schemaVersionMigrationMap = {
     2: migrateV1toV2,
     3: migrateV2ToV3,
@@ -175,7 +202,10 @@ async function initViewSyncerSchema(log, db, shard) {
     // V15 adds desires."inactivatedAtTTLClock" to store TTLClock values
     // directly as DOUBLE PRECISION, avoiding postgres.js TIMESTAMPTZ
     // type conversion issues
-    15: migratedV14ToV15
+    15: migratedV14ToV15,
+    // V16 adds instances."profileID" and a corresponding index for estimating
+    // active user counts more accurately for apps that use memstore.
+    16: migratedV15ToV16
   };
   await runSchemaMigrations(
     log,

package/out/zero-cache/src/services/view-syncer/schema/init.js.map

@@ -1 +1 @@
-{"version":3,"file":"init.js","sources":["../../../../../../../zero-cache/src/services/view-syncer/schema/init.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport type {PendingQuery, Row} from 'postgres';\nimport {\n  runSchemaMigrations,\n  type IncrementalMigrationMap,\n  type Migration,\n} from '../../../db/migration.ts';\nimport type {PostgresDB} from '../../../types/pg.ts';\nimport {cvrSchema, type ShardID} from '../../../types/shards.ts';\nimport {createRowsVersionTable, setupCVRTables} from './cvr.ts';\n\nexport async function initViewSyncerSchema(\n  log: LogContext,\n  db: PostgresDB,\n  shard: ShardID,\n): Promise<void> {\n  const schema = cvrSchema(shard);\n\n  const setupMigration: Migration = {\n    migrateSchema: (lc, tx) => setupCVRTables(lc, tx, shard),\n    minSafeVersion: 1,\n  };\n\n  const migrateV1toV2: Migration = {\n    migrateSchema: async (_, tx) => {\n      await tx`ALTER TABLE ${tx(schema)}.instances ADD \"replicaVersion\" TEXT`;\n    },\n  };\n\n  const migrateV2ToV3: Migration = {\n    migrateSchema: async (_, tx) => {\n      await tx.unsafe(createRowsVersionTable(shard));\n    },\n\n    /** Populates the cvr.rowsVersion table with versions from cvr.instances. */\n    migrateData: async (lc, tx) => {\n      const pending: PendingQuery<Row[]>[] = [];\n      for await (const versions of tx<\n        {clientGroupID: string; version: string}[]\n      >`\n      SELECT \"clientGroupID\", \"version\" FROM ${tx(schema)}.instances`.cursor(\n        5000,\n      )) {\n        for (const version of versions) {\n          pending.push(\n            tx`INSERT INTO ${tx(schema)}.\"rowsVersion\" ${tx(version)} \n               ON CONFLICT (\"clientGroupID\")\n               DO UPDATE SET ${tx(version)}`.execute(),\n          );\n        }\n      }\n      lc.info?.(`initializing rowsVersion for ${pending.length} cvrs`);\n      await Promise.all(pending);\n    },\n  };\n\n  const migrateV3ToV4: Migration = {\n    migrateSchema: async (_, tx) => {\n      await tx`ALTER TABLE ${tx(schema)}.instances ADD \"owner\" TEXT`;\n      await tx`ALTER TABLE ${tx(schema)}.instances ADD \"grantedAt\" TIMESTAMPTZ`;\n    },\n  };\n\n  const migrateV5ToV6: Migration = {\n    migrateSchema: async (_, tx) => {\n      await tx`\n      ALTER TABLE ${tx(schema)}.\"rows\"\n        DROP CONSTRAINT fk_rows_client_group`;\n      await tx`\n      ALTER TABLE ${tx(schema)}.\"rowsVersion\"\n        DROP CONSTRAINT fk_rows_version_client_group`;\n    },\n  };\n\n  const migrateV6ToV7: Migration = {\n    migrateSchema: async (_, tx) => {\n      await tx`ALTER TABLE ${tx(schema)}.desires ADD \"expiresAt\" TIMESTAMPTZ`;\n      await tx`ALTER TABLE ${tx(\n        schema,\n      )}.desires ADD \"inactivatedAt\" TIMESTAMPTZ`;\n      await tx`ALTER TABLE ${tx(schema)}.desires ADD \"ttl\" INTERVAL`;\n\n      await tx`CREATE INDEX desires_expires_at ON ${tx(\n        schema,\n      )}.desires (\"expiresAt\")`;\n      await tx`CREATE INDEX desires_inactivated_at ON ${tx(\n        schema,\n      )}.desires (\"inactivatedAt\")`;\n    },\n  };\n\n  const migrateV7ToV8: Migration = {\n    migrateSchema: async (_, tx) => {\n      await tx`ALTER TABLE ${tx(\n        schema,\n      )}.\"desires\" DROP CONSTRAINT fk_desires_client`;\n    },\n  };\n\n  const migrateV8ToV9: Migration = {\n    migrateSchema: async (_, tx) => {\n      await tx`ALTER TABLE ${tx(schema)}.instances ADD \"clientSchema\" JSONB`;\n    },\n  };\n\n  const migrateV9ToV10: Migration = {\n    migrateSchema: async (_, tx) => {\n      await tx`ALTER TABLE ${tx(schema)}.queries ADD \"queryName\" TEXT`;\n      await tx`ALTER TABLE ${tx(schema)}.queries ADD \"queryArgs\" JSONB`;\n      await tx`ALTER TABLE ${tx(schema)}.queries ALTER COLUMN \"clientAST\" DROP NOT NULL`;\n    },\n  };\n\n  const migrateV10ToV11: Migration = {\n    migrateSchema: async (_, tx) => {\n      await tx`DROP INDEX IF EXISTS ${tx(schema)}.desires_expires_at`;\n      await tx`ALTER TABLE ${tx(schema)}.desires DROP COLUMN \"expiresAt\"`;\n      await tx`DROP INDEX IF EXISTS ${tx(schema)}.client_patch_version`;\n      await tx`ALTER TABLE ${tx(schema)}.clients DROP COLUMN \"patchVersion\"`;\n      await tx`ALTER TABLE ${tx(schema)}.clients DROP COLUMN \"deleted\"`;\n    },\n  };\n\n  const migratedV11ToV12: Migration = {\n    migrateSchema: async (_, tx) => {\n      await tx`ALTER TABLE ${tx(schema)}.queries ALTER COLUMN \"queryArgs\" TYPE JSON USING \"queryArgs\"::JSON`;\n    },\n  };\n\n  const migratedV12ToV13: Migration = {\n    migrateSchema: async (_, tx) => {\n      await tx`ALTER TABLE ${tx(schema)}.instances ADD COLUMN \"ttlClock\" DOUBLE PRECISION NOT NULL DEFAULT 0`;\n    },\n  };\n\n  const migratedV13ToV14: Migration = {\n    migrateSchema: async (_, sql) => {\n      await sql`\n        CREATE INDEX instances_last_active ON ${sql(schema)}.instances (\"lastActive\");\n      `;\n\n      // Update / add foreign key constraints to cascade deletes.\n      for (const [table, reference] of [\n        ['clients', 'instances'],\n        ['queries', 'instances'],\n        ['rows', 'rowsVersion'],\n      ] as [string, string][]) {\n        const constraint = sql(`fk_${table}_client_group`);\n        await sql`\n          ALTER TABLE ${sql(schema)}.${sql(table)} DROP CONSTRAINT IF EXISTS ${constraint}`;\n        await sql`\n          ALTER TABLE ${sql(schema)}.${sql(table)} ADD CONSTRAINT ${constraint}\n            FOREIGN KEY(\"clientGroupID\")\n            REFERENCES ${sql(schema)}.${sql(reference)} (\"clientGroupID\")\n            ON DELETE CASCADE;\n        `;\n      }\n    },\n  };\n\n  const migratedV14ToV15: Migration = {\n    migrateSchema: async (_, sql) => {\n      // Add new columns for storing inactivatedAt and ttl in milliseconds.\n      // This avoids postgres.js type conversion issues with TIMESTAMPTZ and INTERVAL.\n      await sql`ALTER TABLE ${sql(schema)}.desires \n        ADD COLUMN \"inactivatedAtMs\" DOUBLE PRECISION`;\n      await sql`ALTER TABLE ${sql(schema)}.desires \n        ADD COLUMN \"ttlMs\" DOUBLE PRECISION`;\n    },\n    // Migrate existing data: convert TIMESTAMPTZ to milliseconds for inactivatedAt\n    // and INTERVAL to milliseconds for ttl\n    // Note: EXTRACT(EPOCH FROM NULL) returns NULL, so NULL values are preserved\n    migrateData: async (lc, sql) => {\n      lc.info?.(\n        'Migrating desires.inactivatedAt to inactivatedAtMs and ttl to ttlMs',\n      );\n      await sql`\n        UPDATE ${sql(schema)}.desires\n        SET \"inactivatedAtMs\" = EXTRACT(EPOCH FROM \"inactivatedAt\") * 1000,\n            \"ttlMs\" = EXTRACT(EPOCH FROM \"ttl\") * 1000\n      `;\n    },\n  };\n\n  const schemaVersionMigrationMap: IncrementalMigrationMap = {\n    2: migrateV1toV2,\n    3: migrateV2ToV3,\n    4: migrateV3ToV4,\n    // v5 enables asynchronous row-record flushing, and thus relies on\n    // the logic that updates and checks the rowsVersion table in v3.\n    5: {minSafeVersion: 3},\n    6: migrateV5ToV6,\n    7: migrateV6ToV7,\n    8: migrateV7ToV8,\n    9: migrateV8ToV9,\n    // v10 adds queryName and queryArgs to the queries table to support\n    // custom queries. clientAST is now optional to support migrating\n    // off client queries.\n    10: migrateV9ToV10,\n    // V11 removes the deprecated queries.\"expiresAt\", clients.\"patchVersion\",\n    // clients.\"deleted\" columns.\n    11: migrateV10ToV11,\n    12: migratedV11ToV12,\n    // V13 adds instances.\"ttlClock\"\n    13: migratedV12ToV13,\n    // V14 adds an index on instances.\"lastActive\" and a FK constraint\n    // from rows.\"clientGroupID\" to rowsVersion.\"clientGroupID\" for\n    // garbage collection\n    14: migratedV13ToV14,\n    // V15 adds desires.\"inactivatedAtTTLClock\" to store TTLClock values\n    // directly as DOUBLE PRECISION, avoiding postgres.js TIMESTAMPTZ\n    // type conversion issues\n    15: migratedV14ToV15,\n  };\n\n  await runSchemaMigrations(\n    log,\n    'view-syncer',\n    cvrSchema(shard),\n    db,\n    setupMigration,\n    schemaVersionMigrationMap,\n  );\n}\n"],"names":[],"mappings":";;;AAWA,eAAsB,qBACpB,KACA,IACA,OACe;AACf,QAAM,SAAS,UAAU,KAAK;AAE9B,QAAM,iBAA4B;AAAA,IAChC,eAAe,CAAC,IAAI,OAAO,eAAe,IAAI,IAAI,KAAK;AAAA,IACvD,gBAAgB;AAAA,EAAA;AAGlB,QAAM,gBAA2B;AAAA,IAC/B,eAAe,OAAO,GAAG,OAAO;AAC9B,YAAM,iBAAiB,GAAG,MAAM,CAAC;AAAA,IACnC;AAAA,EAAA;AAGF,QAAM,gBAA2B;AAAA,IAC/B,eAAe,OAAO,GAAG,OAAO;AAC9B,YAAM,GAAG,OAAO,uBAAuB,KAAK,CAAC;AAAA,IAC/C;AAAA;AAAA,IAGA,aAAa,OAAO,IAAI,OAAO;AAC7B,YAAM,UAAiC,CAAA;AACvC,uBAAiB,YAAY;AAAA,+CAGY,GAAG,MAAM,CAAC,aAAa;AAAA,QAC9D;AAAA,MAAA,GACC;AACD,mBAAW,WAAW,UAAU;AAC9B,kBAAQ;AAAA,YACN,iBAAiB,GAAG,MAAM,CAAC,kBAAkB,GAAG,OAAO,CAAC;AAAA;AAAA,+BAErC,GAAG,OAAO,CAAC,GAAG,QAAA;AAAA,UAAQ;AAAA,QAE7C;AAAA,MACF;AACA,SAAG,OAAO,gCAAgC,QAAQ,MAAM,OAAO;AAC/D,YAAM,QAAQ,IAAI,OAAO;AAAA,IAC3B;AAAA,EAAA;AAGF,QAAM,gBAA2B;AAAA,IAC/B,eAAe,OAAO,GAAG,OAAO;AAC9B,YAAM,iBAAiB,GAAG,MAAM,CAAC;AACjC,YAAM,iBAAiB,GAAG,MAAM,CAAC;AAAA,IACnC;AAAA,EAAA;AAGF,QAAM,gBAA2B;AAAA,IAC/B,eAAe,OAAO,GAAG,OAAO;AAC9B,YAAM;AAAA,oBACQ,GAAG,MAAM,CAAC;AAAA;AAExB,YAAM;AAAA,oBACQ,GAAG,MAAM,CAAC;AAAA;AAAA,IAE1B;AAAA,EAAA;AAGF,QAAM,gBAA2B;AAAA,IAC/B,eAAe,OAAO,GAAG,OAAO;AAC9B,YAAM,iBAAiB,GAAG,MAAM,CAAC;AACjC,YAAM,iBAAiB;AAAA,QACrB;AAAA,MAAA,CACD;AACD,YAAM,iBAAiB,GAAG,MAAM,CAAC;AAEjC,YAAM,wCAAwC;AAAA,QAC5C;AAAA,MAAA,CACD;AACD,YAAM,4CAA4C;AAAA,QAChD;AAAA,MAAA,CACD;AAAA,IACH;AAAA,EAAA;AAGF,QAAM,gBAA2B;AAAA,IAC/B,eAAe,OAAO,GAAG,OAAO;AAC9B,YAAM,iBAAiB;AAAA,QACrB;AAAA,MAAA,CACD;AAAA,IACH;AAAA,EAAA;AAGF,QAAM,gBAA2B;AAAA,IAC/B,eAAe,OAAO,GAAG,OAAO;AAC9B,YAAM,iBAAiB,GAAG,MAAM,CAAC;AAAA,IACnC;AAAA,EAAA;AAGF,QAAM,iBAA4B;AAAA,IAChC,eAAe,OAAO,GAAG,OAAO;AAC9B,YAAM,iBAAiB,GAAG,MAAM,CAAC;AACjC,YAAM,iBAAiB,GAAG,MAAM,CAAC;AACjC,YAAM,iBAAiB,GAAG,MAAM,CAAC;AAAA,IACnC;AAAA,EAAA;AAGF,QAAM,kBAA6B;AAAA,IACjC,eAAe,OAAO,GAAG,OAAO;AAC9B,YAAM,0BAA0B,GAAG,MAAM,CAAC;AAC1C,YAAM,iBAAiB,GAAG,MAAM,CAAC;AACjC,YAAM,0BAA0B,GAAG,MAAM,CAAC;AAC1C,YAAM,iBAAiB,GAAG,MAAM,CAAC;AACjC,YAAM,iBAAiB,GAAG,MAAM,CAAC;AAAA,IACnC;AAAA,EAAA;AAGF,QAAM,mBAA8B;AAAA,IAClC,eAAe,OAAO,GAAG,OAAO;AAC9B,YAAM,iBAAiB,GAAG,MAAM,CAAC;AAAA,IACnC;AAAA,EAAA;AAGF,QAAM,mBAA8B;AAAA,IAClC,eAAe,OAAO,GAAG,OAAO;AAC9B,YAAM,iBAAiB,GAAG,MAAM,CAAC;AAAA,IACnC;AAAA,EAAA;AAGF,QAAM,mBAA8B;AAAA,IAClC,eAAe,OAAO,GAAG,QAAQ;AAC/B,YAAM;AAAA,gDACoC,IAAI,MAAM,CAAC;AAAA;AAIrD,iBAAW,CAAC,OAAO,SAAS,KAAK;AAAA,QAC/B,CAAC,WAAW,WAAW;AAAA,QACvB,CAAC,WAAW,WAAW;AAAA,QACvB,CAAC,QAAQ,aAAa;AAAA,MAAA,GACC;AACvB,cAAM,aAAa,IAAI,MAAM,KAAK,eAAe;AACjD,cAAM;AAAA,wBACU,IAAI,MAAM,CAAC,IAAI,IAAI,KAAK,CAAC,8BAA8B,UAAU;AACjF,cAAM;AAAA,wBACU,IAAI,MAAM,CAAC,IAAI,IAAI,KAAK,CAAC,mBAAmB,UAAU;AAAA;AAAA,yBAErD,IAAI,MAAM,CAAC,IAAI,IAAI,SAAS,CAAC;AAAA;AAAA;AAAA,MAGhD;AAAA,IACF;AAAA,EAAA;AAGF,QAAM,mBAA8B;AAAA,IAClC,eAAe,OAAO,GAAG,QAAQ;AAG/B,YAAM,kBAAkB,IAAI,MAAM,CAAC;AAAA;AAEnC,YAAM,kBAAkB,IAAI,MAAM,CAAC;AAAA;AAAA,IAErC;AAAA;AAAA;AAAA;AAAA,IAIA,aAAa,OAAO,IAAI,QAAQ;AAC9B,SAAG;AAAA,QACD;AAAA,MAAA;AAEF,YAAM;AAAA,iBACK,IAAI,MAAM,CAAC;AAAA;AAAA;AAAA;AAAA,IAIxB;AAAA,EAAA;AAGF,QAAM,4BAAqD;AAAA,IACzD,GAAG;AAAA,IACH,GAAG;AAAA,IACH,GAAG;AAAA;AAAA;AAAA,IAGH,GAAG,EAAC,gBAAgB,EAAA;AAAA,IACpB,GAAG;AAAA,IACH,GAAG;AAAA,IACH,GAAG;AAAA,IACH,GAAG;AAAA;AAAA;AAAA;AAAA,IAIH,IAAI;AAAA;AAAA;AAAA,IAGJ,IAAI;AAAA,IACJ,IAAI;AAAA;AAAA,IAEJ,IAAI;AAAA;AAAA;AAAA;AAAA,IAIJ,IAAI;AAAA;AAAA;AAAA;AAAA,IAIJ,IAAI;AAAA,EAAA;AAGN,QAAM;AAAA,IACJ;AAAA,IACA;AAAA,IACA,UAAU,KAAK;AAAA,IACf;AAAA,IACA;AAAA,IACA;AAAA,EAAA;AAEJ;"}
+{"version":3,"file":"init.js","sources":["../../../../../../../zero-cache/src/services/view-syncer/schema/init.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport type {PendingQuery, Row} from 'postgres';\nimport {\n  runSchemaMigrations,\n  type IncrementalMigrationMap,\n  type Migration,\n} from '../../../db/migration.ts';\nimport type {PostgresDB} from '../../../types/pg.ts';\nimport {cvrSchema, type ShardID} from '../../../types/shards.ts';\nimport {createRowsVersionTable, setupCVRTables} from './cvr.ts';\n\nexport async function initViewSyncerSchema(\n  log: LogContext,\n  db: PostgresDB,\n  shard: ShardID,\n): Promise<void> {\n  const schema = cvrSchema(shard);\n\n  const setupMigration: Migration = {\n    migrateSchema: (lc, tx) => setupCVRTables(lc, tx, shard),\n    minSafeVersion: 1,\n  };\n\n  const migrateV1toV2: Migration = {\n    migrateSchema: async (_, tx) => {\n      await tx`ALTER TABLE ${tx(schema)}.instances ADD \"replicaVersion\" TEXT`;\n    },\n  };\n\n  const migrateV2ToV3: Migration = {\n    migrateSchema: async (_, tx) => {\n      await tx.unsafe(createRowsVersionTable(shard));\n    },\n\n    /** Populates the cvr.rowsVersion table with versions from cvr.instances. */\n    migrateData: async (lc, tx) => {\n      const pending: PendingQuery<Row[]>[] = [];\n      for await (const versions of tx<\n        {clientGroupID: string; version: string}[]\n      >`\n      SELECT \"clientGroupID\", \"version\" FROM ${tx(schema)}.instances`.cursor(\n        5000,\n      )) {\n        for (const version of versions) {\n          pending.push(\n            tx`INSERT INTO ${tx(schema)}.\"rowsVersion\" ${tx(version)} \n               ON CONFLICT (\"clientGroupID\")\n               DO UPDATE SET ${tx(version)}`.execute(),\n          );\n        }\n      }\n      lc.info?.(`initializing rowsVersion for ${pending.length} cvrs`);\n      await Promise.all(pending);\n    },\n  };\n\n  const migrateV3ToV4: Migration = {\n    migrateSchema: async (_, tx) => {\n      await tx`ALTER TABLE ${tx(schema)}.instances ADD \"owner\" TEXT`;\n      await tx`ALTER TABLE ${tx(schema)}.instances ADD \"grantedAt\" TIMESTAMPTZ`;\n    },\n  };\n\n  const migrateV5ToV6: Migration = {\n    migrateSchema: async (_, tx) => {\n      await tx`\n      ALTER TABLE ${tx(schema)}.\"rows\"\n        DROP CONSTRAINT fk_rows_client_group`;\n      await tx`\n      ALTER TABLE ${tx(schema)}.\"rowsVersion\"\n        DROP CONSTRAINT fk_rows_version_client_group`;\n    },\n  };\n\n  const migrateV6ToV7: Migration = {\n    migrateSchema: async (_, tx) => {\n      await tx`ALTER TABLE ${tx(schema)}.desires ADD \"expiresAt\" TIMESTAMPTZ`;\n      await tx`ALTER TABLE ${tx(\n        schema,\n      )}.desires ADD \"inactivatedAt\" TIMESTAMPTZ`;\n      await tx`ALTER TABLE ${tx(schema)}.desires ADD \"ttl\" INTERVAL`;\n\n      await tx`CREATE INDEX desires_expires_at ON ${tx(\n        schema,\n      )}.desires (\"expiresAt\")`;\n      await tx`CREATE INDEX desires_inactivated_at ON ${tx(\n        schema,\n      )}.desires (\"inactivatedAt\")`;\n    },\n  };\n\n  const migrateV7ToV8: Migration = {\n    migrateSchema: async (_, tx) => {\n      await tx`ALTER TABLE ${tx(\n        schema,\n      )}.\"desires\" DROP CONSTRAINT fk_desires_client`;\n    },\n  };\n\n  const migrateV8ToV9: Migration = {\n    migrateSchema: async (_, tx) => {\n      await tx`ALTER TABLE ${tx(schema)}.instances ADD \"clientSchema\" JSONB`;\n    },\n  };\n\n  const migrateV9ToV10: Migration = {\n    migrateSchema: async (_, tx) => {\n      await tx`ALTER TABLE ${tx(schema)}.queries ADD \"queryName\" TEXT`;\n      await tx`ALTER TABLE ${tx(schema)}.queries ADD \"queryArgs\" JSONB`;\n      await tx`ALTER TABLE ${tx(schema)}.queries ALTER COLUMN \"clientAST\" DROP NOT NULL`;\n    },\n  };\n\n  const migrateV10ToV11: Migration = {\n    migrateSchema: async (_, tx) => {\n      await tx`DROP INDEX IF EXISTS ${tx(schema)}.desires_expires_at`;\n      await tx`ALTER TABLE ${tx(schema)}.desires DROP COLUMN \"expiresAt\"`;\n      await tx`DROP INDEX IF EXISTS ${tx(schema)}.client_patch_version`;\n      await tx`ALTER TABLE ${tx(schema)}.clients DROP COLUMN \"patchVersion\"`;\n      await tx`ALTER TABLE ${tx(schema)}.clients DROP COLUMN \"deleted\"`;\n    },\n  };\n\n  const migratedV11ToV12: Migration = {\n    migrateSchema: async (_, tx) => {\n      await tx`ALTER TABLE ${tx(schema)}.queries ALTER COLUMN \"queryArgs\" TYPE JSON USING \"queryArgs\"::JSON`;\n    },\n  };\n\n  const migratedV12ToV13: Migration = {\n    migrateSchema: async (_, tx) => {\n      await tx`ALTER TABLE ${tx(schema)}.instances ADD COLUMN \"ttlClock\" DOUBLE PRECISION NOT NULL DEFAULT 0`;\n    },\n  };\n\n  const migratedV13ToV14: Migration = {\n    migrateSchema: async (_, sql) => {\n      await sql`\n        CREATE INDEX instances_last_active ON ${sql(schema)}.instances (\"lastActive\");\n      `;\n\n      // Update / add foreign key constraints to cascade deletes.\n      for (const [table, reference] of [\n        ['clients', 'instances'],\n        ['queries', 'instances'],\n        ['rows', 'rowsVersion'],\n      ] as [string, string][]) {\n        const constraint = sql(`fk_${table}_client_group`);\n        await sql`\n          ALTER TABLE ${sql(schema)}.${sql(table)} DROP CONSTRAINT IF EXISTS ${constraint}`;\n        await sql`\n          ALTER TABLE ${sql(schema)}.${sql(table)} ADD CONSTRAINT ${constraint}\n            FOREIGN KEY(\"clientGroupID\")\n            REFERENCES ${sql(schema)}.${sql(reference)} (\"clientGroupID\")\n            ON DELETE CASCADE;\n        `;\n      }\n    },\n  };\n\n  const migratedV14ToV15: Migration = {\n    migrateSchema: async (_, sql) => {\n      // Add new columns for storing inactivatedAt and ttl in milliseconds.\n      // This avoids postgres.js type conversion issues with TIMESTAMPTZ and INTERVAL.\n      await sql`ALTER TABLE ${sql(schema)}.desires \n        ADD COLUMN \"inactivatedAtMs\" DOUBLE PRECISION`;\n      await sql`ALTER TABLE ${sql(schema)}.desires \n        ADD COLUMN \"ttlMs\" DOUBLE PRECISION`;\n    },\n    // Migrate existing data: convert TIMESTAMPTZ to milliseconds for inactivatedAt\n    // and INTERVAL to milliseconds for ttl\n    // Note: EXTRACT(EPOCH FROM NULL) returns NULL, so NULL values are preserved\n    migrateData: async (lc, sql) => {\n      lc.info?.(\n        'Migrating desires.inactivatedAt to inactivatedAtMs and ttl to ttlMs',\n      );\n      await sql`\n        UPDATE ${sql(schema)}.desires\n        SET \"inactivatedAtMs\" = EXTRACT(EPOCH FROM \"inactivatedAt\") * 1000,\n            \"ttlMs\" = EXTRACT(EPOCH FROM \"ttl\") * 1000\n      `;\n    },\n  };\n\n  const migratedV15ToV16: Migration = {\n    migrateSchema: async (_, sql) => {\n      await sql`ALTER TABLE ${sql(schema)}.instances ADD COLUMN \"profileID\" TEXT`;\n      await sql`ALTER TABLE ${sql(schema)}.instances ADD COLUMN \"deleted\" BOOL DEFAULT FALSE`;\n\n      // Recreate the instances_last_active index to exclude tombstones\n      await sql`\n        DROP INDEX IF EXISTS ${sql(schema)}.instances_last_active`;\n      await sql`\n        CREATE INDEX instances_last_active ON ${sql(schema)}.instances (\"lastActive\")\n          WHERE NOT \"deleted\"`;\n      await sql`\n        CREATE INDEX tombstones_last_active ON ${sql(schema)}.instances (\"lastActive\")\n          WHERE \"deleted\"`;\n      await sql`\n        CREATE INDEX profile_ids_last_active ON ${sql(schema)}.instances (\"lastActive\", \"profileID\")\n          WHERE \"profileID\" IS NOT NULL`;\n    },\n\n    // Backfill profileIDs to the `cg${clientGroupID}`, as is done for\n    // client groups from old zero-clients that don't send a profileID.\n    migrateData: async (lc, sql) => {\n      lc.info?.('Backfilling instance.profileIDs');\n      await sql`\n        UPDATE ${sql(schema)}.instances\n        SET \"profileID\" = 'cg' || \"clientGroupID\"\n        WHERE \"profileID\" IS NULL\n      `;\n    },\n  };\n\n  const schemaVersionMigrationMap: IncrementalMigrationMap = {\n    2: migrateV1toV2,\n    3: migrateV2ToV3,\n    4: migrateV3ToV4,\n    // v5 enables asynchronous row-record flushing, and thus relies on\n    // the logic that updates and checks the rowsVersion table in v3.\n    5: {minSafeVersion: 3},\n    6: migrateV5ToV6,\n    7: migrateV6ToV7,\n    8: migrateV7ToV8,\n    9: migrateV8ToV9,\n    // v10 adds queryName and queryArgs to the queries table to support\n    // custom queries. clientAST is now optional to support migrating\n    // off client queries.\n    10: migrateV9ToV10,\n    // V11 removes the deprecated queries.\"expiresAt\", clients.\"patchVersion\",\n    // clients.\"deleted\" columns.\n    11: migrateV10ToV11,\n    12: migratedV11ToV12,\n    // V13 adds instances.\"ttlClock\"\n    13: migratedV12ToV13,\n    // V14 adds an index on instances.\"lastActive\" and a FK constraint\n    // from rows.\"clientGroupID\" to rowsVersion.\"clientGroupID\" for\n    // garbage collection\n    14: migratedV13ToV14,\n    // V15 adds desires.\"inactivatedAtTTLClock\" to store TTLClock values\n    // directly as DOUBLE PRECISION, avoiding postgres.js TIMESTAMPTZ\n    // type conversion issues\n    15: migratedV14ToV15,\n    // V16 adds instances.\"profileID\" and a corresponding index for estimating\n    // active user counts more accurately for apps that use memstore.\n    16: migratedV15ToV16,\n  };\n\n  await runSchemaMigrations(\n    log,\n    'view-syncer',\n    cvrSchema(shard),\n    db,\n    setupMigration,\n    schemaVersionMigrationMap,\n  );\n}\n"],"names":[],"mappings":";;;AAWA,eAAsB,qBACpB,KACA,IACA,OACe;AACf,QAAM,SAAS,UAAU,KAAK;AAE9B,QAAM,iBAA4B;AAAA,IAChC,eAAe,CAAC,IAAI,OAAO,eAAe,IAAI,IAAI,KAAK;AAAA,IACvD,gBAAgB;AAAA,EAAA;AAGlB,QAAM,gBAA2B;AAAA,IAC/B,eAAe,OAAO,GAAG,OAAO;AAC9B,YAAM,iBAAiB,GAAG,MAAM,CAAC;AAAA,IACnC;AAAA,EAAA;AAGF,QAAM,gBAA2B;AAAA,IAC/B,eAAe,OAAO,GAAG,OAAO;AAC9B,YAAM,GAAG,OAAO,uBAAuB,KAAK,CAAC;AAAA,IAC/C;AAAA;AAAA,IAGA,aAAa,OAAO,IAAI,OAAO;AAC7B,YAAM,UAAiC,CAAA;AACvC,uBAAiB,YAAY;AAAA,+CAGY,GAAG,MAAM,CAAC,aAAa;AAAA,QAC9D;AAAA,MAAA,GACC;AACD,mBAAW,WAAW,UAAU;AAC9B,kBAAQ;AAAA,YACN,iBAAiB,GAAG,MAAM,CAAC,kBAAkB,GAAG,OAAO,CAAC;AAAA;AAAA,+BAErC,GAAG,OAAO,CAAC,GAAG,QAAA;AAAA,UAAQ;AAAA,QAE7C;AAAA,MACF;AACA,SAAG,OAAO,gCAAgC,QAAQ,MAAM,OAAO;AAC/D,YAAM,QAAQ,IAAI,OAAO;AAAA,IAC3B;AAAA,EAAA;AAGF,QAAM,gBAA2B;AAAA,IAC/B,eAAe,OAAO,GAAG,OAAO;AAC9B,YAAM,iBAAiB,GAAG,MAAM,CAAC;AACjC,YAAM,iBAAiB,GAAG,MAAM,CAAC;AAAA,IACnC;AAAA,EAAA;AAGF,QAAM,gBAA2B;AAAA,IAC/B,eAAe,OAAO,GAAG,OAAO;AAC9B,YAAM;AAAA,oBACQ,GAAG,MAAM,CAAC;AAAA;AAExB,YAAM;AAAA,oBACQ,GAAG,MAAM,CAAC;AAAA;AAAA,IAE1B;AAAA,EAAA;AAGF,QAAM,gBAA2B;AAAA,IAC/B,eAAe,OAAO,GAAG,OAAO;AAC9B,YAAM,iBAAiB,GAAG,MAAM,CAAC;AACjC,YAAM,iBAAiB;AAAA,QACrB;AAAA,MAAA,CACD;AACD,YAAM,iBAAiB,GAAG,MAAM,CAAC;AAEjC,YAAM,wCAAwC;AAAA,QAC5C;AAAA,MAAA,CACD;AACD,YAAM,4CAA4C;AAAA,QAChD;AAAA,MAAA,CACD;AAAA,IACH;AAAA,EAAA;AAGF,QAAM,gBAA2B;AAAA,IAC/B,eAAe,OAAO,GAAG,OAAO;AAC9B,YAAM,iBAAiB;AAAA,QACrB;AAAA,MAAA,CACD;AAAA,IACH;AAAA,EAAA;AAGF,QAAM,gBAA2B;AAAA,IAC/B,eAAe,OAAO,GAAG,OAAO;AAC9B,YAAM,iBAAiB,GAAG,MAAM,CAAC;AAAA,IACnC;AAAA,EAAA;AAGF,QAAM,iBAA4B;AAAA,IAChC,eAAe,OAAO,GAAG,OAAO;AAC9B,YAAM,iBAAiB,GAAG,MAAM,CAAC;AACjC,YAAM,iBAAiB,GAAG,MAAM,CAAC;AACjC,YAAM,iBAAiB,GAAG,MAAM,CAAC;AAAA,IACnC;AAAA,EAAA;AAGF,QAAM,kBAA6B;AAAA,IACjC,eAAe,OAAO,GAAG,OAAO;AAC9B,YAAM,0BAA0B,GAAG,MAAM,CAAC;AAC1C,YAAM,iBAAiB,GAAG,MAAM,CAAC;AACjC,YAAM,0BAA0B,GAAG,MAAM,CAAC;AAC1C,YAAM,iBAAiB,GAAG,MAAM,CAAC;AACjC,YAAM,iBAAiB,GAAG,MAAM,CAAC;AAAA,IACnC;AAAA,EAAA;AAGF,QAAM,mBAA8B;AAAA,IAClC,eAAe,OAAO,GAAG,OAAO;AAC9B,YAAM,iBAAiB,GAAG,MAAM,CAAC;AAAA,IACnC;AAAA,EAAA;AAGF,QAAM,mBAA8B;AAAA,IAClC,eAAe,OAAO,GAAG,OAAO;AAC9B,YAAM,iBAAiB,GAAG,MAAM,CAAC;AAAA,IACnC;AAAA,EAAA;AAGF,QAAM,mBAA8B;AAAA,IAClC,eAAe,OAAO,GAAG,QAAQ;AAC/B,YAAM;AAAA,gDACoC,IAAI,MAAM,CAAC;AAAA;AAIrD,iBAAW,CAAC,OAAO,SAAS,KAAK;AAAA,QAC/B,CAAC,WAAW,WAAW;AAAA,QACvB,CAAC,WAAW,WAAW;AAAA,QACvB,CAAC,QAAQ,aAAa;AAAA,MAAA,GACC;AACvB,cAAM,aAAa,IAAI,MAAM,KAAK,eAAe;AACjD,cAAM;AAAA,wBACU,IAAI,MAAM,CAAC,IAAI,IAAI,KAAK,CAAC,8BAA8B,UAAU;AACjF,cAAM;AAAA,wBACU,IAAI,MAAM,CAAC,IAAI,IAAI,KAAK,CAAC,mBAAmB,UAAU;AAAA;AAAA,yBAErD,IAAI,MAAM,CAAC,IAAI,IAAI,SAAS,CAAC;AAAA;AAAA;AAAA,MAGhD;AAAA,IACF;AAAA,EAAA;AAGF,QAAM,mBAA8B;AAAA,IAClC,eAAe,OAAO,GAAG,QAAQ;AAG/B,YAAM,kBAAkB,IAAI,MAAM,CAAC;AAAA;AAEnC,YAAM,kBAAkB,IAAI,MAAM,CAAC;AAAA;AAAA,IAErC;AAAA;AAAA;AAAA;AAAA,IAIA,aAAa,OAAO,IAAI,QAAQ;AAC9B,SAAG;AAAA,QACD;AAAA,MAAA;AAEF,YAAM;AAAA,iBACK,IAAI,MAAM,CAAC;AAAA;AAAA;AAAA;AAAA,IAIxB;AAAA,EAAA;AAGF,QAAM,mBAA8B;AAAA,IAClC,eAAe,OAAO,GAAG,QAAQ;AAC/B,YAAM,kBAAkB,IAAI,MAAM,CAAC;AACnC,YAAM,kBAAkB,IAAI,MAAM,CAAC;AAGnC,YAAM;AAAA,+BACmB,IAAI,MAAM,CAAC;AACpC,YAAM;AAAA,gDACoC,IAAI,MAAM,CAAC;AAAA;AAErD,YAAM;AAAA,iDACqC,IAAI,MAAM,CAAC;AAAA;AAEtD,YAAM;AAAA,kDACsC,IAAI,MAAM,CAAC;AAAA;AAAA,IAEzD;AAAA;AAAA;AAAA,IAIA,aAAa,OAAO,IAAI,QAAQ;AAC9B,SAAG,OAAO,iCAAiC;AAC3C,YAAM;AAAA,iBACK,IAAI,MAAM,CAAC;AAAA;AAAA;AAAA;AAAA,IAIxB;AAAA,EAAA;AAGF,QAAM,4BAAqD;AAAA,IACzD,GAAG;AAAA,IACH,GAAG;AAAA,IACH,GAAG;AAAA;AAAA;AAAA,IAGH,GAAG,EAAC,gBAAgB,EAAA;AAAA,IACpB,GAAG;AAAA,IACH,GAAG;AAAA,IACH,GAAG;AAAA,IACH,GAAG;AAAA;AAAA;AAAA;AAAA,IAIH,IAAI;AAAA;AAAA;AAAA,IAGJ,IAAI;AAAA,IACJ,IAAI;AAAA;AAAA,IAEJ,IAAI;AAAA;AAAA;AAAA;AAAA,IAIJ,IAAI;AAAA;AAAA;AAAA;AAAA,IAIJ,IAAI;AAAA;AAAA;AAAA,IAGJ,IAAI;AAAA,EAAA;AAGN,QAAM;AAAA,IACJ;AAAA,IACA;AAAA,IACA,UAAU,KAAK;AAAA,IACf;AAAA,IACA;AAAA,IACA;AAAA,EAAA;AAEJ;"}

package/out/zero-cache/src/services/view-syncer/snapshotter.d.ts

@@ -68,7 +68,7 @@ import type { AppID } from '../../types/shards.ts';
  */
 export declare class Snapshotter {
     #private;
-    constructor(lc: LogContext, dbFile: string, { appID }: AppID);
+    constructor(lc: LogContext, dbFile: string, { appID }: AppID, pageCacheSizeKib?: number | undefined);
     /**
      * Initializes the snapshot to the current head of the database. This must be
      * only be called once. The state of whether a Snapshotter has been initialized
@@ -178,7 +178,7 @@ export declare class ResetPipelinesSignal extends Error {
 }
 declare class Snapshot {
     #private;
-    static create(lc: LogContext, dbFile: string, appID: string): Snapshot;
+    static create(lc: LogContext, dbFile: string, appID: string, pageCacheSizeKib: number | undefined): Snapshot;
     readonly db: StatementRunner;
     readonly version: string;
     readonly schemaVersions: SchemaVersions;

package/out/zero-cache/src/services/view-syncer/snapshotter.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"snapshotter.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/view-syncer/snapshotter.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,EAAY,KAAK,SAAS,EAAC,MAAM,uCAAuC,CAAC;AAGhF,OAAO,KAAK,EAAC,GAAG,EAAC,MAAM,uCAAuC,CAAC;AAC/D,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,sCAAsC,CAAC;AAGrE,OAAO,KAAK,EAAC,cAAc,EAAE,qBAAqB,EAAC,MAAM,mBAAmB,CAAC;AAC7E,OAAO,EAAC,eAAe,EAAC,MAAM,wBAAwB,CAAC;AACvD,OAAO,EAEL,KAAK,MAAM,EACX,KAAK,QAAQ,EACd,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAAC,KAAK,cAAc,EAAC,MAAM,gCAAgC,CAAC;AACnE,OAAO,KAAK,EAAC,KAAK,EAAC,MAAM,uBAAuB,CAAC;AAajD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0DG;AACH,qBAAa,WAAW;;gBAOV,EAAE,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,EAAE,EAAC,KAAK,EAAC,EAAE,KAAK;IAM1D;;;;OAIG;IACH,IAAI,IAAI,IAAI;IAOZ,WAAW,IAAI,OAAO;IAItB,+EAA+E;IAC/E,OAAO,IAAI,QAAQ;IAKnB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAoCG;IACH,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,cAAc,CAAC,GAAG,YAAY;IAK1D,kBAAkB;;;;IAUlB;;;OAGG;IACH,OAAO;CAKR;AAED,MAAM,MAAM,MAAM,GAAG;IACnB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB;;;;;OAKG;IACH,QAAQ,CAAC,UAAU,EAAE,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;IACrC,QAAQ,CAAC,SAAS,EAAE,QAAQ,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC;IACzC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB,CAAC;AAEF;;;;;;GAMG;AACH,MAAM,WAAW,YAAa,SAAQ,QAAQ,CAAC,MAAM,CAAC;IACpD,QAAQ,CAAC,IAAI,EAAE;QACb,QAAQ,CAAC,EAAE,EAAE,eAAe,CAAC;QAC7B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;KAC1B,CAAC;IACF,QAAQ,CAAC,IAAI,EAAE;QACb,QAAQ,CAAC,EAAE,EAAE,eAAe,CAAC;QAC7B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;KAC1B,CAAC;IAEF;;;;;;;;;;OAUG;IACH,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B;AAED;;;;GAIG;AACH,qBAAa,oBAAqB,SAAQ,KAAK;IAC7C,QAAQ,CAAC,IAAI,0BAA0B;gBAE3B,GAAG,EAAE,MAAM;CAGxB;AAQD,cAAM,QAAQ;;IACZ,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM;IAkB3D,QAAQ,CAAC,EAAE,EAAE,eAAe,CAAC;IAE7B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,cAAc,EAAE,cAAc,CAAC;gBAE5B,EAAE,EAAE,eAAe,EAAE,KAAK,EAAE,MAAM;IAa9C,eAAe,CAAC,WAAW,EAAE,MAAM;IAQnC,YAAY,CAAC,WAAW,EAAE,MAAM;;;;IAUhC,MAAM,CAAC,KAAK,EAAE,qBAAqB,EAAE,MAAM,EAAE,SAAS;IAkBtD,OAAO,CAAC,KAAK,EAAE,qBAAqB,EAAE,IAAI,EAAE,UAAU,EAAE,EAAE,GAAG,EAAE,QAAQ;IAmBvE,WAAW,IAAI,QAAQ;CAIxB;AAmKD,qBAAa,gBAAiB,SAAQ,KAAK;gBAC7B,GAAG,EAAE,MAAM;CAGxB"}
+{"version":3,"file":"snapshotter.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/view-syncer/snapshotter.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,EAAY,KAAK,SAAS,EAAC,MAAM,uCAAuC,CAAC;AAGhF,OAAO,KAAK,EAAC,GAAG,EAAC,MAAM,uCAAuC,CAAC;AAC/D,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,sCAAsC,CAAC;AAGrE,OAAO,KAAK,EAAC,cAAc,EAAE,qBAAqB,EAAC,MAAM,mBAAmB,CAAC;AAC7E,OAAO,EAAC,eAAe,EAAC,MAAM,wBAAwB,CAAC;AACvD,OAAO,EAEL,KAAK,MAAM,EACX,KAAK,QAAQ,EACd,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAAC,KAAK,cAAc,EAAC,MAAM,gCAAgC,CAAC;AACnE,OAAO,KAAK,EAAC,KAAK,EAAC,MAAM,uBAAuB,CAAC;AAajD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0DG;AACH,qBAAa,WAAW;;gBASpB,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,MAAM,EACd,EAAC,KAAK,EAAC,EAAE,KAAK,EACd,gBAAgB,CAAC,EAAE,MAAM,GAAG,SAAS;IAQvC;;;;OAIG;IACH,IAAI,IAAI,IAAI;IAYZ,WAAW,IAAI,OAAO;IAItB,+EAA+E;IAC/E,OAAO,IAAI,QAAQ;IAKnB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAoCG;IACH,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,cAAc,CAAC,GAAG,YAAY;IAK1D,kBAAkB;;;;IAelB;;;OAGG;IACH,OAAO;CAKR;AAED,MAAM,MAAM,MAAM,GAAG;IACnB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB;;;;;OAKG;IACH,QAAQ,CAAC,UAAU,EAAE,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;IACrC,QAAQ,CAAC,SAAS,EAAE,QAAQ,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC;IACzC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB,CAAC;AAEF;;;;;;GAMG;AACH,MAAM,WAAW,YAAa,SAAQ,QAAQ,CAAC,MAAM,CAAC;IACpD,QAAQ,CAAC,IAAI,EAAE;QACb,QAAQ,CAAC,EAAE,EAAE,eAAe,CAAC;QAC7B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;KAC1B,CAAC;IACF,QAAQ,CAAC,IAAI,EAAE;QACb,QAAQ,CAAC,EAAE,EAAE,eAAe,CAAC;QAC7B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;KAC1B,CAAC;IAEF;;;;;;;;;;OAUG;IACH,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B;AAED;;;;GAIG;AACH,qBAAa,oBAAqB,SAAQ,KAAK;IAC7C,QAAQ,CAAC,IAAI,0BAA0B;gBAE3B,GAAG,EAAE,MAAM;CAGxB;AAQD,cAAM,QAAQ;;IACZ,MAAM,CAAC,MAAM,CACX,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,MAAM,EACb,gBAAgB,EAAE,MAAM,GAAG,SAAS;IAsBtC,QAAQ,CAAC,EAAE,EAAE,eAAe,CAAC;IAE7B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,cAAc,EAAE,cAAc,CAAC;gBAE5B,EAAE,EAAE,eAAe,EAAE,KAAK,EAAE,MAAM;IAa9C,eAAe,CAAC,WAAW,EAAE,MAAM;IAQnC,YAAY,CAAC,WAAW,EAAE,MAAM;;;;IAUhC,MAAM,CAAC,KAAK,EAAE,qBAAqB,EAAE,MAAM,EAAE,SAAS;IAkBtD,OAAO,CAAC,KAAK,EAAE,qBAAqB,EAAE,IAAI,EAAE,UAAU,EAAE,EAAE,GAAG,EAAE,QAAQ;IAmBvE,WAAW,IAAI,QAAQ;CAIxB;AAmKD,qBAAa,gBAAiB,SAAQ,KAAK;gBAC7B,GAAG,EAAE,MAAM;CAGxB"}

package/out/zero-cache/src/services/view-syncer/snapshotter.js

@@ -15,12 +15,14 @@ class Snapshotter {
   #lc;
   #dbFile;
   #appID;
+  #pageCacheSizeKib;
   #curr;
   #prev;
-  constructor(lc, dbFile, { appID }) {
+  constructor(lc, dbFile, { appID }, pageCacheSizeKib) {
     this.#lc = lc;
     this.#dbFile = dbFile;
     this.#appID = appID;
+    this.#pageCacheSizeKib = pageCacheSizeKib;
   }
   /**
    * Initializes the snapshot to the current head of the database. This must be
@@ -29,7 +31,12 @@ class Snapshotter {
    */
   init() {
     assert(this.#curr === void 0, "Already initialized");
-    this.#curr = Snapshot.create(this.#lc, this.#dbFile, this.#appID);
+    this.#curr = Snapshot.create(
+      this.#lc,
+      this.#dbFile,
+      this.#appID,
+      this.#pageCacheSizeKib
+    );
     this.#lc.debug?.(`Initial snapshot at version ${this.#curr.version}`);
     return this;
   }
@@ -84,7 +91,12 @@ class Snapshotter {
   }
   advanceWithoutDiff() {
     assert(this.#curr !== void 0, "Snapshotter has not been initialized");
-    const next = this.#prev ? this.#prev.resetToHead() : Snapshot.create(this.#lc, this.#curr.db.db.name, this.#appID);
+    const next = this.#prev ? this.#prev.resetToHead() : Snapshot.create(
+      this.#lc,
+      this.#curr.db.db.name,
+      this.#appID,
+      this.#pageCacheSizeKib
+    );
     this.#prev = this.#curr;
     this.#curr = next;
     return { prev: this.#prev, curr: this.#curr };
@@ -111,9 +123,12 @@ function getSchemaVersions(db, appID) {
   );
 }
 class Snapshot {
-  static create(lc, dbFile, appID) {
+  static create(lc, dbFile, appID, pageCacheSizeKib) {
     const conn = new Database(lc, dbFile);
     conn.pragma("synchronous = OFF");
+    if (pageCacheSizeKib !== void 0) {
+      conn.pragma(`cache_size = -${pageCacheSizeKib}`);
+    }
     const [{ journal_mode: mode }] = conn.pragma("journal_mode");
     assert(
       mode === "wal2",

package/out/zero-cache/src/services/view-syncer/snapshotter.js.map

@@ -1 +1 @@
-{"version":3,"file":"snapshotter.js","sources":["../../../../../../zero-cache/src/services/view-syncer/snapshotter.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport {stringify, type JSONValue} from '../../../../shared/src/bigint-json.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport type {Row} from '../../../../zero-protocol/src/data.ts';\nimport type {PrimaryKey} from '../../../../zero-types/src/schema.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {fromSQLiteTypes} from '../../../../zqlite/src/table-source.ts';\nimport type {LiteAndZqlSpec, LiteTableSpecWithKeys} from '../../db/specs.ts';\nimport {StatementRunner} from '../../db/statements.ts';\nimport {\n  normalizedKeyOrder,\n  type RowKey,\n  type RowValue,\n} from '../../types/row-key.ts';\nimport {type SchemaVersions} from '../../types/schema-versions.ts';\nimport type {AppID} from '../../types/shards.ts';\nimport {id} from '../../types/sql.ts';\nimport {\n  RESET_OP,\n  changeLogEntrySchema as schema,\n  SET_OP,\n  TRUNCATE_OP,\n} from '../replicator/schema/change-log.ts';\nimport {\n  getReplicationState,\n  ZERO_VERSION_COLUMN_NAME as ROW_VERSION,\n} from '../replicator/schema/replication-state.ts';\n\n/**\n * A `Snapshotter` manages the progression of database snapshots for a\n * ViewSyncer.\n *\n * The Replicator and ViewSyncers operate on the same SQLite file, with the\n * Replicator being the sole writer to the database. The IVM logic in\n * ViewSyncers, however, rely on incrementally applying changes to the DB to\n * update the state of its pipelines.\n *\n * To avoid coupling the progress of the Replicator and all IVM pipelines on\n * each other, ViewSyncers operate on ephemeral forks of the database by holding\n * [concurrent](https://sqlite.org/src/doc/begin-concurrent/doc/begin_concurrent.md)\n * snapshots of the database and simulating (but ultimately rolling back)\n * mutations on these snapshots.\n *\n * Example:\n * 1. ViewSyncer takes `snapshot_a` at version `t1` of the database and\n *    hydrates its pipeline(s).\n * 2. Replicator applies a new transaction to the database and notifies\n *    subscribers.\n * 3. ViewSyncer takes `snapshot_b` at `t2`, and queries the `ChangeLog` at\n *    that snapshot for changes since `t1`.\n * 4. ViewSyncer applies those changes to `snapshot_a` for IVM, but does not\n *    commit them. (Recall that the Replicator is the sole writer to the db, so\n *    the ViewSyncer never commits any writes.)\n * 5. Replicator applies the next transaction and advances the database to `t3`.\n * 6. ViewSyncer rolls back `snapshot_a` and opens `snapshot_c` at `t3`, using\n *    `snapshot_b` to simulate changes from `t2` to `t3`.\n *\n * ```\n * Replicator:  t1 --------------> t2 --------------> t3 --------------->\n * ViewSyncer:       [snapshot_a] ----> [snapshot_b] ----> [snapshot_c]\n * ```\n *\n * Note that the Replicator (and ViewSyncers) do not wait on the progress of\n * other ViewSyncers. If a ViewSyncer is busy hydrating at `t1`, the Replicator\n * and other ViewSyncers can progress through `t2`, `t3`, etc. independently,\n * as the busy ViewSyncer simply takes its own snapshot when it is ready.\n *\n * ```\n * Replicator:   t1 --------------> t2 --------------> t3 --------------->\n * ViewSyncer1:       [snapshot_a] ----> [snapshot_b] ----> [snapshot_c]\n * ViewSyncer2:       [.......... snapshot_a ..........] ----> [snapshot_b]\n * ```\n *\n * To minimize Database connections (and statement preparation, etc.), the\n * Snapshotter reuses the connection from the previous (rolled back)\n * snapshot when opening the new one.\n *\n * ```\n * Replicator:  t1 --------------> t2 --------------> t3 --------------->\n * ViewSyncer:       [snapshot_a] ----> [snapshot_b] ----> [snapshot_c]\n *                     (conn_1)           (conn_2)           (conn_1)\n * ```\n *\n * In this manner, each ViewSyncer uses two connections that continually\n * \"leapfrog\" each other to replay the timeline of changes in isolation from\n * the Replicator and other ViewSyncers.\n */\nexport class Snapshotter {\n  readonly #lc: LogContext;\n  readonly #dbFile: string;\n  readonly #appID: string;\n  #curr: Snapshot | undefined;\n  #prev: Snapshot | undefined;\n\n  constructor(lc: LogContext, dbFile: string, {appID}: AppID) {\n    this.#lc = lc;\n    this.#dbFile = dbFile;\n    this.#appID = appID;\n  }\n\n  /**\n   * Initializes the snapshot to the current head of the database. This must be\n   * only be called once. The state of whether a Snapshotter has been initialized\n   * can be determined by calling {@link initialized()}.\n   */\n  init(): this {\n    assert(this.#curr === undefined, 'Already initialized');\n    this.#curr = Snapshot.create(this.#lc, this.#dbFile, this.#appID);\n    this.#lc.debug?.(`Initial snapshot at version ${this.#curr.version}`);\n    return this;\n  }\n\n  initialized(): boolean {\n    return this.#curr !== undefined;\n  }\n\n  /** Returns the current snapshot. Asserts if {@link initialized()} is false. */\n  current(): Snapshot {\n    assert(this.#curr !== undefined, 'Snapshotter has not been initialized');\n    return this.#curr;\n  }\n\n  /**\n   * Advances to the head of the Database, returning a diff between the\n   * previously current Snapshot and a new Snapshot at head. This is called\n   * in response to a notification from a Replicator subscription. Subsequent\n   * calls to {@link current()} return the new Snapshot. Note that the Snapshotter\n   * must be initialized before advancing.\n   *\n   * The returned {@link SnapshotDiff} contains snapshots at the endpoints\n   * of the database timeline. Iterating over the diff generates a sequence\n   * of {@link Change}s between the two snapshots.\n   *\n   * Note that this sequence is not chronological; rather, the sequence is\n   * ordered by `<table, row-key>`, such that a row can appear at most once\n   * in the common case, or twice if its table is `TRUNCATE`'d and a new value\n   * is subsequently `INSERT`'ed. This results in dropping most intermediate\n   * changes to a row and bounds the amount of work needed to catch up;\n   * however, as a consequence, a consistent database state is only guaranteed\n   * when the sequence has been fully consumed.\n   *\n   * Note that Change generation relies on the state of the underlying\n   * database connections, and because the connection for the previous snapshot\n   * is reused to produce the next snapshot, the diff object is only valid\n   * until the next call to `advance()`.\n   *\n   * It is okay for the caller to apply `Change`s to the `prev` snapshot\n   * during the iteration (e.g. this is necessary for IVM); the remainder\n   * of the iteration is not affected because a given row can appear at most\n   * once in the sequence (with the exception being TRUNCATE, after which the\n   * deleted rows can be re-inserted, but this will also behave correctly if\n   * the changes are applied).\n   *\n   * Once the changes have been applied, however, a _subsequent_ iteration\n   * will not produce the correct results. In order to perform multiple\n   * change-applying iterations, the caller must (1) create a save point\n   * on `prev` before each iteration, and (2) rollback to the save point after\n   * the iteration.\n   */\n  advance(tables: Map<string, LiteAndZqlSpec>): SnapshotDiff {\n    const {prev, curr} = this.advanceWithoutDiff();\n    return new Diff(this.#appID, tables, prev, curr);\n  }\n\n  advanceWithoutDiff() {\n    assert(this.#curr !== undefined, 'Snapshotter has not been initialized');\n    const next = this.#prev\n      ? this.#prev.resetToHead()\n      : Snapshot.create(this.#lc, this.#curr.db.db.name, this.#appID);\n    this.#prev = this.#curr;\n    this.#curr = next;\n    return {prev: this.#prev, curr: this.#curr};\n  }\n\n  /**\n   * Call this to close the database connections when the Snapshotter is\n   * no longer needed.\n   */\n  destroy() {\n    this.#curr?.db.db.close();\n    this.#prev?.db.db.close();\n    this.#lc.debug?.('closed database connections');\n  }\n}\n\nexport type Change = {\n  readonly table: string;\n  /**\n   * If this change represents a remove the row to remove,\n   * if nextValue is not null then all rows that have a unique constraint\n   * violation with nextValue.\n   * In both cases these rows should be removed.\n   */\n  readonly prevValues: Readonly<Row>[];\n  readonly nextValue: Readonly<Row> | null;\n  readonly rowKey: RowKey;\n};\n\n/**\n * Represents the difference between two database Snapshots.\n * Iterating over the object will produce a sequence of {@link Change}s\n * between the two snapshots.\n *\n * See {@link Snapshotter.advance()} for semantics and usage.\n */\nexport interface SnapshotDiff extends Iterable<Change> {\n  readonly prev: {\n    readonly db: StatementRunner;\n    readonly version: string;\n  };\n  readonly curr: {\n    readonly db: StatementRunner;\n    readonly version: string;\n  };\n\n  /**\n   * The number of ChangeLog entries between the snapshots. Note that this\n   * may not necessarily equal the number of `Change` objects that the iteration\n   * will produce, as `TRUNCATE` entries are counted as a single log entry which\n   * may be expanded into many changes (i.e. row deletes).\n   *\n   * TODO: Determine if it is worth changing the definition to count the\n   *       truncated rows. This would make diff computation more expensive\n   *       (requiring the count to be aggregated by operation type), which\n   *       may not be worth it for a presumable rare operation.\n   */\n  readonly changes: number;\n}\n\n/**\n * Thrown during an iteration of a {@link SnapshotDiff} when a schema\n * change or truncate is encountered, which result in aborting the\n * advancement and resetting / rehydrating the pipelines.\n */\nexport class ResetPipelinesSignal extends Error {\n  readonly name = 'ResetPipelinesSignal';\n\n  constructor(msg: string) {\n    super(msg);\n  }\n}\n\nfunction getSchemaVersions(db: StatementRunner, appID: string): SchemaVersions {\n  return db.get(\n    `SELECT minSupportedVersion, maxSupportedVersion FROM \"${appID}.schemaVersions\"`,\n  );\n}\n\nclass Snapshot {\n  static create(lc: LogContext, dbFile: string, appID: string) {\n    const conn = new Database(lc, dbFile);\n    conn.pragma('synchronous = OFF'); // Applied changes are ephemeral; COMMIT is never called.\n    const [{journal_mode: mode}] = conn.pragma('journal_mode') as [\n      {journal_mode: string},\n    ];\n    // The Snapshotter operates on the replica file with BEGIN CONCURRENT,\n    // which must be used in concert with the replicator using BEGIN CONCURRENT\n    // on a db in the wal2 journal_mode.\n    assert(\n      mode === 'wal2',\n      `replica db must be in wal2 mode (current: ${mode})`,\n    );\n\n    const db = new StatementRunner(conn);\n    return new Snapshot(db, appID);\n  }\n\n  readonly db: StatementRunner;\n  readonly #appID: string;\n  readonly version: string;\n  readonly schemaVersions: SchemaVersions;\n\n  constructor(db: StatementRunner, appID: string) {\n    db.beginConcurrent();\n    // Note: The subsequent read is necessary to acquire the read lock\n    // (which results in the logical creation of the snapshot). Calling\n    // `BEGIN CONCURRENT` alone does not result in acquiring the read lock.\n    const {stateVersion} = getReplicationState(db);\n\n    this.db = db;\n    this.#appID = appID;\n    this.version = stateVersion;\n    this.schemaVersions = getSchemaVersions(db, appID);\n  }\n\n  numChangesSince(prevVersion: string) {\n    const {count} = this.db.get(\n      'SELECT COUNT(*) AS count FROM \"_zero.changeLog\" WHERE stateVersion > ?',\n      prevVersion,\n    );\n    return count;\n  }\n\n  changesSince(prevVersion: string) {\n    const cached = this.db.statementCache.get(\n      'SELECT * FROM \"_zero.changeLog\" WHERE stateVersion > ?',\n    );\n    return {\n      changes: cached.statement.iterate(prevVersion),\n      cleanup: () => this.db.statementCache.return(cached),\n    };\n  }\n\n  getRow(table: LiteTableSpecWithKeys, rowKey: JSONValue) {\n    const key = normalizedKeyOrder(rowKey as RowKey);\n    const conds = Object.keys(key).map(c => `${id(c)}=?`);\n    const cols = Object.keys(table.columns);\n    const cached = this.db.statementCache.get(\n      `SELECT ${cols.map(c => id(c)).join(',')} FROM ${id(\n        table.name,\n      )} WHERE ${conds.join(' AND ')}`,\n    );\n    cached.statement.safeIntegers(true);\n    try {\n      // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n      return cached.statement.get<any>(Object.values(key));\n    } finally {\n      this.db.statementCache.return(cached);\n    }\n  }\n\n  getRows(table: LiteTableSpecWithKeys, keys: PrimaryKey[], row: RowValue) {\n    const conds = keys.map(key => key.map(c => `${id(c)}=?`));\n    const cols = Object.keys(table.columns);\n    const cached = this.db.statementCache.get(\n      `SELECT ${cols.map(c => id(c)).join(',')} FROM ${id(\n        table.name,\n      )} WHERE ${conds.map(cond => cond.join(' AND ')).join(' OR ')}`,\n    );\n    cached.statement.safeIntegers(true);\n    try {\n      // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n      return cached.statement.all<any>(\n        keys.flatMap(key => key.map(column => row[column])),\n      );\n    } finally {\n      this.db.statementCache.return(cached);\n    }\n  }\n\n  resetToHead(): Snapshot {\n    this.db.rollback();\n    return new Snapshot(this.db, this.#appID);\n  }\n}\n\nclass Diff implements SnapshotDiff {\n  readonly #permissionsTable: string;\n  readonly tables: Map<string, LiteAndZqlSpec>;\n  readonly prev: Snapshot;\n  readonly curr: Snapshot;\n  readonly changes: number;\n\n  constructor(\n    appID: string,\n    tables: Map<string, LiteAndZqlSpec>,\n    prev: Snapshot,\n    curr: Snapshot,\n  ) {\n    this.#permissionsTable = `${appID}.permissions`;\n    this.tables = tables;\n    this.prev = prev;\n    this.curr = curr;\n    this.changes = curr.numChangesSince(prev.version);\n  }\n\n  [Symbol.iterator](): Iterator<Change> {\n    const {changes, cleanup: done} = this.curr.changesSince(this.prev.version);\n\n    const cleanup = () => {\n      try {\n        // Allow open iterators to clean up their state.\n        changes.return?.(undefined);\n      } finally {\n        done();\n      }\n    };\n\n    return {\n      next: () => {\n        try {\n          for (;;) {\n            const {value, done} = changes.next();\n            if (done) {\n              cleanup();\n              return {value, done: true};\n            }\n\n            const {table, rowKey, op, stateVersion} = v.parse(value, schema);\n            if (op === RESET_OP) {\n              // The current map of `TableSpec`s may not have the correct or complete information.\n              throw new ResetPipelinesSignal(\n                `schema for table ${table} has changed`,\n              );\n            }\n            if (op === TRUNCATE_OP) {\n              // Truncates are also processed by rehydrating pipelines at current.\n              throw new ResetPipelinesSignal(\n                `table ${table} has been truncated`,\n              );\n            }\n            const {tableSpec, zqlSpec} = must(this.tables.get(table));\n\n            assert(rowKey !== null);\n            const nextValue =\n              op === SET_OP ? this.curr.getRow(tableSpec, rowKey) : null;\n            let prevValues;\n            if (nextValue) {\n              prevValues = this.prev.getRows(\n                tableSpec,\n                tableSpec.uniqueKeys,\n                nextValue,\n              );\n            } else {\n              const prevValue = this.prev.getRow(tableSpec, rowKey);\n              prevValues = prevValue ? [prevValue] : [];\n            }\n            if (nextValue === undefined) {\n              throw new Error(\n                `Missing value for ${table} ${stringify(rowKey)}`,\n              );\n            }\n            // Sanity check detects if the diff is being accessed after the Snapshots have advanced.\n            this.checkThatDiffIsValid(stateVersion, op, prevValues, nextValue);\n\n            if (prevValues.length === 0 && nextValue === null) {\n              // Filter out no-op changes (e.g. a delete of a row that does not exist in prev).\n              // TODO: Consider doing this for deep-equal values.\n              continue;\n            }\n\n            if (\n              table === this.#permissionsTable &&\n              prevValues.find(\n                prevValue => prevValue.permissions !== nextValue.permissions,\n              )\n            ) {\n              throw new ResetPipelinesSignal(\n                `Permissions have changed ${\n                  prevValues.find(\n                    prevValue =>\n                      prevValue.permissions !== nextValue.permissions,\n                  ).hash\n                } => ${nextValue.hash}`,\n              );\n            }\n\n            // Modify the values in place when converting to ZQL rows\n            // This is safe since we're the first node in the iterator chain.\n            // TODO Can we get rid of these RowValue casts?\n            return {\n              value: {\n                table,\n                prevValues: prevValues.map(prevValue =>\n                  fromSQLiteTypes(zqlSpec, prevValue, table),\n                ),\n                nextValue: nextValue\n                  ? fromSQLiteTypes(zqlSpec, nextValue, table)\n                  : null,\n                rowKey,\n              } satisfies Change,\n            };\n          }\n        } catch (e) {\n          // This control flow path is not covered by the return() method (i.e. `break`).\n          cleanup();\n          throw e;\n        }\n      },\n\n      return: (value: unknown) => {\n        cleanup();\n        return {value, done: true};\n      },\n    };\n  }\n\n  checkThatDiffIsValid(\n    stateVersion: string,\n    op: string,\n    prevValues: RowValue[],\n    nextValue: RowValue,\n  ) {\n    // Sanity checks to detect that the diff is not being accessed after\n    // the Snapshots have advanced.\n    if (stateVersion > this.curr.version) {\n      throw new InvalidDiffError(\n        `Diff is no longer valid. curr db has advanced past ${this.curr.version}`,\n      );\n    }\n    if (\n      prevValues.findIndex(\n        prevValue => (prevValue[ROW_VERSION] ?? '~') > this.prev.version,\n      ) !== -1\n    ) {\n      throw new InvalidDiffError(\n        `Diff is no longer valid. prev db has advanced past ${this.prev.version}.`,\n      );\n    }\n    if (op === SET_OP && nextValue[ROW_VERSION] !== stateVersion) {\n      throw new InvalidDiffError(\n        'Diff is no longer valid. curr db has advanced.',\n      );\n    }\n  }\n}\n\nexport class InvalidDiffError extends Error {\n  constructor(msg: string) {\n    super(msg);\n  }\n}\n"],"names":["done","v.parse","schema","ROW_VERSION"],"mappings":";;;;;;;;;;;;;AAyFO,MAAM,YAAY;AAAA,EACd;AAAA,EACA;AAAA,EACA;AAAA,EACT;AAAA,EACA;AAAA,EAEA,YAAY,IAAgB,QAAgB,EAAC,SAAe;AAC1D,SAAK,MAAM;AACX,SAAK,UAAU;AACf,SAAK,SAAS;AAAA,EAChB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,OAAa;AACX,WAAO,KAAK,UAAU,QAAW,qBAAqB;AACtD,SAAK,QAAQ,SAAS,OAAO,KAAK,KAAK,KAAK,SAAS,KAAK,MAAM;AAChE,SAAK,IAAI,QAAQ,+BAA+B,KAAK,MAAM,OAAO,EAAE;AACpE,WAAO;AAAA,EACT;AAAA,EAEA,cAAuB;AACrB,WAAO,KAAK,UAAU;AAAA,EACxB;AAAA;AAAA,EAGA,UAAoB;AAClB,WAAO,KAAK,UAAU,QAAW,sCAAsC;AACvE,WAAO,KAAK;AAAA,EACd;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAuCA,QAAQ,QAAmD;AACzD,UAAM,EAAC,MAAM,SAAQ,KAAK,mBAAA;AAC1B,WAAO,IAAI,KAAK,KAAK,QAAQ,QAAQ,MAAM,IAAI;AAAA,EACjD;AAAA,EAEA,qBAAqB;AACnB,WAAO,KAAK,UAAU,QAAW,sCAAsC;AACvE,UAAM,OAAO,KAAK,QACd,KAAK,MAAM,gBACX,SAAS,OAAO,KAAK,KAAK,KAAK,MAAM,GAAG,GAAG,MAAM,KAAK,MAAM;AAChE,SAAK,QAAQ,KAAK;AAClB,SAAK,QAAQ;AACb,WAAO,EAAC,MAAM,KAAK,OAAO,MAAM,KAAK,MAAA;AAAA,EACvC;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,UAAU;AACR,SAAK,OAAO,GAAG,GAAG,MAAA;AAClB,SAAK,OAAO,GAAG,GAAG,MAAA;AAClB,SAAK,IAAI,QAAQ,6BAA6B;AAAA,EAChD;AACF;AAmDO,MAAM,6BAA6B,MAAM;AAAA,EACrC,OAAO;AAAA,EAEhB,YAAY,KAAa;AACvB,UAAM,GAAG;AAAA,EACX;AACF;AAEA,SAAS,kBAAkB,IAAqB,OAA+B;AAC7E,SAAO,GAAG;AAAA,IACR,yDAAyD,KAAK;AAAA,EAAA;AAElE;AAEA,MAAM,SAAS;AAAA,EACb,OAAO,OAAO,IAAgB,QAAgB,OAAe;AAC3D,UAAM,OAAO,IAAI,SAAS,IAAI,MAAM;AACpC,SAAK,OAAO,mBAAmB;AAC/B,UAAM,CAAC,EAAC,cAAc,KAAA,CAAK,IAAI,KAAK,OAAO,cAAc;AAMzD;AAAA,MACE,SAAS;AAAA,MACT,6CAA6C,IAAI;AAAA,IAAA;AAGnD,UAAM,KAAK,IAAI,gBAAgB,IAAI;AACnC,WAAO,IAAI,SAAS,IAAI,KAAK;AAAA,EAC/B;AAAA,EAES;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAET,YAAY,IAAqB,OAAe;AAC9C,OAAG,gBAAA;AAIH,UAAM,EAAC,aAAA,IAAgB,oBAAoB,EAAE;AAE7C,SAAK,KAAK;AACV,SAAK,SAAS;AACd,SAAK,UAAU;AACf,SAAK,iBAAiB,kBAAkB,IAAI,KAAK;AAAA,EACnD;AAAA,EAEA,gBAAgB,aAAqB;AACnC,UAAM,EAAC,MAAA,IAAS,KAAK,GAAG;AAAA,MACtB;AAAA,MACA;AAAA,IAAA;AAEF,WAAO;AAAA,EACT;AAAA,EAEA,aAAa,aAAqB;AAChC,UAAM,SAAS,KAAK,GAAG,eAAe;AAAA,MACpC;AAAA,IAAA;AAEF,WAAO;AAAA,MACL,SAAS,OAAO,UAAU,QAAQ,WAAW;AAAA,MAC7C,SAAS,MAAM,KAAK,GAAG,eAAe,OAAO,MAAM;AAAA,IAAA;AAAA,EAEvD;AAAA,EAEA,OAAO,OAA8B,QAAmB;AACtD,UAAM,MAAM,mBAAmB,MAAgB;AAC/C,UAAM,QAAQ,OAAO,KAAK,GAAG,EAAE,IAAI,CAAA,MAAK,GAAG,GAAG,CAAC,CAAC,IAAI;AACpD,UAAM,OAAO,OAAO,KAAK,MAAM,OAAO;AACtC,UAAM,SAAS,KAAK,GAAG,eAAe;AAAA,MACpC,UAAU,KAAK,IAAI,CAAA,MAAK,GAAG,CAAC,CAAC,EAAE,KAAK,GAAG,CAAC,SAAS;AAAA,QAC/C,MAAM;AAAA,MAAA,CACP,UAAU,MAAM,KAAK,OAAO,CAAC;AAAA,IAAA;AAEhC,WAAO,UAAU,aAAa,IAAI;AAClC,QAAI;AAEF,aAAO,OAAO,UAAU,IAAS,OAAO,OAAO,GAAG,CAAC;AAAA,IACrD,UAAA;AACE,WAAK,GAAG,eAAe,OAAO,MAAM;AAAA,IACtC;AAAA,EACF;AAAA,EAEA,QAAQ,OAA8B,MAAoB,KAAe;AACvE,UAAM,QAAQ,KAAK,IAAI,CAAA,QAAO,IAAI,IAAI,CAAA,MAAK,GAAG,GAAG,CAAC,CAAC,IAAI,CAAC;AACxD,UAAM,OAAO,OAAO,KAAK,MAAM,OAAO;AACtC,UAAM,SAAS,KAAK,GAAG,eAAe;AAAA,MACpC,UAAU,KAAK,IAAI,CAAA,MAAK,GAAG,CAAC,CAAC,EAAE,KAAK,GAAG,CAAC,SAAS;AAAA,QAC/C,MAAM;AAAA,MAAA,CACP,UAAU,MAAM,IAAI,CAAA,SAAQ,KAAK,KAAK,OAAO,CAAC,EAAE,KAAK,MAAM,CAAC;AAAA,IAAA;AAE/D,WAAO,UAAU,aAAa,IAAI;AAClC,QAAI;AAEF,aAAO,OAAO,UAAU;AAAA,QACtB,KAAK,QAAQ,CAAA,QAAO,IAAI,IAAI,CAAA,WAAU,IAAI,MAAM,CAAC,CAAC;AAAA,MAAA;AAAA,IAEtD,UAAA;AACE,WAAK,GAAG,eAAe,OAAO,MAAM;AAAA,IACtC;AAAA,EACF;AAAA,EAEA,cAAwB;AACtB,SAAK,GAAG,SAAA;AACR,WAAO,IAAI,SAAS,KAAK,IAAI,KAAK,MAAM;AAAA,EAC1C;AACF;AAEA,MAAM,KAA6B;AAAA,EACxB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAET,YACE,OACA,QACA,MACA,MACA;AACA,SAAK,oBAAoB,GAAG,KAAK;AACjC,SAAK,SAAS;AACd,SAAK,OAAO;AACZ,SAAK,OAAO;AACZ,SAAK,UAAU,KAAK,gBAAgB,KAAK,OAAO;AAAA,EAClD;AAAA,EAEA,CAAC,OAAO,QAAQ,IAAsB;AACpC,UAAM,EAAC,SAAS,SAAS,SAAQ,KAAK,KAAK,aAAa,KAAK,KAAK,OAAO;AAEzE,UAAM,UAAU,MAAM;AACpB,UAAI;AAEF,gBAAQ,SAAS,MAAS;AAAA,MAC5B,UAAA;AACE,aAAA;AAAA,MACF;AAAA,IACF;AAEA,WAAO;AAAA,MACL,MAAM,MAAM;AACV,YAAI;AACF,qBAAS;AACP,kBAAM,EAAC,OAAO,MAAAA,MAAAA,IAAQ,QAAQ,KAAA;AAC9B,gBAAIA,OAAM;AACR,sBAAA;AACA,qBAAO,EAAC,OAAO,MAAM,KAAA;AAAA,YACvB;AAEA,kBAAM,EAAC,OAAO,QAAQ,IAAI,iBAAgBC,MAAQ,OAAOC,oBAAM;AAC/D,gBAAI,OAAO,UAAU;AAEnB,oBAAM,IAAI;AAAA,gBACR,oBAAoB,KAAK;AAAA,cAAA;AAAA,YAE7B;AACA,gBAAI,OAAO,aAAa;AAEtB,oBAAM,IAAI;AAAA,gBACR,SAAS,KAAK;AAAA,cAAA;AAAA,YAElB;AACA,kBAAM,EAAC,WAAW,YAAW,KAAK,KAAK,OAAO,IAAI,KAAK,CAAC;AAExD,mBAAO,WAAW,IAAI;AACtB,kBAAM,YACJ,OAAO,SAAS,KAAK,KAAK,OAAO,WAAW,MAAM,IAAI;AACxD,gBAAI;AACJ,gBAAI,WAAW;AACb,2BAAa,KAAK,KAAK;AAAA,gBACrB;AAAA,gBACA,UAAU;AAAA,gBACV;AAAA,cAAA;AAAA,YAEJ,OAAO;AACL,oBAAM,YAAY,KAAK,KAAK,OAAO,WAAW,MAAM;AACpD,2BAAa,YAAY,CAAC,SAAS,IAAI,CAAA;AAAA,YACzC;AACA,gBAAI,cAAc,QAAW;AAC3B,oBAAM,IAAI;AAAA,gBACR,qBAAqB,KAAK,IAAI,UAAU,MAAM,CAAC;AAAA,cAAA;AAAA,YAEnD;AAEA,iBAAK,qBAAqB,cAAc,IAAI,YAAY,SAAS;AAEjE,gBAAI,WAAW,WAAW,KAAK,cAAc,MAAM;AAGjD;AAAA,YACF;AAEA,gBACE,UAAU,KAAK,qBACf,WAAW;AAAA,cACT,CAAA,cAAa,UAAU,gBAAgB,UAAU;AAAA,YAAA,GAEnD;AACA,oBAAM,IAAI;AAAA,gBACR,4BACE,WAAW;AAAA,kBACT,CAAA,cACE,UAAU,gBAAgB,UAAU;AAAA,gBAAA,EACtC,IACJ,OAAO,UAAU,IAAI;AAAA,cAAA;AAAA,YAEzB;AAKA,mBAAO;AAAA,cACL,OAAO;AAAA,gBACL;AAAA,gBACA,YAAY,WAAW;AAAA,kBAAI,CAAA,cACzB,gBAAgB,SAAS,WAAW,KAAK;AAAA,gBAAA;AAAA,gBAE3C,WAAW,YACP,gBAAgB,SAAS,WAAW,KAAK,IACzC;AAAA,gBACJ;AAAA,cAAA;AAAA,YACF;AAAA,UAEJ;AAAA,QACF,SAAS,GAAG;AAEV,kBAAA;AACA,gBAAM;AAAA,QACR;AAAA,MACF;AAAA,MAEA,QAAQ,CAAC,UAAmB;AAC1B,gBAAA;AACA,eAAO,EAAC,OAAO,MAAM,KAAA;AAAA,MACvB;AAAA,IAAA;AAAA,EAEJ;AAAA,EAEA,qBACE,cACA,IACA,YACA,WACA;AAGA,QAAI,eAAe,KAAK,KAAK,SAAS;AACpC,YAAM,IAAI;AAAA,QACR,sDAAsD,KAAK,KAAK,OAAO;AAAA,MAAA;AAAA,IAE3E;AACA,QACE,WAAW;AAAA,MACT,gBAAc,UAAUC,wBAAW,KAAK,OAAO,KAAK,KAAK;AAAA,IAAA,MACrD,IACN;AACA,YAAM,IAAI;AAAA,QACR,sDAAsD,KAAK,KAAK,OAAO;AAAA,MAAA;AAAA,IAE3E;AACA,QAAI,OAAO,UAAU,UAAUA,wBAAW,MAAM,cAAc;AAC5D,YAAM,IAAI;AAAA,QACR;AAAA,MAAA;AAAA,IAEJ;AAAA,EACF;AACF;AAEO,MAAM,yBAAyB,MAAM;AAAA,EAC1C,YAAY,KAAa;AACvB,UAAM,GAAG;AAAA,EACX;AACF;"}
+{"version":3,"file":"snapshotter.js","sources":["../../../../../../zero-cache/src/services/view-syncer/snapshotter.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport {stringify, type JSONValue} from '../../../../shared/src/bigint-json.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport type {Row} from '../../../../zero-protocol/src/data.ts';\nimport type {PrimaryKey} from '../../../../zero-types/src/schema.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {fromSQLiteTypes} from '../../../../zqlite/src/table-source.ts';\nimport type {LiteAndZqlSpec, LiteTableSpecWithKeys} from '../../db/specs.ts';\nimport {StatementRunner} from '../../db/statements.ts';\nimport {\n  normalizedKeyOrder,\n  type RowKey,\n  type RowValue,\n} from '../../types/row-key.ts';\nimport {type SchemaVersions} from '../../types/schema-versions.ts';\nimport type {AppID} from '../../types/shards.ts';\nimport {id} from '../../types/sql.ts';\nimport {\n  RESET_OP,\n  changeLogEntrySchema as schema,\n  SET_OP,\n  TRUNCATE_OP,\n} from '../replicator/schema/change-log.ts';\nimport {\n  getReplicationState,\n  ZERO_VERSION_COLUMN_NAME as ROW_VERSION,\n} from '../replicator/schema/replication-state.ts';\n\n/**\n * A `Snapshotter` manages the progression of database snapshots for a\n * ViewSyncer.\n *\n * The Replicator and ViewSyncers operate on the same SQLite file, with the\n * Replicator being the sole writer to the database. The IVM logic in\n * ViewSyncers, however, rely on incrementally applying changes to the DB to\n * update the state of its pipelines.\n *\n * To avoid coupling the progress of the Replicator and all IVM pipelines on\n * each other, ViewSyncers operate on ephemeral forks of the database by holding\n * [concurrent](https://sqlite.org/src/doc/begin-concurrent/doc/begin_concurrent.md)\n * snapshots of the database and simulating (but ultimately rolling back)\n * mutations on these snapshots.\n *\n * Example:\n * 1. ViewSyncer takes `snapshot_a` at version `t1` of the database and\n *    hydrates its pipeline(s).\n * 2. Replicator applies a new transaction to the database and notifies\n *    subscribers.\n * 3. ViewSyncer takes `snapshot_b` at `t2`, and queries the `ChangeLog` at\n *    that snapshot for changes since `t1`.\n * 4. ViewSyncer applies those changes to `snapshot_a` for IVM, but does not\n *    commit them. (Recall that the Replicator is the sole writer to the db, so\n *    the ViewSyncer never commits any writes.)\n * 5. Replicator applies the next transaction and advances the database to `t3`.\n * 6. ViewSyncer rolls back `snapshot_a` and opens `snapshot_c` at `t3`, using\n *    `snapshot_b` to simulate changes from `t2` to `t3`.\n *\n * ```\n * Replicator:  t1 --------------> t2 --------------> t3 --------------->\n * ViewSyncer:       [snapshot_a] ----> [snapshot_b] ----> [snapshot_c]\n * ```\n *\n * Note that the Replicator (and ViewSyncers) do not wait on the progress of\n * other ViewSyncers. If a ViewSyncer is busy hydrating at `t1`, the Replicator\n * and other ViewSyncers can progress through `t2`, `t3`, etc. independently,\n * as the busy ViewSyncer simply takes its own snapshot when it is ready.\n *\n * ```\n * Replicator:   t1 --------------> t2 --------------> t3 --------------->\n * ViewSyncer1:       [snapshot_a] ----> [snapshot_b] ----> [snapshot_c]\n * ViewSyncer2:       [.......... snapshot_a ..........] ----> [snapshot_b]\n * ```\n *\n * To minimize Database connections (and statement preparation, etc.), the\n * Snapshotter reuses the connection from the previous (rolled back)\n * snapshot when opening the new one.\n *\n * ```\n * Replicator:  t1 --------------> t2 --------------> t3 --------------->\n * ViewSyncer:       [snapshot_a] ----> [snapshot_b] ----> [snapshot_c]\n *                     (conn_1)           (conn_2)           (conn_1)\n * ```\n *\n * In this manner, each ViewSyncer uses two connections that continually\n * \"leapfrog\" each other to replay the timeline of changes in isolation from\n * the Replicator and other ViewSyncers.\n */\nexport class Snapshotter {\n  readonly #lc: LogContext;\n  readonly #dbFile: string;\n  readonly #appID: string;\n  readonly #pageCacheSizeKib: number | undefined;\n  #curr: Snapshot | undefined;\n  #prev: Snapshot | undefined;\n\n  constructor(\n    lc: LogContext,\n    dbFile: string,\n    {appID}: AppID,\n    pageCacheSizeKib?: number | undefined,\n  ) {\n    this.#lc = lc;\n    this.#dbFile = dbFile;\n    this.#appID = appID;\n    this.#pageCacheSizeKib = pageCacheSizeKib;\n  }\n\n  /**\n   * Initializes the snapshot to the current head of the database. This must be\n   * only be called once. The state of whether a Snapshotter has been initialized\n   * can be determined by calling {@link initialized()}.\n   */\n  init(): this {\n    assert(this.#curr === undefined, 'Already initialized');\n    this.#curr = Snapshot.create(\n      this.#lc,\n      this.#dbFile,\n      this.#appID,\n      this.#pageCacheSizeKib,\n    );\n    this.#lc.debug?.(`Initial snapshot at version ${this.#curr.version}`);\n    return this;\n  }\n\n  initialized(): boolean {\n    return this.#curr !== undefined;\n  }\n\n  /** Returns the current snapshot. Asserts if {@link initialized()} is false. */\n  current(): Snapshot {\n    assert(this.#curr !== undefined, 'Snapshotter has not been initialized');\n    return this.#curr;\n  }\n\n  /**\n   * Advances to the head of the Database, returning a diff between the\n   * previously current Snapshot and a new Snapshot at head. This is called\n   * in response to a notification from a Replicator subscription. Subsequent\n   * calls to {@link current()} return the new Snapshot. Note that the Snapshotter\n   * must be initialized before advancing.\n   *\n   * The returned {@link SnapshotDiff} contains snapshots at the endpoints\n   * of the database timeline. Iterating over the diff generates a sequence\n   * of {@link Change}s between the two snapshots.\n   *\n   * Note that this sequence is not chronological; rather, the sequence is\n   * ordered by `<table, row-key>`, such that a row can appear at most once\n   * in the common case, or twice if its table is `TRUNCATE`'d and a new value\n   * is subsequently `INSERT`'ed. This results in dropping most intermediate\n   * changes to a row and bounds the amount of work needed to catch up;\n   * however, as a consequence, a consistent database state is only guaranteed\n   * when the sequence has been fully consumed.\n   *\n   * Note that Change generation relies on the state of the underlying\n   * database connections, and because the connection for the previous snapshot\n   * is reused to produce the next snapshot, the diff object is only valid\n   * until the next call to `advance()`.\n   *\n   * It is okay for the caller to apply `Change`s to the `prev` snapshot\n   * during the iteration (e.g. this is necessary for IVM); the remainder\n   * of the iteration is not affected because a given row can appear at most\n   * once in the sequence (with the exception being TRUNCATE, after which the\n   * deleted rows can be re-inserted, but this will also behave correctly if\n   * the changes are applied).\n   *\n   * Once the changes have been applied, however, a _subsequent_ iteration\n   * will not produce the correct results. In order to perform multiple\n   * change-applying iterations, the caller must (1) create a save point\n   * on `prev` before each iteration, and (2) rollback to the save point after\n   * the iteration.\n   */\n  advance(tables: Map<string, LiteAndZqlSpec>): SnapshotDiff {\n    const {prev, curr} = this.advanceWithoutDiff();\n    return new Diff(this.#appID, tables, prev, curr);\n  }\n\n  advanceWithoutDiff() {\n    assert(this.#curr !== undefined, 'Snapshotter has not been initialized');\n    const next = this.#prev\n      ? this.#prev.resetToHead()\n      : Snapshot.create(\n          this.#lc,\n          this.#curr.db.db.name,\n          this.#appID,\n          this.#pageCacheSizeKib,\n        );\n    this.#prev = this.#curr;\n    this.#curr = next;\n    return {prev: this.#prev, curr: this.#curr};\n  }\n\n  /**\n   * Call this to close the database connections when the Snapshotter is\n   * no longer needed.\n   */\n  destroy() {\n    this.#curr?.db.db.close();\n    this.#prev?.db.db.close();\n    this.#lc.debug?.('closed database connections');\n  }\n}\n\nexport type Change = {\n  readonly table: string;\n  /**\n   * If this change represents a remove the row to remove,\n   * if nextValue is not null then all rows that have a unique constraint\n   * violation with nextValue.\n   * In both cases these rows should be removed.\n   */\n  readonly prevValues: Readonly<Row>[];\n  readonly nextValue: Readonly<Row> | null;\n  readonly rowKey: RowKey;\n};\n\n/**\n * Represents the difference between two database Snapshots.\n * Iterating over the object will produce a sequence of {@link Change}s\n * between the two snapshots.\n *\n * See {@link Snapshotter.advance()} for semantics and usage.\n */\nexport interface SnapshotDiff extends Iterable<Change> {\n  readonly prev: {\n    readonly db: StatementRunner;\n    readonly version: string;\n  };\n  readonly curr: {\n    readonly db: StatementRunner;\n    readonly version: string;\n  };\n\n  /**\n   * The number of ChangeLog entries between the snapshots. Note that this\n   * may not necessarily equal the number of `Change` objects that the iteration\n   * will produce, as `TRUNCATE` entries are counted as a single log entry which\n   * may be expanded into many changes (i.e. row deletes).\n   *\n   * TODO: Determine if it is worth changing the definition to count the\n   *       truncated rows. This would make diff computation more expensive\n   *       (requiring the count to be aggregated by operation type), which\n   *       may not be worth it for a presumable rare operation.\n   */\n  readonly changes: number;\n}\n\n/**\n * Thrown during an iteration of a {@link SnapshotDiff} when a schema\n * change or truncate is encountered, which result in aborting the\n * advancement and resetting / rehydrating the pipelines.\n */\nexport class ResetPipelinesSignal extends Error {\n  readonly name = 'ResetPipelinesSignal';\n\n  constructor(msg: string) {\n    super(msg);\n  }\n}\n\nfunction getSchemaVersions(db: StatementRunner, appID: string): SchemaVersions {\n  return db.get(\n    `SELECT minSupportedVersion, maxSupportedVersion FROM \"${appID}.schemaVersions\"`,\n  );\n}\n\nclass Snapshot {\n  static create(\n    lc: LogContext,\n    dbFile: string,\n    appID: string,\n    pageCacheSizeKib: number | undefined,\n  ) {\n    const conn = new Database(lc, dbFile);\n    conn.pragma('synchronous = OFF'); // Applied changes are ephemeral; COMMIT is never called.\n    if (pageCacheSizeKib !== undefined) {\n      conn.pragma(`cache_size = -${pageCacheSizeKib}`); // Negative = size in KiB\n    }\n    const [{journal_mode: mode}] = conn.pragma('journal_mode') as [\n      {journal_mode: string},\n    ];\n    // The Snapshotter operates on the replica file with BEGIN CONCURRENT,\n    // which must be used in concert with the replicator using BEGIN CONCURRENT\n    // on a db in the wal2 journal_mode.\n    assert(\n      mode === 'wal2',\n      `replica db must be in wal2 mode (current: ${mode})`,\n    );\n\n    const db = new StatementRunner(conn);\n    return new Snapshot(db, appID);\n  }\n\n  readonly db: StatementRunner;\n  readonly #appID: string;\n  readonly version: string;\n  readonly schemaVersions: SchemaVersions;\n\n  constructor(db: StatementRunner, appID: string) {\n    db.beginConcurrent();\n    // Note: The subsequent read is necessary to acquire the read lock\n    // (which results in the logical creation of the snapshot). Calling\n    // `BEGIN CONCURRENT` alone does not result in acquiring the read lock.\n    const {stateVersion} = getReplicationState(db);\n\n    this.db = db;\n    this.#appID = appID;\n    this.version = stateVersion;\n    this.schemaVersions = getSchemaVersions(db, appID);\n  }\n\n  numChangesSince(prevVersion: string) {\n    const {count} = this.db.get(\n      'SELECT COUNT(*) AS count FROM \"_zero.changeLog\" WHERE stateVersion > ?',\n      prevVersion,\n    );\n    return count;\n  }\n\n  changesSince(prevVersion: string) {\n    const cached = this.db.statementCache.get(\n      'SELECT * FROM \"_zero.changeLog\" WHERE stateVersion > ?',\n    );\n    return {\n      changes: cached.statement.iterate(prevVersion),\n      cleanup: () => this.db.statementCache.return(cached),\n    };\n  }\n\n  getRow(table: LiteTableSpecWithKeys, rowKey: JSONValue) {\n    const key = normalizedKeyOrder(rowKey as RowKey);\n    const conds = Object.keys(key).map(c => `${id(c)}=?`);\n    const cols = Object.keys(table.columns);\n    const cached = this.db.statementCache.get(\n      `SELECT ${cols.map(c => id(c)).join(',')} FROM ${id(\n        table.name,\n      )} WHERE ${conds.join(' AND ')}`,\n    );\n    cached.statement.safeIntegers(true);\n    try {\n      // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n      return cached.statement.get<any>(Object.values(key));\n    } finally {\n      this.db.statementCache.return(cached);\n    }\n  }\n\n  getRows(table: LiteTableSpecWithKeys, keys: PrimaryKey[], row: RowValue) {\n    const conds = keys.map(key => key.map(c => `${id(c)}=?`));\n    const cols = Object.keys(table.columns);\n    const cached = this.db.statementCache.get(\n      `SELECT ${cols.map(c => id(c)).join(',')} FROM ${id(\n        table.name,\n      )} WHERE ${conds.map(cond => cond.join(' AND ')).join(' OR ')}`,\n    );\n    cached.statement.safeIntegers(true);\n    try {\n      // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n      return cached.statement.all<any>(\n        keys.flatMap(key => key.map(column => row[column])),\n      );\n    } finally {\n      this.db.statementCache.return(cached);\n    }\n  }\n\n  resetToHead(): Snapshot {\n    this.db.rollback();\n    return new Snapshot(this.db, this.#appID);\n  }\n}\n\nclass Diff implements SnapshotDiff {\n  readonly #permissionsTable: string;\n  readonly tables: Map<string, LiteAndZqlSpec>;\n  readonly prev: Snapshot;\n  readonly curr: Snapshot;\n  readonly changes: number;\n\n  constructor(\n    appID: string,\n    tables: Map<string, LiteAndZqlSpec>,\n    prev: Snapshot,\n    curr: Snapshot,\n  ) {\n    this.#permissionsTable = `${appID}.permissions`;\n    this.tables = tables;\n    this.prev = prev;\n    this.curr = curr;\n    this.changes = curr.numChangesSince(prev.version);\n  }\n\n  [Symbol.iterator](): Iterator<Change> {\n    const {changes, cleanup: done} = this.curr.changesSince(this.prev.version);\n\n    const cleanup = () => {\n      try {\n        // Allow open iterators to clean up their state.\n        changes.return?.(undefined);\n      } finally {\n        done();\n      }\n    };\n\n    return {\n      next: () => {\n        try {\n          for (;;) {\n            const {value, done} = changes.next();\n            if (done) {\n              cleanup();\n              return {value, done: true};\n            }\n\n            const {table, rowKey, op, stateVersion} = v.parse(value, schema);\n            if (op === RESET_OP) {\n              // The current map of `TableSpec`s may not have the correct or complete information.\n              throw new ResetPipelinesSignal(\n                `schema for table ${table} has changed`,\n              );\n            }\n            if (op === TRUNCATE_OP) {\n              // Truncates are also processed by rehydrating pipelines at current.\n              throw new ResetPipelinesSignal(\n                `table ${table} has been truncated`,\n              );\n            }\n            const {tableSpec, zqlSpec} = must(this.tables.get(table));\n\n            assert(rowKey !== null);\n            const nextValue =\n              op === SET_OP ? this.curr.getRow(tableSpec, rowKey) : null;\n            let prevValues;\n            if (nextValue) {\n              prevValues = this.prev.getRows(\n                tableSpec,\n                tableSpec.uniqueKeys,\n                nextValue,\n              );\n            } else {\n              const prevValue = this.prev.getRow(tableSpec, rowKey);\n              prevValues = prevValue ? [prevValue] : [];\n            }\n            if (nextValue === undefined) {\n              throw new Error(\n                `Missing value for ${table} ${stringify(rowKey)}`,\n              );\n            }\n            // Sanity check detects if the diff is being accessed after the Snapshots have advanced.\n            this.checkThatDiffIsValid(stateVersion, op, prevValues, nextValue);\n\n            if (prevValues.length === 0 && nextValue === null) {\n              // Filter out no-op changes (e.g. a delete of a row that does not exist in prev).\n              // TODO: Consider doing this for deep-equal values.\n              continue;\n            }\n\n            if (\n              table === this.#permissionsTable &&\n              prevValues.find(\n                prevValue => prevValue.permissions !== nextValue.permissions,\n              )\n            ) {\n              throw new ResetPipelinesSignal(\n                `Permissions have changed ${\n                  prevValues.find(\n                    prevValue =>\n                      prevValue.permissions !== nextValue.permissions,\n                  ).hash\n                } => ${nextValue.hash}`,\n              );\n            }\n\n            // Modify the values in place when converting to ZQL rows\n            // This is safe since we're the first node in the iterator chain.\n            // TODO Can we get rid of these RowValue casts?\n            return {\n              value: {\n                table,\n                prevValues: prevValues.map(prevValue =>\n                  fromSQLiteTypes(zqlSpec, prevValue, table),\n                ),\n                nextValue: nextValue\n                  ? fromSQLiteTypes(zqlSpec, nextValue, table)\n                  : null,\n                rowKey,\n              } satisfies Change,\n            };\n          }\n        } catch (e) {\n          // This control flow path is not covered by the return() method (i.e. `break`).\n          cleanup();\n          throw e;\n        }\n      },\n\n      return: (value: unknown) => {\n        cleanup();\n        return {value, done: true};\n      },\n    };\n  }\n\n  checkThatDiffIsValid(\n    stateVersion: string,\n    op: string,\n    prevValues: RowValue[],\n    nextValue: RowValue,\n  ) {\n    // Sanity checks to detect that the diff is not being accessed after\n    // the Snapshots have advanced.\n    if (stateVersion > this.curr.version) {\n      throw new InvalidDiffError(\n        `Diff is no longer valid. curr db has advanced past ${this.curr.version}`,\n      );\n    }\n    if (\n      prevValues.findIndex(\n        prevValue => (prevValue[ROW_VERSION] ?? '~') > this.prev.version,\n      ) !== -1\n    ) {\n      throw new InvalidDiffError(\n        `Diff is no longer valid. prev db has advanced past ${this.prev.version}.`,\n      );\n    }\n    if (op === SET_OP && nextValue[ROW_VERSION] !== stateVersion) {\n      throw new InvalidDiffError(\n        'Diff is no longer valid. curr db has advanced.',\n      );\n    }\n  }\n}\n\nexport class InvalidDiffError extends Error {\n  constructor(msg: string) {\n    super(msg);\n  }\n}\n"],"names":["done","v.parse","schema","ROW_VERSION"],"mappings":";;;;;;;;;;;;;AAyFO,MAAM,YAAY;AAAA,EACd;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACT;AAAA,EACA;AAAA,EAEA,YACE,IACA,QACA,EAAC,MAAA,GACD,kBACA;AACA,SAAK,MAAM;AACX,SAAK,UAAU;AACf,SAAK,SAAS;AACd,SAAK,oBAAoB;AAAA,EAC3B;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,OAAa;AACX,WAAO,KAAK,UAAU,QAAW,qBAAqB;AACtD,SAAK,QAAQ,SAAS;AAAA,MACpB,KAAK;AAAA,MACL,KAAK;AAAA,MACL,KAAK;AAAA,MACL,KAAK;AAAA,IAAA;AAEP,SAAK,IAAI,QAAQ,+BAA+B,KAAK,MAAM,OAAO,EAAE;AACpE,WAAO;AAAA,EACT;AAAA,EAEA,cAAuB;AACrB,WAAO,KAAK,UAAU;AAAA,EACxB;AAAA;AAAA,EAGA,UAAoB;AAClB,WAAO,KAAK,UAAU,QAAW,sCAAsC;AACvE,WAAO,KAAK;AAAA,EACd;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAuCA,QAAQ,QAAmD;AACzD,UAAM,EAAC,MAAM,SAAQ,KAAK,mBAAA;AAC1B,WAAO,IAAI,KAAK,KAAK,QAAQ,QAAQ,MAAM,IAAI;AAAA,EACjD;AAAA,EAEA,qBAAqB;AACnB,WAAO,KAAK,UAAU,QAAW,sCAAsC;AACvE,UAAM,OAAO,KAAK,QACd,KAAK,MAAM,YAAA,IACX,SAAS;AAAA,MACP,KAAK;AAAA,MACL,KAAK,MAAM,GAAG,GAAG;AAAA,MACjB,KAAK;AAAA,MACL,KAAK;AAAA,IAAA;AAEX,SAAK,QAAQ,KAAK;AAClB,SAAK,QAAQ;AACb,WAAO,EAAC,MAAM,KAAK,OAAO,MAAM,KAAK,MAAA;AAAA,EACvC;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,UAAU;AACR,SAAK,OAAO,GAAG,GAAG,MAAA;AAClB,SAAK,OAAO,GAAG,GAAG,MAAA;AAClB,SAAK,IAAI,QAAQ,6BAA6B;AAAA,EAChD;AACF;AAmDO,MAAM,6BAA6B,MAAM;AAAA,EACrC,OAAO;AAAA,EAEhB,YAAY,KAAa;AACvB,UAAM,GAAG;AAAA,EACX;AACF;AAEA,SAAS,kBAAkB,IAAqB,OAA+B;AAC7E,SAAO,GAAG;AAAA,IACR,yDAAyD,KAAK;AAAA,EAAA;AAElE;AAEA,MAAM,SAAS;AAAA,EACb,OAAO,OACL,IACA,QACA,OACA,kBACA;AACA,UAAM,OAAO,IAAI,SAAS,IAAI,MAAM;AACpC,SAAK,OAAO,mBAAmB;AAC/B,QAAI,qBAAqB,QAAW;AAClC,WAAK,OAAO,iBAAiB,gBAAgB,EAAE;AAAA,IACjD;AACA,UAAM,CAAC,EAAC,cAAc,KAAA,CAAK,IAAI,KAAK,OAAO,cAAc;AAMzD;AAAA,MACE,SAAS;AAAA,MACT,6CAA6C,IAAI;AAAA,IAAA;AAGnD,UAAM,KAAK,IAAI,gBAAgB,IAAI;AACnC,WAAO,IAAI,SAAS,IAAI,KAAK;AAAA,EAC/B;AAAA,EAES;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAET,YAAY,IAAqB,OAAe;AAC9C,OAAG,gBAAA;AAIH,UAAM,EAAC,aAAA,IAAgB,oBAAoB,EAAE;AAE7C,SAAK,KAAK;AACV,SAAK,SAAS;AACd,SAAK,UAAU;AACf,SAAK,iBAAiB,kBAAkB,IAAI,KAAK;AAAA,EACnD;AAAA,EAEA,gBAAgB,aAAqB;AACnC,UAAM,EAAC,MAAA,IAAS,KAAK,GAAG;AAAA,MACtB;AAAA,MACA;AAAA,IAAA;AAEF,WAAO;AAAA,EACT;AAAA,EAEA,aAAa,aAAqB;AAChC,UAAM,SAAS,KAAK,GAAG,eAAe;AAAA,MACpC;AAAA,IAAA;AAEF,WAAO;AAAA,MACL,SAAS,OAAO,UAAU,QAAQ,WAAW;AAAA,MAC7C,SAAS,MAAM,KAAK,GAAG,eAAe,OAAO,MAAM;AAAA,IAAA;AAAA,EAEvD;AAAA,EAEA,OAAO,OAA8B,QAAmB;AACtD,UAAM,MAAM,mBAAmB,MAAgB;AAC/C,UAAM,QAAQ,OAAO,KAAK,GAAG,EAAE,IAAI,CAAA,MAAK,GAAG,GAAG,CAAC,CAAC,IAAI;AACpD,UAAM,OAAO,OAAO,KAAK,MAAM,OAAO;AACtC,UAAM,SAAS,KAAK,GAAG,eAAe;AAAA,MACpC,UAAU,KAAK,IAAI,CAAA,MAAK,GAAG,CAAC,CAAC,EAAE,KAAK,GAAG,CAAC,SAAS;AAAA,QAC/C,MAAM;AAAA,MAAA,CACP,UAAU,MAAM,KAAK,OAAO,CAAC;AAAA,IAAA;AAEhC,WAAO,UAAU,aAAa,IAAI;AAClC,QAAI;AAEF,aAAO,OAAO,UAAU,IAAS,OAAO,OAAO,GAAG,CAAC;AAAA,IACrD,UAAA;AACE,WAAK,GAAG,eAAe,OAAO,MAAM;AAAA,IACtC;AAAA,EACF;AAAA,EAEA,QAAQ,OAA8B,MAAoB,KAAe;AACvE,UAAM,QAAQ,KAAK,IAAI,CAAA,QAAO,IAAI,IAAI,CAAA,MAAK,GAAG,GAAG,CAAC,CAAC,IAAI,CAAC;AACxD,UAAM,OAAO,OAAO,KAAK,MAAM,OAAO;AACtC,UAAM,SAAS,KAAK,GAAG,eAAe;AAAA,MACpC,UAAU,KAAK,IAAI,CAAA,MAAK,GAAG,CAAC,CAAC,EAAE,KAAK,GAAG,CAAC,SAAS;AAAA,QAC/C,MAAM;AAAA,MAAA,CACP,UAAU,MAAM,IAAI,CAAA,SAAQ,KAAK,KAAK,OAAO,CAAC,EAAE,KAAK,MAAM,CAAC;AAAA,IAAA;AAE/D,WAAO,UAAU,aAAa,IAAI;AAClC,QAAI;AAEF,aAAO,OAAO,UAAU;AAAA,QACtB,KAAK,QAAQ,CAAA,QAAO,IAAI,IAAI,CAAA,WAAU,IAAI,MAAM,CAAC,CAAC;AAAA,MAAA;AAAA,IAEtD,UAAA;AACE,WAAK,GAAG,eAAe,OAAO,MAAM;AAAA,IACtC;AAAA,EACF;AAAA,EAEA,cAAwB;AACtB,SAAK,GAAG,SAAA;AACR,WAAO,IAAI,SAAS,KAAK,IAAI,KAAK,MAAM;AAAA,EAC1C;AACF;AAEA,MAAM,KAA6B;AAAA,EACxB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAET,YACE,OACA,QACA,MACA,MACA;AACA,SAAK,oBAAoB,GAAG,KAAK;AACjC,SAAK,SAAS;AACd,SAAK,OAAO;AACZ,SAAK,OAAO;AACZ,SAAK,UAAU,KAAK,gBAAgB,KAAK,OAAO;AAAA,EAClD;AAAA,EAEA,CAAC,OAAO,QAAQ,IAAsB;AACpC,UAAM,EAAC,SAAS,SAAS,SAAQ,KAAK,KAAK,aAAa,KAAK,KAAK,OAAO;AAEzE,UAAM,UAAU,MAAM;AACpB,UAAI;AAEF,gBAAQ,SAAS,MAAS;AAAA,MAC5B,UAAA;AACE,aAAA;AAAA,MACF;AAAA,IACF;AAEA,WAAO;AAAA,MACL,MAAM,MAAM;AACV,YAAI;AACF,qBAAS;AACP,kBAAM,EAAC,OAAO,MAAAA,MAAAA,IAAQ,QAAQ,KAAA;AAC9B,gBAAIA,OAAM;AACR,sBAAA;AACA,qBAAO,EAAC,OAAO,MAAM,KAAA;AAAA,YACvB;AAEA,kBAAM,EAAC,OAAO,QAAQ,IAAI,iBAAgBC,MAAQ,OAAOC,oBAAM;AAC/D,gBAAI,OAAO,UAAU;AAEnB,oBAAM,IAAI;AAAA,gBACR,oBAAoB,KAAK;AAAA,cAAA;AAAA,YAE7B;AACA,gBAAI,OAAO,aAAa;AAEtB,oBAAM,IAAI;AAAA,gBACR,SAAS,KAAK;AAAA,cAAA;AAAA,YAElB;AACA,kBAAM,EAAC,WAAW,YAAW,KAAK,KAAK,OAAO,IAAI,KAAK,CAAC;AAExD,mBAAO,WAAW,IAAI;AACtB,kBAAM,YACJ,OAAO,SAAS,KAAK,KAAK,OAAO,WAAW,MAAM,IAAI;AACxD,gBAAI;AACJ,gBAAI,WAAW;AACb,2BAAa,KAAK,KAAK;AAAA,gBACrB;AAAA,gBACA,UAAU;AAAA,gBACV;AAAA,cAAA;AAAA,YAEJ,OAAO;AACL,oBAAM,YAAY,KAAK,KAAK,OAAO,WAAW,MAAM;AACpD,2BAAa,YAAY,CAAC,SAAS,IAAI,CAAA;AAAA,YACzC;AACA,gBAAI,cAAc,QAAW;AAC3B,oBAAM,IAAI;AAAA,gBACR,qBAAqB,KAAK,IAAI,UAAU,MAAM,CAAC;AAAA,cAAA;AAAA,YAEnD;AAEA,iBAAK,qBAAqB,cAAc,IAAI,YAAY,SAAS;AAEjE,gBAAI,WAAW,WAAW,KAAK,cAAc,MAAM;AAGjD;AAAA,YACF;AAEA,gBACE,UAAU,KAAK,qBACf,WAAW;AAAA,cACT,CAAA,cAAa,UAAU,gBAAgB,UAAU;AAAA,YAAA,GAEnD;AACA,oBAAM,IAAI;AAAA,gBACR,4BACE,WAAW;AAAA,kBACT,CAAA,cACE,UAAU,gBAAgB,UAAU;AAAA,gBAAA,EACtC,IACJ,OAAO,UAAU,IAAI;AAAA,cAAA;AAAA,YAEzB;AAKA,mBAAO;AAAA,cACL,OAAO;AAAA,gBACL;AAAA,gBACA,YAAY,WAAW;AAAA,kBAAI,CAAA,cACzB,gBAAgB,SAAS,WAAW,KAAK;AAAA,gBAAA;AAAA,gBAE3C,WAAW,YACP,gBAAgB,SAAS,WAAW,KAAK,IACzC;AAAA,gBACJ;AAAA,cAAA;AAAA,YACF;AAAA,UAEJ;AAAA,QACF,SAAS,GAAG;AAEV,kBAAA;AACA,gBAAM;AAAA,QACR;AAAA,MACF;AAAA,MAEA,QAAQ,CAAC,UAAmB;AAC1B,gBAAA;AACA,eAAO,EAAC,OAAO,MAAM,KAAA;AAAA,MACvB;AAAA,IAAA;AAAA,EAEJ;AAAA,EAEA,qBACE,cACA,IACA,YACA,WACA;AAGA,QAAI,eAAe,KAAK,KAAK,SAAS;AACpC,YAAM,IAAI;AAAA,QACR,sDAAsD,KAAK,KAAK,OAAO;AAAA,MAAA;AAAA,IAE3E;AACA,QACE,WAAW;AAAA,MACT,gBAAc,UAAUC,wBAAW,KAAK,OAAO,KAAK,KAAK;AAAA,IAAA,MACrD,IACN;AACA,YAAM,IAAI;AAAA,QACR,sDAAsD,KAAK,KAAK,OAAO;AAAA,MAAA;AAAA,IAE3E;AACA,QAAI,OAAO,UAAU,UAAUA,wBAAW,MAAM,cAAc;AAC5D,YAAM,IAAI;AAAA,QACR;AAAA,MAAA;AAAA,IAEJ;AAAA,EACF;AACF;AAEO,MAAM,yBAAyB,MAAM;AAAA,EAC1C,YAAY,KAAa;AACvB,UAAM,GAAG;AAAA,EACX;AACF;"}

package/out/zero-cache/src/services/view-syncer/view-syncer.d.ts

@@ -24,6 +24,7 @@ export type TokenData = {
 export type SyncContext = {
     readonly clientID: string;
     readonly wsID: string;
+    readonly profileID: string | null;
     readonly baseCookie: string | null;
     readonly protocolVersion: number;
     readonly schemaVersion: number | null;
@@ -83,7 +84,7 @@ export declare class TimeSliceTimer {
     #private;
     start(): Promise<this>;
     startWithoutYielding(): this;
-    yieldProcess(): Promise<void>;
+    yieldProcess(_msgForTesting?: string): Promise<void>;
     elapsedLap(): number;
     /** @returns the total elapsed time */
     stop(): number;

package/out/zero-cache/src/services/view-syncer/view-syncer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"view-syncer.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/view-syncer/view-syncer.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,MAAM,CAAC;AAcrC,OAAO,KAAK,EAAC,2BAA2B,EAAC,MAAM,yDAAyD,CAAC;AACzG,OAAO,KAAK,EAEV,qBAAqB,EACtB,MAAM,0CAA0C,CAAC;AAElD,OAAO,KAAK,EAAC,oBAAoB,EAAC,MAAM,iDAAiD,CAAC;AAC1F,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,uCAAuC,CAAC;AAOtE,OAAO,KAAK,EAEV,gBAAgB,EACjB,MAAM,6CAA6C,CAAC;AAMrD,OAAO,KAAK,EAAC,oBAAoB,EAAC,MAAM,2BAA2B,CAAC;AAEpE,OAAO,KAAK,EAAC,sBAAsB,EAAC,MAAM,yCAAyC,CAAC;AAOpF,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,oCAAoC,CAAC;AAK1E,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,mBAAmB,CAAC;AAElD,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,uBAAuB,CAAC;AACnD,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,wBAAwB,CAAC;AACnD,OAAO,EAAC,YAAY,EAAC,MAAM,6BAA6B,CAAC;AACzD,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,6BAA6B,CAAC;AAE9D,OAAO,KAAK,EAAC,oBAAoB,EAAC,MAAM,eAAe,CAAC;AAiBxD,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,wBAAwB,CAAC;AAE7D,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,sBAAsB,CAAC;AAuBzD,MAAM,MAAM,SAAS,GAAG;IACtB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,kBAAkB;IAClB,QAAQ,CAAC,OAAO,EAAE,UAAU,CAAC;CAC9B,CAAC;AAEF,MAAM,MAAM,WAAW,GAAG;IACxB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC,QAAQ,CAAC,SAAS,EAAE,SAAS,GAAG,SAAS,CAAC;IAC1C,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,SAAS,CAAC;CACzC,CAAC;AAMF,MAAM,WAAW,UAAU;IACzB,cAAc,CACZ,GAAG,EAAE,WAAW,EAChB,GAAG,EAAE,qBAAqB,GACzB,MAAM,CAAC,UAAU,CAAC,CAAC;IAEtB,oBAAoB,CAClB,GAAG,EAAE,WAAW,EAChB,GAAG,EAAE,2BAA2B,GAC/B,OAAO,CAAC,IAAI,CAAC,CAAC;IAEjB,aAAa,CAAC,GAAG,EAAE,WAAW,EAAE,GAAG,EAAE,oBAAoB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC1E,OAAO,CAAC,OAAO,EAAE,WAAW,EAAE,GAAG,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACrE;AAQD,KAAK,UAAU,GAAG,CAChB,EAAE,EAAE,CAAC,GAAG,IAAI,EAAE,OAAO,EAAE,KAAK,IAAI,EAChC,KAAK,CAAC,EAAE,MAAM,KACX,UAAU,CAAC,OAAO,UAAU,CAAC,CAAC;AAEnC;;;;GAIG;AACH,eAAO,MAAM,kBAAkB,QAAS,CAAC;AAEzC;;;;;GAKG;AACH,eAAO,MAAM,oBAAoB,KAAK,CAAC;AAEvC,qBAAa,iBAAkB,YAAW,UAAU,EAAE,oBAAoB;;IACxE,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IAUpB,YAAY,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;gBAsHhC,MAAM,EAAE,oBAAoB,EAC5B,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,MAAM,EAAE,MAAM,EACd,aAAa,EAAE,MAAM,EACrB,KAAK,EAAE,UAAU,EACjB,UAAU,EAAE,UAAU,GAAG,SAAS,EAClC,cAAc,EAAE,cAAc,EAC9B,cAAc,EAAE,YAAY,CAAC,YAAY,CAAC,EAC1C,gBAAgB,EAAE,gBAAgB,EAClC,oBAAoB,EAAE,MAAM,EAC5B,iBAAiB,EAAE,iBAAiB,EACpC,sBAAsB,EAAE,sBAAsB,GAAG,SAAS,EAC1D,WAAW,SAAuB,EAClC,YAAY,GAAE,UAAwC;IA2FxD,UAAU,IAAI,OAAO,CAAC,aAAa,GAAG,UAAU,CAAC;IAO3C,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;IAqH1B;;;;;;;;OAQG;IACH,SAAS,IAAI,OAAO;IAwEpB,cAAc,CACZ,GAAG,EAAE,WAAW,EAChB,qBAAqB,EAAE,qBAAqB,GAC3C,MAAM,CAAC,UAAU,CAAC;IA4Gf,oBAAoB,CACxB,GAAG,EAAE,WAAW,EAChB,GAAG,EAAE,2BAA2B,GAC/B,OAAO,CAAC,IAAI,CAAC;IAIV,aAAa,CACjB,GAAG,EAAE,WAAW,EAChB,GAAG,EAAE,oBAAoB,GACxB,OAAO,CAAC,IAAI,CAAC;IAuuChB,OAAO,CAAC,OAAO,EAAE,WAAW,EAAE,GAAG,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC;IA2BnE,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAqBrB;;;OAGG;IACH,eAAe;CAGhB;AA+ED,wBAAgB,SAAS,CACvB,EAAE,EAAE,UAAU,EACd,aAAa,EAAE,SAAS,GAAG,SAAS,EACpC,QAAQ,EAAE,SAAS,GAAG,SAAS,yBAkDhC;AAyCD,qBAAa,cAAc;;IAInB,KAAK;IAOX,oBAAoB;IAMd,YAAY;IAWlB,UAAU;IAWV,sCAAsC;IACtC,IAAI,IAAI,MAAM;IAKd;;;OAGG;IACH,YAAY,IAAI,MAAM;CAKvB"}
+{"version":3,"file":"view-syncer.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/view-syncer/view-syncer.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,MAAM,CAAC;AAcrC,OAAO,KAAK,EAAC,2BAA2B,EAAC,MAAM,yDAAyD,CAAC;AACzG,OAAO,KAAK,EAEV,qBAAqB,EACtB,MAAM,0CAA0C,CAAC;AAElD,OAAO,KAAK,EAAC,oBAAoB,EAAC,MAAM,iDAAiD,CAAC;AAC1F,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,uCAAuC,CAAC;AAOtE,OAAO,KAAK,EAEV,gBAAgB,EACjB,MAAM,6CAA6C,CAAC;AAMrD,OAAO,KAAK,EAAC,oBAAoB,EAAC,MAAM,2BAA2B,CAAC;AAEpE,OAAO,KAAK,EAAC,sBAAsB,EAAC,MAAM,yCAAyC,CAAC;AAOpF,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,oCAAoC,CAAC;AAK1E,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,mBAAmB,CAAC;AAElD,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,uBAAuB,CAAC;AACnD,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,wBAAwB,CAAC;AACnD,OAAO,EAAC,YAAY,EAAC,MAAM,6BAA6B,CAAC;AACzD,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,6BAA6B,CAAC;AAE9D,OAAO,KAAK,EAAC,oBAAoB,EAAC,MAAM,eAAe,CAAC;AAiBxD,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,wBAAwB,CAAC;AAE7D,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,sBAAsB,CAAC;AAuBzD,MAAM,MAAM,SAAS,GAAG;IACtB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,kBAAkB;IAClB,QAAQ,CAAC,OAAO,EAAE,UAAU,CAAC;CAC9B,CAAC;AAEF,MAAM,MAAM,WAAW,GAAG;IACxB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC,QAAQ,CAAC,SAAS,EAAE,SAAS,GAAG,SAAS,CAAC;IAC1C,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,SAAS,CAAC;CACzC,CAAC;AAMF,MAAM,WAAW,UAAU;IACzB,cAAc,CACZ,GAAG,EAAE,WAAW,EAChB,GAAG,EAAE,qBAAqB,GACzB,MAAM,CAAC,UAAU,CAAC,CAAC;IAEtB,oBAAoB,CAClB,GAAG,EAAE,WAAW,EAChB,GAAG,EAAE,2BAA2B,GAC/B,OAAO,CAAC,IAAI,CAAC,CAAC;IAEjB,aAAa,CAAC,GAAG,EAAE,WAAW,EAAE,GAAG,EAAE,oBAAoB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC1E,OAAO,CAAC,OAAO,EAAE,WAAW,EAAE,GAAG,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACrE;AAQD,KAAK,UAAU,GAAG,CAChB,EAAE,EAAE,CAAC,GAAG,IAAI,EAAE,OAAO,EAAE,KAAK,IAAI,EAChC,KAAK,CAAC,EAAE,MAAM,KACX,UAAU,CAAC,OAAO,UAAU,CAAC,CAAC;AAEnC;;;;GAIG;AACH,eAAO,MAAM,kBAAkB,QAAS,CAAC;AAEzC;;;;;GAKG;AACH,eAAO,MAAM,oBAAoB,KAAK,CAAC;AAEvC,qBAAa,iBAAkB,YAAW,UAAU,EAAE,oBAAoB;;IACxE,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IAUpB,YAAY,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;gBAoHhC,MAAM,EAAE,oBAAoB,EAC5B,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,MAAM,EAAE,MAAM,EACd,aAAa,EAAE,MAAM,EACrB,KAAK,EAAE,UAAU,EACjB,UAAU,EAAE,UAAU,GAAG,SAAS,EAClC,cAAc,EAAE,cAAc,EAC9B,cAAc,EAAE,YAAY,CAAC,YAAY,CAAC,EAC1C,gBAAgB,EAAE,gBAAgB,EAClC,oBAAoB,EAAE,MAAM,EAC5B,iBAAiB,EAAE,iBAAiB,EACpC,sBAAsB,EAAE,sBAAsB,GAAG,SAAS,EAC1D,WAAW,SAAuB,EAClC,YAAY,GAAE,UAAwC;IA2FxD,UAAU,IAAI,OAAO,CAAC,aAAa,GAAG,UAAU,CAAC;IAO3C,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;IAiH1B;;;;;;;;OAQG;IACH,SAAS,IAAI,OAAO;IAwEpB,cAAc,CACZ,GAAG,EAAE,WAAW,EAChB,qBAAqB,EAAE,qBAAqB,GAC3C,MAAM,CAAC,UAAU,CAAC;IAuHf,oBAAoB,CACxB,GAAG,EAAE,WAAW,EAChB,GAAG,EAAE,2BAA2B,GAC/B,OAAO,CAAC,IAAI,CAAC;IAIV,aAAa,CACjB,GAAG,EAAE,WAAW,EAChB,GAAG,EAAE,oBAAoB,GACxB,OAAO,CAAC,IAAI,CAAC;IAyuChB,OAAO,CAAC,OAAO,EAAE,WAAW,EAAE,GAAG,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC;IA2BnE,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAqBrB;;;OAGG;IACH,eAAe;CAGhB;AAuED,wBAAgB,SAAS,CACvB,EAAE,EAAE,UAAU,EACd,aAAa,EAAE,SAAS,GAAG,SAAS,EACpC,QAAQ,EAAE,SAAS,GAAG,SAAS,yBAkDhC;AAyCD,qBAAa,cAAc;;IAInB,KAAK;IAOX,oBAAoB;IAMd,YAAY,CAAC,cAAc,CAAC,EAAE,MAAM;IAW1C,UAAU;IAWV,sCAAsC;IACtC,IAAI,IAAI,MAAM;IAKd;;;OAGG;IACH,YAAY,IAAI,MAAM;CAKvB"}

package/out/zero-cache/src/services/view-syncer/view-syncer.js

@@ -8,7 +8,7 @@ import { stringify } from "../../../../shared/src/bigint-json.js";
 import { CustomKeyMap } from "../../../../shared/src/custom-key-map.js";
 import { must } from "../../../../shared/src/must.js";
 import { randInt } from "../../../../shared/src/rand.js";
-import { Rehome, ClientNotFound, InvalidConnectionRequest, TransformFailed, Unauthorized, InvalidConnectionRequestBaseCookie } from "../../../../zero-protocol/src/error-kind-enum.js";
+import { Rehome, InvalidConnectionRequest, TransformFailed, Unauthorized, InvalidConnectionRequestBaseCookie } from "../../../../zero-protocol/src/error-kind-enum.js";
 import { ZeroCache } from "../../../../zero-protocol/src/error-origin-enum.js";
 import { ProtocolError } from "../../../../zero-protocol/src/error.js";
 import { clampTTL, MAX_TTL_MS } from "../../../../zql/src/query/ttl.js";
@@ -19,7 +19,7 @@ import { rowIDString } from "../../types/row-key.js";
 import { Subscription } from "../../types/subscription.js";
 import "../replicator/schema/replication-state.js";
 import { startPoke, ClientHandler } from "./client-handler.js";
-import { CVRStore } from "./cvr-store.js";
+import { CVRStore, ClientNotFoundError } from "./cvr-store.js";
 import { CVRConfigDrivenUpdater, nextEvictionTime, CVRQueryDrivenUpdater } from "./cvr.js";
 import { handleInspect } from "./inspect-handler.js";
 import "../../../../shared/src/config.js";
@@ -28,6 +28,7 @@ import "../../../../zero-protocol/src/ast.js";
 import "../../../../zero-protocol/src/data.js";
 import "@rocicorp/zero-sqlite3";
 import "../../../../zqlite/src/internal/sql.js";
+import "@databases/escape-identifier";
 import "../../../../shared/src/btree-set.js";
 import "../../../../shared/src/valita.js";
 import "../../../../zero-schema/src/compiled-permissions.js";
@@ -100,8 +101,6 @@ class ViewSyncerService {
   // DEPRECATED: remove `authData` in favor of forwarding
   // auth and cookie headers directly
   #authData;
-  // Not sure why lint can't see that this is used?
-  // oxlint-disable-next-line no-unused-private-class-members
   #httpCookie;
   #expiredQueriesTimer = 0;
   #setTimeout;
@@ -259,11 +258,7 @@ class ViewSyncerService {
           if (cvr.replicaVersion !== null && cvr.version.stateVersion !== "00" && this.#pipelines.replicaVersion < cvr.replicaVersion) {
             const message = `Cannot sync from older replica: CVR=${cvr.replicaVersion}, DB=${this.#pipelines.replicaVersion}`;
             lc.info?.(`resetting CVR: ${message}`);
-            throw new ProtocolErrorWithLevel({
-              kind: ClientNotFound,
-              message,
-              origin: ZeroCache
-            });
+            throw new ClientNotFoundError(message);
           }
           if (this.#pipelinesSynced) {
             const result = await this.#advancePipelines(lc, cvr);
@@ -380,6 +375,7 @@ class ViewSyncerService {
     return startSpan(tracer, "vs.initConnection", () => {
       const {
         clientID,
+        profileID,
         wsID,
         baseCookie,
         schemaVersion,
@@ -447,7 +443,17 @@ class ViewSyncerService {
               origin: ZeroCache
             });
           }
-          await this.#handleConfigUpdate(lc2, clientID2, msg, cvr);
+          await this.#handleConfigUpdate(
+            lc2,
+            clientID2,
+            msg,
+            cvr,
+            // Until the profileID is required in the URL, default it to
+            // `cg${clientGroupID}`, as is done in the schema migration.
+            // As clients update to the zero version with the profileID logic,
+            // the value will be correspondingly in the CVR db.
+            profileID ?? `cg${this.id}`
+          );
           this.#initialized.resolve("initialized");
         },
         newClient
@@ -595,7 +601,7 @@ class ViewSyncerService {
     deleted,
     desiredQueriesPatch,
     activeClients
-  }, cvr) => startAsyncSpan(tracer, "vs.#patchQueries", async () => {
+  }, cvr, profileID) => startAsyncSpan(tracer, "vs.#patchQueries", async () => {
     const deletedClientIDs = [];
     const deletedClientGroupIDs = [];
     cvr = await this.#updateCVRConfig(lc, cvr, clientID, (updater) => {
@@ -604,6 +610,9 @@ class ViewSyncerService {
       if (clientSchema) {
         updater.setClientSchema(lc, clientSchema);
       }
+      if (profileID) {
+        updater.setProfileID(lc, profileID);
+      }
       lc.debug?.(`applying ${desiredQueriesPatch?.length} query patches`);
       if (desiredQueriesPatch?.length) {
         for (const patch of desiredQueriesPatch) {
@@ -787,16 +796,16 @@ class ViewSyncerService {
           span.setAttribute("queryHash", queryID);
           span.setAttribute("transformationHash", transformationHash);
           span.setAttribute("table", transformedAst.table);
-          for (const _ of this.#pipelines.addQuery(
+          for (const change of this.#pipelines.addQuery(
             transformationHash,
             queryID,
             transformedAst,
             await timer.start()
           )) {
-            if (++count % TIME_SLICE_CHECK_SIZE === 0) {
-              if (timer.elapsedLap() > TIME_SLICE_MS) {
-                await timer.yieldProcess();
-              }
+            if (change === "yield") {
+              await timer.yieldProcess("yield in hydrateUnchangedQueries");
+            } else {
+              count++;
             }
           }
         }
@@ -1278,6 +1287,10 @@ class ViewSyncerService {
       });
       await startAsyncSpan(tracer, "loopingChanges", async (span) => {
         for (const change of changes) {
+          if (change === "yield") {
+            await timer.yieldProcess("yield in processChanges");
+            continue;
+          }
           const {
             type,
             queryHash: transformationHash,
@@ -1318,11 +1331,6 @@ class ViewSyncerService {
           if (rows.size % CURSOR_PAGE_SIZE === 0) {
             await processBatch();
           }
-          if (rows.size % TIME_SLICE_CHECK_SIZE === 0) {
-            if (timer.elapsedLap() > TIME_SLICE_MS) {
-              await timer.yieldProcess();
-            }
-          }
         }
         if (rows.size) {
           await processBatch();
@@ -1436,8 +1444,6 @@ class ViewSyncerService {
   }
 }
 const CURSOR_PAGE_SIZE = 1e4;
-const TIME_SLICE_CHECK_SIZE = 100;
-const TIME_SLICE_MS = 500;
 function createHashToIDs(cvr) {
   const hashToIDs = /* @__PURE__ */ new Map();
   for (const { id, transformationHash } of Object.values(cvr.queries)) {
@@ -1466,11 +1472,7 @@ function contentsAndVersion(row) {
 const NEW_CVR_VERSION = { stateVersion: "00" };
 function checkClientAndCVRVersions(client, cvr) {
   if (cmpVersions(cvr, NEW_CVR_VERSION) === 0 && cmpVersions(client, NEW_CVR_VERSION) > 0) {
-    throw new ProtocolErrorWithLevel({
-      kind: ClientNotFound,
-      message: "Client not found",
-      origin: ZeroCache
-    });
+    throw new ClientNotFoundError("Client not found");
   }
   if (cmpVersions(client, cvr) > 0) {
     throw new ProtocolError({
@@ -1554,7 +1556,7 @@ class TimeSliceTimer {
     this.#startLap();
     return this;
   }
-  async yieldProcess() {
+  async yieldProcess(_msgForTesting) {
     this.#stopLap();
     await yieldProcess();
     this.#startLap();