@rocicorp/zero 1.5.0-canary.4 → 1.6.0-canary.0

package/out/zero-cache/src/services/change-source/pg/schema/shard.d.ts

@@ -18,7 +18,6 @@ export declare function replicationSlotPrefix(shard: ShardID): string;
  * in a Postgres `LIKE` operator.
  */
 export declare function replicationSlotExpression(shard: ShardID): string;
-export declare function newReplicationSlot(shard: ShardID): string;
 export declare function metadataPublicationName(appID: string, shardID: string | number): string;
 export declare function ensureGlobalTables(db: PostgresDB, appID: AppID): Promise<void>;
 export declare function getClientsTableDefinition(schema: string): string;
@@ -42,7 +41,9 @@ export type InternalShardConfig = v.Infer<typeof internalShardConfigSchema>;
 declare const replicaSchema: v.ObjectType<Omit<{
     publications: v.ArrayType<v.Type<string>>;
     ddlDetection: v.Type<boolean>;
-}, "initialSyncContext" | "version" | "slot" | "initialSchema" | "subscriberContext"> & {
+}, "id" | "initialSyncContext" | "version" | "rank" | "slot" | "initialSchema" | "subscriberContext"> & {
+    id: v.Type<string>;
+    rank: v.Type<bigint>;
     slot: v.Type<string>;
     version: v.Type<string>;
     initialSchema: v.Type<{
@@ -83,7 +84,20 @@ declare const replicaSchema: v.ObjectType<Omit<{
     subscriberContext: v.Type<Readonly<Record<string, import("../../../../../../shared/src/bigint-json.ts").JSONValue>> | null>;
 }, undefined>;
 export type Replica = v.Infer<typeof replicaSchema>;
-export declare function addReplica(sql: PostgresDB, shard: ShardID, slot: string, replicaVersion: string, { tables, indexes }: PublishedSchema, initialSyncContext: JSONObject): Promise<void>;
+/**
+ * Creates a new replica to mark it as the owner of a specified `slot`.
+ * This should be done with an advisory lock for replica/slot management
+ * to ensure that the slot does not get dropped by concurrent cleanup
+ * logic.
+ *
+ * Once initial sync is complete, {@link initReplica} should be called to
+ * make the replica usable for incremental sync.
+ */
+export declare function createReplica(sql: PostgresDB, shard: ShardID, id: string, slot: string, replicaVersion: string): Promise<void>;
+export declare function initReplica(sql: PostgresDB, shard: ShardID, id: string, { tables, indexes }: PublishedSchema, initialSyncContext: JSONObject): Promise<void>;
+/**
+ * Gets the latest initialized replica at the specified version.
+ */
 export declare function getReplicaAtVersion(lc: LogContext, sql: PostgresDB, shard: ShardID, replicaVersion: string, context?: JSONObject): Promise<Replica | null>;
 export declare function getInternalShardConfig(sql: PostgresDB, shard: ShardID): Promise<InternalShardConfig>;
 /**

package/out/zero-cache/src/services/change-source/pg/schema/shard.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"shard.d.ts","sourceRoot":"","sources":["../../../../../../../../zero-cache/src/services/change-source/pg/schema/shard.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAIjD,OAAO,EAGL,KAAK,UAAU,EAChB,MAAM,6CAA6C,CAAC;AACrD,OAAO,KAAK,CAAC,MAAM,wCAAwC,CAAC;AAE5D,OAAO,KAAK,EAAC,UAAU,EAAE,mBAAmB,EAAC,MAAM,yBAAyB,CAAC;AAC7E,OAAO,KAAK,EAAC,KAAK,EAAE,WAAW,EAAE,OAAO,EAAC,MAAM,6BAA6B,CAAC;AAO7E,OAAO,EAGL,KAAK,eAAe,EACpB,KAAK,eAAe,EACrB,MAAM,gBAAgB,CAAC;AASxB;;;;GAIG;AACH,wBAAgB,uBAAuB,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAY1D;AAED,wBAAgB,yBAAyB,CAAC,EAAC,KAAK,EAAC,EAAE,KAAK,UAEvD;AAED,wBAAgB,qBAAqB,CAAC,EAAC,KAAK,EAAE,QAAQ,EAAC,EAAE,OAAO,UAE/D;AAED,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,OAAO,UAGnD;AAED;;;GAGG;AACH,wBAAgB,yBAAyB,CAAC,KAAK,EAAE,OAAO,UAIvD;AAED,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,OAAO,UAEhD;AAMD,wBAAgB,uBAAuB,CACrC,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,MAAM,GAAG,MAAM,UAGzB;AAoCD,wBAAsB,kBAAkB,CAAC,EAAE,EAAE,UAAU,EAAE,KAAK,EAAE,KAAK,iBAEpE;AAED,wBAAgB,yBAAyB,CAAC,MAAM,EAAE,MAAM,UASvD;AAED;;;;;;;GAOG;AACH,wBAAgB,2BAA2B,CAAC,MAAM,EAAE,MAAM,UASzD;AAED,eAAO,MAAM,kBAAkB,gBAAgB,CAAC;AAEhD,wBAAgB,UAAU,CACxB,WAAW,EAAE,WAAW,EACxB,mBAAmB,EAAE,MAAM,GAC1B,MAAM,CA4CR;AAED,wBAAgB,SAAS,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM,CAYzE;AAED,QAAA,MAAM,yBAAyB;;;aAG7B,CAAC;AAEH,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC;AAE5E,QAAA,MAAM,aAAa;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;aAMjB,CAAC;AAEH,MAAM,MAAM,OAAO,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,aAAa,CAAC,CAAC;AAcpD,wBAAsB,UAAU,CAC9B,GAAG,EAAE,UAAU,EACf,KAAK,EAAE,OAAO,EACd,IAAI,EAAE,MAAM,EACZ,cAAc,EAAE,MAAM,EACtB,EAAC,MAAM,EAAE,OAAO,EAAC,EAAE,eAAe,EAClC,kBAAkB,EAAE,UAAU,iBAQ/B;AAED,wBAAsB,mBAAmB,CACvC,EAAE,EAAE,UAAU,EACd,GAAG,EAAE,UAAU,EACf,KAAK,EAAE,OAAO,EACd,cAAc,EAAE,MAAM,EACtB,OAAO,CAAC,EAAE,UAAU,GACnB,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC,CA2BzB;AAED,wBAAsB,sBAAsB,CAC1C,GAAG,EAAE,UAAU,EACf,KAAK,EAAE,OAAO,GACb,OAAO,CAAC,mBAAmB,CAAC,CAU9B;AAED;;;GAGG;AACH,wBAAsB,yBAAyB,CAC7C,EAAE,EAAE,UAAU,EACd,GAAG,EAAE,mBAAmB,EACxB,SAAS,EAAE,WAAW,iBAyDvB;AAED,wBAAsB,aAAa,CACjC,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,mBAAmB,EACvB,KAAK,EAAE,WAAW,iBAmCnB;AAED,wBAAgB,oBAAoB,CAClC,EAAE,EAAE,UAAU,EACd,SAAS,EAAE,eAAe,QAkB3B;AAED,KAAK,iBAAiB,GAAG;IACvB,KAAK,CAAC,EAAE,EAAE,UAAU,EAAE,EAAE,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACtD,CAAC;AAEF,wBAAgB,4CAA4C,CAC1D,IAAI,EAAE,eAAe,GACpB,iBAAiB,GAAG,SAAS,CA+C/B"}
+{"version":3,"file":"shard.d.ts","sourceRoot":"","sources":["../../../../../../../../zero-cache/src/services/change-source/pg/schema/shard.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAIjD,OAAO,EAGL,KAAK,UAAU,EAChB,MAAM,6CAA6C,CAAC;AACrD,OAAO,KAAK,CAAC,MAAM,wCAAwC,CAAC;AAE5D,OAAO,KAAK,EAAC,UAAU,EAAE,mBAAmB,EAAC,MAAM,yBAAyB,CAAC;AAC7E,OAAO,KAAK,EAAC,KAAK,EAAE,WAAW,EAAE,OAAO,EAAC,MAAM,6BAA6B,CAAC;AAO7E,OAAO,EAGL,KAAK,eAAe,EACpB,KAAK,eAAe,EACrB,MAAM,gBAAgB,CAAC;AASxB;;;;GAIG;AACH,wBAAgB,uBAAuB,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAY1D;AAED,wBAAgB,yBAAyB,CAAC,EAAC,KAAK,EAAC,EAAE,KAAK,UAEvD;AAED,wBAAgB,qBAAqB,CAAC,EAAC,KAAK,EAAE,QAAQ,EAAC,EAAE,OAAO,UAE/D;AAED,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,OAAO,UAGnD;AAED;;;GAGG;AACH,wBAAgB,yBAAyB,CAAC,KAAK,EAAE,OAAO,UAIvD;AAMD,wBAAgB,uBAAuB,CACrC,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,MAAM,GAAG,MAAM,UAGzB;AAoCD,wBAAsB,kBAAkB,CAAC,EAAE,EAAE,UAAU,EAAE,KAAK,EAAE,KAAK,iBAEpE;AAED,wBAAgB,yBAAyB,CAAC,MAAM,EAAE,MAAM,UASvD;AAED;;;;;;;GAOG;AACH,wBAAgB,2BAA2B,CAAC,MAAM,EAAE,MAAM,UASzD;AAED,eAAO,MAAM,kBAAkB,gBAAgB,CAAC;AAEhD,wBAAgB,UAAU,CACxB,WAAW,EAAE,WAAW,EACxB,mBAAmB,EAAE,MAAM,GAC1B,MAAM,CAgDR;AAED,wBAAgB,SAAS,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM,CAYzE;AAED,QAAA,MAAM,yBAAyB;;;aAG7B,CAAC;AAEH,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC;AAE5E,QAAA,MAAM,aAAa;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;aAQjB,CAAC;AAEH,MAAM,MAAM,OAAO,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,aAAa,CAAC,CAAC;AAapD;;;;;;;;GAQG;AACH,wBAAsB,aAAa,CACjC,GAAG,EAAE,UAAU,EACf,KAAK,EAAE,OAAO,EACd,EAAE,EAAE,MAAM,EACV,IAAI,EAAE,MAAM,EACZ,cAAc,EAAE,MAAM,iBAKvB;AAGD,wBAAsB,WAAW,CAC/B,GAAG,EAAE,UAAU,EACf,KAAK,EAAE,OAAO,EACd,EAAE,EAAE,MAAM,EACV,EAAC,MAAM,EAAE,OAAO,EAAC,EAAE,eAAe,EAClC,kBAAkB,EAAE,UAAU,iBAM/B;AAED;;GAEG;AACH,wBAAsB,mBAAmB,CACvC,EAAE,EAAE,UAAU,EACd,GAAG,EAAE,UAAU,EACf,KAAK,EAAE,OAAO,EACd,cAAc,EAAE,MAAM,EACtB,OAAO,CAAC,EAAE,UAAU,GACnB,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC,CA8BzB;AAED,wBAAsB,sBAAsB,CAC1C,GAAG,EAAE,UAAU,EACf,KAAK,EAAE,OAAO,GACb,OAAO,CAAC,mBAAmB,CAAC,CAU9B;AAED;;;GAGG;AACH,wBAAsB,yBAAyB,CAC7C,EAAE,EAAE,UAAU,EACd,GAAG,EAAE,mBAAmB,EACxB,SAAS,EAAE,WAAW,iBAyDvB;AAED,wBAAsB,aAAa,CACjC,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,mBAAmB,EACvB,KAAK,EAAE,WAAW,iBAmCnB;AAED,wBAAgB,oBAAoB,CAClC,EAAE,EAAE,UAAU,EACd,SAAS,EAAE,eAAe,QAkB3B;AAED,KAAK,iBAAiB,GAAG;IACvB,KAAK,CAAC,EAAE,EAAE,UAAU,EAAE,EAAE,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACtD,CAAC;AAEF,wBAAgB,4CAA4C,CAC1D,IAAI,EAAE,eAAe,GACpB,iBAAiB,GAAG,SAAS,CA+C/B"}

package/out/zero-cache/src/services/change-source/pg/schema/shard.js

@@ -41,9 +41,6 @@ function replicationSlotPrefix(shard) {
 function replicationSlotExpression(shard) {
 	return `${replicationSlotPrefix(shard)}%`.replaceAll("_", "\\_");
 }
-function newReplicationSlot(shard) {
-	return replicationSlotPrefix(shard) + Date.now();
-}
 function defaultPublicationName(appID, shardID) {
 	return `_${appID}_public_${shardID}`;
 }
@@ -145,9 +142,13 @@ function shardSetup(shardConfig, metadataPublication) {
     );
 
   CREATE TABLE ${shard}.replicas (
-    "slot"               TEXT PRIMARY KEY,
+    -- The DEFAULT exists purely for backwards compatibility support.
+    -- New code always specifies a value based on Date.now().
+    "id"                 TEXT PRIMARY KEY DEFAULT replace(gen_random_uuid()::text, '-', ''),
+    "rank"               BIGSERIAL,
+    "slot"               TEXT NOT NULL,
     "version"            TEXT NOT NULL,
-    "initialSchema"      JSON NOT NULL,
+    "initialSchema"      JSON,  -- set after initial sync
     "initialSyncContext" JSON,
     "subscriberContext"  JSON
   );
@@ -167,6 +168,8 @@ var internalShardConfigSchema = valita_exports.object({
 	ddlDetection: valita_exports.boolean()
 });
 var replicaSchema = internalShardConfigSchema.extend({
+	id: valita_exports.string(),
+	rank: valita_exports.bigint(),
 	slot: valita_exports.string(),
 	version: valita_exports.string(),
 	initialSchema: publishedSchema,
@@ -177,21 +180,44 @@ function triggerSetup(shard) {
 	const schema = id(upstreamSchema(shard));
 	return createEventTriggerStatements(shard) + `UPDATE ${schema}."shardConfig" SET "ddlDetection" = true;`;
 }
-async function addReplica(sql, shard, slot, replicaVersion, { tables, indexes }, initialSyncContext) {
+/**
+* Creates a new replica to mark it as the owner of a specified `slot`.
+* This should be done with an advisory lock for replica/slot management
+* to ensure that the slot does not get dropped by concurrent cleanup
+* logic.
+*
+* Once initial sync is complete, {@link initReplica} should be called to
+* make the replica usable for incremental sync.
+*/
+async function createReplica(sql, shard, id, slot, replicaVersion) {
 	const schema = upstreamSchema(shard);
-	const synced = {
-		tables,
-		indexes
+	const values = {
+		id,
+		slot,
+		version: replicaVersion
 	};
-	await sql`
-    INSERT INTO ${sql(schema)}.replicas
-      ("slot", "version", "initialSchema", "initialSyncContext")
-      VALUES (${slot}, ${replicaVersion}, ${synced}, ${initialSyncContext})`;
+	await sql`INSERT INTO ${sql(schema)}.replicas ${sql(values)}`;
 }
+async function initReplica(sql, shard, id, { tables, indexes }, initialSyncContext) {
+	const schema = upstreamSchema(shard);
+	const values = {
+		initialSchema: {
+			tables,
+			indexes
+		},
+		initialSyncContext
+	};
+	await sql`UPDATE ${sql(schema)}.replicas SET ${sql(values)} WHERE id = ${id}`;
+}
+/**
+* Gets the latest initialized replica at the specified version.
+*/
 async function getReplicaAtVersion(lc, sql, shard, replicaVersion, context) {
 	const schema = sql(upstreamSchema(shard));
 	const result = await sql`
     SELECT
+      replicas."id",
+      replicas."rank",
       replicas."slot",
       replicas."version",
       replicas."initialSchema",
@@ -200,11 +226,12 @@ async function getReplicaAtVersion(lc, sql, shard, replicaVersion, context) {
       "shardConfig"."publications",
       "shardConfig"."ddlDetection"
     FROM ${schema}.replicas JOIN ${schema}."shardConfig" ON true
-      WHERE version = ${replicaVersion};
+      WHERE version = ${replicaVersion} AND "initialSyncContext" IS NOT NULL
+      ORDER BY rank DESC LIMIT 1;
   `;
 	if (result.length === 0) {
 		const allReplicas = await sql`
-      SELECT slot, version, "initialSyncContext", "subscriberContext" 
+      SELECT id, slot, version, "initialSyncContext", "subscriberContext" 
         FROM ${schema}.replicas`;
 		lc.info?.(`Replica ${replicaVersion} ` + (context ? `(context: ${stringify(context)}) ` : "") + `not found in: ${stringify(allReplicas)}`);
 		return null;
@@ -298,6 +325,6 @@ function replicaIdentitiesForTablesWithoutPrimaryKeys(pubs) {
 	} };
 }
 //#endregion
-export { SHARD_CONFIG_TABLE, addReplica, dropShard, ensureGlobalTables, getInternalShardConfig, getMutationsTableDefinition, getReplicaAtVersion, internalPublicationPrefix, legacyReplicationSlot, metadataPublicationName, newReplicationSlot, replicaIdentitiesForTablesWithoutPrimaryKeys, replicationSlotExpression, setupTablesAndReplication, setupTriggers, validatePublications };
+export { SHARD_CONFIG_TABLE, createReplica, dropShard, ensureGlobalTables, getInternalShardConfig, getMutationsTableDefinition, getReplicaAtVersion, initReplica, internalPublicationPrefix, legacyReplicationSlot, metadataPublicationName, replicaIdentitiesForTablesWithoutPrimaryKeys, replicationSlotExpression, replicationSlotPrefix, setupTablesAndReplication, setupTriggers, validatePublications };
 
 //# sourceMappingURL=shard.js.map

package/out/zero-cache/src/services/change-source/pg/schema/shard.js.map

@@ -1 +1 @@
-{"version":3,"file":"shard.js","names":[],"sources":["../../../../../../../../zero-cache/src/services/change-source/pg/schema/shard.ts"],"sourcesContent":["import {PG_INSUFFICIENT_PRIVILEGE} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport {literal} from 'pg-format';\nimport postgres from 'postgres';\nimport {assert} from '../../../../../../shared/src/asserts.ts';\nimport {\n  jsonObjectSchema,\n  stringify,\n  type JSONObject,\n} from '../../../../../../shared/src/bigint-json.ts';\nimport * as v from '../../../../../../shared/src/valita.ts';\nimport {Default} from '../../../../db/postgres-replica-identity-enum.ts';\nimport type {PostgresDB, PostgresTransaction} from '../../../../types/pg.ts';\nimport type {AppID, ShardConfig, ShardID} from '../../../../types/shards.ts';\nimport {appSchema, check, upstreamSchema} from '../../../../types/shards.ts';\nimport {id} from '../../../../types/sql.ts';\nimport {\n  createEventFunctionStatements,\n  createEventTriggerStatements,\n} from './ddl.ts';\nimport {\n  getPublicationInfo,\n  publishedSchema,\n  type PublicationInfo,\n  type PublishedSchema,\n} from './published.ts';\nimport {validate} from './validation.ts';\n\n/**\n * PostgreSQL unquoted identifiers must start with a letter or underscore\n * and contain only letters, digits, and underscores.\n */\nconst VALID_PUBLICATION_NAME = /^[a-zA-Z_][a-zA-Z0-9_]*$/;\n\n/**\n * Validates that a publication name is a valid PostgreSQL identifier.\n * This provides defense-in-depth against SQL injection when publication\n * names are used in replication commands.\n */\nexport function validatePublicationName(name: string): void {\n  if (!VALID_PUBLICATION_NAME.test(name)) {\n    throw new Error(\n      `Invalid publication name \"${name}\". Publication names must start with a letter or underscore ` +\n        `and contain only letters, digits, and underscores.`,\n    );\n  }\n  if (name.length > 63) {\n    throw new Error(\n      `Publication name \"${name}\" exceeds PostgreSQL's 63-character identifier limit.`,\n    );\n  }\n}\n\nexport function internalPublicationPrefix({appID}: AppID) {\n  return `_${appID}_`;\n}\n\nexport function legacyReplicationSlot({appID, shardNum}: ShardID) {\n  return `${appID}_${shardNum}`;\n}\n\nexport function replicationSlotPrefix(shard: ShardID) {\n  const {appID, shardNum} = check(shard);\n  return `${appID}_${shardNum}_`;\n}\n\n/**\n * An expression used to match replication slots in the shard\n * in a Postgres `LIKE` operator.\n */\nexport function replicationSlotExpression(shard: ShardID) {\n  // Underscores have a special meaning in LIKE values\n  // so they have to be escaped.\n  return `${replicationSlotPrefix(shard)}%`.replaceAll('_', '\\\\_');\n}\n\nexport function newReplicationSlot(shard: ShardID) {\n  return replicationSlotPrefix(shard) + Date.now();\n}\n\nfunction defaultPublicationName(appID: string, shardID: string | number) {\n  return `_${appID}_public_${shardID}`;\n}\n\nexport function metadataPublicationName(\n  appID: string,\n  shardID: string | number,\n) {\n  return `_${appID}_metadata_${shardID}`;\n}\n\n// The GLOBAL_SETUP must be idempotent as it can be run multiple times for different shards.\nfunction globalSetup(appID: AppID): string {\n  const app = id(appSchema(appID));\n\n  return /*sql*/ `\n  CREATE SCHEMA IF NOT EXISTS ${app};\n\n  CREATE TABLE IF NOT EXISTS ${app}.permissions (\n    \"permissions\" JSONB,\n    \"hash\"        TEXT,\n\n    -- Ensure that there is only a single row in the table.\n    -- Application code can be agnostic to this column, and\n    -- simply invoke UPDATE statements on the version columns.\n    \"lock\" BOOL PRIMARY KEY DEFAULT true CHECK (lock)\n  );\n\n  CREATE OR REPLACE FUNCTION ${app}.set_permissions_hash()\n  RETURNS TRIGGER AS $$\n  BEGIN\n      NEW.hash = md5(NEW.permissions::text);\n      RETURN NEW;\n  END;\n  $$ LANGUAGE plpgsql;\n\n  CREATE OR REPLACE TRIGGER on_set_permissions \n    BEFORE INSERT OR UPDATE ON ${app}.permissions\n    FOR EACH ROW\n    EXECUTE FUNCTION ${app}.set_permissions_hash();\n\n  INSERT INTO ${app}.permissions (permissions) VALUES (NULL) ON CONFLICT DO NOTHING;\n`;\n}\n\nexport async function ensureGlobalTables(db: PostgresDB, appID: AppID) {\n  await db.unsafe(globalSetup(appID));\n}\n\nexport function getClientsTableDefinition(schema: string) {\n  return /*sql*/ `\n  CREATE TABLE ${schema}.\"clients\" (\n    \"clientGroupID\"  TEXT NOT NULL,\n    \"clientID\"       TEXT NOT NULL,\n    \"lastMutationID\" BIGINT NOT NULL,\n    \"userID\"         TEXT,\n    PRIMARY KEY(\"clientGroupID\", \"clientID\")\n  );`;\n}\n\n/**\n * Tracks the results of mutations.\n * 1. It is an error for the same mutation ID to be used twice.\n * 2. The result is JSONB to allow for arbitrary results.\n *\n * The tables must be cleaned up as the clients\n * receive the mutation responses and as clients are removed.\n */\nexport function getMutationsTableDefinition(schema: string) {\n  return /*sql*/ `\n  CREATE TABLE ${schema}.\"mutations\" (\n    \"clientGroupID\"  TEXT NOT NULL,\n    \"clientID\"       TEXT NOT NULL,\n    \"mutationID\"     BIGINT NOT NULL,\n    \"result\"         JSON NOT NULL,\n    PRIMARY KEY(\"clientGroupID\", \"clientID\", \"mutationID\")\n  );`;\n}\n\nexport const SHARD_CONFIG_TABLE = 'shardConfig';\n\nexport function shardSetup(\n  shardConfig: ShardConfig,\n  metadataPublication: string,\n): string {\n  const app = id(appSchema(shardConfig));\n  const shard = id(upstreamSchema(shardConfig));\n\n  const pubs = shardConfig.publications.toSorted();\n  assert(\n    pubs.includes(metadataPublication),\n    () => `Publications must include ${metadataPublication}`,\n  );\n\n  return /*sql*/ `\n  CREATE SCHEMA IF NOT EXISTS ${shard};\n\n  ${getClientsTableDefinition(shard)}\n  ${getMutationsTableDefinition(shard)}\n\n  DROP PUBLICATION IF EXISTS ${id(metadataPublication)};\n  CREATE PUBLICATION ${id(metadataPublication)}\n    FOR TABLE ${app}.\"permissions\", TABLE ${shard}.\"clients\", ${shard}.\"mutations\";\n\n  CREATE TABLE ${shard}.\"${SHARD_CONFIG_TABLE}\" (\n    \"publications\"  TEXT[] NOT NULL,\n    \"ddlDetection\"  BOOL NOT NULL,\n\n    -- Ensure that there is only a single row in the table.\n    \"lock\" BOOL PRIMARY KEY DEFAULT true CHECK (lock)\n  );\n\n  INSERT INTO ${shard}.\"${SHARD_CONFIG_TABLE}\" (\n      \"publications\",\n      \"ddlDetection\" \n    ) VALUES (\n      ARRAY[${literal(pubs)}], \n      false  -- set in SAVEPOINT with triggerSetup() statements\n    );\n\n  CREATE TABLE ${shard}.replicas (\n    \"slot\"               TEXT PRIMARY KEY,\n    \"version\"            TEXT NOT NULL,\n    \"initialSchema\"      JSON NOT NULL,\n    \"initialSyncContext\" JSON,\n    \"subscriberContext\"  JSON\n  );\n  `;\n}\n\nexport function dropShard(appID: string, shardID: string | number): string {\n  const schema = `${appID}_${shardID}`;\n  const metadataPublication = metadataPublicationName(appID, shardID);\n  const defaultPublication = defaultPublicationName(appID, shardID);\n\n  // DROP SCHEMA ... CASCADE does not drop dependent PUBLICATIONS,\n  // so PUBLICATIONs must be dropped explicitly.\n  return /*sql*/ `\n    DROP PUBLICATION IF EXISTS ${id(defaultPublication)};\n    DROP PUBLICATION IF EXISTS ${id(metadataPublication)};\n    DROP SCHEMA IF EXISTS ${id(schema)} CASCADE;\n  `;\n}\n\nconst internalShardConfigSchema = v.object({\n  publications: v.array(v.string()),\n  ddlDetection: v.boolean(),\n});\n\nexport type InternalShardConfig = v.Infer<typeof internalShardConfigSchema>;\n\nconst replicaSchema = internalShardConfigSchema.extend({\n  slot: v.string(),\n  version: v.string(),\n  initialSchema: publishedSchema,\n  initialSyncContext: jsonObjectSchema.nullable(),\n  subscriberContext: jsonObjectSchema.nullable(),\n});\n\nexport type Replica = v.Infer<typeof replicaSchema>;\n\n// triggerSetup is run separately in a sub-transaction (i.e. SAVEPOINT) so\n// that a failure (e.g. due to lack of superuser permissions) can be handled\n// by continuing in a degraded mode (ddlDetection = false).\nfunction triggerSetup(shard: ShardConfig): string {\n  const schema = id(upstreamSchema(shard));\n  return (\n    createEventTriggerStatements(shard) +\n    /*sql*/ `UPDATE ${schema}.\"shardConfig\" SET \"ddlDetection\" = true;`\n  );\n}\n\n// Called in initial-sync to store the exact schema that was initially synced.\nexport async function addReplica(\n  sql: PostgresDB,\n  shard: ShardID,\n  slot: string,\n  replicaVersion: string,\n  {tables, indexes}: PublishedSchema,\n  initialSyncContext: JSONObject,\n) {\n  const schema = upstreamSchema(shard);\n  const synced: PublishedSchema = {tables, indexes};\n  await sql`\n    INSERT INTO ${sql(schema)}.replicas\n      (\"slot\", \"version\", \"initialSchema\", \"initialSyncContext\")\n      VALUES (${slot}, ${replicaVersion}, ${synced}, ${initialSyncContext})`;\n}\n\nexport async function getReplicaAtVersion(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardID,\n  replicaVersion: string,\n  context?: JSONObject,\n): Promise<Replica | null> {\n  const schema = sql(upstreamSchema(shard));\n  const result = await sql`\n    SELECT\n      replicas.\"slot\",\n      replicas.\"version\",\n      replicas.\"initialSchema\",\n      replicas.\"initialSyncContext\",\n      replicas.\"subscriberContext\",\n      \"shardConfig\".\"publications\",\n      \"shardConfig\".\"ddlDetection\"\n    FROM ${schema}.replicas JOIN ${schema}.\"shardConfig\" ON true\n      WHERE version = ${replicaVersion};\n  `;\n  if (result.length === 0) {\n    // log out all the replicas and the joined shardConfig\n    const allReplicas = await sql`\n      SELECT slot, version, \"initialSyncContext\", \"subscriberContext\" \n        FROM ${schema}.replicas`;\n    lc.info?.(\n      `Replica ${replicaVersion} ` +\n        (context ? `(context: ${stringify(context)}) ` : '') +\n        `not found in: ${stringify(allReplicas)}`,\n    );\n    return null;\n  }\n  return v.parse(result[0], replicaSchema, 'passthrough');\n}\n\nexport async function getInternalShardConfig(\n  sql: PostgresDB,\n  shard: ShardID,\n): Promise<InternalShardConfig> {\n  const result = await sql`\n    SELECT \"publications\", \"ddlDetection\"\n      FROM ${sql(upstreamSchema(shard))}.\"shardConfig\";\n  `;\n  assert(\n    result.length === 1,\n    () => `Expected exactly one shardConfig row, got ${result.length}`,\n  );\n  return v.parse(result[0], internalShardConfigSchema, 'passthrough');\n}\n\n/**\n * Sets up and returns all publications (including internal ones) for\n * the given shard.\n */\nexport async function setupTablesAndReplication(\n  lc: LogContext,\n  sql: PostgresTransaction,\n  requested: ShardConfig,\n) {\n  const {publications} = requested;\n  // Validate requested publications.\n  for (const pub of publications) {\n    validatePublicationName(pub);\n    if (pub.startsWith('_')) {\n      throw new Error(\n        `Publication names starting with \"_\" are reserved for internal use.\\n` +\n          `Please use a different name for publication \"${pub}\".`,\n      );\n    }\n  }\n  const allPublications: string[] = [];\n\n  // Setup application publications.\n  if (publications.length) {\n    const results = await sql<{pubname: string}[]>`\n    SELECT pubname from pg_publication WHERE pubname IN ${sql(\n      publications,\n    )}`.values();\n\n    if (results.length !== publications.length) {\n      throw new Error(\n        `Unknown or invalid publications. Specified: [${publications}]. Found: [${results.flat()}]`,\n      );\n    }\n    allPublications.push(...publications);\n  } else {\n    const defaultPublication = defaultPublicationName(\n      requested.appID,\n      requested.shardNum,\n    );\n    await sql`\n      DROP PUBLICATION IF EXISTS ${sql(defaultPublication)}`;\n    await sql`\n      CREATE PUBLICATION ${sql(defaultPublication)} \n        FOR TABLES IN SCHEMA public\n        WITH (publish_via_partition_root = true)`;\n    allPublications.push(defaultPublication);\n  }\n\n  const metadataPublication = metadataPublicationName(\n    requested.appID,\n    requested.shardNum,\n  );\n  allPublications.push(metadataPublication);\n\n  const shard = {...requested, publications: allPublications};\n\n  // Setup the global tables and shard tables / publications.\n  await sql.unsafe(globalSetup(shard) + shardSetup(shard, metadataPublication));\n\n  const pubs = await getPublicationInfo(sql, allPublications);\n  await replicaIdentitiesForTablesWithoutPrimaryKeys(pubs)?.apply(lc, sql);\n\n  await setupTriggers(lc, sql, shard);\n}\n\nexport async function setupTriggers(\n  lc: LogContext,\n  tx: PostgresTransaction,\n  shard: ShardConfig,\n) {\n  const schema = upstreamSchema(shard);\n  const [{ddlDetection}] = await tx<InternalShardConfig[]> /*sql*/ `\n    SELECT \"ddlDetection\" FROM ${tx(schema)}.\"shardConfig\"`;\n\n  // The functions invoked by event triggers are installed even if\n  // event triggers are not supported/allowed by the db provider.\n  // This allows users to manually invoke the update_schemas() function\n  // as a workaround.\n  await tx.unsafe(createEventFunctionStatements(shard));\n  try {\n    await tx.savepoint(sub => sub.unsafe(triggerSetup(shard)));\n  } catch (e) {\n    if (ddlDetection) {\n      // If ddlDetection has already been enabled, subsequent failures to\n      // upgrade the trigger should be propagated rather than swallowed.\n      throw e;\n    }\n    if (\n      !(\n        e instanceof postgres.PostgresError &&\n        e.code === PG_INSUFFICIENT_PRIVILEGE\n      )\n    ) {\n      throw e;\n    }\n    // If triggerSetup() fails, replication continues in ddlDetection=false mode.\n    lc.warn?.(\n      `Unable to create event triggers for schema change detection:\\n\\n` +\n        `\"${e.hint ?? e.message}\"\\n\\n` +\n        `Proceeding in degraded mode: schema changes will halt replication,\\n` +\n        `requiring the replica to be reset (manually or with --auto-reset).`,\n    );\n  }\n}\n\nexport function validatePublications(\n  lc: LogContext,\n  published: PublicationInfo,\n) {\n  // Verify that all publications export the proper events.\n  published.publications.forEach(pub => {\n    if (\n      !pub.pubinsert ||\n      !pub.pubupdate ||\n      !pub.pubdelete ||\n      !pub.pubtruncate\n    ) {\n      // TODO: Make APIError?\n      throw new Error(\n        `PUBLICATION ${pub.pubname} must publish insert, update, delete, and truncate`,\n      );\n    }\n  });\n\n  published.tables.forEach(table => validate(lc, table));\n}\n\ntype ReplicaIdentities = {\n  apply(lc: LogContext, db: PostgresDB): Promise<void>;\n};\n\nexport function replicaIdentitiesForTablesWithoutPrimaryKeys(\n  pubs: PublishedSchema,\n): ReplicaIdentities | undefined {\n  const replicaIdentities: {\n    schema: string;\n    tableName: string;\n    indexName: string;\n  }[] = [];\n  for (const table of pubs.tables) {\n    if (!table.primaryKey?.length && table.replicaIdentity === Default) {\n      // Look for an index that can serve as the REPLICA IDENTITY USING INDEX. It must be:\n      // - UNIQUE\n      // - NOT NULL columns\n      // - not deferrable (i.e. isImmediate)\n      // - not partial (are already filtered out)\n      //\n      // https://www.postgresql.org/docs/current/sql-altertable.html#SQL-ALTERTABLE-REPLICA-IDENTITY\n      const {schema, name: tableName} = table;\n      for (const {columns, name: indexName} of pubs.indexes.filter(\n        idx =>\n          idx.schema === schema &&\n          idx.tableName === tableName &&\n          idx.unique &&\n          idx.isImmediate,\n      )) {\n        if (Object.keys(columns).some(col => !table.columns[col].notNull)) {\n          continue; // Only indexes with all NOT NULL columns are suitable.\n        }\n        replicaIdentities.push({schema, tableName, indexName});\n        break;\n      }\n    }\n  }\n\n  if (replicaIdentities.length === 0) {\n    return undefined;\n  }\n  return {\n    apply: async (lc: LogContext, sql: PostgresDB) => {\n      for (const {schema, tableName, indexName} of replicaIdentities) {\n        lc.info?.(\n          `setting \"${indexName}\" as the REPLICA IDENTITY for \"${tableName}\"`,\n        );\n        await sql`\n        ALTER TABLE ${sql(schema)}.${sql(tableName)} \n          REPLICA IDENTITY USING INDEX ${sql(indexName)}`;\n      }\n    },\n  };\n}\n"],"mappings":";;;;;;;;;;;;;;;;AAgCA,IAAM,yBAAyB;;;;;;AAO/B,SAAgB,wBAAwB,MAAoB;AAC1D,KAAI,CAAC,uBAAuB,KAAK,KAAK,CACpC,OAAM,IAAI,MACR,6BAA6B,KAAK,gHAEnC;AAEH,KAAI,KAAK,SAAS,GAChB,OAAM,IAAI,MACR,qBAAqB,KAAK,uDAC3B;;AAIL,SAAgB,0BAA0B,EAAC,SAAe;AACxD,QAAO,IAAI,MAAM;;AAGnB,SAAgB,sBAAsB,EAAC,OAAO,YAAoB;AAChE,QAAO,GAAG,MAAM,GAAG;;AAGrB,SAAgB,sBAAsB,OAAgB;CACpD,MAAM,EAAC,OAAO,aAAY,MAAM,MAAM;AACtC,QAAO,GAAG,MAAM,GAAG,SAAS;;;;;;AAO9B,SAAgB,0BAA0B,OAAgB;AAGxD,QAAO,GAAG,sBAAsB,MAAM,CAAC,GAAG,WAAW,KAAK,MAAM;;AAGlE,SAAgB,mBAAmB,OAAgB;AACjD,QAAO,sBAAsB,MAAM,GAAG,KAAK,KAAK;;AAGlD,SAAS,uBAAuB,OAAe,SAA0B;AACvE,QAAO,IAAI,MAAM,UAAU;;AAG7B,SAAgB,wBACd,OACA,SACA;AACA,QAAO,IAAI,MAAM,YAAY;;AAI/B,SAAS,YAAY,OAAsB;CACzC,MAAM,MAAM,GAAG,UAAU,MAAM,CAAC;AAEhC,QAAe;gCACe,IAAI;;+BAEL,IAAI;;;;;;;;;;+BAUJ,IAAI;;;;;;;;;iCASF,IAAI;;uBAEd,IAAI;;gBAEX,IAAI;;;AAIpB,eAAsB,mBAAmB,IAAgB,OAAc;AACrE,OAAM,GAAG,OAAO,YAAY,MAAM,CAAC;;AAGrC,SAAgB,0BAA0B,QAAgB;AACxD,QAAe;iBACA,OAAO;;;;;;;;;;;;;;;;AAiBxB,SAAgB,4BAA4B,QAAgB;AAC1D,QAAe;iBACA,OAAO;;;;;;;;AASxB,IAAa,qBAAqB;AAElC,SAAgB,WACd,aACA,qBACQ;CACR,MAAM,MAAM,GAAG,UAAU,YAAY,CAAC;CACtC,MAAM,QAAQ,GAAG,eAAe,YAAY,CAAC;CAE7C,MAAM,OAAO,YAAY,aAAa,UAAU;AAChD,QACE,KAAK,SAAS,oBAAoB,QAC5B,6BAA6B,sBACpC;AAED,QAAe;gCACe,MAAM;;IAElC,0BAA0B,MAAM,CAAC;IACjC,4BAA4B,MAAM,CAAC;;+BAER,GAAG,oBAAoB,CAAC;uBAChC,GAAG,oBAAoB,CAAC;gBAC/B,IAAI,wBAAwB,MAAM,cAAc,MAAM;;iBAErD,MAAM,IAAI,mBAAmB;;;;;;;;gBAQ9B,MAAM,IAAI,mBAAmB;;;;cAI/B,QAAQ,KAAK,CAAC;;;;iBAIX,MAAM;;;;;;;;;AAUvB,SAAgB,UAAU,OAAe,SAAkC;CACzE,MAAM,SAAS,GAAG,MAAM,GAAG;CAC3B,MAAM,sBAAsB,wBAAwB,OAAO,QAAQ;AAKnE,QAAe;iCACgB,GALJ,uBAAuB,OAAO,QAAQ,CAKZ,CAAC;iCACvB,GAAG,oBAAoB,CAAC;4BAC7B,GAAG,OAAO,CAAC;;;AAIvC,IAAM,4BAA4B,eAAE,OAAO;CACzC,cAAc,eAAE,MAAM,eAAE,QAAQ,CAAC;CACjC,cAAc,eAAE,SAAS;CAC1B,CAAC;AAIF,IAAM,gBAAgB,0BAA0B,OAAO;CACrD,MAAM,eAAE,QAAQ;CAChB,SAAS,eAAE,QAAQ;CACnB,eAAe;CACf,oBAAoB,iBAAiB,UAAU;CAC/C,mBAAmB,iBAAiB,UAAU;CAC/C,CAAC;AAOF,SAAS,aAAa,OAA4B;CAChD,MAAM,SAAS,GAAG,eAAe,MAAM,CAAC;AACxC,QACE,6BAA6B,MAAM,GAC3B,UAAU,OAAO;;AAK7B,eAAsB,WACpB,KACA,OACA,MACA,gBACA,EAAC,QAAQ,WACT,oBACA;CACA,MAAM,SAAS,eAAe,MAAM;CACpC,MAAM,SAA0B;EAAC;EAAQ;EAAQ;AACjD,OAAM,GAAG;kBACO,IAAI,OAAO,CAAC;;gBAEd,KAAK,IAAI,eAAe,IAAI,OAAO,IAAI,mBAAmB;;AAG1E,eAAsB,oBACpB,IACA,KACA,OACA,gBACA,SACyB;CACzB,MAAM,SAAS,IAAI,eAAe,MAAM,CAAC;CACzC,MAAM,SAAS,MAAM,GAAG;;;;;;;;;WASf,OAAO,iBAAiB,OAAO;wBAClB,eAAe;;AAErC,KAAI,OAAO,WAAW,GAAG;EAEvB,MAAM,cAAc,MAAM,GAAG;;eAElB,OAAO;AAClB,KAAG,OACD,WAAW,eAAe,MACvB,UAAU,aAAa,UAAU,QAAQ,CAAC,MAAM,MACjD,iBAAiB,UAAU,YAAY,GAC1C;AACD,SAAO;;AAET,QAAO,MAAQ,OAAO,IAAI,eAAe,cAAc;;AAGzD,eAAsB,uBACpB,KACA,OAC8B;CAC9B,MAAM,SAAS,MAAM,GAAG;;aAEb,IAAI,eAAe,MAAM,CAAC,CAAC;;AAEtC,QACE,OAAO,WAAW,SACZ,6CAA6C,OAAO,SAC3D;AACD,QAAO,MAAQ,OAAO,IAAI,2BAA2B,cAAc;;;;;;AAOrE,eAAsB,0BACpB,IACA,KACA,WACA;CACA,MAAM,EAAC,iBAAgB;AAEvB,MAAK,MAAM,OAAO,cAAc;AAC9B,0BAAwB,IAAI;AAC5B,MAAI,IAAI,WAAW,IAAI,CACrB,OAAM,IAAI,MACR,oHACkD,IAAI,IACvD;;CAGL,MAAM,kBAA4B,EAAE;AAGpC,KAAI,aAAa,QAAQ;EACvB,MAAM,UAAU,MAAM,GAAwB;0DACQ,IACpD,aACD,GAAG,QAAQ;AAEZ,MAAI,QAAQ,WAAW,aAAa,OAClC,OAAM,IAAI,MACR,gDAAgD,aAAa,aAAa,QAAQ,MAAM,CAAC,GAC1F;AAEH,kBAAgB,KAAK,GAAG,aAAa;QAChC;EACL,MAAM,qBAAqB,uBACzB,UAAU,OACV,UAAU,SACX;AACD,QAAM,GAAG;mCACsB,IAAI,mBAAmB;AACtD,QAAM,GAAG;2BACc,IAAI,mBAAmB,CAAC;;;AAG/C,kBAAgB,KAAK,mBAAmB;;CAG1C,MAAM,sBAAsB,wBAC1B,UAAU,OACV,UAAU,SACX;AACD,iBAAgB,KAAK,oBAAoB;CAEzC,MAAM,QAAQ;EAAC,GAAG;EAAW,cAAc;EAAgB;AAG3D,OAAM,IAAI,OAAO,YAAY,MAAM,GAAG,WAAW,OAAO,oBAAoB,CAAC;AAG7E,OAAM,6CADO,MAAM,mBAAmB,KAAK,gBAAgB,CACH,EAAE,MAAM,IAAI,IAAI;AAExE,OAAM,cAAc,IAAI,KAAK,MAAM;;AAGrC,eAAsB,cACpB,IACA,IACA,OACA;CAEA,MAAM,CAAC,EAAC,kBAAiB,MAAM,EAAkC;iCAClC,GAFhB,eAAe,MAAM,CAEK,CAAC;AAM1C,OAAM,GAAG,OAAO,8BAA8B,MAAM,CAAC;AACrD,KAAI;AACF,QAAM,GAAG,WAAU,QAAO,IAAI,OAAO,aAAa,MAAM,CAAC,CAAC;UACnD,GAAG;AACV,MAAI,aAGF,OAAM;AAER,MACE,EACE,aAAa,SAAS,iBACtB,EAAE,SAAS,2BAGb,OAAM;AAGR,KAAG,OACD,oEACM,EAAE,QAAQ,EAAE,QAAQ,6IAG3B;;;AAIL,SAAgB,qBACd,IACA,WACA;AAEA,WAAU,aAAa,SAAQ,QAAO;AACpC,MACE,CAAC,IAAI,aACL,CAAC,IAAI,aACL,CAAC,IAAI,aACL,CAAC,IAAI,YAGL,OAAM,IAAI,MACR,eAAe,IAAI,QAAQ,oDAC5B;GAEH;AAEF,WAAU,OAAO,SAAQ,UAAS,SAAS,IAAI,MAAM,CAAC;;AAOxD,SAAgB,6CACd,MAC+B;CAC/B,MAAM,oBAIA,EAAE;AACR,MAAK,MAAM,SAAS,KAAK,OACvB,KAAI,CAAC,MAAM,YAAY,UAAU,MAAM,oBAAA,KAA6B;EAQlE,MAAM,EAAC,QAAQ,MAAM,cAAa;AAClC,OAAK,MAAM,EAAC,SAAS,MAAM,eAAc,KAAK,QAAQ,QACpD,QACE,IAAI,WAAW,UACf,IAAI,cAAc,aAClB,IAAI,UACJ,IAAI,YACP,EAAE;AACD,OAAI,OAAO,KAAK,QAAQ,CAAC,MAAK,QAAO,CAAC,MAAM,QAAQ,KAAK,QAAQ,CAC/D;AAEF,qBAAkB,KAAK;IAAC;IAAQ;IAAW;IAAU,CAAC;AACtD;;;AAKN,KAAI,kBAAkB,WAAW,EAC/B;AAEF,QAAO,EACL,OAAO,OAAO,IAAgB,QAAoB;AAChD,OAAK,MAAM,EAAC,QAAQ,WAAW,eAAc,mBAAmB;AAC9D,MAAG,OACD,YAAY,UAAU,iCAAiC,UAAU,GAClE;AACD,SAAM,GAAG;sBACK,IAAI,OAAO,CAAC,GAAG,IAAI,UAAU,CAAC;yCACX,IAAI,UAAU;;IAGpD"}
+{"version":3,"file":"shard.js","names":[],"sources":["../../../../../../../../zero-cache/src/services/change-source/pg/schema/shard.ts"],"sourcesContent":["import {PG_INSUFFICIENT_PRIVILEGE} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport {literal} from 'pg-format';\nimport postgres from 'postgres';\nimport {assert} from '../../../../../../shared/src/asserts.ts';\nimport {\n  jsonObjectSchema,\n  stringify,\n  type JSONObject,\n} from '../../../../../../shared/src/bigint-json.ts';\nimport * as v from '../../../../../../shared/src/valita.ts';\nimport {Default} from '../../../../db/postgres-replica-identity-enum.ts';\nimport type {PostgresDB, PostgresTransaction} from '../../../../types/pg.ts';\nimport type {AppID, ShardConfig, ShardID} from '../../../../types/shards.ts';\nimport {appSchema, check, upstreamSchema} from '../../../../types/shards.ts';\nimport {id} from '../../../../types/sql.ts';\nimport {\n  createEventFunctionStatements,\n  createEventTriggerStatements,\n} from './ddl.ts';\nimport {\n  getPublicationInfo,\n  publishedSchema,\n  type PublicationInfo,\n  type PublishedSchema,\n} from './published.ts';\nimport {validate} from './validation.ts';\n\n/**\n * PostgreSQL unquoted identifiers must start with a letter or underscore\n * and contain only letters, digits, and underscores.\n */\nconst VALID_PUBLICATION_NAME = /^[a-zA-Z_][a-zA-Z0-9_]*$/;\n\n/**\n * Validates that a publication name is a valid PostgreSQL identifier.\n * This provides defense-in-depth against SQL injection when publication\n * names are used in replication commands.\n */\nexport function validatePublicationName(name: string): void {\n  if (!VALID_PUBLICATION_NAME.test(name)) {\n    throw new Error(\n      `Invalid publication name \"${name}\". Publication names must start with a letter or underscore ` +\n        `and contain only letters, digits, and underscores.`,\n    );\n  }\n  if (name.length > 63) {\n    throw new Error(\n      `Publication name \"${name}\" exceeds PostgreSQL's 63-character identifier limit.`,\n    );\n  }\n}\n\nexport function internalPublicationPrefix({appID}: AppID) {\n  return `_${appID}_`;\n}\n\nexport function legacyReplicationSlot({appID, shardNum}: ShardID) {\n  return `${appID}_${shardNum}`;\n}\n\nexport function replicationSlotPrefix(shard: ShardID) {\n  const {appID, shardNum} = check(shard);\n  return `${appID}_${shardNum}_`;\n}\n\n/**\n * An expression used to match replication slots in the shard\n * in a Postgres `LIKE` operator.\n */\nexport function replicationSlotExpression(shard: ShardID) {\n  // Underscores have a special meaning in LIKE values\n  // so they have to be escaped.\n  return `${replicationSlotPrefix(shard)}%`.replaceAll('_', '\\\\_');\n}\n\nfunction defaultPublicationName(appID: string, shardID: string | number) {\n  return `_${appID}_public_${shardID}`;\n}\n\nexport function metadataPublicationName(\n  appID: string,\n  shardID: string | number,\n) {\n  return `_${appID}_metadata_${shardID}`;\n}\n\n// The GLOBAL_SETUP must be idempotent as it can be run multiple times for different shards.\nfunction globalSetup(appID: AppID): string {\n  const app = id(appSchema(appID));\n\n  return /*sql*/ `\n  CREATE SCHEMA IF NOT EXISTS ${app};\n\n  CREATE TABLE IF NOT EXISTS ${app}.permissions (\n    \"permissions\" JSONB,\n    \"hash\"        TEXT,\n\n    -- Ensure that there is only a single row in the table.\n    -- Application code can be agnostic to this column, and\n    -- simply invoke UPDATE statements on the version columns.\n    \"lock\" BOOL PRIMARY KEY DEFAULT true CHECK (lock)\n  );\n\n  CREATE OR REPLACE FUNCTION ${app}.set_permissions_hash()\n  RETURNS TRIGGER AS $$\n  BEGIN\n      NEW.hash = md5(NEW.permissions::text);\n      RETURN NEW;\n  END;\n  $$ LANGUAGE plpgsql;\n\n  CREATE OR REPLACE TRIGGER on_set_permissions \n    BEFORE INSERT OR UPDATE ON ${app}.permissions\n    FOR EACH ROW\n    EXECUTE FUNCTION ${app}.set_permissions_hash();\n\n  INSERT INTO ${app}.permissions (permissions) VALUES (NULL) ON CONFLICT DO NOTHING;\n`;\n}\n\nexport async function ensureGlobalTables(db: PostgresDB, appID: AppID) {\n  await db.unsafe(globalSetup(appID));\n}\n\nexport function getClientsTableDefinition(schema: string) {\n  return /*sql*/ `\n  CREATE TABLE ${schema}.\"clients\" (\n    \"clientGroupID\"  TEXT NOT NULL,\n    \"clientID\"       TEXT NOT NULL,\n    \"lastMutationID\" BIGINT NOT NULL,\n    \"userID\"         TEXT,\n    PRIMARY KEY(\"clientGroupID\", \"clientID\")\n  );`;\n}\n\n/**\n * Tracks the results of mutations.\n * 1. It is an error for the same mutation ID to be used twice.\n * 2. The result is JSONB to allow for arbitrary results.\n *\n * The tables must be cleaned up as the clients\n * receive the mutation responses and as clients are removed.\n */\nexport function getMutationsTableDefinition(schema: string) {\n  return /*sql*/ `\n  CREATE TABLE ${schema}.\"mutations\" (\n    \"clientGroupID\"  TEXT NOT NULL,\n    \"clientID\"       TEXT NOT NULL,\n    \"mutationID\"     BIGINT NOT NULL,\n    \"result\"         JSON NOT NULL,\n    PRIMARY KEY(\"clientGroupID\", \"clientID\", \"mutationID\")\n  );`;\n}\n\nexport const SHARD_CONFIG_TABLE = 'shardConfig';\n\nexport function shardSetup(\n  shardConfig: ShardConfig,\n  metadataPublication: string,\n): string {\n  const app = id(appSchema(shardConfig));\n  const shard = id(upstreamSchema(shardConfig));\n\n  const pubs = shardConfig.publications.toSorted();\n  assert(\n    pubs.includes(metadataPublication),\n    () => `Publications must include ${metadataPublication}`,\n  );\n\n  return /*sql*/ `\n  CREATE SCHEMA IF NOT EXISTS ${shard};\n\n  ${getClientsTableDefinition(shard)}\n  ${getMutationsTableDefinition(shard)}\n\n  DROP PUBLICATION IF EXISTS ${id(metadataPublication)};\n  CREATE PUBLICATION ${id(metadataPublication)}\n    FOR TABLE ${app}.\"permissions\", TABLE ${shard}.\"clients\", ${shard}.\"mutations\";\n\n  CREATE TABLE ${shard}.\"${SHARD_CONFIG_TABLE}\" (\n    \"publications\"  TEXT[] NOT NULL,\n    \"ddlDetection\"  BOOL NOT NULL,\n\n    -- Ensure that there is only a single row in the table.\n    \"lock\" BOOL PRIMARY KEY DEFAULT true CHECK (lock)\n  );\n\n  INSERT INTO ${shard}.\"${SHARD_CONFIG_TABLE}\" (\n      \"publications\",\n      \"ddlDetection\" \n    ) VALUES (\n      ARRAY[${literal(pubs)}], \n      false  -- set in SAVEPOINT with triggerSetup() statements\n    );\n\n  CREATE TABLE ${shard}.replicas (\n    -- The DEFAULT exists purely for backwards compatibility support.\n    -- New code always specifies a value based on Date.now().\n    \"id\"                 TEXT PRIMARY KEY DEFAULT replace(gen_random_uuid()::text, '-', ''),\n    \"rank\"               BIGSERIAL,\n    \"slot\"               TEXT NOT NULL,\n    \"version\"            TEXT NOT NULL,\n    \"initialSchema\"      JSON,  -- set after initial sync\n    \"initialSyncContext\" JSON,\n    \"subscriberContext\"  JSON\n  );\n  `;\n}\n\nexport function dropShard(appID: string, shardID: string | number): string {\n  const schema = `${appID}_${shardID}`;\n  const metadataPublication = metadataPublicationName(appID, shardID);\n  const defaultPublication = defaultPublicationName(appID, shardID);\n\n  // DROP SCHEMA ... CASCADE does not drop dependent PUBLICATIONS,\n  // so PUBLICATIONs must be dropped explicitly.\n  return /*sql*/ `\n    DROP PUBLICATION IF EXISTS ${id(defaultPublication)};\n    DROP PUBLICATION IF EXISTS ${id(metadataPublication)};\n    DROP SCHEMA IF EXISTS ${id(schema)} CASCADE;\n  `;\n}\n\nconst internalShardConfigSchema = v.object({\n  publications: v.array(v.string()),\n  ddlDetection: v.boolean(),\n});\n\nexport type InternalShardConfig = v.Infer<typeof internalShardConfigSchema>;\n\nconst replicaSchema = internalShardConfigSchema.extend({\n  id: v.string(),\n  rank: v.bigint(),\n  slot: v.string(),\n  version: v.string(),\n  initialSchema: publishedSchema,\n  initialSyncContext: jsonObjectSchema.nullable(),\n  subscriberContext: jsonObjectSchema.nullable(),\n});\n\nexport type Replica = v.Infer<typeof replicaSchema>;\n\n// triggerSetup is run separately in a sub-transaction (i.e. SAVEPOINT) so\n// that a failure (e.g. due to lack of superuser permissions) can be handled\n// by continuing in a degraded mode (ddlDetection = false).\nfunction triggerSetup(shard: ShardConfig): string {\n  const schema = id(upstreamSchema(shard));\n  return (\n    createEventTriggerStatements(shard) +\n    /*sql*/ `UPDATE ${schema}.\"shardConfig\" SET \"ddlDetection\" = true;`\n  );\n}\n\n/**\n * Creates a new replica to mark it as the owner of a specified `slot`.\n * This should be done with an advisory lock for replica/slot management\n * to ensure that the slot does not get dropped by concurrent cleanup\n * logic.\n *\n * Once initial sync is complete, {@link initReplica} should be called to\n * make the replica usable for incremental sync.\n */\nexport async function createReplica(\n  sql: PostgresDB,\n  shard: ShardID,\n  id: string,\n  slot: string,\n  replicaVersion: string,\n) {\n  const schema = upstreamSchema(shard);\n  const values = {id, slot, version: replicaVersion};\n  await sql`INSERT INTO ${sql(schema)}.replicas ${sql(values)}`;\n}\n\n// Called in initial-sync to store the exact schema that was initially synced.\nexport async function initReplica(\n  sql: PostgresDB,\n  shard: ShardID,\n  id: string,\n  {tables, indexes}: PublishedSchema,\n  initialSyncContext: JSONObject,\n) {\n  const schema = upstreamSchema(shard);\n  const synced: PublishedSchema = {tables, indexes};\n  const values = {initialSchema: synced, initialSyncContext};\n  await sql`UPDATE ${sql(schema)}.replicas SET ${sql(values)} WHERE id = ${id}`;\n}\n\n/**\n * Gets the latest initialized replica at the specified version.\n */\nexport async function getReplicaAtVersion(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardID,\n  replicaVersion: string,\n  context?: JSONObject,\n): Promise<Replica | null> {\n  const schema = sql(upstreamSchema(shard));\n  const result = await sql`\n    SELECT\n      replicas.\"id\",\n      replicas.\"rank\",\n      replicas.\"slot\",\n      replicas.\"version\",\n      replicas.\"initialSchema\",\n      replicas.\"initialSyncContext\",\n      replicas.\"subscriberContext\",\n      \"shardConfig\".\"publications\",\n      \"shardConfig\".\"ddlDetection\"\n    FROM ${schema}.replicas JOIN ${schema}.\"shardConfig\" ON true\n      WHERE version = ${replicaVersion} AND \"initialSyncContext\" IS NOT NULL\n      ORDER BY rank DESC LIMIT 1;\n  `;\n  if (result.length === 0) {\n    // log out all the replicas and the joined shardConfig\n    const allReplicas = await sql`\n      SELECT id, slot, version, \"initialSyncContext\", \"subscriberContext\" \n        FROM ${schema}.replicas`;\n    lc.info?.(\n      `Replica ${replicaVersion} ` +\n        (context ? `(context: ${stringify(context)}) ` : '') +\n        `not found in: ${stringify(allReplicas)}`,\n    );\n    return null;\n  }\n  return v.parse(result[0], replicaSchema, 'passthrough');\n}\n\nexport async function getInternalShardConfig(\n  sql: PostgresDB,\n  shard: ShardID,\n): Promise<InternalShardConfig> {\n  const result = await sql`\n    SELECT \"publications\", \"ddlDetection\"\n      FROM ${sql(upstreamSchema(shard))}.\"shardConfig\";\n  `;\n  assert(\n    result.length === 1,\n    () => `Expected exactly one shardConfig row, got ${result.length}`,\n  );\n  return v.parse(result[0], internalShardConfigSchema, 'passthrough');\n}\n\n/**\n * Sets up and returns all publications (including internal ones) for\n * the given shard.\n */\nexport async function setupTablesAndReplication(\n  lc: LogContext,\n  sql: PostgresTransaction,\n  requested: ShardConfig,\n) {\n  const {publications} = requested;\n  // Validate requested publications.\n  for (const pub of publications) {\n    validatePublicationName(pub);\n    if (pub.startsWith('_')) {\n      throw new Error(\n        `Publication names starting with \"_\" are reserved for internal use.\\n` +\n          `Please use a different name for publication \"${pub}\".`,\n      );\n    }\n  }\n  const allPublications: string[] = [];\n\n  // Setup application publications.\n  if (publications.length) {\n    const results = await sql<{pubname: string}[]>`\n    SELECT pubname from pg_publication WHERE pubname IN ${sql(\n      publications,\n    )}`.values();\n\n    if (results.length !== publications.length) {\n      throw new Error(\n        `Unknown or invalid publications. Specified: [${publications}]. Found: [${results.flat()}]`,\n      );\n    }\n    allPublications.push(...publications);\n  } else {\n    const defaultPublication = defaultPublicationName(\n      requested.appID,\n      requested.shardNum,\n    );\n    await sql`\n      DROP PUBLICATION IF EXISTS ${sql(defaultPublication)}`;\n    await sql`\n      CREATE PUBLICATION ${sql(defaultPublication)} \n        FOR TABLES IN SCHEMA public\n        WITH (publish_via_partition_root = true)`;\n    allPublications.push(defaultPublication);\n  }\n\n  const metadataPublication = metadataPublicationName(\n    requested.appID,\n    requested.shardNum,\n  );\n  allPublications.push(metadataPublication);\n\n  const shard = {...requested, publications: allPublications};\n\n  // Setup the global tables and shard tables / publications.\n  await sql.unsafe(globalSetup(shard) + shardSetup(shard, metadataPublication));\n\n  const pubs = await getPublicationInfo(sql, allPublications);\n  await replicaIdentitiesForTablesWithoutPrimaryKeys(pubs)?.apply(lc, sql);\n\n  await setupTriggers(lc, sql, shard);\n}\n\nexport async function setupTriggers(\n  lc: LogContext,\n  tx: PostgresTransaction,\n  shard: ShardConfig,\n) {\n  const schema = upstreamSchema(shard);\n  const [{ddlDetection}] = await tx<InternalShardConfig[]> /*sql*/ `\n    SELECT \"ddlDetection\" FROM ${tx(schema)}.\"shardConfig\"`;\n\n  // The functions invoked by event triggers are installed even if\n  // event triggers are not supported/allowed by the db provider.\n  // This allows users to manually invoke the update_schemas() function\n  // as a workaround.\n  await tx.unsafe(createEventFunctionStatements(shard));\n  try {\n    await tx.savepoint(sub => sub.unsafe(triggerSetup(shard)));\n  } catch (e) {\n    if (ddlDetection) {\n      // If ddlDetection has already been enabled, subsequent failures to\n      // upgrade the trigger should be propagated rather than swallowed.\n      throw e;\n    }\n    if (\n      !(\n        e instanceof postgres.PostgresError &&\n        e.code === PG_INSUFFICIENT_PRIVILEGE\n      )\n    ) {\n      throw e;\n    }\n    // If triggerSetup() fails, replication continues in ddlDetection=false mode.\n    lc.warn?.(\n      `Unable to create event triggers for schema change detection:\\n\\n` +\n        `\"${e.hint ?? e.message}\"\\n\\n` +\n        `Proceeding in degraded mode: schema changes will halt replication,\\n` +\n        `requiring the replica to be reset (manually or with --auto-reset).`,\n    );\n  }\n}\n\nexport function validatePublications(\n  lc: LogContext,\n  published: PublicationInfo,\n) {\n  // Verify that all publications export the proper events.\n  published.publications.forEach(pub => {\n    if (\n      !pub.pubinsert ||\n      !pub.pubupdate ||\n      !pub.pubdelete ||\n      !pub.pubtruncate\n    ) {\n      // TODO: Make APIError?\n      throw new Error(\n        `PUBLICATION ${pub.pubname} must publish insert, update, delete, and truncate`,\n      );\n    }\n  });\n\n  published.tables.forEach(table => validate(lc, table));\n}\n\ntype ReplicaIdentities = {\n  apply(lc: LogContext, db: PostgresDB): Promise<void>;\n};\n\nexport function replicaIdentitiesForTablesWithoutPrimaryKeys(\n  pubs: PublishedSchema,\n): ReplicaIdentities | undefined {\n  const replicaIdentities: {\n    schema: string;\n    tableName: string;\n    indexName: string;\n  }[] = [];\n  for (const table of pubs.tables) {\n    if (!table.primaryKey?.length && table.replicaIdentity === Default) {\n      // Look for an index that can serve as the REPLICA IDENTITY USING INDEX. It must be:\n      // - UNIQUE\n      // - NOT NULL columns\n      // - not deferrable (i.e. isImmediate)\n      // - not partial (are already filtered out)\n      //\n      // https://www.postgresql.org/docs/current/sql-altertable.html#SQL-ALTERTABLE-REPLICA-IDENTITY\n      const {schema, name: tableName} = table;\n      for (const {columns, name: indexName} of pubs.indexes.filter(\n        idx =>\n          idx.schema === schema &&\n          idx.tableName === tableName &&\n          idx.unique &&\n          idx.isImmediate,\n      )) {\n        if (Object.keys(columns).some(col => !table.columns[col].notNull)) {\n          continue; // Only indexes with all NOT NULL columns are suitable.\n        }\n        replicaIdentities.push({schema, tableName, indexName});\n        break;\n      }\n    }\n  }\n\n  if (replicaIdentities.length === 0) {\n    return undefined;\n  }\n  return {\n    apply: async (lc: LogContext, sql: PostgresDB) => {\n      for (const {schema, tableName, indexName} of replicaIdentities) {\n        lc.info?.(\n          `setting \"${indexName}\" as the REPLICA IDENTITY for \"${tableName}\"`,\n        );\n        await sql`\n        ALTER TABLE ${sql(schema)}.${sql(tableName)} \n          REPLICA IDENTITY USING INDEX ${sql(indexName)}`;\n      }\n    },\n  };\n}\n"],"mappings":";;;;;;;;;;;;;;;;AAgCA,IAAM,yBAAyB;;;;;;AAO/B,SAAgB,wBAAwB,MAAoB;AAC1D,KAAI,CAAC,uBAAuB,KAAK,KAAK,CACpC,OAAM,IAAI,MACR,6BAA6B,KAAK,gHAEnC;AAEH,KAAI,KAAK,SAAS,GAChB,OAAM,IAAI,MACR,qBAAqB,KAAK,uDAC3B;;AAIL,SAAgB,0BAA0B,EAAC,SAAe;AACxD,QAAO,IAAI,MAAM;;AAGnB,SAAgB,sBAAsB,EAAC,OAAO,YAAoB;AAChE,QAAO,GAAG,MAAM,GAAG;;AAGrB,SAAgB,sBAAsB,OAAgB;CACpD,MAAM,EAAC,OAAO,aAAY,MAAM,MAAM;AACtC,QAAO,GAAG,MAAM,GAAG,SAAS;;;;;;AAO9B,SAAgB,0BAA0B,OAAgB;AAGxD,QAAO,GAAG,sBAAsB,MAAM,CAAC,GAAG,WAAW,KAAK,MAAM;;AAGlE,SAAS,uBAAuB,OAAe,SAA0B;AACvE,QAAO,IAAI,MAAM,UAAU;;AAG7B,SAAgB,wBACd,OACA,SACA;AACA,QAAO,IAAI,MAAM,YAAY;;AAI/B,SAAS,YAAY,OAAsB;CACzC,MAAM,MAAM,GAAG,UAAU,MAAM,CAAC;AAEhC,QAAe;gCACe,IAAI;;+BAEL,IAAI;;;;;;;;;;+BAUJ,IAAI;;;;;;;;;iCASF,IAAI;;uBAEd,IAAI;;gBAEX,IAAI;;;AAIpB,eAAsB,mBAAmB,IAAgB,OAAc;AACrE,OAAM,GAAG,OAAO,YAAY,MAAM,CAAC;;AAGrC,SAAgB,0BAA0B,QAAgB;AACxD,QAAe;iBACA,OAAO;;;;;;;;;;;;;;;;AAiBxB,SAAgB,4BAA4B,QAAgB;AAC1D,QAAe;iBACA,OAAO;;;;;;;;AASxB,IAAa,qBAAqB;AAElC,SAAgB,WACd,aACA,qBACQ;CACR,MAAM,MAAM,GAAG,UAAU,YAAY,CAAC;CACtC,MAAM,QAAQ,GAAG,eAAe,YAAY,CAAC;CAE7C,MAAM,OAAO,YAAY,aAAa,UAAU;AAChD,QACE,KAAK,SAAS,oBAAoB,QAC5B,6BAA6B,sBACpC;AAED,QAAe;gCACe,MAAM;;IAElC,0BAA0B,MAAM,CAAC;IACjC,4BAA4B,MAAM,CAAC;;+BAER,GAAG,oBAAoB,CAAC;uBAChC,GAAG,oBAAoB,CAAC;gBAC/B,IAAI,wBAAwB,MAAM,cAAc,MAAM;;iBAErD,MAAM,IAAI,mBAAmB;;;;;;;;gBAQ9B,MAAM,IAAI,mBAAmB;;;;cAI/B,QAAQ,KAAK,CAAC;;;;iBAIX,MAAM;;;;;;;;;;;;;AAcvB,SAAgB,UAAU,OAAe,SAAkC;CACzE,MAAM,SAAS,GAAG,MAAM,GAAG;CAC3B,MAAM,sBAAsB,wBAAwB,OAAO,QAAQ;AAKnE,QAAe;iCACgB,GALJ,uBAAuB,OAAO,QAAQ,CAKZ,CAAC;iCACvB,GAAG,oBAAoB,CAAC;4BAC7B,GAAG,OAAO,CAAC;;;AAIvC,IAAM,4BAA4B,eAAE,OAAO;CACzC,cAAc,eAAE,MAAM,eAAE,QAAQ,CAAC;CACjC,cAAc,eAAE,SAAS;CAC1B,CAAC;AAIF,IAAM,gBAAgB,0BAA0B,OAAO;CACrD,IAAI,eAAE,QAAQ;CACd,MAAM,eAAE,QAAQ;CAChB,MAAM,eAAE,QAAQ;CAChB,SAAS,eAAE,QAAQ;CACnB,eAAe;CACf,oBAAoB,iBAAiB,UAAU;CAC/C,mBAAmB,iBAAiB,UAAU;CAC/C,CAAC;AAOF,SAAS,aAAa,OAA4B;CAChD,MAAM,SAAS,GAAG,eAAe,MAAM,CAAC;AACxC,QACE,6BAA6B,MAAM,GAC3B,UAAU,OAAO;;;;;;;;;;;AAa7B,eAAsB,cACpB,KACA,OACA,IACA,MACA,gBACA;CACA,MAAM,SAAS,eAAe,MAAM;CACpC,MAAM,SAAS;EAAC;EAAI;EAAM,SAAS;EAAe;AAClD,OAAM,GAAG,eAAe,IAAI,OAAO,CAAC,YAAY,IAAI,OAAO;;AAI7D,eAAsB,YACpB,KACA,OACA,IACA,EAAC,QAAQ,WACT,oBACA;CACA,MAAM,SAAS,eAAe,MAAM;CAEpC,MAAM,SAAS;EAAC,eADgB;GAAC;GAAQ;GAAQ;EACV;EAAmB;AAC1D,OAAM,GAAG,UAAU,IAAI,OAAO,CAAC,gBAAgB,IAAI,OAAO,CAAC,cAAc;;;;;AAM3E,eAAsB,oBACpB,IACA,KACA,OACA,gBACA,SACyB;CACzB,MAAM,SAAS,IAAI,eAAe,MAAM,CAAC;CACzC,MAAM,SAAS,MAAM,GAAG;;;;;;;;;;;WAWf,OAAO,iBAAiB,OAAO;wBAClB,eAAe;;;AAGrC,KAAI,OAAO,WAAW,GAAG;EAEvB,MAAM,cAAc,MAAM,GAAG;;eAElB,OAAO;AAClB,KAAG,OACD,WAAW,eAAe,MACvB,UAAU,aAAa,UAAU,QAAQ,CAAC,MAAM,MACjD,iBAAiB,UAAU,YAAY,GAC1C;AACD,SAAO;;AAET,QAAO,MAAQ,OAAO,IAAI,eAAe,cAAc;;AAGzD,eAAsB,uBACpB,KACA,OAC8B;CAC9B,MAAM,SAAS,MAAM,GAAG;;aAEb,IAAI,eAAe,MAAM,CAAC,CAAC;;AAEtC,QACE,OAAO,WAAW,SACZ,6CAA6C,OAAO,SAC3D;AACD,QAAO,MAAQ,OAAO,IAAI,2BAA2B,cAAc;;;;;;AAOrE,eAAsB,0BACpB,IACA,KACA,WACA;CACA,MAAM,EAAC,iBAAgB;AAEvB,MAAK,MAAM,OAAO,cAAc;AAC9B,0BAAwB,IAAI;AAC5B,MAAI,IAAI,WAAW,IAAI,CACrB,OAAM,IAAI,MACR,oHACkD,IAAI,IACvD;;CAGL,MAAM,kBAA4B,EAAE;AAGpC,KAAI,aAAa,QAAQ;EACvB,MAAM,UAAU,MAAM,GAAwB;0DACQ,IACpD,aACD,GAAG,QAAQ;AAEZ,MAAI,QAAQ,WAAW,aAAa,OAClC,OAAM,IAAI,MACR,gDAAgD,aAAa,aAAa,QAAQ,MAAM,CAAC,GAC1F;AAEH,kBAAgB,KAAK,GAAG,aAAa;QAChC;EACL,MAAM,qBAAqB,uBACzB,UAAU,OACV,UAAU,SACX;AACD,QAAM,GAAG;mCACsB,IAAI,mBAAmB;AACtD,QAAM,GAAG;2BACc,IAAI,mBAAmB,CAAC;;;AAG/C,kBAAgB,KAAK,mBAAmB;;CAG1C,MAAM,sBAAsB,wBAC1B,UAAU,OACV,UAAU,SACX;AACD,iBAAgB,KAAK,oBAAoB;CAEzC,MAAM,QAAQ;EAAC,GAAG;EAAW,cAAc;EAAgB;AAG3D,OAAM,IAAI,OAAO,YAAY,MAAM,GAAG,WAAW,OAAO,oBAAoB,CAAC;AAG7E,OAAM,6CADO,MAAM,mBAAmB,KAAK,gBAAgB,CACH,EAAE,MAAM,IAAI,IAAI;AAExE,OAAM,cAAc,IAAI,KAAK,MAAM;;AAGrC,eAAsB,cACpB,IACA,IACA,OACA;CAEA,MAAM,CAAC,EAAC,kBAAiB,MAAM,EAAkC;iCAClC,GAFhB,eAAe,MAAM,CAEK,CAAC;AAM1C,OAAM,GAAG,OAAO,8BAA8B,MAAM,CAAC;AACrD,KAAI;AACF,QAAM,GAAG,WAAU,QAAO,IAAI,OAAO,aAAa,MAAM,CAAC,CAAC;UACnD,GAAG;AACV,MAAI,aAGF,OAAM;AAER,MACE,EACE,aAAa,SAAS,iBACtB,EAAE,SAAS,2BAGb,OAAM;AAGR,KAAG,OACD,oEACM,EAAE,QAAQ,EAAE,QAAQ,6IAG3B;;;AAIL,SAAgB,qBACd,IACA,WACA;AAEA,WAAU,aAAa,SAAQ,QAAO;AACpC,MACE,CAAC,IAAI,aACL,CAAC,IAAI,aACL,CAAC,IAAI,aACL,CAAC,IAAI,YAGL,OAAM,IAAI,MACR,eAAe,IAAI,QAAQ,oDAC5B;GAEH;AAEF,WAAU,OAAO,SAAQ,UAAS,SAAS,IAAI,MAAM,CAAC;;AAOxD,SAAgB,6CACd,MAC+B;CAC/B,MAAM,oBAIA,EAAE;AACR,MAAK,MAAM,SAAS,KAAK,OACvB,KAAI,CAAC,MAAM,YAAY,UAAU,MAAM,oBAAA,KAA6B;EAQlE,MAAM,EAAC,QAAQ,MAAM,cAAa;AAClC,OAAK,MAAM,EAAC,SAAS,MAAM,eAAc,KAAK,QAAQ,QACpD,QACE,IAAI,WAAW,UACf,IAAI,cAAc,aAClB,IAAI,UACJ,IAAI,YACP,EAAE;AACD,OAAI,OAAO,KAAK,QAAQ,CAAC,MAAK,QAAO,CAAC,MAAM,QAAQ,KAAK,QAAQ,CAC/D;AAEF,qBAAkB,KAAK;IAAC;IAAQ;IAAW;IAAU,CAAC;AACtD;;;AAKN,KAAI,kBAAkB,WAAW,EAC/B;AAEF,QAAO,EACL,OAAO,OAAO,IAAgB,QAAoB;AAChD,OAAK,MAAM,EAAC,QAAQ,WAAW,eAAc,mBAAmB;AAC9D,MAAG,OACD,YAAY,UAAU,iCAAiC,UAAU,GAClE;AACD,SAAM,GAAG;sBACK,IAAI,OAAO,CAAC,GAAG,IAAI,UAAU,CAAC;yCACX,IAAI,UAAU;;IAGpD"}

package/out/zero-cache/src/services/change-streamer/change-streamer-http.d.ts

@@ -4,9 +4,8 @@ import { type Worker } from '../../types/processes.ts';
 import { type ShardID } from '../../types/shards.ts';
 import { type Source } from '../../types/streams.ts';
 import { HttpService } from '../http-service.ts';
-import type { Service } from '../service.ts';
 import type { BackupMonitor } from './backup-monitor.ts';
-import { type ChangeStreamer, type Downstream, type SubscriberContext } from './change-streamer.ts';
+import { type ChangeStreamer, type ChangeStreamerService, type Downstream, type SubscriberContext } from './change-streamer.ts';
 import { type SnapshotMessage } from './snapshot.ts';
 type Options = {
     port: number;
@@ -16,7 +15,7 @@ type Options = {
 export declare class ChangeStreamerHttpServer extends HttpService {
     #private;
     readonly id = "change-streamer-http-server";
-    constructor(lc: LogContext, opts: Options, parent: Worker, changeStreamer: ChangeStreamer & Service, backupMonitor: BackupMonitor | null);
+    constructor(lc: LogContext, opts: Options, parent: Worker, changeStreamer: ChangeStreamerService, backupMonitor: BackupMonitor | null);
     protected _onStart(): void;
     protected _onStop(): Promise<void>;
 }

package/out/zero-cache/src/services/change-streamer/change-streamer-http.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"change-streamer-http.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-http.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,WAAW,CAAC;AAE/C,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAMjD,OAAO,EAAC,KAAK,MAAM,EAAC,MAAM,0BAA0B,CAAC;AACrD,OAAO,EAAC,KAAK,OAAO,EAAC,MAAM,uBAAuB,CAAC;AACnD,OAAO,EAAsB,KAAK,MAAM,EAAC,MAAM,wBAAwB,CAAC;AAIxE,OAAO,EAAC,WAAW,EAAC,MAAM,oBAAoB,CAAC;AAC/C,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,eAAe,CAAC;AAC3C,OAAO,KAAK,EAAC,aAAa,EAAC,MAAM,qBAAqB,CAAC;AACvD,OAAO,EAGL,KAAK,cAAc,EACnB,KAAK,UAAU,EACf,KAAK,iBAAiB,EACvB,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EAAwB,KAAK,eAAe,EAAC,MAAM,eAAe,CAAC;AAW1E,KAAK,OAAO,GAAG;IACb,IAAI,EAAE,MAAM,CAAC;IACb,kBAAkB,EAAE,MAAM,GAAG,SAAS,CAAC;IACvC,cAAc,EAAE,MAAM,CAAC;CACxB,CAAC;AAEF,qBAAa,wBAAyB,SAAQ,WAAW;;IACvD,QAAQ,CAAC,EAAE,iCAAiC;gBAO1C,EAAE,EAAE,UAAU,EACd,IAAI,EAAE,OAAO,EACb,MAAM,EAAE,MAAM,EACd,cAAc,EAAE,cAAc,GAAG,OAAO,EACxC,aAAa,EAAE,aAAa,GAAG,IAAI;cA4GlB,QAAQ,IAAI,IAAI;cAWV,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;CAKlD;AAED,qBAAa,wBAAyB,YAAW,cAAc;;gBAO3D,EAAE,EAAE,UAAU,EACd,OAAO,EAAE,OAAO,EAChB,QAAQ,EAAE,MAAM,EAChB,iBAAiB,EAAE,MAAM,GAAG,SAAS;IA8BjC,eAAe,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;IASjE,SAAS,CAAC,GAAG,EAAE,iBAAiB,GAAG,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;CAQrE;AAED,KAAK,cAAc,GAAG,IAAI,CAAC,eAAe,EAAE,KAAK,GAAG,SAAS,CAAC,CAAC;AAE/D,wBAAgB,oBAAoB,CAAC,GAAG,EAAE,cAAc,GAAG,iBAAiB,CAc3E"}
+{"version":3,"file":"change-streamer-http.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-http.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,WAAW,CAAC;AAE/C,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAMjD,OAAO,EAAC,KAAK,MAAM,EAAC,MAAM,0BAA0B,CAAC;AACrD,OAAO,EAAC,KAAK,OAAO,EAAC,MAAM,uBAAuB,CAAC;AACnD,OAAO,EAIL,KAAK,MAAM,EACZ,MAAM,wBAAwB,CAAC;AAIhC,OAAO,EAAC,WAAW,EAAC,MAAM,oBAAoB,CAAC;AAC/C,OAAO,KAAK,EAAC,aAAa,EAAC,MAAM,qBAAqB,CAAC;AACvD,OAAO,EAGL,KAAK,cAAc,EACnB,KAAK,qBAAqB,EAC1B,KAAK,UAAU,EACf,KAAK,iBAAiB,EACvB,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EAAwB,KAAK,eAAe,EAAC,MAAM,eAAe,CAAC;AAW1E,KAAK,OAAO,GAAG;IACb,IAAI,EAAE,MAAM,CAAC;IACb,kBAAkB,EAAE,MAAM,GAAG,SAAS,CAAC;IACvC,cAAc,EAAE,MAAM,CAAC;CACxB,CAAC;AAEF,qBAAa,wBAAyB,SAAQ,WAAW;;IACvD,QAAQ,CAAC,EAAE,iCAAiC;gBAO1C,EAAE,EAAE,UAAU,EACd,IAAI,EAAE,OAAO,EACb,MAAM,EAAE,MAAM,EACd,cAAc,EAAE,qBAAqB,EACrC,aAAa,EAAE,aAAa,GAAG,IAAI;cA4GlB,QAAQ,IAAI,IAAI;cAWV,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;CAKlD;AAED,qBAAa,wBAAyB,YAAW,cAAc;;gBAO3D,EAAE,EAAE,UAAU,EACd,OAAO,EAAE,OAAO,EAChB,QAAQ,EAAE,MAAM,EAChB,iBAAiB,EAAE,MAAM,GAAG,SAAS;IA8BjC,eAAe,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;IASjE,SAAS,CAAC,GAAG,EAAE,iBAAiB,GAAG,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;CAQrE;AAED,KAAK,cAAc,GAAG,IAAI,CAAC,eAAe,EAAE,KAAK,GAAG,SAAS,CAAC,CAAC;AAE/D,wBAAgB,oBAAoB,CAAC,GAAG,EAAE,cAAc,GAAG,iBAAiB,CAc3E"}

package/out/zero-cache/src/services/change-streamer/change-streamer-http.js

@@ -6,11 +6,11 @@ import { HttpService } from "../http-service.js";
 import { PROTOCOL_ERROR, closeWithError } from "../../types/ws.js";
 import { installWebSocketReceiver } from "../../types/websocket-handoff.js";
 import { discoverChangeStreamerAddress } from "./schema/tables.js";
-import { streamIn, streamOut } from "../../types/streams.js";
+import { streamIn, streamOut, streamOutStringified } from "../../types/streams.js";
 import { URLParams } from "../../types/url-params.js";
 import { downstreamSchema } from "./change-streamer.js";
 import { snapshotMessageSchema } from "./snapshot.js";
-import WebSocket$1 from "ws";
+import WebSocket from "ws";
 import websocket from "@fastify/websocket";
 //#region ../zero-cache/src/services/change-streamer/change-streamer-http.ts
 var MIN_SUPPORTED_PROTOCOL_VERSION = 1;
@@ -67,7 +67,7 @@ var ChangeStreamerHttpServer = class extends HttpService {
 			if (ctx.mode === "serving") this.#ensureChangeStreamerStarted("incoming subscription");
 			const downstream = await this.#changeStreamer.subscribe(ctx);
 			if (ctx.initial && ctx.taskID && this.#backupMonitor) this.#backupMonitor.endReservation(ctx.taskID);
-			streamOut(this._lc, downstream, ws);
+			streamOutStringified(this._lc, downstream, ws);
 		} catch (err) {
 			closeWithError(this._lc, ws, err, PROTOCOL_ERROR);
 		}
@@ -114,11 +114,11 @@ var ChangeStreamerHttpClient = class {
 		return uri;
 	}
 	async reserveSnapshot(taskID) {
-		const ws = new WebSocket$1(await this.#resolveChangeStreamer(SNAPSHOT_PATH) + `?${new URLSearchParams({ taskID }).toString()}`);
+		const ws = new WebSocket(await this.#resolveChangeStreamer(SNAPSHOT_PATH) + `?${new URLSearchParams({ taskID }).toString()}`);
 		return streamIn(this.#lc, ws, snapshotMessageSchema);
 	}
 	async subscribe(ctx) {
-		const ws = new WebSocket$1(await this.#resolveChangeStreamer(CHANGES_PATH) + `?${getParams(ctx).toString()}`);
+		const ws = new WebSocket(await this.#resolveChangeStreamer(CHANGES_PATH) + `?${getParams(ctx).toString()}`);
 		return streamIn(this.#lc, ws, downstreamSchema);
 	}
 };

package/out/zero-cache/src/services/change-streamer/change-streamer-http.js.map

@@ -1 +1 @@
-{"version":3,"file":"change-streamer-http.js","names":["#lc","#opts","#changeStreamer","#backupMonitor","#subscribe","#reserveSnapshot","#receiveWebsocket","#getBackupMonitor","#ensureChangeStreamerStarted","#changeStreamerStarted","#shardID","#changeDB","#changeStreamerURI","#resolveChangeStreamer"],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-http.ts"],"sourcesContent":["import type {IncomingMessage} from 'node:http';\nimport websocket from '@fastify/websocket';\nimport type {LogContext} from '@rocicorp/logger';\nimport WebSocket from 'ws';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport type {IncomingMessageSubset} from '../../types/http.ts';\nimport {pgClient, type PostgresDB} from '../../types/pg.ts';\nimport {type Worker} from '../../types/processes.ts';\nimport {type ShardID} from '../../types/shards.ts';\nimport {streamIn, streamOut, type Source} from '../../types/streams.ts';\nimport {URLParams} from '../../types/url-params.ts';\nimport {installWebSocketReceiver} from '../../types/websocket-handoff.ts';\nimport {closeWithError, PROTOCOL_ERROR} from '../../types/ws.ts';\nimport {HttpService} from '../http-service.ts';\nimport type {Service} from '../service.ts';\nimport type {BackupMonitor} from './backup-monitor.ts';\nimport {\n  downstreamSchema,\n  PROTOCOL_VERSION,\n  type ChangeStreamer,\n  type Downstream,\n  type SubscriberContext,\n} from './change-streamer.ts';\nimport {discoverChangeStreamerAddress} from './schema/tables.ts';\nimport {snapshotMessageSchema, type SnapshotMessage} from './snapshot.ts';\n\nconst MIN_SUPPORTED_PROTOCOL_VERSION = 1;\n\nconst SNAPSHOT_PATH_PATTERN = '/replication/:version/snapshot';\nconst CHANGES_PATH_PATTERN = '/replication/:version/changes';\nconst PATH_REGEX = /\\/replication\\/v(?<version>\\d+)\\/(changes|snapshot)$/;\n\nconst SNAPSHOT_PATH = `/replication/v${PROTOCOL_VERSION}/snapshot`;\nconst CHANGES_PATH = `/replication/v${PROTOCOL_VERSION}/changes`;\n\ntype Options = {\n  port: number;\n  keepaliveTimeoutMs: number | undefined;\n  startupDelayMs: number;\n};\n\nexport class ChangeStreamerHttpServer extends HttpService {\n  readonly id = 'change-streamer-http-server';\n  readonly #lc: LogContext;\n  readonly #opts: Options;\n  readonly #changeStreamer: ChangeStreamer & Service;\n  readonly #backupMonitor: BackupMonitor | null;\n\n  constructor(\n    lc: LogContext,\n    opts: Options,\n    parent: Worker,\n    changeStreamer: ChangeStreamer & Service,\n    backupMonitor: BackupMonitor | null,\n  ) {\n    super('change-streamer-http-server', lc, opts, async fastify => {\n      await fastify.register(websocket);\n\n      fastify.get(CHANGES_PATH_PATTERN, {websocket: true}, this.#subscribe);\n      fastify.get(\n        SNAPSHOT_PATH_PATTERN,\n        {websocket: true},\n        this.#reserveSnapshot,\n      );\n\n      installWebSocketReceiver<'snapshot' | 'changes'>(\n        lc,\n        fastify.websocketServer,\n        this.#receiveWebsocket,\n        parent,\n      );\n    });\n\n    this.#lc = lc;\n    this.#opts = opts;\n    this.#changeStreamer = changeStreamer;\n    this.#backupMonitor = backupMonitor;\n  }\n\n  #getBackupMonitor() {\n    return must(\n      this.#backupMonitor,\n      'replication-manager is not configured with a ZERO_LITESTREAM_BACKUP_URL',\n    );\n  }\n\n  // Called when receiving a web socket via the main dispatcher handoff.\n  readonly #receiveWebsocket = (\n    ws: WebSocket,\n    action: 'changes' | 'snapshot',\n    msg: IncomingMessageSubset,\n  ) => {\n    switch (action) {\n      case 'snapshot':\n        return this.#reserveSnapshot(ws, msg);\n      case 'changes':\n        return this.#subscribe(ws, msg);\n      default:\n        closeWithError(\n          this._lc,\n          ws,\n          `invalid action \"${action}\" received in handoff`,\n        );\n        return;\n    }\n  };\n\n  readonly #reserveSnapshot = (ws: WebSocket, req: RequestHeaders) => {\n    try {\n      const url = new URL(\n        req.url ?? '',\n        req.headers.origin ?? 'http://localhost',\n      );\n      checkProtocolVersion(url.pathname);\n      const taskID = url.searchParams.get('taskID');\n      if (!taskID) {\n        throw new Error('Missing taskID in snapshot request');\n      }\n      const downstream =\n        this.#getBackupMonitor().startSnapshotReservation(taskID);\n      void streamOut(this._lc, downstream, ws);\n    } catch (err) {\n      closeWithError(this._lc, ws, err, PROTOCOL_ERROR);\n    }\n  };\n\n  readonly #subscribe = async (ws: WebSocket, req: RequestHeaders) => {\n    try {\n      const ctx = getSubscriberContext(req);\n      if (ctx.mode === 'serving') {\n        this.#ensureChangeStreamerStarted('incoming subscription');\n      }\n\n      const downstream = await this.#changeStreamer.subscribe(ctx);\n      if (ctx.initial && ctx.taskID && this.#backupMonitor) {\n        // Now that the change-streamer knows about the subscriber and watermark,\n        // end the reservation to safely resume scheduling cleanup.\n        this.#backupMonitor.endReservation(ctx.taskID);\n      }\n      void streamOut(this._lc, downstream, ws);\n    } catch (err) {\n      closeWithError(this._lc, ws, err, PROTOCOL_ERROR);\n    }\n  };\n\n  #changeStreamerStarted = false;\n\n  #ensureChangeStreamerStarted(reason: string) {\n    if (!this.#changeStreamerStarted && this._state.shouldRun()) {\n      this.#lc.info?.(`starting ChangeStreamerService: ${reason}`);\n      void this.#changeStreamer\n        .run()\n        .catch(e =>\n          this.#lc.warn?.(`ChangeStreamerService ended with error`, e),\n        )\n        .finally(() => this.stop());\n\n      this.#changeStreamerStarted = true;\n    }\n  }\n\n  protected override _onStart(): void {\n    const {startupDelayMs} = this.#opts;\n    this._state.setTimeout(\n      () =>\n        this.#ensureChangeStreamerStarted(\n          `startup delay elapsed (${startupDelayMs} ms)`,\n        ),\n      startupDelayMs,\n    );\n  }\n\n  protected override async _onStop(): Promise<void> {\n    if (this.#changeStreamerStarted) {\n      await this.#changeStreamer.stop();\n    }\n  }\n}\n\nexport class ChangeStreamerHttpClient implements ChangeStreamer {\n  readonly #lc: LogContext;\n  readonly #shardID: ShardID;\n  readonly #changeDB: PostgresDB;\n  readonly #changeStreamerURI: string | undefined;\n\n  constructor(\n    lc: LogContext,\n    shardID: ShardID,\n    changeDB: string,\n    changeStreamerURI: string | undefined,\n  ) {\n    this.#lc = lc;\n    this.#shardID = shardID;\n    // Create a pg client with a single short-lived connection for the purpose\n    // of change-streamer discovery (i.e. ChangeDB as DNS).\n    this.#changeDB = pgClient(lc, changeDB, 'change-streamer-discovery', {\n      max: 1,\n      ['idle_timeout']: 15,\n    });\n    this.#changeStreamerURI = changeStreamerURI;\n  }\n\n  async #resolveChangeStreamer(path: string) {\n    let baseURL = this.#changeStreamerURI;\n    if (!baseURL) {\n      const address = await discoverChangeStreamerAddress(\n        this.#shardID,\n        this.#changeDB,\n      );\n      if (!address) {\n        throw new Error(`no change-streamer is running`);\n      }\n      baseURL = address.includes('://') ? `${address}/` : `ws://${address}/`;\n    }\n    const uri = new URL(path, baseURL);\n    this.#lc.info?.(`connecting to change-streamer@${uri}`);\n    return uri;\n  }\n\n  async reserveSnapshot(taskID: string): Promise<Source<SnapshotMessage>> {\n    const uri = await this.#resolveChangeStreamer(SNAPSHOT_PATH);\n\n    const params = new URLSearchParams({taskID});\n    const ws = new WebSocket(uri + `?${params.toString()}`);\n\n    return streamIn(this.#lc, ws, snapshotMessageSchema);\n  }\n\n  async subscribe(ctx: SubscriberContext): Promise<Source<Downstream>> {\n    const uri = await this.#resolveChangeStreamer(CHANGES_PATH);\n\n    const params = getParams(ctx);\n    const ws = new WebSocket(uri + `?${params.toString()}`);\n\n    return streamIn(this.#lc, ws, downstreamSchema);\n  }\n}\n\ntype RequestHeaders = Pick<IncomingMessage, 'url' | 'headers'>;\n\nexport function getSubscriberContext(req: RequestHeaders): SubscriberContext {\n  const url = new URL(req.url ?? '', req.headers.origin ?? 'http://localhost');\n  const protocolVersion = checkProtocolVersion(url.pathname);\n  const params = new URLParams(url);\n\n  return {\n    protocolVersion,\n    id: params.get('id', true),\n    taskID: params.get('taskID', false),\n    mode: params.get('mode', false) === 'backup' ? 'backup' : 'serving',\n    replicaVersion: params.get('replicaVersion', true),\n    watermark: params.get('watermark', true),\n    initial: params.getBoolean('initial'),\n  };\n}\n\nfunction checkProtocolVersion(pathname: string): number {\n  const match = PATH_REGEX.exec(pathname);\n  if (!match) {\n    throw new Error(`invalid path: ${pathname}`);\n  }\n  const v = Number(match.groups?.version);\n  if (\n    Number.isNaN(v) ||\n    v > PROTOCOL_VERSION ||\n    v < MIN_SUPPORTED_PROTOCOL_VERSION\n  ) {\n    throw new Error(\n      `Cannot service client at protocol v${v}. ` +\n        `Supported protocols: [v${MIN_SUPPORTED_PROTOCOL_VERSION} ... v${PROTOCOL_VERSION}]`,\n    );\n  }\n  return v;\n}\n\n// This is called from the client-side (i.e. the replicator).\nfunction getParams(ctx: SubscriberContext): URLSearchParams {\n  // The protocolVersion is hard-coded into the CHANGES_PATH.\n  const {protocolVersion, ...stringParams} = ctx;\n  assert(\n    protocolVersion === PROTOCOL_VERSION,\n    `replicator should be setting protocolVersion to ${PROTOCOL_VERSION}`,\n  );\n  return new URLSearchParams({\n    ...stringParams,\n    taskID: ctx.taskID ? ctx.taskID : '',\n    initial: ctx.initial ? 'true' : 'false',\n  });\n}\n"],"mappings":";;;;;;;;;;;;;;;AA2BA,IAAM,iCAAiC;AAEvC,IAAM,wBAAwB;AAC9B,IAAM,uBAAuB;AAC7B,IAAM,aAAa;AAEnB,IAAM,gBAAgB;AACtB,IAAM,eAAe;AAQrB,IAAa,2BAAb,cAA8C,YAAY;CACxD,KAAc;CACd;CACA;CACA;CACA;CAEA,YACE,IACA,MACA,QACA,gBACA,eACA;AACA,QAAM,+BAA+B,IAAI,MAAM,OAAM,YAAW;AAC9D,SAAM,QAAQ,SAAS,UAAU;AAEjC,WAAQ,IAAI,sBAAsB,EAAC,WAAW,MAAK,EAAE,MAAA,UAAgB;AACrE,WAAQ,IACN,uBACA,EAAC,WAAW,MAAK,EACjB,MAAA,gBACD;AAED,4BACE,IACA,QAAQ,iBACR,MAAA,kBACA,OACD;IACD;AAEF,QAAA,KAAW;AACX,QAAA,OAAa;AACb,QAAA,iBAAuB;AACvB,QAAA,gBAAsB;;CAGxB,oBAAoB;AAClB,SAAO,KACL,MAAA,eACA,0EACD;;CAIH,qBACE,IACA,QACA,QACG;AACH,UAAQ,QAAR;GACE,KAAK,WACH,QAAO,MAAA,gBAAsB,IAAI,IAAI;GACvC,KAAK,UACH,QAAO,MAAA,UAAgB,IAAI,IAAI;GACjC;AACE,mBACE,KAAK,KACL,IACA,mBAAmB,OAAO,uBAC3B;AACD;;;CAIN,oBAA6B,IAAe,QAAwB;AAClE,MAAI;GACF,MAAM,MAAM,IAAI,IACd,IAAI,OAAO,IACX,IAAI,QAAQ,UAAU,mBACvB;AACD,wBAAqB,IAAI,SAAS;GAClC,MAAM,SAAS,IAAI,aAAa,IAAI,SAAS;AAC7C,OAAI,CAAC,OACH,OAAM,IAAI,MAAM,qCAAqC;GAEvD,MAAM,aACJ,MAAA,kBAAwB,CAAC,yBAAyB,OAAO;AACtD,aAAU,KAAK,KAAK,YAAY,GAAG;WACjC,KAAK;AACZ,kBAAe,KAAK,KAAK,IAAI,KAAK,eAAe;;;CAIrD,aAAsB,OAAO,IAAe,QAAwB;AAClE,MAAI;GACF,MAAM,MAAM,qBAAqB,IAAI;AACrC,OAAI,IAAI,SAAS,UACf,OAAA,4BAAkC,wBAAwB;GAG5D,MAAM,aAAa,MAAM,MAAA,eAAqB,UAAU,IAAI;AAC5D,OAAI,IAAI,WAAW,IAAI,UAAU,MAAA,cAG/B,OAAA,cAAoB,eAAe,IAAI,OAAO;AAE3C,aAAU,KAAK,KAAK,YAAY,GAAG;WACjC,KAAK;AACZ,kBAAe,KAAK,KAAK,IAAI,KAAK,eAAe;;;CAIrD,yBAAyB;CAEzB,6BAA6B,QAAgB;AAC3C,MAAI,CAAC,MAAA,yBAA+B,KAAK,OAAO,WAAW,EAAE;AAC3D,SAAA,GAAS,OAAO,mCAAmC,SAAS;AACvD,SAAA,eACF,KAAK,CACL,OAAM,MACL,MAAA,GAAS,OAAO,0CAA0C,EAAE,CAC7D,CACA,cAAc,KAAK,MAAM,CAAC;AAE7B,SAAA,wBAA8B;;;CAIlC,WAAoC;EAClC,MAAM,EAAC,mBAAkB,MAAA;AACzB,OAAK,OAAO,iBAER,MAAA,4BACE,0BAA0B,eAAe,MAC1C,EACH,eACD;;CAGH,MAAyB,UAAyB;AAChD,MAAI,MAAA,sBACF,OAAM,MAAA,eAAqB,MAAM;;;AAKvC,IAAa,2BAAb,MAAgE;CAC9D;CACA;CACA;CACA;CAEA,YACE,IACA,SACA,UACA,mBACA;AACA,QAAA,KAAW;AACX,QAAA,UAAgB;AAGhB,QAAA,WAAiB,SAAS,IAAI,UAAU,6BAA6B;GACnE,KAAK;IACJ,iBAAiB;GACnB,CAAC;AACF,QAAA,oBAA0B;;CAG5B,OAAA,sBAA6B,MAAc;EACzC,IAAI,UAAU,MAAA;AACd,MAAI,CAAC,SAAS;GACZ,MAAM,UAAU,MAAM,8BACpB,MAAA,SACA,MAAA,SACD;AACD,OAAI,CAAC,QACH,OAAM,IAAI,MAAM,gCAAgC;AAElD,aAAU,QAAQ,SAAS,MAAM,GAAG,GAAG,QAAQ,KAAK,QAAQ,QAAQ;;EAEtE,MAAM,MAAM,IAAI,IAAI,MAAM,QAAQ;AAClC,QAAA,GAAS,OAAO,iCAAiC,MAAM;AACvD,SAAO;;CAGT,MAAM,gBAAgB,QAAkD;EAItE,MAAM,KAAK,IAAI,YAHH,MAAM,MAAA,sBAA4B,cAAc,GAG7B,IADhB,IAAI,gBAAgB,EAAC,QAAO,CAAC,CACF,UAAU,GAAG;AAEvD,SAAO,SAAS,MAAA,IAAU,IAAI,sBAAsB;;CAGtD,MAAM,UAAU,KAAqD;EAInE,MAAM,KAAK,IAAI,YAHH,MAAM,MAAA,sBAA4B,aAAa,GAG5B,IADhB,UAAU,IAAI,CACa,UAAU,GAAG;AAEvD,SAAO,SAAS,MAAA,IAAU,IAAI,iBAAiB;;;AAMnD,SAAgB,qBAAqB,KAAwC;CAC3E,MAAM,MAAM,IAAI,IAAI,IAAI,OAAO,IAAI,IAAI,QAAQ,UAAU,mBAAmB;CAC5E,MAAM,kBAAkB,qBAAqB,IAAI,SAAS;CAC1D,MAAM,SAAS,IAAI,UAAU,IAAI;AAEjC,QAAO;EACL;EACA,IAAI,OAAO,IAAI,MAAM,KAAK;EAC1B,QAAQ,OAAO,IAAI,UAAU,MAAM;EACnC,MAAM,OAAO,IAAI,QAAQ,MAAM,KAAK,WAAW,WAAW;EAC1D,gBAAgB,OAAO,IAAI,kBAAkB,KAAK;EAClD,WAAW,OAAO,IAAI,aAAa,KAAK;EACxC,SAAS,OAAO,WAAW,UAAU;EACtC;;AAGH,SAAS,qBAAqB,UAA0B;CACtD,MAAM,QAAQ,WAAW,KAAK,SAAS;AACvC,KAAI,CAAC,MACH,OAAM,IAAI,MAAM,iBAAiB,WAAW;CAE9C,MAAM,IAAI,OAAO,MAAM,QAAQ,QAAQ;AACvC,KACE,OAAO,MAAM,EAAE,IACf,IAAA,KACA,IAAI,+BAEJ,OAAM,IAAI,MACR,sCAAsC,EAAE,2BACZ,+BAA+B,UAC5D;AAEH,QAAO;;AAIT,SAAS,UAAU,KAAyC;CAE1D,MAAM,EAAC,iBAAiB,GAAG,iBAAgB;AAC3C,QACE,oBAAA,GACA,oDACD;AACD,QAAO,IAAI,gBAAgB;EACzB,GAAG;EACH,QAAQ,IAAI,SAAS,IAAI,SAAS;EAClC,SAAS,IAAI,UAAU,SAAS;EACjC,CAAC"}
+{"version":3,"file":"change-streamer-http.js","names":["#lc","#opts","#changeStreamer","#backupMonitor","#subscribe","#reserveSnapshot","#receiveWebsocket","#getBackupMonitor","#ensureChangeStreamerStarted","#changeStreamerStarted","#shardID","#changeDB","#changeStreamerURI","#resolveChangeStreamer"],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-http.ts"],"sourcesContent":["import type {IncomingMessage} from 'node:http';\nimport websocket from '@fastify/websocket';\nimport type {LogContext} from '@rocicorp/logger';\nimport WebSocket from 'ws';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport type {IncomingMessageSubset} from '../../types/http.ts';\nimport {pgClient, type PostgresDB} from '../../types/pg.ts';\nimport {type Worker} from '../../types/processes.ts';\nimport {type ShardID} from '../../types/shards.ts';\nimport {\n  streamIn,\n  streamOut,\n  streamOutStringified,\n  type Source,\n} from '../../types/streams.ts';\nimport {URLParams} from '../../types/url-params.ts';\nimport {installWebSocketReceiver} from '../../types/websocket-handoff.ts';\nimport {closeWithError, PROTOCOL_ERROR} from '../../types/ws.ts';\nimport {HttpService} from '../http-service.ts';\nimport type {BackupMonitor} from './backup-monitor.ts';\nimport {\n  downstreamSchema,\n  PROTOCOL_VERSION,\n  type ChangeStreamer,\n  type ChangeStreamerService,\n  type Downstream,\n  type SubscriberContext,\n} from './change-streamer.ts';\nimport {discoverChangeStreamerAddress} from './schema/tables.ts';\nimport {snapshotMessageSchema, type SnapshotMessage} from './snapshot.ts';\n\nconst MIN_SUPPORTED_PROTOCOL_VERSION = 1;\n\nconst SNAPSHOT_PATH_PATTERN = '/replication/:version/snapshot';\nconst CHANGES_PATH_PATTERN = '/replication/:version/changes';\nconst PATH_REGEX = /\\/replication\\/v(?<version>\\d+)\\/(changes|snapshot)$/;\n\nconst SNAPSHOT_PATH = `/replication/v${PROTOCOL_VERSION}/snapshot`;\nconst CHANGES_PATH = `/replication/v${PROTOCOL_VERSION}/changes`;\n\ntype Options = {\n  port: number;\n  keepaliveTimeoutMs: number | undefined;\n  startupDelayMs: number;\n};\n\nexport class ChangeStreamerHttpServer extends HttpService {\n  readonly id = 'change-streamer-http-server';\n  readonly #lc: LogContext;\n  readonly #opts: Options;\n  readonly #changeStreamer: ChangeStreamerService;\n  readonly #backupMonitor: BackupMonitor | null;\n\n  constructor(\n    lc: LogContext,\n    opts: Options,\n    parent: Worker,\n    changeStreamer: ChangeStreamerService,\n    backupMonitor: BackupMonitor | null,\n  ) {\n    super('change-streamer-http-server', lc, opts, async fastify => {\n      await fastify.register(websocket);\n\n      fastify.get(CHANGES_PATH_PATTERN, {websocket: true}, this.#subscribe);\n      fastify.get(\n        SNAPSHOT_PATH_PATTERN,\n        {websocket: true},\n        this.#reserveSnapshot,\n      );\n\n      installWebSocketReceiver<'snapshot' | 'changes'>(\n        lc,\n        fastify.websocketServer,\n        this.#receiveWebsocket,\n        parent,\n      );\n    });\n\n    this.#lc = lc;\n    this.#opts = opts;\n    this.#changeStreamer = changeStreamer;\n    this.#backupMonitor = backupMonitor;\n  }\n\n  #getBackupMonitor() {\n    return must(\n      this.#backupMonitor,\n      'replication-manager is not configured with a ZERO_LITESTREAM_BACKUP_URL',\n    );\n  }\n\n  // Called when receiving a web socket via the main dispatcher handoff.\n  readonly #receiveWebsocket = (\n    ws: WebSocket,\n    action: 'changes' | 'snapshot',\n    msg: IncomingMessageSubset,\n  ) => {\n    switch (action) {\n      case 'snapshot':\n        return this.#reserveSnapshot(ws, msg);\n      case 'changes':\n        return this.#subscribe(ws, msg);\n      default:\n        closeWithError(\n          this._lc,\n          ws,\n          `invalid action \"${action}\" received in handoff`,\n        );\n        return;\n    }\n  };\n\n  readonly #reserveSnapshot = (ws: WebSocket, req: RequestHeaders) => {\n    try {\n      const url = new URL(\n        req.url ?? '',\n        req.headers.origin ?? 'http://localhost',\n      );\n      checkProtocolVersion(url.pathname);\n      const taskID = url.searchParams.get('taskID');\n      if (!taskID) {\n        throw new Error('Missing taskID in snapshot request');\n      }\n      const downstream =\n        this.#getBackupMonitor().startSnapshotReservation(taskID);\n      void streamOut(this._lc, downstream, ws);\n    } catch (err) {\n      closeWithError(this._lc, ws, err, PROTOCOL_ERROR);\n    }\n  };\n\n  readonly #subscribe = async (ws: WebSocket, req: RequestHeaders) => {\n    try {\n      const ctx = getSubscriberContext(req);\n      if (ctx.mode === 'serving') {\n        this.#ensureChangeStreamerStarted('incoming subscription');\n      }\n\n      const downstream = await this.#changeStreamer.subscribe(ctx);\n      if (ctx.initial && ctx.taskID && this.#backupMonitor) {\n        // Now that the change-streamer knows about the subscriber and watermark,\n        // end the reservation to safely resume scheduling cleanup.\n        this.#backupMonitor.endReservation(ctx.taskID);\n      }\n      void streamOutStringified(this._lc, downstream, ws);\n    } catch (err) {\n      closeWithError(this._lc, ws, err, PROTOCOL_ERROR);\n    }\n  };\n\n  #changeStreamerStarted = false;\n\n  #ensureChangeStreamerStarted(reason: string) {\n    if (!this.#changeStreamerStarted && this._state.shouldRun()) {\n      this.#lc.info?.(`starting ChangeStreamerService: ${reason}`);\n      void this.#changeStreamer\n        .run()\n        .catch(e =>\n          this.#lc.warn?.(`ChangeStreamerService ended with error`, e),\n        )\n        .finally(() => this.stop());\n\n      this.#changeStreamerStarted = true;\n    }\n  }\n\n  protected override _onStart(): void {\n    const {startupDelayMs} = this.#opts;\n    this._state.setTimeout(\n      () =>\n        this.#ensureChangeStreamerStarted(\n          `startup delay elapsed (${startupDelayMs} ms)`,\n        ),\n      startupDelayMs,\n    );\n  }\n\n  protected override async _onStop(): Promise<void> {\n    if (this.#changeStreamerStarted) {\n      await this.#changeStreamer.stop();\n    }\n  }\n}\n\nexport class ChangeStreamerHttpClient implements ChangeStreamer {\n  readonly #lc: LogContext;\n  readonly #shardID: ShardID;\n  readonly #changeDB: PostgresDB;\n  readonly #changeStreamerURI: string | undefined;\n\n  constructor(\n    lc: LogContext,\n    shardID: ShardID,\n    changeDB: string,\n    changeStreamerURI: string | undefined,\n  ) {\n    this.#lc = lc;\n    this.#shardID = shardID;\n    // Create a pg client with a single short-lived connection for the purpose\n    // of change-streamer discovery (i.e. ChangeDB as DNS).\n    this.#changeDB = pgClient(lc, changeDB, 'change-streamer-discovery', {\n      max: 1,\n      ['idle_timeout']: 15,\n    });\n    this.#changeStreamerURI = changeStreamerURI;\n  }\n\n  async #resolveChangeStreamer(path: string) {\n    let baseURL = this.#changeStreamerURI;\n    if (!baseURL) {\n      const address = await discoverChangeStreamerAddress(\n        this.#shardID,\n        this.#changeDB,\n      );\n      if (!address) {\n        throw new Error(`no change-streamer is running`);\n      }\n      baseURL = address.includes('://') ? `${address}/` : `ws://${address}/`;\n    }\n    const uri = new URL(path, baseURL);\n    this.#lc.info?.(`connecting to change-streamer@${uri}`);\n    return uri;\n  }\n\n  async reserveSnapshot(taskID: string): Promise<Source<SnapshotMessage>> {\n    const uri = await this.#resolveChangeStreamer(SNAPSHOT_PATH);\n\n    const params = new URLSearchParams({taskID});\n    const ws = new WebSocket(uri + `?${params.toString()}`);\n\n    return streamIn(this.#lc, ws, snapshotMessageSchema);\n  }\n\n  async subscribe(ctx: SubscriberContext): Promise<Source<Downstream>> {\n    const uri = await this.#resolveChangeStreamer(CHANGES_PATH);\n\n    const params = getParams(ctx);\n    const ws = new WebSocket(uri + `?${params.toString()}`);\n\n    return streamIn(this.#lc, ws, downstreamSchema);\n  }\n}\n\ntype RequestHeaders = Pick<IncomingMessage, 'url' | 'headers'>;\n\nexport function getSubscriberContext(req: RequestHeaders): SubscriberContext {\n  const url = new URL(req.url ?? '', req.headers.origin ?? 'http://localhost');\n  const protocolVersion = checkProtocolVersion(url.pathname);\n  const params = new URLParams(url);\n\n  return {\n    protocolVersion,\n    id: params.get('id', true),\n    taskID: params.get('taskID', false),\n    mode: params.get('mode', false) === 'backup' ? 'backup' : 'serving',\n    replicaVersion: params.get('replicaVersion', true),\n    watermark: params.get('watermark', true),\n    initial: params.getBoolean('initial'),\n  };\n}\n\nfunction checkProtocolVersion(pathname: string): number {\n  const match = PATH_REGEX.exec(pathname);\n  if (!match) {\n    throw new Error(`invalid path: ${pathname}`);\n  }\n  const v = Number(match.groups?.version);\n  if (\n    Number.isNaN(v) ||\n    v > PROTOCOL_VERSION ||\n    v < MIN_SUPPORTED_PROTOCOL_VERSION\n  ) {\n    throw new Error(\n      `Cannot service client at protocol v${v}. ` +\n        `Supported protocols: [v${MIN_SUPPORTED_PROTOCOL_VERSION} ... v${PROTOCOL_VERSION}]`,\n    );\n  }\n  return v;\n}\n\n// This is called from the client-side (i.e. the replicator).\nfunction getParams(ctx: SubscriberContext): URLSearchParams {\n  // The protocolVersion is hard-coded into the CHANGES_PATH.\n  const {protocolVersion, ...stringParams} = ctx;\n  assert(\n    protocolVersion === PROTOCOL_VERSION,\n    `replicator should be setting protocolVersion to ${PROTOCOL_VERSION}`,\n  );\n  return new URLSearchParams({\n    ...stringParams,\n    taskID: ctx.taskID ? ctx.taskID : '',\n    initial: ctx.initial ? 'true' : 'false',\n  });\n}\n"],"mappings":";;;;;;;;;;;;;;;AAgCA,IAAM,iCAAiC;AAEvC,IAAM,wBAAwB;AAC9B,IAAM,uBAAuB;AAC7B,IAAM,aAAa;AAEnB,IAAM,gBAAgB;AACtB,IAAM,eAAe;AAQrB,IAAa,2BAAb,cAA8C,YAAY;CACxD,KAAc;CACd;CACA;CACA;CACA;CAEA,YACE,IACA,MACA,QACA,gBACA,eACA;AACA,QAAM,+BAA+B,IAAI,MAAM,OAAM,YAAW;AAC9D,SAAM,QAAQ,SAAS,UAAU;AAEjC,WAAQ,IAAI,sBAAsB,EAAC,WAAW,MAAK,EAAE,MAAA,UAAgB;AACrE,WAAQ,IACN,uBACA,EAAC,WAAW,MAAK,EACjB,MAAA,gBACD;AAED,4BACE,IACA,QAAQ,iBACR,MAAA,kBACA,OACD;IACD;AAEF,QAAA,KAAW;AACX,QAAA,OAAa;AACb,QAAA,iBAAuB;AACvB,QAAA,gBAAsB;;CAGxB,oBAAoB;AAClB,SAAO,KACL,MAAA,eACA,0EACD;;CAIH,qBACE,IACA,QACA,QACG;AACH,UAAQ,QAAR;GACE,KAAK,WACH,QAAO,MAAA,gBAAsB,IAAI,IAAI;GACvC,KAAK,UACH,QAAO,MAAA,UAAgB,IAAI,IAAI;GACjC;AACE,mBACE,KAAK,KACL,IACA,mBAAmB,OAAO,uBAC3B;AACD;;;CAIN,oBAA6B,IAAe,QAAwB;AAClE,MAAI;GACF,MAAM,MAAM,IAAI,IACd,IAAI,OAAO,IACX,IAAI,QAAQ,UAAU,mBACvB;AACD,wBAAqB,IAAI,SAAS;GAClC,MAAM,SAAS,IAAI,aAAa,IAAI,SAAS;AAC7C,OAAI,CAAC,OACH,OAAM,IAAI,MAAM,qCAAqC;GAEvD,MAAM,aACJ,MAAA,kBAAwB,CAAC,yBAAyB,OAAO;AACtD,aAAU,KAAK,KAAK,YAAY,GAAG;WACjC,KAAK;AACZ,kBAAe,KAAK,KAAK,IAAI,KAAK,eAAe;;;CAIrD,aAAsB,OAAO,IAAe,QAAwB;AAClE,MAAI;GACF,MAAM,MAAM,qBAAqB,IAAI;AACrC,OAAI,IAAI,SAAS,UACf,OAAA,4BAAkC,wBAAwB;GAG5D,MAAM,aAAa,MAAM,MAAA,eAAqB,UAAU,IAAI;AAC5D,OAAI,IAAI,WAAW,IAAI,UAAU,MAAA,cAG/B,OAAA,cAAoB,eAAe,IAAI,OAAO;AAE3C,wBAAqB,KAAK,KAAK,YAAY,GAAG;WAC5C,KAAK;AACZ,kBAAe,KAAK,KAAK,IAAI,KAAK,eAAe;;;CAIrD,yBAAyB;CAEzB,6BAA6B,QAAgB;AAC3C,MAAI,CAAC,MAAA,yBAA+B,KAAK,OAAO,WAAW,EAAE;AAC3D,SAAA,GAAS,OAAO,mCAAmC,SAAS;AACvD,SAAA,eACF,KAAK,CACL,OAAM,MACL,MAAA,GAAS,OAAO,0CAA0C,EAAE,CAC7D,CACA,cAAc,KAAK,MAAM,CAAC;AAE7B,SAAA,wBAA8B;;;CAIlC,WAAoC;EAClC,MAAM,EAAC,mBAAkB,MAAA;AACzB,OAAK,OAAO,iBAER,MAAA,4BACE,0BAA0B,eAAe,MAC1C,EACH,eACD;;CAGH,MAAyB,UAAyB;AAChD,MAAI,MAAA,sBACF,OAAM,MAAA,eAAqB,MAAM;;;AAKvC,IAAa,2BAAb,MAAgE;CAC9D;CACA;CACA;CACA;CAEA,YACE,IACA,SACA,UACA,mBACA;AACA,QAAA,KAAW;AACX,QAAA,UAAgB;AAGhB,QAAA,WAAiB,SAAS,IAAI,UAAU,6BAA6B;GACnE,KAAK;IACJ,iBAAiB;GACnB,CAAC;AACF,QAAA,oBAA0B;;CAG5B,OAAA,sBAA6B,MAAc;EACzC,IAAI,UAAU,MAAA;AACd,MAAI,CAAC,SAAS;GACZ,MAAM,UAAU,MAAM,8BACpB,MAAA,SACA,MAAA,SACD;AACD,OAAI,CAAC,QACH,OAAM,IAAI,MAAM,gCAAgC;AAElD,aAAU,QAAQ,SAAS,MAAM,GAAG,GAAG,QAAQ,KAAK,QAAQ,QAAQ;;EAEtE,MAAM,MAAM,IAAI,IAAI,MAAM,QAAQ;AAClC,QAAA,GAAS,OAAO,iCAAiC,MAAM;AACvD,SAAO;;CAGT,MAAM,gBAAgB,QAAkD;EAItE,MAAM,KAAK,IAAI,UAHH,MAAM,MAAA,sBAA4B,cAAc,GAG7B,IADhB,IAAI,gBAAgB,EAAC,QAAO,CAAC,CACF,UAAU,GAAG;AAEvD,SAAO,SAAS,MAAA,IAAU,IAAI,sBAAsB;;CAGtD,MAAM,UAAU,KAAqD;EAInE,MAAM,KAAK,IAAI,UAHH,MAAM,MAAA,sBAA4B,aAAa,GAG5B,IADhB,UAAU,IAAI,CACa,UAAU,GAAG;AAEvD,SAAO,SAAS,MAAA,IAAU,IAAI,iBAAiB;;;AAMnD,SAAgB,qBAAqB,KAAwC;CAC3E,MAAM,MAAM,IAAI,IAAI,IAAI,OAAO,IAAI,IAAI,QAAQ,UAAU,mBAAmB;CAC5E,MAAM,kBAAkB,qBAAqB,IAAI,SAAS;CAC1D,MAAM,SAAS,IAAI,UAAU,IAAI;AAEjC,QAAO;EACL;EACA,IAAI,OAAO,IAAI,MAAM,KAAK;EAC1B,QAAQ,OAAO,IAAI,UAAU,MAAM;EACnC,MAAM,OAAO,IAAI,QAAQ,MAAM,KAAK,WAAW,WAAW;EAC1D,gBAAgB,OAAO,IAAI,kBAAkB,KAAK;EAClD,WAAW,OAAO,IAAI,aAAa,KAAK;EACxC,SAAS,OAAO,WAAW,UAAU;EACtC;;AAGH,SAAS,qBAAqB,UAA0B;CACtD,MAAM,QAAQ,WAAW,KAAK,SAAS;AACvC,KAAI,CAAC,MACH,OAAM,IAAI,MAAM,iBAAiB,WAAW;CAE9C,MAAM,IAAI,OAAO,MAAM,QAAQ,QAAQ;AACvC,KACE,OAAO,MAAM,EAAE,IACf,IAAA,KACA,IAAI,+BAEJ,OAAM,IAAI,MACR,sCAAsC,EAAE,2BACZ,+BAA+B,UAC5D;AAEH,QAAO;;AAIT,SAAS,UAAU,KAAyC;CAE1D,MAAM,EAAC,iBAAiB,GAAG,iBAAgB;AAC3C,QACE,oBAAA,GACA,oDACD;AACD,QAAO,IAAI,gBAAgB;EACzB,GAAG;EACH,QAAQ,IAAI,SAAS,IAAI,SAAS;EAClC,SAAS,IAAI,UAAU,SAAS;EACjC,CAAC"}

package/out/zero-cache/src/services/change-streamer/change-streamer-service.d.ts

@@ -14,6 +14,7 @@ export type TuningOptions = StorerOptions & {
  * Performs initialization and schema migrations to initialize a ChangeStreamerImpl.
  */
 export declare function initializeStreamer(lc: LogContext, shard: ShardID, taskID: string, discoveryAddress: string, discoveryProtocol: string, changeDB: PostgresDB, changeSource: ChangeSource, replicationStatusPublisher: ReplicationStatusPublisher, subscriptionState: SubscriptionState, purgeLock: PurgeLock | null, autoReset: boolean, opts: TuningOptions, setTimeoutFn?: typeof setTimeout): Promise<ChangeStreamerService>;
+export type ChangeTag = ChangeStreamData[1]['tag'];
 /**
  * Internally all Downstream messages (not just commits) are given a watermark.
  * These are used for internal ordering for:
@@ -23,6 +24,14 @@ export declare function initializeStreamer(lc: LogContext, shard: ShardID, taskI
  * However, only the watermark for `Commit` messages are exposed to
  * subscribers, as that is the only semantically correct watermark to
  * use for tracking a position in a replication stream.
+ *
+ * Additionally, the ChangeStreamData is eagerly stringified once, after which
+ * the string is passed to the changeLog and all subscribers, eliminating
+ * redundant stringification and reducing GC churn.
  */
-export type WatermarkedChange = [watermark: string, ChangeStreamData];
+export type WatermarkedChange = [
+    watermark: string,
+    tag: ChangeTag,
+    json: string
+];
 //# sourceMappingURL=change-streamer-service.d.ts.map

package/out/zero-cache/src/services/change-streamer/change-streamer-service.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"change-streamer-service.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-service.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAWjD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,mBAAmB,CAAC;AAClD,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,uBAAuB,CAAC;AAGnD,OAAO,KAAK,EACV,YAAY,EAEb,MAAM,mCAAmC,CAAC;AAC3C,OAAO,EAEL,KAAK,gBAAgB,EACtB,MAAM,iDAAiD,CAAC;AACzD,OAAO,EAGL,KAAK,0BAA0B,EAChC,MAAM,qCAAqC,CAAC;AAC7C,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,2CAA2C,CAAC;AAMjF,OAAO,EACL,KAAK,qBAAqB,EAI3B,MAAM,sBAAsB,CAAC;AAS9B,OAAO,EAEL,KAAK,SAAS,EACd,KAAK,aAAa,IAAI,aAAa,EACpC,MAAM,aAAa,CAAC;AAGrB,MAAM,MAAM,aAAa,GAAG,aAAa,GAAG;IAC1C,kCAAkC,EAAE,MAAM,CAAC;CAC5C,CAAC;AAEF;;GAEG;AACH,wBAAsB,kBAAkB,CACtC,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,MAAM,EAAE,MAAM,EACd,gBAAgB,EAAE,MAAM,EACxB,iBAAiB,EAAE,MAAM,EACzB,QAAQ,EAAE,UAAU,EACpB,YAAY,EAAE,YAAY,EAC1B,0BAA0B,EAAE,0BAA0B,EACtD,iBAAiB,EAAE,iBAAiB,EACpC,SAAS,EAAE,SAAS,GAAG,IAAI,EAC3B,SAAS,EAAE,OAAO,EAClB,IAAI,EAAE,aAAa,EACnB,YAAY,oBAAa,GACxB,OAAO,CAAC,qBAAqB,CAAC,CA4BhC;AAID;;;;;;;;;GASG;AACH,MAAM,MAAM,iBAAiB,GAAG,CAAC,SAAS,EAAE,MAAM,EAAE,gBAAgB,CAAC,CAAC"}
+{"version":3,"file":"change-streamer-service.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-service.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAWjD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,mBAAmB,CAAC;AAClD,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,uBAAuB,CAAC;AAGnD,OAAO,KAAK,EACV,YAAY,EAEb,MAAM,mCAAmC,CAAC;AAC3C,OAAO,EAEL,KAAK,gBAAgB,EAEtB,MAAM,iDAAiD,CAAC;AACzD,OAAO,EAGL,KAAK,0BAA0B,EAChC,MAAM,qCAAqC,CAAC;AAC7C,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,2CAA2C,CAAC;AAMjF,OAAO,EACL,KAAK,qBAAqB,EAG3B,MAAM,sBAAsB,CAAC;AAS9B,OAAO,EAEL,KAAK,SAAS,EACd,KAAK,aAAa,IAAI,aAAa,EACpC,MAAM,aAAa,CAAC;AAGrB,MAAM,MAAM,aAAa,GAAG,aAAa,GAAG;IAC1C,kCAAkC,EAAE,MAAM,CAAC;CAC5C,CAAC;AAEF;;GAEG;AACH,wBAAsB,kBAAkB,CACtC,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,MAAM,EAAE,MAAM,EACd,gBAAgB,EAAE,MAAM,EACxB,iBAAiB,EAAE,MAAM,EACzB,QAAQ,EAAE,UAAU,EACpB,YAAY,EAAE,YAAY,EAC1B,0BAA0B,EAAE,0BAA0B,EACtD,iBAAiB,EAAE,iBAAiB,EACpC,SAAS,EAAE,SAAS,GAAG,IAAI,EAC3B,SAAS,EAAE,OAAO,EAClB,IAAI,EAAE,aAAa,EACnB,YAAY,oBAAa,GACxB,OAAO,CAAC,qBAAqB,CAAC,CA4BhC;AAID,MAAM,MAAM,SAAS,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;AAEnD;;;;;;;;;;;;;GAaG;AACH,MAAM,MAAM,iBAAiB,GAAG;IAC9B,SAAS,EAAE,MAAM;IACjB,GAAG,EAAE,SAAS;IACd,IAAI,EAAE,MAAM;CACb,CAAC"}

package/out/zero-cache/src/services/change-streamer/change-streamer-service.js

@@ -246,8 +246,13 @@ var ChangeStreamerImpl = class {
 							if (watermark === null) throw new UnrecoverableError(`${type} change (${msg.tag}) received before 'begin' message`);
 							break;
 					}
-					const entry = [watermark, change];
-					unflushedBytes += this.#storer.store(entry);
+					const json = this.#storer.store(watermark, change);
+					const entry = [
+						watermark,
+						change[1].tag,
+						json
+					];
+					unflushedBytes += json.length;
 					if (unflushedBytes < flushBytesThreshold) this.#forwarder.forward(entry);
 					else {
 						await this.#forwarder.forwardWithFlowControl(entry);
@@ -266,7 +271,11 @@ var ChangeStreamerImpl = class {
 			if (watermark) {
 				this.#lc.warn?.(`aborting interrupted transaction ${watermark}`);
 				this.#storer.abort();
-				this.#forwarder.forward([watermark, ["rollback", { tag: "rollback" }]]);
+				this.#forwarder.forward([
+					watermark,
+					"rollback",
+					ROLLBACK_JSON
+				]);
 			}
 			await Promise.all([
 				this.#storer.stop(),
@@ -362,6 +371,7 @@ var ChangeStreamerImpl = class {
 	}
 };
 var CLEANUP_DELAY_MS = DEFAULT_MAX_RETRY_DELAY_MS * 3;
+var ROLLBACK_JSON = JSON.stringify(["rollback", { tag: "rollback" }]);
 //#endregion
 export { initializeStreamer };