@rocicorp/zero 0.26.2-canary.4 → 0.26.2-canary.7

package/out/zero-cache/src/services/change-source/pg/schema/ddl.js

@@ -25,7 +25,53 @@ var ddlUpdateEventSchema = ddlEventSchema.extend({
 	type: valita_exports.literal("ddlUpdate"),
 	event: valita_exports.object({ tag: valita_exports.string() })
 });
-var replicationEventSchema = valita_exports.union(ddlStartEventSchema, ddlUpdateEventSchema);
+/**
+* The `schemaSnapshot` message is a snapshot of a schema taken in response to
+* a `COMMENT ON PUBLICATION` command, which is a hook recognized by zero
+* to manually emit schema snapshots to support detection of schema changes
+* from `ALTER PUBLICATION` commands on supabase, which does not fire event
+* triggers for them (https://github.com/supabase/supautils/issues/123).
+*
+* The hook is exercised by bookmarking the publication change with
+* `COMMENT ON PUBLICATION` statements within e.g.
+*
+* ```sql
+* BEGIN;
+* COMMENT ON PUBLICATION my_publication IS 'whatever';
+* ALTER PUBLICATION my_publication ...;
+* COMMENT ON PUBLICATION my_publication IS 'whatever';
+* COMMIT;
+* ```
+*
+* The `change-source` will perform the diff between a `schemaSnapshot`
+* events and its preceding `schemaSnapshot` (or `ddlUpdate`) within the
+* transaction.
+*
+* In the case where event trigger support is missing, this results in
+* diffing the `schemaSnapshot`s before and after the `ALTER PUBLICATION`
+* statement, thus effecting the same logic that would have been exercised
+* between the `ddlStart` and `ddlEvent` events fired by a database with
+* fully functional event triggers.
+*
+* Note that if the same transaction is run on a database that *does*
+* support event triggers on `ALTER PUBLICATION` statements, the sequence
+* of emitted messages will be:
+*
+* * `schemaSnapshot`
+* * `ddlStart`
+* * `ddlUpdate`
+* * `schemaSnapshot`
+*
+* Since `schemaSnapshot` messages are diffed with the preceding
+* `schemaSnapshot` or `ddlUpdate` event (if any), there will be no schema
+* difference between the `ddlUpdate` and the second `schemaSnapshot`, and
+* thus the extra `COMMENT` statements will effectively be no-ops.
+*/
+var schemaSnapshotEventSchema = ddlEventSchema.extend({
+	type: valita_exports.literal("schemaSnapshot"),
+	event: valita_exports.object({ tag: valita_exports.string() })
+});
+var replicationEventSchema = valita_exports.union(ddlStartEventSchema, ddlUpdateEventSchema, schemaSnapshotEventSchema);
 function append(shardNum) {
 	return (name) => id(name + "_" + String(shardNum));
 }
@@ -116,6 +162,8 @@ DECLARE
   schema_specs TEXT;
   message TEXT;
   event TEXT;
+  event_type TEXT;
+  event_prefix TEXT;
 BEGIN
   publications := ARRAY[${literal(publications)}];
 
@@ -124,6 +172,7 @@ BEGIN
     LIMIT 1 INTO target;
 
   -- Filter DDL updates that are not relevant to the shard (i.e. publications) when possible.
+  SELECT true INTO relevant;
 
   -- Note: ALTER TABLE statements may *remove* the table from the set of published
   --       tables, and there is no way to determine if the table "used to be" in the
@@ -136,10 +185,6 @@ BEGIN
       JOIN pg_publication_tables AS pb ON pb.schemaname = ns.nspname AND pb.tablename = c.relname
       WHERE c.oid = target.objid AND pb.pubname = ANY (publications)
       INTO relevant;
-    IF relevant IS NULL THEN
-      PERFORM ${schema}.notice_ignore(tag, target);
-      RETURN;
-    END IF;
 
   ELSIF target.object_type = 'index' THEN
     SELECT ns.nspname AS "schema", c.relname AS "name" FROM pg_class AS c
@@ -148,30 +193,18 @@ BEGIN
       JOIN pg_publication_tables AS pb ON pb.schemaname = ns.nspname AND pb.tablename = ind.tablename
       WHERE c.oid = target.objid AND pb.pubname = ANY (publications)
       INTO relevant;
-    IF relevant IS NULL THEN
-      PERFORM ${schema}.notice_ignore(tag, target);
-      RETURN;
-    END IF;
 
   ELSIF target.object_type = 'publication relation' THEN
     SELECT pb.pubname FROM pg_publication_rel AS rel
       JOIN pg_publication AS pb ON pb.oid = rel.prpubid
       WHERE rel.oid = target.objid AND pb.pubname = ANY (publications) 
       INTO relevant;
-    IF relevant IS NULL THEN
-      PERFORM ${schema}.notice_ignore(tag, target);
-      RETURN;
-    END IF;
 
   ELSIF target.object_type = 'publication namespace' THEN
     SELECT pb.pubname FROM pg_publication_namespace AS ns
       JOIN pg_publication AS pb ON pb.oid = ns.pnpubid
       WHERE ns.oid = target.objid AND pb.pubname = ANY (publications) 
       INTO relevant;
-    IF relevant IS NULL THEN
-      PERFORM ${schema}.notice_ignore(tag, target);
-      RETURN;
-    END IF;
 
   ELSIF target.object_type = 'schema' THEN
     SELECT ns.nspname AS "schema", c.relname AS "name" FROM pg_class AS c
@@ -179,18 +212,35 @@ BEGIN
       JOIN pg_publication_tables AS pb ON pb.schemaname = ns.nspname AND pb.tablename = c.relname
       WHERE ns.oid = target.objid AND pb.pubname = ANY (publications)
       INTO relevant;
-    IF relevant IS NULL THEN
-      PERFORM ${schema}.notice_ignore(tag, target);
-      RETURN;
-    END IF;
+
+  ELSIF target.object_type = 'publication' THEN
+    SELECT 1 WHERE target.object_identity = ANY (publications)
+      INTO relevant;
 
   -- no-op CREATE IF NOT EXIST statements
   ELSIF tag LIKE 'CREATE %' AND target.object_type IS NULL THEN
+    relevant := NULL;
+  END IF;
+
+  IF relevant IS NULL THEN
     PERFORM ${schema}.notice_ignore(tag, target);
     RETURN;
   END IF;
 
-  RAISE INFO 'Creating ddlUpdate for % %', tag, row_to_json(target);
+  IF tag = 'COMMENT' THEN
+    -- Only make schemaSnapshots for COMMENT ON PUBLICATION
+    IF target.object_type != 'publication' THEN
+      PERFORM ${schema}.notice_ignore(tag, target);
+      RETURN;
+    END IF;
+    event_type := 'schemaSnapshot';
+    event_prefix := '/ddl';
+  ELSE
+    event_type := 'ddlUpdate';
+    event_prefix := '';  -- TODO: Use '/ddl' for both when rollback safe
+  END IF;
+
+  RAISE INFO 'Creating % for % %', event_type, tag, row_to_json(target);
 
   -- Construct and emit the DdlUpdateEvent message.
   SELECT json_build_object('tag', tag) INTO event;
@@ -198,14 +248,14 @@ BEGIN
   SELECT ${schema}.schema_specs() INTO schema_specs;
 
   SELECT json_build_object(
-    'type', 'ddlUpdate',
+    'type', event_type,
     'version', 1,
     'schema', schema_specs::json,
     'event', event::json,
     'context', ${schema}.get_trigger_context()
   ) INTO message;
 
-  PERFORM pg_logical_emit_message(true, ${literal(`${appID}/${shardNum}`)}, message);
+  PERFORM pg_logical_emit_message(true, ${literal(`${appID}/${shardNum}`)} || event_prefix, message);
 END
 $$ LANGUAGE plpgsql;
 `;
@@ -231,7 +281,7 @@ CREATE EVENT TRIGGER ${sharded(`${appID}_ddl_start`)}
   WHEN TAG IN (${literal(TAGS)})
   EXECUTE PROCEDURE ${schema}.emit_ddl_start();
 `);
-	for (const tag of TAGS) {
+	for (const tag of [...TAGS, "COMMENT"]) {
 		const tagID = tag.toLowerCase().replace(" ", "_");
 		triggers.push(`
 CREATE OR REPLACE FUNCTION ${schema}.emit_${tagID}() 
@@ -254,7 +304,7 @@ function dropEventTriggerStatements(appID, shardID) {
 	stmts.push(`
     DROP EVENT TRIGGER IF EXISTS ${id(`${appID}_ddl_start_${shardID}`)};
   `);
-	for (const tag of TAGS) {
+	for (const tag of [...TAGS, "COMMENT"]) {
 		const tagID = tag.toLowerCase().replace(" ", "_");
 		stmts.push(`
       DROP EVENT TRIGGER IF EXISTS ${id(`${appID}_${tagID}_${shardID}`)};

package/out/zero-cache/src/services/change-source/pg/schema/ddl.js.map

@@ -1 +1 @@
-{"version":3,"file":"ddl.js","names":[],"sources":["../../../../../../../../zero-cache/src/services/change-source/pg/schema/ddl.ts"],"sourcesContent":["import {literal as lit} from 'pg-format';\nimport {assert} from '../../../../../../shared/src/asserts.ts';\nimport * as v from '../../../../../../shared/src/valita.ts';\nimport {upstreamSchema, type ShardConfig} from '../../../../types/shards.ts';\nimport {id} from '../../../../types/sql.ts';\nimport {\n  indexDefinitionsQuery,\n  publishedSchema,\n  publishedTableQuery,\n} from './published.ts';\n\n// Sent in the 'version' tag of \"ddlStart\" and \"ddlUpdate\" event messages.\n// This is used to ensure that the message constructed in the upstream\n// Trigger function is compatible with the code processing it in the zero-cache.\n//\n// Increment this when changing the format of the contents of the \"ddl\" events.\n// This will allow old / incompatible code to detect the change and abort.\nexport const PROTOCOL_VERSION = 1;\n\nconst triggerEvent = v.object({\n  context: v.object({query: v.string()}).rest(v.string()),\n});\n\n// All DDL events contain a snapshot of the current tables and indexes that\n// are published / relevant to the shard.\nexport const ddlEventSchema = triggerEvent.extend({\n  version: v.literal(PROTOCOL_VERSION),\n  schema: publishedSchema,\n});\n\n// The `ddlStart` message is computed before every DDL event, regardless of\n// whether the subsequent event affects the shard. Downstream processing should\n// capture the contained schema information in order to determine the schema\n// changes necessary to apply a subsequent `ddlUpdate` message. Note that a\n// `ddlUpdate` message may not follow, as updates determined to be irrelevant\n// to the shard will not result in a message. However, all `ddlUpdate` messages\n// are guaranteed to be preceded by a `ddlStart` message.\nexport const ddlStartEventSchema = ddlEventSchema.extend({\n  type: v.literal('ddlStart'),\n});\n\nexport type DdlStartEvent = v.Infer<typeof ddlStartEventSchema>;\n\n/**\n * The {@link DdlUpdateEvent} contains an updated schema resulting from\n * a particular ddl event. The event type provides information\n * (i.e. constraints) on the difference from the schema of the preceding\n * {@link DdlStartEvent}.\n *\n * Note that in almost all cases (the exception being `CREATE` events),\n * it is possible that there is no relevant difference between the\n * ddl-start schema and the ddl-update schema, as many aspects of the\n * schema (e.g. column constraints) are not relevant to downstream\n * replication.\n */\nexport const ddlUpdateEventSchema = ddlEventSchema.extend({\n  type: v.literal('ddlUpdate'),\n  event: v.object({tag: v.string()}),\n});\n\nexport type DdlUpdateEvent = v.Infer<typeof ddlUpdateEventSchema>;\n\nexport const replicationEventSchema = v.union(\n  ddlStartEventSchema,\n  ddlUpdateEventSchema,\n);\n\nexport type ReplicationEvent = v.Infer<typeof replicationEventSchema>;\n\n// Creates a function that appends `_{shard-num}` to the input and\n// quotes the result to be a valid identifier.\nfunction append(shardNum: number) {\n  return (name: string) => id(name + '_' + String(shardNum));\n}\n\n/**\n * Event trigger functions contain the core logic that are invoked by triggers.\n *\n * Note that although many of these functions can theoretically be parameterized and\n * shared across shards, it is advantageous to keep the functions in each shard\n * isolated from each other in order to avoid the complexity of shared-function\n * versioning.\n *\n * In a sense, shards (and their triggers and functions) should be thought of as\n * execution environments that can be updated at different schedules. If per-shard\n * triggers called into shared functions, we would have to consider versioning the\n * functions when changing their behavior, backwards compatibility, removal of\n * unused versions, etc. (not unlike versioning of npm packages).\n *\n * Instead, we opt for the simplicity and isolation of having each shard\n * completely own (and maintain) the entirety of its trigger/function stack.\n */\nfunction createEventFunctionStatements(shard: ShardConfig) {\n  const {appID, shardNum, publications} = shard;\n  const schema = id(upstreamSchema(shard)); // e.g. \"{APP_ID}_{SHARD_ID}\"\n  return /*sql*/ `\nCREATE SCHEMA IF NOT EXISTS ${schema};\n\nCREATE OR REPLACE FUNCTION ${schema}.get_trigger_context()\nRETURNS record AS $$\nDECLARE\n  result record;\nBEGIN\n  SELECT current_query() AS \"query\" into result;\n  RETURN result;\nEND\n$$ LANGUAGE plpgsql;\n\n\nCREATE OR REPLACE FUNCTION ${schema}.notice_ignore(tag TEXT, target record)\nRETURNS void AS $$\nBEGIN\n  RAISE NOTICE 'zero(%) ignoring % %', ${lit(shardNum)}, tag, row_to_json(target);\nEND\n$$ LANGUAGE plpgsql;\n\n\nCREATE OR REPLACE FUNCTION ${schema}.schema_specs()\nRETURNS TEXT AS $$\nDECLARE\n  tables record;\n  indexes record;\nBEGIN\n  ${publishedTableQuery(publications)} INTO tables;\n  ${indexDefinitionsQuery(publications)} INTO indexes;\n  RETURN json_build_object(\n    'tables', tables.tables,\n    'indexes', indexes.indexes\n  );\nEND\n$$ LANGUAGE plpgsql;\n\n\nCREATE OR REPLACE FUNCTION ${schema}.emit_ddl_start()\nRETURNS event_trigger AS $$\nDECLARE\n  schema_specs TEXT;\n  message TEXT;\nBEGIN\n  SELECT ${schema}.schema_specs() INTO schema_specs;\n\n  SELECT json_build_object(\n    'type', 'ddlStart',\n    'version', ${PROTOCOL_VERSION},\n    'schema', schema_specs::json,\n    'context', ${schema}.get_trigger_context()\n  ) INTO message;\n\n  PERFORM pg_logical_emit_message(true, ${lit(\n    `${appID}/${shardNum}`,\n  )}, message);\nEND\n$$ LANGUAGE plpgsql;\n\n\nCREATE OR REPLACE FUNCTION ${schema}.emit_ddl_end(tag TEXT)\nRETURNS void AS $$\nDECLARE\n  publications TEXT[];\n  target RECORD;\n  relevant RECORD;\n  schema_specs TEXT;\n  message TEXT;\n  event TEXT;\nBEGIN\n  publications := ARRAY[${lit(publications)}];\n\n  SELECT objid, object_type, object_identity \n    FROM pg_event_trigger_ddl_commands() \n    LIMIT 1 INTO target;\n\n  -- Filter DDL updates that are not relevant to the shard (i.e. publications) when possible.\n\n  -- Note: ALTER TABLE statements may *remove* the table from the set of published\n  --       tables, and there is no way to determine if the table \"used to be\" in the\n  --       set. Thus, all ALTER TABLE statements must produce a ddl update, similar to\n  --       any DROP * statement.\n  IF (target.object_type = 'table' AND tag != 'ALTER TABLE') \n     OR target.object_type = 'table column' THEN\n    SELECT ns.nspname AS \"schema\", c.relname AS \"name\" FROM pg_class AS c\n      JOIN pg_namespace AS ns ON c.relnamespace = ns.oid\n      JOIN pg_publication_tables AS pb ON pb.schemaname = ns.nspname AND pb.tablename = c.relname\n      WHERE c.oid = target.objid AND pb.pubname = ANY (publications)\n      INTO relevant;\n    IF relevant IS NULL THEN\n      PERFORM ${schema}.notice_ignore(tag, target);\n      RETURN;\n    END IF;\n\n  ELSIF target.object_type = 'index' THEN\n    SELECT ns.nspname AS \"schema\", c.relname AS \"name\" FROM pg_class AS c\n      JOIN pg_namespace AS ns ON c.relnamespace = ns.oid\n      JOIN pg_indexes as ind ON ind.schemaname = ns.nspname AND ind.indexname = c.relname\n      JOIN pg_publication_tables AS pb ON pb.schemaname = ns.nspname AND pb.tablename = ind.tablename\n      WHERE c.oid = target.objid AND pb.pubname = ANY (publications)\n      INTO relevant;\n    IF relevant IS NULL THEN\n      PERFORM ${schema}.notice_ignore(tag, target);\n      RETURN;\n    END IF;\n\n  ELSIF target.object_type = 'publication relation' THEN\n    SELECT pb.pubname FROM pg_publication_rel AS rel\n      JOIN pg_publication AS pb ON pb.oid = rel.prpubid\n      WHERE rel.oid = target.objid AND pb.pubname = ANY (publications) \n      INTO relevant;\n    IF relevant IS NULL THEN\n      PERFORM ${schema}.notice_ignore(tag, target);\n      RETURN;\n    END IF;\n\n  ELSIF target.object_type = 'publication namespace' THEN\n    SELECT pb.pubname FROM pg_publication_namespace AS ns\n      JOIN pg_publication AS pb ON pb.oid = ns.pnpubid\n      WHERE ns.oid = target.objid AND pb.pubname = ANY (publications) \n      INTO relevant;\n    IF relevant IS NULL THEN\n      PERFORM ${schema}.notice_ignore(tag, target);\n      RETURN;\n    END IF;\n\n  ELSIF target.object_type = 'schema' THEN\n    SELECT ns.nspname AS \"schema\", c.relname AS \"name\" FROM pg_class AS c\n      JOIN pg_namespace AS ns ON c.relnamespace = ns.oid\n      JOIN pg_publication_tables AS pb ON pb.schemaname = ns.nspname AND pb.tablename = c.relname\n      WHERE ns.oid = target.objid AND pb.pubname = ANY (publications)\n      INTO relevant;\n    IF relevant IS NULL THEN\n      PERFORM ${schema}.notice_ignore(tag, target);\n      RETURN;\n    END IF;\n\n  -- no-op CREATE IF NOT EXIST statements\n  ELSIF tag LIKE 'CREATE %' AND target.object_type IS NULL THEN\n    PERFORM ${schema}.notice_ignore(tag, target);\n    RETURN;\n  END IF;\n\n  RAISE INFO 'Creating ddlUpdate for % %', tag, row_to_json(target);\n\n  -- Construct and emit the DdlUpdateEvent message.\n  SELECT json_build_object('tag', tag) INTO event;\n  \n  SELECT ${schema}.schema_specs() INTO schema_specs;\n\n  SELECT json_build_object(\n    'type', 'ddlUpdate',\n    'version', ${PROTOCOL_VERSION},\n    'schema', schema_specs::json,\n    'event', event::json,\n    'context', ${schema}.get_trigger_context()\n  ) INTO message;\n\n  PERFORM pg_logical_emit_message(true, ${lit(\n    `${appID}/${shardNum}`,\n  )}, message);\nEND\n$$ LANGUAGE plpgsql;\n`;\n}\n\n// Exported for testing.\nexport const TAGS = [\n  'CREATE TABLE',\n  'ALTER TABLE',\n  'CREATE INDEX',\n  'DROP TABLE',\n  'DROP INDEX',\n  'ALTER PUBLICATION',\n  'ALTER SCHEMA',\n] as const;\n\nexport function createEventTriggerStatements(shard: ShardConfig) {\n  // Better to assert here than get a cryptic syntax error from Postgres.\n  assert(shard.publications.length, `shard publications must be non-empty`);\n\n  // Unlike functions, which are namespaced in shard-specific schemas,\n  // EVENT TRIGGER names are in the global namespace and thus must include\n  // the appID and shardNum.\n  const {appID, shardNum} = shard;\n  const sharded = append(shardNum);\n  const schema = id(upstreamSchema(shard));\n\n  const triggers = [\n    dropEventTriggerStatements(shard.appID, shard.shardNum),\n    createEventFunctionStatements(shard),\n  ];\n\n  // A single ddl_command_start trigger covering all relevant tags.\n  triggers.push(/*sql*/ `\nCREATE EVENT TRIGGER ${sharded(`${appID}_ddl_start`)}\n  ON ddl_command_start\n  WHEN TAG IN (${lit(TAGS)})\n  EXECUTE PROCEDURE ${schema}.emit_ddl_start();\n`);\n\n  // A per-tag ddl_command_end trigger that dispatches to ${schema}.emit_ddl_end(tag)\n  for (const tag of TAGS) {\n    const tagID = tag.toLowerCase().replace(' ', '_');\n    triggers.push(/*sql*/ `\nCREATE OR REPLACE FUNCTION ${schema}.emit_${tagID}() \nRETURNS event_trigger AS $$\nBEGIN\n  PERFORM ${schema}.emit_ddl_end(${lit(tag)});\nEND\n$$ LANGUAGE plpgsql;\n\nCREATE EVENT TRIGGER ${sharded(`${appID}_${tagID}`)}\n  ON ddl_command_end\n  WHEN TAG IN (${lit(tag)})\n  EXECUTE PROCEDURE ${schema}.emit_${tagID}();\n`);\n  }\n  return triggers.join('');\n}\n\n// Exported for testing.\nexport function dropEventTriggerStatements(\n  appID: string,\n  shardID: string | number,\n) {\n  const stmts: string[] = [];\n  // A single ddl_command_start trigger covering all relevant tags.\n  stmts.push(/*sql*/ `\n    DROP EVENT TRIGGER IF EXISTS ${id(`${appID}_ddl_start_${shardID}`)};\n  `);\n\n  // A per-tag ddl_command_end trigger that dispatches to ${schema}.emit_ddl_end(tag)\n  for (const tag of TAGS) {\n    const tagID = tag.toLowerCase().replace(' ', '_');\n    stmts.push(/*sql*/ `\n      DROP EVENT TRIGGER IF EXISTS ${id(`${appID}_${tagID}_${shardID}`)};\n    `);\n  }\n  return stmts.join('');\n}\n"],"mappings":";;;;;;AAyBA,IAAa,iBANQ,eAAE,OAAO,EAC5B,SAAS,eAAE,OAAO,EAAC,OAAO,eAAE,QAAQ,EAAC,CAAC,CAAC,KAAK,eAAE,QAAQ,CAAC,EACxD,CAAC,CAIyC,OAAO;CAChD,SAAS,eAAE,QAAA,EAAyB;CACpC,QAAQ;CACT,CAAC;AASF,IAAa,sBAAsB,eAAe,OAAO,EACvD,MAAM,eAAE,QAAQ,WAAW,EAC5B,CAAC;;;;;;;;;;;;;AAgBF,IAAa,uBAAuB,eAAe,OAAO;CACxD,MAAM,eAAE,QAAQ,YAAY;CAC5B,OAAO,eAAE,OAAO,EAAC,KAAK,eAAE,QAAQ,EAAC,CAAC;CACnC,CAAC;AAIF,IAAa,yBAAyB,eAAE,MACtC,qBACA,qBACD;AAMD,SAAS,OAAO,UAAkB;AAChC,SAAQ,SAAiB,GAAG,OAAO,MAAM,OAAO,SAAS,CAAC;;;;;;;;;;;;;;;;;;;AAoB5D,SAAS,8BAA8B,OAAoB;CACzD,MAAM,EAAC,OAAO,UAAU,iBAAgB;CACxC,MAAM,SAAS,GAAG,eAAe,MAAM,CAAC;AACxC,QAAe;8BACa,OAAO;;6BAER,OAAO;;;;;;;;;;;6BAWP,OAAO;;;yCAGK,QAAI,SAAS,CAAC;;;;;6BAK1B,OAAO;;;;;;IAMhC,oBAAoB,aAAa,CAAC;IAClC,sBAAsB,aAAa,CAAC;;;;;;;;;6BASX,OAAO;;;;;;WAMzB,OAAO;;;;;;iBAMD,OAAO;;;0CAGkB,QACtC,GAAG,MAAM,GAAG,WACb,CAAC;;;;;6BAKyB,OAAO;;;;;;;;;;0BAUV,QAAI,aAAa,CAAC;;;;;;;;;;;;;;;;;;;;gBAoB5B,OAAO;;;;;;;;;;;;gBAYP,OAAO;;;;;;;;;;gBAUP,OAAO;;;;;;;;;;gBAUP,OAAO;;;;;;;;;;;gBAWP,OAAO;;;;;;cAMT,OAAO;;;;;;;;;WASV,OAAO;;;;;;;iBAOD,OAAO;;;0CAGkB,QACtC,GAAG,MAAM,GAAG,WACb,CAAC;;;;;AAOJ,IAAa,OAAO;CAClB;CACA;CACA;CACA;CACA;CACA;CACA;CACD;AAED,SAAgB,6BAA6B,OAAoB;AAE/D,QAAO,MAAM,aAAa,QAAQ,uCAAuC;CAKzE,MAAM,EAAC,OAAO,aAAY;CAC1B,MAAM,UAAU,OAAO,SAAS;CAChC,MAAM,SAAS,GAAG,eAAe,MAAM,CAAC;CAExC,MAAM,WAAW,CACf,2BAA2B,MAAM,OAAO,MAAM,SAAS,EACvD,8BAA8B,MAAM,CACrC;AAGD,UAAS,KAAa;uBACD,QAAQ,GAAG,MAAM,YAAY,CAAC;;iBAEpC,QAAI,KAAK,CAAC;sBACL,OAAO;EAC3B;AAGA,MAAK,MAAM,OAAO,MAAM;EACtB,MAAM,QAAQ,IAAI,aAAa,CAAC,QAAQ,KAAK,IAAI;AACjD,WAAS,KAAa;6BACG,OAAO,QAAQ,MAAM;;;YAGtC,OAAO,gBAAgB,QAAI,IAAI,CAAC;;;;uBAIrB,QAAQ,GAAG,MAAM,GAAG,QAAQ,CAAC;;iBAEnC,QAAI,IAAI,CAAC;sBACJ,OAAO,QAAQ,MAAM;EACzC;;AAEA,QAAO,SAAS,KAAK,GAAG;;AAI1B,SAAgB,2BACd,OACA,SACA;CACA,MAAM,QAAkB,EAAE;AAE1B,OAAM,KAAa;mCACc,GAAG,GAAG,MAAM,aAAa,UAAU,CAAC;IACnE;AAGF,MAAK,MAAM,OAAO,MAAM;EACtB,MAAM,QAAQ,IAAI,aAAa,CAAC,QAAQ,KAAK,IAAI;AACjD,QAAM,KAAa;qCACc,GAAG,GAAG,MAAM,GAAG,MAAM,GAAG,UAAU,CAAC;MAClE;;AAEJ,QAAO,MAAM,KAAK,GAAG"}
+{"version":3,"file":"ddl.js","names":[],"sources":["../../../../../../../../zero-cache/src/services/change-source/pg/schema/ddl.ts"],"sourcesContent":["import {literal as lit} from 'pg-format';\nimport {assert} from '../../../../../../shared/src/asserts.ts';\nimport * as v from '../../../../../../shared/src/valita.ts';\nimport {upstreamSchema, type ShardConfig} from '../../../../types/shards.ts';\nimport {id} from '../../../../types/sql.ts';\nimport {\n  indexDefinitionsQuery,\n  publishedSchema,\n  publishedTableQuery,\n} from './published.ts';\n\n// Sent in the 'version' tag of \"ddlStart\" and \"ddlUpdate\" event messages.\n// This is used to ensure that the message constructed in the upstream\n// Trigger function is compatible with the code processing it in the zero-cache.\n//\n// Increment this when changing the format of the contents of the \"ddl\" events.\n// This will allow old / incompatible code to detect the change and abort.\nexport const PROTOCOL_VERSION = 1;\n\nconst triggerEvent = v.object({\n  context: v.object({query: v.string()}).rest(v.string()),\n});\n\n// All DDL events contain a snapshot of the current tables and indexes that\n// are published / relevant to the shard.\nexport const ddlEventSchema = triggerEvent.extend({\n  version: v.literal(PROTOCOL_VERSION),\n  schema: publishedSchema,\n});\n\n// The `ddlStart` message is computed before every DDL event, regardless of\n// whether the subsequent event affects the shard. Downstream processing should\n// capture the contained schema information in order to determine the schema\n// changes necessary to apply a subsequent `ddlUpdate` message. Note that a\n// `ddlUpdate` message may not follow, as updates determined to be irrelevant\n// to the shard will not result in a message. However, all `ddlUpdate` messages\n// are guaranteed to be preceded by a `ddlStart` message.\nexport const ddlStartEventSchema = ddlEventSchema.extend({\n  type: v.literal('ddlStart'),\n});\n\nexport type DdlStartEvent = v.Infer<typeof ddlStartEventSchema>;\n\n/**\n * The {@link DdlUpdateEvent} contains an updated schema resulting from\n * a particular ddl event. The event type provides information\n * (i.e. constraints) on the difference from the schema of the preceding\n * {@link DdlStartEvent}.\n *\n * Note that in almost all cases (the exception being `CREATE` events),\n * it is possible that there is no relevant difference between the\n * ddl-start schema and the ddl-update schema, as many aspects of the\n * schema (e.g. column constraints) are not relevant to downstream\n * replication.\n */\nexport const ddlUpdateEventSchema = ddlEventSchema.extend({\n  type: v.literal('ddlUpdate'),\n  event: v.object({tag: v.string()}),\n});\n\nexport type DdlUpdateEvent = v.Infer<typeof ddlUpdateEventSchema>;\n\n/**\n * The `schemaSnapshot` message is a snapshot of a schema taken in response to\n * a `COMMENT ON PUBLICATION` command, which is a hook recognized by zero\n * to manually emit schema snapshots to support detection of schema changes\n * from `ALTER PUBLICATION` commands on supabase, which does not fire event\n * triggers for them (https://github.com/supabase/supautils/issues/123).\n *\n * The hook is exercised by bookmarking the publication change with\n * `COMMENT ON PUBLICATION` statements within e.g.\n *\n * ```sql\n * BEGIN;\n * COMMENT ON PUBLICATION my_publication IS 'whatever';\n * ALTER PUBLICATION my_publication ...;\n * COMMENT ON PUBLICATION my_publication IS 'whatever';\n * COMMIT;\n * ```\n *\n * The `change-source` will perform the diff between a `schemaSnapshot`\n * events and its preceding `schemaSnapshot` (or `ddlUpdate`) within the\n * transaction.\n *\n * In the case where event trigger support is missing, this results in\n * diffing the `schemaSnapshot`s before and after the `ALTER PUBLICATION`\n * statement, thus effecting the same logic that would have been exercised\n * between the `ddlStart` and `ddlEvent` events fired by a database with\n * fully functional event triggers.\n *\n * Note that if the same transaction is run on a database that *does*\n * support event triggers on `ALTER PUBLICATION` statements, the sequence\n * of emitted messages will be:\n *\n * * `schemaSnapshot`\n * * `ddlStart`\n * * `ddlUpdate`\n * * `schemaSnapshot`\n *\n * Since `schemaSnapshot` messages are diffed with the preceding\n * `schemaSnapshot` or `ddlUpdate` event (if any), there will be no schema\n * difference between the `ddlUpdate` and the second `schemaSnapshot`, and\n * thus the extra `COMMENT` statements will effectively be no-ops.\n */\nexport const schemaSnapshotEventSchema = ddlEventSchema.extend({\n  type: v.literal('schemaSnapshot'),\n  event: v.object({tag: v.string()}),\n});\n\nexport type SchemaSnapshotEvent = v.Infer<typeof schemaSnapshotEventSchema>;\n\nexport const replicationEventSchema = v.union(\n  ddlStartEventSchema,\n  ddlUpdateEventSchema,\n  schemaSnapshotEventSchema,\n);\n\nexport type ReplicationEvent = v.Infer<typeof replicationEventSchema>;\n\n// Creates a function that appends `_{shard-num}` to the input and\n// quotes the result to be a valid identifier.\nfunction append(shardNum: number) {\n  return (name: string) => id(name + '_' + String(shardNum));\n}\n\n/**\n * Event trigger functions contain the core logic that are invoked by triggers.\n *\n * Note that although many of these functions can theoretically be parameterized and\n * shared across shards, it is advantageous to keep the functions in each shard\n * isolated from each other in order to avoid the complexity of shared-function\n * versioning.\n *\n * In a sense, shards (and their triggers and functions) should be thought of as\n * execution environments that can be updated at different schedules. If per-shard\n * triggers called into shared functions, we would have to consider versioning the\n * functions when changing their behavior, backwards compatibility, removal of\n * unused versions, etc. (not unlike versioning of npm packages).\n *\n * Instead, we opt for the simplicity and isolation of having each shard\n * completely own (and maintain) the entirety of its trigger/function stack.\n */\nfunction createEventFunctionStatements(shard: ShardConfig) {\n  const {appID, shardNum, publications} = shard;\n  const schema = id(upstreamSchema(shard)); // e.g. \"{APP_ID}_{SHARD_ID}\"\n  return /*sql*/ `\nCREATE SCHEMA IF NOT EXISTS ${schema};\n\nCREATE OR REPLACE FUNCTION ${schema}.get_trigger_context()\nRETURNS record AS $$\nDECLARE\n  result record;\nBEGIN\n  SELECT current_query() AS \"query\" into result;\n  RETURN result;\nEND\n$$ LANGUAGE plpgsql;\n\n\nCREATE OR REPLACE FUNCTION ${schema}.notice_ignore(tag TEXT, target record)\nRETURNS void AS $$\nBEGIN\n  RAISE NOTICE 'zero(%) ignoring % %', ${lit(shardNum)}, tag, row_to_json(target);\nEND\n$$ LANGUAGE plpgsql;\n\n\nCREATE OR REPLACE FUNCTION ${schema}.schema_specs()\nRETURNS TEXT AS $$\nDECLARE\n  tables record;\n  indexes record;\nBEGIN\n  ${publishedTableQuery(publications)} INTO tables;\n  ${indexDefinitionsQuery(publications)} INTO indexes;\n  RETURN json_build_object(\n    'tables', tables.tables,\n    'indexes', indexes.indexes\n  );\nEND\n$$ LANGUAGE plpgsql;\n\n\nCREATE OR REPLACE FUNCTION ${schema}.emit_ddl_start()\nRETURNS event_trigger AS $$\nDECLARE\n  schema_specs TEXT;\n  message TEXT;\nBEGIN\n  SELECT ${schema}.schema_specs() INTO schema_specs;\n\n  SELECT json_build_object(\n    'type', 'ddlStart',\n    'version', ${PROTOCOL_VERSION},\n    'schema', schema_specs::json,\n    'context', ${schema}.get_trigger_context()\n  ) INTO message;\n\n  PERFORM pg_logical_emit_message(true, ${lit(\n    `${appID}/${shardNum}`,\n  )}, message);\nEND\n$$ LANGUAGE plpgsql;\n\n\nCREATE OR REPLACE FUNCTION ${schema}.emit_ddl_end(tag TEXT)\nRETURNS void AS $$\nDECLARE\n  publications TEXT[];\n  target RECORD;\n  relevant RECORD;\n  schema_specs TEXT;\n  message TEXT;\n  event TEXT;\n  event_type TEXT;\n  event_prefix TEXT;\nBEGIN\n  publications := ARRAY[${lit(publications)}];\n\n  SELECT objid, object_type, object_identity \n    FROM pg_event_trigger_ddl_commands() \n    LIMIT 1 INTO target;\n\n  -- Filter DDL updates that are not relevant to the shard (i.e. publications) when possible.\n  SELECT true INTO relevant;\n\n  -- Note: ALTER TABLE statements may *remove* the table from the set of published\n  --       tables, and there is no way to determine if the table \"used to be\" in the\n  --       set. Thus, all ALTER TABLE statements must produce a ddl update, similar to\n  --       any DROP * statement.\n  IF (target.object_type = 'table' AND tag != 'ALTER TABLE') \n     OR target.object_type = 'table column' THEN\n    SELECT ns.nspname AS \"schema\", c.relname AS \"name\" FROM pg_class AS c\n      JOIN pg_namespace AS ns ON c.relnamespace = ns.oid\n      JOIN pg_publication_tables AS pb ON pb.schemaname = ns.nspname AND pb.tablename = c.relname\n      WHERE c.oid = target.objid AND pb.pubname = ANY (publications)\n      INTO relevant;\n\n  ELSIF target.object_type = 'index' THEN\n    SELECT ns.nspname AS \"schema\", c.relname AS \"name\" FROM pg_class AS c\n      JOIN pg_namespace AS ns ON c.relnamespace = ns.oid\n      JOIN pg_indexes as ind ON ind.schemaname = ns.nspname AND ind.indexname = c.relname\n      JOIN pg_publication_tables AS pb ON pb.schemaname = ns.nspname AND pb.tablename = ind.tablename\n      WHERE c.oid = target.objid AND pb.pubname = ANY (publications)\n      INTO relevant;\n\n  ELSIF target.object_type = 'publication relation' THEN\n    SELECT pb.pubname FROM pg_publication_rel AS rel\n      JOIN pg_publication AS pb ON pb.oid = rel.prpubid\n      WHERE rel.oid = target.objid AND pb.pubname = ANY (publications) \n      INTO relevant;\n\n  ELSIF target.object_type = 'publication namespace' THEN\n    SELECT pb.pubname FROM pg_publication_namespace AS ns\n      JOIN pg_publication AS pb ON pb.oid = ns.pnpubid\n      WHERE ns.oid = target.objid AND pb.pubname = ANY (publications) \n      INTO relevant;\n\n  ELSIF target.object_type = 'schema' THEN\n    SELECT ns.nspname AS \"schema\", c.relname AS \"name\" FROM pg_class AS c\n      JOIN pg_namespace AS ns ON c.relnamespace = ns.oid\n      JOIN pg_publication_tables AS pb ON pb.schemaname = ns.nspname AND pb.tablename = c.relname\n      WHERE ns.oid = target.objid AND pb.pubname = ANY (publications)\n      INTO relevant;\n\n  ELSIF target.object_type = 'publication' THEN\n    SELECT 1 WHERE target.object_identity = ANY (publications)\n      INTO relevant;\n\n  -- no-op CREATE IF NOT EXIST statements\n  ELSIF tag LIKE 'CREATE %' AND target.object_type IS NULL THEN\n    relevant := NULL;\n  END IF;\n\n  IF relevant IS NULL THEN\n    PERFORM ${schema}.notice_ignore(tag, target);\n    RETURN;\n  END IF;\n\n  IF tag = 'COMMENT' THEN\n    -- Only make schemaSnapshots for COMMENT ON PUBLICATION\n    IF target.object_type != 'publication' THEN\n      PERFORM ${schema}.notice_ignore(tag, target);\n      RETURN;\n    END IF;\n    event_type := 'schemaSnapshot';\n    event_prefix := '/ddl';\n  ELSE\n    event_type := 'ddlUpdate';\n    event_prefix := '';  -- TODO: Use '/ddl' for both when rollback safe\n  END IF;\n\n  RAISE INFO 'Creating % for % %', event_type, tag, row_to_json(target);\n\n  -- Construct and emit the DdlUpdateEvent message.\n  SELECT json_build_object('tag', tag) INTO event;\n  \n  SELECT ${schema}.schema_specs() INTO schema_specs;\n\n  SELECT json_build_object(\n    'type', event_type,\n    'version', ${PROTOCOL_VERSION},\n    'schema', schema_specs::json,\n    'event', event::json,\n    'context', ${schema}.get_trigger_context()\n  ) INTO message;\n\n  PERFORM pg_logical_emit_message(true, ${lit(\n    `${appID}/${shardNum}`,\n  )} || event_prefix, message);\nEND\n$$ LANGUAGE plpgsql;\n`;\n}\n\n// Exported for testing.\nexport const TAGS = [\n  'CREATE TABLE',\n  'ALTER TABLE',\n  'CREATE INDEX',\n  'DROP TABLE',\n  'DROP INDEX',\n  'ALTER PUBLICATION',\n  'ALTER SCHEMA',\n] as const;\n\nexport function createEventTriggerStatements(shard: ShardConfig) {\n  // Better to assert here than get a cryptic syntax error from Postgres.\n  assert(shard.publications.length, `shard publications must be non-empty`);\n\n  // Unlike functions, which are namespaced in shard-specific schemas,\n  // EVENT TRIGGER names are in the global namespace and thus must include\n  // the appID and shardNum.\n  const {appID, shardNum} = shard;\n  const sharded = append(shardNum);\n  const schema = id(upstreamSchema(shard));\n\n  const triggers = [\n    dropEventTriggerStatements(shard.appID, shard.shardNum),\n    createEventFunctionStatements(shard),\n  ];\n\n  // A single ddl_command_start trigger covering all relevant tags.\n  triggers.push(/*sql*/ `\nCREATE EVENT TRIGGER ${sharded(`${appID}_ddl_start`)}\n  ON ddl_command_start\n  WHEN TAG IN (${lit(TAGS)})\n  EXECUTE PROCEDURE ${schema}.emit_ddl_start();\n`);\n\n  // A per-tag ddl_command_end trigger that dispatches to ${schema}.emit_ddl_end(tag)\n  for (const tag of [...TAGS, 'COMMENT']) {\n    const tagID = tag.toLowerCase().replace(' ', '_');\n    triggers.push(/*sql*/ `\nCREATE OR REPLACE FUNCTION ${schema}.emit_${tagID}() \nRETURNS event_trigger AS $$\nBEGIN\n  PERFORM ${schema}.emit_ddl_end(${lit(tag)});\nEND\n$$ LANGUAGE plpgsql;\n\nCREATE EVENT TRIGGER ${sharded(`${appID}_${tagID}`)}\n  ON ddl_command_end\n  WHEN TAG IN (${lit(tag)})\n  EXECUTE PROCEDURE ${schema}.emit_${tagID}();\n`);\n  }\n  return triggers.join('');\n}\n\n// Exported for testing.\nexport function dropEventTriggerStatements(\n  appID: string,\n  shardID: string | number,\n) {\n  const stmts: string[] = [];\n  // A single ddl_command_start trigger covering all relevant tags.\n  stmts.push(/*sql*/ `\n    DROP EVENT TRIGGER IF EXISTS ${id(`${appID}_ddl_start_${shardID}`)};\n  `);\n\n  // A per-tag ddl_command_end trigger that dispatches to ${schema}.emit_ddl_end(tag)\n  for (const tag of [...TAGS, 'COMMENT']) {\n    const tagID = tag.toLowerCase().replace(' ', '_');\n    stmts.push(/*sql*/ `\n      DROP EVENT TRIGGER IF EXISTS ${id(`${appID}_${tagID}_${shardID}`)};\n    `);\n  }\n  return stmts.join('');\n}\n"],"mappings":";;;;;;AAyBA,IAAa,iBANQ,eAAE,OAAO,EAC5B,SAAS,eAAE,OAAO,EAAC,OAAO,eAAE,QAAQ,EAAC,CAAC,CAAC,KAAK,eAAE,QAAQ,CAAC,EACxD,CAAC,CAIyC,OAAO;CAChD,SAAS,eAAE,QAAA,EAAyB;CACpC,QAAQ;CACT,CAAC;AASF,IAAa,sBAAsB,eAAe,OAAO,EACvD,MAAM,eAAE,QAAQ,WAAW,EAC5B,CAAC;;;;;;;;;;;;;AAgBF,IAAa,uBAAuB,eAAe,OAAO;CACxD,MAAM,eAAE,QAAQ,YAAY;CAC5B,OAAO,eAAE,OAAO,EAAC,KAAK,eAAE,QAAQ,EAAC,CAAC;CACnC,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA8CF,IAAa,4BAA4B,eAAe,OAAO;CAC7D,MAAM,eAAE,QAAQ,iBAAiB;CACjC,OAAO,eAAE,OAAO,EAAC,KAAK,eAAE,QAAQ,EAAC,CAAC;CACnC,CAAC;AAIF,IAAa,yBAAyB,eAAE,MACtC,qBACA,sBACA,0BACD;AAMD,SAAS,OAAO,UAAkB;AAChC,SAAQ,SAAiB,GAAG,OAAO,MAAM,OAAO,SAAS,CAAC;;;;;;;;;;;;;;;;;;;AAoB5D,SAAS,8BAA8B,OAAoB;CACzD,MAAM,EAAC,OAAO,UAAU,iBAAgB;CACxC,MAAM,SAAS,GAAG,eAAe,MAAM,CAAC;AACxC,QAAe;8BACa,OAAO;;6BAER,OAAO;;;;;;;;;;;6BAWP,OAAO;;;yCAGK,QAAI,SAAS,CAAC;;;;;6BAK1B,OAAO;;;;;;IAMhC,oBAAoB,aAAa,CAAC;IAClC,sBAAsB,aAAa,CAAC;;;;;;;;;6BASX,OAAO;;;;;;WAMzB,OAAO;;;;;;iBAMD,OAAO;;;0CAGkB,QACtC,GAAG,MAAM,GAAG,WACb,CAAC;;;;;6BAKyB,OAAO;;;;;;;;;;;;0BAYV,QAAI,aAAa,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;cA0D9B,OAAO;;;;;;;gBAOL,OAAO;;;;;;;;;;;;;;;WAeZ,OAAO;;;;;;;iBAOD,OAAO;;;0CAGkB,QACtC,GAAG,MAAM,GAAG,WACb,CAAC;;;;;AAOJ,IAAa,OAAO;CAClB;CACA;CACA;CACA;CACA;CACA;CACA;CACD;AAED,SAAgB,6BAA6B,OAAoB;AAE/D,QAAO,MAAM,aAAa,QAAQ,uCAAuC;CAKzE,MAAM,EAAC,OAAO,aAAY;CAC1B,MAAM,UAAU,OAAO,SAAS;CAChC,MAAM,SAAS,GAAG,eAAe,MAAM,CAAC;CAExC,MAAM,WAAW,CACf,2BAA2B,MAAM,OAAO,MAAM,SAAS,EACvD,8BAA8B,MAAM,CACrC;AAGD,UAAS,KAAa;uBACD,QAAQ,GAAG,MAAM,YAAY,CAAC;;iBAEpC,QAAI,KAAK,CAAC;sBACL,OAAO;EAC3B;AAGA,MAAK,MAAM,OAAO,CAAC,GAAG,MAAM,UAAU,EAAE;EACtC,MAAM,QAAQ,IAAI,aAAa,CAAC,QAAQ,KAAK,IAAI;AACjD,WAAS,KAAa;6BACG,OAAO,QAAQ,MAAM;;;YAGtC,OAAO,gBAAgB,QAAI,IAAI,CAAC;;;;uBAIrB,QAAQ,GAAG,MAAM,GAAG,QAAQ,CAAC;;iBAEnC,QAAI,IAAI,CAAC;sBACJ,OAAO,QAAQ,MAAM;EACzC;;AAEA,QAAO,SAAS,KAAK,GAAG;;AAI1B,SAAgB,2BACd,OACA,SACA;CACA,MAAM,QAAkB,EAAE;AAE1B,OAAM,KAAa;mCACc,GAAG,GAAG,MAAM,aAAa,UAAU,CAAC;IACnE;AAGF,MAAK,MAAM,OAAO,CAAC,GAAG,MAAM,UAAU,EAAE;EACtC,MAAM,QAAQ,IAAI,aAAa,CAAC,QAAQ,KAAK,IAAI;AACjD,QAAM,KAAa;qCACc,GAAG,GAAG,MAAM,GAAG,MAAM,GAAG,UAAU,CAAC;MAClE;;AAEJ,QAAO,MAAM,KAAK,GAAG"}

package/out/zero-cache/src/services/change-source/pg/schema/init.d.ts

@@ -9,5 +9,6 @@ export declare function ensureShardSchema(lc: LogContext, db: PostgresDB, shard:
  * Updates the schema for an existing shard.
  */
 export declare function updateShardSchema(lc: LogContext, db: PostgresDB, shard: ShardConfig, replicaVersion: string): Promise<void>;
+export declare const CURRENT_SCHEMA_VERSION: number;
 export declare function decommissionLegacyShard(lc: LogContext, db: PostgresDB, shard: ShardConfig): Promise<void>;
 //# sourceMappingURL=init.d.ts.map

package/out/zero-cache/src/services/change-source/pg/schema/init.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"init.d.ts","sourceRoot":"","sources":["../../../../../../../../zero-cache/src/services/change-source/pg/schema/init.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AASjD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,yBAAyB,CAAC;AACxD,OAAO,EAAiB,KAAK,WAAW,EAAC,MAAM,6BAA6B,CAAC;AAa7E;;GAEG;AACH,wBAAsB,iBAAiB,CACrC,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,WAAW,GACjB,OAAO,CAAC,IAAI,CAAC,CAgBf;AAED;;GAEG;AACH,wBAAsB,iBAAiB,CACrC,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,WAAW,EAClB,cAAc,EAAE,MAAM,GACrB,OAAO,CAAC,IAAI,CAAC,CAqBf;AA+KD,wBAAsB,uBAAuB,CAC3C,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,WAAW,iBAYnB"}
+{"version":3,"file":"init.d.ts","sourceRoot":"","sources":["../../../../../../../../zero-cache/src/services/change-source/pg/schema/init.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AASjD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,yBAAyB,CAAC;AACxD,OAAO,EAAiB,KAAK,WAAW,EAAC,MAAM,6BAA6B,CAAC;AAa7E;;GAEG;AACH,wBAAsB,iBAAiB,CACrC,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,WAAW,GACjB,OAAO,CAAC,IAAI,CAAC,CAgBf;AAED;;GAEG;AACH,wBAAsB,iBAAiB,CACrC,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,WAAW,EAClB,cAAc,EAAE,MAAM,GACrB,OAAO,CAAC,IAAI,CAAC,CAqBf;AAsLD,eAAO,MAAM,sBAAsB,QAMwB,CAAC;AAE5D,wBAAsB,uBAAuB,CAC3C,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,WAAW,iBAYnB"}

package/out/zero-cache/src/services/change-source/pg/schema/init.js

@@ -35,7 +35,6 @@ function getIncrementalMigrations(shard, replicaVersion) {
 			},
 			minSafeVersion: 3
 		},
-		5: {},
 		6: { migrateSchema: async (lc, sql) => {
 			assert(replicaVersion, `replicaVersion is always passed for incremental migrations`);
 			await Promise.all([sql`
@@ -82,7 +81,6 @@ function getIncrementalMigrations(shard, replicaVersion) {
 			]);
 			lc.info?.(`Upgraded schema to support non-disruptive resyncs`);
 		} },
-		9: {},
 		10: { migrateSchema: async (lc, sql) => {
 			await sql.unsafe(`
           ${getMutationsTableDefinition(upstreamSchema(shard))}
@@ -90,8 +88,6 @@ function getIncrementalMigrations(shard, replicaVersion) {
         `);
 			lc.info?.("Upgraded schema with new mutations table");
 		} },
-		11: {},
-		12: {},
 		13: { migrateSchema: async (_, sql) => {
 			await sql`
           CREATE TABLE IF NOT EXISTS ${sql(upstreamSchema(shard))}."schemaVersions" (
@@ -106,7 +102,6 @@ function getIncrementalMigrations(shard, replicaVersion) {
             ON CONFLICT DO NOTHING;
         `;
 		} },
-		14: {},
 		15: { migrateSchema: async (_, sql) => {
 			await sql`
           ALTER TABLE ${sql(upstreamSchema(shard))}.replicas
@@ -114,7 +109,7 @@ function getIncrementalMigrations(shard, replicaVersion) {
             ADD COLUMN "subscriberContext" JSON
         `;
 		} },
-		16: { migrateSchema: async (lc, sql) => {
+		18: { migrateSchema: async (lc, sql) => {
 			const [{ publications }] = await sql`
           SELECT publications FROM ${sql(shardConfigTable)}`;
 			await setupTriggers(lc, sql, {
@@ -125,6 +120,11 @@ function getIncrementalMigrations(shard, replicaVersion) {
 		} }
 	};
 }
+Object.keys(getIncrementalMigrations({
+	appID: "unused",
+	shardNum: 0,
+	publications: ["foo"]
+})).reduce((prev, curr) => Math.max(prev, parseInt(curr)), 0);
 async function decommissionLegacyShard(lc, db, shard) {
 	if (shard.appID !== "zero") {
 		const versionHistory = await getVersionHistory(db, `zero_${shard.appID}`);

package/out/zero-cache/src/services/change-source/pg/schema/init.js.map

@@ -1 +1 @@
-{"version":3,"file":"init.js","names":[],"sources":["../../../../../../../../zero-cache/src/services/change-source/pg/schema/init.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../../../../shared/src/asserts.ts';\nimport * as v from '../../../../../../shared/src/valita.ts';\nimport {\n  getVersionHistory,\n  runSchemaMigrations,\n  type IncrementalMigrationMap,\n  type Migration,\n} from '../../../../db/migration.ts';\nimport type {PostgresDB} from '../../../../types/pg.ts';\nimport {upstreamSchema, type ShardConfig} from '../../../../types/shards.ts';\nimport {id} from '../../../../types/sql.ts';\nimport {AutoResetSignal} from '../../../change-streamer/schema/tables.ts';\nimport {decommissionShard} from '../decommission.ts';\nimport {publishedSchema} from './published.ts';\nimport {\n  getMutationsTableDefinition,\n  legacyReplicationSlot,\n  metadataPublicationName,\n  setupTablesAndReplication,\n  setupTriggers,\n} from './shard.ts';\n\n/**\n * Ensures that a shard is set up for initial sync.\n */\nexport async function ensureShardSchema(\n  lc: LogContext,\n  db: PostgresDB,\n  shard: ShardConfig,\n): Promise<void> {\n  const initialSetup: Migration = {\n    migrateSchema: (lc, tx) => setupTablesAndReplication(lc, tx, shard),\n    minSafeVersion: 1,\n  };\n  await runSchemaMigrations(\n    lc,\n    `upstream-shard-${shard.appID}`,\n    upstreamSchema(shard),\n    db,\n    initialSetup,\n    // The incremental migration of any existing replicas will be replaced by\n    // the incoming replica being synced, so the replicaVersion here is\n    // unnecessary.\n    getIncrementalMigrations(shard, 'obsolete'),\n  );\n}\n\n/**\n * Updates the schema for an existing shard.\n */\nexport async function updateShardSchema(\n  lc: LogContext,\n  db: PostgresDB,\n  shard: ShardConfig,\n  replicaVersion: string,\n): Promise<void> {\n  await runSchemaMigrations(\n    lc,\n    `upstream-shard-${shard.appID}`,\n    upstreamSchema(shard),\n    db,\n    {\n      // If the expected existing shard is absent, throw an\n      // AutoResetSignal to backtrack and initial sync.\n      migrateSchema: () => {\n        throw new AutoResetSignal(\n          `upstream shard ${upstreamSchema(shard)} is not initialized`,\n        );\n      },\n    },\n    getIncrementalMigrations(shard, replicaVersion),\n  );\n\n  // The decommission check is run in updateShardSchema so that it happens\n  // after initial sync, and not when the shard schema is initially set up.\n  await decommissionLegacyShard(lc, db, shard);\n}\n\nfunction getIncrementalMigrations(\n  shard: ShardConfig,\n  replicaVersion?: string,\n): IncrementalMigrationMap {\n  const shardConfigTable = `${upstreamSchema(shard)}.shardConfig`;\n\n  return {\n    4: {\n      migrateSchema: () => {\n        throw new AutoResetSignal('resetting to upgrade shard schema');\n      },\n      minSafeVersion: 3,\n    },\n\n    // v5 changes the upstream schema organization from \"zero_{SHARD_ID}\" to\n    // the \"{APP_ID}_0\". An incremental migration indicates that the previous\n    // SHARD_ID was \"0\" and the new APP_ID is \"zero\" (i.e. the default values\n    // for those options). In this case, the upstream format is identical, and\n    // no migration is necessary. However, the version is bumped to v5 to\n    // indicate that it was created with the {APP_ID} configuration and should\n    // not be decommissioned as a legacy shard.\n    5: {},\n\n    6: {\n      migrateSchema: async (lc, sql) => {\n        assert(\n          replicaVersion,\n          `replicaVersion is always passed for incremental migrations`,\n        );\n        await Promise.all([\n          sql`\n          ALTER TABLE ${sql(shardConfigTable)} ADD \"replicaVersion\" TEXT`,\n          sql`\n          UPDATE ${sql(shardConfigTable)} SET ${sql({replicaVersion})}`,\n        ]);\n        lc.info?.(\n          `Recorded replicaVersion ${replicaVersion} in upstream shardConfig`,\n        );\n      },\n    },\n\n    // Updates the DDL event trigger protocol to v2, and adds support for\n    // ALTER SCHEMA x RENAME TO y\n    7: {\n      migrateSchema: async (lc, sql) => {\n        const [{publications}] = await sql<{publications: string[]}[]>`\n          SELECT publications FROM ${sql(shardConfigTable)}`;\n        await setupTriggers(lc, sql, {...shard, publications});\n        lc.info?.(`Upgraded to v2 event triggers`);\n      },\n    },\n\n    // Adds support for non-disruptive resyncs, which tracks multiple\n    // replicas with different slot names.\n    8: {\n      migrateSchema: async (lc, sql) => {\n        const legacyShardConfigSchema = v.object({\n          replicaVersion: v.string().nullable(),\n          initialSchema: publishedSchema.nullable(),\n        });\n        const result = await sql`\n          SELECT \"replicaVersion\", \"initialSchema\" FROM ${sql(shardConfigTable)}`;\n        assert(\n          result.length === 1,\n          () => `Expected exactly one shardConfig row, got ${result.length}`,\n        );\n        const {replicaVersion, initialSchema} = v.parse(\n          result[0],\n          legacyShardConfigSchema,\n          'passthrough',\n        );\n\n        await Promise.all([\n          sql`\n          CREATE TABLE ${sql(upstreamSchema(shard))}.replicas (\n            \"slot\"          TEXT PRIMARY KEY,\n            \"version\"       TEXT NOT NULL,\n            \"initialSchema\" JSON NOT NULL\n          );\n          `,\n          sql`\n          INSERT INTO ${sql(upstreamSchema(shard))}.replicas ${sql({\n            slot: legacyReplicationSlot(shard),\n            version: replicaVersion,\n            initialSchema,\n          })}\n          `,\n          sql`\n          ALTER TABLE ${sql(shardConfigTable)} DROP \"replicaVersion\", DROP \"initialSchema\"\n          `,\n        ]);\n        lc.info?.(`Upgraded schema to support non-disruptive resyncs`);\n      },\n    },\n\n    // Fixes field ordering of compound indexes. This incremental migration\n    // only fixes indexes resulting from new schema changes. A full resync is\n    // required to fix existing indexes.\n    //\n    // The migration has been subsumed by the identical logic for migrating\n    // to v12 (i.e. a trigger upgrade).\n    9: {},\n\n    // Adds the `mutations` table used to track mutation results.\n    10: {\n      migrateSchema: async (lc, sql) => {\n        await sql.unsafe(/*sql*/ `\n          ${getMutationsTableDefinition(upstreamSchema(shard))}\n          ALTER PUBLICATION ${id(metadataPublicationName(shard.appID, shard.shardNum))} ADD TABLE ${id(upstreamSchema(shard))}.\"mutations\";\n        `);\n        lc.info?.('Upgraded schema with new mutations table');\n      },\n    },\n\n    // Formerly dropped the schemaVersions table, but restored in the v13\n    // migration for rollback safety.\n    11: {},\n\n    // Upgrade DDL trigger to query schemaOID, needed information for auto-backfill.\n    // (subsumed by v14)\n    12: {},\n\n    // Recreates the legacy schemaVersions table that was prematurely dropped\n    // in the (former) v11 migration. It needs to remain present for at least one\n    // release in order to be rollback safe.\n    //\n    // TODO: Drop the table once a release that no longer reads the table has\n    // been rolled out.\n    13: {\n      migrateSchema: async (_, sql) => {\n        await sql`\n          CREATE TABLE IF NOT EXISTS ${sql(upstreamSchema(shard))}.\"schemaVersions\" (\n            \"minSupportedVersion\" INT4,\n            \"maxSupportedVersion\" INT4,\n            \"lock\" BOOL PRIMARY KEY DEFAULT true CHECK (lock)\n        );`;\n        await sql`\n          INSERT INTO ${sql(upstreamSchema(shard))}.\"schemaVersions\" \n            (\"lock\", \"minSupportedVersion\", \"maxSupportedVersion\")\n            VALUES (true, 1, 1)\n            ON CONFLICT DO NOTHING;\n        `;\n      },\n    },\n\n    // Upgrade DDL trigger to log more info to PG logs.\n    // (subsumed by v16)\n    14: {},\n\n    // Add initialSyncContext column to replicas table.\n    15: {\n      migrateSchema: async (_, sql) => {\n        await sql`\n          ALTER TABLE ${sql(upstreamSchema(shard))}.replicas\n            ADD COLUMN \"initialSyncContext\" JSON,\n            ADD COLUMN \"subscriberContext\" JSON\n        `;\n      },\n    },\n\n    // Upgrade DDL trigger to fire on all ALTER TABLE statements\n    // to catch the *removal* of a table from the published set.\n    16: {\n      migrateSchema: async (lc, sql) => {\n        const [{publications}] = await sql<{publications: string[]}[]>`\n          SELECT publications FROM ${sql(shardConfigTable)}`;\n        await setupTriggers(lc, sql, {...shard, publications});\n        lc.info?.(`Upgraded DDL event triggers`);\n      },\n    },\n  };\n}\n\nexport async function decommissionLegacyShard(\n  lc: LogContext,\n  db: PostgresDB,\n  shard: ShardConfig,\n) {\n  if (shard.appID !== 'zero') {\n    // When migration from non-default shard ids, e.g. \"zero_prod\" => \"prod_0\",\n    // clean up the old \"zero_prod\" shard if it is pre-v5. Note that the v5\n    // check is important to guard against cleaning up a **new** \"zero_0\" app\n    // that coexists with the current App (with app-id === \"0\").\n    const versionHistory = await getVersionHistory(db, `zero_${shard.appID}`);\n    if (versionHistory !== null && versionHistory.schemaVersion < 5) {\n      await decommissionShard(lc, db, 'zero', shard.appID);\n    }\n  }\n}\n"],"mappings":";;;;;;;;;;;;;AA0BA,eAAsB,kBACpB,IACA,IACA,OACe;AAKf,OAAM,oBACJ,IACA,kBAAkB,MAAM,SACxB,eAAe,MAAM,EACrB,IAR8B;EAC9B,gBAAgB,IAAI,OAAO,0BAA0B,IAAI,IAAI,MAAM;EACnE,gBAAgB;EACjB,EAUC,yBAAyB,OAAO,WAAW,CAC5C;;;;;AAMH,eAAsB,kBACpB,IACA,IACA,OACA,gBACe;AACf,OAAM,oBACJ,IACA,kBAAkB,MAAM,SACxB,eAAe,MAAM,EACrB,IACA,EAGE,qBAAqB;AACnB,QAAM,IAAI,gBACR,kBAAkB,eAAe,MAAM,CAAC,qBACzC;IAEJ,EACD,yBAAyB,OAAO,eAAe,CAChD;AAID,OAAM,wBAAwB,IAAI,IAAI,MAAM;;AAG9C,SAAS,yBACP,OACA,gBACyB;CACzB,MAAM,mBAAmB,GAAG,eAAe,MAAM,CAAC;AAElD,QAAO;EACL,GAAG;GACD,qBAAqB;AACnB,UAAM,IAAI,gBAAgB,oCAAoC;;GAEhE,gBAAgB;GACjB;EASD,GAAG,EAAE;EAEL,GAAG,EACD,eAAe,OAAO,IAAI,QAAQ;AAChC,UACE,gBACA,6DACD;AACD,SAAM,QAAQ,IAAI,CAChB,GAAG;wBACW,IAAI,iBAAiB,CAAC,6BACpC,GAAG;mBACM,IAAI,iBAAiB,CAAC,OAAO,IAAI,EAAC,gBAAe,CAAC,GAC5D,CAAC;AACF,MAAG,OACD,2BAA2B,eAAe,0BAC3C;KAEJ;EAID,GAAG,EACD,eAAe,OAAO,IAAI,QAAQ;GAChC,MAAM,CAAC,EAAC,kBAAiB,MAAM,GAA+B;qCACjC,IAAI,iBAAiB;AAClD,SAAM,cAAc,IAAI,KAAK;IAAC,GAAG;IAAO;IAAa,CAAC;AACtD,MAAG,OAAO,gCAAgC;KAE7C;EAID,GAAG,EACD,eAAe,OAAO,IAAI,QAAQ;GAChC,MAAM,0BAA0B,eAAE,OAAO;IACvC,gBAAgB,eAAE,QAAQ,CAAC,UAAU;IACrC,eAAe,gBAAgB,UAAU;IAC1C,CAAC;GACF,MAAM,SAAS,MAAM,GAAG;0DAC0B,IAAI,iBAAiB;AACvE,UACE,OAAO,WAAW,SACZ,6CAA6C,OAAO,SAC3D;GACD,MAAM,EAAC,gBAAgB,kBAAiB,MACtC,OAAO,IACP,yBACA,cACD;AAED,SAAM,QAAQ,IAAI;IAChB,GAAG;yBACY,IAAI,eAAe,MAAM,CAAC,CAAC;;;;;;IAM1C,GAAG;wBACW,IAAI,eAAe,MAAM,CAAC,CAAC,YAAY,IAAI;KACvD,MAAM,sBAAsB,MAAM;KAClC,SAAS;KACT;KACD,CAAC,CAAC;;IAEH,GAAG;wBACW,IAAI,iBAAiB,CAAC;;IAErC,CAAC;AACF,MAAG,OAAO,oDAAoD;KAEjE;EAQD,GAAG,EAAE;EAGL,IAAI,EACF,eAAe,OAAO,IAAI,QAAQ;AAChC,SAAM,IAAI,OAAe;YACrB,4BAA4B,eAAe,MAAM,CAAC,CAAC;8BACjC,GAAG,wBAAwB,MAAM,OAAO,MAAM,SAAS,CAAC,CAAC,aAAa,GAAG,eAAe,MAAM,CAAC,CAAC;UACpH;AACF,MAAG,OAAO,2CAA2C;KAExD;EAID,IAAI,EAAE;EAIN,IAAI,EAAE;EAQN,IAAI,EACF,eAAe,OAAO,GAAG,QAAQ;AAC/B,SAAM,GAAG;uCACsB,IAAI,eAAe,MAAM,CAAC,CAAC;;;;;AAK1D,SAAM,GAAG;wBACO,IAAI,eAAe,MAAM,CAAC,CAAC;;;;;KAM9C;EAID,IAAI,EAAE;EAGN,IAAI,EACF,eAAe,OAAO,GAAG,QAAQ;AAC/B,SAAM,GAAG;wBACO,IAAI,eAAe,MAAM,CAAC,CAAC;;;;KAK9C;EAID,IAAI,EACF,eAAe,OAAO,IAAI,QAAQ;GAChC,MAAM,CAAC,EAAC,kBAAiB,MAAM,GAA+B;qCACjC,IAAI,iBAAiB;AAClD,SAAM,cAAc,IAAI,KAAK;IAAC,GAAG;IAAO;IAAa,CAAC;AACtD,MAAG,OAAO,8BAA8B;KAE3C;EACF;;AAGH,eAAsB,wBACpB,IACA,IACA,OACA;AACA,KAAI,MAAM,UAAU,QAAQ;EAK1B,MAAM,iBAAiB,MAAM,kBAAkB,IAAI,QAAQ,MAAM,QAAQ;AACzE,MAAI,mBAAmB,QAAQ,eAAe,gBAAgB,EAC5D,OAAM,kBAAkB,IAAI,IAAI,QAAQ,MAAM,MAAM"}
+{"version":3,"file":"init.js","names":[],"sources":["../../../../../../../../zero-cache/src/services/change-source/pg/schema/init.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../../../../shared/src/asserts.ts';\nimport * as v from '../../../../../../shared/src/valita.ts';\nimport {\n  getVersionHistory,\n  runSchemaMigrations,\n  type IncrementalMigrationMap,\n  type Migration,\n} from '../../../../db/migration.ts';\nimport type {PostgresDB} from '../../../../types/pg.ts';\nimport {upstreamSchema, type ShardConfig} from '../../../../types/shards.ts';\nimport {id} from '../../../../types/sql.ts';\nimport {AutoResetSignal} from '../../../change-streamer/schema/tables.ts';\nimport {decommissionShard} from '../decommission.ts';\nimport {publishedSchema} from './published.ts';\nimport {\n  getMutationsTableDefinition,\n  legacyReplicationSlot,\n  metadataPublicationName,\n  setupTablesAndReplication,\n  setupTriggers,\n} from './shard.ts';\n\n/**\n * Ensures that a shard is set up for initial sync.\n */\nexport async function ensureShardSchema(\n  lc: LogContext,\n  db: PostgresDB,\n  shard: ShardConfig,\n): Promise<void> {\n  const initialSetup: Migration = {\n    migrateSchema: (lc, tx) => setupTablesAndReplication(lc, tx, shard),\n    minSafeVersion: 1,\n  };\n  await runSchemaMigrations(\n    lc,\n    `upstream-shard-${shard.appID}`,\n    upstreamSchema(shard),\n    db,\n    initialSetup,\n    // The incremental migration of any existing replicas will be replaced by\n    // the incoming replica being synced, so the replicaVersion here is\n    // unnecessary.\n    getIncrementalMigrations(shard, 'obsolete'),\n  );\n}\n\n/**\n * Updates the schema for an existing shard.\n */\nexport async function updateShardSchema(\n  lc: LogContext,\n  db: PostgresDB,\n  shard: ShardConfig,\n  replicaVersion: string,\n): Promise<void> {\n  await runSchemaMigrations(\n    lc,\n    `upstream-shard-${shard.appID}`,\n    upstreamSchema(shard),\n    db,\n    {\n      // If the expected existing shard is absent, throw an\n      // AutoResetSignal to backtrack and initial sync.\n      migrateSchema: () => {\n        throw new AutoResetSignal(\n          `upstream shard ${upstreamSchema(shard)} is not initialized`,\n        );\n      },\n    },\n    getIncrementalMigrations(shard, replicaVersion),\n  );\n\n  // The decommission check is run in updateShardSchema so that it happens\n  // after initial sync, and not when the shard schema is initially set up.\n  await decommissionLegacyShard(lc, db, shard);\n}\n\nfunction getIncrementalMigrations(\n  shard: ShardConfig,\n  replicaVersion?: string,\n): IncrementalMigrationMap {\n  const shardConfigTable = `${upstreamSchema(shard)}.shardConfig`;\n\n  return {\n    4: {\n      migrateSchema: () => {\n        throw new AutoResetSignal('resetting to upgrade shard schema');\n      },\n      minSafeVersion: 3,\n    },\n\n    // v5: changes the upstream schema organization from \"zero_{SHARD_ID}\" to\n    // the \"{APP_ID}_0\". An incremental migration indicates that the previous\n    // SHARD_ID was \"0\" and the new APP_ID is \"zero\" (i.e. the default values\n    // for those options). In this case, the upstream format is identical, and\n    // no migration is necessary. However, the version is bumped to v5 to\n    // indicate that it was created with the {APP_ID} configuration and should\n    // not be decommissioned as a legacy shard.\n\n    6: {\n      migrateSchema: async (lc, sql) => {\n        assert(\n          replicaVersion,\n          `replicaVersion is always passed for incremental migrations`,\n        );\n        await Promise.all([\n          sql`\n          ALTER TABLE ${sql(shardConfigTable)} ADD \"replicaVersion\" TEXT`,\n          sql`\n          UPDATE ${sql(shardConfigTable)} SET ${sql({replicaVersion})}`,\n        ]);\n        lc.info?.(\n          `Recorded replicaVersion ${replicaVersion} in upstream shardConfig`,\n        );\n      },\n    },\n\n    // Updates the DDL event trigger protocol to v2, and adds support for\n    // ALTER SCHEMA x RENAME TO y\n    7: {\n      migrateSchema: async (lc, sql) => {\n        const [{publications}] = await sql<{publications: string[]}[]>`\n          SELECT publications FROM ${sql(shardConfigTable)}`;\n        await setupTriggers(lc, sql, {...shard, publications});\n        lc.info?.(`Upgraded to v2 event triggers`);\n      },\n    },\n\n    // Adds support for non-disruptive resyncs, which tracks multiple\n    // replicas with different slot names.\n    8: {\n      migrateSchema: async (lc, sql) => {\n        const legacyShardConfigSchema = v.object({\n          replicaVersion: v.string().nullable(),\n          initialSchema: publishedSchema.nullable(),\n        });\n        const result = await sql`\n          SELECT \"replicaVersion\", \"initialSchema\" FROM ${sql(shardConfigTable)}`;\n        assert(\n          result.length === 1,\n          () => `Expected exactly one shardConfig row, got ${result.length}`,\n        );\n        const {replicaVersion, initialSchema} = v.parse(\n          result[0],\n          legacyShardConfigSchema,\n          'passthrough',\n        );\n\n        await Promise.all([\n          sql`\n          CREATE TABLE ${sql(upstreamSchema(shard))}.replicas (\n            \"slot\"          TEXT PRIMARY KEY,\n            \"version\"       TEXT NOT NULL,\n            \"initialSchema\" JSON NOT NULL\n          );\n          `,\n          sql`\n          INSERT INTO ${sql(upstreamSchema(shard))}.replicas ${sql({\n            slot: legacyReplicationSlot(shard),\n            version: replicaVersion,\n            initialSchema,\n          })}\n          `,\n          sql`\n          ALTER TABLE ${sql(shardConfigTable)} DROP \"replicaVersion\", DROP \"initialSchema\"\n          `,\n        ]);\n        lc.info?.(`Upgraded schema to support non-disruptive resyncs`);\n      },\n    },\n\n    // v9: Fixes field ordering of compound indexes. This incremental migration\n    // only fixes indexes resulting from new schema changes. A full resync is\n    // required to fix existing indexes.\n    //\n    // The migration has been subsumed by the identical logic for migrating\n    // to v12 (i.e. a trigger upgrade).\n\n    // Adds the `mutations` table used to track mutation results.\n    10: {\n      migrateSchema: async (lc, sql) => {\n        await sql.unsafe(/*sql*/ `\n          ${getMutationsTableDefinition(upstreamSchema(shard))}\n          ALTER PUBLICATION ${id(metadataPublicationName(shard.appID, shard.shardNum))} ADD TABLE ${id(upstreamSchema(shard))}.\"mutations\";\n        `);\n        lc.info?.('Upgraded schema with new mutations table');\n      },\n    },\n\n    // v11: Formerly dropped the schemaVersions table, but restored in the v13\n    // migration for rollback safety.\n\n    // v12: Upgrade DDL trigger to query schemaOID, needed information for auto-backfill.\n    // (subsumed by v14)\n\n    // Recreates the legacy schemaVersions table that was prematurely dropped\n    // in the (former) v11 migration. It needs to remain present for at least one\n    // release in order to be rollback safe.\n    //\n    // TODO: Drop the table once a release that no longer reads the table has\n    // been rolled out.\n    13: {\n      migrateSchema: async (_, sql) => {\n        await sql`\n          CREATE TABLE IF NOT EXISTS ${sql(upstreamSchema(shard))}.\"schemaVersions\" (\n            \"minSupportedVersion\" INT4,\n            \"maxSupportedVersion\" INT4,\n            \"lock\" BOOL PRIMARY KEY DEFAULT true CHECK (lock)\n        );`;\n        await sql`\n          INSERT INTO ${sql(upstreamSchema(shard))}.\"schemaVersions\" \n            (\"lock\", \"minSupportedVersion\", \"maxSupportedVersion\")\n            VALUES (true, 1, 1)\n            ON CONFLICT DO NOTHING;\n        `;\n      },\n    },\n\n    // v14: Upgrade DDL trigger to log more info to PG logs.\n    // (subsumed by v16)\n\n    // Add initialSyncContext column to replicas table.\n    15: {\n      migrateSchema: async (_, sql) => {\n        await sql`\n          ALTER TABLE ${sql(upstreamSchema(shard))}.replicas\n            ADD COLUMN \"initialSyncContext\" JSON,\n            ADD COLUMN \"subscriberContext\" JSON\n        `;\n      },\n    },\n\n    // v16: Upgrade DDL trigger to fire on all ALTER TABLE statements\n    // to catch the *removal* of a table from the published set.\n    // (subsumed by v17)\n\n    // v17: Upgrade DDL triggers to support the COMMENT ON PUBLICATION hook for\n    // working around the lack of event trigger support for PUBLICATION\n    // changes in supabase.\n    //\n    // This also adds forwards-compatible support for hierarchical logical\n    // message prefixes and unknown ddl event types.\n    // (subsued by v18)\n\n    // v18: Pure refactoring of event trigger code.\n    18: {\n      migrateSchema: async (lc, sql) => {\n        const [{publications}] = await sql<{publications: string[]}[]>`\n          SELECT publications FROM ${sql(shardConfigTable)}`;\n        await setupTriggers(lc, sql, {...shard, publications});\n        lc.info?.(`Upgraded DDL event triggers`);\n      },\n    },\n  };\n}\n\n// Referenced in tests.\nexport const CURRENT_SCHEMA_VERSION = Object.keys(\n  getIncrementalMigrations({\n    appID: 'unused',\n    shardNum: 0,\n    publications: ['foo'],\n  }),\n).reduce((prev, curr) => Math.max(prev, parseInt(curr)), 0);\n\nexport async function decommissionLegacyShard(\n  lc: LogContext,\n  db: PostgresDB,\n  shard: ShardConfig,\n) {\n  if (shard.appID !== 'zero') {\n    // When migration from non-default shard ids, e.g. \"zero_prod\" => \"prod_0\",\n    // clean up the old \"zero_prod\" shard if it is pre-v5. Note that the v5\n    // check is important to guard against cleaning up a **new** \"zero_0\" app\n    // that coexists with the current App (with app-id === \"0\").\n    const versionHistory = await getVersionHistory(db, `zero_${shard.appID}`);\n    if (versionHistory !== null && versionHistory.schemaVersion < 5) {\n      await decommissionShard(lc, db, 'zero', shard.appID);\n    }\n  }\n}\n"],"mappings":";;;;;;;;;;;;;AA0BA,eAAsB,kBACpB,IACA,IACA,OACe;AAKf,OAAM,oBACJ,IACA,kBAAkB,MAAM,SACxB,eAAe,MAAM,EACrB,IAR8B;EAC9B,gBAAgB,IAAI,OAAO,0BAA0B,IAAI,IAAI,MAAM;EACnE,gBAAgB;EACjB,EAUC,yBAAyB,OAAO,WAAW,CAC5C;;;;;AAMH,eAAsB,kBACpB,IACA,IACA,OACA,gBACe;AACf,OAAM,oBACJ,IACA,kBAAkB,MAAM,SACxB,eAAe,MAAM,EACrB,IACA,EAGE,qBAAqB;AACnB,QAAM,IAAI,gBACR,kBAAkB,eAAe,MAAM,CAAC,qBACzC;IAEJ,EACD,yBAAyB,OAAO,eAAe,CAChD;AAID,OAAM,wBAAwB,IAAI,IAAI,MAAM;;AAG9C,SAAS,yBACP,OACA,gBACyB;CACzB,MAAM,mBAAmB,GAAG,eAAe,MAAM,CAAC;AAElD,QAAO;EACL,GAAG;GACD,qBAAqB;AACnB,UAAM,IAAI,gBAAgB,oCAAoC;;GAEhE,gBAAgB;GACjB;EAUD,GAAG,EACD,eAAe,OAAO,IAAI,QAAQ;AAChC,UACE,gBACA,6DACD;AACD,SAAM,QAAQ,IAAI,CAChB,GAAG;wBACW,IAAI,iBAAiB,CAAC,6BACpC,GAAG;mBACM,IAAI,iBAAiB,CAAC,OAAO,IAAI,EAAC,gBAAe,CAAC,GAC5D,CAAC;AACF,MAAG,OACD,2BAA2B,eAAe,0BAC3C;KAEJ;EAID,GAAG,EACD,eAAe,OAAO,IAAI,QAAQ;GAChC,MAAM,CAAC,EAAC,kBAAiB,MAAM,GAA+B;qCACjC,IAAI,iBAAiB;AAClD,SAAM,cAAc,IAAI,KAAK;IAAC,GAAG;IAAO;IAAa,CAAC;AACtD,MAAG,OAAO,gCAAgC;KAE7C;EAID,GAAG,EACD,eAAe,OAAO,IAAI,QAAQ;GAChC,MAAM,0BAA0B,eAAE,OAAO;IACvC,gBAAgB,eAAE,QAAQ,CAAC,UAAU;IACrC,eAAe,gBAAgB,UAAU;IAC1C,CAAC;GACF,MAAM,SAAS,MAAM,GAAG;0DAC0B,IAAI,iBAAiB;AACvE,UACE,OAAO,WAAW,SACZ,6CAA6C,OAAO,SAC3D;GACD,MAAM,EAAC,gBAAgB,kBAAiB,MACtC,OAAO,IACP,yBACA,cACD;AAED,SAAM,QAAQ,IAAI;IAChB,GAAG;yBACY,IAAI,eAAe,MAAM,CAAC,CAAC;;;;;;IAM1C,GAAG;wBACW,IAAI,eAAe,MAAM,CAAC,CAAC,YAAY,IAAI;KACvD,MAAM,sBAAsB,MAAM;KAClC,SAAS;KACT;KACD,CAAC,CAAC;;IAEH,GAAG;wBACW,IAAI,iBAAiB,CAAC;;IAErC,CAAC;AACF,MAAG,OAAO,oDAAoD;KAEjE;EAUD,IAAI,EACF,eAAe,OAAO,IAAI,QAAQ;AAChC,SAAM,IAAI,OAAe;YACrB,4BAA4B,eAAe,MAAM,CAAC,CAAC;8BACjC,GAAG,wBAAwB,MAAM,OAAO,MAAM,SAAS,CAAC,CAAC,aAAa,GAAG,eAAe,MAAM,CAAC,CAAC;UACpH;AACF,MAAG,OAAO,2CAA2C;KAExD;EAcD,IAAI,EACF,eAAe,OAAO,GAAG,QAAQ;AAC/B,SAAM,GAAG;uCACsB,IAAI,eAAe,MAAM,CAAC,CAAC;;;;;AAK1D,SAAM,GAAG;wBACO,IAAI,eAAe,MAAM,CAAC,CAAC;;;;;KAM9C;EAMD,IAAI,EACF,eAAe,OAAO,GAAG,QAAQ;AAC/B,SAAM,GAAG;wBACO,IAAI,eAAe,MAAM,CAAC,CAAC;;;;KAK9C;EAeD,IAAI,EACF,eAAe,OAAO,IAAI,QAAQ;GAChC,MAAM,CAAC,EAAC,kBAAiB,MAAM,GAA+B;qCACjC,IAAI,iBAAiB;AAClD,SAAM,cAAc,IAAI,KAAK;IAAC,GAAG;IAAO;IAAa,CAAC;AACtD,MAAG,OAAO,8BAA8B;KAE3C;EACF;;AAImC,OAAO,KAC3C,yBAAyB;CACvB,OAAO;CACP,UAAU;CACV,cAAc,CAAC,MAAM;CACtB,CAAC,CACH,CAAC,QAAQ,MAAM,SAAS,KAAK,IAAI,MAAM,SAAS,KAAK,CAAC,EAAE,EAAE;AAE3D,eAAsB,wBACpB,IACA,IACA,OACA;AACA,KAAI,MAAM,UAAU,QAAQ;EAK1B,MAAM,iBAAiB,MAAM,kBAAkB,IAAI,QAAQ,MAAM,QAAQ;AACzE,MAAI,mBAAmB,QAAQ,eAAe,gBAAgB,EAC5D,OAAM,kBAAkB,IAAI,IAAI,QAAQ,MAAM,MAAM"}

package/out/zero-cache/src/services/change-streamer/change-streamer-http.js

@@ -10,7 +10,7 @@ import { discoverChangeStreamerAddress } from "./schema/tables.js";
 import { URLParams } from "../../types/url-params.js";
 import { downstreamSchema } from "./change-streamer.js";
 import { snapshotMessageSchema } from "./snapshot.js";
-import WebSocket$1 from "ws";
+import WebSocket from "ws";
 import websocket from "@fastify/websocket";
 //#region ../zero-cache/src/services/change-streamer/change-streamer-http.ts
 var MIN_SUPPORTED_PROTOCOL_VERSION = 1;
@@ -122,11 +122,11 @@ var ChangeStreamerHttpClient = class {
 		return uri;
 	}
 	async reserveSnapshot(taskID) {
-		const ws = new WebSocket$1(await this.#resolveChangeStreamer(SNAPSHOT_PATH) + `?${new URLSearchParams({ taskID }).toString()}`);
+		const ws = new WebSocket(await this.#resolveChangeStreamer(SNAPSHOT_PATH) + `?${new URLSearchParams({ taskID }).toString()}`);
 		return streamIn(this.#lc, ws, snapshotMessageSchema);
 	}
 	async subscribe(ctx) {
-		const ws = new WebSocket$1(await this.#resolveChangeStreamer(CHANGES_PATH) + `?${getParams(ctx).toString()}`);
+		const ws = new WebSocket(await this.#resolveChangeStreamer(CHANGES_PATH) + `?${getParams(ctx).toString()}`);
 		return streamIn(this.#lc, ws, downstreamSchema);
 	}
 };

package/out/zero-cache/src/services/change-streamer/change-streamer-http.js.map

@@ -1 +1 @@
-{"version":3,"file":"change-streamer-http.js","names":["#lc","#opts","#changeStreamer","#backupMonitor","#subscribe","#reserveSnapshot","#receiveWebsocket","#getBackupMonitor","#ensureChangeStreamerStarted","#changeStreamerStarted","#shardID","#changeDB","#changeStreamerURI","#resolveChangeStreamer"],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-http.ts"],"sourcesContent":["import websocket from '@fastify/websocket';\nimport type {LogContext} from '@rocicorp/logger';\nimport type {IncomingMessage} from 'node:http';\nimport WebSocket from 'ws';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport type {ZeroConfig} from '../../config/zero-config.ts';\nimport type {IncomingMessageSubset} from '../../types/http.ts';\nimport {pgClient, type PostgresDB} from '../../types/pg.ts';\nimport {type Worker} from '../../types/processes.ts';\nimport {type ShardID} from '../../types/shards.ts';\nimport {streamIn, streamOut, type Source} from '../../types/streams.ts';\nimport {URLParams} from '../../types/url-params.ts';\nimport {installWebSocketReceiver} from '../../types/websocket-handoff.ts';\nimport {closeWithError, PROTOCOL_ERROR} from '../../types/ws.ts';\nimport {HttpService} from '../http-service.ts';\nimport type {Service} from '../service.ts';\nimport type {BackupMonitor} from './backup-monitor.ts';\nimport {\n  downstreamSchema,\n  PROTOCOL_VERSION,\n  type ChangeStreamer,\n  type Downstream,\n  type SubscriberContext,\n} from './change-streamer.ts';\nimport {discoverChangeStreamerAddress} from './schema/tables.ts';\nimport {snapshotMessageSchema, type SnapshotMessage} from './snapshot.ts';\n\nconst MIN_SUPPORTED_PROTOCOL_VERSION = 1;\n\nconst SNAPSHOT_PATH_PATTERN = '/replication/:version/snapshot';\nconst CHANGES_PATH_PATTERN = '/replication/:version/changes';\nconst PATH_REGEX = /\\/replication\\/v(?<version>\\d+)\\/(changes|snapshot)$/;\n\nconst SNAPSHOT_PATH = `/replication/v${PROTOCOL_VERSION}/snapshot`;\nconst CHANGES_PATH = `/replication/v${PROTOCOL_VERSION}/changes`;\n\ntype Options = {\n  port: number;\n  startupDelayMs: number;\n};\n\nexport class ChangeStreamerHttpServer extends HttpService {\n  readonly id = 'change-streamer-http-server';\n  readonly #lc: LogContext;\n  readonly #opts: Options;\n  readonly #changeStreamer: ChangeStreamer & Service;\n  readonly #backupMonitor: BackupMonitor | null;\n\n  constructor(\n    lc: LogContext,\n    config: ZeroConfig,\n    opts: Options,\n    parent: Worker,\n    changeStreamer: ChangeStreamer & Service,\n    backupMonitor: BackupMonitor | null,\n  ) {\n    super('change-streamer-http-server', lc, opts, async fastify => {\n      const websocketOptions: {perMessageDeflate?: boolean | object} = {};\n      if (config.websocketCompression) {\n        if (config.websocketCompressionOptions) {\n          try {\n            websocketOptions.perMessageDeflate = JSON.parse(\n              config.websocketCompressionOptions,\n            );\n          } catch (e) {\n            throw new Error(\n              `Failed to parse ZERO_WEBSOCKET_COMPRESSION_OPTIONS: ${String(e)}. Expected valid JSON.`,\n            );\n          }\n        } else {\n          websocketOptions.perMessageDeflate = true;\n        }\n      }\n\n      await fastify.register(websocket, {\n        options: websocketOptions,\n      });\n\n      fastify.get(CHANGES_PATH_PATTERN, {websocket: true}, this.#subscribe);\n      fastify.get(\n        SNAPSHOT_PATH_PATTERN,\n        {websocket: true},\n        this.#reserveSnapshot,\n      );\n\n      installWebSocketReceiver<'snapshot' | 'changes'>(\n        lc,\n        fastify.websocketServer,\n        this.#receiveWebsocket,\n        parent,\n      );\n    });\n\n    this.#lc = lc;\n    this.#opts = opts;\n    this.#changeStreamer = changeStreamer;\n    this.#backupMonitor = backupMonitor;\n  }\n\n  #getBackupMonitor() {\n    return must(\n      this.#backupMonitor,\n      'replication-manager is not configured with a ZERO_LITESTREAM_BACKUP_URL',\n    );\n  }\n\n  // Called when receiving a web socket via the main dispatcher handoff.\n  readonly #receiveWebsocket = (\n    ws: WebSocket,\n    action: 'changes' | 'snapshot',\n    msg: IncomingMessageSubset,\n  ) => {\n    switch (action) {\n      case 'snapshot':\n        return this.#reserveSnapshot(ws, msg);\n      case 'changes':\n        return this.#subscribe(ws, msg);\n      default:\n        closeWithError(\n          this._lc,\n          ws,\n          `invalid action \"${action}\" received in handoff`,\n        );\n        return;\n    }\n  };\n\n  readonly #reserveSnapshot = (ws: WebSocket, req: RequestHeaders) => {\n    try {\n      const url = new URL(\n        req.url ?? '',\n        req.headers.origin ?? 'http://localhost',\n      );\n      checkProtocolVersion(url.pathname);\n      const taskID = url.searchParams.get('taskID');\n      if (!taskID) {\n        throw new Error('Missing taskID in snapshot request');\n      }\n      const downstream =\n        this.#getBackupMonitor().startSnapshotReservation(taskID);\n      void streamOut(this._lc, downstream, ws);\n    } catch (err) {\n      closeWithError(this._lc, ws, err, PROTOCOL_ERROR);\n    }\n  };\n\n  readonly #subscribe = async (ws: WebSocket, req: RequestHeaders) => {\n    try {\n      const ctx = getSubscriberContext(req);\n      if (ctx.mode === 'serving') {\n        this.#ensureChangeStreamerStarted('incoming subscription');\n      }\n\n      const downstream = await this.#changeStreamer.subscribe(ctx);\n      if (ctx.initial && ctx.taskID && this.#backupMonitor) {\n        // Now that the change-streamer knows about the subscriber and watermark,\n        // end the reservation to safely resume scheduling cleanup.\n        this.#backupMonitor.endReservation(ctx.taskID);\n      }\n      void streamOut(this._lc, downstream, ws);\n    } catch (err) {\n      closeWithError(this._lc, ws, err, PROTOCOL_ERROR);\n    }\n  };\n\n  #changeStreamerStarted = false;\n\n  #ensureChangeStreamerStarted(reason: string) {\n    if (!this.#changeStreamerStarted && this._state.shouldRun()) {\n      this.#lc.info?.(`starting ChangeStreamerService: ${reason}`);\n      void this.#changeStreamer\n        .run()\n        .catch(e =>\n          this.#lc.warn?.(`ChangeStreamerService ended with error`, e),\n        )\n        .finally(() => this.stop());\n\n      this.#changeStreamerStarted = true;\n    }\n  }\n\n  protected override _onStart(): void {\n    const {startupDelayMs} = this.#opts;\n    this._state.setTimeout(\n      () =>\n        this.#ensureChangeStreamerStarted(\n          `startup delay elapsed (${startupDelayMs} ms)`,\n        ),\n      startupDelayMs,\n    );\n  }\n\n  protected override async _onStop(): Promise<void> {\n    if (this.#changeStreamerStarted) {\n      await this.#changeStreamer.stop();\n    }\n  }\n}\n\nexport class ChangeStreamerHttpClient implements ChangeStreamer {\n  readonly #lc: LogContext;\n  readonly #shardID: ShardID;\n  readonly #changeDB: PostgresDB;\n  readonly #changeStreamerURI: string | undefined;\n\n  constructor(\n    lc: LogContext,\n    shardID: ShardID,\n    changeDB: string,\n    changeStreamerURI: string | undefined,\n  ) {\n    this.#lc = lc;\n    this.#shardID = shardID;\n    // Create a pg client with a single short-lived connection for the purpose\n    // of change-streamer discovery (i.e. ChangeDB as DNS).\n    this.#changeDB = pgClient(lc, changeDB, {\n      max: 1,\n      ['idle_timeout']: 15,\n      connection: {['application_name']: 'change-streamer-discovery'},\n    });\n    this.#changeStreamerURI = changeStreamerURI;\n  }\n\n  async #resolveChangeStreamer(path: string) {\n    let baseURL = this.#changeStreamerURI;\n    if (!baseURL) {\n      const address = await discoverChangeStreamerAddress(\n        this.#shardID,\n        this.#changeDB,\n      );\n      if (!address) {\n        throw new Error(`no change-streamer is running`);\n      }\n      baseURL = address.includes('://') ? `${address}/` : `ws://${address}/`;\n    }\n    const uri = new URL(path, baseURL);\n    this.#lc.info?.(`connecting to change-streamer@${uri}`);\n    return uri;\n  }\n\n  async reserveSnapshot(taskID: string): Promise<Source<SnapshotMessage>> {\n    const uri = await this.#resolveChangeStreamer(SNAPSHOT_PATH);\n\n    const params = new URLSearchParams({taskID});\n    const ws = new WebSocket(uri + `?${params.toString()}`);\n\n    return streamIn(this.#lc, ws, snapshotMessageSchema);\n  }\n\n  async subscribe(ctx: SubscriberContext): Promise<Source<Downstream>> {\n    const uri = await this.#resolveChangeStreamer(CHANGES_PATH);\n\n    const params = getParams(ctx);\n    const ws = new WebSocket(uri + `?${params.toString()}`);\n\n    return streamIn(this.#lc, ws, downstreamSchema);\n  }\n}\n\ntype RequestHeaders = Pick<IncomingMessage, 'url' | 'headers'>;\n\nexport function getSubscriberContext(req: RequestHeaders): SubscriberContext {\n  const url = new URL(req.url ?? '', req.headers.origin ?? 'http://localhost');\n  const protocolVersion = checkProtocolVersion(url.pathname);\n  const params = new URLParams(url);\n\n  return {\n    protocolVersion,\n    id: params.get('id', true),\n    taskID: params.get('taskID', false),\n    mode: params.get('mode', false) === 'backup' ? 'backup' : 'serving',\n    replicaVersion: params.get('replicaVersion', true),\n    watermark: params.get('watermark', true),\n    initial: params.getBoolean('initial'),\n  };\n}\n\nfunction checkProtocolVersion(pathname: string): number {\n  const match = PATH_REGEX.exec(pathname);\n  if (!match) {\n    throw new Error(`invalid path: ${pathname}`);\n  }\n  const v = Number(match.groups?.version);\n  if (\n    Number.isNaN(v) ||\n    v > PROTOCOL_VERSION ||\n    v < MIN_SUPPORTED_PROTOCOL_VERSION\n  ) {\n    throw new Error(\n      `Cannot service client at protocol v${v}. ` +\n        `Supported protocols: [v${MIN_SUPPORTED_PROTOCOL_VERSION} ... v${PROTOCOL_VERSION}]`,\n    );\n  }\n  return v;\n}\n\n// This is called from the client-side (i.e. the replicator).\nfunction getParams(ctx: SubscriberContext): URLSearchParams {\n  // The protocolVersion is hard-coded into the CHANGES_PATH.\n  const {protocolVersion, ...stringParams} = ctx;\n  assert(\n    protocolVersion === PROTOCOL_VERSION,\n    `replicator should be setting protocolVersion to ${PROTOCOL_VERSION}`,\n  );\n  return new URLSearchParams({\n    ...stringParams,\n    taskID: ctx.taskID ? ctx.taskID : '',\n    initial: ctx.initial ? 'true' : 'false',\n  });\n}\n"],"mappings":";;;;;;;;;;;;;;;AA4BA,IAAM,iCAAiC;AAEvC,IAAM,wBAAwB;AAC9B,IAAM,uBAAuB;AAC7B,IAAM,aAAa;AAEnB,IAAM,gBAAgB;AACtB,IAAM,eAAe;AAOrB,IAAa,2BAAb,cAA8C,YAAY;CACxD,KAAc;CACd;CACA;CACA;CACA;CAEA,YACE,IACA,QACA,MACA,QACA,gBACA,eACA;AACA,QAAM,+BAA+B,IAAI,MAAM,OAAM,YAAW;GAC9D,MAAM,mBAA2D,EAAE;AACnE,OAAI,OAAO,qBACT,KAAI,OAAO,4BACT,KAAI;AACF,qBAAiB,oBAAoB,KAAK,MACxC,OAAO,4BACR;YACM,GAAG;AACV,UAAM,IAAI,MACR,uDAAuD,OAAO,EAAE,CAAC,wBAClE;;OAGH,kBAAiB,oBAAoB;AAIzC,SAAM,QAAQ,SAAS,WAAW,EAChC,SAAS,kBACV,CAAC;AAEF,WAAQ,IAAI,sBAAsB,EAAC,WAAW,MAAK,EAAE,MAAA,UAAgB;AACrE,WAAQ,IACN,uBACA,EAAC,WAAW,MAAK,EACjB,MAAA,gBACD;AAED,4BACE,IACA,QAAQ,iBACR,MAAA,kBACA,OACD;IACD;AAEF,QAAA,KAAW;AACX,QAAA,OAAa;AACb,QAAA,iBAAuB;AACvB,QAAA,gBAAsB;;CAGxB,oBAAoB;AAClB,SAAO,KACL,MAAA,eACA,0EACD;;CAIH,qBACE,IACA,QACA,QACG;AACH,UAAQ,QAAR;GACE,KAAK,WACH,QAAO,MAAA,gBAAsB,IAAI,IAAI;GACvC,KAAK,UACH,QAAO,MAAA,UAAgB,IAAI,IAAI;GACjC;AACE,mBACE,KAAK,KACL,IACA,mBAAmB,OAAO,uBAC3B;AACD;;;CAIN,oBAA6B,IAAe,QAAwB;AAClE,MAAI;GACF,MAAM,MAAM,IAAI,IACd,IAAI,OAAO,IACX,IAAI,QAAQ,UAAU,mBACvB;AACD,wBAAqB,IAAI,SAAS;GAClC,MAAM,SAAS,IAAI,aAAa,IAAI,SAAS;AAC7C,OAAI,CAAC,OACH,OAAM,IAAI,MAAM,qCAAqC;GAEvD,MAAM,aACJ,MAAA,kBAAwB,CAAC,yBAAyB,OAAO;AACtD,aAAU,KAAK,KAAK,YAAY,GAAG;WACjC,KAAK;AACZ,kBAAe,KAAK,KAAK,IAAI,KAAK,eAAe;;;CAIrD,aAAsB,OAAO,IAAe,QAAwB;AAClE,MAAI;GACF,MAAM,MAAM,qBAAqB,IAAI;AACrC,OAAI,IAAI,SAAS,UACf,OAAA,4BAAkC,wBAAwB;GAG5D,MAAM,aAAa,MAAM,MAAA,eAAqB,UAAU,IAAI;AAC5D,OAAI,IAAI,WAAW,IAAI,UAAU,MAAA,cAG/B,OAAA,cAAoB,eAAe,IAAI,OAAO;AAE3C,aAAU,KAAK,KAAK,YAAY,GAAG;WACjC,KAAK;AACZ,kBAAe,KAAK,KAAK,IAAI,KAAK,eAAe;;;CAIrD,yBAAyB;CAEzB,6BAA6B,QAAgB;AAC3C,MAAI,CAAC,MAAA,yBAA+B,KAAK,OAAO,WAAW,EAAE;AAC3D,SAAA,GAAS,OAAO,mCAAmC,SAAS;AACvD,SAAA,eACF,KAAK,CACL,OAAM,MACL,MAAA,GAAS,OAAO,0CAA0C,EAAE,CAC7D,CACA,cAAc,KAAK,MAAM,CAAC;AAE7B,SAAA,wBAA8B;;;CAIlC,WAAoC;EAClC,MAAM,EAAC,mBAAkB,MAAA;AACzB,OAAK,OAAO,iBAER,MAAA,4BACE,0BAA0B,eAAe,MAC1C,EACH,eACD;;CAGH,MAAyB,UAAyB;AAChD,MAAI,MAAA,sBACF,OAAM,MAAA,eAAqB,MAAM;;;AAKvC,IAAa,2BAAb,MAAgE;CAC9D;CACA;CACA;CACA;CAEA,YACE,IACA,SACA,UACA,mBACA;AACA,QAAA,KAAW;AACX,QAAA,UAAgB;AAGhB,QAAA,WAAiB,SAAS,IAAI,UAAU;GACtC,KAAK;IACJ,iBAAiB;GAClB,YAAY,GAAE,qBAAqB,6BAA4B;GAChE,CAAC;AACF,QAAA,oBAA0B;;CAG5B,OAAA,sBAA6B,MAAc;EACzC,IAAI,UAAU,MAAA;AACd,MAAI,CAAC,SAAS;GACZ,MAAM,UAAU,MAAM,8BACpB,MAAA,SACA,MAAA,SACD;AACD,OAAI,CAAC,QACH,OAAM,IAAI,MAAM,gCAAgC;AAElD,aAAU,QAAQ,SAAS,MAAM,GAAG,GAAG,QAAQ,KAAK,QAAQ,QAAQ;;EAEtE,MAAM,MAAM,IAAI,IAAI,MAAM,QAAQ;AAClC,QAAA,GAAS,OAAO,iCAAiC,MAAM;AACvD,SAAO;;CAGT,MAAM,gBAAgB,QAAkD;EAItE,MAAM,KAAK,IAAI,YAHH,MAAM,MAAA,sBAA4B,cAAc,GAG7B,IADhB,IAAI,gBAAgB,EAAC,QAAO,CAAC,CACF,UAAU,GAAG;AAEvD,SAAO,SAAS,MAAA,IAAU,IAAI,sBAAsB;;CAGtD,MAAM,UAAU,KAAqD;EAInE,MAAM,KAAK,IAAI,YAHH,MAAM,MAAA,sBAA4B,aAAa,GAG5B,IADhB,UAAU,IAAI,CACa,UAAU,GAAG;AAEvD,SAAO,SAAS,MAAA,IAAU,IAAI,iBAAiB;;;AAMnD,SAAgB,qBAAqB,KAAwC;CAC3E,MAAM,MAAM,IAAI,IAAI,IAAI,OAAO,IAAI,IAAI,QAAQ,UAAU,mBAAmB;CAC5E,MAAM,kBAAkB,qBAAqB,IAAI,SAAS;CAC1D,MAAM,SAAS,IAAI,UAAU,IAAI;AAEjC,QAAO;EACL;EACA,IAAI,OAAO,IAAI,MAAM,KAAK;EAC1B,QAAQ,OAAO,IAAI,UAAU,MAAM;EACnC,MAAM,OAAO,IAAI,QAAQ,MAAM,KAAK,WAAW,WAAW;EAC1D,gBAAgB,OAAO,IAAI,kBAAkB,KAAK;EAClD,WAAW,OAAO,IAAI,aAAa,KAAK;EACxC,SAAS,OAAO,WAAW,UAAU;EACtC;;AAGH,SAAS,qBAAqB,UAA0B;CACtD,MAAM,QAAQ,WAAW,KAAK,SAAS;AACvC,KAAI,CAAC,MACH,OAAM,IAAI,MAAM,iBAAiB,WAAW;CAE9C,MAAM,IAAI,OAAO,MAAM,QAAQ,QAAQ;AACvC,KACE,OAAO,MAAM,EAAE,IACf,IAAA,KACA,IAAI,+BAEJ,OAAM,IAAI,MACR,sCAAsC,EAAE,2BACZ,+BAA+B,UAC5D;AAEH,QAAO;;AAIT,SAAS,UAAU,KAAyC;CAE1D,MAAM,EAAC,iBAAiB,GAAG,iBAAgB;AAC3C,QACE,oBAAA,GACA,oDACD;AACD,QAAO,IAAI,gBAAgB;EACzB,GAAG;EACH,QAAQ,IAAI,SAAS,IAAI,SAAS;EAClC,SAAS,IAAI,UAAU,SAAS;EACjC,CAAC"}
+{"version":3,"file":"change-streamer-http.js","names":["#lc","#opts","#changeStreamer","#backupMonitor","#subscribe","#reserveSnapshot","#receiveWebsocket","#getBackupMonitor","#ensureChangeStreamerStarted","#changeStreamerStarted","#shardID","#changeDB","#changeStreamerURI","#resolveChangeStreamer"],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-http.ts"],"sourcesContent":["import websocket from '@fastify/websocket';\nimport type {LogContext} from '@rocicorp/logger';\nimport type {IncomingMessage} from 'node:http';\nimport WebSocket from 'ws';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport type {ZeroConfig} from '../../config/zero-config.ts';\nimport type {IncomingMessageSubset} from '../../types/http.ts';\nimport {pgClient, type PostgresDB} from '../../types/pg.ts';\nimport {type Worker} from '../../types/processes.ts';\nimport {type ShardID} from '../../types/shards.ts';\nimport {streamIn, streamOut, type Source} from '../../types/streams.ts';\nimport {URLParams} from '../../types/url-params.ts';\nimport {installWebSocketReceiver} from '../../types/websocket-handoff.ts';\nimport {closeWithError, PROTOCOL_ERROR} from '../../types/ws.ts';\nimport {HttpService} from '../http-service.ts';\nimport type {Service} from '../service.ts';\nimport type {BackupMonitor} from './backup-monitor.ts';\nimport {\n  downstreamSchema,\n  PROTOCOL_VERSION,\n  type ChangeStreamer,\n  type Downstream,\n  type SubscriberContext,\n} from './change-streamer.ts';\nimport {discoverChangeStreamerAddress} from './schema/tables.ts';\nimport {snapshotMessageSchema, type SnapshotMessage} from './snapshot.ts';\n\nconst MIN_SUPPORTED_PROTOCOL_VERSION = 1;\n\nconst SNAPSHOT_PATH_PATTERN = '/replication/:version/snapshot';\nconst CHANGES_PATH_PATTERN = '/replication/:version/changes';\nconst PATH_REGEX = /\\/replication\\/v(?<version>\\d+)\\/(changes|snapshot)$/;\n\nconst SNAPSHOT_PATH = `/replication/v${PROTOCOL_VERSION}/snapshot`;\nconst CHANGES_PATH = `/replication/v${PROTOCOL_VERSION}/changes`;\n\ntype Options = {\n  port: number;\n  startupDelayMs: number;\n};\n\nexport class ChangeStreamerHttpServer extends HttpService {\n  readonly id = 'change-streamer-http-server';\n  readonly #lc: LogContext;\n  readonly #opts: Options;\n  readonly #changeStreamer: ChangeStreamer & Service;\n  readonly #backupMonitor: BackupMonitor | null;\n\n  constructor(\n    lc: LogContext,\n    config: ZeroConfig,\n    opts: Options,\n    parent: Worker,\n    changeStreamer: ChangeStreamer & Service,\n    backupMonitor: BackupMonitor | null,\n  ) {\n    super('change-streamer-http-server', lc, opts, async fastify => {\n      const websocketOptions: {perMessageDeflate?: boolean | object} = {};\n      if (config.websocketCompression) {\n        if (config.websocketCompressionOptions) {\n          try {\n            websocketOptions.perMessageDeflate = JSON.parse(\n              config.websocketCompressionOptions,\n            );\n          } catch (e) {\n            throw new Error(\n              `Failed to parse ZERO_WEBSOCKET_COMPRESSION_OPTIONS: ${String(e)}. Expected valid JSON.`,\n            );\n          }\n        } else {\n          websocketOptions.perMessageDeflate = true;\n        }\n      }\n\n      await fastify.register(websocket, {\n        options: websocketOptions,\n      });\n\n      fastify.get(CHANGES_PATH_PATTERN, {websocket: true}, this.#subscribe);\n      fastify.get(\n        SNAPSHOT_PATH_PATTERN,\n        {websocket: true},\n        this.#reserveSnapshot,\n      );\n\n      installWebSocketReceiver<'snapshot' | 'changes'>(\n        lc,\n        fastify.websocketServer,\n        this.#receiveWebsocket,\n        parent,\n      );\n    });\n\n    this.#lc = lc;\n    this.#opts = opts;\n    this.#changeStreamer = changeStreamer;\n    this.#backupMonitor = backupMonitor;\n  }\n\n  #getBackupMonitor() {\n    return must(\n      this.#backupMonitor,\n      'replication-manager is not configured with a ZERO_LITESTREAM_BACKUP_URL',\n    );\n  }\n\n  // Called when receiving a web socket via the main dispatcher handoff.\n  readonly #receiveWebsocket = (\n    ws: WebSocket,\n    action: 'changes' | 'snapshot',\n    msg: IncomingMessageSubset,\n  ) => {\n    switch (action) {\n      case 'snapshot':\n        return this.#reserveSnapshot(ws, msg);\n      case 'changes':\n        return this.#subscribe(ws, msg);\n      default:\n        closeWithError(\n          this._lc,\n          ws,\n          `invalid action \"${action}\" received in handoff`,\n        );\n        return;\n    }\n  };\n\n  readonly #reserveSnapshot = (ws: WebSocket, req: RequestHeaders) => {\n    try {\n      const url = new URL(\n        req.url ?? '',\n        req.headers.origin ?? 'http://localhost',\n      );\n      checkProtocolVersion(url.pathname);\n      const taskID = url.searchParams.get('taskID');\n      if (!taskID) {\n        throw new Error('Missing taskID in snapshot request');\n      }\n      const downstream =\n        this.#getBackupMonitor().startSnapshotReservation(taskID);\n      void streamOut(this._lc, downstream, ws);\n    } catch (err) {\n      closeWithError(this._lc, ws, err, PROTOCOL_ERROR);\n    }\n  };\n\n  readonly #subscribe = async (ws: WebSocket, req: RequestHeaders) => {\n    try {\n      const ctx = getSubscriberContext(req);\n      if (ctx.mode === 'serving') {\n        this.#ensureChangeStreamerStarted('incoming subscription');\n      }\n\n      const downstream = await this.#changeStreamer.subscribe(ctx);\n      if (ctx.initial && ctx.taskID && this.#backupMonitor) {\n        // Now that the change-streamer knows about the subscriber and watermark,\n        // end the reservation to safely resume scheduling cleanup.\n        this.#backupMonitor.endReservation(ctx.taskID);\n      }\n      void streamOut(this._lc, downstream, ws);\n    } catch (err) {\n      closeWithError(this._lc, ws, err, PROTOCOL_ERROR);\n    }\n  };\n\n  #changeStreamerStarted = false;\n\n  #ensureChangeStreamerStarted(reason: string) {\n    if (!this.#changeStreamerStarted && this._state.shouldRun()) {\n      this.#lc.info?.(`starting ChangeStreamerService: ${reason}`);\n      void this.#changeStreamer\n        .run()\n        .catch(e =>\n          this.#lc.warn?.(`ChangeStreamerService ended with error`, e),\n        )\n        .finally(() => this.stop());\n\n      this.#changeStreamerStarted = true;\n    }\n  }\n\n  protected override _onStart(): void {\n    const {startupDelayMs} = this.#opts;\n    this._state.setTimeout(\n      () =>\n        this.#ensureChangeStreamerStarted(\n          `startup delay elapsed (${startupDelayMs} ms)`,\n        ),\n      startupDelayMs,\n    );\n  }\n\n  protected override async _onStop(): Promise<void> {\n    if (this.#changeStreamerStarted) {\n      await this.#changeStreamer.stop();\n    }\n  }\n}\n\nexport class ChangeStreamerHttpClient implements ChangeStreamer {\n  readonly #lc: LogContext;\n  readonly #shardID: ShardID;\n  readonly #changeDB: PostgresDB;\n  readonly #changeStreamerURI: string | undefined;\n\n  constructor(\n    lc: LogContext,\n    shardID: ShardID,\n    changeDB: string,\n    changeStreamerURI: string | undefined,\n  ) {\n    this.#lc = lc;\n    this.#shardID = shardID;\n    // Create a pg client with a single short-lived connection for the purpose\n    // of change-streamer discovery (i.e. ChangeDB as DNS).\n    this.#changeDB = pgClient(lc, changeDB, {\n      max: 1,\n      ['idle_timeout']: 15,\n      connection: {['application_name']: 'change-streamer-discovery'},\n    });\n    this.#changeStreamerURI = changeStreamerURI;\n  }\n\n  async #resolveChangeStreamer(path: string) {\n    let baseURL = this.#changeStreamerURI;\n    if (!baseURL) {\n      const address = await discoverChangeStreamerAddress(\n        this.#shardID,\n        this.#changeDB,\n      );\n      if (!address) {\n        throw new Error(`no change-streamer is running`);\n      }\n      baseURL = address.includes('://') ? `${address}/` : `ws://${address}/`;\n    }\n    const uri = new URL(path, baseURL);\n    this.#lc.info?.(`connecting to change-streamer@${uri}`);\n    return uri;\n  }\n\n  async reserveSnapshot(taskID: string): Promise<Source<SnapshotMessage>> {\n    const uri = await this.#resolveChangeStreamer(SNAPSHOT_PATH);\n\n    const params = new URLSearchParams({taskID});\n    const ws = new WebSocket(uri + `?${params.toString()}`);\n\n    return streamIn(this.#lc, ws, snapshotMessageSchema);\n  }\n\n  async subscribe(ctx: SubscriberContext): Promise<Source<Downstream>> {\n    const uri = await this.#resolveChangeStreamer(CHANGES_PATH);\n\n    const params = getParams(ctx);\n    const ws = new WebSocket(uri + `?${params.toString()}`);\n\n    return streamIn(this.#lc, ws, downstreamSchema);\n  }\n}\n\ntype RequestHeaders = Pick<IncomingMessage, 'url' | 'headers'>;\n\nexport function getSubscriberContext(req: RequestHeaders): SubscriberContext {\n  const url = new URL(req.url ?? '', req.headers.origin ?? 'http://localhost');\n  const protocolVersion = checkProtocolVersion(url.pathname);\n  const params = new URLParams(url);\n\n  return {\n    protocolVersion,\n    id: params.get('id', true),\n    taskID: params.get('taskID', false),\n    mode: params.get('mode', false) === 'backup' ? 'backup' : 'serving',\n    replicaVersion: params.get('replicaVersion', true),\n    watermark: params.get('watermark', true),\n    initial: params.getBoolean('initial'),\n  };\n}\n\nfunction checkProtocolVersion(pathname: string): number {\n  const match = PATH_REGEX.exec(pathname);\n  if (!match) {\n    throw new Error(`invalid path: ${pathname}`);\n  }\n  const v = Number(match.groups?.version);\n  if (\n    Number.isNaN(v) ||\n    v > PROTOCOL_VERSION ||\n    v < MIN_SUPPORTED_PROTOCOL_VERSION\n  ) {\n    throw new Error(\n      `Cannot service client at protocol v${v}. ` +\n        `Supported protocols: [v${MIN_SUPPORTED_PROTOCOL_VERSION} ... v${PROTOCOL_VERSION}]`,\n    );\n  }\n  return v;\n}\n\n// This is called from the client-side (i.e. the replicator).\nfunction getParams(ctx: SubscriberContext): URLSearchParams {\n  // The protocolVersion is hard-coded into the CHANGES_PATH.\n  const {protocolVersion, ...stringParams} = ctx;\n  assert(\n    protocolVersion === PROTOCOL_VERSION,\n    `replicator should be setting protocolVersion to ${PROTOCOL_VERSION}`,\n  );\n  return new URLSearchParams({\n    ...stringParams,\n    taskID: ctx.taskID ? ctx.taskID : '',\n    initial: ctx.initial ? 'true' : 'false',\n  });\n}\n"],"mappings":";;;;;;;;;;;;;;;AA4BA,IAAM,iCAAiC;AAEvC,IAAM,wBAAwB;AAC9B,IAAM,uBAAuB;AAC7B,IAAM,aAAa;AAEnB,IAAM,gBAAgB;AACtB,IAAM,eAAe;AAOrB,IAAa,2BAAb,cAA8C,YAAY;CACxD,KAAc;CACd;CACA;CACA;CACA;CAEA,YACE,IACA,QACA,MACA,QACA,gBACA,eACA;AACA,QAAM,+BAA+B,IAAI,MAAM,OAAM,YAAW;GAC9D,MAAM,mBAA2D,EAAE;AACnE,OAAI,OAAO,qBACT,KAAI,OAAO,4BACT,KAAI;AACF,qBAAiB,oBAAoB,KAAK,MACxC,OAAO,4BACR;YACM,GAAG;AACV,UAAM,IAAI,MACR,uDAAuD,OAAO,EAAE,CAAC,wBAClE;;OAGH,kBAAiB,oBAAoB;AAIzC,SAAM,QAAQ,SAAS,WAAW,EAChC,SAAS,kBACV,CAAC;AAEF,WAAQ,IAAI,sBAAsB,EAAC,WAAW,MAAK,EAAE,MAAA,UAAgB;AACrE,WAAQ,IACN,uBACA,EAAC,WAAW,MAAK,EACjB,MAAA,gBACD;AAED,4BACE,IACA,QAAQ,iBACR,MAAA,kBACA,OACD;IACD;AAEF,QAAA,KAAW;AACX,QAAA,OAAa;AACb,QAAA,iBAAuB;AACvB,QAAA,gBAAsB;;CAGxB,oBAAoB;AAClB,SAAO,KACL,MAAA,eACA,0EACD;;CAIH,qBACE,IACA,QACA,QACG;AACH,UAAQ,QAAR;GACE,KAAK,WACH,QAAO,MAAA,gBAAsB,IAAI,IAAI;GACvC,KAAK,UACH,QAAO,MAAA,UAAgB,IAAI,IAAI;GACjC;AACE,mBACE,KAAK,KACL,IACA,mBAAmB,OAAO,uBAC3B;AACD;;;CAIN,oBAA6B,IAAe,QAAwB;AAClE,MAAI;GACF,MAAM,MAAM,IAAI,IACd,IAAI,OAAO,IACX,IAAI,QAAQ,UAAU,mBACvB;AACD,wBAAqB,IAAI,SAAS;GAClC,MAAM,SAAS,IAAI,aAAa,IAAI,SAAS;AAC7C,OAAI,CAAC,OACH,OAAM,IAAI,MAAM,qCAAqC;GAEvD,MAAM,aACJ,MAAA,kBAAwB,CAAC,yBAAyB,OAAO;AACtD,aAAU,KAAK,KAAK,YAAY,GAAG;WACjC,KAAK;AACZ,kBAAe,KAAK,KAAK,IAAI,KAAK,eAAe;;;CAIrD,aAAsB,OAAO,IAAe,QAAwB;AAClE,MAAI;GACF,MAAM,MAAM,qBAAqB,IAAI;AACrC,OAAI,IAAI,SAAS,UACf,OAAA,4BAAkC,wBAAwB;GAG5D,MAAM,aAAa,MAAM,MAAA,eAAqB,UAAU,IAAI;AAC5D,OAAI,IAAI,WAAW,IAAI,UAAU,MAAA,cAG/B,OAAA,cAAoB,eAAe,IAAI,OAAO;AAE3C,aAAU,KAAK,KAAK,YAAY,GAAG;WACjC,KAAK;AACZ,kBAAe,KAAK,KAAK,IAAI,KAAK,eAAe;;;CAIrD,yBAAyB;CAEzB,6BAA6B,QAAgB;AAC3C,MAAI,CAAC,MAAA,yBAA+B,KAAK,OAAO,WAAW,EAAE;AAC3D,SAAA,GAAS,OAAO,mCAAmC,SAAS;AACvD,SAAA,eACF,KAAK,CACL,OAAM,MACL,MAAA,GAAS,OAAO,0CAA0C,EAAE,CAC7D,CACA,cAAc,KAAK,MAAM,CAAC;AAE7B,SAAA,wBAA8B;;;CAIlC,WAAoC;EAClC,MAAM,EAAC,mBAAkB,MAAA;AACzB,OAAK,OAAO,iBAER,MAAA,4BACE,0BAA0B,eAAe,MAC1C,EACH,eACD;;CAGH,MAAyB,UAAyB;AAChD,MAAI,MAAA,sBACF,OAAM,MAAA,eAAqB,MAAM;;;AAKvC,IAAa,2BAAb,MAAgE;CAC9D;CACA;CACA;CACA;CAEA,YACE,IACA,SACA,UACA,mBACA;AACA,QAAA,KAAW;AACX,QAAA,UAAgB;AAGhB,QAAA,WAAiB,SAAS,IAAI,UAAU;GACtC,KAAK;IACJ,iBAAiB;GAClB,YAAY,GAAE,qBAAqB,6BAA4B;GAChE,CAAC;AACF,QAAA,oBAA0B;;CAG5B,OAAA,sBAA6B,MAAc;EACzC,IAAI,UAAU,MAAA;AACd,MAAI,CAAC,SAAS;GACZ,MAAM,UAAU,MAAM,8BACpB,MAAA,SACA,MAAA,SACD;AACD,OAAI,CAAC,QACH,OAAM,IAAI,MAAM,gCAAgC;AAElD,aAAU,QAAQ,SAAS,MAAM,GAAG,GAAG,QAAQ,KAAK,QAAQ,QAAQ;;EAEtE,MAAM,MAAM,IAAI,IAAI,MAAM,QAAQ;AAClC,QAAA,GAAS,OAAO,iCAAiC,MAAM;AACvD,SAAO;;CAGT,MAAM,gBAAgB,QAAkD;EAItE,MAAM,KAAK,IAAI,UAHH,MAAM,MAAA,sBAA4B,cAAc,GAG7B,IADhB,IAAI,gBAAgB,EAAC,QAAO,CAAC,CACF,UAAU,GAAG;AAEvD,SAAO,SAAS,MAAA,IAAU,IAAI,sBAAsB;;CAGtD,MAAM,UAAU,KAAqD;EAInE,MAAM,KAAK,IAAI,UAHH,MAAM,MAAA,sBAA4B,aAAa,GAG5B,IADhB,UAAU,IAAI,CACa,UAAU,GAAG;AAEvD,SAAO,SAAS,MAAA,IAAU,IAAI,iBAAiB;;;AAMnD,SAAgB,qBAAqB,KAAwC;CAC3E,MAAM,MAAM,IAAI,IAAI,IAAI,OAAO,IAAI,IAAI,QAAQ,UAAU,mBAAmB;CAC5E,MAAM,kBAAkB,qBAAqB,IAAI,SAAS;CAC1D,MAAM,SAAS,IAAI,UAAU,IAAI;AAEjC,QAAO;EACL;EACA,IAAI,OAAO,IAAI,MAAM,KAAK;EAC1B,QAAQ,OAAO,IAAI,UAAU,MAAM;EACnC,MAAM,OAAO,IAAI,QAAQ,MAAM,KAAK,WAAW,WAAW;EAC1D,gBAAgB,OAAO,IAAI,kBAAkB,KAAK;EAClD,WAAW,OAAO,IAAI,aAAa,KAAK;EACxC,SAAS,OAAO,WAAW,UAAU;EACtC;;AAGH,SAAS,qBAAqB,UAA0B;CACtD,MAAM,QAAQ,WAAW,KAAK,SAAS;AACvC,KAAI,CAAC,MACH,OAAM,IAAI,MAAM,iBAAiB,WAAW;CAE9C,MAAM,IAAI,OAAO,MAAM,QAAQ,QAAQ;AACvC,KACE,OAAO,MAAM,EAAE,IACf,IAAA,KACA,IAAI,+BAEJ,OAAM,IAAI,MACR,sCAAsC,EAAE,2BACZ,+BAA+B,UAC5D;AAEH,QAAO;;AAIT,SAAS,UAAU,KAAyC;CAE1D,MAAM,EAAC,iBAAiB,GAAG,iBAAgB;AAC3C,QACE,oBAAA,GACA,oDACD;AACD,QAAO,IAAI,gBAAgB;EACzB,GAAG;EACH,QAAQ,IAAI,SAAS,IAAI,SAAS;EAClC,SAAS,IAAI,UAAU,SAAS;EACjC,CAAC"}

package/out/zero-cache/src/services/change-streamer/change-streamer-service.d.ts

@@ -3,12 +3,13 @@ import type { PostgresDB } from '../../types/pg.ts';
 import type { ShardID } from '../../types/shards.ts';
 import type { ChangeSource } from '../change-source/change-source.ts';
 import { type ChangeStreamData } from '../change-source/protocol/current/downstream.ts';
+import { type ReplicationStatusPublisher } from '../replicator/replication-status.ts';
 import type { SubscriptionState } from '../replicator/schema/replication-state.ts';
 import { type ChangeStreamerService } from './change-streamer.ts';
 /**
  * Performs initialization and schema migrations to initialize a ChangeStreamerImpl.
  */
-export declare function initializeStreamer(lc: LogContext, shard: ShardID, taskID: string, discoveryAddress: string, discoveryProtocol: string, changeDB: PostgresDB, changeSource: ChangeSource, subscriptionState: SubscriptionState, autoReset: boolean, backPressureLimitHeapProportion: number, flowControlConsensusPaddingSeconds: number, setTimeoutFn?: typeof setTimeout): Promise<ChangeStreamerService>;
+export declare function initializeStreamer(lc: LogContext, shard: ShardID, taskID: string, discoveryAddress: string, discoveryProtocol: string, changeDB: PostgresDB, changeSource: ChangeSource, replicationStatusPublisher: ReplicationStatusPublisher, subscriptionState: SubscriptionState, autoReset: boolean, backPressureLimitHeapProportion: number, flowControlConsensusPaddingSeconds: number, setTimeoutFn?: typeof setTimeout): Promise<ChangeStreamerService>;
 /**
  * Internally all Downstream messages (not just commits) are given a watermark.
  * These are used for internal ordering for:

package/out/zero-cache/src/services/change-streamer/change-streamer-service.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"change-streamer-service.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-service.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAYjD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,mBAAmB,CAAC;AAClD,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,uBAAuB,CAAC;AAGnD,OAAO,KAAK,EACV,YAAY,EAEb,MAAM,mCAAmC,CAAC;AAC3C,OAAO,EAEL,KAAK,gBAAgB,EACtB,MAAM,iDAAiD,CAAC;AAKzD,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,2CAA2C,CAAC;AAMjF,OAAO,EACL,KAAK,qBAAqB,EAG3B,MAAM,sBAAsB,CAAC;AAY9B;;GAEG;AACH,wBAAsB,kBAAkB,CACtC,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,MAAM,EAAE,MAAM,EACd,gBAAgB,EAAE,MAAM,EACxB,iBAAiB,EAAE,MAAM,EACzB,QAAQ,EAAE,UAAU,EACpB,YAAY,EAAE,YAAY,EAC1B,iBAAiB,EAAE,iBAAiB,EACpC,SAAS,EAAE,OAAO,EAClB,+BAA+B,EAAE,MAAM,EACvC,kCAAkC,EAAE,MAAM,EAC1C,YAAY,oBAAa,GACxB,OAAO,CAAC,qBAAqB,CAAC,CA2BhC;AAED;;;;;;;;;GASG;AACH,MAAM,MAAM,iBAAiB,GAAG,CAAC,SAAS,EAAE,MAAM,EAAE,gBAAgB,CAAC,CAAC"}
+{"version":3,"file":"change-streamer-service.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-service.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAYjD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,mBAAmB,CAAC;AAClD,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,uBAAuB,CAAC;AAGnD,OAAO,KAAK,EACV,YAAY,EAEb,MAAM,mCAAmC,CAAC;AAC3C,OAAO,EAEL,KAAK,gBAAgB,EACtB,MAAM,iDAAiD,CAAC;AACzD,OAAO,EAGL,KAAK,0BAA0B,EAChC,MAAM,qCAAqC,CAAC;AAC7C,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,2CAA2C,CAAC;AAMjF,OAAO,EACL,KAAK,qBAAqB,EAG3B,MAAM,sBAAsB,CAAC;AAY9B;;GAEG;AACH,wBAAsB,kBAAkB,CACtC,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,MAAM,EAAE,MAAM,EACd,gBAAgB,EAAE,MAAM,EACxB,iBAAiB,EAAE,MAAM,EACzB,QAAQ,EAAE,UAAU,EACpB,YAAY,EAAE,YAAY,EAC1B,0BAA0B,EAAE,0BAA0B,EACtD,iBAAiB,EAAE,iBAAiB,EACpC,SAAS,EAAE,OAAO,EAClB,+BAA+B,EAAE,MAAM,EACvC,kCAAkC,EAAE,MAAM,EAC1C,YAAY,oBAAa,GACxB,OAAO,CAAC,qBAAqB,CAAC,CA4BhC;AAED;;;;;;;;;GASG;AACH,MAAM,MAAM,iBAAiB,GAAG,CAAC,SAAS,EAAE,MAAM,EAAE,gBAAgB,CAAC,CAAC"}

package/out/zero-cache/src/services/change-streamer/change-streamer-service.js

@@ -19,11 +19,11 @@ import { getDefaultHighWaterMark } from "node:stream";
 /**
 * Performs initialization and schema migrations to initialize a ChangeStreamerImpl.
 */
-async function initializeStreamer(lc, shard, taskID, discoveryAddress, discoveryProtocol, changeDB, changeSource, subscriptionState, autoReset, backPressureLimitHeapProportion, flowControlConsensusPaddingSeconds, setTimeoutFn = setTimeout) {
+async function initializeStreamer(lc, shard, taskID, discoveryAddress, discoveryProtocol, changeDB, changeSource, replicationStatusPublisher, subscriptionState, autoReset, backPressureLimitHeapProportion, flowControlConsensusPaddingSeconds, setTimeoutFn = setTimeout) {
 	await initChangeStreamerSchema(lc, changeDB, shard);
 	await ensureReplicationConfig(lc, changeDB, subscriptionState, shard, autoReset, setTimeoutFn);
 	const { replicaVersion } = subscriptionState;
-	return new ChangeStreamerImpl(lc, shard, taskID, discoveryAddress, discoveryProtocol, changeDB, replicaVersion, changeSource, autoReset, backPressureLimitHeapProportion, flowControlConsensusPaddingSeconds, setTimeoutFn);
+	return new ChangeStreamerImpl(lc, shard, taskID, discoveryAddress, discoveryProtocol, changeDB, replicaVersion, changeSource, replicationStatusPublisher, autoReset, backPressureLimitHeapProportion, flowControlConsensusPaddingSeconds, setTimeoutFn);
 }
 /**
 * Upstream-agnostic dispatch of messages in a {@link ChangeStreamMessage} to a
@@ -172,13 +172,14 @@ var ChangeStreamerImpl = class {
 	#source;
 	#storer;
 	#forwarder;
+	#replicationStatusPublisher;
 	#autoReset;
 	#state;
 	#initialWatermarks = /* @__PURE__ */ new Set();
 	#serving = resolver();
 	#txCounter = getOrCreateCounter("replication", "transactions", "Count of replicated transactions");
 	#stream;
-	constructor(lc, shard, taskID, discoveryAddress, discoveryProtocol, changeDB, replicaVersion, source, autoReset, backPressureLimitHeapProportion, flowControlConsensusPaddingSeconds, setTimeoutFn = setTimeout) {
+	constructor(lc, shard, taskID, discoveryAddress, discoveryProtocol, changeDB, replicaVersion, source, replicationStatusPublisher, autoReset, backPressureLimitHeapProportion, flowControlConsensusPaddingSeconds, setTimeoutFn = setTimeout) {
 		this.id = `change-streamer`;
 		this.#lc = lc.withContext("component", "change-streamer");
 		this.#shard = shard;
@@ -191,6 +192,7 @@ var ChangeStreamerImpl = class {
 			consumed[2]
 		]), (err) => this.stop(err), backPressureLimitHeapProportion);
 		this.#forwarder = new Forwarder(lc, { flowControlConsensusPaddingSeconds });
+		this.#replicationStatusPublisher = replicationStatusPublisher;
 		this.#autoReset = autoReset;
 		this.#state = new RunningState(this.id, void 0, setTimeoutFn);
 	}
@@ -209,6 +211,7 @@ var ChangeStreamerImpl = class {
 				this.#storer.run().catch((e) => stream.changes.cancel(e));
 				this.#stream = stream;
 				this.#state.resetBackoff();
+				this.#replicationStatusPublisher.publish(this.#lc, "Replicating", `Replicating from ${lastWatermark}`);
 				watermark = null;
 				for await (const change of stream.changes) {
 					const [type, msg] = change;