@rocicorp/zero 1.9.0-head-177d6c1f-20260713 → 1.9.0

package/out/zero-cache/src/services/change-source/pg/schema/ddl.js.map

@@ -1 +1 @@
-{"version":3,"file":"ddl.js","names":[],"sources":["../../../../../../../../zero-cache/src/services/change-source/pg/schema/ddl.ts"],"sourcesContent":["import {literal as lit} from 'pg-format';\nimport {assert} from '../../../../../../shared/src/asserts.ts';\nimport * as v from '../../../../../../shared/src/valita.ts';\nimport {upstreamSchema, type ShardConfig} from '../../../../types/shards.ts';\nimport {id} from '../../../../types/sql.ts';\nimport {publishedSchema, publishedSchemaQuery} from './published.ts';\n\n// Sent in the 'version' tag of \"ddlStart\" and \"ddlUpdate\" event messages.\n// This is used to ensure that the message constructed in the upstream\n// Trigger function is compatible with the code processing it in the zero-cache.\n//\n// Increment this when changing the format of the contents of the \"ddl\" events.\n// This will allow old / incompatible code to detect the change and abort.\nexport const PROTOCOL_VERSION = 1;\n\nconst triggerEvent = v.object({\n  context: v.object({query: v.string()}).rest(v.string()),\n});\n\n// All DDL events contain a snapshot of the current tables and indexes that\n// are published / relevant to the shard.\nexport const ddlEventSchema = triggerEvent.extend({\n  version: v.literal(PROTOCOL_VERSION),\n  schema: publishedSchema,\n  event: v.object({tag: v.string()}),\n  // Maps the OID of each published table to the `attnum`s of published\n  // columns that were created in the (upstream) transaction that emitted\n  // the event. Such columns are guaranteed to have the column default in\n  // all pre-existing rows, and can thus be replicated without backfill if\n  // the default value itself is replicable. Tables whose publication\n  // entries (e.g. column lists) were modified in the same transaction are\n  // excluded, since a newly *published* (as opposed to newly *created*)\n  // column may hold arbitrary values in existing rows.\n  //\n  // The field is absent in messages from older versions of the upstream\n  // functions (in which case backfill decisions fall back to the command\n  // tag heuristic), and `null` when there are no such columns.\n  newColumns: v.record(v.array(v.number())).nullable().optional(),\n});\n\n/**\n * A {@link DdlStartEvent} message is emitted before every DDL event, containing\n * the current `schema` and the command `tag`.\n *\n * In most cases, the `DdlStartEvent` itself will not be associated with a\n * schema change, in which case `previousSchema` will be `null`. However, the\n * message is still emitted, both for backwards compatibility and to provide\n * the command `tag` context in case an immediately following `DdlStartEvent`\n * tag is emitted with a schema change (which can happen when another event\n * trigger results in a nested ddl statement).\n *\n * In such cases, the `previousSchema` and `schema` fields of the latter event\n * are used to determine the necessary schema change operations (as they are\n * with `ddlUpdate` and `schemaSnapshot` events), and the `tag` of the\n * preceding start event indicates the command that precipitated the schema\n * change (e.g. a CREATE vs ALTER) to determine whether a backfill is\n * necessary.\n */\nexport const ddlStartEventSchema = ddlEventSchema.extend({\n  type: v.literal('ddlStart'),\n  // For ddlStart messages, previousSchema is `null` if there was no change\n  // in schema detected. The `schema` field is always set to the current\n  // schema.\n  //\n  // In 1.5.0 it is always present, and can be made non-optional when\n  // rollback safe.\n  previousSchema: publishedSchema.nullable().optional(),\n  // For backwards compatibility with previous versions of the trigger,\n  // default an absent `event` field with a semantic equivalent. This\n  // field override can be removed in a version that is rollback safe\n  // with 1.4.0.\n  event: v.object({tag: v.string()}).optional(() => ({tag: 'UNKNOWN'})),\n});\n\nexport type DdlStartEvent = v.Infer<typeof ddlStartEventSchema>;\n\n/**\n * A {@link DdlUpdateEvent} is emitted if there was a change in the schema.\n * It always contains `previousSchema` and (current) `schema` fields, leaving\n * it to the receiver to compute the necessary schema change operations.\n */\nexport const ddlUpdateEventSchema = ddlEventSchema.extend({\n  type: v.literal('ddlUpdate'),\n  // ddlUpdate messages are only emitted if the schema changed, with the\n  // `previousSchema` containing the schema before the change.\n  //\n  // In 1.5.0 it is always set, and can be made non-optional when\n  // rollback safe.\n  previousSchema: publishedSchema.optional(),\n});\n\nexport type DdlUpdateEvent = v.Infer<typeof ddlUpdateEventSchema>;\n\n/**\n * The `schemaSnapshot` message is a snapshot of a schema taken in response to\n * a `COMMENT ON PUBLICATION` command, which is a hook recognized by zero\n * to manually emit `previousSchema` and `schema` snapshots when a difference\n * is detected. This is a workaround provided to support detection of schema\n * changes from `ALTER PUBLICATION` commands on supabase, which does not fire\n * event triggers for them (https://github.com/supabase/supautils/issues/123).\n *\n * The hook is exercised by trailing the publication change with a\n * `COMMENT ON PUBLICATION` statement, e.g.\n *\n * ```sql\n * BEGIN;\n * ALTER PUBLICATION my_publication ...;\n * COMMENT ON PUBLICATION my_publication IS 'whatever';\n * COMMIT;\n * ```\n *\n * Note that it is fine to invoke `COMMENT ON PUBLICATION` statements\n * on a database that *does* support event triggers on\n * `ALTER PUBLICATION` statements, as it will simply be a no-op.\n */\nexport const schemaSnapshotEventSchema = ddlEventSchema.extend({\n  type: v.literal('schemaSnapshot'),\n  previousSchema: publishedSchema.optional(),\n});\n\nexport type SchemaSnapshotEvent = v.Infer<typeof schemaSnapshotEventSchema>;\n\nexport const replicationEventSchema = v.union(\n  ddlStartEventSchema,\n  ddlUpdateEventSchema,\n  schemaSnapshotEventSchema,\n);\n\nexport type ReplicationEvent = v.Infer<typeof replicationEventSchema>;\n\n// Creates a function that appends `_{shard-num}` to the input and\n// quotes the result to be a valid identifier.\nfunction append(shardNum: number) {\n  return (name: string) => id(name + '_' + String(shardNum));\n}\n\n// pg_advisory_xact_lock key for serializing ddl statements in order to\n// produce correct schema change diffs.\nconst DDL_SERIALIZATION_LOCK = 0x3c6b8468f1bac0b0n;\n\n/**\n * Event trigger functions contain the core logic that are invoked by triggers.\n *\n * Note that although many of these functions can theoretically be parameterized and\n * shared across shards, it is advantageous to keep the functions in each shard\n * isolated from each other in order to avoid the complexity of shared-function\n * versioning.\n *\n * In a sense, shards (and their triggers and functions) should be thought of as\n * execution environments that can be updated at different schedules. If per-shard\n * triggers called into shared functions, we would have to consider versioning the\n * functions when changing their behavior, backwards compatibility, removal of\n * unused versions, etc. (not unlike versioning of npm packages).\n *\n * Instead, we opt for the simplicity and isolation of having each shard\n * completely own (and maintain) the entirety of its trigger/function stack.\n */\nexport function createEventFunctionStatements(shard: ShardConfig) {\n  const {appID, shardNum, publications} = shard;\n  const schema = id(upstreamSchema(shard)); // e.g. \"{APP_ID}_{SHARD_ID}\"\n  return /*sql*/ `\nCREATE SCHEMA IF NOT EXISTS ${schema};\n\nCREATE OR REPLACE FUNCTION ${schema}.get_trigger_context()\nRETURNS record AS $$\nDECLARE\n  result record;\nBEGIN\n  SELECT COALESCE(current_query(), 'current_query() returned NULL') AS \"query\" into result;\n  RETURN result;\nEND\n$$ LANGUAGE plpgsql;\n\n\nCREATE OR REPLACE FUNCTION ${schema}.notice_ignore(reason TEXT, tag TEXT, target record)\nRETURNS void AS $$\nBEGIN\n  RAISE NOTICE '${appID}_${shardNum} ignoring % % %', reason, tag, \n    COALESCE(row_to_json(target)::text, '');\nEND\n$$ LANGUAGE plpgsql;\n\n\n-- Note: DROP and CREATE to upgrade from v20 to v21 because the\n-- return type has changed. This can be simplified to CREATE OR REPLACE\n-- once 1.5.0 is rollback safe.\nDROP FUNCTION IF EXISTS ${schema}.schema_specs();\nCREATE FUNCTION ${schema}.schema_specs()\nRETURNS JSON \nSTABLE\nAS $$\n  ${publishedSchemaQuery(publications)}\n$$ LANGUAGE sql;\n\n\n-- Stores the most recent published schema\nCREATE TABLE IF NOT EXISTS ${schema}.\"publishedSchema\" (\n  current JSON,\n  exists BOOL PRIMARY KEY DEFAULT true CHECK (exists)\n);\n\nINSERT INTO ${schema}.\"publishedSchema\" (current) VALUES (${schema}.schema_specs())\n  ON CONFLICT (exists) DO \n  UPDATE SET current = excluded.current;\n\n\nCREATE OR REPLACE FUNCTION ${schema}.update_schemas(event_type text, tag text, target record)\nRETURNS void AS $$\nDECLARE\n  prev_schema_specs JSON;\n  schema_specs JSON;\n  message TEXT;\nBEGIN\n  SELECT current FROM ${schema}.\"publishedSchema\" INTO prev_schema_specs;\n  SELECT ${schema}.schema_specs() INTO schema_specs;\n  \n  IF prev_schema_specs::text != schema_specs::text THEN\n    UPDATE ${schema}.\"publishedSchema\" SET current = schema_specs;\n  ELSIF event_type = 'ddlStart' THEN\n    -- ddlStart events are always be emitted to allow the zero-cache\n    -- to track the context of the current command tag in the face of\n    -- nested event triggers (e.g. start->start->end->end).\n    prev_schema_specs = NULL;\n  ELSIF event_type = 'ddlUpdate' THEN\n    -- TODO: fold 'schemaSnapshot' into this condition too (i.e. make it \"ELSE\")\n    -- when 1.5.0 is rollback safe. Until then, noop schemaSnapshots are sent\n    -- for compatibility with 1.0.0 ~ 1.4.0.\n    PERFORM ${schema}.notice_ignore('noop', tag, target);\n    RETURN;\n  END IF;\n\n  SELECT json_build_object(\n    'type', event_type,\n    'version', ${PROTOCOL_VERSION},\n    'previousSchema', prev_schema_specs,\n    'schema', schema_specs,\n    'event', json_build_object('tag', tag),\n    'context', ${schema}.get_trigger_context()\n  ) INTO message;\n\n  PERFORM pg_logical_emit_message(true, '${appID}/${shardNum}/ddl', message);\n\n  RAISE NOTICE 'Emitted ${appID}_${shardNum} % for % %', event_type, tag, \n    COALESCE(row_to_json(target)::text, '');\nEND\n$$ LANGUAGE plpgsql;\n\n\n-- Hook/workaround to manually trigger replication of schema changes on DBs \n-- that do not support/allow event triggers.\nCREATE OR REPLACE FUNCTION ${schema}.update_schemas()\nRETURNS void AS $$\nBEGIN\n  PERFORM ${schema}.update_schemas('schemaSnapshot', 'MANUAL', NULL);\nEND\n$$ LANGUAGE plpgsql;\n\n\nCREATE OR REPLACE FUNCTION ${schema}.emit_ddl_start()\nRETURNS event_trigger AS $$\nDECLARE\n  schema_specs JSON;\n  message TEXT;\nBEGIN\n  -- serialize DDL statements to compute correct schema change diffs\n  PERFORM pg_advisory_xact_lock(${DDL_SERIALIZATION_LOCK});\n  PERFORM ${schema}.update_schemas('ddlStart', TG_TAG, NULL);\nEND\n$$ LANGUAGE plpgsql;\n\n\nCREATE OR REPLACE FUNCTION ${schema}.emit_ddl_end()\nRETURNS event_trigger AS $$\nDECLARE\n  publications TEXT[];\n  target RECORD;\n  relevant RECORD;\n  schema_specs JSON;\n  message TEXT;\n  event TEXT;\nBEGIN\n  publications := ARRAY[${lit(publications)}];\n\n  SELECT objid, object_type, object_identity \n    FROM pg_event_trigger_ddl_commands() \n    LIMIT 1 INTO target;\n\n  -- Filter DDL updates that are not relevant to the shard (i.e. publications) when possible.\n  SELECT true INTO relevant;\n\n  -- Note: ALTER TABLE statements may *remove* the table from the set of published\n  --       tables, and there is no way to determine if the table \"used to be\" in the\n  --       set. Thus, all ALTER TABLE statements must produce a ddl update, similar to\n  --       any DROP * statement.\n  IF (target.object_type = 'table' AND TG_TAG != 'ALTER TABLE') \n     OR target.object_type = 'table column' THEN\n    SELECT ns.nspname AS \"schema\", c.relname AS \"name\" FROM pg_class AS c\n      JOIN pg_namespace AS ns ON c.relnamespace = ns.oid\n      JOIN pg_publication_tables AS pb ON pb.schemaname = ns.nspname AND pb.tablename = c.relname\n      WHERE c.oid = target.objid AND pb.pubname = ANY (publications)\n      INTO relevant;\n\n  ELSIF target.object_type = 'index' THEN\n    SELECT ns.nspname AS \"schema\", c.relname AS \"name\" FROM pg_class AS c\n      JOIN pg_namespace AS ns ON c.relnamespace = ns.oid\n      JOIN pg_indexes as ind ON ind.schemaname = ns.nspname AND ind.indexname = c.relname\n      JOIN pg_publication_tables AS pb ON pb.schemaname = ns.nspname AND pb.tablename = ind.tablename\n      WHERE c.oid = target.objid AND pb.pubname = ANY (publications)\n      INTO relevant;\n\n  ELSIF target.object_type = 'publication relation' THEN\n    SELECT pb.pubname FROM pg_publication_rel AS rel\n      JOIN pg_publication AS pb ON pb.oid = rel.prpubid\n      WHERE rel.oid = target.objid AND pb.pubname = ANY (publications) \n      INTO relevant;\n\n  ELSIF target.object_type = 'publication namespace' THEN\n    SELECT pb.pubname FROM pg_publication_namespace AS ns\n      JOIN pg_publication AS pb ON pb.oid = ns.pnpubid\n      WHERE ns.oid = target.objid AND pb.pubname = ANY (publications) \n      INTO relevant;\n\n  ELSIF target.object_type = 'schema' THEN\n    SELECT ns.nspname AS \"schema\", c.relname AS \"name\" FROM pg_class AS c\n      JOIN pg_namespace AS ns ON c.relnamespace = ns.oid\n      JOIN pg_publication_tables AS pb ON pb.schemaname = ns.nspname AND pb.tablename = c.relname\n      WHERE ns.oid = target.objid AND pb.pubname = ANY (publications)\n      INTO relevant;\n\n  ELSIF target.object_type = 'publication' THEN\n    SELECT 1 WHERE target.object_identity = ANY (publications)\n      INTO relevant;\n\n  -- no-op CREATE IF NOT EXIST statements\n  ELSIF TG_TAG LIKE 'CREATE %' AND target.object_type IS NULL THEN\n    relevant := NULL;\n  END IF;\n\n  IF relevant IS NULL THEN\n    PERFORM ${schema}.notice_ignore('irrelevant', TG_TAG, target);\n    RETURN;\n  END IF;\n\n  IF TG_TAG = 'COMMENT' THEN\n    -- Only make schemaSnapshots for COMMENT ON PUBLICATION\n    IF target.object_type != 'publication' THEN\n      PERFORM ${schema}.notice_ignore('irrelevant', TG_TAG, target);\n      RETURN;\n    END IF;\n    PERFORM ${schema}.update_schemas('schemaSnapshot', TG_TAG, target);\n  ELSE\n    PERFORM ${schema}.update_schemas('ddlUpdate', TG_TAG, target);\n  END IF;\n\nEND\n$$ LANGUAGE plpgsql;\n`;\n}\n\n// Exported for testing.\nexport const TAGS = [\n  'CREATE TABLE',\n  'ALTER TABLE',\n  'CREATE INDEX',\n  'DROP TABLE',\n  'DROP INDEX',\n  'ALTER PUBLICATION',\n  'ALTER SCHEMA',\n] as const;\n\nexport function createEventTriggerStatements(shard: ShardConfig) {\n  // Better to assert here than get a cryptic syntax error from Postgres.\n  assert(shard.publications.length, `shard publications must be non-empty`);\n\n  // Unlike functions, which are namespaced in shard-specific schemas,\n  // EVENT TRIGGER names are in the global namespace and thus must include\n  // the appID and shardNum.\n  const {appID, shardNum} = shard;\n  const sharded = append(shardNum);\n  const schema = id(upstreamSchema(shard));\n\n  const triggers = [\n    dropEventTriggerStatements(shard.appID, shard.shardNum),\n    /*sql*/ `\nCREATE EVENT TRIGGER ${sharded(`${appID}_ddl_start`)}\n  ON ddl_command_start\n  WHEN TAG IN (${lit(TAGS)})\n  EXECUTE PROCEDURE ${schema}.emit_ddl_start();\n\nCREATE EVENT TRIGGER ${sharded(`${appID}_ddl_end`)}\n  ON ddl_command_end\n  WHEN TAG IN (${lit([...TAGS, 'COMMENT'])})\n  EXECUTE PROCEDURE ${schema}.emit_ddl_end();\n`,\n  ];\n\n  // Drop legacy functions / triggers.\n  triggers.push(\n    `DROP FUNCTION IF EXISTS ${schema}.emit_ddl_end(text) CASCADE;`,\n    `DROP FUNCTION IF EXISTS ${schema}.notice_ignore(text, record);`,\n  );\n  for (const tag of [...TAGS, 'COMMENT']) {\n    const tagID = tag.toLowerCase().replace(' ', '_');\n    triggers.push(`DROP FUNCTION IF EXISTS ${schema}.emit_${tagID}() CASCADE;`);\n  }\n  return triggers.join('');\n}\n\n// Exported for testing.\nexport function dropEventTriggerStatements(\n  appID: string,\n  shardID: string | number,\n) {\n  return /*sql*/ `\n    DROP EVENT TRIGGER IF EXISTS ${id(`${appID}_ddl_start_${shardID}`)};\n    DROP EVENT TRIGGER IF EXISTS ${id(`${appID}_ddl_end_${shardID}`)};\n  `;\n}\n"],"mappings":";;;;;;AAqBA,IAAa,iBANQ,eAAE,OAAO,EAC5B,SAAS,eAAE,OAAO,EAAC,OAAO,eAAE,OAAO,EAAC,CAAC,EAAE,KAAK,eAAE,OAAO,CAAC,EACxD,CAI8B,EAAa,OAAO;CAChD,SAAS,eAAE,QAAA,CAAwB;CACnC,QAAQ;CACR,OAAO,eAAE,OAAO,EAAC,KAAK,eAAE,OAAO,EAAC,CAAC;CAajC,YAAY,eAAE,OAAO,eAAE,MAAM,eAAE,OAAO,CAAC,CAAC,EAAE,SAAS,EAAE,SAAS;AAChE,CAAC;;;;;;;;;;;;;;;;;;;AAoBD,IAAa,sBAAsB,eAAe,OAAO;CACvD,MAAM,eAAE,QAAQ,UAAU;CAO1B,gBAAgB,gBAAgB,SAAS,EAAE,SAAS;CAKpD,OAAO,eAAE,OAAO,EAAC,KAAK,eAAE,OAAO,EAAC,CAAC,EAAE,gBAAgB,EAAC,KAAK,UAAS,EAAE;AACtE,CAAC;;;;;;AASD,IAAa,uBAAuB,eAAe,OAAO;CACxD,MAAM,eAAE,QAAQ,WAAW;CAM3B,gBAAgB,gBAAgB,SAAS;AAC3C,CAAC;;;;;;;;;;;;;;;;;;;;;;;AA0BD,IAAa,4BAA4B,eAAe,OAAO;CAC7D,MAAM,eAAE,QAAQ,gBAAgB;CAChC,gBAAgB,gBAAgB,SAAS;AAC3C,CAAC;AAID,IAAa,yBAAyB,eAAE,MACtC,qBACA,sBACA,yBACF;AAMA,SAAS,OAAO,UAAkB;CAChC,QAAQ,SAAiB,GAAG,OAAO,MAAM,OAAO,QAAQ,CAAC;AAC3D;AAIA,IAAM,yBAAyB;;;;;;;;;;;;;;;;;;AAmB/B,SAAgB,8BAA8B,OAAoB;CAChE,MAAM,EAAC,OAAO,UAAU,iBAAgB;CACxC,MAAM,SAAS,GAAG,eAAe,KAAK,CAAC;CACvC,OAAe;8BACa,OAAO;;6BAER,OAAO;;;;;;;;;;;6BAWP,OAAO;;;kBAGlB,MAAM,GAAG,SAAS;;;;;;;;;0BASV,OAAO;kBACf,OAAO;;;;IAIrB,qBAAqB,YAAY,EAAE;;;;;6BAKV,OAAO;;;;;cAKtB,OAAO,uCAAuC,OAAO;;;;;6BAKtC,OAAO;;;;;;;wBAOZ,OAAO;WACpB,OAAO;;;aAGL,OAAO;;;;;;;;;;cAUN,OAAO;;;;;;;;;;iBAUJ,OAAO;;;2CAGmB,MAAM,GAAG,SAAS;;0BAEnC,MAAM,GAAG,SAAS;;;;;;;;6BAQf,OAAO;;;YAGxB,OAAO;;;;;6BAKU,OAAO;;;;;;;kCAOF,uBAAuB;YAC7C,OAAO;;;;;6BAKU,OAAO;;;;;;;;;;0BAUV,QAAI,YAAY,EAAE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;cA0D9B,OAAO;;;;;;;gBAOL,OAAO;;;cAGT,OAAO;;cAEP,OAAO;;;;;;AAMrB;AAGA,IAAa,OAAO;CAClB;CACA;CACA;CACA;CACA;CACA;CACA;AACF;AAEA,SAAgB,6BAA6B,OAAoB;CAE/D,OAAO,MAAM,aAAa,QAAQ,sCAAsC;CAKxE,MAAM,EAAC,OAAO,aAAY;CAC1B,MAAM,UAAU,OAAO,QAAQ;CAC/B,MAAM,SAAS,GAAG,eAAe,KAAK,CAAC;CAEvC,MAAM,WAAW,CACf,2BAA2B,MAAM,OAAO,MAAM,QAAQ,GAC9C;uBACW,QAAQ,GAAG,MAAM,WAAW,EAAE;;iBAEpC,QAAI,IAAI,EAAE;sBACL,OAAO;;uBAEN,QAAQ,GAAG,MAAM,SAAS,EAAE;;iBAElC,QAAI,CAAC,GAAG,MAAM,SAAS,CAAC,EAAE;sBACrB,OAAO;CAE3B;CAGA,SAAS,KACP,2BAA2B,OAAO,+BAClC,2BAA2B,OAAO,8BACpC;CACA,KAAK,MAAM,OAAO,CAAC,GAAG,MAAM,SAAS,GAAG;EACtC,MAAM,QAAQ,IAAI,YAAY,EAAE,QAAQ,KAAK,GAAG;EAChD,SAAS,KAAK,2BAA2B,OAAO,QAAQ,MAAM,YAAY;CAC5E;CACA,OAAO,SAAS,KAAK,EAAE;AACzB;AAGA,SAAgB,2BACd,OACA,SACA;CACA,OAAe;mCACkB,GAAG,GAAG,MAAM,aAAa,SAAS,EAAE;mCACpC,GAAG,GAAG,MAAM,WAAW,SAAS,EAAE;;AAErE"}
+{"version":3,"file":"ddl.js","names":[],"sources":["../../../../../../../../zero-cache/src/services/change-source/pg/schema/ddl.ts"],"sourcesContent":["import {literal as lit} from 'pg-format';\nimport {assert} from '../../../../../../shared/src/asserts.ts';\nimport * as v from '../../../../../../shared/src/valita.ts';\nimport {upstreamSchema, type ShardConfig} from '../../../../types/shards.ts';\nimport {id} from '../../../../types/sql.ts';\nimport {publishedSchema, publishedSchemaQuery} from './published.ts';\n\n// Sent in the 'version' tag of \"ddlStart\" and \"ddlUpdate\" event messages.\n// This is used to ensure that the message constructed in the upstream\n// Trigger function is compatible with the code processing it in the zero-cache.\n//\n// Increment this when changing the format of the contents of the \"ddl\" events.\n// This will allow old / incompatible code to detect the change and abort.\nexport const PROTOCOL_VERSION = 1;\n\n// In protocol v2 (planned, not yet emitted), \"ddlStart\" events that are not\n// associated with a schema change will be context-only, omitting the\n// `schema` snapshot entirely. This avoids bloating the WAL with (large)\n// redundant schema snapshots for DDL commands that do not affect the\n// published schema, e.g. the CREATE/ALTER/DROP TABLE sub-commands executed\n// by REFRESH MATERIALIZED VIEW CONCURRENTLY.\n//\n// This release already accepts v2 events so that the (subsequent) release\n// that upgrades the triggers to emit them remains rollback safe with respect\n// to this one.\nconst versionSchema = v.literalUnion(PROTOCOL_VERSION, 2);\n\nconst triggerEvent = v.object({\n  context: v.object({query: v.string()}).rest(v.string()),\n});\n\nexport const ddlEventSchema = triggerEvent.extend({\n  version: versionSchema,\n  event: v.object({tag: v.string()}),\n  // Maps the OID of each published table to the `attnum`s of published\n  // columns that were created in the (upstream) transaction that emitted\n  // the event. Such columns are guaranteed to have the column default in\n  // all pre-existing rows, and can thus be replicated without backfill if\n  // the default value itself is replicable. Tables whose publication\n  // entries (e.g. column lists) were modified in the same transaction are\n  // excluded, since a newly *published* (as opposed to newly *created*)\n  // column may hold arbitrary values in existing rows.\n  //\n  // The field is absent in messages from older versions of the upstream\n  // functions (in which case backfill decisions fall back to the command\n  // tag heuristic), and `null` when there are no such columns.\n  newColumns: v.record(v.array(v.number())).nullable().optional(),\n});\n\n/**\n * A {@link DdlStartEvent} message is emitted before every DDL event,\n * containing the command `tag`.\n *\n * In most cases, the `DdlStartEvent` itself will not be associated with a\n * schema change, in which case the event is context-only (in protocol v2,\n * planned, both `previousSchema` and `schema` will be absent; in v1,\n * `schema` is set and `previousSchema` is `null`). The message is still\n * emitted to provide the command `tag` context in case an immediately\n * following `DdlStartEvent` tag is emitted with a schema change (which can\n * happen when another event trigger results in a nested ddl statement).\n *\n * In such cases, the `previousSchema` and `schema` fields of the latter event\n * are used to determine the necessary schema change operations (as they are\n * with `ddlUpdate` and `schemaSnapshot` events), and the `tag` of the\n * preceding start event indicates the command that precipitated the schema\n * change (e.g. a CREATE vs ALTER) to determine whether a backfill is\n * necessary.\n */\nexport const ddlStartEventSchema = ddlEventSchema.extend({\n  type: v.literal('ddlStart'),\n  // Set (along with `previousSchema`) only if the ddlStart event itself is\n  // associated with a schema change. Absent in context-only (protocol v2,\n  // planned) events. v1 events always contain the current `schema`.\n  schema: publishedSchema.optional(),\n  // For ddlStart messages, previousSchema is `null` (v1) or absent\n  // (protocol v2, planned) if there was no change in schema detected.\n  previousSchema: publishedSchema.nullable().optional(),\n  // For backwards compatibility with previous versions of the trigger,\n  // default an absent `event` field with a semantic equivalent. This\n  // field override can be removed in a version that is rollback safe\n  // with 1.4.0.\n  event: v.object({tag: v.string()}).optional(() => ({tag: 'UNKNOWN'})),\n});\n\nexport type DdlStartEvent = v.Infer<typeof ddlStartEventSchema>;\n\n/**\n * A {@link DdlUpdateEvent} is emitted if there was a change in the schema.\n * It always contains `previousSchema` and (current) `schema` fields, leaving\n * it to the receiver to compute the necessary schema change operations.\n */\nexport const ddlUpdateEventSchema = ddlEventSchema.extend({\n  type: v.literal('ddlUpdate'),\n  // ddlUpdate messages are only emitted if the schema changed. `schema`\n  // contains the current (i.e. post-change) snapshot.\n  schema: publishedSchema,\n  // The `previousSchema` contains the schema before the change.\n  //\n  // In 1.5.0 it is always set, and can be made non-optional when\n  // rollback safe.\n  previousSchema: publishedSchema.optional(),\n});\n\nexport type DdlUpdateEvent = v.Infer<typeof ddlUpdateEventSchema>;\n\n/**\n * The `schemaSnapshot` message is a snapshot of a schema taken in response to\n * a `COMMENT ON PUBLICATION` command, which is a hook recognized by zero\n * to manually emit `previousSchema` and `schema` snapshots when a difference\n * is detected. This is a workaround provided to support detection of schema\n * changes from `ALTER PUBLICATION` commands on supabase, which does not fire\n * event triggers for them (https://github.com/supabase/supautils/issues/123).\n *\n * The hook is exercised by trailing the publication change with a\n * `COMMENT ON PUBLICATION` statement, e.g.\n *\n * ```sql\n * BEGIN;\n * ALTER PUBLICATION my_publication ...;\n * COMMENT ON PUBLICATION my_publication IS 'whatever';\n * COMMIT;\n * ```\n *\n * Note that it is fine to invoke `COMMENT ON PUBLICATION` statements\n * on a database that *does* support event triggers on\n * `ALTER PUBLICATION` statements, as it will simply be a no-op.\n */\nexport const schemaSnapshotEventSchema = ddlEventSchema.extend({\n  type: v.literal('schemaSnapshot'),\n  schema: publishedSchema,\n  previousSchema: publishedSchema.optional(),\n});\n\nexport type SchemaSnapshotEvent = v.Infer<typeof schemaSnapshotEventSchema>;\n\nexport const replicationEventSchema = v.union(\n  ddlStartEventSchema,\n  ddlUpdateEventSchema,\n  schemaSnapshotEventSchema,\n);\n\nexport type ReplicationEvent = v.Infer<typeof replicationEventSchema>;\n\n// Creates a function that appends `_{shard-num}` to the input and\n// quotes the result to be a valid identifier.\nfunction append(shardNum: number) {\n  return (name: string) => id(name + '_' + String(shardNum));\n}\n\n// pg_advisory_xact_lock key for serializing ddl statements in order to\n// produce correct schema change diffs.\nconst DDL_SERIALIZATION_LOCK = 0x3c6b8468f1bac0b0n;\n\n/**\n * Event trigger functions contain the core logic that are invoked by triggers.\n *\n * Note that although many of these functions can theoretically be parameterized and\n * shared across shards, it is advantageous to keep the functions in each shard\n * isolated from each other in order to avoid the complexity of shared-function\n * versioning.\n *\n * In a sense, shards (and their triggers and functions) should be thought of as\n * execution environments that can be updated at different schedules. If per-shard\n * triggers called into shared functions, we would have to consider versioning the\n * functions when changing their behavior, backwards compatibility, removal of\n * unused versions, etc. (not unlike versioning of npm packages).\n *\n * Instead, we opt for the simplicity and isolation of having each shard\n * completely own (and maintain) the entirety of its trigger/function stack.\n */\nexport function createEventFunctionStatements(shard: ShardConfig) {\n  const {appID, shardNum, publications} = shard;\n  const schema = id(upstreamSchema(shard)); // e.g. \"{APP_ID}_{SHARD_ID}\"\n  return /*sql*/ `\nCREATE SCHEMA IF NOT EXISTS ${schema};\n\nCREATE OR REPLACE FUNCTION ${schema}.get_trigger_context()\nRETURNS record AS $$\nDECLARE\n  result record;\nBEGIN\n  SELECT COALESCE(current_query(), 'current_query() returned NULL') AS \"query\" into result;\n  RETURN result;\nEND\n$$ LANGUAGE plpgsql;\n\n\nCREATE OR REPLACE FUNCTION ${schema}.notice_ignore(reason TEXT, tag TEXT, target record)\nRETURNS void AS $$\nBEGIN\n  RAISE NOTICE '${appID}_${shardNum} ignoring % % %', reason, tag, \n    COALESCE(row_to_json(target)::text, '');\nEND\n$$ LANGUAGE plpgsql;\n\n\n-- Note: DROP and CREATE to upgrade from v20 to v21 because the\n-- return type has changed. This can be simplified to CREATE OR REPLACE\n-- once 1.5.0 is rollback safe.\nDROP FUNCTION IF EXISTS ${schema}.schema_specs();\nCREATE FUNCTION ${schema}.schema_specs()\nRETURNS JSON \nSTABLE\nAS $$\n  ${publishedSchemaQuery(publications)}\n$$ LANGUAGE sql;\n\n\n-- Stores the most recent published schema\nCREATE TABLE IF NOT EXISTS ${schema}.\"publishedSchema\" (\n  current JSON,\n  exists BOOL PRIMARY KEY DEFAULT true CHECK (exists)\n);\n\nINSERT INTO ${schema}.\"publishedSchema\" (current) VALUES (${schema}.schema_specs())\n  ON CONFLICT (exists) DO \n  UPDATE SET current = excluded.current;\n\n\nCREATE OR REPLACE FUNCTION ${schema}.update_schemas(event_type text, tag text, target record)\nRETURNS void AS $$\nDECLARE\n  prev_schema_specs JSON;\n  schema_specs JSON;\n  message TEXT;\nBEGIN\n  SELECT current FROM ${schema}.\"publishedSchema\" INTO prev_schema_specs;\n  SELECT ${schema}.schema_specs() INTO schema_specs;\n  \n  IF prev_schema_specs::text != schema_specs::text THEN\n    UPDATE ${schema}.\"publishedSchema\" SET current = schema_specs;\n  ELSIF event_type = 'ddlStart' THEN\n    -- ddlStart events are always be emitted to allow the zero-cache\n    -- to track the context of the current command tag in the face of\n    -- nested event triggers (e.g. start->start->end->end).\n    prev_schema_specs = NULL;\n  ELSIF event_type = 'ddlUpdate' THEN\n    -- TODO: fold 'schemaSnapshot' into this condition too (i.e. make it \"ELSE\")\n    -- when 1.5.0 is rollback safe. Until then, noop schemaSnapshots are sent\n    -- for compatibility with 1.0.0 ~ 1.4.0.\n    PERFORM ${schema}.notice_ignore('noop', tag, target);\n    RETURN;\n  END IF;\n\n  SELECT json_build_object(\n    'type', event_type,\n    'version', ${PROTOCOL_VERSION},\n    'previousSchema', prev_schema_specs,\n    'schema', schema_specs,\n    'event', json_build_object('tag', tag),\n    'context', ${schema}.get_trigger_context()\n  ) INTO message;\n\n  PERFORM pg_logical_emit_message(true, '${appID}/${shardNum}/ddl', message);\n\n  RAISE NOTICE 'Emitted ${appID}_${shardNum} % for % %', event_type, tag,\n    COALESCE(row_to_json(target)::text, '');\nEND\n$$ LANGUAGE plpgsql;\n\n\n-- Hook/workaround to manually trigger replication of schema changes on DBs \n-- that do not support/allow event triggers.\nCREATE OR REPLACE FUNCTION ${schema}.update_schemas()\nRETURNS void AS $$\nBEGIN\n  PERFORM ${schema}.update_schemas('schemaSnapshot', 'MANUAL', NULL);\nEND\n$$ LANGUAGE plpgsql;\n\n\nCREATE OR REPLACE FUNCTION ${schema}.emit_ddl_start()\nRETURNS event_trigger AS $$\nDECLARE\n  schema_specs JSON;\n  message TEXT;\nBEGIN\n  -- serialize DDL statements to compute correct schema change diffs\n  PERFORM pg_advisory_xact_lock(${DDL_SERIALIZATION_LOCK});\n  PERFORM ${schema}.update_schemas('ddlStart', TG_TAG, NULL);\nEND\n$$ LANGUAGE plpgsql;\n\n\nCREATE OR REPLACE FUNCTION ${schema}.emit_ddl_end()\nRETURNS event_trigger AS $$\nDECLARE\n  publications TEXT[];\n  target RECORD;\n  relevant RECORD;\n  schema_specs JSON;\n  message TEXT;\n  event TEXT;\nBEGIN\n  publications := ARRAY[${lit(publications)}];\n\n  SELECT objid, object_type, object_identity \n    FROM pg_event_trigger_ddl_commands() \n    LIMIT 1 INTO target;\n\n  -- Filter DDL updates that are not relevant to the shard (i.e. publications) when possible.\n  SELECT true INTO relevant;\n\n  -- Note: ALTER TABLE statements may *remove* the table from the set of published\n  --       tables, and there is no way to determine if the table \"used to be\" in the\n  --       set. Thus, all ALTER TABLE statements must produce a ddl update, similar to\n  --       any DROP * statement.\n  IF (target.object_type = 'table' AND TG_TAG != 'ALTER TABLE') \n     OR target.object_type = 'table column' THEN\n    SELECT ns.nspname AS \"schema\", c.relname AS \"name\" FROM pg_class AS c\n      JOIN pg_namespace AS ns ON c.relnamespace = ns.oid\n      JOIN pg_publication_tables AS pb ON pb.schemaname = ns.nspname AND pb.tablename = c.relname\n      WHERE c.oid = target.objid AND pb.pubname = ANY (publications)\n      INTO relevant;\n\n  ELSIF target.object_type = 'index' THEN\n    SELECT ns.nspname AS \"schema\", c.relname AS \"name\" FROM pg_class AS c\n      JOIN pg_namespace AS ns ON c.relnamespace = ns.oid\n      JOIN pg_indexes as ind ON ind.schemaname = ns.nspname AND ind.indexname = c.relname\n      JOIN pg_publication_tables AS pb ON pb.schemaname = ns.nspname AND pb.tablename = ind.tablename\n      WHERE c.oid = target.objid AND pb.pubname = ANY (publications)\n      INTO relevant;\n\n  ELSIF target.object_type = 'publication relation' THEN\n    SELECT pb.pubname FROM pg_publication_rel AS rel\n      JOIN pg_publication AS pb ON pb.oid = rel.prpubid\n      WHERE rel.oid = target.objid AND pb.pubname = ANY (publications) \n      INTO relevant;\n\n  ELSIF target.object_type = 'publication namespace' THEN\n    SELECT pb.pubname FROM pg_publication_namespace AS ns\n      JOIN pg_publication AS pb ON pb.oid = ns.pnpubid\n      WHERE ns.oid = target.objid AND pb.pubname = ANY (publications) \n      INTO relevant;\n\n  ELSIF target.object_type = 'schema' THEN\n    SELECT ns.nspname AS \"schema\", c.relname AS \"name\" FROM pg_class AS c\n      JOIN pg_namespace AS ns ON c.relnamespace = ns.oid\n      JOIN pg_publication_tables AS pb ON pb.schemaname = ns.nspname AND pb.tablename = c.relname\n      WHERE ns.oid = target.objid AND pb.pubname = ANY (publications)\n      INTO relevant;\n\n  ELSIF target.object_type = 'publication' THEN\n    SELECT 1 WHERE target.object_identity = ANY (publications)\n      INTO relevant;\n\n  -- no-op CREATE IF NOT EXIST statements\n  ELSIF TG_TAG LIKE 'CREATE %' AND target.object_type IS NULL THEN\n    relevant := NULL;\n  END IF;\n\n  IF relevant IS NULL THEN\n    PERFORM ${schema}.notice_ignore('irrelevant', TG_TAG, target);\n    RETURN;\n  END IF;\n\n  IF TG_TAG = 'COMMENT' THEN\n    -- Only make schemaSnapshots for COMMENT ON PUBLICATION\n    IF target.object_type != 'publication' THEN\n      PERFORM ${schema}.notice_ignore('irrelevant', TG_TAG, target);\n      RETURN;\n    END IF;\n    PERFORM ${schema}.update_schemas('schemaSnapshot', TG_TAG, target);\n  ELSE\n    PERFORM ${schema}.update_schemas('ddlUpdate', TG_TAG, target);\n  END IF;\n\nEND\n$$ LANGUAGE plpgsql;\n`;\n}\n\n// Exported for testing.\nexport const TAGS = [\n  'CREATE TABLE',\n  'ALTER TABLE',\n  'CREATE INDEX',\n  'DROP TABLE',\n  'DROP INDEX',\n  'ALTER PUBLICATION',\n  'ALTER SCHEMA',\n] as const;\n\nexport function createEventTriggerStatements(shard: ShardConfig) {\n  // Better to assert here than get a cryptic syntax error from Postgres.\n  assert(shard.publications.length, `shard publications must be non-empty`);\n\n  // Unlike functions, which are namespaced in shard-specific schemas,\n  // EVENT TRIGGER names are in the global namespace and thus must include\n  // the appID and shardNum.\n  const {appID, shardNum} = shard;\n  const sharded = append(shardNum);\n  const schema = id(upstreamSchema(shard));\n\n  const triggers = [\n    dropEventTriggerStatements(shard.appID, shard.shardNum),\n    /*sql*/ `\nCREATE EVENT TRIGGER ${sharded(`${appID}_ddl_start`)}\n  ON ddl_command_start\n  WHEN TAG IN (${lit(TAGS)})\n  EXECUTE PROCEDURE ${schema}.emit_ddl_start();\n\nCREATE EVENT TRIGGER ${sharded(`${appID}_ddl_end`)}\n  ON ddl_command_end\n  WHEN TAG IN (${lit([...TAGS, 'COMMENT'])})\n  EXECUTE PROCEDURE ${schema}.emit_ddl_end();\n`,\n  ];\n\n  // Drop legacy functions / triggers.\n  triggers.push(\n    `DROP FUNCTION IF EXISTS ${schema}.emit_ddl_end(text) CASCADE;`,\n    `DROP FUNCTION IF EXISTS ${schema}.notice_ignore(text, record);`,\n  );\n  for (const tag of [...TAGS, 'COMMENT']) {\n    const tagID = tag.toLowerCase().replace(' ', '_');\n    triggers.push(`DROP FUNCTION IF EXISTS ${schema}.emit_${tagID}() CASCADE;`);\n  }\n  return triggers.join('');\n}\n\n// Exported for testing.\nexport function dropEventTriggerStatements(\n  appID: string,\n  shardID: string | number,\n) {\n  return /*sql*/ `\n    DROP EVENT TRIGGER IF EXISTS ${id(`${appID}_ddl_start_${shardID}`)};\n    DROP EVENT TRIGGER IF EXISTS ${id(`${appID}_ddl_end_${shardID}`)};\n  `;\n}\n"],"mappings":";;;;;;AAyBA,IAAM,gBAAgB,aAAA,GAAiC,CAAC;AAMxD,IAAa,iBAJQ,eAAE,OAAO,EAC5B,SAAS,eAAE,OAAO,EAAC,OAAO,eAAE,OAAO,EAAC,CAAC,EAAE,KAAK,eAAE,OAAO,CAAC,EACxD,CAE8B,EAAa,OAAO;CAChD,SAAS;CACT,OAAO,eAAE,OAAO,EAAC,KAAK,eAAE,OAAO,EAAC,CAAC;CAajC,YAAY,eAAE,OAAO,eAAE,MAAM,eAAE,OAAO,CAAC,CAAC,EAAE,SAAS,EAAE,SAAS;AAChE,CAAC;;;;;;;;;;;;;;;;;;;;AAqBD,IAAa,sBAAsB,eAAe,OAAO;CACvD,MAAM,eAAE,QAAQ,UAAU;CAI1B,QAAQ,gBAAgB,SAAS;CAGjC,gBAAgB,gBAAgB,SAAS,EAAE,SAAS;CAKpD,OAAO,eAAE,OAAO,EAAC,KAAK,eAAE,OAAO,EAAC,CAAC,EAAE,gBAAgB,EAAC,KAAK,UAAS,EAAE;AACtE,CAAC;;;;;;AASD,IAAa,uBAAuB,eAAe,OAAO;CACxD,MAAM,eAAE,QAAQ,WAAW;CAG3B,QAAQ;CAKR,gBAAgB,gBAAgB,SAAS;AAC3C,CAAC;;;;;;;;;;;;;;;;;;;;;;;AA0BD,IAAa,4BAA4B,eAAe,OAAO;CAC7D,MAAM,eAAE,QAAQ,gBAAgB;CAChC,QAAQ;CACR,gBAAgB,gBAAgB,SAAS;AAC3C,CAAC;AAID,IAAa,yBAAyB,eAAE,MACtC,qBACA,sBACA,yBACF;AAMA,SAAS,OAAO,UAAkB;CAChC,QAAQ,SAAiB,GAAG,OAAO,MAAM,OAAO,QAAQ,CAAC;AAC3D;AAIA,IAAM,yBAAyB;;;;;;;;;;;;;;;;;;AAmB/B,SAAgB,8BAA8B,OAAoB;CAChE,MAAM,EAAC,OAAO,UAAU,iBAAgB;CACxC,MAAM,SAAS,GAAG,eAAe,KAAK,CAAC;CACvC,OAAe;8BACa,OAAO;;6BAER,OAAO;;;;;;;;;;;6BAWP,OAAO;;;kBAGlB,MAAM,GAAG,SAAS;;;;;;;;;0BASV,OAAO;kBACf,OAAO;;;;IAIrB,qBAAqB,YAAY,EAAE;;;;;6BAKV,OAAO;;;;;cAKtB,OAAO,uCAAuC,OAAO;;;;;6BAKtC,OAAO;;;;;;;wBAOZ,OAAO;WACpB,OAAO;;;aAGL,OAAO;;;;;;;;;;cAUN,OAAO;;;;;;;;;;iBAUJ,OAAO;;;2CAGmB,MAAM,GAAG,SAAS;;0BAEnC,MAAM,GAAG,SAAS;;;;;;;;6BAQf,OAAO;;;YAGxB,OAAO;;;;;6BAKU,OAAO;;;;;;;kCAOF,uBAAuB;YAC7C,OAAO;;;;;6BAKU,OAAO;;;;;;;;;;0BAUV,QAAI,YAAY,EAAE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;cA0D9B,OAAO;;;;;;;gBAOL,OAAO;;;cAGT,OAAO;;cAEP,OAAO;;;;;;AAMrB;AAGA,IAAa,OAAO;CAClB;CACA;CACA;CACA;CACA;CACA;CACA;AACF;AAEA,SAAgB,6BAA6B,OAAoB;CAE/D,OAAO,MAAM,aAAa,QAAQ,sCAAsC;CAKxE,MAAM,EAAC,OAAO,aAAY;CAC1B,MAAM,UAAU,OAAO,QAAQ;CAC/B,MAAM,SAAS,GAAG,eAAe,KAAK,CAAC;CAEvC,MAAM,WAAW,CACf,2BAA2B,MAAM,OAAO,MAAM,QAAQ,GAC9C;uBACW,QAAQ,GAAG,MAAM,WAAW,EAAE;;iBAEpC,QAAI,IAAI,EAAE;sBACL,OAAO;;uBAEN,QAAQ,GAAG,MAAM,SAAS,EAAE;;iBAElC,QAAI,CAAC,GAAG,MAAM,SAAS,CAAC,EAAE;sBACrB,OAAO;CAE3B;CAGA,SAAS,KACP,2BAA2B,OAAO,+BAClC,2BAA2B,OAAO,8BACpC;CACA,KAAK,MAAM,OAAO,CAAC,GAAG,MAAM,SAAS,GAAG;EACtC,MAAM,QAAQ,IAAI,YAAY,EAAE,QAAQ,KAAK,GAAG;EAChD,SAAS,KAAK,2BAA2B,OAAO,QAAQ,MAAM,YAAY;CAC5E;CACA,OAAO,SAAS,KAAK,EAAE;AACzB;AAGA,SAAgB,2BACd,OACA,SACA;CACA,OAAe;mCACkB,GAAG,GAAG,MAAM,aAAa,SAAS,EAAE;mCACpC,GAAG,GAAG,MAAM,WAAW,SAAS,EAAE;;AAErE"}

package/out/zero-cache/src/services/change-source/protocol/current/data.d.ts

@@ -13,6 +13,7 @@ export declare const beginSchema: v.ObjectType<{
 }, undefined>;
 export declare const commitSchema: v.ObjectType<{
     tag: v.Type<"commit">;
+    commitTimeMs: v.Optional<number>;
 }, undefined>;
 export declare const rollbackSchema: v.ObjectType<{
     tag: v.Type<"rollback">;

package/out/zero-cache/src/services/change-source/protocol/current/data.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../../../../../zero-cache/src/services/change-source/protocol/current/data.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAEL,KAAK,UAAU,EAChB,MAAM,6CAA6C,CAAC;AAErD,OAAO,KAAK,CAAC,MAAM,wCAAwC,CAAC;AAE5D,OAAO,KAAK,EAAC,SAAS,EAAC,MAAM,gCAAgC,CAAC;AAG9D,eAAO,MAAM,WAAW;;;;aAgBtB,CAAC;AAEH,eAAO,MAAM,YAAY;;aAEvB,CAAC;AAEH,eAAO,MAAM,cAAc;;aAEzB,CAAC;AAcH,eAAO,MAAM,cAAc;;;;;;;;;EA2BvB,CAAC;AAGL,eAAO,MAAM,iBAAiB;;;;;;;aAK5B,CAAC;AAUH,eAAO,MAAM,mBAAmB;;2EAER,CAAC;AAEzB,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,mBAAmB,CAAC,CAAC;AAEhE,eAAO,MAAM,SAAS,yFAA4B,CAAC;AAEnD,eAAO,MAAM,YAAY;;;;;;;;;;;;;aAIvB,CAAC;AAEH,eAAO,MAAM,YAAY;;;;;;;;;;;;;;aAUvB,CAAC;AAEH,eAAO,MAAM,YAAY;;;;;;;;;;;;;aAKvB,CAAC;AAEH,eAAO,MAAM,cAAc;;;;;;;;;;;;aAGzB,CAAC;AAEH,eAAO,MAAM,gBAAgB;;;aAG3B,CAAC;AAEH,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAU1D,eAAO,MAAM,gBAAgB,mEAAmB,CAAC;AAEjD,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAE1D,eAAO,MAAM,iBAAiB;;;;;;;;;;;;;;;;;;;;;;aA4B5B,CAAC;AAEH,eAAO,MAAM,iBAAiB;;;;;;;;;;aAI5B,CAAC;AAEH,eAAO,MAAM,yBAAyB;;;;;;;;;;;;aAKpC,CAAC;AAOH,eAAO,MAAM,eAAe;;;;;;;;;;;;;;;;;;;;;;;aAa1B,CAAC;AAEH,eAAO,MAAM,kBAAkB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;aAK7B,CAAC;AAEH,eAAO,MAAM,gBAAgB;;;;;;;aAI3B,CAAC;AAEH,eAAO,MAAM,eAAe;;;;;;aAG1B,CAAC;AAEH,eAAO,MAAM,iBAAiB;;;;;;;;;;aAG5B,CAAC;AAEH,eAAO,MAAM,eAAe;;;;;;aAG1B,CAAC;AAEH,eAAO,MAAM,oBAAoB;;;;aAI/B,CAAC;AAEH,MAAM,MAAM,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,oBAAoB,CAAC,CAAC;AAIlE,eAAO,MAAM,cAAc;;;;;;;;;;;;;;;;;;aA8BzB,CAAC;AAIH,eAAO,MAAM,uBAAuB;;;;;;;;;;;;;;;;;aAkBlC,CAAC;AAEH,MAAM,MAAM,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,WAAW,CAAC,CAAC;AACvD,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,YAAY,CAAC,CAAC;AACzD,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAC;AAE7D,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAC;AAC7D,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,YAAY,CAAC,CAAC;AACzD,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,YAAY,CAAC,CAAC;AACzD,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,YAAY,CAAC,CAAC;AACzD,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAC;AAE7D,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAC;AAE7D,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAC5D,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAC5D,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC;AAC5E,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AACxD,MAAM,MAAM,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB,CAAC,CAAC;AAC9D,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAC1D,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AACxD,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAC5D,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AACxD,MAAM,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC;AAExE,eAAO,MAAM,gBAAgB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;eAM5B,CAAC;AAWF,QAAA,MAAM,oBAAoB,kEAAoC,CAAC;AAE/D,MAAM,MAAM,UAAU,GAAG,SAAS,CAChC,UAAU,EAAE,+CAA+C;AAC3D,CAAC,CAAC,KAAK,CAAC,OAAO,gBAAgB,CAAC,CACjC,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,oBAAoB,CAAC,CAAC;AA6BjE,eAAO,MAAM,kBAAkB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;eAA4B,CAAC;AAE5D,QAAA,MAAM,sBAAsB,0LAAsC,CAAC;AAEnE,MAAM,MAAM,YAAY,GAAG,SAAS,CAClC,UAAU,EACV,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB,CAAC,CACnC,CAAC;AAEF,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,sBAAsB,CAAC,CAAC;AAErE,MAAM,MAAM,kBAAkB,GAAG,UAAU,GAAG,YAAY,CAAC;AAE3D,MAAM,MAAM,MAAM,GACd,YAAY,GACZ,kBAAkB,GAClB,aAAa,GACb,eAAe,CAAC;AAEpB,MAAM,MAAM,SAAS,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;AAItC,wBAAgB,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,IAAI,YAAY,CAErE;AAID,wBAAgB,YAAY,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,IAAI,UAAU,CAEjE"}
+{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../../../../../zero-cache/src/services/change-source/protocol/current/data.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAEL,KAAK,UAAU,EAChB,MAAM,6CAA6C,CAAC;AAErD,OAAO,KAAK,CAAC,MAAM,wCAAwC,CAAC;AAE5D,OAAO,KAAK,EAAC,SAAS,EAAC,MAAM,gCAAgC,CAAC;AAG9D,eAAO,MAAM,WAAW;;;;aAgBtB,CAAC;AAEH,eAAO,MAAM,YAAY;;;aAevB,CAAC;AAEH,eAAO,MAAM,cAAc;;aAEzB,CAAC;AAcH,eAAO,MAAM,cAAc;;;;;;;;;EA2BvB,CAAC;AAGL,eAAO,MAAM,iBAAiB;;;;;;;aAK5B,CAAC;AAUH,eAAO,MAAM,mBAAmB;;2EAER,CAAC;AAEzB,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,mBAAmB,CAAC,CAAC;AAEhE,eAAO,MAAM,SAAS,yFAA4B,CAAC;AAEnD,eAAO,MAAM,YAAY;;;;;;;;;;;;;aAIvB,CAAC;AAEH,eAAO,MAAM,YAAY;;;;;;;;;;;;;;aAUvB,CAAC;AAEH,eAAO,MAAM,YAAY;;;;;;;;;;;;;aAKvB,CAAC;AAEH,eAAO,MAAM,cAAc;;;;;;;;;;;;aAGzB,CAAC;AAEH,eAAO,MAAM,gBAAgB;;;aAG3B,CAAC;AAEH,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAU1D,eAAO,MAAM,gBAAgB,mEAAmB,CAAC;AAEjD,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAE1D,eAAO,MAAM,iBAAiB;;;;;;;;;;;;;;;;;;;;;;aA4B5B,CAAC;AAEH,eAAO,MAAM,iBAAiB;;;;;;;;;;aAI5B,CAAC;AAEH,eAAO,MAAM,yBAAyB;;;;;;;;;;;;aAKpC,CAAC;AAOH,eAAO,MAAM,eAAe;;;;;;;;;;;;;;;;;;;;;;;aAa1B,CAAC;AAEH,eAAO,MAAM,kBAAkB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;aAK7B,CAAC;AAEH,eAAO,MAAM,gBAAgB;;;;;;;aAI3B,CAAC;AAEH,eAAO,MAAM,eAAe;;;;;;aAG1B,CAAC;AAEH,eAAO,MAAM,iBAAiB;;;;;;;;;;aAG5B,CAAC;AAEH,eAAO,MAAM,eAAe;;;;;;aAG1B,CAAC;AAEH,eAAO,MAAM,oBAAoB;;;;aAI/B,CAAC;AAEH,MAAM,MAAM,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,oBAAoB,CAAC,CAAC;AAIlE,eAAO,MAAM,cAAc;;;;;;;;;;;;;;;;;;aA8BzB,CAAC;AAIH,eAAO,MAAM,uBAAuB;;;;;;;;;;;;;;;;;aAkBlC,CAAC;AAEH,MAAM,MAAM,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,WAAW,CAAC,CAAC;AACvD,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,YAAY,CAAC,CAAC;AACzD,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAC;AAE7D,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAC;AAC7D,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,YAAY,CAAC,CAAC;AACzD,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,YAAY,CAAC,CAAC;AACzD,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,YAAY,CAAC,CAAC;AACzD,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAC;AAE7D,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAC;AAE7D,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAC5D,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAC5D,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC;AAC5E,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AACxD,MAAM,MAAM,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB,CAAC,CAAC;AAC9D,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAC1D,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AACxD,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAC5D,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AACxD,MAAM,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC;AAExE,eAAO,MAAM,gBAAgB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;eAM5B,CAAC;AAWF,QAAA,MAAM,oBAAoB,kEAAoC,CAAC;AAE/D,MAAM,MAAM,UAAU,GAAG,SAAS,CAChC,UAAU,EAAE,+CAA+C;AAC3D,CAAC,CAAC,KAAK,CAAC,OAAO,gBAAgB,CAAC,CACjC,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,oBAAoB,CAAC,CAAC;AA6BjE,eAAO,MAAM,kBAAkB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;eAA4B,CAAC;AAE5D,QAAA,MAAM,sBAAsB,0LAAsC,CAAC;AAEnE,MAAM,MAAM,YAAY,GAAG,SAAS,CAClC,UAAU,EACV,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB,CAAC,CACnC,CAAC;AAEF,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,sBAAsB,CAAC,CAAC;AAErE,MAAM,MAAM,kBAAkB,GAAG,UAAU,GAAG,YAAY,CAAC;AAE3D,MAAM,MAAM,MAAM,GACd,YAAY,GACZ,kBAAkB,GAClB,aAAa,GACb,eAAe,CAAC;AAEpB,MAAM,MAAM,SAAS,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;AAItC,wBAAgB,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,IAAI,YAAY,CAErE;AAID,wBAAgB,YAAY,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,IAAI,UAAU,CAEjE"}

package/out/zero-cache/src/services/change-source/protocol/current/data.js

@@ -14,7 +14,10 @@ var beginSchema = valita_exports.object({
 	json: literalUnion("p", "s").optional(),
 	skipAck: valita_exports.boolean().optional()
 });
-var commitSchema = valita_exports.object({ tag: valita_exports.literal("commit") });
+var commitSchema = valita_exports.object({
+	tag: valita_exports.literal("commit"),
+	commitTimeMs: valita_exports.number().optional()
+});
 var rollbackSchema = valita_exports.object({ tag: valita_exports.literal("rollback") });
 var rowKeySchema = valita_exports.object({
 	columns: valita_exports.array(valita_exports.string()),

package/out/zero-cache/src/services/change-source/protocol/current/data.js.map

@@ -1 +1 @@
-{"version":3,"file":"data.js","names":[],"sources":["../../../../../../../../zero-cache/src/services/change-source/protocol/current/data.ts"],"sourcesContent":["/**\n * Data plane messages encapsulate changes that are sent by ChangeSources,\n * forwarded / fanned out to subscribers by the ChangeStreamerService, and\n * stored in the Change DB for catchup of old subscribers.\n */\n\nimport {\n  jsonValueSchema,\n  type JSONObject,\n} from '../../../../../../shared/src/bigint-json.ts';\nimport {must} from '../../../../../../shared/src/must.ts';\nimport * as v from '../../../../../../shared/src/valita.ts';\nimport {columnSpec, indexSpec, tableSpec} from '../../../../db/specs.ts';\nimport type {Satisfies} from '../../../../types/satisfies.ts';\nimport {jsonObjectSchema} from './json.ts';\n\nexport const beginSchema = v.object({\n  tag: v.literal('begin'),\n  // The format of values of \"json\"-typed columns (e.g. \"JSON\" and \"JSONB\").\n  // - 'p' is for parsed JSON, which may include JSON values or JSON objects.\n  //   These values are parsed and stringified at every process boundary\n  //   between the change-source and the replica.\n  // - 's' is for stringified JSON. These values skip the parsing and\n  //   stringification, and are directly ferried to the replica as a JSON\n  //   string. For JSON values this improves performance by 20~25% in the\n  //   change-streamer and 25~30% in the replicator.\n  //\n  // If absent, the format is assumed to be 'p' (parsed JSON objects/values).\n  json: v.literalUnion('p', 's').optional(),\n\n  // Directs the change-streamer to skip the ACK for the corresponding commit.\n  skipAck: v.boolean().optional(),\n});\n\nexport const commitSchema = v.object({\n  tag: v.literal('commit'),\n});\n\nexport const rollbackSchema = v.object({\n  tag: v.literal('rollback'),\n});\n\nconst rowKeySchema = v.object({\n  // The columns used to identify a row in insert, update, and delete changes.\n  columns: v.array(v.string()),\n\n  // An optional qualifier identifying how the key is chosen. Currently this\n  // is postgres-specific, describing the REPLICA IDENTITY, for which replica\n  // identity 'full' (FULL) is handled differently; the replicator handles\n  // these tables by extracting a row key from the full row based on the\n  // table's PRIMARY KEY or UNIQUE INDEX.\n  type: v.literalUnion('default', 'nothing', 'full', 'index').optional(),\n});\n\nexport const relationSchema = v\n  .object({\n    schema: v.string(),\n    name: v.string(),\n\n    // This will become required.\n    rowKey: rowKeySchema.optional(),\n\n    /** Deprecated: set the rowKey.columns instead. */\n    keyColumns: v.array(v.string()).optional(),\n    /** Deprecated: set the rowKey.columns instead. */\n    replicaIdentity: v\n      .literalUnion('default', 'nothing', 'full', 'index')\n      .optional(),\n  })\n  .map(rel => {\n    const {rowKey, ...rest} = rel;\n    if (rowKey) {\n      return {...rest, rowKey};\n    }\n    return {\n      ...rest,\n      rowKey: {\n        columns: must(rel.keyColumns),\n        type: rel.replicaIdentity,\n      },\n    };\n  });\n\n// The eventual fate of relationSchema\nexport const newRelationSchema = v.object({\n  schema: v.string(),\n  name: v.string(),\n\n  rowKey: rowKeySchema,\n});\n\n// TableMetadata contains table-related configuration that does not affect the\n// actual data in the table, but rather how the table's change messages are\n// handled. The is an opaque object that clients must track (and update) based\n// on `create-table`, `add-column`, and `table-update-metadata` messages, and\n// pass in BackfillRequests when there are columns to be backfilled.\n//\n// Note that the backfill-related change-source implementation does, however,\n// rely on the rowKey (columns) being specified in the message.\nexport const tableMetadataSchema = v\n  .object({rowKey: v.record(jsonValueSchema)})\n  .rest(jsonValueSchema);\n\nexport type TableMetadata = v.Infer<typeof tableMetadataSchema>;\n\nexport const rowSchema = v.record(jsonValueSchema);\n\nexport const insertSchema = v.object({\n  tag: v.literal('insert'),\n  relation: relationSchema,\n  new: rowSchema,\n});\n\nexport const updateSchema = v.object({\n  tag: v.literal('update'),\n  relation: relationSchema,\n  // `key` is present if the update changed the key of the row, or if the\n  // table's replicaIdentity === 'full'\n  key: rowSchema.nullable(),\n  // `new` is the full row (and not just the updated columns). This is\n  // necessary for \"catchup\" replication scenarios such as adding tables\n  // to a publication, or resharding.\n  new: rowSchema,\n});\n\nexport const deleteSchema = v.object({\n  tag: v.literal('delete'),\n  relation: relationSchema,\n  // key is the full row if replicaIdentity === 'full'\n  key: rowSchema,\n});\n\nexport const truncateSchema = v.object({\n  tag: v.literal('truncate'),\n  relations: v.array(relationSchema),\n});\n\nexport const identifierSchema = v.object({\n  schema: v.string(),\n  name: v.string(),\n});\n\nexport type Identifier = v.Infer<typeof identifierSchema>;\n\n// A BackfillID is an upstream specific stable identifier for a column\n// that needs backfilling. This id is used to ensure that the schema, table,\n// and column names of a requested backfill still match the original\n// underlying upstream ID.\n//\n// The change-streamer stores these IDs as opaque values while a column is\n// being backfilled, and initiates new change-source streams with the IDs\n// in order to restart backfills that did not complete in previous sessions.\nexport const backfillIDSchema = jsonObjectSchema;\n\nexport type BackfillID = v.Infer<typeof backfillIDSchema>;\n\nexport const createTableSchema = v.object({\n  tag: v.literal('create-table'),\n  spec: tableSpec,\n\n  // This must be set by change source implementations that support\n  // table/column backfill.\n  //\n  // TODO: to simplify the protocol, see if we can make this required\n  metadata: tableMetadataSchema.optional(),\n\n  // Indicate that columns of the table require backfilling. These columns\n  // should be created on the replica but not yet synced the clients.\n  //\n  // ## State Persistence\n  //\n  // To obviate the need for change-source implementations to persist state\n  // related to backfill progress, the change-source only tracks backfills\n  // **for the current session**. In the event that the session is interrupted\n  // before columns have been fully backfilled, it is the responsibility of the\n  // change-streamer to send {@link BackfillRequest}s when it reconnects.\n  //\n  // This means that the change-streamer must track and persist:\n  // * the backfill IDs of the columns requiring backfilling\n  // * the most current table metadata of the associated table(s)\n  //\n  // The change-streamer then uses this information to send backfill requests\n  // when it reconnects.\n  backfill: v.record(backfillIDSchema).optional(),\n});\n\nexport const renameTableSchema = v.object({\n  tag: v.literal('rename-table'),\n  old: identifierSchema,\n  new: identifierSchema,\n});\n\nexport const updateTableMetadataSchema = v.object({\n  tag: v.literal('update-table-metadata'),\n  table: identifierSchema,\n  old: tableMetadataSchema,\n  new: tableMetadataSchema,\n});\n\nconst columnSchema = v.object({\n  name: v.string(),\n  spec: columnSpec,\n});\n\nexport const addColumnSchema = v.object({\n  tag: v.literal('add-column'),\n  table: identifierSchema,\n  column: columnSchema,\n\n  // This must be set by change source implementations that support\n  // table/column backfill.\n  //\n  // TODO: to simplify the protocol, see if we can make this required\n  tableMetadata: tableMetadataSchema.optional(),\n\n  // See documentation for the `backfill` field of the `create-table` change.\n  backfill: backfillIDSchema.optional(),\n});\n\nexport const updateColumnSchema = v.object({\n  tag: v.literal('update-column'),\n  table: identifierSchema,\n  old: columnSchema,\n  new: columnSchema,\n});\n\nexport const dropColumnSchema = v.object({\n  tag: v.literal('drop-column'),\n  table: identifierSchema,\n  column: v.string(),\n});\n\nexport const dropTableSchema = v.object({\n  tag: v.literal('drop-table'),\n  id: identifierSchema,\n});\n\nexport const createIndexSchema = v.object({\n  tag: v.literal('create-index'),\n  spec: indexSpec,\n});\n\nexport const dropIndexSchema = v.object({\n  tag: v.literal('drop-index'),\n  id: identifierSchema,\n});\n\nexport const downloadStatusSchema = v.object({\n  rows: v.number(),\n  totalRows: v.number(),\n  totalBytes: v.number().optional(),\n});\n\nexport type DownloadStatus = v.Infer<typeof downloadStatusSchema>;\n\n// A batch of rows from a single table containing column values\n// to be backfilled.\nexport const backfillSchema = v.object({\n  tag: v.literal('backfill'),\n\n  relation: newRelationSchema,\n\n  // The columns to be backfilled. `rowKey` columns are automatically excluded,\n  // which means that this field may be empty.\n  columns: v.array(v.string()),\n\n  // The watermark at which the backfill data was queried. Note that this\n  // generally will be different from the commit watermarks of the main change\n  // stream, and in particular, the commit watermark of the backfill change's\n  // enclosing transaction.\n  watermark: v.string(),\n\n  // A batch of row values, each row consisting of the `rowKey`\n  // values, followed by the `column` values, in the same order in which\n  // the column names appear in their respective fields, e.g.\n  //\n  // ```\n  // [\n  //   [...rowKeyValues, ...columnValues],  // row 1\n  //   [...rowKeyValues, ...columnValues],  // row 2\n  // ]\n  // ```\n  rowValues: v.array(v.array(jsonValueSchema)),\n\n  // Optionally includes the progress of the backfill operation,\n  // for display purposes.\n  status: downloadStatusSchema.optional(),\n});\n\n// Indicates that the backfill for the specified columns have\n// been successfully backfilled and can be published to clients.\nexport const backfillCompletedSchema = v.object({\n  tag: v.literal('backfill-completed'),\n\n  relation: newRelationSchema,\n\n  // The columns to be backfilled. `rowKey` columns are automatically excluded,\n  // which means that this field may be empty.\n  columns: v.array(v.string()),\n\n  // The watermark at which the backfill data was queried. Note that this\n  // generally will be different from the commit watermarks of the main change\n  // stream, and in particular, the commit watermark of the backfill change's\n  // enclosing transaction.\n  watermark: v.string(),\n\n  // Optionally includes the final status of the backfill operation,\n  // for display purposes.\n  status: downloadStatusSchema.optional(),\n});\n\nexport type MessageBegin = v.Infer<typeof beginSchema>;\nexport type MessageCommit = v.Infer<typeof commitSchema>;\nexport type MessageRollback = v.Infer<typeof rollbackSchema>;\n\nexport type MessageRelation = v.Infer<typeof relationSchema>;\nexport type MessageInsert = v.Infer<typeof insertSchema>;\nexport type MessageUpdate = v.Infer<typeof updateSchema>;\nexport type MessageDelete = v.Infer<typeof deleteSchema>;\nexport type MessageTruncate = v.Infer<typeof truncateSchema>;\n\nexport type MessageBackfill = v.Infer<typeof backfillSchema>;\n\nexport type TableCreate = v.Infer<typeof createTableSchema>;\nexport type TableRename = v.Infer<typeof renameTableSchema>;\nexport type TableUpdateMetadata = v.Infer<typeof updateTableMetadataSchema>;\nexport type ColumnAdd = v.Infer<typeof addColumnSchema>;\nexport type ColumnUpdate = v.Infer<typeof updateColumnSchema>;\nexport type ColumnDrop = v.Infer<typeof dropColumnSchema>;\nexport type TableDrop = v.Infer<typeof dropTableSchema>;\nexport type IndexCreate = v.Infer<typeof createIndexSchema>;\nexport type IndexDrop = v.Infer<typeof dropIndexSchema>;\nexport type BackfillCompleted = v.Infer<typeof backfillCompletedSchema>;\n\nexport const dataChangeSchema = v.union(\n  insertSchema,\n  updateSchema,\n  deleteSchema,\n  truncateSchema,\n  backfillSchema,\n);\n\n// Note: keep in sync or the tag tests will fail\nconst dataChangeTags = [\n  'insert',\n  'update',\n  'delete',\n  'truncate',\n  'backfill',\n] as const;\n\nconst dataChangeTagsSchema = v.literalUnion(...dataChangeTags);\n\nexport type DataChange = Satisfies<\n  JSONObject, // guarantees serialization over IPC or network\n  v.Infer<typeof dataChangeSchema>\n>;\n\nexport type DataChangeTag = v.Infer<typeof dataChangeTagsSchema>;\n\nconst schemaChanges = [\n  createTableSchema,\n  renameTableSchema,\n  updateTableMetadataSchema,\n  addColumnSchema,\n  updateColumnSchema,\n  dropColumnSchema,\n  dropTableSchema,\n  createIndexSchema,\n  dropIndexSchema,\n  backfillCompletedSchema,\n] as const;\n\n// Note: keep in sync or the tag tests will fail\nconst schemaChangeTags = [\n  'create-table',\n  'rename-table',\n  'update-table-metadata',\n  'add-column',\n  'update-column',\n  'drop-column',\n  'drop-table',\n  'create-index',\n  'drop-index',\n  'backfill-completed',\n] as const;\n\nexport const schemaChangeSchema = v.union(...schemaChanges);\n\nconst schemaChangeTagsSchema = v.literalUnion(...schemaChangeTags);\n\nexport type SchemaChange = Satisfies<\n  JSONObject,\n  v.Infer<typeof schemaChangeSchema>\n>;\n\nexport type SchemaChangeTag = v.Infer<typeof schemaChangeTagsSchema>;\n\nexport type DataOrSchemaChange = DataChange | SchemaChange;\n\nexport type Change =\n  | MessageBegin\n  | DataOrSchemaChange\n  | MessageCommit\n  | MessageRollback;\n\nexport type ChangeTag = Change['tag'];\n\nconst schemaChangeTagSet = new Set<string>(schemaChangeTags);\n\nexport function isSchemaChange(change: Change): change is SchemaChange {\n  return schemaChangeTagSet.has(change.tag);\n}\n\nconst dataChangeTagSet = new Set<string>(dataChangeTags);\n\nexport function isDataChange(change: Change): change is DataChange {\n  return dataChangeTagSet.has(change.tag);\n}\n"],"mappings":";;;;;;;;;;;AAgBA,IAAa,cAAc,eAAE,OAAO;CAClC,KAAK,eAAE,QAAQ,OAAO;CAWtB,MAAM,aAAe,KAAK,GAAG,EAAE,SAAS;CAGxC,SAAS,eAAE,QAAQ,EAAE,SAAS;AAChC,CAAC;AAED,IAAa,eAAe,eAAE,OAAO,EACnC,KAAK,eAAE,QAAQ,QAAQ,EACzB,CAAC;AAED,IAAa,iBAAiB,eAAE,OAAO,EACrC,KAAK,eAAE,QAAQ,UAAU,EAC3B,CAAC;AAED,IAAM,eAAe,eAAE,OAAO;CAE5B,SAAS,eAAE,MAAM,eAAE,OAAO,CAAC;CAO3B,MAAM,aAAe,WAAW,WAAW,QAAQ,OAAO,EAAE,SAAS;AACvE,CAAC;AAED,IAAa,iBAAiB,eAC3B,OAAO;CACN,QAAQ,eAAE,OAAO;CACjB,MAAM,eAAE,OAAO;CAGf,QAAQ,aAAa,SAAS;;CAG9B,YAAY,eAAE,MAAM,eAAE,OAAO,CAAC,EAAE,SAAS;;CAEzC,iBAAiB,aACD,WAAW,WAAW,QAAQ,OAAO,EAClD,SAAS;AACd,CAAC,EACA,KAAI,QAAO;CACV,MAAM,EAAC,QAAQ,GAAG,SAAQ;CAC1B,IAAI,QACF,OAAO;EAAC,GAAG;EAAM;CAAM;CAEzB,OAAO;EACL,GAAG;EACH,QAAQ;GACN,SAAS,KAAK,IAAI,UAAU;GAC5B,MAAM,IAAI;EACZ;CACF;AACF,CAAC;AAGH,IAAa,oBAAoB,eAAE,OAAO;CACxC,QAAQ,eAAE,OAAO;CACjB,MAAM,eAAE,OAAO;CAEf,QAAQ;AACV,CAAC;AAUD,IAAa,sBAAsB,eAChC,OAAO,EAAC,QAAQ,eAAE,OAAO,eAAe,EAAC,CAAC,EAC1C,KAAK,eAAe;AAIvB,IAAa,YAAY,eAAE,OAAO,eAAe;AAEjD,IAAa,eAAe,eAAE,OAAO;CACnC,KAAK,eAAE,QAAQ,QAAQ;CACvB,UAAU;CACV,KAAK;AACP,CAAC;AAED,IAAa,eAAe,eAAE,OAAO;CACnC,KAAK,eAAE,QAAQ,QAAQ;CACvB,UAAU;CAGV,KAAK,UAAU,SAAS;CAIxB,KAAK;AACP,CAAC;AAED,IAAa,eAAe,eAAE,OAAO;CACnC,KAAK,eAAE,QAAQ,QAAQ;CACvB,UAAU;CAEV,KAAK;AACP,CAAC;AAED,IAAa,iBAAiB,eAAE,OAAO;CACrC,KAAK,eAAE,QAAQ,UAAU;CACzB,WAAW,eAAE,MAAM,cAAc;AACnC,CAAC;AAED,IAAa,mBAAmB,eAAE,OAAO;CACvC,QAAQ,eAAE,OAAO;CACjB,MAAM,eAAE,OAAO;AACjB,CAAC;AAYD,IAAa,mBAAmB;AAIhC,IAAa,oBAAoB,eAAE,OAAO;CACxC,KAAK,eAAE,QAAQ,cAAc;CAC7B,MAAM;CAMN,UAAU,oBAAoB,SAAS;CAmBvC,UAAU,eAAE,OAAO,gBAAgB,EAAE,SAAS;AAChD,CAAC;AAED,IAAa,oBAAoB,eAAE,OAAO;CACxC,KAAK,eAAE,QAAQ,cAAc;CAC7B,KAAK;CACL,KAAK;AACP,CAAC;AAED,IAAa,4BAA4B,eAAE,OAAO;CAChD,KAAK,eAAE,QAAQ,uBAAuB;CACtC,OAAO;CACP,KAAK;CACL,KAAK;AACP,CAAC;AAED,IAAM,eAAe,eAAE,OAAO;CAC5B,MAAM,eAAE,OAAO;CACf,MAAM;AACR,CAAC;AAED,IAAa,kBAAkB,eAAE,OAAO;CACtC,KAAK,eAAE,QAAQ,YAAY;CAC3B,OAAO;CACP,QAAQ;CAMR,eAAe,oBAAoB,SAAS;CAG5C,UAAU,iBAAiB,SAAS;AACtC,CAAC;AAED,IAAa,qBAAqB,eAAE,OAAO;CACzC,KAAK,eAAE,QAAQ,eAAe;CAC9B,OAAO;CACP,KAAK;CACL,KAAK;AACP,CAAC;AAED,IAAa,mBAAmB,eAAE,OAAO;CACvC,KAAK,eAAE,QAAQ,aAAa;CAC5B,OAAO;CACP,QAAQ,eAAE,OAAO;AACnB,CAAC;AAED,IAAa,kBAAkB,eAAE,OAAO;CACtC,KAAK,eAAE,QAAQ,YAAY;CAC3B,IAAI;AACN,CAAC;AAED,IAAa,oBAAoB,eAAE,OAAO;CACxC,KAAK,eAAE,QAAQ,cAAc;CAC7B,MAAM;AACR,CAAC;AAED,IAAa,kBAAkB,eAAE,OAAO;CACtC,KAAK,eAAE,QAAQ,YAAY;CAC3B,IAAI;AACN,CAAC;AAED,IAAa,uBAAuB,eAAE,OAAO;CAC3C,MAAM,eAAE,OAAO;CACf,WAAW,eAAE,OAAO;CACpB,YAAY,eAAE,OAAO,EAAE,SAAS;AAClC,CAAC;AAMD,IAAa,iBAAiB,eAAE,OAAO;CACrC,KAAK,eAAE,QAAQ,UAAU;CAEzB,UAAU;CAIV,SAAS,eAAE,MAAM,eAAE,OAAO,CAAC;CAM3B,WAAW,eAAE,OAAO;CAYpB,WAAW,eAAE,MAAM,eAAE,MAAM,eAAe,CAAC;CAI3C,QAAQ,qBAAqB,SAAS;AACxC,CAAC;AAID,IAAa,0BAA0B,eAAE,OAAO;CAC9C,KAAK,eAAE,QAAQ,oBAAoB;CAEnC,UAAU;CAIV,SAAS,eAAE,MAAM,eAAE,OAAO,CAAC;CAM3B,WAAW,eAAE,OAAO;CAIpB,QAAQ,qBAAqB,SAAS;AACxC,CAAC;AAyBD,IAAa,mBAAmB,eAAE,MAChC,cACA,cACA,cACA,gBACA,cACF;AAGA,IAAM,iBAAiB;CACrB;CACA;CACA;CACA;CACA;AACF;AAE6B,aAAe,GAAG,cAAc;AAS7D,IAAM,gBAAgB;CACpB;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;AACF;AAGA,IAAM,mBAAmB;CACvB;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;AACF;AAEA,IAAa,qBAAqB,eAAE,MAAM,GAAG,aAAa;AAE3B,aAAe,GAAG,gBAAgB;AAmBjE,IAAM,qBAAqB,IAAI,IAAY,gBAAgB;AAE3D,SAAgB,eAAe,QAAwC;CACrE,OAAO,mBAAmB,IAAI,OAAO,GAAG;AAC1C;AAEA,IAAM,mBAAmB,IAAI,IAAY,cAAc;AAEvD,SAAgB,aAAa,QAAsC;CACjE,OAAO,iBAAiB,IAAI,OAAO,GAAG;AACxC"}
+{"version":3,"file":"data.js","names":[],"sources":["../../../../../../../../zero-cache/src/services/change-source/protocol/current/data.ts"],"sourcesContent":["/**\n * Data plane messages encapsulate changes that are sent by ChangeSources,\n * forwarded / fanned out to subscribers by the ChangeStreamerService, and\n * stored in the Change DB for catchup of old subscribers.\n */\n\nimport {\n  jsonValueSchema,\n  type JSONObject,\n} from '../../../../../../shared/src/bigint-json.ts';\nimport {must} from '../../../../../../shared/src/must.ts';\nimport * as v from '../../../../../../shared/src/valita.ts';\nimport {columnSpec, indexSpec, tableSpec} from '../../../../db/specs.ts';\nimport type {Satisfies} from '../../../../types/satisfies.ts';\nimport {jsonObjectSchema} from './json.ts';\n\nexport const beginSchema = v.object({\n  tag: v.literal('begin'),\n  // The format of values of \"json\"-typed columns (e.g. \"JSON\" and \"JSONB\").\n  // - 'p' is for parsed JSON, which may include JSON values or JSON objects.\n  //   These values are parsed and stringified at every process boundary\n  //   between the change-source and the replica.\n  // - 's' is for stringified JSON. These values skip the parsing and\n  //   stringification, and are directly ferried to the replica as a JSON\n  //   string. For JSON values this improves performance by 20~25% in the\n  //   change-streamer and 25~30% in the replicator.\n  //\n  // If absent, the format is assumed to be 'p' (parsed JSON objects/values).\n  json: v.literalUnion('p', 's').optional(),\n\n  // Directs the change-streamer to skip the ACK for the corresponding commit.\n  skipAck: v.boolean().optional(),\n});\n\nexport const commitSchema = v.object({\n  tag: v.literal('commit'),\n\n  // Millisecond epoch at which the transaction committed upstream, as reported\n  // by the upstream database (e.g. the `commitTime` of the Postgres logical\n  // replication Commit message).\n  //\n  // This is the origin timestamp of the end-to-end serving lag measurement:\n  // it is carried through the replication stream to the ViewSyncer, which\n  // records `now - commitTimeMs` once the change has been poked to clients.\n  //\n  // Optional because it is absent from changes written to the Change DB by\n  // older versions, which are replayed verbatim during catchup, and from\n  // ChangeSources that do not report a commit time.\n  commitTimeMs: v.number().optional(),\n});\n\nexport const rollbackSchema = v.object({\n  tag: v.literal('rollback'),\n});\n\nconst rowKeySchema = v.object({\n  // The columns used to identify a row in insert, update, and delete changes.\n  columns: v.array(v.string()),\n\n  // An optional qualifier identifying how the key is chosen. Currently this\n  // is postgres-specific, describing the REPLICA IDENTITY, for which replica\n  // identity 'full' (FULL) is handled differently; the replicator handles\n  // these tables by extracting a row key from the full row based on the\n  // table's PRIMARY KEY or UNIQUE INDEX.\n  type: v.literalUnion('default', 'nothing', 'full', 'index').optional(),\n});\n\nexport const relationSchema = v\n  .object({\n    schema: v.string(),\n    name: v.string(),\n\n    // This will become required.\n    rowKey: rowKeySchema.optional(),\n\n    /** Deprecated: set the rowKey.columns instead. */\n    keyColumns: v.array(v.string()).optional(),\n    /** Deprecated: set the rowKey.columns instead. */\n    replicaIdentity: v\n      .literalUnion('default', 'nothing', 'full', 'index')\n      .optional(),\n  })\n  .map(rel => {\n    const {rowKey, ...rest} = rel;\n    if (rowKey) {\n      return {...rest, rowKey};\n    }\n    return {\n      ...rest,\n      rowKey: {\n        columns: must(rel.keyColumns),\n        type: rel.replicaIdentity,\n      },\n    };\n  });\n\n// The eventual fate of relationSchema\nexport const newRelationSchema = v.object({\n  schema: v.string(),\n  name: v.string(),\n\n  rowKey: rowKeySchema,\n});\n\n// TableMetadata contains table-related configuration that does not affect the\n// actual data in the table, but rather how the table's change messages are\n// handled. The is an opaque object that clients must track (and update) based\n// on `create-table`, `add-column`, and `table-update-metadata` messages, and\n// pass in BackfillRequests when there are columns to be backfilled.\n//\n// Note that the backfill-related change-source implementation does, however,\n// rely on the rowKey (columns) being specified in the message.\nexport const tableMetadataSchema = v\n  .object({rowKey: v.record(jsonValueSchema)})\n  .rest(jsonValueSchema);\n\nexport type TableMetadata = v.Infer<typeof tableMetadataSchema>;\n\nexport const rowSchema = v.record(jsonValueSchema);\n\nexport const insertSchema = v.object({\n  tag: v.literal('insert'),\n  relation: relationSchema,\n  new: rowSchema,\n});\n\nexport const updateSchema = v.object({\n  tag: v.literal('update'),\n  relation: relationSchema,\n  // `key` is present if the update changed the key of the row, or if the\n  // table's replicaIdentity === 'full'\n  key: rowSchema.nullable(),\n  // `new` is the full row (and not just the updated columns). This is\n  // necessary for \"catchup\" replication scenarios such as adding tables\n  // to a publication, or resharding.\n  new: rowSchema,\n});\n\nexport const deleteSchema = v.object({\n  tag: v.literal('delete'),\n  relation: relationSchema,\n  // key is the full row if replicaIdentity === 'full'\n  key: rowSchema,\n});\n\nexport const truncateSchema = v.object({\n  tag: v.literal('truncate'),\n  relations: v.array(relationSchema),\n});\n\nexport const identifierSchema = v.object({\n  schema: v.string(),\n  name: v.string(),\n});\n\nexport type Identifier = v.Infer<typeof identifierSchema>;\n\n// A BackfillID is an upstream specific stable identifier for a column\n// that needs backfilling. This id is used to ensure that the schema, table,\n// and column names of a requested backfill still match the original\n// underlying upstream ID.\n//\n// The change-streamer stores these IDs as opaque values while a column is\n// being backfilled, and initiates new change-source streams with the IDs\n// in order to restart backfills that did not complete in previous sessions.\nexport const backfillIDSchema = jsonObjectSchema;\n\nexport type BackfillID = v.Infer<typeof backfillIDSchema>;\n\nexport const createTableSchema = v.object({\n  tag: v.literal('create-table'),\n  spec: tableSpec,\n\n  // This must be set by change source implementations that support\n  // table/column backfill.\n  //\n  // TODO: to simplify the protocol, see if we can make this required\n  metadata: tableMetadataSchema.optional(),\n\n  // Indicate that columns of the table require backfilling. These columns\n  // should be created on the replica but not yet synced the clients.\n  //\n  // ## State Persistence\n  //\n  // To obviate the need for change-source implementations to persist state\n  // related to backfill progress, the change-source only tracks backfills\n  // **for the current session**. In the event that the session is interrupted\n  // before columns have been fully backfilled, it is the responsibility of the\n  // change-streamer to send {@link BackfillRequest}s when it reconnects.\n  //\n  // This means that the change-streamer must track and persist:\n  // * the backfill IDs of the columns requiring backfilling\n  // * the most current table metadata of the associated table(s)\n  //\n  // The change-streamer then uses this information to send backfill requests\n  // when it reconnects.\n  backfill: v.record(backfillIDSchema).optional(),\n});\n\nexport const renameTableSchema = v.object({\n  tag: v.literal('rename-table'),\n  old: identifierSchema,\n  new: identifierSchema,\n});\n\nexport const updateTableMetadataSchema = v.object({\n  tag: v.literal('update-table-metadata'),\n  table: identifierSchema,\n  old: tableMetadataSchema,\n  new: tableMetadataSchema,\n});\n\nconst columnSchema = v.object({\n  name: v.string(),\n  spec: columnSpec,\n});\n\nexport const addColumnSchema = v.object({\n  tag: v.literal('add-column'),\n  table: identifierSchema,\n  column: columnSchema,\n\n  // This must be set by change source implementations that support\n  // table/column backfill.\n  //\n  // TODO: to simplify the protocol, see if we can make this required\n  tableMetadata: tableMetadataSchema.optional(),\n\n  // See documentation for the `backfill` field of the `create-table` change.\n  backfill: backfillIDSchema.optional(),\n});\n\nexport const updateColumnSchema = v.object({\n  tag: v.literal('update-column'),\n  table: identifierSchema,\n  old: columnSchema,\n  new: columnSchema,\n});\n\nexport const dropColumnSchema = v.object({\n  tag: v.literal('drop-column'),\n  table: identifierSchema,\n  column: v.string(),\n});\n\nexport const dropTableSchema = v.object({\n  tag: v.literal('drop-table'),\n  id: identifierSchema,\n});\n\nexport const createIndexSchema = v.object({\n  tag: v.literal('create-index'),\n  spec: indexSpec,\n});\n\nexport const dropIndexSchema = v.object({\n  tag: v.literal('drop-index'),\n  id: identifierSchema,\n});\n\nexport const downloadStatusSchema = v.object({\n  rows: v.number(),\n  totalRows: v.number(),\n  totalBytes: v.number().optional(),\n});\n\nexport type DownloadStatus = v.Infer<typeof downloadStatusSchema>;\n\n// A batch of rows from a single table containing column values\n// to be backfilled.\nexport const backfillSchema = v.object({\n  tag: v.literal('backfill'),\n\n  relation: newRelationSchema,\n\n  // The columns to be backfilled. `rowKey` columns are automatically excluded,\n  // which means that this field may be empty.\n  columns: v.array(v.string()),\n\n  // The watermark at which the backfill data was queried. Note that this\n  // generally will be different from the commit watermarks of the main change\n  // stream, and in particular, the commit watermark of the backfill change's\n  // enclosing transaction.\n  watermark: v.string(),\n\n  // A batch of row values, each row consisting of the `rowKey`\n  // values, followed by the `column` values, in the same order in which\n  // the column names appear in their respective fields, e.g.\n  //\n  // ```\n  // [\n  //   [...rowKeyValues, ...columnValues],  // row 1\n  //   [...rowKeyValues, ...columnValues],  // row 2\n  // ]\n  // ```\n  rowValues: v.array(v.array(jsonValueSchema)),\n\n  // Optionally includes the progress of the backfill operation,\n  // for display purposes.\n  status: downloadStatusSchema.optional(),\n});\n\n// Indicates that the backfill for the specified columns have\n// been successfully backfilled and can be published to clients.\nexport const backfillCompletedSchema = v.object({\n  tag: v.literal('backfill-completed'),\n\n  relation: newRelationSchema,\n\n  // The columns to be backfilled. `rowKey` columns are automatically excluded,\n  // which means that this field may be empty.\n  columns: v.array(v.string()),\n\n  // The watermark at which the backfill data was queried. Note that this\n  // generally will be different from the commit watermarks of the main change\n  // stream, and in particular, the commit watermark of the backfill change's\n  // enclosing transaction.\n  watermark: v.string(),\n\n  // Optionally includes the final status of the backfill operation,\n  // for display purposes.\n  status: downloadStatusSchema.optional(),\n});\n\nexport type MessageBegin = v.Infer<typeof beginSchema>;\nexport type MessageCommit = v.Infer<typeof commitSchema>;\nexport type MessageRollback = v.Infer<typeof rollbackSchema>;\n\nexport type MessageRelation = v.Infer<typeof relationSchema>;\nexport type MessageInsert = v.Infer<typeof insertSchema>;\nexport type MessageUpdate = v.Infer<typeof updateSchema>;\nexport type MessageDelete = v.Infer<typeof deleteSchema>;\nexport type MessageTruncate = v.Infer<typeof truncateSchema>;\n\nexport type MessageBackfill = v.Infer<typeof backfillSchema>;\n\nexport type TableCreate = v.Infer<typeof createTableSchema>;\nexport type TableRename = v.Infer<typeof renameTableSchema>;\nexport type TableUpdateMetadata = v.Infer<typeof updateTableMetadataSchema>;\nexport type ColumnAdd = v.Infer<typeof addColumnSchema>;\nexport type ColumnUpdate = v.Infer<typeof updateColumnSchema>;\nexport type ColumnDrop = v.Infer<typeof dropColumnSchema>;\nexport type TableDrop = v.Infer<typeof dropTableSchema>;\nexport type IndexCreate = v.Infer<typeof createIndexSchema>;\nexport type IndexDrop = v.Infer<typeof dropIndexSchema>;\nexport type BackfillCompleted = v.Infer<typeof backfillCompletedSchema>;\n\nexport const dataChangeSchema = v.union(\n  insertSchema,\n  updateSchema,\n  deleteSchema,\n  truncateSchema,\n  backfillSchema,\n);\n\n// Note: keep in sync or the tag tests will fail\nconst dataChangeTags = [\n  'insert',\n  'update',\n  'delete',\n  'truncate',\n  'backfill',\n] as const;\n\nconst dataChangeTagsSchema = v.literalUnion(...dataChangeTags);\n\nexport type DataChange = Satisfies<\n  JSONObject, // guarantees serialization over IPC or network\n  v.Infer<typeof dataChangeSchema>\n>;\n\nexport type DataChangeTag = v.Infer<typeof dataChangeTagsSchema>;\n\nconst schemaChanges = [\n  createTableSchema,\n  renameTableSchema,\n  updateTableMetadataSchema,\n  addColumnSchema,\n  updateColumnSchema,\n  dropColumnSchema,\n  dropTableSchema,\n  createIndexSchema,\n  dropIndexSchema,\n  backfillCompletedSchema,\n] as const;\n\n// Note: keep in sync or the tag tests will fail\nconst schemaChangeTags = [\n  'create-table',\n  'rename-table',\n  'update-table-metadata',\n  'add-column',\n  'update-column',\n  'drop-column',\n  'drop-table',\n  'create-index',\n  'drop-index',\n  'backfill-completed',\n] as const;\n\nexport const schemaChangeSchema = v.union(...schemaChanges);\n\nconst schemaChangeTagsSchema = v.literalUnion(...schemaChangeTags);\n\nexport type SchemaChange = Satisfies<\n  JSONObject,\n  v.Infer<typeof schemaChangeSchema>\n>;\n\nexport type SchemaChangeTag = v.Infer<typeof schemaChangeTagsSchema>;\n\nexport type DataOrSchemaChange = DataChange | SchemaChange;\n\nexport type Change =\n  | MessageBegin\n  | DataOrSchemaChange\n  | MessageCommit\n  | MessageRollback;\n\nexport type ChangeTag = Change['tag'];\n\nconst schemaChangeTagSet = new Set<string>(schemaChangeTags);\n\nexport function isSchemaChange(change: Change): change is SchemaChange {\n  return schemaChangeTagSet.has(change.tag);\n}\n\nconst dataChangeTagSet = new Set<string>(dataChangeTags);\n\nexport function isDataChange(change: Change): change is DataChange {\n  return dataChangeTagSet.has(change.tag);\n}\n"],"mappings":";;;;;;;;;;;AAgBA,IAAa,cAAc,eAAE,OAAO;CAClC,KAAK,eAAE,QAAQ,OAAO;CAWtB,MAAM,aAAe,KAAK,GAAG,EAAE,SAAS;CAGxC,SAAS,eAAE,QAAQ,EAAE,SAAS;AAChC,CAAC;AAED,IAAa,eAAe,eAAE,OAAO;CACnC,KAAK,eAAE,QAAQ,QAAQ;CAavB,cAAc,eAAE,OAAO,EAAE,SAAS;AACpC,CAAC;AAED,IAAa,iBAAiB,eAAE,OAAO,EACrC,KAAK,eAAE,QAAQ,UAAU,EAC3B,CAAC;AAED,IAAM,eAAe,eAAE,OAAO;CAE5B,SAAS,eAAE,MAAM,eAAE,OAAO,CAAC;CAO3B,MAAM,aAAe,WAAW,WAAW,QAAQ,OAAO,EAAE,SAAS;AACvE,CAAC;AAED,IAAa,iBAAiB,eAC3B,OAAO;CACN,QAAQ,eAAE,OAAO;CACjB,MAAM,eAAE,OAAO;CAGf,QAAQ,aAAa,SAAS;;CAG9B,YAAY,eAAE,MAAM,eAAE,OAAO,CAAC,EAAE,SAAS;;CAEzC,iBAAiB,aACD,WAAW,WAAW,QAAQ,OAAO,EAClD,SAAS;AACd,CAAC,EACA,KAAI,QAAO;CACV,MAAM,EAAC,QAAQ,GAAG,SAAQ;CAC1B,IAAI,QACF,OAAO;EAAC,GAAG;EAAM;CAAM;CAEzB,OAAO;EACL,GAAG;EACH,QAAQ;GACN,SAAS,KAAK,IAAI,UAAU;GAC5B,MAAM,IAAI;EACZ;CACF;AACF,CAAC;AAGH,IAAa,oBAAoB,eAAE,OAAO;CACxC,QAAQ,eAAE,OAAO;CACjB,MAAM,eAAE,OAAO;CAEf,QAAQ;AACV,CAAC;AAUD,IAAa,sBAAsB,eAChC,OAAO,EAAC,QAAQ,eAAE,OAAO,eAAe,EAAC,CAAC,EAC1C,KAAK,eAAe;AAIvB,IAAa,YAAY,eAAE,OAAO,eAAe;AAEjD,IAAa,eAAe,eAAE,OAAO;CACnC,KAAK,eAAE,QAAQ,QAAQ;CACvB,UAAU;CACV,KAAK;AACP,CAAC;AAED,IAAa,eAAe,eAAE,OAAO;CACnC,KAAK,eAAE,QAAQ,QAAQ;CACvB,UAAU;CAGV,KAAK,UAAU,SAAS;CAIxB,KAAK;AACP,CAAC;AAED,IAAa,eAAe,eAAE,OAAO;CACnC,KAAK,eAAE,QAAQ,QAAQ;CACvB,UAAU;CAEV,KAAK;AACP,CAAC;AAED,IAAa,iBAAiB,eAAE,OAAO;CACrC,KAAK,eAAE,QAAQ,UAAU;CACzB,WAAW,eAAE,MAAM,cAAc;AACnC,CAAC;AAED,IAAa,mBAAmB,eAAE,OAAO;CACvC,QAAQ,eAAE,OAAO;CACjB,MAAM,eAAE,OAAO;AACjB,CAAC;AAYD,IAAa,mBAAmB;AAIhC,IAAa,oBAAoB,eAAE,OAAO;CACxC,KAAK,eAAE,QAAQ,cAAc;CAC7B,MAAM;CAMN,UAAU,oBAAoB,SAAS;CAmBvC,UAAU,eAAE,OAAO,gBAAgB,EAAE,SAAS;AAChD,CAAC;AAED,IAAa,oBAAoB,eAAE,OAAO;CACxC,KAAK,eAAE,QAAQ,cAAc;CAC7B,KAAK;CACL,KAAK;AACP,CAAC;AAED,IAAa,4BAA4B,eAAE,OAAO;CAChD,KAAK,eAAE,QAAQ,uBAAuB;CACtC,OAAO;CACP,KAAK;CACL,KAAK;AACP,CAAC;AAED,IAAM,eAAe,eAAE,OAAO;CAC5B,MAAM,eAAE,OAAO;CACf,MAAM;AACR,CAAC;AAED,IAAa,kBAAkB,eAAE,OAAO;CACtC,KAAK,eAAE,QAAQ,YAAY;CAC3B,OAAO;CACP,QAAQ;CAMR,eAAe,oBAAoB,SAAS;CAG5C,UAAU,iBAAiB,SAAS;AACtC,CAAC;AAED,IAAa,qBAAqB,eAAE,OAAO;CACzC,KAAK,eAAE,QAAQ,eAAe;CAC9B,OAAO;CACP,KAAK;CACL,KAAK;AACP,CAAC;AAED,IAAa,mBAAmB,eAAE,OAAO;CACvC,KAAK,eAAE,QAAQ,aAAa;CAC5B,OAAO;CACP,QAAQ,eAAE,OAAO;AACnB,CAAC;AAED,IAAa,kBAAkB,eAAE,OAAO;CACtC,KAAK,eAAE,QAAQ,YAAY;CAC3B,IAAI;AACN,CAAC;AAED,IAAa,oBAAoB,eAAE,OAAO;CACxC,KAAK,eAAE,QAAQ,cAAc;CAC7B,MAAM;AACR,CAAC;AAED,IAAa,kBAAkB,eAAE,OAAO;CACtC,KAAK,eAAE,QAAQ,YAAY;CAC3B,IAAI;AACN,CAAC;AAED,IAAa,uBAAuB,eAAE,OAAO;CAC3C,MAAM,eAAE,OAAO;CACf,WAAW,eAAE,OAAO;CACpB,YAAY,eAAE,OAAO,EAAE,SAAS;AAClC,CAAC;AAMD,IAAa,iBAAiB,eAAE,OAAO;CACrC,KAAK,eAAE,QAAQ,UAAU;CAEzB,UAAU;CAIV,SAAS,eAAE,MAAM,eAAE,OAAO,CAAC;CAM3B,WAAW,eAAE,OAAO;CAYpB,WAAW,eAAE,MAAM,eAAE,MAAM,eAAe,CAAC;CAI3C,QAAQ,qBAAqB,SAAS;AACxC,CAAC;AAID,IAAa,0BAA0B,eAAE,OAAO;CAC9C,KAAK,eAAE,QAAQ,oBAAoB;CAEnC,UAAU;CAIV,SAAS,eAAE,MAAM,eAAE,OAAO,CAAC;CAM3B,WAAW,eAAE,OAAO;CAIpB,QAAQ,qBAAqB,SAAS;AACxC,CAAC;AAyBD,IAAa,mBAAmB,eAAE,MAChC,cACA,cACA,cACA,gBACA,cACF;AAGA,IAAM,iBAAiB;CACrB;CACA;CACA;CACA;CACA;AACF;AAE6B,aAAe,GAAG,cAAc;AAS7D,IAAM,gBAAgB;CACpB;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;AACF;AAGA,IAAM,mBAAmB;CACvB;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;AACF;AAEA,IAAa,qBAAqB,eAAE,MAAM,GAAG,aAAa;AAE3B,aAAe,GAAG,gBAAgB;AAmBjE,IAAM,qBAAqB,IAAI,IAAY,gBAAgB;AAE3D,SAAgB,eAAe,QAAwC;CACrE,OAAO,mBAAmB,IAAI,OAAO,GAAG;AAC1C;AAEA,IAAM,mBAAmB,IAAI,IAAY,cAAc;AAEvD,SAAgB,aAAa,QAAsC;CACjE,OAAO,iBAAiB,IAAI,OAAO,GAAG;AACxC"}

package/out/zero-cache/src/services/change-source/protocol/current/downstream.d.ts

@@ -222,6 +222,7 @@ declare const data: v.TupleType<[v.Type<"data">, v.UnionType<[v.UnionType<[v.Obj
 }, undefined>]>]>]>;
 declare const commit: v.TupleType<[v.Type<"commit">, v.ObjectType<{
     tag: v.Type<"commit">;
+    commitTimeMs: v.Optional<number>;
 }, undefined>, v.ObjectType<{
     watermark: v.Type<string>;
 }, undefined>]>;
@@ -453,6 +454,7 @@ export declare const changeStreamDataSchema: v.UnionType<[v.TupleType<[v.Type<"b
     }>;
 }, undefined>]>]>]>, v.TupleType<[v.Type<"commit">, v.ObjectType<{
     tag: v.Type<"commit">;
+    commitTimeMs: v.Optional<number>;
 }, undefined>, v.ObjectType<{
     watermark: v.Type<string>;
 }, undefined>]>, v.TupleType<[v.Type<"rollback">, v.ObjectType<{
@@ -687,6 +689,7 @@ export declare const changeStreamMessageSchema: v.UnionType<[v.UnionType<[v.Tupl
     }>;
 }, undefined>]>]>]>, v.TupleType<[v.Type<"commit">, v.ObjectType<{
     tag: v.Type<"commit">;
+    commitTimeMs: v.Optional<number>;
 }, undefined>, v.ObjectType<{
     watermark: v.Type<string>;
 }, undefined>]>, v.TupleType<[v.Type<"rollback">, v.ObjectType<{

package/out/zero-cache/src/services/change-source/protocol/current/downstream.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"downstream.d.ts","sourceRoot":"","sources":["../../../../../../../../zero-cache/src/services/change-source/protocol/current/downstream.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,CAAC,MAAM,wCAAwC,CAAC;AAW5D,QAAA,MAAM,KAAK;;;;;;eAIT,CAAC;AACH,QAAA,MAAM,IAAI;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;mBAGR,CAAC;AACH,QAAA,MAAM,MAAM;;;;eAIV,CAAC;AACH,QAAA,MAAM,QAAQ;;eAAmD,CAAC;AAElE,MAAM,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,KAAK,CAAC,CAAC;AAC1C,MAAM,MAAM,IAAI,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,IAAI,CAAC,CAAC;AACxC,MAAM,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,MAAM,CAAC,CAAC;AAC5C,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,QAAQ,CAAC,CAAC;AAEhD,eAAO,MAAM,sBAAsB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAAyC,CAAC;AAC7E,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,sBAAsB,CAAC,CAAC;AAEtE,eAAO,MAAM,yBAAyB;;;;eAGpC,CAAC;AACH,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC;AAE5E,4EAA4E;AAC5E,eAAO,MAAM,yBAAyB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAIrC,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC"}
+{"version":3,"file":"downstream.d.ts","sourceRoot":"","sources":["../../../../../../../../zero-cache/src/services/change-source/protocol/current/downstream.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,CAAC,MAAM,wCAAwC,CAAC;AAW5D,QAAA,MAAM,KAAK;;;;;;eAIT,CAAC;AACH,QAAA,MAAM,IAAI;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;mBAGR,CAAC;AACH,QAAA,MAAM,MAAM;;;;;eAIV,CAAC;AACH,QAAA,MAAM,QAAQ;;eAAmD,CAAC;AAElE,MAAM,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,KAAK,CAAC,CAAC;AAC1C,MAAM,MAAM,IAAI,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,IAAI,CAAC,CAAC;AACxC,MAAM,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,MAAM,CAAC,CAAC;AAC5C,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,QAAQ,CAAC,CAAC;AAEhD,eAAO,MAAM,sBAAsB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAAyC,CAAC;AAC7E,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,sBAAsB,CAAC,CAAC;AAEtE,eAAO,MAAM,yBAAyB;;;;eAGpC,CAAC;AACH,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC;AAE5E,4EAA4E;AAC5E,eAAO,MAAM,yBAAyB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAIrC,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC"}

package/out/zero-cache/src/services/change-source/protocol/current/status.d.ts

@@ -47,6 +47,7 @@ export declare const upstreamStatusMessageSchema: v.TupleType<[v.Type<"status">,
     }>;
 }, undefined>, v.ObjectType<{
     tag: v.Type<"commit">;
+    commitTimeMs: v.Optional<number>;
 }, undefined>]>, v.ObjectType<{
     watermark: v.Type<string>;
 }, undefined>]>;

package/out/zero-cache/src/services/change-source/protocol/current/status.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"status.d.ts","sourceRoot":"","sources":["../../../../../../../../zero-cache/src/services/change-source/protocol/current/status.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,CAAC,MAAM,wCAAwC,CAAC;AAI5D;;;;GAIG;AACH,eAAO,MAAM,sBAAsB;;;;;;;;;;aAQjC,CAAC;AAEH,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,sBAAsB,CAAC,CAAC;AAEtE,eAAO,MAAM,6BAA6B;;;;;;;;;;;;eAIxC,CAAC;AAEH;;;;;GAKG;AACH,eAAO,MAAM,2BAA2B;;;;;;;;;;;;;;eAItC,CAAC;AAEH;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,MAAM,MAAM,uBAAuB,GAAG,CAAC,CAAC,KAAK,CAC3C,OAAO,6BAA6B,CACrC,CAAC;AACF,MAAM,MAAM,qBAAqB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,2BAA2B,CAAC,CAAC"}
+{"version":3,"file":"status.d.ts","sourceRoot":"","sources":["../../../../../../../../zero-cache/src/services/change-source/protocol/current/status.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,CAAC,MAAM,wCAAwC,CAAC;AAI5D;;;;GAIG;AACH,eAAO,MAAM,sBAAsB;;;;;;;;;;aAQjC,CAAC;AAEH,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,sBAAsB,CAAC,CAAC;AAEtE,eAAO,MAAM,6BAA6B;;;;;;;;;;;;eAIxC,CAAC;AAEH;;;;;GAKG;AACH,eAAO,MAAM,2BAA2B;;;;;;;;;;;;;;;eAItC,CAAC;AAEH;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,MAAM,MAAM,uBAAuB,GAAG,CAAC,CAAC,KAAK,CAC3C,OAAO,6BAA6B,CACrC,CAAC;AACF,MAAM,MAAM,qBAAqB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,2BAA2B,CAAC,CAAC"}

package/out/zero-cache/src/services/change-source/protocol/current/upstream.d.ts

@@ -12,6 +12,7 @@ export declare const changeSourceUpstreamSchema: v.TupleType<[v.Type<"status">,
     }>;
 }, undefined>, v.ObjectType<{
     tag: v.Type<"commit">;
+    commitTimeMs: v.Optional<number>;
 }, undefined>]>, v.ObjectType<{
     watermark: v.Type<string>;
 }, undefined>]>;

package/out/zero-cache/src/services/change-source/protocol/current/upstream.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"upstream.d.ts","sourceRoot":"","sources":["../../../../../../../../zero-cache/src/services/change-source/protocol/current/upstream.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,CAAC,MAAM,wCAAwC,CAAC;AAQ5D,sEAAsE;AACtE,eAAO,MAAM,0BAA0B;;;;;;;;;;;;;;eAA8B,CAAC;AACtE,MAAM,MAAM,oBAAoB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,0BAA0B,CAAC,CAAC;AAE9E;;;;;;;;;;;GAWG;AACH,eAAO,MAAM,qBAAqB;;;;;;;;;;;aAOhC,CAAC;AAEH,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,qBAAqB,CAAC,CAAC"}
+{"version":3,"file":"upstream.d.ts","sourceRoot":"","sources":["../../../../../../../../zero-cache/src/services/change-source/protocol/current/upstream.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,CAAC,MAAM,wCAAwC,CAAC;AAQ5D,sEAAsE;AACtE,eAAO,MAAM,0BAA0B;;;;;;;;;;;;;;;eAA8B,CAAC;AACtE,MAAM,MAAM,oBAAoB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,0BAA0B,CAAC,CAAC;AAE9E;;;;;;;;;;;GAWG;AACH,eAAO,MAAM,qBAAqB;;;;;;;;;;;aAOhC,CAAC;AAEH,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,qBAAqB,CAAC,CAAC"}

package/out/zero-cache/src/services/change-streamer/change-streamer-service.js

@@ -255,12 +255,13 @@ var ChangeStreamerImpl = class {
 					unflushedBytes += json.length;
 					if (unflushedBytes < flushBytesThreshold) this.#forwarder.forward(entry);
 					else {
-						await this.#forwarder.forwardWithFlowControl(entry);
+						const forwarded = this.#forwarder.forwardWithFlowControl(entry);
+						await stream.changes.doneOr(forwarded);
 						unflushedBytes = 0;
 					}
 					if (type === "commit" || type === "rollback") watermark = null;
 					const readyForMore = this.#storer.readyForMore();
-					if (readyForMore) await readyForMore;
+					if (readyForMore) await stream.changes.doneOr(readyForMore);
 				}
 			} catch (e) {
 				err = e;
@@ -366,8 +367,7 @@ var ChangeStreamerImpl = class {
 	async stop(err) {
 		this.#state.stop(this.#lc, err);
 		this.#stream?.changes.cancel();
-		await this.#storer.stop();
-		await this.#source.stop();
+		await Promise.allSettled([this.#storer.stop(), this.#source.stop()]);
 	}
 };
 var CLEANUP_DELAY_MS = DEFAULT_MAX_RETRY_DELAY_MS * 3;

package/out/zero-cache/src/services/change-streamer/change-streamer-service.js.map

@@ -1 +1 @@
-{"version":3,"file":"change-streamer-service.js","names":["#lc","#shard","#changeDB","#replicaVersion","#source","#storer","#forwarder","#replicationStatusPublisher","#autoReset","#state","#initialWatermarks","#serving","#txCounter","#changeCounter","#stream","#purgeLock","#latestStatus","#handleControlMessage","#purgeOldChanges"],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-service.ts"],"sourcesContent":["import {getDefaultHighWaterMark} from 'node:stream';\nimport type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {unreachable} from '../../../../shared/src/asserts.ts';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport {publishCriticalEvent} from '../../observability/events.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport {\n  min,\n  type AtLeastOne,\n  type LexiVersion,\n} from '../../types/lexi-version.ts';\nimport type {PostgresDB} from '../../types/pg.ts';\nimport type {ShardID} from '../../types/shards.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport type {\n  ChangeSource,\n  ChangeStream,\n} from '../change-source/change-source.ts';\nimport {\n  type ChangeStreamControl,\n  type ChangeStreamData,\n  type Rollback,\n} from '../change-source/protocol/current/downstream.ts';\nimport {\n  publishReplicationError,\n  replicationStatusError,\n  type ReplicationStatusPublisher,\n} from '../replicator/replication-status.ts';\nimport type {SubscriptionState} from '../replicator/schema/replication-state.ts';\nimport {\n  DEFAULT_MAX_RETRY_DELAY_MS,\n  RunningState,\n  UnrecoverableError,\n} from '../running-state.ts';\nimport {\n  type ChangeStreamerService,\n  type Status,\n  type SubscriberContext,\n} from './change-streamer.ts';\nimport * as ErrorType from './error-type-enum.ts';\nimport {Forwarder} from './forwarder.ts';\nimport {initChangeStreamerSchema} from './schema/init.ts';\nimport {\n  AutoResetSignal,\n  ensureReplicationConfig,\n  markResetRequired,\n} from './schema/tables.ts';\nimport {\n  Storer,\n  type PurgeLock,\n  type TuningOptions as StorerOptions,\n} from './storer.ts';\nimport {Subscriber} from './subscriber.ts';\n\nexport type TuningOptions = StorerOptions & {\n  flowControlConsensusPaddingSeconds: number;\n};\n\n/**\n * Performs initialization and schema migrations to initialize a ChangeStreamerImpl.\n */\nexport async function initializeStreamer(\n  lc: LogContext,\n  shard: ShardID,\n  taskID: string,\n  discoveryAddress: string,\n  discoveryProtocol: string,\n  changeDB: PostgresDB,\n  changeSource: ChangeSource,\n  replicationStatusPublisher: ReplicationStatusPublisher,\n  subscriptionState: SubscriptionState,\n  purgeLock: PurgeLock | null,\n  autoReset: boolean,\n  opts: TuningOptions,\n  setTimeoutFn = setTimeout,\n): Promise<ChangeStreamerService> {\n  // Make sure the ChangeLog DB is set up.\n  await initChangeStreamerSchema(lc, changeDB, shard);\n  await ensureReplicationConfig(\n    lc,\n    changeDB,\n    subscriptionState,\n    shard,\n    autoReset,\n    setTimeoutFn,\n  );\n\n  const {replicaVersion} = subscriptionState;\n  return new ChangeStreamerImpl(\n    lc,\n    shard,\n    taskID,\n    discoveryAddress,\n    discoveryProtocol,\n    changeDB,\n    replicaVersion,\n    changeSource,\n    replicationStatusPublisher,\n    purgeLock,\n    autoReset,\n    opts,\n    setTimeoutFn,\n  );\n}\n\nconst REPLICATION_STATUS_ERROR_DELAY_THRESHOLD_MS = 5000;\n\nexport type ChangeTag = ChangeStreamData[1]['tag'];\n\n/**\n * Internally all Downstream messages (not just commits) are given a watermark.\n * These are used for internal ordering for:\n * 1. Replaying new changes in the Storer\n * 2. Filtering old changes in the Subscriber\n *\n * However, only the watermark for `Commit` messages are exposed to\n * subscribers, as that is the only semantically correct watermark to\n * use for tracking a position in a replication stream.\n *\n * Additionally, the ChangeStreamData is eagerly stringified once, after which\n * the string is passed to the changeLog and all subscribers, eliminating\n * redundant stringification and reducing GC churn.\n */\nexport type WatermarkedChange = [\n  watermark: string,\n  tag: ChangeTag,\n  json: string,\n];\n\n/**\n * Upstream-agnostic dispatch of messages in a {@link ChangeStreamMessage} to a\n * {@link Forwarder} and {@link Storer} to execute the forward-store-ack\n * procedure described in {@link ChangeStreamer}.\n *\n * ### Subscriber Catchup\n *\n * Connecting clients first need to be \"caught up\" to the current watermark\n * (from stored change log entries) before new entries are forwarded to\n * them. This is non-trivial because the replication stream may be in the\n * middle of a pending streamed Transaction for which some entries have\n * already been forwarded but are not yet committed to the store.\n *\n *\n * ```\n * ------------------------------- - - - - - - - - - - - - - - - - - - -\n * | Historic changes in storage |  Pending (streamed) tx  |   Next tx\n * ------------------------------- - - - - - - - - - - - - - - - - - - -\n *                                           Replication stream\n *                                           >  >  >  >  >  >  >  >  >\n *           ^  ---> required catchup --->   ^\n * Subscriber watermark               Subscription begins\n * ```\n *\n * Preemptively buffering the changes of every pending transaction\n * would be wasteful and consume too much memory for large transactions.\n *\n * Instead, the streamer synchronously dispatches changes and subscriptions\n * to the {@link Forwarder} and the {@link Storer} such that the two\n * components are aligned as to where in the stream the subscription started.\n * The two components then coordinate catchup and handoff via the\n * {@link Subscriber} object with the following algorithm:\n *\n * * If the streamer is in the middle of a pending Transaction, the\n *   Subscriber is \"queued\" on both the Forwarder and the Storer. In this\n *   state, new changes are *not* forwarded to the Subscriber, and catchup\n *   is not yet executed.\n * * Once the commit message for the pending Transaction is processed\n *   by the Storer, it begins catchup on the Subscriber (with a READONLY\n *   snapshot so that it does not block subsequent storage operations).\n *   This catchup is thus guaranteed to load the change log entries of\n *   that last Transaction.\n * * When the Forwarder processes that same commit message, it moves the\n *   Subscriber from the \"queued\" to the \"active\" set of clients such that\n *   the Subscriber begins receiving new changes, starting from the next\n *   Transaction.\n * * The Subscriber does not forward those changes, however, if its catchup\n *   is not complete. Until then, it buffers the changes in memory.\n * * Once catchup is complete, the buffered changes are immediately sent\n *   and the Subscriber henceforth forwards changes as they are received.\n *\n * In the (common) case where the streamer is not in the middle of a pending\n * transaction when a subscription begins, the Storer begins catchup\n * immediately and the Forwarder directly adds the Subscriber to its active\n * set. However, the Subscriber still buffers any forwarded messages until\n * its catchup is complete.\n *\n * ### Watermarks and ordering\n *\n * The ChangeStreamerService depends on its {@link ChangeSource} to send\n * changes in contiguous [`begin`, `data` ..., `data`, `commit`] sequences\n * in commit order. This follows Postgres's Logical Replication Protocol\n * Message Flow:\n *\n * https://www.postgresql.org/docs/16/protocol-logical-replication.html#PROTOCOL-LOGICAL-MESSAGES-FLOW\n *\n * > The logical replication protocol sends individual transactions one by one.\n * > This means that all messages between a pair of Begin and Commit messages belong to the same transaction.\n *\n * In order to correctly replay (new) and filter (old) messages to subscribers\n * at different points in the replication stream, these changes must be assigned\n * watermarks such that they preserve the order in which they were received\n * from the ChangeSource.\n *\n * A previous implementation incorrectly derived these watermarks from the Postgres\n * Log Sequence Numbers (LSN) of each message. However, LSNs from concurrent,\n * non-conflicting transactions can overlap, which can result in a `begin` message\n * with an earlier LSN arriving after a `commit` message. For example, the\n * changes for these transactions:\n *\n * ```\n * LSN:   1     2     3  4    5   6   7     8   9      10\n * tx1: begin  data     data     data     commit\n * tx2:             begin    data    data      data  commit\n * ```\n *\n * will arrive as:\n *\n * ```\n * begin1, data2, data4, data6, commit8, begin3, data5, data7, data9, commit10\n * ```\n *\n * Thus, LSN of non-commit messages are not suitable for tracking the sorting\n * order of the replication stream.\n *\n * Instead, the ChangeStreamer uses the following algorithm for deterministic\n * catchup and filtering of changes:\n *\n * * A `commit` message is assigned to a watermark corresponding to its LSN.\n *   These are guaranteed to be in commit order by definition.\n *\n * * `begin` and `data` messages are assigned to the watermark of the\n *   preceding `commit` (the previous transaction, or the replication\n *   slot's starting LSN) plus 1. This guarantees that they will be sorted\n *   after the previously commit transaction even if their LSNs came before it.\n *   This is referred to as the `preCommitWatermark`.\n *\n * * In the ChangeLog DB, messages have a secondary sort column `pos`, which is\n *   the position of the message within its transaction, with the `begin` message\n *   starting at `0`. This guarantees that `begin` and `data` messages will be\n *   fetched in the original ChangeSource order during catchup.\n *\n * `begin` and `data` messages share the same watermark, but this is sufficient for\n * Subscriber filtering because subscribers only know about the `commit` watermarks\n * exposed in the `Downstream` `Commit` message. The Subscriber object thus compares\n * the internal watermarks of the incoming messages against the commit watermark of\n * the caller, updating the watermark at every `Commit` message that is forwarded.\n *\n * ### Cleanup\n *\n * As mentioned in the {@link ChangeStreamer} documentation: \"the ChangeStreamer\n * uses a combination of [the \"initial\", i.e. backup-derived watermark and] ACK\n * responses from connected subscribers to determine the watermark up\n * to which it is safe to purge old change log entries.\"\n *\n * More concretely:\n *\n * * The `initial`, backup-derived watermark is the earliest to which cleanup\n *   should ever happen.\n *\n * * However, it is possible for the replica backup to be *ahead* of a connected\n *   subscriber; and if a network error causes that subscriber to retry from its\n *   last watermark, the change streamer must support it.\n *\n * Thus, before cleaning up to an `initial` backup-derived watermark, the change\n * streamer first confirms that all connected subscribers have also passed\n * that watermark.\n */\nclass ChangeStreamerImpl implements ChangeStreamerService {\n  readonly id: string;\n  readonly #lc: LogContext;\n  readonly #shard: ShardID;\n  readonly #changeDB: PostgresDB;\n  readonly #replicaVersion: string;\n  readonly #source: ChangeSource;\n  readonly #storer: Storer;\n  readonly #forwarder: Forwarder;\n  readonly #replicationStatusPublisher: ReplicationStatusPublisher;\n\n  readonly #autoReset: boolean;\n  readonly #state: RunningState;\n  readonly #initialWatermarks = new Set<string>();\n\n  // Starting the (Postgres) ChangeStream results in killing the previous\n  // Postgres subscriber, potentially creating a gap in which the old\n  // change-streamer has shut down and the new change-streamer has not yet\n  // been recognized as \"healthy\" (and thus does not get any requests).\n  //\n  // To minimize this gap, delay starting the ChangeStream until the first\n  // request from a `serving` replicator, indicating that higher level\n  // load-balancing / routing logic has begun routing requests to this task.\n  readonly #serving = resolver();\n\n  readonly #txCounter = getOrCreateCounter(\n    'replication',\n    'transactions',\n    'Count of replicated transactions',\n  );\n  readonly #changeCounter = getOrCreateCounter(\n    'replication',\n    'changes',\n    'Count of replicated changes (DML or DDL statements)',\n  );\n\n  #latestStatus: Status;\n  #purgeLock: PurgeLock | null;\n  #stream: ChangeStream | undefined;\n\n  constructor(\n    lc: LogContext,\n    shard: ShardID,\n    taskID: string,\n    discoveryAddress: string,\n    discoveryProtocol: string,\n    changeDB: PostgresDB,\n    replicaVersion: string,\n    source: ChangeSource,\n    replicationStatusPublisher: ReplicationStatusPublisher,\n    initialPurgeLock: PurgeLock | null,\n    autoReset: boolean,\n    opts: TuningOptions,\n    setTimeoutFn = setTimeout,\n  ) {\n    this.id = `change-streamer`;\n    this.#lc = lc.withContext('component', 'change-streamer');\n    this.#shard = shard;\n    this.#changeDB = changeDB;\n    this.#replicaVersion = replicaVersion;\n    this.#source = source;\n    this.#storer = new Storer(\n      lc,\n      shard,\n      taskID,\n      discoveryAddress,\n      discoveryProtocol,\n      changeDB,\n      replicaVersion,\n      consumed => this.#stream?.acks.push(['status', consumed[1], consumed[2]]),\n      err => this.stop(err),\n      opts,\n    );\n    this.#forwarder = new Forwarder(lc, {\n      flowControlConsensusPaddingSeconds:\n        opts.flowControlConsensusPaddingSeconds,\n    });\n    this.#replicationStatusPublisher = replicationStatusPublisher;\n    this.#purgeLock = initialPurgeLock;\n    this.#autoReset = autoReset;\n    this.#state = new RunningState(this.id, undefined, setTimeoutFn);\n    this.#latestStatus = {tag: 'status'};\n  }\n\n  async run() {\n    this.#lc.info?.('starting change stream');\n\n    this.#forwarder.startProgressMonitor();\n\n    const lagReport = await this.#source.startLagReporter();\n    if (lagReport) {\n      this.#latestStatus.lagReport = lagReport;\n    }\n\n    // Once this change-streamer acquires \"ownership\" of the change DB,\n    // it is safe to start the storer.\n    await this.#storer.assumeOwnership(this.#purgeLock);\n    this.#purgeLock = null;\n\n    // The threshold in (estimated number of) bytes to send() on subscriber\n    // websockets before `await`-ing the I/O buffers to be ready for more.\n    const flushBytesThreshold = getDefaultHighWaterMark(false);\n\n    while (this.#state.shouldRun()) {\n      let err: unknown;\n      let watermark: string | null = null;\n      let unflushedBytes = 0;\n      try {\n        const {lastWatermark, backfillRequests} =\n          await this.#storer.getStartStreamInitializationParameters();\n        const stream = await this.#source.startStream(\n          lastWatermark,\n          backfillRequests,\n        );\n        this.#storer.run().catch(e => stream.changes.cancel(e));\n\n        this.#stream = stream;\n        if (\n          this.#state.resetBackoff() >\n          REPLICATION_STATUS_ERROR_DELAY_THRESHOLD_MS\n        ) {\n          // After recovering from a backoff for which a replication status\n          // error was published, publish an OK status\n          this.#replicationStatusPublisher.publish(\n            this.#lc,\n            'Replicating',\n            `Replicating from ${lastWatermark}`,\n          );\n        }\n        watermark = null;\n\n        for await (const change of stream.changes) {\n          const [type, msg] = change;\n          switch (type) {\n            case 'status':\n              if (msg.ack) {\n                this.#storer.status(change); // storer acks once it gets through its queue\n              }\n              if (msg.lagReport) {\n                // Lag reports are not stored in the cdc change log, but rather\n                // only forwarded on \"live\" connections. When a new subscriber\n                // is catching up, it is initialized with the #latestStatus\n                // from which it can measure lag while catching up.\n                this.#latestStatus.lagReport = msg.lagReport;\n                this.#forwarder.sendStatus(this.#latestStatus);\n              }\n              continue;\n            case 'control':\n              await this.#handleControlMessage(msg);\n              continue; // control messages are not stored/forwarded\n            case 'begin':\n              watermark = change[2].commitWatermark;\n              break;\n            case 'commit':\n              if (watermark !== change[2].watermark) {\n                throw new UnrecoverableError(\n                  `commit watermark ${change[2].watermark} does not match 'begin' watermark ${watermark}`,\n                );\n              }\n              this.#txCounter.add(1);\n              break;\n            default:\n              if (type === 'data') {\n                this.#changeCounter.add(1);\n              }\n              if (watermark === null) {\n                throw new UnrecoverableError(\n                  `${type} change (${msg.tag}) received before 'begin' message`,\n                );\n              }\n              break;\n          }\n\n          const json = this.#storer.store(watermark, change);\n          const entry: WatermarkedChange = [watermark, change[1].tag, json];\n          unflushedBytes += json.length;\n          if (unflushedBytes < flushBytesThreshold) {\n            // pipeline changes until flushBytesThreshold\n            this.#forwarder.forward(entry);\n          } else {\n            // Wait for messages to clear socket buffers to ensure that they\n            // make their way to subscribers. Without this `await`, the\n            // messages end up being buffered in this process, which:\n            // (1) results in memory pressure and increased GC activity\n            // (2) prevents subscribers from processing the messages as they\n            //     arrive, instead getting them in a large batch after being\n            //     idle while they were queued (causing further delays).\n            await this.#forwarder.forwardWithFlowControl(entry);\n            unflushedBytes = 0;\n          }\n\n          if (type === 'commit' || type === 'rollback') {\n            watermark = null;\n          }\n\n          // Allow the storer to exert back pressure.\n          const readyForMore = this.#storer.readyForMore();\n          if (readyForMore) {\n            await readyForMore;\n          }\n        }\n      } catch (e) {\n        err = e;\n      } finally {\n        this.#stream?.changes.cancel();\n        this.#stream = undefined;\n      }\n\n      // When the change stream is interrupted, abort any pending transaction.\n      if (watermark) {\n        this.#lc.warn?.(`aborting interrupted transaction ${watermark}`);\n        this.#storer.abort();\n        this.#forwarder.forward([watermark, 'rollback', ROLLBACK_JSON]);\n      }\n\n      // Backoff and drain any pending entries in the storer before reconnecting.\n      await Promise.all([\n        this.#storer.stop(),\n        this.#state.backoff(this.#lc, err),\n        this.#state.retryDelay > REPLICATION_STATUS_ERROR_DELAY_THRESHOLD_MS\n          ? publishCriticalEvent(\n              this.#lc,\n              replicationStatusError(this.#lc, 'Replicating', err),\n            )\n          : promiseVoid,\n      ]);\n    }\n\n    this.#forwarder.stopProgressMonitor();\n    this.#lc.info?.('ChangeStreamer stopped');\n  }\n\n  async #handleControlMessage(msg: ChangeStreamControl[1]) {\n    this.#lc.info?.('received control message', msg);\n    const {tag} = msg;\n\n    switch (tag) {\n      case 'reset-required':\n        await markResetRequired(this.#changeDB, this.#shard);\n        await publishReplicationError(\n          this.#lc,\n          'Replicating',\n          msg.message ?? 'Resync required',\n          msg.errorDetails,\n        );\n        if (this.#autoReset) {\n          this.#lc.warn?.('shutting down for auto-reset');\n          await this.stop(new AutoResetSignal());\n        }\n        break;\n      default:\n        unreachable(tag);\n    }\n  }\n\n  subscribe(ctx: SubscriberContext): Promise<Source<string>> {\n    const {protocolVersion, id, mode, replicaVersion, watermark} = ctx;\n    if (mode === 'serving') {\n      this.#serving.resolve();\n    }\n    const downstream = Subscription.create<string>({\n      cleanup: () => this.#forwarder.remove(subscriber),\n    });\n    const subscriber = new Subscriber(\n      protocolVersion,\n      id,\n      watermark,\n      downstream,\n      () => this.#latestStatus,\n    );\n    if (replicaVersion !== this.#replicaVersion) {\n      this.#lc.warn?.(\n        `rejecting subscriber at replica version ${replicaVersion}`,\n      );\n      subscriber.close(\n        ErrorType.WrongReplicaVersion,\n        `current replica version is ${\n          this.#replicaVersion\n        } (requested ${replicaVersion})`,\n      );\n    } else {\n      this.#lc.debug?.(`adding subscriber ${subscriber.id}`);\n\n      this.#forwarder.add(subscriber);\n      this.#storer.catchup(subscriber, mode);\n    }\n    return Promise.resolve(downstream);\n  }\n\n  scheduleCleanup(watermark: string) {\n    const origSize = this.#initialWatermarks.size;\n    this.#initialWatermarks.add(watermark);\n\n    if (origSize === 0) {\n      this.#state.setTimeout(() => this.#purgeOldChanges(), CLEANUP_DELAY_MS);\n    }\n  }\n\n  async getChangeLogState(): Promise<{\n    replicaVersion: string;\n    minWatermark: string;\n  }> {\n    const minWatermark = await this.#storer.getMinWatermarkForCatchup();\n    if (!minWatermark) {\n      this.#lc.warn?.(\n        `Unexpected empty changeLog. Resync if \"Local replica watermark\" errors arise`,\n      );\n    }\n    return {\n      replicaVersion: this.#replicaVersion,\n      minWatermark: minWatermark ?? this.#replicaVersion,\n    };\n  }\n\n  /**\n   * Makes a best effort to purge the change log. In the event of a database\n   * error, exceptions will be logged and swallowed, so this method is safe\n   * to run in a timeout.\n   */\n  async #purgeOldChanges(): Promise<void> {\n    const initial = [...this.#initialWatermarks];\n    if (initial.length === 0) {\n      this.#lc.warn?.('No initial watermarks to check for cleanup'); // Not expected.\n      return;\n    }\n    const current = [...this.#forwarder.getAcks()];\n    if (current.length === 0) {\n      // Also not expected, but possible (e.g. subscriber connects, then disconnects).\n      // Bail to be safe.\n      this.#lc.warn?.('No subscribers to confirm cleanup');\n      return;\n    }\n    try {\n      const earliestInitial = min(...(initial as AtLeastOne<LexiVersion>));\n      const earliestCurrent = min(...(current as AtLeastOne<LexiVersion>));\n      if (earliestCurrent < earliestInitial) {\n        this.#lc.info?.(\n          `At least one client is behind backup (${earliestCurrent} < ${earliestInitial})`,\n        );\n      } else {\n        this.#lc.info?.(`Purging changes before ${earliestInitial} ...`);\n        const start = performance.now();\n        const deleted = await this.#storer.purgeRecordsBefore(earliestInitial);\n        const elapsed = (performance.now() - start).toFixed(2);\n        this.#lc.info?.(\n          `Purged ${deleted} changes before ${earliestInitial} (${elapsed} ms)`,\n        );\n        this.#initialWatermarks.delete(earliestInitial);\n      }\n    } catch (e) {\n      this.#lc.warn?.(`error purging change log`, e);\n    } finally {\n      if (this.#initialWatermarks.size) {\n        // If there are unpurged watermarks to check, schedule the next purge.\n        this.#state.setTimeout(() => this.#purgeOldChanges(), CLEANUP_DELAY_MS);\n      }\n    }\n  }\n\n  async stop(err?: unknown) {\n    this.#state.stop(this.#lc, err);\n    this.#stream?.changes.cancel();\n    await this.#storer.stop();\n    await this.#source.stop();\n  }\n}\n\n// The delay between receiving an initial, backup-based watermark\n// and performing a check of whether to purge records before it.\n// This delay should be long enough to handle situations like the following:\n//\n// 1. `litestream restore` downloads a backup for the `replication-manager`\n// 2. `replication-manager` starts up and runs this `change-streamer`\n// 3. `zero-cache`s that are running on a different replica connect to this\n//    `change-streamer` after exponential backoff retries.\n//\n// It is possible for a `zero-cache`[3] to be behind the backup restored [1].\n// This cleanup delay (30 seconds) is thus set to be a value comfortably\n// longer than the max delay for exponential backoff (10 seconds) in\n// `services/running-state.ts`. This allows the `zero-cache` [3] to reconnect\n// so that the `change-streamer` can track its progress and know when it has\n// surpassed the initial watermark of the backup [1].\nconst CLEANUP_DELAY_MS = DEFAULT_MAX_RETRY_DELAY_MS * 3;\n\nconst ROLLBACK_JSON = JSON.stringify([\n  'rollback',\n  {tag: 'rollback'},\n] satisfies Rollback);\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AA+DA,eAAsB,mBACpB,IACA,OACA,QACA,kBACA,mBACA,UACA,cACA,4BACA,mBACA,WACA,WACA,MACA,eAAe,YACiB;CAEhC,MAAM,yBAAyB,IAAI,UAAU,KAAK;CAClD,MAAM,wBACJ,IACA,UACA,mBACA,OACA,WACA,YACF;CAEA,MAAM,EAAC,mBAAkB;CACzB,OAAO,IAAI,mBACT,IACA,OACA,QACA,kBACA,mBACA,UACA,gBACA,cACA,4BACA,WACA,WACA,MACA,YACF;AACF;AAEA,IAAM,8CAA8C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAkKpD,IAAM,qBAAN,MAA0D;CACxD;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAEA;CACA;CACA,qCAA8B,IAAI,IAAY;CAU9C,WAAoB,SAAS;CAE7B,aAAsB,mBACpB,eACA,gBACA,kCACF;CACA,iBAA0B,mBACxB,eACA,WACA,qDACF;CAEA;CACA;CACA;CAEA,YACE,IACA,OACA,QACA,kBACA,mBACA,UACA,gBACA,QACA,4BACA,kBACA,WACA,MACA,eAAe,YACf;EACA,KAAK,KAAK;EACV,KAAKA,MAAM,GAAG,YAAY,aAAa,iBAAiB;EACxD,KAAKC,SAAS;EACd,KAAKC,YAAY;EACjB,KAAKC,kBAAkB;EACvB,KAAKC,UAAU;EACf,KAAKC,UAAU,IAAI,OACjB,IACA,OACA,QACA,kBACA,mBACA,UACA,iBACA,aAAY,KAAKS,SAAS,KAAK,KAAK;GAAC;GAAU,SAAS;GAAI,SAAS;EAAE,CAAC,IACxE,QAAO,KAAK,KAAK,GAAG,GACpB,IACF;EACA,KAAKR,aAAa,IAAI,UAAU,IAAI,EAClC,oCACE,KAAK,mCACT,CAAC;EACD,KAAKC,8BAA8B;EACnC,KAAKQ,aAAa;EAClB,KAAKP,aAAa;EAClB,KAAKC,SAAS,IAAI,aAAa,KAAK,IAAI,KAAA,GAAW,YAAY;EAC/D,KAAKO,gBAAgB,EAAC,KAAK,SAAQ;CACrC;CAEA,MAAM,MAAM;EACV,KAAKhB,IAAI,OAAO,wBAAwB;EAExC,KAAKM,WAAW,qBAAqB;EAErC,MAAM,YAAY,MAAM,KAAKF,QAAQ,iBAAiB;EACtD,IAAI,WACF,KAAKY,cAAc,YAAY;EAKjC,MAAM,KAAKX,QAAQ,gBAAgB,KAAKU,UAAU;EAClD,KAAKA,aAAa;EAIlB,MAAM,sBAAsB,wBAAwB,KAAK;EAEzD,OAAO,KAAKN,OAAO,UAAU,GAAG;GAC9B,IAAI;GACJ,IAAI,YAA2B;GAC/B,IAAI,iBAAiB;GACrB,IAAI;IACF,MAAM,EAAC,eAAe,qBACpB,MAAM,KAAKJ,QAAQ,uCAAuC;IAC5D,MAAM,SAAS,MAAM,KAAKD,QAAQ,YAChC,eACA,gBACF;IACA,KAAKC,QAAQ,IAAI,EAAE,OAAM,MAAK,OAAO,QAAQ,OAAO,CAAC,CAAC;IAEtD,KAAKS,UAAU;IACf,IACE,KAAKL,OAAO,aAAa,IACzB,6CAIA,KAAKF,4BAA4B,QAC/B,KAAKP,KACL,eACA,oBAAoB,eACtB;IAEF,YAAY;IAEZ,WAAW,MAAM,UAAU,OAAO,SAAS;KACzC,MAAM,CAAC,MAAM,OAAO;KACpB,QAAQ,MAAR;MACE,KAAK;OACH,IAAI,IAAI,KACN,KAAKK,QAAQ,OAAO,MAAM;OAE5B,IAAI,IAAI,WAAW;QAKjB,KAAKW,cAAc,YAAY,IAAI;QACnC,KAAKV,WAAW,WAAW,KAAKU,aAAa;OAC/C;OACA;MACF,KAAK;OACH,MAAM,KAAKC,sBAAsB,GAAG;OACpC;MACF,KAAK;OACH,YAAY,OAAO,GAAG;OACtB;MACF,KAAK;OACH,IAAI,cAAc,OAAO,GAAG,WAC1B,MAAM,IAAI,mBACR,oBAAoB,OAAO,GAAG,UAAU,oCAAoC,WAC9E;OAEF,KAAKL,WAAW,IAAI,CAAC;OACrB;MACF;OACE,IAAI,SAAS,QACX,KAAKC,eAAe,IAAI,CAAC;OAE3B,IAAI,cAAc,MAChB,MAAM,IAAI,mBACR,GAAG,KAAK,WAAW,IAAI,IAAI,kCAC7B;OAEF;KACJ;KAEA,MAAM,OAAO,KAAKR,QAAQ,MAAM,WAAW,MAAM;KACjD,MAAM,QAA2B;MAAC;MAAW,OAAO,GAAG;MAAK;KAAI;KAChE,kBAAkB,KAAK;KACvB,IAAI,iBAAiB,qBAEnB,KAAKC,WAAW,QAAQ,KAAK;UACxB;MAQL,MAAM,KAAKA,WAAW,uBAAuB,KAAK;MAClD,iBAAiB;KACnB;KAEA,IAAI,SAAS,YAAY,SAAS,YAChC,YAAY;KAId,MAAM,eAAe,KAAKD,QAAQ,aAAa;KAC/C,IAAI,cACF,MAAM;IAEV;GACF,SAAS,GAAG;IACV,MAAM;GACR,UAAU;IACR,KAAKS,SAAS,QAAQ,OAAO;IAC7B,KAAKA,UAAU,KAAA;GACjB;GAGA,IAAI,WAAW;IACb,KAAKd,IAAI,OAAO,oCAAoC,WAAW;IAC/D,KAAKK,QAAQ,MAAM;IACnB,KAAKC,WAAW,QAAQ;KAAC;KAAW;KAAY;IAAa,CAAC;GAChE;GAGA,MAAM,QAAQ,IAAI;IAChB,KAAKD,QAAQ,KAAK;IAClB,KAAKI,OAAO,QAAQ,KAAKT,KAAK,GAAG;IACjC,KAAKS,OAAO,aAAa,8CACrB,qBACE,KAAKT,KACL,uBAAuB,KAAKA,KAAK,eAAe,GAAG,CACrD,IACA;GACN,CAAC;EACH;EAEA,KAAKM,WAAW,oBAAoB;EACpC,KAAKN,IAAI,OAAO,wBAAwB;CAC1C;CAEA,MAAMiB,sBAAsB,KAA6B;EACvD,KAAKjB,IAAI,OAAO,4BAA4B,GAAG;EAC/C,MAAM,EAAC,QAAO;EAEd,QAAQ,KAAR;GACE,KAAK;IACH,MAAM,kBAAkB,KAAKE,WAAW,KAAKD,MAAM;IACnD,MAAM,wBACJ,KAAKD,KACL,eACA,IAAI,WAAW,mBACf,IAAI,YACN;IACA,IAAI,KAAKQ,YAAY;KACnB,KAAKR,IAAI,OAAO,8BAA8B;KAC9C,MAAM,KAAK,KAAK,IAAI,gBAAgB,CAAC;IACvC;IACA;GACF,SACE,YAAY,GAAG;EACnB;CACF;CAEA,UAAU,KAAiD;EACzD,MAAM,EAAC,iBAAiB,IAAI,MAAM,gBAAgB,cAAa;EAC/D,IAAI,SAAS,WACX,KAAKW,SAAS,QAAQ;EAExB,MAAM,aAAa,aAAa,OAAe,EAC7C,eAAe,KAAKL,WAAW,OAAO,UAAU,EAClD,CAAC;EACD,MAAM,aAAa,IAAI,WACrB,iBACA,IACA,WACA,kBACM,KAAKU,aACb;EACA,IAAI,mBAAmB,KAAKb,iBAAiB;GAC3C,KAAKH,IAAI,OACP,2CAA2C,gBAC7C;GACA,WAAW,MACT,GACA,8BACE,KAAKG,gBACN,cAAc,eAAe,EAChC;EACF,OAAO;GACL,KAAKH,IAAI,QAAQ,qBAAqB,WAAW,IAAI;GAErD,KAAKM,WAAW,IAAI,UAAU;GAC9B,KAAKD,QAAQ,QAAQ,YAAY,IAAI;EACvC;EACA,OAAO,QAAQ,QAAQ,UAAU;CACnC;CAEA,gBAAgB,WAAmB;EACjC,MAAM,WAAW,KAAKK,mBAAmB;EACzC,KAAKA,mBAAmB,IAAI,SAAS;EAErC,IAAI,aAAa,GACf,KAAKD,OAAO,iBAAiB,KAAKS,iBAAiB,GAAG,gBAAgB;CAE1E;CAEA,MAAM,oBAGH;EACD,MAAM,eAAe,MAAM,KAAKb,QAAQ,0BAA0B;EAClE,IAAI,CAAC,cACH,KAAKL,IAAI,OACP,8EACF;EAEF,OAAO;GACL,gBAAgB,KAAKG;GACrB,cAAc,gBAAgB,KAAKA;EACrC;CACF;;;;;;CAOA,MAAMe,mBAAkC;EACtC,MAAM,UAAU,CAAC,GAAG,KAAKR,kBAAkB;EAC3C,IAAI,QAAQ,WAAW,GAAG;GACxB,KAAKV,IAAI,OAAO,4CAA4C;GAC5D;EACF;EACA,MAAM,UAAU,CAAC,GAAG,KAAKM,WAAW,QAAQ,CAAC;EAC7C,IAAI,QAAQ,WAAW,GAAG;GAGxB,KAAKN,IAAI,OAAO,mCAAmC;GACnD;EACF;EACA,IAAI;GACF,MAAM,kBAAkB,IAAI,GAAI,OAAmC;GACnE,MAAM,kBAAkB,IAAI,GAAI,OAAmC;GACnE,IAAI,kBAAkB,iBACpB,KAAKA,IAAI,OACP,yCAAyC,gBAAgB,KAAK,gBAAgB,EAChF;QACK;IACL,KAAKA,IAAI,OAAO,0BAA0B,gBAAgB,KAAK;IAC/D,MAAM,QAAQ,YAAY,IAAI;IAC9B,MAAM,UAAU,MAAM,KAAKK,QAAQ,mBAAmB,eAAe;IACrE,MAAM,WAAW,YAAY,IAAI,IAAI,OAAO,QAAQ,CAAC;IACrD,KAAKL,IAAI,OACP,UAAU,QAAQ,kBAAkB,gBAAgB,IAAI,QAAQ,KAClE;IACA,KAAKU,mBAAmB,OAAO,eAAe;GAChD;EACF,SAAS,GAAG;GACV,KAAKV,IAAI,OAAO,4BAA4B,CAAC;EAC/C,UAAU;GACR,IAAI,KAAKU,mBAAmB,MAE1B,KAAKD,OAAO,iBAAiB,KAAKS,iBAAiB,GAAG,gBAAgB;EAE1E;CACF;CAEA,MAAM,KAAK,KAAe;EACxB,KAAKT,OAAO,KAAK,KAAKT,KAAK,GAAG;EAC9B,KAAKc,SAAS,QAAQ,OAAO;EAC7B,MAAM,KAAKT,QAAQ,KAAK;EACxB,MAAM,KAAKD,QAAQ,KAAK;CAC1B;AACF;AAiBA,IAAM,mBAAmB,6BAA6B;AAEtD,IAAM,gBAAgB,KAAK,UAAU,CACnC,YACA,EAAC,KAAK,WAAU,CAClB,CAAoB"}
+{"version":3,"file":"change-streamer-service.js","names":["#lc","#shard","#changeDB","#replicaVersion","#source","#storer","#forwarder","#replicationStatusPublisher","#autoReset","#state","#initialWatermarks","#serving","#txCounter","#changeCounter","#stream","#purgeLock","#latestStatus","#handleControlMessage","#purgeOldChanges"],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-service.ts"],"sourcesContent":["import {getDefaultHighWaterMark} from 'node:stream';\nimport type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {unreachable} from '../../../../shared/src/asserts.ts';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport {publishCriticalEvent} from '../../observability/events.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport {\n  min,\n  type AtLeastOne,\n  type LexiVersion,\n} from '../../types/lexi-version.ts';\nimport type {PostgresDB} from '../../types/pg.ts';\nimport type {ShardID} from '../../types/shards.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport type {\n  ChangeSource,\n  ChangeStream,\n} from '../change-source/change-source.ts';\nimport {\n  type ChangeStreamControl,\n  type ChangeStreamData,\n  type Rollback,\n} from '../change-source/protocol/current/downstream.ts';\nimport {\n  publishReplicationError,\n  replicationStatusError,\n  type ReplicationStatusPublisher,\n} from '../replicator/replication-status.ts';\nimport type {SubscriptionState} from '../replicator/schema/replication-state.ts';\nimport {\n  DEFAULT_MAX_RETRY_DELAY_MS,\n  RunningState,\n  UnrecoverableError,\n} from '../running-state.ts';\nimport {\n  type ChangeStreamerService,\n  type Status,\n  type SubscriberContext,\n} from './change-streamer.ts';\nimport * as ErrorType from './error-type-enum.ts';\nimport {Forwarder} from './forwarder.ts';\nimport {initChangeStreamerSchema} from './schema/init.ts';\nimport {\n  AutoResetSignal,\n  ensureReplicationConfig,\n  markResetRequired,\n} from './schema/tables.ts';\nimport {\n  Storer,\n  type PurgeLock,\n  type TuningOptions as StorerOptions,\n} from './storer.ts';\nimport {Subscriber} from './subscriber.ts';\n\nexport type TuningOptions = StorerOptions & {\n  flowControlConsensusPaddingSeconds: number;\n};\n\n/**\n * Performs initialization and schema migrations to initialize a ChangeStreamerImpl.\n */\nexport async function initializeStreamer(\n  lc: LogContext,\n  shard: ShardID,\n  taskID: string,\n  discoveryAddress: string,\n  discoveryProtocol: string,\n  changeDB: PostgresDB,\n  changeSource: ChangeSource,\n  replicationStatusPublisher: ReplicationStatusPublisher,\n  subscriptionState: SubscriptionState,\n  purgeLock: PurgeLock | null,\n  autoReset: boolean,\n  opts: TuningOptions,\n  setTimeoutFn = setTimeout,\n): Promise<ChangeStreamerService> {\n  // Make sure the ChangeLog DB is set up.\n  await initChangeStreamerSchema(lc, changeDB, shard);\n  await ensureReplicationConfig(\n    lc,\n    changeDB,\n    subscriptionState,\n    shard,\n    autoReset,\n    setTimeoutFn,\n  );\n\n  const {replicaVersion} = subscriptionState;\n  return new ChangeStreamerImpl(\n    lc,\n    shard,\n    taskID,\n    discoveryAddress,\n    discoveryProtocol,\n    changeDB,\n    replicaVersion,\n    changeSource,\n    replicationStatusPublisher,\n    purgeLock,\n    autoReset,\n    opts,\n    setTimeoutFn,\n  );\n}\n\nconst REPLICATION_STATUS_ERROR_DELAY_THRESHOLD_MS = 5000;\n\nexport type ChangeTag = ChangeStreamData[1]['tag'];\n\n/**\n * Internally all Downstream messages (not just commits) are given a watermark.\n * These are used for internal ordering for:\n * 1. Replaying new changes in the Storer\n * 2. Filtering old changes in the Subscriber\n *\n * However, only the watermark for `Commit` messages are exposed to\n * subscribers, as that is the only semantically correct watermark to\n * use for tracking a position in a replication stream.\n *\n * Additionally, the ChangeStreamData is eagerly stringified once, after which\n * the string is passed to the changeLog and all subscribers, eliminating\n * redundant stringification and reducing GC churn.\n */\nexport type WatermarkedChange = [\n  watermark: string,\n  tag: ChangeTag,\n  json: string,\n];\n\n/**\n * Upstream-agnostic dispatch of messages in a {@link ChangeStreamMessage} to a\n * {@link Forwarder} and {@link Storer} to execute the forward-store-ack\n * procedure described in {@link ChangeStreamer}.\n *\n * ### Subscriber Catchup\n *\n * Connecting clients first need to be \"caught up\" to the current watermark\n * (from stored change log entries) before new entries are forwarded to\n * them. This is non-trivial because the replication stream may be in the\n * middle of a pending streamed Transaction for which some entries have\n * already been forwarded but are not yet committed to the store.\n *\n *\n * ```\n * ------------------------------- - - - - - - - - - - - - - - - - - - -\n * | Historic changes in storage |  Pending (streamed) tx  |   Next tx\n * ------------------------------- - - - - - - - - - - - - - - - - - - -\n *                                           Replication stream\n *                                           >  >  >  >  >  >  >  >  >\n *           ^  ---> required catchup --->   ^\n * Subscriber watermark               Subscription begins\n * ```\n *\n * Preemptively buffering the changes of every pending transaction\n * would be wasteful and consume too much memory for large transactions.\n *\n * Instead, the streamer synchronously dispatches changes and subscriptions\n * to the {@link Forwarder} and the {@link Storer} such that the two\n * components are aligned as to where in the stream the subscription started.\n * The two components then coordinate catchup and handoff via the\n * {@link Subscriber} object with the following algorithm:\n *\n * * If the streamer is in the middle of a pending Transaction, the\n *   Subscriber is \"queued\" on both the Forwarder and the Storer. In this\n *   state, new changes are *not* forwarded to the Subscriber, and catchup\n *   is not yet executed.\n * * Once the commit message for the pending Transaction is processed\n *   by the Storer, it begins catchup on the Subscriber (with a READONLY\n *   snapshot so that it does not block subsequent storage operations).\n *   This catchup is thus guaranteed to load the change log entries of\n *   that last Transaction.\n * * When the Forwarder processes that same commit message, it moves the\n *   Subscriber from the \"queued\" to the \"active\" set of clients such that\n *   the Subscriber begins receiving new changes, starting from the next\n *   Transaction.\n * * The Subscriber does not forward those changes, however, if its catchup\n *   is not complete. Until then, it buffers the changes in memory.\n * * Once catchup is complete, the buffered changes are immediately sent\n *   and the Subscriber henceforth forwards changes as they are received.\n *\n * In the (common) case where the streamer is not in the middle of a pending\n * transaction when a subscription begins, the Storer begins catchup\n * immediately and the Forwarder directly adds the Subscriber to its active\n * set. However, the Subscriber still buffers any forwarded messages until\n * its catchup is complete.\n *\n * ### Watermarks and ordering\n *\n * The ChangeStreamerService depends on its {@link ChangeSource} to send\n * changes in contiguous [`begin`, `data` ..., `data`, `commit`] sequences\n * in commit order. This follows Postgres's Logical Replication Protocol\n * Message Flow:\n *\n * https://www.postgresql.org/docs/16/protocol-logical-replication.html#PROTOCOL-LOGICAL-MESSAGES-FLOW\n *\n * > The logical replication protocol sends individual transactions one by one.\n * > This means that all messages between a pair of Begin and Commit messages belong to the same transaction.\n *\n * In order to correctly replay (new) and filter (old) messages to subscribers\n * at different points in the replication stream, these changes must be assigned\n * watermarks such that they preserve the order in which they were received\n * from the ChangeSource.\n *\n * A previous implementation incorrectly derived these watermarks from the Postgres\n * Log Sequence Numbers (LSN) of each message. However, LSNs from concurrent,\n * non-conflicting transactions can overlap, which can result in a `begin` message\n * with an earlier LSN arriving after a `commit` message. For example, the\n * changes for these transactions:\n *\n * ```\n * LSN:   1     2     3  4    5   6   7     8   9      10\n * tx1: begin  data     data     data     commit\n * tx2:             begin    data    data      data  commit\n * ```\n *\n * will arrive as:\n *\n * ```\n * begin1, data2, data4, data6, commit8, begin3, data5, data7, data9, commit10\n * ```\n *\n * Thus, LSN of non-commit messages are not suitable for tracking the sorting\n * order of the replication stream.\n *\n * Instead, the ChangeStreamer uses the following algorithm for deterministic\n * catchup and filtering of changes:\n *\n * * A `commit` message is assigned to a watermark corresponding to its LSN.\n *   These are guaranteed to be in commit order by definition.\n *\n * * `begin` and `data` messages are assigned to the watermark of the\n *   preceding `commit` (the previous transaction, or the replication\n *   slot's starting LSN) plus 1. This guarantees that they will be sorted\n *   after the previously commit transaction even if their LSNs came before it.\n *   This is referred to as the `preCommitWatermark`.\n *\n * * In the ChangeLog DB, messages have a secondary sort column `pos`, which is\n *   the position of the message within its transaction, with the `begin` message\n *   starting at `0`. This guarantees that `begin` and `data` messages will be\n *   fetched in the original ChangeSource order during catchup.\n *\n * `begin` and `data` messages share the same watermark, but this is sufficient for\n * Subscriber filtering because subscribers only know about the `commit` watermarks\n * exposed in the `Downstream` `Commit` message. The Subscriber object thus compares\n * the internal watermarks of the incoming messages against the commit watermark of\n * the caller, updating the watermark at every `Commit` message that is forwarded.\n *\n * ### Cleanup\n *\n * As mentioned in the {@link ChangeStreamer} documentation: \"the ChangeStreamer\n * uses a combination of [the \"initial\", i.e. backup-derived watermark and] ACK\n * responses from connected subscribers to determine the watermark up\n * to which it is safe to purge old change log entries.\"\n *\n * More concretely:\n *\n * * The `initial`, backup-derived watermark is the earliest to which cleanup\n *   should ever happen.\n *\n * * However, it is possible for the replica backup to be *ahead* of a connected\n *   subscriber; and if a network error causes that subscriber to retry from its\n *   last watermark, the change streamer must support it.\n *\n * Thus, before cleaning up to an `initial` backup-derived watermark, the change\n * streamer first confirms that all connected subscribers have also passed\n * that watermark.\n */\nclass ChangeStreamerImpl implements ChangeStreamerService {\n  readonly id: string;\n  readonly #lc: LogContext;\n  readonly #shard: ShardID;\n  readonly #changeDB: PostgresDB;\n  readonly #replicaVersion: string;\n  readonly #source: ChangeSource;\n  readonly #storer: Storer;\n  readonly #forwarder: Forwarder;\n  readonly #replicationStatusPublisher: ReplicationStatusPublisher;\n\n  readonly #autoReset: boolean;\n  readonly #state: RunningState;\n  readonly #initialWatermarks = new Set<string>();\n\n  // Starting the (Postgres) ChangeStream results in killing the previous\n  // Postgres subscriber, potentially creating a gap in which the old\n  // change-streamer has shut down and the new change-streamer has not yet\n  // been recognized as \"healthy\" (and thus does not get any requests).\n  //\n  // To minimize this gap, delay starting the ChangeStream until the first\n  // request from a `serving` replicator, indicating that higher level\n  // load-balancing / routing logic has begun routing requests to this task.\n  readonly #serving = resolver();\n\n  readonly #txCounter = getOrCreateCounter(\n    'replication',\n    'transactions',\n    'Count of replicated transactions',\n  );\n  readonly #changeCounter = getOrCreateCounter(\n    'replication',\n    'changes',\n    'Count of replicated changes (DML or DDL statements)',\n  );\n\n  #latestStatus: Status;\n  #purgeLock: PurgeLock | null;\n  #stream: ChangeStream | undefined;\n\n  constructor(\n    lc: LogContext,\n    shard: ShardID,\n    taskID: string,\n    discoveryAddress: string,\n    discoveryProtocol: string,\n    changeDB: PostgresDB,\n    replicaVersion: string,\n    source: ChangeSource,\n    replicationStatusPublisher: ReplicationStatusPublisher,\n    initialPurgeLock: PurgeLock | null,\n    autoReset: boolean,\n    opts: TuningOptions,\n    setTimeoutFn = setTimeout,\n  ) {\n    this.id = `change-streamer`;\n    this.#lc = lc.withContext('component', 'change-streamer');\n    this.#shard = shard;\n    this.#changeDB = changeDB;\n    this.#replicaVersion = replicaVersion;\n    this.#source = source;\n    this.#storer = new Storer(\n      lc,\n      shard,\n      taskID,\n      discoveryAddress,\n      discoveryProtocol,\n      changeDB,\n      replicaVersion,\n      consumed => this.#stream?.acks.push(['status', consumed[1], consumed[2]]),\n      err => this.stop(err),\n      opts,\n    );\n    this.#forwarder = new Forwarder(lc, {\n      flowControlConsensusPaddingSeconds:\n        opts.flowControlConsensusPaddingSeconds,\n    });\n    this.#replicationStatusPublisher = replicationStatusPublisher;\n    this.#purgeLock = initialPurgeLock;\n    this.#autoReset = autoReset;\n    this.#state = new RunningState(this.id, undefined, setTimeoutFn);\n    this.#latestStatus = {tag: 'status'};\n  }\n\n  async run() {\n    this.#lc.info?.('starting change stream');\n\n    this.#forwarder.startProgressMonitor();\n\n    const lagReport = await this.#source.startLagReporter();\n    if (lagReport) {\n      this.#latestStatus.lagReport = lagReport;\n    }\n\n    // Once this change-streamer acquires \"ownership\" of the change DB,\n    // it is safe to start the storer.\n    await this.#storer.assumeOwnership(this.#purgeLock);\n    this.#purgeLock = null;\n\n    // The threshold in (estimated number of) bytes to send() on subscriber\n    // websockets before `await`-ing the I/O buffers to be ready for more.\n    const flushBytesThreshold = getDefaultHighWaterMark(false);\n\n    while (this.#state.shouldRun()) {\n      let err: unknown;\n      let watermark: string | null = null;\n      let unflushedBytes = 0;\n      try {\n        const {lastWatermark, backfillRequests} =\n          await this.#storer.getStartStreamInitializationParameters();\n        const stream = await this.#source.startStream(\n          lastWatermark,\n          backfillRequests,\n        );\n        this.#storer.run().catch(e => stream.changes.cancel(e));\n\n        this.#stream = stream;\n        if (\n          this.#state.resetBackoff() >\n          REPLICATION_STATUS_ERROR_DELAY_THRESHOLD_MS\n        ) {\n          // After recovering from a backoff for which a replication status\n          // error was published, publish an OK status\n          this.#replicationStatusPublisher.publish(\n            this.#lc,\n            'Replicating',\n            `Replicating from ${lastWatermark}`,\n          );\n        }\n        watermark = null;\n\n        for await (const change of stream.changes) {\n          const [type, msg] = change;\n          switch (type) {\n            case 'status':\n              if (msg.ack) {\n                this.#storer.status(change); // storer acks once it gets through its queue\n              }\n              if (msg.lagReport) {\n                // Lag reports are not stored in the cdc change log, but rather\n                // only forwarded on \"live\" connections. When a new subscriber\n                // is catching up, it is initialized with the #latestStatus\n                // from which it can measure lag while catching up.\n                this.#latestStatus.lagReport = msg.lagReport;\n                this.#forwarder.sendStatus(this.#latestStatus);\n              }\n              continue;\n            case 'control':\n              await this.#handleControlMessage(msg);\n              continue; // control messages are not stored/forwarded\n            case 'begin':\n              watermark = change[2].commitWatermark;\n              break;\n            case 'commit':\n              if (watermark !== change[2].watermark) {\n                throw new UnrecoverableError(\n                  `commit watermark ${change[2].watermark} does not match 'begin' watermark ${watermark}`,\n                );\n              }\n              this.#txCounter.add(1);\n              break;\n            default:\n              if (type === 'data') {\n                this.#changeCounter.add(1);\n              }\n              if (watermark === null) {\n                throw new UnrecoverableError(\n                  `${type} change (${msg.tag}) received before 'begin' message`,\n                );\n              }\n              break;\n          }\n\n          const json = this.#storer.store(watermark, change);\n          const entry: WatermarkedChange = [watermark, change[1].tag, json];\n          unflushedBytes += json.length;\n          if (unflushedBytes < flushBytesThreshold) {\n            // pipeline changes until flushBytesThreshold\n            this.#forwarder.forward(entry);\n          } else {\n            // Wait for messages to clear socket buffers to ensure that they\n            // make their way to subscribers. Without this `await`, the\n            // messages end up being buffered in this process, which:\n            // (1) results in memory pressure and increased GC activity\n            // (2) prevents subscribers from processing the messages as they\n            //     arrive, instead getting them in a large batch after being\n            //     idle while they were queued (causing further delays).\n            const forwarded = this.#forwarder.forwardWithFlowControl(entry);\n            await stream.changes.doneOr(forwarded);\n            unflushedBytes = 0;\n          }\n\n          if (type === 'commit' || type === 'rollback') {\n            watermark = null;\n          }\n\n          // Allow the storer to exert back pressure.\n          const readyForMore = this.#storer.readyForMore();\n          if (readyForMore) {\n            await stream.changes.doneOr(readyForMore);\n          }\n        }\n      } catch (e) {\n        err = e;\n      } finally {\n        this.#stream?.changes.cancel();\n        this.#stream = undefined;\n      }\n\n      // When the change stream is interrupted, abort any pending transaction.\n      if (watermark) {\n        this.#lc.warn?.(`aborting interrupted transaction ${watermark}`);\n        this.#storer.abort();\n        this.#forwarder.forward([watermark, 'rollback', ROLLBACK_JSON]);\n      }\n\n      // Backoff and drain any pending entries in the storer before reconnecting.\n      await Promise.all([\n        this.#storer.stop(),\n        this.#state.backoff(this.#lc, err),\n        this.#state.retryDelay > REPLICATION_STATUS_ERROR_DELAY_THRESHOLD_MS\n          ? publishCriticalEvent(\n              this.#lc,\n              replicationStatusError(this.#lc, 'Replicating', err),\n            )\n          : promiseVoid,\n      ]);\n    }\n\n    this.#forwarder.stopProgressMonitor();\n    this.#lc.info?.('ChangeStreamer stopped');\n  }\n\n  async #handleControlMessage(msg: ChangeStreamControl[1]) {\n    this.#lc.info?.('received control message', msg);\n    const {tag} = msg;\n\n    switch (tag) {\n      case 'reset-required':\n        await markResetRequired(this.#changeDB, this.#shard);\n        await publishReplicationError(\n          this.#lc,\n          'Replicating',\n          msg.message ?? 'Resync required',\n          msg.errorDetails,\n        );\n        if (this.#autoReset) {\n          this.#lc.warn?.('shutting down for auto-reset');\n          await this.stop(new AutoResetSignal());\n        }\n        break;\n      default:\n        unreachable(tag);\n    }\n  }\n\n  subscribe(ctx: SubscriberContext): Promise<Source<string>> {\n    const {protocolVersion, id, mode, replicaVersion, watermark} = ctx;\n    if (mode === 'serving') {\n      this.#serving.resolve();\n    }\n    const downstream = Subscription.create<string>({\n      cleanup: () => this.#forwarder.remove(subscriber),\n    });\n    const subscriber = new Subscriber(\n      protocolVersion,\n      id,\n      watermark,\n      downstream,\n      () => this.#latestStatus,\n    );\n    if (replicaVersion !== this.#replicaVersion) {\n      this.#lc.warn?.(\n        `rejecting subscriber at replica version ${replicaVersion}`,\n      );\n      subscriber.close(\n        ErrorType.WrongReplicaVersion,\n        `current replica version is ${\n          this.#replicaVersion\n        } (requested ${replicaVersion})`,\n      );\n    } else {\n      this.#lc.debug?.(`adding subscriber ${subscriber.id}`);\n\n      this.#forwarder.add(subscriber);\n      this.#storer.catchup(subscriber, mode);\n    }\n    return Promise.resolve(downstream);\n  }\n\n  scheduleCleanup(watermark: string) {\n    const origSize = this.#initialWatermarks.size;\n    this.#initialWatermarks.add(watermark);\n\n    if (origSize === 0) {\n      this.#state.setTimeout(() => this.#purgeOldChanges(), CLEANUP_DELAY_MS);\n    }\n  }\n\n  async getChangeLogState(): Promise<{\n    replicaVersion: string;\n    minWatermark: string;\n  }> {\n    const minWatermark = await this.#storer.getMinWatermarkForCatchup();\n    if (!minWatermark) {\n      this.#lc.warn?.(\n        `Unexpected empty changeLog. Resync if \"Local replica watermark\" errors arise`,\n      );\n    }\n    return {\n      replicaVersion: this.#replicaVersion,\n      minWatermark: minWatermark ?? this.#replicaVersion,\n    };\n  }\n\n  /**\n   * Makes a best effort to purge the change log. In the event of a database\n   * error, exceptions will be logged and swallowed, so this method is safe\n   * to run in a timeout.\n   */\n  async #purgeOldChanges(): Promise<void> {\n    const initial = [...this.#initialWatermarks];\n    if (initial.length === 0) {\n      this.#lc.warn?.('No initial watermarks to check for cleanup'); // Not expected.\n      return;\n    }\n    const current = [...this.#forwarder.getAcks()];\n    if (current.length === 0) {\n      // Also not expected, but possible (e.g. subscriber connects, then disconnects).\n      // Bail to be safe.\n      this.#lc.warn?.('No subscribers to confirm cleanup');\n      return;\n    }\n    try {\n      const earliestInitial = min(...(initial as AtLeastOne<LexiVersion>));\n      const earliestCurrent = min(...(current as AtLeastOne<LexiVersion>));\n      if (earliestCurrent < earliestInitial) {\n        this.#lc.info?.(\n          `At least one client is behind backup (${earliestCurrent} < ${earliestInitial})`,\n        );\n      } else {\n        this.#lc.info?.(`Purging changes before ${earliestInitial} ...`);\n        const start = performance.now();\n        const deleted = await this.#storer.purgeRecordsBefore(earliestInitial);\n        const elapsed = (performance.now() - start).toFixed(2);\n        this.#lc.info?.(\n          `Purged ${deleted} changes before ${earliestInitial} (${elapsed} ms)`,\n        );\n        this.#initialWatermarks.delete(earliestInitial);\n      }\n    } catch (e) {\n      this.#lc.warn?.(`error purging change log`, e);\n    } finally {\n      if (this.#initialWatermarks.size) {\n        // If there are unpurged watermarks to check, schedule the next purge.\n        this.#state.setTimeout(() => this.#purgeOldChanges(), CLEANUP_DELAY_MS);\n      }\n    }\n  }\n\n  async stop(err?: unknown) {\n    this.#state.stop(this.#lc, err);\n    this.#stream?.changes.cancel();\n    await Promise.allSettled([this.#storer.stop(), this.#source.stop()]);\n  }\n}\n\n// The delay between receiving an initial, backup-based watermark\n// and performing a check of whether to purge records before it.\n// This delay should be long enough to handle situations like the following:\n//\n// 1. `litestream restore` downloads a backup for the `replication-manager`\n// 2. `replication-manager` starts up and runs this `change-streamer`\n// 3. `zero-cache`s that are running on a different replica connect to this\n//    `change-streamer` after exponential backoff retries.\n//\n// It is possible for a `zero-cache`[3] to be behind the backup restored [1].\n// This cleanup delay (30 seconds) is thus set to be a value comfortably\n// longer than the max delay for exponential backoff (10 seconds) in\n// `services/running-state.ts`. This allows the `zero-cache` [3] to reconnect\n// so that the `change-streamer` can track its progress and know when it has\n// surpassed the initial watermark of the backup [1].\nconst CLEANUP_DELAY_MS = DEFAULT_MAX_RETRY_DELAY_MS * 3;\n\nconst ROLLBACK_JSON = JSON.stringify([\n  'rollback',\n  {tag: 'rollback'},\n] satisfies Rollback);\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AA+DA,eAAsB,mBACpB,IACA,OACA,QACA,kBACA,mBACA,UACA,cACA,4BACA,mBACA,WACA,WACA,MACA,eAAe,YACiB;CAEhC,MAAM,yBAAyB,IAAI,UAAU,KAAK;CAClD,MAAM,wBACJ,IACA,UACA,mBACA,OACA,WACA,YACF;CAEA,MAAM,EAAC,mBAAkB;CACzB,OAAO,IAAI,mBACT,IACA,OACA,QACA,kBACA,mBACA,UACA,gBACA,cACA,4BACA,WACA,WACA,MACA,YACF;AACF;AAEA,IAAM,8CAA8C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAkKpD,IAAM,qBAAN,MAA0D;CACxD;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAEA;CACA;CACA,qCAA8B,IAAI,IAAY;CAU9C,WAAoB,SAAS;CAE7B,aAAsB,mBACpB,eACA,gBACA,kCACF;CACA,iBAA0B,mBACxB,eACA,WACA,qDACF;CAEA;CACA;CACA;CAEA,YACE,IACA,OACA,QACA,kBACA,mBACA,UACA,gBACA,QACA,4BACA,kBACA,WACA,MACA,eAAe,YACf;EACA,KAAK,KAAK;EACV,KAAKA,MAAM,GAAG,YAAY,aAAa,iBAAiB;EACxD,KAAKC,SAAS;EACd,KAAKC,YAAY;EACjB,KAAKC,kBAAkB;EACvB,KAAKC,UAAU;EACf,KAAKC,UAAU,IAAI,OACjB,IACA,OACA,QACA,kBACA,mBACA,UACA,iBACA,aAAY,KAAKS,SAAS,KAAK,KAAK;GAAC;GAAU,SAAS;GAAI,SAAS;EAAE,CAAC,IACxE,QAAO,KAAK,KAAK,GAAG,GACpB,IACF;EACA,KAAKR,aAAa,IAAI,UAAU,IAAI,EAClC,oCACE,KAAK,mCACT,CAAC;EACD,KAAKC,8BAA8B;EACnC,KAAKQ,aAAa;EAClB,KAAKP,aAAa;EAClB,KAAKC,SAAS,IAAI,aAAa,KAAK,IAAI,KAAA,GAAW,YAAY;EAC/D,KAAKO,gBAAgB,EAAC,KAAK,SAAQ;CACrC;CAEA,MAAM,MAAM;EACV,KAAKhB,IAAI,OAAO,wBAAwB;EAExC,KAAKM,WAAW,qBAAqB;EAErC,MAAM,YAAY,MAAM,KAAKF,QAAQ,iBAAiB;EACtD,IAAI,WACF,KAAKY,cAAc,YAAY;EAKjC,MAAM,KAAKX,QAAQ,gBAAgB,KAAKU,UAAU;EAClD,KAAKA,aAAa;EAIlB,MAAM,sBAAsB,wBAAwB,KAAK;EAEzD,OAAO,KAAKN,OAAO,UAAU,GAAG;GAC9B,IAAI;GACJ,IAAI,YAA2B;GAC/B,IAAI,iBAAiB;GACrB,IAAI;IACF,MAAM,EAAC,eAAe,qBACpB,MAAM,KAAKJ,QAAQ,uCAAuC;IAC5D,MAAM,SAAS,MAAM,KAAKD,QAAQ,YAChC,eACA,gBACF;IACA,KAAKC,QAAQ,IAAI,EAAE,OAAM,MAAK,OAAO,QAAQ,OAAO,CAAC,CAAC;IAEtD,KAAKS,UAAU;IACf,IACE,KAAKL,OAAO,aAAa,IACzB,6CAIA,KAAKF,4BAA4B,QAC/B,KAAKP,KACL,eACA,oBAAoB,eACtB;IAEF,YAAY;IAEZ,WAAW,MAAM,UAAU,OAAO,SAAS;KACzC,MAAM,CAAC,MAAM,OAAO;KACpB,QAAQ,MAAR;MACE,KAAK;OACH,IAAI,IAAI,KACN,KAAKK,QAAQ,OAAO,MAAM;OAE5B,IAAI,IAAI,WAAW;QAKjB,KAAKW,cAAc,YAAY,IAAI;QACnC,KAAKV,WAAW,WAAW,KAAKU,aAAa;OAC/C;OACA;MACF,KAAK;OACH,MAAM,KAAKC,sBAAsB,GAAG;OACpC;MACF,KAAK;OACH,YAAY,OAAO,GAAG;OACtB;MACF,KAAK;OACH,IAAI,cAAc,OAAO,GAAG,WAC1B,MAAM,IAAI,mBACR,oBAAoB,OAAO,GAAG,UAAU,oCAAoC,WAC9E;OAEF,KAAKL,WAAW,IAAI,CAAC;OACrB;MACF;OACE,IAAI,SAAS,QACX,KAAKC,eAAe,IAAI,CAAC;OAE3B,IAAI,cAAc,MAChB,MAAM,IAAI,mBACR,GAAG,KAAK,WAAW,IAAI,IAAI,kCAC7B;OAEF;KACJ;KAEA,MAAM,OAAO,KAAKR,QAAQ,MAAM,WAAW,MAAM;KACjD,MAAM,QAA2B;MAAC;MAAW,OAAO,GAAG;MAAK;KAAI;KAChE,kBAAkB,KAAK;KACvB,IAAI,iBAAiB,qBAEnB,KAAKC,WAAW,QAAQ,KAAK;UACxB;MAQL,MAAM,YAAY,KAAKA,WAAW,uBAAuB,KAAK;MAC9D,MAAM,OAAO,QAAQ,OAAO,SAAS;MACrC,iBAAiB;KACnB;KAEA,IAAI,SAAS,YAAY,SAAS,YAChC,YAAY;KAId,MAAM,eAAe,KAAKD,QAAQ,aAAa;KAC/C,IAAI,cACF,MAAM,OAAO,QAAQ,OAAO,YAAY;IAE5C;GACF,SAAS,GAAG;IACV,MAAM;GACR,UAAU;IACR,KAAKS,SAAS,QAAQ,OAAO;IAC7B,KAAKA,UAAU,KAAA;GACjB;GAGA,IAAI,WAAW;IACb,KAAKd,IAAI,OAAO,oCAAoC,WAAW;IAC/D,KAAKK,QAAQ,MAAM;IACnB,KAAKC,WAAW,QAAQ;KAAC;KAAW;KAAY;IAAa,CAAC;GAChE;GAGA,MAAM,QAAQ,IAAI;IAChB,KAAKD,QAAQ,KAAK;IAClB,KAAKI,OAAO,QAAQ,KAAKT,KAAK,GAAG;IACjC,KAAKS,OAAO,aAAa,8CACrB,qBACE,KAAKT,KACL,uBAAuB,KAAKA,KAAK,eAAe,GAAG,CACrD,IACA;GACN,CAAC;EACH;EAEA,KAAKM,WAAW,oBAAoB;EACpC,KAAKN,IAAI,OAAO,wBAAwB;CAC1C;CAEA,MAAMiB,sBAAsB,KAA6B;EACvD,KAAKjB,IAAI,OAAO,4BAA4B,GAAG;EAC/C,MAAM,EAAC,QAAO;EAEd,QAAQ,KAAR;GACE,KAAK;IACH,MAAM,kBAAkB,KAAKE,WAAW,KAAKD,MAAM;IACnD,MAAM,wBACJ,KAAKD,KACL,eACA,IAAI,WAAW,mBACf,IAAI,YACN;IACA,IAAI,KAAKQ,YAAY;KACnB,KAAKR,IAAI,OAAO,8BAA8B;KAC9C,MAAM,KAAK,KAAK,IAAI,gBAAgB,CAAC;IACvC;IACA;GACF,SACE,YAAY,GAAG;EACnB;CACF;CAEA,UAAU,KAAiD;EACzD,MAAM,EAAC,iBAAiB,IAAI,MAAM,gBAAgB,cAAa;EAC/D,IAAI,SAAS,WACX,KAAKW,SAAS,QAAQ;EAExB,MAAM,aAAa,aAAa,OAAe,EAC7C,eAAe,KAAKL,WAAW,OAAO,UAAU,EAClD,CAAC;EACD,MAAM,aAAa,IAAI,WACrB,iBACA,IACA,WACA,kBACM,KAAKU,aACb;EACA,IAAI,mBAAmB,KAAKb,iBAAiB;GAC3C,KAAKH,IAAI,OACP,2CAA2C,gBAC7C;GACA,WAAW,MACT,GACA,8BACE,KAAKG,gBACN,cAAc,eAAe,EAChC;EACF,OAAO;GACL,KAAKH,IAAI,QAAQ,qBAAqB,WAAW,IAAI;GAErD,KAAKM,WAAW,IAAI,UAAU;GAC9B,KAAKD,QAAQ,QAAQ,YAAY,IAAI;EACvC;EACA,OAAO,QAAQ,QAAQ,UAAU;CACnC;CAEA,gBAAgB,WAAmB;EACjC,MAAM,WAAW,KAAKK,mBAAmB;EACzC,KAAKA,mBAAmB,IAAI,SAAS;EAErC,IAAI,aAAa,GACf,KAAKD,OAAO,iBAAiB,KAAKS,iBAAiB,GAAG,gBAAgB;CAE1E;CAEA,MAAM,oBAGH;EACD,MAAM,eAAe,MAAM,KAAKb,QAAQ,0BAA0B;EAClE,IAAI,CAAC,cACH,KAAKL,IAAI,OACP,8EACF;EAEF,OAAO;GACL,gBAAgB,KAAKG;GACrB,cAAc,gBAAgB,KAAKA;EACrC;CACF;;;;;;CAOA,MAAMe,mBAAkC;EACtC,MAAM,UAAU,CAAC,GAAG,KAAKR,kBAAkB;EAC3C,IAAI,QAAQ,WAAW,GAAG;GACxB,KAAKV,IAAI,OAAO,4CAA4C;GAC5D;EACF;EACA,MAAM,UAAU,CAAC,GAAG,KAAKM,WAAW,QAAQ,CAAC;EAC7C,IAAI,QAAQ,WAAW,GAAG;GAGxB,KAAKN,IAAI,OAAO,mCAAmC;GACnD;EACF;EACA,IAAI;GACF,MAAM,kBAAkB,IAAI,GAAI,OAAmC;GACnE,MAAM,kBAAkB,IAAI,GAAI,OAAmC;GACnE,IAAI,kBAAkB,iBACpB,KAAKA,IAAI,OACP,yCAAyC,gBAAgB,KAAK,gBAAgB,EAChF;QACK;IACL,KAAKA,IAAI,OAAO,0BAA0B,gBAAgB,KAAK;IAC/D,MAAM,QAAQ,YAAY,IAAI;IAC9B,MAAM,UAAU,MAAM,KAAKK,QAAQ,mBAAmB,eAAe;IACrE,MAAM,WAAW,YAAY,IAAI,IAAI,OAAO,QAAQ,CAAC;IACrD,KAAKL,IAAI,OACP,UAAU,QAAQ,kBAAkB,gBAAgB,IAAI,QAAQ,KAClE;IACA,KAAKU,mBAAmB,OAAO,eAAe;GAChD;EACF,SAAS,GAAG;GACV,KAAKV,IAAI,OAAO,4BAA4B,CAAC;EAC/C,UAAU;GACR,IAAI,KAAKU,mBAAmB,MAE1B,KAAKD,OAAO,iBAAiB,KAAKS,iBAAiB,GAAG,gBAAgB;EAE1E;CACF;CAEA,MAAM,KAAK,KAAe;EACxB,KAAKT,OAAO,KAAK,KAAKT,KAAK,GAAG;EAC9B,KAAKc,SAAS,QAAQ,OAAO;EAC7B,MAAM,QAAQ,WAAW,CAAC,KAAKT,QAAQ,KAAK,GAAG,KAAKD,QAAQ,KAAK,CAAC,CAAC;CACrE;AACF;AAiBA,IAAM,mBAAmB,6BAA6B;AAEtD,IAAM,gBAAgB,KAAK,UAAU,CACnC,YACA,EAAC,KAAK,WAAU,CAClB,CAAoB"}

package/out/zero-cache/src/services/change-streamer/change-streamer.d.ts

@@ -361,6 +361,7 @@ export declare const downstreamSchema: v.UnionType<[v.TupleType<[v.Type<"status"
     }>;
 }, undefined>]>]>]>, v.TupleType<[v.Type<"commit">, v.ObjectType<{
     tag: v.Type<"commit">;
+    commitTimeMs: v.Optional<number>;
 }, undefined>, v.ObjectType<{
     watermark: v.Type<string>;
 }, undefined>]>, v.TupleType<[v.Type<"rollback">, v.ObjectType<{

package/out/zero-cache/src/services/change-streamer/change-streamer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"change-streamer.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,IAAI,EAAC,MAAM,gCAAgC,CAAC;AACzD,OAAO,KAAK,CAAC,MAAM,kCAAkC,CAAC;AACtD,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,wBAAwB,CAAC;AAEnD,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,6BAA6B,CAAC;AAEhE,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,eAAe,CAAC;AAC3C,OAAO,KAAK,SAAS,MAAM,sBAAsB,CAAC;AAElD,KAAK,SAAS,GAAG,IAAI,CAAC,OAAO,SAAS,CAAC,CAAC;AAExC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AACH,MAAM,WAAW,cAAc;IAC7B;;;;OAIG;IACH,SAAS,CAAC,GAAG,EAAE,iBAAiB,GAAG,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC;CAChE;AA0BD,eAAO,MAAM,gBAAgB,IAAI,CAAC;AAElC,MAAM,MAAM,iBAAiB,GAAG;IAC9B;;OAEG;IACH,eAAe,EAAE,MAAM,CAAC;IAExB;;;OAGG;IACH,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IAEtB;;OAEG;IACH,EAAE,EAAE,MAAM,CAAC;IAEX;;;;OAIG;IACH,IAAI,EAAE,cAAc,CAAC;IAErB;;;;OAIG;IACH,cAAc,EAAE,MAAM,CAAC;IAEvB;;;OAGG;IACH,SAAS,EAAE,MAAM,CAAC;IAElB;;;;;OAKG;IACH,OAAO,EAAE,OAAO,CAAC;CAClB,CAAC;AAEF;;;GAGG;AACH,eAAO,MAAM,YAAY;;;;;;;;;;aASvB,CAAC;AAEH,MAAM,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,YAAY,CAAC,CAAC;AAElD,eAAO,MAAM,mBAAmB;;;;;;;;;;eAA+C,CAAC;AAEhF;;;;;GAKG;AACH,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,mBAAmB,CAAC,CAAC;AAEhE,QAAA,MAAM,uBAAuB;;;aAG3B,CAAC;AAEH,MAAM,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC;AAExE,QAAA,MAAM,WAAW;;;eAAyD,CAAC;AAE3E,eAAO,MAAM,gBAAgB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAI5B,CAAC;AAEF,MAAM,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,WAAW,CAAC,CAAC;AAEhD,wBAAgB,uBAAuB,CAAC,GAAG,EAAE,SAAS,yDAWrD;AAED;;;;;;;;;;GAUG;AACH,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAE1D,MAAM,WAAW,qBACf,SAAQ,IAAI,CAAC,cAAc,EAAE,WAAW,CAAC,EAAE,OAAO;IAClD;;;OAGG;IACH,SAAS,CAAC,GAAG,EAAE,iBAAiB,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;IAE3D;;;;OAIG;IACH,eAAe,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IAEzC,iBAAiB,IAAI,OAAO,CAAC;QAC3B,cAAc,EAAE,MAAM,CAAC;QACvB,YAAY,EAAE,MAAM,CAAC;KACtB,CAAC,CAAC;CACJ"}
+{"version":3,"file":"change-streamer.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,IAAI,EAAC,MAAM,gCAAgC,CAAC;AACzD,OAAO,KAAK,CAAC,MAAM,kCAAkC,CAAC;AACtD,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,wBAAwB,CAAC;AAEnD,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,6BAA6B,CAAC;AAEhE,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,eAAe,CAAC;AAC3C,OAAO,KAAK,SAAS,MAAM,sBAAsB,CAAC;AAElD,KAAK,SAAS,GAAG,IAAI,CAAC,OAAO,SAAS,CAAC,CAAC;AAExC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AACH,MAAM,WAAW,cAAc;IAC7B;;;;OAIG;IACH,SAAS,CAAC,GAAG,EAAE,iBAAiB,GAAG,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC;CAChE;AAgCD,eAAO,MAAM,gBAAgB,IAAI,CAAC;AAElC,MAAM,MAAM,iBAAiB,GAAG;IAC9B;;OAEG;IACH,eAAe,EAAE,MAAM,CAAC;IAExB;;;OAGG;IACH,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IAEtB;;OAEG;IACH,EAAE,EAAE,MAAM,CAAC;IAEX;;;;OAIG;IACH,IAAI,EAAE,cAAc,CAAC;IAErB;;;;OAIG;IACH,cAAc,EAAE,MAAM,CAAC;IAEvB;;;OAGG;IACH,SAAS,EAAE,MAAM,CAAC;IAElB;;;;;OAKG;IACH,OAAO,EAAE,OAAO,CAAC;CAClB,CAAC;AAEF;;;GAGG;AACH,eAAO,MAAM,YAAY;;;;;;;;;;aASvB,CAAC;AAEH,MAAM,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,YAAY,CAAC,CAAC;AAElD,eAAO,MAAM,mBAAmB;;;;;;;;;;eAA+C,CAAC;AAEhF;;;;;GAKG;AACH,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,mBAAmB,CAAC,CAAC;AAEhE,QAAA,MAAM,uBAAuB;;;aAG3B,CAAC;AAEH,MAAM,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC;AAExE,QAAA,MAAM,WAAW;;;eAAyD,CAAC;AAE3E,eAAO,MAAM,gBAAgB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAI5B,CAAC;AAEF,MAAM,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,WAAW,CAAC,CAAC;AAEhD,wBAAgB,uBAAuB,CAAC,GAAG,EAAE,SAAS,yDAWrD;AAED;;;;;;;;;;GAUG;AACH,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAE1D,MAAM,WAAW,qBACf,SAAQ,IAAI,CAAC,cAAc,EAAE,WAAW,CAAC,EAAE,OAAO;IAClD;;;OAGG;IACH,SAAS,CAAC,GAAG,EAAE,iBAAiB,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;IAE3D;;;;OAIG;IACH,eAAe,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IAEzC,iBAAiB,IAAI,OAAO,CAAC;QAC3B,cAAc,EAAE,MAAM,CAAC;QACvB,YAAY,EAAE,MAAM,CAAC;KACtB,CAAC,CAAC;CACJ"}

package/out/zero-cache/src/services/change-streamer/change-streamer.js.map

@@ -1 +1 @@
-{"version":3,"file":"change-streamer.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer.ts"],"sourcesContent":["import type {Enum} from '../../../../shared/src/enum.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {changeStreamDataSchema} from '../change-source/protocol/current/downstream.ts';\nimport type {ReplicatorMode} from '../replicator/replicator.ts';\nimport {changeSourceTimingsSchema} from '../replicator/reporter/report-schema.ts';\nimport type {Service} from '../service.ts';\nimport * as ErrorType from './error-type-enum.ts';\n\ntype ErrorType = Enum<typeof ErrorType>;\n\n/**\n * The ChangeStreamer is the component between replicators (\"subscribers\")\n * and a canonical upstream source of changes (e.g. a Postgres logical\n * replication slot). It facilitates multiple subscribers without incurring\n * the associated upstream expense (e.g. PG replication slots are resource\n * intensive) with a \"forward-store-ack\" procedure.\n *\n * * Changes from the upstream source are immediately **forwarded** to\n *   connected subscribers to minimize latency.\n *\n * * They are then **stored** in a separate DB to facilitate catchup\n *   of connecting subscribers that are behind.\n *\n * * **Acknowledgements** are sent upstream after they are successfully\n *   stored.\n *\n * Unlike Postgres replication slots, in which the progress of a static\n * subscriber is tracked in the replication slot, the ChangeStreamer\n * supports a dynamic set of subscribers (i.e.. zero-caches) that can\n * can continually change.\n *\n * However, it is not the case that the ChangeStreamer needs to support\n * arbitrarily old subscribers. Because the replica is continually\n * backed up to a global location and used to initialize new subscriber\n * tasks, an initial subscription request from a subscriber constitutes\n * a signal for how \"behind\" a new subscriber task can be. This is\n * reflected in the {@link SubscriberContext}, which indicates whether\n * the watermark corresponds to an \"initial\" watermark derived from the\n * replica at task startup.\n *\n * The ChangeStreamer uses a combination of this signal with ACK\n * responses from connected subscribers to determine the watermark up\n * to which it is safe to purge old change log entries.\n */\nexport interface ChangeStreamer {\n  /**\n   * Subscribes to changes based on the supplied subscriber `ctx`,\n   * which indicates the watermark at which the subscriber is up to\n   * date.\n   */\n  subscribe(ctx: SubscriberContext): Promise<Source<Downstream>>;\n}\n\n// v1: v0.18\n//   - Client-side support for JSON_FORMAT. Introduced in 0.18.\n// v2: v0.19\n//   - Adds the \"status\" message which is initially used to signal that the\n//     subscription is valid (i.e. starting at the requested watermark).\n// v3: v0.20\n//   - Adds the \"taskID\" to the subscription context, and support for\n//     the backup monitor-mediated \"/snapshot\" request.\n// v4: v0.25\n//   - Adds the \"replicaVersion\" and \"minWatermark\" fields to the \"/snapshot\"\n//     status request so that a subscriber can verify whether its replica,\n//     whether it be restored or existing in a permanent volume, is compatible\n//     with the change-streamer.\n// v5: v0.26\n//   - Moves relation.keyColumns and relation.replicaIdentity to\n//     relation.rowKey: { columns, type }.\n//   - Adds `metadata` to `create-table` message\n//   - Adds `tableMetadata` to `add-column` message\n//   - Adds `table-update-metadata` message\n// v6: v0.26\n//   - Adds support for `backfill` messages\n// v6: v1.0.1  (backwards compatible, no version change)\n//   - Adds lag reporting to status messages\n\nexport const PROTOCOL_VERSION = 6;\n\nexport type SubscriberContext = {\n  /**\n   * The supported change-streamer protocol version.\n   */\n  protocolVersion: number;\n\n  /**\n   * Task ID. This is used to link the request with a preceding snapshot\n   * reservation.\n   */\n  taskID: string | null; // TODO: Make required when v3 is min.\n\n  /**\n   * Subscriber id. This is only used for debugging.\n   */\n  id: string;\n\n  /**\n   * The ReplicatorMode of the subscriber. 'backup' indicates that the\n   * subscriber is local to the `change-streamer` in the `replication-manager`,\n   * while 'serving' indicates that user-facing requests depend on the subscriber.\n   */\n  mode: ReplicatorMode;\n\n  /**\n   * The ChangeStreamer will return an Error if the subscriber is\n   * on a different replica version (i.e. the initial snapshot associated\n   * with the replication slot).\n   */\n  replicaVersion: string;\n\n  /**\n   * The watermark up to which the subscriber is up to date.\n   * Only changes after the watermark will be streamed.\n   */\n  watermark: string;\n\n  /**\n   * Whether this is the first subscription request made by the task,\n   * i.e. indicating that the watermark comes from a restored replica\n   * backup. The ChangeStreamer uses this to determine which changes\n   * are safe to purge from the Storer.\n   */\n  initial: boolean;\n};\n\n/**\n * The StatusMessage payload for now is empty, but can be extended to\n * include meta-level information in the future.\n */\nexport const statusSchema = v.object({\n  tag: v.literal('status'),\n\n  lagReport: v\n    .object({\n      lastTimings: changeSourceTimingsSchema.optional(),\n      nextSendTimeMs: v.number(),\n    })\n    .optional(),\n});\n\nexport type Status = v.Infer<typeof statusSchema>;\n\nexport const statusMessageSchema = v.tuple([v.literal('status'), statusSchema]);\n\n/**\n * A StatusMessage will be immediately sent on a (v2+) subscription to\n * indicate that the subscription is valid (i.e. starting at the requested\n * watermark). Invalid subscriptions will instead result in a\n * SubscriptionError as the first message.\n */\nexport type StatusMessage = v.Infer<typeof statusMessageSchema>;\n\nconst subscriptionErrorSchema = v.object({\n  type: v.number(), // ErrorType\n  message: v.string().optional(),\n});\n\nexport type SubscriptionError = v.Infer<typeof subscriptionErrorSchema>;\n\nconst errorSchema = v.tuple([v.literal('error'), subscriptionErrorSchema]);\n\nexport const downstreamSchema = v.union(\n  statusMessageSchema,\n  changeStreamDataSchema,\n  errorSchema,\n);\n\nexport type Error = v.Infer<typeof errorSchema>;\n\nexport function errorTypeToReadableName(val: ErrorType) {\n  switch (val) {\n    case ErrorType.WrongReplicaVersion:\n      return 'WrongReplicaVersion';\n    case ErrorType.WatermarkTooOld:\n      return 'WatermarkTooOld';\n    case ErrorType.Unknown:\n      return 'Unknown';\n    default:\n      return 'Unknown';\n  }\n}\n\n/**\n * A stream of transactions, each starting with a {@link Begin} message,\n * containing one or more {@link Data} messages, and ending with a\n * {@link Commit} or {@link Rollback} message. The 'commit' tuple\n * includes a `watermark` that should be stored with the committed\n * data and used for resuming a subscription (e.g. in the\n * {@link SubscriberContext}).\n *\n * A {@link SubscriptionError} indicates an unrecoverable error that requires\n * manual intervention (e.g. configuration / operational error).\n */\nexport type Downstream = v.Infer<typeof downstreamSchema>;\n\nexport interface ChangeStreamerService\n  extends Omit<ChangeStreamer, 'subscribe'>, Service {\n  /**\n   * The server-side interface overrides `subscribe()` to return a stream\n   * of already-stringified {@link Downstream} payloads.\n   */\n  subscribe(ctx: SubscriberContext): Promise<Source<string>>;\n\n  /**\n   * Notifies the change streamer of a watermark that has been backed up,\n   * indicating that changes before the watermark can be purged if active\n   * subscribers have progressed beyond the watermark.\n   */\n  scheduleCleanup(watermark: string): void;\n\n  getChangeLogState(): Promise<{\n    replicaVersion: string;\n    minWatermark: string;\n  }>;\n}\n"],"mappings":";;;;;;;;AAkIA,IAAa,eAAe,eAAE,OAAO;CACnC,KAAK,eAAE,QAAQ,QAAQ;CAEvB,WAAW,eACR,OAAO;EACN,aAAa,0BAA0B,SAAS;EAChD,gBAAgB,eAAE,OAAO;CAC3B,CAAC,EACA,SAAS;AACd,CAAC;AAID,IAAa,sBAAsB,eAAE,MAAM,CAAC,eAAE,QAAQ,QAAQ,GAAG,YAAY,CAAC;AAU9E,IAAM,0BAA0B,eAAE,OAAO;CACvC,MAAM,eAAE,OAAO;CACf,SAAS,eAAE,OAAO,EAAE,SAAS;AAC/B,CAAC;AAID,IAAM,cAAc,eAAE,MAAM,CAAC,eAAE,QAAQ,OAAO,GAAG,uBAAuB,CAAC;AAEzE,IAAa,mBAAmB,eAAE,MAChC,qBACA,wBACA,WACF;AAIA,SAAgB,wBAAwB,KAAgB;CACtD,QAAQ,KAAR;EACE,KAAK,GACH,OAAO;EACT,KAAK,GACH,OAAO;EACT,KAAK,GACH,OAAO;EACT,SACE,OAAO;CACX;AACF"}
+{"version":3,"file":"change-streamer.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer.ts"],"sourcesContent":["import type {Enum} from '../../../../shared/src/enum.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {changeStreamDataSchema} from '../change-source/protocol/current/downstream.ts';\nimport type {ReplicatorMode} from '../replicator/replicator.ts';\nimport {changeSourceTimingsSchema} from '../replicator/reporter/report-schema.ts';\nimport type {Service} from '../service.ts';\nimport * as ErrorType from './error-type-enum.ts';\n\ntype ErrorType = Enum<typeof ErrorType>;\n\n/**\n * The ChangeStreamer is the component between replicators (\"subscribers\")\n * and a canonical upstream source of changes (e.g. a Postgres logical\n * replication slot). It facilitates multiple subscribers without incurring\n * the associated upstream expense (e.g. PG replication slots are resource\n * intensive) with a \"forward-store-ack\" procedure.\n *\n * * Changes from the upstream source are immediately **forwarded** to\n *   connected subscribers to minimize latency.\n *\n * * They are then **stored** in a separate DB to facilitate catchup\n *   of connecting subscribers that are behind.\n *\n * * **Acknowledgements** are sent upstream after they are successfully\n *   stored.\n *\n * Unlike Postgres replication slots, in which the progress of a static\n * subscriber is tracked in the replication slot, the ChangeStreamer\n * supports a dynamic set of subscribers (i.e.. zero-caches) that can\n * can continually change.\n *\n * However, it is not the case that the ChangeStreamer needs to support\n * arbitrarily old subscribers. Because the replica is continually\n * backed up to a global location and used to initialize new subscriber\n * tasks, an initial subscription request from a subscriber constitutes\n * a signal for how \"behind\" a new subscriber task can be. This is\n * reflected in the {@link SubscriberContext}, which indicates whether\n * the watermark corresponds to an \"initial\" watermark derived from the\n * replica at task startup.\n *\n * The ChangeStreamer uses a combination of this signal with ACK\n * responses from connected subscribers to determine the watermark up\n * to which it is safe to purge old change log entries.\n */\nexport interface ChangeStreamer {\n  /**\n   * Subscribes to changes based on the supplied subscriber `ctx`,\n   * which indicates the watermark at which the subscriber is up to\n   * date.\n   */\n  subscribe(ctx: SubscriberContext): Promise<Source<Downstream>>;\n}\n\n// v1: v0.18\n//   - Client-side support for JSON_FORMAT. Introduced in 0.18.\n// v2: v0.19\n//   - Adds the \"status\" message which is initially used to signal that the\n//     subscription is valid (i.e. starting at the requested watermark).\n// v3: v0.20\n//   - Adds the \"taskID\" to the subscription context, and support for\n//     the backup monitor-mediated \"/snapshot\" request.\n// v4: v0.25\n//   - Adds the \"replicaVersion\" and \"minWatermark\" fields to the \"/snapshot\"\n//     status request so that a subscriber can verify whether its replica,\n//     whether it be restored or existing in a permanent volume, is compatible\n//     with the change-streamer.\n// v5: v0.26\n//   - Moves relation.keyColumns and relation.replicaIdentity to\n//     relation.rowKey: { columns, type }.\n//   - Adds `metadata` to `create-table` message\n//   - Adds `tableMetadata` to `add-column` message\n//   - Adds `table-update-metadata` message\n// v6: v0.26\n//   - Adds support for `backfill` messages\n// v6: v1.0.1  (backwards compatible, no version change)\n//   - Adds lag reporting to status messages\n// v6: (backwards compatible, no version change)\n//   - Adds the optional `commitTimeMs` field to `commit` messages, carrying\n//     the upstream commit timestamp for end-to-end serving lag measurement.\n//     The stream is parsed in 'passthrough' mode, so an older peer ignores the\n//     field, and a newer peer treats its absence (including in changes\n//     replayed from the Change DB) as \"no commit time reported\".\n\nexport const PROTOCOL_VERSION = 6;\n\nexport type SubscriberContext = {\n  /**\n   * The supported change-streamer protocol version.\n   */\n  protocolVersion: number;\n\n  /**\n   * Task ID. This is used to link the request with a preceding snapshot\n   * reservation.\n   */\n  taskID: string | null; // TODO: Make required when v3 is min.\n\n  /**\n   * Subscriber id. This is only used for debugging.\n   */\n  id: string;\n\n  /**\n   * The ReplicatorMode of the subscriber. 'backup' indicates that the\n   * subscriber is local to the `change-streamer` in the `replication-manager`,\n   * while 'serving' indicates that user-facing requests depend on the subscriber.\n   */\n  mode: ReplicatorMode;\n\n  /**\n   * The ChangeStreamer will return an Error if the subscriber is\n   * on a different replica version (i.e. the initial snapshot associated\n   * with the replication slot).\n   */\n  replicaVersion: string;\n\n  /**\n   * The watermark up to which the subscriber is up to date.\n   * Only changes after the watermark will be streamed.\n   */\n  watermark: string;\n\n  /**\n   * Whether this is the first subscription request made by the task,\n   * i.e. indicating that the watermark comes from a restored replica\n   * backup. The ChangeStreamer uses this to determine which changes\n   * are safe to purge from the Storer.\n   */\n  initial: boolean;\n};\n\n/**\n * The StatusMessage payload for now is empty, but can be extended to\n * include meta-level information in the future.\n */\nexport const statusSchema = v.object({\n  tag: v.literal('status'),\n\n  lagReport: v\n    .object({\n      lastTimings: changeSourceTimingsSchema.optional(),\n      nextSendTimeMs: v.number(),\n    })\n    .optional(),\n});\n\nexport type Status = v.Infer<typeof statusSchema>;\n\nexport const statusMessageSchema = v.tuple([v.literal('status'), statusSchema]);\n\n/**\n * A StatusMessage will be immediately sent on a (v2+) subscription to\n * indicate that the subscription is valid (i.e. starting at the requested\n * watermark). Invalid subscriptions will instead result in a\n * SubscriptionError as the first message.\n */\nexport type StatusMessage = v.Infer<typeof statusMessageSchema>;\n\nconst subscriptionErrorSchema = v.object({\n  type: v.number(), // ErrorType\n  message: v.string().optional(),\n});\n\nexport type SubscriptionError = v.Infer<typeof subscriptionErrorSchema>;\n\nconst errorSchema = v.tuple([v.literal('error'), subscriptionErrorSchema]);\n\nexport const downstreamSchema = v.union(\n  statusMessageSchema,\n  changeStreamDataSchema,\n  errorSchema,\n);\n\nexport type Error = v.Infer<typeof errorSchema>;\n\nexport function errorTypeToReadableName(val: ErrorType) {\n  switch (val) {\n    case ErrorType.WrongReplicaVersion:\n      return 'WrongReplicaVersion';\n    case ErrorType.WatermarkTooOld:\n      return 'WatermarkTooOld';\n    case ErrorType.Unknown:\n      return 'Unknown';\n    default:\n      return 'Unknown';\n  }\n}\n\n/**\n * A stream of transactions, each starting with a {@link Begin} message,\n * containing one or more {@link Data} messages, and ending with a\n * {@link Commit} or {@link Rollback} message. The 'commit' tuple\n * includes a `watermark` that should be stored with the committed\n * data and used for resuming a subscription (e.g. in the\n * {@link SubscriberContext}).\n *\n * A {@link SubscriptionError} indicates an unrecoverable error that requires\n * manual intervention (e.g. configuration / operational error).\n */\nexport type Downstream = v.Infer<typeof downstreamSchema>;\n\nexport interface ChangeStreamerService\n  extends Omit<ChangeStreamer, 'subscribe'>, Service {\n  /**\n   * The server-side interface overrides `subscribe()` to return a stream\n   * of already-stringified {@link Downstream} payloads.\n   */\n  subscribe(ctx: SubscriberContext): Promise<Source<string>>;\n\n  /**\n   * Notifies the change streamer of a watermark that has been backed up,\n   * indicating that changes before the watermark can be purged if active\n   * subscribers have progressed beyond the watermark.\n   */\n  scheduleCleanup(watermark: string): void;\n\n  getChangeLogState(): Promise<{\n    replicaVersion: string;\n    minWatermark: string;\n  }>;\n}\n"],"mappings":";;;;;;;;AAwIA,IAAa,eAAe,eAAE,OAAO;CACnC,KAAK,eAAE,QAAQ,QAAQ;CAEvB,WAAW,eACR,OAAO;EACN,aAAa,0BAA0B,SAAS;EAChD,gBAAgB,eAAE,OAAO;CAC3B,CAAC,EACA,SAAS;AACd,CAAC;AAID,IAAa,sBAAsB,eAAE,MAAM,CAAC,eAAE,QAAQ,QAAQ,GAAG,YAAY,CAAC;AAU9E,IAAM,0BAA0B,eAAE,OAAO;CACvC,MAAM,eAAE,OAAO;CACf,SAAS,eAAE,OAAO,EAAE,SAAS;AAC/B,CAAC;AAID,IAAM,cAAc,eAAE,MAAM,CAAC,eAAE,QAAQ,OAAO,GAAG,uBAAuB,CAAC;AAEzE,IAAa,mBAAmB,eAAE,MAChC,qBACA,wBACA,WACF;AAIA,SAAgB,wBAAwB,KAAgB;CACtD,QAAQ,KAAR;EACE,KAAK,GACH,OAAO;EACT,KAAK,GACH,OAAO;EACT,KAAK,GACH,OAAO;EACT,SACE,OAAO;CACX;AACF"}

package/out/zero-cache/src/services/change-streamer/storer.d.ts

@@ -12,6 +12,7 @@ export type TuningOptions = {
     backPressureLimitHeapProportion: number;
     statementTimeoutMs: number;
     changeLogBatchSize: number;
+    drainTimeoutMs?: number | undefined;
 };
 /**
  * Handles the storage of changes and the catchup of subscribers
@@ -46,7 +47,7 @@ export type TuningOptions = {
 export declare class Storer implements Service {
     #private;
     readonly id = "storer";
-    constructor(lc: LogContext, shard: ShardID, taskID: string, discoveryAddress: string, discoveryProtocol: string, db: PostgresDB, replicaVersion: string, onConsumed: (c: Commit | UpstreamStatusMessage) => void, onFatal: (err: Error) => void, { backPressureLimitHeapProportion, statementTimeoutMs, changeLogBatchSize, }: TuningOptions);
+    constructor(lc: LogContext, shard: ShardID, taskID: string, discoveryAddress: string, discoveryProtocol: string, db: PostgresDB, replicaVersion: string, onConsumed: (c: Commit | UpstreamStatusMessage) => void, onFatal: (err: Error) => void, { backPressureLimitHeapProportion, statementTimeoutMs, changeLogBatchSize, drainTimeoutMs, }: TuningOptions);
     assumeOwnership(purgeLock?: PurgeLock | null): Promise<void>;
     getStartStreamInitializationParameters(): Promise<{
         lastWatermark: string;
@@ -72,6 +73,11 @@ export declare class Storer implements Service {
      * This is only used in tests.
      */
     allProcessed(): Promise<void>;
+    /**
+     * Stops the storer and waits up to a drain timeout for entries to drain,
+     * throwing an exception if it doesn't drain in time, in order to abort
+     * the server when PG (or the connection to it) appears to be wedged.
+     */
     stop(): Promise<void>;
 }
 /**