@rocicorp/zero 0.25.0-canary.0 → 0.25.0-canary.10

package/out/zero-cache/src/services/change-streamer/change-streamer-service.js.map

@@ -1 +1 @@
-{"version":3,"file":"change-streamer-service.js","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-service.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAC,QAAQ,EAAC,MAAM,oBAAoB,CAAC;AAC5C,OAAO,EAAC,WAAW,EAAC,MAAM,mCAAmC,CAAC;AAC9D,OAAO,EAAC,kBAAkB,EAAC,MAAM,gCAAgC,CAAC;AAClE,OAAO,EACL,GAAG,GAGJ,MAAM,6BAA6B,CAAC;AAIrC,OAAO,EAAC,YAAY,EAAC,MAAM,6BAA6B,CAAC;AACzD,OAAO,EAIN,MAAM,iDAAiD,CAAC;AAGzD,OAAO,EACL,0BAA0B,EAC1B,YAAY,EACZ,kBAAkB,GACnB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAIN,MAAM,sBAAsB,CAAC;AAC9B,OAAO,KAAK,SAAS,MAAM,sBAAsB,CAAC;AAClD,OAAO,EAAC,SAAS,EAAC,MAAM,gBAAgB,CAAC;AACzC,OAAO,EAAC,wBAAwB,EAAC,MAAM,kBAAkB,CAAC;AAC1D,OAAO,EACL,eAAe,EACf,uBAAuB,EACvB,iBAAiB,GAClB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAC,MAAM,EAAC,MAAM,aAAa,CAAC;AACnC,OAAO,EAAC,UAAU,EAAC,MAAM,iBAAiB,CAAC;AAE3C;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,EAAc,EACd,KAAc,EACd,MAAc,EACd,gBAAwB,EACxB,iBAAyB,EACzB,QAAoB,EACpB,YAA0B,EAC1B,iBAAoC,EACpC,SAAkB,EAClB,YAAY,GAAG,UAAU;IAEzB,wCAAwC;IACxC,MAAM,wBAAwB,CAAC,EAAE,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC;IACpD,MAAM,uBAAuB,CAC3B,EAAE,EACF,QAAQ,EACR,iBAAiB,EACjB,KAAK,EACL,SAAS,CACV,CAAC;IAEF,MAAM,EAAC,cAAc,EAAC,GAAG,iBAAiB,CAAC;IAC3C,OAAO,IAAI,kBAAkB,CAC3B,EAAE,EACF,KAAK,EACL,MAAM,EACN,gBAAgB,EAChB,iBAAiB,EACjB,QAAQ,EACR,cAAc,EACd,YAAY,EACZ,SAAS,EACT,YAAY,CACb,CAAC;AACJ,CAAC;AAkCD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyIG;AACH,MAAM,kBAAkB;IACb,EAAE,CAAS;IACX,GAAG,CAAa;IAChB,MAAM,CAAU;IAChB,SAAS,CAAa;IACtB,eAAe,CAAS;IACxB,OAAO,CAAe;IACtB,OAAO,CAAS;IAChB,UAAU,CAAY;IAEtB,UAAU,CAAU;IACpB,MAAM,CAAe;IACrB,kBAAkB,GAAG,IAAI,GAAG,EAAU,CAAC;IAEhD,uEAAuE;IACvE,mEAAmE;IACnE,wEAAwE;IACxE,qEAAqE;IACrE,EAAE;IACF,wEAAwE;IACxE,oEAAoE;IACpE,0EAA0E;IACjE,QAAQ,GAAG,QAAQ,EAAE,CAAC;IAEtB,UAAU,GAAG,kBAAkB,CACtC,aAAa,EACb,cAAc,EACd,kCAAkC,CACnC,CAAC;IAEF,OAAO,CAA2B;IAElC,YACE,EAAc,EACd,KAAc,EACd,MAAc,EACd,gBAAwB,EACxB,iBAAyB,EACzB,QAAoB,EACpB,cAAsB,EACtB,MAAoB,EACpB,SAAkB,EAClB,YAAY,GAAG,UAAU;QAEzB,IAAI,CAAC,EAAE,GAAG,iBAAiB,CAAC;QAC5B,IAAI,CAAC,GAAG,GAAG,EAAE,CAAC,WAAW,CAAC,WAAW,EAAE,iBAAiB,CAAC,CAAC;QAC1D,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC;QACpB,IAAI,CAAC,SAAS,GAAG,QAAQ,CAAC;QAC1B,IAAI,CAAC,eAAe,GAAG,cAAc,CAAC;QACtC,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;QACtB,IAAI,CAAC,OAAO,GAAG,IAAI,MAAM,CACvB,EAAE,EACF,KAAK,EACL,MAAM,EACN,gBAAgB,EAChB,iBAAiB,EACjB,QAAQ,EACR,cAAc,EACd,QAAQ,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,EACzE,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CACtB,CAAC;QACF,IAAI,CAAC,UAAU,GAAG,IAAI,SAAS,EAAE,CAAC;QAClC,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;QAC5B,IAAI,CAAC,MAAM,GAAG,IAAI,YAAY,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,YAAY,CAAC,CAAC;IACnE,CAAC;IAED,KAAK,CAAC,GAAG;QACP,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,wBAAwB,CAAC,CAAC;QAE1C,mEAAmE;QACnE,kCAAkC;QAClC,MAAM,IAAI,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;QACrC,iEAAiE;QACjE,qCAAqC;QACrC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QAE5C,OAAO,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,EAAE,CAAC;YAC/B,IAAI,GAAY,CAAC;YACjB,IAAI,SAAS,GAAkB,IAAI,CAAC;YACpC,IAAI,CAAC;gBACH,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,6BAA6B,EAAE,CAAC;gBACtE,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;gBAC1D,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;gBACtB,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC;gBAC3B,SAAS,GAAG,IAAI,CAAC;gBAEjB,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;oBAC1C,MAAM,CAAC,IAAI,EAAE,GAAG,CAAC,GAAG,MAAM,CAAC;oBAC3B,QAAQ,IAAI,EAAE,CAAC;wBACb,KAAK,QAAQ;4BACX,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,6CAA6C;4BAC1E,SAAS;wBACX,KAAK,SAAS;4BACZ,MAAM,IAAI,CAAC,qBAAqB,CAAC,GAAG,CAAC,CAAC;4BACtC,SAAS,CAAC,4CAA4C;wBACxD,KAAK,OAAO;4BACV,SAAS,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC;4BACtC,MAAM;wBACR,KAAK,QAAQ;4BACX,IAAI,SAAS,KAAK,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC;gCACtC,MAAM,IAAI,kBAAkB,CAC1B,oBAAoB,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS,qCAAqC,SAAS,EAAE,CACxF,CAAC;4BACJ,CAAC;4BACD,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;4BACvB,MAAM;wBACR;4BACE,IAAI,SAAS,KAAK,IAAI,EAAE,CAAC;gCACvB,MAAM,IAAI,kBAAkB,CAC1B,GAAG,IAAI,YAAY,GAAG,CAAC,GAAG,mCAAmC,CAC9D,CAAC;4BACJ,CAAC;4BACD,MAAM;oBACV,CAAC;oBAED,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC;oBACxC,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC;oBAE7C,IAAI,IAAI,KAAK,QAAQ,EAAE,CAAC;wBACtB,SAAS,GAAG,IAAI,CAAC;oBACnB,CAAC;oBAED,2CAA2C;oBAC3C,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,YAAY,EAAE,CAAC;oBACjD,IAAI,YAAY,EAAE,CAAC;wBACjB,MAAM,YAAY,CAAC;oBACrB,CAAC;gBACH,CAAC;YACH,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,GAAG,GAAG,CAAC,CAAC;YACV,CAAC;oBAAS,CAAC;gBACT,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;gBAC/B,IAAI,CAAC,OAAO,GAAG,SAAS,CAAC;YAC3B,CAAC;YAED,wEAAwE;YACxE,IAAI,SAAS,EAAE,CAAC;gBACd,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,oCAAoC,SAAS,EAAE,CAAC,CAAC;gBACjE,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;gBACrB,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,SAAS,EAAE,CAAC,UAAU,EAAE,EAAC,GAAG,EAAE,UAAU,EAAC,CAAC,CAAC,CAAC,CAAC;YACxE,CAAC;YAED,MAAM,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;QAC3C,CAAC;QACD,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,wBAAwB,CAAC,CAAC;IAC5C,CAAC;IAED,KAAK,CAAC,qBAAqB,CAAC,GAA2B;QACrD,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,0BAA0B,EAAE,GAAG,CAAC,CAAC;QACjD,MAAM,EAAC,GAAG,EAAC,GAAG,GAAG,CAAC;QAElB,QAAQ,GAAG,EAAE,CAAC;YACZ,KAAK,gBAAgB;gBACnB,MAAM,iBAAiB,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;gBACrD,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;oBACpB,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,8BAA8B,CAAC,CAAC;oBAChD,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,eAAe,EAAE,CAAC,CAAC;gBACzC,CAAC;gBACD,MAAM;YACR;gBACE,WAAW,CAAC,GAAG,CAAC,CAAC;QACrB,CAAC;IACH,CAAC;IAED,SAAS,CAAC,GAAsB;QAC9B,MAAM,EAAC,eAAe,EAAE,EAAE,EAAE,IAAI,EAAE,cAAc,EAAE,SAAS,EAAE,OAAO,EAAC,GAAG,GAAG,CAAC;QAC5E,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YACvB,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC;QAC1B,CAAC;QACD,MAAM,UAAU,GAAG,YAAY,CAAC,MAAM,CAAa;YACjD,OAAO,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,UAAU,CAAC;SAClD,CAAC,CAAC;QACH,MAAM,UAAU,GAAG,IAAI,UAAU,CAC/B,eAAe,EACf,EAAE,EACF,SAAS,EACT,UAAU,CACX,CAAC;QACF,IAAI,cAAc,KAAK,IAAI,CAAC,eAAe,EAAE,CAAC;YAC5C,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CACb,2CAA2C,cAAc,EAAE,CAC5D,CAAC;YACF,UAAU,CAAC,KAAK,CACd,SAAS,CAAC,mBAAmB,EAC7B,8BACE,IAAI,CAAC,eACP,eAAe,cAAc,GAAG,CACjC,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,qBAAqB,UAAU,CAAC,EAAE,EAAE,CAAC,CAAC;YAEvD,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;YAChC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;YAEvC,IAAI,OAAO,EAAE,CAAC;gBACZ,IAAI,CAAC,eAAe,CAAC,SAAS,CAAC,CAAC;YAClC,CAAC;QACH,CAAC;QACD,OAAO,OAAO,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IACrC,CAAC;IAED,eAAe,CAAC,SAAiB;QAC/B,MAAM,QAAQ,GAAG,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC;QAC9C,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAEvC,IAAI,QAAQ,KAAK,CAAC,EAAE,CAAC;YACnB,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,gBAAgB,EAAE,EAAE,gBAAgB,CAAC,CAAC;QAC1E,CAAC;IACH,CAAC;IAED,KAAK,CAAC,gBAAgB;QACpB,MAAM,OAAO,GAAG,CAAC,GAAG,IAAI,CAAC,kBAAkB,CAAC,CAAC;QAC7C,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzB,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,4CAA4C,CAAC,CAAC,CAAC,gBAAgB;YAC/E,OAAO;QACT,CAAC;QACD,MAAM,OAAO,GAAG,CAAC,GAAG,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC,CAAC;QAC/C,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzB,gFAAgF;YAChF,mBAAmB;YACnB,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,mCAAmC,CAAC,CAAC;YACrD,OAAO;QACT,CAAC;QACD,IAAI,CAAC;YACH,MAAM,eAAe,GAAG,GAAG,CAAC,GAAI,OAAmC,CAAC,CAAC;YACrE,MAAM,eAAe,GAAG,GAAG,CAAC,GAAI,OAAmC,CAAC,CAAC;YACrE,IAAI,eAAe,GAAG,eAAe,EAAE,CAAC;gBACtC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CACb,yCAAyC,eAAe,MAAM,eAAe,GAAG,CACjF,CAAC;YACJ,CAAC;iBAAM,CAAC;gBACN,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC,eAAe,CAAC,CAAC;gBACvE,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,UAAU,OAAO,mBAAmB,eAAe,EAAE,CAAC,CAAC;gBACvE,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;YAClD,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,IAAI,IAAI,CAAC,kBAAkB,CAAC,IAAI,EAAE,CAAC;gBACjC,sEAAsE;gBACtE,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,gBAAgB,EAAE,EAAE,gBAAgB,CAAC,CAAC;YAC1E,CAAC;QACH,CAAC;IACH,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,GAAa;QACtB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;QAChC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;QAC/B,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;IAC5B,CAAC;CACF;AAED,iEAAiE;AACjE,gEAAgE;AAChE,4EAA4E;AAC5E,EAAE;AACF,2EAA2E;AAC3E,qEAAqE;AACrE,2EAA2E;AAC3E,0DAA0D;AAC1D,EAAE;AACF,6EAA6E;AAC7E,wEAAwE;AACxE,oEAAoE;AACpE,6EAA6E;AAC7E,4EAA4E;AAC5E,qDAAqD;AACrD,MAAM,gBAAgB,GAAG,0BAA0B,GAAG,CAAC,CAAC"}
+{"version":3,"file":"change-streamer-service.js","sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-service.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {unreachable} from '../../../../shared/src/asserts.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport {\n  min,\n  type AtLeastOne,\n  type LexiVersion,\n} from '../../types/lexi-version.ts';\nimport type {PostgresDB} from '../../types/pg.ts';\nimport type {ShardID} from '../../types/shards.ts';\nimport type {Sink, Source} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport {\n  type ChangeStreamControl,\n  type ChangeStreamData,\n  type ChangeStreamMessage,\n} from '../change-source/protocol/current/downstream.ts';\nimport type {ChangeSourceUpstream} from '../change-source/protocol/current/upstream.ts';\nimport type {SubscriptionState} from '../replicator/schema/replication-state.ts';\nimport {\n  DEFAULT_MAX_RETRY_DELAY_MS,\n  RunningState,\n  UnrecoverableError,\n} from '../running-state.ts';\nimport {\n  type ChangeStreamerService,\n  type Downstream,\n  type SubscriberContext,\n} from './change-streamer.ts';\nimport * as ErrorType from './error-type-enum.ts';\nimport {Forwarder} from './forwarder.ts';\nimport {initChangeStreamerSchema} from './schema/init.ts';\nimport {\n  AutoResetSignal,\n  ensureReplicationConfig,\n  markResetRequired,\n} from './schema/tables.ts';\nimport {Storer} from './storer.ts';\nimport {Subscriber} from './subscriber.ts';\n\n/**\n * Performs initialization and schema migrations to initialize a ChangeStreamerImpl.\n */\nexport async function initializeStreamer(\n  lc: LogContext,\n  shard: ShardID,\n  taskID: string,\n  discoveryAddress: string,\n  discoveryProtocol: string,\n  changeDB: PostgresDB,\n  changeSource: ChangeSource,\n  subscriptionState: SubscriptionState,\n  autoReset: boolean,\n  setTimeoutFn = setTimeout,\n): Promise<ChangeStreamerService> {\n  // Make sure the ChangeLog DB is set up.\n  await initChangeStreamerSchema(lc, changeDB, shard);\n  await ensureReplicationConfig(\n    lc,\n    changeDB,\n    subscriptionState,\n    shard,\n    autoReset,\n  );\n\n  const {replicaVersion} = subscriptionState;\n  return new ChangeStreamerImpl(\n    lc,\n    shard,\n    taskID,\n    discoveryAddress,\n    discoveryProtocol,\n    changeDB,\n    replicaVersion,\n    changeSource,\n    autoReset,\n    setTimeoutFn,\n  );\n}\n\n/**\n * Internally all Downstream messages (not just commits) are given a watermark.\n * These are used for internal ordering for:\n * 1. Replaying new changes in the Storer\n * 2. Filtering old changes in the Subscriber\n *\n * However, only the watermark for `Commit` messages are exposed to\n * subscribers, as that is the only semantically correct watermark to\n * use for tracking a position in a replication stream.\n */\nexport type WatermarkedChange = [watermark: string, ChangeStreamData];\n\nexport type ChangeStream = {\n  changes: Source<ChangeStreamMessage>;\n\n  /**\n   * A Sink to push the {@link StatusMessage}s that reflect Commits\n   * that have been successfully stored by the {@link Storer}, or\n   * downstream {@link StatusMessage}s henceforth.\n   */\n  acks: Sink<ChangeSourceUpstream>;\n};\n\n/** Encapsulates an upstream-specific implementation of a stream of Changes. */\nexport interface ChangeSource {\n  /**\n   * Starts a stream of changes starting after the specific watermark,\n   * with a corresponding sink for upstream acknowledgements.\n   */\n  startStream(afterWatermark: string): Promise<ChangeStream>;\n}\n\n/**\n * Upstream-agnostic dispatch of messages in a {@link ChangeStreamMessage} to a\n * {@link Forwarder} and {@link Storer} to execute the forward-store-ack\n * procedure described in {@link ChangeStreamer}.\n *\n * ### Subscriber Catchup\n *\n * Connecting clients first need to be \"caught up\" to the current watermark\n * (from stored change log entries) before new entries are forwarded to\n * them. This is non-trivial because the replication stream may be in the\n * middle of a pending streamed Transaction for which some entries have\n * already been forwarded but are not yet committed to the store.\n *\n *\n * ```\n * ------------------------------- - - - - - - - - - - - - - - - - - - -\n * | Historic changes in storage |  Pending (streamed) tx  |   Next tx\n * ------------------------------- - - - - - - - - - - - - - - - - - - -\n *                                           Replication stream\n *                                           >  >  >  >  >  >  >  >  >\n *           ^  ---> required catchup --->   ^\n * Subscriber watermark               Subscription begins\n * ```\n *\n * Preemptively buffering the changes of every pending transaction\n * would be wasteful and consume too much memory for large transactions.\n *\n * Instead, the streamer synchronously dispatches changes and subscriptions\n * to the {@link Forwarder} and the {@link Storer} such that the two\n * components are aligned as to where in the stream the subscription started.\n * The two components then coordinate catchup and handoff via the\n * {@link Subscriber} object with the following algorithm:\n *\n * * If the streamer is in the middle of a pending Transaction, the\n *   Subscriber is \"queued\" on both the Forwarder and the Storer. In this\n *   state, new changes are *not* forwarded to the Subscriber, and catchup\n *   is not yet executed.\n * * Once the commit message for the pending Transaction is processed\n *   by the Storer, it begins catchup on the Subscriber (with a READONLY\n *   snapshot so that it does not block subsequent storage operations).\n *   This catchup is thus guaranteed to load the change log entries of\n *   that last Transaction.\n * * When the Forwarder processes that same commit message, it moves the\n *   Subscriber from the \"queued\" to the \"active\" set of clients such that\n *   the Subscriber begins receiving new changes, starting from the next\n *   Transaction.\n * * The Subscriber does not forward those changes, however, if its catchup\n *   is not complete. Until then, it buffers the changes in memory.\n * * Once catchup is complete, the buffered changes are immediately sent\n *   and the Subscriber henceforth forwards changes as they are received.\n *\n * In the (common) case where the streamer is not in the middle of a pending\n * transaction when a subscription begins, the Storer begins catchup\n * immediately and the Forwarder directly adds the Subscriber to its active\n * set. However, the Subscriber still buffers any forwarded messages until\n * its catchup is complete.\n *\n * ### Watermarks and ordering\n *\n * The ChangeStreamerService depends on its {@link ChangeSource} to send\n * changes in contiguous [`begin`, `data` ..., `data`, `commit`] sequences\n * in commit order. This follows Postgres's Logical Replication Protocol\n * Message Flow:\n *\n * https://www.postgresql.org/docs/16/protocol-logical-replication.html#PROTOCOL-LOGICAL-MESSAGES-FLOW\n *\n * > The logical replication protocol sends individual transactions one by one.\n * > This means that all messages between a pair of Begin and Commit messages belong to the same transaction.\n *\n * In order to correctly replay (new) and filter (old) messages to subscribers\n * at different points in the replication stream, these changes must be assigned\n * watermarks such that they preserve the order in which they were received\n * from the ChangeSource.\n *\n * A previous implementation incorrectly derived these watermarks from the Postgres\n * Log Sequence Numbers (LSN) of each message. However, LSNs from concurrent,\n * non-conflicting transactions can overlap, which can result in a `begin` message\n * with an earlier LSN arriving after a `commit` message. For example, the\n * changes for these transactions:\n *\n * ```\n * LSN:   1     2     3  4    5   6   7     8   9      10\n * tx1: begin  data     data     data     commit\n * tx2:             begin    data    data      data  commit\n * ```\n *\n * will arrive as:\n *\n * ```\n * begin1, data2, data4, data6, commit8, begin3, data5, data7, data9, commit10\n * ```\n *\n * Thus, LSN of non-commit messages are not suitable for tracking the sorting\n * order of the replication stream.\n *\n * Instead, the ChangeStreamer uses the following algorithm for deterministic\n * catchup and filtering of changes:\n *\n * * A `commit` message is assigned to a watermark corresponding to its LSN.\n *   These are guaranteed to be in commit order by definition.\n *\n * * `begin` and `data` messages are assigned to the watermark of the\n *   preceding `commit` (the previous transaction, or the replication\n *   slot's starting LSN) plus 1. This guarantees that they will be sorted\n *   after the previously commit transaction even if their LSNs came before it.\n *   This is referred to as the `preCommitWatermark`.\n *\n * * In the ChangeLog DB, messages have a secondary sort column `pos`, which is\n *   the position of the message within its transaction, with the `begin` message\n *   starting at `0`. This guarantees that `begin` and `data` messages will be\n *   fetched in the original ChangeSource order during catchup.\n *\n * `begin` and `data` messages share the same watermark, but this is sufficient for\n * Subscriber filtering because subscribers only know about the `commit` watermarks\n * exposed in the `Downstream` `Commit` message. The Subscriber object thus compares\n * the internal watermarks of the incoming messages against the commit watermark of\n * the caller, updating the watermark at every `Commit` message that is forwarded.\n *\n * ### Cleanup\n *\n * As mentioned in the {@link ChangeStreamer} documentation: \"the ChangeStreamer\n * uses a combination of [the \"initial\", i.e. backup-derived watermark and] ACK\n * responses from connected subscribers to determine the watermark up\n * to which it is safe to purge old change log entries.\"\n *\n * More concretely:\n *\n * * The `initial`, backup-derived watermark is the earliest to which cleanup\n *   should ever happen.\n *\n * * However, it is possible for the replica backup to be *ahead* of a connected\n *   subscriber; and if a network error causes that subscriber to retry from its\n *   last watermark, the change streamer must support it.\n *\n * Thus, before cleaning up to an `initial` backup-derived watermark, the change\n * streamer first confirms that all connected subscribers have also passed\n * that watermark.\n */\nclass ChangeStreamerImpl implements ChangeStreamerService {\n  readonly id: string;\n  readonly #lc: LogContext;\n  readonly #shard: ShardID;\n  readonly #changeDB: PostgresDB;\n  readonly #replicaVersion: string;\n  readonly #source: ChangeSource;\n  readonly #storer: Storer;\n  readonly #forwarder: Forwarder;\n\n  readonly #autoReset: boolean;\n  readonly #state: RunningState;\n  readonly #initialWatermarks = new Set<string>();\n\n  // Starting the (Postgres) ChangeStream results in killing the previous\n  // Postgres subscriber, potentially creating a gap in which the old\n  // change-streamer has shut down and the new change-streamer has not yet\n  // been recognized as \"healthy\" (and thus does not get any requests).\n  //\n  // To minimize this gap, delay starting the ChangeStream until the first\n  // request from a `serving` replicator, indicating that higher level\n  // load-balancing / routing logic has begun routing requests to this task.\n  readonly #serving = resolver();\n\n  readonly #txCounter = getOrCreateCounter(\n    'replication',\n    'transactions',\n    'Count of replicated transactions',\n  );\n\n  #stream: ChangeStream | undefined;\n\n  constructor(\n    lc: LogContext,\n    shard: ShardID,\n    taskID: string,\n    discoveryAddress: string,\n    discoveryProtocol: string,\n    changeDB: PostgresDB,\n    replicaVersion: string,\n    source: ChangeSource,\n    autoReset: boolean,\n    setTimeoutFn = setTimeout,\n  ) {\n    this.id = `change-streamer`;\n    this.#lc = lc.withContext('component', 'change-streamer');\n    this.#shard = shard;\n    this.#changeDB = changeDB;\n    this.#replicaVersion = replicaVersion;\n    this.#source = source;\n    this.#storer = new Storer(\n      lc,\n      shard,\n      taskID,\n      discoveryAddress,\n      discoveryProtocol,\n      changeDB,\n      replicaVersion,\n      consumed => this.#stream?.acks.push(['status', consumed[1], consumed[2]]),\n      err => this.stop(err),\n    );\n    this.#forwarder = new Forwarder();\n    this.#autoReset = autoReset;\n    this.#state = new RunningState(this.id, undefined, setTimeoutFn);\n  }\n\n  async run() {\n    this.#lc.info?.('starting change stream');\n\n    // Once this change-streamer acquires \"ownership\" of the change DB,\n    // it is safe to start the storer.\n    await this.#storer.assumeOwnership();\n    // The storer will, in turn, detect changes to ownership and stop\n    // the change-streamer appropriately.\n    this.#storer.run().catch(e => this.stop(e));\n\n    while (this.#state.shouldRun()) {\n      let err: unknown;\n      let watermark: string | null = null;\n      try {\n        const startAfter = await this.#storer.getLastWatermarkToStartStream();\n        const stream = await this.#source.startStream(startAfter);\n        this.#stream = stream;\n        this.#state.resetBackoff();\n        watermark = null;\n\n        for await (const change of stream.changes) {\n          const [type, msg] = change;\n          switch (type) {\n            case 'status':\n              this.#storer.status(change); // storer acks once it gets through its queue\n              continue;\n            case 'control':\n              await this.#handleControlMessage(msg);\n              continue; // control messages are not stored/forwarded\n            case 'begin':\n              watermark = change[2].commitWatermark;\n              break;\n            case 'commit':\n              if (watermark !== change[2].watermark) {\n                throw new UnrecoverableError(\n                  `commit watermark ${change[2].watermark} does not match 'begin' watermark ${watermark}`,\n                );\n              }\n              this.#txCounter.add(1);\n              break;\n            default:\n              if (watermark === null) {\n                throw new UnrecoverableError(\n                  `${type} change (${msg.tag}) received before 'begin' message`,\n                );\n              }\n              break;\n          }\n\n          this.#storer.store([watermark, change]);\n          this.#forwarder.forward([watermark, change]);\n\n          if (type === 'commit') {\n            watermark = null;\n          }\n\n          // Allow the storer to exert back pressure.\n          const readyForMore = this.#storer.readyForMore();\n          if (readyForMore) {\n            await readyForMore;\n          }\n        }\n      } catch (e) {\n        err = e;\n      } finally {\n        this.#stream?.changes.cancel();\n        this.#stream = undefined;\n      }\n\n      // When the change stream is interrupted, abort any pending transaction.\n      if (watermark) {\n        this.#lc.warn?.(`aborting interrupted transaction ${watermark}`);\n        this.#storer.abort();\n        this.#forwarder.forward([watermark, ['rollback', {tag: 'rollback'}]]);\n      }\n\n      await this.#state.backoff(this.#lc, err);\n    }\n    this.#lc.info?.('ChangeStreamer stopped');\n  }\n\n  async #handleControlMessage(msg: ChangeStreamControl[1]) {\n    this.#lc.info?.('received control message', msg);\n    const {tag} = msg;\n\n    switch (tag) {\n      case 'reset-required':\n        await markResetRequired(this.#changeDB, this.#shard);\n        if (this.#autoReset) {\n          this.#lc.warn?.('shutting down for auto-reset');\n          await this.stop(new AutoResetSignal());\n        }\n        break;\n      default:\n        unreachable(tag);\n    }\n  }\n\n  subscribe(ctx: SubscriberContext): Promise<Source<Downstream>> {\n    const {protocolVersion, id, mode, replicaVersion, watermark, initial} = ctx;\n    if (mode === 'serving') {\n      this.#serving.resolve();\n    }\n    const downstream = Subscription.create<Downstream>({\n      cleanup: () => this.#forwarder.remove(subscriber),\n    });\n    const subscriber = new Subscriber(\n      protocolVersion,\n      id,\n      watermark,\n      downstream,\n    );\n    if (replicaVersion !== this.#replicaVersion) {\n      this.#lc.warn?.(\n        `rejecting subscriber at replica version ${replicaVersion}`,\n      );\n      subscriber.close(\n        ErrorType.WrongReplicaVersion,\n        `current replica version is ${\n          this.#replicaVersion\n        } (requested ${replicaVersion})`,\n      );\n    } else {\n      this.#lc.debug?.(`adding subscriber ${subscriber.id}`);\n\n      this.#forwarder.add(subscriber);\n      this.#storer.catchup(subscriber, mode);\n\n      if (initial) {\n        this.scheduleCleanup(watermark);\n      }\n    }\n    return Promise.resolve(downstream);\n  }\n\n  scheduleCleanup(watermark: string) {\n    const origSize = this.#initialWatermarks.size;\n    this.#initialWatermarks.add(watermark);\n\n    if (origSize === 0) {\n      this.#state.setTimeout(() => this.#purgeOldChanges(), CLEANUP_DELAY_MS);\n    }\n  }\n\n  async getChangeLogState(): Promise<{\n    replicaVersion: string;\n    minWatermark: string;\n  }> {\n    const minWatermark = await this.#storer.getMinWatermarkForCatchup();\n    return {\n      replicaVersion: this.#replicaVersion,\n      minWatermark: minWatermark ?? this.#replicaVersion,\n    };\n  }\n\n  async #purgeOldChanges(): Promise<void> {\n    const initial = [...this.#initialWatermarks];\n    if (initial.length === 0) {\n      this.#lc.warn?.('No initial watermarks to check for cleanup'); // Not expected.\n      return;\n    }\n    const current = [...this.#forwarder.getAcks()];\n    if (current.length === 0) {\n      // Also not expected, but possible (e.g. subscriber connects, then disconnects).\n      // Bail to be safe.\n      this.#lc.warn?.('No subscribers to confirm cleanup');\n      return;\n    }\n    try {\n      const earliestInitial = min(...(initial as AtLeastOne<LexiVersion>));\n      const earliestCurrent = min(...(current as AtLeastOne<LexiVersion>));\n      if (earliestCurrent < earliestInitial) {\n        this.#lc.info?.(\n          `At least one client is behind backup (${earliestCurrent} < ${earliestInitial})`,\n        );\n      } else {\n        const deleted = await this.#storer.purgeRecordsBefore(earliestInitial);\n        this.#lc.info?.(`Purged ${deleted} changes before ${earliestInitial}`);\n        this.#initialWatermarks.delete(earliestInitial);\n      }\n    } finally {\n      if (this.#initialWatermarks.size) {\n        // If there are unpurged watermarks to check, schedule the next purge.\n        this.#state.setTimeout(() => this.#purgeOldChanges(), CLEANUP_DELAY_MS);\n      }\n    }\n  }\n\n  async stop(err?: unknown) {\n    this.#state.stop(this.#lc, err);\n    this.#stream?.changes.cancel();\n    await this.#storer.stop();\n  }\n}\n\n// The delay between receiving an initial, backup-based watermark\n// and performing a check of whether to purge records before it.\n// This delay should be long enough to handle situations like the following:\n//\n// 1. `litestream restore` downloads a backup for the `replication-manager`\n// 2. `replication-manager` starts up and runs this `change-streamer`\n// 3. `zero-cache`s that are running on a different replica connect to this\n//    `change-streamer` after exponential backoff retries.\n//\n// It is possible for a `zero-cache`[3] to be behind the backup restored [1].\n// This cleanup delay (30 seconds) is thus set to be a value comfortably\n// longer than the max delay for exponential backoff (10 seconds) in\n// `services/running-state.ts`. This allows the `zero-cache` [3] to reconnect\n// so that the `change-streamer` can track its progress and know when it has\n// surpassed the initial watermark of the backup [1].\nconst CLEANUP_DELAY_MS = DEFAULT_MAX_RETRY_DELAY_MS * 3;\n"],"names":["ErrorType.WrongReplicaVersion"],"mappings":";;;;;;;;;;;;;;AA4CA,eAAsB,mBACpB,IACA,OACA,QACA,kBACA,mBACA,UACA,cACA,mBACA,WACA,eAAe,YACiB;AAEhC,QAAM,yBAAyB,IAAI,UAAU,KAAK;AAClD,QAAM;AAAA,IACJ;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EAAA;AAGF,QAAM,EAAC,mBAAkB;AACzB,SAAO,IAAI;AAAA,IACT;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EAAA;AAEJ;AA4KA,MAAM,mBAAoD;AAAA,EAC/C;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAEA;AAAA,EACA;AAAA,EACA,yCAAyB,IAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAUzB,WAAW,SAAA;AAAA,EAEX,aAAa;AAAA,IACpB;AAAA,IACA;AAAA,IACA;AAAA,EAAA;AAAA,EAGF;AAAA,EAEA,YACE,IACA,OACA,QACA,kBACA,mBACA,UACA,gBACA,QACA,WACA,eAAe,YACf;AACA,SAAK,KAAK;AACV,SAAK,MAAM,GAAG,YAAY,aAAa,iBAAiB;AACxD,SAAK,SAAS;AACd,SAAK,YAAY;AACjB,SAAK,kBAAkB;AACvB,SAAK,UAAU;AACf,SAAK,UAAU,IAAI;AAAA,MACjB;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,CAAA,aAAY,KAAK,SAAS,KAAK,KAAK,CAAC,UAAU,SAAS,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC;AAAA,MACxE,CAAA,QAAO,KAAK,KAAK,GAAG;AAAA,IAAA;AAEtB,SAAK,aAAa,IAAI,UAAA;AACtB,SAAK,aAAa;AAClB,SAAK,SAAS,IAAI,aAAa,KAAK,IAAI,QAAW,YAAY;AAAA,EACjE;AAAA,EAEA,MAAM,MAAM;AACV,SAAK,IAAI,OAAO,wBAAwB;AAIxC,UAAM,KAAK,QAAQ,gBAAA;AAGnB,SAAK,QAAQ,MAAM,MAAM,OAAK,KAAK,KAAK,CAAC,CAAC;AAE1C,WAAO,KAAK,OAAO,aAAa;AAC9B,UAAI;AACJ,UAAI,YAA2B;AAC/B,UAAI;AACF,cAAM,aAAa,MAAM,KAAK,QAAQ,8BAAA;AACtC,cAAM,SAAS,MAAM,KAAK,QAAQ,YAAY,UAAU;AACxD,aAAK,UAAU;AACf,aAAK,OAAO,aAAA;AACZ,oBAAY;AAEZ,yBAAiB,UAAU,OAAO,SAAS;AACzC,gBAAM,CAAC,MAAM,GAAG,IAAI;AACpB,kBAAQ,MAAA;AAAA,YACN,KAAK;AACH,mBAAK,QAAQ,OAAO,MAAM;AAC1B;AAAA,YACF,KAAK;AACH,oBAAM,KAAK,sBAAsB,GAAG;AACpC;AAAA;AAAA,YACF,KAAK;AACH,0BAAY,OAAO,CAAC,EAAE;AACtB;AAAA,YACF,KAAK;AACH,kBAAI,cAAc,OAAO,CAAC,EAAE,WAAW;AACrC,sBAAM,IAAI;AAAA,kBACR,oBAAoB,OAAO,CAAC,EAAE,SAAS,qCAAqC,SAAS;AAAA,gBAAA;AAAA,cAEzF;AACA,mBAAK,WAAW,IAAI,CAAC;AACrB;AAAA,YACF;AACE,kBAAI,cAAc,MAAM;AACtB,sBAAM,IAAI;AAAA,kBACR,GAAG,IAAI,YAAY,IAAI,GAAG;AAAA,gBAAA;AAAA,cAE9B;AACA;AAAA,UAAA;AAGJ,eAAK,QAAQ,MAAM,CAAC,WAAW,MAAM,CAAC;AACtC,eAAK,WAAW,QAAQ,CAAC,WAAW,MAAM,CAAC;AAE3C,cAAI,SAAS,UAAU;AACrB,wBAAY;AAAA,UACd;AAGA,gBAAM,eAAe,KAAK,QAAQ,aAAA;AAClC,cAAI,cAAc;AAChB,kBAAM;AAAA,UACR;AAAA,QACF;AAAA,MACF,SAAS,GAAG;AACV,cAAM;AAAA,MACR,UAAA;AACE,aAAK,SAAS,QAAQ,OAAA;AACtB,aAAK,UAAU;AAAA,MACjB;AAGA,UAAI,WAAW;AACb,aAAK,IAAI,OAAO,oCAAoC,SAAS,EAAE;AAC/D,aAAK,QAAQ,MAAA;AACb,aAAK,WAAW,QAAQ,CAAC,WAAW,CAAC,YAAY,EAAC,KAAK,WAAA,CAAW,CAAC,CAAC;AAAA,MACtE;AAEA,YAAM,KAAK,OAAO,QAAQ,KAAK,KAAK,GAAG;AAAA,IACzC;AACA,SAAK,IAAI,OAAO,wBAAwB;AAAA,EAC1C;AAAA,EAEA,MAAM,sBAAsB,KAA6B;AACvD,SAAK,IAAI,OAAO,4BAA4B,GAAG;AAC/C,UAAM,EAAC,QAAO;AAEd,YAAQ,KAAA;AAAA,MACN,KAAK;AACH,cAAM,kBAAkB,KAAK,WAAW,KAAK,MAAM;AACnD,YAAI,KAAK,YAAY;AACnB,eAAK,IAAI,OAAO,8BAA8B;AAC9C,gBAAM,KAAK,KAAK,IAAI,iBAAiB;AAAA,QACvC;AACA;AAAA,MACF;AACE,oBAAe;AAAA,IAAA;AAAA,EAErB;AAAA,EAEA,UAAU,KAAqD;AAC7D,UAAM,EAAC,iBAAiB,IAAI,MAAM,gBAAgB,WAAW,YAAW;AACxE,QAAI,SAAS,WAAW;AACtB,WAAK,SAAS,QAAA;AAAA,IAChB;AACA,UAAM,aAAa,aAAa,OAAmB;AAAA,MACjD,SAAS,MAAM,KAAK,WAAW,OAAO,UAAU;AAAA,IAAA,CACjD;AACD,UAAM,aAAa,IAAI;AAAA,MACrB;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IAAA;AAEF,QAAI,mBAAmB,KAAK,iBAAiB;AAC3C,WAAK,IAAI;AAAA,QACP,2CAA2C,cAAc;AAAA,MAAA;AAE3D,iBAAW;AAAA,QACTA;AAAAA,QACA,8BACE,KAAK,eACP,eAAe,cAAc;AAAA,MAAA;AAAA,IAEjC,OAAO;AACL,WAAK,IAAI,QAAQ,qBAAqB,WAAW,EAAE,EAAE;AAErD,WAAK,WAAW,IAAI,UAAU;AAC9B,WAAK,QAAQ,QAAQ,YAAY,IAAI;AAErC,UAAI,SAAS;AACX,aAAK,gBAAgB,SAAS;AAAA,MAChC;AAAA,IACF;AACA,WAAO,QAAQ,QAAQ,UAAU;AAAA,EACnC;AAAA,EAEA,gBAAgB,WAAmB;AACjC,UAAM,WAAW,KAAK,mBAAmB;AACzC,SAAK,mBAAmB,IAAI,SAAS;AAErC,QAAI,aAAa,GAAG;AAClB,WAAK,OAAO,WAAW,MAAM,KAAK,iBAAA,GAAoB,gBAAgB;AAAA,IACxE;AAAA,EACF;AAAA,EAEA,MAAM,oBAGH;AACD,UAAM,eAAe,MAAM,KAAK,QAAQ,0BAAA;AACxC,WAAO;AAAA,MACL,gBAAgB,KAAK;AAAA,MACrB,cAAc,gBAAgB,KAAK;AAAA,IAAA;AAAA,EAEvC;AAAA,EAEA,MAAM,mBAAkC;AACtC,UAAM,UAAU,CAAC,GAAG,KAAK,kBAAkB;AAC3C,QAAI,QAAQ,WAAW,GAAG;AACxB,WAAK,IAAI,OAAO,4CAA4C;AAC5D;AAAA,IACF;AACA,UAAM,UAAU,CAAC,GAAG,KAAK,WAAW,SAAS;AAC7C,QAAI,QAAQ,WAAW,GAAG;AAGxB,WAAK,IAAI,OAAO,mCAAmC;AACnD;AAAA,IACF;AACA,QAAI;AACF,YAAM,kBAAkB,IAAI,GAAI,OAAmC;AACnE,YAAM,kBAAkB,IAAI,GAAI,OAAmC;AACnE,UAAI,kBAAkB,iBAAiB;AACrC,aAAK,IAAI;AAAA,UACP,yCAAyC,eAAe,MAAM,eAAe;AAAA,QAAA;AAAA,MAEjF,OAAO;AACL,cAAM,UAAU,MAAM,KAAK,QAAQ,mBAAmB,eAAe;AACrE,aAAK,IAAI,OAAO,UAAU,OAAO,mBAAmB,eAAe,EAAE;AACrE,aAAK,mBAAmB,OAAO,eAAe;AAAA,MAChD;AAAA,IACF,UAAA;AACE,UAAI,KAAK,mBAAmB,MAAM;AAEhC,aAAK,OAAO,WAAW,MAAM,KAAK,iBAAA,GAAoB,gBAAgB;AAAA,MACxE;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAM,KAAK,KAAe;AACxB,SAAK,OAAO,KAAK,KAAK,KAAK,GAAG;AAC9B,SAAK,SAAS,QAAQ,OAAA;AACtB,UAAM,KAAK,QAAQ,KAAA;AAAA,EACrB;AACF;AAiBA,MAAM,mBAAmB,6BAA6B;"}

package/out/zero-cache/src/services/change-streamer/change-streamer.d.ts

@@ -45,7 +45,7 @@ export interface ChangeStreamer {
      */
     subscribe(ctx: SubscriberContext): Promise<Source<Downstream>>;
 }
-export declare const PROTOCOL_VERSION = 3;
+export declare const PROTOCOL_VERSION = 4;
 export type SubscriberContext = {
     /**
      * The supported change-streamer protocol version.
@@ -168,8 +168,8 @@ export declare const downstreamSchema: v.UnionType<[v.TupleType<[v.Type<"status"
     spec: v.ObjectType<Omit<{
         name: v.Type<string>;
         columns: v.Type<Record<string, {
-            pgTypeClass?: "e" | "b" | "d" | "c" | "p" | "r" | "m" | undefined;
-            elemPgTypeClass?: "e" | "b" | "d" | "c" | "p" | "r" | "m" | null | undefined;
+            pgTypeClass?: "e" | "d" | "b" | "c" | "p" | "r" | "m" | undefined;
+            elemPgTypeClass?: "e" | "d" | "b" | "c" | "p" | "r" | "m" | null | undefined;
             characterMaximumLength?: number | null | undefined;
             notNull?: boolean | null | undefined;
             dflt?: string | null | undefined;
@@ -201,8 +201,8 @@ export declare const downstreamSchema: v.UnionType<[v.TupleType<[v.Type<"status"
         spec: v.ObjectType<{
             pos: v.Type<number>;
             dataType: v.Type<string>;
-            pgTypeClass: v.Optional<"e" | "b" | "d" | "c" | "p" | "r" | "m">;
-            elemPgTypeClass: v.Optional<"e" | "b" | "d" | "c" | "p" | "r" | "m" | null>;
+            pgTypeClass: v.Optional<"e" | "d" | "b" | "c" | "p" | "r" | "m">;
+            elemPgTypeClass: v.Optional<"e" | "d" | "b" | "c" | "p" | "r" | "m" | null>;
             characterMaximumLength: v.Optional<number | null>;
             notNull: v.Optional<boolean | null>;
             dflt: v.Optional<string | null>;
@@ -219,8 +219,8 @@ export declare const downstreamSchema: v.UnionType<[v.TupleType<[v.Type<"status"
         spec: v.ObjectType<{
             pos: v.Type<number>;
             dataType: v.Type<string>;
-            pgTypeClass: v.Optional<"e" | "b" | "d" | "c" | "p" | "r" | "m">;
-            elemPgTypeClass: v.Optional<"e" | "b" | "d" | "c" | "p" | "r" | "m" | null>;
+            pgTypeClass: v.Optional<"e" | "d" | "b" | "c" | "p" | "r" | "m">;
+            elemPgTypeClass: v.Optional<"e" | "d" | "b" | "c" | "p" | "r" | "m" | null>;
             characterMaximumLength: v.Optional<number | null>;
             notNull: v.Optional<boolean | null>;
             dflt: v.Optional<string | null>;
@@ -231,8 +231,8 @@ export declare const downstreamSchema: v.UnionType<[v.TupleType<[v.Type<"status"
         spec: v.ObjectType<{
             pos: v.Type<number>;
             dataType: v.Type<string>;
-            pgTypeClass: v.Optional<"e" | "b" | "d" | "c" | "p" | "r" | "m">;
-            elemPgTypeClass: v.Optional<"e" | "b" | "d" | "c" | "p" | "r" | "m" | null>;
+            pgTypeClass: v.Optional<"e" | "d" | "b" | "c" | "p" | "r" | "m">;
+            elemPgTypeClass: v.Optional<"e" | "d" | "b" | "c" | "p" | "r" | "m" | null>;
             characterMaximumLength: v.Optional<number | null>;
             notNull: v.Optional<boolean | null>;
             dflt: v.Optional<string | null>;
@@ -297,6 +297,10 @@ export interface ChangeStreamerService extends ChangeStreamer, Service {
      * subscribers have progressed beyond the watermark.
      */
     scheduleCleanup(watermark: string): void;
+    getChangeLogState(): Promise<{
+        replicaVersion: string;
+        minWatermark: string;
+    }>;
 }
 export {};
 //# sourceMappingURL=change-streamer.d.ts.map

package/out/zero-cache/src/services/change-streamer/change-streamer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"change-streamer.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,CAAC,MAAM,kCAAkC,CAAC;AACtD,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,wBAAwB,CAAC;AACnD,OAAO,EAAC,KAAK,MAAM,EAAC,MAAM,2CAA2C,CAAC;AAEtE,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,6BAA6B,CAAC;AAChE,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,eAAe,CAAC;AAE3C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AACH,MAAM,WAAW,cAAc;IAC7B;;;;OAIG;IACH,SAAS,CAAC,GAAG,EAAE,iBAAiB,GAAG,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC;CAChE;AASD,eAAO,MAAM,gBAAgB,IAAI,CAAC;AAElC,MAAM,MAAM,iBAAiB,GAAG;IAC9B;;OAEG;IACH,eAAe,EAAE,MAAM,CAAC;IAExB;;;OAGG;IACH,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IAEtB;;OAEG;IACH,EAAE,EAAE,MAAM,CAAC;IAEX;;;;OAIG;IACH,IAAI,EAAE,cAAc,CAAC;IAErB;;;;OAIG;IACH,cAAc,EAAE,MAAM,CAAC;IAEvB;;;OAGG;IACH,SAAS,EAAE,MAAM,CAAC;IAElB;;;;;OAKG;IACH,OAAO,EAAE,OAAO,CAAC;CAClB,CAAC;AAEF,MAAM,MAAM,WAAW,GAAG;IACxB,MAAM,EAAE,MAAM,CAAC;IAEf;;;;OAIG;IACH,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF;;;GAGG;AACH,eAAO,MAAM,YAAY;;aAEvB,CAAC;AAEH,eAAO,MAAM,mBAAmB;;eAA+C,CAAC;AAEhF;;;;;GAKG;AACH,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,mBAAmB,CAAC,CAAC;AAEhE,QAAA,MAAM,uBAAuB;;;aAG3B,CAAC;AAEH,MAAM,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC;AAExE,QAAA,MAAM,WAAW;;;eAAyD,CAAC;AAE3E,eAAO,MAAM,gBAAgB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAI5B,CAAC;AAEF,MAAM,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,WAAW,CAAC,CAAC;AAEhD;;;;;;;;;;GAUG;AACH,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAE1D,MAAM,WAAW,qBAAsB,SAAQ,cAAc,EAAE,OAAO;IACpE;;;;OAIG;IACH,eAAe,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;CAC1C"}
+{"version":3,"file":"change-streamer.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,CAAC,MAAM,kCAAkC,CAAC;AACtD,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,wBAAwB,CAAC;AACnD,OAAO,EAAC,KAAK,MAAM,EAAC,MAAM,2CAA2C,CAAC;AAEtE,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,6BAA6B,CAAC;AAChE,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,eAAe,CAAC;AAE3C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AACH,MAAM,WAAW,cAAc;IAC7B;;;;OAIG;IACH,SAAS,CAAC,GAAG,EAAE,iBAAiB,GAAG,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC;CAChE;AAaD,eAAO,MAAM,gBAAgB,IAAI,CAAC;AAElC,MAAM,MAAM,iBAAiB,GAAG;IAC9B;;OAEG;IACH,eAAe,EAAE,MAAM,CAAC;IAExB;;;OAGG;IACH,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IAEtB;;OAEG;IACH,EAAE,EAAE,MAAM,CAAC;IAEX;;;;OAIG;IACH,IAAI,EAAE,cAAc,CAAC;IAErB;;;;OAIG;IACH,cAAc,EAAE,MAAM,CAAC;IAEvB;;;OAGG;IACH,SAAS,EAAE,MAAM,CAAC;IAElB;;;;;OAKG;IACH,OAAO,EAAE,OAAO,CAAC;CAClB,CAAC;AAEF,MAAM,MAAM,WAAW,GAAG;IACxB,MAAM,EAAE,MAAM,CAAC;IAEf;;;;OAIG;IACH,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF;;;GAGG;AACH,eAAO,MAAM,YAAY;;aAEvB,CAAC;AAEH,eAAO,MAAM,mBAAmB;;eAA+C,CAAC;AAEhF;;;;;GAKG;AACH,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,mBAAmB,CAAC,CAAC;AAEhE,QAAA,MAAM,uBAAuB;;;aAG3B,CAAC;AAEH,MAAM,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC;AAExE,QAAA,MAAM,WAAW;;;eAAyD,CAAC;AAE3E,eAAO,MAAM,gBAAgB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAI5B,CAAC;AAEF,MAAM,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,WAAW,CAAC,CAAC;AAEhD;;;;;;;;;;GAUG;AACH,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAE1D,MAAM,WAAW,qBAAsB,SAAQ,cAAc,EAAE,OAAO;IACpE;;;;OAIG;IACH,eAAe,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IAEzC,iBAAiB,IAAI,OAAO,CAAC;QAC3B,cAAc,EAAE,MAAM,CAAC;QACvB,YAAY,EAAE,MAAM,CAAC;KACtB,CAAC,CAAC;CACJ"}

package/out/zero-cache/src/services/change-streamer/change-streamer.js

@@ -1,25 +1,27 @@
-import * as v from "../../../../shared/src/valita.js";
-import {} from "../change-source/protocol/current/data.js";
+import "../../../../shared/src/valita.js";
+import "../change-source/protocol/current/data.js";
 import { changeStreamDataSchema } from "../change-source/protocol/current/downstream.js";
-// v1: Client-side support for JSON_FORMAT. Introduced in 0.18.
-// v2: Adds the "status" message which is initially used to signal that the
-//     subscription is valid (i.e. starting at the requested watermark).
-//     Introduced in 0.19.
-// v3: Adds the "taskID" to the subscription context, and support for
-//     the BackupMonitor-mediated "/snapshot" request.
-export const PROTOCOL_VERSION = 3;
-/**
- * The StatusMessage payload for now is empty, but can be extended to
- * include meta-level information in the future.
- */
-export const statusSchema = v.object({
-    tag: v.literal('status'),
+import { object, literal, tuple, string, number, union } from "@badrap/valita";
+const PROTOCOL_VERSION = 4;
+const statusSchema = object({
+  tag: literal("status")
 });
-export const statusMessageSchema = v.tuple([v.literal('status'), statusSchema]);
-const subscriptionErrorSchema = v.object({
-    type: v.number(), // ErrorType
-    message: v.string().optional(),
+const statusMessageSchema = tuple([literal("status"), statusSchema]);
+const subscriptionErrorSchema = object({
+  type: number(),
+  // ErrorType
+  message: string().optional()
 });
-const errorSchema = v.tuple([v.literal('error'), subscriptionErrorSchema]);
-export const downstreamSchema = v.union(statusMessageSchema, changeStreamDataSchema, errorSchema);
-//# sourceMappingURL=change-streamer.js.map
+const errorSchema = tuple([literal("error"), subscriptionErrorSchema]);
+const downstreamSchema = union(
+  statusMessageSchema,
+  changeStreamDataSchema,
+  errorSchema
+);
+export {
+  PROTOCOL_VERSION,
+  downstreamSchema,
+  statusMessageSchema,
+  statusSchema
+};
+//# sourceMappingURL=change-streamer.js.map

package/out/zero-cache/src/services/change-streamer/change-streamer.js.map

@@ -1 +1 @@
-{"version":3,"file":"change-streamer.js","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,CAAC,MAAM,kCAAkC,CAAC;AAEtD,OAAO,EAAa,MAAM,2CAA2C,CAAC;AACtE,OAAO,EAAC,sBAAsB,EAAC,MAAM,iDAAiD,CAAC;AA+CvF,+DAA+D;AAC/D,2EAA2E;AAC3E,wEAAwE;AACxE,0BAA0B;AAC1B,qEAAqE;AACrE,sDAAsD;AAEtD,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,CAAC;AA2DlC;;;GAGG;AACH,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,CAAC,MAAM,CAAC;IACnC,GAAG,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC;CACzB,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,YAAY,CAAC,CAAC,CAAC;AAUhF,MAAM,uBAAuB,GAAG,CAAC,CAAC,MAAM,CAAC;IACvC,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,YAAY;IAC9B,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;CAC/B,CAAC,CAAC;AAIH,MAAM,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,uBAAuB,CAAC,CAAC,CAAC;AAE3E,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CACrC,mBAAmB,EACnB,sBAAsB,EACtB,WAAW,CACZ,CAAC"}
+{"version":3,"file":"change-streamer.js","sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer.ts"],"sourcesContent":["import * as v from '../../../../shared/src/valita.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {type Change} from '../change-source/protocol/current/data.ts';\nimport {changeStreamDataSchema} from '../change-source/protocol/current/downstream.ts';\nimport type {ReplicatorMode} from '../replicator/replicator.ts';\nimport type {Service} from '../service.ts';\n\n/**\n * The ChangeStreamer is the component between replicators (\"subscribers\")\n * and a canonical upstream source of changes (e.g. a Postgres logical\n * replication slot). It facilitates multiple subscribers without incurring\n * the associated upstream expense (e.g. PG replication slots are resource\n * intensive) with a \"forward-store-ack\" procedure.\n *\n * * Changes from the upstream source are immediately **forwarded** to\n *   connected subscribers to minimize latency.\n *\n * * They are then **stored** in a separate DB to facilitate catchup\n *   of connecting subscribers that are behind.\n *\n * * **Acknowledgements** are sent upstream after they are successfully\n *   stored.\n *\n * Unlike Postgres replication slots, in which the progress of a static\n * subscriber is tracked in the replication slot, the ChangeStreamer\n * supports a dynamic set of subscribers (i.e.. zero-caches) that can\n * can continually change.\n *\n * However, it is not the case that the ChangeStreamer needs to support\n * arbitrarily old subscribers. Because the replica is continually\n * backed up to a global location and used to initialize new subscriber\n * tasks, an initial subscription request from a subscriber constitutes\n * a signal for how \"behind\" a new subscriber task can be. This is\n * reflected in the {@link SubscriberContext}, which indicates whether\n * the watermark corresponds to an \"initial\" watermark derived from the\n * replica at task startup.\n *\n * The ChangeStreamer uses a combination of this signal with ACK\n * responses from connected subscribers to determine the watermark up\n * to which it is safe to purge old change log entries.\n */\nexport interface ChangeStreamer {\n  /**\n   * Subscribes to changes based on the supplied subscriber `ctx`,\n   * which indicates the watermark at which the subscriber is up to\n   * date.\n   */\n  subscribe(ctx: SubscriberContext): Promise<Source<Downstream>>;\n}\n\n// v1: Client-side support for JSON_FORMAT. Introduced in 0.18.\n// v2: Adds the \"status\" message which is initially used to signal that the\n//     subscription is valid (i.e. starting at the requested watermark).\n//     Introduced in 0.19.\n// v3: Adds the \"taskID\" to the subscription context, and support for\n//     the BackupMonitor-mediated \"/snapshot\" request.\n// v4: Adds the \"replicaVersion\" and \"minWatermark\" fields to the \"/snapshot\"\n//     status request so that a subscriber can verify whether its replica,\n//     whether it be restored or existing in a permanent volume, is compatible\n//     with the change-streamer.\n\nexport const PROTOCOL_VERSION = 4;\n\nexport type SubscriberContext = {\n  /**\n   * The supported change-streamer protocol version.\n   */\n  protocolVersion: number;\n\n  /**\n   * Task ID. This is used to link the request with a preceding snapshot\n   * reservation.\n   */\n  taskID: string | null; // TODO: Make required when v3 is min.\n\n  /**\n   * Subscriber id. This is only used for debugging.\n   */\n  id: string;\n\n  /**\n   * The ReplicatorMode of the subscriber. 'backup' indicates that the\n   * subscriber is local to the `change-streamer` in the `replication-manager`,\n   * while 'serving' indicates that user-facing requests depend on the subscriber.\n   */\n  mode: ReplicatorMode;\n\n  /**\n   * The ChangeStreamer will return an Error if the subscriber is\n   * on a different replica version (i.e. the initial snapshot associated\n   * with the replication slot).\n   */\n  replicaVersion: string;\n\n  /**\n   * The watermark up to which the subscriber is up to date.\n   * Only changes after the watermark will be streamed.\n   */\n  watermark: string;\n\n  /**\n   * Whether this is the first subscription request made by the task,\n   * i.e. indicating that the watermark comes from a restored replica\n   * backup. The ChangeStreamer uses this to determine which changes\n   * are safe to purge from the Storer.\n   */\n  initial: boolean;\n};\n\nexport type ChangeEntry = {\n  change: Change;\n\n  /**\n   * Note that it is technically possible for multiple changes to have\n   * the same watermark, but that of a commit is guaranteed to be final,\n   * so subscribers should only store the watermark of commit changes.\n   */\n  watermark: string;\n};\n\n/**\n * The StatusMessage payload for now is empty, but can be extended to\n * include meta-level information in the future.\n */\nexport const statusSchema = v.object({\n  tag: v.literal('status'),\n});\n\nexport const statusMessageSchema = v.tuple([v.literal('status'), statusSchema]);\n\n/**\n * A StatusMessage will be immediately sent on a (v2+) subscription to\n * indicate that the subscription is valid (i.e. starting at the requested\n * watermark). Invalid subscriptions will instead result in a\n * SubscriptionError as the first message.\n */\nexport type StatusMessage = v.Infer<typeof statusMessageSchema>;\n\nconst subscriptionErrorSchema = v.object({\n  type: v.number(), // ErrorType\n  message: v.string().optional(),\n});\n\nexport type SubscriptionError = v.Infer<typeof subscriptionErrorSchema>;\n\nconst errorSchema = v.tuple([v.literal('error'), subscriptionErrorSchema]);\n\nexport const downstreamSchema = v.union(\n  statusMessageSchema,\n  changeStreamDataSchema,\n  errorSchema,\n);\n\nexport type Error = v.Infer<typeof errorSchema>;\n\n/**\n * A stream of transactions, each starting with a {@link Begin} message,\n * containing one or more {@link Data} messages, and ending with a\n * {@link Commit} or {@link Rollback} message. The 'commit' tuple\n * includes a `watermark` that should be stored with the committed\n * data and used for resuming a subscription (e.g. in the\n * {@link SubscriberContext}).\n *\n * A {@link SubscriptionError} indicates an unrecoverable error that requires\n * manual intervention (e.g. configuration / operational error).\n */\nexport type Downstream = v.Infer<typeof downstreamSchema>;\n\nexport interface ChangeStreamerService extends ChangeStreamer, Service {\n  /**\n   * Notifies the change streamer of a watermark that has been backed up,\n   * indicating that changes before the watermark can be purged if active\n   * subscribers have progressed beyond the watermark.\n   */\n  scheduleCleanup(watermark: string): void;\n\n  getChangeLogState(): Promise<{\n    replicaVersion: string;\n    minWatermark: string;\n  }>;\n}\n"],"names":["v.object","v.literal","v.tuple","v.number","v.string","v.union"],"mappings":";;;;AA6DO,MAAM,mBAAmB;AA+DzB,MAAM,eAAeA,OAAS;AAAA,EACnC,KAAKC,QAAU,QAAQ;AACzB,CAAC;AAEM,MAAM,sBAAsBC,MAAQ,CAACD,QAAU,QAAQ,GAAG,YAAY,CAAC;AAU9E,MAAM,0BAA0BD,OAAS;AAAA,EACvC,MAAMG,OAAE;AAAA;AAAA,EACR,SAASC,OAAE,EAAS,SAAA;AACtB,CAAC;AAID,MAAM,cAAcF,MAAQ,CAACD,QAAU,OAAO,GAAG,uBAAuB,CAAC;AAElE,MAAM,mBAAmBI;AAAAA,EAC9B;AAAA,EACA;AAAA,EACA;AACF;"}

package/out/zero-cache/src/services/change-streamer/error-type-enum.js

@@ -1,4 +1,9 @@
-export const Unknown = 0;
-export const WrongReplicaVersion = 1;
-export const WatermarkTooOld = 2;
-//# sourceMappingURL=error-type-enum.js.map
+const Unknown = 0;
+const WrongReplicaVersion = 1;
+const WatermarkTooOld = 2;
+export {
+  Unknown,
+  WatermarkTooOld,
+  WrongReplicaVersion
+};
+//# sourceMappingURL=error-type-enum.js.map

package/out/zero-cache/src/services/change-streamer/error-type-enum.js.map

@@ -1 +1 @@
-{"version":3,"file":"error-type-enum.js","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/error-type-enum.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,OAAO,GAAG,CAAC,CAAC;AACzB,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,CAAC;AACrC,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,CAAC"}
+{"version":3,"file":"error-type-enum.js","sources":["../../../../../../zero-cache/src/services/change-streamer/error-type-enum.ts"],"sourcesContent":["export const Unknown = 0;\nexport const WrongReplicaVersion = 1;\nexport const WatermarkTooOld = 2;\n\nexport type Unknown = typeof Unknown;\nexport type WrongReplicaVersion = typeof WrongReplicaVersion;\nexport type WatermarkTooOld = typeof WatermarkTooOld;\n"],"names":[],"mappings":"AAAO,MAAM,UAAU;AAChB,MAAM,sBAAsB;AAC5B,MAAM,kBAAkB;"}

package/out/zero-cache/src/services/change-streamer/forwarder.d.ts

@@ -1,5 +1,5 @@
 import type { WatermarkedChange } from './change-streamer-service.ts';
-import { Subscriber } from './subscriber.ts';
+import type { Subscriber } from './subscriber.ts';
 export declare class Forwarder {
     #private;
     /**

package/out/zero-cache/src/services/change-streamer/forwarder.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"forwarder.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/forwarder.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,8BAA8B,CAAC;AACpE,OAAO,EAAC,UAAU,EAAC,MAAM,iBAAiB,CAAC;AAE3C,qBAAa,SAAS;;IAKpB;;;;OAIG;IACH,GAAG,CAAC,GAAG,EAAE,UAAU;IAQnB,MAAM,CAAC,GAAG,EAAE,UAAU;IAMtB;;;;OAIG;IACH,OAAO,CAAC,KAAK,EAAE,iBAAiB;IA8BhC,OAAO,IAAI,GAAG,CAAC,MAAM,CAAC;CAQvB"}
+{"version":3,"file":"forwarder.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/forwarder.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,8BAA8B,CAAC;AACpE,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,iBAAiB,CAAC;AAEhD,qBAAa,SAAS;;IAKpB;;;;OAIG;IACH,GAAG,CAAC,GAAG,EAAE,UAAU;IAQnB,MAAM,CAAC,GAAG,EAAE,UAAU;IAMtB;;;;OAIG;IACH,OAAO,CAAC,KAAK,EAAE,iBAAiB;IA8BhC,OAAO,IAAI,GAAG,CAAC,MAAM,CAAC;CAQvB"}

package/out/zero-cache/src/services/change-streamer/forwarder.js

@@ -1,63 +1,59 @@
 import { joinIterables, wrapIterable } from "../../../../shared/src/iterables.js";
-import { Subscriber } from "./subscriber.js";
-export class Forwarder {
-    #active = new Set();
-    #queued = new Set();
-    #inTransaction = false;
-    /**
-     * `add()` is called in lock step with `Storer.catchup()` so that the
-     * two components have an equivalent interpretation of whether a Transaction is
-     * currently being streamed.
-     */
-    add(sub) {
-        if (this.#inTransaction) {
-            this.#queued.add(sub);
-        }
-        else {
-            this.#active.add(sub);
-        }
+class Forwarder {
+  #active = /* @__PURE__ */ new Set();
+  #queued = /* @__PURE__ */ new Set();
+  #inTransaction = false;
+  /**
+   * `add()` is called in lock step with `Storer.catchup()` so that the
+   * two components have an equivalent interpretation of whether a Transaction is
+   * currently being streamed.
+   */
+  add(sub) {
+    if (this.#inTransaction) {
+      this.#queued.add(sub);
+    } else {
+      this.#active.add(sub);
     }
-    remove(sub) {
-        this.#active.delete(sub);
-        this.#queued.delete(sub);
-        sub.close();
+  }
+  remove(sub) {
+    this.#active.delete(sub);
+    this.#queued.delete(sub);
+    sub.close();
+  }
+  /**
+   * `forward()` is called in lockstep with `Storer.store()` so that the
+   * two components have an equivalent interpretation of whether a Transaction is
+   * currently being streamed.
+   */
+  forward(entry) {
+    const [type] = entry[1];
+    for (const active of this.#active.values()) {
+      active.send(entry);
     }
-    /**
-     * `forward()` is called in lockstep with `Storer.store()` so that the
-     * two components have an equivalent interpretation of whether a Transaction is
-     * currently being streamed.
-     */
-    forward(entry) {
-        const [type] = entry[1];
-        for (const active of this.#active.values()) {
-            active.send(entry);
+    switch (type) {
+      case "begin":
+        this.#inTransaction = true;
+        break;
+      case "commit":
+      case "rollback":
+        this.#inTransaction = false;
+        for (const sub of this.#queued.values()) {
+          this.#active.add(sub);
         }
-        switch (type) {
-            case 'begin':
-                // While in a Transaction, all added subscribers are "queued" so that no
-                // messages are forwarded to them. This state corresponds to being queued
-                // for catchup in the Storer, which will retrieve historic changes
-                // and call catchup() once the current transaction is committed.
-                this.#inTransaction = true;
-                break;
-            case 'commit':
-            case 'rollback':
-                // Upon commit or rollback, all queued subscribers are transferred to
-                // the active set. This means that they can receive messages starting
-                // from the next transaction.
-                //
-                // Note that if catchup is still in progress (in the Storer), these messages
-                // will be buffered in the backlog until catchup completes.
-                this.#inTransaction = false;
-                for (const sub of this.#queued.values()) {
-                    this.#active.add(sub);
-                }
-                this.#queued.clear();
-                break;
-        }
-    }
-    getAcks() {
-        return new Set(joinIterables(wrapIterable(this.#active).map(s => s.acked), wrapIterable(this.#queued).map(s => s.acked)));
+        this.#queued.clear();
+        break;
     }
+  }
+  getAcks() {
+    return new Set(
+      joinIterables(
+        wrapIterable(this.#active).map((s) => s.acked),
+        wrapIterable(this.#queued).map((s) => s.acked)
+      )
+    );
+  }
 }
-//# sourceMappingURL=forwarder.js.map
+export {
+  Forwarder
+};
+//# sourceMappingURL=forwarder.js.map

package/out/zero-cache/src/services/change-streamer/forwarder.js.map

@@ -1 +1 @@
-{"version":3,"file":"forwarder.js","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/forwarder.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,aAAa,EAAE,YAAY,EAAC,MAAM,qCAAqC,CAAC;AAEhF,OAAO,EAAC,UAAU,EAAC,MAAM,iBAAiB,CAAC;AAE3C,MAAM,OAAO,SAAS;IACX,OAAO,GAAG,IAAI,GAAG,EAAc,CAAC;IAChC,OAAO,GAAG,IAAI,GAAG,EAAc,CAAC;IACzC,cAAc,GAAG,KAAK,CAAC;IAEvB;;;;OAIG;IACH,GAAG,CAAC,GAAe;QACjB,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACxB,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACxB,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACxB,CAAC;IACH,CAAC;IAED,MAAM,CAAC,GAAe;QACpB,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACzB,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACzB,GAAG,CAAC,KAAK,EAAE,CAAC;IACd,CAAC;IAED;;;;OAIG;IACH,OAAO,CAAC,KAAwB;QAC9B,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACxB,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;YAC3C,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACrB,CAAC;QACD,QAAQ,IAAI,EAAE,CAAC;YACb,KAAK,OAAO;gBACV,wEAAwE;gBACxE,yEAAyE;gBACzE,kEAAkE;gBAClE,gEAAgE;gBAChE,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;gBAC3B,MAAM;YACR,KAAK,QAAQ,CAAC;YACd,KAAK,UAAU;gBACb,qEAAqE;gBACrE,qEAAqE;gBACrE,6BAA6B;gBAC7B,EAAE;gBACF,4EAA4E;gBAC5E,2DAA2D;gBAC3D,IAAI,CAAC,cAAc,GAAG,KAAK,CAAC;gBAC5B,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;oBACxC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBACxB,CAAC;gBACD,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;gBACrB,MAAM;QACV,CAAC;IACH,CAAC;IAED,OAAO;QACL,OAAO,IAAI,GAAG,CACZ,aAAa,CACX,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,EAC5C,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAC7C,CACF,CAAC;IACJ,CAAC;CACF"}
+{"version":3,"file":"forwarder.js","sources":["../../../../../../zero-cache/src/services/change-streamer/forwarder.ts"],"sourcesContent":["import {joinIterables, wrapIterable} from '../../../../shared/src/iterables.ts';\nimport type {WatermarkedChange} from './change-streamer-service.ts';\nimport type {Subscriber} from './subscriber.ts';\n\nexport class Forwarder {\n  readonly #active = new Set<Subscriber>();\n  readonly #queued = new Set<Subscriber>();\n  #inTransaction = false;\n\n  /**\n   * `add()` is called in lock step with `Storer.catchup()` so that the\n   * two components have an equivalent interpretation of whether a Transaction is\n   * currently being streamed.\n   */\n  add(sub: Subscriber) {\n    if (this.#inTransaction) {\n      this.#queued.add(sub);\n    } else {\n      this.#active.add(sub);\n    }\n  }\n\n  remove(sub: Subscriber) {\n    this.#active.delete(sub);\n    this.#queued.delete(sub);\n    sub.close();\n  }\n\n  /**\n   * `forward()` is called in lockstep with `Storer.store()` so that the\n   * two components have an equivalent interpretation of whether a Transaction is\n   * currently being streamed.\n   */\n  forward(entry: WatermarkedChange) {\n    const [type] = entry[1];\n    for (const active of this.#active.values()) {\n      active.send(entry);\n    }\n    switch (type) {\n      case 'begin':\n        // While in a Transaction, all added subscribers are \"queued\" so that no\n        // messages are forwarded to them. This state corresponds to being queued\n        // for catchup in the Storer, which will retrieve historic changes\n        // and call catchup() once the current transaction is committed.\n        this.#inTransaction = true;\n        break;\n      case 'commit':\n      case 'rollback':\n        // Upon commit or rollback, all queued subscribers are transferred to\n        // the active set. This means that they can receive messages starting\n        // from the next transaction.\n        //\n        // Note that if catchup is still in progress (in the Storer), these messages\n        // will be buffered in the backlog until catchup completes.\n        this.#inTransaction = false;\n        for (const sub of this.#queued.values()) {\n          this.#active.add(sub);\n        }\n        this.#queued.clear();\n        break;\n    }\n  }\n\n  getAcks(): Set<string> {\n    return new Set(\n      joinIterables(\n        wrapIterable(this.#active).map(s => s.acked),\n        wrapIterable(this.#queued).map(s => s.acked),\n      ),\n    );\n  }\n}\n"],"names":[],"mappings":";AAIO,MAAM,UAAU;AAAA,EACZ,8BAAc,IAAA;AAAA,EACd,8BAAc,IAAA;AAAA,EACvB,iBAAiB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOjB,IAAI,KAAiB;AACnB,QAAI,KAAK,gBAAgB;AACvB,WAAK,QAAQ,IAAI,GAAG;AAAA,IACtB,OAAO;AACL,WAAK,QAAQ,IAAI,GAAG;AAAA,IACtB;AAAA,EACF;AAAA,EAEA,OAAO,KAAiB;AACtB,SAAK,QAAQ,OAAO,GAAG;AACvB,SAAK,QAAQ,OAAO,GAAG;AACvB,QAAI,MAAA;AAAA,EACN;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,QAAQ,OAA0B;AAChC,UAAM,CAAC,IAAI,IAAI,MAAM,CAAC;AACtB,eAAW,UAAU,KAAK,QAAQ,OAAA,GAAU;AAC1C,aAAO,KAAK,KAAK;AAAA,IACnB;AACA,YAAQ,MAAA;AAAA,MACN,KAAK;AAKH,aAAK,iBAAiB;AACtB;AAAA,MACF,KAAK;AAAA,MACL,KAAK;AAOH,aAAK,iBAAiB;AACtB,mBAAW,OAAO,KAAK,QAAQ,OAAA,GAAU;AACvC,eAAK,QAAQ,IAAI,GAAG;AAAA,QACtB;AACA,aAAK,QAAQ,MAAA;AACb;AAAA,IAAA;AAAA,EAEN;AAAA,EAEA,UAAuB;AACrB,WAAO,IAAI;AAAA,MACT;AAAA,QACE,aAAa,KAAK,OAAO,EAAE,IAAI,CAAA,MAAK,EAAE,KAAK;AAAA,QAC3C,aAAa,KAAK,OAAO,EAAE,IAAI,CAAA,MAAK,EAAE,KAAK;AAAA,MAAA;AAAA,IAC7C;AAAA,EAEJ;AACF;"}

package/out/zero-cache/src/services/change-streamer/replica-monitor.d.ts

@@ -0,0 +1,16 @@
+import type { LogContext } from '@rocicorp/logger';
+import type { Service } from '../service.ts';
+import type { ChangeStreamerService } from './change-streamer.ts';
+/**
+ * The single-node equivalent of the {@link BackupMonitor} polls the replica
+ * file every 30 seconds and schedules cleanup when the watermark
+ * (i.e. stateVersion) moves forward.
+ */
+export declare class ReplicaMonitor implements Service {
+    #private;
+    readonly id = "replica-monitor";
+    constructor(lc: LogContext, replicaFile: string, changeStreamer: ChangeStreamerService);
+    run(): Promise<void>;
+    stop(): Promise<void>;
+}
+//# sourceMappingURL=replica-monitor.d.ts.map

package/out/zero-cache/src/services/change-streamer/replica-monitor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"replica-monitor.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/replica-monitor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAMjD,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,eAAe,CAAC;AAC3C,OAAO,KAAK,EAAC,qBAAqB,EAAC,MAAM,sBAAsB,CAAC;AAIhE;;;;GAIG;AACH,qBAAa,cAAe,YAAW,OAAO;;IAC5C,QAAQ,CAAC,EAAE,qBAAqB;gBAS9B,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,cAAc,EAAE,qBAAqB;IAOjC,GAAG;IAwBT,IAAI;CAIL"}

package/out/zero-cache/src/services/change-streamer/replica-monitor.js

@@ -0,0 +1,48 @@
+import { promiseVoid } from "../../../../shared/src/resolved-promises.js";
+import { Database } from "../../../../zqlite/src/db.js";
+import { StatementRunner } from "../../db/statements.js";
+import { getReplicationState } from "../replicator/schema/replication-state.js";
+import { RunningState } from "../running-state.js";
+const CHECK_INTERVAL_MS = 30 * 1e3;
+class ReplicaMonitor {
+  id = "replica-monitor";
+  #lc;
+  #replicaFile;
+  #changeStreamer;
+  #state = new RunningState(this.id);
+  #lastWatermark = "";
+  constructor(lc, replicaFile, changeStreamer) {
+    this.#lc = lc.withContext("component", this.id);
+    this.#replicaFile = replicaFile;
+    this.#changeStreamer = changeStreamer;
+  }
+  async run() {
+    this.#lc.info?.(`starting replica monitor`);
+    await this.#state.sleep(CHECK_INTERVAL_MS);
+    while (this.#state.shouldRun()) {
+      const db = new Database(this.#lc, this.#replicaFile);
+      try {
+        const { stateVersion } = getReplicationState(new StatementRunner(db));
+        if (stateVersion !== this.#lastWatermark) {
+          this.#lastWatermark = stateVersion;
+          this.#lc.debug?.(`replicated up to watermark ${stateVersion}`);
+          this.#changeStreamer.scheduleCleanup(stateVersion);
+        }
+      } catch (e) {
+        this.#lc.error?.(`Unable to read watermark from replica`, e);
+      } finally {
+        db.close();
+      }
+      await this.#state.sleep(CHECK_INTERVAL_MS);
+    }
+    this.#lc.info?.(`replica monitor stopped`);
+  }
+  stop() {
+    this.#state.stop(this.#lc);
+    return promiseVoid;
+  }
+}
+export {
+  ReplicaMonitor
+};
+//# sourceMappingURL=replica-monitor.js.map

package/out/zero-cache/src/services/change-streamer/replica-monitor.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"replica-monitor.js","sources":["../../../../../../zero-cache/src/services/change-streamer/replica-monitor.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {StatementRunner} from '../../db/statements.ts';\nimport {getReplicationState} from '../replicator/schema/replication-state.ts';\nimport {RunningState} from '../running-state.ts';\nimport type {Service} from '../service.ts';\nimport type {ChangeStreamerService} from './change-streamer.ts';\n\nconst CHECK_INTERVAL_MS = 30 * 1000;\n\n/**\n * The single-node equivalent of the {@link BackupMonitor} polls the replica\n * file every 30 seconds and schedules cleanup when the watermark\n * (i.e. stateVersion) moves forward.\n */\nexport class ReplicaMonitor implements Service {\n  readonly id = 'replica-monitor';\n  readonly #lc: LogContext;\n  readonly #replicaFile: string;\n  readonly #changeStreamer: ChangeStreamerService;\n  readonly #state = new RunningState(this.id);\n\n  #lastWatermark: string = '';\n\n  constructor(\n    lc: LogContext,\n    replicaFile: string,\n    changeStreamer: ChangeStreamerService,\n  ) {\n    this.#lc = lc.withContext('component', this.id);\n    this.#replicaFile = replicaFile;\n    this.#changeStreamer = changeStreamer;\n  }\n\n  async run() {\n    this.#lc.info?.(`starting replica monitor`);\n    await this.#state.sleep(CHECK_INTERVAL_MS);\n\n    while (this.#state.shouldRun()) {\n      const db = new Database(this.#lc, this.#replicaFile);\n      try {\n        const {stateVersion} = getReplicationState(new StatementRunner(db));\n        if (stateVersion !== this.#lastWatermark) {\n          this.#lastWatermark = stateVersion;\n          this.#lc.debug?.(`replicated up to watermark ${stateVersion}`);\n          this.#changeStreamer.scheduleCleanup(stateVersion);\n        }\n      } catch (e) {\n        this.#lc.error?.(`Unable to read watermark from replica`, e);\n      } finally {\n        db.close();\n      }\n\n      await this.#state.sleep(CHECK_INTERVAL_MS);\n    }\n    this.#lc.info?.(`replica monitor stopped`);\n  }\n\n  stop() {\n    this.#state.stop(this.#lc);\n    return promiseVoid;\n  }\n}\n"],"names":[],"mappings":";;;;;AASA,MAAM,oBAAoB,KAAK;AAOxB,MAAM,eAAkC;AAAA,EACpC,KAAK;AAAA,EACL;AAAA,EACA;AAAA,EACA;AAAA,EACA,SAAS,IAAI,aAAa,KAAK,EAAE;AAAA,EAE1C,iBAAyB;AAAA,EAEzB,YACE,IACA,aACA,gBACA;AACA,SAAK,MAAM,GAAG,YAAY,aAAa,KAAK,EAAE;AAC9C,SAAK,eAAe;AACpB,SAAK,kBAAkB;AAAA,EACzB;AAAA,EAEA,MAAM,MAAM;AACV,SAAK,IAAI,OAAO,0BAA0B;AAC1C,UAAM,KAAK,OAAO,MAAM,iBAAiB;AAEzC,WAAO,KAAK,OAAO,aAAa;AAC9B,YAAM,KAAK,IAAI,SAAS,KAAK,KAAK,KAAK,YAAY;AACnD,UAAI;AACF,cAAM,EAAC,aAAA,IAAgB,oBAAoB,IAAI,gBAAgB,EAAE,CAAC;AAClE,YAAI,iBAAiB,KAAK,gBAAgB;AACxC,eAAK,iBAAiB;AACtB,eAAK,IAAI,QAAQ,8BAA8B,YAAY,EAAE;AAC7D,eAAK,gBAAgB,gBAAgB,YAAY;AAAA,QACnD;AAAA,MACF,SAAS,GAAG;AACV,aAAK,IAAI,QAAQ,yCAAyC,CAAC;AAAA,MAC7D,UAAA;AACE,WAAG,MAAA;AAAA,MACL;AAEA,YAAM,KAAK,OAAO,MAAM,iBAAiB;AAAA,IAC3C;AACA,SAAK,IAAI,OAAO,yBAAyB;AAAA,EAC3C;AAAA,EAEA,OAAO;AACL,SAAK,OAAO,KAAK,KAAK,GAAG;AACzB,WAAO;AAAA,EACT;AACF;"}