@rocicorp/zero 0.26.0-canary.0 → 0.26.0-canary.2

package/out/zero-cache/src/services/change-streamer/storer.js

@@ -14,7 +14,6 @@ import "../change-source/protocol/current/downstream.js";
 import "./change-streamer.js";
 import { WatermarkTooOld } from "./error-type-enum.js";
 import { AutoResetSignal, markResetRequired } from "./schema/tables.js";
-const QUEUE_SIZE_BACK_PRESSURE_THRESHOLD = 1e5;
 class Storer {
   id = "storer";
   #lc;
@@ -27,8 +26,9 @@ class Storer {
   #onConsumed;
   #onFatal;
   #queue = new Queue();
+  #backPressureThreshold;
   #running = false;
-  constructor(lc, shard, taskID, discoveryAddress, discoveryProtocol, db, replicaVersion, onConsumed, onFatal) {
+  constructor(lc, shard, taskID, discoveryAddress, discoveryProtocol, db, replicaVersion, onConsumed, onFatal, backPressureThreshold) {
     this.#lc = lc;
     this.#shard = shard;
     this.#taskID = taskID;
@@ -38,6 +38,7 @@ class Storer {
     this.#replicaVersion = replicaVersion;
     this.#onConsumed = onConsumed;
     this.#onFatal = onFatal;
+    this.#backPressureThreshold = backPressureThreshold;
   }
   // For readability in SQL statements.
   #cdc(table) {
@@ -101,9 +102,19 @@ class Storer {
     if (!this.#running) {
       return void 0;
     }
-    if (this.#readyForMore === null && this.#queue.size() > QUEUE_SIZE_BACK_PRESSURE_THRESHOLD) {
+    if (this.#readyForMore === null && this.#queue.size() > this.#backPressureThreshold) {
       this.#lc.warn?.(
-        `applying back pressure with ${this.#queue.size()} queued changes`
+        `applying back pressure with ${this.#queue.size()} queued changes (threshold: ${this.#backPressureThreshold})
+
+To inspect changeLog backlog in your CVR database:
+  SELECT
+    (change->'relation'->>'schema') || '.' || (change->'relation'->>'name') AS table_name,
+    change->>'tag' AS operation,
+    COUNT(*) AS count
+  FROM "<app_id>/cdc"."changeLog"
+  GROUP BY 1, 2
+  ORDER BY 3 DESC
+  LIMIT 20;`
       );
       this.#readyForMore = resolver();
     }
@@ -111,7 +122,7 @@ class Storer {
   }
   #maybeReleaseBackPressure() {
     if (this.#readyForMore !== null && // Wait for at least 10% of the threshold to free up.
-    this.#queue.size() < QUEUE_SIZE_BACK_PRESSURE_THRESHOLD * 0.9) {
+    this.#queue.size() < this.#backPressureThreshold * 0.9) {
       this.#lc.info?.(
         `releasing back pressure with ${this.#queue.size()} queued changes`
       );

package/out/zero-cache/src/services/change-streamer/storer.js.map

@@ -1 +1 @@
-{"version":3,"file":"storer.js","sources":["../../../../../../zero-cache/src/services/change-streamer/storer.ts"],"sourcesContent":["import {PG_SERIALIZATION_FAILURE} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport {resolver, type Resolver} from '@rocicorp/resolver';\nimport postgres from 'postgres';\nimport {AbortError} from '../../../../shared/src/abort-error.ts';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport {type JSONValue} from '../../../../shared/src/bigint-json.ts';\nimport {Queue} from '../../../../shared/src/queue.ts';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport * as Mode from '../../db/mode-enum.ts';\nimport {TransactionPool} from '../../db/transaction-pool.ts';\nimport {disableStatementTimeout, type PostgresDB} from '../../types/pg.ts';\nimport {cdcSchema, type ShardID} from '../../types/shards.ts';\nimport {type Commit} from '../change-source/protocol/current/downstream.ts';\nimport type {StatusMessage} from '../change-source/protocol/current/status.ts';\nimport type {ReplicatorMode} from '../replicator/replicator.ts';\nimport type {Service} from '../service.ts';\nimport type {WatermarkedChange} from './change-streamer-service.ts';\nimport {type ChangeEntry} from './change-streamer.ts';\nimport * as ErrorType from './error-type-enum.ts';\nimport {\n  AutoResetSignal,\n  markResetRequired,\n  type ReplicationState,\n} from './schema/tables.ts';\nimport type {Subscriber} from './subscriber.ts';\n\ntype SubscriberAndMode = {\n  subscriber: Subscriber;\n  mode: ReplicatorMode;\n};\n\ntype QueueEntry =\n  | ['change', WatermarkedChange]\n  | ['ready', callback: () => void]\n  | ['subscriber', SubscriberAndMode]\n  | StatusMessage\n  | ['abort']\n  | 'stop';\n\ntype PendingTransaction = {\n  pool: TransactionPool;\n  preCommitWatermark: string;\n  pos: number;\n  startingReplicationState: Promise<ReplicationState>;\n};\n\n// Technically, any threshold is fine because the point of back pressure\n// is to adjust the rate of incoming messages, and the size of the pending\n// work queue does not affect that mechanism.\n//\n// However, it is theoretically possible to exceed the available memory if\n// the size of changes is very large. This threshold can be improved by\n// roughly measuring the size of the enqueued contents and setting the\n// threshold based on available memory.\n//\n// TODO: switch to a message size-based thresholding when migrating over\n// to stringified JSON messages, which will bound the computation involved\n// in measuring the size of row messages.\nconst QUEUE_SIZE_BACK_PRESSURE_THRESHOLD = 100_000;\n\n/**\n * Handles the storage of changes and the catchup of subscribers\n * that are behind.\n *\n * In the context of catchup and cleanup, it is the responsibility of the\n * Storer to decide whether a client can be caught up, or whether the\n * changes needed to catch a client up have been purged.\n *\n * **Maintained invariant**: The Change DB is only empty for a\n * completely new replica (i.e. initial-sync with no changes from the\n * replication stream).\n * * In this case, all new subscribers are expected start from the\n *   `replicaVersion`, which is the version at which initial sync\n *   was performed, and any attempts to catchup from a different\n *   point fail.\n *\n * Conversely, if non-initial changes have flowed through the system\n * (i.e. via the replication stream), the ChangeDB must *not* be empty,\n * and the earliest change in the `changeLog` represents the earliest\n * \"commit\" from (after) which a subscriber can be caught up.\n * * Any attempts to catchup from an earlier point must fail with\n *   a `WatermarkTooOld` error.\n * * Failure to do so could result in streaming changes to the\n *   subscriber such that there is a gap in its replication history.\n *\n * Note: Subscribers (i.e. `incremental-syncer`) consider an \"error\" signal\n * an unrecoverable error and shut down in response. This allows the\n * production system to replace it with a new task and fresh copy of the\n * replica backup.\n */\nexport class Storer implements Service {\n  readonly id = 'storer';\n  readonly #lc: LogContext;\n  readonly #shard: ShardID;\n  readonly #taskID: string;\n  readonly #discoveryAddress: string;\n  readonly #discoveryProtocol: string;\n  readonly #db: PostgresDB;\n  readonly #replicaVersion: string;\n  readonly #onConsumed: (c: Commit | StatusMessage) => void;\n  readonly #onFatal: (err: Error) => void;\n  readonly #queue = new Queue<QueueEntry>();\n\n  #running = false;\n\n  constructor(\n    lc: LogContext,\n    shard: ShardID,\n    taskID: string,\n    discoveryAddress: string,\n    discoveryProtocol: string,\n    db: PostgresDB,\n    replicaVersion: string,\n    onConsumed: (c: Commit | StatusMessage) => void,\n    onFatal: (err: Error) => void,\n  ) {\n    this.#lc = lc;\n    this.#shard = shard;\n    this.#taskID = taskID;\n    this.#discoveryAddress = discoveryAddress;\n    this.#discoveryProtocol = discoveryProtocol;\n    this.#db = db;\n    this.#replicaVersion = replicaVersion;\n    this.#onConsumed = onConsumed;\n    this.#onFatal = onFatal;\n  }\n\n  // For readability in SQL statements.\n  #cdc(table: string) {\n    return this.#db(`${cdcSchema(this.#shard)}.${table}`);\n  }\n\n  async assumeOwnership() {\n    const db = this.#db;\n    const owner = this.#taskID;\n    const ownerAddress = this.#discoveryAddress;\n    const ownerProtocol = this.#discoveryProtocol;\n    // we omit `ws://` so that old view syncer versions that are not expecting the protocol continue to not get it\n    const addressWithProtocol =\n      ownerProtocol === 'ws'\n        ? ownerAddress\n        : `${ownerProtocol}://${ownerAddress}`;\n    await db`UPDATE ${this.#cdc('replicationState')} SET ${db({owner, ownerAddress: addressWithProtocol})}`;\n    this.#lc.info?.(`assumed ownership at ${addressWithProtocol}`);\n  }\n\n  async getLastWatermarkToStartStream(): Promise<string> {\n    // Before starting or restarting a stream from the change source,\n    // wait for all queued changes to be processed so that we pick up\n    // from the right spot.\n    const {promise: ready, resolve} = resolver();\n    this.#queue.enqueue(['ready', resolve]);\n    await ready;\n\n    const [{lastWatermark}] = await this.#db<{lastWatermark: string}[]>`\n      SELECT \"lastWatermark\" FROM ${this.#cdc('replicationState')}`;\n    return lastWatermark;\n  }\n\n  async getMinWatermarkForCatchup(): Promise<string | null> {\n    const [{minWatermark}] = await this.#db<\n      {minWatermark: string | null}[]\n    > /*sql*/ `\n      SELECT min(watermark) as \"minWatermark\" FROM ${this.#cdc('changeLog')}`;\n    return minWatermark;\n  }\n\n  purgeRecordsBefore(watermark: string): Promise<number> {\n    return this.#db.begin(Mode.SERIALIZABLE, async sql => {\n      disableStatementTimeout(sql);\n\n      // Check ownership before performing the purge. The server is expected to\n      // exit immediately when an ownership change is detected, but checking\n      // explicitly guards against race conditions.\n      const [{owner}] = await sql<ReplicationState[]>`\n        SELECT * FROM ${this.#cdc('replicationState')}`;\n      if (owner !== this.#taskID) {\n        this.#lc.warn?.(\n          `Ignoring change log purge request (${watermark}) while not owner`,\n        );\n        return 0;\n      }\n\n      const [{deleted}] = await sql<{deleted: bigint}[]>`\n        WITH purged AS (\n          DELETE FROM ${this.#cdc('changeLog')} WHERE watermark < ${watermark} \n            RETURNING watermark, pos\n        ) SELECT COUNT(*) as deleted FROM purged;`;\n      return Number(deleted);\n    });\n  }\n\n  store(entry: WatermarkedChange) {\n    this.#queue.enqueue(['change', entry]);\n  }\n\n  abort() {\n    this.#queue.enqueue(['abort']);\n  }\n\n  status(s: StatusMessage) {\n    this.#queue.enqueue(s);\n  }\n\n  catchup(subscriber: Subscriber, mode: ReplicatorMode) {\n    this.#queue.enqueue(['subscriber', {subscriber, mode}]);\n  }\n\n  #readyForMore: Resolver<void> | null = null;\n\n  readyForMore(): Promise<void> | undefined {\n    if (!this.#running) {\n      return undefined;\n    }\n    if (\n      this.#readyForMore === null &&\n      this.#queue.size() > QUEUE_SIZE_BACK_PRESSURE_THRESHOLD\n    ) {\n      this.#lc.warn?.(\n        `applying back pressure with ${this.#queue.size()} queued changes`,\n      );\n      this.#readyForMore = resolver();\n    }\n    return this.#readyForMore?.promise;\n  }\n\n  #maybeReleaseBackPressure() {\n    if (\n      this.#readyForMore !== null &&\n      // Wait for at least 10% of the threshold to free up.\n      this.#queue.size() < QUEUE_SIZE_BACK_PRESSURE_THRESHOLD * 0.9\n    ) {\n      this.#lc.info?.(\n        `releasing back pressure with ${this.#queue.size()} queued changes`,\n      );\n      this.#readyForMore.resolve();\n      this.#readyForMore = null;\n    }\n  }\n\n  async run() {\n    this.#running = true;\n    try {\n      await this.#processQueue();\n    } finally {\n      this.#running = false;\n      this.#lc.info?.('storer stopped');\n    }\n  }\n\n  async #processQueue() {\n    let tx: PendingTransaction | null = null;\n    let msg: QueueEntry | false;\n\n    const catchupQueue: SubscriberAndMode[] = [];\n    while ((msg = await this.#queue.dequeue()) !== 'stop') {\n      this.#maybeReleaseBackPressure();\n\n      const [msgType] = msg;\n      switch (msgType) {\n        case 'ready': {\n          const signalReady = msg[1];\n          signalReady();\n          continue;\n        }\n        case 'subscriber': {\n          const subscriber = msg[1];\n          if (tx) {\n            catchupQueue.push(subscriber); // Wait for the current tx to complete.\n          } else {\n            await this.#startCatchup([subscriber]); // Catch up immediately.\n          }\n          continue;\n        }\n        case 'status':\n          this.#onConsumed(msg);\n          continue;\n        case 'abort': {\n          if (tx) {\n            tx.pool.abort();\n            await tx.pool.done();\n            tx = null;\n          }\n          continue;\n        }\n      }\n      // msgType === 'change'\n      const [watermark, downstream] = msg[1];\n      const [tag, change] = downstream;\n      if (tag === 'begin') {\n        assert(!tx, 'received BEGIN in the middle of a transaction');\n        const {promise, resolve, reject} = resolver<ReplicationState>();\n        tx = {\n          pool: new TransactionPool(\n            this.#lc.withContext('watermark', watermark),\n            Mode.SERIALIZABLE,\n          ),\n          preCommitWatermark: watermark,\n          pos: 0,\n          startingReplicationState: promise,\n        };\n        tx.pool.run(this.#db);\n        // Pipeline a read of the current ReplicationState,\n        // which will be checked before committing.\n        void tx.pool.process(tx => {\n          tx<ReplicationState[]>`\n          SELECT * FROM ${this.#cdc('replicationState')}`.then(\n            ([result]) => resolve(result),\n            reject,\n          );\n          return [];\n        });\n      } else {\n        assert(tx, `received ${tag} outside of transaction`);\n        tx.pos++;\n      }\n\n      const entry = {\n        watermark: tag === 'commit' ? watermark : tx.preCommitWatermark,\n        precommit: tag === 'commit' ? tx.preCommitWatermark : null,\n        pos: tx.pos,\n        change: change as unknown as JSONValue,\n      };\n\n      const processed = tx.pool.process(tx => [\n        tx`\n        INSERT INTO ${this.#cdc('changeLog')} ${tx(entry)}`,\n      ]);\n\n      if (tag === 'data' && tx.pos % 10_000 === 0) {\n        // Backpressure is exerted on commit when awaiting tx.pool.done().\n        // However, backpressure checks need to be regularly done for\n        // very large transactions in order to avoid memory blowup.\n        await processed;\n      }\n\n      if (tag === 'commit') {\n        const {owner} = await tx.startingReplicationState;\n        if (owner !== this.#taskID) {\n          // Ownership change reflected in the replicationState read in 'begin'.\n          tx.pool.fail(\n            new AbortError(`changeLog ownership has been assumed by ${owner}`),\n          );\n        } else {\n          // Update the replication state.\n          const lastWatermark = watermark;\n          void tx.pool.process(tx => [\n            tx`\n            UPDATE ${this.#cdc('replicationState')} SET ${tx({lastWatermark})}`,\n          ]);\n          tx.pool.setDone();\n        }\n\n        try {\n          await tx.pool.done();\n        } catch (e) {\n          if (\n            e instanceof postgres.PostgresError &&\n            e.code === PG_SERIALIZATION_FAILURE\n          ) {\n            // Ownership change happened after the replicationState was read in 'begin'.\n            throw new AbortError(`changeLog ownership has changed`, {cause: e});\n          }\n          throw e;\n        }\n\n        tx = null;\n\n        // ACK the LSN to the upstream Postgres.\n        this.#onConsumed(downstream);\n\n        // Before beginning the next transaction, open a READONLY snapshot to\n        // concurrently catchup any queued subscribers.\n        await this.#startCatchup(catchupQueue.splice(0));\n      } else if (tag === 'rollback') {\n        // Aborted transactions are not stored in the changeLog. Abort the current tx\n        // and process catchup of subscribers that were waiting for it to end.\n        tx.pool.abort();\n        await tx.pool.done();\n        tx = null;\n\n        await this.#startCatchup(catchupQueue.splice(0));\n      }\n    }\n  }\n\n  async #startCatchup(subs: SubscriberAndMode[]) {\n    if (subs.length === 0) {\n      return;\n    }\n\n    const reader = new TransactionPool(\n      this.#lc.withContext('pool', 'catchup'),\n      Mode.READONLY,\n    );\n    reader.run(this.#db);\n\n    // Ensure that the transaction has started (and is thus holding a snapshot\n    // of the database) before continuing on to commit more changes. This is\n    // done by waiting for a no-op task to be processed by the pool, which\n    // indicates that the BEGIN statement has been sent to the database.\n    await reader.processReadTask(() => {});\n\n    // Run the actual catchup queries in the background. Errors are handled in\n    // #catchup() by disconnecting the associated subscriber.\n    void Promise.all(subs.map(sub => this.#catchup(sub, reader))).finally(() =>\n      reader.setDone(),\n    );\n  }\n\n  async #catchup(\n    {subscriber: sub, mode}: SubscriberAndMode,\n    reader: TransactionPool,\n  ) {\n    try {\n      await reader.processReadTask(async tx => {\n        const start = Date.now();\n\n        // When starting from initial-sync, there won't be a change with a watermark\n        // equal to the replica version. This is the empty changeLog scenario.\n        let watermarkFound = sub.watermark === this.#replicaVersion;\n        let count = 0;\n        let lastBatchConsumed: Promise<unknown> | undefined;\n\n        for await (const entries of tx<ChangeEntry[]>`\n          SELECT watermark, change FROM ${this.#cdc('changeLog')}\n           WHERE watermark >= ${sub.watermark}\n           ORDER BY watermark, pos`.cursor(2000)) {\n          // Wait for the last batch of entries to be consumed by the\n          // subscriber before sending down the current batch. This pipelining\n          // allows one batch of changes to be received from the change-db\n          // while the previous batch of changes are sent to the subscriber,\n          // resulting in flow control that caps the number of changes\n          // referenced in memory to 2 * batch-size.\n          const start = performance.now();\n          await lastBatchConsumed;\n          const elapsed = performance.now() - start;\n          if (lastBatchConsumed) {\n            (elapsed > 100 ? this.#lc.info : this.#lc.debug)?.(\n              `waited ${elapsed.toFixed(3)} ms for ${sub.id} to consume last batch of catchup entries`,\n            );\n          }\n\n          for (const entry of entries) {\n            if (entry.watermark === sub.watermark) {\n              // This should be the first entry.\n              // Catchup starts from *after* the watermark.\n              watermarkFound = true;\n            } else if (watermarkFound) {\n              lastBatchConsumed = sub.catchup(toDownstream(entry)).result;\n              count++;\n            } else if (mode === 'backup') {\n              throw new AutoResetSignal(\n                `backup replica at watermark ${sub.watermark} is behind change db: ${entry.watermark})`,\n              );\n            } else {\n              this.#lc.warn?.(\n                `rejecting subscriber at watermark ${sub.watermark} (earliest watermark: ${entry.watermark})`,\n              );\n              sub.close(\n                ErrorType.WatermarkTooOld,\n                `earliest supported watermark is ${entry.watermark} (requested ${sub.watermark})`,\n              );\n              return;\n            }\n          }\n        }\n        if (watermarkFound) {\n          await lastBatchConsumed;\n          this.#lc.info?.(\n            `caught up ${sub.id} with ${count} changes (${\n              Date.now() - start\n            } ms)`,\n          );\n        } else {\n          this.#lc.warn?.(\n            `subscriber at watermark ${sub.watermark} is ahead of latest watermark`,\n          );\n        }\n        // Flushes the backlog of messages buffered during catchup and\n        // allows the subscription to forward subsequent messages immediately.\n        sub.setCaughtUp();\n      });\n    } catch (err) {\n      this.#lc.error?.(`error while catching up subscriber ${sub.id}`, err);\n      if (err instanceof AutoResetSignal) {\n        await markResetRequired(this.#db, this.#shard);\n        this.#onFatal(err);\n      }\n      sub.fail(err);\n    }\n  }\n\n  stop() {\n    this.#queue.enqueue('stop');\n    return promiseVoid;\n  }\n}\n\nfunction toDownstream(entry: ChangeEntry): WatermarkedChange {\n  const {watermark, change} = entry;\n  switch (change.tag) {\n    case 'begin':\n      return [watermark, ['begin', change, {commitWatermark: watermark}]];\n    case 'commit':\n      return [watermark, ['commit', change, {watermark}]];\n    case 'rollback':\n      return [watermark, ['rollback', change]];\n    default:\n      return [watermark, ['data', change]];\n  }\n}\n"],"names":["Mode.SERIALIZABLE","tx","Mode.READONLY","start","ErrorType.WatermarkTooOld"],"mappings":";;;;;;;;;;;;;;;;AA2DA,MAAM,qCAAqC;AAgCpC,MAAM,OAA0B;AAAA,EAC5B,KAAK;AAAA,EACL;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA,SAAS,IAAI,MAAA;AAAA,EAEtB,WAAW;AAAA,EAEX,YACE,IACA,OACA,QACA,kBACA,mBACA,IACA,gBACA,YACA,SACA;AACA,SAAK,MAAM;AACX,SAAK,SAAS;AACd,SAAK,UAAU;AACf,SAAK,oBAAoB;AACzB,SAAK,qBAAqB;AAC1B,SAAK,MAAM;AACX,SAAK,kBAAkB;AACvB,SAAK,cAAc;AACnB,SAAK,WAAW;AAAA,EAClB;AAAA;AAAA,EAGA,KAAK,OAAe;AAClB,WAAO,KAAK,IAAI,GAAG,UAAU,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE;AAAA,EACtD;AAAA,EAEA,MAAM,kBAAkB;AACtB,UAAM,KAAK,KAAK;AAChB,UAAM,QAAQ,KAAK;AACnB,UAAM,eAAe,KAAK;AAC1B,UAAM,gBAAgB,KAAK;AAE3B,UAAM,sBACJ,kBAAkB,OACd,eACA,GAAG,aAAa,MAAM,YAAY;AACxC,UAAM,YAAY,KAAK,KAAK,kBAAkB,CAAC,QAAQ,GAAG,EAAC,OAAO,cAAc,oBAAA,CAAoB,CAAC;AACrG,SAAK,IAAI,OAAO,wBAAwB,mBAAmB,EAAE;AAAA,EAC/D;AAAA,EAEA,MAAM,gCAAiD;AAIrD,UAAM,EAAC,SAAS,OAAO,QAAA,IAAW,SAAA;AAClC,SAAK,OAAO,QAAQ,CAAC,SAAS,OAAO,CAAC;AACtC,UAAM;AAEN,UAAM,CAAC,EAAC,cAAA,CAAc,IAAI,MAAM,KAAK;AAAA,oCACL,KAAK,KAAK,kBAAkB,CAAC;AAC7D,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,4BAAoD;AACxD,UAAM,CAAC,EAAC,aAAA,CAAa,IAAI,MAAM,KAAK;AAAA,qDAGa,KAAK,KAAK,WAAW,CAAC;AACvE,WAAO;AAAA,EACT;AAAA,EAEA,mBAAmB,WAAoC;AACrD,WAAO,KAAK,IAAI,MAAMA,cAAmB,OAAM,QAAO;AACpD,8BAAwB,GAAG;AAK3B,YAAM,CAAC,EAAC,OAAM,IAAI,MAAM;AAAA,wBACN,KAAK,KAAK,kBAAkB,CAAC;AAC/C,UAAI,UAAU,KAAK,SAAS;AAC1B,aAAK,IAAI;AAAA,UACP,sCAAsC,SAAS;AAAA,QAAA;AAEjD,eAAO;AAAA,MACT;AAEA,YAAM,CAAC,EAAC,SAAQ,IAAI,MAAM;AAAA;AAAA,wBAER,KAAK,KAAK,WAAW,CAAC,sBAAsB,SAAS;AAAA;AAAA;AAGvE,aAAO,OAAO,OAAO;AAAA,IACvB,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,OAA0B;AAC9B,SAAK,OAAO,QAAQ,CAAC,UAAU,KAAK,CAAC;AAAA,EACvC;AAAA,EAEA,QAAQ;AACN,SAAK,OAAO,QAAQ,CAAC,OAAO,CAAC;AAAA,EAC/B;AAAA,EAEA,OAAO,GAAkB;AACvB,SAAK,OAAO,QAAQ,CAAC;AAAA,EACvB;AAAA,EAEA,QAAQ,YAAwB,MAAsB;AACpD,SAAK,OAAO,QAAQ,CAAC,cAAc,EAAC,YAAY,KAAA,CAAK,CAAC;AAAA,EACxD;AAAA,EAEA,gBAAuC;AAAA,EAEvC,eAA0C;AACxC,QAAI,CAAC,KAAK,UAAU;AAClB,aAAO;AAAA,IACT;AACA,QACE,KAAK,kBAAkB,QACvB,KAAK,OAAO,KAAA,IAAS,oCACrB;AACA,WAAK,IAAI;AAAA,QACP,+BAA+B,KAAK,OAAO,KAAA,CAAM;AAAA,MAAA;AAEnD,WAAK,gBAAgB,SAAA;AAAA,IACvB;AACA,WAAO,KAAK,eAAe;AAAA,EAC7B;AAAA,EAEA,4BAA4B;AAC1B,QACE,KAAK,kBAAkB;AAAA,IAEvB,KAAK,OAAO,SAAS,qCAAqC,KAC1D;AACA,WAAK,IAAI;AAAA,QACP,gCAAgC,KAAK,OAAO,KAAA,CAAM;AAAA,MAAA;AAEpD,WAAK,cAAc,QAAA;AACnB,WAAK,gBAAgB;AAAA,IACvB;AAAA,EACF;AAAA,EAEA,MAAM,MAAM;AACV,SAAK,WAAW;AAChB,QAAI;AACF,YAAM,KAAK,cAAA;AAAA,IACb,UAAA;AACE,WAAK,WAAW;AAChB,WAAK,IAAI,OAAO,gBAAgB;AAAA,IAClC;AAAA,EACF;AAAA,EAEA,MAAM,gBAAgB;AACpB,QAAI,KAAgC;AACpC,QAAI;AAEJ,UAAM,eAAoC,CAAA;AAC1C,YAAQ,MAAM,MAAM,KAAK,OAAO,QAAA,OAAe,QAAQ;AACrD,WAAK,0BAAA;AAEL,YAAM,CAAC,OAAO,IAAI;AAClB,cAAQ,SAAA;AAAA,QACN,KAAK,SAAS;AACZ,gBAAM,cAAc,IAAI,CAAC;AACzB,sBAAA;AACA;AAAA,QACF;AAAA,QACA,KAAK,cAAc;AACjB,gBAAM,aAAa,IAAI,CAAC;AACxB,cAAI,IAAI;AACN,yBAAa,KAAK,UAAU;AAAA,UAC9B,OAAO;AACL,kBAAM,KAAK,cAAc,CAAC,UAAU,CAAC;AAAA,UACvC;AACA;AAAA,QACF;AAAA,QACA,KAAK;AACH,eAAK,YAAY,GAAG;AACpB;AAAA,QACF,KAAK,SAAS;AACZ,cAAI,IAAI;AACN,eAAG,KAAK,MAAA;AACR,kBAAM,GAAG,KAAK,KAAA;AACd,iBAAK;AAAA,UACP;AACA;AAAA,QACF;AAAA,MAAA;AAGF,YAAM,CAAC,WAAW,UAAU,IAAI,IAAI,CAAC;AACrC,YAAM,CAAC,KAAK,MAAM,IAAI;AACtB,UAAI,QAAQ,SAAS;AACnB,eAAO,CAAC,IAAI,+CAA+C;AAC3D,cAAM,EAAC,SAAS,SAAS,OAAA,IAAU,SAAA;AACnC,aAAK;AAAA,UACH,MAAM,IAAI;AAAA,YACR,KAAK,IAAI,YAAY,aAAa,SAAS;AAAA,YAC3CA;AAAAA,UAAK;AAAA,UAEP,oBAAoB;AAAA,UACpB,KAAK;AAAA,UACL,0BAA0B;AAAA,QAAA;AAE5B,WAAG,KAAK,IAAI,KAAK,GAAG;AAGpB,aAAK,GAAG,KAAK,QAAQ,CAAAC,QAAM;AACzBA;AAAAA,0BACgB,KAAK,KAAK,kBAAkB,CAAC,GAAG;AAAA,YAC9C,CAAC,CAAC,MAAM,MAAM,QAAQ,MAAM;AAAA,YAC5B;AAAA,UAAA;AAEF,iBAAO,CAAA;AAAA,QACT,CAAC;AAAA,MACH,OAAO;AACL,eAAO,IAAI,YAAY,GAAG,yBAAyB;AACnD,WAAG;AAAA,MACL;AAEA,YAAM,QAAQ;AAAA,QACZ,WAAW,QAAQ,WAAW,YAAY,GAAG;AAAA,QAC7C,WAAW,QAAQ,WAAW,GAAG,qBAAqB;AAAA,QACtD,KAAK,GAAG;AAAA,QACR;AAAA,MAAA;AAGF,YAAM,YAAY,GAAG,KAAK,QAAQ,CAAAA,QAAM;AAAA,QACtCA;AAAAA,sBACc,KAAK,KAAK,WAAW,CAAC,IAAIA,IAAG,KAAK,CAAC;AAAA,MAAA,CAClD;AAED,UAAI,QAAQ,UAAU,GAAG,MAAM,QAAW,GAAG;AAI3C,cAAM;AAAA,MACR;AAEA,UAAI,QAAQ,UAAU;AACpB,cAAM,EAAC,MAAA,IAAS,MAAM,GAAG;AACzB,YAAI,UAAU,KAAK,SAAS;AAE1B,aAAG,KAAK;AAAA,YACN,IAAI,WAAW,2CAA2C,KAAK,EAAE;AAAA,UAAA;AAAA,QAErE,OAAO;AAEL,gBAAM,gBAAgB;AACtB,eAAK,GAAG,KAAK,QAAQ,CAAAA,QAAM;AAAA,YACzBA;AAAAA,qBACS,KAAK,KAAK,kBAAkB,CAAC,QAAQA,IAAG,EAAC,cAAA,CAAc,CAAC;AAAA,UAAA,CAClE;AACD,aAAG,KAAK,QAAA;AAAA,QACV;AAEA,YAAI;AACF,gBAAM,GAAG,KAAK,KAAA;AAAA,QAChB,SAAS,GAAG;AACV,cACE,aAAa,SAAS,iBACtB,EAAE,SAAS,0BACX;AAEA,kBAAM,IAAI,WAAW,mCAAmC,EAAC,OAAO,GAAE;AAAA,UACpE;AACA,gBAAM;AAAA,QACR;AAEA,aAAK;AAGL,aAAK,YAAY,UAAU;AAI3B,cAAM,KAAK,cAAc,aAAa,OAAO,CAAC,CAAC;AAAA,MACjD,WAAW,QAAQ,YAAY;AAG7B,WAAG,KAAK,MAAA;AACR,cAAM,GAAG,KAAK,KAAA;AACd,aAAK;AAEL,cAAM,KAAK,cAAc,aAAa,OAAO,CAAC,CAAC;AAAA,MACjD;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAM,cAAc,MAA2B;AAC7C,QAAI,KAAK,WAAW,GAAG;AACrB;AAAA,IACF;AAEA,UAAM,SAAS,IAAI;AAAA,MACjB,KAAK,IAAI,YAAY,QAAQ,SAAS;AAAA,MACtCC;AAAAA,IAAK;AAEP,WAAO,IAAI,KAAK,GAAG;AAMnB,UAAM,OAAO,gBAAgB,MAAM;AAAA,IAAC,CAAC;AAIrC,SAAK,QAAQ,IAAI,KAAK,IAAI,CAAA,QAAO,KAAK,SAAS,KAAK,MAAM,CAAC,CAAC,EAAE;AAAA,MAAQ,MACpE,OAAO,QAAA;AAAA,IAAQ;AAAA,EAEnB;AAAA,EAEA,MAAM,SACJ,EAAC,YAAY,KAAK,KAAA,GAClB,QACA;AACA,QAAI;AACF,YAAM,OAAO,gBAAgB,OAAM,OAAM;AACvC,cAAM,QAAQ,KAAK,IAAA;AAInB,YAAI,iBAAiB,IAAI,cAAc,KAAK;AAC5C,YAAI,QAAQ;AACZ,YAAI;AAEJ,yBAAiB,WAAW;AAAA,0CACM,KAAK,KAAK,WAAW,CAAC;AAAA,gCAChC,IAAI,SAAS;AAAA,oCACT,OAAO,GAAI,GAAG;AAOxC,gBAAMC,SAAQ,YAAY,IAAA;AAC1B,gBAAM;AACN,gBAAM,UAAU,YAAY,IAAA,IAAQA;AACpC,cAAI,mBAAmB;AACrB,aAAC,UAAU,MAAM,KAAK,IAAI,OAAO,KAAK,IAAI;AAAA,cACxC,UAAU,QAAQ,QAAQ,CAAC,CAAC,WAAW,IAAI,EAAE;AAAA,YAAA;AAAA,UAEjD;AAEA,qBAAW,SAAS,SAAS;AAC3B,gBAAI,MAAM,cAAc,IAAI,WAAW;AAGrC,+BAAiB;AAAA,YACnB,WAAW,gBAAgB;AACzB,kCAAoB,IAAI,QAAQ,aAAa,KAAK,CAAC,EAAE;AACrD;AAAA,YACF,WAAW,SAAS,UAAU;AAC5B,oBAAM,IAAI;AAAA,gBACR,+BAA+B,IAAI,SAAS,yBAAyB,MAAM,SAAS;AAAA,cAAA;AAAA,YAExF,OAAO;AACL,mBAAK,IAAI;AAAA,gBACP,qCAAqC,IAAI,SAAS,yBAAyB,MAAM,SAAS;AAAA,cAAA;AAE5F,kBAAI;AAAA,gBACFC;AAAAA,gBACA,mCAAmC,MAAM,SAAS,eAAe,IAAI,SAAS;AAAA,cAAA;AAEhF;AAAA,YACF;AAAA,UACF;AAAA,QACF;AACA,YAAI,gBAAgB;AAClB,gBAAM;AACN,eAAK,IAAI;AAAA,YACP,aAAa,IAAI,EAAE,SAAS,KAAK,aAC/B,KAAK,QAAQ,KACf;AAAA,UAAA;AAAA,QAEJ,OAAO;AACL,eAAK,IAAI;AAAA,YACP,2BAA2B,IAAI,SAAS;AAAA,UAAA;AAAA,QAE5C;AAGA,YAAI,YAAA;AAAA,MACN,CAAC;AAAA,IACH,SAAS,KAAK;AACZ,WAAK,IAAI,QAAQ,sCAAsC,IAAI,EAAE,IAAI,GAAG;AACpE,UAAI,eAAe,iBAAiB;AAClC,cAAM,kBAAkB,KAAK,KAAK,KAAK,MAAM;AAC7C,aAAK,SAAS,GAAG;AAAA,MACnB;AACA,UAAI,KAAK,GAAG;AAAA,IACd;AAAA,EACF;AAAA,EAEA,OAAO;AACL,SAAK,OAAO,QAAQ,MAAM;AAC1B,WAAO;AAAA,EACT;AACF;AAEA,SAAS,aAAa,OAAuC;AAC3D,QAAM,EAAC,WAAW,OAAA,IAAU;AAC5B,UAAQ,OAAO,KAAA;AAAA,IACb,KAAK;AACH,aAAO,CAAC,WAAW,CAAC,SAAS,QAAQ,EAAC,iBAAiB,UAAA,CAAU,CAAC;AAAA,IACpE,KAAK;AACH,aAAO,CAAC,WAAW,CAAC,UAAU,QAAQ,EAAC,UAAA,CAAU,CAAC;AAAA,IACpD,KAAK;AACH,aAAO,CAAC,WAAW,CAAC,YAAY,MAAM,CAAC;AAAA,IACzC;AACE,aAAO,CAAC,WAAW,CAAC,QAAQ,MAAM,CAAC;AAAA,EAAA;AAEzC;"}
+{"version":3,"file":"storer.js","sources":["../../../../../../zero-cache/src/services/change-streamer/storer.ts"],"sourcesContent":["import {PG_SERIALIZATION_FAILURE} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport {resolver, type Resolver} from '@rocicorp/resolver';\nimport postgres from 'postgres';\nimport {AbortError} from '../../../../shared/src/abort-error.ts';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport {type JSONValue} from '../../../../shared/src/bigint-json.ts';\nimport {Queue} from '../../../../shared/src/queue.ts';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport * as Mode from '../../db/mode-enum.ts';\nimport {TransactionPool} from '../../db/transaction-pool.ts';\nimport {disableStatementTimeout, type PostgresDB} from '../../types/pg.ts';\nimport {cdcSchema, type ShardID} from '../../types/shards.ts';\nimport {type Commit} from '../change-source/protocol/current/downstream.ts';\nimport type {StatusMessage} from '../change-source/protocol/current/status.ts';\nimport type {ReplicatorMode} from '../replicator/replicator.ts';\nimport type {Service} from '../service.ts';\nimport type {WatermarkedChange} from './change-streamer-service.ts';\nimport {type ChangeEntry} from './change-streamer.ts';\nimport * as ErrorType from './error-type-enum.ts';\nimport {\n  AutoResetSignal,\n  markResetRequired,\n  type ReplicationState,\n} from './schema/tables.ts';\nimport type {Subscriber} from './subscriber.ts';\n\ntype SubscriberAndMode = {\n  subscriber: Subscriber;\n  mode: ReplicatorMode;\n};\n\ntype QueueEntry =\n  | ['change', WatermarkedChange]\n  | ['ready', callback: () => void]\n  | ['subscriber', SubscriberAndMode]\n  | StatusMessage\n  | ['abort']\n  | 'stop';\n\ntype PendingTransaction = {\n  pool: TransactionPool;\n  preCommitWatermark: string;\n  pos: number;\n  startingReplicationState: Promise<ReplicationState>;\n};\n\n/**\n * Handles the storage of changes and the catchup of subscribers\n * that are behind.\n *\n * In the context of catchup and cleanup, it is the responsibility of the\n * Storer to decide whether a client can be caught up, or whether the\n * changes needed to catch a client up have been purged.\n *\n * **Maintained invariant**: The Change DB is only empty for a\n * completely new replica (i.e. initial-sync with no changes from the\n * replication stream).\n * * In this case, all new subscribers are expected start from the\n *   `replicaVersion`, which is the version at which initial sync\n *   was performed, and any attempts to catchup from a different\n *   point fail.\n *\n * Conversely, if non-initial changes have flowed through the system\n * (i.e. via the replication stream), the ChangeDB must *not* be empty,\n * and the earliest change in the `changeLog` represents the earliest\n * \"commit\" from (after) which a subscriber can be caught up.\n * * Any attempts to catchup from an earlier point must fail with\n *   a `WatermarkTooOld` error.\n * * Failure to do so could result in streaming changes to the\n *   subscriber such that there is a gap in its replication history.\n *\n * Note: Subscribers (i.e. `incremental-syncer`) consider an \"error\" signal\n * an unrecoverable error and shut down in response. This allows the\n * production system to replace it with a new task and fresh copy of the\n * replica backup.\n */\nexport class Storer implements Service {\n  readonly id = 'storer';\n  readonly #lc: LogContext;\n  readonly #shard: ShardID;\n  readonly #taskID: string;\n  readonly #discoveryAddress: string;\n  readonly #discoveryProtocol: string;\n  readonly #db: PostgresDB;\n  readonly #replicaVersion: string;\n  readonly #onConsumed: (c: Commit | StatusMessage) => void;\n  readonly #onFatal: (err: Error) => void;\n  readonly #queue = new Queue<QueueEntry>();\n  readonly #backPressureThreshold: number;\n\n  #running = false;\n\n  constructor(\n    lc: LogContext,\n    shard: ShardID,\n    taskID: string,\n    discoveryAddress: string,\n    discoveryProtocol: string,\n    db: PostgresDB,\n    replicaVersion: string,\n    onConsumed: (c: Commit | StatusMessage) => void,\n    onFatal: (err: Error) => void,\n    backPressureThreshold: number,\n  ) {\n    this.#lc = lc;\n    this.#shard = shard;\n    this.#taskID = taskID;\n    this.#discoveryAddress = discoveryAddress;\n    this.#discoveryProtocol = discoveryProtocol;\n    this.#db = db;\n    this.#replicaVersion = replicaVersion;\n    this.#onConsumed = onConsumed;\n    this.#onFatal = onFatal;\n    this.#backPressureThreshold = backPressureThreshold;\n  }\n\n  // For readability in SQL statements.\n  #cdc(table: string) {\n    return this.#db(`${cdcSchema(this.#shard)}.${table}`);\n  }\n\n  async assumeOwnership() {\n    const db = this.#db;\n    const owner = this.#taskID;\n    const ownerAddress = this.#discoveryAddress;\n    const ownerProtocol = this.#discoveryProtocol;\n    // we omit `ws://` so that old view syncer versions that are not expecting the protocol continue to not get it\n    const addressWithProtocol =\n      ownerProtocol === 'ws'\n        ? ownerAddress\n        : `${ownerProtocol}://${ownerAddress}`;\n    await db`UPDATE ${this.#cdc('replicationState')} SET ${db({owner, ownerAddress: addressWithProtocol})}`;\n    this.#lc.info?.(`assumed ownership at ${addressWithProtocol}`);\n  }\n\n  async getLastWatermarkToStartStream(): Promise<string> {\n    // Before starting or restarting a stream from the change source,\n    // wait for all queued changes to be processed so that we pick up\n    // from the right spot.\n    const {promise: ready, resolve} = resolver();\n    this.#queue.enqueue(['ready', resolve]);\n    await ready;\n\n    const [{lastWatermark}] = await this.#db<{lastWatermark: string}[]>`\n      SELECT \"lastWatermark\" FROM ${this.#cdc('replicationState')}`;\n    return lastWatermark;\n  }\n\n  async getMinWatermarkForCatchup(): Promise<string | null> {\n    const [{minWatermark}] = await this.#db<\n      {minWatermark: string | null}[]\n    > /*sql*/ `\n      SELECT min(watermark) as \"minWatermark\" FROM ${this.#cdc('changeLog')}`;\n    return minWatermark;\n  }\n\n  purgeRecordsBefore(watermark: string): Promise<number> {\n    return this.#db.begin(Mode.SERIALIZABLE, async sql => {\n      disableStatementTimeout(sql);\n\n      // Check ownership before performing the purge. The server is expected to\n      // exit immediately when an ownership change is detected, but checking\n      // explicitly guards against race conditions.\n      const [{owner}] = await sql<ReplicationState[]>`\n        SELECT * FROM ${this.#cdc('replicationState')}`;\n      if (owner !== this.#taskID) {\n        this.#lc.warn?.(\n          `Ignoring change log purge request (${watermark}) while not owner`,\n        );\n        return 0;\n      }\n\n      const [{deleted}] = await sql<{deleted: bigint}[]>`\n        WITH purged AS (\n          DELETE FROM ${this.#cdc('changeLog')} WHERE watermark < ${watermark} \n            RETURNING watermark, pos\n        ) SELECT COUNT(*) as deleted FROM purged;`;\n      return Number(deleted);\n    });\n  }\n\n  store(entry: WatermarkedChange) {\n    this.#queue.enqueue(['change', entry]);\n  }\n\n  abort() {\n    this.#queue.enqueue(['abort']);\n  }\n\n  status(s: StatusMessage) {\n    this.#queue.enqueue(s);\n  }\n\n  catchup(subscriber: Subscriber, mode: ReplicatorMode) {\n    this.#queue.enqueue(['subscriber', {subscriber, mode}]);\n  }\n\n  #readyForMore: Resolver<void> | null = null;\n\n  readyForMore(): Promise<void> | undefined {\n    if (!this.#running) {\n      return undefined;\n    }\n    if (\n      this.#readyForMore === null &&\n      this.#queue.size() > this.#backPressureThreshold\n    ) {\n      this.#lc.warn?.(\n        `applying back pressure with ${this.#queue.size()} queued changes (threshold: ${this.#backPressureThreshold})\\n` +\n          `\\n` +\n          `To inspect changeLog backlog in your CVR database:\\n` +\n          `  SELECT\\n` +\n          `    (change->'relation'->>'schema') || '.' || (change->'relation'->>'name') AS table_name,\\n` +\n          `    change->>'tag' AS operation,\\n` +\n          `    COUNT(*) AS count\\n` +\n          `  FROM \"<app_id>/cdc\".\"changeLog\"\\n` +\n          `  GROUP BY 1, 2\\n` +\n          `  ORDER BY 3 DESC\\n` +\n          `  LIMIT 20;`,\n      );\n      this.#readyForMore = resolver();\n    }\n    return this.#readyForMore?.promise;\n  }\n\n  #maybeReleaseBackPressure() {\n    if (\n      this.#readyForMore !== null &&\n      // Wait for at least 10% of the threshold to free up.\n      this.#queue.size() < this.#backPressureThreshold * 0.9\n    ) {\n      this.#lc.info?.(\n        `releasing back pressure with ${this.#queue.size()} queued changes`,\n      );\n      this.#readyForMore.resolve();\n      this.#readyForMore = null;\n    }\n  }\n\n  async run() {\n    this.#running = true;\n    try {\n      await this.#processQueue();\n    } finally {\n      this.#running = false;\n      this.#lc.info?.('storer stopped');\n    }\n  }\n\n  async #processQueue() {\n    let tx: PendingTransaction | null = null;\n    let msg: QueueEntry | false;\n\n    const catchupQueue: SubscriberAndMode[] = [];\n    while ((msg = await this.#queue.dequeue()) !== 'stop') {\n      this.#maybeReleaseBackPressure();\n\n      const [msgType] = msg;\n      switch (msgType) {\n        case 'ready': {\n          const signalReady = msg[1];\n          signalReady();\n          continue;\n        }\n        case 'subscriber': {\n          const subscriber = msg[1];\n          if (tx) {\n            catchupQueue.push(subscriber); // Wait for the current tx to complete.\n          } else {\n            await this.#startCatchup([subscriber]); // Catch up immediately.\n          }\n          continue;\n        }\n        case 'status':\n          this.#onConsumed(msg);\n          continue;\n        case 'abort': {\n          if (tx) {\n            tx.pool.abort();\n            await tx.pool.done();\n            tx = null;\n          }\n          continue;\n        }\n      }\n      // msgType === 'change'\n      const [watermark, downstream] = msg[1];\n      const [tag, change] = downstream;\n      if (tag === 'begin') {\n        assert(!tx, 'received BEGIN in the middle of a transaction');\n        const {promise, resolve, reject} = resolver<ReplicationState>();\n        tx = {\n          pool: new TransactionPool(\n            this.#lc.withContext('watermark', watermark),\n            Mode.SERIALIZABLE,\n          ),\n          preCommitWatermark: watermark,\n          pos: 0,\n          startingReplicationState: promise,\n        };\n        tx.pool.run(this.#db);\n        // Pipeline a read of the current ReplicationState,\n        // which will be checked before committing.\n        void tx.pool.process(tx => {\n          tx<ReplicationState[]>`\n          SELECT * FROM ${this.#cdc('replicationState')}`.then(\n            ([result]) => resolve(result),\n            reject,\n          );\n          return [];\n        });\n      } else {\n        assert(tx, `received ${tag} outside of transaction`);\n        tx.pos++;\n      }\n\n      const entry = {\n        watermark: tag === 'commit' ? watermark : tx.preCommitWatermark,\n        precommit: tag === 'commit' ? tx.preCommitWatermark : null,\n        pos: tx.pos,\n        change: change as unknown as JSONValue,\n      };\n\n      const processed = tx.pool.process(tx => [\n        tx`\n        INSERT INTO ${this.#cdc('changeLog')} ${tx(entry)}`,\n      ]);\n\n      if (tag === 'data' && tx.pos % 10_000 === 0) {\n        // Backpressure is exerted on commit when awaiting tx.pool.done().\n        // However, backpressure checks need to be regularly done for\n        // very large transactions in order to avoid memory blowup.\n        await processed;\n      }\n\n      if (tag === 'commit') {\n        const {owner} = await tx.startingReplicationState;\n        if (owner !== this.#taskID) {\n          // Ownership change reflected in the replicationState read in 'begin'.\n          tx.pool.fail(\n            new AbortError(`changeLog ownership has been assumed by ${owner}`),\n          );\n        } else {\n          // Update the replication state.\n          const lastWatermark = watermark;\n          void tx.pool.process(tx => [\n            tx`\n            UPDATE ${this.#cdc('replicationState')} SET ${tx({lastWatermark})}`,\n          ]);\n          tx.pool.setDone();\n        }\n\n        try {\n          await tx.pool.done();\n        } catch (e) {\n          if (\n            e instanceof postgres.PostgresError &&\n            e.code === PG_SERIALIZATION_FAILURE\n          ) {\n            // Ownership change happened after the replicationState was read in 'begin'.\n            throw new AbortError(`changeLog ownership has changed`, {cause: e});\n          }\n          throw e;\n        }\n\n        tx = null;\n\n        // ACK the LSN to the upstream Postgres.\n        this.#onConsumed(downstream);\n\n        // Before beginning the next transaction, open a READONLY snapshot to\n        // concurrently catchup any queued subscribers.\n        await this.#startCatchup(catchupQueue.splice(0));\n      } else if (tag === 'rollback') {\n        // Aborted transactions are not stored in the changeLog. Abort the current tx\n        // and process catchup of subscribers that were waiting for it to end.\n        tx.pool.abort();\n        await tx.pool.done();\n        tx = null;\n\n        await this.#startCatchup(catchupQueue.splice(0));\n      }\n    }\n  }\n\n  async #startCatchup(subs: SubscriberAndMode[]) {\n    if (subs.length === 0) {\n      return;\n    }\n\n    const reader = new TransactionPool(\n      this.#lc.withContext('pool', 'catchup'),\n      Mode.READONLY,\n    );\n    reader.run(this.#db);\n\n    // Ensure that the transaction has started (and is thus holding a snapshot\n    // of the database) before continuing on to commit more changes. This is\n    // done by waiting for a no-op task to be processed by the pool, which\n    // indicates that the BEGIN statement has been sent to the database.\n    await reader.processReadTask(() => {});\n\n    // Run the actual catchup queries in the background. Errors are handled in\n    // #catchup() by disconnecting the associated subscriber.\n    void Promise.all(subs.map(sub => this.#catchup(sub, reader))).finally(() =>\n      reader.setDone(),\n    );\n  }\n\n  async #catchup(\n    {subscriber: sub, mode}: SubscriberAndMode,\n    reader: TransactionPool,\n  ) {\n    try {\n      await reader.processReadTask(async tx => {\n        const start = Date.now();\n\n        // When starting from initial-sync, there won't be a change with a watermark\n        // equal to the replica version. This is the empty changeLog scenario.\n        let watermarkFound = sub.watermark === this.#replicaVersion;\n        let count = 0;\n        let lastBatchConsumed: Promise<unknown> | undefined;\n\n        for await (const entries of tx<ChangeEntry[]>`\n          SELECT watermark, change FROM ${this.#cdc('changeLog')}\n           WHERE watermark >= ${sub.watermark}\n           ORDER BY watermark, pos`.cursor(2000)) {\n          // Wait for the last batch of entries to be consumed by the\n          // subscriber before sending down the current batch. This pipelining\n          // allows one batch of changes to be received from the change-db\n          // while the previous batch of changes are sent to the subscriber,\n          // resulting in flow control that caps the number of changes\n          // referenced in memory to 2 * batch-size.\n          const start = performance.now();\n          await lastBatchConsumed;\n          const elapsed = performance.now() - start;\n          if (lastBatchConsumed) {\n            (elapsed > 100 ? this.#lc.info : this.#lc.debug)?.(\n              `waited ${elapsed.toFixed(3)} ms for ${sub.id} to consume last batch of catchup entries`,\n            );\n          }\n\n          for (const entry of entries) {\n            if (entry.watermark === sub.watermark) {\n              // This should be the first entry.\n              // Catchup starts from *after* the watermark.\n              watermarkFound = true;\n            } else if (watermarkFound) {\n              lastBatchConsumed = sub.catchup(toDownstream(entry)).result;\n              count++;\n            } else if (mode === 'backup') {\n              throw new AutoResetSignal(\n                `backup replica at watermark ${sub.watermark} is behind change db: ${entry.watermark})`,\n              );\n            } else {\n              this.#lc.warn?.(\n                `rejecting subscriber at watermark ${sub.watermark} (earliest watermark: ${entry.watermark})`,\n              );\n              sub.close(\n                ErrorType.WatermarkTooOld,\n                `earliest supported watermark is ${entry.watermark} (requested ${sub.watermark})`,\n              );\n              return;\n            }\n          }\n        }\n        if (watermarkFound) {\n          await lastBatchConsumed;\n          this.#lc.info?.(\n            `caught up ${sub.id} with ${count} changes (${\n              Date.now() - start\n            } ms)`,\n          );\n        } else {\n          this.#lc.warn?.(\n            `subscriber at watermark ${sub.watermark} is ahead of latest watermark`,\n          );\n        }\n        // Flushes the backlog of messages buffered during catchup and\n        // allows the subscription to forward subsequent messages immediately.\n        sub.setCaughtUp();\n      });\n    } catch (err) {\n      this.#lc.error?.(`error while catching up subscriber ${sub.id}`, err);\n      if (err instanceof AutoResetSignal) {\n        await markResetRequired(this.#db, this.#shard);\n        this.#onFatal(err);\n      }\n      sub.fail(err);\n    }\n  }\n\n  stop() {\n    this.#queue.enqueue('stop');\n    return promiseVoid;\n  }\n}\n\nfunction toDownstream(entry: ChangeEntry): WatermarkedChange {\n  const {watermark, change} = entry;\n  switch (change.tag) {\n    case 'begin':\n      return [watermark, ['begin', change, {commitWatermark: watermark}]];\n    case 'commit':\n      return [watermark, ['commit', change, {watermark}]];\n    case 'rollback':\n      return [watermark, ['rollback', change]];\n    default:\n      return [watermark, ['data', change]];\n  }\n}\n"],"names":["Mode.SERIALIZABLE","tx","Mode.READONLY","start","ErrorType.WatermarkTooOld"],"mappings":";;;;;;;;;;;;;;;;AA6EO,MAAM,OAA0B;AAAA,EAC5B,KAAK;AAAA,EACL;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA,SAAS,IAAI,MAAA;AAAA,EACb;AAAA,EAET,WAAW;AAAA,EAEX,YACE,IACA,OACA,QACA,kBACA,mBACA,IACA,gBACA,YACA,SACA,uBACA;AACA,SAAK,MAAM;AACX,SAAK,SAAS;AACd,SAAK,UAAU;AACf,SAAK,oBAAoB;AACzB,SAAK,qBAAqB;AAC1B,SAAK,MAAM;AACX,SAAK,kBAAkB;AACvB,SAAK,cAAc;AACnB,SAAK,WAAW;AAChB,SAAK,yBAAyB;AAAA,EAChC;AAAA;AAAA,EAGA,KAAK,OAAe;AAClB,WAAO,KAAK,IAAI,GAAG,UAAU,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE;AAAA,EACtD;AAAA,EAEA,MAAM,kBAAkB;AACtB,UAAM,KAAK,KAAK;AAChB,UAAM,QAAQ,KAAK;AACnB,UAAM,eAAe,KAAK;AAC1B,UAAM,gBAAgB,KAAK;AAE3B,UAAM,sBACJ,kBAAkB,OACd,eACA,GAAG,aAAa,MAAM,YAAY;AACxC,UAAM,YAAY,KAAK,KAAK,kBAAkB,CAAC,QAAQ,GAAG,EAAC,OAAO,cAAc,oBAAA,CAAoB,CAAC;AACrG,SAAK,IAAI,OAAO,wBAAwB,mBAAmB,EAAE;AAAA,EAC/D;AAAA,EAEA,MAAM,gCAAiD;AAIrD,UAAM,EAAC,SAAS,OAAO,QAAA,IAAW,SAAA;AAClC,SAAK,OAAO,QAAQ,CAAC,SAAS,OAAO,CAAC;AACtC,UAAM;AAEN,UAAM,CAAC,EAAC,cAAA,CAAc,IAAI,MAAM,KAAK;AAAA,oCACL,KAAK,KAAK,kBAAkB,CAAC;AAC7D,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,4BAAoD;AACxD,UAAM,CAAC,EAAC,aAAA,CAAa,IAAI,MAAM,KAAK;AAAA,qDAGa,KAAK,KAAK,WAAW,CAAC;AACvE,WAAO;AAAA,EACT;AAAA,EAEA,mBAAmB,WAAoC;AACrD,WAAO,KAAK,IAAI,MAAMA,cAAmB,OAAM,QAAO;AACpD,8BAAwB,GAAG;AAK3B,YAAM,CAAC,EAAC,OAAM,IAAI,MAAM;AAAA,wBACN,KAAK,KAAK,kBAAkB,CAAC;AAC/C,UAAI,UAAU,KAAK,SAAS;AAC1B,aAAK,IAAI;AAAA,UACP,sCAAsC,SAAS;AAAA,QAAA;AAEjD,eAAO;AAAA,MACT;AAEA,YAAM,CAAC,EAAC,SAAQ,IAAI,MAAM;AAAA;AAAA,wBAER,KAAK,KAAK,WAAW,CAAC,sBAAsB,SAAS;AAAA;AAAA;AAGvE,aAAO,OAAO,OAAO;AAAA,IACvB,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,OAA0B;AAC9B,SAAK,OAAO,QAAQ,CAAC,UAAU,KAAK,CAAC;AAAA,EACvC;AAAA,EAEA,QAAQ;AACN,SAAK,OAAO,QAAQ,CAAC,OAAO,CAAC;AAAA,EAC/B;AAAA,EAEA,OAAO,GAAkB;AACvB,SAAK,OAAO,QAAQ,CAAC;AAAA,EACvB;AAAA,EAEA,QAAQ,YAAwB,MAAsB;AACpD,SAAK,OAAO,QAAQ,CAAC,cAAc,EAAC,YAAY,KAAA,CAAK,CAAC;AAAA,EACxD;AAAA,EAEA,gBAAuC;AAAA,EAEvC,eAA0C;AACxC,QAAI,CAAC,KAAK,UAAU;AAClB,aAAO;AAAA,IACT;AACA,QACE,KAAK,kBAAkB,QACvB,KAAK,OAAO,KAAA,IAAS,KAAK,wBAC1B;AACA,WAAK,IAAI;AAAA,QACP,+BAA+B,KAAK,OAAO,MAAM,+BAA+B,KAAK,sBAAsB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,MAAA;AAY7G,WAAK,gBAAgB,SAAA;AAAA,IACvB;AACA,WAAO,KAAK,eAAe;AAAA,EAC7B;AAAA,EAEA,4BAA4B;AAC1B,QACE,KAAK,kBAAkB;AAAA,IAEvB,KAAK,OAAO,KAAA,IAAS,KAAK,yBAAyB,KACnD;AACA,WAAK,IAAI;AAAA,QACP,gCAAgC,KAAK,OAAO,KAAA,CAAM;AAAA,MAAA;AAEpD,WAAK,cAAc,QAAA;AACnB,WAAK,gBAAgB;AAAA,IACvB;AAAA,EACF;AAAA,EAEA,MAAM,MAAM;AACV,SAAK,WAAW;AAChB,QAAI;AACF,YAAM,KAAK,cAAA;AAAA,IACb,UAAA;AACE,WAAK,WAAW;AAChB,WAAK,IAAI,OAAO,gBAAgB;AAAA,IAClC;AAAA,EACF;AAAA,EAEA,MAAM,gBAAgB;AACpB,QAAI,KAAgC;AACpC,QAAI;AAEJ,UAAM,eAAoC,CAAA;AAC1C,YAAQ,MAAM,MAAM,KAAK,OAAO,QAAA,OAAe,QAAQ;AACrD,WAAK,0BAAA;AAEL,YAAM,CAAC,OAAO,IAAI;AAClB,cAAQ,SAAA;AAAA,QACN,KAAK,SAAS;AACZ,gBAAM,cAAc,IAAI,CAAC;AACzB,sBAAA;AACA;AAAA,QACF;AAAA,QACA,KAAK,cAAc;AACjB,gBAAM,aAAa,IAAI,CAAC;AACxB,cAAI,IAAI;AACN,yBAAa,KAAK,UAAU;AAAA,UAC9B,OAAO;AACL,kBAAM,KAAK,cAAc,CAAC,UAAU,CAAC;AAAA,UACvC;AACA;AAAA,QACF;AAAA,QACA,KAAK;AACH,eAAK,YAAY,GAAG;AACpB;AAAA,QACF,KAAK,SAAS;AACZ,cAAI,IAAI;AACN,eAAG,KAAK,MAAA;AACR,kBAAM,GAAG,KAAK,KAAA;AACd,iBAAK;AAAA,UACP;AACA;AAAA,QACF;AAAA,MAAA;AAGF,YAAM,CAAC,WAAW,UAAU,IAAI,IAAI,CAAC;AACrC,YAAM,CAAC,KAAK,MAAM,IAAI;AACtB,UAAI,QAAQ,SAAS;AACnB,eAAO,CAAC,IAAI,+CAA+C;AAC3D,cAAM,EAAC,SAAS,SAAS,OAAA,IAAU,SAAA;AACnC,aAAK;AAAA,UACH,MAAM,IAAI;AAAA,YACR,KAAK,IAAI,YAAY,aAAa,SAAS;AAAA,YAC3CA;AAAAA,UAAK;AAAA,UAEP,oBAAoB;AAAA,UACpB,KAAK;AAAA,UACL,0BAA0B;AAAA,QAAA;AAE5B,WAAG,KAAK,IAAI,KAAK,GAAG;AAGpB,aAAK,GAAG,KAAK,QAAQ,CAAAC,QAAM;AACzBA;AAAAA,0BACgB,KAAK,KAAK,kBAAkB,CAAC,GAAG;AAAA,YAC9C,CAAC,CAAC,MAAM,MAAM,QAAQ,MAAM;AAAA,YAC5B;AAAA,UAAA;AAEF,iBAAO,CAAA;AAAA,QACT,CAAC;AAAA,MACH,OAAO;AACL,eAAO,IAAI,YAAY,GAAG,yBAAyB;AACnD,WAAG;AAAA,MACL;AAEA,YAAM,QAAQ;AAAA,QACZ,WAAW,QAAQ,WAAW,YAAY,GAAG;AAAA,QAC7C,WAAW,QAAQ,WAAW,GAAG,qBAAqB;AAAA,QACtD,KAAK,GAAG;AAAA,QACR;AAAA,MAAA;AAGF,YAAM,YAAY,GAAG,KAAK,QAAQ,CAAAA,QAAM;AAAA,QACtCA;AAAAA,sBACc,KAAK,KAAK,WAAW,CAAC,IAAIA,IAAG,KAAK,CAAC;AAAA,MAAA,CAClD;AAED,UAAI,QAAQ,UAAU,GAAG,MAAM,QAAW,GAAG;AAI3C,cAAM;AAAA,MACR;AAEA,UAAI,QAAQ,UAAU;AACpB,cAAM,EAAC,MAAA,IAAS,MAAM,GAAG;AACzB,YAAI,UAAU,KAAK,SAAS;AAE1B,aAAG,KAAK;AAAA,YACN,IAAI,WAAW,2CAA2C,KAAK,EAAE;AAAA,UAAA;AAAA,QAErE,OAAO;AAEL,gBAAM,gBAAgB;AACtB,eAAK,GAAG,KAAK,QAAQ,CAAAA,QAAM;AAAA,YACzBA;AAAAA,qBACS,KAAK,KAAK,kBAAkB,CAAC,QAAQA,IAAG,EAAC,cAAA,CAAc,CAAC;AAAA,UAAA,CAClE;AACD,aAAG,KAAK,QAAA;AAAA,QACV;AAEA,YAAI;AACF,gBAAM,GAAG,KAAK,KAAA;AAAA,QAChB,SAAS,GAAG;AACV,cACE,aAAa,SAAS,iBACtB,EAAE,SAAS,0BACX;AAEA,kBAAM,IAAI,WAAW,mCAAmC,EAAC,OAAO,GAAE;AAAA,UACpE;AACA,gBAAM;AAAA,QACR;AAEA,aAAK;AAGL,aAAK,YAAY,UAAU;AAI3B,cAAM,KAAK,cAAc,aAAa,OAAO,CAAC,CAAC;AAAA,MACjD,WAAW,QAAQ,YAAY;AAG7B,WAAG,KAAK,MAAA;AACR,cAAM,GAAG,KAAK,KAAA;AACd,aAAK;AAEL,cAAM,KAAK,cAAc,aAAa,OAAO,CAAC,CAAC;AAAA,MACjD;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAM,cAAc,MAA2B;AAC7C,QAAI,KAAK,WAAW,GAAG;AACrB;AAAA,IACF;AAEA,UAAM,SAAS,IAAI;AAAA,MACjB,KAAK,IAAI,YAAY,QAAQ,SAAS;AAAA,MACtCC;AAAAA,IAAK;AAEP,WAAO,IAAI,KAAK,GAAG;AAMnB,UAAM,OAAO,gBAAgB,MAAM;AAAA,IAAC,CAAC;AAIrC,SAAK,QAAQ,IAAI,KAAK,IAAI,CAAA,QAAO,KAAK,SAAS,KAAK,MAAM,CAAC,CAAC,EAAE;AAAA,MAAQ,MACpE,OAAO,QAAA;AAAA,IAAQ;AAAA,EAEnB;AAAA,EAEA,MAAM,SACJ,EAAC,YAAY,KAAK,KAAA,GAClB,QACA;AACA,QAAI;AACF,YAAM,OAAO,gBAAgB,OAAM,OAAM;AACvC,cAAM,QAAQ,KAAK,IAAA;AAInB,YAAI,iBAAiB,IAAI,cAAc,KAAK;AAC5C,YAAI,QAAQ;AACZ,YAAI;AAEJ,yBAAiB,WAAW;AAAA,0CACM,KAAK,KAAK,WAAW,CAAC;AAAA,gCAChC,IAAI,SAAS;AAAA,oCACT,OAAO,GAAI,GAAG;AAOxC,gBAAMC,SAAQ,YAAY,IAAA;AAC1B,gBAAM;AACN,gBAAM,UAAU,YAAY,IAAA,IAAQA;AACpC,cAAI,mBAAmB;AACrB,aAAC,UAAU,MAAM,KAAK,IAAI,OAAO,KAAK,IAAI;AAAA,cACxC,UAAU,QAAQ,QAAQ,CAAC,CAAC,WAAW,IAAI,EAAE;AAAA,YAAA;AAAA,UAEjD;AAEA,qBAAW,SAAS,SAAS;AAC3B,gBAAI,MAAM,cAAc,IAAI,WAAW;AAGrC,+BAAiB;AAAA,YACnB,WAAW,gBAAgB;AACzB,kCAAoB,IAAI,QAAQ,aAAa,KAAK,CAAC,EAAE;AACrD;AAAA,YACF,WAAW,SAAS,UAAU;AAC5B,oBAAM,IAAI;AAAA,gBACR,+BAA+B,IAAI,SAAS,yBAAyB,MAAM,SAAS;AAAA,cAAA;AAAA,YAExF,OAAO;AACL,mBAAK,IAAI;AAAA,gBACP,qCAAqC,IAAI,SAAS,yBAAyB,MAAM,SAAS;AAAA,cAAA;AAE5F,kBAAI;AAAA,gBACFC;AAAAA,gBACA,mCAAmC,MAAM,SAAS,eAAe,IAAI,SAAS;AAAA,cAAA;AAEhF;AAAA,YACF;AAAA,UACF;AAAA,QACF;AACA,YAAI,gBAAgB;AAClB,gBAAM;AACN,eAAK,IAAI;AAAA,YACP,aAAa,IAAI,EAAE,SAAS,KAAK,aAC/B,KAAK,QAAQ,KACf;AAAA,UAAA;AAAA,QAEJ,OAAO;AACL,eAAK,IAAI;AAAA,YACP,2BAA2B,IAAI,SAAS;AAAA,UAAA;AAAA,QAE5C;AAGA,YAAI,YAAA;AAAA,MACN,CAAC;AAAA,IACH,SAAS,KAAK;AACZ,WAAK,IAAI,QAAQ,sCAAsC,IAAI,EAAE,IAAI,GAAG;AACpE,UAAI,eAAe,iBAAiB;AAClC,cAAM,kBAAkB,KAAK,KAAK,KAAK,MAAM;AAC7C,aAAK,SAAS,GAAG;AAAA,MACnB;AACA,UAAI,KAAK,GAAG;AAAA,IACd;AAAA,EACF;AAAA,EAEA,OAAO;AACL,SAAK,OAAO,QAAQ,MAAM;AAC1B,WAAO;AAAA,EACT;AACF;AAEA,SAAS,aAAa,OAAuC;AAC3D,QAAM,EAAC,WAAW,OAAA,IAAU;AAC5B,UAAQ,OAAO,KAAA;AAAA,IACb,KAAK;AACH,aAAO,CAAC,WAAW,CAAC,SAAS,QAAQ,EAAC,iBAAiB,UAAA,CAAU,CAAC;AAAA,IACpE,KAAK;AACH,aAAO,CAAC,WAAW,CAAC,UAAU,QAAQ,EAAC,UAAA,CAAU,CAAC;AAAA,IACpD,KAAK;AACH,aAAO,CAAC,WAAW,CAAC,YAAY,MAAM,CAAC;AAAA,IACzC;AACE,aAAO,CAAC,WAAW,CAAC,QAAQ,MAAM,CAAC;AAAA,EAAA;AAEzC;"}

package/out/zero-cache/src/services/life-cycle.d.ts

@@ -39,7 +39,7 @@ export declare class ProcessManager {
  *
  * @returns a Promise that resolves/rejects when any of the services stops/throws.
  */
-export declare function runUntilKilled(lc: LogContext, parent: Worker | NodeJS.Process, ...services: SingletonService[]): Promise<void>;
+export declare function runUntilKilled(lc: LogContext, parent: EventEmitter, ...services: SingletonService[]): Promise<void>;
 export declare function exitAfter(run: () => Promise<void>): Promise<void>;
 /**
  * The HeartbeatMonitor monitors the cadence heartbeats (e.g. "/keepalive"

package/out/zero-cache/src/services/life-cycle.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"life-cycle.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/services/life-cycle.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,mBAAmB,EAAC,MAAM,WAAW,CAAC;AAEnD,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,QAAQ,CAAC;AACzC,OAAO,EAEL,KAAK,UAAU,EACf,KAAK,MAAM,EACZ,MAAM,uBAAuB,CAAC;AAE/B,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,cAAc,CAAC;AAEnD;;;;;;;;;;;GAWG;AACH,MAAM,MAAM,UAAU,GAAG,aAAa,GAAG,YAAY,CAAC;AAEtD,eAAO,MAAM,iBAAiB,gCAAiC,CAAC;AAChE,eAAO,MAAM,iBAAiB,sBAAuB,CAAC;AAEtD;;;GAGG;AACH,qBAAa,cAAc;;gBAWb,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,YAAY;IAiC9C,IAAI;IAoBJ,aAAa,CAAC,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM;IAiB9D,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM;IAiBjE,YAAY,IAAI,MAAM,EAAE;IAIlB,eAAe;IAIrB,eAAe,CAAC,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM;CAqE3C;AAED;;;;;;GAMG;AAEH,wBAAsB,cAAc,CAClC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,OAAO,EAC/B,GAAG,QAAQ,EAAE,gBAAgB,EAAE,GAC9B,OAAO,CAAC,IAAI,CAAC,CA6Bf;AAED,wBAAsB,SAAS,CAAC,GAAG,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,iBAWvD;AAID;;;;;;;;;;GAUG;AACH,qBAAa,gBAAgB;;gBAQf,EAAE,EAAE,UAAU,EAAE,YAAY,SAA2B;IAKnE,WAAW,CAAC,UAAU,EAAE,mBAAmB;IAsD3C,IAAI;CAML"}
+{"version":3,"file":"life-cycle.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/services/life-cycle.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,mBAAmB,EAAC,MAAM,WAAW,CAAC;AAEnD,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,QAAQ,CAAC;AACzC,OAAO,EAEL,KAAK,UAAU,EACf,KAAK,MAAM,EACZ,MAAM,uBAAuB,CAAC;AAE/B,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,cAAc,CAAC;AAEnD;;;;;;;;;;;GAWG;AACH,MAAM,MAAM,UAAU,GAAG,aAAa,GAAG,YAAY,CAAC;AAEtD,eAAO,MAAM,iBAAiB,gCAAiC,CAAC;AAChE,eAAO,MAAM,iBAAiB,sBAAuB,CAAC;AAEtD;;;GAGG;AACH,qBAAa,cAAc;;gBAWb,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,YAAY;IAiC9C,IAAI;IAoBJ,aAAa,CAAC,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM;IAiB9D,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM;IAiBjE,YAAY,IAAI,MAAM,EAAE;IAIlB,eAAe;IAIrB,eAAe,CAAC,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM;CAqE3C;AAED;;;;;;GAMG;AAEH,wBAAsB,cAAc,CAClC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,YAAY,EACpB,GAAG,QAAQ,EAAE,gBAAgB,EAAE,GAC9B,OAAO,CAAC,IAAI,CAAC,CA6Bf;AAED,wBAAsB,SAAS,CAAC,GAAG,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,iBAWvD;AAID;;;;;;;;;;GAUG;AACH,qBAAa,gBAAgB;;gBAQf,EAAE,EAAE,UAAU,EAAE,YAAY,SAA2B;IAKnE,WAAW,CAAC,UAAU,EAAE,mBAAmB;IAsD3C,IAAI;CAML"}

package/out/zero-cache/src/services/life-cycle.js.map

@@ -1 +1 @@
-{"version":3,"file":"life-cycle.js","sources":["../../../../../zero-cache/src/services/life-cycle.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport type {IncomingHttpHeaders} from 'node:http';\nimport {pid} from 'node:process';\nimport type {EventEmitter} from 'stream';\nimport {\n  singleProcessMode,\n  type Subprocess,\n  type Worker,\n} from '../types/processes.ts';\nimport {RunningState} from './running-state.ts';\nimport type {SingletonService} from './service.ts';\n\n/**\n * * `user-facing` workers serve external requests and are the first to\n *   receive a `SIGTERM` or `SIGINT` signal for graceful shutdown.\n *\n * * `supporting` workers support `user-facing` workers and are sent\n *   the `SIGTERM` signal only after all `user-facing` workers have\n *   exited.\n *\n * For other kill signals, such as `SIGQUIT`, all workers\n * are stopped without draining. Additionally, if any worker exits\n * unexpectedly, all workers sent an immediate `SIGQUIT` signal.\n */\nexport type WorkerType = 'user-facing' | 'supporting';\n\nexport const GRACEFUL_SHUTDOWN = ['SIGTERM', 'SIGINT'] as const;\nexport const FORCEFUL_SHUTDOWN = ['SIGQUIT'] as const;\n\n/**\n * Handles readiness, termination signals, and coordination of graceful\n * shutdown.\n */\nexport class ProcessManager {\n  readonly #lc: LogContext;\n  readonly #userFacing = new Set<Subprocess>();\n  readonly #all = new Set<Subprocess>();\n  readonly #exitImpl: (code: number) => never;\n  readonly #start = Date.now();\n  readonly #ready: Promise<void>[] = [];\n\n  #runningState = new RunningState('process-manager');\n  #drainStart = 0;\n\n  constructor(lc: LogContext, proc: EventEmitter) {\n    this.#lc = lc.withContext('component', 'process-manager');\n\n    // Propagate `SIGTERM` and `SIGINT` to all user-facing workers,\n    // initiating a graceful shutdown. The parent process will\n    // exit once all user-facing workers have exited ...\n    for (const signal of GRACEFUL_SHUTDOWN) {\n      proc.on(signal, () => this.#startDrain(signal));\n    }\n\n    // ... which will result in sending `SIGTERM` to the remaining workers.\n    proc.on('exit', code =>\n      this.#kill(\n        this.#all,\n        code === 0 ? GRACEFUL_SHUTDOWN[0] : FORCEFUL_SHUTDOWN[0],\n      ),\n    );\n\n    // For other (catchable) kill signals, exit with a non-zero error code\n    // to send a `SIGQUIT` to all workers. For this signal, workers are\n    // stopped immediately without draining. See `runUntilKilled()`.\n    for (const signal of FORCEFUL_SHUTDOWN) {\n      proc.on(signal, () => this.#exit(-1));\n    }\n\n    this.#exitImpl = (code: number) => {\n      if (singleProcessMode()) {\n        return proc.emit('exit', code) as never; // For unit / integration tests.\n      }\n      process.exit(code);\n    };\n  }\n\n  done() {\n    return this.#runningState.stopped();\n  }\n\n  #exit(code: number) {\n    this.#lc.info?.('exiting with code', code);\n    this.#runningState.stop(this.#lc);\n    void this.#lc.flush().finally(() => this.#exitImpl(code));\n  }\n\n  #startDrain(signal: 'SIGTERM' | 'SIGINT' = 'SIGTERM') {\n    this.#lc.info?.(`initiating drain (${signal})`);\n    this.#drainStart = Date.now();\n    if (this.#userFacing.size) {\n      this.#kill(this.#userFacing, signal);\n    } else {\n      this.#kill(this.#all, signal);\n    }\n  }\n\n  addSubprocess(proc: Subprocess, type: WorkerType, name: string) {\n    if (type === 'user-facing') {\n      this.#userFacing.add(proc);\n    }\n    this.#all.add(proc);\n\n    proc.on('error', err =>\n      this.#lc.error?.(`error from ${name} ${proc.pid}`, err),\n    );\n    proc.on('close', (code, signal) =>\n      this.#onExit(code, signal, null, type, name, proc),\n    );\n  }\n\n  readonly #initializing = new Map<number, string>();\n  #nextID = 0;\n\n  addWorker(worker: Worker, type: WorkerType, name: string): Worker {\n    this.addSubprocess(worker, type, name);\n\n    const id = ++this.#nextID;\n    this.#initializing.set(id, name);\n    const {promise, resolve} = resolver();\n    this.#ready.push(promise);\n\n    worker.onceMessageType('ready', () => {\n      this.#lc.debug?.(`${name} ready (${Date.now() - this.#start} ms)`);\n      this.#initializing.delete(id);\n      resolve();\n    });\n\n    return worker;\n  }\n\n  initializing(): string[] {\n    return [...this.#initializing.values()];\n  }\n\n  async allWorkersReady() {\n    await Promise.all(this.#ready);\n  }\n\n  logErrorAndExit(err: unknown, name: string) {\n    // only accessible by the main (i.e. user-facing) process.\n    this.#onExit(-1, null, err, 'user-facing', name, undefined);\n  }\n\n  #onExit(\n    code: number,\n    sig: NodeJS.Signals | null,\n    err: unknown | null,\n    type: WorkerType,\n    name: string,\n    worker: Subprocess | undefined,\n  ) {\n    // Remove the worker from maps to avoid attempting to send more signals to it.\n    if (worker) {\n      this.#userFacing.delete(worker);\n      this.#all.delete(worker);\n    }\n\n    const pid = worker?.pid ?? process.pid;\n\n    if (type === 'supporting') {\n      // The replication-manager has no user-facing workers.\n      // In this case, code === 0 shutdowns are not errors.\n      // Non-zero exits are warnings (not errors) since they're often transient issues.\n      const log = code === 0 && this.#userFacing.size === 0 ? 'info' : 'warn';\n      this.#lc[log]?.(`${name} (${pid}) exited with code (${code})`, err ?? '');\n      return this.#exit(log === 'info' ? code : -1);\n    }\n\n    const log = this.#drainStart === 0 ? 'error' : 'warn';\n    if (sig) {\n      this.#lc[log]?.(`${name} (${pid}) killed with (${sig})`, err ?? '');\n    } else if (code !== 0) {\n      this.#lc[log]?.(`${name} (${pid}) exited with code (${code})`, err ?? '');\n    } else {\n      this.#lc.info?.(`${name} (${pid}) exited with code (${code})`);\n    }\n\n    // user-facing workers exited or finished draining.\n    if (this.#userFacing.size === 0) {\n      this.#lc.info?.(\n        this.#drainStart\n          ? `all user-facing workers drained (${\n              Date.now() - this.#drainStart\n            } ms)`\n          : `all user-facing workers exited`,\n      );\n      return this.#exit(0);\n    }\n\n    // Exit only if not draining. If a user-facing worker exits unexpectedly\n    // during a drain, log a warning but let other user-facing workers drain.\n    if (log === 'error') {\n      return this.#exit(code || -1);\n    }\n\n    return undefined;\n  }\n\n  #kill(workers: Iterable<Subprocess>, signal: NodeJS.Signals) {\n    for (const worker of workers) {\n      try {\n        worker.kill(signal);\n      } catch (e) {\n        this.#lc.error?.(e);\n      }\n    }\n  }\n}\n\n/**\n * Runs the specified services, stopping them on `SIGTERM` or `SIGINT` with\n * an optional {@link SingletonService.drain drain()}, or stopping them\n * without draining for `SIGQUIT`.\n *\n * @returns a Promise that resolves/rejects when any of the services stops/throws.\n */\n\nexport async function runUntilKilled(\n  lc: LogContext,\n  parent: Worker | NodeJS.Process,\n  ...services: SingletonService[]\n): Promise<void> {\n  if (services.length === 0) {\n    return;\n  }\n  for (const signal of [...GRACEFUL_SHUTDOWN, ...FORCEFUL_SHUTDOWN]) {\n    parent.once(signal, () => {\n      const GRACEFUL_SIGNALS = GRACEFUL_SHUTDOWN as readonly NodeJS.Signals[];\n\n      services.forEach(async svc => {\n        if (GRACEFUL_SIGNALS.includes(signal) && svc.drain) {\n          lc.info?.(`draining ${svc.constructor.name} ${svc.id} (${signal})`);\n          await svc.drain();\n        }\n        lc.info?.(`stopping ${svc.constructor.name} ${svc.id} (${signal})`);\n        await svc.stop();\n      });\n    });\n  }\n\n  try {\n    // Run all services and resolve when any of them stops.\n    const svc = await Promise.race(\n      services.map(svc => svc.run().then(() => svc)),\n    );\n    lc.info?.(`${svc.constructor.name} (${svc.id}) stopped`);\n  } catch (e) {\n    lc.error?.(`exiting on error`, e);\n    throw e;\n  }\n}\n\nexport async function exitAfter(run: () => Promise<void>) {\n  try {\n    await run();\n    // oxlint-disable-next-line no-console\n    console.info(`pid ${pid} exiting normally`);\n    process.exit(0);\n  } catch (e) {\n    // oxlint-disable-next-line no-console\n    console.error(`pid ${pid} exiting with error`, e);\n    process.exit(-1);\n  }\n}\n\nconst DEFAULT_STOP_INTERVAL_MS = 20_000;\n\n/**\n * The HeartbeatMonitor monitors the cadence heartbeats (e.g. \"/keepalive\"\n * health checks made to HttpServices) that signal that the server\n * should continue processing requests. When a configurable `stopInterval`\n * elapses without receiving these heartbeats, the monitor initiates a\n * graceful shutdown of the server. This works with common load balancing\n * frameworks such as AWS Elastic Load Balancing.\n *\n * The HeartbeatMonitor is **opt-in** in that it only kicks in after it\n * starts receiving keepalives.\n */\nexport class HeartbeatMonitor {\n  readonly #stopInterval: number;\n\n  #lc: LogContext;\n  #checkIntervalTimer: NodeJS.Timeout | undefined;\n  #checkImmediateTimer: NodeJS.Immediate | undefined;\n  #lastHeartbeat = 0;\n\n  constructor(lc: LogContext, stopInterval = DEFAULT_STOP_INTERVAL_MS) {\n    this.#lc = lc;\n    this.#stopInterval = stopInterval;\n  }\n\n  onHeartbeat(reqHeaders: IncomingHttpHeaders) {\n    this.#lastHeartbeat = Date.now();\n    if (this.#checkIntervalTimer === undefined) {\n      this.#lc.info?.(\n        `starting heartbeat monitor at ${\n          this.#stopInterval / 1000\n        } second interval`,\n        reqHeaders,\n      );\n      // e.g. check every 5 seconds to see if it's been over 20 seconds\n      //      since the last heartbeat.\n      this.#checkIntervalTimer = setInterval(\n        this.#checkStopInterval,\n        this.#stopInterval / 4,\n      );\n    }\n  }\n\n  #checkStopInterval = () => {\n    // In the Node.js event loop, timers like setInterval and setTimeout\n    // run *before* I/O events coming from network sockets or file reads/writes.\n    // When this process gets starved of CPU resources for long periods of time,\n    // for example when other processes are monopolizing all available cores,\n    // pathological behavior can emerge:\n    // - keepalive network request comes in, but is queued in Node internals waiting\n    //   for time on the event loop\n    // - CPU is starved/monopolized by other processes for longer than the time\n    //   configured via this.#stopInterval\n    // - When CPU becomes available and the event loop wakes up, this stop interval\n    //   check is run *before* the keepalive request is processed. The value of\n    //   this.#lastHeartbeat is now very stale, and erroneously triggers a shutdown\n    //   even though keepalive requests were about to be processed and update\n    //   this.#lastHeartbeat. Downtime ensues.\n    //\n    // To avoid this, we push the check out to a phase of the event loop *after*\n    // I/O events are processed, using setImmediate():\n    // https://nodejs.org/en/learn/asynchronous-work/event-loop-timers-and-nexttick#setimmediate-vs-settimeout\n    //\n    // This ensures we see a value for this.#lastHeartbeat that reflects\n    // any keepalive requests that came in during the current event loop turn.\n    this.#checkImmediateTimer = setImmediate(() => {\n      this.#checkImmediateTimer = undefined;\n      const timeSinceLastHeartbeat = Date.now() - this.#lastHeartbeat;\n      if (timeSinceLastHeartbeat >= this.#stopInterval) {\n        this.#lc.info?.(\n          `last heartbeat received ${\n            timeSinceLastHeartbeat / 1000\n          } seconds ago. draining.`,\n        );\n        process.kill(process.pid, GRACEFUL_SHUTDOWN[0]);\n      }\n    });\n  };\n\n  stop() {\n    clearTimeout(this.#checkIntervalTimer);\n    if (this.#checkImmediateTimer) {\n      clearImmediate(this.#checkImmediateTimer);\n    }\n  }\n}\n"],"names":["pid","log","svc"],"mappings":";;;;AA2BO,MAAM,oBAAoB,CAAC,WAAW,QAAQ;AAC9C,MAAM,oBAAoB,CAAC,SAAS;AAMpC,MAAM,eAAe;AAAA,EACjB;AAAA,EACA,kCAAkB,IAAA;AAAA,EAClB,2BAAW,IAAA;AAAA,EACX;AAAA,EACA,SAAS,KAAK,IAAA;AAAA,EACd,SAA0B,CAAA;AAAA,EAEnC,gBAAgB,IAAI,aAAa,iBAAiB;AAAA,EAClD,cAAc;AAAA,EAEd,YAAY,IAAgB,MAAoB;AAC9C,SAAK,MAAM,GAAG,YAAY,aAAa,iBAAiB;AAKxD,eAAW,UAAU,mBAAmB;AACtC,WAAK,GAAG,QAAQ,MAAM,KAAK,YAAY,MAAM,CAAC;AAAA,IAChD;AAGA,SAAK;AAAA,MAAG;AAAA,MAAQ,UACd,KAAK;AAAA,QACH,KAAK;AAAA,QACL,SAAS,IAAI,kBAAkB,CAAC,IAAI,kBAAkB,CAAC;AAAA,MAAA;AAAA,IACzD;AAMF,eAAW,UAAU,mBAAmB;AACtC,WAAK,GAAG,QAAQ,MAAM,KAAK,MAAM,EAAE,CAAC;AAAA,IACtC;AAEA,SAAK,YAAY,CAAC,SAAiB;AACjC,UAAI,qBAAqB;AACvB,eAAO,KAAK,KAAK,QAAQ,IAAI;AAAA,MAC/B;AACA,cAAQ,KAAK,IAAI;AAAA,IACnB;AAAA,EACF;AAAA,EAEA,OAAO;AACL,WAAO,KAAK,cAAc,QAAA;AAAA,EAC5B;AAAA,EAEA,MAAM,MAAc;AAClB,SAAK,IAAI,OAAO,qBAAqB,IAAI;AACzC,SAAK,cAAc,KAAK,KAAK,GAAG;AAChC,SAAK,KAAK,IAAI,MAAA,EAAQ,QAAQ,MAAM,KAAK,UAAU,IAAI,CAAC;AAAA,EAC1D;AAAA,EAEA,YAAY,SAA+B,WAAW;AACpD,SAAK,IAAI,OAAO,qBAAqB,MAAM,GAAG;AAC9C,SAAK,cAAc,KAAK,IAAA;AACxB,QAAI,KAAK,YAAY,MAAM;AACzB,WAAK,MAAM,KAAK,aAAa,MAAM;AAAA,IACrC,OAAO;AACL,WAAK,MAAM,KAAK,MAAM,MAAM;AAAA,IAC9B;AAAA,EACF;AAAA,EAEA,cAAc,MAAkB,MAAkB,MAAc;AAC9D,QAAI,SAAS,eAAe;AAC1B,WAAK,YAAY,IAAI,IAAI;AAAA,IAC3B;AACA,SAAK,KAAK,IAAI,IAAI;AAElB,SAAK;AAAA,MAAG;AAAA,MAAS,CAAA,QACf,KAAK,IAAI,QAAQ,cAAc,IAAI,IAAI,KAAK,GAAG,IAAI,GAAG;AAAA,IAAA;AAExD,SAAK;AAAA,MAAG;AAAA,MAAS,CAAC,MAAM,WACtB,KAAK,QAAQ,MAAM,QAAQ,MAAM,MAAM,MAAM,IAAI;AAAA,IAAA;AAAA,EAErD;AAAA,EAES,oCAAoB,IAAA;AAAA,EAC7B,UAAU;AAAA,EAEV,UAAU,QAAgB,MAAkB,MAAsB;AAChE,SAAK,cAAc,QAAQ,MAAM,IAAI;AAErC,UAAM,KAAK,EAAE,KAAK;AAClB,SAAK,cAAc,IAAI,IAAI,IAAI;AAC/B,UAAM,EAAC,SAAS,QAAA,IAAW,SAAA;AAC3B,SAAK,OAAO,KAAK,OAAO;AAExB,WAAO,gBAAgB,SAAS,MAAM;AACpC,WAAK,IAAI,QAAQ,GAAG,IAAI,WAAW,KAAK,IAAA,IAAQ,KAAK,MAAM,MAAM;AACjE,WAAK,cAAc,OAAO,EAAE;AAC5B,cAAA;AAAA,IACF,CAAC;AAED,WAAO;AAAA,EACT;AAAA,EAEA,eAAyB;AACvB,WAAO,CAAC,GAAG,KAAK,cAAc,QAAQ;AAAA,EACxC;AAAA,EAEA,MAAM,kBAAkB;AACtB,UAAM,QAAQ,IAAI,KAAK,MAAM;AAAA,EAC/B;AAAA,EAEA,gBAAgB,KAAc,MAAc;AAE1C,SAAK,QAAQ,IAAI,MAAM,KAAK,eAAe,MAAM,MAAS;AAAA,EAC5D;AAAA,EAEA,QACE,MACA,KACA,KACA,MACA,MACA,QACA;AAEA,QAAI,QAAQ;AACV,WAAK,YAAY,OAAO,MAAM;AAC9B,WAAK,KAAK,OAAO,MAAM;AAAA,IACzB;AAEA,UAAMA,OAAM,QAAQ,OAAO,QAAQ;AAEnC,QAAI,SAAS,cAAc;AAIzB,YAAMC,OAAM,SAAS,KAAK,KAAK,YAAY,SAAS,IAAI,SAAS;AACjE,WAAK,IAAIA,IAAG,IAAI,GAAG,IAAI,KAAKD,IAAG,uBAAuB,IAAI,KAAK,OAAO,EAAE;AACxE,aAAO,KAAK,MAAMC,SAAQ,SAAS,OAAO,EAAE;AAAA,IAC9C;AAEA,UAAM,MAAM,KAAK,gBAAgB,IAAI,UAAU;AAC/C,QAAI,KAAK;AACP,WAAK,IAAI,GAAG,IAAI,GAAG,IAAI,KAAKD,IAAG,kBAAkB,GAAG,KAAK,OAAO,EAAE;AAAA,IACpE,WAAW,SAAS,GAAG;AACrB,WAAK,IAAI,GAAG,IAAI,GAAG,IAAI,KAAKA,IAAG,uBAAuB,IAAI,KAAK,OAAO,EAAE;AAAA,IAC1E,OAAO;AACL,WAAK,IAAI,OAAO,GAAG,IAAI,KAAKA,IAAG,uBAAuB,IAAI,GAAG;AAAA,IAC/D;AAGA,QAAI,KAAK,YAAY,SAAS,GAAG;AAC/B,WAAK,IAAI;AAAA,QACP,KAAK,cACD,oCACE,KAAK,QAAQ,KAAK,WACpB,SACA;AAAA,MAAA;AAEN,aAAO,KAAK,MAAM,CAAC;AAAA,IACrB;AAIA,QAAI,QAAQ,SAAS;AACnB,aAAO,KAAK,MAAM,QAAQ,EAAE;AAAA,IAC9B;AAEA,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,SAA+B,QAAwB;AAC3D,eAAW,UAAU,SAAS;AAC5B,UAAI;AACF,eAAO,KAAK,MAAM;AAAA,MACpB,SAAS,GAAG;AACV,aAAK,IAAI,QAAQ,CAAC;AAAA,MACpB;AAAA,IACF;AAAA,EACF;AACF;AAUA,eAAsB,eACpB,IACA,WACG,UACY;AACf,MAAI,SAAS,WAAW,GAAG;AACzB;AAAA,EACF;AACA,aAAW,UAAU,CAAC,GAAG,mBAAmB,GAAG,iBAAiB,GAAG;AACjE,WAAO,KAAK,QAAQ,MAAM;AACxB,YAAM,mBAAmB;AAEzB,eAAS,QAAQ,OAAM,QAAO;AAC5B,YAAI,iBAAiB,SAAS,MAAM,KAAK,IAAI,OAAO;AAClD,aAAG,OAAO,YAAY,IAAI,YAAY,IAAI,IAAI,IAAI,EAAE,KAAK,MAAM,GAAG;AAClE,gBAAM,IAAI,MAAA;AAAA,QACZ;AACA,WAAG,OAAO,YAAY,IAAI,YAAY,IAAI,IAAI,IAAI,EAAE,KAAK,MAAM,GAAG;AAClE,cAAM,IAAI,KAAA;AAAA,MACZ,CAAC;AAAA,IACH,CAAC;AAAA,EACH;AAEA,MAAI;AAEF,UAAM,MAAM,MAAM,QAAQ;AAAA,MACxB,SAAS,IAAI,CAAAE,SAAOA,KAAI,MAAM,KAAK,MAAMA,IAAG,CAAC;AAAA,IAAA;AAE/C,OAAG,OAAO,GAAG,IAAI,YAAY,IAAI,KAAK,IAAI,EAAE,WAAW;AAAA,EACzD,SAAS,GAAG;AACV,OAAG,QAAQ,oBAAoB,CAAC;AAChC,UAAM;AAAA,EACR;AACF;AAEA,eAAsB,UAAU,KAA0B;AACxD,MAAI;AACF,UAAM,IAAA;AAEN,YAAQ,KAAK,OAAO,GAAG,mBAAmB;AAC1C,YAAQ,KAAK,CAAC;AAAA,EAChB,SAAS,GAAG;AAEV,YAAQ,MAAM,OAAO,GAAG,uBAAuB,CAAC;AAChD,YAAQ,KAAK,EAAE;AAAA,EACjB;AACF;AAEA,MAAM,2BAA2B;AAa1B,MAAM,iBAAiB;AAAA,EACnB;AAAA,EAET;AAAA,EACA;AAAA,EACA;AAAA,EACA,iBAAiB;AAAA,EAEjB,YAAY,IAAgB,eAAe,0BAA0B;AACnE,SAAK,MAAM;AACX,SAAK,gBAAgB;AAAA,EACvB;AAAA,EAEA,YAAY,YAAiC;AAC3C,SAAK,iBAAiB,KAAK,IAAA;AAC3B,QAAI,KAAK,wBAAwB,QAAW;AAC1C,WAAK,IAAI;AAAA,QACP,iCACE,KAAK,gBAAgB,GACvB;AAAA,QACA;AAAA,MAAA;AAIF,WAAK,sBAAsB;AAAA,QACzB,KAAK;AAAA,QACL,KAAK,gBAAgB;AAAA,MAAA;AAAA,IAEzB;AAAA,EACF;AAAA,EAEA,qBAAqB,MAAM;AAsBzB,SAAK,uBAAuB,aAAa,MAAM;AAC7C,WAAK,uBAAuB;AAC5B,YAAM,yBAAyB,KAAK,IAAA,IAAQ,KAAK;AACjD,UAAI,0BAA0B,KAAK,eAAe;AAChD,aAAK,IAAI;AAAA,UACP,2BACE,yBAAyB,GAC3B;AAAA,QAAA;AAEF,gBAAQ,KAAK,QAAQ,KAAK,kBAAkB,CAAC,CAAC;AAAA,MAChD;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,OAAO;AACL,iBAAa,KAAK,mBAAmB;AACrC,QAAI,KAAK,sBAAsB;AAC7B,qBAAe,KAAK,oBAAoB;AAAA,IAC1C;AAAA,EACF;AACF;"}
+{"version":3,"file":"life-cycle.js","sources":["../../../../../zero-cache/src/services/life-cycle.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport type {IncomingHttpHeaders} from 'node:http';\nimport {pid} from 'node:process';\nimport type {EventEmitter} from 'stream';\nimport {\n  singleProcessMode,\n  type Subprocess,\n  type Worker,\n} from '../types/processes.ts';\nimport {RunningState} from './running-state.ts';\nimport type {SingletonService} from './service.ts';\n\n/**\n * * `user-facing` workers serve external requests and are the first to\n *   receive a `SIGTERM` or `SIGINT` signal for graceful shutdown.\n *\n * * `supporting` workers support `user-facing` workers and are sent\n *   the `SIGTERM` signal only after all `user-facing` workers have\n *   exited.\n *\n * For other kill signals, such as `SIGQUIT`, all workers\n * are stopped without draining. Additionally, if any worker exits\n * unexpectedly, all workers sent an immediate `SIGQUIT` signal.\n */\nexport type WorkerType = 'user-facing' | 'supporting';\n\nexport const GRACEFUL_SHUTDOWN = ['SIGTERM', 'SIGINT'] as const;\nexport const FORCEFUL_SHUTDOWN = ['SIGQUIT'] as const;\n\n/**\n * Handles readiness, termination signals, and coordination of graceful\n * shutdown.\n */\nexport class ProcessManager {\n  readonly #lc: LogContext;\n  readonly #userFacing = new Set<Subprocess>();\n  readonly #all = new Set<Subprocess>();\n  readonly #exitImpl: (code: number) => never;\n  readonly #start = Date.now();\n  readonly #ready: Promise<void>[] = [];\n\n  #runningState = new RunningState('process-manager');\n  #drainStart = 0;\n\n  constructor(lc: LogContext, proc: EventEmitter) {\n    this.#lc = lc.withContext('component', 'process-manager');\n\n    // Propagate `SIGTERM` and `SIGINT` to all user-facing workers,\n    // initiating a graceful shutdown. The parent process will\n    // exit once all user-facing workers have exited ...\n    for (const signal of GRACEFUL_SHUTDOWN) {\n      proc.on(signal, () => this.#startDrain(signal));\n    }\n\n    // ... which will result in sending `SIGTERM` to the remaining workers.\n    proc.on('exit', code =>\n      this.#kill(\n        this.#all,\n        code === 0 ? GRACEFUL_SHUTDOWN[0] : FORCEFUL_SHUTDOWN[0],\n      ),\n    );\n\n    // For other (catchable) kill signals, exit with a non-zero error code\n    // to send a `SIGQUIT` to all workers. For this signal, workers are\n    // stopped immediately without draining. See `runUntilKilled()`.\n    for (const signal of FORCEFUL_SHUTDOWN) {\n      proc.on(signal, () => this.#exit(-1));\n    }\n\n    this.#exitImpl = (code: number) => {\n      if (singleProcessMode()) {\n        return proc.emit('exit', code) as never; // For unit / integration tests.\n      }\n      process.exit(code);\n    };\n  }\n\n  done() {\n    return this.#runningState.stopped();\n  }\n\n  #exit(code: number) {\n    this.#lc.info?.('exiting with code', code);\n    this.#runningState.stop(this.#lc);\n    void this.#lc.flush().finally(() => this.#exitImpl(code));\n  }\n\n  #startDrain(signal: 'SIGTERM' | 'SIGINT' = 'SIGTERM') {\n    this.#lc.info?.(`initiating drain (${signal})`);\n    this.#drainStart = Date.now();\n    if (this.#userFacing.size) {\n      this.#kill(this.#userFacing, signal);\n    } else {\n      this.#kill(this.#all, signal);\n    }\n  }\n\n  addSubprocess(proc: Subprocess, type: WorkerType, name: string) {\n    if (type === 'user-facing') {\n      this.#userFacing.add(proc);\n    }\n    this.#all.add(proc);\n\n    proc.on('error', err =>\n      this.#lc.error?.(`error from ${name} ${proc.pid}`, err),\n    );\n    proc.on('close', (code, signal) =>\n      this.#onExit(code, signal, null, type, name, proc),\n    );\n  }\n\n  readonly #initializing = new Map<number, string>();\n  #nextID = 0;\n\n  addWorker(worker: Worker, type: WorkerType, name: string): Worker {\n    this.addSubprocess(worker, type, name);\n\n    const id = ++this.#nextID;\n    this.#initializing.set(id, name);\n    const {promise, resolve} = resolver();\n    this.#ready.push(promise);\n\n    worker.onceMessageType('ready', () => {\n      this.#lc.debug?.(`${name} ready (${Date.now() - this.#start} ms)`);\n      this.#initializing.delete(id);\n      resolve();\n    });\n\n    return worker;\n  }\n\n  initializing(): string[] {\n    return [...this.#initializing.values()];\n  }\n\n  async allWorkersReady() {\n    await Promise.all(this.#ready);\n  }\n\n  logErrorAndExit(err: unknown, name: string) {\n    // only accessible by the main (i.e. user-facing) process.\n    this.#onExit(-1, null, err, 'user-facing', name, undefined);\n  }\n\n  #onExit(\n    code: number,\n    sig: NodeJS.Signals | null,\n    err: unknown | null,\n    type: WorkerType,\n    name: string,\n    worker: Subprocess | undefined,\n  ) {\n    // Remove the worker from maps to avoid attempting to send more signals to it.\n    if (worker) {\n      this.#userFacing.delete(worker);\n      this.#all.delete(worker);\n    }\n\n    const pid = worker?.pid ?? process.pid;\n\n    if (type === 'supporting') {\n      // The replication-manager has no user-facing workers.\n      // In this case, code === 0 shutdowns are not errors.\n      // Non-zero exits are warnings (not errors) since they're often transient issues.\n      const log = code === 0 && this.#userFacing.size === 0 ? 'info' : 'warn';\n      this.#lc[log]?.(`${name} (${pid}) exited with code (${code})`, err ?? '');\n      return this.#exit(log === 'info' ? code : -1);\n    }\n\n    const log = this.#drainStart === 0 ? 'error' : 'warn';\n    if (sig) {\n      this.#lc[log]?.(`${name} (${pid}) killed with (${sig})`, err ?? '');\n    } else if (code !== 0) {\n      this.#lc[log]?.(`${name} (${pid}) exited with code (${code})`, err ?? '');\n    } else {\n      this.#lc.info?.(`${name} (${pid}) exited with code (${code})`);\n    }\n\n    // user-facing workers exited or finished draining.\n    if (this.#userFacing.size === 0) {\n      this.#lc.info?.(\n        this.#drainStart\n          ? `all user-facing workers drained (${\n              Date.now() - this.#drainStart\n            } ms)`\n          : `all user-facing workers exited`,\n      );\n      return this.#exit(0);\n    }\n\n    // Exit only if not draining. If a user-facing worker exits unexpectedly\n    // during a drain, log a warning but let other user-facing workers drain.\n    if (log === 'error') {\n      return this.#exit(code || -1);\n    }\n\n    return undefined;\n  }\n\n  #kill(workers: Iterable<Subprocess>, signal: NodeJS.Signals) {\n    for (const worker of workers) {\n      try {\n        worker.kill(signal);\n      } catch (e) {\n        this.#lc.error?.(e);\n      }\n    }\n  }\n}\n\n/**\n * Runs the specified services, stopping them on `SIGTERM` or `SIGINT` with\n * an optional {@link SingletonService.drain drain()}, or stopping them\n * without draining for `SIGQUIT`.\n *\n * @returns a Promise that resolves/rejects when any of the services stops/throws.\n */\n\nexport async function runUntilKilled(\n  lc: LogContext,\n  parent: EventEmitter,\n  ...services: SingletonService[]\n): Promise<void> {\n  if (services.length === 0) {\n    return;\n  }\n  for (const signal of [...GRACEFUL_SHUTDOWN, ...FORCEFUL_SHUTDOWN]) {\n    parent.once(signal, () => {\n      const GRACEFUL_SIGNALS = GRACEFUL_SHUTDOWN as readonly NodeJS.Signals[];\n\n      services.forEach(async svc => {\n        if (GRACEFUL_SIGNALS.includes(signal) && svc.drain) {\n          lc.info?.(`draining ${svc.constructor.name} ${svc.id} (${signal})`);\n          await svc.drain();\n        }\n        lc.info?.(`stopping ${svc.constructor.name} ${svc.id} (${signal})`);\n        await svc.stop();\n      });\n    });\n  }\n\n  try {\n    // Run all services and resolve when any of them stops.\n    const svc = await Promise.race(\n      services.map(svc => svc.run().then(() => svc)),\n    );\n    lc.info?.(`${svc.constructor.name} (${svc.id}) stopped`);\n  } catch (e) {\n    lc.error?.(`exiting on error`, e);\n    throw e;\n  }\n}\n\nexport async function exitAfter(run: () => Promise<void>) {\n  try {\n    await run();\n    // oxlint-disable-next-line no-console\n    console.info(`pid ${pid} exiting normally`);\n    process.exit(0);\n  } catch (e) {\n    // oxlint-disable-next-line no-console\n    console.error(`pid ${pid} exiting with error`, e);\n    process.exit(-1);\n  }\n}\n\nconst DEFAULT_STOP_INTERVAL_MS = 20_000;\n\n/**\n * The HeartbeatMonitor monitors the cadence heartbeats (e.g. \"/keepalive\"\n * health checks made to HttpServices) that signal that the server\n * should continue processing requests. When a configurable `stopInterval`\n * elapses without receiving these heartbeats, the monitor initiates a\n * graceful shutdown of the server. This works with common load balancing\n * frameworks such as AWS Elastic Load Balancing.\n *\n * The HeartbeatMonitor is **opt-in** in that it only kicks in after it\n * starts receiving keepalives.\n */\nexport class HeartbeatMonitor {\n  readonly #stopInterval: number;\n\n  #lc: LogContext;\n  #checkIntervalTimer: NodeJS.Timeout | undefined;\n  #checkImmediateTimer: NodeJS.Immediate | undefined;\n  #lastHeartbeat = 0;\n\n  constructor(lc: LogContext, stopInterval = DEFAULT_STOP_INTERVAL_MS) {\n    this.#lc = lc;\n    this.#stopInterval = stopInterval;\n  }\n\n  onHeartbeat(reqHeaders: IncomingHttpHeaders) {\n    this.#lastHeartbeat = Date.now();\n    if (this.#checkIntervalTimer === undefined) {\n      this.#lc.info?.(\n        `starting heartbeat monitor at ${\n          this.#stopInterval / 1000\n        } second interval`,\n        reqHeaders,\n      );\n      // e.g. check every 5 seconds to see if it's been over 20 seconds\n      //      since the last heartbeat.\n      this.#checkIntervalTimer = setInterval(\n        this.#checkStopInterval,\n        this.#stopInterval / 4,\n      );\n    }\n  }\n\n  #checkStopInterval = () => {\n    // In the Node.js event loop, timers like setInterval and setTimeout\n    // run *before* I/O events coming from network sockets or file reads/writes.\n    // When this process gets starved of CPU resources for long periods of time,\n    // for example when other processes are monopolizing all available cores,\n    // pathological behavior can emerge:\n    // - keepalive network request comes in, but is queued in Node internals waiting\n    //   for time on the event loop\n    // - CPU is starved/monopolized by other processes for longer than the time\n    //   configured via this.#stopInterval\n    // - When CPU becomes available and the event loop wakes up, this stop interval\n    //   check is run *before* the keepalive request is processed. The value of\n    //   this.#lastHeartbeat is now very stale, and erroneously triggers a shutdown\n    //   even though keepalive requests were about to be processed and update\n    //   this.#lastHeartbeat. Downtime ensues.\n    //\n    // To avoid this, we push the check out to a phase of the event loop *after*\n    // I/O events are processed, using setImmediate():\n    // https://nodejs.org/en/learn/asynchronous-work/event-loop-timers-and-nexttick#setimmediate-vs-settimeout\n    //\n    // This ensures we see a value for this.#lastHeartbeat that reflects\n    // any keepalive requests that came in during the current event loop turn.\n    this.#checkImmediateTimer = setImmediate(() => {\n      this.#checkImmediateTimer = undefined;\n      const timeSinceLastHeartbeat = Date.now() - this.#lastHeartbeat;\n      if (timeSinceLastHeartbeat >= this.#stopInterval) {\n        this.#lc.info?.(\n          `last heartbeat received ${\n            timeSinceLastHeartbeat / 1000\n          } seconds ago. draining.`,\n        );\n        process.kill(process.pid, GRACEFUL_SHUTDOWN[0]);\n      }\n    });\n  };\n\n  stop() {\n    clearTimeout(this.#checkIntervalTimer);\n    if (this.#checkImmediateTimer) {\n      clearImmediate(this.#checkImmediateTimer);\n    }\n  }\n}\n"],"names":["pid","log","svc"],"mappings":";;;;AA2BO,MAAM,oBAAoB,CAAC,WAAW,QAAQ;AAC9C,MAAM,oBAAoB,CAAC,SAAS;AAMpC,MAAM,eAAe;AAAA,EACjB;AAAA,EACA,kCAAkB,IAAA;AAAA,EAClB,2BAAW,IAAA;AAAA,EACX;AAAA,EACA,SAAS,KAAK,IAAA;AAAA,EACd,SAA0B,CAAA;AAAA,EAEnC,gBAAgB,IAAI,aAAa,iBAAiB;AAAA,EAClD,cAAc;AAAA,EAEd,YAAY,IAAgB,MAAoB;AAC9C,SAAK,MAAM,GAAG,YAAY,aAAa,iBAAiB;AAKxD,eAAW,UAAU,mBAAmB;AACtC,WAAK,GAAG,QAAQ,MAAM,KAAK,YAAY,MAAM,CAAC;AAAA,IAChD;AAGA,SAAK;AAAA,MAAG;AAAA,MAAQ,UACd,KAAK;AAAA,QACH,KAAK;AAAA,QACL,SAAS,IAAI,kBAAkB,CAAC,IAAI,kBAAkB,CAAC;AAAA,MAAA;AAAA,IACzD;AAMF,eAAW,UAAU,mBAAmB;AACtC,WAAK,GAAG,QAAQ,MAAM,KAAK,MAAM,EAAE,CAAC;AAAA,IACtC;AAEA,SAAK,YAAY,CAAC,SAAiB;AACjC,UAAI,qBAAqB;AACvB,eAAO,KAAK,KAAK,QAAQ,IAAI;AAAA,MAC/B;AACA,cAAQ,KAAK,IAAI;AAAA,IACnB;AAAA,EACF;AAAA,EAEA,OAAO;AACL,WAAO,KAAK,cAAc,QAAA;AAAA,EAC5B;AAAA,EAEA,MAAM,MAAc;AAClB,SAAK,IAAI,OAAO,qBAAqB,IAAI;AACzC,SAAK,cAAc,KAAK,KAAK,GAAG;AAChC,SAAK,KAAK,IAAI,MAAA,EAAQ,QAAQ,MAAM,KAAK,UAAU,IAAI,CAAC;AAAA,EAC1D;AAAA,EAEA,YAAY,SAA+B,WAAW;AACpD,SAAK,IAAI,OAAO,qBAAqB,MAAM,GAAG;AAC9C,SAAK,cAAc,KAAK,IAAA;AACxB,QAAI,KAAK,YAAY,MAAM;AACzB,WAAK,MAAM,KAAK,aAAa,MAAM;AAAA,IACrC,OAAO;AACL,WAAK,MAAM,KAAK,MAAM,MAAM;AAAA,IAC9B;AAAA,EACF;AAAA,EAEA,cAAc,MAAkB,MAAkB,MAAc;AAC9D,QAAI,SAAS,eAAe;AAC1B,WAAK,YAAY,IAAI,IAAI;AAAA,IAC3B;AACA,SAAK,KAAK,IAAI,IAAI;AAElB,SAAK;AAAA,MAAG;AAAA,MAAS,CAAA,QACf,KAAK,IAAI,QAAQ,cAAc,IAAI,IAAI,KAAK,GAAG,IAAI,GAAG;AAAA,IAAA;AAExD,SAAK;AAAA,MAAG;AAAA,MAAS,CAAC,MAAM,WACtB,KAAK,QAAQ,MAAM,QAAQ,MAAM,MAAM,MAAM,IAAI;AAAA,IAAA;AAAA,EAErD;AAAA,EAES,oCAAoB,IAAA;AAAA,EAC7B,UAAU;AAAA,EAEV,UAAU,QAAgB,MAAkB,MAAsB;AAChE,SAAK,cAAc,QAAQ,MAAM,IAAI;AAErC,UAAM,KAAK,EAAE,KAAK;AAClB,SAAK,cAAc,IAAI,IAAI,IAAI;AAC/B,UAAM,EAAC,SAAS,QAAA,IAAW,SAAA;AAC3B,SAAK,OAAO,KAAK,OAAO;AAExB,WAAO,gBAAgB,SAAS,MAAM;AACpC,WAAK,IAAI,QAAQ,GAAG,IAAI,WAAW,KAAK,IAAA,IAAQ,KAAK,MAAM,MAAM;AACjE,WAAK,cAAc,OAAO,EAAE;AAC5B,cAAA;AAAA,IACF,CAAC;AAED,WAAO;AAAA,EACT;AAAA,EAEA,eAAyB;AACvB,WAAO,CAAC,GAAG,KAAK,cAAc,QAAQ;AAAA,EACxC;AAAA,EAEA,MAAM,kBAAkB;AACtB,UAAM,QAAQ,IAAI,KAAK,MAAM;AAAA,EAC/B;AAAA,EAEA,gBAAgB,KAAc,MAAc;AAE1C,SAAK,QAAQ,IAAI,MAAM,KAAK,eAAe,MAAM,MAAS;AAAA,EAC5D;AAAA,EAEA,QACE,MACA,KACA,KACA,MACA,MACA,QACA;AAEA,QAAI,QAAQ;AACV,WAAK,YAAY,OAAO,MAAM;AAC9B,WAAK,KAAK,OAAO,MAAM;AAAA,IACzB;AAEA,UAAMA,OAAM,QAAQ,OAAO,QAAQ;AAEnC,QAAI,SAAS,cAAc;AAIzB,YAAMC,OAAM,SAAS,KAAK,KAAK,YAAY,SAAS,IAAI,SAAS;AACjE,WAAK,IAAIA,IAAG,IAAI,GAAG,IAAI,KAAKD,IAAG,uBAAuB,IAAI,KAAK,OAAO,EAAE;AACxE,aAAO,KAAK,MAAMC,SAAQ,SAAS,OAAO,EAAE;AAAA,IAC9C;AAEA,UAAM,MAAM,KAAK,gBAAgB,IAAI,UAAU;AAC/C,QAAI,KAAK;AACP,WAAK,IAAI,GAAG,IAAI,GAAG,IAAI,KAAKD,IAAG,kBAAkB,GAAG,KAAK,OAAO,EAAE;AAAA,IACpE,WAAW,SAAS,GAAG;AACrB,WAAK,IAAI,GAAG,IAAI,GAAG,IAAI,KAAKA,IAAG,uBAAuB,IAAI,KAAK,OAAO,EAAE;AAAA,IAC1E,OAAO;AACL,WAAK,IAAI,OAAO,GAAG,IAAI,KAAKA,IAAG,uBAAuB,IAAI,GAAG;AAAA,IAC/D;AAGA,QAAI,KAAK,YAAY,SAAS,GAAG;AAC/B,WAAK,IAAI;AAAA,QACP,KAAK,cACD,oCACE,KAAK,QAAQ,KAAK,WACpB,SACA;AAAA,MAAA;AAEN,aAAO,KAAK,MAAM,CAAC;AAAA,IACrB;AAIA,QAAI,QAAQ,SAAS;AACnB,aAAO,KAAK,MAAM,QAAQ,EAAE;AAAA,IAC9B;AAEA,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,SAA+B,QAAwB;AAC3D,eAAW,UAAU,SAAS;AAC5B,UAAI;AACF,eAAO,KAAK,MAAM;AAAA,MACpB,SAAS,GAAG;AACV,aAAK,IAAI,QAAQ,CAAC;AAAA,MACpB;AAAA,IACF;AAAA,EACF;AACF;AAUA,eAAsB,eACpB,IACA,WACG,UACY;AACf,MAAI,SAAS,WAAW,GAAG;AACzB;AAAA,EACF;AACA,aAAW,UAAU,CAAC,GAAG,mBAAmB,GAAG,iBAAiB,GAAG;AACjE,WAAO,KAAK,QAAQ,MAAM;AACxB,YAAM,mBAAmB;AAEzB,eAAS,QAAQ,OAAM,QAAO;AAC5B,YAAI,iBAAiB,SAAS,MAAM,KAAK,IAAI,OAAO;AAClD,aAAG,OAAO,YAAY,IAAI,YAAY,IAAI,IAAI,IAAI,EAAE,KAAK,MAAM,GAAG;AAClE,gBAAM,IAAI,MAAA;AAAA,QACZ;AACA,WAAG,OAAO,YAAY,IAAI,YAAY,IAAI,IAAI,IAAI,EAAE,KAAK,MAAM,GAAG;AAClE,cAAM,IAAI,KAAA;AAAA,MACZ,CAAC;AAAA,IACH,CAAC;AAAA,EACH;AAEA,MAAI;AAEF,UAAM,MAAM,MAAM,QAAQ;AAAA,MACxB,SAAS,IAAI,CAAAE,SAAOA,KAAI,MAAM,KAAK,MAAMA,IAAG,CAAC;AAAA,IAAA;AAE/C,OAAG,OAAO,GAAG,IAAI,YAAY,IAAI,KAAK,IAAI,EAAE,WAAW;AAAA,EACzD,SAAS,GAAG;AACV,OAAG,QAAQ,oBAAoB,CAAC;AAChC,UAAM;AAAA,EACR;AACF;AAEA,eAAsB,UAAU,KAA0B;AACxD,MAAI;AACF,UAAM,IAAA;AAEN,YAAQ,KAAK,OAAO,GAAG,mBAAmB;AAC1C,YAAQ,KAAK,CAAC;AAAA,EAChB,SAAS,GAAG;AAEV,YAAQ,MAAM,OAAO,GAAG,uBAAuB,CAAC;AAChD,YAAQ,KAAK,EAAE;AAAA,EACjB;AACF;AAEA,MAAM,2BAA2B;AAa1B,MAAM,iBAAiB;AAAA,EACnB;AAAA,EAET;AAAA,EACA;AAAA,EACA;AAAA,EACA,iBAAiB;AAAA,EAEjB,YAAY,IAAgB,eAAe,0BAA0B;AACnE,SAAK,MAAM;AACX,SAAK,gBAAgB;AAAA,EACvB;AAAA,EAEA,YAAY,YAAiC;AAC3C,SAAK,iBAAiB,KAAK,IAAA;AAC3B,QAAI,KAAK,wBAAwB,QAAW;AAC1C,WAAK,IAAI;AAAA,QACP,iCACE,KAAK,gBAAgB,GACvB;AAAA,QACA;AAAA,MAAA;AAIF,WAAK,sBAAsB;AAAA,QACzB,KAAK;AAAA,QACL,KAAK,gBAAgB;AAAA,MAAA;AAAA,IAEzB;AAAA,EACF;AAAA,EAEA,qBAAqB,MAAM;AAsBzB,SAAK,uBAAuB,aAAa,MAAM;AAC7C,WAAK,uBAAuB;AAC5B,YAAM,yBAAyB,KAAK,IAAA,IAAQ,KAAK;AACjD,UAAI,0BAA0B,KAAK,eAAe;AAChD,aAAK,IAAI;AAAA,UACP,2BACE,yBAAyB,GAC3B;AAAA,QAAA;AAEF,gBAAQ,KAAK,QAAQ,KAAK,kBAAkB,CAAC,CAAC;AAAA,MAChD;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,OAAO;AACL,iBAAa,KAAK,mBAAmB;AACrC,QAAI,KAAK,sBAAsB;AAC7B,qBAAe,KAAK,oBAAoB;AAAA,IAC1C;AAAA,EACF;AACF;"}

package/out/zero-cache/src/services/litestream/commands.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"commands.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/litestream/commands.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAW,MAAM,kBAAkB,CAAC;AAE3D,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAC;AAOrD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,6BAA6B,CAAC;AAiB5D;;;GAGG;AACH,wBAAsB,cAAc,CAClC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,UAAU,GACjB,OAAO,CAAC,IAAI,CAAC,CAwBf;AA4JD,wBAAgB,yBAAyB,CAAC,MAAM,EAAE,UAAU,GAAG,YAAY,CAO1E"}
+{"version":3,"file":"commands.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/litestream/commands.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAW,MAAM,kBAAkB,CAAC;AAE3D,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAC;AAOrD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,6BAA6B,CAAC;AAiB5D;;;GAGG;AACH,wBAAsB,cAAc,CAClC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,UAAU,GACjB,OAAO,CAAC,IAAI,CAAC,CAwBf;AA8JD,wBAAgB,yBAAyB,CAAC,MAAM,EAAE,UAAU,GAAG,YAAY,CAO1E"}

package/out/zero-cache/src/services/litestream/commands.js

@@ -39,6 +39,7 @@ function getLitestream(config, logLevelOverride, backupURLOverride) {
     backupURL,
     logLevel,
     configPath,
+    endpoint,
     port = config.port + 2,
     checkpointThresholdMB,
     minCheckpointPageCount = checkpointThresholdMB * 250,
@@ -73,7 +74,8 @@ function getLitestream(config, logLevelOverride, backupURLOverride) {
       ["ZERO_LITESTREAM_MULTIPART_SIZE"]: String(multipartSize),
       ["ZERO_LOG_FORMAT"]: config.log.format,
       ["LITESTREAM_CONFIG"]: configPath,
-      ["LITESTREAM_PORT"]: String(port)
+      ["LITESTREAM_PORT"]: String(port),
+      ...endpoint ? { ["ZERO_LITESTREAM_ENDPOINT"]: endpoint } : {}
     }
   };
 }

package/out/zero-cache/src/services/litestream/commands.js.map

@@ -1 +1 @@
-{"version":3,"file":"commands.js","sources":["../../../../../../zero-cache/src/services/litestream/commands.ts"],"sourcesContent":["import type {LogContext, LogLevel} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport type {ChildProcess} from 'node:child_process';\nimport {spawn} from 'node:child_process';\nimport {existsSync} from 'node:fs';\nimport {must} from '../../../../shared/src/must.ts';\nimport {sleep} from '../../../../shared/src/sleep.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {assertNormalized} from '../../config/normalize.ts';\nimport type {ZeroConfig} from '../../config/zero-config.ts';\nimport {deleteLiteDB} from '../../db/delete-lite-db.ts';\nimport {StatementRunner} from '../../db/statements.ts';\nimport {getShardConfig} from '../../types/shards.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {ChangeStreamerHttpClient} from '../change-streamer/change-streamer-http.ts';\nimport type {\n  SnapshotMessage,\n  SnapshotStatus,\n} from '../change-streamer/snapshot.ts';\nimport {getSubscriptionState} from '../replicator/schema/replication-state.ts';\n\n// Retry for up to 3 minutes (60 times with 3 second delay).\n// Beyond that, let the container runner restart the task.\nconst MAX_RETRIES = 60;\nconst RETRY_INTERVAL_MS = 3000;\n\n/**\n * @returns The time at which the last restore started\n *          (i.e. not counting failed attempts).\n */\nexport async function restoreReplica(\n  lc: LogContext,\n  config: ZeroConfig,\n): Promise<Date> {\n  const {changeStreamer} = config;\n\n  for (let i = 0; i < MAX_RETRIES; i++) {\n    if (i > 0) {\n      lc.info?.(\n        `replica not found. retrying in ${RETRY_INTERVAL_MS / 1000} seconds`,\n      );\n      await sleep(RETRY_INTERVAL_MS);\n    }\n    const start = new Date();\n    const restored = await tryRestore(lc, config);\n    if (restored) {\n      return start;\n    }\n    if (\n      changeStreamer.mode === 'dedicated' &&\n      changeStreamer.uri === undefined\n    ) {\n      lc.info?.('no litestream backup found');\n      return start;\n    }\n  }\n  throw new Error(`max attempts exceeded restoring replica`);\n}\n\nfunction getLitestream(\n  config: ZeroConfig,\n  logLevelOverride?: LogLevel,\n  backupURLOverride?: string,\n): {\n  litestream: string;\n  env: NodeJS.ProcessEnv;\n} {\n  const {\n    executable,\n    backupURL,\n    logLevel,\n    configPath,\n    port = config.port + 2,\n    checkpointThresholdMB,\n    minCheckpointPageCount = checkpointThresholdMB * 250, // SQLite page size is 4KB\n    maxCheckpointPageCount = minCheckpointPageCount * 10,\n    incrementalBackupIntervalMinutes,\n    snapshotBackupIntervalHours,\n    multipartConcurrency,\n    multipartSize,\n  } = config.litestream;\n\n  // Set the snapshot interval to something smaller than x hours so that\n  // the hourly check triggers on the hour, rather than the hour after.\n  const snapshotBackupIntervalMinutes = snapshotBackupIntervalHours * 60 - 5;\n\n  return {\n    litestream: must(executable, `Missing --litestream-executable`),\n    env: {\n      ...process.env,\n      ['ZERO_REPLICA_FILE']: config.replica.file,\n      ['ZERO_LITESTREAM_BACKUP_URL']: must(backupURLOverride ?? backupURL),\n      ['ZERO_LITESTREAM_MIN_CHECKPOINT_PAGE_COUNT']: String(\n        minCheckpointPageCount,\n      ),\n      ['ZERO_LITESTREAM_MAX_CHECKPOINT_PAGE_COUNT']: String(\n        maxCheckpointPageCount,\n      ),\n      ['ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_MINUTES']: String(\n        incrementalBackupIntervalMinutes,\n      ),\n      ['ZERO_LITESTREAM_LOG_LEVEL']: logLevelOverride ?? logLevel,\n      ['ZERO_LITESTREAM_SNAPSHOT_BACKUP_INTERVAL_MINUTES']: String(\n        snapshotBackupIntervalMinutes,\n      ),\n      ['ZERO_LITESTREAM_MULTIPART_CONCURRENCY']: String(multipartConcurrency),\n      ['ZERO_LITESTREAM_MULTIPART_SIZE']: String(multipartSize),\n      ['ZERO_LOG_FORMAT']: config.log.format,\n      ['LITESTREAM_CONFIG']: configPath,\n      ['LITESTREAM_PORT']: String(port),\n    },\n  };\n}\n\nasync function tryRestore(lc: LogContext, config: ZeroConfig) {\n  const {changeStreamer} = config;\n\n  const isViewSyncer =\n    changeStreamer.mode === 'discover' || changeStreamer.uri !== undefined;\n\n  // Fire off a snapshot reservation to the current replication-manager\n  // (if there is one).\n  const firstMessage = reserveAndGetSnapshotStatus(lc, config, isViewSyncer);\n  let snapshotStatus: SnapshotStatus | undefined;\n  if (isViewSyncer) {\n    // The return value is required by view-syncers ...\n    snapshotStatus = await firstMessage;\n    lc.info?.(`restoring backup from ${snapshotStatus.backupURL}`);\n  } else {\n    // but it is also useful to pause change-log cleanup when a new\n    // replication-manager is starting up. In this case, the request is\n    // best-effort. In particular, there may not be a previous\n    // replication-manager running at all.\n    void firstMessage.catch(e => lc.debug?.(e));\n  }\n\n  const {litestream, env} = getLitestream(\n    config,\n    'debug', // Include all output from `litestream restore`, as it's minimal.\n    snapshotStatus?.backupURL,\n  );\n  const {restoreParallelism: parallelism} = config.litestream;\n  const proc = spawn(\n    litestream,\n    [\n      'restore',\n      '-if-db-not-exists',\n      '-if-replica-exists',\n      '-parallelism',\n      String(parallelism),\n      config.replica.file,\n    ],\n    {env, stdio: 'inherit', windowsHide: true},\n  );\n  const {promise, resolve, reject} = resolver();\n  proc.on('error', reject);\n  proc.on('close', (code, signal) => {\n    if (signal) {\n      reject(`litestream killed with ${signal}`);\n    } else if (code !== 0) {\n      reject(`litestream exited with code ${code}`);\n    } else {\n      resolve();\n    }\n  });\n  await promise;\n  if (!existsSync(config.replica.file)) {\n    return false;\n  }\n  if (\n    snapshotStatus &&\n    !replicaIsValid(lc, config.replica.file, snapshotStatus)\n  ) {\n    lc.info?.(`Deleting local replica and retrying restore`);\n    deleteLiteDB(config.replica.file);\n    return false;\n  }\n  return true;\n}\n\nfunction replicaIsValid(\n  lc: LogContext,\n  replica: string,\n  snapshot: SnapshotStatus,\n) {\n  const db = new Database(lc, replica);\n  try {\n    const {replicaVersion, watermark} = getSubscriptionState(\n      new StatementRunner(db),\n    );\n    if (replicaVersion !== snapshot.replicaVersion) {\n      lc.warn?.(\n        `Local replica version ${replicaVersion} does not match change-streamer replicaVersion ${snapshot.replicaVersion}`,\n        snapshot,\n      );\n      return false;\n    }\n    if (watermark < snapshot.minWatermark) {\n      lc.warn?.(\n        `Local replica watermark ${watermark} is earlier than change-streamer minWatermark ${snapshot.minWatermark}`,\n      );\n      return false;\n    }\n    lc.info?.(\n      `Local replica at version ${replicaVersion} and watermark ${watermark} is compatible with change-streamer`,\n      snapshot,\n    );\n    return true;\n  } finally {\n    db.close();\n  }\n}\n\nexport function startReplicaBackupProcess(config: ZeroConfig): ChildProcess {\n  const {litestream, env} = getLitestream(config);\n  return spawn(litestream, ['replicate'], {\n    env,\n    stdio: 'inherit',\n    windowsHide: true,\n  });\n}\n\nfunction reserveAndGetSnapshotStatus(\n  lc: LogContext,\n  config: ZeroConfig,\n  isViewSyncer: boolean,\n): Promise<SnapshotStatus> {\n  const {promise: status, resolve, reject} = resolver<SnapshotStatus>();\n\n  void (async function () {\n    const abort = new AbortController();\n    process.on('SIGINT', () => abort.abort());\n    process.on('SIGTERM', () => abort.abort());\n\n    for (let i = 0; ; i++) {\n      let err: unknown | string = '';\n      try {\n        let resolved = false;\n        const stream = await reserveSnapshot(lc, config);\n        for await (const msg of stream) {\n          // Capture the value of the status message that the change-streamer\n          // (i.e. BackupMonitor) returns, and hold the connection open to\n          // \"reserve\" the snapshot and prevent change log cleanup.\n          resolve(msg[1]);\n          resolved = true;\n        }\n        // The change-streamer itself closes the connection when the\n        // subscription is started (or the reservation retried).\n        if (resolved) {\n          break;\n        }\n      } catch (e) {\n        err = e;\n      }\n      if (!isViewSyncer) {\n        return reject(err);\n      }\n      // Retry in the view-syncer since it cannot proceed until it connects\n      // to a (compatible) replication-manager. In particular, a\n      // replication-manager that does not support the view-syncer's\n      // change-streamer protocol will close the stream with an error; this\n      // retry logic essentially delays the startup of a view-syncer until\n      // a compatible replication-manager has been rolled out, allowing\n      // replication-manager and view-syncer services to be updated in\n      // parallel.\n      lc.warn?.(\n        `Unable to reserve snapshot (attempt ${i + 1}). Retrying in 5 seconds.`,\n        String(err),\n      );\n      try {\n        await sleep(5000, abort.signal);\n      } catch (e) {\n        return reject(e);\n      }\n    }\n  })();\n\n  return status;\n}\n\nfunction reserveSnapshot(\n  lc: LogContext,\n  config: ZeroConfig,\n): Promise<Source<SnapshotMessage>> {\n  assertNormalized(config);\n  const {taskID, change, changeStreamer} = config;\n  const shardID = getShardConfig(config);\n\n  const changeStreamerClient = new ChangeStreamerHttpClient(\n    lc,\n    shardID,\n    change.db,\n    changeStreamer.uri,\n  );\n\n  return changeStreamerClient.reserveSnapshot(taskID);\n}\n"],"names":[],"mappings":";;;;;;;;;;;;AAuBA,MAAM,cAAc;AACpB,MAAM,oBAAoB;AAM1B,eAAsB,eACpB,IACA,QACe;AACf,QAAM,EAAC,mBAAkB;AAEzB,WAAS,IAAI,GAAG,IAAI,aAAa,KAAK;AACpC,QAAI,IAAI,GAAG;AACT,SAAG;AAAA,QACD,kCAAkC,oBAAoB,GAAI;AAAA,MAAA;AAE5D,YAAM,MAAM,iBAAiB;AAAA,IAC/B;AACA,UAAM,4BAAY,KAAA;AAClB,UAAM,WAAW,MAAM,WAAW,IAAI,MAAM;AAC5C,QAAI,UAAU;AACZ,aAAO;AAAA,IACT;AACA,QACE,eAAe,SAAS,eACxB,eAAe,QAAQ,QACvB;AACA,SAAG,OAAO,4BAA4B;AACtC,aAAO;AAAA,IACT;AAAA,EACF;AACA,QAAM,IAAI,MAAM,yCAAyC;AAC3D;AAEA,SAAS,cACP,QACA,kBACA,mBAIA;AACA,QAAM;AAAA,IACJ;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,OAAO,OAAO,OAAO;AAAA,IACrB;AAAA,IACA,yBAAyB,wBAAwB;AAAA;AAAA,IACjD,yBAAyB,yBAAyB;AAAA,IAClD;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EAAA,IACE,OAAO;AAIX,QAAM,gCAAgC,8BAA8B,KAAK;AAEzE,SAAO;AAAA,IACL,YAAY,KAAK,YAAY,iCAAiC;AAAA,IAC9D,KAAK;AAAA,MACH,GAAG,QAAQ;AAAA,MACX,CAAC,mBAAmB,GAAG,OAAO,QAAQ;AAAA,MACtC,CAAC,4BAA4B,GAAG,KAAK,qBAAqB,SAAS;AAAA,MACnE,CAAC,2CAA2C,GAAG;AAAA,QAC7C;AAAA,MAAA;AAAA,MAEF,CAAC,2CAA2C,GAAG;AAAA,QAC7C;AAAA,MAAA;AAAA,MAEF,CAAC,qDAAqD,GAAG;AAAA,QACvD;AAAA,MAAA;AAAA,MAEF,CAAC,2BAA2B,GAAG,oBAAoB;AAAA,MACnD,CAAC,kDAAkD,GAAG;AAAA,QACpD;AAAA,MAAA;AAAA,MAEF,CAAC,uCAAuC,GAAG,OAAO,oBAAoB;AAAA,MACtE,CAAC,gCAAgC,GAAG,OAAO,aAAa;AAAA,MACxD,CAAC,iBAAiB,GAAG,OAAO,IAAI;AAAA,MAChC,CAAC,mBAAmB,GAAG;AAAA,MACvB,CAAC,iBAAiB,GAAG,OAAO,IAAI;AAAA,IAAA;AAAA,EAClC;AAEJ;AAEA,eAAe,WAAW,IAAgB,QAAoB;AAC5D,QAAM,EAAC,mBAAkB;AAEzB,QAAM,eACJ,eAAe,SAAS,cAAc,eAAe,QAAQ;AAI/D,QAAM,eAAe,4BAA4B,IAAI,QAAQ,YAAY;AACzE,MAAI;AACJ,MAAI,cAAc;AAEhB,qBAAiB,MAAM;AACvB,OAAG,OAAO,yBAAyB,eAAe,SAAS,EAAE;AAAA,EAC/D,OAAO;AAKL,SAAK,aAAa,MAAM,CAAA,MAAK,GAAG,QAAQ,CAAC,CAAC;AAAA,EAC5C;AAEA,QAAM,EAAC,YAAY,IAAA,IAAO;AAAA,IACxB;AAAA,IACA;AAAA;AAAA,IACA,gBAAgB;AAAA,EAAA;AAElB,QAAM,EAAC,oBAAoB,YAAA,IAAe,OAAO;AACjD,QAAM,OAAO;AAAA,IACX;AAAA,IACA;AAAA,MACE;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,OAAO,WAAW;AAAA,MAClB,OAAO,QAAQ;AAAA,IAAA;AAAA,IAEjB,EAAC,KAAK,OAAO,WAAW,aAAa,KAAA;AAAA,EAAI;AAE3C,QAAM,EAAC,SAAS,SAAS,OAAA,IAAU,SAAA;AACnC,OAAK,GAAG,SAAS,MAAM;AACvB,OAAK,GAAG,SAAS,CAAC,MAAM,WAAW;AACjC,QAAI,QAAQ;AACV,aAAO,0BAA0B,MAAM,EAAE;AAAA,IAC3C,WAAW,SAAS,GAAG;AACrB,aAAO,+BAA+B,IAAI,EAAE;AAAA,IAC9C,OAAO;AACL,cAAA;AAAA,IACF;AAAA,EACF,CAAC;AACD,QAAM;AACN,MAAI,CAAC,WAAW,OAAO,QAAQ,IAAI,GAAG;AACpC,WAAO;AAAA,EACT;AACA,MACE,kBACA,CAAC,eAAe,IAAI,OAAO,QAAQ,MAAM,cAAc,GACvD;AACA,OAAG,OAAO,6CAA6C;AACvD,iBAAa,OAAO,QAAQ,IAAI;AAChC,WAAO;AAAA,EACT;AACA,SAAO;AACT;AAEA,SAAS,eACP,IACA,SACA,UACA;AACA,QAAM,KAAK,IAAI,SAAS,IAAI,OAAO;AACnC,MAAI;AACF,UAAM,EAAC,gBAAgB,UAAA,IAAa;AAAA,MAClC,IAAI,gBAAgB,EAAE;AAAA,IAAA;AAExB,QAAI,mBAAmB,SAAS,gBAAgB;AAC9C,SAAG;AAAA,QACD,yBAAyB,cAAc,kDAAkD,SAAS,cAAc;AAAA,QAChH;AAAA,MAAA;AAEF,aAAO;AAAA,IACT;AACA,QAAI,YAAY,SAAS,cAAc;AACrC,SAAG;AAAA,QACD,2BAA2B,SAAS,iDAAiD,SAAS,YAAY;AAAA,MAAA;AAE5G,aAAO;AAAA,IACT;AACA,OAAG;AAAA,MACD,4BAA4B,cAAc,kBAAkB,SAAS;AAAA,MACrE;AAAA,IAAA;AAEF,WAAO;AAAA,EACT,UAAA;AACE,OAAG,MAAA;AAAA,EACL;AACF;AAEO,SAAS,0BAA0B,QAAkC;AAC1E,QAAM,EAAC,YAAY,QAAO,cAAc,MAAM;AAC9C,SAAO,MAAM,YAAY,CAAC,WAAW,GAAG;AAAA,IACtC;AAAA,IACA,OAAO;AAAA,IACP,aAAa;AAAA,EAAA,CACd;AACH;AAEA,SAAS,4BACP,IACA,QACA,cACyB;AACzB,QAAM,EAAC,SAAS,QAAQ,SAAS,OAAA,IAAU,SAAA;AAE3C,QAAM,iBAAkB;AACtB,UAAM,QAAQ,IAAI,gBAAA;AAClB,YAAQ,GAAG,UAAU,MAAM,MAAM,OAAO;AACxC,YAAQ,GAAG,WAAW,MAAM,MAAM,OAAO;AAEzC,aAAS,IAAI,KAAK,KAAK;AACrB,UAAI,MAAwB;AAC5B,UAAI;AACF,YAAI,WAAW;AACf,cAAM,SAAS,MAAM,gBAAgB,IAAI,MAAM;AAC/C,yBAAiB,OAAO,QAAQ;AAI9B,kBAAQ,IAAI,CAAC,CAAC;AACd,qBAAW;AAAA,QACb;AAGA,YAAI,UAAU;AACZ;AAAA,QACF;AAAA,MACF,SAAS,GAAG;AACV,cAAM;AAAA,MACR;AACA,UAAI,CAAC,cAAc;AACjB,eAAO,OAAO,GAAG;AAAA,MACnB;AASA,SAAG;AAAA,QACD,uCAAuC,IAAI,CAAC;AAAA,QAC5C,OAAO,GAAG;AAAA,MAAA;AAEZ,UAAI;AACF,cAAM,MAAM,KAAM,MAAM,MAAM;AAAA,MAChC,SAAS,GAAG;AACV,eAAO,OAAO,CAAC;AAAA,MACjB;AAAA,IACF;AAAA,EACF,GAAA;AAEA,SAAO;AACT;AAEA,SAAS,gBACP,IACA,QACkC;AAClC,mBAAiB,MAAM;AACvB,QAAM,EAAC,QAAQ,QAAQ,eAAA,IAAkB;AACzC,QAAM,UAAU,eAAe,MAAM;AAErC,QAAM,uBAAuB,IAAI;AAAA,IAC/B;AAAA,IACA;AAAA,IACA,OAAO;AAAA,IACP,eAAe;AAAA,EAAA;AAGjB,SAAO,qBAAqB,gBAAgB,MAAM;AACpD;"}
+{"version":3,"file":"commands.js","sources":["../../../../../../zero-cache/src/services/litestream/commands.ts"],"sourcesContent":["import type {LogContext, LogLevel} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport type {ChildProcess} from 'node:child_process';\nimport {spawn} from 'node:child_process';\nimport {existsSync} from 'node:fs';\nimport {must} from '../../../../shared/src/must.ts';\nimport {sleep} from '../../../../shared/src/sleep.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {assertNormalized} from '../../config/normalize.ts';\nimport type {ZeroConfig} from '../../config/zero-config.ts';\nimport {deleteLiteDB} from '../../db/delete-lite-db.ts';\nimport {StatementRunner} from '../../db/statements.ts';\nimport {getShardConfig} from '../../types/shards.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {ChangeStreamerHttpClient} from '../change-streamer/change-streamer-http.ts';\nimport type {\n  SnapshotMessage,\n  SnapshotStatus,\n} from '../change-streamer/snapshot.ts';\nimport {getSubscriptionState} from '../replicator/schema/replication-state.ts';\n\n// Retry for up to 3 minutes (60 times with 3 second delay).\n// Beyond that, let the container runner restart the task.\nconst MAX_RETRIES = 60;\nconst RETRY_INTERVAL_MS = 3000;\n\n/**\n * @returns The time at which the last restore started\n *          (i.e. not counting failed attempts).\n */\nexport async function restoreReplica(\n  lc: LogContext,\n  config: ZeroConfig,\n): Promise<Date> {\n  const {changeStreamer} = config;\n\n  for (let i = 0; i < MAX_RETRIES; i++) {\n    if (i > 0) {\n      lc.info?.(\n        `replica not found. retrying in ${RETRY_INTERVAL_MS / 1000} seconds`,\n      );\n      await sleep(RETRY_INTERVAL_MS);\n    }\n    const start = new Date();\n    const restored = await tryRestore(lc, config);\n    if (restored) {\n      return start;\n    }\n    if (\n      changeStreamer.mode === 'dedicated' &&\n      changeStreamer.uri === undefined\n    ) {\n      lc.info?.('no litestream backup found');\n      return start;\n    }\n  }\n  throw new Error(`max attempts exceeded restoring replica`);\n}\n\nfunction getLitestream(\n  config: ZeroConfig,\n  logLevelOverride?: LogLevel,\n  backupURLOverride?: string,\n): {\n  litestream: string;\n  env: NodeJS.ProcessEnv;\n} {\n  const {\n    executable,\n    backupURL,\n    logLevel,\n    configPath,\n    endpoint,\n    port = config.port + 2,\n    checkpointThresholdMB,\n    minCheckpointPageCount = checkpointThresholdMB * 250, // SQLite page size is 4KB\n    maxCheckpointPageCount = minCheckpointPageCount * 10,\n    incrementalBackupIntervalMinutes,\n    snapshotBackupIntervalHours,\n    multipartConcurrency,\n    multipartSize,\n  } = config.litestream;\n\n  // Set the snapshot interval to something smaller than x hours so that\n  // the hourly check triggers on the hour, rather than the hour after.\n  const snapshotBackupIntervalMinutes = snapshotBackupIntervalHours * 60 - 5;\n\n  return {\n    litestream: must(executable, `Missing --litestream-executable`),\n    env: {\n      ...process.env,\n      ['ZERO_REPLICA_FILE']: config.replica.file,\n      ['ZERO_LITESTREAM_BACKUP_URL']: must(backupURLOverride ?? backupURL),\n      ['ZERO_LITESTREAM_MIN_CHECKPOINT_PAGE_COUNT']: String(\n        minCheckpointPageCount,\n      ),\n      ['ZERO_LITESTREAM_MAX_CHECKPOINT_PAGE_COUNT']: String(\n        maxCheckpointPageCount,\n      ),\n      ['ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_MINUTES']: String(\n        incrementalBackupIntervalMinutes,\n      ),\n      ['ZERO_LITESTREAM_LOG_LEVEL']: logLevelOverride ?? logLevel,\n      ['ZERO_LITESTREAM_SNAPSHOT_BACKUP_INTERVAL_MINUTES']: String(\n        snapshotBackupIntervalMinutes,\n      ),\n      ['ZERO_LITESTREAM_MULTIPART_CONCURRENCY']: String(multipartConcurrency),\n      ['ZERO_LITESTREAM_MULTIPART_SIZE']: String(multipartSize),\n      ['ZERO_LOG_FORMAT']: config.log.format,\n      ['LITESTREAM_CONFIG']: configPath,\n      ['LITESTREAM_PORT']: String(port),\n      ...(endpoint ? {['ZERO_LITESTREAM_ENDPOINT']: endpoint} : {}),\n    },\n  };\n}\n\nasync function tryRestore(lc: LogContext, config: ZeroConfig) {\n  const {changeStreamer} = config;\n\n  const isViewSyncer =\n    changeStreamer.mode === 'discover' || changeStreamer.uri !== undefined;\n\n  // Fire off a snapshot reservation to the current replication-manager\n  // (if there is one).\n  const firstMessage = reserveAndGetSnapshotStatus(lc, config, isViewSyncer);\n  let snapshotStatus: SnapshotStatus | undefined;\n  if (isViewSyncer) {\n    // The return value is required by view-syncers ...\n    snapshotStatus = await firstMessage;\n    lc.info?.(`restoring backup from ${snapshotStatus.backupURL}`);\n  } else {\n    // but it is also useful to pause change-log cleanup when a new\n    // replication-manager is starting up. In this case, the request is\n    // best-effort. In particular, there may not be a previous\n    // replication-manager running at all.\n    void firstMessage.catch(e => lc.debug?.(e));\n  }\n\n  const {litestream, env} = getLitestream(\n    config,\n    'debug', // Include all output from `litestream restore`, as it's minimal.\n    snapshotStatus?.backupURL,\n  );\n  const {restoreParallelism: parallelism} = config.litestream;\n  const proc = spawn(\n    litestream,\n    [\n      'restore',\n      '-if-db-not-exists',\n      '-if-replica-exists',\n      '-parallelism',\n      String(parallelism),\n      config.replica.file,\n    ],\n    {env, stdio: 'inherit', windowsHide: true},\n  );\n  const {promise, resolve, reject} = resolver();\n  proc.on('error', reject);\n  proc.on('close', (code, signal) => {\n    if (signal) {\n      reject(`litestream killed with ${signal}`);\n    } else if (code !== 0) {\n      reject(`litestream exited with code ${code}`);\n    } else {\n      resolve();\n    }\n  });\n  await promise;\n  if (!existsSync(config.replica.file)) {\n    return false;\n  }\n  if (\n    snapshotStatus &&\n    !replicaIsValid(lc, config.replica.file, snapshotStatus)\n  ) {\n    lc.info?.(`Deleting local replica and retrying restore`);\n    deleteLiteDB(config.replica.file);\n    return false;\n  }\n  return true;\n}\n\nfunction replicaIsValid(\n  lc: LogContext,\n  replica: string,\n  snapshot: SnapshotStatus,\n) {\n  const db = new Database(lc, replica);\n  try {\n    const {replicaVersion, watermark} = getSubscriptionState(\n      new StatementRunner(db),\n    );\n    if (replicaVersion !== snapshot.replicaVersion) {\n      lc.warn?.(\n        `Local replica version ${replicaVersion} does not match change-streamer replicaVersion ${snapshot.replicaVersion}`,\n        snapshot,\n      );\n      return false;\n    }\n    if (watermark < snapshot.minWatermark) {\n      lc.warn?.(\n        `Local replica watermark ${watermark} is earlier than change-streamer minWatermark ${snapshot.minWatermark}`,\n      );\n      return false;\n    }\n    lc.info?.(\n      `Local replica at version ${replicaVersion} and watermark ${watermark} is compatible with change-streamer`,\n      snapshot,\n    );\n    return true;\n  } finally {\n    db.close();\n  }\n}\n\nexport function startReplicaBackupProcess(config: ZeroConfig): ChildProcess {\n  const {litestream, env} = getLitestream(config);\n  return spawn(litestream, ['replicate'], {\n    env,\n    stdio: 'inherit',\n    windowsHide: true,\n  });\n}\n\nfunction reserveAndGetSnapshotStatus(\n  lc: LogContext,\n  config: ZeroConfig,\n  isViewSyncer: boolean,\n): Promise<SnapshotStatus> {\n  const {promise: status, resolve, reject} = resolver<SnapshotStatus>();\n\n  void (async function () {\n    const abort = new AbortController();\n    process.on('SIGINT', () => abort.abort());\n    process.on('SIGTERM', () => abort.abort());\n\n    for (let i = 0; ; i++) {\n      let err: unknown | string = '';\n      try {\n        let resolved = false;\n        const stream = await reserveSnapshot(lc, config);\n        for await (const msg of stream) {\n          // Capture the value of the status message that the change-streamer\n          // (i.e. BackupMonitor) returns, and hold the connection open to\n          // \"reserve\" the snapshot and prevent change log cleanup.\n          resolve(msg[1]);\n          resolved = true;\n        }\n        // The change-streamer itself closes the connection when the\n        // subscription is started (or the reservation retried).\n        if (resolved) {\n          break;\n        }\n      } catch (e) {\n        err = e;\n      }\n      if (!isViewSyncer) {\n        return reject(err);\n      }\n      // Retry in the view-syncer since it cannot proceed until it connects\n      // to a (compatible) replication-manager. In particular, a\n      // replication-manager that does not support the view-syncer's\n      // change-streamer protocol will close the stream with an error; this\n      // retry logic essentially delays the startup of a view-syncer until\n      // a compatible replication-manager has been rolled out, allowing\n      // replication-manager and view-syncer services to be updated in\n      // parallel.\n      lc.warn?.(\n        `Unable to reserve snapshot (attempt ${i + 1}). Retrying in 5 seconds.`,\n        String(err),\n      );\n      try {\n        await sleep(5000, abort.signal);\n      } catch (e) {\n        return reject(e);\n      }\n    }\n  })();\n\n  return status;\n}\n\nfunction reserveSnapshot(\n  lc: LogContext,\n  config: ZeroConfig,\n): Promise<Source<SnapshotMessage>> {\n  assertNormalized(config);\n  const {taskID, change, changeStreamer} = config;\n  const shardID = getShardConfig(config);\n\n  const changeStreamerClient = new ChangeStreamerHttpClient(\n    lc,\n    shardID,\n    change.db,\n    changeStreamer.uri,\n  );\n\n  return changeStreamerClient.reserveSnapshot(taskID);\n}\n"],"names":[],"mappings":";;;;;;;;;;;;AAuBA,MAAM,cAAc;AACpB,MAAM,oBAAoB;AAM1B,eAAsB,eACpB,IACA,QACe;AACf,QAAM,EAAC,mBAAkB;AAEzB,WAAS,IAAI,GAAG,IAAI,aAAa,KAAK;AACpC,QAAI,IAAI,GAAG;AACT,SAAG;AAAA,QACD,kCAAkC,oBAAoB,GAAI;AAAA,MAAA;AAE5D,YAAM,MAAM,iBAAiB;AAAA,IAC/B;AACA,UAAM,4BAAY,KAAA;AAClB,UAAM,WAAW,MAAM,WAAW,IAAI,MAAM;AAC5C,QAAI,UAAU;AACZ,aAAO;AAAA,IACT;AACA,QACE,eAAe,SAAS,eACxB,eAAe,QAAQ,QACvB;AACA,SAAG,OAAO,4BAA4B;AACtC,aAAO;AAAA,IACT;AAAA,EACF;AACA,QAAM,IAAI,MAAM,yCAAyC;AAC3D;AAEA,SAAS,cACP,QACA,kBACA,mBAIA;AACA,QAAM;AAAA,IACJ;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,OAAO,OAAO,OAAO;AAAA,IACrB;AAAA,IACA,yBAAyB,wBAAwB;AAAA;AAAA,IACjD,yBAAyB,yBAAyB;AAAA,IAClD;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EAAA,IACE,OAAO;AAIX,QAAM,gCAAgC,8BAA8B,KAAK;AAEzE,SAAO;AAAA,IACL,YAAY,KAAK,YAAY,iCAAiC;AAAA,IAC9D,KAAK;AAAA,MACH,GAAG,QAAQ;AAAA,MACX,CAAC,mBAAmB,GAAG,OAAO,QAAQ;AAAA,MACtC,CAAC,4BAA4B,GAAG,KAAK,qBAAqB,SAAS;AAAA,MACnE,CAAC,2CAA2C,GAAG;AAAA,QAC7C;AAAA,MAAA;AAAA,MAEF,CAAC,2CAA2C,GAAG;AAAA,QAC7C;AAAA,MAAA;AAAA,MAEF,CAAC,qDAAqD,GAAG;AAAA,QACvD;AAAA,MAAA;AAAA,MAEF,CAAC,2BAA2B,GAAG,oBAAoB;AAAA,MACnD,CAAC,kDAAkD,GAAG;AAAA,QACpD;AAAA,MAAA;AAAA,MAEF,CAAC,uCAAuC,GAAG,OAAO,oBAAoB;AAAA,MACtE,CAAC,gCAAgC,GAAG,OAAO,aAAa;AAAA,MACxD,CAAC,iBAAiB,GAAG,OAAO,IAAI;AAAA,MAChC,CAAC,mBAAmB,GAAG;AAAA,MACvB,CAAC,iBAAiB,GAAG,OAAO,IAAI;AAAA,MAChC,GAAI,WAAW,EAAC,CAAC,0BAA0B,GAAG,SAAA,IAAY,CAAA;AAAA,IAAC;AAAA,EAC7D;AAEJ;AAEA,eAAe,WAAW,IAAgB,QAAoB;AAC5D,QAAM,EAAC,mBAAkB;AAEzB,QAAM,eACJ,eAAe,SAAS,cAAc,eAAe,QAAQ;AAI/D,QAAM,eAAe,4BAA4B,IAAI,QAAQ,YAAY;AACzE,MAAI;AACJ,MAAI,cAAc;AAEhB,qBAAiB,MAAM;AACvB,OAAG,OAAO,yBAAyB,eAAe,SAAS,EAAE;AAAA,EAC/D,OAAO;AAKL,SAAK,aAAa,MAAM,CAAA,MAAK,GAAG,QAAQ,CAAC,CAAC;AAAA,EAC5C;AAEA,QAAM,EAAC,YAAY,IAAA,IAAO;AAAA,IACxB;AAAA,IACA;AAAA;AAAA,IACA,gBAAgB;AAAA,EAAA;AAElB,QAAM,EAAC,oBAAoB,YAAA,IAAe,OAAO;AACjD,QAAM,OAAO;AAAA,IACX;AAAA,IACA;AAAA,MACE;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,OAAO,WAAW;AAAA,MAClB,OAAO,QAAQ;AAAA,IAAA;AAAA,IAEjB,EAAC,KAAK,OAAO,WAAW,aAAa,KAAA;AAAA,EAAI;AAE3C,QAAM,EAAC,SAAS,SAAS,OAAA,IAAU,SAAA;AACnC,OAAK,GAAG,SAAS,MAAM;AACvB,OAAK,GAAG,SAAS,CAAC,MAAM,WAAW;AACjC,QAAI,QAAQ;AACV,aAAO,0BAA0B,MAAM,EAAE;AAAA,IAC3C,WAAW,SAAS,GAAG;AACrB,aAAO,+BAA+B,IAAI,EAAE;AAAA,IAC9C,OAAO;AACL,cAAA;AAAA,IACF;AAAA,EACF,CAAC;AACD,QAAM;AACN,MAAI,CAAC,WAAW,OAAO,QAAQ,IAAI,GAAG;AACpC,WAAO;AAAA,EACT;AACA,MACE,kBACA,CAAC,eAAe,IAAI,OAAO,QAAQ,MAAM,cAAc,GACvD;AACA,OAAG,OAAO,6CAA6C;AACvD,iBAAa,OAAO,QAAQ,IAAI;AAChC,WAAO;AAAA,EACT;AACA,SAAO;AACT;AAEA,SAAS,eACP,IACA,SACA,UACA;AACA,QAAM,KAAK,IAAI,SAAS,IAAI,OAAO;AACnC,MAAI;AACF,UAAM,EAAC,gBAAgB,UAAA,IAAa;AAAA,MAClC,IAAI,gBAAgB,EAAE;AAAA,IAAA;AAExB,QAAI,mBAAmB,SAAS,gBAAgB;AAC9C,SAAG;AAAA,QACD,yBAAyB,cAAc,kDAAkD,SAAS,cAAc;AAAA,QAChH;AAAA,MAAA;AAEF,aAAO;AAAA,IACT;AACA,QAAI,YAAY,SAAS,cAAc;AACrC,SAAG;AAAA,QACD,2BAA2B,SAAS,iDAAiD,SAAS,YAAY;AAAA,MAAA;AAE5G,aAAO;AAAA,IACT;AACA,OAAG;AAAA,MACD,4BAA4B,cAAc,kBAAkB,SAAS;AAAA,MACrE;AAAA,IAAA;AAEF,WAAO;AAAA,EACT,UAAA;AACE,OAAG,MAAA;AAAA,EACL;AACF;AAEO,SAAS,0BAA0B,QAAkC;AAC1E,QAAM,EAAC,YAAY,QAAO,cAAc,MAAM;AAC9C,SAAO,MAAM,YAAY,CAAC,WAAW,GAAG;AAAA,IACtC;AAAA,IACA,OAAO;AAAA,IACP,aAAa;AAAA,EAAA,CACd;AACH;AAEA,SAAS,4BACP,IACA,QACA,cACyB;AACzB,QAAM,EAAC,SAAS,QAAQ,SAAS,OAAA,IAAU,SAAA;AAE3C,QAAM,iBAAkB;AACtB,UAAM,QAAQ,IAAI,gBAAA;AAClB,YAAQ,GAAG,UAAU,MAAM,MAAM,OAAO;AACxC,YAAQ,GAAG,WAAW,MAAM,MAAM,OAAO;AAEzC,aAAS,IAAI,KAAK,KAAK;AACrB,UAAI,MAAwB;AAC5B,UAAI;AACF,YAAI,WAAW;AACf,cAAM,SAAS,MAAM,gBAAgB,IAAI,MAAM;AAC/C,yBAAiB,OAAO,QAAQ;AAI9B,kBAAQ,IAAI,CAAC,CAAC;AACd,qBAAW;AAAA,QACb;AAGA,YAAI,UAAU;AACZ;AAAA,QACF;AAAA,MACF,SAAS,GAAG;AACV,cAAM;AAAA,MACR;AACA,UAAI,CAAC,cAAc;AACjB,eAAO,OAAO,GAAG;AAAA,MACnB;AASA,SAAG;AAAA,QACD,uCAAuC,IAAI,CAAC;AAAA,QAC5C,OAAO,GAAG;AAAA,MAAA;AAEZ,UAAI;AACF,cAAM,MAAM,KAAM,MAAM,MAAM;AAAA,MAChC,SAAS,GAAG;AACV,eAAO,OAAO,CAAC;AAAA,MACjB;AAAA,IACF;AAAA,EACF,GAAA;AAEA,SAAO;AACT;AAEA,SAAS,gBACP,IACA,QACkC;AAClC,mBAAiB,MAAM;AACvB,QAAM,EAAC,QAAQ,QAAQ,eAAA,IAAkB;AACzC,QAAM,UAAU,eAAe,MAAM;AAErC,QAAM,uBAAuB,IAAI;AAAA,IAC/B;AAAA,IACA;AAAA,IACA,OAAO;AAAA,IACP,eAAe;AAAA,EAAA;AAGjB,SAAO,qBAAqB,gBAAgB,MAAM;AACpD;"}

package/out/zero-cache/src/services/litestream/config.yml

@@ -6,6 +6,7 @@ dbs:
     watermark-column: stateVersion
     replicas:
       - url: ${ZERO_LITESTREAM_BACKUP_URL}
+        endpoint: ${ZERO_LITESTREAM_ENDPOINT}
         retention: ${ZERO_LITESTREAM_SNAPSHOT_BACKUP_INTERVAL_MINUTES}m
         retention-check-interval: 1h
         sync-interval: ${ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_MINUTES}m

package/out/zero-cache/src/services/mutagen/mutagen.d.ts

@@ -14,7 +14,7 @@ export type MutationError = [
     desc: string
 ];
 export interface Mutagen extends RefCountedService {
-    processMutation(mutation: Mutation, authData: JWTPayload | undefined, schemaVersion: number | undefined, customMutatorsEnabled: boolean): Promise<MutationError | undefined>;
+    processMutation(mutation: Mutation, authData: JWTPayload | undefined, customMutatorsEnabled: boolean): Promise<MutationError | undefined>;
 }
 export declare class MutagenService implements Mutagen, Service {
     #private;
@@ -23,13 +23,13 @@ export declare class MutagenService implements Mutagen, Service {
     ref(): void;
     unref(): void;
     hasRefs(): boolean;
-    processMutation(mutation: Mutation, authData: JWTPayload | undefined, schemaVersion: number | undefined, customMutatorsEnabled?: boolean): Promise<MutationError | undefined>;
+    processMutation(mutation: Mutation, authData: JWTPayload | undefined, customMutatorsEnabled?: boolean): Promise<MutationError | undefined>;
     run(): Promise<void>;
     stop(): Promise<void>;
 }
-export declare function processMutation(lc: LogContext, authData: JWTPayload | undefined, db: PostgresDB, shard: ShardID, clientGroupID: string, mutation: Mutation, writeAuthorizer: WriteAuthorizer, schemaVersion: number | undefined, onTxStart?: () => void | Promise<void>, // for testing
+export declare function processMutation(lc: LogContext, authData: JWTPayload | undefined, db: PostgresDB, shard: ShardID, clientGroupID: string, mutation: Mutation, writeAuthorizer: WriteAuthorizer, onTxStart?: () => void | Promise<void>, // for testing
 customMutatorsEnabled?: boolean): Promise<MutationError | undefined>;
-export declare function processMutationWithTx(lc: LogContext, tx: PostgresTransaction, authData: JWTPayload | undefined, shard: ShardID, clientGroupID: string, schemaVersion: number | undefined, mutation: CRUDMutation, errorMode: boolean, authorizer: WriteAuthorizer): Promise<void>;
+export declare function processMutationWithTx(lc: LogContext, tx: PostgresTransaction, authData: JWTPayload | undefined, shard: ShardID, clientGroupID: string, mutation: CRUDMutation, errorMode: boolean, authorizer: WriteAuthorizer): Promise<void>;
 export declare function getInsertSQL(tx: postgres.TransactionSql, create: InsertOp): postgres.PendingQuery<postgres.Row[]>;
 export declare function getUpsertSQL(tx: postgres.TransactionSql, set: UpsertOp): postgres.PendingQuery<postgres.Row[]>;
 //# sourceMappingURL=mutagen.d.ts.map

package/out/zero-cache/src/services/mutagen/mutagen.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"mutagen.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/mutagen/mutagen.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,MAAM,CAAC;AACrC,OAAO,QAAQ,MAAM,UAAU,CAAC;AAGhC,OAAO,EAAC,SAAS,EAAC,MAAM,6CAA6C,CAAC;AAMtE,OAAO,EACL,KAAK,YAAY,EAEjB,KAAK,QAAQ,EACb,KAAK,QAAQ,EAEb,KAAK,QAAQ,EACd,MAAM,uCAAuC,CAAC;AAE/C,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,4CAA4C,CAAC;AAChF,OAAO,EAEL,KAAK,eAAe,EACrB,MAAM,gCAAgC,CAAC;AACxC,OAAO,EAAC,KAAK,UAAU,EAAC,MAAM,6BAA6B,CAAC;AAI5D,OAAO,KAAK,EAAC,UAAU,EAAE,mBAAmB,EAAC,MAAM,mBAAmB,CAAC;AAEvE,OAAO,EAA4B,KAAK,OAAO,EAAC,MAAM,uBAAuB,CAAC;AAE9E,OAAO,KAAK,EAAC,iBAAiB,EAAE,OAAO,EAAC,MAAM,eAAe,CAAC;AAU9D,MAAM,MAAM,aAAa,GAAG;IAC1B,IAAI,EAAE,SAAS,CAAC,cAAc,GAAG,SAAS,CAAC,mBAAmB;IAC9D,IAAI,EAAE,MAAM;CACb,CAAC;AAEF,MAAM,WAAW,OAAQ,SAAQ,iBAAiB;IAChD,eAAe,CACb,QAAQ,EAAE,QAAQ,EAClB,QAAQ,EAAE,UAAU,GAAG,SAAS,EAChC,aAAa,EAAE,MAAM,GAAG,SAAS,EACjC,qBAAqB,EAAE,OAAO,GAC7B,OAAO,CAAC,aAAa,GAAG,SAAS,CAAC,CAAC;CACvC;AAED,qBAAa,cAAe,YAAW,OAAO,EAAE,OAAO;;IACrD,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;gBAkBlB,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,aAAa,EAAE,MAAM,EACrB,QAAQ,EAAE,UAAU,EACpB,MAAM,EAAE,UAAU,EAClB,iBAAiB,EAAE,eAAe;IA0BpC,GAAG;IAKH,KAAK;IAQL,OAAO,IAAI,OAAO;IAIlB,eAAe,CACb,QAAQ,EAAE,QAAQ,EAClB,QAAQ,EAAE,UAAU,GAAG,SAAS,EAChC,aAAa,EAAE,MAAM,GAAG,SAAS,EACjC,qBAAqB,UAAQ,GAC5B,OAAO,CAAC,aAAa,GAAG,SAAS,CAAC;IAwBrC,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;IAIpB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;CAStB;AAID,wBAAsB,eAAe,CACnC,EAAE,EAAE,UAAU,EACd,QAAQ,EAAE,UAAU,GAAG,SAAS,EAChC,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,aAAa,EAAE,MAAM,EACrB,QAAQ,EAAE,QAAQ,EAClB,eAAe,EAAE,eAAe,EAChC,aAAa,EAAE,MAAM,GAAG,SAAS,EACjC,SAAS,CAAC,EAAE,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAAE,cAAc;AACtD,qBAAqB,UAAQ,GAC5B,OAAO,CAAC,aAAa,GAAG,SAAS,CAAC,CAkIpC;AAED,wBAAsB,qBAAqB,CACzC,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,mBAAmB,EACvB,QAAQ,EAAE,UAAU,GAAG,SAAS,EAChC,KAAK,EAAE,OAAO,EACd,aAAa,EAAE,MAAM,EACrB,aAAa,EAAE,MAAM,GAAG,SAAS,EACjC,QAAQ,EAAE,YAAY,EACtB,SAAS,EAAE,OAAO,EAClB,UAAU,EAAE,eAAe,iBA2D5B;AAED,wBAAgB,YAAY,CAC1B,EAAE,EAAE,QAAQ,CAAC,cAAc,EAC3B,MAAM,EAAE,QAAQ,GACf,QAAQ,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC,CAEvC;AAED,wBAAgB,YAAY,CAC1B,EAAE,EAAE,QAAQ,CAAC,cAAc,EAC3B,GAAG,EAAE,QAAQ,GACZ,QAAQ,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC,CAOvC"}
+{"version":3,"file":"mutagen.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/mutagen/mutagen.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,MAAM,CAAC;AACrC,OAAO,QAAQ,MAAM,UAAU,CAAC;AAGhC,OAAO,EAAC,SAAS,EAAC,MAAM,6CAA6C,CAAC;AAWtE,OAAO,EACL,KAAK,YAAY,EAEjB,KAAK,QAAQ,EACb,KAAK,QAAQ,EAEb,KAAK,QAAQ,EACd,MAAM,uCAAuC,CAAC;AAC/C,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,4CAA4C,CAAC;AAEhF,OAAO,EAEL,KAAK,eAAe,EACrB,MAAM,gCAAgC,CAAC;AACxC,OAAO,EAAC,KAAK,UAAU,EAAC,MAAM,6BAA6B,CAAC;AAI5D,OAAO,KAAK,EAAC,UAAU,EAAE,mBAAmB,EAAC,MAAM,mBAAmB,CAAC;AACvE,OAAO,EAAiB,KAAK,OAAO,EAAC,MAAM,uBAAuB,CAAC;AAEnE,OAAO,KAAK,EAAC,iBAAiB,EAAE,OAAO,EAAC,MAAM,eAAe,CAAC;AAK9D,MAAM,MAAM,aAAa,GAAG;IAC1B,IAAI,EAAE,SAAS,CAAC,cAAc,GAAG,SAAS,CAAC,mBAAmB;IAC9D,IAAI,EAAE,MAAM;CACb,CAAC;AAEF,MAAM,WAAW,OAAQ,SAAQ,iBAAiB;IAChD,eAAe,CACb,QAAQ,EAAE,QAAQ,EAClB,QAAQ,EAAE,UAAU,GAAG,SAAS,EAChC,qBAAqB,EAAE,OAAO,GAC7B,OAAO,CAAC,aAAa,GAAG,SAAS,CAAC,CAAC;CACvC;AAED,qBAAa,cAAe,YAAW,OAAO,EAAE,OAAO;;IACrD,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;gBAkBlB,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,aAAa,EAAE,MAAM,EACrB,QAAQ,EAAE,UAAU,EACpB,MAAM,EAAE,UAAU,EAClB,iBAAiB,EAAE,eAAe;IA0BpC,GAAG;IAKH,KAAK;IAQL,OAAO,IAAI,OAAO;IAIlB,eAAe,CACb,QAAQ,EAAE,QAAQ,EAClB,QAAQ,EAAE,UAAU,GAAG,SAAS,EAChC,qBAAqB,UAAQ,GAC5B,OAAO,CAAC,aAAa,GAAG,SAAS,CAAC;IAuBrC,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;IAIpB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;CAStB;AAID,wBAAsB,eAAe,CACnC,EAAE,EAAE,UAAU,EACd,QAAQ,EAAE,UAAU,GAAG,SAAS,EAChC,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,aAAa,EAAE,MAAM,EACrB,QAAQ,EAAE,QAAQ,EAClB,eAAe,EAAE,eAAe,EAChC,SAAS,CAAC,EAAE,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAAE,cAAc;AACtD,qBAAqB,UAAQ,GAC5B,OAAO,CAAC,aAAa,GAAG,SAAS,CAAC,CAiIpC;AAED,wBAAsB,qBAAqB,CACzC,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,mBAAmB,EACvB,QAAQ,EAAE,UAAU,GAAG,SAAS,EAChC,KAAK,EAAE,OAAO,EACd,aAAa,EAAE,MAAM,EACrB,QAAQ,EAAE,YAAY,EACtB,SAAS,EAAE,OAAO,EAClB,UAAU,EAAE,eAAe,iBA0D5B;AAED,wBAAgB,YAAY,CAC1B,EAAE,EAAE,QAAQ,CAAC,cAAc,EAC3B,MAAM,EAAE,QAAQ,GACf,QAAQ,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC,CAEvC;AAED,wBAAgB,YAAY,CAC1B,EAAE,EAAE,QAAQ,CAAC,cAAc,EAC3B,GAAG,EAAE,QAAQ,GACZ,QAAQ,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC,CAOvC"}

package/out/zero-cache/src/services/mutagen/mutagen.js

@@ -4,6 +4,8 @@ import postgres from "postgres";
 import { assert, unreachable } from "../../../../shared/src/asserts.js";
 import { parse } from "../../../../shared/src/valita.js";
 import { MutationRateLimited, InvalidPush, MutationFailed } from "../../../../zero-protocol/src/error-kind-enum.js";
+import { ZeroCache } from "../../../../zero-protocol/src/error-origin-enum.js";
+import { isProtocolError, ProtocolError } from "../../../../zero-protocol/src/error.js";
 import { CRUD } from "../../../../zero-protocol/src/mutation-type-enum.js";
 import { primaryKeyValueSchema } from "../../../../zero-protocol/src/primary-key.js";
 import "../../../../zero-protocol/src/push.js";
@@ -13,12 +15,9 @@ import "../../config/zero-config.js";
 import { SERIALIZABLE } from "../../db/mode-enum.js";
 import { getOrCreateCounter } from "../../observability/metrics.js";
 import { recordMutation } from "../../server/anonymous-otel-start.js";
-import { throwProtocolErrorIfSchemaVersionNotSupported } from "../../types/schema-versions.js";
-import { upstreamSchema, appSchema } from "../../types/shards.js";
+import { upstreamSchema } from "../../types/shards.js";
 import { SlidingWindowLimiter } from "../limiter/sliding-window-limiter.js";
 import { MutationAlreadyProcessedError } from "./error.js";
-import { isProtocolError, ProtocolError } from "../../../../zero-protocol/src/error.js";
-import { ZeroCache } from "../../../../zero-protocol/src/error-origin-enum.js";
 class MutagenService {
   id;
   #lc;
@@ -72,7 +71,7 @@ class MutagenService {
   hasRefs() {
     return this.#refCount > 0;
   }
-  processMutation(mutation, authData, schemaVersion, customMutatorsEnabled = false) {
+  processMutation(mutation, authData, customMutatorsEnabled = false) {
     if (this.#limiter?.canDo() === false) {
       return Promise.resolve([
         MutationRateLimited,
@@ -90,7 +89,6 @@ class MutagenService {
       this.id,
       mutation,
       this.#writeAuthorizer,
-      schemaVersion,
       void 0,
       customMutatorsEnabled
     );
@@ -109,7 +107,7 @@ class MutagenService {
   }
 }
 const MAX_SERIALIZATION_ATTEMPTS = 10;
-async function processMutation(lc, authData, db, shard, clientGroupID, mutation, writeAuthorizer, schemaVersion, onTxStart, customMutatorsEnabled = false) {
+async function processMutation(lc, authData, db, shard, clientGroupID, mutation, writeAuthorizer, onTxStart, customMutatorsEnabled = false) {
   assert(
     mutation.type === CRUD,
     "Only CRUD mutations are supported"
@@ -133,7 +131,6 @@ async function processMutation(lc, authData, db, shard, clientGroupID, mutation,
               authData,
               shard,
               clientGroupID,
-              schemaVersion,
               mutation,
               errorMode,
               writeAuthorizer
@@ -181,7 +178,7 @@ async function processMutation(lc, authData, db, shard, clientGroupID, mutation,
   }
   return result;
 }
-async function processMutationWithTx(lc, tx, authData, shard, clientGroupID, schemaVersion, mutation, errorMode, authorizer) {
+async function processMutationWithTx(lc, tx, authData, shard, clientGroupID, mutation, errorMode, authorizer) {
   const tasks = [];
   async function execute(stmt) {
     try {
@@ -225,7 +222,6 @@ async function processMutationWithTx(lc, tx, authData, shard, clientGroupID, sch
       tx,
       shard,
       clientGroupID,
-      schemaVersion,
       mutation.clientID,
       mutation.id
     )
@@ -267,19 +263,15 @@ function getDeleteSQL(tx, deleteOp) {
   }
   return tx`DELETE FROM ${tx(tableName)} WHERE ${conditions}`;
 }
-async function checkSchemaVersionAndIncrementLastMutationID(tx, shard, clientGroupID, schemaVersion, clientID, receivedMutationID) {
-  const [[{ lastMutationID }], supportedVersionRange] = await Promise.all([
-    tx`
+async function checkSchemaVersionAndIncrementLastMutationID(tx, shard, clientGroupID, clientID, receivedMutationID) {
+  const [{ lastMutationID }] = await tx`
     INSERT INTO ${tx(upstreamSchema(shard))}.clients 
       as current ("clientGroupID", "clientID", "lastMutationID")
           VALUES (${clientGroupID}, ${clientID}, ${1})
       ON CONFLICT ("clientGroupID", "clientID")
       DO UPDATE SET "lastMutationID" = current."lastMutationID" + 1
       RETURNING "lastMutationID"
-  `,
-    schemaVersion === void 0 ? void 0 : tx`SELECT "minSupportedVersion", "maxSupportedVersion" 
-        FROM ${tx(appSchema(shard))}."schemaVersions"`
-  ]);
+  `;
   if (receivedMutationID < lastMutationID) {
     throw new MutationAlreadyProcessedError(
       clientID,
@@ -293,13 +285,6 @@ async function checkSchemaVersionAndIncrementLastMutationID(tx, shard, clientGro
       origin: ZeroCache
     });
   }
-  if (schemaVersion !== void 0 && supportedVersionRange !== void 0) {
-    assert(supportedVersionRange.length === 1);
-    throwProtocolErrorIfSchemaVersionNotSupported(
-      schemaVersion,
-      supportedVersionRange[0]
-    );
-  }
 }
 export {
   MutagenService,