@rocicorp/zero 1.4.0 → 1.5.0-canary.1

package/out/zero-cache/src/services/change-streamer/change-streamer-http.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"change-streamer-http.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-http.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,WAAW,CAAC;AAE/C,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAIjD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,6BAA6B,CAAC;AAG5D,OAAO,EAAC,KAAK,MAAM,EAAC,MAAM,0BAA0B,CAAC;AACrD,OAAO,EAAC,KAAK,OAAO,EAAC,MAAM,uBAAuB,CAAC;AACnD,OAAO,EAAsB,KAAK,MAAM,EAAC,MAAM,wBAAwB,CAAC;AAIxE,OAAO,EAAC,WAAW,EAAC,MAAM,oBAAoB,CAAC;AAC/C,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,eAAe,CAAC;AAC3C,OAAO,KAAK,EAAC,aAAa,EAAC,MAAM,qBAAqB,CAAC;AACvD,OAAO,EAGL,KAAK,cAAc,EACnB,KAAK,UAAU,EACf,KAAK,iBAAiB,EACvB,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EAAwB,KAAK,eAAe,EAAC,MAAM,eAAe,CAAC;AAW1E,KAAK,OAAO,GAAG;IACb,IAAI,EAAE,MAAM,CAAC;IACb,cAAc,EAAE,MAAM,CAAC;CACxB,CAAC;AAEF,qBAAa,wBAAyB,SAAQ,WAAW;;IACvD,QAAQ,CAAC,EAAE,iCAAiC;gBAO1C,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,UAAU,EAClB,IAAI,EAAE,OAAO,EACb,MAAM,EAAE,MAAM,EACd,cAAc,EAAE,cAAc,GAAG,OAAO,EACxC,aAAa,EAAE,aAAa,GAAG,IAAI;cA+HlB,QAAQ,IAAI,IAAI;cAWV,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;CAKlD;AAED,qBAAa,wBAAyB,YAAW,cAAc;;gBAO3D,EAAE,EAAE,UAAU,EACd,OAAO,EAAE,OAAO,EAChB,QAAQ,EAAE,MAAM,EAChB,iBAAiB,EAAE,MAAM,GAAG,SAAS;IA8BjC,eAAe,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;IASjE,SAAS,CAAC,GAAG,EAAE,iBAAiB,GAAG,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;CAQrE;AAED,KAAK,cAAc,GAAG,IAAI,CAAC,eAAe,EAAE,KAAK,GAAG,SAAS,CAAC,CAAC;AAE/D,wBAAgB,oBAAoB,CAAC,GAAG,EAAE,cAAc,GAAG,iBAAiB,CAc3E"}
+{"version":3,"file":"change-streamer-http.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-http.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,WAAW,CAAC;AAE/C,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAIjD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,6BAA6B,CAAC;AAG5D,OAAO,EAAC,KAAK,MAAM,EAAC,MAAM,0BAA0B,CAAC;AACrD,OAAO,EAAC,KAAK,OAAO,EAAC,MAAM,uBAAuB,CAAC;AACnD,OAAO,EAAsB,KAAK,MAAM,EAAC,MAAM,wBAAwB,CAAC;AAIxE,OAAO,EAAC,WAAW,EAAC,MAAM,oBAAoB,CAAC;AAC/C,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,eAAe,CAAC;AAC3C,OAAO,KAAK,EAAC,aAAa,EAAC,MAAM,qBAAqB,CAAC;AACvD,OAAO,EAGL,KAAK,cAAc,EACnB,KAAK,UAAU,EACf,KAAK,iBAAiB,EACvB,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EAAwB,KAAK,eAAe,EAAC,MAAM,eAAe,CAAC;AAW1E,KAAK,OAAO,GAAG;IACb,IAAI,EAAE,MAAM,CAAC;IACb,kBAAkB,EAAE,MAAM,GAAG,SAAS,CAAC;IACvC,cAAc,EAAE,MAAM,CAAC;CACxB,CAAC;AAEF,qBAAa,wBAAyB,SAAQ,WAAW;;IACvD,QAAQ,CAAC,EAAE,iCAAiC;gBAO1C,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,UAAU,EAClB,IAAI,EAAE,OAAO,EACb,MAAM,EAAE,MAAM,EACd,cAAc,EAAE,cAAc,GAAG,OAAO,EACxC,aAAa,EAAE,aAAa,GAAG,IAAI;cA+HlB,QAAQ,IAAI,IAAI;cAWV,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;CAKlD;AAED,qBAAa,wBAAyB,YAAW,cAAc;;gBAO3D,EAAE,EAAE,UAAU,EACd,OAAO,EAAE,OAAO,EAChB,QAAQ,EAAE,MAAM,EAChB,iBAAiB,EAAE,MAAM,GAAG,SAAS;IA8BjC,eAAe,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;IASjE,SAAS,CAAC,GAAG,EAAE,iBAAiB,GAAG,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;CAQrE;AAED,KAAK,cAAc,GAAG,IAAI,CAAC,eAAe,EAAE,KAAK,GAAG,SAAS,CAAC,CAAC;AAE/D,wBAAgB,oBAAoB,CAAC,GAAG,EAAE,cAAc,GAAG,iBAAiB,CAc3E"}

package/out/zero-cache/src/services/change-streamer/change-streamer-http.js

@@ -10,7 +10,7 @@ import { streamIn, streamOut } from "../../types/streams.js";
 import { URLParams } from "../../types/url-params.js";
 import { downstreamSchema } from "./change-streamer.js";
 import { snapshotMessageSchema } from "./snapshot.js";
-import WebSocket from "ws";
+import WebSocket$1 from "ws";
 import websocket from "@fastify/websocket";
 //#region ../zero-cache/src/services/change-streamer/change-streamer-http.ts
 var MIN_SUPPORTED_PROTOCOL_VERSION = 1;
@@ -121,11 +121,11 @@ var ChangeStreamerHttpClient = class {
 		return uri;
 	}
 	async reserveSnapshot(taskID) {
-		const ws = new WebSocket(await this.#resolveChangeStreamer(SNAPSHOT_PATH) + `?${new URLSearchParams({ taskID }).toString()}`);
+		const ws = new WebSocket$1(await this.#resolveChangeStreamer(SNAPSHOT_PATH) + `?${new URLSearchParams({ taskID }).toString()}`);
 		return streamIn(this.#lc, ws, snapshotMessageSchema);
 	}
 	async subscribe(ctx) {
-		const ws = new WebSocket(await this.#resolveChangeStreamer(CHANGES_PATH) + `?${getParams(ctx).toString()}`);
+		const ws = new WebSocket$1(await this.#resolveChangeStreamer(CHANGES_PATH) + `?${getParams(ctx).toString()}`);
 		return streamIn(this.#lc, ws, downstreamSchema);
 	}
 };

package/out/zero-cache/src/services/change-streamer/change-streamer-http.js.map

@@ -1 +1 @@
-{"version":3,"file":"change-streamer-http.js","names":["#lc","#opts","#changeStreamer","#backupMonitor","#subscribe","#reserveSnapshot","#receiveWebsocket","#getBackupMonitor","#ensureChangeStreamerStarted","#changeStreamerStarted","#shardID","#changeDB","#changeStreamerURI","#resolveChangeStreamer"],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-http.ts"],"sourcesContent":["import type {IncomingMessage} from 'node:http';\nimport websocket from '@fastify/websocket';\nimport type {LogContext} from '@rocicorp/logger';\nimport WebSocket from 'ws';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport type {ZeroConfig} from '../../config/zero-config.ts';\nimport type {IncomingMessageSubset} from '../../types/http.ts';\nimport {pgClient, type PostgresDB} from '../../types/pg.ts';\nimport {type Worker} from '../../types/processes.ts';\nimport {type ShardID} from '../../types/shards.ts';\nimport {streamIn, streamOut, type Source} from '../../types/streams.ts';\nimport {URLParams} from '../../types/url-params.ts';\nimport {installWebSocketReceiver} from '../../types/websocket-handoff.ts';\nimport {closeWithError, PROTOCOL_ERROR} from '../../types/ws.ts';\nimport {HttpService} from '../http-service.ts';\nimport type {Service} from '../service.ts';\nimport type {BackupMonitor} from './backup-monitor.ts';\nimport {\n  downstreamSchema,\n  PROTOCOL_VERSION,\n  type ChangeStreamer,\n  type Downstream,\n  type SubscriberContext,\n} from './change-streamer.ts';\nimport {discoverChangeStreamerAddress} from './schema/tables.ts';\nimport {snapshotMessageSchema, type SnapshotMessage} from './snapshot.ts';\n\nconst MIN_SUPPORTED_PROTOCOL_VERSION = 1;\n\nconst SNAPSHOT_PATH_PATTERN = '/replication/:version/snapshot';\nconst CHANGES_PATH_PATTERN = '/replication/:version/changes';\nconst PATH_REGEX = /\\/replication\\/v(?<version>\\d+)\\/(changes|snapshot)$/;\n\nconst SNAPSHOT_PATH = `/replication/v${PROTOCOL_VERSION}/snapshot`;\nconst CHANGES_PATH = `/replication/v${PROTOCOL_VERSION}/changes`;\n\ntype Options = {\n  port: number;\n  startupDelayMs: number;\n};\n\nexport class ChangeStreamerHttpServer extends HttpService {\n  readonly id = 'change-streamer-http-server';\n  readonly #lc: LogContext;\n  readonly #opts: Options;\n  readonly #changeStreamer: ChangeStreamer & Service;\n  readonly #backupMonitor: BackupMonitor | null;\n\n  constructor(\n    lc: LogContext,\n    config: ZeroConfig,\n    opts: Options,\n    parent: Worker,\n    changeStreamer: ChangeStreamer & Service,\n    backupMonitor: BackupMonitor | null,\n  ) {\n    super('change-streamer-http-server', lc, opts, async fastify => {\n      const websocketOptions: {perMessageDeflate?: boolean | object} = {};\n      if (config.websocketCompression) {\n        if (config.websocketCompressionOptions) {\n          try {\n            websocketOptions.perMessageDeflate = JSON.parse(\n              config.websocketCompressionOptions,\n            );\n          } catch (e) {\n            throw new Error(\n              `Failed to parse ZERO_WEBSOCKET_COMPRESSION_OPTIONS: ${String(e)}. Expected valid JSON.`,\n            );\n          }\n        } else {\n          websocketOptions.perMessageDeflate = true;\n        }\n      }\n\n      await fastify.register(websocket, {\n        options: websocketOptions,\n      });\n\n      fastify.get(CHANGES_PATH_PATTERN, {websocket: true}, this.#subscribe);\n      fastify.get(\n        SNAPSHOT_PATH_PATTERN,\n        {websocket: true},\n        this.#reserveSnapshot,\n      );\n\n      installWebSocketReceiver<'snapshot' | 'changes'>(\n        lc,\n        fastify.websocketServer,\n        this.#receiveWebsocket,\n        parent,\n      );\n    });\n\n    this.#lc = lc;\n    this.#opts = opts;\n    this.#changeStreamer = changeStreamer;\n    this.#backupMonitor = backupMonitor;\n  }\n\n  #getBackupMonitor() {\n    return must(\n      this.#backupMonitor,\n      'replication-manager is not configured with a ZERO_LITESTREAM_BACKUP_URL',\n    );\n  }\n\n  // Called when receiving a web socket via the main dispatcher handoff.\n  readonly #receiveWebsocket = (\n    ws: WebSocket,\n    action: 'changes' | 'snapshot',\n    msg: IncomingMessageSubset,\n  ) => {\n    switch (action) {\n      case 'snapshot':\n        return this.#reserveSnapshot(ws, msg);\n      case 'changes':\n        return this.#subscribe(ws, msg);\n      default:\n        closeWithError(\n          this._lc,\n          ws,\n          `invalid action \"${action}\" received in handoff`,\n        );\n        return;\n    }\n  };\n\n  readonly #reserveSnapshot = (ws: WebSocket, req: RequestHeaders) => {\n    try {\n      const url = new URL(\n        req.url ?? '',\n        req.headers.origin ?? 'http://localhost',\n      );\n      checkProtocolVersion(url.pathname);\n      const taskID = url.searchParams.get('taskID');\n      if (!taskID) {\n        throw new Error('Missing taskID in snapshot request');\n      }\n      const downstream =\n        this.#getBackupMonitor().startSnapshotReservation(taskID);\n      void streamOut(this._lc, downstream, ws);\n    } catch (err) {\n      closeWithError(this._lc, ws, err, PROTOCOL_ERROR);\n    }\n  };\n\n  readonly #subscribe = async (ws: WebSocket, req: RequestHeaders) => {\n    try {\n      const ctx = getSubscriberContext(req);\n      if (ctx.mode === 'serving') {\n        this.#ensureChangeStreamerStarted('incoming subscription');\n      }\n\n      const downstream = await this.#changeStreamer.subscribe(ctx);\n      if (ctx.initial && ctx.taskID && this.#backupMonitor) {\n        // Now that the change-streamer knows about the subscriber and watermark,\n        // end the reservation to safely resume scheduling cleanup.\n        this.#backupMonitor.endReservation(ctx.taskID);\n      }\n      void streamOut(this._lc, downstream, ws);\n    } catch (err) {\n      closeWithError(this._lc, ws, err, PROTOCOL_ERROR);\n    }\n  };\n\n  #changeStreamerStarted = false;\n\n  #ensureChangeStreamerStarted(reason: string) {\n    if (!this.#changeStreamerStarted && this._state.shouldRun()) {\n      this.#lc.info?.(`starting ChangeStreamerService: ${reason}`);\n      void this.#changeStreamer\n        .run()\n        .catch(e =>\n          this.#lc.warn?.(`ChangeStreamerService ended with error`, e),\n        )\n        .finally(() => this.stop());\n\n      this.#changeStreamerStarted = true;\n    }\n  }\n\n  protected override _onStart(): void {\n    const {startupDelayMs} = this.#opts;\n    this._state.setTimeout(\n      () =>\n        this.#ensureChangeStreamerStarted(\n          `startup delay elapsed (${startupDelayMs} ms)`,\n        ),\n      startupDelayMs,\n    );\n  }\n\n  protected override async _onStop(): Promise<void> {\n    if (this.#changeStreamerStarted) {\n      await this.#changeStreamer.stop();\n    }\n  }\n}\n\nexport class ChangeStreamerHttpClient implements ChangeStreamer {\n  readonly #lc: LogContext;\n  readonly #shardID: ShardID;\n  readonly #changeDB: PostgresDB;\n  readonly #changeStreamerURI: string | undefined;\n\n  constructor(\n    lc: LogContext,\n    shardID: ShardID,\n    changeDB: string,\n    changeStreamerURI: string | undefined,\n  ) {\n    this.#lc = lc;\n    this.#shardID = shardID;\n    // Create a pg client with a single short-lived connection for the purpose\n    // of change-streamer discovery (i.e. ChangeDB as DNS).\n    this.#changeDB = pgClient(lc, changeDB, 'change-streamer-discovery', {\n      max: 1,\n      ['idle_timeout']: 15,\n    });\n    this.#changeStreamerURI = changeStreamerURI;\n  }\n\n  async #resolveChangeStreamer(path: string) {\n    let baseURL = this.#changeStreamerURI;\n    if (!baseURL) {\n      const address = await discoverChangeStreamerAddress(\n        this.#shardID,\n        this.#changeDB,\n      );\n      if (!address) {\n        throw new Error(`no change-streamer is running`);\n      }\n      baseURL = address.includes('://') ? `${address}/` : `ws://${address}/`;\n    }\n    const uri = new URL(path, baseURL);\n    this.#lc.info?.(`connecting to change-streamer@${uri}`);\n    return uri;\n  }\n\n  async reserveSnapshot(taskID: string): Promise<Source<SnapshotMessage>> {\n    const uri = await this.#resolveChangeStreamer(SNAPSHOT_PATH);\n\n    const params = new URLSearchParams({taskID});\n    const ws = new WebSocket(uri + `?${params.toString()}`);\n\n    return streamIn(this.#lc, ws, snapshotMessageSchema);\n  }\n\n  async subscribe(ctx: SubscriberContext): Promise<Source<Downstream>> {\n    const uri = await this.#resolveChangeStreamer(CHANGES_PATH);\n\n    const params = getParams(ctx);\n    const ws = new WebSocket(uri + `?${params.toString()}`);\n\n    return streamIn(this.#lc, ws, downstreamSchema);\n  }\n}\n\ntype RequestHeaders = Pick<IncomingMessage, 'url' | 'headers'>;\n\nexport function getSubscriberContext(req: RequestHeaders): SubscriberContext {\n  const url = new URL(req.url ?? '', req.headers.origin ?? 'http://localhost');\n  const protocolVersion = checkProtocolVersion(url.pathname);\n  const params = new URLParams(url);\n\n  return {\n    protocolVersion,\n    id: params.get('id', true),\n    taskID: params.get('taskID', false),\n    mode: params.get('mode', false) === 'backup' ? 'backup' : 'serving',\n    replicaVersion: params.get('replicaVersion', true),\n    watermark: params.get('watermark', true),\n    initial: params.getBoolean('initial'),\n  };\n}\n\nfunction checkProtocolVersion(pathname: string): number {\n  const match = PATH_REGEX.exec(pathname);\n  if (!match) {\n    throw new Error(`invalid path: ${pathname}`);\n  }\n  const v = Number(match.groups?.version);\n  if (\n    Number.isNaN(v) ||\n    v > PROTOCOL_VERSION ||\n    v < MIN_SUPPORTED_PROTOCOL_VERSION\n  ) {\n    throw new Error(\n      `Cannot service client at protocol v${v}. ` +\n        `Supported protocols: [v${MIN_SUPPORTED_PROTOCOL_VERSION} ... v${PROTOCOL_VERSION}]`,\n    );\n  }\n  return v;\n}\n\n// This is called from the client-side (i.e. the replicator).\nfunction getParams(ctx: SubscriberContext): URLSearchParams {\n  // The protocolVersion is hard-coded into the CHANGES_PATH.\n  const {protocolVersion, ...stringParams} = ctx;\n  assert(\n    protocolVersion === PROTOCOL_VERSION,\n    `replicator should be setting protocolVersion to ${PROTOCOL_VERSION}`,\n  );\n  return new URLSearchParams({\n    ...stringParams,\n    taskID: ctx.taskID ? ctx.taskID : '',\n    initial: ctx.initial ? 'true' : 'false',\n  });\n}\n"],"mappings":";;;;;;;;;;;;;;;AA4BA,IAAM,iCAAiC;AAEvC,IAAM,wBAAwB;AAC9B,IAAM,uBAAuB;AAC7B,IAAM,aAAa;AAEnB,IAAM,gBAAgB;AACtB,IAAM,eAAe;AAOrB,IAAa,2BAAb,cAA8C,YAAY;CACxD,KAAc;CACd;CACA;CACA;CACA;CAEA,YACE,IACA,QACA,MACA,QACA,gBACA,eACA;AACA,QAAM,+BAA+B,IAAI,MAAM,OAAM,YAAW;GAC9D,MAAM,mBAA2D,EAAE;AACnE,OAAI,OAAO,qBACT,KAAI,OAAO,4BACT,KAAI;AACF,qBAAiB,oBAAoB,KAAK,MACxC,OAAO,4BACR;YACM,GAAG;AACV,UAAM,IAAI,MACR,uDAAuD,OAAO,EAAE,CAAC,wBAClE;;OAGH,kBAAiB,oBAAoB;AAIzC,SAAM,QAAQ,SAAS,WAAW,EAChC,SAAS,kBACV,CAAC;AAEF,WAAQ,IAAI,sBAAsB,EAAC,WAAW,MAAK,EAAE,MAAA,UAAgB;AACrE,WAAQ,IACN,uBACA,EAAC,WAAW,MAAK,EACjB,MAAA,gBACD;AAED,4BACE,IACA,QAAQ,iBACR,MAAA,kBACA,OACD;IACD;AAEF,QAAA,KAAW;AACX,QAAA,OAAa;AACb,QAAA,iBAAuB;AACvB,QAAA,gBAAsB;;CAGxB,oBAAoB;AAClB,SAAO,KACL,MAAA,eACA,0EACD;;CAIH,qBACE,IACA,QACA,QACG;AACH,UAAQ,QAAR;GACE,KAAK,WACH,QAAO,MAAA,gBAAsB,IAAI,IAAI;GACvC,KAAK,UACH,QAAO,MAAA,UAAgB,IAAI,IAAI;GACjC;AACE,mBACE,KAAK,KACL,IACA,mBAAmB,OAAO,uBAC3B;AACD;;;CAIN,oBAA6B,IAAe,QAAwB;AAClE,MAAI;GACF,MAAM,MAAM,IAAI,IACd,IAAI,OAAO,IACX,IAAI,QAAQ,UAAU,mBACvB;AACD,wBAAqB,IAAI,SAAS;GAClC,MAAM,SAAS,IAAI,aAAa,IAAI,SAAS;AAC7C,OAAI,CAAC,OACH,OAAM,IAAI,MAAM,qCAAqC;GAEvD,MAAM,aACJ,MAAA,kBAAwB,CAAC,yBAAyB,OAAO;AACtD,aAAU,KAAK,KAAK,YAAY,GAAG;WACjC,KAAK;AACZ,kBAAe,KAAK,KAAK,IAAI,KAAK,eAAe;;;CAIrD,aAAsB,OAAO,IAAe,QAAwB;AAClE,MAAI;GACF,MAAM,MAAM,qBAAqB,IAAI;AACrC,OAAI,IAAI,SAAS,UACf,OAAA,4BAAkC,wBAAwB;GAG5D,MAAM,aAAa,MAAM,MAAA,eAAqB,UAAU,IAAI;AAC5D,OAAI,IAAI,WAAW,IAAI,UAAU,MAAA,cAG/B,OAAA,cAAoB,eAAe,IAAI,OAAO;AAE3C,aAAU,KAAK,KAAK,YAAY,GAAG;WACjC,KAAK;AACZ,kBAAe,KAAK,KAAK,IAAI,KAAK,eAAe;;;CAIrD,yBAAyB;CAEzB,6BAA6B,QAAgB;AAC3C,MAAI,CAAC,MAAA,yBAA+B,KAAK,OAAO,WAAW,EAAE;AAC3D,SAAA,GAAS,OAAO,mCAAmC,SAAS;AACvD,SAAA,eACF,KAAK,CACL,OAAM,MACL,MAAA,GAAS,OAAO,0CAA0C,EAAE,CAC7D,CACA,cAAc,KAAK,MAAM,CAAC;AAE7B,SAAA,wBAA8B;;;CAIlC,WAAoC;EAClC,MAAM,EAAC,mBAAkB,MAAA;AACzB,OAAK,OAAO,iBAER,MAAA,4BACE,0BAA0B,eAAe,MAC1C,EACH,eACD;;CAGH,MAAyB,UAAyB;AAChD,MAAI,MAAA,sBACF,OAAM,MAAA,eAAqB,MAAM;;;AAKvC,IAAa,2BAAb,MAAgE;CAC9D;CACA;CACA;CACA;CAEA,YACE,IACA,SACA,UACA,mBACA;AACA,QAAA,KAAW;AACX,QAAA,UAAgB;AAGhB,QAAA,WAAiB,SAAS,IAAI,UAAU,6BAA6B;GACnE,KAAK;IACJ,iBAAiB;GACnB,CAAC;AACF,QAAA,oBAA0B;;CAG5B,OAAA,sBAA6B,MAAc;EACzC,IAAI,UAAU,MAAA;AACd,MAAI,CAAC,SAAS;GACZ,MAAM,UAAU,MAAM,8BACpB,MAAA,SACA,MAAA,SACD;AACD,OAAI,CAAC,QACH,OAAM,IAAI,MAAM,gCAAgC;AAElD,aAAU,QAAQ,SAAS,MAAM,GAAG,GAAG,QAAQ,KAAK,QAAQ,QAAQ;;EAEtE,MAAM,MAAM,IAAI,IAAI,MAAM,QAAQ;AAClC,QAAA,GAAS,OAAO,iCAAiC,MAAM;AACvD,SAAO;;CAGT,MAAM,gBAAgB,QAAkD;EAItE,MAAM,KAAK,IAAI,UAHH,MAAM,MAAA,sBAA4B,cAAc,GAG7B,IADhB,IAAI,gBAAgB,EAAC,QAAO,CAAC,CACF,UAAU,GAAG;AAEvD,SAAO,SAAS,MAAA,IAAU,IAAI,sBAAsB;;CAGtD,MAAM,UAAU,KAAqD;EAInE,MAAM,KAAK,IAAI,UAHH,MAAM,MAAA,sBAA4B,aAAa,GAG5B,IADhB,UAAU,IAAI,CACa,UAAU,GAAG;AAEvD,SAAO,SAAS,MAAA,IAAU,IAAI,iBAAiB;;;AAMnD,SAAgB,qBAAqB,KAAwC;CAC3E,MAAM,MAAM,IAAI,IAAI,IAAI,OAAO,IAAI,IAAI,QAAQ,UAAU,mBAAmB;CAC5E,MAAM,kBAAkB,qBAAqB,IAAI,SAAS;CAC1D,MAAM,SAAS,IAAI,UAAU,IAAI;AAEjC,QAAO;EACL;EACA,IAAI,OAAO,IAAI,MAAM,KAAK;EAC1B,QAAQ,OAAO,IAAI,UAAU,MAAM;EACnC,MAAM,OAAO,IAAI,QAAQ,MAAM,KAAK,WAAW,WAAW;EAC1D,gBAAgB,OAAO,IAAI,kBAAkB,KAAK;EAClD,WAAW,OAAO,IAAI,aAAa,KAAK;EACxC,SAAS,OAAO,WAAW,UAAU;EACtC;;AAGH,SAAS,qBAAqB,UAA0B;CACtD,MAAM,QAAQ,WAAW,KAAK,SAAS;AACvC,KAAI,CAAC,MACH,OAAM,IAAI,MAAM,iBAAiB,WAAW;CAE9C,MAAM,IAAI,OAAO,MAAM,QAAQ,QAAQ;AACvC,KACE,OAAO,MAAM,EAAE,IACf,IAAA,KACA,IAAI,+BAEJ,OAAM,IAAI,MACR,sCAAsC,EAAE,2BACZ,+BAA+B,UAC5D;AAEH,QAAO;;AAIT,SAAS,UAAU,KAAyC;CAE1D,MAAM,EAAC,iBAAiB,GAAG,iBAAgB;AAC3C,QACE,oBAAA,GACA,oDACD;AACD,QAAO,IAAI,gBAAgB;EACzB,GAAG;EACH,QAAQ,IAAI,SAAS,IAAI,SAAS;EAClC,SAAS,IAAI,UAAU,SAAS;EACjC,CAAC"}
+{"version":3,"file":"change-streamer-http.js","names":["#lc","#opts","#changeStreamer","#backupMonitor","#subscribe","#reserveSnapshot","#receiveWebsocket","#getBackupMonitor","#ensureChangeStreamerStarted","#changeStreamerStarted","#shardID","#changeDB","#changeStreamerURI","#resolveChangeStreamer"],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-http.ts"],"sourcesContent":["import type {IncomingMessage} from 'node:http';\nimport websocket from '@fastify/websocket';\nimport type {LogContext} from '@rocicorp/logger';\nimport WebSocket from 'ws';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport type {ZeroConfig} from '../../config/zero-config.ts';\nimport type {IncomingMessageSubset} from '../../types/http.ts';\nimport {pgClient, type PostgresDB} from '../../types/pg.ts';\nimport {type Worker} from '../../types/processes.ts';\nimport {type ShardID} from '../../types/shards.ts';\nimport {streamIn, streamOut, type Source} from '../../types/streams.ts';\nimport {URLParams} from '../../types/url-params.ts';\nimport {installWebSocketReceiver} from '../../types/websocket-handoff.ts';\nimport {closeWithError, PROTOCOL_ERROR} from '../../types/ws.ts';\nimport {HttpService} from '../http-service.ts';\nimport type {Service} from '../service.ts';\nimport type {BackupMonitor} from './backup-monitor.ts';\nimport {\n  downstreamSchema,\n  PROTOCOL_VERSION,\n  type ChangeStreamer,\n  type Downstream,\n  type SubscriberContext,\n} from './change-streamer.ts';\nimport {discoverChangeStreamerAddress} from './schema/tables.ts';\nimport {snapshotMessageSchema, type SnapshotMessage} from './snapshot.ts';\n\nconst MIN_SUPPORTED_PROTOCOL_VERSION = 1;\n\nconst SNAPSHOT_PATH_PATTERN = '/replication/:version/snapshot';\nconst CHANGES_PATH_PATTERN = '/replication/:version/changes';\nconst PATH_REGEX = /\\/replication\\/v(?<version>\\d+)\\/(changes|snapshot)$/;\n\nconst SNAPSHOT_PATH = `/replication/v${PROTOCOL_VERSION}/snapshot`;\nconst CHANGES_PATH = `/replication/v${PROTOCOL_VERSION}/changes`;\n\ntype Options = {\n  port: number;\n  keepaliveTimeoutMs: number | undefined;\n  startupDelayMs: number;\n};\n\nexport class ChangeStreamerHttpServer extends HttpService {\n  readonly id = 'change-streamer-http-server';\n  readonly #lc: LogContext;\n  readonly #opts: Options;\n  readonly #changeStreamer: ChangeStreamer & Service;\n  readonly #backupMonitor: BackupMonitor | null;\n\n  constructor(\n    lc: LogContext,\n    config: ZeroConfig,\n    opts: Options,\n    parent: Worker,\n    changeStreamer: ChangeStreamer & Service,\n    backupMonitor: BackupMonitor | null,\n  ) {\n    super('change-streamer-http-server', lc, opts, async fastify => {\n      const websocketOptions: {perMessageDeflate?: boolean | object} = {};\n      if (config.websocketCompression) {\n        if (config.websocketCompressionOptions) {\n          try {\n            websocketOptions.perMessageDeflate = JSON.parse(\n              config.websocketCompressionOptions,\n            );\n          } catch (e) {\n            throw new Error(\n              `Failed to parse ZERO_WEBSOCKET_COMPRESSION_OPTIONS: ${String(e)}. Expected valid JSON.`,\n            );\n          }\n        } else {\n          websocketOptions.perMessageDeflate = true;\n        }\n      }\n\n      await fastify.register(websocket, {\n        options: websocketOptions,\n      });\n\n      fastify.get(CHANGES_PATH_PATTERN, {websocket: true}, this.#subscribe);\n      fastify.get(\n        SNAPSHOT_PATH_PATTERN,\n        {websocket: true},\n        this.#reserveSnapshot,\n      );\n\n      installWebSocketReceiver<'snapshot' | 'changes'>(\n        lc,\n        fastify.websocketServer,\n        this.#receiveWebsocket,\n        parent,\n      );\n    });\n\n    this.#lc = lc;\n    this.#opts = opts;\n    this.#changeStreamer = changeStreamer;\n    this.#backupMonitor = backupMonitor;\n  }\n\n  #getBackupMonitor() {\n    return must(\n      this.#backupMonitor,\n      'replication-manager is not configured with a ZERO_LITESTREAM_BACKUP_URL',\n    );\n  }\n\n  // Called when receiving a web socket via the main dispatcher handoff.\n  readonly #receiveWebsocket = (\n    ws: WebSocket,\n    action: 'changes' | 'snapshot',\n    msg: IncomingMessageSubset,\n  ) => {\n    switch (action) {\n      case 'snapshot':\n        return this.#reserveSnapshot(ws, msg);\n      case 'changes':\n        return this.#subscribe(ws, msg);\n      default:\n        closeWithError(\n          this._lc,\n          ws,\n          `invalid action \"${action}\" received in handoff`,\n        );\n        return;\n    }\n  };\n\n  readonly #reserveSnapshot = (ws: WebSocket, req: RequestHeaders) => {\n    try {\n      const url = new URL(\n        req.url ?? '',\n        req.headers.origin ?? 'http://localhost',\n      );\n      checkProtocolVersion(url.pathname);\n      const taskID = url.searchParams.get('taskID');\n      if (!taskID) {\n        throw new Error('Missing taskID in snapshot request');\n      }\n      const downstream =\n        this.#getBackupMonitor().startSnapshotReservation(taskID);\n      void streamOut(this._lc, downstream, ws);\n    } catch (err) {\n      closeWithError(this._lc, ws, err, PROTOCOL_ERROR);\n    }\n  };\n\n  readonly #subscribe = async (ws: WebSocket, req: RequestHeaders) => {\n    try {\n      const ctx = getSubscriberContext(req);\n      if (ctx.mode === 'serving') {\n        this.#ensureChangeStreamerStarted('incoming subscription');\n      }\n\n      const downstream = await this.#changeStreamer.subscribe(ctx);\n      if (ctx.initial && ctx.taskID && this.#backupMonitor) {\n        // Now that the change-streamer knows about the subscriber and watermark,\n        // end the reservation to safely resume scheduling cleanup.\n        this.#backupMonitor.endReservation(ctx.taskID);\n      }\n      void streamOut(this._lc, downstream, ws);\n    } catch (err) {\n      closeWithError(this._lc, ws, err, PROTOCOL_ERROR);\n    }\n  };\n\n  #changeStreamerStarted = false;\n\n  #ensureChangeStreamerStarted(reason: string) {\n    if (!this.#changeStreamerStarted && this._state.shouldRun()) {\n      this.#lc.info?.(`starting ChangeStreamerService: ${reason}`);\n      void this.#changeStreamer\n        .run()\n        .catch(e =>\n          this.#lc.warn?.(`ChangeStreamerService ended with error`, e),\n        )\n        .finally(() => this.stop());\n\n      this.#changeStreamerStarted = true;\n    }\n  }\n\n  protected override _onStart(): void {\n    const {startupDelayMs} = this.#opts;\n    this._state.setTimeout(\n      () =>\n        this.#ensureChangeStreamerStarted(\n          `startup delay elapsed (${startupDelayMs} ms)`,\n        ),\n      startupDelayMs,\n    );\n  }\n\n  protected override async _onStop(): Promise<void> {\n    if (this.#changeStreamerStarted) {\n      await this.#changeStreamer.stop();\n    }\n  }\n}\n\nexport class ChangeStreamerHttpClient implements ChangeStreamer {\n  readonly #lc: LogContext;\n  readonly #shardID: ShardID;\n  readonly #changeDB: PostgresDB;\n  readonly #changeStreamerURI: string | undefined;\n\n  constructor(\n    lc: LogContext,\n    shardID: ShardID,\n    changeDB: string,\n    changeStreamerURI: string | undefined,\n  ) {\n    this.#lc = lc;\n    this.#shardID = shardID;\n    // Create a pg client with a single short-lived connection for the purpose\n    // of change-streamer discovery (i.e. ChangeDB as DNS).\n    this.#changeDB = pgClient(lc, changeDB, 'change-streamer-discovery', {\n      max: 1,\n      ['idle_timeout']: 15,\n    });\n    this.#changeStreamerURI = changeStreamerURI;\n  }\n\n  async #resolveChangeStreamer(path: string) {\n    let baseURL = this.#changeStreamerURI;\n    if (!baseURL) {\n      const address = await discoverChangeStreamerAddress(\n        this.#shardID,\n        this.#changeDB,\n      );\n      if (!address) {\n        throw new Error(`no change-streamer is running`);\n      }\n      baseURL = address.includes('://') ? `${address}/` : `ws://${address}/`;\n    }\n    const uri = new URL(path, baseURL);\n    this.#lc.info?.(`connecting to change-streamer@${uri}`);\n    return uri;\n  }\n\n  async reserveSnapshot(taskID: string): Promise<Source<SnapshotMessage>> {\n    const uri = await this.#resolveChangeStreamer(SNAPSHOT_PATH);\n\n    const params = new URLSearchParams({taskID});\n    const ws = new WebSocket(uri + `?${params.toString()}`);\n\n    return streamIn(this.#lc, ws, snapshotMessageSchema);\n  }\n\n  async subscribe(ctx: SubscriberContext): Promise<Source<Downstream>> {\n    const uri = await this.#resolveChangeStreamer(CHANGES_PATH);\n\n    const params = getParams(ctx);\n    const ws = new WebSocket(uri + `?${params.toString()}`);\n\n    return streamIn(this.#lc, ws, downstreamSchema);\n  }\n}\n\ntype RequestHeaders = Pick<IncomingMessage, 'url' | 'headers'>;\n\nexport function getSubscriberContext(req: RequestHeaders): SubscriberContext {\n  const url = new URL(req.url ?? '', req.headers.origin ?? 'http://localhost');\n  const protocolVersion = checkProtocolVersion(url.pathname);\n  const params = new URLParams(url);\n\n  return {\n    protocolVersion,\n    id: params.get('id', true),\n    taskID: params.get('taskID', false),\n    mode: params.get('mode', false) === 'backup' ? 'backup' : 'serving',\n    replicaVersion: params.get('replicaVersion', true),\n    watermark: params.get('watermark', true),\n    initial: params.getBoolean('initial'),\n  };\n}\n\nfunction checkProtocolVersion(pathname: string): number {\n  const match = PATH_REGEX.exec(pathname);\n  if (!match) {\n    throw new Error(`invalid path: ${pathname}`);\n  }\n  const v = Number(match.groups?.version);\n  if (\n    Number.isNaN(v) ||\n    v > PROTOCOL_VERSION ||\n    v < MIN_SUPPORTED_PROTOCOL_VERSION\n  ) {\n    throw new Error(\n      `Cannot service client at protocol v${v}. ` +\n        `Supported protocols: [v${MIN_SUPPORTED_PROTOCOL_VERSION} ... v${PROTOCOL_VERSION}]`,\n    );\n  }\n  return v;\n}\n\n// This is called from the client-side (i.e. the replicator).\nfunction getParams(ctx: SubscriberContext): URLSearchParams {\n  // The protocolVersion is hard-coded into the CHANGES_PATH.\n  const {protocolVersion, ...stringParams} = ctx;\n  assert(\n    protocolVersion === PROTOCOL_VERSION,\n    `replicator should be setting protocolVersion to ${PROTOCOL_VERSION}`,\n  );\n  return new URLSearchParams({\n    ...stringParams,\n    taskID: ctx.taskID ? ctx.taskID : '',\n    initial: ctx.initial ? 'true' : 'false',\n  });\n}\n"],"mappings":";;;;;;;;;;;;;;;AA4BA,IAAM,iCAAiC;AAEvC,IAAM,wBAAwB;AAC9B,IAAM,uBAAuB;AAC7B,IAAM,aAAa;AAEnB,IAAM,gBAAgB;AACtB,IAAM,eAAe;AAQrB,IAAa,2BAAb,cAA8C,YAAY;CACxD,KAAc;CACd;CACA;CACA;CACA;CAEA,YACE,IACA,QACA,MACA,QACA,gBACA,eACA;AACA,QAAM,+BAA+B,IAAI,MAAM,OAAM,YAAW;GAC9D,MAAM,mBAA2D,EAAE;AACnE,OAAI,OAAO,qBACT,KAAI,OAAO,4BACT,KAAI;AACF,qBAAiB,oBAAoB,KAAK,MACxC,OAAO,4BACR;YACM,GAAG;AACV,UAAM,IAAI,MACR,uDAAuD,OAAO,EAAE,CAAC,wBAClE;;OAGH,kBAAiB,oBAAoB;AAIzC,SAAM,QAAQ,SAAS,WAAW,EAChC,SAAS,kBACV,CAAC;AAEF,WAAQ,IAAI,sBAAsB,EAAC,WAAW,MAAK,EAAE,MAAA,UAAgB;AACrE,WAAQ,IACN,uBACA,EAAC,WAAW,MAAK,EACjB,MAAA,gBACD;AAED,4BACE,IACA,QAAQ,iBACR,MAAA,kBACA,OACD;IACD;AAEF,QAAA,KAAW;AACX,QAAA,OAAa;AACb,QAAA,iBAAuB;AACvB,QAAA,gBAAsB;;CAGxB,oBAAoB;AAClB,SAAO,KACL,MAAA,eACA,0EACD;;CAIH,qBACE,IACA,QACA,QACG;AACH,UAAQ,QAAR;GACE,KAAK,WACH,QAAO,MAAA,gBAAsB,IAAI,IAAI;GACvC,KAAK,UACH,QAAO,MAAA,UAAgB,IAAI,IAAI;GACjC;AACE,mBACE,KAAK,KACL,IACA,mBAAmB,OAAO,uBAC3B;AACD;;;CAIN,oBAA6B,IAAe,QAAwB;AAClE,MAAI;GACF,MAAM,MAAM,IAAI,IACd,IAAI,OAAO,IACX,IAAI,QAAQ,UAAU,mBACvB;AACD,wBAAqB,IAAI,SAAS;GAClC,MAAM,SAAS,IAAI,aAAa,IAAI,SAAS;AAC7C,OAAI,CAAC,OACH,OAAM,IAAI,MAAM,qCAAqC;GAEvD,MAAM,aACJ,MAAA,kBAAwB,CAAC,yBAAyB,OAAO;AACtD,aAAU,KAAK,KAAK,YAAY,GAAG;WACjC,KAAK;AACZ,kBAAe,KAAK,KAAK,IAAI,KAAK,eAAe;;;CAIrD,aAAsB,OAAO,IAAe,QAAwB;AAClE,MAAI;GACF,MAAM,MAAM,qBAAqB,IAAI;AACrC,OAAI,IAAI,SAAS,UACf,OAAA,4BAAkC,wBAAwB;GAG5D,MAAM,aAAa,MAAM,MAAA,eAAqB,UAAU,IAAI;AAC5D,OAAI,IAAI,WAAW,IAAI,UAAU,MAAA,cAG/B,OAAA,cAAoB,eAAe,IAAI,OAAO;AAE3C,aAAU,KAAK,KAAK,YAAY,GAAG;WACjC,KAAK;AACZ,kBAAe,KAAK,KAAK,IAAI,KAAK,eAAe;;;CAIrD,yBAAyB;CAEzB,6BAA6B,QAAgB;AAC3C,MAAI,CAAC,MAAA,yBAA+B,KAAK,OAAO,WAAW,EAAE;AAC3D,SAAA,GAAS,OAAO,mCAAmC,SAAS;AACvD,SAAA,eACF,KAAK,CACL,OAAM,MACL,MAAA,GAAS,OAAO,0CAA0C,EAAE,CAC7D,CACA,cAAc,KAAK,MAAM,CAAC;AAE7B,SAAA,wBAA8B;;;CAIlC,WAAoC;EAClC,MAAM,EAAC,mBAAkB,MAAA;AACzB,OAAK,OAAO,iBAER,MAAA,4BACE,0BAA0B,eAAe,MAC1C,EACH,eACD;;CAGH,MAAyB,UAAyB;AAChD,MAAI,MAAA,sBACF,OAAM,MAAA,eAAqB,MAAM;;;AAKvC,IAAa,2BAAb,MAAgE;CAC9D;CACA;CACA;CACA;CAEA,YACE,IACA,SACA,UACA,mBACA;AACA,QAAA,KAAW;AACX,QAAA,UAAgB;AAGhB,QAAA,WAAiB,SAAS,IAAI,UAAU,6BAA6B;GACnE,KAAK;IACJ,iBAAiB;GACnB,CAAC;AACF,QAAA,oBAA0B;;CAG5B,OAAA,sBAA6B,MAAc;EACzC,IAAI,UAAU,MAAA;AACd,MAAI,CAAC,SAAS;GACZ,MAAM,UAAU,MAAM,8BACpB,MAAA,SACA,MAAA,SACD;AACD,OAAI,CAAC,QACH,OAAM,IAAI,MAAM,gCAAgC;AAElD,aAAU,QAAQ,SAAS,MAAM,GAAG,GAAG,QAAQ,KAAK,QAAQ,QAAQ;;EAEtE,MAAM,MAAM,IAAI,IAAI,MAAM,QAAQ;AAClC,QAAA,GAAS,OAAO,iCAAiC,MAAM;AACvD,SAAO;;CAGT,MAAM,gBAAgB,QAAkD;EAItE,MAAM,KAAK,IAAI,YAHH,MAAM,MAAA,sBAA4B,cAAc,GAG7B,IADhB,IAAI,gBAAgB,EAAC,QAAO,CAAC,CACF,UAAU,GAAG;AAEvD,SAAO,SAAS,MAAA,IAAU,IAAI,sBAAsB;;CAGtD,MAAM,UAAU,KAAqD;EAInE,MAAM,KAAK,IAAI,YAHH,MAAM,MAAA,sBAA4B,aAAa,GAG5B,IADhB,UAAU,IAAI,CACa,UAAU,GAAG;AAEvD,SAAO,SAAS,MAAA,IAAU,IAAI,iBAAiB;;;AAMnD,SAAgB,qBAAqB,KAAwC;CAC3E,MAAM,MAAM,IAAI,IAAI,IAAI,OAAO,IAAI,IAAI,QAAQ,UAAU,mBAAmB;CAC5E,MAAM,kBAAkB,qBAAqB,IAAI,SAAS;CAC1D,MAAM,SAAS,IAAI,UAAU,IAAI;AAEjC,QAAO;EACL;EACA,IAAI,OAAO,IAAI,MAAM,KAAK;EAC1B,QAAQ,OAAO,IAAI,UAAU,MAAM;EACnC,MAAM,OAAO,IAAI,QAAQ,MAAM,KAAK,WAAW,WAAW;EAC1D,gBAAgB,OAAO,IAAI,kBAAkB,KAAK;EAClD,WAAW,OAAO,IAAI,aAAa,KAAK;EACxC,SAAS,OAAO,WAAW,UAAU;EACtC;;AAGH,SAAS,qBAAqB,UAA0B;CACtD,MAAM,QAAQ,WAAW,KAAK,SAAS;AACvC,KAAI,CAAC,MACH,OAAM,IAAI,MAAM,iBAAiB,WAAW;CAE9C,MAAM,IAAI,OAAO,MAAM,QAAQ,QAAQ;AACvC,KACE,OAAO,MAAM,EAAE,IACf,IAAA,KACA,IAAI,+BAEJ,OAAM,IAAI,MACR,sCAAsC,EAAE,2BACZ,+BAA+B,UAC5D;AAEH,QAAO;;AAIT,SAAS,UAAU,KAAyC;CAE1D,MAAM,EAAC,iBAAiB,GAAG,iBAAgB;AAC3C,QACE,oBAAA,GACA,oDACD;AACD,QAAO,IAAI,gBAAgB;EACzB,GAAG;EACH,QAAQ,IAAI,SAAS,IAAI,SAAS;EAClC,SAAS,IAAI,UAAU,SAAS;EACjC,CAAC"}

package/out/zero-cache/src/services/http-service.d.ts

@@ -4,6 +4,7 @@ import { RunningState } from './running-state.ts';
 import type { Service } from './service.ts';
 export type Options = {
     port: number;
+    keepaliveTimeoutMs: number | undefined;
 };
 /**
  * Common functionality for all HttpServices. These include:

package/out/zero-cache/src/services/http-service.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"http-service.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/services/http-service.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAgB,EAAC,KAAK,eAAe,EAAC,MAAM,SAAS,CAAC;AAGtD,OAAO,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAC;AAChD,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,cAAc,CAAC;AAE1C,MAAM,MAAM,OAAO,GAAG;IACpB,IAAI,EAAE,MAAM,CAAC;CACd,CAAC;AAEF;;;;GAIG;AACH,qBAAa,WAAY,YAAW,OAAO;;IACzC,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,QAAQ,CAAC,GAAG,EAAE,UAAU,CAAC;IAGnC,SAAS,CAAC,QAAQ,CAAC,MAAM,EAAE,YAAY,CAAC;gBAKtC,EAAE,EAAE,MAAM,EACV,EAAE,EAAE,UAAU,EACd,IAAI,EAAE,OAAO,EACb,IAAI,EAAE,CAAC,OAAO,EAAE,eAAe,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAY1D,SAAS,CAAC,QAAQ;IAClB,SAAS,CAAC,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAK5B,KAAK,IAAI,OAAO,CAAC,MAAM,CAAC;IAgBxB,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;IAKpB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;CAO5B"}
+{"version":3,"file":"http-service.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/services/http-service.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAgB,EAAC,KAAK,eAAe,EAAC,MAAM,SAAS,CAAC;AAGtD,OAAO,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAC;AAChD,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,cAAc,CAAC;AAE1C,MAAM,MAAM,OAAO,GAAG;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,kBAAkB,EAAE,MAAM,GAAG,SAAS,CAAC;CACxC,CAAC;AAEF;;;;GAIG;AACH,qBAAa,WAAY,YAAW,OAAO;;IACzC,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,QAAQ,CAAC,GAAG,EAAE,UAAU,CAAC;IAGnC,SAAS,CAAC,QAAQ,CAAC,MAAM,EAAE,YAAY,CAAC;gBAKtC,EAAE,EAAE,MAAM,EACV,EAAE,EAAE,UAAU,EACd,IAAI,EAAE,OAAO,EACb,IAAI,EAAE,CAAC,OAAO,EAAE,eAAe,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAe1D,SAAS,CAAC,QAAQ;IAClB,SAAS,CAAC,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAK5B,KAAK,IAAI,OAAO,CAAC,MAAM,CAAC;IAgBxB,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;IAKpB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;CAO5B"}

package/out/zero-cache/src/services/http-service.js

@@ -17,13 +17,14 @@ var HttpService = class {
 	#heartbeatMonitor;
 	#init;
 	constructor(id, lc, opts, init) {
+		const { port, keepaliveTimeoutMs } = opts;
 		this.id = id;
 		this._lc = lc.withContext("component", this.id);
 		this.#fastify = Fastify();
-		this.#port = opts.port;
+		this.#port = port;
 		this.#init = init;
 		this._state = new RunningState(id);
-		this.#heartbeatMonitor = new HeartbeatMonitor(this._lc);
+		this.#heartbeatMonitor = keepaliveTimeoutMs ? new HeartbeatMonitor(this._lc, keepaliveTimeoutMs) : void 0;
 	}
 	_onStart() {}
 	_onStop() {
@@ -32,7 +33,7 @@ var HttpService = class {
 	async start() {
 		this.#fastify.get("/", (_req, res) => res.send("OK"));
 		this.#fastify.get("/keepalive", ({ headers }, res) => {
-			this.#heartbeatMonitor.onHeartbeat(headers);
+			this.#heartbeatMonitor?.onHeartbeat(headers);
 			return res.send("OK");
 		});
 		await this.#init(this.#fastify);
@@ -50,7 +51,7 @@ var HttpService = class {
 	}
 	async stop() {
 		this._lc.info?.(`${this.id}: no longer accepting connections`);
-		this.#heartbeatMonitor.stop();
+		this.#heartbeatMonitor?.stop();
 		this._state.stop(this._lc);
 		await this.#fastify.close();
 		await this._onStop();

package/out/zero-cache/src/services/http-service.js.map

@@ -1 +1 @@
-{"version":3,"file":"http-service.js","names":["#fastify","#port","#heartbeatMonitor","#init"],"sources":["../../../../../zero-cache/src/services/http-service.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport Fastify, {type FastifyInstance} from 'fastify';\nimport {promiseVoid} from '../../../shared/src/resolved-promises.ts';\nimport {HeartbeatMonitor} from './life-cycle.ts';\nimport {RunningState} from './running-state.ts';\nimport type {Service} from './service.ts';\n\nexport type Options = {\n  port: number;\n};\n\n/**\n * Common functionality for all HttpServices. These include:\n * * Responding to health checks at \"/\"\n * * Tracking optional heartbeats at \"/keepalive\" and draining when they stop.\n */\nexport class HttpService implements Service {\n  readonly id: string;\n  protected readonly _lc: LogContext;\n  readonly #fastify: FastifyInstance;\n  readonly #port: number;\n  protected readonly _state: RunningState;\n  readonly #heartbeatMonitor: HeartbeatMonitor;\n  readonly #init: (fastify: FastifyInstance) => void | Promise<void>;\n\n  constructor(\n    id: string,\n    lc: LogContext,\n    opts: Options,\n    init: (fastify: FastifyInstance) => void | Promise<void>,\n  ) {\n    this.id = id;\n    this._lc = lc.withContext('component', this.id);\n    this.#fastify = Fastify();\n    this.#port = opts.port;\n    this.#init = init;\n    this._state = new RunningState(id);\n    this.#heartbeatMonitor = new HeartbeatMonitor(this._lc);\n  }\n\n  // Life-cycle hooks for subclass implementations\n  protected _onStart() {}\n  protected _onStop(): Promise<void> {\n    return promiseVoid;\n  }\n  // start() is used in unit tests.\n  // run() is the lifecycle method called by the ServiceRunner.\n  async start(): Promise<string> {\n    this.#fastify.get('/', (_req, res) => res.send('OK'));\n    this.#fastify.get('/keepalive', ({headers}, res) => {\n      this.#heartbeatMonitor.onHeartbeat(headers);\n      return res.send('OK');\n    });\n    await this.#init(this.#fastify);\n    const address = await this.#fastify.listen({\n      host: '::',\n      port: this.#port,\n    });\n    this._lc.info?.(`${this.id} listening at ${address}`);\n    this._onStart();\n    return address;\n  }\n\n  async run(): Promise<void> {\n    await this.start();\n    await this._state.stopped();\n  }\n\n  async stop(): Promise<void> {\n    this._lc.info?.(`${this.id}: no longer accepting connections`);\n    this.#heartbeatMonitor.stop();\n    this._state.stop(this._lc);\n    await this.#fastify.close();\n    await this._onStop();\n  }\n}\n"],"mappings":";;;;;;;;;;AAgBA,IAAa,cAAb,MAA4C;CAC1C;CACA;CACA;CACA;CACA;CACA;CACA;CAEA,YACE,IACA,IACA,MACA,MACA;AACA,OAAK,KAAK;AACV,OAAK,MAAM,GAAG,YAAY,aAAa,KAAK,GAAG;AAC/C,QAAA,UAAgB,SAAS;AACzB,QAAA,OAAa,KAAK;AAClB,QAAA,OAAa;AACb,OAAK,SAAS,IAAI,aAAa,GAAG;AAClC,QAAA,mBAAyB,IAAI,iBAAiB,KAAK,IAAI;;CAIzD,WAAqB;CACrB,UAAmC;AACjC,SAAO;;CAIT,MAAM,QAAyB;AAC7B,QAAA,QAAc,IAAI,MAAM,MAAM,QAAQ,IAAI,KAAK,KAAK,CAAC;AACrD,QAAA,QAAc,IAAI,eAAe,EAAC,WAAU,QAAQ;AAClD,SAAA,iBAAuB,YAAY,QAAQ;AAC3C,UAAO,IAAI,KAAK,KAAK;IACrB;AACF,QAAM,MAAA,KAAW,MAAA,QAAc;EAC/B,MAAM,UAAU,MAAM,MAAA,QAAc,OAAO;GACzC,MAAM;GACN,MAAM,MAAA;GACP,CAAC;AACF,OAAK,IAAI,OAAO,GAAG,KAAK,GAAG,gBAAgB,UAAU;AACrD,OAAK,UAAU;AACf,SAAO;;CAGT,MAAM,MAAqB;AACzB,QAAM,KAAK,OAAO;AAClB,QAAM,KAAK,OAAO,SAAS;;CAG7B,MAAM,OAAsB;AAC1B,OAAK,IAAI,OAAO,GAAG,KAAK,GAAG,mCAAmC;AAC9D,QAAA,iBAAuB,MAAM;AAC7B,OAAK,OAAO,KAAK,KAAK,IAAI;AAC1B,QAAM,MAAA,QAAc,OAAO;AAC3B,QAAM,KAAK,SAAS"}
+{"version":3,"file":"http-service.js","names":["#fastify","#port","#heartbeatMonitor","#init"],"sources":["../../../../../zero-cache/src/services/http-service.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport Fastify, {type FastifyInstance} from 'fastify';\nimport {promiseVoid} from '../../../shared/src/resolved-promises.ts';\nimport {HeartbeatMonitor} from './life-cycle.ts';\nimport {RunningState} from './running-state.ts';\nimport type {Service} from './service.ts';\n\nexport type Options = {\n  port: number;\n  keepaliveTimeoutMs: number | undefined;\n};\n\n/**\n * Common functionality for all HttpServices. These include:\n * * Responding to health checks at \"/\"\n * * Tracking optional heartbeats at \"/keepalive\" and draining when they stop.\n */\nexport class HttpService implements Service {\n  readonly id: string;\n  protected readonly _lc: LogContext;\n  readonly #fastify: FastifyInstance;\n  readonly #port: number;\n  protected readonly _state: RunningState;\n  readonly #heartbeatMonitor: HeartbeatMonitor | undefined;\n  readonly #init: (fastify: FastifyInstance) => void | Promise<void>;\n\n  constructor(\n    id: string,\n    lc: LogContext,\n    opts: Options,\n    init: (fastify: FastifyInstance) => void | Promise<void>,\n  ) {\n    const {port, keepaliveTimeoutMs} = opts;\n    this.id = id;\n    this._lc = lc.withContext('component', this.id);\n    this.#fastify = Fastify();\n    this.#port = port;\n    this.#init = init;\n    this._state = new RunningState(id);\n    this.#heartbeatMonitor = keepaliveTimeoutMs\n      ? new HeartbeatMonitor(this._lc, keepaliveTimeoutMs)\n      : undefined;\n  }\n\n  // Life-cycle hooks for subclass implementations\n  protected _onStart() {}\n  protected _onStop(): Promise<void> {\n    return promiseVoid;\n  }\n  // start() is used in unit tests.\n  // run() is the lifecycle method called by the ServiceRunner.\n  async start(): Promise<string> {\n    this.#fastify.get('/', (_req, res) => res.send('OK'));\n    this.#fastify.get('/keepalive', ({headers}, res) => {\n      this.#heartbeatMonitor?.onHeartbeat(headers);\n      return res.send('OK');\n    });\n    await this.#init(this.#fastify);\n    const address = await this.#fastify.listen({\n      host: '::',\n      port: this.#port,\n    });\n    this._lc.info?.(`${this.id} listening at ${address}`);\n    this._onStart();\n    return address;\n  }\n\n  async run(): Promise<void> {\n    await this.start();\n    await this._state.stopped();\n  }\n\n  async stop(): Promise<void> {\n    this._lc.info?.(`${this.id}: no longer accepting connections`);\n    this.#heartbeatMonitor?.stop();\n    this._state.stop(this._lc);\n    await this.#fastify.close();\n    await this._onStop();\n  }\n}\n"],"mappings":";;;;;;;;;;AAiBA,IAAa,cAAb,MAA4C;CAC1C;CACA;CACA;CACA;CACA;CACA;CACA;CAEA,YACE,IACA,IACA,MACA,MACA;EACA,MAAM,EAAC,MAAM,uBAAsB;AACnC,OAAK,KAAK;AACV,OAAK,MAAM,GAAG,YAAY,aAAa,KAAK,GAAG;AAC/C,QAAA,UAAgB,SAAS;AACzB,QAAA,OAAa;AACb,QAAA,OAAa;AACb,OAAK,SAAS,IAAI,aAAa,GAAG;AAClC,QAAA,mBAAyB,qBACrB,IAAI,iBAAiB,KAAK,KAAK,mBAAmB,GAClD,KAAA;;CAIN,WAAqB;CACrB,UAAmC;AACjC,SAAO;;CAIT,MAAM,QAAyB;AAC7B,QAAA,QAAc,IAAI,MAAM,MAAM,QAAQ,IAAI,KAAK,KAAK,CAAC;AACrD,QAAA,QAAc,IAAI,eAAe,EAAC,WAAU,QAAQ;AAClD,SAAA,kBAAwB,YAAY,QAAQ;AAC5C,UAAO,IAAI,KAAK,KAAK;IACrB;AACF,QAAM,MAAA,KAAW,MAAA,QAAc;EAC/B,MAAM,UAAU,MAAM,MAAA,QAAc,OAAO;GACzC,MAAM;GACN,MAAM,MAAA;GACP,CAAC;AACF,OAAK,IAAI,OAAO,GAAG,KAAK,GAAG,gBAAgB,UAAU;AACrD,OAAK,UAAU;AACf,SAAO;;CAGT,MAAM,MAAqB;AACzB,QAAM,KAAK,OAAO;AAClB,QAAM,KAAK,OAAO,SAAS;;CAG7B,MAAM,OAAsB;AAC1B,OAAK,IAAI,OAAO,GAAG,KAAK,GAAG,mCAAmC;AAC9D,QAAA,kBAAwB,MAAM;AAC9B,OAAK,OAAO,KAAK,KAAK,IAAI;AAC1B,QAAM,MAAA,QAAc,OAAO;AAC3B,QAAM,KAAK,SAAS"}

package/out/zero-cache/src/services/life-cycle.d.ts

@@ -57,7 +57,7 @@ export declare function exitAfter(run: () => Promise<void>): Promise<void>;
  */
 export declare class HeartbeatMonitor {
     #private;
-    constructor(lc: LogContext, stopInterval?: number);
+    constructor(lc: LogContext, stopInterval: number);
     onHeartbeat(reqHeaders: IncomingHttpHeaders): void;
     stop(): void;
 }

package/out/zero-cache/src/services/life-cycle.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"life-cycle.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/services/life-cycle.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,mBAAmB,EAAC,MAAM,WAAW,CAAC;AAEnD,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,QAAQ,CAAC;AACzC,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,EAEL,KAAK,UAAU,EACf,KAAK,MAAM,EACZ,MAAM,uBAAuB,CAAC;AAE/B,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,cAAc,CAAC;AAEnD;;;;;;;;;;;;GAYG;AACH,MAAM,MAAM,UAAU,GAAG,aAAa,GAAG,YAAY,CAAC;AAEtD,eAAO,MAAM,iBAAiB,gCAAiC,CAAC;AAChE,eAAO,MAAM,iBAAiB,iCAAkC,CAAC;AAOjE,eAAO,MAAM,8BAA8B,KAAK,CAAC;AAKjD,eAAO,MAAM,+BAA+B,KAAK,CAAC;AAElD;;;GAGG;AACH,qBAAa,cAAc;;gBAWb,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,YAAY;IA2C9C,IAAI;IAyBJ,aAAa,CAAC,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM;IA2B9D,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM;IAiBjE,YAAY,IAAI,MAAM,EAAE;IAIlB,eAAe;IAIrB,eAAe,CAAC,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM;CAiF3C;AAED;;;;;;GAMG;AAEH,wBAAsB,cAAc,CAClC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,YAAY,EACpB,GAAG,QAAQ,EAAE,gBAAgB,EAAE,GAC9B,OAAO,CAAC,IAAI,CAAC,CA6Bf;AAED,wBAAsB,SAAS,CAAC,GAAG,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,iBAWvD;AAID;;;;;;;;;;GAUG;AACH,qBAAa,gBAAgB;;gBAQf,EAAE,EAAE,UAAU,EAAE,YAAY,SAA2B;IAKnE,WAAW,CAAC,UAAU,EAAE,mBAAmB;IAsD3C,IAAI;CAML"}
+{"version":3,"file":"life-cycle.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/services/life-cycle.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,mBAAmB,EAAC,MAAM,WAAW,CAAC;AAEnD,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,QAAQ,CAAC;AACzC,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,EAEL,KAAK,UAAU,EACf,KAAK,MAAM,EACZ,MAAM,uBAAuB,CAAC;AAE/B,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,cAAc,CAAC;AAEnD;;;;;;;;;;;;GAYG;AACH,MAAM,MAAM,UAAU,GAAG,aAAa,GAAG,YAAY,CAAC;AAEtD,eAAO,MAAM,iBAAiB,gCAAiC,CAAC;AAChE,eAAO,MAAM,iBAAiB,iCAAkC,CAAC;AAOjE,eAAO,MAAM,8BAA8B,KAAK,CAAC;AAKjD,eAAO,MAAM,+BAA+B,KAAK,CAAC;AAElD;;;GAGG;AACH,qBAAa,cAAc;;gBAWb,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,YAAY;IA2C9C,IAAI;IAyBJ,aAAa,CAAC,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM;IA2B9D,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM;IAiBjE,YAAY,IAAI,MAAM,EAAE;IAIlB,eAAe;IAIrB,eAAe,CAAC,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM;CAiF3C;AAED;;;;;;GAMG;AAEH,wBAAsB,cAAc,CAClC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,YAAY,EACpB,GAAG,QAAQ,EAAE,gBAAgB,EAAE,GAC9B,OAAO,CAAC,IAAI,CAAC,CA6Bf;AAED,wBAAsB,SAAS,CAAC,GAAG,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,iBAWvD;AAED;;;;;;;;;;GAUG;AACH,qBAAa,gBAAgB;;gBAQf,EAAE,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM;IAKhD,WAAW,CAAC,UAAU,EAAE,mBAAmB;IAsD3C,IAAI;CAML"}

package/out/zero-cache/src/services/life-cycle.js

@@ -146,7 +146,6 @@ async function exitAfter(run) {
 		process.exit(-1);
 	}
 }
-var DEFAULT_STOP_INTERVAL_MS = 2e4;
 /**
 * The HeartbeatMonitor monitors the cadence heartbeats (e.g. "/keepalive"
 * health checks made to HttpServices) that signal that the server
@@ -164,7 +163,7 @@ var HeartbeatMonitor = class {
 	#checkIntervalTimer;
 	#checkImmediateTimer;
 	#lastHeartbeat = 0;
-	constructor(lc, stopInterval = DEFAULT_STOP_INTERVAL_MS) {
+	constructor(lc, stopInterval) {
 		this.#lc = lc;
 		this.#stopInterval = stopInterval;
 	}

package/out/zero-cache/src/services/life-cycle.js.map

@@ -1 +1 @@
-{"version":3,"file":"life-cycle.js","names":["#lc","#userFacing","#all","#exitImpl","#start","#ready","#startDrain","#kill","#exit","#runningState","#drainStart","#onExit","#initializing","#nextID","#stopInterval","#lastHeartbeat","#checkIntervalTimer","#checkStopInterval","#checkImmediateTimer"],"sources":["../../../../../zero-cache/src/services/life-cycle.ts"],"sourcesContent":["import type {IncomingHttpHeaders} from 'node:http';\nimport {pid} from 'node:process';\nimport type {EventEmitter} from 'stream';\nimport type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {\n  singleProcessMode,\n  type Subprocess,\n  type Worker,\n} from '../types/processes.ts';\nimport {RunningState} from './running-state.ts';\nimport type {SingletonService} from './service.ts';\n\n/**\n * * `user-facing` workers serve external requests and are the first to\n *   receive a `SIGTERM` or `SIGINT` signal for graceful shutdown.\n *\n * * `supporting` workers support `user-facing` workers and are sent\n *   the `SIGTERM` signal only after all `user-facing` workers have\n *   exited.\n *\n * For other kill signals, such as `SIGQUIT` and `SIGABRT`, all workers\n * are stopped without draining. `SIGQUIT` is used to represent an\n * intentional shutdown (for which draining is not beneficial), whereas\n * `SIGABRT` is used for unexpected process exits.\n */\nexport type WorkerType = 'user-facing' | 'supporting';\n\nexport const GRACEFUL_SHUTDOWN = ['SIGTERM', 'SIGINT'] as const;\nexport const FORCEFUL_SHUTDOWN = ['SIGQUIT', 'SIGABRT'] as const;\n\ntype GracefulShutdownSignal = (typeof GRACEFUL_SHUTDOWN)[number];\n\n// An internal error code used to indicate that a message has already been\n// logged at level ERRROR. When a process exits with this error code, the\n// parent process logs the exit at level WARN instead of ERROR.\nexport const UNHANDLED_EXCEPTION_ERROR_CODE = 13;\n\n// An internal error code used to indicate that the server should exit\n// without draining (e.g. due to a supporting worker get a signal to shut\n// down), but the exit is otherwise intentional.\nexport const INTENTIONAL_SHUTDOWN_ERROR_CODE = 14;\n\n/**\n * Handles readiness, termination signals, and coordination of graceful\n * shutdown.\n */\nexport class ProcessManager {\n  readonly #lc: LogContext;\n  readonly #userFacing = new Set<Subprocess>();\n  readonly #all = new Set<Subprocess>();\n  readonly #exitImpl: (code: number) => never;\n  readonly #start = Date.now();\n  readonly #ready: Promise<void>[] = [];\n\n  #runningState = new RunningState('process-manager');\n  #drainStart = 0;\n\n  constructor(lc: LogContext, proc: EventEmitter) {\n    this.#lc = lc.withContext('component', 'process-manager');\n\n    // Propagate `SIGTERM` and `SIGINT` to all user-facing workers,\n    // initiating a graceful shutdown. The parent process will\n    // exit once all user-facing workers have exited ...\n    for (const signal of GRACEFUL_SHUTDOWN) {\n      proc.on(signal, () => this.#startDrain(signal));\n    }\n\n    // ... which will result in sending `SIGTERM` to the remaining workers.\n    proc.on('exit', code =>\n      this.#kill(\n        this.#all,\n        code === 0\n          ? 'SIGTERM' // graceful, drained shutdown\n          : code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n            ? 'SIGQUIT' // intentional abort without drain\n            : 'SIGABRT', // unintentional shutdown, alertable error\n      ),\n    );\n\n    // For other (catchable) kill signals, exit with a non-zero error code\n    // to send a `SIGQUIT` (intentional shutdown) or `SIGABRT` (unexpected\n    // shutdown) to all workers. For these signals, workers are stopped\n    // immediately without draining, since there is no merit to slowly draining\n    // when supporting workers have stopped.\n    //\n    // The logic for handling these signals is in `runUntilKilled()`.\n    for (const signal of FORCEFUL_SHUTDOWN) {\n      proc.on(signal, () =>\n        this.#exit(signal === 'SIGQUIT' ? INTENTIONAL_SHUTDOWN_ERROR_CODE : -1),\n      );\n    }\n\n    this.#exitImpl = (code: number) => {\n      if (singleProcessMode()) {\n        return proc.emit('exit', code) as never; // For unit / integration tests.\n      }\n      process.exit(code);\n    };\n  }\n\n  done() {\n    return this.#runningState.stopped();\n  }\n\n  #exit(code: number) {\n    this.#lc.info?.('exiting with code', code);\n    this.#runningState.stop(this.#lc);\n    void this.#lc.flush().finally(() => this.#exitImpl(code));\n  }\n\n  #startDrain(signal: GracefulShutdownSignal) {\n    if (this.#all.size === 0) {\n      // Shutdown if a signal is received before any subprocesses are added.\n      this.#lc.info?.(`exiting on ${signal}`);\n      this.#exit(0);\n    }\n    this.#lc.info?.(`initiating drain (${signal})`);\n    this.#drainStart = Date.now();\n    if (this.#userFacing.size) {\n      this.#kill(this.#userFacing, signal);\n    } else {\n      this.#kill(this.#all, signal);\n    }\n  }\n\n  addSubprocess(proc: Subprocess, type: WorkerType, name: string) {\n    if (type === 'user-facing') {\n      this.#userFacing.add(proc);\n    }\n    this.#all.add(proc);\n\n    let isOpen = true;\n    proc.on('close', (code, signal) => {\n      isOpen = false;\n      this.#onExit(code, signal, null, type, name, proc);\n    });\n\n    // As per https://nodejs.org/api/child_process.html#event-error\n    // 'error' events can happen when sending a message to a child process\n    // fails. This is not really an error when the server is shutting down,\n    // so log any post-close errors at 'warn'.\n    proc.on('error', err =>\n      this.#lc[!isOpen || this.#drainStart > 0 ? 'warn' : 'error']?.(\n        `error from ${name} ${proc.pid}`,\n        err,\n      ),\n    );\n  }\n\n  readonly #initializing = new Map<number, string>();\n  #nextID = 0;\n\n  addWorker(worker: Worker, type: WorkerType, name: string): Worker {\n    this.addSubprocess(worker, type, name);\n\n    const id = ++this.#nextID;\n    this.#initializing.set(id, name);\n    const {promise, resolve} = resolver();\n    this.#ready.push(promise);\n\n    worker.onceMessageType('ready', () => {\n      this.#lc.debug?.(`${name} ready (${Date.now() - this.#start} ms)`);\n      this.#initializing.delete(id);\n      resolve();\n    });\n\n    return worker;\n  }\n\n  initializing(): string[] {\n    return [...this.#initializing.values()];\n  }\n\n  async allWorkersReady() {\n    await Promise.all(this.#ready);\n  }\n\n  logErrorAndExit(err: unknown, name: string) {\n    // only accessible by the main (i.e. user-facing) process.\n    this.#onExit(-1, null, err, 'user-facing', name, undefined);\n  }\n\n  #onExit(\n    code: number,\n    sig: NodeJS.Signals | null,\n    err: unknown,\n    type: WorkerType,\n    name: string,\n    worker: Subprocess | undefined,\n  ) {\n    // Remove the worker from maps to avoid attempting to send more signals to it.\n    if (worker) {\n      this.#userFacing.delete(worker);\n      this.#all.delete(worker);\n    }\n\n    const pid = worker?.pid ?? process.pid;\n\n    if (type === 'supporting') {\n      // Supporting workers like the replication-manager shut down without a\n      // drain signal when receiving protocol-specific instructions (like auto\n      // reset). In this case, a special error code is used to signal that the\n      // server should be shut down without draining, but it is otherwise not\n      // considered an unexpected/alertable error.\n      if (code === 0 && (this.#drainStart === 0 || this.#userFacing.size > 0)) {\n        code = INTENTIONAL_SHUTDOWN_ERROR_CODE;\n      }\n      const log =\n        code === 0 || code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n          ? 'info'\n          : 'warn';\n      this.#lc[log]?.(`${name} (${pid}) exited with code (${code})`, err ?? '');\n      return this.#exit(code);\n    }\n\n    const log =\n      code === 0 || code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n        ? 'info'\n        : this.#drainStart > 0 || code === UNHANDLED_EXCEPTION_ERROR_CODE\n          ? 'warn'\n          : 'error';\n    this.#lc[log]?.(\n      sig\n        ? `${name} (${pid}) killed with (${sig})`\n        : `${name} (${pid}) exited with code (${code})`,\n      err ?? '',\n    );\n\n    // user-facing workers exited or finished draining.\n    if (this.#userFacing.size === 0) {\n      this.#lc.info?.(\n        this.#drainStart\n          ? `all user-facing workers drained (${\n              Date.now() - this.#drainStart\n            } ms)`\n          : `all user-facing workers exited`,\n      );\n      return this.#exit(0);\n    }\n\n    if (this.#drainStart === 0) {\n      // If a user-facing worker exits without receiving a drain signal,\n      // shutdown the server.\n      return this.#exit(code || -1);\n    }\n\n    return undefined;\n  }\n\n  #kill(workers: Iterable<Subprocess>, signal: NodeJS.Signals) {\n    for (const worker of workers) {\n      try {\n        worker.kill(signal);\n      } catch (e) {\n        this.#lc.error?.(e);\n      }\n    }\n  }\n}\n\n/**\n * Runs the specified services, stopping them on `SIGTERM` or `SIGINT` with\n * an optional {@link SingletonService.drain drain()}, or stopping them\n * without draining for `SIGQUIT`.\n *\n * @returns a Promise that resolves/rejects when any of the services stops/throws.\n */\n\nexport async function runUntilKilled(\n  lc: LogContext,\n  parent: EventEmitter,\n  ...services: SingletonService[]\n): Promise<void> {\n  if (services.length === 0) {\n    return;\n  }\n  for (const signal of [...GRACEFUL_SHUTDOWN, ...FORCEFUL_SHUTDOWN]) {\n    parent.once(signal, () => {\n      const GRACEFUL_SIGNALS = GRACEFUL_SHUTDOWN as readonly NodeJS.Signals[];\n\n      services.forEach(async svc => {\n        if (GRACEFUL_SIGNALS.includes(signal) && svc.drain) {\n          lc.info?.(`draining ${svc.constructor.name} ${svc.id} (${signal})`);\n          await svc.drain();\n        }\n        lc.info?.(`stopping ${svc.constructor.name} ${svc.id} (${signal})`);\n        await svc.stop();\n      });\n    });\n  }\n\n  try {\n    // Run all services and resolve when any of them stops.\n    const svc = await Promise.race(\n      services.map(svc => svc.run().then(() => svc)),\n    );\n    lc.info?.(`${svc.constructor.name} (${svc.id}) stopped`);\n  } catch (e) {\n    lc.error?.(`exiting on error`, e);\n    throw e;\n  }\n}\n\nexport async function exitAfter(run: () => Promise<void>) {\n  try {\n    await run();\n    // oxlint-disable-next-line no-console\n    console.info(`pid ${pid} exiting normally`);\n    process.exit(0);\n  } catch (e) {\n    // oxlint-disable-next-line no-console\n    console.error(`pid ${pid} exiting with error`, e);\n    process.exit(-1);\n  }\n}\n\nconst DEFAULT_STOP_INTERVAL_MS = 20_000;\n\n/**\n * The HeartbeatMonitor monitors the cadence heartbeats (e.g. \"/keepalive\"\n * health checks made to HttpServices) that signal that the server\n * should continue processing requests. When a configurable `stopInterval`\n * elapses without receiving these heartbeats, the monitor initiates a\n * graceful shutdown of the server. This works with common load balancing\n * frameworks such as AWS Elastic Load Balancing.\n *\n * The HeartbeatMonitor is **opt-in** in that it only kicks in after it\n * starts receiving keepalives.\n */\nexport class HeartbeatMonitor {\n  readonly #stopInterval: number;\n\n  #lc: LogContext;\n  #checkIntervalTimer: NodeJS.Timeout | undefined;\n  #checkImmediateTimer: NodeJS.Immediate | undefined;\n  #lastHeartbeat = 0;\n\n  constructor(lc: LogContext, stopInterval = DEFAULT_STOP_INTERVAL_MS) {\n    this.#lc = lc;\n    this.#stopInterval = stopInterval;\n  }\n\n  onHeartbeat(reqHeaders: IncomingHttpHeaders) {\n    this.#lastHeartbeat = Date.now();\n    if (this.#checkIntervalTimer === undefined) {\n      this.#lc.info?.(\n        `starting heartbeat monitor at ${\n          this.#stopInterval / 1000\n        } second interval`,\n        reqHeaders,\n      );\n      // e.g. check every 5 seconds to see if it's been over 20 seconds\n      //      since the last heartbeat.\n      this.#checkIntervalTimer = setInterval(\n        this.#checkStopInterval,\n        this.#stopInterval / 4,\n      );\n    }\n  }\n\n  #checkStopInterval = () => {\n    // In the Node.js event loop, timers like setInterval and setTimeout\n    // run *before* I/O events coming from network sockets or file reads/writes.\n    // When this process gets starved of CPU resources for long periods of time,\n    // for example when other processes are monopolizing all available cores,\n    // pathological behavior can emerge:\n    // - keepalive network request comes in, but is queued in Node internals waiting\n    //   for time on the event loop\n    // - CPU is starved/monopolized by other processes for longer than the time\n    //   configured via this.#stopInterval\n    // - When CPU becomes available and the event loop wakes up, this stop interval\n    //   check is run *before* the keepalive request is processed. The value of\n    //   this.#lastHeartbeat is now very stale, and erroneously triggers a shutdown\n    //   even though keepalive requests were about to be processed and update\n    //   this.#lastHeartbeat. Downtime ensues.\n    //\n    // To avoid this, we push the check out to a phase of the event loop *after*\n    // I/O events are processed, using setImmediate():\n    // https://nodejs.org/en/learn/asynchronous-work/event-loop-timers-and-nexttick#setimmediate-vs-settimeout\n    //\n    // This ensures we see a value for this.#lastHeartbeat that reflects\n    // any keepalive requests that came in during the current event loop turn.\n    this.#checkImmediateTimer = setImmediate(() => {\n      this.#checkImmediateTimer = undefined;\n      const timeSinceLastHeartbeat = Date.now() - this.#lastHeartbeat;\n      if (timeSinceLastHeartbeat >= this.#stopInterval) {\n        this.#lc.info?.(\n          `last heartbeat received ${\n            timeSinceLastHeartbeat / 1000\n          } seconds ago. draining.`,\n        );\n        process.kill(process.pid, GRACEFUL_SHUTDOWN[0]);\n      }\n    });\n  };\n\n  stop() {\n    clearTimeout(this.#checkIntervalTimer);\n    if (this.#checkImmediateTimer) {\n      clearImmediate(this.#checkImmediateTimer);\n    }\n  }\n}\n"],"mappings":";;;;;AA4BA,IAAa,oBAAoB,CAAC,WAAW,SAAS;AACtD,IAAa,oBAAoB,CAAC,WAAW,UAAU;;;;;AAkBvD,IAAa,iBAAb,MAA4B;CAC1B;CACA,8BAAuB,IAAI,KAAiB;CAC5C,uBAAgB,IAAI,KAAiB;CACrC;CACA,SAAkB,KAAK,KAAK;CAC5B,SAAmC,EAAE;CAErC,gBAAgB,IAAI,aAAa,kBAAkB;CACnD,cAAc;CAEd,YAAY,IAAgB,MAAoB;AAC9C,QAAA,KAAW,GAAG,YAAY,aAAa,kBAAkB;AAKzD,OAAK,MAAM,UAAU,kBACnB,MAAK,GAAG,cAAc,MAAA,WAAiB,OAAO,CAAC;AAIjD,OAAK,GAAG,SAAQ,SACd,MAAA,KACE,MAAA,KACA,SAAS,IACL,YACA,SAAA,KACE,YACA,UACP,CACF;AASD,OAAK,MAAM,UAAU,kBACnB,MAAK,GAAG,cACN,MAAA,KAAW,WAAW,YAAA,KAA8C,GAAG,CACxE;AAGH,QAAA,YAAkB,SAAiB;AACjC,OAAI,mBAAmB,CACrB,QAAO,KAAK,KAAK,QAAQ,KAAK;AAEhC,WAAQ,KAAK,KAAK;;;CAItB,OAAO;AACL,SAAO,MAAA,aAAmB,SAAS;;CAGrC,MAAM,MAAc;AAClB,QAAA,GAAS,OAAO,qBAAqB,KAAK;AAC1C,QAAA,aAAmB,KAAK,MAAA,GAAS;AAC5B,QAAA,GAAS,OAAO,CAAC,cAAc,MAAA,SAAe,KAAK,CAAC;;CAG3D,YAAY,QAAgC;AAC1C,MAAI,MAAA,IAAU,SAAS,GAAG;AAExB,SAAA,GAAS,OAAO,cAAc,SAAS;AACvC,SAAA,KAAW,EAAE;;AAEf,QAAA,GAAS,OAAO,qBAAqB,OAAO,GAAG;AAC/C,QAAA,aAAmB,KAAK,KAAK;AAC7B,MAAI,MAAA,WAAiB,KACnB,OAAA,KAAW,MAAA,YAAkB,OAAO;MAEpC,OAAA,KAAW,MAAA,KAAW,OAAO;;CAIjC,cAAc,MAAkB,MAAkB,MAAc;AAC9D,MAAI,SAAS,cACX,OAAA,WAAiB,IAAI,KAAK;AAE5B,QAAA,IAAU,IAAI,KAAK;EAEnB,IAAI,SAAS;AACb,OAAK,GAAG,UAAU,MAAM,WAAW;AACjC,YAAS;AACT,SAAA,OAAa,MAAM,QAAQ,MAAM,MAAM,MAAM,KAAK;IAClD;AAMF,OAAK,GAAG,UAAS,QACf,MAAA,GAAS,CAAC,UAAU,MAAA,aAAmB,IAAI,SAAS,WAClD,cAAc,KAAK,GAAG,KAAK,OAC3B,IACD,CACF;;CAGH,gCAAyB,IAAI,KAAqB;CAClD,UAAU;CAEV,UAAU,QAAgB,MAAkB,MAAsB;AAChE,OAAK,cAAc,QAAQ,MAAM,KAAK;EAEtC,MAAM,KAAK,EAAE,MAAA;AACb,QAAA,aAAmB,IAAI,IAAI,KAAK;EAChC,MAAM,EAAC,SAAS,YAAW,UAAU;AACrC,QAAA,MAAY,KAAK,QAAQ;AAEzB,SAAO,gBAAgB,eAAe;AACpC,SAAA,GAAS,QAAQ,GAAG,KAAK,UAAU,KAAK,KAAK,GAAG,MAAA,MAAY,MAAM;AAClE,SAAA,aAAmB,OAAO,GAAG;AAC7B,YAAS;IACT;AAEF,SAAO;;CAGT,eAAyB;AACvB,SAAO,CAAC,GAAG,MAAA,aAAmB,QAAQ,CAAC;;CAGzC,MAAM,kBAAkB;AACtB,QAAM,QAAQ,IAAI,MAAA,MAAY;;CAGhC,gBAAgB,KAAc,MAAc;AAE1C,QAAA,OAAa,IAAI,MAAM,KAAK,eAAe,MAAM,KAAA,EAAU;;CAG7D,QACE,MACA,KACA,KACA,MACA,MACA,QACA;AAEA,MAAI,QAAQ;AACV,SAAA,WAAiB,OAAO,OAAO;AAC/B,SAAA,IAAU,OAAO,OAAO;;EAG1B,MAAM,MAAM,QAAQ,OAAO,QAAQ;AAEnC,MAAI,SAAS,cAAc;AAMzB,OAAI,SAAS,MAAM,MAAA,eAAqB,KAAK,MAAA,WAAiB,OAAO,GACnE,QAAA;GAEF,MAAM,MACJ,SAAS,KAAK,SAAA,KACV,SACA;AACN,SAAA,GAAS,OAAO,GAAG,KAAK,IAAI,IAAI,sBAAsB,KAAK,IAAI,OAAO,GAAG;AACzE,UAAO,MAAA,KAAW,KAAK;;EAGzB,MAAM,MACJ,SAAS,KAAK,SAAA,KACV,SACA,MAAA,aAAmB,KAAK,SAAA,KACtB,SACA;AACR,QAAA,GAAS,OACP,MACI,GAAG,KAAK,IAAI,IAAI,iBAAiB,IAAI,KACrC,GAAG,KAAK,IAAI,IAAI,sBAAsB,KAAK,IAC/C,OAAO,GACR;AAGD,MAAI,MAAA,WAAiB,SAAS,GAAG;AAC/B,SAAA,GAAS,OACP,MAAA,aACI,oCACE,KAAK,KAAK,GAAG,MAAA,WACd,QACD,iCACL;AACD,UAAO,MAAA,KAAW,EAAE;;AAGtB,MAAI,MAAA,eAAqB,EAGvB,QAAO,MAAA,KAAW,QAAQ,GAAG;;CAMjC,MAAM,SAA+B,QAAwB;AAC3D,OAAK,MAAM,UAAU,QACnB,KAAI;AACF,UAAO,KAAK,OAAO;WACZ,GAAG;AACV,SAAA,GAAS,QAAQ,EAAE;;;;;;;;;;;AAc3B,eAAsB,eACpB,IACA,QACA,GAAG,UACY;AACf,KAAI,SAAS,WAAW,EACtB;AAEF,MAAK,MAAM,UAAU,CAAC,GAAG,mBAAmB,GAAG,kBAAkB,CAC/D,QAAO,KAAK,cAAc;EACxB,MAAM,mBAAmB;AAEzB,WAAS,QAAQ,OAAM,QAAO;AAC5B,OAAI,iBAAiB,SAAS,OAAO,IAAI,IAAI,OAAO;AAClD,OAAG,OAAO,YAAY,IAAI,YAAY,KAAK,GAAG,IAAI,GAAG,IAAI,OAAO,GAAG;AACnE,UAAM,IAAI,OAAO;;AAEnB,MAAG,OAAO,YAAY,IAAI,YAAY,KAAK,GAAG,IAAI,GAAG,IAAI,OAAO,GAAG;AACnE,SAAM,IAAI,MAAM;IAChB;GACF;AAGJ,KAAI;EAEF,MAAM,MAAM,MAAM,QAAQ,KACxB,SAAS,KAAI,QAAO,IAAI,KAAK,CAAC,WAAW,IAAI,CAAC,CAC/C;AACD,KAAG,OAAO,GAAG,IAAI,YAAY,KAAK,IAAI,IAAI,GAAG,WAAW;UACjD,GAAG;AACV,KAAG,QAAQ,oBAAoB,EAAE;AACjC,QAAM;;;AAIV,eAAsB,UAAU,KAA0B;AACxD,KAAI;AACF,QAAM,KAAK;AAEX,UAAQ,KAAK,OAAO,IAAI,mBAAmB;AAC3C,UAAQ,KAAK,EAAE;UACR,GAAG;AAEV,UAAQ,MAAM,OAAO,IAAI,sBAAsB,EAAE;AACjD,UAAQ,KAAK,GAAG;;;AAIpB,IAAM,2BAA2B;;;;;;;;;;;;AAajC,IAAa,mBAAb,MAA8B;CAC5B;CAEA;CACA;CACA;CACA,iBAAiB;CAEjB,YAAY,IAAgB,eAAe,0BAA0B;AACnE,QAAA,KAAW;AACX,QAAA,eAAqB;;CAGvB,YAAY,YAAiC;AAC3C,QAAA,gBAAsB,KAAK,KAAK;AAChC,MAAI,MAAA,uBAA6B,KAAA,GAAW;AAC1C,SAAA,GAAS,OACP,iCACE,MAAA,eAAqB,IACtB,mBACD,WACD;AAGD,SAAA,qBAA2B,YACzB,MAAA,mBACA,MAAA,eAAqB,EACtB;;;CAIL,2BAA2B;AAsBzB,QAAA,sBAA4B,mBAAmB;AAC7C,SAAA,sBAA4B,KAAA;GAC5B,MAAM,yBAAyB,KAAK,KAAK,GAAG,MAAA;AAC5C,OAAI,0BAA0B,MAAA,cAAoB;AAChD,UAAA,GAAS,OACP,2BACE,yBAAyB,IAC1B,yBACF;AACD,YAAQ,KAAK,QAAQ,KAAK,kBAAkB,GAAG;;IAEjD;;CAGJ,OAAO;AACL,eAAa,MAAA,mBAAyB;AACtC,MAAI,MAAA,oBACF,gBAAe,MAAA,oBAA0B"}
+{"version":3,"file":"life-cycle.js","names":["#lc","#userFacing","#all","#exitImpl","#start","#ready","#startDrain","#kill","#exit","#runningState","#drainStart","#onExit","#initializing","#nextID","#stopInterval","#lastHeartbeat","#checkIntervalTimer","#checkStopInterval","#checkImmediateTimer"],"sources":["../../../../../zero-cache/src/services/life-cycle.ts"],"sourcesContent":["import type {IncomingHttpHeaders} from 'node:http';\nimport {pid} from 'node:process';\nimport type {EventEmitter} from 'stream';\nimport type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {\n  singleProcessMode,\n  type Subprocess,\n  type Worker,\n} from '../types/processes.ts';\nimport {RunningState} from './running-state.ts';\nimport type {SingletonService} from './service.ts';\n\n/**\n * * `user-facing` workers serve external requests and are the first to\n *   receive a `SIGTERM` or `SIGINT` signal for graceful shutdown.\n *\n * * `supporting` workers support `user-facing` workers and are sent\n *   the `SIGTERM` signal only after all `user-facing` workers have\n *   exited.\n *\n * For other kill signals, such as `SIGQUIT` and `SIGABRT`, all workers\n * are stopped without draining. `SIGQUIT` is used to represent an\n * intentional shutdown (for which draining is not beneficial), whereas\n * `SIGABRT` is used for unexpected process exits.\n */\nexport type WorkerType = 'user-facing' | 'supporting';\n\nexport const GRACEFUL_SHUTDOWN = ['SIGTERM', 'SIGINT'] as const;\nexport const FORCEFUL_SHUTDOWN = ['SIGQUIT', 'SIGABRT'] as const;\n\ntype GracefulShutdownSignal = (typeof GRACEFUL_SHUTDOWN)[number];\n\n// An internal error code used to indicate that a message has already been\n// logged at level ERRROR. When a process exits with this error code, the\n// parent process logs the exit at level WARN instead of ERROR.\nexport const UNHANDLED_EXCEPTION_ERROR_CODE = 13;\n\n// An internal error code used to indicate that the server should exit\n// without draining (e.g. due to a supporting worker get a signal to shut\n// down), but the exit is otherwise intentional.\nexport const INTENTIONAL_SHUTDOWN_ERROR_CODE = 14;\n\n/**\n * Handles readiness, termination signals, and coordination of graceful\n * shutdown.\n */\nexport class ProcessManager {\n  readonly #lc: LogContext;\n  readonly #userFacing = new Set<Subprocess>();\n  readonly #all = new Set<Subprocess>();\n  readonly #exitImpl: (code: number) => never;\n  readonly #start = Date.now();\n  readonly #ready: Promise<void>[] = [];\n\n  #runningState = new RunningState('process-manager');\n  #drainStart = 0;\n\n  constructor(lc: LogContext, proc: EventEmitter) {\n    this.#lc = lc.withContext('component', 'process-manager');\n\n    // Propagate `SIGTERM` and `SIGINT` to all user-facing workers,\n    // initiating a graceful shutdown. The parent process will\n    // exit once all user-facing workers have exited ...\n    for (const signal of GRACEFUL_SHUTDOWN) {\n      proc.on(signal, () => this.#startDrain(signal));\n    }\n\n    // ... which will result in sending `SIGTERM` to the remaining workers.\n    proc.on('exit', code =>\n      this.#kill(\n        this.#all,\n        code === 0\n          ? 'SIGTERM' // graceful, drained shutdown\n          : code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n            ? 'SIGQUIT' // intentional abort without drain\n            : 'SIGABRT', // unintentional shutdown, alertable error\n      ),\n    );\n\n    // For other (catchable) kill signals, exit with a non-zero error code\n    // to send a `SIGQUIT` (intentional shutdown) or `SIGABRT` (unexpected\n    // shutdown) to all workers. For these signals, workers are stopped\n    // immediately without draining, since there is no merit to slowly draining\n    // when supporting workers have stopped.\n    //\n    // The logic for handling these signals is in `runUntilKilled()`.\n    for (const signal of FORCEFUL_SHUTDOWN) {\n      proc.on(signal, () =>\n        this.#exit(signal === 'SIGQUIT' ? INTENTIONAL_SHUTDOWN_ERROR_CODE : -1),\n      );\n    }\n\n    this.#exitImpl = (code: number) => {\n      if (singleProcessMode()) {\n        return proc.emit('exit', code) as never; // For unit / integration tests.\n      }\n      process.exit(code);\n    };\n  }\n\n  done() {\n    return this.#runningState.stopped();\n  }\n\n  #exit(code: number) {\n    this.#lc.info?.('exiting with code', code);\n    this.#runningState.stop(this.#lc);\n    void this.#lc.flush().finally(() => this.#exitImpl(code));\n  }\n\n  #startDrain(signal: GracefulShutdownSignal) {\n    if (this.#all.size === 0) {\n      // Shutdown if a signal is received before any subprocesses are added.\n      this.#lc.info?.(`exiting on ${signal}`);\n      this.#exit(0);\n    }\n    this.#lc.info?.(`initiating drain (${signal})`);\n    this.#drainStart = Date.now();\n    if (this.#userFacing.size) {\n      this.#kill(this.#userFacing, signal);\n    } else {\n      this.#kill(this.#all, signal);\n    }\n  }\n\n  addSubprocess(proc: Subprocess, type: WorkerType, name: string) {\n    if (type === 'user-facing') {\n      this.#userFacing.add(proc);\n    }\n    this.#all.add(proc);\n\n    let isOpen = true;\n    proc.on('close', (code, signal) => {\n      isOpen = false;\n      this.#onExit(code, signal, null, type, name, proc);\n    });\n\n    // As per https://nodejs.org/api/child_process.html#event-error\n    // 'error' events can happen when sending a message to a child process\n    // fails. This is not really an error when the server is shutting down,\n    // so log any post-close errors at 'warn'.\n    proc.on('error', err =>\n      this.#lc[!isOpen || this.#drainStart > 0 ? 'warn' : 'error']?.(\n        `error from ${name} ${proc.pid}`,\n        err,\n      ),\n    );\n  }\n\n  readonly #initializing = new Map<number, string>();\n  #nextID = 0;\n\n  addWorker(worker: Worker, type: WorkerType, name: string): Worker {\n    this.addSubprocess(worker, type, name);\n\n    const id = ++this.#nextID;\n    this.#initializing.set(id, name);\n    const {promise, resolve} = resolver();\n    this.#ready.push(promise);\n\n    worker.onceMessageType('ready', () => {\n      this.#lc.debug?.(`${name} ready (${Date.now() - this.#start} ms)`);\n      this.#initializing.delete(id);\n      resolve();\n    });\n\n    return worker;\n  }\n\n  initializing(): string[] {\n    return [...this.#initializing.values()];\n  }\n\n  async allWorkersReady() {\n    await Promise.all(this.#ready);\n  }\n\n  logErrorAndExit(err: unknown, name: string) {\n    // only accessible by the main (i.e. user-facing) process.\n    this.#onExit(-1, null, err, 'user-facing', name, undefined);\n  }\n\n  #onExit(\n    code: number,\n    sig: NodeJS.Signals | null,\n    err: unknown,\n    type: WorkerType,\n    name: string,\n    worker: Subprocess | undefined,\n  ) {\n    // Remove the worker from maps to avoid attempting to send more signals to it.\n    if (worker) {\n      this.#userFacing.delete(worker);\n      this.#all.delete(worker);\n    }\n\n    const pid = worker?.pid ?? process.pid;\n\n    if (type === 'supporting') {\n      // Supporting workers like the replication-manager shut down without a\n      // drain signal when receiving protocol-specific instructions (like auto\n      // reset). In this case, a special error code is used to signal that the\n      // server should be shut down without draining, but it is otherwise not\n      // considered an unexpected/alertable error.\n      if (code === 0 && (this.#drainStart === 0 || this.#userFacing.size > 0)) {\n        code = INTENTIONAL_SHUTDOWN_ERROR_CODE;\n      }\n      const log =\n        code === 0 || code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n          ? 'info'\n          : 'warn';\n      this.#lc[log]?.(`${name} (${pid}) exited with code (${code})`, err ?? '');\n      return this.#exit(code);\n    }\n\n    const log =\n      code === 0 || code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n        ? 'info'\n        : this.#drainStart > 0 || code === UNHANDLED_EXCEPTION_ERROR_CODE\n          ? 'warn'\n          : 'error';\n    this.#lc[log]?.(\n      sig\n        ? `${name} (${pid}) killed with (${sig})`\n        : `${name} (${pid}) exited with code (${code})`,\n      err ?? '',\n    );\n\n    // user-facing workers exited or finished draining.\n    if (this.#userFacing.size === 0) {\n      this.#lc.info?.(\n        this.#drainStart\n          ? `all user-facing workers drained (${\n              Date.now() - this.#drainStart\n            } ms)`\n          : `all user-facing workers exited`,\n      );\n      return this.#exit(0);\n    }\n\n    if (this.#drainStart === 0) {\n      // If a user-facing worker exits without receiving a drain signal,\n      // shutdown the server.\n      return this.#exit(code || -1);\n    }\n\n    return undefined;\n  }\n\n  #kill(workers: Iterable<Subprocess>, signal: NodeJS.Signals) {\n    for (const worker of workers) {\n      try {\n        worker.kill(signal);\n      } catch (e) {\n        this.#lc.error?.(e);\n      }\n    }\n  }\n}\n\n/**\n * Runs the specified services, stopping them on `SIGTERM` or `SIGINT` with\n * an optional {@link SingletonService.drain drain()}, or stopping them\n * without draining for `SIGQUIT`.\n *\n * @returns a Promise that resolves/rejects when any of the services stops/throws.\n */\n\nexport async function runUntilKilled(\n  lc: LogContext,\n  parent: EventEmitter,\n  ...services: SingletonService[]\n): Promise<void> {\n  if (services.length === 0) {\n    return;\n  }\n  for (const signal of [...GRACEFUL_SHUTDOWN, ...FORCEFUL_SHUTDOWN]) {\n    parent.once(signal, () => {\n      const GRACEFUL_SIGNALS = GRACEFUL_SHUTDOWN as readonly NodeJS.Signals[];\n\n      services.forEach(async svc => {\n        if (GRACEFUL_SIGNALS.includes(signal) && svc.drain) {\n          lc.info?.(`draining ${svc.constructor.name} ${svc.id} (${signal})`);\n          await svc.drain();\n        }\n        lc.info?.(`stopping ${svc.constructor.name} ${svc.id} (${signal})`);\n        await svc.stop();\n      });\n    });\n  }\n\n  try {\n    // Run all services and resolve when any of them stops.\n    const svc = await Promise.race(\n      services.map(svc => svc.run().then(() => svc)),\n    );\n    lc.info?.(`${svc.constructor.name} (${svc.id}) stopped`);\n  } catch (e) {\n    lc.error?.(`exiting on error`, e);\n    throw e;\n  }\n}\n\nexport async function exitAfter(run: () => Promise<void>) {\n  try {\n    await run();\n    // oxlint-disable-next-line no-console\n    console.info(`pid ${pid} exiting normally`);\n    process.exit(0);\n  } catch (e) {\n    // oxlint-disable-next-line no-console\n    console.error(`pid ${pid} exiting with error`, e);\n    process.exit(-1);\n  }\n}\n\n/**\n * The HeartbeatMonitor monitors the cadence heartbeats (e.g. \"/keepalive\"\n * health checks made to HttpServices) that signal that the server\n * should continue processing requests. When a configurable `stopInterval`\n * elapses without receiving these heartbeats, the monitor initiates a\n * graceful shutdown of the server. This works with common load balancing\n * frameworks such as AWS Elastic Load Balancing.\n *\n * The HeartbeatMonitor is **opt-in** in that it only kicks in after it\n * starts receiving keepalives.\n */\nexport class HeartbeatMonitor {\n  readonly #stopInterval: number;\n\n  #lc: LogContext;\n  #checkIntervalTimer: NodeJS.Timeout | undefined;\n  #checkImmediateTimer: NodeJS.Immediate | undefined;\n  #lastHeartbeat = 0;\n\n  constructor(lc: LogContext, stopInterval: number) {\n    this.#lc = lc;\n    this.#stopInterval = stopInterval;\n  }\n\n  onHeartbeat(reqHeaders: IncomingHttpHeaders) {\n    this.#lastHeartbeat = Date.now();\n    if (this.#checkIntervalTimer === undefined) {\n      this.#lc.info?.(\n        `starting heartbeat monitor at ${\n          this.#stopInterval / 1000\n        } second interval`,\n        reqHeaders,\n      );\n      // e.g. check every 5 seconds to see if it's been over 20 seconds\n      //      since the last heartbeat.\n      this.#checkIntervalTimer = setInterval(\n        this.#checkStopInterval,\n        this.#stopInterval / 4,\n      );\n    }\n  }\n\n  #checkStopInterval = () => {\n    // In the Node.js event loop, timers like setInterval and setTimeout\n    // run *before* I/O events coming from network sockets or file reads/writes.\n    // When this process gets starved of CPU resources for long periods of time,\n    // for example when other processes are monopolizing all available cores,\n    // pathological behavior can emerge:\n    // - keepalive network request comes in, but is queued in Node internals waiting\n    //   for time on the event loop\n    // - CPU is starved/monopolized by other processes for longer than the time\n    //   configured via this.#stopInterval\n    // - When CPU becomes available and the event loop wakes up, this stop interval\n    //   check is run *before* the keepalive request is processed. The value of\n    //   this.#lastHeartbeat is now very stale, and erroneously triggers a shutdown\n    //   even though keepalive requests were about to be processed and update\n    //   this.#lastHeartbeat. Downtime ensues.\n    //\n    // To avoid this, we push the check out to a phase of the event loop *after*\n    // I/O events are processed, using setImmediate():\n    // https://nodejs.org/en/learn/asynchronous-work/event-loop-timers-and-nexttick#setimmediate-vs-settimeout\n    //\n    // This ensures we see a value for this.#lastHeartbeat that reflects\n    // any keepalive requests that came in during the current event loop turn.\n    this.#checkImmediateTimer = setImmediate(() => {\n      this.#checkImmediateTimer = undefined;\n      const timeSinceLastHeartbeat = Date.now() - this.#lastHeartbeat;\n      if (timeSinceLastHeartbeat >= this.#stopInterval) {\n        this.#lc.info?.(\n          `last heartbeat received ${\n            timeSinceLastHeartbeat / 1000\n          } seconds ago. draining.`,\n        );\n        process.kill(process.pid, GRACEFUL_SHUTDOWN[0]);\n      }\n    });\n  };\n\n  stop() {\n    clearTimeout(this.#checkIntervalTimer);\n    if (this.#checkImmediateTimer) {\n      clearImmediate(this.#checkImmediateTimer);\n    }\n  }\n}\n"],"mappings":";;;;;AA4BA,IAAa,oBAAoB,CAAC,WAAW,SAAS;AACtD,IAAa,oBAAoB,CAAC,WAAW,UAAU;;;;;AAkBvD,IAAa,iBAAb,MAA4B;CAC1B;CACA,8BAAuB,IAAI,KAAiB;CAC5C,uBAAgB,IAAI,KAAiB;CACrC;CACA,SAAkB,KAAK,KAAK;CAC5B,SAAmC,EAAE;CAErC,gBAAgB,IAAI,aAAa,kBAAkB;CACnD,cAAc;CAEd,YAAY,IAAgB,MAAoB;AAC9C,QAAA,KAAW,GAAG,YAAY,aAAa,kBAAkB;AAKzD,OAAK,MAAM,UAAU,kBACnB,MAAK,GAAG,cAAc,MAAA,WAAiB,OAAO,CAAC;AAIjD,OAAK,GAAG,SAAQ,SACd,MAAA,KACE,MAAA,KACA,SAAS,IACL,YACA,SAAA,KACE,YACA,UACP,CACF;AASD,OAAK,MAAM,UAAU,kBACnB,MAAK,GAAG,cACN,MAAA,KAAW,WAAW,YAAA,KAA8C,GAAG,CACxE;AAGH,QAAA,YAAkB,SAAiB;AACjC,OAAI,mBAAmB,CACrB,QAAO,KAAK,KAAK,QAAQ,KAAK;AAEhC,WAAQ,KAAK,KAAK;;;CAItB,OAAO;AACL,SAAO,MAAA,aAAmB,SAAS;;CAGrC,MAAM,MAAc;AAClB,QAAA,GAAS,OAAO,qBAAqB,KAAK;AAC1C,QAAA,aAAmB,KAAK,MAAA,GAAS;AAC5B,QAAA,GAAS,OAAO,CAAC,cAAc,MAAA,SAAe,KAAK,CAAC;;CAG3D,YAAY,QAAgC;AAC1C,MAAI,MAAA,IAAU,SAAS,GAAG;AAExB,SAAA,GAAS,OAAO,cAAc,SAAS;AACvC,SAAA,KAAW,EAAE;;AAEf,QAAA,GAAS,OAAO,qBAAqB,OAAO,GAAG;AAC/C,QAAA,aAAmB,KAAK,KAAK;AAC7B,MAAI,MAAA,WAAiB,KACnB,OAAA,KAAW,MAAA,YAAkB,OAAO;MAEpC,OAAA,KAAW,MAAA,KAAW,OAAO;;CAIjC,cAAc,MAAkB,MAAkB,MAAc;AAC9D,MAAI,SAAS,cACX,OAAA,WAAiB,IAAI,KAAK;AAE5B,QAAA,IAAU,IAAI,KAAK;EAEnB,IAAI,SAAS;AACb,OAAK,GAAG,UAAU,MAAM,WAAW;AACjC,YAAS;AACT,SAAA,OAAa,MAAM,QAAQ,MAAM,MAAM,MAAM,KAAK;IAClD;AAMF,OAAK,GAAG,UAAS,QACf,MAAA,GAAS,CAAC,UAAU,MAAA,aAAmB,IAAI,SAAS,WAClD,cAAc,KAAK,GAAG,KAAK,OAC3B,IACD,CACF;;CAGH,gCAAyB,IAAI,KAAqB;CAClD,UAAU;CAEV,UAAU,QAAgB,MAAkB,MAAsB;AAChE,OAAK,cAAc,QAAQ,MAAM,KAAK;EAEtC,MAAM,KAAK,EAAE,MAAA;AACb,QAAA,aAAmB,IAAI,IAAI,KAAK;EAChC,MAAM,EAAC,SAAS,YAAW,UAAU;AACrC,QAAA,MAAY,KAAK,QAAQ;AAEzB,SAAO,gBAAgB,eAAe;AACpC,SAAA,GAAS,QAAQ,GAAG,KAAK,UAAU,KAAK,KAAK,GAAG,MAAA,MAAY,MAAM;AAClE,SAAA,aAAmB,OAAO,GAAG;AAC7B,YAAS;IACT;AAEF,SAAO;;CAGT,eAAyB;AACvB,SAAO,CAAC,GAAG,MAAA,aAAmB,QAAQ,CAAC;;CAGzC,MAAM,kBAAkB;AACtB,QAAM,QAAQ,IAAI,MAAA,MAAY;;CAGhC,gBAAgB,KAAc,MAAc;AAE1C,QAAA,OAAa,IAAI,MAAM,KAAK,eAAe,MAAM,KAAA,EAAU;;CAG7D,QACE,MACA,KACA,KACA,MACA,MACA,QACA;AAEA,MAAI,QAAQ;AACV,SAAA,WAAiB,OAAO,OAAO;AAC/B,SAAA,IAAU,OAAO,OAAO;;EAG1B,MAAM,MAAM,QAAQ,OAAO,QAAQ;AAEnC,MAAI,SAAS,cAAc;AAMzB,OAAI,SAAS,MAAM,MAAA,eAAqB,KAAK,MAAA,WAAiB,OAAO,GACnE,QAAA;GAEF,MAAM,MACJ,SAAS,KAAK,SAAA,KACV,SACA;AACN,SAAA,GAAS,OAAO,GAAG,KAAK,IAAI,IAAI,sBAAsB,KAAK,IAAI,OAAO,GAAG;AACzE,UAAO,MAAA,KAAW,KAAK;;EAGzB,MAAM,MACJ,SAAS,KAAK,SAAA,KACV,SACA,MAAA,aAAmB,KAAK,SAAA,KACtB,SACA;AACR,QAAA,GAAS,OACP,MACI,GAAG,KAAK,IAAI,IAAI,iBAAiB,IAAI,KACrC,GAAG,KAAK,IAAI,IAAI,sBAAsB,KAAK,IAC/C,OAAO,GACR;AAGD,MAAI,MAAA,WAAiB,SAAS,GAAG;AAC/B,SAAA,GAAS,OACP,MAAA,aACI,oCACE,KAAK,KAAK,GAAG,MAAA,WACd,QACD,iCACL;AACD,UAAO,MAAA,KAAW,EAAE;;AAGtB,MAAI,MAAA,eAAqB,EAGvB,QAAO,MAAA,KAAW,QAAQ,GAAG;;CAMjC,MAAM,SAA+B,QAAwB;AAC3D,OAAK,MAAM,UAAU,QACnB,KAAI;AACF,UAAO,KAAK,OAAO;WACZ,GAAG;AACV,SAAA,GAAS,QAAQ,EAAE;;;;;;;;;;;AAc3B,eAAsB,eACpB,IACA,QACA,GAAG,UACY;AACf,KAAI,SAAS,WAAW,EACtB;AAEF,MAAK,MAAM,UAAU,CAAC,GAAG,mBAAmB,GAAG,kBAAkB,CAC/D,QAAO,KAAK,cAAc;EACxB,MAAM,mBAAmB;AAEzB,WAAS,QAAQ,OAAM,QAAO;AAC5B,OAAI,iBAAiB,SAAS,OAAO,IAAI,IAAI,OAAO;AAClD,OAAG,OAAO,YAAY,IAAI,YAAY,KAAK,GAAG,IAAI,GAAG,IAAI,OAAO,GAAG;AACnE,UAAM,IAAI,OAAO;;AAEnB,MAAG,OAAO,YAAY,IAAI,YAAY,KAAK,GAAG,IAAI,GAAG,IAAI,OAAO,GAAG;AACnE,SAAM,IAAI,MAAM;IAChB;GACF;AAGJ,KAAI;EAEF,MAAM,MAAM,MAAM,QAAQ,KACxB,SAAS,KAAI,QAAO,IAAI,KAAK,CAAC,WAAW,IAAI,CAAC,CAC/C;AACD,KAAG,OAAO,GAAG,IAAI,YAAY,KAAK,IAAI,IAAI,GAAG,WAAW;UACjD,GAAG;AACV,KAAG,QAAQ,oBAAoB,EAAE;AACjC,QAAM;;;AAIV,eAAsB,UAAU,KAA0B;AACxD,KAAI;AACF,QAAM,KAAK;AAEX,UAAQ,KAAK,OAAO,IAAI,mBAAmB;AAC3C,UAAQ,KAAK,EAAE;UACR,GAAG;AAEV,UAAQ,MAAM,OAAO,IAAI,sBAAsB,EAAE;AACjD,UAAQ,KAAK,GAAG;;;;;;;;;;;;;;AAepB,IAAa,mBAAb,MAA8B;CAC5B;CAEA;CACA;CACA;CACA,iBAAiB;CAEjB,YAAY,IAAgB,cAAsB;AAChD,QAAA,KAAW;AACX,QAAA,eAAqB;;CAGvB,YAAY,YAAiC;AAC3C,QAAA,gBAAsB,KAAK,KAAK;AAChC,MAAI,MAAA,uBAA6B,KAAA,GAAW;AAC1C,SAAA,GAAS,OACP,iCACE,MAAA,eAAqB,IACtB,mBACD,WACD;AAGD,SAAA,qBAA2B,YACzB,MAAA,mBACA,MAAA,eAAqB,EACtB;;;CAIL,2BAA2B;AAsBzB,QAAA,sBAA4B,mBAAmB;AAC7C,SAAA,sBAA4B,KAAA;GAC5B,MAAM,yBAAyB,KAAK,KAAK,GAAG,MAAA;AAC5C,OAAI,0BAA0B,MAAA,cAAoB;AAChD,UAAA,GAAS,OACP,2BACE,yBAAyB,IAC1B,yBACF;AACD,YAAQ,KAAK,QAAQ,KAAK,kBAAkB,GAAG;;IAEjD;;CAGJ,OAAO;AACL,eAAa,MAAA,mBAAyB;AACtC,MAAI,MAAA,oBACF,gBAAe,MAAA,oBAA0B"}

package/out/zero-cache/src/services/mutagen/mutagen.d.ts

@@ -2,7 +2,7 @@ import type { LogContext } from '@rocicorp/logger';
 import type { JWTPayload } from 'jose';
 import postgres from 'postgres';
 import { ErrorKind } from '../../../../zero-protocol/src/error-kind.ts';
-import { type CRUDMutation, type InsertOp, type Mutation, type UpsertOp } from '../../../../zero-protocol/src/push.ts';
+import { type CRUDMutation, type InsertOp, type Mutation, type UpsertOp } from '../../../../zero-protocol/src/mutation.ts';
 import type { DatabaseStorage } from '../../../../zqlite/src/database-storage.ts';
 import { type WriteAuthorizer } from '../../auth/write-authorizer.ts';
 import { type ZeroConfig } from '../../config/zero-config.ts';

package/out/zero-cache/src/services/mutagen/mutagen.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"mutagen.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/mutagen/mutagen.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,MAAM,CAAC;AACrC,OAAO,QAAQ,MAAM,UAAU,CAAC;AAGhC,OAAO,EAAC,SAAS,EAAC,MAAM,6CAA6C,CAAC;AAWtE,OAAO,EACL,KAAK,YAAY,EAEjB,KAAK,QAAQ,EACb,KAAK,QAAQ,EAEb,KAAK,QAAQ,EACd,MAAM,uCAAuC,CAAC;AAC/C,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,4CAA4C,CAAC;AAEhF,OAAO,EAEL,KAAK,eAAe,EACrB,MAAM,gCAAgC,CAAC;AACxC,OAAO,EAAC,KAAK,UAAU,EAAC,MAAM,6BAA6B,CAAC;AAK5D,OAAO,KAAK,EAAC,UAAU,EAAE,mBAAmB,EAAC,MAAM,mBAAmB,CAAC;AACvE,OAAO,EAAiB,KAAK,OAAO,EAAC,MAAM,uBAAuB,CAAC;AAEnE,OAAO,KAAK,EAAC,iBAAiB,EAAE,OAAO,EAAC,MAAM,eAAe,CAAC;AAK9D,MAAM,MAAM,aAAa,GAAG;IAC1B,IAAI,EAAE,SAAS,CAAC,cAAc,GAAG,SAAS,CAAC,mBAAmB;IAC9D,IAAI,EAAE,MAAM;CACb,CAAC;AAEF,MAAM,WAAW,OAAQ,SAAQ,iBAAiB;IAChD,eAAe,CACb,QAAQ,EAAE,QAAQ,EAClB,QAAQ,EAAE,UAAU,GAAG,SAAS,EAChC,qBAAqB,EAAE,OAAO,GAC7B,OAAO,CAAC,aAAa,GAAG,SAAS,CAAC,CAAC;CACvC;AAED,qBAAa,cAAe,YAAW,OAAO,EAAE,OAAO;;IACrD,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;gBAkBlB,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,aAAa,EAAE,MAAM,EACrB,QAAQ,EAAE,UAAU,EACpB,MAAM,EAAE,UAAU,EAClB,iBAAiB,EAAE,eAAe;IA0BpC,GAAG;IAKH,KAAK;IAQL,OAAO,IAAI,OAAO;IAIlB,eAAe,CACb,QAAQ,EAAE,QAAQ,EAClB,QAAQ,EAAE,UAAU,GAAG,SAAS,EAChC,qBAAqB,UAAQ,GAC5B,OAAO,CAAC,aAAa,GAAG,SAAS,CAAC;IAuBrC,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;IAIpB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;CAStB;AAID,wBAAsB,eAAe,CACnC,EAAE,EAAE,UAAU,EACd,QAAQ,EAAE,UAAU,GAAG,SAAS,EAChC,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,aAAa,EAAE,MAAM,EACrB,QAAQ,EAAE,QAAQ,EAClB,eAAe,EAAE,eAAe,EAChC,SAAS,CAAC,EAAE,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAAE,cAAc;AACtD,qBAAqB,UAAQ,GAC5B,OAAO,CAAC,aAAa,GAAG,SAAS,CAAC,CAqIpC;AAED,wBAAsB,qBAAqB,CACzC,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,mBAAmB,EACvB,QAAQ,EAAE,UAAU,GAAG,SAAS,EAChC,KAAK,EAAE,OAAO,EACd,aAAa,EAAE,MAAM,EACrB,QAAQ,EAAE,YAAY,EACtB,SAAS,EAAE,OAAO,EAClB,UAAU,EAAE,eAAe,iBA8D5B;AAED,wBAAgB,YAAY,CAC1B,EAAE,EAAE,QAAQ,CAAC,cAAc,EAC3B,MAAM,EAAE,QAAQ,GACf,QAAQ,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC,CAEvC;AAED,wBAAgB,YAAY,CAC1B,EAAE,EAAE,QAAQ,CAAC,cAAc,EAC3B,GAAG,EAAE,QAAQ,GACZ,QAAQ,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC,CAOvC"}
+{"version":3,"file":"mutagen.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/mutagen/mutagen.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,MAAM,CAAC;AACrC,OAAO,QAAQ,MAAM,UAAU,CAAC;AAGhC,OAAO,EAAC,SAAS,EAAC,MAAM,6CAA6C,CAAC;AAOtE,OAAO,EACL,KAAK,YAAY,EAEjB,KAAK,QAAQ,EACb,KAAK,QAAQ,EAEb,KAAK,QAAQ,EACd,MAAM,2CAA2C,CAAC;AAKnD,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,4CAA4C,CAAC;AAEhF,OAAO,EAEL,KAAK,eAAe,EACrB,MAAM,gCAAgC,CAAC;AACxC,OAAO,EAAC,KAAK,UAAU,EAAC,MAAM,6BAA6B,CAAC;AAK5D,OAAO,KAAK,EAAC,UAAU,EAAE,mBAAmB,EAAC,MAAM,mBAAmB,CAAC;AACvE,OAAO,EAAiB,KAAK,OAAO,EAAC,MAAM,uBAAuB,CAAC;AAEnE,OAAO,KAAK,EAAC,iBAAiB,EAAE,OAAO,EAAC,MAAM,eAAe,CAAC;AAK9D,MAAM,MAAM,aAAa,GAAG;IAC1B,IAAI,EAAE,SAAS,CAAC,cAAc,GAAG,SAAS,CAAC,mBAAmB;IAC9D,IAAI,EAAE,MAAM;CACb,CAAC;AAEF,MAAM,WAAW,OAAQ,SAAQ,iBAAiB;IAChD,eAAe,CACb,QAAQ,EAAE,QAAQ,EAClB,QAAQ,EAAE,UAAU,GAAG,SAAS,EAChC,qBAAqB,EAAE,OAAO,GAC7B,OAAO,CAAC,aAAa,GAAG,SAAS,CAAC,CAAC;CACvC;AAED,qBAAa,cAAe,YAAW,OAAO,EAAE,OAAO;;IACrD,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;gBAkBlB,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,aAAa,EAAE,MAAM,EACrB,QAAQ,EAAE,UAAU,EACpB,MAAM,EAAE,UAAU,EAClB,iBAAiB,EAAE,eAAe;IA0BpC,GAAG;IAKH,KAAK;IAQL,OAAO,IAAI,OAAO;IAIlB,eAAe,CACb,QAAQ,EAAE,QAAQ,EAClB,QAAQ,EAAE,UAAU,GAAG,SAAS,EAChC,qBAAqB,UAAQ,GAC5B,OAAO,CAAC,aAAa,GAAG,SAAS,CAAC;IAuBrC,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;IAIpB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;CAStB;AAID,wBAAsB,eAAe,CACnC,EAAE,EAAE,UAAU,EACd,QAAQ,EAAE,UAAU,GAAG,SAAS,EAChC,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,aAAa,EAAE,MAAM,EACrB,QAAQ,EAAE,QAAQ,EAClB,eAAe,EAAE,eAAe,EAChC,SAAS,CAAC,EAAE,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAAE,cAAc;AACtD,qBAAqB,UAAQ,GAC5B,OAAO,CAAC,aAAa,GAAG,SAAS,CAAC,CAqIpC;AAED,wBAAsB,qBAAqB,CACzC,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,mBAAmB,EACvB,QAAQ,EAAE,UAAU,GAAG,SAAS,EAChC,KAAK,EAAE,OAAO,EACd,aAAa,EAAE,MAAM,EACrB,QAAQ,EAAE,YAAY,EACtB,SAAS,EAAE,OAAO,EAClB,UAAU,EAAE,eAAe,iBA8D5B;AAED,wBAAgB,YAAY,CAC1B,EAAE,EAAE,QAAQ,CAAC,cAAc,EAC3B,MAAM,EAAE,QAAQ,GACf,QAAQ,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC,CAEvC;AAED,wBAAgB,YAAY,CAC1B,EAAE,EAAE,QAAQ,CAAC,cAAc,EAC3B,GAAG,EAAE,QAAQ,GACZ,QAAQ,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC,CAOvC"}

package/out/zero-cache/src/services/mutagen/mutagen.js

@@ -5,7 +5,7 @@ import { ZeroCache } from "../../../../zero-protocol/src/error-origin-enum.js";
 import { ProtocolError, isProtocolError } from "../../../../zero-protocol/src/error.js";
 import { CRUD } from "../../../../zero-protocol/src/mutation-type-enum.js";
 import { primaryKeyValueSchema } from "../../../../zero-protocol/src/primary-key.js";
-import "../../../../zero-protocol/src/push.js";
+import "../../../../zero-protocol/src/mutation.js";
 import { upstreamSchema } from "../../types/shards.js";
 import "../../config/zero-config.js";
 import { MutationAlreadyProcessedError } from "./error.js";

package/out/zero-cache/src/services/mutagen/mutagen.js.map

@@ -1 +1 @@
-{"version":3,"file":"mutagen.js","names":["#lc","#upstream","#shard","#stopped","#replica","#writeAuthorizer","#limiter","#crudMutations","#isStopped","#refCount"],"sources":["../../../../../../zero-cache/src/services/mutagen/mutagen.ts"],"sourcesContent":["import {PG_SERIALIZATION_FAILURE} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport type {JWTPayload} from 'jose';\nimport postgres from 'postgres';\nimport {assert, unreachable} from '../../../../shared/src/asserts.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport {ErrorKind} from '../../../../zero-protocol/src/error-kind.ts';\nimport {ErrorOrigin} from '../../../../zero-protocol/src/error-origin.ts';\nimport {\n  isProtocolError,\n  ProtocolError,\n} from '../../../../zero-protocol/src/error.ts';\nimport * as MutationType from '../../../../zero-protocol/src/mutation-type-enum.ts';\nimport {\n  primaryKeyValueSchema,\n  type PrimaryKeyValue,\n} from '../../../../zero-protocol/src/primary-key.ts';\nimport {\n  type CRUDMutation,\n  type DeleteOp,\n  type InsertOp,\n  type Mutation,\n  type UpdateOp,\n  type UpsertOp,\n} from '../../../../zero-protocol/src/push.ts';\nimport type {DatabaseStorage} from '../../../../zqlite/src/database-storage.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {\n  WriteAuthorizerImpl,\n  type WriteAuthorizer,\n} from '../../auth/write-authorizer.ts';\nimport {type ZeroConfig} from '../../config/zero-config.ts';\nimport * as Mode from '../../db/mode-enum.ts';\nimport {runTx} from '../../db/run-transaction.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport {recordMutation} from '../../server/anonymous-otel-start.ts';\nimport type {PostgresDB, PostgresTransaction} from '../../types/pg.ts';\nimport {upstreamSchema, type ShardID} from '../../types/shards.ts';\nimport {SlidingWindowLimiter} from '../limiter/sliding-window-limiter.ts';\nimport type {RefCountedService, Service} from '../service.ts';\nimport {MutationAlreadyProcessedError} from './error.ts';\n\n// An error encountered processing a mutation.\n// Returned back to application for display to user.\nexport type MutationError = [\n  kind: ErrorKind.MutationFailed | ErrorKind.MutationRateLimited,\n  desc: string,\n];\n\nexport interface Mutagen extends RefCountedService {\n  processMutation(\n    mutation: Mutation,\n    authData: JWTPayload | undefined,\n    customMutatorsEnabled: boolean,\n  ): Promise<MutationError | undefined>;\n}\n\nexport class MutagenService implements Mutagen, Service {\n  readonly id: string;\n  readonly #lc: LogContext;\n  readonly #upstream: PostgresDB;\n  readonly #shard: ShardID;\n  readonly #stopped = resolver();\n  readonly #replica: Database;\n  readonly #writeAuthorizer: WriteAuthorizerImpl;\n  readonly #limiter: SlidingWindowLimiter | undefined;\n  #refCount = 0;\n  #isStopped = false;\n\n  readonly #crudMutations = getOrCreateCounter(\n    'mutation',\n    'crud',\n    'Number of CRUD mutations processed',\n  );\n\n  constructor(\n    lc: LogContext,\n    shard: ShardID,\n    clientGroupID: string,\n    upstream: PostgresDB,\n    config: ZeroConfig,\n    writeAuthzStorage: DatabaseStorage,\n  ) {\n    this.id = clientGroupID;\n    this.#lc = lc;\n    this.#upstream = upstream;\n    this.#shard = shard;\n    this.#replica = new Database(this.#lc, config.replica.file, {\n      fileMustExist: true,\n    });\n    this.#writeAuthorizer = new WriteAuthorizerImpl(\n      this.#lc,\n      config,\n      this.#replica,\n      shard.appID,\n      clientGroupID,\n      writeAuthzStorage,\n    );\n\n    if (config.perUserMutationLimit.max !== undefined) {\n      this.#limiter = new SlidingWindowLimiter(\n        config.perUserMutationLimit.windowMs,\n        config.perUserMutationLimit.max,\n      );\n    }\n  }\n\n  ref() {\n    assert(!this.#isStopped, 'MutagenService is already stopped');\n    ++this.#refCount;\n  }\n\n  unref() {\n    assert(!this.#isStopped, 'MutagenService is already stopped');\n    --this.#refCount;\n    if (this.#refCount <= 0) {\n      void this.stop();\n    }\n  }\n\n  hasRefs(): boolean {\n    return this.#refCount > 0;\n  }\n\n  processMutation(\n    mutation: Mutation,\n    authData: JWTPayload | undefined,\n    customMutatorsEnabled = false,\n  ): Promise<MutationError | undefined> {\n    if (this.#limiter?.canDo() === false) {\n      return Promise.resolve([\n        ErrorKind.MutationRateLimited,\n        'Rate limit exceeded',\n      ]);\n    }\n    this.#crudMutations.add(1, {\n      clientGroupID: this.id,\n    });\n    return processMutation(\n      this.#lc,\n      authData,\n      this.#upstream,\n      this.#shard,\n      this.id,\n      mutation,\n      this.#writeAuthorizer,\n      undefined,\n      customMutatorsEnabled,\n    );\n  }\n\n  run(): Promise<void> {\n    return this.#stopped.promise;\n  }\n\n  stop(): Promise<void> {\n    if (this.#isStopped) {\n      return this.#stopped.promise;\n    }\n    this.#writeAuthorizer.destroy();\n    this.#isStopped = true;\n    this.#stopped.resolve();\n    return this.#stopped.promise;\n  }\n}\n\nconst MAX_SERIALIZATION_ATTEMPTS = 10;\n\nexport async function processMutation(\n  lc: LogContext,\n  authData: JWTPayload | undefined,\n  db: PostgresDB,\n  shard: ShardID,\n  clientGroupID: string,\n  mutation: Mutation,\n  writeAuthorizer: WriteAuthorizer,\n  onTxStart?: () => void | Promise<void>, // for testing\n  customMutatorsEnabled = false,\n): Promise<MutationError | undefined> {\n  assert(\n    mutation.type === MutationType.CRUD,\n    'Only CRUD mutations are supported',\n  );\n  lc = lc.withContext('mutationID', mutation.id);\n  lc = lc.withContext('processMutation');\n  lc.debug?.('Process mutation start', mutation);\n\n  // Record mutation processing attempt for telemetry (regardless of success/failure)\n  recordMutation('crud');\n\n  let result: MutationError | undefined;\n\n  const start = Date.now();\n  try {\n    // Mutations can fail for a variety of reasons:\n    //\n    // - application error\n    // - network/db error\n    // - zero bug\n    //\n    // For application errors what we want is to re-run the mutation in\n    // \"error mode\", which skips the actual mutation and just updates the\n    // lastMutationID. Then return the error to the app.\n    //\n    // However, it's hard to tell the difference between application errors\n    // and the other types.\n    //\n    // A reasonable policy ends up being to just retry every mutation once\n    // in error mode. If the error mode mutation succeeds then we assume it\n    // was an application error and return the error to the app. Otherwise,\n    // we know it was something internal and we log it.\n    //\n    // This is not 100% correct - there are theoretical cases where we\n    // return an internal error to the app that shouldn't have been. But it\n    // would have to be a crazy coincidence: we'd have to have a network\n    // error on the first attempt that resolves by the second attempt.\n    //\n    // One might ask why not try/catch just the calls to the mutators and\n    // consider those application errors. That is actually what we do in\n    // Replicache:\n    //\n    // https://github.com/rocicorp/todo-row-versioning/blob/9a0a79dc2d2de32c4fac61b5d1634bd9a9e66b7c/server/src/push.ts#L131\n    //\n    // We don't do it here because:\n    //\n    // 1. It's still not perfect. It's hard to isolate SQL errors in\n    //    mutators due to app developer mistakes from SQL errors due to\n    //    Zero mistakes.\n    // 2. It's not possible to do this with the pg library we're using in\n    //    Zero anyway: https://github.com/porsager/postgres/issues/455.\n    //\n    // Personally I think this simple retry policy is nice.\n    let errorMode = false;\n    for (let i = 0; i < MAX_SERIALIZATION_ATTEMPTS; i++) {\n      try {\n        await runTx(\n          db,\n          async tx => {\n            // Simulates a concurrent request for testing. In production this is a noop.\n            const done = onTxStart?.();\n            try {\n              return await processMutationWithTx(\n                lc,\n                tx,\n                authData,\n                shard,\n                clientGroupID,\n                mutation,\n                errorMode,\n                writeAuthorizer,\n              );\n            } finally {\n              await done;\n            }\n          },\n          {mode: Mode.SERIALIZABLE},\n        );\n        if (errorMode) {\n          lc.debug?.('Ran mutation successfully in error mode');\n        }\n        break;\n      } catch (e) {\n        if (e instanceof MutationAlreadyProcessedError) {\n          lc.debug?.(e.message);\n          // Don't double-count already processed mutations, but they were counted above\n          return undefined;\n        }\n        if (\n          isProtocolError(e) &&\n          !errorMode &&\n          e.kind === ErrorKind.InvalidPush &&\n          customMutatorsEnabled &&\n          i < 2\n        ) {\n          // We're temporarily supporting custom mutators AND CRUD mutators at the same time.\n          // This can create a lot of OOO mutation errors since we do not know when the API server\n          // has applied a custom mutation before moving on to process CRUD mutations.\n          // The temporary workaround (since CRUD is being deprecated) is to retry the mutation\n          // after a small delay. Users are not expected to be running both CRUD and Custom mutators.\n          // They should migrate completely to custom mutators.\n          lc.info?.(\n            'Both CRUD and Custom mutators are being used at once. This is supported for now but IS NOT RECOMMENDED. Migrate completely to custom mutators.',\n            e,\n          );\n          await new Promise(resolve => setTimeout(resolve, 100));\n          continue;\n        }\n        if (isProtocolError(e) || errorMode) {\n          lc.error?.('Process mutation error', e);\n          throw e;\n        }\n        if (\n          e instanceof postgres.PostgresError &&\n          e.code === PG_SERIALIZATION_FAILURE\n        ) {\n          lc.info?.(`attempt ${i + 1}: ${String(e)}`, e);\n          continue; // Retry up to MAX_SERIALIZATION_ATTEMPTS.\n        }\n        result = [ErrorKind.MutationFailed, String(e)];\n        if (errorMode) {\n          break;\n        }\n        lc.error?.('Got error running mutation, re-running in error mode', e);\n        errorMode = true;\n        i--;\n      }\n    }\n  } finally {\n    lc.debug?.('Process mutation complete in', Date.now() - start);\n  }\n  return result;\n}\n\nexport async function processMutationWithTx(\n  lc: LogContext,\n  tx: PostgresTransaction,\n  authData: JWTPayload | undefined,\n  shard: ShardID,\n  clientGroupID: string,\n  mutation: CRUDMutation,\n  errorMode: boolean,\n  authorizer: WriteAuthorizer,\n) {\n  const tasks: (() => Promise<unknown>)[] = [];\n\n  async function execute(stmt: postgres.PendingQuery<postgres.Row[]>) {\n    try {\n      return await stmt.execute();\n    } finally {\n      const q = stmt as unknown as Query;\n      lc.debug?.(`${q.string}: ${JSON.stringify(q.parameters)}`);\n    }\n  }\n\n  authorizer.reloadPermissions();\n\n  if (!errorMode) {\n    const {ops} = mutation.args[0];\n\n    // Validate all table names before processing\n    authorizer.validateTableNames(ops);\n\n    const normalizedOps = authorizer.normalizeOps(ops);\n    const [canPre, canPost] = await Promise.all([\n      authorizer.canPreMutation(authData, normalizedOps),\n      authorizer.canPostMutation(authData, normalizedOps),\n    ]);\n    if (canPre && canPost) {\n      for (const op of ops) {\n        switch (op.op) {\n          case 'insert':\n            tasks.push(() => execute(getInsertSQL(tx, op)));\n            break;\n          case 'upsert':\n            tasks.push(() => execute(getUpsertSQL(tx, op)));\n            break;\n          case 'update':\n            tasks.push(() => execute(getUpdateSQL(tx, op)));\n            break;\n          case 'delete':\n            tasks.push(() => execute(getDeleteSQL(tx, op)));\n            break;\n          default:\n            unreachable(op);\n        }\n      }\n    }\n  }\n\n  // Confirm the mutation even though it may have been blocked by the authorizer.\n  // Authorizer blocking a mutation is not an error but the correct result of the mutation.\n  tasks.unshift(() =>\n    checkSchemaVersionAndIncrementLastMutationID(\n      tx,\n      shard,\n      clientGroupID,\n      mutation.clientID,\n      mutation.id,\n    ),\n  );\n\n  // Note: An error thrown from any Promise aborts the entire transaction.\n  await Promise.all(tasks.map(task => task()));\n}\n\nexport function getInsertSQL(\n  tx: postgres.TransactionSql,\n  create: InsertOp,\n): postgres.PendingQuery<postgres.Row[]> {\n  return tx`INSERT INTO ${tx(create.tableName)} ${tx(create.value)}`;\n}\n\nexport function getUpsertSQL(\n  tx: postgres.TransactionSql,\n  set: UpsertOp,\n): postgres.PendingQuery<postgres.Row[]> {\n  const {tableName, primaryKey, value} = set;\n  return tx`\n    INSERT INTO ${tx(tableName)} ${tx(value)}\n    ON CONFLICT (${tx(primaryKey)})\n    DO UPDATE SET ${tx(value)}\n  `;\n}\n\nfunction getUpdateSQL(\n  tx: postgres.TransactionSql,\n  update: UpdateOp,\n): postgres.PendingQuery<postgres.Row[]> {\n  const table = update.tableName;\n  const {primaryKey, value} = update;\n  const id: Record<string, PrimaryKeyValue> = {};\n  for (const key of primaryKey) {\n    id[key] = v.parse(value[key], primaryKeyValueSchema);\n  }\n  return tx`UPDATE ${tx(table)} SET ${tx(value)} WHERE ${Object.entries(\n    id,\n  ).flatMap(([key, value], i) =>\n    i ? [tx`AND`, tx`${tx(key)} = ${value}`] : tx`${tx(key)} = ${value}`,\n  )}`;\n}\n\nfunction getDeleteSQL(\n  tx: postgres.TransactionSql,\n  deleteOp: DeleteOp,\n): postgres.PendingQuery<postgres.Row[]> {\n  const {tableName, primaryKey, value} = deleteOp;\n\n  const conditions = [];\n  for (const key of primaryKey) {\n    if (conditions.length > 0) {\n      conditions.push(tx`AND`);\n    }\n    conditions.push(tx`${tx(key)} = ${value[key]}`);\n  }\n\n  return tx`DELETE FROM ${tx(tableName)} WHERE ${conditions}`;\n}\n\nasync function checkSchemaVersionAndIncrementLastMutationID(\n  tx: PostgresTransaction,\n  shard: ShardID,\n  clientGroupID: string,\n  clientID: string,\n  receivedMutationID: number,\n) {\n  const [{lastMutationID}] = await tx<{lastMutationID: bigint}[]>`\n    INSERT INTO ${tx(upstreamSchema(shard))}.clients \n      as current (\"clientGroupID\", \"clientID\", \"lastMutationID\")\n          VALUES (${clientGroupID}, ${clientID}, ${1})\n      ON CONFLICT (\"clientGroupID\", \"clientID\")\n      DO UPDATE SET \"lastMutationID\" = current.\"lastMutationID\" + 1\n      RETURNING \"lastMutationID\"\n  `;\n\n  // ABORT if the resulting lastMutationID is not equal to the receivedMutationID.\n  if (receivedMutationID < lastMutationID) {\n    throw new MutationAlreadyProcessedError(\n      clientID,\n      receivedMutationID,\n      lastMutationID,\n    );\n  } else if (receivedMutationID > lastMutationID) {\n    throw new ProtocolError({\n      kind: ErrorKind.InvalidPush,\n      message: `Push contains unexpected mutation id ${receivedMutationID} for client ${clientID}. Expected mutation id ${lastMutationID.toString()}.`,\n      origin: ErrorOrigin.ZeroCache,\n    });\n  }\n}\n\n// The slice of information from the Query object in Postgres.js that gets logged for debugging.\n// https://github.com/porsager/postgres/blob/f58cd4f3affd3e8ce8f53e42799672d86cd2c70b/src/connection.js#L219\ntype Query = {string: string; parameters: object[]};\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;AA0DA,IAAa,iBAAb,MAAwD;CACtD;CACA;CACA;CACA;CACA,WAAoB,UAAU;CAC9B;CACA;CACA;CACA,YAAY;CACZ,aAAa;CAEb,iBAA0B,mBACxB,YACA,QACA,qCACD;CAED,YACE,IACA,OACA,eACA,UACA,QACA,mBACA;AACA,OAAK,KAAK;AACV,QAAA,KAAW;AACX,QAAA,WAAiB;AACjB,QAAA,QAAc;AACd,QAAA,UAAgB,IAAI,SAAS,MAAA,IAAU,OAAO,QAAQ,MAAM,EAC1D,eAAe,MAChB,CAAC;AACF,QAAA,kBAAwB,IAAI,oBAC1B,MAAA,IACA,QACA,MAAA,SACA,MAAM,OACN,eACA,kBACD;AAED,MAAI,OAAO,qBAAqB,QAAQ,KAAA,EACtC,OAAA,UAAgB,IAAI,qBAClB,OAAO,qBAAqB,UAC5B,OAAO,qBAAqB,IAC7B;;CAIL,MAAM;AACJ,SAAO,CAAC,MAAA,WAAiB,oCAAoC;AAC7D,IAAE,MAAA;;CAGJ,QAAQ;AACN,SAAO,CAAC,MAAA,WAAiB,oCAAoC;AAC7D,IAAE,MAAA;AACF,MAAI,MAAA,YAAkB,EACf,MAAK,MAAM;;CAIpB,UAAmB;AACjB,SAAO,MAAA,WAAiB;;CAG1B,gBACE,UACA,UACA,wBAAwB,OACY;AACpC,MAAI,MAAA,SAAe,OAAO,KAAK,MAC7B,QAAO,QAAQ,QAAQ,CACrB,qBACA,sBACD,CAAC;AAEJ,QAAA,cAAoB,IAAI,GAAG,EACzB,eAAe,KAAK,IACrB,CAAC;AACF,SAAO,gBACL,MAAA,IACA,UACA,MAAA,UACA,MAAA,OACA,KAAK,IACL,UACA,MAAA,iBACA,KAAA,GACA,sBACD;;CAGH,MAAqB;AACnB,SAAO,MAAA,QAAc;;CAGvB,OAAsB;AACpB,MAAI,MAAA,UACF,QAAO,MAAA,QAAc;AAEvB,QAAA,gBAAsB,SAAS;AAC/B,QAAA,YAAkB;AAClB,QAAA,QAAc,SAAS;AACvB,SAAO,MAAA,QAAc;;;AAIzB,IAAM,6BAA6B;AAEnC,eAAsB,gBACpB,IACA,UACA,IACA,OACA,eACA,UACA,iBACA,WACA,wBAAwB,OACY;AACpC,QACE,SAAS,SAAS,MAClB,oCACD;AACD,MAAK,GAAG,YAAY,cAAc,SAAS,GAAG;AAC9C,MAAK,GAAG,YAAY,kBAAkB;AACtC,IAAG,QAAQ,0BAA0B,SAAS;AAG9C,gBAAe,OAAO;CAEtB,IAAI;CAEJ,MAAM,QAAQ,KAAK,KAAK;AACxB,KAAI;EAuCF,IAAI,YAAY;AAChB,OAAK,IAAI,IAAI,GAAG,IAAI,4BAA4B,IAC9C,KAAI;AACF,SAAM,MACJ,IACA,OAAM,OAAM;IAEV,MAAM,OAAO,aAAa;AAC1B,QAAI;AACF,YAAO,MAAM,sBACX,IACA,IACA,UACA,OACA,eACA,UACA,WACA,gBACD;cACO;AACR,WAAM;;MAGV,EAAC,MAAM,cAAkB,CAC1B;AACD,OAAI,UACF,IAAG,QAAQ,0CAA0C;AAEvD;WACO,GAAG;AACV,OAAI,aAAa,+BAA+B;AAC9C,OAAG,QAAQ,EAAE,QAAQ;AAErB;;AAEF,OACE,gBAAgB,EAAE,IAClB,CAAC,aACD,EAAE,SAAS,iBACX,yBACA,IAAI,GACJ;AAOA,OAAG,OACD,kJACA,EACD;AACD,UAAM,IAAI,SAAQ,YAAW,WAAW,SAAS,IAAI,CAAC;AACtD;;AAEF,OAAI,gBAAgB,EAAE,IAAI,WAAW;AACnC,OAAG,QAAQ,0BAA0B,EAAE;AACvC,UAAM;;AAER,OACE,aAAa,SAAS,iBACtB,EAAE,SAAS,0BACX;AACA,OAAG,OAAO,WAAW,IAAI,EAAE,IAAI,OAAO,EAAE,IAAI,EAAE;AAC9C;;AAEF,YAAS,CAAC,gBAA0B,OAAO,EAAE,CAAC;AAC9C,OAAI,UACF;AAEF,MAAG,QAAQ,wDAAwD,EAAE;AACrE,eAAY;AACZ;;WAGI;AACR,KAAG,QAAQ,gCAAgC,KAAK,KAAK,GAAG,MAAM;;AAEhE,QAAO;;AAGT,eAAsB,sBACpB,IACA,IACA,UACA,OACA,eACA,UACA,WACA,YACA;CACA,MAAM,QAAoC,EAAE;CAE5C,eAAe,QAAQ,MAA6C;AAClE,MAAI;AACF,UAAO,MAAM,KAAK,SAAS;YACnB;GACR,MAAM,IAAI;AACV,MAAG,QAAQ,GAAG,EAAE,OAAO,IAAI,KAAK,UAAU,EAAE,WAAW,GAAG;;;AAI9D,YAAW,mBAAmB;AAE9B,KAAI,CAAC,WAAW;EACd,MAAM,EAAC,QAAO,SAAS,KAAK;AAG5B,aAAW,mBAAmB,IAAI;EAElC,MAAM,gBAAgB,WAAW,aAAa,IAAI;EAClD,MAAM,CAAC,QAAQ,WAAW,MAAM,QAAQ,IAAI,CAC1C,WAAW,eAAe,UAAU,cAAc,EAClD,WAAW,gBAAgB,UAAU,cAAc,CACpD,CAAC;AACF,MAAI,UAAU,QACZ,MAAK,MAAM,MAAM,IACf,SAAQ,GAAG,IAAX;GACE,KAAK;AACH,UAAM,WAAW,QAAQ,aAAa,IAAI,GAAG,CAAC,CAAC;AAC/C;GACF,KAAK;AACH,UAAM,WAAW,QAAQ,aAAa,IAAI,GAAG,CAAC,CAAC;AAC/C;GACF,KAAK;AACH,UAAM,WAAW,QAAQ,aAAa,IAAI,GAAG,CAAC,CAAC;AAC/C;GACF,KAAK;AACH,UAAM,WAAW,QAAQ,aAAa,IAAI,GAAG,CAAC,CAAC;AAC/C;GACF,QACE,aAAY,GAAG;;;AAQzB,OAAM,cACJ,6CACE,IACA,OACA,eACA,SAAS,UACT,SAAS,GACV,CACF;AAGD,OAAM,QAAQ,IAAI,MAAM,KAAI,SAAQ,MAAM,CAAC,CAAC;;AAG9C,SAAgB,aACd,IACA,QACuC;AACvC,QAAO,EAAE,eAAe,GAAG,OAAO,UAAU,CAAC,GAAG,GAAG,OAAO,MAAM;;AAGlE,SAAgB,aACd,IACA,KACuC;CACvC,MAAM,EAAC,WAAW,YAAY,UAAS;AACvC,QAAO,EAAE;kBACO,GAAG,UAAU,CAAC,GAAG,GAAG,MAAM,CAAC;mBAC1B,GAAG,WAAW,CAAC;oBACd,GAAG,MAAM,CAAC;;;AAI9B,SAAS,aACP,IACA,QACuC;CACvC,MAAM,QAAQ,OAAO;CACrB,MAAM,EAAC,YAAY,UAAS;CAC5B,MAAM,KAAsC,EAAE;AAC9C,MAAK,MAAM,OAAO,WAChB,IAAG,OAAO,MAAQ,MAAM,MAAM,sBAAsB;AAEtD,QAAO,EAAE,UAAU,GAAG,MAAM,CAAC,OAAO,GAAG,MAAM,CAAC,SAAS,OAAO,QAC5D,GACD,CAAC,SAAS,CAAC,KAAK,QAAQ,MACvB,IAAI,CAAC,EAAE,OAAO,EAAE,GAAG,GAAG,IAAI,CAAC,KAAK,QAAQ,GAAG,EAAE,GAAG,GAAG,IAAI,CAAC,KAAK,QAC9D;;AAGH,SAAS,aACP,IACA,UACuC;CACvC,MAAM,EAAC,WAAW,YAAY,UAAS;CAEvC,MAAM,aAAa,EAAE;AACrB,MAAK,MAAM,OAAO,YAAY;AAC5B,MAAI,WAAW,SAAS,EACtB,YAAW,KAAK,EAAE,MAAM;AAE1B,aAAW,KAAK,EAAE,GAAG,GAAG,IAAI,CAAC,KAAK,MAAM,OAAO;;AAGjD,QAAO,EAAE,eAAe,GAAG,UAAU,CAAC,SAAS;;AAGjD,eAAe,6CACb,IACA,OACA,eACA,UACA,oBACA;CACA,MAAM,CAAC,EAAC,oBAAmB,MAAM,EAA8B;kBAC/C,GAAG,eAAe,MAAM,CAAC,CAAC;;oBAExB,cAAc,IAAI,SAAS,IAAI,EAAE;;;;;AAOnD,KAAI,qBAAqB,eACvB,OAAM,IAAI,8BACR,UACA,oBACA,eACD;UACQ,qBAAqB,eAC9B,OAAM,IAAI,cAAc;EACtB,MAAM;EACN,SAAS,wCAAwC,mBAAmB,cAAc,SAAS,yBAAyB,eAAe,UAAU,CAAC;EAC9I,QAAQ;EACT,CAAC"}
+{"version":3,"file":"mutagen.js","names":["#lc","#upstream","#shard","#stopped","#replica","#writeAuthorizer","#limiter","#crudMutations","#isStopped","#refCount"],"sources":["../../../../../../zero-cache/src/services/mutagen/mutagen.ts"],"sourcesContent":["import {PG_SERIALIZATION_FAILURE} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport type {JWTPayload} from 'jose';\nimport postgres from 'postgres';\nimport {assert, unreachable} from '../../../../shared/src/asserts.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport {ErrorKind} from '../../../../zero-protocol/src/error-kind.ts';\nimport {ErrorOrigin} from '../../../../zero-protocol/src/error-origin.ts';\nimport {\n  isProtocolError,\n  ProtocolError,\n} from '../../../../zero-protocol/src/error.ts';\nimport * as MutationType from '../../../../zero-protocol/src/mutation-type-enum.ts';\nimport {\n  type CRUDMutation,\n  type DeleteOp,\n  type InsertOp,\n  type Mutation,\n  type UpdateOp,\n  type UpsertOp,\n} from '../../../../zero-protocol/src/mutation.ts';\nimport {\n  primaryKeyValueSchema,\n  type PrimaryKeyValue,\n} from '../../../../zero-protocol/src/primary-key.ts';\nimport type {DatabaseStorage} from '../../../../zqlite/src/database-storage.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {\n  WriteAuthorizerImpl,\n  type WriteAuthorizer,\n} from '../../auth/write-authorizer.ts';\nimport {type ZeroConfig} from '../../config/zero-config.ts';\nimport * as Mode from '../../db/mode-enum.ts';\nimport {runTx} from '../../db/run-transaction.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport {recordMutation} from '../../server/anonymous-otel-start.ts';\nimport type {PostgresDB, PostgresTransaction} from '../../types/pg.ts';\nimport {upstreamSchema, type ShardID} from '../../types/shards.ts';\nimport {SlidingWindowLimiter} from '../limiter/sliding-window-limiter.ts';\nimport type {RefCountedService, Service} from '../service.ts';\nimport {MutationAlreadyProcessedError} from './error.ts';\n\n// An error encountered processing a mutation.\n// Returned back to application for display to user.\nexport type MutationError = [\n  kind: ErrorKind.MutationFailed | ErrorKind.MutationRateLimited,\n  desc: string,\n];\n\nexport interface Mutagen extends RefCountedService {\n  processMutation(\n    mutation: Mutation,\n    authData: JWTPayload | undefined,\n    customMutatorsEnabled: boolean,\n  ): Promise<MutationError | undefined>;\n}\n\nexport class MutagenService implements Mutagen, Service {\n  readonly id: string;\n  readonly #lc: LogContext;\n  readonly #upstream: PostgresDB;\n  readonly #shard: ShardID;\n  readonly #stopped = resolver();\n  readonly #replica: Database;\n  readonly #writeAuthorizer: WriteAuthorizerImpl;\n  readonly #limiter: SlidingWindowLimiter | undefined;\n  #refCount = 0;\n  #isStopped = false;\n\n  readonly #crudMutations = getOrCreateCounter(\n    'mutation',\n    'crud',\n    'Number of CRUD mutations processed',\n  );\n\n  constructor(\n    lc: LogContext,\n    shard: ShardID,\n    clientGroupID: string,\n    upstream: PostgresDB,\n    config: ZeroConfig,\n    writeAuthzStorage: DatabaseStorage,\n  ) {\n    this.id = clientGroupID;\n    this.#lc = lc;\n    this.#upstream = upstream;\n    this.#shard = shard;\n    this.#replica = new Database(this.#lc, config.replica.file, {\n      fileMustExist: true,\n    });\n    this.#writeAuthorizer = new WriteAuthorizerImpl(\n      this.#lc,\n      config,\n      this.#replica,\n      shard.appID,\n      clientGroupID,\n      writeAuthzStorage,\n    );\n\n    if (config.perUserMutationLimit.max !== undefined) {\n      this.#limiter = new SlidingWindowLimiter(\n        config.perUserMutationLimit.windowMs,\n        config.perUserMutationLimit.max,\n      );\n    }\n  }\n\n  ref() {\n    assert(!this.#isStopped, 'MutagenService is already stopped');\n    ++this.#refCount;\n  }\n\n  unref() {\n    assert(!this.#isStopped, 'MutagenService is already stopped');\n    --this.#refCount;\n    if (this.#refCount <= 0) {\n      void this.stop();\n    }\n  }\n\n  hasRefs(): boolean {\n    return this.#refCount > 0;\n  }\n\n  processMutation(\n    mutation: Mutation,\n    authData: JWTPayload | undefined,\n    customMutatorsEnabled = false,\n  ): Promise<MutationError | undefined> {\n    if (this.#limiter?.canDo() === false) {\n      return Promise.resolve([\n        ErrorKind.MutationRateLimited,\n        'Rate limit exceeded',\n      ]);\n    }\n    this.#crudMutations.add(1, {\n      clientGroupID: this.id,\n    });\n    return processMutation(\n      this.#lc,\n      authData,\n      this.#upstream,\n      this.#shard,\n      this.id,\n      mutation,\n      this.#writeAuthorizer,\n      undefined,\n      customMutatorsEnabled,\n    );\n  }\n\n  run(): Promise<void> {\n    return this.#stopped.promise;\n  }\n\n  stop(): Promise<void> {\n    if (this.#isStopped) {\n      return this.#stopped.promise;\n    }\n    this.#writeAuthorizer.destroy();\n    this.#isStopped = true;\n    this.#stopped.resolve();\n    return this.#stopped.promise;\n  }\n}\n\nconst MAX_SERIALIZATION_ATTEMPTS = 10;\n\nexport async function processMutation(\n  lc: LogContext,\n  authData: JWTPayload | undefined,\n  db: PostgresDB,\n  shard: ShardID,\n  clientGroupID: string,\n  mutation: Mutation,\n  writeAuthorizer: WriteAuthorizer,\n  onTxStart?: () => void | Promise<void>, // for testing\n  customMutatorsEnabled = false,\n): Promise<MutationError | undefined> {\n  assert(\n    mutation.type === MutationType.CRUD,\n    'Only CRUD mutations are supported',\n  );\n  lc = lc.withContext('mutationID', mutation.id);\n  lc = lc.withContext('processMutation');\n  lc.debug?.('Process mutation start', mutation);\n\n  // Record mutation processing attempt for telemetry (regardless of success/failure)\n  recordMutation('crud');\n\n  let result: MutationError | undefined;\n\n  const start = Date.now();\n  try {\n    // Mutations can fail for a variety of reasons:\n    //\n    // - application error\n    // - network/db error\n    // - zero bug\n    //\n    // For application errors what we want is to re-run the mutation in\n    // \"error mode\", which skips the actual mutation and just updates the\n    // lastMutationID. Then return the error to the app.\n    //\n    // However, it's hard to tell the difference between application errors\n    // and the other types.\n    //\n    // A reasonable policy ends up being to just retry every mutation once\n    // in error mode. If the error mode mutation succeeds then we assume it\n    // was an application error and return the error to the app. Otherwise,\n    // we know it was something internal and we log it.\n    //\n    // This is not 100% correct - there are theoretical cases where we\n    // return an internal error to the app that shouldn't have been. But it\n    // would have to be a crazy coincidence: we'd have to have a network\n    // error on the first attempt that resolves by the second attempt.\n    //\n    // One might ask why not try/catch just the calls to the mutators and\n    // consider those application errors. That is actually what we do in\n    // Replicache:\n    //\n    // https://github.com/rocicorp/todo-row-versioning/blob/9a0a79dc2d2de32c4fac61b5d1634bd9a9e66b7c/server/src/push.ts#L131\n    //\n    // We don't do it here because:\n    //\n    // 1. It's still not perfect. It's hard to isolate SQL errors in\n    //    mutators due to app developer mistakes from SQL errors due to\n    //    Zero mistakes.\n    // 2. It's not possible to do this with the pg library we're using in\n    //    Zero anyway: https://github.com/porsager/postgres/issues/455.\n    //\n    // Personally I think this simple retry policy is nice.\n    let errorMode = false;\n    for (let i = 0; i < MAX_SERIALIZATION_ATTEMPTS; i++) {\n      try {\n        await runTx(\n          db,\n          async tx => {\n            // Simulates a concurrent request for testing. In production this is a noop.\n            const done = onTxStart?.();\n            try {\n              return await processMutationWithTx(\n                lc,\n                tx,\n                authData,\n                shard,\n                clientGroupID,\n                mutation,\n                errorMode,\n                writeAuthorizer,\n              );\n            } finally {\n              await done;\n            }\n          },\n          {mode: Mode.SERIALIZABLE},\n        );\n        if (errorMode) {\n          lc.debug?.('Ran mutation successfully in error mode');\n        }\n        break;\n      } catch (e) {\n        if (e instanceof MutationAlreadyProcessedError) {\n          lc.debug?.(e.message);\n          // Don't double-count already processed mutations, but they were counted above\n          return undefined;\n        }\n        if (\n          isProtocolError(e) &&\n          !errorMode &&\n          e.kind === ErrorKind.InvalidPush &&\n          customMutatorsEnabled &&\n          i < 2\n        ) {\n          // We're temporarily supporting custom mutators AND CRUD mutators at the same time.\n          // This can create a lot of OOO mutation errors since we do not know when the API server\n          // has applied a custom mutation before moving on to process CRUD mutations.\n          // The temporary workaround (since CRUD is being deprecated) is to retry the mutation\n          // after a small delay. Users are not expected to be running both CRUD and Custom mutators.\n          // They should migrate completely to custom mutators.\n          lc.info?.(\n            'Both CRUD and Custom mutators are being used at once. This is supported for now but IS NOT RECOMMENDED. Migrate completely to custom mutators.',\n            e,\n          );\n          await new Promise(resolve => setTimeout(resolve, 100));\n          continue;\n        }\n        if (isProtocolError(e) || errorMode) {\n          lc.error?.('Process mutation error', e);\n          throw e;\n        }\n        if (\n          e instanceof postgres.PostgresError &&\n          e.code === PG_SERIALIZATION_FAILURE\n        ) {\n          lc.info?.(`attempt ${i + 1}: ${String(e)}`, e);\n          continue; // Retry up to MAX_SERIALIZATION_ATTEMPTS.\n        }\n        result = [ErrorKind.MutationFailed, String(e)];\n        if (errorMode) {\n          break;\n        }\n        lc.error?.('Got error running mutation, re-running in error mode', e);\n        errorMode = true;\n        i--;\n      }\n    }\n  } finally {\n    lc.debug?.('Process mutation complete in', Date.now() - start);\n  }\n  return result;\n}\n\nexport async function processMutationWithTx(\n  lc: LogContext,\n  tx: PostgresTransaction,\n  authData: JWTPayload | undefined,\n  shard: ShardID,\n  clientGroupID: string,\n  mutation: CRUDMutation,\n  errorMode: boolean,\n  authorizer: WriteAuthorizer,\n) {\n  const tasks: (() => Promise<unknown>)[] = [];\n\n  async function execute(stmt: postgres.PendingQuery<postgres.Row[]>) {\n    try {\n      return await stmt.execute();\n    } finally {\n      const q = stmt as unknown as Query;\n      lc.debug?.(`${q.string}: ${JSON.stringify(q.parameters)}`);\n    }\n  }\n\n  authorizer.reloadPermissions();\n\n  if (!errorMode) {\n    const {ops} = mutation.args[0];\n\n    // Validate all table names before processing\n    authorizer.validateTableNames(ops);\n\n    const normalizedOps = authorizer.normalizeOps(ops);\n    const [canPre, canPost] = await Promise.all([\n      authorizer.canPreMutation(authData, normalizedOps),\n      authorizer.canPostMutation(authData, normalizedOps),\n    ]);\n    if (canPre && canPost) {\n      for (const op of ops) {\n        switch (op.op) {\n          case 'insert':\n            tasks.push(() => execute(getInsertSQL(tx, op)));\n            break;\n          case 'upsert':\n            tasks.push(() => execute(getUpsertSQL(tx, op)));\n            break;\n          case 'update':\n            tasks.push(() => execute(getUpdateSQL(tx, op)));\n            break;\n          case 'delete':\n            tasks.push(() => execute(getDeleteSQL(tx, op)));\n            break;\n          default:\n            unreachable(op);\n        }\n      }\n    }\n  }\n\n  // Confirm the mutation even though it may have been blocked by the authorizer.\n  // Authorizer blocking a mutation is not an error but the correct result of the mutation.\n  tasks.unshift(() =>\n    checkSchemaVersionAndIncrementLastMutationID(\n      tx,\n      shard,\n      clientGroupID,\n      mutation.clientID,\n      mutation.id,\n    ),\n  );\n\n  // Note: An error thrown from any Promise aborts the entire transaction.\n  await Promise.all(tasks.map(task => task()));\n}\n\nexport function getInsertSQL(\n  tx: postgres.TransactionSql,\n  create: InsertOp,\n): postgres.PendingQuery<postgres.Row[]> {\n  return tx`INSERT INTO ${tx(create.tableName)} ${tx(create.value)}`;\n}\n\nexport function getUpsertSQL(\n  tx: postgres.TransactionSql,\n  set: UpsertOp,\n): postgres.PendingQuery<postgres.Row[]> {\n  const {tableName, primaryKey, value} = set;\n  return tx`\n    INSERT INTO ${tx(tableName)} ${tx(value)}\n    ON CONFLICT (${tx(primaryKey)})\n    DO UPDATE SET ${tx(value)}\n  `;\n}\n\nfunction getUpdateSQL(\n  tx: postgres.TransactionSql,\n  update: UpdateOp,\n): postgres.PendingQuery<postgres.Row[]> {\n  const table = update.tableName;\n  const {primaryKey, value} = update;\n  const id: Record<string, PrimaryKeyValue> = {};\n  for (const key of primaryKey) {\n    id[key] = v.parse(value[key], primaryKeyValueSchema);\n  }\n  return tx`UPDATE ${tx(table)} SET ${tx(value)} WHERE ${Object.entries(\n    id,\n  ).flatMap(([key, value], i) =>\n    i ? [tx`AND`, tx`${tx(key)} = ${value}`] : tx`${tx(key)} = ${value}`,\n  )}`;\n}\n\nfunction getDeleteSQL(\n  tx: postgres.TransactionSql,\n  deleteOp: DeleteOp,\n): postgres.PendingQuery<postgres.Row[]> {\n  const {tableName, primaryKey, value} = deleteOp;\n\n  const conditions = [];\n  for (const key of primaryKey) {\n    if (conditions.length > 0) {\n      conditions.push(tx`AND`);\n    }\n    conditions.push(tx`${tx(key)} = ${value[key]}`);\n  }\n\n  return tx`DELETE FROM ${tx(tableName)} WHERE ${conditions}`;\n}\n\nasync function checkSchemaVersionAndIncrementLastMutationID(\n  tx: PostgresTransaction,\n  shard: ShardID,\n  clientGroupID: string,\n  clientID: string,\n  receivedMutationID: number,\n) {\n  const [{lastMutationID}] = await tx<{lastMutationID: bigint}[]>`\n    INSERT INTO ${tx(upstreamSchema(shard))}.clients \n      as current (\"clientGroupID\", \"clientID\", \"lastMutationID\")\n          VALUES (${clientGroupID}, ${clientID}, ${1})\n      ON CONFLICT (\"clientGroupID\", \"clientID\")\n      DO UPDATE SET \"lastMutationID\" = current.\"lastMutationID\" + 1\n      RETURNING \"lastMutationID\"\n  `;\n\n  // ABORT if the resulting lastMutationID is not equal to the receivedMutationID.\n  if (receivedMutationID < lastMutationID) {\n    throw new MutationAlreadyProcessedError(\n      clientID,\n      receivedMutationID,\n      lastMutationID,\n    );\n  } else if (receivedMutationID > lastMutationID) {\n    throw new ProtocolError({\n      kind: ErrorKind.InvalidPush,\n      message: `Push contains unexpected mutation id ${receivedMutationID} for client ${clientID}. Expected mutation id ${lastMutationID.toString()}.`,\n      origin: ErrorOrigin.ZeroCache,\n    });\n  }\n}\n\n// The slice of information from the Query object in Postgres.js that gets logged for debugging.\n// https://github.com/porsager/postgres/blob/f58cd4f3affd3e8ce8f53e42799672d86cd2c70b/src/connection.js#L219\ntype Query = {string: string; parameters: object[]};\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;AA0DA,IAAa,iBAAb,MAAwD;CACtD;CACA;CACA;CACA;CACA,WAAoB,UAAU;CAC9B;CACA;CACA;CACA,YAAY;CACZ,aAAa;CAEb,iBAA0B,mBACxB,YACA,QACA,qCACD;CAED,YACE,IACA,OACA,eACA,UACA,QACA,mBACA;AACA,OAAK,KAAK;AACV,QAAA,KAAW;AACX,QAAA,WAAiB;AACjB,QAAA,QAAc;AACd,QAAA,UAAgB,IAAI,SAAS,MAAA,IAAU,OAAO,QAAQ,MAAM,EAC1D,eAAe,MAChB,CAAC;AACF,QAAA,kBAAwB,IAAI,oBAC1B,MAAA,IACA,QACA,MAAA,SACA,MAAM,OACN,eACA,kBACD;AAED,MAAI,OAAO,qBAAqB,QAAQ,KAAA,EACtC,OAAA,UAAgB,IAAI,qBAClB,OAAO,qBAAqB,UAC5B,OAAO,qBAAqB,IAC7B;;CAIL,MAAM;AACJ,SAAO,CAAC,MAAA,WAAiB,oCAAoC;AAC7D,IAAE,MAAA;;CAGJ,QAAQ;AACN,SAAO,CAAC,MAAA,WAAiB,oCAAoC;AAC7D,IAAE,MAAA;AACF,MAAI,MAAA,YAAkB,EACf,MAAK,MAAM;;CAIpB,UAAmB;AACjB,SAAO,MAAA,WAAiB;;CAG1B,gBACE,UACA,UACA,wBAAwB,OACY;AACpC,MAAI,MAAA,SAAe,OAAO,KAAK,MAC7B,QAAO,QAAQ,QAAQ,CACrB,qBACA,sBACD,CAAC;AAEJ,QAAA,cAAoB,IAAI,GAAG,EACzB,eAAe,KAAK,IACrB,CAAC;AACF,SAAO,gBACL,MAAA,IACA,UACA,MAAA,UACA,MAAA,OACA,KAAK,IACL,UACA,MAAA,iBACA,KAAA,GACA,sBACD;;CAGH,MAAqB;AACnB,SAAO,MAAA,QAAc;;CAGvB,OAAsB;AACpB,MAAI,MAAA,UACF,QAAO,MAAA,QAAc;AAEvB,QAAA,gBAAsB,SAAS;AAC/B,QAAA,YAAkB;AAClB,QAAA,QAAc,SAAS;AACvB,SAAO,MAAA,QAAc;;;AAIzB,IAAM,6BAA6B;AAEnC,eAAsB,gBACpB,IACA,UACA,IACA,OACA,eACA,UACA,iBACA,WACA,wBAAwB,OACY;AACpC,QACE,SAAS,SAAS,MAClB,oCACD;AACD,MAAK,GAAG,YAAY,cAAc,SAAS,GAAG;AAC9C,MAAK,GAAG,YAAY,kBAAkB;AACtC,IAAG,QAAQ,0BAA0B,SAAS;AAG9C,gBAAe,OAAO;CAEtB,IAAI;CAEJ,MAAM,QAAQ,KAAK,KAAK;AACxB,KAAI;EAuCF,IAAI,YAAY;AAChB,OAAK,IAAI,IAAI,GAAG,IAAI,4BAA4B,IAC9C,KAAI;AACF,SAAM,MACJ,IACA,OAAM,OAAM;IAEV,MAAM,OAAO,aAAa;AAC1B,QAAI;AACF,YAAO,MAAM,sBACX,IACA,IACA,UACA,OACA,eACA,UACA,WACA,gBACD;cACO;AACR,WAAM;;MAGV,EAAC,MAAM,cAAkB,CAC1B;AACD,OAAI,UACF,IAAG,QAAQ,0CAA0C;AAEvD;WACO,GAAG;AACV,OAAI,aAAa,+BAA+B;AAC9C,OAAG,QAAQ,EAAE,QAAQ;AAErB;;AAEF,OACE,gBAAgB,EAAE,IAClB,CAAC,aACD,EAAE,SAAS,iBACX,yBACA,IAAI,GACJ;AAOA,OAAG,OACD,kJACA,EACD;AACD,UAAM,IAAI,SAAQ,YAAW,WAAW,SAAS,IAAI,CAAC;AACtD;;AAEF,OAAI,gBAAgB,EAAE,IAAI,WAAW;AACnC,OAAG,QAAQ,0BAA0B,EAAE;AACvC,UAAM;;AAER,OACE,aAAa,SAAS,iBACtB,EAAE,SAAS,0BACX;AACA,OAAG,OAAO,WAAW,IAAI,EAAE,IAAI,OAAO,EAAE,IAAI,EAAE;AAC9C;;AAEF,YAAS,CAAC,gBAA0B,OAAO,EAAE,CAAC;AAC9C,OAAI,UACF;AAEF,MAAG,QAAQ,wDAAwD,EAAE;AACrE,eAAY;AACZ;;WAGI;AACR,KAAG,QAAQ,gCAAgC,KAAK,KAAK,GAAG,MAAM;;AAEhE,QAAO;;AAGT,eAAsB,sBACpB,IACA,IACA,UACA,OACA,eACA,UACA,WACA,YACA;CACA,MAAM,QAAoC,EAAE;CAE5C,eAAe,QAAQ,MAA6C;AAClE,MAAI;AACF,UAAO,MAAM,KAAK,SAAS;YACnB;GACR,MAAM,IAAI;AACV,MAAG,QAAQ,GAAG,EAAE,OAAO,IAAI,KAAK,UAAU,EAAE,WAAW,GAAG;;;AAI9D,YAAW,mBAAmB;AAE9B,KAAI,CAAC,WAAW;EACd,MAAM,EAAC,QAAO,SAAS,KAAK;AAG5B,aAAW,mBAAmB,IAAI;EAElC,MAAM,gBAAgB,WAAW,aAAa,IAAI;EAClD,MAAM,CAAC,QAAQ,WAAW,MAAM,QAAQ,IAAI,CAC1C,WAAW,eAAe,UAAU,cAAc,EAClD,WAAW,gBAAgB,UAAU,cAAc,CACpD,CAAC;AACF,MAAI,UAAU,QACZ,MAAK,MAAM,MAAM,IACf,SAAQ,GAAG,IAAX;GACE,KAAK;AACH,UAAM,WAAW,QAAQ,aAAa,IAAI,GAAG,CAAC,CAAC;AAC/C;GACF,KAAK;AACH,UAAM,WAAW,QAAQ,aAAa,IAAI,GAAG,CAAC,CAAC;AAC/C;GACF,KAAK;AACH,UAAM,WAAW,QAAQ,aAAa,IAAI,GAAG,CAAC,CAAC;AAC/C;GACF,KAAK;AACH,UAAM,WAAW,QAAQ,aAAa,IAAI,GAAG,CAAC,CAAC;AAC/C;GACF,QACE,aAAY,GAAG;;;AAQzB,OAAM,cACJ,6CACE,IACA,OACA,eACA,SAAS,UACT,SAAS,GACV,CACF;AAGD,OAAM,QAAQ,IAAI,MAAM,KAAI,SAAQ,MAAM,CAAC,CAAC;;AAG9C,SAAgB,aACd,IACA,QACuC;AACvC,QAAO,EAAE,eAAe,GAAG,OAAO,UAAU,CAAC,GAAG,GAAG,OAAO,MAAM;;AAGlE,SAAgB,aACd,IACA,KACuC;CACvC,MAAM,EAAC,WAAW,YAAY,UAAS;AACvC,QAAO,EAAE;kBACO,GAAG,UAAU,CAAC,GAAG,GAAG,MAAM,CAAC;mBAC1B,GAAG,WAAW,CAAC;oBACd,GAAG,MAAM,CAAC;;;AAI9B,SAAS,aACP,IACA,QACuC;CACvC,MAAM,QAAQ,OAAO;CACrB,MAAM,EAAC,YAAY,UAAS;CAC5B,MAAM,KAAsC,EAAE;AAC9C,MAAK,MAAM,OAAO,WAChB,IAAG,OAAO,MAAQ,MAAM,MAAM,sBAAsB;AAEtD,QAAO,EAAE,UAAU,GAAG,MAAM,CAAC,OAAO,GAAG,MAAM,CAAC,SAAS,OAAO,QAC5D,GACD,CAAC,SAAS,CAAC,KAAK,QAAQ,MACvB,IAAI,CAAC,EAAE,OAAO,EAAE,GAAG,GAAG,IAAI,CAAC,KAAK,QAAQ,GAAG,EAAE,GAAG,GAAG,IAAI,CAAC,KAAK,QAC9D;;AAGH,SAAS,aACP,IACA,UACuC;CACvC,MAAM,EAAC,WAAW,YAAY,UAAS;CAEvC,MAAM,aAAa,EAAE;AACrB,MAAK,MAAM,OAAO,YAAY;AAC5B,MAAI,WAAW,SAAS,EACtB,YAAW,KAAK,EAAE,MAAM;AAE1B,aAAW,KAAK,EAAE,GAAG,GAAG,IAAI,CAAC,KAAK,MAAM,OAAO;;AAGjD,QAAO,EAAE,eAAe,GAAG,UAAU,CAAC,SAAS;;AAGjD,eAAe,6CACb,IACA,OACA,eACA,UACA,oBACA;CACA,MAAM,CAAC,EAAC,oBAAmB,MAAM,EAA8B;kBAC/C,GAAG,eAAe,MAAM,CAAC,CAAC;;oBAExB,cAAc,IAAI,SAAS,IAAI,EAAE;;;;;AAOnD,KAAI,qBAAqB,eACvB,OAAM,IAAI,8BACR,UACA,oBACA,eACD;UACQ,qBAAqB,eAC9B,OAAM,IAAI,cAAc;EACtB,MAAM;EACN,SAAS,wCAAwC,mBAAmB,cAAc,SAAS,yBAAyB,eAAe,UAAU,CAAC;EAC9I,QAAQ;EACT,CAAC"}

package/out/zero-cache/src/services/mutagen/pusher.d.ts

@@ -1,6 +1,7 @@
 import type { LogContext } from '@rocicorp/logger';
 import type { Downstream } from '../../../../zero-protocol/src/down.ts';
-import { type MutationID, type PushBody } from '../../../../zero-protocol/src/push.ts';
+import type { MutationID } from '../../../../zero-protocol/src/mutation-id.ts';
+import { type PushBody } from '../../../../zero-protocol/src/push.ts';
 import { type ZeroConfig } from '../../config/zero-config.ts';
 import type { Source } from '../../types/streams.ts';
 import { Subscription } from '../../types/subscription.ts';
@@ -30,7 +31,7 @@ type Config = Pick<ZeroConfig, 'app' | 'shard'>;
 export declare class PusherService implements Service, Pusher {
     #private;
     readonly id: string;
-    constructor(appConfig: Config, lc: LogContext, clientGroupID: string, contextManager: ConnectionContextManager);
+    constructor(appConfig: Config, lc: LogContext, clientGroupID: string, connContextManager: ConnectionContextManager);
     initConnection(selector: ConnectionSelector): Subscription<["deleteClients", {
         readonly clientIDs?: readonly string[] | undefined;
         readonly clientGroupIDs?: readonly string[] | undefined;
@@ -647,7 +648,7 @@ export declare class PusherService implements Service, Pusher {
 }
 type PusherEntry = {
     push: PushBody;
-    context: ConnectionContext;
+    connCtx: ConnectionContext;
 };
 type PusherEntryOrStop = PusherEntry | 'stop';
 /**

package/out/zero-cache/src/services/mutagen/pusher.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"pusher.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/mutagen/pusher.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAMjD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,uCAAuC,CAAC;AAStE,OAAO,EAGL,KAAK,UAAU,EACf,KAAK,QAAQ,EAEd,MAAM,uCAAuC,CAAC;AAE/C,OAAO,EAAC,KAAK,UAAU,EAAC,MAAM,6BAA6B,CAAC;AAK5D,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,wBAAwB,CAAC;AACnD,OAAO,EAAC,YAAY,EAAC,MAAM,6BAA6B,CAAC;AACzD,OAAO,KAAK,EAAC,aAAa,EAAE,YAAY,EAAC,MAAM,6BAA6B,CAAC;AAC7E,OAAO,KAAK,EAAC,iBAAiB,EAAE,OAAO,EAAC,MAAM,eAAe,CAAC;AAC9D,OAAO,KAAK,EACV,iBAAiB,EACjB,wBAAwB,EACxB,kBAAkB,EACnB,MAAM,8CAA8C,CAAC;AAEtD,MAAM,WAAW,MAAO,SAAQ,iBAAiB;IAC/C,cAAc,CAAC,QAAQ,EAAE,kBAAkB,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC;IACjE,WAAW,CAAC,QAAQ,EAAE,kBAAkB,EAAE,IAAI,EAAE,QAAQ,GAAG,aAAa,CAAC;IACzE,oBAAoB,CAClB,SAAS,EAAE,kBAAkB,EAC7B,MAAM,EAAE,UAAU,GACjB,OAAO,CAAC,IAAI,CAAC,CAAC;IACjB,qBAAqB,CACnB,SAAS,EAAE,kBAAkB,EAC7B,SAAS,EAAE,MAAM,EAAE,GAClB,OAAO,CAAC,IAAI,CAAC,CAAC;CAClB;AAED,KAAK,MAAM,GAAG,IAAI,CAAC,UAAU,EAAE,KAAK,GAAG,OAAO,CAAC,CAAC;AAEhD;;;;;;;;;;;;GAYG;AACH,qBAAa,aAAc,YAAW,OAAO,EAAE,MAAM;;IACnD,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;gBAWlB,SAAS,EAAE,MAAM,EACjB,EAAE,EAAE,UAAU,EACd,aAAa,EAAE,MAAM,EACrB,cAAc,EAAE,wBAAwB;IAe1C,cAAc,CAAC,QAAQ,EAAE,kBAAkB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAI3C,WAAW,CACT,QAAQ,EAAE,kBAAkB,EAC5B,IAAI,EAAE,QAAQ,GACb,OAAO,CAAC,aAAa,EAAE,YAAY,CAAC;IAWjC,oBAAoB,CACxB,SAAS,EAAE,kBAAkB,EAC7B,MAAM,EAAE,UAAU,GACjB,OAAO,CAAC,IAAI,CAAC;IA+ChB;;;;;OAKG;IACG,qBAAqB,CACzB,SAAS,EAAE,kBAAkB,EAC7B,SAAS,EAAE,MAAM,EAAE,GAClB,OAAO,CAAC,IAAI,CAAC;IAkDhB,GAAG;IAKH,KAAK;IAQL,OAAO,IAAI,OAAO;IAIlB,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;IAKpB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;CAQtB;AAED,KAAK,WAAW,GAAG;IACjB,IAAI,EAAE,QAAQ,CAAC;IACf,OAAO,EAAE,iBAAiB,CAAC;CAC5B,CAAC;AACF,KAAK,iBAAiB,GAAG,WAAW,GAAG,MAAM,CAAC;AAsU9C;;;;;GAKG;AACH,wBAAgB,aAAa,CAC3B,OAAO,EAAE,SAAS,CAAC,iBAAiB,GAAG,SAAS,CAAC,EAAE,GAClD,CAAC,WAAW,EAAE,EAAE,OAAO,CAAC,CAqC1B"}
+{"version":3,"file":"pusher.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/mutagen/pusher.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAMjD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,uCAAuC,CAAC;AAYtE,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,8CAA8C,CAAC;AAG7E,OAAO,EAAC,KAAK,QAAQ,EAAC,MAAM,uCAAuC,CAAC;AAEpE,OAAO,EAAC,KAAK,UAAU,EAAC,MAAM,6BAA6B,CAAC;AAK5D,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,wBAAwB,CAAC;AACnD,OAAO,EAAC,YAAY,EAAC,MAAM,6BAA6B,CAAC;AACzD,OAAO,KAAK,EAAC,aAAa,EAAE,YAAY,EAAC,MAAM,6BAA6B,CAAC;AAC7E,OAAO,KAAK,EAAC,iBAAiB,EAAE,OAAO,EAAC,MAAM,eAAe,CAAC;AAC9D,OAAO,KAAK,EACV,iBAAiB,EACjB,wBAAwB,EACxB,kBAAkB,EACnB,MAAM,8CAA8C,CAAC;AAEtD,MAAM,WAAW,MAAO,SAAQ,iBAAiB;IAC/C,cAAc,CAAC,QAAQ,EAAE,kBAAkB,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC;IACjE,WAAW,CAAC,QAAQ,EAAE,kBAAkB,EAAE,IAAI,EAAE,QAAQ,GAAG,aAAa,CAAC;IACzE,oBAAoB,CAClB,SAAS,EAAE,kBAAkB,EAC7B,MAAM,EAAE,UAAU,GACjB,OAAO,CAAC,IAAI,CAAC,CAAC;IACjB,qBAAqB,CACnB,SAAS,EAAE,kBAAkB,EAC7B,SAAS,EAAE,MAAM,EAAE,GAClB,OAAO,CAAC,IAAI,CAAC,CAAC;CAClB;AAED,KAAK,MAAM,GAAG,IAAI,CAAC,UAAU,EAAE,KAAK,GAAG,OAAO,CAAC,CAAC;AAEhD;;;;;;;;;;;;GAYG;AACH,qBAAa,aAAc,YAAW,OAAO,EAAE,MAAM;;IACnD,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;gBAWlB,SAAS,EAAE,MAAM,EACjB,EAAE,EAAE,UAAU,EACd,aAAa,EAAE,MAAM,EACrB,kBAAkB,EAAE,wBAAwB;IAe9C,cAAc,CAAC,QAAQ,EAAE,kBAAkB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAI3C,WAAW,CACT,QAAQ,EAAE,kBAAkB,EAC5B,IAAI,EAAE,QAAQ,GACb,OAAO,CAAC,aAAa,EAAE,YAAY,CAAC;IAWjC,oBAAoB,CACxB,SAAS,EAAE,kBAAkB,EAC7B,MAAM,EAAE,UAAU,GACjB,OAAO,CAAC,IAAI,CAAC;IA+ChB;;;;;OAKG;IACG,qBAAqB,CACzB,SAAS,EAAE,kBAAkB,EAC7B,SAAS,EAAE,MAAM,EAAE,GAClB,OAAO,CAAC,IAAI,CAAC;IAkDhB,GAAG;IAKH,KAAK;IAQL,OAAO,IAAI,OAAO;IAIlB,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;IAKpB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;CAQtB;AAED,KAAK,WAAW,GAAG;IACjB,IAAI,EAAE,QAAQ,CAAC;IACf,OAAO,EAAE,iBAAiB,CAAC;CAC5B,CAAC;AACF,KAAK,iBAAiB,GAAG,WAAW,GAAG,MAAM,CAAC;AAoW9C;;;;;GAKG;AACH,wBAAgB,aAAa,CAC3B,OAAO,EAAE,SAAS,CAAC,iBAAiB,GAAG,SAAS,CAAC,EAAE,GAClD,CAAC,WAAW,EAAE,EAAE,OAAO,CAAC,CAqC1B"}