@rocicorp/zero 1.11.3 → 1.11.5

package/out/zero-cache/src/services/replicator/incremental-sync.js.map

@@ -1 +1 @@
-{"version":3,"file":"incremental-sync.js","names":[],"sources":["../../../../../../zero-cache/src/services/replicator/incremental-sync.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {AbortError} from '../../../../shared/src/abort-error.ts';\nimport type {Enum} from '../../../../shared/src/enum.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {mapPostgresToLiteIndex} from '../../db/pg-to-lite.ts';\nimport {StatementRunner} from '../../db/statements.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport type {Source} from '../../types/streams.ts';\nimport type {DownloadStatus} from '../change-source/protocol/current.ts';\nimport type {ChangeStreamData} from '../change-source/protocol/current/downstream.ts';\nimport type {ReservationFollowup} from '../change-streamer/change-streamer-http.ts';\nimport {\n  errorTypeToReadableName,\n  PROTOCOL_VERSION,\n  type ChangeStreamer,\n  type SubscriberContext,\n} from '../change-streamer/change-streamer.ts';\nimport type * as ErrorType from '../change-streamer/error-type-enum.ts';\nimport {WrongReplicaVersion} from '../change-streamer/error-type-enum.ts';\nimport type {SubscribeDownstream} from '../change-streamer/subscribe.ts';\nimport {RunningState} from '../running-state.ts';\nimport type {CommitResult} from './change-processor.ts';\nimport {\n  DrainHandler,\n  type Downstream,\n  type SwitchoverOptions,\n} from './drain-handler.ts';\nimport {Notifier} from './notifier.ts';\nimport {\n  IndexingProgress,\n  type ReplicationStatusPublisher,\n} from './replication-status.ts';\nimport type {ReplicaState, ReplicatorMode} from './replicator.ts';\nimport {\n  ReplicationReportRecorder,\n  type ReplicationLagReadinessOptions,\n} from './reporter/recorder.ts';\nimport type {ReplicationReport} from './reporter/report-schema.ts';\nimport {readReplicaBackfills} from './schema/backfilling.ts';\nimport {getSubscriptionState} from './schema/replication-state.ts';\nimport type {WriteWorkerClient} from './write-worker-client.ts';\n\ntype ErrorType = Enum<typeof ErrorType>;\n\nconst MAX_WORKER_BATCH_MESSAGES = 256;\nconst MAX_WORKER_BATCH_SIZE = 1024 * 1024;\n\n// The number of times to reconnect after being rejected by a\n// replication-manager with an older replica version (e.g. one that has yet\n// to be replaced by a resync), before accepting the rejection and restarting\n// (e.g. if the upstream was legitimately replaced by one with a lower replica\n// version).\nconst MAX_OLDER_REPLICA_VERSION_REJECTIONS = 20;\n\n/**\n * The {@link IncrementalSyncer} manages a logical replication stream from upstream,\n * handling application lifecycle events (start, stop) and retrying the\n * connection with exponential backoff. The actual handling of the logical\n * replication messages is done by the {@link ChangeProcessor}, which runs\n * in a worker thread via the {@link WriteWorkerClient}.\n */\nexport class IncrementalSyncer {\n  readonly #lc: LogContext;\n  readonly #taskID: string;\n  readonly #id: string;\n  readonly #changeStreamer: ChangeStreamer;\n  readonly #worker: WriteWorkerClient;\n  readonly #mode: ReplicatorMode;\n  readonly #statusPublisher: ReplicationStatusPublisher | null;\n  readonly #notifier: Notifier;\n  readonly #reporter: ReplicationReportRecorder;\n  // Consumed (and cleared) by the first subscribe() in run()'s loop; every\n  // subsequent (re)connection goes through #changeStreamer.subscribe().\n  #initialConnection: ReservationFollowup | undefined;\n\n  // Only set for serving replicas that switch over between\n  // replication-managers.\n  readonly #drainHandler: DrainHandler | undefined;\n  #olderReplicaVersionRejections = 0;\n\n  readonly #state = new RunningState('IncrementalSyncer');\n\n  readonly #replicationEvents = getOrCreateCounter(\n    'replication',\n    'events',\n    'Number of replication events processed',\n  );\n\n  constructor(\n    lc: LogContext,\n    taskID: string,\n    id: string,\n    changeStreamer: ChangeStreamer,\n    worker: WriteWorkerClient,\n    mode: ReplicatorMode,\n    statusPublisher: ReplicationStatusPublisher | null,\n    replicationLagReadinessOptions: ReplicationLagReadinessOptions = {},\n    initialConnection?: ReservationFollowup,\n    switchover?: SwitchoverOptions,\n  ) {\n    this.#lc = lc;\n    this.#taskID = taskID;\n    this.#id = id;\n    this.#changeStreamer = changeStreamer;\n    this.#worker = worker;\n    this.#mode = mode;\n    this.#statusPublisher = statusPublisher;\n    this.#notifier = new Notifier();\n    this.#reporter = new ReplicationReportRecorder(\n      lc,\n      replicationLagReadinessOptions,\n    );\n    this.#initialConnection = initialConnection;\n    // Only serving replicas switch over between replication-managers. In\n    // particular, the in-RM backup-replicator never does.\n    this.#drainHandler =\n      mode === 'serving' && switchover\n        ? new DrainHandler(lc, changeStreamer, this.#state, switchover, () =>\n            this.#committedContext(switchover.replicaFile),\n          )\n        : undefined;\n  }\n\n  ready(): Promise<void> {\n    return this.#reporter.ready();\n  }\n\n  async run() {\n    const lc = this.#lc;\n    let workerError: Error | undefined;\n    this.#worker.onError(err => {\n      workerError ??= err;\n      this.#state.stop(lc, err);\n    });\n    lc.info?.(`Starting IncrementalSyncer`);\n    const {watermark: initialWatermark} =\n      await this.#worker.getSubscriptionState();\n\n    // Notify any waiting subscribers that the replica is ready to be read.\n    // This initial notification intentionally omits replicaReadyTimeMs because\n    // it represents already-current state, not newly-unserved work.\n    void this.#notifier.notifySubscribers({\n      state: 'version-ready',\n      watermark: initialWatermark,\n    });\n\n    while (this.#state.shouldRun()) {\n      const {replicaVersion, watermark, backfills} =\n        await this.#worker.getSubscriptionState();\n\n      let downstream: Downstream | undefined;\n      let unregister = () => {};\n      let err: unknown | undefined;\n\n      try {\n        const ctx = this.#subscriberContext(\n          replicaVersion,\n          watermark,\n          backfills,\n        );\n        // A successful switchover has already established the subscription\n        // on a new connection.\n        const switched = this.#drainHandler?.getNextConnection();\n        // The first connection reuses the still-open reservation connection\n        // (if any) from the backup restore, so the replication-manager that\n        // reserved the change log is the one that serves the subscription.\n        // Every reconnect after that opens a fresh connection.\n        const initialConnection = this.#initialConnection;\n        this.#initialConnection = undefined;\n        downstream =\n          switched ??\n          (initialConnection\n            ? await initialConnection.subscribe(ctx)\n            : await this.#changeStreamer.subscribe(ctx));\n        unregister = this.#state.cancelOnStop(downstream);\n        this.#statusPublisher?.publish(\n          lc,\n          'Replicating',\n          `Replicating from ${watermark}`,\n        );\n\n        let backfill:\n          | {status: DownloadStatus; table: string; columns: string[]}\n          | undefined;\n        let writeBatch: ChangeStreamData[] = [];\n        let writeBatchSize = 0;\n        let inTransaction = false;\n        // Indexes created in the current run of consecutive index creations.\n        let indexing: IndexingProgress | undefined;\n\n        const publishBackfillStatus = (table: string) =>\n          this.#statusPublisher?.publish(\n            lc,\n            'Replicating',\n            `Backfilling ${table} table`,\n            3000,\n            () =>\n              backfill\n                ? {\n                    downloadStatus: [\n                      {\n                        ...backfill.status,\n                        table: backfill.table,\n                        columns: backfill.columns,\n                      },\n                    ],\n                  }\n                : {},\n          );\n\n        const flushWrites = async () => {\n          if (writeBatch.length === 0) {\n            return;\n          }\n          const batch = writeBatch;\n          writeBatch = [];\n          writeBatchSize = 0;\n\n          const result = await this.#worker.processMessages(batch);\n          this.#handleResult(lc, result);\n          if (result?.completedBackfill) {\n            backfill = undefined;\n          }\n        };\n\n        // Whether the current transaction is being skipped.\n        let skipping = false;\n\n        // A subscription established by a switchover starts at the\n        // watermark at which it was requested, and may thus include\n        // transactions that were since committed from the previous\n        // subscription.\n        const shouldSkip = (message: SubscribeDownstream) => {\n          const type = message[0];\n          if (type === 'begin' && message[2].commitWatermark <= watermark) {\n            lc.debug?.(\n              `skipping committed transaction ${message[2].commitWatermark}`,\n            );\n            skipping = true;\n          }\n          if (\n            skipping &&\n            (type === 'begin' ||\n              type === 'data' ||\n              type === 'commit' ||\n              type === 'rollback')\n          ) {\n            skipping = type !== 'commit' && type !== 'rollback';\n            return true;\n          }\n          return false;\n        };\n\n        for await (const {data: message, size} of downstream) {\n          if (shouldSkip(message)) {\n            continue;\n          }\n          // Only reset backoff after receiving an actual message.\n          this.#state.resetBackoff();\n\n          this.#replicationEvents.add(1);\n          switch (message[0]) {\n            case 'reserved':\n              lc.warn?.(`unexpected snapshot reservation message received`, {\n                message: message[0],\n              });\n              break;\n            case 'status': {\n              const {lagReport, draining} = message[1];\n              this.#olderReplicaVersionRejections = 0;\n              if (draining) {\n                this.#drainHandler?.onChangeStreamerDrain(downstream);\n              }\n              if (lagReport) {\n                const report: ReplicationReport = {\n                  nextSendTimeMs: lagReport.nextSendTimeMs,\n                };\n                if (lagReport.lastTimings) {\n                  report.lastTimings = {\n                    ...lagReport.lastTimings,\n                    replicateTimeMs: Date.now(),\n                  };\n                }\n                this.#reporter.record(report);\n              }\n              break;\n            }\n            case 'error': {\n              const {\n                type,\n                message: msg,\n                replicaVersion: rmVersion,\n              } = message[1];\n              if (\n                type === WrongReplicaVersion &&\n                rmVersion !== undefined &&\n                rmVersion < replicaVersion &&\n                this.#olderReplicaVersionRejections++ <\n                  MAX_OLDER_REPLICA_VERSION_REJECTIONS\n              ) {\n                // The replication-manager is behind this replica (e.g. it\n                // has yet to be replaced by a resync). Rather than restoring\n                // its older replica, reconnect (with backoff), which may\n                // reach a different replication-manager.\n                lc.warn?.(\n                  `replication-manager is at an older replica version ` +\n                    `(${rmVersion}) than ${replicaVersion}. reconnecting`,\n                );\n                break;\n              }\n              // Signal from the replication-manager that the view-syncer must\n              // shut down and restore a new backup from litestream.\n              this.stop(\n                lc,\n                // Note: The AbortError indicates a clean / intentional shutdown.\n                new AbortError(\n                  `${errorTypeToReadableName(type as ErrorType)}: ${msg}`,\n                ),\n              );\n              break;\n            }\n            default: {\n              const msg = message[1];\n              if (msg.tag === 'backfill' && msg.status) {\n                const {status, relation} = msg;\n                const first = !backfill;\n                backfill = {\n                  status, // Update the current status\n                  table: relation.name,\n                  columns: [...relation.rowKey.columns, ...msg.columns],\n                };\n                if (first) {\n                  // Start publishing the status every 3 seconds.\n                  publishBackfillStatus(relation.name);\n                }\n              }\n\n              if (msg.tag === 'create-index' && this.#statusPublisher) {\n                // Creating an index on an existing table can take a long\n                // time. Flush preceding changes so that the index creation\n                // is processed (and timed) on its own.\n                await flushWrites();\n                const index = mapPostgresToLiteIndex(msg.spec);\n                indexing ??= new IndexingProgress();\n                indexing.start(index);\n                this.#statusPublisher.publish(\n                  lc,\n                  'Replicating',\n                  `Creating index ${index.name} on ${index.tableName}`,\n                  3000,\n                  indexing.state,\n                );\n                // Exclude the time spent publishing the status.\n                indexing.restartTimer();\n                writeBatch.push(message as ChangeStreamData);\n                writeBatchSize += size;\n                await flushWrites();\n                const elapsed = indexing.finish();\n                lc.info?.(\n                  `Created index ${index.name} (${elapsed.toFixed(3)} ms)`,\n                );\n                this.#statusPublisher.publish(\n                  lc,\n                  'Replicating',\n                  `Created index ${index.name} on ${index.tableName}`,\n                  0,\n                  indexing.state,\n                );\n                if (backfill) {\n                  // Resume reporting the progress of the ongoing backfill.\n                  publishBackfillStatus(backfill.table);\n                }\n                break;\n              }\n              indexing = undefined;\n\n              const type = message[0];\n              const invalidTransactionSequence =\n                type === 'begin' ? inTransaction : !inTransaction;\n              if (type === 'begin') {\n                inTransaction = true;\n              } else if (type === 'commit' || type === 'rollback') {\n                inTransaction = false;\n              }\n\n              writeBatch.push(message as ChangeStreamData);\n              writeBatchSize += size;\n              if (\n                // Waiting here ensures that consuming the commit, which ACKs\n                // it upstream, only happens after the SQLite commit finishes.\n                type === 'commit' ||\n                type === 'rollback' ||\n                // Promptly surface malformed transaction sequences instead\n                // of leaving a partial batch waiting for another message.\n                invalidTransactionSequence ||\n                writeBatch.length >= MAX_WORKER_BATCH_MESSAGES ||\n                writeBatchSize >= MAX_WORKER_BATCH_SIZE\n              ) {\n                await flushWrites();\n              }\n              break;\n            }\n          }\n        }\n        this.#worker.abort();\n      } catch (e) {\n        err = e;\n        this.#worker.abort();\n      } finally {\n        downstream?.cancel();\n        unregister();\n        this.#statusPublisher?.stop();\n      }\n      if (err === undefined && this.#drainHandler?.hasNextConnection) {\n        continue; // Switched over to a new subscription; no backoff.\n      }\n      this.#drainHandler?.getNextConnection()?.cancel();\n      await this.#state.backoff(lc, err);\n    }\n    lc.info?.('IncrementalSyncer stopped');\n    if (workerError) {\n      throw workerError;\n    }\n  }\n\n  #handleResult(lc: LogContext, result: CommitResult | null) {\n    if (!result) {\n      return;\n    }\n    if (result.completedBackfill) {\n      // Publish the final status\n      const status = result.completedBackfill;\n      this.#statusPublisher?.publish(\n        lc,\n        'Replicating',\n        `Backfilled ${status.table} table`,\n        0,\n        () => ({downloadStatus: [status]}),\n      );\n    } else if (result.schemaUpdated) {\n      this.#statusPublisher?.publish(lc, 'Replicating', 'Schema updated');\n    }\n    if (result.watermark && result.changeLogUpdated) {\n      void this.#notifier.notifySubscribers({\n        state: 'version-ready',\n        watermark: result.watermark,\n        replicaReadyTimeMs: Date.now(),\n        upstreamCommitTimeMs: result.upstreamCommitTimeMs,\n      });\n    }\n  }\n\n  subscribe(): Source<ReplicaState> {\n    return this.#notifier.subscribe();\n  }\n\n  #subscriberContext(\n    replicaVersion: string,\n    watermark: string,\n    backfills: SubscriberContext['backfills'],\n  ): SubscriberContext {\n    return {\n      protocolVersion: PROTOCOL_VERSION,\n      taskID: this.#taskID,\n      id: this.#id,\n      mode: this.#mode,\n      watermark,\n      replicaVersion,\n      backfills,\n      ...(this.#drainHandler ? {drainable: true} : {}),\n    };\n  }\n\n  /**\n   * Returns the subscriber context as of the last committed transaction,\n   * read from the replica file (i.e. not from the write worker, whose\n   * connection may be in the middle of a transaction).\n   */\n  #committedContext(replicaFile: string): SubscriberContext {\n    const db = new Database(this.#lc, replicaFile, {readonly: true});\n    try {\n      const {replicaVersion, watermark, backfills} = db.transaction(() => ({\n        ...getSubscriptionState(new StatementRunner(db)),\n        backfills: readReplicaBackfills(db),\n      }));\n      return this.#subscriberContext(replicaVersion, watermark, backfills);\n    } finally {\n      db.close();\n    }\n  }\n\n  stop(lc: LogContext, err?: unknown) {\n    this.#state.stop(lc, err);\n    // Abort any polling loop that the write worker may be in.\n    this.#worker.abort();\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;AA4CA,IAAM,4BAA4B;AAClC,IAAM,wBAAwB;AAO9B,IAAM,uCAAuC;;;;;;;;AAS7C,IAAa,oBAAb,MAA+B;CAC7B;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAGA;CAIA;CACA,iCAAiC;CAEjC,SAAkB,IAAI,aAAa,mBAAmB;CAEtD,qBAA8B,mBAC5B,eACA,UACA,wCACF;CAEA,YACE,IACA,QACA,IACA,gBACA,QACA,MACA,iBACA,iCAAiE,CAAC,GAClE,mBACA,YACA;EACA,KAAK,MAAM;EACX,KAAK,UAAU;EACf,KAAK,MAAM;EACX,KAAK,kBAAkB;EACvB,KAAK,UAAU;EACf,KAAK,QAAQ;EACb,KAAK,mBAAmB;EACxB,KAAK,YAAY,IAAI,SAAS;EAC9B,KAAK,YAAY,IAAI,0BACnB,IACA,8BACF;EACA,KAAK,qBAAqB;EAG1B,KAAK,gBACH,SAAS,aAAa,aAClB,IAAI,aAAa,IAAI,gBAAgB,KAAK,QAAQ,kBAChD,KAAK,kBAAkB,WAAW,WAAW,CAC/C,IACA,KAAA;CACR;CAEA,QAAuB;EACrB,OAAO,KAAK,UAAU,MAAM;CAC9B;CAEA,MAAM,MAAM;EACV,MAAM,KAAK,KAAK;EAChB,IAAI;EACJ,KAAK,QAAQ,SAAQ,QAAO;GAC1B,gBAAgB;GAChB,KAAK,OAAO,KAAK,IAAI,GAAG;EAC1B,CAAC;EACD,GAAG,OAAO,4BAA4B;EACtC,MAAM,EAAC,WAAW,qBAChB,MAAM,KAAK,QAAQ,qBAAqB;EAK1C,KAAU,UAAU,kBAAkB;GACpC,OAAO;GACP,WAAW;EACb,CAAC;EAED,OAAO,KAAK,OAAO,UAAU,GAAG;GAC9B,MAAM,EAAC,gBAAgB,WAAW,cAChC,MAAM,KAAK,QAAQ,qBAAqB;GAE1C,IAAI;GACJ,IAAI,mBAAmB,CAAC;GACxB,IAAI;GAEJ,IAAI;IACF,MAAM,MAAM,KAAK,mBACf,gBACA,WACA,SACF;IAGA,MAAM,WAAW,KAAK,eAAe,kBAAkB;IAKvD,MAAM,oBAAoB,KAAK;IAC/B,KAAK,qBAAqB,KAAA;IAC1B,aACE,aACC,oBACG,MAAM,kBAAkB,UAAU,GAAG,IACrC,MAAM,KAAK,gBAAgB,UAAU,GAAG;IAC9C,aAAa,KAAK,OAAO,aAAa,UAAU;IAChD,KAAK,kBAAkB,QACrB,IACA,eACA,oBAAoB,WACtB;IAEA,IAAI;IAGJ,IAAI,aAAiC,CAAC;IACtC,IAAI,iBAAiB;IACrB,IAAI,gBAAgB;IAEpB,IAAI;IAEJ,MAAM,yBAAyB,UAC7B,KAAK,kBAAkB,QACrB,IACA,eACA,eAAe,MAAM,SACrB,WAEE,WACI,EACE,gBAAgB,CACd;KACE,GAAG,SAAS;KACZ,OAAO,SAAS;KAChB,SAAS,SAAS;IACpB,CACF,EACF,IACA,CAAC,CACT;IAEF,MAAM,cAAc,YAAY;KAC9B,IAAI,WAAW,WAAW,GACxB;KAEF,MAAM,QAAQ;KACd,aAAa,CAAC;KACd,iBAAiB;KAEjB,MAAM,SAAS,MAAM,KAAK,QAAQ,gBAAgB,KAAK;KACvD,KAAK,cAAc,IAAI,MAAM;KAC7B,IAAI,QAAQ,mBACV,WAAW,KAAA;IAEf;IAGA,IAAI,WAAW;IAMf,MAAM,cAAc,YAAiC;KACnD,MAAM,OAAO,QAAQ;KACrB,IAAI,SAAS,WAAW,QAAQ,EAAE,CAAC,mBAAmB,WAAW;MAC/D,GAAG,QACD,kCAAkC,QAAQ,EAAE,CAAC,iBAC/C;MACA,WAAW;KACb;KACA,IACE,aACC,SAAS,WACR,SAAS,UACT,SAAS,YACT,SAAS,aACX;MACA,WAAW,SAAS,YAAY,SAAS;MACzC,OAAO;KACT;KACA,OAAO;IACT;IAEA,WAAW,MAAM,EAAC,MAAM,SAAS,UAAS,YAAY;KACpD,IAAI,WAAW,OAAO,GACpB;KAGF,KAAK,OAAO,aAAa;KAEzB,KAAK,mBAAmB,IAAI,CAAC;KAC7B,QAAQ,QAAQ,IAAhB;MACE,KAAK;OACH,GAAG,OAAO,oDAAoD,EAC5D,SAAS,QAAQ,GACnB,CAAC;OACD;MACF,KAAK,UAAU;OACb,MAAM,EAAC,WAAW,aAAY,QAAQ;OACtC,KAAK,iCAAiC;OACtC,IAAI,UACF,KAAK,eAAe,sBAAsB,UAAU;OAEtD,IAAI,WAAW;QACb,MAAM,SAA4B,EAChC,gBAAgB,UAAU,eAC5B;QACA,IAAI,UAAU,aACZ,OAAO,cAAc;SACnB,GAAG,UAAU;SACb,iBAAiB,KAAK,IAAI;QAC5B;QAEF,KAAK,UAAU,OAAO,MAAM;OAC9B;OACA;MACF;MACA,KAAK,SAAS;OACZ,MAAM,EACJ,MACA,SAAS,KACT,gBAAgB,cACd,QAAQ;OACZ,IACE,SAAA,KACA,cAAc,KAAA,KACd,YAAY,kBACZ,KAAK,mCACH,sCACF;QAKA,GAAG,OACD,uDACM,UAAU,SAAS,eAAe,eAC1C;QACA;OACF;OAGA,KAAK,KACH,IAEA,IAAI,WACF,GAAG,wBAAwB,IAAiB,EAAE,IAAI,KACpD,CACF;OACA;MACF;MACA,SAAS;OACP,MAAM,MAAM,QAAQ;OACpB,IAAI,IAAI,QAAQ,cAAc,IAAI,QAAQ;QACxC,MAAM,EAAC,QAAQ,aAAY;QAC3B,MAAM,QAAQ,CAAC;QACf,WAAW;SACT;SACA,OAAO,SAAS;SAChB,SAAS,CAAC,GAAG,SAAS,OAAO,SAAS,GAAG,IAAI,OAAO;QACtD;QACA,IAAI,OAEF,sBAAsB,SAAS,IAAI;OAEvC;OAEA,IAAI,IAAI,QAAQ,kBAAkB,KAAK,kBAAkB;QAIvD,MAAM,YAAY;QAClB,MAAM,QAAQ,uBAAuB,IAAI,IAAI;QAC7C,aAAa,IAAI,iBAAiB;QAClC,SAAS,MAAM,KAAK;QACpB,KAAK,iBAAiB,QACpB,IACA,eACA,kBAAkB,MAAM,KAAK,MAAM,MAAM,aACzC,KACA,SAAS,KACX;QAEA,SAAS,aAAa;QACtB,WAAW,KAAK,OAA2B;QAC3C,kBAAkB;QAClB,MAAM,YAAY;QAClB,MAAM,UAAU,SAAS,OAAO;QAChC,GAAG,OACD,iBAAiB,MAAM,KAAK,IAAI,QAAQ,QAAQ,CAAC,EAAE,KACrD;QACA,KAAK,iBAAiB,QACpB,IACA,eACA,iBAAiB,MAAM,KAAK,MAAM,MAAM,aACxC,GACA,SAAS,KACX;QACA,IAAI,UAEF,sBAAsB,SAAS,KAAK;QAEtC;OACF;OACA,WAAW,KAAA;OAEX,MAAM,OAAO,QAAQ;OACrB,MAAM,6BACJ,SAAS,UAAU,gBAAgB,CAAC;OACtC,IAAI,SAAS,SACX,gBAAgB;YACX,IAAI,SAAS,YAAY,SAAS,YACvC,gBAAgB;OAGlB,WAAW,KAAK,OAA2B;OAC3C,kBAAkB;OAClB,IAGE,SAAS,YACT,SAAS,cAGT,8BACA,WAAW,UAAU,6BACrB,kBAAkB,uBAElB,MAAM,YAAY;OAEpB;MACF;KACF;IACF;IACA,KAAK,QAAQ,MAAM;GACrB,SAAS,GAAG;IACV,MAAM;IACN,KAAK,QAAQ,MAAM;GACrB,UAAU;IACR,YAAY,OAAO;IACnB,WAAW;IACX,KAAK,kBAAkB,KAAK;GAC9B;GACA,IAAI,QAAQ,KAAA,KAAa,KAAK,eAAe,mBAC3C;GAEF,KAAK,eAAe,kBAAkB,CAAC,EAAE,OAAO;GAChD,MAAM,KAAK,OAAO,QAAQ,IAAI,GAAG;EACnC;EACA,GAAG,OAAO,2BAA2B;EACrC,IAAI,aACF,MAAM;CAEV;CAEA,cAAc,IAAgB,QAA6B;EACzD,IAAI,CAAC,QACH;EAEF,IAAI,OAAO,mBAAmB;GAE5B,MAAM,SAAS,OAAO;GACtB,KAAK,kBAAkB,QACrB,IACA,eACA,cAAc,OAAO,MAAM,SAC3B,UACO,EAAC,gBAAgB,CAAC,MAAM,EAAC,EAClC;EACF,OAAO,IAAI,OAAO,eAChB,KAAK,kBAAkB,QAAQ,IAAI,eAAe,gBAAgB;EAEpE,IAAI,OAAO,aAAa,OAAO,kBAC7B,KAAU,UAAU,kBAAkB;GACpC,OAAO;GACP,WAAW,OAAO;GAClB,oBAAoB,KAAK,IAAI;GAC7B,sBAAsB,OAAO;EAC/B,CAAC;CAEL;CAEA,YAAkC;EAChC,OAAO,KAAK,UAAU,UAAU;CAClC;CAEA,mBACE,gBACA,WACA,WACmB;EACnB,OAAO;GACL,iBAAA;GACA,QAAQ,KAAK;GACb,IAAI,KAAK;GACT,MAAM,KAAK;GACX;GACA;GACA;GACA,GAAI,KAAK,gBAAgB,EAAC,WAAW,KAAI,IAAI,CAAC;EAChD;CACF;;;;;;CAOA,kBAAkB,aAAwC;EACxD,MAAM,KAAK,IAAI,SAAS,KAAK,KAAK,aAAa,EAAC,UAAU,KAAI,CAAC;EAC/D,IAAI;GACF,MAAM,EAAC,gBAAgB,WAAW,cAAa,GAAG,mBAAmB;IACnE,GAAG,qBAAqB,IAAI,gBAAgB,EAAE,CAAC;IAC/C,WAAW,qBAAqB,EAAE;GACpC,EAAE;GACF,OAAO,KAAK,mBAAmB,gBAAgB,WAAW,SAAS;EACrE,UAAU;GACR,GAAG,MAAM;EACX;CACF;CAEA,KAAK,IAAgB,KAAe;EAClC,KAAK,OAAO,KAAK,IAAI,GAAG;EAExB,KAAK,QAAQ,MAAM;CACrB;AACF"}
+{"version":3,"file":"incremental-sync.js","names":[],"sources":["../../../../../../zero-cache/src/services/replicator/incremental-sync.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {AbortError} from '../../../../shared/src/abort-error.ts';\nimport type {Enum} from '../../../../shared/src/enum.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {mapPostgresToLiteIndex} from '../../db/pg-to-lite.ts';\nimport {StatementRunner} from '../../db/statements.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport type {Source} from '../../types/streams.ts';\nimport type {DownloadStatus} from '../change-source/protocol/current.ts';\nimport type {ChangeStreamData} from '../change-source/protocol/current/downstream.ts';\nimport type {ReservationFollowup} from '../change-streamer/change-streamer-http.ts';\nimport {\n  errorTypeToReadableName,\n  PROTOCOL_VERSION,\n  type ChangeStreamer,\n  type SubscriberContext,\n} from '../change-streamer/change-streamer.ts';\nimport type * as ErrorType from '../change-streamer/error-type-enum.ts';\nimport {WrongReplicaVersion} from '../change-streamer/error-type-enum.ts';\nimport type {SubscribeDownstream} from '../change-streamer/subscribe.ts';\nimport {RunningState} from '../running-state.ts';\nimport type {CommitResult} from './change-processor.ts';\nimport {\n  DrainHandler,\n  type Downstream,\n  type SwitchoverOptions,\n} from './drain-handler.ts';\nimport {Notifier} from './notifier.ts';\nimport {\n  IndexingProgress,\n  type ReplicationStatusPublisher,\n} from './replication-status.ts';\nimport type {ReplicaState, ReplicatorMode} from './replicator.ts';\nimport {\n  ReplicationReportRecorder,\n  type ReplicationLagReadinessOptions,\n} from './reporter/recorder.ts';\nimport type {ReplicationReport} from './reporter/report-schema.ts';\nimport {readReplicaBackfills} from './schema/backfilling.ts';\nimport {getSubscriptionState} from './schema/replication-state.ts';\nimport type {WriteWorkerClient} from './write-worker-client.ts';\n\ntype ErrorType = Enum<typeof ErrorType>;\n\nconst MAX_WORKER_BATCH_MESSAGES = 256;\nconst MAX_WORKER_BATCH_SIZE = 1024 * 1024;\n\n// The number of times to reconnect after being rejected by a\n// replication-manager with an older replica version (e.g. one that has yet\n// to be replaced by a resync), before accepting the rejection and restarting\n// (e.g. if the upstream was legitimately replaced by one with a lower replica\n// version).\nconst MAX_OLDER_REPLICA_VERSION_REJECTIONS = 20;\n\n/**\n * The {@link IncrementalSyncer} manages a logical replication stream from upstream,\n * handling application lifecycle events (start, stop) and retrying the\n * connection with exponential backoff. The actual handling of the logical\n * replication messages is done by the {@link ChangeProcessor}, which runs\n * in a worker thread via the {@link WriteWorkerClient}.\n */\nexport class IncrementalSyncer {\n  readonly #lc: LogContext;\n  readonly #taskID: string;\n  readonly #id: string;\n  readonly #changeStreamer: ChangeStreamer;\n  readonly #worker: WriteWorkerClient;\n  readonly #mode: ReplicatorMode;\n  readonly #statusPublisher: ReplicationStatusPublisher | null;\n  readonly #notifier: Notifier;\n  readonly #reporter: ReplicationReportRecorder;\n  // Consumed (and cleared) by the first subscribe() in run()'s loop; every\n  // subsequent (re)connection goes through #changeStreamer.subscribe().\n  #initialConnection: ReservationFollowup | undefined;\n\n  // Only set for serving replicas that switch over between\n  // replication-managers.\n  readonly #drainHandler: DrainHandler | undefined;\n  #olderReplicaVersionRejections = 0;\n\n  readonly #state = new RunningState('IncrementalSyncer');\n\n  readonly #replicationEvents = getOrCreateCounter(\n    'replication',\n    'events',\n    'Number of replication events processed',\n  );\n\n  constructor(\n    lc: LogContext,\n    taskID: string,\n    id: string,\n    changeStreamer: ChangeStreamer,\n    worker: WriteWorkerClient,\n    mode: ReplicatorMode,\n    statusPublisher: ReplicationStatusPublisher | null,\n    replicationLagReadinessOptions: ReplicationLagReadinessOptions = {},\n    initialConnection?: ReservationFollowup,\n    switchover?: SwitchoverOptions,\n  ) {\n    this.#lc = lc;\n    this.#taskID = taskID;\n    this.#id = id;\n    this.#changeStreamer = changeStreamer;\n    this.#worker = worker;\n    this.#mode = mode;\n    this.#statusPublisher = statusPublisher;\n    this.#notifier = new Notifier();\n    this.#reporter = new ReplicationReportRecorder(\n      lc,\n      replicationLagReadinessOptions,\n    );\n    this.#initialConnection = initialConnection;\n    // Only serving replicas switch over between replication-managers. In\n    // particular, the in-RM backup-replicator never does.\n    this.#drainHandler =\n      mode === 'serving' && switchover\n        ? new DrainHandler(lc, changeStreamer, this.#state, switchover, () =>\n            this.#committedContext(switchover.replicaFile),\n          )\n        : undefined;\n  }\n\n  ready(): Promise<void> {\n    return this.#reporter.ready();\n  }\n\n  async run() {\n    const lc = this.#lc;\n    let workerError: Error | undefined;\n    this.#worker.onError(err => {\n      workerError ??= err;\n      this.#state.stop(lc, err);\n    });\n    lc.info?.(`Starting IncrementalSyncer`);\n    const {watermark: initialWatermark} =\n      await this.#worker.getSubscriptionState();\n\n    // Notify any waiting subscribers that the replica is ready to be read.\n    // This initial notification intentionally omits replicaReadyTimeMs because\n    // it represents already-current state, not newly-unserved work.\n    void this.#notifier.notifySubscribers({\n      state: 'version-ready',\n      watermark: initialWatermark,\n    });\n\n    while (this.#state.shouldRun()) {\n      const {replicaVersion, watermark, backfills} =\n        await this.#worker.getSubscriptionState();\n\n      let downstream: Downstream | undefined;\n      let unregister = () => {};\n      let err: unknown | undefined;\n\n      try {\n        const ctx = this.#subscriberContext(\n          replicaVersion,\n          watermark,\n          backfills,\n        );\n        // A successful switchover has already established the subscription\n        // on a new connection.\n        const switched = this.#drainHandler?.getNextConnection();\n        // The first connection reuses the still-open reservation connection\n        // (if any) from the backup restore, so the replication-manager that\n        // reserved the change log is the one that serves the subscription.\n        // Every reconnect after that opens a fresh connection.\n        const initialConnection = this.#initialConnection;\n        this.#initialConnection = undefined;\n        downstream =\n          switched ??\n          (initialConnection\n            ? await initialConnection.subscribe(ctx)\n            : await this.#changeStreamer.subscribe(ctx));\n        unregister = this.#state.cancelOnStop(downstream);\n        this.#statusPublisher?.publish(\n          lc,\n          'Replicating',\n          `Replicating from ${watermark}`,\n        );\n\n        let backfill:\n          | {status: DownloadStatus; table: string; columns: string[]}\n          | undefined;\n        let writeBatch: ChangeStreamData[] = [];\n        let writeBatchSize = 0;\n        let inTransaction = false;\n        // Indexes created in the current run of consecutive index creations.\n        let indexing: IndexingProgress | undefined;\n\n        const publishBackfillStatus = (table: string) =>\n          this.#statusPublisher?.publish(\n            lc,\n            'Replicating',\n            `Backfilling ${table} table`,\n            3000,\n            () =>\n              backfill\n                ? {\n                    downloadStatus: [\n                      {\n                        ...backfill.status,\n                        table: backfill.table,\n                        columns: backfill.columns,\n                      },\n                    ],\n                  }\n                : {},\n          );\n\n        const flushWrites = async () => {\n          if (writeBatch.length === 0) {\n            return;\n          }\n          const batch = writeBatch;\n          writeBatch = [];\n          writeBatchSize = 0;\n\n          const result = await this.#worker.processMessages(batch);\n          this.#handleResult(lc, result);\n          if (result?.completedBackfill) {\n            backfill = undefined;\n          }\n        };\n\n        // Whether the current transaction is being skipped.\n        let skipping = false;\n\n        // A subscription established by a switchover starts at the\n        // watermark at which it was requested, and may thus include\n        // transactions that were since committed from the previous\n        // subscription.\n        const shouldSkip = (message: SubscribeDownstream) => {\n          const type = message[0];\n          if (type === 'begin' && message[2].commitWatermark <= watermark) {\n            lc.debug?.(\n              `skipping committed transaction ${message[2].commitWatermark}`,\n            );\n            skipping = true;\n          }\n          if (\n            skipping &&\n            (type === 'begin' ||\n              type === 'data' ||\n              type === 'commit' ||\n              type === 'rollback')\n          ) {\n            skipping = type !== 'commit' && type !== 'rollback';\n            return true;\n          }\n          return false;\n        };\n\n        for await (const {data: message, size} of downstream) {\n          if (shouldSkip(message)) {\n            continue;\n          }\n          // Only reset backoff after receiving an actual message.\n          this.#state.resetBackoff();\n\n          this.#replicationEvents.add(1);\n          switch (message[0]) {\n            case 'reserved':\n              lc.warn?.(`unexpected snapshot reservation message received`, {\n                message: message[0],\n              });\n              break;\n            case 'status': {\n              const {lagReport, draining} = message[1];\n              this.#olderReplicaVersionRejections = 0;\n              if (draining) {\n                this.#drainHandler?.onChangeStreamerDrain(downstream);\n              }\n              if (lagReport) {\n                const report: ReplicationReport = {\n                  nextSendTimeMs: lagReport.nextSendTimeMs,\n                };\n                if (lagReport.lastTimings) {\n                  report.lastTimings = {\n                    ...lagReport.lastTimings,\n                    replicateTimeMs: Date.now(),\n                  };\n                }\n                this.#reporter.record(report);\n              }\n              break;\n            }\n            case 'error': {\n              const {\n                type,\n                message: msg,\n                olderReplica,\n                replicaVersion: rmVersion,\n              } = message[1];\n              if (\n                type === WrongReplicaVersion &&\n                // Older replication-managers disclose their version instead.\n                (olderReplica ??\n                  (rmVersion !== undefined && rmVersion < replicaVersion)) &&\n                this.#olderReplicaVersionRejections++ <\n                  MAX_OLDER_REPLICA_VERSION_REJECTIONS\n              ) {\n                // The replication-manager is behind this replica (e.g. it\n                // has yet to be replaced by a resync). Rather than restoring\n                // its older replica, reconnect (with backoff), which may\n                // reach a different replication-manager.\n                lc.warn?.(\n                  `replication-manager is at an older replica version ` +\n                    `than ${replicaVersion}. reconnecting`,\n                );\n                break;\n              }\n              // Signal from the replication-manager that the view-syncer must\n              // shut down and restore a new backup from litestream.\n              this.stop(\n                lc,\n                // Note: The AbortError indicates a clean / intentional shutdown.\n                new AbortError(\n                  `${errorTypeToReadableName(type as ErrorType)}: ${msg}`,\n                ),\n              );\n              break;\n            }\n            default: {\n              const msg = message[1];\n              if (msg.tag === 'backfill' && msg.status) {\n                const {status, relation} = msg;\n                const first = !backfill;\n                backfill = {\n                  status, // Update the current status\n                  table: relation.name,\n                  columns: [...relation.rowKey.columns, ...msg.columns],\n                };\n                if (first) {\n                  // Start publishing the status every 3 seconds.\n                  publishBackfillStatus(relation.name);\n                }\n              }\n\n              if (msg.tag === 'create-index' && this.#statusPublisher) {\n                // Creating an index on an existing table can take a long\n                // time. Flush preceding changes so that the index creation\n                // is processed (and timed) on its own.\n                await flushWrites();\n                const index = mapPostgresToLiteIndex(msg.spec);\n                indexing ??= new IndexingProgress();\n                indexing.start(index);\n                this.#statusPublisher.publish(\n                  lc,\n                  'Replicating',\n                  `Creating index ${index.name} on ${index.tableName}`,\n                  3000,\n                  indexing.state,\n                );\n                // Exclude the time spent publishing the status.\n                indexing.restartTimer();\n                writeBatch.push(message as ChangeStreamData);\n                writeBatchSize += size;\n                await flushWrites();\n                const elapsed = indexing.finish();\n                lc.info?.(\n                  `Created index ${index.name} (${elapsed.toFixed(3)} ms)`,\n                );\n                this.#statusPublisher.publish(\n                  lc,\n                  'Replicating',\n                  `Created index ${index.name} on ${index.tableName}`,\n                  0,\n                  indexing.state,\n                );\n                if (backfill) {\n                  // Resume reporting the progress of the ongoing backfill.\n                  publishBackfillStatus(backfill.table);\n                }\n                break;\n              }\n              indexing = undefined;\n\n              const type = message[0];\n              const invalidTransactionSequence =\n                type === 'begin' ? inTransaction : !inTransaction;\n              if (type === 'begin') {\n                inTransaction = true;\n              } else if (type === 'commit' || type === 'rollback') {\n                inTransaction = false;\n              }\n\n              writeBatch.push(message as ChangeStreamData);\n              writeBatchSize += size;\n              if (\n                // Waiting here ensures that consuming the commit, which ACKs\n                // it upstream, only happens after the SQLite commit finishes.\n                type === 'commit' ||\n                type === 'rollback' ||\n                // Promptly surface malformed transaction sequences instead\n                // of leaving a partial batch waiting for another message.\n                invalidTransactionSequence ||\n                writeBatch.length >= MAX_WORKER_BATCH_MESSAGES ||\n                writeBatchSize >= MAX_WORKER_BATCH_SIZE\n              ) {\n                await flushWrites();\n              }\n              break;\n            }\n          }\n        }\n        this.#worker.abort();\n      } catch (e) {\n        err = e;\n        this.#worker.abort();\n      } finally {\n        downstream?.cancel();\n        unregister();\n        this.#statusPublisher?.stop();\n      }\n      if (err === undefined && this.#drainHandler?.hasNextConnection) {\n        continue; // Switched over to a new subscription; no backoff.\n      }\n      this.#drainHandler?.getNextConnection()?.cancel();\n      await this.#state.backoff(lc, err);\n    }\n    lc.info?.('IncrementalSyncer stopped');\n    if (workerError) {\n      throw workerError;\n    }\n  }\n\n  #handleResult(lc: LogContext, result: CommitResult | null) {\n    if (!result) {\n      return;\n    }\n    if (result.completedBackfill) {\n      // Publish the final status\n      const status = result.completedBackfill;\n      this.#statusPublisher?.publish(\n        lc,\n        'Replicating',\n        `Backfilled ${status.table} table`,\n        0,\n        () => ({downloadStatus: [status]}),\n      );\n    } else if (result.schemaUpdated) {\n      this.#statusPublisher?.publish(lc, 'Replicating', 'Schema updated');\n    }\n    if (result.watermark && result.changeLogUpdated) {\n      void this.#notifier.notifySubscribers({\n        state: 'version-ready',\n        watermark: result.watermark,\n        replicaReadyTimeMs: Date.now(),\n        upstreamCommitTimeMs: result.upstreamCommitTimeMs,\n      });\n    }\n  }\n\n  subscribe(): Source<ReplicaState> {\n    return this.#notifier.subscribe();\n  }\n\n  #subscriberContext(\n    replicaVersion: string,\n    watermark: string,\n    backfills: SubscriberContext['backfills'],\n  ): SubscriberContext {\n    return {\n      protocolVersion: PROTOCOL_VERSION,\n      taskID: this.#taskID,\n      id: this.#id,\n      mode: this.#mode,\n      watermark,\n      replicaVersion,\n      backfills,\n      ...(this.#drainHandler ? {drainable: true} : {}),\n    };\n  }\n\n  /**\n   * Returns the subscriber context as of the last committed transaction,\n   * read from the replica file (i.e. not from the write worker, whose\n   * connection may be in the middle of a transaction).\n   */\n  #committedContext(replicaFile: string): SubscriberContext {\n    const db = new Database(this.#lc, replicaFile, {readonly: true});\n    try {\n      const {replicaVersion, watermark, backfills} = db.transaction(() => ({\n        ...getSubscriptionState(new StatementRunner(db)),\n        backfills: readReplicaBackfills(db),\n      }));\n      return this.#subscriberContext(replicaVersion, watermark, backfills);\n    } finally {\n      db.close();\n    }\n  }\n\n  stop(lc: LogContext, err?: unknown) {\n    this.#state.stop(lc, err);\n    // Abort any polling loop that the write worker may be in.\n    this.#worker.abort();\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;AA4CA,IAAM,4BAA4B;AAClC,IAAM,wBAAwB;AAO9B,IAAM,uCAAuC;;;;;;;;AAS7C,IAAa,oBAAb,MAA+B;CAC7B;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAGA;CAIA;CACA,iCAAiC;CAEjC,SAAkB,IAAI,aAAa,mBAAmB;CAEtD,qBAA8B,mBAC5B,eACA,UACA,wCACF;CAEA,YACE,IACA,QACA,IACA,gBACA,QACA,MACA,iBACA,iCAAiE,CAAC,GAClE,mBACA,YACA;EACA,KAAK,MAAM;EACX,KAAK,UAAU;EACf,KAAK,MAAM;EACX,KAAK,kBAAkB;EACvB,KAAK,UAAU;EACf,KAAK,QAAQ;EACb,KAAK,mBAAmB;EACxB,KAAK,YAAY,IAAI,SAAS;EAC9B,KAAK,YAAY,IAAI,0BACnB,IACA,8BACF;EACA,KAAK,qBAAqB;EAG1B,KAAK,gBACH,SAAS,aAAa,aAClB,IAAI,aAAa,IAAI,gBAAgB,KAAK,QAAQ,kBAChD,KAAK,kBAAkB,WAAW,WAAW,CAC/C,IACA,KAAA;CACR;CAEA,QAAuB;EACrB,OAAO,KAAK,UAAU,MAAM;CAC9B;CAEA,MAAM,MAAM;EACV,MAAM,KAAK,KAAK;EAChB,IAAI;EACJ,KAAK,QAAQ,SAAQ,QAAO;GAC1B,gBAAgB;GAChB,KAAK,OAAO,KAAK,IAAI,GAAG;EAC1B,CAAC;EACD,GAAG,OAAO,4BAA4B;EACtC,MAAM,EAAC,WAAW,qBAChB,MAAM,KAAK,QAAQ,qBAAqB;EAK1C,KAAU,UAAU,kBAAkB;GACpC,OAAO;GACP,WAAW;EACb,CAAC;EAED,OAAO,KAAK,OAAO,UAAU,GAAG;GAC9B,MAAM,EAAC,gBAAgB,WAAW,cAChC,MAAM,KAAK,QAAQ,qBAAqB;GAE1C,IAAI;GACJ,IAAI,mBAAmB,CAAC;GACxB,IAAI;GAEJ,IAAI;IACF,MAAM,MAAM,KAAK,mBACf,gBACA,WACA,SACF;IAGA,MAAM,WAAW,KAAK,eAAe,kBAAkB;IAKvD,MAAM,oBAAoB,KAAK;IAC/B,KAAK,qBAAqB,KAAA;IAC1B,aACE,aACC,oBACG,MAAM,kBAAkB,UAAU,GAAG,IACrC,MAAM,KAAK,gBAAgB,UAAU,GAAG;IAC9C,aAAa,KAAK,OAAO,aAAa,UAAU;IAChD,KAAK,kBAAkB,QACrB,IACA,eACA,oBAAoB,WACtB;IAEA,IAAI;IAGJ,IAAI,aAAiC,CAAC;IACtC,IAAI,iBAAiB;IACrB,IAAI,gBAAgB;IAEpB,IAAI;IAEJ,MAAM,yBAAyB,UAC7B,KAAK,kBAAkB,QACrB,IACA,eACA,eAAe,MAAM,SACrB,WAEE,WACI,EACE,gBAAgB,CACd;KACE,GAAG,SAAS;KACZ,OAAO,SAAS;KAChB,SAAS,SAAS;IACpB,CACF,EACF,IACA,CAAC,CACT;IAEF,MAAM,cAAc,YAAY;KAC9B,IAAI,WAAW,WAAW,GACxB;KAEF,MAAM,QAAQ;KACd,aAAa,CAAC;KACd,iBAAiB;KAEjB,MAAM,SAAS,MAAM,KAAK,QAAQ,gBAAgB,KAAK;KACvD,KAAK,cAAc,IAAI,MAAM;KAC7B,IAAI,QAAQ,mBACV,WAAW,KAAA;IAEf;IAGA,IAAI,WAAW;IAMf,MAAM,cAAc,YAAiC;KACnD,MAAM,OAAO,QAAQ;KACrB,IAAI,SAAS,WAAW,QAAQ,EAAE,CAAC,mBAAmB,WAAW;MAC/D,GAAG,QACD,kCAAkC,QAAQ,EAAE,CAAC,iBAC/C;MACA,WAAW;KACb;KACA,IACE,aACC,SAAS,WACR,SAAS,UACT,SAAS,YACT,SAAS,aACX;MACA,WAAW,SAAS,YAAY,SAAS;MACzC,OAAO;KACT;KACA,OAAO;IACT;IAEA,WAAW,MAAM,EAAC,MAAM,SAAS,UAAS,YAAY;KACpD,IAAI,WAAW,OAAO,GACpB;KAGF,KAAK,OAAO,aAAa;KAEzB,KAAK,mBAAmB,IAAI,CAAC;KAC7B,QAAQ,QAAQ,IAAhB;MACE,KAAK;OACH,GAAG,OAAO,oDAAoD,EAC5D,SAAS,QAAQ,GACnB,CAAC;OACD;MACF,KAAK,UAAU;OACb,MAAM,EAAC,WAAW,aAAY,QAAQ;OACtC,KAAK,iCAAiC;OACtC,IAAI,UACF,KAAK,eAAe,sBAAsB,UAAU;OAEtD,IAAI,WAAW;QACb,MAAM,SAA4B,EAChC,gBAAgB,UAAU,eAC5B;QACA,IAAI,UAAU,aACZ,OAAO,cAAc;SACnB,GAAG,UAAU;SACb,iBAAiB,KAAK,IAAI;QAC5B;QAEF,KAAK,UAAU,OAAO,MAAM;OAC9B;OACA;MACF;MACA,KAAK,SAAS;OACZ,MAAM,EACJ,MACA,SAAS,KACT,cACA,gBAAgB,cACd,QAAQ;OACZ,IACE,SAAA,MAEC,iBACE,cAAc,KAAA,KAAa,YAAY,oBAC1C,KAAK,mCACH,sCACF;QAKA,GAAG,OACD,2DACU,eAAe,eAC3B;QACA;OACF;OAGA,KAAK,KACH,IAEA,IAAI,WACF,GAAG,wBAAwB,IAAiB,EAAE,IAAI,KACpD,CACF;OACA;MACF;MACA,SAAS;OACP,MAAM,MAAM,QAAQ;OACpB,IAAI,IAAI,QAAQ,cAAc,IAAI,QAAQ;QACxC,MAAM,EAAC,QAAQ,aAAY;QAC3B,MAAM,QAAQ,CAAC;QACf,WAAW;SACT;SACA,OAAO,SAAS;SAChB,SAAS,CAAC,GAAG,SAAS,OAAO,SAAS,GAAG,IAAI,OAAO;QACtD;QACA,IAAI,OAEF,sBAAsB,SAAS,IAAI;OAEvC;OAEA,IAAI,IAAI,QAAQ,kBAAkB,KAAK,kBAAkB;QAIvD,MAAM,YAAY;QAClB,MAAM,QAAQ,uBAAuB,IAAI,IAAI;QAC7C,aAAa,IAAI,iBAAiB;QAClC,SAAS,MAAM,KAAK;QACpB,KAAK,iBAAiB,QACpB,IACA,eACA,kBAAkB,MAAM,KAAK,MAAM,MAAM,aACzC,KACA,SAAS,KACX;QAEA,SAAS,aAAa;QACtB,WAAW,KAAK,OAA2B;QAC3C,kBAAkB;QAClB,MAAM,YAAY;QAClB,MAAM,UAAU,SAAS,OAAO;QAChC,GAAG,OACD,iBAAiB,MAAM,KAAK,IAAI,QAAQ,QAAQ,CAAC,EAAE,KACrD;QACA,KAAK,iBAAiB,QACpB,IACA,eACA,iBAAiB,MAAM,KAAK,MAAM,MAAM,aACxC,GACA,SAAS,KACX;QACA,IAAI,UAEF,sBAAsB,SAAS,KAAK;QAEtC;OACF;OACA,WAAW,KAAA;OAEX,MAAM,OAAO,QAAQ;OACrB,MAAM,6BACJ,SAAS,UAAU,gBAAgB,CAAC;OACtC,IAAI,SAAS,SACX,gBAAgB;YACX,IAAI,SAAS,YAAY,SAAS,YACvC,gBAAgB;OAGlB,WAAW,KAAK,OAA2B;OAC3C,kBAAkB;OAClB,IAGE,SAAS,YACT,SAAS,cAGT,8BACA,WAAW,UAAU,6BACrB,kBAAkB,uBAElB,MAAM,YAAY;OAEpB;MACF;KACF;IACF;IACA,KAAK,QAAQ,MAAM;GACrB,SAAS,GAAG;IACV,MAAM;IACN,KAAK,QAAQ,MAAM;GACrB,UAAU;IACR,YAAY,OAAO;IACnB,WAAW;IACX,KAAK,kBAAkB,KAAK;GAC9B;GACA,IAAI,QAAQ,KAAA,KAAa,KAAK,eAAe,mBAC3C;GAEF,KAAK,eAAe,kBAAkB,CAAC,EAAE,OAAO;GAChD,MAAM,KAAK,OAAO,QAAQ,IAAI,GAAG;EACnC;EACA,GAAG,OAAO,2BAA2B;EACrC,IAAI,aACF,MAAM;CAEV;CAEA,cAAc,IAAgB,QAA6B;EACzD,IAAI,CAAC,QACH;EAEF,IAAI,OAAO,mBAAmB;GAE5B,MAAM,SAAS,OAAO;GACtB,KAAK,kBAAkB,QACrB,IACA,eACA,cAAc,OAAO,MAAM,SAC3B,UACO,EAAC,gBAAgB,CAAC,MAAM,EAAC,EAClC;EACF,OAAO,IAAI,OAAO,eAChB,KAAK,kBAAkB,QAAQ,IAAI,eAAe,gBAAgB;EAEpE,IAAI,OAAO,aAAa,OAAO,kBAC7B,KAAU,UAAU,kBAAkB;GACpC,OAAO;GACP,WAAW,OAAO;GAClB,oBAAoB,KAAK,IAAI;GAC7B,sBAAsB,OAAO;EAC/B,CAAC;CAEL;CAEA,YAAkC;EAChC,OAAO,KAAK,UAAU,UAAU;CAClC;CAEA,mBACE,gBACA,WACA,WACmB;EACnB,OAAO;GACL,iBAAA;GACA,QAAQ,KAAK;GACb,IAAI,KAAK;GACT,MAAM,KAAK;GACX;GACA;GACA;GACA,GAAI,KAAK,gBAAgB,EAAC,WAAW,KAAI,IAAI,CAAC;EAChD;CACF;;;;;;CAOA,kBAAkB,aAAwC;EACxD,MAAM,KAAK,IAAI,SAAS,KAAK,KAAK,aAAa,EAAC,UAAU,KAAI,CAAC;EAC/D,IAAI;GACF,MAAM,EAAC,gBAAgB,WAAW,cAAa,GAAG,mBAAmB;IACnE,GAAG,qBAAqB,IAAI,gBAAgB,EAAE,CAAC;IAC/C,WAAW,qBAAqB,EAAE;GACpC,EAAE;GACF,OAAO,KAAK,mBAAmB,gBAAgB,WAAW,SAAS;EACrE,UAAU;GACR,GAAG,MAAM;EACX;CACF;CAEA,KAAK,IAAgB,KAAe;EAClC,KAAK,OAAO,KAAK,IAAI,GAAG;EAExB,KAAK,QAAQ,MAAM;CACrB;AACF"}

package/out/zero-cache/src/services/statz.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"statz.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/services/statz.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,YAAY,EAAE,cAAc,EAAC,MAAM,SAAS,CAAC;AAG1D,OAAO,KAAK,EAAC,oBAAoB,IAAI,UAAU,EAAC,MAAM,wBAAwB,CAAC;AAiT/E;;;;;;;;;GASG;AACH,wBAAsB,kBAAkB,CACtC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,UAAU,EAClB,GAAG,EAAE,cAAc,EACnB,GAAG,EAAE,YAAY,iBAuDlB"}
+{"version":3,"file":"statz.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/services/statz.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,EAAC,YAAY,EAAE,cAAc,EAAC,MAAM,SAAS,CAAC;AAG1D,OAAO,KAAK,EAAC,oBAAoB,IAAI,UAAU,EAAC,MAAM,wBAAwB,CAAC;AAiT/E;;;;;;;;;GASG;AACH,wBAAsB,kBAAkB,CACtC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,UAAU,EAClB,GAAG,EAAE,cAAc,EACnB,GAAG,EAAE,YAAY,iBA2DlB"}

package/out/zero-cache/src/services/statz.js

@@ -1,14 +1,13 @@
 import { BigIntJSON } from "../../../shared/src/bigint-json.js";
 import { getShardID, upstreamSchema } from "../types/shards.js";
-import { getOperatorAccess } from "../config/zero-config.js";
 import { Database } from "../../../zqlite/src/db.js";
 import { getReplicationState } from "./replicator/schema/replication-state.js";
 import { StatementRunner } from "../db/statements.js";
 import { pgClient } from "../types/pg.js";
+import { authorizeAdminRequest } from "./admin-auth.js";
 import { fromStateVersionString } from "./change-source/pg/lsn.js";
 import os from "os";
 import fs from "fs";
-import auth from "basic-auth";
 //#region ../zero-cache/src/services/statz.ts
 async function upstreamStats(lc, config) {
 	const schema = upstreamSchema(getShardID(config));
@@ -221,11 +220,7 @@ function printStats(group, statsObject) {
 * * `pretty`: formats the JSON object with indentation
 */
 async function handleStatzRequest(lc, config, req, res) {
-	const credentials = auth(req);
-	if (getOperatorAccess(lc, config, credentials?.pass) === void 0) {
-		res.code(401).header("WWW-Authenticate", "Basic realm=\"Statz Protected Area\"").send("Unauthorized");
-		return;
-	}
+	if (authorizeAdminRequest(lc, config, req, res, "Statz Protected Area", "operator") === void 0) return;
 	const statsFns = {
 		upstream: () => upstreamStats(lc, config),
 		cvr: () => cvrStats(lc, config),

package/out/zero-cache/src/services/statz.js.map

@@ -1 +1 @@
-{"version":3,"file":"statz.js","names":[],"sources":["../../../../../zero-cache/src/services/statz.ts"],"sourcesContent":["import fs from 'fs';\nimport os from 'os';\nimport type {LogContext} from '@rocicorp/logger';\nimport auth from 'basic-auth';\nimport type {FastifyReply, FastifyRequest} from 'fastify';\nimport {BigIntJSON} from '../../../shared/src/bigint-json.ts';\nimport {Database} from '../../../zqlite/src/db.ts';\nimport type {NormalizedZeroConfig as ZeroConfig} from '../config/normalize.ts';\nimport {getOperatorAccess} from '../config/zero-config.ts';\nimport {StatementRunner} from '../db/statements.ts';\nimport {pgClient} from '../types/pg.ts';\nimport {getShardID, upstreamSchema} from '../types/shards.ts';\nimport {fromStateVersionString} from './change-source/pg/lsn.ts';\nimport {getReplicationState} from './replicator/schema/replication-state.ts';\n\nasync function upstreamStats(lc: LogContext, config: ZeroConfig) {\n  const schema = upstreamSchema(getShardID(config));\n  const sql = pgClient(lc, config.upstream.db, 'statz-upstream');\n  try {\n    return await getPgStats([\n      [\n        'numReplicas',\n        sql`SELECT COUNT(*) as \"c\" FROM ${sql(schema)}.\"replicas\"`,\n      ],\n      [\n        'numClientsWithMutations',\n        sql`SELECT COUNT(*) as \"c\" FROM ${sql(schema)}.\"clients\"`,\n      ],\n      [\n        'numMutationsProcessed',\n        sql`SELECT SUM(\"lastMutationID\") as \"c\" FROM ${sql(schema)}.\"clients\"`,\n      ],\n    ]);\n  } finally {\n    await sql.end();\n  }\n}\n\nasync function cvrStats(lc: LogContext, config: ZeroConfig) {\n  const schema = upstreamSchema(getShardID(config)) + '/cvr';\n  const sql = pgClient(lc, config.cvr.db, 'statz-cvr');\n\n  function numQueriesPerClientGroup(\n    active: boolean,\n  ): ReturnType<ReturnType<typeof pgClient>> {\n    const filter = active\n      ? sql`WHERE \"inactivatedAt\" IS NULL AND deleted = false`\n      : sql`WHERE \"inactivatedAt\" IS NOT NULL AND (\"inactivatedAt\" + \"ttl\") > NOW()`;\n    return sql`WITH\n    group_counts AS (\n      SELECT\n        \"clientGroupID\",\n        COUNT(*) AS num_queries\n      FROM ${sql(schema)}.\"desires\"\n      ${filter}\n      GROUP BY \"clientGroupID\"\n    ),\n    -- Count distinct clientIDs per clientGroupID\n    client_per_group_counts AS (\n      SELECT\n        \"clientGroupID\",\n        COUNT(DISTINCT \"clientID\") AS num_clients\n      FROM ${sql(schema)}.\"desires\"\n      ${filter}\n      GROUP BY \"clientGroupID\"\n    )\n    -- Combine all the information\n    SELECT\n      g.\"clientGroupID\",\n      cpg.num_clients,\n      g.num_queries\n    FROM group_counts g\n    JOIN client_per_group_counts cpg ON g.\"clientGroupID\" = cpg.\"clientGroupID\"\n    ORDER BY g.num_queries DESC;`;\n  }\n\n  try {\n    return await getPgStats([\n      [\n        'totalNumQueries',\n        sql`SELECT COUNT(*) as \"c\" FROM ${sql(schema)}.\"desires\"`,\n      ],\n      [\n        'numUniqueQueryHashes',\n        sql`SELECT COUNT(DISTINCT \"queryHash\") as \"c\" FROM ${sql(\n          schema,\n        )}.\"desires\"`,\n      ],\n      [\n        'numActiveQueries',\n        sql`SELECT COUNT(*) as \"c\" FROM ${sql(schema)}.\"desires\" WHERE \"inactivatedAt\" IS NULL AND \"deleted\" = false`,\n      ],\n      [\n        'numInactiveQueries',\n        sql`SELECT COUNT(*) as \"c\" FROM ${sql(schema)}.\"desires\" WHERE \"inactivatedAt\" IS NOT NULL AND (\"inactivatedAt\" + \"ttl\") > NOW()`,\n      ],\n      [\n        'numDeletedQueries',\n        sql`SELECT COUNT(*) as \"c\" FROM ${sql(schema)}.\"desires\" WHERE \"deleted\" = true`,\n      ],\n      [\n        'freshQueriesPercentiles',\n        sql`WITH client_group_counts AS (\n        -- Count inactive desires per clientGroupID\n        SELECT\n          \"clientGroupID\",\n          COUNT(*) AS fresh_count\n        FROM ${sql(schema)}.\"desires\"\n        WHERE\n          (\"inactivatedAt\" IS NOT NULL\n          AND (\"inactivatedAt\" + \"ttl\") > NOW()) OR (\"inactivatedAt\" IS NULL\n          AND deleted = false)\n        GROUP BY \"clientGroupID\"\n      )\n\n      SELECT\n        percentile_cont(0.50) WITHIN GROUP (ORDER BY fresh_count) AS \"p50\",\n        percentile_cont(0.75) WITHIN GROUP (ORDER BY fresh_count) AS \"p75\",\n        percentile_cont(0.90) WITHIN GROUP (ORDER BY fresh_count) AS \"p90\",\n        percentile_cont(0.95) WITHIN GROUP (ORDER BY fresh_count) AS \"p95\",\n        percentile_cont(0.99) WITHIN GROUP (ORDER BY fresh_count) AS \"p99\",\n        MIN(fresh_count) AS \"min\",\n        MAX(fresh_count) AS \"max\",\n        AVG(fresh_count) AS \"avg\"\n      FROM client_group_counts;`,\n      ],\n      [\n        'rowsPerClientGroupPercentiles',\n        sql`WITH client_group_counts AS (\n        -- Count inactive desires per clientGroupID\n        SELECT\n          \"clientGroupID\",\n          COUNT(*) AS row_count\n        FROM ${sql(schema)}.\"rows\"\n        GROUP BY \"clientGroupID\"\n      )\n      SELECT\n        percentile_cont(0.50) WITHIN GROUP (ORDER BY row_count) AS \"p50\",\n        percentile_cont(0.75) WITHIN GROUP (ORDER BY row_count) AS \"p75\",\n        percentile_cont(0.90) WITHIN GROUP (ORDER BY row_count) AS \"p90\",\n        percentile_cont(0.95) WITHIN GROUP (ORDER BY row_count) AS \"p95\",\n        percentile_cont(0.99) WITHIN GROUP (ORDER BY row_count) AS \"p99\",\n        MIN(row_count) AS \"min\",\n        MAX(row_count) AS \"max\",\n        AVG(row_count) AS \"avg\"\n      FROM client_group_counts;`,\n      ],\n      [\n        // check for AST blowup due to DNF conversion.\n        'astSizes',\n        sql`SELECT\n        percentile_cont(0.25) WITHIN GROUP (ORDER BY length(\"clientAST\"::text)) AS \"25th_percentile\",\n        percentile_cont(0.5) WITHIN GROUP (ORDER BY length(\"clientAST\"::text)) AS \"50th_percentile\",\n        percentile_cont(0.75) WITHIN GROUP (ORDER BY length(\"clientAST\"::text)) AS \"75th_percentile\",\n        percentile_cont(0.9) WITHIN GROUP (ORDER BY length(\"clientAST\"::text)) AS \"90th_percentile\",\n        percentile_cont(0.95) WITHIN GROUP (ORDER BY length(\"clientAST\"::text)) AS \"95th_percentile\",\n        percentile_cont(0.99) WITHIN GROUP (ORDER BY length(\"clientAST\"::text)) AS \"99th_percentile\",\n        MIN(length(\"clientAST\"::text)) AS \"minimum_length\",\n        MAX(length(\"clientAST\"::text)) AS \"maximum_length\",\n        AVG(length(\"clientAST\"::text))::integer AS \"average_length\",\n        COUNT(*) AS \"total_records\"\n      FROM ${sql(schema)}.\"queries\";`,\n      ],\n      [\n        // output the hash of the largest AST\n        'biggestAstHash',\n        sql`SELECT \"queryHash\", length(\"clientAST\"::text) AS \"ast_length\"\n      FROM ${sql(schema)}.\"queries\"\n      ORDER BY length(\"clientAST\"::text) DESC\n      LIMIT 1;`,\n      ],\n      [\n        'totalActiveQueriesPerClientAndClientGroup',\n        numQueriesPerClientGroup(true),\n      ],\n      [\n        'totalInactiveQueriesPerClientAndClientGroup',\n        numQueriesPerClientGroup(false),\n      ],\n      [\n        'totalRowsPerClientGroup',\n        sql`SELECT \"clientGroupID\", COUNT(*) as \"c\" FROM ${sql(\n          schema,\n        )}.\"rows\" GROUP BY \"clientGroupID\" ORDER BY \"c\" DESC`,\n      ],\n      [\n        'numRowsPerQuery',\n        sql`SELECT\n        k.key AS \"queryHash\",\n        COUNT(*) AS row_count\n      FROM ${sql(schema)}.\"rows\" r,\n      LATERAL jsonb_each(r.\"refCounts\") k\n      GROUP BY k.key\n      ORDER BY row_count DESC;`,\n      ],\n    ] satisfies [\n      name: string,\n      query: ReturnType<ReturnType<typeof pgClient>>,\n    ][]);\n  } finally {\n    await sql.end();\n  }\n}\n\nasync function changeLogStats(lc: LogContext, config: ZeroConfig) {\n  const schema = upstreamSchema(getShardID(config)) + '/cdc';\n  const sql = pgClient(lc, config.change.db, 'statz-change');\n\n  try {\n    return await getPgStats([\n      [\n        'changeLogSize',\n        sql`SELECT COUNT(*) as \"change_log_size\" FROM ${sql(schema)}.\"changeLog\"`,\n      ],\n    ]);\n  } finally {\n    await sql.end();\n  }\n}\n\n// The replica is opened read-only: /statz is available to the operator\n// password, and must not change the replica. Among other things, a writable\n// connection would run `PRAGMA optimize` when closed.\nfunction replicaStats(lc: LogContext, config: ZeroConfig) {\n  const db = new Database(lc, config.replica.file, {readonly: true});\n  try {\n    return Object.fromEntries([\n      ['wal file sizes', walFileSizes(config.replica.file)],\n      ['page count', pick(first(db.pragma('PAGE_COUNT')))],\n      ['page size', pick(first(db.pragma('PAGE_SIZE')))],\n      ['journal mode', pick(first(db.pragma('JOURNAL_MODE')))],\n      ['synchronous', pick(first(db.pragma('SYNCHRONOUS')))],\n      ['cache size', pick(first(db.pragma('CACHE_SIZE')))],\n      ['auto vacuum', pick(first(db.pragma('AUTO_VACUUM')))],\n      ['freelist count', pick(first(db.pragma('FREELIST_COUNT')))],\n      ['wal autocheckpoint', pick(first(db.pragma('WAL_AUTOCHECKPOINT')))],\n      ['db file stats', fs.statSync(config.replica.file)],\n    ] as const);\n  } finally {\n    db.close();\n  }\n}\n\nfunction replicationStats(lc: LogContext, config: ZeroConfig) {\n  const db = new Database(lc, config.replica.file, {readonly: true});\n  try {\n    return getReplicationStats(db);\n  } finally {\n    db.close();\n  }\n}\n\n/**\n * The sizes of the replica's WAL files, by suffix, read from the filesystem.\n * `PRAGMA wal_checkpoint` would report the WAL's frame counts, but it runs a\n * checkpoint to do so, and fails on a read-only connection in WAL mode.\n */\nfunction walFileSizes(file: string): Record<string, number> {\n  const sizes: Record<string, number> = {};\n  // `-wal` is used in WAL mode, and both files in WAL2 mode.\n  for (const suffix of ['-wal', '-wal2']) {\n    const stats = fs.statSync(file + suffix, {throwIfNoEntry: false});\n    if (stats) {\n      sizes[suffix] = stats.size;\n    }\n  }\n  return sizes;\n}\n\nfunction getReplicationStats(db: Database) {\n  const {stateVersion} = getReplicationState(new StatementRunner(db));\n  const lsn = fromStateVersionString(stateVersion);\n  return {lsn};\n}\n\nfunction osStats() {\n  return Object.fromEntries([\n    ['load avg', os.loadavg()],\n    ['uptime', os.uptime()],\n    ['total mem', os.totalmem()],\n    ['free mem', os.freemem()],\n    ['cpus', os.cpus().length],\n    ['available parallelism', os.availableParallelism()],\n    ['platform', os.platform()],\n    ['arch', os.arch()],\n    ['release', os.release()],\n  ] as const);\n}\n\nasync function getPgStats(\n  pendingQueries: [\n    name: string,\n    query: ReturnType<ReturnType<typeof pgClient>>,\n  ][],\n) {\n  const results = await Promise.all(\n    pendingQueries.map(async ([name, query]) => [name, await query] as const),\n  );\n  return Object.fromEntries(results);\n}\n\ntype StatsObject = Record<string, unknown>;\n\nfunction printStats(group: string, statsObject: StatsObject): string {\n  const lines: string[] = ['\\n' + header(group)];\n  for (const [name, result] of Object.entries(statsObject)) {\n    lines.push('\\n' + name + ': ' + BigIntJSON.stringify(result, null, 2));\n  }\n  lines.push('\\n');\n  return lines.join('');\n}\n\n/**\n * Serves counts and sizes. This is available to the operator password as well\n * as the admin password (see `--operator-password`), so it must never return\n * application data: no row values, query ASTs or query arguments.\n *\n * HTTP query parameters:\n * * `group`: restricts the groups for which stats are computed\n * * `format=json`: returns the stats as a JSON object\n * * `pretty`: formats the JSON object with indentation\n */\nexport async function handleStatzRequest(\n  lc: LogContext,\n  config: ZeroConfig,\n  req: FastifyRequest,\n  res: FastifyReply,\n) {\n  const credentials = auth(req);\n  if (getOperatorAccess(lc, config, credentials?.pass) === undefined) {\n    void res\n      .code(401)\n      .header('WWW-Authenticate', 'Basic realm=\"Statz Protected Area\"')\n      .send('Unauthorized');\n    return;\n  }\n\n  const statsFns: Record<string, () => Promise<StatsObject> | StatsObject> = {\n    upstream: () => upstreamStats(lc, config),\n    cvr: () => cvrStats(lc, config),\n    changeLog: () => changeLogStats(lc, config),\n    replica: () => replicaStats(lc, config),\n    replication: () => replicationStats(lc, config),\n    os: () => osStats(),\n  };\n\n  async function computeStats(group: string): Promise<[string, StatsObject]> {\n    try {\n      return [group, await statsFns[group]()];\n    } catch (e) {\n      lc.error?.(`error computing ${group} stats`, e);\n      return [group, {error: String(e)}];\n    }\n  }\n\n  const query = req.query as Record<string, unknown>;\n  const groups =\n    typeof query.group === 'string'\n      ? query.group.split(',')\n      : Array.isArray(query.group)\n        ? query.group\n        : undefined;\n\n  const stats = await Promise.all(\n    groups\n      ? groups.filter(g => g in statsFns).map(computeStats)\n      : Object.keys(statsFns).map(computeStats),\n  );\n\n  if (query.format === 'json') {\n    const indent = query.pretty !== undefined ? 2 : undefined;\n    await res\n      .header('Content-Type', 'application/json')\n      .send(BigIntJSON.stringify(Object.fromEntries(stats), null, indent));\n    return;\n  } else {\n    const body = stats\n      .map(([group, statsObject]) => printStats(group, statsObject))\n      .join('');\n    await res.header('Content-Type', 'text/plain; charset=utf-8').send(body);\n  }\n}\n\nfunction first(x: object[]): object {\n  return x[0];\n}\n\nfunction pick(x: object): unknown {\n  return Object.values(x)[0];\n}\n\nfunction header(name: string): string {\n  return `=== ${name} ===\\n`;\n}\n"],"mappings":";;;;;;;;;;;;AAeA,eAAe,cAAc,IAAgB,QAAoB;CAC/D,MAAM,SAAS,eAAe,WAAW,MAAM,CAAC;CAChD,MAAM,MAAM,SAAS,IAAI,OAAO,SAAS,IAAI,gBAAgB;CAC7D,IAAI;EACF,OAAO,MAAM,WAAW;GACtB,CACE,eACA,GAAG,+BAA+B,IAAI,MAAM,EAAE,YAChD;GACA,CACE,2BACA,GAAG,+BAA+B,IAAI,MAAM,EAAE,WAChD;GACA,CACE,yBACA,GAAG,4CAA4C,IAAI,MAAM,EAAE,WAC7D;EACF,CAAC;CACH,UAAU;EACR,MAAM,IAAI,IAAI;CAChB;AACF;AAEA,eAAe,SAAS,IAAgB,QAAoB;CAC1D,MAAM,SAAS,eAAe,WAAW,MAAM,CAAC,IAAI;CACpD,MAAM,MAAM,SAAS,IAAI,OAAO,IAAI,IAAI,WAAW;CAEnD,SAAS,yBACP,QACyC;EACzC,MAAM,SAAS,SACX,GAAG,sDACH,GAAG;EACP,OAAO,GAAG;;;;;aAKD,IAAI,MAAM,EAAE;QACjB,OAAO;;;;;;;;aAQF,IAAI,MAAM,EAAE;QACjB,OAAO;;;;;;;;;;;CAWb;CAEA,IAAI;EACF,OAAO,MAAM,WAAW;GACtB,CACE,mBACA,GAAG,+BAA+B,IAAI,MAAM,EAAE,WAChD;GACA,CACE,wBACA,GAAG,kDAAkD,IACnD,MACF,EAAE,WACJ;GACA,CACE,oBACA,GAAG,+BAA+B,IAAI,MAAM,EAAE,+DAChD;GACA,CACE,sBACA,GAAG,+BAA+B,IAAI,MAAM,EAAE,mFAChD;GACA,CACE,qBACA,GAAG,+BAA+B,IAAI,MAAM,EAAE,kCAChD;GACA,CACE,2BACA,GAAG;;;;;eAKI,IAAI,MAAM,EAAE;;;;;;;;;;;;;;;;;gCAkBrB;GACA,CACE,iCACA,GAAG;;;;;eAKI,IAAI,MAAM,EAAE;;;;;;;;;;;;gCAarB;GACA,CAEE,YACA,GAAG;;;;;;;;;;;aAWE,IAAI,MAAM,EAAE,YACnB;GACA,CAEE,kBACA,GAAG;aACE,IAAI,MAAM,EAAE;;eAGnB;GACA,CACE,6CACA,yBAAyB,IAAI,CAC/B;GACA,CACE,+CACA,yBAAyB,KAAK,CAChC;GACA,CACE,2BACA,GAAG,gDAAgD,IACjD,MACF,EAAE,mDACJ;GACA,CACE,mBACA,GAAG;;;aAGE,IAAI,MAAM,EAAE;;;+BAInB;EACF,CAGG;CACL,UAAU;EACR,MAAM,IAAI,IAAI;CAChB;AACF;AAEA,eAAe,eAAe,IAAgB,QAAoB;CAChE,MAAM,SAAS,eAAe,WAAW,MAAM,CAAC,IAAI;CACpD,MAAM,MAAM,SAAS,IAAI,OAAO,OAAO,IAAI,cAAc;CAEzD,IAAI;EACF,OAAO,MAAM,WAAW,CACtB,CACE,iBACA,GAAG,6CAA6C,IAAI,MAAM,EAAE,aAC9D,CACF,CAAC;CACH,UAAU;EACR,MAAM,IAAI,IAAI;CAChB;AACF;AAKA,SAAS,aAAa,IAAgB,QAAoB;CACxD,MAAM,KAAK,IAAI,SAAS,IAAI,OAAO,QAAQ,MAAM,EAAC,UAAU,KAAI,CAAC;CACjE,IAAI;EACF,OAAO,OAAO,YAAY;GACxB,CAAC,kBAAkB,aAAa,OAAO,QAAQ,IAAI,CAAC;GACpD,CAAC,cAAc,KAAK,MAAM,GAAG,OAAO,YAAY,CAAC,CAAC,CAAC;GACnD,CAAC,aAAa,KAAK,MAAM,GAAG,OAAO,WAAW,CAAC,CAAC,CAAC;GACjD,CAAC,gBAAgB,KAAK,MAAM,GAAG,OAAO,cAAc,CAAC,CAAC,CAAC;GACvD,CAAC,eAAe,KAAK,MAAM,GAAG,OAAO,aAAa,CAAC,CAAC,CAAC;GACrD,CAAC,cAAc,KAAK,MAAM,GAAG,OAAO,YAAY,CAAC,CAAC,CAAC;GACnD,CAAC,eAAe,KAAK,MAAM,GAAG,OAAO,aAAa,CAAC,CAAC,CAAC;GACrD,CAAC,kBAAkB,KAAK,MAAM,GAAG,OAAO,gBAAgB,CAAC,CAAC,CAAC;GAC3D,CAAC,sBAAsB,KAAK,MAAM,GAAG,OAAO,oBAAoB,CAAC,CAAC,CAAC;GACnE,CAAC,iBAAiB,GAAG,SAAS,OAAO,QAAQ,IAAI,CAAC;EACpD,CAAU;CACZ,UAAU;EACR,GAAG,MAAM;CACX;AACF;AAEA,SAAS,iBAAiB,IAAgB,QAAoB;CAC5D,MAAM,KAAK,IAAI,SAAS,IAAI,OAAO,QAAQ,MAAM,EAAC,UAAU,KAAI,CAAC;CACjE,IAAI;EACF,OAAO,oBAAoB,EAAE;CAC/B,UAAU;EACR,GAAG,MAAM;CACX;AACF;;;;;;AAOA,SAAS,aAAa,MAAsC;CAC1D,MAAM,QAAgC,CAAC;CAEvC,KAAK,MAAM,UAAU,CAAC,QAAQ,OAAO,GAAG;EACtC,MAAM,QAAQ,GAAG,SAAS,OAAO,QAAQ,EAAC,gBAAgB,MAAK,CAAC;EAChE,IAAI,OACF,MAAM,UAAU,MAAM;CAE1B;CACA,OAAO;AACT;AAEA,SAAS,oBAAoB,IAAc;CACzC,MAAM,EAAC,iBAAgB,oBAAoB,IAAI,gBAAgB,EAAE,CAAC;CAElE,OAAO,EAAC,KADI,uBAAuB,YAC3B,EAAG;AACb;AAEA,SAAS,UAAU;CACjB,OAAO,OAAO,YAAY;EACxB,CAAC,YAAY,GAAG,QAAQ,CAAC;EACzB,CAAC,UAAU,GAAG,OAAO,CAAC;EACtB,CAAC,aAAa,GAAG,SAAS,CAAC;EAC3B,CAAC,YAAY,GAAG,QAAQ,CAAC;EACzB,CAAC,QAAQ,GAAG,KAAK,CAAC,CAAC,MAAM;EACzB,CAAC,yBAAyB,GAAG,qBAAqB,CAAC;EACnD,CAAC,YAAY,GAAG,SAAS,CAAC;EAC1B,CAAC,QAAQ,GAAG,KAAK,CAAC;EAClB,CAAC,WAAW,GAAG,QAAQ,CAAC;CAC1B,CAAU;AACZ;AAEA,eAAe,WACb,gBAIA;CACA,MAAM,UAAU,MAAM,QAAQ,IAC5B,eAAe,IAAI,OAAO,CAAC,MAAM,WAAW,CAAC,MAAM,MAAM,KAAK,CAAU,CAC1E;CACA,OAAO,OAAO,YAAY,OAAO;AACnC;AAIA,SAAS,WAAW,OAAe,aAAkC;CACnE,MAAM,QAAkB,CAAC,OAAO,OAAO,KAAK,CAAC;CAC7C,KAAK,MAAM,CAAC,MAAM,WAAW,OAAO,QAAQ,WAAW,GACrD,MAAM,KAAK,OAAO,OAAO,OAAO,WAAW,UAAU,QAAQ,MAAM,CAAC,CAAC;CAEvE,MAAM,KAAK,IAAI;CACf,OAAO,MAAM,KAAK,EAAE;AACtB;;;;;;;;;;;AAYA,eAAsB,mBACpB,IACA,QACA,KACA,KACA;CACA,MAAM,cAAc,KAAK,GAAG;CAC5B,IAAI,kBAAkB,IAAI,QAAQ,aAAa,IAAI,MAAM,KAAA,GAAW;EAClE,IACG,KAAK,GAAG,CAAC,CACT,OAAO,oBAAoB,sCAAoC,CAAC,CAChE,KAAK,cAAc;EACtB;CACF;CAEA,MAAM,WAAqE;EACzE,gBAAgB,cAAc,IAAI,MAAM;EACxC,WAAW,SAAS,IAAI,MAAM;EAC9B,iBAAiB,eAAe,IAAI,MAAM;EAC1C,eAAe,aAAa,IAAI,MAAM;EACtC,mBAAmB,iBAAiB,IAAI,MAAM;EAC9C,UAAU,QAAQ;CACpB;CAEA,eAAe,aAAa,OAA+C;EACzE,IAAI;GACF,OAAO,CAAC,OAAO,MAAM,SAAS,MAAM,CAAC,CAAC;EACxC,SAAS,GAAG;GACV,GAAG,QAAQ,mBAAmB,MAAM,SAAS,CAAC;GAC9C,OAAO,CAAC,OAAO,EAAC,OAAO,OAAO,CAAC,EAAC,CAAC;EACnC;CACF;CAEA,MAAM,QAAQ,IAAI;CAClB,MAAM,SACJ,OAAO,MAAM,UAAU,WACnB,MAAM,MAAM,MAAM,GAAG,IACrB,MAAM,QAAQ,MAAM,KAAK,IACvB,MAAM,QACN,KAAA;CAER,MAAM,QAAQ,MAAM,QAAQ,IAC1B,SACI,OAAO,QAAO,MAAK,KAAK,QAAQ,CAAC,CAAC,IAAI,YAAY,IAClD,OAAO,KAAK,QAAQ,CAAC,CAAC,IAAI,YAAY,CAC5C;CAEA,IAAI,MAAM,WAAW,QAAQ;EAC3B,MAAM,SAAS,MAAM,WAAW,KAAA,IAAY,IAAI,KAAA;EAChD,MAAM,IACH,OAAO,gBAAgB,kBAAkB,CAAC,CAC1C,KAAK,WAAW,UAAU,OAAO,YAAY,KAAK,GAAG,MAAM,MAAM,CAAC;EACrE;CACF,OAAO;EACL,MAAM,OAAO,MACV,KAAK,CAAC,OAAO,iBAAiB,WAAW,OAAO,WAAW,CAAC,CAAC,CAC7D,KAAK,EAAE;EACV,MAAM,IAAI,OAAO,gBAAgB,2BAA2B,CAAC,CAAC,KAAK,IAAI;CACzE;AACF;AAEA,SAAS,MAAM,GAAqB;CAClC,OAAO,EAAE;AACX;AAEA,SAAS,KAAK,GAAoB;CAChC,OAAO,OAAO,OAAO,CAAC,CAAC,CAAC;AAC1B;AAEA,SAAS,OAAO,MAAsB;CACpC,OAAO,OAAO,KAAK;AACrB"}
+{"version":3,"file":"statz.js","names":[],"sources":["../../../../../zero-cache/src/services/statz.ts"],"sourcesContent":["import fs from 'fs';\nimport os from 'os';\nimport type {LogContext} from '@rocicorp/logger';\nimport type {FastifyReply, FastifyRequest} from 'fastify';\nimport {BigIntJSON} from '../../../shared/src/bigint-json.ts';\nimport {Database} from '../../../zqlite/src/db.ts';\nimport type {NormalizedZeroConfig as ZeroConfig} from '../config/normalize.ts';\nimport {StatementRunner} from '../db/statements.ts';\nimport {pgClient} from '../types/pg.ts';\nimport {getShardID, upstreamSchema} from '../types/shards.ts';\nimport {authorizeAdminRequest} from './admin-auth.ts';\nimport {fromStateVersionString} from './change-source/pg/lsn.ts';\nimport {getReplicationState} from './replicator/schema/replication-state.ts';\n\nasync function upstreamStats(lc: LogContext, config: ZeroConfig) {\n  const schema = upstreamSchema(getShardID(config));\n  const sql = pgClient(lc, config.upstream.db, 'statz-upstream');\n  try {\n    return await getPgStats([\n      [\n        'numReplicas',\n        sql`SELECT COUNT(*) as \"c\" FROM ${sql(schema)}.\"replicas\"`,\n      ],\n      [\n        'numClientsWithMutations',\n        sql`SELECT COUNT(*) as \"c\" FROM ${sql(schema)}.\"clients\"`,\n      ],\n      [\n        'numMutationsProcessed',\n        sql`SELECT SUM(\"lastMutationID\") as \"c\" FROM ${sql(schema)}.\"clients\"`,\n      ],\n    ]);\n  } finally {\n    await sql.end();\n  }\n}\n\nasync function cvrStats(lc: LogContext, config: ZeroConfig) {\n  const schema = upstreamSchema(getShardID(config)) + '/cvr';\n  const sql = pgClient(lc, config.cvr.db, 'statz-cvr');\n\n  function numQueriesPerClientGroup(\n    active: boolean,\n  ): ReturnType<ReturnType<typeof pgClient>> {\n    const filter = active\n      ? sql`WHERE \"inactivatedAt\" IS NULL AND deleted = false`\n      : sql`WHERE \"inactivatedAt\" IS NOT NULL AND (\"inactivatedAt\" + \"ttl\") > NOW()`;\n    return sql`WITH\n    group_counts AS (\n      SELECT\n        \"clientGroupID\",\n        COUNT(*) AS num_queries\n      FROM ${sql(schema)}.\"desires\"\n      ${filter}\n      GROUP BY \"clientGroupID\"\n    ),\n    -- Count distinct clientIDs per clientGroupID\n    client_per_group_counts AS (\n      SELECT\n        \"clientGroupID\",\n        COUNT(DISTINCT \"clientID\") AS num_clients\n      FROM ${sql(schema)}.\"desires\"\n      ${filter}\n      GROUP BY \"clientGroupID\"\n    )\n    -- Combine all the information\n    SELECT\n      g.\"clientGroupID\",\n      cpg.num_clients,\n      g.num_queries\n    FROM group_counts g\n    JOIN client_per_group_counts cpg ON g.\"clientGroupID\" = cpg.\"clientGroupID\"\n    ORDER BY g.num_queries DESC;`;\n  }\n\n  try {\n    return await getPgStats([\n      [\n        'totalNumQueries',\n        sql`SELECT COUNT(*) as \"c\" FROM ${sql(schema)}.\"desires\"`,\n      ],\n      [\n        'numUniqueQueryHashes',\n        sql`SELECT COUNT(DISTINCT \"queryHash\") as \"c\" FROM ${sql(\n          schema,\n        )}.\"desires\"`,\n      ],\n      [\n        'numActiveQueries',\n        sql`SELECT COUNT(*) as \"c\" FROM ${sql(schema)}.\"desires\" WHERE \"inactivatedAt\" IS NULL AND \"deleted\" = false`,\n      ],\n      [\n        'numInactiveQueries',\n        sql`SELECT COUNT(*) as \"c\" FROM ${sql(schema)}.\"desires\" WHERE \"inactivatedAt\" IS NOT NULL AND (\"inactivatedAt\" + \"ttl\") > NOW()`,\n      ],\n      [\n        'numDeletedQueries',\n        sql`SELECT COUNT(*) as \"c\" FROM ${sql(schema)}.\"desires\" WHERE \"deleted\" = true`,\n      ],\n      [\n        'freshQueriesPercentiles',\n        sql`WITH client_group_counts AS (\n        -- Count inactive desires per clientGroupID\n        SELECT\n          \"clientGroupID\",\n          COUNT(*) AS fresh_count\n        FROM ${sql(schema)}.\"desires\"\n        WHERE\n          (\"inactivatedAt\" IS NOT NULL\n          AND (\"inactivatedAt\" + \"ttl\") > NOW()) OR (\"inactivatedAt\" IS NULL\n          AND deleted = false)\n        GROUP BY \"clientGroupID\"\n      )\n\n      SELECT\n        percentile_cont(0.50) WITHIN GROUP (ORDER BY fresh_count) AS \"p50\",\n        percentile_cont(0.75) WITHIN GROUP (ORDER BY fresh_count) AS \"p75\",\n        percentile_cont(0.90) WITHIN GROUP (ORDER BY fresh_count) AS \"p90\",\n        percentile_cont(0.95) WITHIN GROUP (ORDER BY fresh_count) AS \"p95\",\n        percentile_cont(0.99) WITHIN GROUP (ORDER BY fresh_count) AS \"p99\",\n        MIN(fresh_count) AS \"min\",\n        MAX(fresh_count) AS \"max\",\n        AVG(fresh_count) AS \"avg\"\n      FROM client_group_counts;`,\n      ],\n      [\n        'rowsPerClientGroupPercentiles',\n        sql`WITH client_group_counts AS (\n        -- Count inactive desires per clientGroupID\n        SELECT\n          \"clientGroupID\",\n          COUNT(*) AS row_count\n        FROM ${sql(schema)}.\"rows\"\n        GROUP BY \"clientGroupID\"\n      )\n      SELECT\n        percentile_cont(0.50) WITHIN GROUP (ORDER BY row_count) AS \"p50\",\n        percentile_cont(0.75) WITHIN GROUP (ORDER BY row_count) AS \"p75\",\n        percentile_cont(0.90) WITHIN GROUP (ORDER BY row_count) AS \"p90\",\n        percentile_cont(0.95) WITHIN GROUP (ORDER BY row_count) AS \"p95\",\n        percentile_cont(0.99) WITHIN GROUP (ORDER BY row_count) AS \"p99\",\n        MIN(row_count) AS \"min\",\n        MAX(row_count) AS \"max\",\n        AVG(row_count) AS \"avg\"\n      FROM client_group_counts;`,\n      ],\n      [\n        // check for AST blowup due to DNF conversion.\n        'astSizes',\n        sql`SELECT\n        percentile_cont(0.25) WITHIN GROUP (ORDER BY length(\"clientAST\"::text)) AS \"25th_percentile\",\n        percentile_cont(0.5) WITHIN GROUP (ORDER BY length(\"clientAST\"::text)) AS \"50th_percentile\",\n        percentile_cont(0.75) WITHIN GROUP (ORDER BY length(\"clientAST\"::text)) AS \"75th_percentile\",\n        percentile_cont(0.9) WITHIN GROUP (ORDER BY length(\"clientAST\"::text)) AS \"90th_percentile\",\n        percentile_cont(0.95) WITHIN GROUP (ORDER BY length(\"clientAST\"::text)) AS \"95th_percentile\",\n        percentile_cont(0.99) WITHIN GROUP (ORDER BY length(\"clientAST\"::text)) AS \"99th_percentile\",\n        MIN(length(\"clientAST\"::text)) AS \"minimum_length\",\n        MAX(length(\"clientAST\"::text)) AS \"maximum_length\",\n        AVG(length(\"clientAST\"::text))::integer AS \"average_length\",\n        COUNT(*) AS \"total_records\"\n      FROM ${sql(schema)}.\"queries\";`,\n      ],\n      [\n        // output the hash of the largest AST\n        'biggestAstHash',\n        sql`SELECT \"queryHash\", length(\"clientAST\"::text) AS \"ast_length\"\n      FROM ${sql(schema)}.\"queries\"\n      ORDER BY length(\"clientAST\"::text) DESC\n      LIMIT 1;`,\n      ],\n      [\n        'totalActiveQueriesPerClientAndClientGroup',\n        numQueriesPerClientGroup(true),\n      ],\n      [\n        'totalInactiveQueriesPerClientAndClientGroup',\n        numQueriesPerClientGroup(false),\n      ],\n      [\n        'totalRowsPerClientGroup',\n        sql`SELECT \"clientGroupID\", COUNT(*) as \"c\" FROM ${sql(\n          schema,\n        )}.\"rows\" GROUP BY \"clientGroupID\" ORDER BY \"c\" DESC`,\n      ],\n      [\n        'numRowsPerQuery',\n        sql`SELECT\n        k.key AS \"queryHash\",\n        COUNT(*) AS row_count\n      FROM ${sql(schema)}.\"rows\" r,\n      LATERAL jsonb_each(r.\"refCounts\") k\n      GROUP BY k.key\n      ORDER BY row_count DESC;`,\n      ],\n    ] satisfies [\n      name: string,\n      query: ReturnType<ReturnType<typeof pgClient>>,\n    ][]);\n  } finally {\n    await sql.end();\n  }\n}\n\nasync function changeLogStats(lc: LogContext, config: ZeroConfig) {\n  const schema = upstreamSchema(getShardID(config)) + '/cdc';\n  const sql = pgClient(lc, config.change.db, 'statz-change');\n\n  try {\n    return await getPgStats([\n      [\n        'changeLogSize',\n        sql`SELECT COUNT(*) as \"change_log_size\" FROM ${sql(schema)}.\"changeLog\"`,\n      ],\n    ]);\n  } finally {\n    await sql.end();\n  }\n}\n\n// The replica is opened read-only: /statz is available to the operator\n// password, and must not change the replica. Among other things, a writable\n// connection would run `PRAGMA optimize` when closed.\nfunction replicaStats(lc: LogContext, config: ZeroConfig) {\n  const db = new Database(lc, config.replica.file, {readonly: true});\n  try {\n    return Object.fromEntries([\n      ['wal file sizes', walFileSizes(config.replica.file)],\n      ['page count', pick(first(db.pragma('PAGE_COUNT')))],\n      ['page size', pick(first(db.pragma('PAGE_SIZE')))],\n      ['journal mode', pick(first(db.pragma('JOURNAL_MODE')))],\n      ['synchronous', pick(first(db.pragma('SYNCHRONOUS')))],\n      ['cache size', pick(first(db.pragma('CACHE_SIZE')))],\n      ['auto vacuum', pick(first(db.pragma('AUTO_VACUUM')))],\n      ['freelist count', pick(first(db.pragma('FREELIST_COUNT')))],\n      ['wal autocheckpoint', pick(first(db.pragma('WAL_AUTOCHECKPOINT')))],\n      ['db file stats', fs.statSync(config.replica.file)],\n    ] as const);\n  } finally {\n    db.close();\n  }\n}\n\nfunction replicationStats(lc: LogContext, config: ZeroConfig) {\n  const db = new Database(lc, config.replica.file, {readonly: true});\n  try {\n    return getReplicationStats(db);\n  } finally {\n    db.close();\n  }\n}\n\n/**\n * The sizes of the replica's WAL files, by suffix, read from the filesystem.\n * `PRAGMA wal_checkpoint` would report the WAL's frame counts, but it runs a\n * checkpoint to do so, and fails on a read-only connection in WAL mode.\n */\nfunction walFileSizes(file: string): Record<string, number> {\n  const sizes: Record<string, number> = {};\n  // `-wal` is used in WAL mode, and both files in WAL2 mode.\n  for (const suffix of ['-wal', '-wal2']) {\n    const stats = fs.statSync(file + suffix, {throwIfNoEntry: false});\n    if (stats) {\n      sizes[suffix] = stats.size;\n    }\n  }\n  return sizes;\n}\n\nfunction getReplicationStats(db: Database) {\n  const {stateVersion} = getReplicationState(new StatementRunner(db));\n  const lsn = fromStateVersionString(stateVersion);\n  return {lsn};\n}\n\nfunction osStats() {\n  return Object.fromEntries([\n    ['load avg', os.loadavg()],\n    ['uptime', os.uptime()],\n    ['total mem', os.totalmem()],\n    ['free mem', os.freemem()],\n    ['cpus', os.cpus().length],\n    ['available parallelism', os.availableParallelism()],\n    ['platform', os.platform()],\n    ['arch', os.arch()],\n    ['release', os.release()],\n  ] as const);\n}\n\nasync function getPgStats(\n  pendingQueries: [\n    name: string,\n    query: ReturnType<ReturnType<typeof pgClient>>,\n  ][],\n) {\n  const results = await Promise.all(\n    pendingQueries.map(async ([name, query]) => [name, await query] as const),\n  );\n  return Object.fromEntries(results);\n}\n\ntype StatsObject = Record<string, unknown>;\n\nfunction printStats(group: string, statsObject: StatsObject): string {\n  const lines: string[] = ['\\n' + header(group)];\n  for (const [name, result] of Object.entries(statsObject)) {\n    lines.push('\\n' + name + ': ' + BigIntJSON.stringify(result, null, 2));\n  }\n  lines.push('\\n');\n  return lines.join('');\n}\n\n/**\n * Serves counts and sizes. This is available to the operator password as well\n * as the admin password (see `--operator-password`), so it must never return\n * application data: no row values, query ASTs or query arguments.\n *\n * HTTP query parameters:\n * * `group`: restricts the groups for which stats are computed\n * * `format=json`: returns the stats as a JSON object\n * * `pretty`: formats the JSON object with indentation\n */\nexport async function handleStatzRequest(\n  lc: LogContext,\n  config: ZeroConfig,\n  req: FastifyRequest,\n  res: FastifyReply,\n) {\n  if (\n    authorizeAdminRequest(\n      lc,\n      config,\n      req,\n      res,\n      'Statz Protected Area',\n      'operator',\n    ) === undefined\n  ) {\n    return;\n  }\n\n  const statsFns: Record<string, () => Promise<StatsObject> | StatsObject> = {\n    upstream: () => upstreamStats(lc, config),\n    cvr: () => cvrStats(lc, config),\n    changeLog: () => changeLogStats(lc, config),\n    replica: () => replicaStats(lc, config),\n    replication: () => replicationStats(lc, config),\n    os: () => osStats(),\n  };\n\n  async function computeStats(group: string): Promise<[string, StatsObject]> {\n    try {\n      return [group, await statsFns[group]()];\n    } catch (e) {\n      lc.error?.(`error computing ${group} stats`, e);\n      return [group, {error: String(e)}];\n    }\n  }\n\n  const query = req.query as Record<string, unknown>;\n  const groups =\n    typeof query.group === 'string'\n      ? query.group.split(',')\n      : Array.isArray(query.group)\n        ? query.group\n        : undefined;\n\n  const stats = await Promise.all(\n    groups\n      ? groups.filter(g => g in statsFns).map(computeStats)\n      : Object.keys(statsFns).map(computeStats),\n  );\n\n  if (query.format === 'json') {\n    const indent = query.pretty !== undefined ? 2 : undefined;\n    await res\n      .header('Content-Type', 'application/json')\n      .send(BigIntJSON.stringify(Object.fromEntries(stats), null, indent));\n    return;\n  } else {\n    const body = stats\n      .map(([group, statsObject]) => printStats(group, statsObject))\n      .join('');\n    await res.header('Content-Type', 'text/plain; charset=utf-8').send(body);\n  }\n}\n\nfunction first(x: object[]): object {\n  return x[0];\n}\n\nfunction pick(x: object): unknown {\n  return Object.values(x)[0];\n}\n\nfunction header(name: string): string {\n  return `=== ${name} ===\\n`;\n}\n"],"mappings":";;;;;;;;;;;AAcA,eAAe,cAAc,IAAgB,QAAoB;CAC/D,MAAM,SAAS,eAAe,WAAW,MAAM,CAAC;CAChD,MAAM,MAAM,SAAS,IAAI,OAAO,SAAS,IAAI,gBAAgB;CAC7D,IAAI;EACF,OAAO,MAAM,WAAW;GACtB,CACE,eACA,GAAG,+BAA+B,IAAI,MAAM,EAAE,YAChD;GACA,CACE,2BACA,GAAG,+BAA+B,IAAI,MAAM,EAAE,WAChD;GACA,CACE,yBACA,GAAG,4CAA4C,IAAI,MAAM,EAAE,WAC7D;EACF,CAAC;CACH,UAAU;EACR,MAAM,IAAI,IAAI;CAChB;AACF;AAEA,eAAe,SAAS,IAAgB,QAAoB;CAC1D,MAAM,SAAS,eAAe,WAAW,MAAM,CAAC,IAAI;CACpD,MAAM,MAAM,SAAS,IAAI,OAAO,IAAI,IAAI,WAAW;CAEnD,SAAS,yBACP,QACyC;EACzC,MAAM,SAAS,SACX,GAAG,sDACH,GAAG;EACP,OAAO,GAAG;;;;;aAKD,IAAI,MAAM,EAAE;QACjB,OAAO;;;;;;;;aAQF,IAAI,MAAM,EAAE;QACjB,OAAO;;;;;;;;;;;CAWb;CAEA,IAAI;EACF,OAAO,MAAM,WAAW;GACtB,CACE,mBACA,GAAG,+BAA+B,IAAI,MAAM,EAAE,WAChD;GACA,CACE,wBACA,GAAG,kDAAkD,IACnD,MACF,EAAE,WACJ;GACA,CACE,oBACA,GAAG,+BAA+B,IAAI,MAAM,EAAE,+DAChD;GACA,CACE,sBACA,GAAG,+BAA+B,IAAI,MAAM,EAAE,mFAChD;GACA,CACE,qBACA,GAAG,+BAA+B,IAAI,MAAM,EAAE,kCAChD;GACA,CACE,2BACA,GAAG;;;;;eAKI,IAAI,MAAM,EAAE;;;;;;;;;;;;;;;;;gCAkBrB;GACA,CACE,iCACA,GAAG;;;;;eAKI,IAAI,MAAM,EAAE;;;;;;;;;;;;gCAarB;GACA,CAEE,YACA,GAAG;;;;;;;;;;;aAWE,IAAI,MAAM,EAAE,YACnB;GACA,CAEE,kBACA,GAAG;aACE,IAAI,MAAM,EAAE;;eAGnB;GACA,CACE,6CACA,yBAAyB,IAAI,CAC/B;GACA,CACE,+CACA,yBAAyB,KAAK,CAChC;GACA,CACE,2BACA,GAAG,gDAAgD,IACjD,MACF,EAAE,mDACJ;GACA,CACE,mBACA,GAAG;;;aAGE,IAAI,MAAM,EAAE;;;+BAInB;EACF,CAGG;CACL,UAAU;EACR,MAAM,IAAI,IAAI;CAChB;AACF;AAEA,eAAe,eAAe,IAAgB,QAAoB;CAChE,MAAM,SAAS,eAAe,WAAW,MAAM,CAAC,IAAI;CACpD,MAAM,MAAM,SAAS,IAAI,OAAO,OAAO,IAAI,cAAc;CAEzD,IAAI;EACF,OAAO,MAAM,WAAW,CACtB,CACE,iBACA,GAAG,6CAA6C,IAAI,MAAM,EAAE,aAC9D,CACF,CAAC;CACH,UAAU;EACR,MAAM,IAAI,IAAI;CAChB;AACF;AAKA,SAAS,aAAa,IAAgB,QAAoB;CACxD,MAAM,KAAK,IAAI,SAAS,IAAI,OAAO,QAAQ,MAAM,EAAC,UAAU,KAAI,CAAC;CACjE,IAAI;EACF,OAAO,OAAO,YAAY;GACxB,CAAC,kBAAkB,aAAa,OAAO,QAAQ,IAAI,CAAC;GACpD,CAAC,cAAc,KAAK,MAAM,GAAG,OAAO,YAAY,CAAC,CAAC,CAAC;GACnD,CAAC,aAAa,KAAK,MAAM,GAAG,OAAO,WAAW,CAAC,CAAC,CAAC;GACjD,CAAC,gBAAgB,KAAK,MAAM,GAAG,OAAO,cAAc,CAAC,CAAC,CAAC;GACvD,CAAC,eAAe,KAAK,MAAM,GAAG,OAAO,aAAa,CAAC,CAAC,CAAC;GACrD,CAAC,cAAc,KAAK,MAAM,GAAG,OAAO,YAAY,CAAC,CAAC,CAAC;GACnD,CAAC,eAAe,KAAK,MAAM,GAAG,OAAO,aAAa,CAAC,CAAC,CAAC;GACrD,CAAC,kBAAkB,KAAK,MAAM,GAAG,OAAO,gBAAgB,CAAC,CAAC,CAAC;GAC3D,CAAC,sBAAsB,KAAK,MAAM,GAAG,OAAO,oBAAoB,CAAC,CAAC,CAAC;GACnE,CAAC,iBAAiB,GAAG,SAAS,OAAO,QAAQ,IAAI,CAAC;EACpD,CAAU;CACZ,UAAU;EACR,GAAG,MAAM;CACX;AACF;AAEA,SAAS,iBAAiB,IAAgB,QAAoB;CAC5D,MAAM,KAAK,IAAI,SAAS,IAAI,OAAO,QAAQ,MAAM,EAAC,UAAU,KAAI,CAAC;CACjE,IAAI;EACF,OAAO,oBAAoB,EAAE;CAC/B,UAAU;EACR,GAAG,MAAM;CACX;AACF;;;;;;AAOA,SAAS,aAAa,MAAsC;CAC1D,MAAM,QAAgC,CAAC;CAEvC,KAAK,MAAM,UAAU,CAAC,QAAQ,OAAO,GAAG;EACtC,MAAM,QAAQ,GAAG,SAAS,OAAO,QAAQ,EAAC,gBAAgB,MAAK,CAAC;EAChE,IAAI,OACF,MAAM,UAAU,MAAM;CAE1B;CACA,OAAO;AACT;AAEA,SAAS,oBAAoB,IAAc;CACzC,MAAM,EAAC,iBAAgB,oBAAoB,IAAI,gBAAgB,EAAE,CAAC;CAElE,OAAO,EAAC,KADI,uBAAuB,YAC3B,EAAG;AACb;AAEA,SAAS,UAAU;CACjB,OAAO,OAAO,YAAY;EACxB,CAAC,YAAY,GAAG,QAAQ,CAAC;EACzB,CAAC,UAAU,GAAG,OAAO,CAAC;EACtB,CAAC,aAAa,GAAG,SAAS,CAAC;EAC3B,CAAC,YAAY,GAAG,QAAQ,CAAC;EACzB,CAAC,QAAQ,GAAG,KAAK,CAAC,CAAC,MAAM;EACzB,CAAC,yBAAyB,GAAG,qBAAqB,CAAC;EACnD,CAAC,YAAY,GAAG,SAAS,CAAC;EAC1B,CAAC,QAAQ,GAAG,KAAK,CAAC;EAClB,CAAC,WAAW,GAAG,QAAQ,CAAC;CAC1B,CAAU;AACZ;AAEA,eAAe,WACb,gBAIA;CACA,MAAM,UAAU,MAAM,QAAQ,IAC5B,eAAe,IAAI,OAAO,CAAC,MAAM,WAAW,CAAC,MAAM,MAAM,KAAK,CAAU,CAC1E;CACA,OAAO,OAAO,YAAY,OAAO;AACnC;AAIA,SAAS,WAAW,OAAe,aAAkC;CACnE,MAAM,QAAkB,CAAC,OAAO,OAAO,KAAK,CAAC;CAC7C,KAAK,MAAM,CAAC,MAAM,WAAW,OAAO,QAAQ,WAAW,GACrD,MAAM,KAAK,OAAO,OAAO,OAAO,WAAW,UAAU,QAAQ,MAAM,CAAC,CAAC;CAEvE,MAAM,KAAK,IAAI;CACf,OAAO,MAAM,KAAK,EAAE;AACtB;;;;;;;;;;;AAYA,eAAsB,mBACpB,IACA,QACA,KACA,KACA;CACA,IACE,sBACE,IACA,QACA,KACA,KACA,wBACA,UACF,MAAM,KAAA,GAEN;CAGF,MAAM,WAAqE;EACzE,gBAAgB,cAAc,IAAI,MAAM;EACxC,WAAW,SAAS,IAAI,MAAM;EAC9B,iBAAiB,eAAe,IAAI,MAAM;EAC1C,eAAe,aAAa,IAAI,MAAM;EACtC,mBAAmB,iBAAiB,IAAI,MAAM;EAC9C,UAAU,QAAQ;CACpB;CAEA,eAAe,aAAa,OAA+C;EACzE,IAAI;GACF,OAAO,CAAC,OAAO,MAAM,SAAS,MAAM,CAAC,CAAC;EACxC,SAAS,GAAG;GACV,GAAG,QAAQ,mBAAmB,MAAM,SAAS,CAAC;GAC9C,OAAO,CAAC,OAAO,EAAC,OAAO,OAAO,CAAC,EAAC,CAAC;EACnC;CACF;CAEA,MAAM,QAAQ,IAAI;CAClB,MAAM,SACJ,OAAO,MAAM,UAAU,WACnB,MAAM,MAAM,MAAM,GAAG,IACrB,MAAM,QAAQ,MAAM,KAAK,IACvB,MAAM,QACN,KAAA;CAER,MAAM,QAAQ,MAAM,QAAQ,IAC1B,SACI,OAAO,QAAO,MAAK,KAAK,QAAQ,CAAC,CAAC,IAAI,YAAY,IAClD,OAAO,KAAK,QAAQ,CAAC,CAAC,IAAI,YAAY,CAC5C;CAEA,IAAI,MAAM,WAAW,QAAQ;EAC3B,MAAM,SAAS,MAAM,WAAW,KAAA,IAAY,IAAI,KAAA;EAChD,MAAM,IACH,OAAO,gBAAgB,kBAAkB,CAAC,CAC1C,KAAK,WAAW,UAAU,OAAO,YAAY,KAAK,GAAG,MAAM,MAAM,CAAC;EACrE;CACF,OAAO;EACL,MAAM,OAAO,MACV,KAAK,CAAC,OAAO,iBAAiB,WAAW,OAAO,WAAW,CAAC,CAAC,CAC7D,KAAK,EAAE;EACV,MAAM,IAAI,OAAO,gBAAgB,2BAA2B,CAAC,CAAC,KAAK,IAAI;CACzE;AACF;AAEA,SAAS,MAAM,GAAqB;CAClC,OAAO,EAAE;AACX;AAEA,SAAS,KAAK,GAAoB;CAChC,OAAO,OAAO,OAAO,CAAC,CAAC,CAAC;AAC1B;AAEA,SAAS,OAAO,MAAsB;CACpC,OAAO,OAAO,KAAK;AACrB"}

package/out/zero-cache/src/services/view-syncer/client-handler.d.ts

@@ -18,6 +18,13 @@ export type DeleteRowPatch = {
     type: 'row';
     op: 'del';
     id: RowID;
+    /**
+     * Set when the deletion replays a row the CVR already records as deleted,
+     * to catch up a client that fell behind. Unlike a deletion computed while
+     * advancing, it can only undo a put the client received in an earlier poke,
+     * so a client with no base version has nothing it can apply to.
+     */
+    catchup?: boolean | undefined;
 };
 export type RowPatch = PutRowPatch | DeleteRowPatch;
 export type ConfigPatch = DelQueryPatch | PutQueryPatch;
@@ -60,6 +67,15 @@ export declare class ClientHandler {
     readonly wsID: string;
     constructor(lc: LogContext, clientGroupID: string, clientID: string, wsID: string, shard: ShardID, baseCookie: string | null, downstream: Subscription<ViewSyncerDownstream>);
     version(): NullableCVRVersion;
+    validated(): boolean;
+    markValidated(): void;
+    /**
+     * Whether this connection has presented the admin password to the
+     * inspector. Authentication is per connection: it is not shared with other
+     * connections of the client group and does not survive a reconnect.
+     */
+    isInspectorAuthenticated(): boolean;
+    setInspectorAuthenticated(authenticated: boolean): void;
     fail(e: unknown): void;
     close(reason: string): void;
     startPoke(tentativeVersion: CVRVersion): ClientPokeHandler;

package/out/zero-cache/src/services/view-syncer/client-handler.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"client-handler.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/view-syncer/client-handler.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,uCAAuC,CAAC;AACtE,OAAO,EAEL,KAAK,UAAU,IAAI,cAAc,EAClC,MAAM,gCAAgC,CAAC;AAIxC,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,iDAAiD,CAAC;AAIlF,OAAO,EAEL,KAAK,mBAAmB,EACzB,MAAM,wCAAwC,CAAC;AAChD,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,+CAA+C,CAAC;AAanF,OAAO,KAAK,EAAC,oBAAoB,EAAC,MAAM,2BAA2B,CAAC;AAKpE,OAAO,EAAiB,KAAK,OAAO,EAAC,MAAM,uBAAuB,CAAC;AACnE,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,6BAA6B,CAAC;AAC9D,OAAO,EAKL,KAAK,UAAU,EACf,KAAK,aAAa,EAClB,KAAK,kBAAkB,EACvB,KAAK,aAAa,EAClB,KAAK,KAAK,EACX,MAAM,mBAAmB,CAAC;AAE3B,MAAM,MAAM,WAAW,GAAG;IACxB,IAAI,EAAE,KAAK,CAAC;IACZ,EAAE,EAAE,KAAK,CAAC;IACV,EAAE,EAAE,KAAK,CAAC;IACV,QAAQ,EAAE,UAAU,CAAC;CACtB,CAAC;AAEF,MAAM,MAAM,cAAc,GAAG;IAC3B,IAAI,EAAE,KAAK,CAAC;IACZ,EAAE,EAAE,KAAK,CAAC;IACV,EAAE,EAAE,KAAK,CAAC;CACX,CAAC;AAEF,MAAM,MAAM,QAAQ,GAAG,WAAW,GAAG,cAAc,CAAC;AACpD,MAAM,MAAM,WAAW,GAAG,aAAa,GAAG,aAAa,CAAC;AAExD,MAAM,MAAM,KAAK,GAAG,WAAW,GAAG,QAAQ,CAAC;AAE3C,MAAM,MAAM,cAAc,GAAG;IAC3B,KAAK,EAAE,KAAK,CAAC;IACb,SAAS,EAAE,UAAU,CAAC;CACvB,CAAC;AAEF,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,KAAK,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC/C,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACxB,GAAG,CAAC,YAAY,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAC9C;AAED,iDAAiD;AACjD,MAAM,WAAW,iBAAkB,SAAQ,WAAW;IACpD;;;;OAIG;IACH,QAAQ,CAAC,WAAW,EAAE,OAAO,CAAC;CAC/B;AAED;;;GAGG;AACH,MAAM,WAAW,gBAAiB,SAAQ,WAAW;IACnD,2DAA2D;IAC3D,QAAQ,CAAC,WAAW,EAAE,OAAO,CAAC;CAC/B;AASD,uEAAuE;AACvE,wBAAgB,SAAS,CACvB,EAAE,EAAE,UAAU,EACd,OAAO,EAAE,aAAa,EAAE,EACxB,gBAAgB,EAAE,UAAU,GAC3B,gBAAgB,CAsClB;AAKD,eAAO,MAAM,+BAA+B,QAAc,CAAC;AAG3D;;GAEG;AACH,qBAAa,aAAa;;IAExB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IA6BtB,YACE,EAAE,EAAE,UAAU,EACd,aAAa,EAAE,MAAM,EACrB,QAAQ,EAAE,MAAM,EAChB,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,OAAO,EACd,UAAU,EAAE,MAAM,GAAG,IAAI,EACzB,UAAU,EAAE,YAAY,CAAC,oBAAoB,CAAC,EAW/C;IAED,OAAO,IAAI,kBAAkB,CAE5B;IAOD,IAAI,CAAC,CAAC,EAAE,OAAO,QAMd;IAED,KAAK,CAAC,MAAM,EAAE,MAAM,QAGnB;IAED,SAAS,CAAC,gBAAgB,EAAE,UAAU,GAAG,iBAAiB,CAqLzD;IAEK,iBAAiB,CACrB,EAAE,EAAE,UAAU,EACd,gBAAgB,EAAE,MAAM,EAAE,EAC1B,qBAAqB,EAAE,MAAM,EAAE,iBAWhC;IAED,mCAAmC,CAAC,MAAM,EAAE,YAAY,EAAE,QAEzD;IAED,6BAA6B,CAAC,KAAK,EAAE,mBAAmB,QAEvD;IAED,mBAAmB,CAAC,EAAE,EAAE,UAAU,EAAE,QAAQ,EAAE,eAAe,GAAG,IAAI,CAMnE;CAuBF;AA0CD;;;;;;GAMG;AACH,wBAAgB,cAAc,CAAC,GAAG,EAAE,UAAU,GAAG,cAAc,CAkB9D"}
+{"version":3,"file":"client-handler.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/view-syncer/client-handler.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,uCAAuC,CAAC;AACtE,OAAO,EAEL,KAAK,UAAU,IAAI,cAAc,EAClC,MAAM,gCAAgC,CAAC;AAIxC,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,iDAAiD,CAAC;AAIlF,OAAO,EAEL,KAAK,mBAAmB,EACzB,MAAM,wCAAwC,CAAC;AAChD,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,+CAA+C,CAAC;AAcnF,OAAO,KAAK,EAAC,oBAAoB,EAAC,MAAM,2BAA2B,CAAC;AAKpE,OAAO,EAAiB,KAAK,OAAO,EAAC,MAAM,uBAAuB,CAAC;AACnE,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,6BAA6B,CAAC;AAC9D,OAAO,EAKL,KAAK,UAAU,EACf,KAAK,aAAa,EAClB,KAAK,kBAAkB,EACvB,KAAK,aAAa,EAClB,KAAK,KAAK,EACX,MAAM,mBAAmB,CAAC;AAE3B,MAAM,MAAM,WAAW,GAAG;IACxB,IAAI,EAAE,KAAK,CAAC;IACZ,EAAE,EAAE,KAAK,CAAC;IACV,EAAE,EAAE,KAAK,CAAC;IACV,QAAQ,EAAE,UAAU,CAAC;CACtB,CAAC;AAEF,MAAM,MAAM,cAAc,GAAG;IAC3B,IAAI,EAAE,KAAK,CAAC;IACZ,EAAE,EAAE,KAAK,CAAC;IACV,EAAE,EAAE,KAAK,CAAC;IACV;;;;;OAKG;IACH,OAAO,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;CAC/B,CAAC;AAEF,MAAM,MAAM,QAAQ,GAAG,WAAW,GAAG,cAAc,CAAC;AACpD,MAAM,MAAM,WAAW,GAAG,aAAa,GAAG,aAAa,CAAC;AAExD,MAAM,MAAM,KAAK,GAAG,WAAW,GAAG,QAAQ,CAAC;AAE3C,MAAM,MAAM,cAAc,GAAG;IAC3B,KAAK,EAAE,KAAK,CAAC;IACb,SAAS,EAAE,UAAU,CAAC;CACvB,CAAC;AAEF,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,KAAK,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC/C,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACxB,GAAG,CAAC,YAAY,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAC9C;AAED,iDAAiD;AACjD,MAAM,WAAW,iBAAkB,SAAQ,WAAW;IACpD;;;;OAIG;IACH,QAAQ,CAAC,WAAW,EAAE,OAAO,CAAC;CAC/B;AAED;;;GAGG;AACH,MAAM,WAAW,gBAAiB,SAAQ,WAAW;IACnD,2DAA2D;IAC3D,QAAQ,CAAC,WAAW,EAAE,OAAO,CAAC;CAC/B;AASD,uEAAuE;AACvE,wBAAgB,SAAS,CACvB,EAAE,EAAE,UAAU,EACd,OAAO,EAAE,aAAa,EAAE,EACxB,gBAAgB,EAAE,UAAU,GAC3B,gBAAgB,CAsClB;AAKD,eAAO,MAAM,+BAA+B,QAAc,CAAC;AAG3D;;GAEG;AACH,qBAAa,aAAa;;IAExB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IAiCtB,YACE,EAAE,EAAE,UAAU,EACd,aAAa,EAAE,MAAM,EACrB,QAAQ,EAAE,MAAM,EAChB,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,OAAO,EACd,UAAU,EAAE,MAAM,GAAG,IAAI,EACzB,UAAU,EAAE,YAAY,CAAC,oBAAoB,CAAC,EAW/C;IAED,OAAO,IAAI,kBAAkB,CAE5B;IAED,SAAS,IAAI,OAAO,CAEnB;IAED,aAAa,IAAI,IAAI,CAEpB;IAED;;;;OAIG;IACH,wBAAwB,IAAI,OAAO,CAElC;IAED,yBAAyB,CAAC,aAAa,EAAE,OAAO,GAAG,IAAI,CAEtD;IAOD,IAAI,CAAC,CAAC,EAAE,OAAO,QAMd;IAED,KAAK,CAAC,MAAM,EAAE,MAAM,QAGnB;IAED,SAAS,CAAC,gBAAgB,EAAE,UAAU,GAAG,iBAAiB,CA+LzD;IAEK,iBAAiB,CACrB,EAAE,EAAE,UAAU,EACd,gBAAgB,EAAE,MAAM,EAAE,EAC1B,qBAAqB,EAAE,MAAM,EAAE,iBAWhC;IAED,mCAAmC,CAAC,MAAM,EAAE,YAAY,EAAE,QAEzD;IAED,6BAA6B,CAAC,KAAK,EAAE,mBAAmB,QAEvD;IAED,mBAAmB,CAAC,EAAE,EAAE,UAAU,EAAE,QAAQ,EAAE,eAAe,GAAG,IAAI,CAMnE;CAuBF;AA0CD;;;;;;GAMG;AACH,wBAAgB,cAAc,CAAC,GAAG,EAAE,UAAU,GAAG,cAAc,CAkB9D"}

package/out/zero-cache/src/services/view-syncer/client-handler.js

@@ -8,6 +8,7 @@ import { primaryKeyValueRecordSchema } from "../../../../zero-protocol/src/prima
 import { mutationResultSchema } from "../../../../zero-protocol/src/mutation.js";
 import { upstreamSchema } from "../../types/shards.js";
 import { getOrCreateCounter, getOrCreateLatencyHistogram } from "../../observability/metrics.js";
+import { clientGroupLogID } from "../../types/client-group-log-id.js";
 import { cmpVersions, cookieToVersion, versionToCookie, versionToNullableCookie } from "./schema/types.js";
 import { getLogLevel, wrapWithProtocolError } from "../../types/error-with-level.js";
 //#region ../zero-cache/src/services/view-syncer/client-handler.ts
@@ -46,7 +47,9 @@ var ClientHandler = class {
 	#lc;
 	#downstream;
 	#baseVersion;
+	#validated = false;
 	#everPoked = false;
+	#inspectorAuthenticated = false;
 	#pokeTime = getOrCreateLatencyHistogram("sync", "poke.time", "Time elapsed for each poke transaction. Canceled / noop pokes are excluded.");
 	#pokeTransactions = getOrCreateCounter("sync", "poke.transactions", "Count of poke transactions.");
 	#pokedRows = getOrCreateCounter("sync", "poke.rows", "Count of poked rows.");
@@ -64,6 +67,23 @@ var ClientHandler = class {
 	version() {
 		return this.#baseVersion;
 	}
+	validated() {
+		return this.#validated;
+	}
+	markValidated() {
+		this.#validated = true;
+	}
+	/**
+	* Whether this connection has presented the admin password to the
+	* inspector. Authentication is per connection: it is not shared with other
+	* connections of the client group and does not survive a reconnect.
+	*/
+	isInspectorAuthenticated() {
+		return this.#inspectorAuthenticated;
+	}
+	setInspectorAuthenticated(authenticated) {
+		this.#inspectorAuthenticated = authenticated;
+	}
 	async #push(msg, serialized) {
 		const { result } = this.#downstream.push({
 			message: msg,
@@ -119,6 +139,7 @@ var ClientHandler = class {
 		const addPatch = async (patchToVersion) => {
 			const { patch, toVersion } = patchToVersion;
 			if (cmpVersions(toVersion, this.#baseVersion) <= 0) return;
+			if (patch.type === "row" && patch.op === "del" && patch.catchup && this.#baseVersion === null) return;
 			const target = await ensureBody();
 			const { type, op } = patch;
 			switch (type) {
@@ -238,7 +259,7 @@ var ClientHandler = class {
 		if (patch.op === "put") {
 			const row = ensureSafeJSON(patch.contents);
 			const { clientGroupID, clientID, lastMutationID } = parse(row, lmidRowSchema, "passthrough");
-			if (clientGroupID !== this.#clientGroupID) this.#lc.error?.(`Received clients row for wrong clientGroupID. Ignoring.`, clientGroupID);
+			if (clientGroupID !== this.#clientGroupID) this.#lc.error?.(`Received clients row for wrong clientGroupID. Ignoring.`, clientGroupLogID(clientGroupID));
 			else lmids[clientID] = lastMutationID;
 		} else patch.op;
 	}

package/out/zero-cache/src/services/view-syncer/client-handler.js.map

@@ -1 +1 @@
-{"version":3,"file":"client-handler.js","names":["v.parse","v"],"sources":["../../../../../../zero-cache/src/services/view-syncer/client-handler.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {assert, unreachable} from '../../../../shared/src/asserts.ts';\nimport type {JSONObject} from '../../../../shared/src/bigint-json.ts';\nimport {\n  assertJSONValue,\n  type JSONObject as SafeJSONObject,\n} from '../../../../shared/src/json.ts';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport type {Writable} from '../../../../shared/src/writable.ts';\nimport type {ErroredQuery} from '../../../../zero-protocol/src/custom-queries.ts';\nimport {rowSchema} from '../../../../zero-protocol/src/data.ts';\nimport type {DeleteClientsBody} from '../../../../zero-protocol/src/delete-clients.ts';\nimport type {Downstream} from '../../../../zero-protocol/src/down.ts';\nimport {\n  ProtocolError,\n  type TransformFailedBody,\n} from '../../../../zero-protocol/src/error.ts';\nimport type {InspectDownBody} from '../../../../zero-protocol/src/inspect-down.ts';\nimport {mutationResultSchema} from '../../../../zero-protocol/src/mutation.ts';\nimport type {\n  PokePartBody,\n  PokePartMessage,\n  PokeStartBody,\n} from '../../../../zero-protocol/src/poke.ts';\nimport {primaryKeyValueRecordSchema} from '../../../../zero-protocol/src/primary-key.ts';\nimport type {RowPatchOp} from '../../../../zero-protocol/src/row-patch.ts';\nimport {\n  getOrCreateCounter,\n  getOrCreateLatencyHistogram,\n} from '../../observability/metrics.ts';\nimport type {ViewSyncerDownstream} from '../../types/downstream.ts';\nimport {\n  getLogLevel,\n  wrapWithProtocolError,\n} from '../../types/error-with-level.ts';\nimport {upstreamSchema, type ShardID} from '../../types/shards.ts';\nimport type {Subscription} from '../../types/subscription.ts';\nimport {\n  cmpVersions,\n  cookieToVersion,\n  versionToCookie,\n  versionToNullableCookie,\n  type CVRVersion,\n  type DelQueryPatch,\n  type NullableCVRVersion,\n  type PutQueryPatch,\n  type RowID,\n} from './schema/types.ts';\n\nexport type PutRowPatch = {\n  type: 'row';\n  op: 'put';\n  id: RowID;\n  contents: JSONObject;\n};\n\nexport type DeleteRowPatch = {\n  type: 'row';\n  op: 'del';\n  id: RowID;\n};\n\nexport type RowPatch = PutRowPatch | DeleteRowPatch;\nexport type ConfigPatch = DelQueryPatch | PutQueryPatch;\n\nexport type Patch = ConfigPatch | RowPatch;\n\nexport type PatchToVersion = {\n  patch: Patch;\n  toVersion: CVRVersion;\n};\n\nexport interface PokeHandler {\n  addPatch(patch: PatchToVersion): Promise<void>;\n  cancel(): Promise<void>;\n  end(finalVersion: CVRVersion): Promise<void>;\n}\n\n/** A {@link PokeHandler} for a single client. */\nexport interface ClientPokeHandler extends PokeHandler {\n  /**\n   * Whether a patch has been sent to the client. Once one has, the poke can\n   * only be ended with a `finalVersion` that is ahead of the client's base\n   * version.\n   */\n  readonly patchesSent: boolean;\n}\n\n/**\n * A {@link PokeHandler} for multiple clients, as returned by\n * {@link startPoke}.\n */\nexport interface MultiPokeHandler extends PokeHandler {\n  /** Whether a patch has been sent to any of the clients. */\n  readonly patchesSent: boolean;\n}\n\nconst NOOP: ClientPokeHandler = {\n  patchesSent: false,\n  addPatch: () => promiseVoid,\n  cancel: () => promiseVoid,\n  end: () => promiseVoid,\n};\n\n/** Wraps PokeHandlers for multiple clients in a single PokeHandler. */\nexport function startPoke(\n  lc: LogContext,\n  clients: ClientHandler[],\n  tentativeVersion: CVRVersion,\n): MultiPokeHandler {\n  const pokers = clients.map(c => c.startPoke(tentativeVersion));\n\n  // Promise.allSettled() ensures that a failed (e.g. disconnected) client\n  // does not prevent other clients from receiving the pokes. However, the\n  // rate (per client group) will be limited by the slowest connection.\n  //\n  // The individual pokers report their own failures (by failing the\n  // connection), so a rejection here is not expected. Log it rather than\n  // dropping it on the floor.\n  const settle = async (op: string, results: Promise<void>[]) => {\n    for (const result of await Promise.allSettled(results)) {\n      if (result.status === 'rejected') {\n        lc.error?.(`unhandled error poking client (${op})`, result.reason);\n      }\n    }\n  };\n\n  return {\n    get patchesSent() {\n      return pokers.some(poker => poker.patchesSent);\n    },\n    addPatch: patch =>\n      settle(\n        'addPatch',\n        pokers.map(poker => poker.addPatch(patch)),\n      ),\n    cancel: () =>\n      settle(\n        'cancel',\n        pokers.map(poker => poker.cancel()),\n      ),\n    end: finalVersion =>\n      settle(\n        'end',\n        pokers.map(poker => poker.end(finalVersion)),\n      ),\n  };\n}\n\n// Soft limit on the serialized row characters accumulated in a poke part.\n// This bounds view-syncer memory and keeps legacy JSON messages reasonably\n// sized. PokeChunkEncoder separately enforces the binary frame limit in bytes.\nexport const POKE_PART_FLUSH_THRESHOLD_CHARS = 1024 * 1024;\nconst PART_COUNT_FLUSH_THRESHOLD = 100;\n\n/**\n * Handles a single `ViewSyncer` connection.\n */\nexport class ClientHandler {\n  readonly #clientGroupID: string;\n  readonly clientID: string;\n  readonly wsID: string;\n  readonly #zeroClientsTable: string;\n  readonly #zeroMutationsTable: string;\n  readonly #lc: LogContext;\n  readonly #downstream: Subscription<ViewSyncerDownstream>;\n  #baseVersion: NullableCVRVersion;\n  // We will send a poke on connect even if the client is already caught up, so that it can learn its\n  // got-queries state has been reconciled with the server. After that, we will only send a poke if\n  // the client is behind.\n  #everPoked = false;\n\n  readonly #pokeTime = getOrCreateLatencyHistogram(\n    'sync',\n    'poke.time',\n    'Time elapsed for each poke transaction. Canceled / noop pokes are excluded.',\n  );\n\n  readonly #pokeTransactions = getOrCreateCounter(\n    'sync',\n    'poke.transactions',\n    'Count of poke transactions.',\n  );\n\n  readonly #pokedRows = getOrCreateCounter(\n    'sync',\n    'poke.rows',\n    'Count of poked rows.',\n  );\n\n  constructor(\n    lc: LogContext,\n    clientGroupID: string,\n    clientID: string,\n    wsID: string,\n    shard: ShardID,\n    baseCookie: string | null,\n    downstream: Subscription<ViewSyncerDownstream>,\n  ) {\n    lc.debug?.('new client handler');\n    this.#clientGroupID = clientGroupID;\n    this.clientID = clientID;\n    this.wsID = wsID;\n    this.#zeroClientsTable = `${upstreamSchema(shard)}.clients`;\n    this.#zeroMutationsTable = `${upstreamSchema(shard)}.mutations`;\n    this.#lc = lc;\n    this.#downstream = downstream;\n    this.#baseVersion = cookieToVersion(baseCookie);\n  }\n\n  version(): NullableCVRVersion {\n    return this.#baseVersion;\n  }\n\n  async #push(msg: Downstream, serialized?: string | undefined): Promise<void> {\n    const {result} = this.#downstream.push({message: msg, serialized});\n    await result;\n  }\n\n  fail(e: unknown) {\n    this.#lc[getLogLevel(e)]?.(\n      `view-syncer closing connection with error: ${String(e)}`,\n      e,\n    );\n    this.#downstream.fail(wrapWithProtocolError(e));\n  }\n\n  close(reason: string) {\n    this.#lc.debug?.(`view-syncer closing connection: ${reason}`);\n    this.#downstream.cancel();\n  }\n\n  startPoke(tentativeVersion: CVRVersion): ClientPokeHandler {\n    const pokeID = versionToCookie(tentativeVersion);\n    const lc = this.#lc.withContext('pokeID', pokeID);\n\n    // Force an (empty) initial poke even when already caught up; see #everPoked.\n    const forceInitialPoke = !this.#everPoked;\n    const cmp = cmpVersions(this.#baseVersion, tentativeVersion);\n    if (cmp > 0 || (cmp === 0 && !forceInitialPoke)) {\n      lc.info?.(`already caught up, not sending poke.`);\n      return NOOP;\n    }\n\n    const baseCookie = versionToNullableCookie(this.#baseVersion);\n    const cookie = versionToCookie(tentativeVersion);\n    lc.debug?.(`starting poke from ${baseCookie} to ${cookie}`);\n\n    const start = performance.now();\n\n    const pokeStart: PokeStartBody = {pokeID, baseCookie};\n\n    let pokeStarted = false;\n    let body: PokePartBody | undefined;\n    let partCount = 0;\n    let serializedRowsLength = 0;\n\n    const ensureBody = async () => {\n      if (!pokeStarted) {\n        await this.#push(['pokeStart', pokeStart]);\n        pokeStarted = true;\n      }\n      return (body ??= {pokeID});\n    };\n\n    const flushBody = async () => {\n      if (body) {\n        const message = ['pokePart', body] satisfies PokePartMessage;\n        await this.#push(message, JSON.stringify(message));\n        body = undefined;\n        partCount = 0;\n        serializedRowsLength = 0;\n      }\n    };\n\n    const addPatch = async (patchToVersion: PatchToVersion) => {\n      const {patch, toVersion} = patchToVersion;\n      if (cmpVersions(toVersion, this.#baseVersion) <= 0) {\n        return;\n      }\n      const target = await ensureBody();\n\n      const {type, op} = patch;\n      switch (type) {\n        case 'query': {\n          const patches = patch.clientID\n            ? ((target.desiredQueriesPatches ??= {})[patch.clientID] ??= [])\n            : (target.gotQueriesPatch ??= []);\n          patches.push({op, hash: patch.id});\n          break;\n        }\n        case 'row':\n          if (patch.id.table === this.#zeroClientsTable) {\n            this.#updateLMIDs((target.lastMutationIDChanges ??= {}), patch);\n          } else if (patch.id.table === this.#zeroMutationsTable) {\n            const patches = (target.mutationsPatch ??= []);\n            if (op === 'put') {\n              const row = v.parse(\n                ensureSafeJSON(patch.contents),\n                mutationRowSchema,\n                'passthrough',\n              );\n              patches.push({\n                op: 'put',\n                mutation: {\n                  id: {\n                    clientID: row.clientID,\n                    id: row.mutationID,\n                  },\n                  result: row.result,\n                },\n              });\n            } else {\n              const {clientID, mutationID} = patch.id.rowKey;\n              assert(\n                typeof clientID === 'string',\n                'client id must be a string',\n              );\n              const id = Number(mutationID);\n              assert(\n                !Number.isNaN(id) && Number.isFinite(id) && id >= 0,\n                'mutation id must be a finite number',\n              );\n              patches.push({\n                op: 'del',\n                id: {\n                  clientID,\n                  id,\n                },\n              });\n            }\n          } else {\n            const rowPatch = makeRowPatch(patch);\n            (target.rowsPatch ??= []).push(rowPatch);\n            serializedRowsLength += JSON.stringify(rowPatch).length;\n          }\n          break;\n        default:\n          unreachable(patch);\n      }\n\n      partCount++;\n      if (\n        serializedRowsLength >= POKE_PART_FLUSH_THRESHOLD_CHARS ||\n        partCount >= PART_COUNT_FLUSH_THRESHOLD\n      ) {\n        await flushBody();\n      }\n    };\n\n    // A poke that was started must always be ended. If it isn't, the\n    // connection is left mid-poke and the *next* pokeStart fails the\n    // DownstreamSender's in-progress check, surfacing the error far from\n    // its cause. Fail the connection instead; the client reconnects and\n    // resyncs from its baseVersion.\n    return {\n      get patchesSent() {\n        return pokeStarted;\n      },\n      addPatch: async (patchToVersion: PatchToVersion) => {\n        try {\n          await addPatch(patchToVersion);\n          if (patchToVersion.patch.type === 'row') {\n            this.#pokedRows.add(1);\n          }\n        } catch (e) {\n          this.fail(e);\n        }\n      },\n\n      cancel: async () => {\n        try {\n          if (pokeStarted) {\n            await this.#push(['pokeEnd', {pokeID, cookie: '', cancel: true}]);\n          }\n        } catch (e) {\n          this.fail(e);\n        }\n      },\n\n      end: async (finalVersion: CVRVersion) => {\n        try {\n          const cookie = versionToCookie(finalVersion);\n          if (!pokeStarted) {\n            if (\n              cmpVersions(this.#baseVersion, finalVersion) === 0 &&\n              !forceInitialPoke\n            ) {\n              return; // Nothing changed and nothing was sent.\n            }\n            await this.#push(['pokeStart', pokeStart]);\n          } else if (cmpVersions(this.#baseVersion, finalVersion) >= 0) {\n            // Sanity check: If the poke was started, the finalVersion\n            // must be > #baseVersion.\n            throw new Error(\n              `Patches were sent but finalVersion ${cookie} is not ` +\n                `greater than baseVersion ` +\n                `${versionToNullableCookie(this.#baseVersion)}`,\n            );\n          }\n          await flushBody();\n          await this.#push(['pokeEnd', {pokeID, cookie}]);\n          this.#baseVersion = finalVersion;\n          this.#everPoked = true;\n\n          const elapsed = performance.now() - start;\n          this.#pokeTransactions.add(1);\n          this.#pokeTime.recordMs(elapsed);\n        } catch (e) {\n          this.fail(e);\n        }\n      },\n    };\n  }\n\n  async sendDeleteClients(\n    lc: LogContext,\n    deletedClientIDs: string[],\n    deletedClientGroupIDs: string[],\n  ) {\n    const deleteClientsBody: Writable<DeleteClientsBody> = {};\n    if (deletedClientIDs.length > 0) {\n      deleteClientsBody.clientIDs = deletedClientIDs;\n    }\n    if (deletedClientGroupIDs.length > 0) {\n      deleteClientsBody.clientGroupIDs = deletedClientGroupIDs;\n    }\n    lc.debug?.('sending deleteClients', deleteClientsBody);\n    await this.#push(['deleteClients', deleteClientsBody]);\n  }\n\n  sendQueryTransformApplicationErrors(errors: ErroredQuery[]) {\n    void this.#push(['transformError', errors]);\n  }\n\n  sendQueryTransformFailedError(error: TransformFailedBody) {\n    this.fail(new ProtocolError(error));\n  }\n\n  sendInspectResponse(lc: LogContext, response: InspectDownBody): void {\n    lc.debug?.('sending inspect response', response);\n    this.#downstream.push({\n      message: ['inspect', response],\n      serialized: undefined,\n    });\n  }\n\n  #updateLMIDs(lmids: Record<string, number>, patch: RowPatch) {\n    if (patch.op === 'put') {\n      const row = ensureSafeJSON(patch.contents);\n      const {clientGroupID, clientID, lastMutationID} = v.parse(\n        row,\n        lmidRowSchema,\n        'passthrough',\n      );\n      if (clientGroupID !== this.#clientGroupID) {\n        this.#lc.error?.(\n          `Received clients row for wrong clientGroupID. Ignoring.`,\n          clientGroupID,\n        );\n      } else {\n        lmids[clientID] = lastMutationID;\n      }\n    } else {\n      // The 'constrain' and 'del' ops for clients can be ignored.\n      patch.op satisfies 'constrain' | 'del';\n    }\n  }\n}\n\n// Note: The {APP_ID}_{SHARD_ID}.clients table is set up in replicator/initial-sync.ts.\nconst lmidRowSchema = v.object({\n  clientGroupID: v.string(),\n  clientID: v.string(),\n  lastMutationID: v.number(), // Actually returned as a bigint, but converted by ensureSafeJSON().\n});\n\nconst mutationRowSchema = v.object({\n  clientGroupID: v.string(),\n  clientID: v.string(),\n  mutationID: v.number(),\n  result: mutationResultSchema,\n});\n\nfunction makeRowPatch(patch: RowPatch): RowPatchOp {\n  const {\n    op,\n    id: {table: tableName, rowKey: id},\n  } = patch;\n\n  switch (op) {\n    case 'put':\n      return {\n        op: 'put',\n        tableName,\n        value: v.parse(ensureSafeJSON(patch.contents), rowSchema),\n      };\n\n    case 'del':\n      return {\n        op,\n        tableName,\n        id: v.parse(id, primaryKeyValueRecordSchema),\n      };\n\n    default:\n      unreachable(op);\n  }\n}\n\n/**\n * Column values of type INT8 are returned as the `bigint` from the\n * Postgres library. These are converted to `number` if they are within\n * the safe Number range, allowing the protocol to support numbers larger\n * than 32-bits. Values outside of the safe number range (e.g. > 2^53) will\n * result in an Error.\n */\nexport function ensureSafeJSON(row: JSONObject): SafeJSONObject {\n  const modified = Object.entries(row)\n    .filter(([k, v]) => {\n      if (typeof v === 'bigint') {\n        if (v >= Number.MIN_SAFE_INTEGER && v <= Number.MAX_SAFE_INTEGER) {\n          return true; // send this entry onto the next map() step.\n        }\n        throw new Error(`Value of \"${k}\" exceeds safe Number range (${v})`);\n      } else if (typeof v === 'object') {\n        assertJSONValue(v);\n      }\n      return false;\n    })\n    .map(([k, v]) => [k, Number(v)]);\n\n  return modified.length\n    ? {...row, ...Object.fromEntries(modified)}\n    : (row as SafeJSONObject);\n}\n"],"mappings":";;;;;;;;;;;;;AAkGA,IAAM,OAA0B;CAC9B,aAAa;CACb,gBAAgB;CAChB,cAAc;CACd,WAAW;AACb;;AAGA,SAAgB,UACd,IACA,SACA,kBACkB;CAClB,MAAM,SAAS,QAAQ,KAAI,MAAK,EAAE,UAAU,gBAAgB,CAAC;CAS7D,MAAM,SAAS,OAAO,IAAY,YAA6B;EAC7D,KAAK,MAAM,UAAU,MAAM,QAAQ,WAAW,OAAO,GACnD,IAAI,OAAO,WAAW,YACpB,GAAG,QAAQ,kCAAkC,GAAG,IAAI,OAAO,MAAM;CAGvE;CAEA,OAAO;EACL,IAAI,cAAc;GAChB,OAAO,OAAO,MAAK,UAAS,MAAM,WAAW;EAC/C;EACA,WAAU,UACR,OACE,YACA,OAAO,KAAI,UAAS,MAAM,SAAS,KAAK,CAAC,CAC3C;EACF,cACE,OACE,UACA,OAAO,KAAI,UAAS,MAAM,OAAO,CAAC,CACpC;EACF,MAAK,iBACH,OACE,OACA,OAAO,KAAI,UAAS,MAAM,IAAI,YAAY,CAAC,CAC7C;CACJ;AACF;AAKA,IAAa,kCAAkC;AAC/C,IAAM,6BAA6B;;;;AAKnC,IAAa,gBAAb,MAA2B;CACzB;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAIA,aAAa;CAEb,YAAqB,4BACnB,QACA,aACA,6EACF;CAEA,oBAA6B,mBAC3B,QACA,qBACA,6BACF;CAEA,aAAsB,mBACpB,QACA,aACA,sBACF;CAEA,YACE,IACA,eACA,UACA,MACA,OACA,YACA,YACA;EACA,GAAG,QAAQ,oBAAoB;EAC/B,KAAK,iBAAiB;EACtB,KAAK,WAAW;EAChB,KAAK,OAAO;EACZ,KAAK,oBAAoB,GAAG,eAAe,KAAK,EAAE;EAClD,KAAK,sBAAsB,GAAG,eAAe,KAAK,EAAE;EACpD,KAAK,MAAM;EACX,KAAK,cAAc;EACnB,KAAK,eAAe,gBAAgB,UAAU;CAChD;CAEA,UAA8B;EAC5B,OAAO,KAAK;CACd;CAEA,MAAM,MAAM,KAAiB,YAAgD;EAC3E,MAAM,EAAC,WAAU,KAAK,YAAY,KAAK;GAAC,SAAS;GAAK;EAAU,CAAC;EACjE,MAAM;CACR;CAEA,KAAK,GAAY;EACf,KAAK,IAAI,YAAY,CAAC,EAAE,GACtB,8CAA8C,OAAO,CAAC,KACtD,CACF;EACA,KAAK,YAAY,KAAK,sBAAsB,CAAC,CAAC;CAChD;CAEA,MAAM,QAAgB;EACpB,KAAK,IAAI,QAAQ,mCAAmC,QAAQ;EAC5D,KAAK,YAAY,OAAO;CAC1B;CAEA,UAAU,kBAAiD;EACzD,MAAM,SAAS,gBAAgB,gBAAgB;EAC/C,MAAM,KAAK,KAAK,IAAI,YAAY,UAAU,MAAM;EAGhD,MAAM,mBAAmB,CAAC,KAAK;EAC/B,MAAM,MAAM,YAAY,KAAK,cAAc,gBAAgB;EAC3D,IAAI,MAAM,KAAM,QAAQ,KAAK,CAAC,kBAAmB;GAC/C,GAAG,OAAO,sCAAsC;GAChD,OAAO;EACT;EAEA,MAAM,aAAa,wBAAwB,KAAK,YAAY;EAC5D,MAAM,SAAS,gBAAgB,gBAAgB;EAC/C,GAAG,QAAQ,sBAAsB,WAAW,MAAM,QAAQ;EAE1D,MAAM,QAAQ,YAAY,IAAI;EAE9B,MAAM,YAA2B;GAAC;GAAQ;EAAU;EAEpD,IAAI,cAAc;EAClB,IAAI;EACJ,IAAI,YAAY;EAChB,IAAI,uBAAuB;EAE3B,MAAM,aAAa,YAAY;GAC7B,IAAI,CAAC,aAAa;IAChB,MAAM,KAAK,MAAM,CAAC,aAAa,SAAS,CAAC;IACzC,cAAc;GAChB;GACA,OAAQ,SAAS,EAAC,OAAM;EAC1B;EAEA,MAAM,YAAY,YAAY;GAC5B,IAAI,MAAM;IACR,MAAM,UAAU,CAAC,YAAY,IAAI;IACjC,MAAM,KAAK,MAAM,SAAS,KAAK,UAAU,OAAO,CAAC;IACjD,OAAO,KAAA;IACP,YAAY;IACZ,uBAAuB;GACzB;EACF;EAEA,MAAM,WAAW,OAAO,mBAAmC;GACzD,MAAM,EAAC,OAAO,cAAa;GAC3B,IAAI,YAAY,WAAW,KAAK,YAAY,KAAK,GAC/C;GAEF,MAAM,SAAS,MAAM,WAAW;GAEhC,MAAM,EAAC,MAAM,OAAM;GACnB,QAAQ,MAAR;IACE,KAAK;KAIH,CAHgB,MAAM,WACjB,CAAC,OAAO,0BAA0B,CAAC,EAAA,CAAG,MAAM,cAAc,CAAC,IAC3D,OAAO,oBAAoB,CAAC,EAAA,CACzB,KAAK;MAAC;MAAI,MAAM,MAAM;KAAE,CAAC;KACjC;IAEF,KAAK;KACH,IAAI,MAAM,GAAG,UAAU,KAAK,mBAC1B,KAAK,aAAc,OAAO,0BAA0B,CAAC,GAAI,KAAK;UACzD,IAAI,MAAM,GAAG,UAAU,KAAK,qBAAqB;MACtD,MAAM,UAAW,OAAO,mBAAmB,CAAC;MAC5C,IAAI,OAAO,OAAO;OAChB,MAAM,MAAMA,MACV,eAAe,MAAM,QAAQ,GAC7B,mBACA,aACF;OACA,QAAQ,KAAK;QACX,IAAI;QACJ,UAAU;SACR,IAAI;UACF,UAAU,IAAI;UACd,IAAI,IAAI;SACV;SACA,QAAQ,IAAI;QACd;OACF,CAAC;MACH,OAAO;OACL,MAAM,EAAC,UAAU,eAAc,MAAM,GAAG;OACxC,OACE,OAAO,aAAa,UACpB,4BACF;OACA,MAAM,KAAK,OAAO,UAAU;OAC5B,OACE,CAAC,OAAO,MAAM,EAAE,KAAK,OAAO,SAAS,EAAE,KAAK,MAAM,GAClD,qCACF;OACA,QAAQ,KAAK;QACX,IAAI;QACJ,IAAI;SACF;SACA;QACF;OACF,CAAC;MACH;KACF,OAAO;MACL,MAAM,WAAW,aAAa,KAAK;MACnC,CAAC,OAAO,cAAc,CAAC,EAAA,CAAG,KAAK,QAAQ;MACvC,wBAAwB,KAAK,UAAU,QAAQ,CAAC,CAAC;KACnD;KACA;IACF,SACE,YAAY,KAAK;GACrB;GAEA;GACA,IACE,wBAAA,WACA,aAAa,4BAEb,MAAM,UAAU;EAEpB;EAOA,OAAO;GACL,IAAI,cAAc;IAChB,OAAO;GACT;GACA,UAAU,OAAO,mBAAmC;IAClD,IAAI;KACF,MAAM,SAAS,cAAc;KAC7B,IAAI,eAAe,MAAM,SAAS,OAChC,KAAK,WAAW,IAAI,CAAC;IAEzB,SAAS,GAAG;KACV,KAAK,KAAK,CAAC;IACb;GACF;GAEA,QAAQ,YAAY;IAClB,IAAI;KACF,IAAI,aACF,MAAM,KAAK,MAAM,CAAC,WAAW;MAAC;MAAQ,QAAQ;MAAI,QAAQ;KAAI,CAAC,CAAC;IAEpE,SAAS,GAAG;KACV,KAAK,KAAK,CAAC;IACb;GACF;GAEA,KAAK,OAAO,iBAA6B;IACvC,IAAI;KACF,MAAM,SAAS,gBAAgB,YAAY;KAC3C,IAAI,CAAC,aAAa;MAChB,IACE,YAAY,KAAK,cAAc,YAAY,MAAM,KACjD,CAAC,kBAED;MAEF,MAAM,KAAK,MAAM,CAAC,aAAa,SAAS,CAAC;KAC3C,OAAO,IAAI,YAAY,KAAK,cAAc,YAAY,KAAK,GAGzD,MAAM,IAAI,MACR,sCAAsC,OAAO,mCAExC,wBAAwB,KAAK,YAAY,GAChD;KAEF,MAAM,UAAU;KAChB,MAAM,KAAK,MAAM,CAAC,WAAW;MAAC;MAAQ;KAAM,CAAC,CAAC;KAC9C,KAAK,eAAe;KACpB,KAAK,aAAa;KAElB,MAAM,UAAU,YAAY,IAAI,IAAI;KACpC,KAAK,kBAAkB,IAAI,CAAC;KAC5B,KAAK,UAAU,SAAS,OAAO;IACjC,SAAS,GAAG;KACV,KAAK,KAAK,CAAC;IACb;GACF;EACF;CACF;CAEA,MAAM,kBACJ,IACA,kBACA,uBACA;EACA,MAAM,oBAAiD,CAAC;EACxD,IAAI,iBAAiB,SAAS,GAC5B,kBAAkB,YAAY;EAEhC,IAAI,sBAAsB,SAAS,GACjC,kBAAkB,iBAAiB;EAErC,GAAG,QAAQ,yBAAyB,iBAAiB;EACrD,MAAM,KAAK,MAAM,CAAC,iBAAiB,iBAAiB,CAAC;CACvD;CAEA,oCAAoC,QAAwB;EAC1D,KAAU,MAAM,CAAC,kBAAkB,MAAM,CAAC;CAC5C;CAEA,8BAA8B,OAA4B;EACxD,KAAK,KAAK,IAAI,cAAc,KAAK,CAAC;CACpC;CAEA,oBAAoB,IAAgB,UAAiC;EACnE,GAAG,QAAQ,4BAA4B,QAAQ;EAC/C,KAAK,YAAY,KAAK;GACpB,SAAS,CAAC,WAAW,QAAQ;GAC7B,YAAY,KAAA;EACd,CAAC;CACH;CAEA,aAAa,OAA+B,OAAiB;EAC3D,IAAI,MAAM,OAAO,OAAO;GACtB,MAAM,MAAM,eAAe,MAAM,QAAQ;GACzC,MAAM,EAAC,eAAe,UAAU,mBAAkBA,MAChD,KACA,eACA,aACF;GACA,IAAI,kBAAkB,KAAK,gBACzB,KAAK,IAAI,QACP,2DACA,aACF;QAEA,MAAM,YAAY;EAEtB,OAEE,MAAM;CAEV;AACF;AAGA,IAAM,gBAAgBC,eAAE,OAAO;CAC7B,eAAeA,eAAE,OAAO;CACxB,UAAUA,eAAE,OAAO;CACnB,gBAAgBA,eAAE,OAAO;AAC3B,CAAC;AAED,IAAM,oBAAoBA,eAAE,OAAO;CACjC,eAAeA,eAAE,OAAO;CACxB,UAAUA,eAAE,OAAO;CACnB,YAAYA,eAAE,OAAO;CACrB,QAAQ;AACV,CAAC;AAED,SAAS,aAAa,OAA6B;CACjD,MAAM,EACJ,IACA,IAAI,EAAC,OAAO,WAAW,QAAQ,SAC7B;CAEJ,QAAQ,IAAR;EACE,KAAK,OACH,OAAO;GACL,IAAI;GACJ;GACA,OAAOD,MAAQ,eAAe,MAAM,QAAQ,GAAG,SAAS;EAC1D;EAEF,KAAK,OACH,OAAO;GACL;GACA;GACA,IAAIA,MAAQ,IAAI,2BAA2B;EAC7C;EAEF,SACE,YAAY,EAAE;CAClB;AACF;;;;;;;;AASA,SAAgB,eAAe,KAAiC;CAC9D,MAAM,WAAW,OAAO,QAAQ,GAAG,CAAC,CACjC,QAAQ,CAAC,GAAG,OAAO;EAClB,IAAI,OAAO,MAAM,UAAU;GACzB,IAAI,KAAK,OAAO,oBAAoB,KAAK,OAAO,kBAC9C,OAAO;GAET,MAAM,IAAI,MAAM,aAAa,EAAE,+BAA+B,EAAE,EAAE;EACpE,OAAO,IAAI,OAAO,MAAM,UACtB,gBAAgB,CAAC;EAEnB,OAAO;CACT,CAAC,CAAC,CACD,KAAK,CAAC,GAAG,OAAO,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;CAEjC,OAAO,SAAS,SACZ;EAAC,GAAG;EAAK,GAAG,OAAO,YAAY,QAAQ;CAAC,IACvC;AACP"}
+{"version":3,"file":"client-handler.js","names":["v.parse","v"],"sources":["../../../../../../zero-cache/src/services/view-syncer/client-handler.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {assert, unreachable} from '../../../../shared/src/asserts.ts';\nimport type {JSONObject} from '../../../../shared/src/bigint-json.ts';\nimport {\n  assertJSONValue,\n  type JSONObject as SafeJSONObject,\n} from '../../../../shared/src/json.ts';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport type {Writable} from '../../../../shared/src/writable.ts';\nimport type {ErroredQuery} from '../../../../zero-protocol/src/custom-queries.ts';\nimport {rowSchema} from '../../../../zero-protocol/src/data.ts';\nimport type {DeleteClientsBody} from '../../../../zero-protocol/src/delete-clients.ts';\nimport type {Downstream} from '../../../../zero-protocol/src/down.ts';\nimport {\n  ProtocolError,\n  type TransformFailedBody,\n} from '../../../../zero-protocol/src/error.ts';\nimport type {InspectDownBody} from '../../../../zero-protocol/src/inspect-down.ts';\nimport {mutationResultSchema} from '../../../../zero-protocol/src/mutation.ts';\nimport type {\n  PokePartBody,\n  PokePartMessage,\n  PokeStartBody,\n} from '../../../../zero-protocol/src/poke.ts';\nimport {primaryKeyValueRecordSchema} from '../../../../zero-protocol/src/primary-key.ts';\nimport type {RowPatchOp} from '../../../../zero-protocol/src/row-patch.ts';\nimport {\n  getOrCreateCounter,\n  getOrCreateLatencyHistogram,\n} from '../../observability/metrics.ts';\nimport {clientGroupLogID} from '../../types/client-group-log-id.ts';\nimport type {ViewSyncerDownstream} from '../../types/downstream.ts';\nimport {\n  getLogLevel,\n  wrapWithProtocolError,\n} from '../../types/error-with-level.ts';\nimport {upstreamSchema, type ShardID} from '../../types/shards.ts';\nimport type {Subscription} from '../../types/subscription.ts';\nimport {\n  cmpVersions,\n  cookieToVersion,\n  versionToCookie,\n  versionToNullableCookie,\n  type CVRVersion,\n  type DelQueryPatch,\n  type NullableCVRVersion,\n  type PutQueryPatch,\n  type RowID,\n} from './schema/types.ts';\n\nexport type PutRowPatch = {\n  type: 'row';\n  op: 'put';\n  id: RowID;\n  contents: JSONObject;\n};\n\nexport type DeleteRowPatch = {\n  type: 'row';\n  op: 'del';\n  id: RowID;\n  /**\n   * Set when the deletion replays a row the CVR already records as deleted,\n   * to catch up a client that fell behind. Unlike a deletion computed while\n   * advancing, it can only undo a put the client received in an earlier poke,\n   * so a client with no base version has nothing it can apply to.\n   */\n  catchup?: boolean | undefined;\n};\n\nexport type RowPatch = PutRowPatch | DeleteRowPatch;\nexport type ConfigPatch = DelQueryPatch | PutQueryPatch;\n\nexport type Patch = ConfigPatch | RowPatch;\n\nexport type PatchToVersion = {\n  patch: Patch;\n  toVersion: CVRVersion;\n};\n\nexport interface PokeHandler {\n  addPatch(patch: PatchToVersion): Promise<void>;\n  cancel(): Promise<void>;\n  end(finalVersion: CVRVersion): Promise<void>;\n}\n\n/** A {@link PokeHandler} for a single client. */\nexport interface ClientPokeHandler extends PokeHandler {\n  /**\n   * Whether a patch has been sent to the client. Once one has, the poke can\n   * only be ended with a `finalVersion` that is ahead of the client's base\n   * version.\n   */\n  readonly patchesSent: boolean;\n}\n\n/**\n * A {@link PokeHandler} for multiple clients, as returned by\n * {@link startPoke}.\n */\nexport interface MultiPokeHandler extends PokeHandler {\n  /** Whether a patch has been sent to any of the clients. */\n  readonly patchesSent: boolean;\n}\n\nconst NOOP: ClientPokeHandler = {\n  patchesSent: false,\n  addPatch: () => promiseVoid,\n  cancel: () => promiseVoid,\n  end: () => promiseVoid,\n};\n\n/** Wraps PokeHandlers for multiple clients in a single PokeHandler. */\nexport function startPoke(\n  lc: LogContext,\n  clients: ClientHandler[],\n  tentativeVersion: CVRVersion,\n): MultiPokeHandler {\n  const pokers = clients.map(c => c.startPoke(tentativeVersion));\n\n  // Promise.allSettled() ensures that a failed (e.g. disconnected) client\n  // does not prevent other clients from receiving the pokes. However, the\n  // rate (per client group) will be limited by the slowest connection.\n  //\n  // The individual pokers report their own failures (by failing the\n  // connection), so a rejection here is not expected. Log it rather than\n  // dropping it on the floor.\n  const settle = async (op: string, results: Promise<void>[]) => {\n    for (const result of await Promise.allSettled(results)) {\n      if (result.status === 'rejected') {\n        lc.error?.(`unhandled error poking client (${op})`, result.reason);\n      }\n    }\n  };\n\n  return {\n    get patchesSent() {\n      return pokers.some(poker => poker.patchesSent);\n    },\n    addPatch: patch =>\n      settle(\n        'addPatch',\n        pokers.map(poker => poker.addPatch(patch)),\n      ),\n    cancel: () =>\n      settle(\n        'cancel',\n        pokers.map(poker => poker.cancel()),\n      ),\n    end: finalVersion =>\n      settle(\n        'end',\n        pokers.map(poker => poker.end(finalVersion)),\n      ),\n  };\n}\n\n// Soft limit on the serialized row characters accumulated in a poke part.\n// This bounds view-syncer memory and keeps legacy JSON messages reasonably\n// sized. PokeChunkEncoder separately enforces the binary frame limit in bytes.\nexport const POKE_PART_FLUSH_THRESHOLD_CHARS = 1024 * 1024;\nconst PART_COUNT_FLUSH_THRESHOLD = 100;\n\n/**\n * Handles a single `ViewSyncer` connection.\n */\nexport class ClientHandler {\n  readonly #clientGroupID: string;\n  readonly clientID: string;\n  readonly wsID: string;\n  readonly #zeroClientsTable: string;\n  readonly #zeroMutationsTable: string;\n  readonly #lc: LogContext;\n  readonly #downstream: Subscription<ViewSyncerDownstream>;\n  #baseVersion: NullableCVRVersion;\n  // Set once the view-syncer has validated this connection. Until then the\n  // client is excluded from pokes and catch-up; see ViewSyncerService.#getClients().\n  #validated = false;\n  // We will send a poke on connect even if the client is already caught up, so that it can learn its\n  // got-queries state has been reconciled with the server. After that, we will only send a poke if\n  // the client is behind.\n  #everPoked = false;\n  #inspectorAuthenticated = false;\n\n  readonly #pokeTime = getOrCreateLatencyHistogram(\n    'sync',\n    'poke.time',\n    'Time elapsed for each poke transaction. Canceled / noop pokes are excluded.',\n  );\n\n  readonly #pokeTransactions = getOrCreateCounter(\n    'sync',\n    'poke.transactions',\n    'Count of poke transactions.',\n  );\n\n  readonly #pokedRows = getOrCreateCounter(\n    'sync',\n    'poke.rows',\n    'Count of poked rows.',\n  );\n\n  constructor(\n    lc: LogContext,\n    clientGroupID: string,\n    clientID: string,\n    wsID: string,\n    shard: ShardID,\n    baseCookie: string | null,\n    downstream: Subscription<ViewSyncerDownstream>,\n  ) {\n    lc.debug?.('new client handler');\n    this.#clientGroupID = clientGroupID;\n    this.clientID = clientID;\n    this.wsID = wsID;\n    this.#zeroClientsTable = `${upstreamSchema(shard)}.clients`;\n    this.#zeroMutationsTable = `${upstreamSchema(shard)}.mutations`;\n    this.#lc = lc;\n    this.#downstream = downstream;\n    this.#baseVersion = cookieToVersion(baseCookie);\n  }\n\n  version(): NullableCVRVersion {\n    return this.#baseVersion;\n  }\n\n  validated(): boolean {\n    return this.#validated;\n  }\n\n  markValidated(): void {\n    this.#validated = true;\n  }\n\n  /**\n   * Whether this connection has presented the admin password to the\n   * inspector. Authentication is per connection: it is not shared with other\n   * connections of the client group and does not survive a reconnect.\n   */\n  isInspectorAuthenticated(): boolean {\n    return this.#inspectorAuthenticated;\n  }\n\n  setInspectorAuthenticated(authenticated: boolean): void {\n    this.#inspectorAuthenticated = authenticated;\n  }\n\n  async #push(msg: Downstream, serialized?: string | undefined): Promise<void> {\n    const {result} = this.#downstream.push({message: msg, serialized});\n    await result;\n  }\n\n  fail(e: unknown) {\n    this.#lc[getLogLevel(e)]?.(\n      `view-syncer closing connection with error: ${String(e)}`,\n      e,\n    );\n    this.#downstream.fail(wrapWithProtocolError(e));\n  }\n\n  close(reason: string) {\n    this.#lc.debug?.(`view-syncer closing connection: ${reason}`);\n    this.#downstream.cancel();\n  }\n\n  startPoke(tentativeVersion: CVRVersion): ClientPokeHandler {\n    const pokeID = versionToCookie(tentativeVersion);\n    const lc = this.#lc.withContext('pokeID', pokeID);\n\n    // Force an (empty) initial poke even when already caught up; see #everPoked.\n    const forceInitialPoke = !this.#everPoked;\n    const cmp = cmpVersions(this.#baseVersion, tentativeVersion);\n    if (cmp > 0 || (cmp === 0 && !forceInitialPoke)) {\n      lc.info?.(`already caught up, not sending poke.`);\n      return NOOP;\n    }\n\n    const baseCookie = versionToNullableCookie(this.#baseVersion);\n    const cookie = versionToCookie(tentativeVersion);\n    lc.debug?.(`starting poke from ${baseCookie} to ${cookie}`);\n\n    const start = performance.now();\n\n    const pokeStart: PokeStartBody = {pokeID, baseCookie};\n\n    let pokeStarted = false;\n    let body: PokePartBody | undefined;\n    let partCount = 0;\n    let serializedRowsLength = 0;\n\n    const ensureBody = async () => {\n      if (!pokeStarted) {\n        await this.#push(['pokeStart', pokeStart]);\n        pokeStarted = true;\n      }\n      return (body ??= {pokeID});\n    };\n\n    const flushBody = async () => {\n      if (body) {\n        const message = ['pokePart', body] satisfies PokePartMessage;\n        await this.#push(message, JSON.stringify(message));\n        body = undefined;\n        partCount = 0;\n        serializedRowsLength = 0;\n      }\n    };\n\n    const addPatch = async (patchToVersion: PatchToVersion) => {\n      const {patch, toVersion} = patchToVersion;\n      if (cmpVersions(toVersion, this.#baseVersion) <= 0) {\n        return;\n      }\n      if (\n        patch.type === 'row' &&\n        patch.op === 'del' &&\n        patch.catchup &&\n        this.#baseVersion === null\n      ) {\n        // The client has no rows; deleting one it never had only tells it\n        // the row's key.\n        return;\n      }\n      const target = await ensureBody();\n\n      const {type, op} = patch;\n      switch (type) {\n        case 'query': {\n          const patches = patch.clientID\n            ? ((target.desiredQueriesPatches ??= {})[patch.clientID] ??= [])\n            : (target.gotQueriesPatch ??= []);\n          patches.push({op, hash: patch.id});\n          break;\n        }\n        case 'row':\n          if (patch.id.table === this.#zeroClientsTable) {\n            this.#updateLMIDs((target.lastMutationIDChanges ??= {}), patch);\n          } else if (patch.id.table === this.#zeroMutationsTable) {\n            const patches = (target.mutationsPatch ??= []);\n            if (op === 'put') {\n              const row = v.parse(\n                ensureSafeJSON(patch.contents),\n                mutationRowSchema,\n                'passthrough',\n              );\n              patches.push({\n                op: 'put',\n                mutation: {\n                  id: {\n                    clientID: row.clientID,\n                    id: row.mutationID,\n                  },\n                  result: row.result,\n                },\n              });\n            } else {\n              const {clientID, mutationID} = patch.id.rowKey;\n              assert(\n                typeof clientID === 'string',\n                'client id must be a string',\n              );\n              const id = Number(mutationID);\n              assert(\n                !Number.isNaN(id) && Number.isFinite(id) && id >= 0,\n                'mutation id must be a finite number',\n              );\n              patches.push({\n                op: 'del',\n                id: {\n                  clientID,\n                  id,\n                },\n              });\n            }\n          } else {\n            const rowPatch = makeRowPatch(patch);\n            (target.rowsPatch ??= []).push(rowPatch);\n            serializedRowsLength += JSON.stringify(rowPatch).length;\n          }\n          break;\n        default:\n          unreachable(patch);\n      }\n\n      partCount++;\n      if (\n        serializedRowsLength >= POKE_PART_FLUSH_THRESHOLD_CHARS ||\n        partCount >= PART_COUNT_FLUSH_THRESHOLD\n      ) {\n        await flushBody();\n      }\n    };\n\n    // A poke that was started must always be ended. If it isn't, the\n    // connection is left mid-poke and the *next* pokeStart fails the\n    // DownstreamSender's in-progress check, surfacing the error far from\n    // its cause. Fail the connection instead; the client reconnects and\n    // resyncs from its baseVersion.\n    return {\n      get patchesSent() {\n        return pokeStarted;\n      },\n      addPatch: async (patchToVersion: PatchToVersion) => {\n        try {\n          await addPatch(patchToVersion);\n          if (patchToVersion.patch.type === 'row') {\n            this.#pokedRows.add(1);\n          }\n        } catch (e) {\n          this.fail(e);\n        }\n      },\n\n      cancel: async () => {\n        try {\n          if (pokeStarted) {\n            await this.#push(['pokeEnd', {pokeID, cookie: '', cancel: true}]);\n          }\n        } catch (e) {\n          this.fail(e);\n        }\n      },\n\n      end: async (finalVersion: CVRVersion) => {\n        try {\n          const cookie = versionToCookie(finalVersion);\n          if (!pokeStarted) {\n            if (\n              cmpVersions(this.#baseVersion, finalVersion) === 0 &&\n              !forceInitialPoke\n            ) {\n              return; // Nothing changed and nothing was sent.\n            }\n            await this.#push(['pokeStart', pokeStart]);\n          } else if (cmpVersions(this.#baseVersion, finalVersion) >= 0) {\n            // Sanity check: If the poke was started, the finalVersion\n            // must be > #baseVersion.\n            throw new Error(\n              `Patches were sent but finalVersion ${cookie} is not ` +\n                `greater than baseVersion ` +\n                `${versionToNullableCookie(this.#baseVersion)}`,\n            );\n          }\n          await flushBody();\n          await this.#push(['pokeEnd', {pokeID, cookie}]);\n          this.#baseVersion = finalVersion;\n          this.#everPoked = true;\n\n          const elapsed = performance.now() - start;\n          this.#pokeTransactions.add(1);\n          this.#pokeTime.recordMs(elapsed);\n        } catch (e) {\n          this.fail(e);\n        }\n      },\n    };\n  }\n\n  async sendDeleteClients(\n    lc: LogContext,\n    deletedClientIDs: string[],\n    deletedClientGroupIDs: string[],\n  ) {\n    const deleteClientsBody: Writable<DeleteClientsBody> = {};\n    if (deletedClientIDs.length > 0) {\n      deleteClientsBody.clientIDs = deletedClientIDs;\n    }\n    if (deletedClientGroupIDs.length > 0) {\n      deleteClientsBody.clientGroupIDs = deletedClientGroupIDs;\n    }\n    lc.debug?.('sending deleteClients', deleteClientsBody);\n    await this.#push(['deleteClients', deleteClientsBody]);\n  }\n\n  sendQueryTransformApplicationErrors(errors: ErroredQuery[]) {\n    void this.#push(['transformError', errors]);\n  }\n\n  sendQueryTransformFailedError(error: TransformFailedBody) {\n    this.fail(new ProtocolError(error));\n  }\n\n  sendInspectResponse(lc: LogContext, response: InspectDownBody): void {\n    lc.debug?.('sending inspect response', response);\n    this.#downstream.push({\n      message: ['inspect', response],\n      serialized: undefined,\n    });\n  }\n\n  #updateLMIDs(lmids: Record<string, number>, patch: RowPatch) {\n    if (patch.op === 'put') {\n      const row = ensureSafeJSON(patch.contents);\n      const {clientGroupID, clientID, lastMutationID} = v.parse(\n        row,\n        lmidRowSchema,\n        'passthrough',\n      );\n      if (clientGroupID !== this.#clientGroupID) {\n        this.#lc.error?.(\n          `Received clients row for wrong clientGroupID. Ignoring.`,\n          clientGroupLogID(clientGroupID),\n        );\n      } else {\n        lmids[clientID] = lastMutationID;\n      }\n    } else {\n      // The 'constrain' and 'del' ops for clients can be ignored.\n      patch.op satisfies 'constrain' | 'del';\n    }\n  }\n}\n\n// Note: The {APP_ID}_{SHARD_ID}.clients table is set up in replicator/initial-sync.ts.\nconst lmidRowSchema = v.object({\n  clientGroupID: v.string(),\n  clientID: v.string(),\n  lastMutationID: v.number(), // Actually returned as a bigint, but converted by ensureSafeJSON().\n});\n\nconst mutationRowSchema = v.object({\n  clientGroupID: v.string(),\n  clientID: v.string(),\n  mutationID: v.number(),\n  result: mutationResultSchema,\n});\n\nfunction makeRowPatch(patch: RowPatch): RowPatchOp {\n  const {\n    op,\n    id: {table: tableName, rowKey: id},\n  } = patch;\n\n  switch (op) {\n    case 'put':\n      return {\n        op: 'put',\n        tableName,\n        value: v.parse(ensureSafeJSON(patch.contents), rowSchema),\n      };\n\n    case 'del':\n      return {\n        op,\n        tableName,\n        id: v.parse(id, primaryKeyValueRecordSchema),\n      };\n\n    default:\n      unreachable(op);\n  }\n}\n\n/**\n * Column values of type INT8 are returned as the `bigint` from the\n * Postgres library. These are converted to `number` if they are within\n * the safe Number range, allowing the protocol to support numbers larger\n * than 32-bits. Values outside of the safe number range (e.g. > 2^53) will\n * result in an Error.\n */\nexport function ensureSafeJSON(row: JSONObject): SafeJSONObject {\n  const modified = Object.entries(row)\n    .filter(([k, v]) => {\n      if (typeof v === 'bigint') {\n        if (v >= Number.MIN_SAFE_INTEGER && v <= Number.MAX_SAFE_INTEGER) {\n          return true; // send this entry onto the next map() step.\n        }\n        throw new Error(`Value of \"${k}\" exceeds safe Number range (${v})`);\n      } else if (typeof v === 'object') {\n        assertJSONValue(v);\n      }\n      return false;\n    })\n    .map(([k, v]) => [k, Number(v)]);\n\n  return modified.length\n    ? {...row, ...Object.fromEntries(modified)}\n    : (row as SafeJSONObject);\n}\n"],"mappings":";;;;;;;;;;;;;;AA0GA,IAAM,OAA0B;CAC9B,aAAa;CACb,gBAAgB;CAChB,cAAc;CACd,WAAW;AACb;;AAGA,SAAgB,UACd,IACA,SACA,kBACkB;CAClB,MAAM,SAAS,QAAQ,KAAI,MAAK,EAAE,UAAU,gBAAgB,CAAC;CAS7D,MAAM,SAAS,OAAO,IAAY,YAA6B;EAC7D,KAAK,MAAM,UAAU,MAAM,QAAQ,WAAW,OAAO,GACnD,IAAI,OAAO,WAAW,YACpB,GAAG,QAAQ,kCAAkC,GAAG,IAAI,OAAO,MAAM;CAGvE;CAEA,OAAO;EACL,IAAI,cAAc;GAChB,OAAO,OAAO,MAAK,UAAS,MAAM,WAAW;EAC/C;EACA,WAAU,UACR,OACE,YACA,OAAO,KAAI,UAAS,MAAM,SAAS,KAAK,CAAC,CAC3C;EACF,cACE,OACE,UACA,OAAO,KAAI,UAAS,MAAM,OAAO,CAAC,CACpC;EACF,MAAK,iBACH,OACE,OACA,OAAO,KAAI,UAAS,MAAM,IAAI,YAAY,CAAC,CAC7C;CACJ;AACF;AAKA,IAAa,kCAAkC;AAC/C,IAAM,6BAA6B;;;;AAKnC,IAAa,gBAAb,MAA2B;CACzB;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAGA,aAAa;CAIb,aAAa;CACb,0BAA0B;CAE1B,YAAqB,4BACnB,QACA,aACA,6EACF;CAEA,oBAA6B,mBAC3B,QACA,qBACA,6BACF;CAEA,aAAsB,mBACpB,QACA,aACA,sBACF;CAEA,YACE,IACA,eACA,UACA,MACA,OACA,YACA,YACA;EACA,GAAG,QAAQ,oBAAoB;EAC/B,KAAK,iBAAiB;EACtB,KAAK,WAAW;EAChB,KAAK,OAAO;EACZ,KAAK,oBAAoB,GAAG,eAAe,KAAK,EAAE;EAClD,KAAK,sBAAsB,GAAG,eAAe,KAAK,EAAE;EACpD,KAAK,MAAM;EACX,KAAK,cAAc;EACnB,KAAK,eAAe,gBAAgB,UAAU;CAChD;CAEA,UAA8B;EAC5B,OAAO,KAAK;CACd;CAEA,YAAqB;EACnB,OAAO,KAAK;CACd;CAEA,gBAAsB;EACpB,KAAK,aAAa;CACpB;;;;;;CAOA,2BAAoC;EAClC,OAAO,KAAK;CACd;CAEA,0BAA0B,eAA8B;EACtD,KAAK,0BAA0B;CACjC;CAEA,MAAM,MAAM,KAAiB,YAAgD;EAC3E,MAAM,EAAC,WAAU,KAAK,YAAY,KAAK;GAAC,SAAS;GAAK;EAAU,CAAC;EACjE,MAAM;CACR;CAEA,KAAK,GAAY;EACf,KAAK,IAAI,YAAY,CAAC,EAAE,GACtB,8CAA8C,OAAO,CAAC,KACtD,CACF;EACA,KAAK,YAAY,KAAK,sBAAsB,CAAC,CAAC;CAChD;CAEA,MAAM,QAAgB;EACpB,KAAK,IAAI,QAAQ,mCAAmC,QAAQ;EAC5D,KAAK,YAAY,OAAO;CAC1B;CAEA,UAAU,kBAAiD;EACzD,MAAM,SAAS,gBAAgB,gBAAgB;EAC/C,MAAM,KAAK,KAAK,IAAI,YAAY,UAAU,MAAM;EAGhD,MAAM,mBAAmB,CAAC,KAAK;EAC/B,MAAM,MAAM,YAAY,KAAK,cAAc,gBAAgB;EAC3D,IAAI,MAAM,KAAM,QAAQ,KAAK,CAAC,kBAAmB;GAC/C,GAAG,OAAO,sCAAsC;GAChD,OAAO;EACT;EAEA,MAAM,aAAa,wBAAwB,KAAK,YAAY;EAC5D,MAAM,SAAS,gBAAgB,gBAAgB;EAC/C,GAAG,QAAQ,sBAAsB,WAAW,MAAM,QAAQ;EAE1D,MAAM,QAAQ,YAAY,IAAI;EAE9B,MAAM,YAA2B;GAAC;GAAQ;EAAU;EAEpD,IAAI,cAAc;EAClB,IAAI;EACJ,IAAI,YAAY;EAChB,IAAI,uBAAuB;EAE3B,MAAM,aAAa,YAAY;GAC7B,IAAI,CAAC,aAAa;IAChB,MAAM,KAAK,MAAM,CAAC,aAAa,SAAS,CAAC;IACzC,cAAc;GAChB;GACA,OAAQ,SAAS,EAAC,OAAM;EAC1B;EAEA,MAAM,YAAY,YAAY;GAC5B,IAAI,MAAM;IACR,MAAM,UAAU,CAAC,YAAY,IAAI;IACjC,MAAM,KAAK,MAAM,SAAS,KAAK,UAAU,OAAO,CAAC;IACjD,OAAO,KAAA;IACP,YAAY;IACZ,uBAAuB;GACzB;EACF;EAEA,MAAM,WAAW,OAAO,mBAAmC;GACzD,MAAM,EAAC,OAAO,cAAa;GAC3B,IAAI,YAAY,WAAW,KAAK,YAAY,KAAK,GAC/C;GAEF,IACE,MAAM,SAAS,SACf,MAAM,OAAO,SACb,MAAM,WACN,KAAK,iBAAiB,MAItB;GAEF,MAAM,SAAS,MAAM,WAAW;GAEhC,MAAM,EAAC,MAAM,OAAM;GACnB,QAAQ,MAAR;IACE,KAAK;KAIH,CAHgB,MAAM,WACjB,CAAC,OAAO,0BAA0B,CAAC,EAAA,CAAG,MAAM,cAAc,CAAC,IAC3D,OAAO,oBAAoB,CAAC,EAAA,CACzB,KAAK;MAAC;MAAI,MAAM,MAAM;KAAE,CAAC;KACjC;IAEF,KAAK;KACH,IAAI,MAAM,GAAG,UAAU,KAAK,mBAC1B,KAAK,aAAc,OAAO,0BAA0B,CAAC,GAAI,KAAK;UACzD,IAAI,MAAM,GAAG,UAAU,KAAK,qBAAqB;MACtD,MAAM,UAAW,OAAO,mBAAmB,CAAC;MAC5C,IAAI,OAAO,OAAO;OAChB,MAAM,MAAMA,MACV,eAAe,MAAM,QAAQ,GAC7B,mBACA,aACF;OACA,QAAQ,KAAK;QACX,IAAI;QACJ,UAAU;SACR,IAAI;UACF,UAAU,IAAI;UACd,IAAI,IAAI;SACV;SACA,QAAQ,IAAI;QACd;OACF,CAAC;MACH,OAAO;OACL,MAAM,EAAC,UAAU,eAAc,MAAM,GAAG;OACxC,OACE,OAAO,aAAa,UACpB,4BACF;OACA,MAAM,KAAK,OAAO,UAAU;OAC5B,OACE,CAAC,OAAO,MAAM,EAAE,KAAK,OAAO,SAAS,EAAE,KAAK,MAAM,GAClD,qCACF;OACA,QAAQ,KAAK;QACX,IAAI;QACJ,IAAI;SACF;SACA;QACF;OACF,CAAC;MACH;KACF,OAAO;MACL,MAAM,WAAW,aAAa,KAAK;MACnC,CAAC,OAAO,cAAc,CAAC,EAAA,CAAG,KAAK,QAAQ;MACvC,wBAAwB,KAAK,UAAU,QAAQ,CAAC,CAAC;KACnD;KACA;IACF,SACE,YAAY,KAAK;GACrB;GAEA;GACA,IACE,wBAAA,WACA,aAAa,4BAEb,MAAM,UAAU;EAEpB;EAOA,OAAO;GACL,IAAI,cAAc;IAChB,OAAO;GACT;GACA,UAAU,OAAO,mBAAmC;IAClD,IAAI;KACF,MAAM,SAAS,cAAc;KAC7B,IAAI,eAAe,MAAM,SAAS,OAChC,KAAK,WAAW,IAAI,CAAC;IAEzB,SAAS,GAAG;KACV,KAAK,KAAK,CAAC;IACb;GACF;GAEA,QAAQ,YAAY;IAClB,IAAI;KACF,IAAI,aACF,MAAM,KAAK,MAAM,CAAC,WAAW;MAAC;MAAQ,QAAQ;MAAI,QAAQ;KAAI,CAAC,CAAC;IAEpE,SAAS,GAAG;KACV,KAAK,KAAK,CAAC;IACb;GACF;GAEA,KAAK,OAAO,iBAA6B;IACvC,IAAI;KACF,MAAM,SAAS,gBAAgB,YAAY;KAC3C,IAAI,CAAC,aAAa;MAChB,IACE,YAAY,KAAK,cAAc,YAAY,MAAM,KACjD,CAAC,kBAED;MAEF,MAAM,KAAK,MAAM,CAAC,aAAa,SAAS,CAAC;KAC3C,OAAO,IAAI,YAAY,KAAK,cAAc,YAAY,KAAK,GAGzD,MAAM,IAAI,MACR,sCAAsC,OAAO,mCAExC,wBAAwB,KAAK,YAAY,GAChD;KAEF,MAAM,UAAU;KAChB,MAAM,KAAK,MAAM,CAAC,WAAW;MAAC;MAAQ;KAAM,CAAC,CAAC;KAC9C,KAAK,eAAe;KACpB,KAAK,aAAa;KAElB,MAAM,UAAU,YAAY,IAAI,IAAI;KACpC,KAAK,kBAAkB,IAAI,CAAC;KAC5B,KAAK,UAAU,SAAS,OAAO;IACjC,SAAS,GAAG;KACV,KAAK,KAAK,CAAC;IACb;GACF;EACF;CACF;CAEA,MAAM,kBACJ,IACA,kBACA,uBACA;EACA,MAAM,oBAAiD,CAAC;EACxD,IAAI,iBAAiB,SAAS,GAC5B,kBAAkB,YAAY;EAEhC,IAAI,sBAAsB,SAAS,GACjC,kBAAkB,iBAAiB;EAErC,GAAG,QAAQ,yBAAyB,iBAAiB;EACrD,MAAM,KAAK,MAAM,CAAC,iBAAiB,iBAAiB,CAAC;CACvD;CAEA,oCAAoC,QAAwB;EAC1D,KAAU,MAAM,CAAC,kBAAkB,MAAM,CAAC;CAC5C;CAEA,8BAA8B,OAA4B;EACxD,KAAK,KAAK,IAAI,cAAc,KAAK,CAAC;CACpC;CAEA,oBAAoB,IAAgB,UAAiC;EACnE,GAAG,QAAQ,4BAA4B,QAAQ;EAC/C,KAAK,YAAY,KAAK;GACpB,SAAS,CAAC,WAAW,QAAQ;GAC7B,YAAY,KAAA;EACd,CAAC;CACH;CAEA,aAAa,OAA+B,OAAiB;EAC3D,IAAI,MAAM,OAAO,OAAO;GACtB,MAAM,MAAM,eAAe,MAAM,QAAQ;GACzC,MAAM,EAAC,eAAe,UAAU,mBAAkBA,MAChD,KACA,eACA,aACF;GACA,IAAI,kBAAkB,KAAK,gBACzB,KAAK,IAAI,QACP,2DACA,iBAAiB,aAAa,CAChC;QAEA,MAAM,YAAY;EAEtB,OAEE,MAAM;CAEV;AACF;AAGA,IAAM,gBAAgBC,eAAE,OAAO;CAC7B,eAAeA,eAAE,OAAO;CACxB,UAAUA,eAAE,OAAO;CACnB,gBAAgBA,eAAE,OAAO;AAC3B,CAAC;AAED,IAAM,oBAAoBA,eAAE,OAAO;CACjC,eAAeA,eAAE,OAAO;CACxB,UAAUA,eAAE,OAAO;CACnB,YAAYA,eAAE,OAAO;CACrB,QAAQ;AACV,CAAC;AAED,SAAS,aAAa,OAA6B;CACjD,MAAM,EACJ,IACA,IAAI,EAAC,OAAO,WAAW,QAAQ,SAC7B;CAEJ,QAAQ,IAAR;EACE,KAAK,OACH,OAAO;GACL,IAAI;GACJ;GACA,OAAOD,MAAQ,eAAe,MAAM,QAAQ,GAAG,SAAS;EAC1D;EAEF,KAAK,OACH,OAAO;GACL;GACA;GACA,IAAIA,MAAQ,IAAI,2BAA2B;EAC7C;EAEF,SACE,YAAY,EAAE;CAClB;AACF;;;;;;;;AASA,SAAgB,eAAe,KAAiC;CAC9D,MAAM,WAAW,OAAO,QAAQ,GAAG,CAAC,CACjC,QAAQ,CAAC,GAAG,OAAO;EAClB,IAAI,OAAO,MAAM,UAAU;GACzB,IAAI,KAAK,OAAO,oBAAoB,KAAK,OAAO,kBAC9C,OAAO;GAET,MAAM,IAAI,MAAM,aAAa,EAAE,+BAA+B,EAAE,EAAE;EACpE,OAAO,IAAI,OAAO,MAAM,UACtB,gBAAgB,CAAC;EAEnB,OAAO;CACT,CAAC,CAAC,CACD,KAAK,CAAC,GAAG,OAAO,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;CAEjC,OAAO,SAAS,SACZ;EAAC,GAAG;EAAK,GAAG,OAAO,YAAY,QAAQ;CAAC,IACvC;AACP"}

package/out/zero-cache/src/services/view-syncer/connection-context-manager.d.ts

@@ -13,12 +13,24 @@ export type UserState = {
     readonly id: string | null;
 };
 /**
- * Delineates the two paths for validating a connection: either server can validate
- * the user's identity and return a definitive userID to trust, or we fall back to
- * trusting the one provided by the client in the incoming query params.
+ * How a connection's `userID` was established:
+ *
+ * - `server-validated`: the API server returned the authoritative userID
+ *   (`null` for a logged-out caller). The connection's claimed userID must
+ *   match it.
+ * - `server-unverified`: the API server answered without a userID, so it did
+ *   not vouch for the caller. With `requireServerUserID`, only a logged-out
+ *   connection, or a userID that a legacy JWT has already verified, is
+ *   accepted. Without it, the claimed userID is trusted and a warning is
+ *   logged.
+ * - `client-fallback`: no API server is involved, so the userID provided by
+ *   the client in the connection params is trusted as-is (a legacy JWT, when
+ *   configured, has verified it).
  */
 export type ConnectionValidation = {
     kind: 'client-fallback';
+} | {
+    kind: 'server-unverified';
 } | {
     kind: 'server-validated';
     validatedUserID: string | null;
@@ -86,6 +98,7 @@ export type ConnectionContextManager = {
         connection: ConnectionContext;
         group: GroupAuthState;
     }> | undefined;
+    restorePinnedUser(pinnedUser: UserState): void;
     failConnection(selector: ConnectionSelector, revision: number): Readonly<ConnectionContext> | undefined;
     closeConnection(selector: ConnectionSelector): Readonly<ConnectionContext> | undefined;
     markBackgroundRetransformSuccess(selector: ConnectionSelector, revision: number): void;
@@ -112,7 +125,7 @@ export type ConnectionContextManager = {
  */
 export declare class ConnectionContextManagerImpl implements ConnectionContextManager {
     #private;
-    constructor(lc: LogContext, revalidateIntervalSeconds?: number, retransformIntervalSeconds?: number, queryConfig?: FetchConfig, pushConfig?: FetchConfig, validateLegacyJWT?: ValidateLegacyJWT, now?: () => number);
+    constructor(lc: LogContext, revalidateIntervalSeconds?: number, retransformIntervalSeconds?: number, queryConfig?: FetchConfig, pushConfig?: FetchConfig, validateLegacyJWT?: ValidateLegacyJWT, now?: () => number, requireServerUserID?: boolean);
     /**
      * Creates or replaces the live record for a websocket connection.
      *
@@ -145,6 +158,18 @@ export declare class ConnectionContextManagerImpl implements ConnectionContextMa
         connection: ConnectionContext;
         group: GroupAuthState;
     }> | undefined;
+    /**
+     * Pins the group to the user persisted with its CVR, as bound by a validated
+     * connection to an earlier instance of the group's view-syncer. Later
+     * validations by this instance must match it, as they would had the group
+     * been pinned here.
+     *
+     * Throws a `Rehome` error if the group is already pinned to another user:
+     * the stored pin wins, so the connections validated here can no longer be
+     * trusted and the view-syncer should stop, leaving its clients to reconnect
+     * to a fresh instance that validates them against the stored pin.
+     */
+    restorePinnedUser(pinnedUser: UserState): void;
     /** Removes one connection due to failed auth and updates all derived background/deadline state. */
     failConnection(selector: ConnectionSelector, revision: number): ConnectionContext | undefined;
     /** Removes one disconnected connection and updates all derived background/deadline state. */

package/out/zero-cache/src/services/view-syncer/connection-context-manager.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"connection-context-manager.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/view-syncer/connection-context-manager.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,EAAC,kBAAkB,EAAC,MAAM,0CAA0C,CAAC;AAGjF,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,8CAA8C,CAAC;AACjF,OAAO,EAGL,KAAK,IAAI,EACT,KAAK,iBAAiB,EACvB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,6BAA6B,CAAC;AAG5D,OAAO,KAAK,EAAC,aAAa,EAAC,MAAM,iCAAiC,CAAC;AAEnE,MAAM,MAAM,eAAe,GAAG,aAAa,GAAG,WAAW,CAAC;AAE1D;;;GAGG;AACH,MAAM,MAAM,SAAS,GAAG;IAAC,QAAQ,CAAC,EAAE,EAAE,MAAM,GAAG,IAAI,CAAA;CAAC,CAAC;AAErD;;;;GAIG;AACH,MAAM,MAAM,oBAAoB,GAC5B;IAAC,IAAI,EAAE,iBAAiB,CAAA;CAAC,GACzB;IAAC,IAAI,EAAE,kBAAkB,CAAC;IAAC,eAAe,EAAE,MAAM,GAAG,IAAI,CAAA;CAAC,CAAC;AAE/D;;GAEG;AACH,MAAM,MAAM,kBAAkB,GAAG;IAC/B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;CACvB,CAAC;AAEF,KAAK,WAAW,GAAG,UAAU,CAAC,OAAO,CAAC,CAAC;AAEvC,MAAM,MAAM,aAAa,GAAG;IAC1B,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACrC,0DAA0D;IAC1D,QAAQ,CAAC,aAAa,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,GAAG,SAAS,CAAC;IACtE,0DAA0D;IAC1D,QAAQ,CAAC,cAAc,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,GAAG,SAAS,CAAC;IACvE,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACrC,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACtC,CAAC;AAEF,MAAM,MAAM,sBAAsB,GAAG;IACnC,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG,SAAS,CAAC;IACjC,QAAQ,CAAC,kBAAkB,EAAE,SAAS,UAAU,EAAE,GAAG,SAAS,CAAC;IAC/D,QAAQ,CAAC,aAAa,EAAE,aAAa,CAAC;CACvC,CAAC;AAEF;;;;GAIG;AACH,MAAM,MAAM,iBAAiB,GAAG;IAC9B,QAAQ,CAAC,KAAK,EAAE,eAAe,CAAC;IAEhC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC;IAEzB,QAAQ,CAAC,IAAI,EAAE,IAAI,GAAG,SAAS,CAAC;IAEhC,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IAEjC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAE1B,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,SAAS,CAAC;IAE1C,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAEhC,QAAQ,CAAC,YAAY,EAAE,sBAAsB,CAAC;IAC9C,QAAQ,CAAC,aAAa,EAAE,sBAAsB,CAAC;CAChD,CAAC;AAEF;;;;;;GAMG;AACH,MAAM,MAAM,cAAc,GAAG;IAC3B,QAAQ,CAAC,UAAU,EAAE,SAAS,GAAG,SAAS,CAAC;IAE3C,QAAQ,CAAC,oBAAoB,EAAE,kBAAkB,GAAG,SAAS,CAAC;IAC9D,QAAQ,CAAC,aAAa,EAAE,MAAM,GAAG,SAAS,CAAC;IAE3C,QAAQ,CAAC,sBAAsB,EAAE,MAAM,GAAG,SAAS,CAAC;CACrD,CAAC;AAEF,MAAM,MAAM,wBAAwB,GAAG;IACrC,kBAAkB,CAChB,QAAQ,EAAE,kBAAkB,EAC5B,aAAa,EAAE,aAAa,EAC5B,IAAI,CAAC,EAAE,IAAI,GACV,QAAQ,CAAC,iBAAiB,CAAC,CAAC;IAE/B,cAAc,CACZ,QAAQ,EAAE,kBAAkB,EAC5B,IAAI,EAAE,kBAAkB,GACvB,QAAQ,CAAC,iBAAiB,CAAC,CAAC;IAE/B,UAAU,CACR,QAAQ,EAAE,kBAAkB,EAC5B,IAAI,EAAE,cAAc,GACnB,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAC,CAAC;IAExC,kBAAkB,CAChB,QAAQ,EAAE,kBAAkB,EAC5B,QAAQ,EAAE,MAAM,EAChB,UAAU,EAAE,oBAAoB,GAE9B,QAAQ,CAAC;QACP,UAAU,EAAE,iBAAiB,CAAC;QAC9B,KAAK,EAAE,cAAc,CAAC;KACvB,CAAC,GACF,SAAS,CAAC;IAEd,cAAc,CACZ,QAAQ,EAAE,kBAAkB,EAC5B,QAAQ,EAAE,MAAM,GACf,QAAQ,CAAC,iBAAiB,CAAC,GAAG,SAAS,CAAC;IAC3C,eAAe,CACb,QAAQ,EAAE,kBAAkB,GAC3B,QAAQ,CAAC,iBAAiB,CAAC,GAAG,SAAS,CAAC;IAE3C,gCAAgC,CAC9B,QAAQ,EAAE,kBAAkB,EAC5B,QAAQ,EAAE,MAAM,GACf,IAAI,CAAC;IAER,yBAAyB,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI,CAAC;IAEhD,gBAAgB,CAAC,IAAI,EAAE,YAAY,GAAG,aAAa,GAAG,IAAI,CAAC;IAE3D,oBAAoB,CAClB,QAAQ,EAAE,kBAAkB,GAC3B,QAAQ,CAAC,iBAAiB,CAAC,GAAG,SAAS,CAAC;IAC3C,wBAAwB,CACtB,QAAQ,EAAE,kBAAkB,GAC3B,QAAQ,CAAC,iBAAiB,CAAC,CAAC;IAE/B,8BAA8B,IAAI,QAAQ,CAAC,iBAAiB,CAAC,GAAG,SAAS,CAAC;IAC1E,kCAAkC,IAAI,QAAQ,CAAC,iBAAiB,CAAC,CAAC;IAElE,aAAa,IAAI,QAAQ,CAAC,cAAc,CAAC,CAAC;IAE1C,eAAe,IAAI;QACjB,gBAAgB,EAAE,QAAQ,CAAC,iBAAiB,CAAC,EAAE,CAAC;QAChD,cAAc,EAAE,OAAO,CAAC;QACxB,kBAAkB,EAAE,MAAM,GAAG,SAAS,CAAC;KACxC,CAAC;CACH,CAAC;AAEF;;;;;;;GAOG;AACH,qBAAa,4BAA6B,YAAW,wBAAwB;;IAsB3E,YACE,EAAE,EAAE,UAAU,EACd,yBAAyB,CAAC,EAAE,MAAM,EAClC,0BAA0B,CAAC,EAAE,MAAM,EACnC,WAAW,CAAC,EAAE,WAAW,EACzB,UAAU,CAAC,EAAE,WAAW,EACxB,iBAAiB,CAAC,EAAE,iBAAiB,EACrC,GAAG,CAAC,EAAE,MAAM,MAAM,EAenB;IAED;;;;;OAKG;IACH,kBAAkB,CAChB,QAAQ,EAAE,kBAAkB,EAC5B,aAAa,EAAE,aAAa,EAC5B,IAAI,CAAC,EAAE,IAAI,GACV,QAAQ,CAAC,iBAAiB,CAAC,CA8C7B;IAED;;;;;OAKG;IACH,cAAc,CACZ,QAAQ,EAAE,kBAAkB,EAC5B,IAAI,EAAE,kBAAkB,GACvB,QAAQ,CAAC,iBAAiB,CAAC,CAiD7B;IAED;;;OAGG;IACG,UAAU,CACd,QAAQ,EAAE,kBAAkB,EAC5B,IAAI,EAAE,cAAc,GACnB,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAC,CA4BtC;IAED;;;;;;;;OAQG;IACH,kBAAkB,CAChB,QAAQ,EAAE,kBAAkB,EAC5B,QAAQ,EAAE,MAAM,EAChB,UAAU,EAAE,oBAAoB,GAE9B,QAAQ,CAAC;QACP,UAAU,EAAE,iBAAiB,CAAC;QAC9B,KAAK,EAAE,cAAc,CAAC;KACvB,CAAC,GACF,SAAS,CA6EZ;IAED,mGAAmG;IACnG,cAAc,CACZ,QAAQ,EAAE,kBAAkB,EAC5B,QAAQ,EAAE,MAAM,GACf,iBAAiB,GAAG,SAAS,CAE/B;IAED,6FAA6F;IAC7F,eAAe,CAAC,QAAQ,EAAE,kBAAkB,GAAG,iBAAiB,GAAG,SAAS,CAE3E;IAED;;;;OAIG;IACH,gCAAgC,CAC9B,QAAQ,EAAE,kBAAkB,EAC5B,QAAQ,EAAE,MAAM,GACf,IAAI,CAaN;IAED,yBAAyB,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI,CAM9C;IAED,gBAAgB,CAAC,IAAI,EAAE,YAAY,GAAG,aAAa,GAAG,IAAI,CAezD;IAED,iEAAiE;IACjE,oBAAoB,CAClB,QAAQ,EAAE,kBAAkB,GAC3B,QAAQ,CAAC,iBAAiB,CAAC,GAAG,SAAS,CAEzC;IAED,gFAAgF;IAChF,wBAAwB,CACtB,QAAQ,EAAE,kBAAkB,GAC3B,QAAQ,CAAC,iBAAiB,CAAC,CAE7B;IAED,gEAAgE;IAChE,8BAA8B,IAAI,QAAQ,CAAC,iBAAiB,CAAC,GAAG,SAAS,CAExE;IAED,kCAAkC,IAAI,QAAQ,CAAC,iBAAiB,CAAC,CAchE;IAED,2CAA2C;IAC3C,aAAa,IAAI,QAAQ,CAAC,cAAc,CAAC,CAExC;IAED;;;;;;;OAOG;IACH,eAAe,IAAI;QACjB,gBAAgB,EAAE,QAAQ,CAAC,iBAAiB,CAAC,EAAE,CAAC;QAChD,cAAc,EAAE,OAAO,CAAC;QACxB,kBAAkB,EAAE,MAAM,GAAG,SAAS,CAAC;KACxC,CA8CA;CAiNF"}
+{"version":3,"file":"connection-context-manager.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/view-syncer/connection-context-manager.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,EAAC,kBAAkB,EAAC,MAAM,0CAA0C,CAAC;AAGjF,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,8CAA8C,CAAC;AACjF,OAAO,EAGL,KAAK,IAAI,EACT,KAAK,iBAAiB,EACvB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,6BAA6B,CAAC;AAG5D,OAAO,KAAK,EAAC,aAAa,EAAC,MAAM,iCAAiC,CAAC;AAEnE,MAAM,MAAM,eAAe,GAAG,aAAa,GAAG,WAAW,CAAC;AAE1D;;;GAGG;AACH,MAAM,MAAM,SAAS,GAAG;IAAC,QAAQ,CAAC,EAAE,EAAE,MAAM,GAAG,IAAI,CAAA;CAAC,CAAC;AAErD;;;;;;;;;;;;;;GAcG;AACH,MAAM,MAAM,oBAAoB,GAC5B;IAAC,IAAI,EAAE,iBAAiB,CAAA;CAAC,GACzB;IAAC,IAAI,EAAE,mBAAmB,CAAA;CAAC,GAC3B;IAAC,IAAI,EAAE,kBAAkB,CAAC;IAAC,eAAe,EAAE,MAAM,GAAG,IAAI,CAAA;CAAC,CAAC;AAE/D;;GAEG;AACH,MAAM,MAAM,kBAAkB,GAAG;IAC/B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;CACvB,CAAC;AAEF,KAAK,WAAW,GAAG,UAAU,CAAC,OAAO,CAAC,CAAC;AAEvC,MAAM,MAAM,aAAa,GAAG;IAC1B,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACrC,0DAA0D;IAC1D,QAAQ,CAAC,aAAa,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,GAAG,SAAS,CAAC;IACtE,0DAA0D;IAC1D,QAAQ,CAAC,cAAc,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,GAAG,SAAS,CAAC;IACvE,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACrC,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACtC,CAAC;AAEF,MAAM,MAAM,sBAAsB,GAAG;IACnC,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG,SAAS,CAAC;IACjC,QAAQ,CAAC,kBAAkB,EAAE,SAAS,UAAU,EAAE,GAAG,SAAS,CAAC;IAC/D,QAAQ,CAAC,aAAa,EAAE,aAAa,CAAC;CACvC,CAAC;AAEF;;;;GAIG;AACH,MAAM,MAAM,iBAAiB,GAAG;IAC9B,QAAQ,CAAC,KAAK,EAAE,eAAe,CAAC;IAEhC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC;IAEzB,QAAQ,CAAC,IAAI,EAAE,IAAI,GAAG,SAAS,CAAC;IAEhC,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IAEjC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAE1B,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,SAAS,CAAC;IAE1C,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAEhC,QAAQ,CAAC,YAAY,EAAE,sBAAsB,CAAC;IAC9C,QAAQ,CAAC,aAAa,EAAE,sBAAsB,CAAC;CAChD,CAAC;AAEF;;;;;;GAMG;AACH,MAAM,MAAM,cAAc,GAAG;IAC3B,QAAQ,CAAC,UAAU,EAAE,SAAS,GAAG,SAAS,CAAC;IAE3C,QAAQ,CAAC,oBAAoB,EAAE,kBAAkB,GAAG,SAAS,CAAC;IAC9D,QAAQ,CAAC,aAAa,EAAE,MAAM,GAAG,SAAS,CAAC;IAE3C,QAAQ,CAAC,sBAAsB,EAAE,MAAM,GAAG,SAAS,CAAC;CACrD,CAAC;AAEF,MAAM,MAAM,wBAAwB,GAAG;IACrC,kBAAkB,CAChB,QAAQ,EAAE,kBAAkB,EAC5B,aAAa,EAAE,aAAa,EAC5B,IAAI,CAAC,EAAE,IAAI,GACV,QAAQ,CAAC,iBAAiB,CAAC,CAAC;IAE/B,cAAc,CACZ,QAAQ,EAAE,kBAAkB,EAC5B,IAAI,EAAE,kBAAkB,GACvB,QAAQ,CAAC,iBAAiB,CAAC,CAAC;IAE/B,UAAU,CACR,QAAQ,EAAE,kBAAkB,EAC5B,IAAI,EAAE,cAAc,GACnB,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAC,CAAC;IAExC,kBAAkB,CAChB,QAAQ,EAAE,kBAAkB,EAC5B,QAAQ,EAAE,MAAM,EAChB,UAAU,EAAE,oBAAoB,GAE9B,QAAQ,CAAC;QACP,UAAU,EAAE,iBAAiB,CAAC;QAC9B,KAAK,EAAE,cAAc,CAAC;KACvB,CAAC,GACF,SAAS,CAAC;IAEd,iBAAiB,CAAC,UAAU,EAAE,SAAS,GAAG,IAAI,CAAC;IAE/C,cAAc,CACZ,QAAQ,EAAE,kBAAkB,EAC5B,QAAQ,EAAE,MAAM,GACf,QAAQ,CAAC,iBAAiB,CAAC,GAAG,SAAS,CAAC;IAC3C,eAAe,CACb,QAAQ,EAAE,kBAAkB,GAC3B,QAAQ,CAAC,iBAAiB,CAAC,GAAG,SAAS,CAAC;IAE3C,gCAAgC,CAC9B,QAAQ,EAAE,kBAAkB,EAC5B,QAAQ,EAAE,MAAM,GACf,IAAI,CAAC;IAER,yBAAyB,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI,CAAC;IAEhD,gBAAgB,CAAC,IAAI,EAAE,YAAY,GAAG,aAAa,GAAG,IAAI,CAAC;IAE3D,oBAAoB,CAClB,QAAQ,EAAE,kBAAkB,GAC3B,QAAQ,CAAC,iBAAiB,CAAC,GAAG,SAAS,CAAC;IAC3C,wBAAwB,CACtB,QAAQ,EAAE,kBAAkB,GAC3B,QAAQ,CAAC,iBAAiB,CAAC,CAAC;IAE/B,8BAA8B,IAAI,QAAQ,CAAC,iBAAiB,CAAC,GAAG,SAAS,CAAC;IAC1E,kCAAkC,IAAI,QAAQ,CAAC,iBAAiB,CAAC,CAAC;IAElE,aAAa,IAAI,QAAQ,CAAC,cAAc,CAAC,CAAC;IAE1C,eAAe,IAAI;QACjB,gBAAgB,EAAE,QAAQ,CAAC,iBAAiB,CAAC,EAAE,CAAC;QAChD,cAAc,EAAE,OAAO,CAAC;QACxB,kBAAkB,EAAE,MAAM,GAAG,SAAS,CAAC;KACxC,CAAC;CACH,CAAC;AAEF;;;;;;;GAOG;AACH,qBAAa,4BAA6B,YAAW,wBAAwB;;IAwB3E,YACE,EAAE,EAAE,UAAU,EACd,yBAAyB,CAAC,EAAE,MAAM,EAClC,0BAA0B,CAAC,EAAE,MAAM,EACnC,WAAW,CAAC,EAAE,WAAW,EACzB,UAAU,CAAC,EAAE,WAAW,EACxB,iBAAiB,CAAC,EAAE,iBAAiB,EACrC,GAAG,CAAC,EAAE,MAAM,MAAM,EAClB,mBAAmB,UAAQ,EAgB5B;IAED;;;;;OAKG;IACH,kBAAkB,CAChB,QAAQ,EAAE,kBAAkB,EAC5B,aAAa,EAAE,aAAa,EAC5B,IAAI,CAAC,EAAE,IAAI,GACV,QAAQ,CAAC,iBAAiB,CAAC,CA8C7B;IAED;;;;;OAKG;IACH,cAAc,CACZ,QAAQ,EAAE,kBAAkB,EAC5B,IAAI,EAAE,kBAAkB,GACvB,QAAQ,CAAC,iBAAiB,CAAC,CAiD7B;IAED;;;OAGG;IACG,UAAU,CACd,QAAQ,EAAE,kBAAkB,EAC5B,IAAI,EAAE,cAAc,GACnB,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAC,CA4BtC;IAED;;;;;;;;OAQG;IACH,kBAAkB,CAChB,QAAQ,EAAE,kBAAkB,EAC5B,QAAQ,EAAE,MAAM,EAChB,UAAU,EAAE,oBAAoB,GAE9B,QAAQ,CAAC;QACP,UAAU,EAAE,iBAAiB,CAAC;QAC9B,KAAK,EAAE,cAAc,CAAC;KACvB,CAAC,GACF,SAAS,CAwGZ;IAED;;;;;;;;;;OAUG;IACH,iBAAiB,CAAC,UAAU,EAAE,SAAS,GAAG,IAAI,CAgB7C;IAED,mGAAmG;IACnG,cAAc,CACZ,QAAQ,EAAE,kBAAkB,EAC5B,QAAQ,EAAE,MAAM,GACf,iBAAiB,GAAG,SAAS,CAE/B;IAED,6FAA6F;IAC7F,eAAe,CAAC,QAAQ,EAAE,kBAAkB,GAAG,iBAAiB,GAAG,SAAS,CAE3E;IAED;;;;OAIG;IACH,gCAAgC,CAC9B,QAAQ,EAAE,kBAAkB,EAC5B,QAAQ,EAAE,MAAM,GACf,IAAI,CAaN;IAED,yBAAyB,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI,CAM9C;IAED,gBAAgB,CAAC,IAAI,EAAE,YAAY,GAAG,aAAa,GAAG,IAAI,CAezD;IAED,iEAAiE;IACjE,oBAAoB,CAClB,QAAQ,EAAE,kBAAkB,GAC3B,QAAQ,CAAC,iBAAiB,CAAC,GAAG,SAAS,CAEzC;IAED,gFAAgF;IAChF,wBAAwB,CACtB,QAAQ,EAAE,kBAAkB,GAC3B,QAAQ,CAAC,iBAAiB,CAAC,CAE7B;IAED,gEAAgE;IAChE,8BAA8B,IAAI,QAAQ,CAAC,iBAAiB,CAAC,GAAG,SAAS,CAExE;IAED,kCAAkC,IAAI,QAAQ,CAAC,iBAAiB,CAAC,CAchE;IAED,2CAA2C;IAC3C,aAAa,IAAI,QAAQ,CAAC,cAAc,CAAC,CAExC;IAED;;;;;;;OAOG;IACH,eAAe,IAAI;QACjB,gBAAgB,EAAE,QAAQ,CAAC,iBAAiB,CAAC,EAAE,CAAC;QAChD,cAAc,EAAE,OAAO,CAAC;QACxB,kBAAkB,EAAE,MAAM,GAAG,SAAS,CAAC;KACxC,CA8CA;CAiNF"}