@rocicorp/zero 1.11.0-canary.35 → 1.11.0-canary.36

package/out/zero-cache/src/services/change-source/pg/replication-slots.js.map

@@ -1 +1 @@
-{"version":3,"file":"replication-slots.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/pg/replication-slots.ts"],"sourcesContent":["import type {Readable} from 'node:stream';\nimport {\n  PG_CONFIGURATION_LIMIT_EXCEEDED,\n  PG_INSUFFICIENT_PRIVILEGE,\n} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport type postgres from 'postgres';\nimport type {JSONObject} from '../../../../../shared/src/bigint-json.ts';\nimport {must} from '../../../../../shared/src/must.ts';\nimport {runTx} from '../../../db/run-transaction.ts';\nimport {PG_17} from '../../../types/pg-versions.ts';\nimport {isPostgresError, type PostgresDB} from '../../../types/pg.ts';\nimport {upstreamSchema, type ShardID} from '../../../types/shards.ts';\nimport {orTimeout} from '../../../types/timeout.ts';\nimport {AutoResetSignal} from '../../change-streamer/schema/tables.ts';\nimport {\n  createReplicationSessionFor,\n  endSession,\n  extractConnectionConfig,\n  keepSlotActiveUntilTakenOver,\n} from './logical-replication/stream.ts';\nimport {toBigInt, toStateVersionString} from './lsn.ts';\nimport {\n  InitialSync,\n  Restore,\n  type ReplicaStage,\n} from './schema/replica-stage-enum.ts';\nimport {\n  createReplica,\n  getReplicaState,\n  getRestoreCandidates,\n  metadataPublicationName,\n  replicationSlotExpression,\n  replicationSlotPrefix,\n  type BackupOptions,\n  type ReplicaState,\n} from './schema/shard.ts';\n\n// Record returned by `CREATE_REPLICATION_SLOT`\nexport type ReplicationSlot = {\n  slot_name: string;\n  consistent_point: string;\n  snapshot_name: string;\n  output_plugin: string;\n};\n\nexport type ReplicationSlotResult<T> = {\n  slot: ReplicationSlot;\n  capturedSnapshot: T;\n  initialSession: Readable;\n  replica: ReplicaState;\n};\n\nexport type ReservedSlot = {\n  slot: string;\n  reservation: Readable;\n  replica: ReplicaState;\n};\n\nexport type CreateSlotSpec = {\n  slotName: string;\n\n  // Note: ignored if pgVersion < PG_17.\n  failover?: boolean;\n\n  // Create a temporary slot (i.e. not persisted, and automatically\n  // cleaned up when the replication session ends).\n  temporary?: boolean;\n\n  // For overriding in tests.\n  lockTimeout?: number;\n\n  // A (non-replication) connection to the same upstream instance, used to\n  // log what slot creation was waiting on if it times out.\n  diagnosticsDB?: PostgresDB | undefined;\n};\n\n// When creating a replication slot, Postgres waits for open transactions\n// to complete before reserving a consistent_point (LSN) in the WAL and creating\n// a matching transaction snapshot. As such, it can technically take an arbitrary\n// amount of time (e.g. DDL operations, table-wide operations, etc.).\n//\n// However, to detect pathological situations, bound the amount of time that\n// the server waits for replication slot creation, so that a continual failure to\n// create a replication slot is surfaced by errors / alerts.\nexport const CREATE_REPLICATION_SLOT_TIMEOUT_MS = 60_000;\n\n// The lock_timeout is set 1s before the client-side orTimeout so that\n// Postgres reliably aborts first and tears down the walsender cleanly.\n// The client-side timeout remains as a fallback for network-level failures.\nconst SERVER_LOCK_TIMEOUT_MS = CREATE_REPLICATION_SLOT_TIMEOUT_MS - 1_000;\n\n// Note: The replication connection does not support the extended query protocol,\n//       so all commands must be sent using sql.unsafe(). This is technically safe\n//       because all placeholder values are under our control (i.e. \"slotName\").\nexport async function createReplicationSlot(\n  lc: LogContext,\n  session: postgres.Sql,\n  {\n    slotName,\n    failover,\n    temporary,\n    lockTimeout = SERVER_LOCK_TIMEOUT_MS,\n    diagnosticsDB,\n  }: CreateSlotSpec,\n): Promise<ReplicationSlot> {\n  // CREATE_REPLICATION_SLOT can hang indefinitely waiting for long-running\n  // transactions to finish: internally it calls SnapBuildWaitSnapshot →\n  // XactLockTableWait → LockAcquire on each running XID. statement_timeout\n  // does NOT apply to replication commands, but lock_timeout does (it governs\n  // the heavyweight lock wait inside LockAcquire). Setting it here causes\n  // Postgres to raise ERRCODE_LOCK_NOT_AVAILABLE and cleanly tear down the\n  // walsender, rather than relying solely on the client-side orTimeout\n  // which can leave an orphaned backend.\n  //\n  // An orphaned walsender is actively harmful: by this point the replication\n  // slot has already been created and is pinning WAL retention and catalog_xmin.\n  // Worse, the slot is marked `active` (the walsender PID is still alive), so\n  // the existing cleanup code (which drops inactive slots on retry) can't\n  // reclaim it. Without lock_timeout the orphan persists until TCP keepalive\n  // fires (~2h default) or the blocking transaction finishes.\n  await session.unsafe(`SET lock_timeout = ${lockTimeout}`);\n\n  const {pgVersion, pid: walsenderPID} = (\n    await session.unsafe<{pgVersion: number; pid: number}[]>(`\n      SELECT current_setting('server_version_num') as \"pgVersion\",\n             pg_backend_pid() as \"pid\";\n  `)\n  )[0];\n\n  const maybeTemporary = temporary ? 'TEMPORARY' : '';\n  const options = failover && pgVersion >= PG_17 ? '(FAILOVER)' : '';\n  const createSlot = session.unsafe<ReplicationSlot[]>(/*sql*/ `\n    CREATE_REPLICATION_SLOT \"${slotName}\" ${maybeTemporary} LOGICAL pgoutput ${options}`);\n\n  const raced = await orTimeout(createSlot, CREATE_REPLICATION_SLOT_TIMEOUT_MS);\n  if (raced === 'timed-out') {\n    if (diagnosticsDB) {\n      await logSlotCreationBlockers(lc, diagnosticsDB, walsenderPID, slotName);\n    }\n    // Create slot can block indefinitely waiting for old transactions. End\n    // this connection in the background and fail fast so the process restarts.\n    void session\n      .end()\n      .catch(e =>\n        lc.warn?.(`Error closing timed out replication slot session`, e),\n      );\n    throw new Error(\n      `Timed out after ${CREATE_REPLICATION_SLOT_TIMEOUT_MS} ms creating replication slot ${slotName}.`,\n    );\n  }\n  const [slot] = raced;\n  lc.info?.(`Created replication slot ${slotName}`, slot);\n  return slot;\n}\n\nconst SLOT_DIAGNOSTICS_TIMEOUT_MS = 5_000;\nconst SLOT_DIAGNOSTICS_MAX_TRANSACTIONS = 20;\n\n/**\n * Best-effort logging of what a timed out CREATE_REPLICATION_SLOT was waiting\n * on: the walsender's wait event and blocking pids, and the oldest\n * transactions holding an xid. Slot creation waits for every such transaction\n * on the instance (in all databases), each with its own lock_timeout, so a\n * chain of overlapping transactions can exceed the client-side timeout\n * without the lock_timeout ever firing.\n *\n * Query text is deliberately omitted since it can contain row values.\n *\n * Exported for testing.\n */\nexport async function logSlotCreationBlockers(\n  lc: LogContext,\n  sql: PostgresDB,\n  walsenderPID: number,\n  slotName: string,\n) {\n  try {\n    const result = await orTimeout(\n      Promise.all([\n        sql<JSONObject[]> /*sql*/ `\n          SELECT pid, wait_event_type as \"waitEventType\", wait_event as \"waitEvent\",\n                 pg_blocking_pids(pid) as \"blockingPIDs\"\n            FROM pg_stat_activity WHERE pid = ${walsenderPID}`,\n        sql<JSONObject[]> /*sql*/ `\n          SELECT pid, datname, usename, application_name as \"applicationName\",\n                 backend_type as \"backendType\", state,\n                 wait_event_type as \"waitEventType\", wait_event as \"waitEvent\",\n                 backend_xid::text as xid,\n                 xact_start::text as \"xactStart\",\n                 extract(epoch from now() - xact_start)::float8 as \"xactAgeSec\"\n            FROM pg_stat_activity\n            WHERE backend_xid IS NOT NULL AND pid <> ${walsenderPID}\n            ORDER BY xact_start NULLS LAST\n            LIMIT ${SLOT_DIAGNOSTICS_MAX_TRANSACTIONS}`,\n      ]),\n      SLOT_DIAGNOSTICS_TIMEOUT_MS,\n    );\n    if (result === 'timed-out') {\n      lc.warn?.(`Timed out collecting diagnostics for slot ${slotName}`);\n      return;\n    }\n    const [[walsender], transactions] = result;\n    lc.warn?.(`Replication slot ${slotName} creation blocked`, {\n      walsender: walsender ?? null,\n      transactions,\n    });\n  } catch (e) {\n    lc.warn?.(`Unable to collect diagnostics for slot ${slotName}`, e);\n  }\n}\n\n/**\n * Replica and slot creation involves two sessions for proper coordination\n * with other replica management logic:\n *\n * * A normal transaction is started and acquires an advisory lock for\n *   replica slot management. This is the same lock that cleanup logic\n *   acquires before cleaning up replication slots.\n * * With the lock held, a new replication slot is created in a\n *   replication session. The API of CREATE_REPLICATION_SLOT is such\n *   that it cannot be done in a transaction, and cannot be followed by\n *   any writes, or else its snapshot (which is needed for initial sync)\n *   would be invalidated.\n * * Once the slot is created, the slot and replica information are recorded\n *   in the `replicas` table before releasing the lock.\n *\n * This locking ensures that:\n * 1. multiple replication managers attempting to create a replication slot\n *    will not use the same name for the replication slot (which is selected\n *    from a pool of reused names).\n * 2. Running replication managers (which use an earlier replica of a lower\n *    rank) will not delete the new slot during their cleanup logic, since\n *    the slot will belong to a replica of a higher rank.\n *\n * When the replication slot is created, `captureSnapshot` callback is first\n * run while the snapshot at the consistent point is available. The callback\n * must capture the snapshot in postgres (e.g. `SET TRANSACTION SNAPSHOT`)\n * in order to use it. Once the callback completes, a placeholder replication\n * session (i.e. that does not consume any messages) is started, in order to\n * reserve the slot by mark it \"active\", effectively distinguishing the\n * initializing session from a failed or abandoned session.\n *\n * Note that starting the replication session releases the initial snapshot,\n * which is why the callback must capture it synchronously, before the\n * replication session is started.\n *\n * The placeholder replication session is closed when the\n * PostgresChangeSource takes over the slot to process the stream in earnest.\n * It is also returned for cleanup in the case of failures (and for control\n * in unit tests).\n *\n * For replica's created in the `Restore` stage, the `generation` is\n * initialized to an empty string instead of the slot's consistent point,\n * since the slot is not being used for initial sync. It is the\n * responsibiliy of the caller to call {@link initRestoreReplica} when\n * the source replica has been selected for restoring, which will carry\n * over the `generation` and other relevant values from the source.\n */\nexport async function createReplicaAndSlot<T>(\n  lc: LogContext,\n  sql: PostgresDB,\n  sessionName: string,\n  shard: ShardID,\n  epoch: number,\n  replicaID: string,\n  failover: boolean,\n  backupOptions: BackupOptions,\n  captureSnapshot: (snapshot: string) => Promise<T>,\n  stage: ReplicaStage,\n): Promise<ReplicationSlotResult<T>> {\n  // Note: The replicationSession is used to create the replication slot\n  // and closed immediately afterwards, or on error.\n  const replicationSession = createReplicationSessionFor(sql, sessionName);\n  const lockName = replicationSlotManagementLock(shard);\n  const slotPoolPrefix = replicationSlotPrefix(shard);\n\n  for (let first = true; ; first = false) {\n    await dropUnclaimedSlots(lc, sql, shard);\n\n    let slotName: string | undefined;\n    let initialSession: Readable | undefined;\n    try {\n      return await runTx(sql, async tx => {\n        await tx`SELECT pg_advisory_xact_lock(hashtext(${lockName}))`;\n\n        if (stage === InitialSync) {\n          // With the lock acquired, ensure that only one initial sync is\n          // active at a time. getRestoreCandidates() orders its results\n          // with stage = InitialSync (and active = true) first.\n          const others = await getRestoreCandidates(tx, shard, epoch);\n          lc.info?.(`current replicas at epoch ${epoch}`, {replicas: others});\n          if (others.length) {\n            const [{id, stage, active}] = others;\n            if (stage === InitialSync && active) {\n              throw new AutoResetSignal(\n                `another replica (${id}) is performing initial sync`,\n              );\n            }\n          }\n        }\n\n        // Pick an available slotName from the slotPoolPrefix pool.\n        const names = await tx<{name: string}[]> /*sql*/ `\n          SELECT slot_name as name FROM pg_replication_slots\n            WHERE slot_name LIKE ${slotPoolPrefix + '%'};\n        `.values();\n        const inUse = new Set(names.flat());\n        for (let next = 0; ; next++) {\n          const candidateName = `${slotPoolPrefix}${slotPoolSuffix(next)}`;\n          if (!inUse.has(candidateName)) {\n            slotName = candidateName;\n            break;\n          }\n        }\n\n        const slot = await createReplicationSlot(lc, replicationSession, {\n          slotName,\n          failover,\n          diagnosticsDB: sql,\n        });\n        const capturedSnapshot = await captureSnapshot(slot.snapshot_name);\n        await replicationSession.end();\n        initialSession = await keepSlotActiveUntilTakenOver(lc, {\n          db: extractConnectionConfig(sql),\n          slot: slot.slot_name,\n          dummyPublication: metadataPublicationName(\n            shard.appID,\n            shard.shardNum,\n          ),\n          lsn: String(toBigInt(slot.consistent_point)),\n        });\n\n        await createReplica(\n          tx,\n          shard,\n          replicaID,\n          slot.slot_name,\n          epoch,\n          stage === InitialSync\n            ? toStateVersionString(slot.consistent_point)\n            : '', // initialized with initRestoreReplica\n          backupOptions,\n          stage,\n        );\n\n        const replica = must(\n          await getReplicaState(tx, shard, replicaID),\n          `replica ${replicaID} was not created`,\n        );\n\n        return {slot, capturedSnapshot, initialSession, replica};\n      });\n    } catch (e) {\n      if (first && isPostgresError(e, PG_INSUFFICIENT_PRIVILEGE)) {\n        // Some Postgres variants (e.g. Google Cloud SQL) require that\n        // the user have the REPLICATION role in order to create a slot.\n        // Note that this must be done by the upstreamDB connection, and\n        // does not work in the replicationSession itself.\n        await sql`ALTER ROLE current_user WITH REPLICATION`;\n        lc.info?.(`Added the REPLICATION role to database user`);\n        continue;\n      }\n      // Note: This is currently manually tested since max_replication_slots\n      //       is a PG startup parameter that other tests depend on.\n      // TODO: Figure out a way to unit test this (with the full PG setup).\n      if (first && isPostgresError(e, PG_CONFIGURATION_LIMIT_EXCEEDED)) {\n        lc.warn?.(\n          `Reached max replication slots. Attempting to clean up unused slots`,\n          e,\n        );\n        // Drop any inactive replicas from failed initial syncs (e.g. inactive slots).\n        const replicasTable = `${upstreamSchema(shard)}.replicas`;\n        await sql`\n          DELETE FROM ${sql(replicasTable)} USING pg_replication_slots slots\n            WHERE replicas.slot = slots.slot_name AND NOT slots.active`;\n        continue; // then let dropUnclaimedSlots() perform its cleanup\n      }\n      if (initialSession) {\n        try {\n          await endSession(initialSession);\n        } catch {}\n      }\n      // Otherwise, clean up any created slot if something went wrong. The\n      // slot may not exist (e.g. if its creation failed), and its name may\n      // have been reused since the lock was released, so leave it to\n      // dropUnclaimedSlots(), which only drops slots without a replica.\n      if (slotName) {\n        lc.warn?.(`deleting slot ${slotName} due to error`, e);\n        try {\n          await replicationSession.end();\n        } catch {}\n        await dropUnclaimedSlots(lc, sql, shard);\n      }\n      throw e;\n    }\n  }\n}\n\n/**\n * Claims the inactive slot of the given replica for use by this task, opening\n * a replication session to mark it as active. The process should follow up by\n * taking over the slot via the PostgresChangeSource.\n *\n * The replica is identified by its ID (and not just its slot) because rows of\n * replicas whose slots were dropped outside of zero-cache's control may share\n * the slot name.\n */\nexport async function claimSlotForResumption(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardID,\n  _sessionName: string,\n  {id: replicaID, slot}: Pick<ReplicaState, 'id' | 'slot'>,\n): Promise<ReservedSlot | null> {\n  const lockName = replicationSlotManagementLock(shard);\n  const replicasTable = `${upstreamSchema(shard)}.replicas`;\n\n  let reservation: Readable | undefined;\n  try {\n    const reserved = await runTx(sql, async tx => {\n      await tx`SELECT pg_advisory_xact_lock(hashtext(${lockName}))`;\n\n      const replicas = await tx<{active: boolean; lsn: string}[]> /*sql*/ `\n      SELECT active, confirmed_flush_lsn as lsn\n        FROM pg_replication_slots\n        JOIN ${tx(replicasTable)} replica on slot_name = slot\n        WHERE slot_name = ${slot} AND replica.id = ${replicaID};\n    `;\n      if (replicas.length === 0) {\n        lc.warn?.(`no replica ${replicaID} found for slot ${slot}`);\n        return null;\n      }\n      const [{active, lsn}] = replicas;\n      if (active) {\n        lc.warn?.(\n          `replica ${replicaID} for slot ${slot} is active and cannot be claimed`,\n        );\n        return null;\n      }\n      // Mark the replica as Restoring.\n      await tx`\n        UPDATE ${tx(replicasTable)} SET \"stage\" = ${Restore} WHERE id = ${replicaID};\n      `;\n      const replica = must(\n        await getReplicaState(tx, shard, replicaID),\n        `replica ${replicaID} disappeared`,\n      );\n\n      reservation = await keepSlotActiveUntilTakenOver(lc, {\n        db: extractConnectionConfig(sql),\n        slot,\n        dummyPublication: metadataPublicationName(shard.appID, shard.shardNum),\n        lsn: String(toBigInt(lsn)),\n      });\n      return {\n        slot,\n        replica,\n        reservation,\n      };\n    });\n    lc.info?.(`successfully claimed slot ${slot}`);\n    return reserved;\n  } catch (e) {\n    if (reservation) {\n      try {\n        await endSession(reservation);\n      } catch {}\n    }\n    lc.error?.(`unable to claim replication slot ${slot}`, e);\n    return null;\n  }\n}\n\nexport function dropInactiveSlotsAndReplicas(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardID,\n  slots: string[],\n) {\n  const lockName = replicationSlotManagementLock(shard);\n  const replicasTable = `${upstreamSchema(shard)}.replicas`;\n\n  return runTx(sql, async tx => {\n    await tx`SELECT pg_advisory_xact_lock(hashtext(${lockName}))`;\n\n    const dropped = await tx<{slot: string; replicaID: string | null}[]>\n    /*sql*/ `\n      SELECT slot_name as slot, replica.id as \"replicaID\", pg_drop_replication_slot(slot_name) \n        FROM pg_replication_slots\n        LEFT JOIN ${tx(replicasTable)} replica on slot_name = slot\n        WHERE slot_name IN ${tx(slots)} AND NOT active;\n    `;\n    if (dropped.length) {\n      lc.info?.(`dropped ${dropped.length} inactive replication slot(s)`, {\n        dropped,\n      });\n    }\n\n    const replicas = dropped\n      .map(({replicaID}) => replicaID)\n      .filter(id => id !== null);\n    if (replicas.length) {\n      // Delete replicas associated with the inactive (and now dropped) slots.\n      await tx\n      /*sql*/ `DELETE FROM ${tx(replicasTable)} WHERE id IN ${tx(replicas)}`;\n      lc.info?.(`deleted ${replicas.length} old replica(s)`, {replicas});\n    }\n  });\n}\n\nfunction dropUnclaimedSlots(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardID,\n): Promise<{dropped: number; active: number; draining: number}> {\n  // The slot / replica cleanup happens within a transaction while holding\n  // the replication slot management lock for this shard, to ensure that no\n  // slot that belongs to a newer replica is dropped.\n  const lockName = replicationSlotManagementLock(shard);\n  const slotExpression = replicationSlotExpression(shard);\n  const replicasTable = `${upstreamSchema(shard)}.replicas`;\n\n  return runTx(sql, async tx => {\n    await tx`SELECT pg_advisory_xact_lock(hashtext(${lockName}))`;\n\n    const dropped = await tx /*sql*/ `\n      SELECT slot_name as slot, pg_drop_replication_slot(slot_name) \n        FROM pg_replication_slots\n        LEFT JOIN ${tx(replicasTable)} replica on slot_name = slot\n        WHERE slot_name LIKE ${slotExpression} \n          AND NOT active\n          AND replica.id IS NULL;\n    `;\n    if (dropped.length) {\n      lc.info?.(`dropped inactive replication slots`, {dropped});\n    }\n\n    // Conversely, delete replicas whose slots no longer exist, e.g. because\n    // they were dropped outside of zero-cache's control (such as in an\n    // upstream failover). Slot names are reused, so such rows would otherwise\n    // be associated with a new slot of the same name. This is done (and\n    // committed) before the slot is created, both so that the rows are never\n    // associated with it, and because CREATE_REPLICATION_SLOT waits for\n    // transactions that have written (i.e. been assigned an xid).\n    const orphaned = await tx<{id: string}[]> /*sql*/ `\n      DELETE FROM ${tx(replicasTable)} replica\n        WHERE NOT EXISTS (\n          SELECT 1 FROM pg_replication_slots WHERE slot_name = replica.slot\n        )\n        RETURNING id;\n    `;\n    if (orphaned.length) {\n      lc.warn?.(`deleted replicas whose slots no longer exist`, {\n        replicas: orphaned.map(({id}) => id),\n      });\n    }\n\n    const remaining = await tx<\n      {slot: string; pid: number | null; id: string | null}[]\n    > /*sql*/ `\n      SELECT slot_name as slot, active_pid as pid, replica.id as id\n        FROM pg_replication_slots\n        LEFT JOIN ${tx(replicasTable)} replica on slot_name = slot\n        WHERE slot_name LIKE ${slotExpression};\n    `;\n    if (remaining.length) {\n      lc.info?.(`remaining replication slots`, {remaining});\n    }\n\n    let active = 0;\n    let draining = 0;\n    for (const {id} of remaining) {\n      if (id === null) {\n        draining++;\n      } else {\n        active++;\n      }\n    }\n\n    return {\n      dropped: dropped.length,\n      active,\n      draining,\n    };\n  });\n}\n\nconst ALPHABET = 'abcdefghijklmnopqrstuvwxyz';\n\n// Alphabetic notation is used as the slot pool suffix to distinguish\n// it from the (numeric) shard num that's also encoded in the slot name.\nexport function slotPoolSuffix(n: number) {\n  n++; // Adjust for 0-based indexing\n\n  let suffix = '';\n  while (n > 0) {\n    n--;\n    suffix = ALPHABET[n % 26] + suffix;\n    n = Math.floor(n / 26);\n  }\n  return suffix;\n}\n\nfunction replicationSlotManagementLock(shard: ShardID) {\n  return `replication-slot-management:${shard.appID}_${shard.shardNum}`;\n}\n"],"mappings":";;;;;;;;;;;;;AAqFA,IAAa,qCAAqC;AAKlD,IAAM,yBAAyB,qCAAqC;AAKpE,eAAsB,sBACpB,IACA,SACA,EACE,UACA,UACA,WACA,cAAc,wBACd,iBAEwB;CAgB1B,MAAM,QAAQ,OAAO,sBAAsB,aAAa;CAExD,MAAM,EAAC,WAAW,KAAK,kBACrB,MAAM,QAAQ,OAA2C;;;GAG1D,GACC;CAEF,MAAM,iBAAiB,YAAY,cAAc;CACjD,MAAM,UAAU,YAAY,aAAA,OAAqB,eAAe;CAIhE,MAAM,QAAQ,MAAM,UAHD,QAAQ,OAAkC;+BAChC,SAAS,IAAI,eAAe,oBAAoB,SAE/C,GAAY,kCAAkC;CAC5E,IAAI,UAAU,aAAa;EACzB,IAAI,eACF,MAAM,wBAAwB,IAAI,eAAe,cAAc,QAAQ;EAIzE,QACG,IAAI,EACJ,OAAM,MACL,GAAG,OAAO,oDAAoD,CAAC,CACjE;EACF,MAAM,IAAI,MACR,mBAAmB,mCAAmC,gCAAgC,SAAS,EACjG;CACF;CACA,MAAM,CAAC,QAAQ;CACf,GAAG,OAAO,4BAA4B,YAAY,IAAI;CACtD,OAAO;AACT;AAEA,IAAM,8BAA8B;AACpC,IAAM,oCAAoC;;;;;;;;;;;;;AAc1C,eAAsB,wBACpB,IACA,KACA,cACA,UACA;CACA,IAAI;EACF,MAAM,SAAS,MAAM,UACnB,QAAQ,IAAI,CACV,GAA0B;;;gDAGc,gBACxC,GAA0B;;;;;;;;uDAQqB,aAAa;;oBAEhD,mCACd,CAAC,GACD,2BACF;EACA,IAAI,WAAW,aAAa;GAC1B,GAAG,OAAO,6CAA6C,UAAU;GACjE;EACF;EACA,MAAM,CAAC,CAAC,YAAY,gBAAgB;EACpC,GAAG,OAAO,oBAAoB,SAAS,oBAAoB;GACzD,WAAW,aAAa;GACxB;EACF,CAAC;CACH,SAAS,GAAG;EACV,GAAG,OAAO,0CAA0C,YAAY,CAAC;CACnE;AACF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAiDA,eAAsB,qBACpB,IACA,KACA,aACA,OACA,OACA,WACA,UACA,eACA,iBACA,OACmC;CAGnC,MAAM,qBAAqB,4BAA4B,KAAK,WAAW;CACvE,MAAM,WAAW,8BAA8B,KAAK;CACpD,MAAM,iBAAiB,sBAAsB,KAAK;CAElD,KAAK,IAAI,QAAQ,OAAQ,QAAQ,OAAO;EACtC,MAAM,mBAAmB,IAAI,KAAK,KAAK;EAEvC,IAAI;EACJ,IAAI;EACJ,IAAI;GACF,OAAO,MAAM,MAAM,KAAK,OAAM,OAAM;IAClC,MAAM,EAAE,yCAAyC,SAAS;IAE1D,IAAI,UAAA,GAAuB;KAIzB,MAAM,SAAS,MAAM,qBAAqB,IAAI,OAAO,KAAK;KAC1D,GAAG,OAAO,6BAA6B,SAAS,EAAC,UAAU,OAAM,CAAC;KAClE,IAAI,OAAO,QAAQ;MACjB,MAAM,CAAC,EAAC,IAAI,OAAO,YAAW;MAC9B,IAAI,UAAA,KAAyB,QAC3B,MAAM,IAAI,gBACR,oBAAoB,GAAG,6BACzB;KAEJ;IACF;IAGA,MAAM,QAAQ,MAAM,EAA6B;;mCAEtB,iBAAiB,IAAI;UAC9C,OAAO;IACT,MAAM,QAAQ,IAAI,IAAI,MAAM,KAAK,CAAC;IAClC,KAAK,IAAI,OAAO,IAAK,QAAQ;KAC3B,MAAM,gBAAgB,GAAG,iBAAiB,eAAe,IAAI;KAC7D,IAAI,CAAC,MAAM,IAAI,aAAa,GAAG;MAC7B,WAAW;MACX;KACF;IACF;IAEA,MAAM,OAAO,MAAM,sBAAsB,IAAI,oBAAoB;KAC/D;KACA;KACA,eAAe;IACjB,CAAC;IACD,MAAM,mBAAmB,MAAM,gBAAgB,KAAK,aAAa;IACjE,MAAM,mBAAmB,IAAI;IAC7B,iBAAiB,MAAM,6BAA6B,IAAI;KACtD,IAAI,wBAAwB,GAAG;KAC/B,MAAM,KAAK;KACX,kBAAkB,wBAChB,MAAM,OACN,MAAM,QACR;KACA,KAAK,OAAO,SAAS,KAAK,gBAAgB,CAAC;IAC7C,CAAC;IAED,MAAM,cACJ,IACA,OACA,WACA,KAAK,WACL,OACA,UAAA,IACI,qBAAqB,KAAK,gBAAgB,IAC1C,IACJ,eACA,KACF;IAEA,MAAM,UAAU,KACd,MAAM,gBAAgB,IAAI,OAAO,SAAS,GAC1C,WAAW,UAAU,iBACvB;IAEA,OAAO;KAAC;KAAM;KAAkB;KAAgB;IAAO;GACzD,CAAC;EACH,SAAS,GAAG;GACV,IAAI,SAAS,gBAAgB,GAAG,yBAAyB,GAAG;IAK1D,MAAM,GAAG;IACT,GAAG,OAAO,6CAA6C;IACvD;GACF;GAIA,IAAI,SAAS,gBAAgB,GAAG,+BAA+B,GAAG;IAChE,GAAG,OACD,sEACA,CACF;IAGA,MAAM,GAAG;wBACO,IAAI,GAFK,eAAe,KAAK,EAAE,UAEd,EAAE;;IAEnC;GACF;GACA,IAAI,gBACF,IAAI;IACF,MAAM,WAAW,cAAc;GACjC,QAAQ,CAAC;GAMX,IAAI,UAAU;IACZ,GAAG,OAAO,iBAAiB,SAAS,gBAAgB,CAAC;IACrD,IAAI;KACF,MAAM,mBAAmB,IAAI;IAC/B,QAAQ,CAAC;IACT,MAAM,mBAAmB,IAAI,KAAK,KAAK;GACzC;GACA,MAAM;EACR;CACF;AACF;;;;;;;;;;AAWA,eAAsB,uBACpB,IACA,KACA,OACA,cACA,EAAC,IAAI,WAAW,QACc;CAC9B,MAAM,WAAW,8BAA8B,KAAK;CACpD,MAAM,gBAAgB,GAAG,eAAe,KAAK,EAAE;CAE/C,IAAI;CACJ,IAAI;EACF,MAAM,WAAW,MAAM,MAAM,KAAK,OAAM,OAAM;GAC5C,MAAM,EAAE,yCAAyC,SAAS;GAE1D,MAAM,WAAW,MAAM,EAA6C;;;eAG3D,GAAG,aAAa,EAAE;4BACL,KAAK,oBAAoB,UAAU;;GAEzD,IAAI,SAAS,WAAW,GAAG;IACzB,GAAG,OAAO,cAAc,UAAU,kBAAkB,MAAM;IAC1D,OAAO;GACT;GACA,MAAM,CAAC,EAAC,QAAQ,SAAQ;GACxB,IAAI,QAAQ;IACV,GAAG,OACD,WAAW,UAAU,YAAY,KAAK,iCACxC;IACA,OAAO;GACT;GAEA,MAAM,EAAE;iBACG,GAAG,aAAa,EAAE,iBAAA,EAAyB,cAAc,UAAU;;GAE9E,MAAM,UAAU,KACd,MAAM,gBAAgB,IAAI,OAAO,SAAS,GAC1C,WAAW,UAAU,aACvB;GAEA,cAAc,MAAM,6BAA6B,IAAI;IACnD,IAAI,wBAAwB,GAAG;IAC/B;IACA,kBAAkB,wBAAwB,MAAM,OAAO,MAAM,QAAQ;IACrE,KAAK,OAAO,SAAS,GAAG,CAAC;GAC3B,CAAC;GACD,OAAO;IACL;IACA;IACA;GACF;EACF,CAAC;EACD,GAAG,OAAO,6BAA6B,MAAM;EAC7C,OAAO;CACT,SAAS,GAAG;EACV,IAAI,aACF,IAAI;GACF,MAAM,WAAW,WAAW;EAC9B,QAAQ,CAAC;EAEX,GAAG,QAAQ,oCAAoC,QAAQ,CAAC;EACxD,OAAO;CACT;AACF;AAEA,SAAgB,6BACd,IACA,KACA,OACA,OACA;CACA,MAAM,WAAW,8BAA8B,KAAK;CACpD,MAAM,gBAAgB,GAAG,eAAe,KAAK,EAAE;CAE/C,OAAO,MAAM,KAAK,OAAM,OAAM;EAC5B,MAAM,EAAE,yCAAyC,SAAS;EAE1D,MAAM,UAAU,MAAM,EACd;;;oBAGQ,GAAG,aAAa,EAAE;6BACT,GAAG,KAAK,EAAE;;EAEnC,IAAI,QAAQ,QACV,GAAG,OAAO,WAAW,QAAQ,OAAO,gCAAgC,EAClE,QACF,CAAC;EAGH,MAAM,WAAW,QACd,KAAK,EAAC,gBAAe,SAAS,EAC9B,QAAO,OAAM,OAAO,IAAI;EAC3B,IAAI,SAAS,QAAQ;GAEnB,MAAM,EACE,eAAe,GAAG,aAAa,EAAE,eAAe,GAAG,QAAQ;GACnE,GAAG,OAAO,WAAW,SAAS,OAAO,kBAAkB,EAAC,SAAQ,CAAC;EACnE;CACF,CAAC;AACH;AAEA,SAAS,mBACP,IACA,KACA,OAC8D;CAI9D,MAAM,WAAW,8BAA8B,KAAK;CACpD,MAAM,iBAAiB,0BAA0B,KAAK;CACtD,MAAM,gBAAgB,GAAG,eAAe,KAAK,EAAE;CAE/C,OAAO,MAAM,KAAK,OAAM,OAAM;EAC5B,MAAM,EAAE,yCAAyC,SAAS;EAE1D,MAAM,UAAU,MAAM,EAAW;;;oBAGjB,GAAG,aAAa,EAAE;+BACP,eAAe;;;;EAI1C,IAAI,QAAQ,QACV,GAAG,OAAO,sCAAsC,EAAC,QAAO,CAAC;EAU3D,MAAM,WAAW,MAAM,EAA2B;oBAClC,GAAG,aAAa,EAAE;;;;;;EAMlC,IAAI,SAAS,QACX,GAAG,OAAO,gDAAgD,EACxD,UAAU,SAAS,KAAK,EAAC,SAAQ,EAAE,EACrC,CAAC;EAGH,MAAM,YAAY,MAAM,EAEd;;;oBAGM,GAAG,aAAa,EAAE;+BACP,eAAe;;EAE1C,IAAI,UAAU,QACZ,GAAG,OAAO,+BAA+B,EAAC,UAAS,CAAC;EAGtD,IAAI,SAAS;EACb,IAAI,WAAW;EACf,KAAK,MAAM,EAAC,QAAO,WACjB,IAAI,OAAO,MACT;OAEA;EAIJ,OAAO;GACL,SAAS,QAAQ;GACjB;GACA;EACF;CACF,CAAC;AACH;AAEA,IAAM,WAAW;AAIjB,SAAgB,eAAe,GAAW;CACxC;CAEA,IAAI,SAAS;CACb,OAAO,IAAI,GAAG;EACZ;EACA,SAAS,SAAS,IAAI,MAAM;EAC5B,IAAI,KAAK,MAAM,IAAI,EAAE;CACvB;CACA,OAAO;AACT;AAEA,SAAS,8BAA8B,OAAgB;CACrD,OAAO,+BAA+B,MAAM,MAAM,GAAG,MAAM;AAC7D"}
+{"version":3,"file":"replication-slots.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/pg/replication-slots.ts"],"sourcesContent":["import type {Readable} from 'node:stream';\nimport {\n  PG_CONFIGURATION_LIMIT_EXCEEDED,\n  PG_INSUFFICIENT_PRIVILEGE,\n  PG_LOCK_NOT_AVAILABLE,\n} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport {defu} from 'defu';\nimport postgres, {type Options, type PostgresType} from 'postgres';\nimport type {JSONObject} from '../../../../../shared/src/bigint-json.ts';\nimport {must} from '../../../../../shared/src/must.ts';\nimport {sleep} from '../../../../../shared/src/sleep.ts';\nimport {runTx} from '../../../db/run-transaction.ts';\nimport {PG_17} from '../../../types/pg-versions.ts';\nimport {isPostgresError, type PostgresDB} from '../../../types/pg.ts';\nimport {upstreamSchema, type ShardID} from '../../../types/shards.ts';\nimport {orTimeout} from '../../../types/timeout.ts';\nimport {AutoResetSignal} from '../../change-streamer/schema/tables.ts';\nimport {\n  createReplicationSessionFor,\n  endSession,\n  extractConnectionConfig,\n  keepSlotActiveUntilTakenOver,\n} from './logical-replication/stream.ts';\nimport {toBigInt, toStateVersionString} from './lsn.ts';\nimport {\n  InitialSync,\n  Restore,\n  type ReplicaStage,\n} from './schema/replica-stage-enum.ts';\nimport {\n  createReplica,\n  getReplicaState,\n  getRestoreCandidates,\n  metadataPublicationName,\n  replicationSlotExpression,\n  replicationSlotPrefix,\n  type BackupOptions,\n  type ReplicaState,\n} from './schema/shard.ts';\n\n// Record returned by `CREATE_REPLICATION_SLOT`\nexport type ReplicationSlot = {\n  slot_name: string;\n  consistent_point: string;\n  snapshot_name: string;\n  output_plugin: string;\n};\n\nexport type ReplicationSlotResult<T> = {\n  slot: ReplicationSlot;\n  capturedSnapshot: T;\n  initialSession: Readable;\n  replica: ReplicaState;\n};\n\nexport type ReservedSlot = {\n  slot: string;\n  reservation: Readable;\n  replica: ReplicaState;\n};\n\nexport type CreateSlotSpec = {\n  slotName: string;\n\n  // Note: ignored if pgVersion < PG_17.\n  failover?: boolean;\n\n  // Create a temporary slot (i.e. not persisted, and automatically\n  // cleaned up when the replication session ends).\n  temporary?: boolean;\n\n  // For overriding in tests.\n  lockTimeout?: number | undefined;\n\n  // A (non-replication) connection to the same upstream instance, used to\n  // log what slot creation was waiting on if it times out.\n  diagnosticsDB?: PostgresDB | undefined;\n};\n\n// When creating a replication slot, Postgres waits for open transactions\n// to complete before reserving a consistent_point (LSN) in the WAL and creating\n// a matching transaction snapshot. As such, it can technically take an arbitrary\n// amount of time (e.g. DDL operations, table-wide operations, etc.).\n//\n// However, to detect pathological situations, bound the amount of time that\n// the server waits for replication slot creation, so that a continual failure to\n// create a replication slot is surfaced by errors / alerts.\nexport const CREATE_REPLICATION_SLOT_TIMEOUT_MS = 60_000;\n\n// The lock_timeout is set 1s before the client-side orTimeout so that\n// Postgres reliably aborts first and tears down the walsender cleanly.\n// The client-side timeout remains as a fallback for network-level failures.\nconst SERVER_LOCK_TIMEOUT_MS = CREATE_REPLICATION_SLOT_TIMEOUT_MS - 1_000;\n\n/** Thrown when the client-side timeout for creating a slot is reached. */\nexport class SlotCreationTimeoutError extends Error {\n  readonly name = 'SlotCreationTimeoutError';\n}\n\n// Note: The replication connection does not support the extended query protocol,\n//       so all commands must be sent using sql.unsafe(). This is technically safe\n//       because all placeholder values are under our control (i.e. \"slotName\").\nexport async function createReplicationSlot(\n  lc: LogContext,\n  session: postgres.Sql,\n  {\n    slotName,\n    failover,\n    temporary,\n    lockTimeout = SERVER_LOCK_TIMEOUT_MS,\n    diagnosticsDB,\n  }: CreateSlotSpec,\n): Promise<ReplicationSlot> {\n  // CREATE_REPLICATION_SLOT can hang indefinitely waiting for long-running\n  // transactions to finish: internally it calls SnapBuildWaitSnapshot →\n  // XactLockTableWait → LockAcquire on each running XID. statement_timeout\n  // does NOT apply to replication commands, but lock_timeout does (it governs\n  // the heavyweight lock wait inside LockAcquire). Setting it here causes\n  // Postgres to raise ERRCODE_LOCK_NOT_AVAILABLE and cleanly tear down the\n  // walsender, rather than relying solely on the client-side orTimeout\n  // which can leave an orphaned backend.\n  //\n  // An orphaned walsender is actively harmful: by this point the replication\n  // slot has already been created and is pinning WAL retention and catalog_xmin.\n  // Worse, the slot is marked `active` (the walsender PID is still alive), so\n  // the existing cleanup code (which drops inactive slots on retry) can't\n  // reclaim it. Without lock_timeout the orphan persists until TCP keepalive\n  // fires (~2h default) or the blocking transaction finishes.\n  await session.unsafe(`SET lock_timeout = ${lockTimeout}`);\n\n  const {pgVersion, pid: walsenderPID} = (\n    await session.unsafe<{pgVersion: number; pid: number}[]>(`\n      SELECT current_setting('server_version_num') as \"pgVersion\",\n             pg_backend_pid() as \"pid\";\n  `)\n  )[0];\n\n  const maybeTemporary = temporary ? 'TEMPORARY' : '';\n  const options = failover && pgVersion >= PG_17 ? '(FAILOVER)' : '';\n  const createSlot = session.unsafe<ReplicationSlot[]>(/*sql*/ `\n    CREATE_REPLICATION_SLOT \"${slotName}\" ${maybeTemporary} LOGICAL pgoutput ${options}`);\n\n  try {\n    const raced = await orTimeout(\n      createSlot,\n      CREATE_REPLICATION_SLOT_TIMEOUT_MS,\n    );\n    if (raced === 'timed-out') {\n      throw new SlotCreationTimeoutError(\n        `Timed out after ${CREATE_REPLICATION_SLOT_TIMEOUT_MS} ms creating replication slot ${slotName}.`,\n      );\n    }\n    const [slot] = raced;\n    lc.info?.(`Created replication slot ${slotName}`, slot);\n    return slot;\n  } catch (e) {\n    if (\n      diagnosticsDB &&\n      (isPostgresError(e, PG_LOCK_NOT_AVAILABLE) ||\n        e instanceof SlotCreationTimeoutError)\n    ) {\n      // After a lock_timeout, the walsender is no longer waiting, but the\n      // transactions it was waiting on are likely still running. After the\n      // client-side timeout, it may still be waiting on them.\n      await logSlotCreationBlockers(lc, diagnosticsDB, walsenderPID, slotName);\n    }\n    throw e;\n  }\n}\n\nconst SLOT_DIAGNOSTICS_TIMEOUT_MS = 5_000;\nconst SLOT_DIAGNOSTICS_MAX_TRANSACTIONS = 20;\n\n/**\n * Best-effort logging of what a timed out CREATE_REPLICATION_SLOT was waiting\n * on: the walsender's wait event and blocking pids, and the oldest\n * transactions holding an xid. Slot creation waits for every such transaction\n * on the instance (in all databases), each with its own lock_timeout, so a\n * chain of overlapping transactions can exceed the client-side timeout\n * without the lock_timeout ever firing.\n *\n * Query text is deliberately omitted since it can contain row values.\n *\n * Exported for testing.\n */\nexport async function logSlotCreationBlockers(\n  lc: LogContext,\n  sql: PostgresDB,\n  walsenderPID: number,\n  slotName: string,\n) {\n  try {\n    const result = await orTimeout(\n      Promise.all([\n        sql<JSONObject[]> /*sql*/ `\n          SELECT pid, wait_event_type as \"waitEventType\", wait_event as \"waitEvent\",\n                 pg_blocking_pids(pid) as \"blockingPIDs\"\n            FROM pg_stat_activity WHERE pid = ${walsenderPID}`,\n        sql<JSONObject[]> /*sql*/ `\n          SELECT pid, datname, usename, application_name as \"applicationName\",\n                 backend_type as \"backendType\", state,\n                 wait_event_type as \"waitEventType\", wait_event as \"waitEvent\",\n                 backend_xid::text as xid,\n                 xact_start::text as \"xactStart\",\n                 extract(epoch from now() - xact_start)::float8 as \"xactAgeSec\"\n            FROM pg_stat_activity\n            WHERE backend_xid IS NOT NULL AND pid <> ${walsenderPID}\n            ORDER BY xact_start NULLS LAST\n            LIMIT ${SLOT_DIAGNOSTICS_MAX_TRANSACTIONS}`,\n      ]),\n      SLOT_DIAGNOSTICS_TIMEOUT_MS,\n    );\n    if (result === 'timed-out') {\n      lc.warn?.(`Timed out collecting diagnostics for slot ${slotName}`);\n      return;\n    }\n    const [[walsender], transactions] = result;\n    lc.warn?.(`Replication slot ${slotName} creation blocked`, {\n      walsender: walsender ?? null,\n      transactions,\n    });\n  } catch (e) {\n    lc.warn?.(`Unable to collect diagnostics for slot ${slotName}`, e);\n  }\n}\n\n/**\n * Whether slot creation failed because it was waiting on transactions on the\n * upstream instance: the server's lock_timeout fired, the client-side timeout\n * was reached, or the replication session was closed while it was waiting.\n * Such failures are expected on busy upstreams, and leave no durable state,\n * so the attempt can simply be retried.\n */\nfunction isSlotCreationBlocked(e: unknown): boolean {\n  return (\n    isPostgresError(e, PG_LOCK_NOT_AVAILABLE) ||\n    e instanceof SlotCreationTimeoutError ||\n    (e instanceof Error && (e as {code?: unknown}).code === 'CONNECTION_CLOSED')\n  );\n}\n\n// Slot creation is retried while it is blocked by upstream transactions, but\n// a slot that cannot be created for this long is surfaced as an error (i.e.\n// by exiting the process).\nconst DEFAULT_MAX_SLOT_CREATION_BLOCKED_MS = 30 * 60 * 1000;\n\n// Base delay between retries of blocked slot creation. The actual delay is\n// jittered to between 1x and 5x of this value.\nconst DEFAULT_SLOT_CREATION_RETRY_DELAY_MS = 1_000;\n\nexport type CreateReplicaAndSlotOptions = {\n  // For overriding in tests.\n  lockTimeout?: number | undefined;\n  maxBlockedMs?: number | undefined;\n  retryDelayMs?: number | undefined;\n};\n\n/**\n * Creates a dedicated (non-pooled) session for holding the session-level\n * replication slot management lock. Ending the session releases the lock,\n * which guarantees that the lock never outlives the attempt.\n */\nfunction createLockSessionFor(db: PostgresDB, applicationName: string) {\n  return postgres(\n    defu(\n      {\n        max: 1,\n        ['idle_timeout']: null,\n        ['max_lifetime']: null as unknown as number,\n        connection: {['application_name']: applicationName},\n      },\n      // See createReplicationSessionFor() for why this cast is necessary.\n      db.options as unknown as Options<Record<string, PostgresType>>,\n    ),\n  );\n}\n\n/**\n * Replica and slot creation involves several sessions for proper\n * coordination with other replica management logic:\n *\n * * A dedicated session acquires a (session-level) advisory lock for\n *   replica slot management. This is the same lock that cleanup logic\n *   acquires before cleaning up replication slots. The lock is held\n *   outside of a transaction so that the session is not \"idle in\n *   transaction\" while slot creation waits on upstream transactions,\n *   which some upstreams terminate (e.g. idle_in_transaction_session_timeout).\n * * With the lock held, a new replication slot is created in a\n *   replication session. The API of CREATE_REPLICATION_SLOT is such\n *   that it cannot be done in a transaction, and cannot be followed by\n *   any writes, or else its snapshot (which is needed for initial sync)\n *   would be invalidated.\n * * Once the slot is created, the slot and replica information are recorded\n *   in the `replicas` table before releasing the lock.\n *\n * Slot creation waits for all transactions on the upstream instance that\n * hold an xid, so it can be blocked by long-running transactions. A blocked\n * attempt is abandoned (see {@link CREATE_REPLICATION_SLOT_TIMEOUT_MS}) and\n * retried in-process, as it leaves no durable state: the `replicas` row is\n * only inserted after the slot has been created. Retries continue until the\n * slot is created or until `maxBlockedMs` has elapsed, after which the error\n * is thrown.\n *\n * This locking ensures that:\n * 1. multiple replication managers attempting to create a replication slot\n *    will not use the same name for the replication slot (which is selected\n *    from a pool of reused names).\n * 2. Running replication managers (which use an earlier replica of a lower\n *    rank) will not delete the new slot during their cleanup logic, since\n *    the slot will belong to a replica of a higher rank.\n *\n * When the replication slot is created, `captureSnapshot` callback is first\n * run while the snapshot at the consistent point is available. The callback\n * must capture the snapshot in postgres (e.g. `SET TRANSACTION SNAPSHOT`)\n * in order to use it. Once the callback completes, a placeholder replication\n * session (i.e. that does not consume any messages) is started, in order to\n * reserve the slot by mark it \"active\", effectively distinguishing the\n * initializing session from a failed or abandoned session.\n *\n * Note that starting the replication session releases the initial snapshot,\n * which is why the callback must capture it synchronously, before the\n * replication session is started.\n *\n * The placeholder replication session is closed when the\n * PostgresChangeSource takes over the slot to process the stream in earnest.\n * It is also returned for cleanup in the case of failures (and for control\n * in unit tests).\n *\n * For replica's created in the `Restore` stage, the `generation` is\n * initialized to an empty string instead of the slot's consistent point,\n * since the slot is not being used for initial sync. It is the\n * responsibiliy of the caller to call {@link initRestoreReplica} when\n * the source replica has been selected for restoring, which will carry\n * over the `generation` and other relevant values from the source.\n */\nexport async function createReplicaAndSlot<T>(\n  lc: LogContext,\n  sql: PostgresDB,\n  sessionName: string,\n  shard: ShardID,\n  epoch: number,\n  replicaID: string,\n  failover: boolean,\n  backupOptions: BackupOptions,\n  captureSnapshot: (snapshot: string) => Promise<T>,\n  stage: ReplicaStage,\n  {\n    lockTimeout,\n    maxBlockedMs = DEFAULT_MAX_SLOT_CREATION_BLOCKED_MS,\n    retryDelayMs = DEFAULT_SLOT_CREATION_RETRY_DELAY_MS,\n  }: CreateReplicaAndSlotOptions = {},\n): Promise<ReplicationSlotResult<T>> {\n  const lockName = replicationSlotManagementLock(shard);\n  const slotPoolPrefix = replicationSlotPrefix(shard);\n  const start = Date.now();\n  let addedReplicationRole = false;\n  let cleanedUpForSlotLimit = false;\n\n  for (let attempt = 1; ; attempt++) {\n    await dropUnclaimedSlots(lc, sql, shard);\n\n    // Note: The replicationSession is used to create the replication slot\n    // and closed immediately afterwards, or on error. A new session is used\n    // for each attempt, since a failed attempt closes it.\n    const replicationSession = createReplicationSessionFor(sql, sessionName);\n    const lockSession = createLockSessionFor(sql, `${sessionName}-lock`);\n    // Ending the lock session releases the replication slot management lock.\n    // The sessions are ended with a timeout in case a hung slot creation\n    // (e.g. after a network partition) is still pending.\n    const endSessions = () =>\n      Promise.allSettled([\n        replicationSession.end({timeout: 5}),\n        lockSession.end({timeout: 5}),\n      ]);\n\n    let slotName: string | undefined;\n    let initialSession: Readable | undefined;\n    let creatingSlot = false;\n    try {\n      await lockSession`SELECT pg_advisory_lock(hashtext(${lockName}))`;\n\n      if (stage === InitialSync) {\n        // With the lock acquired, ensure that only one initial sync is\n        // active at a time. getRestoreCandidates() orders its results\n        // with stage = InitialSync (and active = true) first.\n        const others = await getRestoreCandidates(sql, shard, epoch);\n        lc.info?.(`current replicas at epoch ${epoch}`, {replicas: others});\n        if (others.length) {\n          const [{id, stage, active}] = others;\n          if (stage === InitialSync && active) {\n            throw new AutoResetSignal(\n              `another replica (${id}) is performing initial sync`,\n            );\n          }\n        }\n      }\n\n      // Pick an available slotName from the slotPoolPrefix pool.\n      const names = await sql<{name: string}[]> /*sql*/ `\n        SELECT slot_name as name FROM pg_replication_slots\n          WHERE slot_name LIKE ${slotPoolPrefix + '%'};\n      `.values();\n      const inUse = new Set(names.flat());\n      for (let next = 0; ; next++) {\n        const candidateName = `${slotPoolPrefix}${slotPoolSuffix(next)}`;\n        if (!inUse.has(candidateName)) {\n          slotName = candidateName;\n          break;\n        }\n      }\n\n      creatingSlot = true;\n      const slot = await createReplicationSlot(lc, replicationSession, {\n        slotName,\n        failover,\n        lockTimeout,\n        diagnosticsDB: sql,\n      });\n      creatingSlot = false;\n      const capturedSnapshot = await captureSnapshot(slot.snapshot_name);\n      await replicationSession.end({timeout: 5});\n      initialSession = await keepSlotActiveUntilTakenOver(lc, {\n        db: extractConnectionConfig(sql),\n        slot: slot.slot_name,\n        dummyPublication: metadataPublicationName(shard.appID, shard.shardNum),\n        lsn: String(toBigInt(slot.consistent_point)),\n      });\n\n      const replica = await runTx(sql, async tx => {\n        await createReplica(\n          tx,\n          shard,\n          replicaID,\n          slot.slot_name,\n          epoch,\n          stage === InitialSync\n            ? toStateVersionString(slot.consistent_point)\n            : '', // initialized with initRestoreReplica\n          backupOptions,\n          stage,\n        );\n        return must(\n          await getReplicaState(tx, shard, replicaID),\n          `replica ${replicaID} was not created`,\n        );\n      });\n\n      return {slot, capturedSnapshot, initialSession, replica};\n    } catch (e) {\n      // Release the lock (and end the attempt's sessions) before handling the\n      // error, e.g. as dropUnclaimedSlots() acquires the lock from a different\n      // session.\n      await endSessions();\n      if (\n        !addedReplicationRole &&\n        isPostgresError(e, PG_INSUFFICIENT_PRIVILEGE)\n      ) {\n        // Some Postgres variants (e.g. Google Cloud SQL) require that\n        // the user have the REPLICATION role in order to create a slot.\n        // Note that this must be done by the upstreamDB connection, and\n        // does not work in the replicationSession itself.\n        await sql`ALTER ROLE current_user WITH REPLICATION`;\n        lc.info?.(`Added the REPLICATION role to database user`);\n        addedReplicationRole = true;\n        continue;\n      }\n      // Note: This is currently manually tested since max_replication_slots\n      //       is a PG startup parameter that other tests depend on.\n      // TODO: Figure out a way to unit test this (with the full PG setup).\n      if (\n        !cleanedUpForSlotLimit &&\n        isPostgresError(e, PG_CONFIGURATION_LIMIT_EXCEEDED)\n      ) {\n        lc.warn?.(\n          `Reached max replication slots. Attempting to clean up unused slots`,\n          e,\n        );\n        // Drop any inactive replicas from failed initial syncs (e.g. inactive slots).\n        const replicasTable = `${upstreamSchema(shard)}.replicas`;\n        await sql`\n          DELETE FROM ${sql(replicasTable)} USING pg_replication_slots slots\n            WHERE replicas.slot = slots.slot_name AND NOT slots.active`;\n        cleanedUpForSlotLimit = true;\n        continue; // then let dropUnclaimedSlots() perform its cleanup\n      }\n      if (initialSession) {\n        try {\n          await endSession(initialSession);\n        } catch {}\n      }\n      // The slot may not exist (e.g. if its creation failed), and its name\n      // may be reused once the lock is released, so any created slot is\n      // left to dropUnclaimedSlots(), which only drops slots without a\n      // replica. This runs at the start of the next attempt.\n      const blockedMs = Date.now() - start;\n      if (\n        creatingSlot &&\n        isSlotCreationBlocked(e) &&\n        blockedMs < maxBlockedMs\n      ) {\n        lc.warn?.(\n          `Replication slot ${slotName} creation blocked by upstream transactions ` +\n            `(attempt ${attempt}, ${blockedMs} ms). Retrying.`,\n          e,\n        );\n        await sleep(retryDelayMs * (1 + 4 * Math.random()));\n        continue;\n      }\n      if (slotName) {\n        lc.warn?.(`deleting slot ${slotName} due to error`, e);\n        await dropUnclaimedSlots(lc, sql, shard);\n      }\n      throw e;\n    } finally {\n      await endSessions();\n    }\n  }\n}\n\n/**\n * Claims the inactive slot of the given replica for use by this task, opening\n * a replication session to mark it as active. The process should follow up by\n * taking over the slot via the PostgresChangeSource.\n *\n * The replica is identified by its ID (and not just its slot) because rows of\n * replicas whose slots were dropped outside of zero-cache's control may share\n * the slot name.\n */\nexport async function claimSlotForResumption(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardID,\n  _sessionName: string,\n  {id: replicaID, slot}: Pick<ReplicaState, 'id' | 'slot'>,\n): Promise<ReservedSlot | null> {\n  const lockName = replicationSlotManagementLock(shard);\n  const replicasTable = `${upstreamSchema(shard)}.replicas`;\n\n  let reservation: Readable | undefined;\n  try {\n    const reserved = await runTx(sql, async tx => {\n      await tx`SELECT pg_advisory_xact_lock(hashtext(${lockName}))`;\n\n      const replicas = await tx<{active: boolean; lsn: string}[]> /*sql*/ `\n      SELECT active, confirmed_flush_lsn as lsn\n        FROM pg_replication_slots\n        JOIN ${tx(replicasTable)} replica on slot_name = slot\n        WHERE slot_name = ${slot} AND replica.id = ${replicaID};\n    `;\n      if (replicas.length === 0) {\n        lc.warn?.(`no replica ${replicaID} found for slot ${slot}`);\n        return null;\n      }\n      const [{active, lsn}] = replicas;\n      if (active) {\n        lc.warn?.(\n          `replica ${replicaID} for slot ${slot} is active and cannot be claimed`,\n        );\n        return null;\n      }\n      // Mark the replica as Restoring.\n      await tx`\n        UPDATE ${tx(replicasTable)} SET \"stage\" = ${Restore} WHERE id = ${replicaID};\n      `;\n      const replica = must(\n        await getReplicaState(tx, shard, replicaID),\n        `replica ${replicaID} disappeared`,\n      );\n\n      reservation = await keepSlotActiveUntilTakenOver(lc, {\n        db: extractConnectionConfig(sql),\n        slot,\n        dummyPublication: metadataPublicationName(shard.appID, shard.shardNum),\n        lsn: String(toBigInt(lsn)),\n      });\n      return {\n        slot,\n        replica,\n        reservation,\n      };\n    });\n    lc.info?.(`successfully claimed slot ${slot}`);\n    return reserved;\n  } catch (e) {\n    if (reservation) {\n      try {\n        await endSession(reservation);\n      } catch {}\n    }\n    lc.error?.(`unable to claim replication slot ${slot}`, e);\n    return null;\n  }\n}\n\nexport function dropInactiveSlotsAndReplicas(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardID,\n  slots: string[],\n) {\n  const lockName = replicationSlotManagementLock(shard);\n  const replicasTable = `${upstreamSchema(shard)}.replicas`;\n\n  return runTx(sql, async tx => {\n    await tx`SELECT pg_advisory_xact_lock(hashtext(${lockName}))`;\n\n    const dropped = await tx<{slot: string; replicaID: string | null}[]>\n    /*sql*/ `\n      SELECT slot_name as slot, replica.id as \"replicaID\", pg_drop_replication_slot(slot_name) \n        FROM pg_replication_slots\n        LEFT JOIN ${tx(replicasTable)} replica on slot_name = slot\n        WHERE slot_name IN ${tx(slots)} AND NOT active;\n    `;\n    if (dropped.length) {\n      lc.info?.(`dropped ${dropped.length} inactive replication slot(s)`, {\n        dropped,\n      });\n    }\n\n    const replicas = dropped\n      .map(({replicaID}) => replicaID)\n      .filter(id => id !== null);\n    if (replicas.length) {\n      // Delete replicas associated with the inactive (and now dropped) slots.\n      await tx\n      /*sql*/ `DELETE FROM ${tx(replicasTable)} WHERE id IN ${tx(replicas)}`;\n      lc.info?.(`deleted ${replicas.length} old replica(s)`, {replicas});\n    }\n  });\n}\n\nfunction dropUnclaimedSlots(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardID,\n): Promise<{dropped: number; active: number; draining: number}> {\n  // The slot / replica cleanup happens within a transaction while holding\n  // the replication slot management lock for this shard, to ensure that no\n  // slot that belongs to a newer replica is dropped.\n  const lockName = replicationSlotManagementLock(shard);\n  const slotExpression = replicationSlotExpression(shard);\n  const replicasTable = `${upstreamSchema(shard)}.replicas`;\n\n  return runTx(sql, async tx => {\n    await tx`SELECT pg_advisory_xact_lock(hashtext(${lockName}))`;\n\n    const dropped = await tx /*sql*/ `\n      SELECT slot_name as slot, pg_drop_replication_slot(slot_name) \n        FROM pg_replication_slots\n        LEFT JOIN ${tx(replicasTable)} replica on slot_name = slot\n        WHERE slot_name LIKE ${slotExpression} \n          AND NOT active\n          AND replica.id IS NULL;\n    `;\n    if (dropped.length) {\n      lc.info?.(`dropped inactive replication slots`, {dropped});\n    }\n\n    // Conversely, delete replicas whose slots no longer exist, e.g. because\n    // they were dropped outside of zero-cache's control (such as in an\n    // upstream failover). Slot names are reused, so such rows would otherwise\n    // be associated with a new slot of the same name. This is done (and\n    // committed) before the slot is created, both so that the rows are never\n    // associated with it, and because CREATE_REPLICATION_SLOT waits for\n    // transactions that have written (i.e. been assigned an xid).\n    const orphaned = await tx<{id: string}[]> /*sql*/ `\n      DELETE FROM ${tx(replicasTable)} replica\n        WHERE NOT EXISTS (\n          SELECT 1 FROM pg_replication_slots WHERE slot_name = replica.slot\n        )\n        RETURNING id;\n    `;\n    if (orphaned.length) {\n      lc.warn?.(`deleted replicas whose slots no longer exist`, {\n        replicas: orphaned.map(({id}) => id),\n      });\n    }\n\n    const remaining = await tx<\n      {slot: string; pid: number | null; id: string | null}[]\n    > /*sql*/ `\n      SELECT slot_name as slot, active_pid as pid, replica.id as id\n        FROM pg_replication_slots\n        LEFT JOIN ${tx(replicasTable)} replica on slot_name = slot\n        WHERE slot_name LIKE ${slotExpression};\n    `;\n    if (remaining.length) {\n      lc.info?.(`remaining replication slots`, {remaining});\n    }\n\n    let active = 0;\n    let draining = 0;\n    for (const {id} of remaining) {\n      if (id === null) {\n        draining++;\n      } else {\n        active++;\n      }\n    }\n\n    return {\n      dropped: dropped.length,\n      active,\n      draining,\n    };\n  });\n}\n\nconst ALPHABET = 'abcdefghijklmnopqrstuvwxyz';\n\n// Alphabetic notation is used as the slot pool suffix to distinguish\n// it from the (numeric) shard num that's also encoded in the slot name.\nexport function slotPoolSuffix(n: number) {\n  n++; // Adjust for 0-based indexing\n\n  let suffix = '';\n  while (n > 0) {\n    n--;\n    suffix = ALPHABET[n % 26] + suffix;\n    n = Math.floor(n / 26);\n  }\n  return suffix;\n}\n\nfunction replicationSlotManagementLock(shard: ShardID) {\n  return `replication-slot-management:${shard.appID}_${shard.shardNum}`;\n}\n"],"mappings":";;;;;;;;;;;;;;;;AAwFA,IAAa,qCAAqC;AAKlD,IAAM,yBAAyB,qCAAqC;;AAGpE,IAAa,2BAAb,cAA8C,MAAM;CAClD,OAAgB;AAClB;AAKA,eAAsB,sBACpB,IACA,SACA,EACE,UACA,UACA,WACA,cAAc,wBACd,iBAEwB;CAgB1B,MAAM,QAAQ,OAAO,sBAAsB,aAAa;CAExD,MAAM,EAAC,WAAW,KAAK,kBACrB,MAAM,QAAQ,OAA2C;;;GAG1D,GACC;CAEF,MAAM,iBAAiB,YAAY,cAAc;CACjD,MAAM,UAAU,YAAY,aAAA,OAAqB,eAAe;CAChE,MAAM,aAAa,QAAQ,OAAkC;+BAChC,SAAS,IAAI,eAAe,oBAAoB,SAAS;CAEtF,IAAI;EACF,MAAM,QAAQ,MAAM,UAClB,YACA,kCACF;EACA,IAAI,UAAU,aACZ,MAAM,IAAI,yBACR,mBAAmB,mCAAmC,gCAAgC,SAAS,EACjG;EAEF,MAAM,CAAC,QAAQ;EACf,GAAG,OAAO,4BAA4B,YAAY,IAAI;EACtD,OAAO;CACT,SAAS,GAAG;EACV,IACE,kBACC,gBAAgB,GAAG,qBAAqB,KACvC,aAAa,2BAKf,MAAM,wBAAwB,IAAI,eAAe,cAAc,QAAQ;EAEzE,MAAM;CACR;AACF;AAEA,IAAM,8BAA8B;AACpC,IAAM,oCAAoC;;;;;;;;;;;;;AAc1C,eAAsB,wBACpB,IACA,KACA,cACA,UACA;CACA,IAAI;EACF,MAAM,SAAS,MAAM,UACnB,QAAQ,IAAI,CACV,GAA0B;;;gDAGc,gBACxC,GAA0B;;;;;;;;uDAQqB,aAAa;;oBAEhD,mCACd,CAAC,GACD,2BACF;EACA,IAAI,WAAW,aAAa;GAC1B,GAAG,OAAO,6CAA6C,UAAU;GACjE;EACF;EACA,MAAM,CAAC,CAAC,YAAY,gBAAgB;EACpC,GAAG,OAAO,oBAAoB,SAAS,oBAAoB;GACzD,WAAW,aAAa;GACxB;EACF,CAAC;CACH,SAAS,GAAG;EACV,GAAG,OAAO,0CAA0C,YAAY,CAAC;CACnE;AACF;;;;;;;;AASA,SAAS,sBAAsB,GAAqB;CAClD,OACE,gBAAgB,GAAG,qBAAqB,KACxC,aAAa,4BACZ,aAAa,SAAU,EAAuB,SAAS;AAE5D;AAKA,IAAM,uCAAuC,OAAU;AAIvD,IAAM,uCAAuC;;;;;;AAc7C,SAAS,qBAAqB,IAAgB,iBAAyB;CACrE,OAAO,SACL,KACE;EACE,KAAK;GACJ,iBAAiB;GACjB,iBAAiB;EAClB,YAAY,GAAE,qBAAqB,gBAAe;CACpD,GAEA,GAAG,OACL,CACF;AACF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA4DA,eAAsB,qBACpB,IACA,KACA,aACA,OACA,OACA,WACA,UACA,eACA,iBACA,OACA,EACE,aACA,eAAe,sCACf,eAAe,yCACgB,CAAC,GACC;CACnC,MAAM,WAAW,8BAA8B,KAAK;CACpD,MAAM,iBAAiB,sBAAsB,KAAK;CAClD,MAAM,QAAQ,KAAK,IAAI;CACvB,IAAI,uBAAuB;CAC3B,IAAI,wBAAwB;CAE5B,KAAK,IAAI,UAAU,IAAK,WAAW;EACjC,MAAM,mBAAmB,IAAI,KAAK,KAAK;EAKvC,MAAM,qBAAqB,4BAA4B,KAAK,WAAW;EACvE,MAAM,cAAc,qBAAqB,KAAK,GAAG,YAAY,MAAM;EAInE,MAAM,oBACJ,QAAQ,WAAW,CACjB,mBAAmB,IAAI,EAAC,SAAS,EAAC,CAAC,GACnC,YAAY,IAAI,EAAC,SAAS,EAAC,CAAC,CAC9B,CAAC;EAEH,IAAI;EACJ,IAAI;EACJ,IAAI,eAAe;EACnB,IAAI;GACF,MAAM,WAAW,oCAAoC,SAAS;GAE9D,IAAI,UAAA,GAAuB;IAIzB,MAAM,SAAS,MAAM,qBAAqB,KAAK,OAAO,KAAK;IAC3D,GAAG,OAAO,6BAA6B,SAAS,EAAC,UAAU,OAAM,CAAC;IAClE,IAAI,OAAO,QAAQ;KACjB,MAAM,CAAC,EAAC,IAAI,OAAO,YAAW;KAC9B,IAAI,UAAA,KAAyB,QAC3B,MAAM,IAAI,gBACR,oBAAoB,GAAG,6BACzB;IAEJ;GACF;GAGA,MAAM,QAAQ,MAAM,GAA8B;;iCAEvB,iBAAiB,IAAI;QAC9C,OAAO;GACT,MAAM,QAAQ,IAAI,IAAI,MAAM,KAAK,CAAC;GAClC,KAAK,IAAI,OAAO,IAAK,QAAQ;IAC3B,MAAM,gBAAgB,GAAG,iBAAiB,eAAe,IAAI;IAC7D,IAAI,CAAC,MAAM,IAAI,aAAa,GAAG;KAC7B,WAAW;KACX;IACF;GACF;GAEA,eAAe;GACf,MAAM,OAAO,MAAM,sBAAsB,IAAI,oBAAoB;IAC/D;IACA;IACA;IACA,eAAe;GACjB,CAAC;GACD,eAAe;GACf,MAAM,mBAAmB,MAAM,gBAAgB,KAAK,aAAa;GACjE,MAAM,mBAAmB,IAAI,EAAC,SAAS,EAAC,CAAC;GACzC,iBAAiB,MAAM,6BAA6B,IAAI;IACtD,IAAI,wBAAwB,GAAG;IAC/B,MAAM,KAAK;IACX,kBAAkB,wBAAwB,MAAM,OAAO,MAAM,QAAQ;IACrE,KAAK,OAAO,SAAS,KAAK,gBAAgB,CAAC;GAC7C,CAAC;GAED,MAAM,UAAU,MAAM,MAAM,KAAK,OAAM,OAAM;IAC3C,MAAM,cACJ,IACA,OACA,WACA,KAAK,WACL,OACA,UAAA,IACI,qBAAqB,KAAK,gBAAgB,IAC1C,IACJ,eACA,KACF;IACA,OAAO,KACL,MAAM,gBAAgB,IAAI,OAAO,SAAS,GAC1C,WAAW,UAAU,iBACvB;GACF,CAAC;GAED,OAAO;IAAC;IAAM;IAAkB;IAAgB;GAAO;EACzD,SAAS,GAAG;GAIV,MAAM,YAAY;GAClB,IACE,CAAC,wBACD,gBAAgB,GAAG,yBAAyB,GAC5C;IAKA,MAAM,GAAG;IACT,GAAG,OAAO,6CAA6C;IACvD,uBAAuB;IACvB;GACF;GAIA,IACE,CAAC,yBACD,gBAAgB,GAAG,+BAA+B,GAClD;IACA,GAAG,OACD,sEACA,CACF;IAGA,MAAM,GAAG;wBACO,IAAI,GAFK,eAAe,KAAK,EAAE,UAEd,EAAE;;IAEnC,wBAAwB;IACxB;GACF;GACA,IAAI,gBACF,IAAI;IACF,MAAM,WAAW,cAAc;GACjC,QAAQ,CAAC;GAMX,MAAM,YAAY,KAAK,IAAI,IAAI;GAC/B,IACE,gBACA,sBAAsB,CAAC,KACvB,YAAY,cACZ;IACA,GAAG,OACD,oBAAoB,SAAS,sDACf,QAAQ,IAAI,UAAU,kBACpC,CACF;IACA,MAAM,MAAM,gBAAgB,IAAI,IAAI,KAAK,OAAO,EAAE;IAClD;GACF;GACA,IAAI,UAAU;IACZ,GAAG,OAAO,iBAAiB,SAAS,gBAAgB,CAAC;IACrD,MAAM,mBAAmB,IAAI,KAAK,KAAK;GACzC;GACA,MAAM;EACR,UAAU;GACR,MAAM,YAAY;EACpB;CACF;AACF;;;;;;;;;;AAWA,eAAsB,uBACpB,IACA,KACA,OACA,cACA,EAAC,IAAI,WAAW,QACc;CAC9B,MAAM,WAAW,8BAA8B,KAAK;CACpD,MAAM,gBAAgB,GAAG,eAAe,KAAK,EAAE;CAE/C,IAAI;CACJ,IAAI;EACF,MAAM,WAAW,MAAM,MAAM,KAAK,OAAM,OAAM;GAC5C,MAAM,EAAE,yCAAyC,SAAS;GAE1D,MAAM,WAAW,MAAM,EAA6C;;;eAG3D,GAAG,aAAa,EAAE;4BACL,KAAK,oBAAoB,UAAU;;GAEzD,IAAI,SAAS,WAAW,GAAG;IACzB,GAAG,OAAO,cAAc,UAAU,kBAAkB,MAAM;IAC1D,OAAO;GACT;GACA,MAAM,CAAC,EAAC,QAAQ,SAAQ;GACxB,IAAI,QAAQ;IACV,GAAG,OACD,WAAW,UAAU,YAAY,KAAK,iCACxC;IACA,OAAO;GACT;GAEA,MAAM,EAAE;iBACG,GAAG,aAAa,EAAE,iBAAA,EAAyB,cAAc,UAAU;;GAE9E,MAAM,UAAU,KACd,MAAM,gBAAgB,IAAI,OAAO,SAAS,GAC1C,WAAW,UAAU,aACvB;GAEA,cAAc,MAAM,6BAA6B,IAAI;IACnD,IAAI,wBAAwB,GAAG;IAC/B;IACA,kBAAkB,wBAAwB,MAAM,OAAO,MAAM,QAAQ;IACrE,KAAK,OAAO,SAAS,GAAG,CAAC;GAC3B,CAAC;GACD,OAAO;IACL;IACA;IACA;GACF;EACF,CAAC;EACD,GAAG,OAAO,6BAA6B,MAAM;EAC7C,OAAO;CACT,SAAS,GAAG;EACV,IAAI,aACF,IAAI;GACF,MAAM,WAAW,WAAW;EAC9B,QAAQ,CAAC;EAEX,GAAG,QAAQ,oCAAoC,QAAQ,CAAC;EACxD,OAAO;CACT;AACF;AAEA,SAAgB,6BACd,IACA,KACA,OACA,OACA;CACA,MAAM,WAAW,8BAA8B,KAAK;CACpD,MAAM,gBAAgB,GAAG,eAAe,KAAK,EAAE;CAE/C,OAAO,MAAM,KAAK,OAAM,OAAM;EAC5B,MAAM,EAAE,yCAAyC,SAAS;EAE1D,MAAM,UAAU,MAAM,EACd;;;oBAGQ,GAAG,aAAa,EAAE;6BACT,GAAG,KAAK,EAAE;;EAEnC,IAAI,QAAQ,QACV,GAAG,OAAO,WAAW,QAAQ,OAAO,gCAAgC,EAClE,QACF,CAAC;EAGH,MAAM,WAAW,QACd,KAAK,EAAC,gBAAe,SAAS,EAC9B,QAAO,OAAM,OAAO,IAAI;EAC3B,IAAI,SAAS,QAAQ;GAEnB,MAAM,EACE,eAAe,GAAG,aAAa,EAAE,eAAe,GAAG,QAAQ;GACnE,GAAG,OAAO,WAAW,SAAS,OAAO,kBAAkB,EAAC,SAAQ,CAAC;EACnE;CACF,CAAC;AACH;AAEA,SAAS,mBACP,IACA,KACA,OAC8D;CAI9D,MAAM,WAAW,8BAA8B,KAAK;CACpD,MAAM,iBAAiB,0BAA0B,KAAK;CACtD,MAAM,gBAAgB,GAAG,eAAe,KAAK,EAAE;CAE/C,OAAO,MAAM,KAAK,OAAM,OAAM;EAC5B,MAAM,EAAE,yCAAyC,SAAS;EAE1D,MAAM,UAAU,MAAM,EAAW;;;oBAGjB,GAAG,aAAa,EAAE;+BACP,eAAe;;;;EAI1C,IAAI,QAAQ,QACV,GAAG,OAAO,sCAAsC,EAAC,QAAO,CAAC;EAU3D,MAAM,WAAW,MAAM,EAA2B;oBAClC,GAAG,aAAa,EAAE;;;;;;EAMlC,IAAI,SAAS,QACX,GAAG,OAAO,gDAAgD,EACxD,UAAU,SAAS,KAAK,EAAC,SAAQ,EAAE,EACrC,CAAC;EAGH,MAAM,YAAY,MAAM,EAEd;;;oBAGM,GAAG,aAAa,EAAE;+BACP,eAAe;;EAE1C,IAAI,UAAU,QACZ,GAAG,OAAO,+BAA+B,EAAC,UAAS,CAAC;EAGtD,IAAI,SAAS;EACb,IAAI,WAAW;EACf,KAAK,MAAM,EAAC,QAAO,WACjB,IAAI,OAAO,MACT;OAEA;EAIJ,OAAO;GACL,SAAS,QAAQ;GACjB;GACA;EACF;CACF,CAAC;AACH;AAEA,IAAM,WAAW;AAIjB,SAAgB,eAAe,GAAW;CACxC;CAEA,IAAI,SAAS;CACb,OAAO,IAAI,GAAG;EACZ;EACA,SAAS,SAAS,IAAI,MAAM;EAC5B,IAAI,KAAK,MAAM,IAAI,EAAE;CACvB;CACA,OAAO;AACT;AAEA,SAAS,8BAA8B,OAAgB;CACrD,OAAO,+BAA+B,MAAM,MAAM,GAAG,MAAM;AAC7D"}

package/out/zero-cache/src/services/view-syncer/batch-pacer.d.ts

@@ -0,0 +1,79 @@
+import type { LogContext } from '@rocicorp/logger';
+export type BatchPacerOptions = {
+    /**
+     * The largest share of the worker's time that it may be busy before its
+     * client groups are paced.
+     */
+    readonly shareTarget: number;
+    /** The longest a client group waits between batches. */
+    readonly maxIntervalMs: number;
+    readonly now?: (() => number) | undefined;
+    /**
+     * The share of the time since the previous call that the worker was busy.
+     * By default, the utilization of the event loop.
+     */
+    readonly utilization?: (() => number) | undefined;
+    /** Where the worker says that it is pacing its client groups. */
+    readonly lc?: LogContext | undefined;
+};
+/**
+ * Paces the batches (advancements, including the queries they hydrate
+ * again) of the client groups of a sync worker, so that the worker is left
+ * with time for new hydrations and connections, and gives the hydrations of
+ * new queries priority over the batches.
+ *
+ * **Pacing.** After each batch, a client group waits {@link waitMs} before
+ * it starts the next. The wait is a multiple of the batch's processing
+ * time, so a client group that always has changes waiting is busy at most
+ * `1 / (1 + k)` of the time, where `k` is the {@link multiplier}. Because
+ * the wait is in proportion to a client group's own processing time, those
+ * that cause the load are slowed the most, and cheap ones barely at all.
+ *
+ * The multiplier is shared by the client groups of the worker. Once per
+ * second, it is scaled by how far the share of its time that the worker was
+ * busy, smoothed over a few seconds, is from the target, and by more while
+ * the worker has no time to spare. It never goes below 0, where no client
+ * group waits. The share is the utilization of the worker's event loop, i.e. all
+ * of what the worker does: what a batch costs is spread over the pipelines,
+ * the CVR, the pokes and garbage collection, and is not measurable batch by
+ * batch.
+ *
+ * **Priority of new hydrations.** The worker's time slices are handed out
+ * by the pacer (see {@link yieldSlice}), and while the hydration of new
+ * queries is waiting for one, most of them go to it rather than to batches.
+ */
+export declare class BatchPacer {
+    #private;
+    constructor({ shareTarget, maxIntervalMs, now, utilization, lc, }: BatchPacerOptions);
+    /**
+     * How long a client group should wait before its next batch, after one
+     * that took `processingMs` to process. The multiplier changes while a
+     * client group waits, so a long wait should be taken in steps, asking
+     * again after each.
+     */
+    waitMs(processingMs: number): number;
+    /** `k`: a client group waits `k` times the processing time of a batch. */
+    get multiplier(): number;
+    /**
+     * The share of its time that the worker was busy, smoothed over the last
+     * few seconds, as of the last update of the multiplier.
+     */
+    get share(): number;
+    /**
+     * Yields the worker to the other work that is waiting for a time slice,
+     * and resolves when it is the caller's turn again. One piece of work runs
+     * per turn of the event loop, as when the work of a worker's client groups
+     * simply takes turns, except that the hydration of new queries goes first:
+     * while both kinds of work are waiting, `HYDRATION_PRIORITY` slices go to
+     * hydrations for each one that goes to a batch.
+     *
+     * Batches are never made to wait for anything but a slice. If no
+     * hydration is waiting for one (because none is running, or because those
+     * that are running are waiting for I/O), batches get every slice. And
+     * however many hydrations there are, batches keep a share of the slices,
+     * so they keep moving, and so do the snapshots of the replica that they
+     * hold.
+     */
+    yieldSlice(kind: 'hydration' | 'batch'): Promise<void>;
+}
+//# sourceMappingURL=batch-pacer.d.ts.map

package/out/zero-cache/src/services/view-syncer/batch-pacer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"batch-pacer.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/view-syncer/batch-pacer.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAyDjD,MAAM,MAAM,iBAAiB,GAAG;IAC9B;;;OAGG;IACH,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,wDAAwD;IACxD,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,MAAM,CAAC,GAAG,SAAS,CAAC;IAC1C;;;OAGG;IACH,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC,MAAM,MAAM,CAAC,GAAG,SAAS,CAAC;IAClD,iEAAiE;IACjE,QAAQ,CAAC,EAAE,CAAC,EAAE,UAAU,GAAG,SAAS,CAAC;CACtC,CAAC;AAYF;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,qBAAa,UAAU;;IA8BrB,YAAY,EACV,WAAW,EACX,aAAa,EACb,GAAG,EACH,WAAW,EACX,EAAE,GACH,EAAE,iBAAiB,EAOnB;IAED;;;;;OAKG;IACH,MAAM,CAAC,YAAY,EAAE,MAAM,GAAG,MAAM,CASnC;IAED,0EAA0E;IAC1E,IAAI,UAAU,IAAI,MAAM,CAGvB;IAED;;;OAGG;IACH,IAAI,KAAK,IAAI,MAAM,CAGlB;IA+FD;;;;;;;;;;;;;;OAcG;IACH,UAAU,CAAC,IAAI,EAAE,WAAW,GAAG,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAQrD;CAuBF"}

package/out/zero-cache/src/services/view-syncer/batch-pacer.js

@@ -0,0 +1,249 @@
+import { performance } from "node:perf_hooks";
+//#region ../zero-cache/src/services/view-syncer/batch-pacer.ts
+/** How often the multiplier is updated. */
+var UPDATE_INTERVAL_MS = 1e3;
+/**
+* The weight of the latest second in the worker's smoothed busy share.
+* Client groups that wait for seconds between batches make the share of
+* any one second jump between idle and saturated, and a worker that keeps
+* up has the odd busy second. Neither says what the load is.
+*/
+var SHARE_SMOOTHING = .3;
+/**
+* How much of the difference between the smoothed busy share and the target
+* each update corrects for. The smoothed share only shows part of what the
+* previous updates did, so correcting for all of the difference each time
+* would overshoot and oscillate.
+*/
+var CORRECTION_EXPONENT = .3;
+/**
+* The share of its time above which a worker has no time to spare. A share
+* target has to stay well clear of this to be held, which is why
+* `syncUpdateShareTarget` is at most 0.9 (or 1, which no worker is over).
+*/
+var SATURATED_SHARE = .95;
+/** The seconds that a worker has to be saturated for to be treated as such. */
+var SATURATED_SECONDS = 2;
+/**
+* How much `1 + k` is scaled by each second that the worker is saturated.
+* A saturated worker does not show how much more its client groups would
+* use, which can be many times what it has, so the multiplier is raised by
+* more than the difference from the target calls for until it does.
+*/
+var SATURATED_CORRECTION = 1.3;
+/**
+* How often a worker that is pacing its client groups says so in its log,
+* and how long it has to have stopped for to say that it has.
+*/
+var PACING_LOG_INTERVAL_MS = 6e4;
+/**
+* The most updates that are made up for when nothing has asked the pacer
+* for a wait in a while, which is enough for the multiplier to get to 0.
+*/
+var MAX_CATCH_UP_UPDATES = 60;
+/**
+* Bounds the multiplier, which otherwise has no limit while a worker stays
+* over the target however long its client groups wait.
+*/
+var MAX_MULTIPLIER = 1e6;
+function eventLoopUtilization() {
+	let last = performance.eventLoopUtilization();
+	return () => {
+		const current = performance.eventLoopUtilization();
+		const { utilization } = performance.eventLoopUtilization(current, last);
+		last = current;
+		return utilization;
+	};
+}
+/**
+* Paces the batches (advancements, including the queries they hydrate
+* again) of the client groups of a sync worker, so that the worker is left
+* with time for new hydrations and connections, and gives the hydrations of
+* new queries priority over the batches.
+*
+* **Pacing.** After each batch, a client group waits {@link waitMs} before
+* it starts the next. The wait is a multiple of the batch's processing
+* time, so a client group that always has changes waiting is busy at most
+* `1 / (1 + k)` of the time, where `k` is the {@link multiplier}. Because
+* the wait is in proportion to a client group's own processing time, those
+* that cause the load are slowed the most, and cheap ones barely at all.
+*
+* The multiplier is shared by the client groups of the worker. Once per
+* second, it is scaled by how far the share of its time that the worker was
+* busy, smoothed over a few seconds, is from the target, and by more while
+* the worker has no time to spare. It never goes below 0, where no client
+* group waits. The share is the utilization of the worker's event loop, i.e. all
+* of what the worker does: what a batch costs is spread over the pipelines,
+* the CVR, the pokes and garbage collection, and is not measurable batch by
+* batch.
+*
+* **Priority of new hydrations.** The worker's time slices are handed out
+* by the pacer (see {@link yieldSlice}), and while the hydration of new
+* queries is waiting for one, most of them go to it rather than to batches.
+*/
+var BatchPacer = class {
+	#shareTarget;
+	#maxIntervalMs;
+	#now;
+	#utilization;
+	#lc;
+	/** When the worker last logged that it is pacing, while it still is. */
+	#pacingLoggedAt;
+	/** Since when the worker has not been pacing, after it logged that it was. */
+	#notPacingSince;
+	#intervalStart;
+	#multiplier = 0;
+	/** The smoothed share of its time that the worker was busy. */
+	#share = 0;
+	#saturatedSeconds = 0;
+	/** The waits handed out since the last update, by whether at the cap. */
+	#waitsBelowMax = 0;
+	#waitsAtMax = 0;
+	/** The work that is waiting for a time slice, in the order it yielded. */
+	#waiting = {
+		hydration: [],
+		batch: []
+	};
+	#handingOutSlices = false;
+	/** The slices that have gone to hydrations since one went to a batch. */
+	#hydrationSlices = 0;
+	constructor({ shareTarget, maxIntervalMs, now, utilization, lc }) {
+		this.#shareTarget = shareTarget;
+		this.#maxIntervalMs = maxIntervalMs;
+		this.#now = now ?? (() => performance.now());
+		this.#utilization = utilization ?? eventLoopUtilization();
+		this.#lc = lc;
+		this.#intervalStart = this.#now();
+	}
+	/**
+	* How long a client group should wait before its next batch, after one
+	* that took `processingMs` to process. The multiplier changes while a
+	* client group waits, so a long wait should be taken in steps, asking
+	* again after each.
+	*/
+	waitMs(processingMs) {
+		this.#update();
+		const wait = this.#multiplier * processingMs;
+		if (wait < this.#maxIntervalMs) {
+			this.#waitsBelowMax++;
+			return wait;
+		}
+		this.#waitsAtMax++;
+		return this.#maxIntervalMs;
+	}
+	/** `k`: a client group waits `k` times the processing time of a batch. */
+	get multiplier() {
+		this.#update();
+		return this.#multiplier;
+	}
+	/**
+	* The share of its time that the worker was busy, smoothed over the last
+	* few seconds, as of the last update of the multiplier.
+	*/
+	get share() {
+		this.#update();
+		return this.#share;
+	}
+	/** Updates the multiplier for each interval that has ended. */
+	#update() {
+		const now = this.#now();
+		const ended = Math.floor((now - this.#intervalStart) / UPDATE_INTERVAL_MS);
+		if (ended <= 0) return;
+		this.#intervalStart += ended * UPDATE_INTERVAL_MS;
+		const latest = this.#utilization();
+		const allAtMax = this.#waitsAtMax > 0 && this.#waitsBelowMax === 0;
+		this.#waitsAtMax = 0;
+		this.#waitsBelowMax = 0;
+		for (let i = Math.min(ended, MAX_CATCH_UP_UPDATES); i > 0; i--) {
+			this.#share += SHARE_SMOOTHING * (latest - this.#share);
+			this.#saturatedSeconds = latest >= SATURATED_SHARE && latest > this.#shareTarget ? this.#saturatedSeconds + 1 : 0;
+			let correction = this.#saturatedSeconds >= SATURATED_SECONDS ? SATURATED_CORRECTION : (this.#share / this.#shareTarget) ** CORRECTION_EXPONENT;
+			if (allAtMax && correction > 1) correction = 1;
+			this.#multiplier = Math.min(Math.max((1 + this.#multiplier) * correction - 1, 0), MAX_MULTIPLIER);
+		}
+		this.#logPacing(now, allAtMax);
+	}
+	/**
+	* Says in the worker's log when it starts to pace its client groups, once
+	* a minute while it does, and when it has stopped. The pacing of a client
+	* group is logged by its view syncer.
+	*/
+	#logPacing(now, allAtMax) {
+		const lc = this.#lc;
+		if (lc === void 0) return;
+		const state = {
+			busyShare: this.#share,
+			shareTarget: this.#shareTarget,
+			waitMultiplier: this.#multiplier,
+			maxBatchIntervalMs: this.#maxIntervalMs
+		};
+		if (this.#multiplier > 0) {
+			this.#notPacingSince = void 0;
+			if (this.#pacingLoggedAt === void 0) {
+				lc.info?.("Sync worker is over its share target: its client groups wait between batches", state);
+				this.#pacingLoggedAt = now;
+			} else if (now - this.#pacingLoggedAt >= PACING_LOG_INTERVAL_MS) {
+				if (allAtMax && this.#share > this.#shareTarget) lc.warn?.("Sync worker is over its share target although every client group waits the longest it may", state);
+				else lc.info?.("Sync worker is pacing its client groups", state);
+				this.#pacingLoggedAt = now;
+			}
+		} else if (this.#pacingLoggedAt !== void 0) {
+			this.#notPacingSince ??= now;
+			if (now - this.#notPacingSince >= PACING_LOG_INTERVAL_MS) {
+				lc.info?.("Sync worker is under its share target: its client groups no longer wait", state);
+				this.#pacingLoggedAt = void 0;
+				this.#notPacingSince = void 0;
+			}
+		}
+	}
+	/**
+	* Yields the worker to the other work that is waiting for a time slice,
+	* and resolves when it is the caller's turn again. One piece of work runs
+	* per turn of the event loop, as when the work of a worker's client groups
+	* simply takes turns, except that the hydration of new queries goes first:
+	* while both kinds of work are waiting, `HYDRATION_PRIORITY` slices go to
+	* hydrations for each one that goes to a batch.
+	*
+	* Batches are never made to wait for anything but a slice. If no
+	* hydration is waiting for one (because none is running, or because those
+	* that are running are waiting for I/O), batches get every slice. And
+	* however many hydrations there are, batches keep a share of the slices,
+	* so they keep moving, and so do the snapshots of the replica that they
+	* hold.
+	*/
+	yieldSlice(kind) {
+		return new Promise((resolve) => {
+			this.#waiting[kind].push(resolve);
+			if (!this.#handingOutSlices) {
+				this.#handingOutSlices = true;
+				setImmediate(this.#nextSlice);
+			}
+		});
+	}
+	#nextSlice = () => {
+		const { hydration, batch } = this.#waiting;
+		let next;
+		if (hydration.length > 0 && (batch.length === 0 || this.#hydrationSlices < HYDRATION_PRIORITY)) {
+			next = hydration.shift();
+			this.#hydrationSlices++;
+		} else if (batch.length > 0) {
+			next = batch.shift();
+			this.#hydrationSlices = 0;
+		}
+		if (next === void 0) {
+			this.#handingOutSlices = false;
+			return;
+		}
+		next();
+		setImmediate(this.#nextSlice);
+	};
+};
+/**
+* How many time slices go to the hydration of new queries for each one that
+* goes to a batch, while both are waiting for one.
+*/
+var HYDRATION_PRIORITY = 9;
+//#endregion
+export { BatchPacer };
+
+//# sourceMappingURL=batch-pacer.js.map

package/out/zero-cache/src/services/view-syncer/batch-pacer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"batch-pacer.js","names":["#shareTarget","#maxIntervalMs","#now","#utilization","#lc","#waiting","#intervalStart","#update","#multiplier","#waitsBelowMax","#waitsAtMax","#share","#saturatedSeconds","#logPacing","#notPacingSince","#pacingLoggedAt","#handingOutSlices","#nextSlice","#hydrationSlices"],"sources":["../../../../../../zero-cache/src/services/view-syncer/batch-pacer.ts"],"sourcesContent":["import {performance} from 'node:perf_hooks';\nimport type {LogContext} from '@rocicorp/logger';\n\n/** How often the multiplier is updated. */\nconst UPDATE_INTERVAL_MS = 1000;\n\n/**\n * The weight of the latest second in the worker's smoothed busy share.\n * Client groups that wait for seconds between batches make the share of\n * any one second jump between idle and saturated, and a worker that keeps\n * up has the odd busy second. Neither says what the load is.\n */\nconst SHARE_SMOOTHING = 0.3;\n\n/**\n * How much of the difference between the smoothed busy share and the target\n * each update corrects for. The smoothed share only shows part of what the\n * previous updates did, so correcting for all of the difference each time\n * would overshoot and oscillate.\n */\nconst CORRECTION_EXPONENT = 0.3;\n\n/**\n * The share of its time above which a worker has no time to spare. A share\n * target has to stay well clear of this to be held, which is why\n * `syncUpdateShareTarget` is at most 0.9 (or 1, which no worker is over).\n */\nconst SATURATED_SHARE = 0.95;\n\n/** The seconds that a worker has to be saturated for to be treated as such. */\nconst SATURATED_SECONDS = 2;\n\n/**\n * How much `1 + k` is scaled by each second that the worker is saturated.\n * A saturated worker does not show how much more its client groups would\n * use, which can be many times what it has, so the multiplier is raised by\n * more than the difference from the target calls for until it does.\n */\nconst SATURATED_CORRECTION = 1.3;\n\n/**\n * How often a worker that is pacing its client groups says so in its log,\n * and how long it has to have stopped for to say that it has.\n */\nconst PACING_LOG_INTERVAL_MS = 60_000;\n\n/**\n * The most updates that are made up for when nothing has asked the pacer\n * for a wait in a while, which is enough for the multiplier to get to 0.\n */\nconst MAX_CATCH_UP_UPDATES = 60;\n\n/**\n * Bounds the multiplier, which otherwise has no limit while a worker stays\n * over the target however long its client groups wait.\n */\nconst MAX_MULTIPLIER = 1_000_000;\n\nexport type BatchPacerOptions = {\n  /**\n   * The largest share of the worker's time that it may be busy before its\n   * client groups are paced.\n   */\n  readonly shareTarget: number;\n  /** The longest a client group waits between batches. */\n  readonly maxIntervalMs: number;\n  readonly now?: (() => number) | undefined;\n  /**\n   * The share of the time since the previous call that the worker was busy.\n   * By default, the utilization of the event loop.\n   */\n  readonly utilization?: (() => number) | undefined;\n  /** Where the worker says that it is pacing its client groups. */\n  readonly lc?: LogContext | undefined;\n};\n\nfunction eventLoopUtilization(): () => number {\n  let last = performance.eventLoopUtilization();\n  return () => {\n    const current = performance.eventLoopUtilization();\n    const {utilization} = performance.eventLoopUtilization(current, last);\n    last = current;\n    return utilization;\n  };\n}\n\n/**\n * Paces the batches (advancements, including the queries they hydrate\n * again) of the client groups of a sync worker, so that the worker is left\n * with time for new hydrations and connections, and gives the hydrations of\n * new queries priority over the batches.\n *\n * **Pacing.** After each batch, a client group waits {@link waitMs} before\n * it starts the next. The wait is a multiple of the batch's processing\n * time, so a client group that always has changes waiting is busy at most\n * `1 / (1 + k)` of the time, where `k` is the {@link multiplier}. Because\n * the wait is in proportion to a client group's own processing time, those\n * that cause the load are slowed the most, and cheap ones barely at all.\n *\n * The multiplier is shared by the client groups of the worker. Once per\n * second, it is scaled by how far the share of its time that the worker was\n * busy, smoothed over a few seconds, is from the target, and by more while\n * the worker has no time to spare. It never goes below 0, where no client\n * group waits. The share is the utilization of the worker's event loop, i.e. all\n * of what the worker does: what a batch costs is spread over the pipelines,\n * the CVR, the pokes and garbage collection, and is not measurable batch by\n * batch.\n *\n * **Priority of new hydrations.** The worker's time slices are handed out\n * by the pacer (see {@link yieldSlice}), and while the hydration of new\n * queries is waiting for one, most of them go to it rather than to batches.\n */\nexport class BatchPacer {\n  readonly #shareTarget: number;\n  readonly #maxIntervalMs: number;\n  readonly #now: () => number;\n  readonly #utilization: () => number;\n  readonly #lc: LogContext | undefined;\n\n  /** When the worker last logged that it is pacing, while it still is. */\n  #pacingLoggedAt: number | undefined;\n  /** Since when the worker has not been pacing, after it logged that it was. */\n  #notPacingSince: number | undefined;\n\n  #intervalStart: number;\n  #multiplier = 0;\n  /** The smoothed share of its time that the worker was busy. */\n  #share = 0;\n  #saturatedSeconds = 0;\n  /** The waits handed out since the last update, by whether at the cap. */\n  #waitsBelowMax = 0;\n  #waitsAtMax = 0;\n\n  /** The work that is waiting for a time slice, in the order it yielded. */\n  readonly #waiting = {\n    hydration: [] as (() => void)[],\n    batch: [] as (() => void)[],\n  };\n  #handingOutSlices = false;\n  /** The slices that have gone to hydrations since one went to a batch. */\n  #hydrationSlices = 0;\n\n  constructor({\n    shareTarget,\n    maxIntervalMs,\n    now,\n    utilization,\n    lc,\n  }: BatchPacerOptions) {\n    this.#shareTarget = shareTarget;\n    this.#maxIntervalMs = maxIntervalMs;\n    this.#now = now ?? (() => performance.now());\n    this.#utilization = utilization ?? eventLoopUtilization();\n    this.#lc = lc;\n    this.#intervalStart = this.#now();\n  }\n\n  /**\n   * How long a client group should wait before its next batch, after one\n   * that took `processingMs` to process. The multiplier changes while a\n   * client group waits, so a long wait should be taken in steps, asking\n   * again after each.\n   */\n  waitMs(processingMs: number): number {\n    this.#update();\n    const wait = this.#multiplier * processingMs;\n    if (wait < this.#maxIntervalMs) {\n      this.#waitsBelowMax++;\n      return wait;\n    }\n    this.#waitsAtMax++;\n    return this.#maxIntervalMs;\n  }\n\n  /** `k`: a client group waits `k` times the processing time of a batch. */\n  get multiplier(): number {\n    this.#update();\n    return this.#multiplier;\n  }\n\n  /**\n   * The share of its time that the worker was busy, smoothed over the last\n   * few seconds, as of the last update of the multiplier.\n   */\n  get share(): number {\n    this.#update();\n    return this.#share;\n  }\n\n  /** Updates the multiplier for each interval that has ended. */\n  #update(): void {\n    const now = this.#now();\n    const ended = Math.floor((now - this.#intervalStart) / UPDATE_INTERVAL_MS);\n    if (ended <= 0) {\n      return;\n    }\n    this.#intervalStart += ended * UPDATE_INTERVAL_MS;\n    // If more than one interval has ended, nothing asked the pacer for a\n    // wait in the meantime, and the utilization since then is that of each.\n    const latest = this.#utilization();\n    // Once every client group is told to wait the longest it can, raising\n    // the multiplier changes nothing, and only makes it take longer to come\n    // back down. A worker that is still over its target then is busy with\n    // something that making its client groups wait does not relieve.\n    const allAtMax = this.#waitsAtMax > 0 && this.#waitsBelowMax === 0;\n    this.#waitsAtMax = 0;\n    this.#waitsBelowMax = 0;\n    for (let i = Math.min(ended, MAX_CATCH_UP_UPDATES); i > 0; i--) {\n      this.#share += SHARE_SMOOTHING * (latest - this.#share);\n      // A worker that is allowed to be this busy is only raised the\n      // multiplier for what it is over its target by.\n      this.#saturatedSeconds =\n        latest >= SATURATED_SHARE && latest > this.#shareTarget\n          ? this.#saturatedSeconds + 1\n          : 0;\n      let correction =\n        this.#saturatedSeconds >= SATURATED_SECONDS\n          ? SATURATED_CORRECTION\n          : (this.#share / this.#shareTarget) ** CORRECTION_EXPONENT;\n      if (allAtMax && correction > 1) {\n        correction = 1;\n      }\n      this.#multiplier = Math.min(\n        Math.max((1 + this.#multiplier) * correction - 1, 0),\n        MAX_MULTIPLIER,\n      );\n    }\n    this.#logPacing(now, allAtMax);\n  }\n\n  /**\n   * Says in the worker's log when it starts to pace its client groups, once\n   * a minute while it does, and when it has stopped. The pacing of a client\n   * group is logged by its view syncer.\n   */\n  #logPacing(now: number, allAtMax: boolean): void {\n    const lc = this.#lc;\n    if (lc === undefined) {\n      return;\n    }\n    const state = {\n      busyShare: this.#share,\n      shareTarget: this.#shareTarget,\n      waitMultiplier: this.#multiplier,\n      maxBatchIntervalMs: this.#maxIntervalMs,\n    };\n    if (this.#multiplier > 0) {\n      this.#notPacingSince = undefined;\n      if (this.#pacingLoggedAt === undefined) {\n        lc.info?.(\n          'Sync worker is over its share target: its client groups wait between batches',\n          state,\n        );\n        this.#pacingLoggedAt = now;\n      } else if (now - this.#pacingLoggedAt >= PACING_LOG_INTERVAL_MS) {\n        if (allAtMax && this.#share > this.#shareTarget) {\n          // Waiting longer is not an option, so pacing cannot bring the\n          // worker to its target: it has more to do than it has time for.\n          lc.warn?.(\n            'Sync worker is over its share target although every client group waits the longest it may',\n            state,\n          );\n        } else {\n          lc.info?.('Sync worker is pacing its client groups', state);\n        }\n        this.#pacingLoggedAt = now;\n      }\n    } else if (this.#pacingLoggedAt !== undefined) {\n      // A worker around its target goes in and out of pacing. It only says\n      // that it has stopped once it has stayed out.\n      this.#notPacingSince ??= now;\n      if (now - this.#notPacingSince >= PACING_LOG_INTERVAL_MS) {\n        lc.info?.(\n          'Sync worker is under its share target: its client groups no longer wait',\n          state,\n        );\n        this.#pacingLoggedAt = undefined;\n        this.#notPacingSince = undefined;\n      }\n    }\n  }\n\n  /**\n   * Yields the worker to the other work that is waiting for a time slice,\n   * and resolves when it is the caller's turn again. One piece of work runs\n   * per turn of the event loop, as when the work of a worker's client groups\n   * simply takes turns, except that the hydration of new queries goes first:\n   * while both kinds of work are waiting, `HYDRATION_PRIORITY` slices go to\n   * hydrations for each one that goes to a batch.\n   *\n   * Batches are never made to wait for anything but a slice. If no\n   * hydration is waiting for one (because none is running, or because those\n   * that are running are waiting for I/O), batches get every slice. And\n   * however many hydrations there are, batches keep a share of the slices,\n   * so they keep moving, and so do the snapshots of the replica that they\n   * hold.\n   */\n  yieldSlice(kind: 'hydration' | 'batch'): Promise<void> {\n    return new Promise(resolve => {\n      this.#waiting[kind].push(resolve);\n      if (!this.#handingOutSlices) {\n        this.#handingOutSlices = true;\n        setImmediate(this.#nextSlice);\n      }\n    });\n  }\n\n  readonly #nextSlice = () => {\n    const {hydration, batch} = this.#waiting;\n    let next: (() => void) | undefined;\n    if (\n      hydration.length > 0 &&\n      (batch.length === 0 || this.#hydrationSlices < HYDRATION_PRIORITY)\n    ) {\n      next = hydration.shift();\n      this.#hydrationSlices++;\n    } else if (batch.length > 0) {\n      next = batch.shift();\n      this.#hydrationSlices = 0;\n    }\n    if (next === undefined) {\n      this.#handingOutSlices = false;\n      return;\n    }\n    next();\n    // After the work that was just resumed has run its slice.\n    setImmediate(this.#nextSlice);\n  };\n}\n\n/**\n * How many time slices go to the hydration of new queries for each one that\n * goes to a batch, while both are waiting for one.\n */\nconst HYDRATION_PRIORITY = 9;\n"],"mappings":";;;AAIA,IAAM,qBAAqB;;;;;;;AAQ3B,IAAM,kBAAkB;;;;;;;AAQxB,IAAM,sBAAsB;;;;;;AAO5B,IAAM,kBAAkB;;AAGxB,IAAM,oBAAoB;;;;;;;AAQ1B,IAAM,uBAAuB;;;;;AAM7B,IAAM,yBAAyB;;;;;AAM/B,IAAM,uBAAuB;;;;;AAM7B,IAAM,iBAAiB;AAoBvB,SAAS,uBAAqC;CAC5C,IAAI,OAAO,YAAY,qBAAqB;CAC5C,aAAa;EACX,MAAM,UAAU,YAAY,qBAAqB;EACjD,MAAM,EAAC,gBAAe,YAAY,qBAAqB,SAAS,IAAI;EACpE,OAAO;EACP,OAAO;CACT;AACF;;;;;;;;;;;;;;;;;;;;;;;;;;;AA4BA,IAAa,aAAb,MAAwB;CACtB;CACA;CACA;CACA;CACA;;CAGA;;CAEA;CAEA;CACA,cAAc;;CAEd,SAAS;CACT,oBAAoB;;CAEpB,iBAAiB;CACjB,cAAc;;CAGd,WAAoB;EAClB,WAAW,CAAC;EACZ,OAAO,CAAC;CACV;CACA,oBAAoB;;CAEpB,mBAAmB;CAEnB,YAAY,EACV,aACA,eACA,KACA,aACA,MACoB;EACpB,KAAKA,eAAe;EACpB,KAAKC,iBAAiB;EACtB,KAAKC,OAAO,cAAc,YAAY,IAAI;EAC1C,KAAKC,eAAe,eAAe,qBAAqB;EACxD,KAAKC,MAAM;EACX,KAAKE,iBAAiB,KAAKJ,KAAK;CAClC;;;;;;;CAQA,OAAO,cAA8B;EACnC,KAAKK,QAAQ;EACb,MAAM,OAAO,KAAKC,cAAc;EAChC,IAAI,OAAO,KAAKP,gBAAgB;GAC9B,KAAKQ;GACL,OAAO;EACT;EACA,KAAKC;EACL,OAAO,KAAKT;CACd;;CAGA,IAAI,aAAqB;EACvB,KAAKM,QAAQ;EACb,OAAO,KAAKC;CACd;;;;;CAMA,IAAI,QAAgB;EAClB,KAAKD,QAAQ;EACb,OAAO,KAAKI;CACd;;CAGA,UAAgB;EACd,MAAM,MAAM,KAAKT,KAAK;EACtB,MAAM,QAAQ,KAAK,OAAO,MAAM,KAAKI,kBAAkB,kBAAkB;EACzE,IAAI,SAAS,GACX;EAEF,KAAKA,kBAAkB,QAAQ;EAG/B,MAAM,SAAS,KAAKH,aAAa;EAKjC,MAAM,WAAW,KAAKO,cAAc,KAAK,KAAKD,mBAAmB;EACjE,KAAKC,cAAc;EACnB,KAAKD,iBAAiB;EACtB,KAAK,IAAI,IAAI,KAAK,IAAI,OAAO,oBAAoB,GAAG,IAAI,GAAG,KAAK;GAC9D,KAAKE,UAAU,mBAAmB,SAAS,KAAKA;GAGhD,KAAKC,oBACH,UAAU,mBAAmB,SAAS,KAAKZ,eACvC,KAAKY,oBAAoB,IACzB;GACN,IAAI,aACF,KAAKA,qBAAqB,oBACtB,wBACC,KAAKD,SAAS,KAAKX,iBAAiB;GAC3C,IAAI,YAAY,aAAa,GAC3B,aAAa;GAEf,KAAKQ,cAAc,KAAK,IACtB,KAAK,KAAK,IAAI,KAAKA,eAAe,aAAa,GAAG,CAAC,GACnD,cACF;EACF;EACA,KAAKK,WAAW,KAAK,QAAQ;CAC/B;;;;;;CAOA,WAAW,KAAa,UAAyB;EAC/C,MAAM,KAAK,KAAKT;EAChB,IAAI,OAAO,KAAA,GACT;EAEF,MAAM,QAAQ;GACZ,WAAW,KAAKO;GAChB,aAAa,KAAKX;GAClB,gBAAgB,KAAKQ;GACrB,oBAAoB,KAAKP;EAC3B;EACA,IAAI,KAAKO,cAAc,GAAG;GACxB,KAAKM,kBAAkB,KAAA;GACvB,IAAI,KAAKC,oBAAoB,KAAA,GAAW;IACtC,GAAG,OACD,gFACA,KACF;IACA,KAAKA,kBAAkB;GACzB,OAAO,IAAI,MAAM,KAAKA,mBAAmB,wBAAwB;IAC/D,IAAI,YAAY,KAAKJ,SAAS,KAAKX,cAGjC,GAAG,OACD,6FACA,KACF;SAEA,GAAG,OAAO,2CAA2C,KAAK;IAE5D,KAAKe,kBAAkB;GACzB;EACF,OAAO,IAAI,KAAKA,oBAAoB,KAAA,GAAW;GAG7C,KAAKD,oBAAoB;GACzB,IAAI,MAAM,KAAKA,mBAAmB,wBAAwB;IACxD,GAAG,OACD,2EACA,KACF;IACA,KAAKC,kBAAkB,KAAA;IACvB,KAAKD,kBAAkB,KAAA;GACzB;EACF;CACF;;;;;;;;;;;;;;;;CAiBA,WAAW,MAA4C;EACrD,OAAO,IAAI,SAAQ,YAAW;GAC5B,KAAKT,SAAS,MAAM,KAAK,OAAO;GAChC,IAAI,CAAC,KAAKW,mBAAmB;IAC3B,KAAKA,oBAAoB;IACzB,aAAa,KAAKC,UAAU;GAC9B;EACF,CAAC;CACH;CAEA,mBAA4B;EAC1B,MAAM,EAAC,WAAW,UAAS,KAAKZ;EAChC,IAAI;EACJ,IACE,UAAU,SAAS,MAClB,MAAM,WAAW,KAAK,KAAKa,mBAAmB,qBAC/C;GACA,OAAO,UAAU,MAAM;GACvB,KAAKA;EACP,OAAO,IAAI,MAAM,SAAS,GAAG;GAC3B,OAAO,MAAM,MAAM;GACnB,KAAKA,mBAAmB;EAC1B;EACA,IAAI,SAAS,KAAA,GAAW;GACtB,KAAKF,oBAAoB;GACzB;EACF;EACA,KAAK;EAEL,aAAa,KAAKC,UAAU;CAC9B;AACF;;;;;AAMA,IAAM,qBAAqB"}

package/out/zero-cache/src/services/view-syncer/cvr.d.ts

@@ -204,6 +204,32 @@ export declare class CVRQueryDrivenUpdater extends CVRUpdater {
      * rows of the aborted queries have been {@link received}.
      */
     abortExecutedQueries(lc: LogContext, queryIDs: readonly string[]): Promise<PatchToVersion[]>;
+    /**
+     * Retracts what queries have contributed to this update, so that they can
+     * be executed again from scratch in the same update. This is for a query
+     * whose changes were being received incrementally (see
+     * `PipelineDriver.advance()`) until its pipeline was abandoned in favor of
+     * hydrating it again.
+     *
+     * Rows received so far in this update have the queries' references
+     * stripped, as in {@link abortExecutedQueries}, except that a row left
+     * with no references is only deleted by {@link deleteUnreferencedRows}, if
+     * it is still unreferenced by then. Until then the clients keep the row
+     * and the patches they received for it in this update, so a row that is
+     * received again unchanged needs no patch at all. The queries are then
+     * tracked as executed, without changing them in the CVR: the references
+     * they held before this update are replaced, rather than adjusted, by the
+     * rows subsequently {@link received} for them, and
+     * {@link deleteUnreferencedRows} drops the references of rows that are not
+     * received again.
+     *
+     * Unlike {@link trackQueries}, this can be called on an updater that has
+     * not tracked any queries, and after rows have been received. All rows
+     * that the queries' pipelines have output must have been {@link received}
+     * first, and {@link deleteUnreferencedRows} must be called once the
+     * queries' rows have been received again.
+     */
+    retractQueries(lc: LogContext, queryIDs: readonly string[]): Promise<PatchToVersion[]>;
     updatedVersion(): CVRVersion;
     /**
      * Public alias for {@link _ensureNewVersion}. Forces a `configVersion` bump
@@ -221,7 +247,7 @@ export declare class CVRQueryDrivenUpdater extends CVRUpdater {
      * returns (put) patches to be returned to update their state, versioned by
      * patchVersion so that only the patches new to the clients are sent.
      */
-    received(_lc: LogContext, rows: Map<RowID, RowUpdate>): Promise<PatchToVersion[]>;
+    received(lc: LogContext, rows: Map<RowID, RowUpdate>): Promise<PatchToVersion[]>;
     /**
      * Computes and updates the row records based on:
      * * The {@link #executed} queries

package/out/zero-cache/src/services/view-syncer/cvr.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"cvr.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/view-syncer/cvr.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAGjD,OAAO,EAAC,KAAK,UAAU,EAAC,MAAM,uCAAuC,CAAC;AAGtE,OAAO,EAEL,KAAK,iBAAiB,EACvB,MAAM,gCAAgC,CAAC;AAQxC,OAAO,KAAK,EAAC,GAAG,EAAC,MAAM,sCAAsC,CAAC;AAC9D,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,gDAAgD,CAAC;AASjF,OAAO,KAAK,EAAC,WAAW,EAAC,MAAM,6BAA6B,CAAC;AAE7D,OAAO,EAAiB,KAAK,OAAO,EAAC,MAAM,uBAAuB,CAAC;AACnE,OAAO,KAAK,EAAQ,cAAc,EAAC,MAAM,qBAAqB,CAAC;AAC/D,OAAO,EAAC,KAAK,aAAa,EAAE,KAAK,QAAQ,EAAC,MAAM,gBAAgB,CAAC;AACjE,OAAO,EAML,KAAK,YAAY,EAEjB,KAAK,UAAU,EACf,KAAK,mBAAmB,EACxB,KAAK,WAAW,EAChB,KAAK,KAAK,EAEX,MAAM,mBAAmB,CAAC;AAE3B,OAAO,EAAmB,KAAK,QAAQ,EAAC,MAAM,gBAAgB,CAAC;AAE/D,MAAM,MAAM,SAAS,GAAG;IACtB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,UAAU,CAAC;IACtB,SAAS,EAAE;QAAC,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAAA;KAAC,CAAC;CACrC,CAAC;AAEF,wCAAwC;AACxC,MAAM,MAAM,GAAG,GAAG;IAChB,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,UAAU,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,QAAQ,CAAC;IACnB,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;IACtC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;IACrC,YAAY,EAAE,YAAY,GAAG,IAAI,CAAC;IAClC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;CAC1B,CAAC;AAEF,mCAAmC;AAEnC,MAAM,MAAM,WAAW,GAAG;IACxB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,OAAO,EAAE,UAAU,CAAC;IAC7B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC;IAC5B,QAAQ,CAAC,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IACvC,QAAQ,CAAC,OAAO,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC,CAAC;IACzD,QAAQ,CAAC,OAAO,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC,CAAC;IACxD,QAAQ,CAAC,YAAY,EAAE,YAAY,GAAG,IAAI,CAAC;IAC3C,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;CACnC,CAAC;AAcF,wBAAgB,uBAAuB,CACrC,cAAc,EAAE,MAAM,EACtB,aAAa,EAAE,MAAM,GACpB,mBAAmB,CA+BrB;AAED;;;;;;;;GAQG;AACH,qBAAa,UAAU;IACrB,SAAS,CAAC,QAAQ,CAAC,KAAK,EAAE,WAAW,CAAC;IACtC,SAAS,CAAC,QAAQ,CAAC,IAAI,EAAE,GAAG,CAAC;IAE7B,SAAS,CAAC,QAAQ,CAAC,SAAS,EAAE,QAAQ,CAAC;IAEvC;;;OAGG;IACH,YACE,QAAQ,EAAE,QAAQ,EAClB,GAAG,EAAE,WAAW,EAChB,cAAc,EAAE,MAAM,GAAG,IAAI,EAQ9B;IAED,SAAS,CAAC,eAAe,IAAI,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC,CAKxD;IAED,SAAS,CAAC,cAAc,CAAC,EAAE,EAAE,MAAM,GAAG,YAAY,CAYjD;IAED,SAAS,CAAC,eAAe,IAAI,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,CAKvD;IAED,SAAS,CAAC,aAAa,CAAC,EAAE,EAAE,MAAM,GAAG,WAAW,CAc/C;IAED,SAAS,CAAC,WAAW,CAAC,OAAO,EAAE,UAAU;;;MAOxC;IAED;;;;OAIG;IACH,SAAS,CAAC,iBAAiB,IAAI,UAAU,CAKxC;IAED;;;OAGG;IACG,KAAK,CACT,EAAE,EAAE,UAAU,EACd,eAAe,EAAE,MAAM,EACvB,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,QAAQ,EAClB,UAAU,UAAQ,GACjB,OAAO,CAAC;QACT,GAAG,EAAE,WAAW,CAAC;QACjB,OAAO,EAAE,aAAa,GAAG,KAAK,CAAC;KAChC,CAAC,CAcD;CACF;AAED;;;;GAIG;AACH,qBAAa,sBAAuB,SAAQ,UAAU;;IAGpD,YAAY,QAAQ,EAAE,QAAQ,EAAE,GAAG,EAAE,WAAW,EAAE,KAAK,EAAE,OAAO,EAG/D;IAED,YAAY,CAAC,EAAE,EAAE,MAAM,GAAG,YAAY,CAmDrC;IAED,eAAe,CAAC,EAAE,EAAE,UAAU,EAAE,YAAY,EAAE,YAAY,QAwBzD;IAED,YAAY,CAAC,EAAE,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,QAgB7C;IAED,iBAAiB,CACf,QAAQ,EAAE,MAAM,EAChB,OAAO,EAAE,QAAQ,CAAC;QAChB,IAAI,EAAE,MAAM,CAAC;QACb,GAAG,CAAC,EAAE,GAAG,GAAG,SAAS,CAAC;QACtB,IAAI,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;QAC1B,IAAI,CAAC,EAAE,SAAS,iBAAiB,EAAE,GAAG,SAAS,CAAC;QAChD,GAAG,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;KAC1B,CAAC,EAAE,GACH,cAAc,EAAE,CA4FlB;IAED,4BAA4B,CAC1B,QAAQ,EAAE,MAAM,EAChB,WAAW,EAAE,MAAM,EAAE,EACrB,QAAQ,EAAE,QAAQ,GACjB,cAAc,EAAE,CAElB;IAED,oBAAoB,CAClB,QAAQ,EAAE,MAAM,EAChB,WAAW,EAAE,MAAM,EAAE,GACpB,cAAc,EAAE,CAElB;IAuED,mBAAmB,CAAC,QAAQ,EAAE,MAAM,GAAG,cAAc,EAAE,CAGtD;IAED,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,GAAG,cAAc,EAAE,CAwBnE;CACF;AAED,KAAK,IAAI,GAAG,MAAM,CAAC;AACnB,MAAM,MAAM,MAAM,GAAG,MAAM,CAAC;AAC5B,MAAM,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;AAO7C;;;;GAIG;AACH,MAAM,WAAW,uBAAuB;IACtC;;;;OAIG;IACH,uBAAuB,IAAI,QAAQ,CAAC,MAAM,CAAC,CAAC;IAE5C;;;OAGG;IACH,eAAe,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAAC;IAErD,6DAA6D;IAC7D,4BAA4B,IAAI,IAAI,CAAC;CACtC;AAED;;;;;;;;;;;;;;GAcG;AACH,qBAAa,qBAAsB,SAAQ,UAAU;;IAWnD;;;;;OAKG;IACH,YACE,QAAQ,EAAE,QAAQ,EAClB,GAAG,EAAE,WAAW,EAChB,YAAY,EAAE,WAAW,EACzB,cAAc,EAAE,MAAM,EACtB,eAAe,CAAC,EAAE,uBAAuB,EAoB1C;IAED;;;;;;;;;;;;OAYG;IACH,YAAY,CACV,EAAE,EAAE,UAAU,EACd,QAAQ,EAAE;QAAC,EAAE,EAAE,MAAM,CAAC;QAAC,kBAAkB,EAAE,MAAM,CAAA;KAAC,EAAE,EACpD,OAAO,EAAE;QAAC,EAAE,EAAE,MAAM,CAAA;KAAC,EAAE,GACtB;QAAC,UAAU,EAAE,UAAU,CAAC;QAAC,YAAY,EAAE,cAAc,EAAE,CAAA;KAAC,CA6B1D;IAuFD;;;;;;;;;;;OAWG;IACH,oBAAoB,CAAC,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,GAAG,cAAc,EAAE,CAgCjE;IAED;;;;;;;;;;;;;;;OAeG;IACG,oBAAoB,CACxB,EAAE,EAAE,UAAU,EACd,QAAQ,EAAE,SAAS,MAAM,EAAE,GAC1B,OAAO,CAAC,cAAc,EAAE,CAAC,CAqD3B;IA2DD,cAAc,IAAI,UAAU,CAE3B;IAED;;;;OAIG;IACH,gBAAgB,IAAI,UAAU,CAE7B;IAEc,KAAK,CAClB,EAAE,EAAE,UAAU,EACd,eAAe,EAAE,MAAM,EACvB,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,QAAQ,EAClB,UAAU,UAAQ,GACjB,OAAO,CAAC;QAAC,GAAG,EAAE,WAAW,CAAC;QAAC,OAAO,EAAE,aAAa,GAAG,KAAK,CAAA;KAAC,CAAC,CAsC7D;IAED;;;;;OAKG;IACH,QAAQ,CACN,GAAG,EAAE,UAAU,EACf,IAAI,EAAE,GAAG,CAAC,KAAK,EAAE,SAAS,CAAC,GAC1B,OAAO,CAAC,cAAc,EAAE,CAAC,CA0G3B;IAED;;;;;;;;;;;OAWG;IACH,sBAAsB,CAAC,EAAE,CAAC,EAAE,UAAU,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC,CA2CjE;CA8CF;AAgCD;;;;;;;GAOG;AACH,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,GAAG,GAAG;IAC5C,IAAI,EAAE,MAAM,CAAC;IACb,aAAa,EAAE,QAAQ,CAAC;IACxB,GAAG,EAAE,MAAM,CAAC;CACb,EAAE,CA+DF;AAED,MAAM,MAAM,oBAAoB,CAAC,CAAC,SAAS,WAAW,GAAG,WAAW,IAAI;IACtE,QAAQ,EAAE,CAAC,EAAE,CAAC;IACd;;;OAGG;IACH,QAAQ,EAAE,OAAO,CAAC,CAAC,EAAE,mBAAmB,CAAC,EAAE,CAAC;CAC7C,CAAC;AAEF;;;;GAIG;AACH,wBAAgB,2BAA2B,CAAC,CAAC,SAAS,WAAW,EAC/D,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC,GACnB,oBAAoB,CAAC,CAAC,CAAC,CA2CzB;AAED,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,GAAG,GAAG,QAAQ,GAAG,SAAS,CAS/D"}
+{"version":3,"file":"cvr.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/view-syncer/cvr.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAGjD,OAAO,EAAC,KAAK,UAAU,EAAC,MAAM,uCAAuC,CAAC;AAGtE,OAAO,EAEL,KAAK,iBAAiB,EACvB,MAAM,gCAAgC,CAAC;AAQxC,OAAO,KAAK,EAAC,GAAG,EAAC,MAAM,sCAAsC,CAAC;AAC9D,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,gDAAgD,CAAC;AASjF,OAAO,KAAK,EAAC,WAAW,EAAC,MAAM,6BAA6B,CAAC;AAE7D,OAAO,EAAiB,KAAK,OAAO,EAAC,MAAM,uBAAuB,CAAC;AACnE,OAAO,KAAK,EAAQ,cAAc,EAAC,MAAM,qBAAqB,CAAC;AAC/D,OAAO,EAAC,KAAK,aAAa,EAAE,KAAK,QAAQ,EAAC,MAAM,gBAAgB,CAAC;AACjE,OAAO,EAML,KAAK,YAAY,EAEjB,KAAK,UAAU,EACf,KAAK,mBAAmB,EACxB,KAAK,WAAW,EAChB,KAAK,KAAK,EAEX,MAAM,mBAAmB,CAAC;AAE3B,OAAO,EAAmB,KAAK,QAAQ,EAAC,MAAM,gBAAgB,CAAC;AAE/D,MAAM,MAAM,SAAS,GAAG;IACtB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,UAAU,CAAC;IACtB,SAAS,EAAE;QAAC,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAAA;KAAC,CAAC;CACrC,CAAC;AAEF,wCAAwC;AACxC,MAAM,MAAM,GAAG,GAAG;IAChB,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,UAAU,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,QAAQ,CAAC;IACnB,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;IACtC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;IACrC,YAAY,EAAE,YAAY,GAAG,IAAI,CAAC;IAClC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;CAC1B,CAAC;AAEF,mCAAmC;AAEnC,MAAM,MAAM,WAAW,GAAG;IACxB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,OAAO,EAAE,UAAU,CAAC;IAC7B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC;IAC5B,QAAQ,CAAC,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IACvC,QAAQ,CAAC,OAAO,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC,CAAC;IACzD,QAAQ,CAAC,OAAO,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC,CAAC;IACxD,QAAQ,CAAC,YAAY,EAAE,YAAY,GAAG,IAAI,CAAC;IAC3C,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;CACnC,CAAC;AAcF,wBAAgB,uBAAuB,CACrC,cAAc,EAAE,MAAM,EACtB,aAAa,EAAE,MAAM,GACpB,mBAAmB,CA+BrB;AAED;;;;;;;;GAQG;AACH,qBAAa,UAAU;IACrB,SAAS,CAAC,QAAQ,CAAC,KAAK,EAAE,WAAW,CAAC;IACtC,SAAS,CAAC,QAAQ,CAAC,IAAI,EAAE,GAAG,CAAC;IAE7B,SAAS,CAAC,QAAQ,CAAC,SAAS,EAAE,QAAQ,CAAC;IAEvC;;;OAGG;IACH,YACE,QAAQ,EAAE,QAAQ,EAClB,GAAG,EAAE,WAAW,EAChB,cAAc,EAAE,MAAM,GAAG,IAAI,EAQ9B;IAED,SAAS,CAAC,eAAe,IAAI,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC,CAKxD;IAED,SAAS,CAAC,cAAc,CAAC,EAAE,EAAE,MAAM,GAAG,YAAY,CAYjD;IAED,SAAS,CAAC,eAAe,IAAI,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,CAKvD;IAED,SAAS,CAAC,aAAa,CAAC,EAAE,EAAE,MAAM,GAAG,WAAW,CAc/C;IAED,SAAS,CAAC,WAAW,CAAC,OAAO,EAAE,UAAU;;;MAOxC;IAED;;;;OAIG;IACH,SAAS,CAAC,iBAAiB,IAAI,UAAU,CAKxC;IAED;;;OAGG;IACG,KAAK,CACT,EAAE,EAAE,UAAU,EACd,eAAe,EAAE,MAAM,EACvB,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,QAAQ,EAClB,UAAU,UAAQ,GACjB,OAAO,CAAC;QACT,GAAG,EAAE,WAAW,CAAC;QACjB,OAAO,EAAE,aAAa,GAAG,KAAK,CAAC;KAChC,CAAC,CAcD;CACF;AAED;;;;GAIG;AACH,qBAAa,sBAAuB,SAAQ,UAAU;;IAGpD,YAAY,QAAQ,EAAE,QAAQ,EAAE,GAAG,EAAE,WAAW,EAAE,KAAK,EAAE,OAAO,EAG/D;IAED,YAAY,CAAC,EAAE,EAAE,MAAM,GAAG,YAAY,CAmDrC;IAED,eAAe,CAAC,EAAE,EAAE,UAAU,EAAE,YAAY,EAAE,YAAY,QAwBzD;IAED,YAAY,CAAC,EAAE,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,QAgB7C;IAED,iBAAiB,CACf,QAAQ,EAAE,MAAM,EAChB,OAAO,EAAE,QAAQ,CAAC;QAChB,IAAI,EAAE,MAAM,CAAC;QACb,GAAG,CAAC,EAAE,GAAG,GAAG,SAAS,CAAC;QACtB,IAAI,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;QAC1B,IAAI,CAAC,EAAE,SAAS,iBAAiB,EAAE,GAAG,SAAS,CAAC;QAChD,GAAG,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;KAC1B,CAAC,EAAE,GACH,cAAc,EAAE,CA4FlB;IAED,4BAA4B,CAC1B,QAAQ,EAAE,MAAM,EAChB,WAAW,EAAE,MAAM,EAAE,EACrB,QAAQ,EAAE,QAAQ,GACjB,cAAc,EAAE,CAElB;IAED,oBAAoB,CAClB,QAAQ,EAAE,MAAM,EAChB,WAAW,EAAE,MAAM,EAAE,GACpB,cAAc,EAAE,CAElB;IAuED,mBAAmB,CAAC,QAAQ,EAAE,MAAM,GAAG,cAAc,EAAE,CAGtD;IAED,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,GAAG,cAAc,EAAE,CAwBnE;CACF;AAED,KAAK,IAAI,GAAG,MAAM,CAAC;AACnB,MAAM,MAAM,MAAM,GAAG,MAAM,CAAC;AAC5B,MAAM,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;AAO7C;;;;GAIG;AACH,MAAM,WAAW,uBAAuB;IACtC;;;;OAIG;IACH,uBAAuB,IAAI,QAAQ,CAAC,MAAM,CAAC,CAAC;IAE5C;;;OAGG;IACH,eAAe,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAAC;IAErD,6DAA6D;IAC7D,4BAA4B,IAAI,IAAI,CAAC;CACtC;AAED;;;;;;;;;;;;;;GAcG;AACH,qBAAa,qBAAsB,SAAQ,UAAU;;IAkBnD;;;;;OAKG;IACH,YACE,QAAQ,EAAE,QAAQ,EAClB,GAAG,EAAE,WAAW,EAChB,YAAY,EAAE,WAAW,EACzB,cAAc,EAAE,MAAM,EACtB,eAAe,CAAC,EAAE,uBAAuB,EAoB1C;IAED;;;;;;;;;;;;OAYG;IACH,YAAY,CACV,EAAE,EAAE,UAAU,EACd,QAAQ,EAAE;QAAC,EAAE,EAAE,MAAM,CAAC;QAAC,kBAAkB,EAAE,MAAM,CAAA;KAAC,EAAE,EACpD,OAAO,EAAE;QAAC,EAAE,EAAE,MAAM,CAAA;KAAC,EAAE,GACtB;QAAC,UAAU,EAAE,UAAU,CAAC;QAAC,YAAY,EAAE,cAAc,EAAE,CAAA;KAAC,CA6B1D;IAuFD;;;;;;;;;;;OAWG;IACH,oBAAoB,CAAC,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,GAAG,cAAc,EAAE,CAgCjE;IAED;;;;;;;;;;;;;;;OAeG;IACG,oBAAoB,CACxB,EAAE,EAAE,UAAU,EACd,QAAQ,EAAE,SAAS,MAAM,EAAE,GAC1B,OAAO,CAAC,cAAc,EAAE,CAAC,CAyB3B;IAwCD;;;;;;;;;;;;;;;;;;;;;;;;OAwBG;IACG,cAAc,CAClB,EAAE,EAAE,UAAU,EACd,QAAQ,EAAE,SAAS,MAAM,EAAE,GAC1B,OAAO,CAAC,cAAc,EAAE,CAAC,CAoC3B;IA2DD,cAAc,IAAI,UAAU,CAE3B;IAED;;;;OAIG;IACH,gBAAgB,IAAI,UAAU,CAE7B;IAEc,KAAK,CAClB,EAAE,EAAE,UAAU,EACd,eAAe,EAAE,MAAM,EACvB,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,QAAQ,EAClB,UAAU,UAAQ,GACjB,OAAO,CAAC;QAAC,GAAG,EAAE,WAAW,CAAC;QAAC,OAAO,EAAE,aAAa,GAAG,KAAK,CAAA;KAAC,CAAC,CAsC7D;IAED;;;;;OAKG;IACH,QAAQ,CACN,EAAE,EAAE,UAAU,EACd,IAAI,EAAE,GAAG,CAAC,KAAK,EAAE,SAAS,CAAC,GAC1B,OAAO,CAAC,cAAc,EAAE,CAAC,CAE3B;IA4HD;;;;;;;;;;;OAWG;IACH,sBAAsB,CAAC,EAAE,CAAC,EAAE,UAAU,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC,CAwDjE;CA8CF;AAgCD;;;;;;;GAOG;AACH,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,GAAG,GAAG;IAC5C,IAAI,EAAE,MAAM,CAAC;IACb,aAAa,EAAE,QAAQ,CAAC;IACxB,GAAG,EAAE,MAAM,CAAC;CACb,EAAE,CA+DF;AAED,MAAM,MAAM,oBAAoB,CAAC,CAAC,SAAS,WAAW,GAAG,WAAW,IAAI;IACtE,QAAQ,EAAE,CAAC,EAAE,CAAC;IACd;;;OAGG;IACH,QAAQ,EAAE,OAAO,CAAC,CAAC,EAAE,mBAAmB,CAAC,EAAE,CAAC;CAC7C,CAAC;AAEF;;;;GAIG;AACH,wBAAgB,2BAA2B,CAAC,CAAC,SAAS,WAAW,EAC/D,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC,GACnB,oBAAoB,CAAC,CAAC,CAAC,CA2CzB;AAED,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,GAAG,GAAG,QAAQ,GAAG,SAAS,CAS/D"}