@rocicorp/zero 0.26.0-canary.7 → 0.26.0

package/out/zero-cache/src/services/change-source/pg/schema/shard.js.map

@@ -1 +1 @@
-{"version":3,"file":"shard.js","sources":["../../../../../../../../zero-cache/src/services/change-source/pg/schema/shard.ts"],"sourcesContent":["import {PG_INSUFFICIENT_PRIVILEGE} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport {literal} from 'pg-format';\nimport postgres from 'postgres';\nimport {assert} from '../../../../../../shared/src/asserts.ts';\nimport * as v from '../../../../../../shared/src/valita.ts';\nimport {Default} from '../../../../db/postgres-replica-identity-enum.ts';\nimport type {PostgresDB, PostgresTransaction} from '../../../../types/pg.ts';\nimport type {AppID, ShardConfig, ShardID} from '../../../../types/shards.ts';\nimport {appSchema, check, upstreamSchema} from '../../../../types/shards.ts';\nimport {id} from '../../../../types/sql.ts';\nimport {createEventTriggerStatements} from './ddl.ts';\nimport {\n  getPublicationInfo,\n  publishedSchema,\n  type PublicationInfo,\n  type PublishedSchema,\n} from './published.ts';\nimport {validate} from './validation.ts';\n\n/**\n * PostgreSQL unquoted identifiers must start with a letter or underscore\n * and contain only letters, digits, and underscores.\n */\nconst VALID_PUBLICATION_NAME = /^[a-zA-Z_][a-zA-Z0-9_]*$/;\n\n/**\n * Validates that a publication name is a valid PostgreSQL identifier.\n * This provides defense-in-depth against SQL injection when publication\n * names are used in replication commands.\n */\nexport function validatePublicationName(name: string): void {\n  if (!VALID_PUBLICATION_NAME.test(name)) {\n    throw new Error(\n      `Invalid publication name \"${name}\". Publication names must start with a letter or underscore ` +\n        `and contain only letters, digits, and underscores.`,\n    );\n  }\n  if (name.length > 63) {\n    throw new Error(\n      `Publication name \"${name}\" exceeds PostgreSQL's 63-character identifier limit.`,\n    );\n  }\n}\n\nexport function internalPublicationPrefix({appID}: AppID) {\n  return `_${appID}_`;\n}\n\nexport function legacyReplicationSlot({appID, shardNum}: ShardID) {\n  return `${appID}_${shardNum}`;\n}\n\nexport function replicationSlotPrefix(shard: ShardID) {\n  const {appID, shardNum} = check(shard);\n  return `${appID}_${shardNum}_`;\n}\n\n/**\n * An expression used to match replication slots in the shard\n * in a Postgres `LIKE` operator.\n */\nexport function replicationSlotExpression(shard: ShardID) {\n  // Underscores have a special meaning in LIKE values\n  // so they have to be escaped.\n  return `${replicationSlotPrefix(shard)}%`.replaceAll('_', '\\\\_');\n}\n\nexport function newReplicationSlot(shard: ShardID) {\n  return replicationSlotPrefix(shard) + Date.now();\n}\n\nfunction defaultPublicationName(appID: string, shardID: string | number) {\n  return `_${appID}_public_${shardID}`;\n}\n\nexport function metadataPublicationName(\n  appID: string,\n  shardID: string | number,\n) {\n  return `_${appID}_metadata_${shardID}`;\n}\n\n// The GLOBAL_SETUP must be idempotent as it can be run multiple times for different shards.\nfunction globalSetup(appID: AppID): string {\n  const app = id(appSchema(appID));\n\n  return /*sql*/ `\n  CREATE SCHEMA IF NOT EXISTS ${app};\n\n  CREATE TABLE IF NOT EXISTS ${app}.permissions (\n    \"permissions\" JSONB,\n    \"hash\"        TEXT,\n\n    -- Ensure that there is only a single row in the table.\n    -- Application code can be agnostic to this column, and\n    -- simply invoke UPDATE statements on the version columns.\n    \"lock\" BOOL PRIMARY KEY DEFAULT true CHECK (lock)\n  );\n\n  CREATE OR REPLACE FUNCTION ${app}.set_permissions_hash()\n  RETURNS TRIGGER AS $$\n  BEGIN\n      NEW.hash = md5(NEW.permissions::text);\n      RETURN NEW;\n  END;\n  $$ LANGUAGE plpgsql;\n\n  CREATE OR REPLACE TRIGGER on_set_permissions \n    BEFORE INSERT OR UPDATE ON ${app}.permissions\n    FOR EACH ROW\n    EXECUTE FUNCTION ${app}.set_permissions_hash();\n\n  INSERT INTO ${app}.permissions (permissions) VALUES (NULL) ON CONFLICT DO NOTHING;\n`;\n}\n\nexport async function ensureGlobalTables(db: PostgresDB, appID: AppID) {\n  await db.unsafe(globalSetup(appID));\n}\n\nexport function getClientsTableDefinition(schema: string) {\n  return /*sql*/ `\n  CREATE TABLE ${schema}.\"clients\" (\n    \"clientGroupID\"  TEXT NOT NULL,\n    \"clientID\"       TEXT NOT NULL,\n    \"lastMutationID\" BIGINT NOT NULL,\n    \"userID\"         TEXT,\n    PRIMARY KEY(\"clientGroupID\", \"clientID\")\n  );`;\n}\n\n/**\n * Tracks the results of mutations.\n * 1. It is an error for the same mutation ID to be used twice.\n * 2. The result is JSONB to allow for arbitrary results.\n *\n * The tables must be cleaned up as the clients\n * receive the mutation responses and as clients are removed.\n */\nexport function getMutationsTableDefinition(schema: string) {\n  return /*sql*/ `\n  CREATE TABLE ${schema}.\"mutations\" (\n    \"clientGroupID\"  TEXT NOT NULL,\n    \"clientID\"       TEXT NOT NULL,\n    \"mutationID\"     BIGINT NOT NULL,\n    \"result\"         JSON NOT NULL,\n    PRIMARY KEY(\"clientGroupID\", \"clientID\", \"mutationID\")\n  );`;\n}\n\nexport const SHARD_CONFIG_TABLE = 'shardConfig';\n\nexport function shardSetup(\n  shardConfig: ShardConfig,\n  metadataPublication: string,\n): string {\n  const app = id(appSchema(shardConfig));\n  const shard = id(upstreamSchema(shardConfig));\n\n  const pubs = [...shardConfig.publications].sort();\n  assert(\n    pubs.includes(metadataPublication),\n    () => `Publications must include ${metadataPublication}`,\n  );\n\n  return /*sql*/ `\n  CREATE SCHEMA IF NOT EXISTS ${shard};\n\n  ${getClientsTableDefinition(shard)}\n  ${getMutationsTableDefinition(shard)}\n\n  DROP PUBLICATION IF EXISTS ${id(metadataPublication)};\n  CREATE PUBLICATION ${id(metadataPublication)}\n    FOR TABLE ${app}.\"permissions\", TABLE ${shard}.\"clients\", ${shard}.\"mutations\";\n\n  CREATE TABLE ${shard}.\"${SHARD_CONFIG_TABLE}\" (\n    \"publications\"  TEXT[] NOT NULL,\n    \"ddlDetection\"  BOOL NOT NULL,\n\n    -- Ensure that there is only a single row in the table.\n    \"lock\" BOOL PRIMARY KEY DEFAULT true CHECK (lock)\n  );\n\n  INSERT INTO ${shard}.\"${SHARD_CONFIG_TABLE}\" (\n      \"publications\",\n      \"ddlDetection\" \n    ) VALUES (\n      ARRAY[${literal(pubs)}], \n      false  -- set in SAVEPOINT with triggerSetup() statements\n    );\n\n  CREATE TABLE ${shard}.replicas (\n    \"slot\"          TEXT PRIMARY KEY,\n    \"version\"       TEXT NOT NULL,\n    \"initialSchema\" JSON NOT NULL\n  );\n  `;\n}\n\nexport function dropShard(appID: string, shardID: string | number): string {\n  const schema = `${appID}_${shardID}`;\n  const metadataPublication = metadataPublicationName(appID, shardID);\n  const defaultPublication = defaultPublicationName(appID, shardID);\n\n  // DROP SCHEMA ... CASCADE does not drop dependent PUBLICATIONS,\n  // so PUBLICATIONs must be dropped explicitly.\n  return /*sql*/ `\n    DROP PUBLICATION IF EXISTS ${id(defaultPublication)};\n    DROP PUBLICATION IF EXISTS ${id(metadataPublication)};\n    DROP SCHEMA IF EXISTS ${id(schema)} CASCADE;\n  `;\n}\n\nconst internalShardConfigSchema = v.object({\n  publications: v.array(v.string()),\n  ddlDetection: v.boolean(),\n});\n\nexport type InternalShardConfig = v.Infer<typeof internalShardConfigSchema>;\n\nconst replicaSchema = internalShardConfigSchema.extend({\n  slot: v.string(),\n  version: v.string(),\n  initialSchema: publishedSchema,\n});\n\nexport type Replica = v.Infer<typeof replicaSchema>;\n\n// triggerSetup is run separately in a sub-transaction (i.e. SAVEPOINT) so\n// that a failure (e.g. due to lack of superuser permissions) can be handled\n// by continuing in a degraded mode (ddlDetection = false).\nfunction triggerSetup(shard: ShardConfig): string {\n  const schema = id(upstreamSchema(shard));\n  return (\n    createEventTriggerStatements(shard) +\n    /*sql*/ `UPDATE ${schema}.\"shardConfig\" SET \"ddlDetection\" = true;`\n  );\n}\n\n// Called in initial-sync to store the exact schema that was initially synced.\nexport async function addReplica(\n  sql: PostgresDB,\n  shard: ShardID,\n  slot: string,\n  replicaVersion: string,\n  {tables, indexes}: PublishedSchema,\n) {\n  const schema = upstreamSchema(shard);\n  const synced: PublishedSchema = {tables, indexes};\n  await sql`\n    INSERT INTO ${sql(schema)}.replicas (\"slot\", \"version\", \"initialSchema\")\n      VALUES (${slot}, ${replicaVersion}, ${synced})`;\n}\n\nexport async function getReplicaAtVersion(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardID,\n  replicaVersion: string,\n): Promise<Replica | null> {\n  const schema = sql(upstreamSchema(shard));\n  const result = await sql`\n    SELECT * FROM ${schema}.replicas JOIN ${schema}.\"shardConfig\" ON true\n      WHERE version = ${replicaVersion};\n  `;\n  if (result.length === 0) {\n    // log out all the replicas and the joined shardConfig\n    const allReplicas = await sql`\n      SELECT * FROM ${schema}.replicas JOIN ${schema}.\"shardConfig\" ON true`;\n    lc.debug?.(`Replica not found in: ${JSON.stringify(allReplicas)}`);\n    return null;\n  }\n  return v.parse(result[0], replicaSchema, 'passthrough');\n}\n\nexport async function getInternalShardConfig(\n  sql: PostgresDB,\n  shard: ShardID,\n): Promise<InternalShardConfig> {\n  const result = await sql`\n    SELECT \"publications\", \"ddlDetection\"\n      FROM ${sql(upstreamSchema(shard))}.\"shardConfig\";\n  `;\n  assert(\n    result.length === 1,\n    () => `Expected exactly one shardConfig row, got ${result.length}`,\n  );\n  return v.parse(result[0], internalShardConfigSchema, 'passthrough');\n}\n\n/**\n * Sets up and returns all publications (including internal ones) for\n * the given shard.\n */\nexport async function setupTablesAndReplication(\n  lc: LogContext,\n  sql: PostgresTransaction,\n  requested: ShardConfig,\n) {\n  const {publications} = requested;\n  // Validate requested publications.\n  for (const pub of publications) {\n    validatePublicationName(pub);\n    if (pub.startsWith('_')) {\n      throw new Error(\n        `Publication names starting with \"_\" are reserved for internal use.\\n` +\n          `Please use a different name for publication \"${pub}\".`,\n      );\n    }\n  }\n  const allPublications: string[] = [];\n\n  // Setup application publications.\n  if (publications.length) {\n    const results = await sql<{pubname: string}[]>`\n    SELECT pubname from pg_publication WHERE pubname IN ${sql(\n      publications,\n    )}`.values();\n\n    if (results.length !== publications.length) {\n      throw new Error(\n        `Unknown or invalid publications. Specified: [${publications}]. Found: [${results.flat()}]`,\n      );\n    }\n    allPublications.push(...publications);\n  } else {\n    const defaultPublication = defaultPublicationName(\n      requested.appID,\n      requested.shardNum,\n    );\n    await sql`\n      DROP PUBLICATION IF EXISTS ${sql(defaultPublication)}`;\n    await sql`\n      CREATE PUBLICATION ${sql(defaultPublication)} \n        FOR TABLES IN SCHEMA public\n        WITH (publish_via_partition_root = true)`;\n    allPublications.push(defaultPublication);\n  }\n\n  const metadataPublication = metadataPublicationName(\n    requested.appID,\n    requested.shardNum,\n  );\n  allPublications.push(metadataPublication);\n\n  const shard = {...requested, publications: allPublications};\n\n  // Setup the global tables and shard tables / publications.\n  await sql.unsafe(globalSetup(shard) + shardSetup(shard, metadataPublication));\n\n  const pubs = await getPublicationInfo(sql, allPublications);\n  await replicaIdentitiesForTablesWithoutPrimaryKeys(pubs)?.apply(lc, sql);\n\n  await setupTriggers(lc, sql, shard);\n}\n\nexport async function setupTriggers(\n  lc: LogContext,\n  tx: PostgresTransaction,\n  shard: ShardConfig,\n) {\n  try {\n    await tx.savepoint(sub => sub.unsafe(triggerSetup(shard)));\n  } catch (e) {\n    if (\n      !(\n        e instanceof postgres.PostgresError &&\n        e.code === PG_INSUFFICIENT_PRIVILEGE\n      )\n    ) {\n      throw e;\n    }\n    // If triggerSetup() fails, replication continues in ddlDetection=false mode.\n    lc.warn?.(\n      `Unable to create event triggers for schema change detection:\\n\\n` +\n        `\"${e.hint ?? e.message}\"\\n\\n` +\n        `Proceeding in degraded mode: schema changes will halt replication,\\n` +\n        `requiring the replica to be reset (manually or with --auto-reset).`,\n    );\n  }\n}\n\nexport function validatePublications(\n  lc: LogContext,\n  published: PublicationInfo,\n) {\n  // Verify that all publications export the proper events.\n  published.publications.forEach(pub => {\n    if (\n      !pub.pubinsert ||\n      !pub.pubtruncate ||\n      !pub.pubdelete ||\n      !pub.pubtruncate\n    ) {\n      // TODO: Make APIError?\n      throw new Error(\n        `PUBLICATION ${pub.pubname} must publish insert, update, delete, and truncate`,\n      );\n    }\n  });\n\n  published.tables.forEach(table => validate(lc, table));\n}\n\ntype ReplicaIdentities = {\n  apply(lc: LogContext, db: PostgresDB): Promise<void>;\n};\n\nexport function replicaIdentitiesForTablesWithoutPrimaryKeys(\n  pubs: PublishedSchema,\n): ReplicaIdentities | undefined {\n  const replicaIdentities: {\n    schema: string;\n    tableName: string;\n    indexName: string;\n  }[] = [];\n  for (const table of pubs.tables) {\n    if (!table.primaryKey?.length && table.replicaIdentity === Default) {\n      // Look for an index that can serve as the REPLICA IDENTITY USING INDEX. It must be:\n      // - UNIQUE\n      // - NOT NULL columns\n      // - not deferrable (i.e. isImmediate)\n      // - not partial (are already filtered out)\n      //\n      // https://www.postgresql.org/docs/current/sql-altertable.html#SQL-ALTERTABLE-REPLICA-IDENTITY\n      const {schema, name: tableName} = table;\n      for (const {columns, name: indexName} of pubs.indexes.filter(\n        idx =>\n          idx.schema === schema &&\n          idx.tableName === tableName &&\n          idx.unique &&\n          idx.isImmediate,\n      )) {\n        if (Object.keys(columns).some(col => !table.columns[col].notNull)) {\n          continue; // Only indexes with all NOT NULL columns are suitable.\n        }\n        replicaIdentities.push({schema, tableName, indexName});\n        break;\n      }\n    }\n  }\n\n  if (replicaIdentities.length === 0) {\n    return undefined;\n  }\n  return {\n    apply: async (lc: LogContext, sql: PostgresDB) => {\n      for (const {schema, tableName, indexName} of replicaIdentities) {\n        lc.info?.(\n          `setting \"${indexName}\" as the REPLICA IDENTITY for \"${tableName}\"`,\n        );\n        await sql`\n        ALTER TABLE ${sql(schema)}.${sql(tableName)} \n          REPLICA IDENTITY USING INDEX ${sql(indexName)}`;\n      }\n    },\n  };\n}\n"],"names":["v.object","v.array","v.string","v.boolean","v.parse"],"mappings":";;;;;;;;;;;;AAwBA,MAAM,yBAAyB;AAOxB,SAAS,wBAAwB,MAAoB;AAC1D,MAAI,CAAC,uBAAuB,KAAK,IAAI,GAAG;AACtC,UAAM,IAAI;AAAA,MACR,6BAA6B,IAAI;AAAA,IAAA;AAAA,EAGrC;AACA,MAAI,KAAK,SAAS,IAAI;AACpB,UAAM,IAAI;AAAA,MACR,qBAAqB,IAAI;AAAA,IAAA;AAAA,EAE7B;AACF;AAEO,SAAS,0BAA0B,EAAC,SAAe;AACxD,SAAO,IAAI,KAAK;AAClB;AAEO,SAAS,sBAAsB,EAAC,OAAO,YAAoB;AAChE,SAAO,GAAG,KAAK,IAAI,QAAQ;AAC7B;AAEO,SAAS,sBAAsB,OAAgB;AACpD,QAAM,EAAC,OAAO,aAAY,MAAM,KAAK;AACrC,SAAO,GAAG,KAAK,IAAI,QAAQ;AAC7B;AAMO,SAAS,0BAA0B,OAAgB;AAGxD,SAAO,GAAG,sBAAsB,KAAK,CAAC,IAAI,WAAW,KAAK,KAAK;AACjE;AAEO,SAAS,mBAAmB,OAAgB;AACjD,SAAO,sBAAsB,KAAK,IAAI,KAAK,IAAA;AAC7C;AAEA,SAAS,uBAAuB,OAAe,SAA0B;AACvE,SAAO,IAAI,KAAK,WAAW,OAAO;AACpC;AAEO,SAAS,wBACd,OACA,SACA;AACA,SAAO,IAAI,KAAK,aAAa,OAAO;AACtC;AAGA,SAAS,YAAY,OAAsB;AACzC,QAAM,MAAM,GAAG,UAAU,KAAK,CAAC;AAE/B;AAAA;AAAA,IAAe;AAAA,gCACe,GAAG;AAAA;AAAA,+BAEJ,GAAG;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,+BAUH,GAAG;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,iCASD,GAAG;AAAA;AAAA,uBAEb,GAAG;AAAA;AAAA,gBAEV,GAAG;AAAA;AAAA;AAEnB;AAEA,eAAsB,mBAAmB,IAAgB,OAAc;AACrE,QAAM,GAAG,OAAO,YAAY,KAAK,CAAC;AACpC;AAEO,SAAS,0BAA0B,QAAgB;AACxD;AAAA;AAAA,IAAe;AAAA,iBACA,MAAM;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAOvB;AAUO,SAAS,4BAA4B,QAAgB;AAC1D;AAAA;AAAA,IAAe;AAAA,iBACA,MAAM;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAOvB;AAEO,MAAM,qBAAqB;AAE3B,SAAS,WACd,aACA,qBACQ;AACR,QAAM,MAAM,GAAG,UAAU,WAAW,CAAC;AACrC,QAAM,QAAQ,GAAG,eAAe,WAAW,CAAC;AAE5C,QAAM,OAAO,CAAC,GAAG,YAAY,YAAY,EAAE,KAAA;AAC3C;AAAA,IACE,KAAK,SAAS,mBAAmB;AAAA,IACjC,MAAM,6BAA6B,mBAAmB;AAAA,EAAA;AAGxD;AAAA;AAAA,IAAe;AAAA,gCACe,KAAK;AAAA;AAAA,IAEjC,0BAA0B,KAAK,CAAC;AAAA,IAChC,4BAA4B,KAAK,CAAC;AAAA;AAAA,+BAEP,GAAG,mBAAmB,CAAC;AAAA,uBAC/B,GAAG,mBAAmB,CAAC;AAAA,gBAC9B,GAAG,yBAAyB,KAAK,eAAe,KAAK;AAAA;AAAA,iBAEpD,KAAK,KAAK,kBAAkB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,gBAQ7B,KAAK,KAAK,kBAAkB;AAAA;AAAA;AAAA;AAAA,cAI9B,QAAQ,IAAI,CAAC;AAAA;AAAA;AAAA;AAAA,iBAIV,KAAK;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAMtB;AAEO,SAAS,UAAU,OAAe,SAAkC;AACzE,QAAM,SAAS,GAAG,KAAK,IAAI,OAAO;AAClC,QAAM,sBAAsB,wBAAwB,OAAO,OAAO;AAClE,QAAM,qBAAqB,uBAAuB,OAAO,OAAO;AAIhE;AAAA;AAAA,IAAe;AAAA,iCACgB,GAAG,kBAAkB,CAAC;AAAA,iCACtB,GAAG,mBAAmB,CAAC;AAAA,4BAC5B,GAAG,MAAM,CAAC;AAAA;AAAA;AAEtC;AAEA,MAAM,4BAA4BA,OAAS;AAAA,EACzC,cAAcC,MAAQC,QAAU;AAAA,EAChC,cAAcC,QAAE;AAClB,CAAC;AAID,MAAM,gBAAgB,0BAA0B,OAAO;AAAA,EACrD,MAAMD,OAAE;AAAA,EACR,SAASA,OAAE;AAAA,EACX,eAAe;AACjB,CAAC;AAOD,SAAS,aAAa,OAA4B;AAChD,QAAM,SAAS,GAAG,eAAe,KAAK,CAAC;AACvC,SACE,6BAA6B,KAAK;AAAA,EAC1B,UAAU,MAAM;AAE5B;AAGA,eAAsB,WACpB,KACA,OACA,MACA,gBACA,EAAC,QAAQ,WACT;AACA,QAAM,SAAS,eAAe,KAAK;AACnC,QAAM,SAA0B,EAAC,QAAQ,QAAA;AACzC,QAAM;AAAA,kBACU,IAAI,MAAM,CAAC;AAAA,gBACb,IAAI,KAAK,cAAc,KAAK,MAAM;AAClD;AAEA,eAAsB,oBACpB,IACA,KACA,OACA,gBACyB;AACzB,QAAM,SAAS,IAAI,eAAe,KAAK,CAAC;AACxC,QAAM,SAAS,MAAM;AAAA,oBACH,MAAM,kBAAkB,MAAM;AAAA,wBAC1B,cAAc;AAAA;AAEpC,MAAI,OAAO,WAAW,GAAG;AAEvB,UAAM,cAAc,MAAM;AAAA,sBACR,MAAM,kBAAkB,MAAM;AAChD,OAAG,QAAQ,yBAAyB,KAAK,UAAU,WAAW,CAAC,EAAE;AACjE,WAAO;AAAA,EACT;AACA,SAAOE,MAAQ,OAAO,CAAC,GAAG,eAAe,aAAa;AACxD;AAEA,eAAsB,uBACpB,KACA,OAC8B;AAC9B,QAAM,SAAS,MAAM;AAAA;AAAA,aAEV,IAAI,eAAe,KAAK,CAAC,CAAC;AAAA;AAErC;AAAA,IACE,OAAO,WAAW;AAAA,IAClB,MAAM,6CAA6C,OAAO,MAAM;AAAA,EAAA;AAElE,SAAOA,MAAQ,OAAO,CAAC,GAAG,2BAA2B,aAAa;AACpE;AAMA,eAAsB,0BACpB,IACA,KACA,WACA;AACA,QAAM,EAAC,iBAAgB;AAEvB,aAAW,OAAO,cAAc;AAC9B,4BAAwB,GAAG;AAC3B,QAAI,IAAI,WAAW,GAAG,GAAG;AACvB,YAAM,IAAI;AAAA,QACR;AAAA,+CACkD,GAAG;AAAA,MAAA;AAAA,IAEzD;AAAA,EACF;AACA,QAAM,kBAA4B,CAAA;AAGlC,MAAI,aAAa,QAAQ;AACvB,UAAM,UAAU,MAAM;AAAA,0DACgC;AAAA,MACpD;AAAA,IAAA,CACD,GAAG,OAAA;AAEJ,QAAI,QAAQ,WAAW,aAAa,QAAQ;AAC1C,YAAM,IAAI;AAAA,QACR,gDAAgD,YAAY,cAAc,QAAQ,MAAM;AAAA,MAAA;AAAA,IAE5F;AACA,oBAAgB,KAAK,GAAG,YAAY;AAAA,EACtC,OAAO;AACL,UAAM,qBAAqB;AAAA,MACzB,UAAU;AAAA,MACV,UAAU;AAAA,IAAA;AAEZ,UAAM;AAAA,mCACyB,IAAI,kBAAkB,CAAC;AACtD,UAAM;AAAA,2BACiB,IAAI,kBAAkB,CAAC;AAAA;AAAA;AAG9C,oBAAgB,KAAK,kBAAkB;AAAA,EACzC;AAEA,QAAM,sBAAsB;AAAA,IAC1B,UAAU;AAAA,IACV,UAAU;AAAA,EAAA;AAEZ,kBAAgB,KAAK,mBAAmB;AAExC,QAAM,QAAQ,EAAC,GAAG,WAAW,cAAc,gBAAA;AAG3C,QAAM,IAAI,OAAO,YAAY,KAAK,IAAI,WAAW,OAAO,mBAAmB,CAAC;AAE5E,QAAM,OAAO,MAAM,mBAAmB,KAAK,eAAe;AAC1D,QAAM,6CAA6C,IAAI,GAAG,MAAM,IAAI,GAAG;AAEvE,QAAM,cAAc,IAAI,KAAK,KAAK;AACpC;AAEA,eAAsB,cACpB,IACA,IACA,OACA;AACA,MAAI;AACF,UAAM,GAAG,UAAU,CAAA,QAAO,IAAI,OAAO,aAAa,KAAK,CAAC,CAAC;AAAA,EAC3D,SAAS,GAAG;AACV,QACE,EACE,aAAa,SAAS,iBACtB,EAAE,SAAS,4BAEb;AACA,YAAM;AAAA,IACR;AAEA,OAAG;AAAA,MACD;AAAA;AAAA,GACM,EAAE,QAAQ,EAAE,OAAO;AAAA;AAAA;AAAA;AAAA,IAAA;AAAA,EAI7B;AACF;AAEO,SAAS,qBACd,IACA,WACA;AAEA,YAAU,aAAa,QAAQ,CAAA,QAAO;AACpC,QACE,CAAC,IAAI,aACL,CAAC,IAAI,eACL,CAAC,IAAI,aACL,CAAC,IAAI,aACL;AAEA,YAAM,IAAI;AAAA,QACR,eAAe,IAAI,OAAO;AAAA,MAAA;AAAA,IAE9B;AAAA,EACF,CAAC;AAED,YAAU,OAAO,QAAQ,CAAA,UAAS,SAAS,IAAI,KAAK,CAAC;AACvD;AAMO,SAAS,6CACd,MAC+B;AAC/B,QAAM,oBAIA,CAAA;AACN,aAAW,SAAS,KAAK,QAAQ;AAC/B,QAAI,CAAC,MAAM,YAAY,UAAU,MAAM,oBAAoB,SAAS;AAQlE,YAAM,EAAC,QAAQ,MAAM,UAAA,IAAa;AAClC,iBAAW,EAAC,SAAS,MAAM,UAAA,KAAc,KAAK,QAAQ;AAAA,QACpD,CAAA,QACE,IAAI,WAAW,UACf,IAAI,cAAc,aAClB,IAAI,UACJ,IAAI;AAAA,MAAA,GACL;AACD,YAAI,OAAO,KAAK,OAAO,EAAE,KAAK,CAAA,QAAO,CAAC,MAAM,QAAQ,GAAG,EAAE,OAAO,GAAG;AACjE;AAAA,QACF;AACA,0BAAkB,KAAK,EAAC,QAAQ,WAAW,WAAU;AACrD;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAEA,MAAI,kBAAkB,WAAW,GAAG;AAClC,WAAO;AAAA,EACT;AACA,SAAO;AAAA,IACL,OAAO,OAAO,IAAgB,QAAoB;AAChD,iBAAW,EAAC,QAAQ,WAAW,UAAA,KAAc,mBAAmB;AAC9D,WAAG;AAAA,UACD,YAAY,SAAS,kCAAkC,SAAS;AAAA,QAAA;AAElE,cAAM;AAAA,sBACQ,IAAI,MAAM,CAAC,IAAI,IAAI,SAAS,CAAC;AAAA,yCACV,IAAI,SAAS,CAAC;AAAA,MACjD;AAAA,IACF;AAAA,EAAA;AAEJ;"}
+{"version":3,"file":"shard.js","sources":["../../../../../../../../zero-cache/src/services/change-source/pg/schema/shard.ts"],"sourcesContent":["import {PG_INSUFFICIENT_PRIVILEGE} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport {literal} from 'pg-format';\nimport postgres from 'postgres';\nimport {assert} from '../../../../../../shared/src/asserts.ts';\nimport {\n  jsonObjectSchema,\n  stringify,\n  type JSONObject,\n} from '../../../../../../shared/src/bigint-json.ts';\nimport * as v from '../../../../../../shared/src/valita.ts';\nimport {Default} from '../../../../db/postgres-replica-identity-enum.ts';\nimport type {PostgresDB, PostgresTransaction} from '../../../../types/pg.ts';\nimport type {AppID, ShardConfig, ShardID} from '../../../../types/shards.ts';\nimport {appSchema, check, upstreamSchema} from '../../../../types/shards.ts';\nimport {id} from '../../../../types/sql.ts';\nimport {createEventTriggerStatements} from './ddl.ts';\nimport {\n  getPublicationInfo,\n  publishedSchema,\n  type PublicationInfo,\n  type PublishedSchema,\n} from './published.ts';\nimport {validate} from './validation.ts';\n\n/**\n * PostgreSQL unquoted identifiers must start with a letter or underscore\n * and contain only letters, digits, and underscores.\n */\nconst VALID_PUBLICATION_NAME = /^[a-zA-Z_][a-zA-Z0-9_]*$/;\n\n/**\n * Validates that a publication name is a valid PostgreSQL identifier.\n * This provides defense-in-depth against SQL injection when publication\n * names are used in replication commands.\n */\nexport function validatePublicationName(name: string): void {\n  if (!VALID_PUBLICATION_NAME.test(name)) {\n    throw new Error(\n      `Invalid publication name \"${name}\". Publication names must start with a letter or underscore ` +\n        `and contain only letters, digits, and underscores.`,\n    );\n  }\n  if (name.length > 63) {\n    throw new Error(\n      `Publication name \"${name}\" exceeds PostgreSQL's 63-character identifier limit.`,\n    );\n  }\n}\n\nexport function internalPublicationPrefix({appID}: AppID) {\n  return `_${appID}_`;\n}\n\nexport function legacyReplicationSlot({appID, shardNum}: ShardID) {\n  return `${appID}_${shardNum}`;\n}\n\nexport function replicationSlotPrefix(shard: ShardID) {\n  const {appID, shardNum} = check(shard);\n  return `${appID}_${shardNum}_`;\n}\n\n/**\n * An expression used to match replication slots in the shard\n * in a Postgres `LIKE` operator.\n */\nexport function replicationSlotExpression(shard: ShardID) {\n  // Underscores have a special meaning in LIKE values\n  // so they have to be escaped.\n  return `${replicationSlotPrefix(shard)}%`.replaceAll('_', '\\\\_');\n}\n\nexport function newReplicationSlot(shard: ShardID) {\n  return replicationSlotPrefix(shard) + Date.now();\n}\n\nfunction defaultPublicationName(appID: string, shardID: string | number) {\n  return `_${appID}_public_${shardID}`;\n}\n\nexport function metadataPublicationName(\n  appID: string,\n  shardID: string | number,\n) {\n  return `_${appID}_metadata_${shardID}`;\n}\n\n// The GLOBAL_SETUP must be idempotent as it can be run multiple times for different shards.\nfunction globalSetup(appID: AppID): string {\n  const app = id(appSchema(appID));\n\n  return /*sql*/ `\n  CREATE SCHEMA IF NOT EXISTS ${app};\n\n  CREATE TABLE IF NOT EXISTS ${app}.permissions (\n    \"permissions\" JSONB,\n    \"hash\"        TEXT,\n\n    -- Ensure that there is only a single row in the table.\n    -- Application code can be agnostic to this column, and\n    -- simply invoke UPDATE statements on the version columns.\n    \"lock\" BOOL PRIMARY KEY DEFAULT true CHECK (lock)\n  );\n\n  CREATE OR REPLACE FUNCTION ${app}.set_permissions_hash()\n  RETURNS TRIGGER AS $$\n  BEGIN\n      NEW.hash = md5(NEW.permissions::text);\n      RETURN NEW;\n  END;\n  $$ LANGUAGE plpgsql;\n\n  CREATE OR REPLACE TRIGGER on_set_permissions \n    BEFORE INSERT OR UPDATE ON ${app}.permissions\n    FOR EACH ROW\n    EXECUTE FUNCTION ${app}.set_permissions_hash();\n\n  INSERT INTO ${app}.permissions (permissions) VALUES (NULL) ON CONFLICT DO NOTHING;\n`;\n}\n\nexport async function ensureGlobalTables(db: PostgresDB, appID: AppID) {\n  await db.unsafe(globalSetup(appID));\n}\n\nexport function getClientsTableDefinition(schema: string) {\n  return /*sql*/ `\n  CREATE TABLE ${schema}.\"clients\" (\n    \"clientGroupID\"  TEXT NOT NULL,\n    \"clientID\"       TEXT NOT NULL,\n    \"lastMutationID\" BIGINT NOT NULL,\n    \"userID\"         TEXT,\n    PRIMARY KEY(\"clientGroupID\", \"clientID\")\n  );`;\n}\n\n/**\n * Tracks the results of mutations.\n * 1. It is an error for the same mutation ID to be used twice.\n * 2. The result is JSONB to allow for arbitrary results.\n *\n * The tables must be cleaned up as the clients\n * receive the mutation responses and as clients are removed.\n */\nexport function getMutationsTableDefinition(schema: string) {\n  return /*sql*/ `\n  CREATE TABLE ${schema}.\"mutations\" (\n    \"clientGroupID\"  TEXT NOT NULL,\n    \"clientID\"       TEXT NOT NULL,\n    \"mutationID\"     BIGINT NOT NULL,\n    \"result\"         JSON NOT NULL,\n    PRIMARY KEY(\"clientGroupID\", \"clientID\", \"mutationID\")\n  );`;\n}\n\nexport const SHARD_CONFIG_TABLE = 'shardConfig';\n\nexport function shardSetup(\n  shardConfig: ShardConfig,\n  metadataPublication: string,\n): string {\n  const app = id(appSchema(shardConfig));\n  const shard = id(upstreamSchema(shardConfig));\n\n  const pubs = [...shardConfig.publications].sort();\n  assert(\n    pubs.includes(metadataPublication),\n    () => `Publications must include ${metadataPublication}`,\n  );\n\n  return /*sql*/ `\n  CREATE SCHEMA IF NOT EXISTS ${shard};\n\n  ${getClientsTableDefinition(shard)}\n  ${getMutationsTableDefinition(shard)}\n\n  DROP PUBLICATION IF EXISTS ${id(metadataPublication)};\n  CREATE PUBLICATION ${id(metadataPublication)}\n    FOR TABLE ${app}.\"permissions\", TABLE ${shard}.\"clients\", ${shard}.\"mutations\";\n\n  CREATE TABLE ${shard}.\"${SHARD_CONFIG_TABLE}\" (\n    \"publications\"  TEXT[] NOT NULL,\n    \"ddlDetection\"  BOOL NOT NULL,\n\n    -- Ensure that there is only a single row in the table.\n    \"lock\" BOOL PRIMARY KEY DEFAULT true CHECK (lock)\n  );\n\n  INSERT INTO ${shard}.\"${SHARD_CONFIG_TABLE}\" (\n      \"publications\",\n      \"ddlDetection\" \n    ) VALUES (\n      ARRAY[${literal(pubs)}], \n      false  -- set in SAVEPOINT with triggerSetup() statements\n    );\n\n  CREATE TABLE ${shard}.replicas (\n    \"slot\"               TEXT PRIMARY KEY,\n    \"version\"            TEXT NOT NULL,\n    \"initialSchema\"      JSON NOT NULL,\n    \"initialSyncContext\" JSON,\n    \"subscriberContext\"  JSON\n  );\n  `;\n}\n\nexport function dropShard(appID: string, shardID: string | number): string {\n  const schema = `${appID}_${shardID}`;\n  const metadataPublication = metadataPublicationName(appID, shardID);\n  const defaultPublication = defaultPublicationName(appID, shardID);\n\n  // DROP SCHEMA ... CASCADE does not drop dependent PUBLICATIONS,\n  // so PUBLICATIONs must be dropped explicitly.\n  return /*sql*/ `\n    DROP PUBLICATION IF EXISTS ${id(defaultPublication)};\n    DROP PUBLICATION IF EXISTS ${id(metadataPublication)};\n    DROP SCHEMA IF EXISTS ${id(schema)} CASCADE;\n  `;\n}\n\nconst internalShardConfigSchema = v.object({\n  publications: v.array(v.string()),\n  ddlDetection: v.boolean(),\n});\n\nexport type InternalShardConfig = v.Infer<typeof internalShardConfigSchema>;\n\nconst replicaSchema = internalShardConfigSchema.extend({\n  slot: v.string(),\n  version: v.string(),\n  initialSchema: publishedSchema,\n  initialSyncContext: jsonObjectSchema.nullable(),\n  subscriberContext: jsonObjectSchema.nullable(),\n});\n\nexport type Replica = v.Infer<typeof replicaSchema>;\n\n// triggerSetup is run separately in a sub-transaction (i.e. SAVEPOINT) so\n// that a failure (e.g. due to lack of superuser permissions) can be handled\n// by continuing in a degraded mode (ddlDetection = false).\nfunction triggerSetup(shard: ShardConfig): string {\n  const schema = id(upstreamSchema(shard));\n  return (\n    createEventTriggerStatements(shard) +\n    /*sql*/ `UPDATE ${schema}.\"shardConfig\" SET \"ddlDetection\" = true;`\n  );\n}\n\n// Called in initial-sync to store the exact schema that was initially synced.\nexport async function addReplica(\n  sql: PostgresDB,\n  shard: ShardID,\n  slot: string,\n  replicaVersion: string,\n  {tables, indexes}: PublishedSchema,\n  initialSyncContext: JSONObject,\n) {\n  const schema = upstreamSchema(shard);\n  const synced: PublishedSchema = {tables, indexes};\n  await sql`\n    INSERT INTO ${sql(schema)}.replicas\n      (\"slot\", \"version\", \"initialSchema\", \"initialSyncContext\")\n      VALUES (${slot}, ${replicaVersion}, ${synced}, ${initialSyncContext})`;\n}\n\nexport async function getReplicaAtVersion(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardID,\n  replicaVersion: string,\n  context?: JSONObject,\n): Promise<Replica | null> {\n  const schema = sql(upstreamSchema(shard));\n  const result = await sql`\n    SELECT * FROM ${schema}.replicas JOIN ${schema}.\"shardConfig\" ON true\n      WHERE version = ${replicaVersion};\n  `;\n  if (result.length === 0) {\n    // log out all the replicas and the joined shardConfig\n    const allReplicas = await sql`\n      SELECT slot, version, \"initialSyncContext\", \"subscriberContext\" \n        FROM ${schema}.replicas`;\n    lc.info?.(\n      `Replica ${replicaVersion} ` +\n        (context ? `(context: ${stringify(context)}) ` : '') +\n        `not found in: ${stringify(allReplicas)}`,\n    );\n    return null;\n  }\n  return v.parse(result[0], replicaSchema, 'passthrough');\n}\n\nexport async function getInternalShardConfig(\n  sql: PostgresDB,\n  shard: ShardID,\n): Promise<InternalShardConfig> {\n  const result = await sql`\n    SELECT \"publications\", \"ddlDetection\"\n      FROM ${sql(upstreamSchema(shard))}.\"shardConfig\";\n  `;\n  assert(\n    result.length === 1,\n    () => `Expected exactly one shardConfig row, got ${result.length}`,\n  );\n  return v.parse(result[0], internalShardConfigSchema, 'passthrough');\n}\n\n/**\n * Sets up and returns all publications (including internal ones) for\n * the given shard.\n */\nexport async function setupTablesAndReplication(\n  lc: LogContext,\n  sql: PostgresTransaction,\n  requested: ShardConfig,\n) {\n  const {publications} = requested;\n  // Validate requested publications.\n  for (const pub of publications) {\n    validatePublicationName(pub);\n    if (pub.startsWith('_')) {\n      throw new Error(\n        `Publication names starting with \"_\" are reserved for internal use.\\n` +\n          `Please use a different name for publication \"${pub}\".`,\n      );\n    }\n  }\n  const allPublications: string[] = [];\n\n  // Setup application publications.\n  if (publications.length) {\n    const results = await sql<{pubname: string}[]>`\n    SELECT pubname from pg_publication WHERE pubname IN ${sql(\n      publications,\n    )}`.values();\n\n    if (results.length !== publications.length) {\n      throw new Error(\n        `Unknown or invalid publications. Specified: [${publications}]. Found: [${results.flat()}]`,\n      );\n    }\n    allPublications.push(...publications);\n  } else {\n    const defaultPublication = defaultPublicationName(\n      requested.appID,\n      requested.shardNum,\n    );\n    await sql`\n      DROP PUBLICATION IF EXISTS ${sql(defaultPublication)}`;\n    await sql`\n      CREATE PUBLICATION ${sql(defaultPublication)} \n        FOR TABLES IN SCHEMA public\n        WITH (publish_via_partition_root = true)`;\n    allPublications.push(defaultPublication);\n  }\n\n  const metadataPublication = metadataPublicationName(\n    requested.appID,\n    requested.shardNum,\n  );\n  allPublications.push(metadataPublication);\n\n  const shard = {...requested, publications: allPublications};\n\n  // Setup the global tables and shard tables / publications.\n  await sql.unsafe(globalSetup(shard) + shardSetup(shard, metadataPublication));\n\n  const pubs = await getPublicationInfo(sql, allPublications);\n  await replicaIdentitiesForTablesWithoutPrimaryKeys(pubs)?.apply(lc, sql);\n\n  await setupTriggers(lc, sql, shard);\n}\n\nexport async function setupTriggers(\n  lc: LogContext,\n  tx: PostgresTransaction,\n  shard: ShardConfig,\n) {\n  try {\n    await tx.savepoint(sub => sub.unsafe(triggerSetup(shard)));\n  } catch (e) {\n    if (\n      !(\n        e instanceof postgres.PostgresError &&\n        e.code === PG_INSUFFICIENT_PRIVILEGE\n      )\n    ) {\n      throw e;\n    }\n    // If triggerSetup() fails, replication continues in ddlDetection=false mode.\n    lc.warn?.(\n      `Unable to create event triggers for schema change detection:\\n\\n` +\n        `\"${e.hint ?? e.message}\"\\n\\n` +\n        `Proceeding in degraded mode: schema changes will halt replication,\\n` +\n        `requiring the replica to be reset (manually or with --auto-reset).`,\n    );\n  }\n}\n\nexport function validatePublications(\n  lc: LogContext,\n  published: PublicationInfo,\n) {\n  // Verify that all publications export the proper events.\n  published.publications.forEach(pub => {\n    if (\n      !pub.pubinsert ||\n      !pub.pubtruncate ||\n      !pub.pubdelete ||\n      !pub.pubtruncate\n    ) {\n      // TODO: Make APIError?\n      throw new Error(\n        `PUBLICATION ${pub.pubname} must publish insert, update, delete, and truncate`,\n      );\n    }\n  });\n\n  published.tables.forEach(table => validate(lc, table));\n}\n\ntype ReplicaIdentities = {\n  apply(lc: LogContext, db: PostgresDB): Promise<void>;\n};\n\nexport function replicaIdentitiesForTablesWithoutPrimaryKeys(\n  pubs: PublishedSchema,\n): ReplicaIdentities | undefined {\n  const replicaIdentities: {\n    schema: string;\n    tableName: string;\n    indexName: string;\n  }[] = [];\n  for (const table of pubs.tables) {\n    if (!table.primaryKey?.length && table.replicaIdentity === Default) {\n      // Look for an index that can serve as the REPLICA IDENTITY USING INDEX. It must be:\n      // - UNIQUE\n      // - NOT NULL columns\n      // - not deferrable (i.e. isImmediate)\n      // - not partial (are already filtered out)\n      //\n      // https://www.postgresql.org/docs/current/sql-altertable.html#SQL-ALTERTABLE-REPLICA-IDENTITY\n      const {schema, name: tableName} = table;\n      for (const {columns, name: indexName} of pubs.indexes.filter(\n        idx =>\n          idx.schema === schema &&\n          idx.tableName === tableName &&\n          idx.unique &&\n          idx.isImmediate,\n      )) {\n        if (Object.keys(columns).some(col => !table.columns[col].notNull)) {\n          continue; // Only indexes with all NOT NULL columns are suitable.\n        }\n        replicaIdentities.push({schema, tableName, indexName});\n        break;\n      }\n    }\n  }\n\n  if (replicaIdentities.length === 0) {\n    return undefined;\n  }\n  return {\n    apply: async (lc: LogContext, sql: PostgresDB) => {\n      for (const {schema, tableName, indexName} of replicaIdentities) {\n        lc.info?.(\n          `setting \"${indexName}\" as the REPLICA IDENTITY for \"${tableName}\"`,\n        );\n        await sql`\n        ALTER TABLE ${sql(schema)}.${sql(tableName)} \n          REPLICA IDENTITY USING INDEX ${sql(indexName)}`;\n      }\n    },\n  };\n}\n"],"names":["v.object","v.array","v.string","v.boolean","v.parse"],"mappings":";;;;;;;;;;;;;AA6BA,MAAM,yBAAyB;AAOxB,SAAS,wBAAwB,MAAoB;AAC1D,MAAI,CAAC,uBAAuB,KAAK,IAAI,GAAG;AACtC,UAAM,IAAI;AAAA,MACR,6BAA6B,IAAI;AAAA,IAAA;AAAA,EAGrC;AACA,MAAI,KAAK,SAAS,IAAI;AACpB,UAAM,IAAI;AAAA,MACR,qBAAqB,IAAI;AAAA,IAAA;AAAA,EAE7B;AACF;AAEO,SAAS,0BAA0B,EAAC,SAAe;AACxD,SAAO,IAAI,KAAK;AAClB;AAEO,SAAS,sBAAsB,EAAC,OAAO,YAAoB;AAChE,SAAO,GAAG,KAAK,IAAI,QAAQ;AAC7B;AAEO,SAAS,sBAAsB,OAAgB;AACpD,QAAM,EAAC,OAAO,aAAY,MAAM,KAAK;AACrC,SAAO,GAAG,KAAK,IAAI,QAAQ;AAC7B;AAMO,SAAS,0BAA0B,OAAgB;AAGxD,SAAO,GAAG,sBAAsB,KAAK,CAAC,IAAI,WAAW,KAAK,KAAK;AACjE;AAEO,SAAS,mBAAmB,OAAgB;AACjD,SAAO,sBAAsB,KAAK,IAAI,KAAK,IAAA;AAC7C;AAEA,SAAS,uBAAuB,OAAe,SAA0B;AACvE,SAAO,IAAI,KAAK,WAAW,OAAO;AACpC;AAEO,SAAS,wBACd,OACA,SACA;AACA,SAAO,IAAI,KAAK,aAAa,OAAO;AACtC;AAGA,SAAS,YAAY,OAAsB;AACzC,QAAM,MAAM,GAAG,UAAU,KAAK,CAAC;AAE/B;AAAA;AAAA,IAAe;AAAA,gCACe,GAAG;AAAA;AAAA,+BAEJ,GAAG;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,+BAUH,GAAG;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,iCASD,GAAG;AAAA;AAAA,uBAEb,GAAG;AAAA;AAAA,gBAEV,GAAG;AAAA;AAAA;AAEnB;AAEA,eAAsB,mBAAmB,IAAgB,OAAc;AACrE,QAAM,GAAG,OAAO,YAAY,KAAK,CAAC;AACpC;AAEO,SAAS,0BAA0B,QAAgB;AACxD;AAAA;AAAA,IAAe;AAAA,iBACA,MAAM;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAOvB;AAUO,SAAS,4BAA4B,QAAgB;AAC1D;AAAA;AAAA,IAAe;AAAA,iBACA,MAAM;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAOvB;AAEO,MAAM,qBAAqB;AAE3B,SAAS,WACd,aACA,qBACQ;AACR,QAAM,MAAM,GAAG,UAAU,WAAW,CAAC;AACrC,QAAM,QAAQ,GAAG,eAAe,WAAW,CAAC;AAE5C,QAAM,OAAO,CAAC,GAAG,YAAY,YAAY,EAAE,KAAA;AAC3C;AAAA,IACE,KAAK,SAAS,mBAAmB;AAAA,IACjC,MAAM,6BAA6B,mBAAmB;AAAA,EAAA;AAGxD;AAAA;AAAA,IAAe;AAAA,gCACe,KAAK;AAAA;AAAA,IAEjC,0BAA0B,KAAK,CAAC;AAAA,IAChC,4BAA4B,KAAK,CAAC;AAAA;AAAA,+BAEP,GAAG,mBAAmB,CAAC;AAAA,uBAC/B,GAAG,mBAAmB,CAAC;AAAA,gBAC9B,GAAG,yBAAyB,KAAK,eAAe,KAAK;AAAA;AAAA,iBAEpD,KAAK,KAAK,kBAAkB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,gBAQ7B,KAAK,KAAK,kBAAkB;AAAA;AAAA;AAAA;AAAA,cAI9B,QAAQ,IAAI,CAAC;AAAA;AAAA;AAAA;AAAA,iBAIV,KAAK;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAQtB;AAEO,SAAS,UAAU,OAAe,SAAkC;AACzE,QAAM,SAAS,GAAG,KAAK,IAAI,OAAO;AAClC,QAAM,sBAAsB,wBAAwB,OAAO,OAAO;AAClE,QAAM,qBAAqB,uBAAuB,OAAO,OAAO;AAIhE;AAAA;AAAA,IAAe;AAAA,iCACgB,GAAG,kBAAkB,CAAC;AAAA,iCACtB,GAAG,mBAAmB,CAAC;AAAA,4BAC5B,GAAG,MAAM,CAAC;AAAA;AAAA;AAEtC;AAEA,MAAM,4BAA4BA,OAAS;AAAA,EACzC,cAAcC,MAAQC,QAAU;AAAA,EAChC,cAAcC,QAAE;AAClB,CAAC;AAID,MAAM,gBAAgB,0BAA0B,OAAO;AAAA,EACrD,MAAMD,OAAE;AAAA,EACR,SAASA,OAAE;AAAA,EACX,eAAe;AAAA,EACf,oBAAoB,iBAAiB,SAAA;AAAA,EACrC,mBAAmB,iBAAiB,SAAA;AACtC,CAAC;AAOD,SAAS,aAAa,OAA4B;AAChD,QAAM,SAAS,GAAG,eAAe,KAAK,CAAC;AACvC,SACE,6BAA6B,KAAK;AAAA,EAC1B,UAAU,MAAM;AAE5B;AAGA,eAAsB,WACpB,KACA,OACA,MACA,gBACA,EAAC,QAAQ,QAAA,GACT,oBACA;AACA,QAAM,SAAS,eAAe,KAAK;AACnC,QAAM,SAA0B,EAAC,QAAQ,QAAA;AACzC,QAAM;AAAA,kBACU,IAAI,MAAM,CAAC;AAAA;AAAA,gBAEb,IAAI,KAAK,cAAc,KAAK,MAAM,KAAK,kBAAkB;AACzE;AAEA,eAAsB,oBACpB,IACA,KACA,OACA,gBACA,SACyB;AACzB,QAAM,SAAS,IAAI,eAAe,KAAK,CAAC;AACxC,QAAM,SAAS,MAAM;AAAA,oBACH,MAAM,kBAAkB,MAAM;AAAA,wBAC1B,cAAc;AAAA;AAEpC,MAAI,OAAO,WAAW,GAAG;AAEvB,UAAM,cAAc,MAAM;AAAA;AAAA,eAEf,MAAM;AACjB,OAAG;AAAA,MACD,WAAW,cAAc,OACtB,UAAU,aAAa,UAAU,OAAO,CAAC,OAAO,MACjD,iBAAiB,UAAU,WAAW,CAAC;AAAA,IAAA;AAE3C,WAAO;AAAA,EACT;AACA,SAAOE,MAAQ,OAAO,CAAC,GAAG,eAAe,aAAa;AACxD;AAEA,eAAsB,uBACpB,KACA,OAC8B;AAC9B,QAAM,SAAS,MAAM;AAAA;AAAA,aAEV,IAAI,eAAe,KAAK,CAAC,CAAC;AAAA;AAErC;AAAA,IACE,OAAO,WAAW;AAAA,IAClB,MAAM,6CAA6C,OAAO,MAAM;AAAA,EAAA;AAElE,SAAOA,MAAQ,OAAO,CAAC,GAAG,2BAA2B,aAAa;AACpE;AAMA,eAAsB,0BACpB,IACA,KACA,WACA;AACA,QAAM,EAAC,iBAAgB;AAEvB,aAAW,OAAO,cAAc;AAC9B,4BAAwB,GAAG;AAC3B,QAAI,IAAI,WAAW,GAAG,GAAG;AACvB,YAAM,IAAI;AAAA,QACR;AAAA,+CACkD,GAAG;AAAA,MAAA;AAAA,IAEzD;AAAA,EACF;AACA,QAAM,kBAA4B,CAAA;AAGlC,MAAI,aAAa,QAAQ;AACvB,UAAM,UAAU,MAAM;AAAA,0DACgC;AAAA,MACpD;AAAA,IAAA,CACD,GAAG,OAAA;AAEJ,QAAI,QAAQ,WAAW,aAAa,QAAQ;AAC1C,YAAM,IAAI;AAAA,QACR,gDAAgD,YAAY,cAAc,QAAQ,MAAM;AAAA,MAAA;AAAA,IAE5F;AACA,oBAAgB,KAAK,GAAG,YAAY;AAAA,EACtC,OAAO;AACL,UAAM,qBAAqB;AAAA,MACzB,UAAU;AAAA,MACV,UAAU;AAAA,IAAA;AAEZ,UAAM;AAAA,mCACyB,IAAI,kBAAkB,CAAC;AACtD,UAAM;AAAA,2BACiB,IAAI,kBAAkB,CAAC;AAAA;AAAA;AAG9C,oBAAgB,KAAK,kBAAkB;AAAA,EACzC;AAEA,QAAM,sBAAsB;AAAA,IAC1B,UAAU;AAAA,IACV,UAAU;AAAA,EAAA;AAEZ,kBAAgB,KAAK,mBAAmB;AAExC,QAAM,QAAQ,EAAC,GAAG,WAAW,cAAc,gBAAA;AAG3C,QAAM,IAAI,OAAO,YAAY,KAAK,IAAI,WAAW,OAAO,mBAAmB,CAAC;AAE5E,QAAM,OAAO,MAAM,mBAAmB,KAAK,eAAe;AAC1D,QAAM,6CAA6C,IAAI,GAAG,MAAM,IAAI,GAAG;AAEvE,QAAM,cAAc,IAAI,KAAK,KAAK;AACpC;AAEA,eAAsB,cACpB,IACA,IACA,OACA;AACA,MAAI;AACF,UAAM,GAAG,UAAU,CAAA,QAAO,IAAI,OAAO,aAAa,KAAK,CAAC,CAAC;AAAA,EAC3D,SAAS,GAAG;AACV,QACE,EACE,aAAa,SAAS,iBACtB,EAAE,SAAS,4BAEb;AACA,YAAM;AAAA,IACR;AAEA,OAAG;AAAA,MACD;AAAA;AAAA,GACM,EAAE,QAAQ,EAAE,OAAO;AAAA;AAAA;AAAA;AAAA,IAAA;AAAA,EAI7B;AACF;AAEO,SAAS,qBACd,IACA,WACA;AAEA,YAAU,aAAa,QAAQ,CAAA,QAAO;AACpC,QACE,CAAC,IAAI,aACL,CAAC,IAAI,eACL,CAAC,IAAI,aACL,CAAC,IAAI,aACL;AAEA,YAAM,IAAI;AAAA,QACR,eAAe,IAAI,OAAO;AAAA,MAAA;AAAA,IAE9B;AAAA,EACF,CAAC;AAED,YAAU,OAAO,QAAQ,CAAA,UAAS,SAAS,IAAI,KAAK,CAAC;AACvD;AAMO,SAAS,6CACd,MAC+B;AAC/B,QAAM,oBAIA,CAAA;AACN,aAAW,SAAS,KAAK,QAAQ;AAC/B,QAAI,CAAC,MAAM,YAAY,UAAU,MAAM,oBAAoB,SAAS;AAQlE,YAAM,EAAC,QAAQ,MAAM,UAAA,IAAa;AAClC,iBAAW,EAAC,SAAS,MAAM,UAAA,KAAc,KAAK,QAAQ;AAAA,QACpD,CAAA,QACE,IAAI,WAAW,UACf,IAAI,cAAc,aAClB,IAAI,UACJ,IAAI;AAAA,MAAA,GACL;AACD,YAAI,OAAO,KAAK,OAAO,EAAE,KAAK,CAAA,QAAO,CAAC,MAAM,QAAQ,GAAG,EAAE,OAAO,GAAG;AACjE;AAAA,QACF;AACA,0BAAkB,KAAK,EAAC,QAAQ,WAAW,WAAU;AACrD;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAEA,MAAI,kBAAkB,WAAW,GAAG;AAClC,WAAO;AAAA,EACT;AACA,SAAO;AAAA,IACL,OAAO,OAAO,IAAgB,QAAoB;AAChD,iBAAW,EAAC,QAAQ,WAAW,UAAA,KAAc,mBAAmB;AAC9D,WAAG;AAAA,UACD,YAAY,SAAS,kCAAkC,SAAS;AAAA,QAAA;AAElE,cAAM;AAAA,sBACQ,IAAI,MAAM,CAAC,IAAI,IAAI,SAAS,CAAC;AAAA,yCACV,IAAI,SAAS,CAAC;AAAA,MACjD;AAAA,IACF;AAAA,EAAA;AAEJ;"}

package/out/zero-cache/src/services/change-source/protocol/current/data.d.ts

@@ -9,6 +9,7 @@ import type { Satisfies } from '../../../../types/satisfies.ts';
 export declare const beginSchema: v.ObjectType<{
     tag: v.Type<"begin">;
     json: v.Optional<"p" | "s">;
+    skipAck: v.Optional<boolean>;
 }, undefined>;
 export declare const commitSchema: v.ObjectType<{
     tag: v.Type<"commit">;

package/out/zero-cache/src/services/change-source/protocol/current/data.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../../../../../zero-cache/src/services/change-source/protocol/current/data.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAEL,KAAK,UAAU,EAChB,MAAM,6CAA6C,CAAC;AAErD,OAAO,KAAK,CAAC,MAAM,wCAAwC,CAAC;AAE5D,OAAO,KAAK,EAAC,SAAS,EAAC,MAAM,gCAAgC,CAAC;AAG9D,eAAO,MAAM,WAAW;;;aAatB,CAAC;AAEH,eAAO,MAAM,YAAY;;aAEvB,CAAC;AAEH,eAAO,MAAM,cAAc;;aAEzB,CAAC;AAcH,eAAO,MAAM,cAAc;;;;;;;;;EA2BvB,CAAC;AAGL,eAAO,MAAM,iBAAiB;;;;;;;aAK5B,CAAC;AAUH,eAAO,MAAM,mBAAmB;;2EAER,CAAC;AAEzB,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,mBAAmB,CAAC,CAAC;AAEhE,eAAO,MAAM,SAAS,yFAA4B,CAAC;AAEnD,eAAO,MAAM,YAAY;;;;;;;;;;;;;aAIvB,CAAC;AAEH,eAAO,MAAM,YAAY;;;;;;;;;;;;;;aAUvB,CAAC;AAEH,eAAO,MAAM,YAAY;;;;;;;;;;;;;aAKvB,CAAC;AAEH,eAAO,MAAM,cAAc;;;;;;;;;;;;aAGzB,CAAC;AAEH,eAAO,MAAM,gBAAgB;;;aAG3B,CAAC;AAEH,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAU1D,eAAO,MAAM,gBAAgB,mEAAmB,CAAC;AAEjD,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAE1D,eAAO,MAAM,iBAAiB;;;;;;;;;;;;;;;;;;;;;;aA4B5B,CAAC;AAEH,eAAO,MAAM,iBAAiB;;;;;;;;;;aAI5B,CAAC;AAEH,eAAO,MAAM,yBAAyB;;;;;;;;;;;;aAKpC,CAAC;AAOH,eAAO,MAAM,eAAe;;;;;;;;;;;;;;;;;;;;;;;aAa1B,CAAC;AAEH,eAAO,MAAM,kBAAkB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;aAK7B,CAAC;AAEH,eAAO,MAAM,gBAAgB;;;;;;;aAI3B,CAAC;AAEH,eAAO,MAAM,eAAe;;;;;;aAG1B,CAAC;AAEH,eAAO,MAAM,iBAAiB;;;;;;;;;;aAG5B,CAAC;AAEH,eAAO,MAAM,eAAe;;;;;;aAG1B,CAAC;AAIH,eAAO,MAAM,cAAc;;;;;;;;;;;;;aA0BzB,CAAC;AAIH,eAAO,MAAM,uBAAuB;;;;;;;;;;;;aAclC,CAAC;AAEH,MAAM,MAAM,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,WAAW,CAAC,CAAC;AACvD,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,YAAY,CAAC,CAAC;AACzD,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAC;AAE7D,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAC;AAC7D,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,YAAY,CAAC,CAAC;AACzD,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,YAAY,CAAC,CAAC;AACzD,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,YAAY,CAAC,CAAC;AACzD,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAC;AAE7D,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAC;AAE7D,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAC5D,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAC5D,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC;AAC5E,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AACxD,MAAM,MAAM,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB,CAAC,CAAC;AAC9D,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAC1D,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AACxD,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAC5D,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AACxD,MAAM,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC;AAExE,eAAO,MAAM,gBAAgB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;eAM5B,CAAC;AAWF,QAAA,MAAM,oBAAoB,kEAAoC,CAAC;AAE/D,MAAM,MAAM,UAAU,GAAG,SAAS,CAChC,UAAU,EAAE,+CAA+C;AAC3D,CAAC,CAAC,KAAK,CAAC,OAAO,gBAAgB,CAAC,CACjC,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,oBAAoB,CAAC,CAAC;AA6BjE,eAAO,MAAM,kBAAkB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;eAA4B,CAAC;AAE5D,QAAA,MAAM,sBAAsB,0LAAsC,CAAC;AAEnE,MAAM,MAAM,YAAY,GAAG,SAAS,CAClC,UAAU,EACV,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB,CAAC,CACnC,CAAC;AAEF,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,sBAAsB,CAAC,CAAC;AAErE,MAAM,MAAM,kBAAkB,GAAG,UAAU,GAAG,YAAY,CAAC;AAE3D,MAAM,MAAM,MAAM,GACd,YAAY,GACZ,kBAAkB,GAClB,aAAa,GACb,eAAe,CAAC;AAEpB,MAAM,MAAM,SAAS,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;AAItC,wBAAgB,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,IAAI,YAAY,CAErE;AAID,wBAAgB,YAAY,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,IAAI,UAAU,CAEjE"}
+{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../../../../../zero-cache/src/services/change-source/protocol/current/data.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAEL,KAAK,UAAU,EAChB,MAAM,6CAA6C,CAAC;AAErD,OAAO,KAAK,CAAC,MAAM,wCAAwC,CAAC;AAE5D,OAAO,KAAK,EAAC,SAAS,EAAC,MAAM,gCAAgC,CAAC;AAG9D,eAAO,MAAM,WAAW;;;;aAgBtB,CAAC;AAEH,eAAO,MAAM,YAAY;;aAEvB,CAAC;AAEH,eAAO,MAAM,cAAc;;aAEzB,CAAC;AAcH,eAAO,MAAM,cAAc;;;;;;;;;EA2BvB,CAAC;AAGL,eAAO,MAAM,iBAAiB;;;;;;;aAK5B,CAAC;AAUH,eAAO,MAAM,mBAAmB;;2EAER,CAAC;AAEzB,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,mBAAmB,CAAC,CAAC;AAEhE,eAAO,MAAM,SAAS,yFAA4B,CAAC;AAEnD,eAAO,MAAM,YAAY;;;;;;;;;;;;;aAIvB,CAAC;AAEH,eAAO,MAAM,YAAY;;;;;;;;;;;;;;aAUvB,CAAC;AAEH,eAAO,MAAM,YAAY;;;;;;;;;;;;;aAKvB,CAAC;AAEH,eAAO,MAAM,cAAc;;;;;;;;;;;;aAGzB,CAAC;AAEH,eAAO,MAAM,gBAAgB;;;aAG3B,CAAC;AAEH,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAU1D,eAAO,MAAM,gBAAgB,mEAAmB,CAAC;AAEjD,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAE1D,eAAO,MAAM,iBAAiB;;;;;;;;;;;;;;;;;;;;;;aA4B5B,CAAC;AAEH,eAAO,MAAM,iBAAiB;;;;;;;;;;aAI5B,CAAC;AAEH,eAAO,MAAM,yBAAyB;;;;;;;;;;;;aAKpC,CAAC;AAOH,eAAO,MAAM,eAAe;;;;;;;;;;;;;;;;;;;;;;;aAa1B,CAAC;AAEH,eAAO,MAAM,kBAAkB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;aAK7B,CAAC;AAEH,eAAO,MAAM,gBAAgB;;;;;;;aAI3B,CAAC;AAEH,eAAO,MAAM,eAAe;;;;;;aAG1B,CAAC;AAEH,eAAO,MAAM,iBAAiB;;;;;;;;;;aAG5B,CAAC;AAEH,eAAO,MAAM,eAAe;;;;;;aAG1B,CAAC;AAIH,eAAO,MAAM,cAAc;;;;;;;;;;;;;aA0BzB,CAAC;AAIH,eAAO,MAAM,uBAAuB;;;;;;;;;;;;aAclC,CAAC;AAEH,MAAM,MAAM,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,WAAW,CAAC,CAAC;AACvD,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,YAAY,CAAC,CAAC;AACzD,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAC;AAE7D,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAC;AAC7D,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,YAAY,CAAC,CAAC;AACzD,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,YAAY,CAAC,CAAC;AACzD,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,YAAY,CAAC,CAAC;AACzD,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAC;AAE7D,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAC;AAE7D,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAC5D,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAC5D,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC;AAC5E,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AACxD,MAAM,MAAM,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB,CAAC,CAAC;AAC9D,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAC1D,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AACxD,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAC5D,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AACxD,MAAM,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC;AAExE,eAAO,MAAM,gBAAgB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;eAM5B,CAAC;AAWF,QAAA,MAAM,oBAAoB,kEAAoC,CAAC;AAE/D,MAAM,MAAM,UAAU,GAAG,SAAS,CAChC,UAAU,EAAE,+CAA+C;AAC3D,CAAC,CAAC,KAAK,CAAC,OAAO,gBAAgB,CAAC,CACjC,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,oBAAoB,CAAC,CAAC;AA6BjE,eAAO,MAAM,kBAAkB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;eAA4B,CAAC;AAE5D,QAAA,MAAM,sBAAsB,0LAAsC,CAAC;AAEnE,MAAM,MAAM,YAAY,GAAG,SAAS,CAClC,UAAU,EACV,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB,CAAC,CACnC,CAAC;AAEF,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,sBAAsB,CAAC,CAAC;AAErE,MAAM,MAAM,kBAAkB,GAAG,UAAU,GAAG,YAAY,CAAC;AAE3D,MAAM,MAAM,MAAM,GACd,YAAY,GACZ,kBAAkB,GAClB,aAAa,GACb,eAAe,CAAC;AAEpB,MAAM,MAAM,SAAS,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;AAItC,wBAAgB,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,IAAI,YAAY,CAErE;AAID,wBAAgB,YAAY,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,IAAI,UAAU,CAEjE"}

package/out/zero-cache/src/services/change-source/protocol/current/data.js

@@ -3,7 +3,7 @@ import { must } from "../../../../../../shared/src/must.js";
 import { literalUnion } from "../../../../../../shared/src/valita.js";
 import { tableSpec, columnSpec, indexSpec } from "../../../../db/specs.js";
 import { jsonObjectSchema } from "./json.js";
-import { object, literal, array, string, record, union } from "@badrap/valita";
+import { object, boolean, literal, array, string, record, union } from "@badrap/valita";
 const beginSchema = object({
   tag: literal("begin"),
   // The format of values of "json"-typed columns (e.g. "JSON" and "JSONB").
@@ -16,7 +16,9 @@ const beginSchema = object({
   //   change-streamer and 25~30% in the replicator.
   //
   // If absent, the format is assumed to be 'p' (parsed JSON objects/values).
-  json: literalUnion("p", "s").optional()
+  json: literalUnion("p", "s").optional(),
+  // Directs the change-streamer to skip the ACK for the corresponding commit.
+  skipAck: boolean().optional()
 });
 const commitSchema = object({
   tag: literal("commit")

package/out/zero-cache/src/services/change-source/protocol/current/data.js.map

@@ -1 +1 @@
-{"version":3,"file":"data.js","sources":["../../../../../../../../zero-cache/src/services/change-source/protocol/current/data.ts"],"sourcesContent":["/**\n * Data plane messages encapsulate changes that are sent by ChangeSources,\n * forwarded / fanned out to subscribers by the ChangeStreamerService, and\n * stored in the Change DB for catchup of old subscribers.\n */\n\nimport {\n  jsonValueSchema,\n  type JSONObject,\n} from '../../../../../../shared/src/bigint-json.ts';\nimport {must} from '../../../../../../shared/src/must.ts';\nimport * as v from '../../../../../../shared/src/valita.ts';\nimport {columnSpec, indexSpec, tableSpec} from '../../../../db/specs.ts';\nimport type {Satisfies} from '../../../../types/satisfies.ts';\nimport {jsonObjectSchema} from './json.ts';\n\nexport const beginSchema = v.object({\n  tag: v.literal('begin'),\n  // The format of values of \"json\"-typed columns (e.g. \"JSON\" and \"JSONB\").\n  // - 'p' is for parsed JSON, which may include JSON values or JSON objects.\n  //   These values are parsed and stringified at every process boundary\n  //   between the change-source and the replica.\n  // - 's' is for stringified JSON. These values skip the parsing and\n  //   stringification, and are directly ferried to the replica as a JSON\n  //   string. For JSON values this improves performance by 20~25% in the\n  //   change-streamer and 25~30% in the replicator.\n  //\n  // If absent, the format is assumed to be 'p' (parsed JSON objects/values).\n  json: v.literalUnion('p', 's').optional(),\n});\n\nexport const commitSchema = v.object({\n  tag: v.literal('commit'),\n});\n\nexport const rollbackSchema = v.object({\n  tag: v.literal('rollback'),\n});\n\nconst rowKeySchema = v.object({\n  // The columns used to identify a row in insert, update, and delete changes.\n  columns: v.array(v.string()),\n\n  // An optional qualifier identifying how the key is chosen. Currently this\n  // is postgres-specific, describing the REPLICA IDENTITY, for which replica\n  // identity 'full' (FULL) is handled differently; the replicator handles\n  // these tables by extracting a row key from the full row based on the\n  // table's PRIMARY KEY or UNIQUE INDEX.\n  type: v.literalUnion('default', 'nothing', 'full', 'index').optional(),\n});\n\nexport const relationSchema = v\n  .object({\n    schema: v.string(),\n    name: v.string(),\n\n    // This will become required.\n    rowKey: rowKeySchema.optional(),\n\n    /** Deprecated: set the rowKey.columns instead. */\n    keyColumns: v.array(v.string()).optional(),\n    /** Deprecated: set the rowKey.columns instead. */\n    replicaIdentity: v\n      .literalUnion('default', 'nothing', 'full', 'index')\n      .optional(),\n  })\n  .map(rel => {\n    const {rowKey, ...rest} = rel;\n    if (rowKey) {\n      return {...rest, rowKey};\n    }\n    return {\n      ...rest,\n      rowKey: {\n        columns: must(rel.keyColumns),\n        type: rel.replicaIdentity,\n      },\n    };\n  });\n\n// The eventual fate of relationSchema\nexport const newRelationSchema = v.object({\n  schema: v.string(),\n  name: v.string(),\n\n  rowKey: rowKeySchema,\n});\n\n// TableMetadata contains table-related configuration that does not affect the\n// actual data in the table, but rather how the table's change messages are\n// handled. The is an opaque object that clients must track (and update) based\n// on `create-table`, `add-column`, and `table-update-metadata` messages, and\n// pass in BackfillRequests when there are columns to be backfilled.\n//\n// Note that the backfill-related change-source implementation does, however,\n// rely on the rowKey (columns) being specified in the message.\nexport const tableMetadataSchema = v\n  .object({rowKey: v.record(jsonValueSchema)})\n  .rest(jsonValueSchema);\n\nexport type TableMetadata = v.Infer<typeof tableMetadataSchema>;\n\nexport const rowSchema = v.record(jsonValueSchema);\n\nexport const insertSchema = v.object({\n  tag: v.literal('insert'),\n  relation: relationSchema,\n  new: rowSchema,\n});\n\nexport const updateSchema = v.object({\n  tag: v.literal('update'),\n  relation: relationSchema,\n  // `key` is present if the update changed the key of the row, or if the\n  // table's replicaIdentity === 'full'\n  key: rowSchema.nullable(),\n  // `new` is the full row (and not just the updated columns). This is\n  // necessary for \"catchup\" replication scenarios such as adding tables\n  // to a publication, or resharding.\n  new: rowSchema,\n});\n\nexport const deleteSchema = v.object({\n  tag: v.literal('delete'),\n  relation: relationSchema,\n  // key is the full row if replicaIdentity === 'full'\n  key: rowSchema,\n});\n\nexport const truncateSchema = v.object({\n  tag: v.literal('truncate'),\n  relations: v.array(relationSchema),\n});\n\nexport const identifierSchema = v.object({\n  schema: v.string(),\n  name: v.string(),\n});\n\nexport type Identifier = v.Infer<typeof identifierSchema>;\n\n// A BackfillID is an upstream specific stable identifier for a column\n// that needs backfilling. This id is used to ensure that the schema, table,\n// and column names of a requested backfill still match the original\n// underlying upstream ID.\n//\n// The change-streamer stores these IDs as opaque values while a column is\n// being backfilled, and initiates new change-source streams with the IDs\n// in order to restart backfills that did not complete in previous sessions.\nexport const backfillIDSchema = jsonObjectSchema;\n\nexport type BackfillID = v.Infer<typeof backfillIDSchema>;\n\nexport const createTableSchema = v.object({\n  tag: v.literal('create-table'),\n  spec: tableSpec,\n\n  // This must be set by change source implementations that support\n  // table/column backfill.\n  //\n  // TODO: to simplify the protocol, see if we can make this required\n  metadata: tableMetadataSchema.optional(),\n\n  // Indicate that columns of the table require backfilling. These columns\n  // should be created on the replica but not yet synced the clients.\n  //\n  // ## State Persistence\n  //\n  // To obviate the need for change-source implementations to persist state\n  // related to backfill progress, the change-source only tracks backfills\n  // **for the current session**. In the event that the session is interrupted\n  // before columns have been fully backfilled, it is the responsibility of the\n  // change-streamer to send {@link BackfillRequest}s when it reconnects.\n  //\n  // This means that the change-streamer must track and persist:\n  // * the backfill IDs of the columns requiring backfilling\n  // * the most current table metadata of the associated table(s)\n  //\n  // The change-streamer then uses this information to send backfill requests\n  // when it reconnects.\n  backfill: v.record(backfillIDSchema).optional(),\n});\n\nexport const renameTableSchema = v.object({\n  tag: v.literal('rename-table'),\n  old: identifierSchema,\n  new: identifierSchema,\n});\n\nexport const updateTableMetadataSchema = v.object({\n  tag: v.literal('update-table-metadata'),\n  table: identifierSchema,\n  old: tableMetadataSchema,\n  new: tableMetadataSchema,\n});\n\nconst columnSchema = v.object({\n  name: v.string(),\n  spec: columnSpec,\n});\n\nexport const addColumnSchema = v.object({\n  tag: v.literal('add-column'),\n  table: identifierSchema,\n  column: columnSchema,\n\n  // This must be set by change source implementations that support\n  // table/column backfill.\n  //\n  // TODO: to simplify the protocol, see if we can make this required\n  tableMetadata: tableMetadataSchema.optional(),\n\n  // See documentation for the `backfill` field of the `create-table` change.\n  backfill: backfillIDSchema.optional(),\n});\n\nexport const updateColumnSchema = v.object({\n  tag: v.literal('update-column'),\n  table: identifierSchema,\n  old: columnSchema,\n  new: columnSchema,\n});\n\nexport const dropColumnSchema = v.object({\n  tag: v.literal('drop-column'),\n  table: identifierSchema,\n  column: v.string(),\n});\n\nexport const dropTableSchema = v.object({\n  tag: v.literal('drop-table'),\n  id: identifierSchema,\n});\n\nexport const createIndexSchema = v.object({\n  tag: v.literal('create-index'),\n  spec: indexSpec,\n});\n\nexport const dropIndexSchema = v.object({\n  tag: v.literal('drop-index'),\n  id: identifierSchema,\n});\n\n// A batch of rows from a single table containing column values\n// to be backfilled.\nexport const backfillSchema = v.object({\n  tag: v.literal('backfill'),\n\n  relation: newRelationSchema,\n\n  // The columns to be backfilled. `rowKey` columns are automatically excluded,\n  // which means that this field may be empty.\n  columns: v.array(v.string()),\n\n  // The watermark at which the backfill data was queried. Note that this\n  // generally will be different from the commit watermarks of the main change\n  // stream, and in particular, the commit watermark of the backfill change's\n  // enclosing transaction.\n  watermark: v.string(),\n\n  // A batch of row values, each row consisting of the `rowKey`\n  // values, followed by the `column` values, in the same order in which\n  // the column names appear in their respective fields, e.g.\n  //\n  // ```\n  // [\n  //   [...rowKeyValues, ...columnValues],  // row 1\n  //   [...rowKeyValues, ...columnValues],  // row 2\n  // ]\n  // ```\n  rowValues: v.array(v.array(jsonValueSchema)),\n});\n\n// Indicates that the backfill for the specified columns have\n// been successfully backfilled and can be published to clients.\nexport const backfillCompletedSchema = v.object({\n  tag: v.literal('backfill-completed'),\n\n  relation: newRelationSchema,\n\n  // The columns to be backfilled. `rowKey` columns are automatically excluded,\n  // which means that this field may be empty.\n  columns: v.array(v.string()),\n\n  // The watermark at which the backfill data was queried. Note that this\n  // generally will be different from the commit watermarks of the main change\n  // stream, and in particular, the commit watermark of the backfill change's\n  // enclosing transaction.\n  watermark: v.string(),\n});\n\nexport type MessageBegin = v.Infer<typeof beginSchema>;\nexport type MessageCommit = v.Infer<typeof commitSchema>;\nexport type MessageRollback = v.Infer<typeof rollbackSchema>;\n\nexport type MessageRelation = v.Infer<typeof relationSchema>;\nexport type MessageInsert = v.Infer<typeof insertSchema>;\nexport type MessageUpdate = v.Infer<typeof updateSchema>;\nexport type MessageDelete = v.Infer<typeof deleteSchema>;\nexport type MessageTruncate = v.Infer<typeof truncateSchema>;\n\nexport type MessageBackfill = v.Infer<typeof backfillSchema>;\n\nexport type TableCreate = v.Infer<typeof createTableSchema>;\nexport type TableRename = v.Infer<typeof renameTableSchema>;\nexport type TableUpdateMetadata = v.Infer<typeof updateTableMetadataSchema>;\nexport type ColumnAdd = v.Infer<typeof addColumnSchema>;\nexport type ColumnUpdate = v.Infer<typeof updateColumnSchema>;\nexport type ColumnDrop = v.Infer<typeof dropColumnSchema>;\nexport type TableDrop = v.Infer<typeof dropTableSchema>;\nexport type IndexCreate = v.Infer<typeof createIndexSchema>;\nexport type IndexDrop = v.Infer<typeof dropIndexSchema>;\nexport type BackfillCompleted = v.Infer<typeof backfillCompletedSchema>;\n\nexport const dataChangeSchema = v.union(\n  insertSchema,\n  updateSchema,\n  deleteSchema,\n  truncateSchema,\n  backfillSchema,\n);\n\n// Note: keep in sync or the tag tests will fail\nconst dataChangeTags = [\n  'insert',\n  'update',\n  'delete',\n  'truncate',\n  'backfill',\n] as const;\n\nconst dataChangeTagsSchema = v.literalUnion(...dataChangeTags);\n\nexport type DataChange = Satisfies<\n  JSONObject, // guarantees serialization over IPC or network\n  v.Infer<typeof dataChangeSchema>\n>;\n\nexport type DataChangeTag = v.Infer<typeof dataChangeTagsSchema>;\n\nconst schemaChanges = [\n  createTableSchema,\n  renameTableSchema,\n  updateTableMetadataSchema,\n  addColumnSchema,\n  updateColumnSchema,\n  dropColumnSchema,\n  dropTableSchema,\n  createIndexSchema,\n  dropIndexSchema,\n  backfillCompletedSchema,\n] as const;\n\n// Note: keep in sync or the tag tests will fail\nconst schemaChangeTags = [\n  'create-table',\n  'rename-table',\n  'update-table-metadata',\n  'add-column',\n  'update-column',\n  'drop-column',\n  'drop-table',\n  'create-index',\n  'drop-index',\n  'backfill-completed',\n] as const;\n\nexport const schemaChangeSchema = v.union(...schemaChanges);\n\nconst schemaChangeTagsSchema = v.literalUnion(...schemaChangeTags);\n\nexport type SchemaChange = Satisfies<\n  JSONObject,\n  v.Infer<typeof schemaChangeSchema>\n>;\n\nexport type SchemaChangeTag = v.Infer<typeof schemaChangeTagsSchema>;\n\nexport type DataOrSchemaChange = DataChange | SchemaChange;\n\nexport type Change =\n  | MessageBegin\n  | DataOrSchemaChange\n  | MessageCommit\n  | MessageRollback;\n\nexport type ChangeTag = Change['tag'];\n\nconst schemaChangeTagSet = new Set<string>(schemaChangeTags);\n\nexport function isSchemaChange(change: Change): change is SchemaChange {\n  return schemaChangeTagSet.has(change.tag);\n}\n\nconst dataChangeTagSet = new Set<string>(dataChangeTags);\n\nexport function isDataChange(change: Change): change is DataChange {\n  return dataChangeTagSet.has(change.tag);\n}\n"],"names":["v.object","v.literal","v.literalUnion","v.array","v.string","v.record","v.union"],"mappings":";;;;;;AAgBO,MAAM,cAAcA,OAAS;AAAA,EAClC,KAAKC,QAAU,OAAO;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAWtB,MAAMC,aAAe,KAAK,GAAG,EAAE,SAAA;AACjC,CAAC;AAEM,MAAM,eAAeF,OAAS;AAAA,EACnC,KAAKC,QAAU,QAAQ;AACzB,CAAC;AAEM,MAAM,iBAAiBD,OAAS;AAAA,EACrC,KAAKC,QAAU,UAAU;AAC3B,CAAC;AAED,MAAM,eAAeD,OAAS;AAAA;AAAA,EAE5B,SAASG,MAAQC,QAAU;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAO3B,MAAMF,aAAe,WAAW,WAAW,QAAQ,OAAO,EAAE,SAAA;AAC9D,CAAC;AAEM,MAAM,iBAAiBF,OACpB;AAAA,EACN,QAAQI,OAAE;AAAA,EACV,MAAMA,OAAE;AAAA;AAAA,EAGR,QAAQ,aAAa,SAAA;AAAA;AAAA,EAGrB,YAAYD,MAAQC,OAAE,CAAQ,EAAE,SAAA;AAAA;AAAA,EAEhC,iBAAiBF,aACD,WAAW,WAAW,QAAQ,OAAO,EAClD,SAAA;AACL,CAAC,EACA,IAAI,CAAA,QAAO;AACV,QAAM,EAAC,QAAQ,GAAG,KAAA,IAAQ;AAC1B,MAAI,QAAQ;AACV,WAAO,EAAC,GAAG,MAAM,OAAA;AAAA,EACnB;AACA,SAAO;AAAA,IACL,GAAG;AAAA,IACH,QAAQ;AAAA,MACN,SAAS,KAAK,IAAI,UAAU;AAAA,MAC5B,MAAM,IAAI;AAAA,IAAA;AAAA,EACZ;AAEJ,CAAC;AAGI,MAAM,oBAAoBF,OAAS;AAAA,EACxC,QAAQI,OAAE;AAAA,EACV,MAAMA,OAAE;AAAA,EAER,QAAQ;AACV,CAAC;AAUM,MAAM,sBAAsBJ,OACzB,EAAC,QAAQK,OAAS,eAAe,EAAA,CAAE,EAC1C,KAAK,eAAe;AAIhB,MAAM,YAAYA,OAAS,eAAe;AAE1C,MAAM,eAAeL,OAAS;AAAA,EACnC,KAAKC,QAAU,QAAQ;AAAA,EACvB,UAAU;AAAA,EACV,KAAK;AACP,CAAC;AAEM,MAAM,eAAeD,OAAS;AAAA,EACnC,KAAKC,QAAU,QAAQ;AAAA,EACvB,UAAU;AAAA;AAAA;AAAA,EAGV,KAAK,UAAU,SAAA;AAAA;AAAA;AAAA;AAAA,EAIf,KAAK;AACP,CAAC;AAEM,MAAM,eAAeD,OAAS;AAAA,EACnC,KAAKC,QAAU,QAAQ;AAAA,EACvB,UAAU;AAAA;AAAA,EAEV,KAAK;AACP,CAAC;AAEM,MAAM,iBAAiBD,OAAS;AAAA,EACrC,KAAKC,QAAU,UAAU;AAAA,EACzB,WAAWE,MAAQ,cAAc;AACnC,CAAC;AAEM,MAAM,mBAAmBH,OAAS;AAAA,EACvC,QAAQI,OAAE;AAAA,EACV,MAAMA,OAAE;AACV,CAAC;AAYM,MAAM,mBAAmB;AAIzB,MAAM,oBAAoBJ,OAAS;AAAA,EACxC,KAAKC,QAAU,cAAc;AAAA,EAC7B,MAAM;AAAA;AAAA;AAAA;AAAA;AAAA,EAMN,UAAU,oBAAoB,SAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAmB9B,UAAUI,OAAS,gBAAgB,EAAE,SAAA;AACvC,CAAC;AAEM,MAAM,oBAAoBL,OAAS;AAAA,EACxC,KAAKC,QAAU,cAAc;AAAA,EAC7B,KAAK;AAAA,EACL,KAAK;AACP,CAAC;AAEM,MAAM,4BAA4BD,OAAS;AAAA,EAChD,KAAKC,QAAU,uBAAuB;AAAA,EACtC,OAAO;AAAA,EACP,KAAK;AAAA,EACL,KAAK;AACP,CAAC;AAED,MAAM,eAAeD,OAAS;AAAA,EAC5B,MAAMI,OAAE;AAAA,EACR,MAAM;AACR,CAAC;AAEM,MAAM,kBAAkBJ,OAAS;AAAA,EACtC,KAAKC,QAAU,YAAY;AAAA,EAC3B,OAAO;AAAA,EACP,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA,EAMR,eAAe,oBAAoB,SAAA;AAAA;AAAA,EAGnC,UAAU,iBAAiB,SAAA;AAC7B,CAAC;AAEM,MAAM,qBAAqBD,OAAS;AAAA,EACzC,KAAKC,QAAU,eAAe;AAAA,EAC9B,OAAO;AAAA,EACP,KAAK;AAAA,EACL,KAAK;AACP,CAAC;AAEM,MAAM,mBAAmBD,OAAS;AAAA,EACvC,KAAKC,QAAU,aAAa;AAAA,EAC5B,OAAO;AAAA,EACP,QAAQG,OAAE;AACZ,CAAC;AAEM,MAAM,kBAAkBJ,OAAS;AAAA,EACtC,KAAKC,QAAU,YAAY;AAAA,EAC3B,IAAI;AACN,CAAC;AAEM,MAAM,oBAAoBD,OAAS;AAAA,EACxC,KAAKC,QAAU,cAAc;AAAA,EAC7B,MAAM;AACR,CAAC;AAEM,MAAM,kBAAkBD,OAAS;AAAA,EACtC,KAAKC,QAAU,YAAY;AAAA,EAC3B,IAAI;AACN,CAAC;AAIM,MAAM,iBAAiBD,OAAS;AAAA,EACrC,KAAKC,QAAU,UAAU;AAAA,EAEzB,UAAU;AAAA;AAAA;AAAA,EAIV,SAASE,MAAQC,QAAU;AAAA;AAAA;AAAA;AAAA;AAAA,EAM3B,WAAWA,OAAE;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAYb,WAAWD,MAAQA,MAAQ,eAAe,CAAC;AAC7C,CAAC;AAIM,MAAM,0BAA0BH,OAAS;AAAA,EAC9C,KAAKC,QAAU,oBAAoB;AAAA,EAEnC,UAAU;AAAA;AAAA;AAAA,EAIV,SAASE,MAAQC,QAAU;AAAA;AAAA;AAAA;AAAA;AAAA,EAM3B,WAAWA,OAAE;AACf,CAAC;AAyBM,MAAM,mBAAmBE;AAAAA,EAC9B;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAGA,MAAM,iBAAiB;AAAA,EACrB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAE6BJ,aAAe,GAAG,cAAc;AAS7D,MAAM,gBAAgB;AAAA,EACpB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAGA,MAAM,mBAAmB;AAAA,EACvB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAEO,MAAM,qBAAqBI,MAAQ,GAAG,aAAa;AAE3BJ,aAAe,GAAG,gBAAgB;AAmBjE,MAAM,qBAAqB,IAAI,IAAY,gBAAgB;AAEpD,SAAS,eAAe,QAAwC;AACrE,SAAO,mBAAmB,IAAI,OAAO,GAAG;AAC1C;AAEA,MAAM,mBAAmB,IAAI,IAAY,cAAc;AAEhD,SAAS,aAAa,QAAsC;AACjE,SAAO,iBAAiB,IAAI,OAAO,GAAG;AACxC;"}
+{"version":3,"file":"data.js","sources":["../../../../../../../../zero-cache/src/services/change-source/protocol/current/data.ts"],"sourcesContent":["/**\n * Data plane messages encapsulate changes that are sent by ChangeSources,\n * forwarded / fanned out to subscribers by the ChangeStreamerService, and\n * stored in the Change DB for catchup of old subscribers.\n */\n\nimport {\n  jsonValueSchema,\n  type JSONObject,\n} from '../../../../../../shared/src/bigint-json.ts';\nimport {must} from '../../../../../../shared/src/must.ts';\nimport * as v from '../../../../../../shared/src/valita.ts';\nimport {columnSpec, indexSpec, tableSpec} from '../../../../db/specs.ts';\nimport type {Satisfies} from '../../../../types/satisfies.ts';\nimport {jsonObjectSchema} from './json.ts';\n\nexport const beginSchema = v.object({\n  tag: v.literal('begin'),\n  // The format of values of \"json\"-typed columns (e.g. \"JSON\" and \"JSONB\").\n  // - 'p' is for parsed JSON, which may include JSON values or JSON objects.\n  //   These values are parsed and stringified at every process boundary\n  //   between the change-source and the replica.\n  // - 's' is for stringified JSON. These values skip the parsing and\n  //   stringification, and are directly ferried to the replica as a JSON\n  //   string. For JSON values this improves performance by 20~25% in the\n  //   change-streamer and 25~30% in the replicator.\n  //\n  // If absent, the format is assumed to be 'p' (parsed JSON objects/values).\n  json: v.literalUnion('p', 's').optional(),\n\n  // Directs the change-streamer to skip the ACK for the corresponding commit.\n  skipAck: v.boolean().optional(),\n});\n\nexport const commitSchema = v.object({\n  tag: v.literal('commit'),\n});\n\nexport const rollbackSchema = v.object({\n  tag: v.literal('rollback'),\n});\n\nconst rowKeySchema = v.object({\n  // The columns used to identify a row in insert, update, and delete changes.\n  columns: v.array(v.string()),\n\n  // An optional qualifier identifying how the key is chosen. Currently this\n  // is postgres-specific, describing the REPLICA IDENTITY, for which replica\n  // identity 'full' (FULL) is handled differently; the replicator handles\n  // these tables by extracting a row key from the full row based on the\n  // table's PRIMARY KEY or UNIQUE INDEX.\n  type: v.literalUnion('default', 'nothing', 'full', 'index').optional(),\n});\n\nexport const relationSchema = v\n  .object({\n    schema: v.string(),\n    name: v.string(),\n\n    // This will become required.\n    rowKey: rowKeySchema.optional(),\n\n    /** Deprecated: set the rowKey.columns instead. */\n    keyColumns: v.array(v.string()).optional(),\n    /** Deprecated: set the rowKey.columns instead. */\n    replicaIdentity: v\n      .literalUnion('default', 'nothing', 'full', 'index')\n      .optional(),\n  })\n  .map(rel => {\n    const {rowKey, ...rest} = rel;\n    if (rowKey) {\n      return {...rest, rowKey};\n    }\n    return {\n      ...rest,\n      rowKey: {\n        columns: must(rel.keyColumns),\n        type: rel.replicaIdentity,\n      },\n    };\n  });\n\n// The eventual fate of relationSchema\nexport const newRelationSchema = v.object({\n  schema: v.string(),\n  name: v.string(),\n\n  rowKey: rowKeySchema,\n});\n\n// TableMetadata contains table-related configuration that does not affect the\n// actual data in the table, but rather how the table's change messages are\n// handled. The is an opaque object that clients must track (and update) based\n// on `create-table`, `add-column`, and `table-update-metadata` messages, and\n// pass in BackfillRequests when there are columns to be backfilled.\n//\n// Note that the backfill-related change-source implementation does, however,\n// rely on the rowKey (columns) being specified in the message.\nexport const tableMetadataSchema = v\n  .object({rowKey: v.record(jsonValueSchema)})\n  .rest(jsonValueSchema);\n\nexport type TableMetadata = v.Infer<typeof tableMetadataSchema>;\n\nexport const rowSchema = v.record(jsonValueSchema);\n\nexport const insertSchema = v.object({\n  tag: v.literal('insert'),\n  relation: relationSchema,\n  new: rowSchema,\n});\n\nexport const updateSchema = v.object({\n  tag: v.literal('update'),\n  relation: relationSchema,\n  // `key` is present if the update changed the key of the row, or if the\n  // table's replicaIdentity === 'full'\n  key: rowSchema.nullable(),\n  // `new` is the full row (and not just the updated columns). This is\n  // necessary for \"catchup\" replication scenarios such as adding tables\n  // to a publication, or resharding.\n  new: rowSchema,\n});\n\nexport const deleteSchema = v.object({\n  tag: v.literal('delete'),\n  relation: relationSchema,\n  // key is the full row if replicaIdentity === 'full'\n  key: rowSchema,\n});\n\nexport const truncateSchema = v.object({\n  tag: v.literal('truncate'),\n  relations: v.array(relationSchema),\n});\n\nexport const identifierSchema = v.object({\n  schema: v.string(),\n  name: v.string(),\n});\n\nexport type Identifier = v.Infer<typeof identifierSchema>;\n\n// A BackfillID is an upstream specific stable identifier for a column\n// that needs backfilling. This id is used to ensure that the schema, table,\n// and column names of a requested backfill still match the original\n// underlying upstream ID.\n//\n// The change-streamer stores these IDs as opaque values while a column is\n// being backfilled, and initiates new change-source streams with the IDs\n// in order to restart backfills that did not complete in previous sessions.\nexport const backfillIDSchema = jsonObjectSchema;\n\nexport type BackfillID = v.Infer<typeof backfillIDSchema>;\n\nexport const createTableSchema = v.object({\n  tag: v.literal('create-table'),\n  spec: tableSpec,\n\n  // This must be set by change source implementations that support\n  // table/column backfill.\n  //\n  // TODO: to simplify the protocol, see if we can make this required\n  metadata: tableMetadataSchema.optional(),\n\n  // Indicate that columns of the table require backfilling. These columns\n  // should be created on the replica but not yet synced the clients.\n  //\n  // ## State Persistence\n  //\n  // To obviate the need for change-source implementations to persist state\n  // related to backfill progress, the change-source only tracks backfills\n  // **for the current session**. In the event that the session is interrupted\n  // before columns have been fully backfilled, it is the responsibility of the\n  // change-streamer to send {@link BackfillRequest}s when it reconnects.\n  //\n  // This means that the change-streamer must track and persist:\n  // * the backfill IDs of the columns requiring backfilling\n  // * the most current table metadata of the associated table(s)\n  //\n  // The change-streamer then uses this information to send backfill requests\n  // when it reconnects.\n  backfill: v.record(backfillIDSchema).optional(),\n});\n\nexport const renameTableSchema = v.object({\n  tag: v.literal('rename-table'),\n  old: identifierSchema,\n  new: identifierSchema,\n});\n\nexport const updateTableMetadataSchema = v.object({\n  tag: v.literal('update-table-metadata'),\n  table: identifierSchema,\n  old: tableMetadataSchema,\n  new: tableMetadataSchema,\n});\n\nconst columnSchema = v.object({\n  name: v.string(),\n  spec: columnSpec,\n});\n\nexport const addColumnSchema = v.object({\n  tag: v.literal('add-column'),\n  table: identifierSchema,\n  column: columnSchema,\n\n  // This must be set by change source implementations that support\n  // table/column backfill.\n  //\n  // TODO: to simplify the protocol, see if we can make this required\n  tableMetadata: tableMetadataSchema.optional(),\n\n  // See documentation for the `backfill` field of the `create-table` change.\n  backfill: backfillIDSchema.optional(),\n});\n\nexport const updateColumnSchema = v.object({\n  tag: v.literal('update-column'),\n  table: identifierSchema,\n  old: columnSchema,\n  new: columnSchema,\n});\n\nexport const dropColumnSchema = v.object({\n  tag: v.literal('drop-column'),\n  table: identifierSchema,\n  column: v.string(),\n});\n\nexport const dropTableSchema = v.object({\n  tag: v.literal('drop-table'),\n  id: identifierSchema,\n});\n\nexport const createIndexSchema = v.object({\n  tag: v.literal('create-index'),\n  spec: indexSpec,\n});\n\nexport const dropIndexSchema = v.object({\n  tag: v.literal('drop-index'),\n  id: identifierSchema,\n});\n\n// A batch of rows from a single table containing column values\n// to be backfilled.\nexport const backfillSchema = v.object({\n  tag: v.literal('backfill'),\n\n  relation: newRelationSchema,\n\n  // The columns to be backfilled. `rowKey` columns are automatically excluded,\n  // which means that this field may be empty.\n  columns: v.array(v.string()),\n\n  // The watermark at which the backfill data was queried. Note that this\n  // generally will be different from the commit watermarks of the main change\n  // stream, and in particular, the commit watermark of the backfill change's\n  // enclosing transaction.\n  watermark: v.string(),\n\n  // A batch of row values, each row consisting of the `rowKey`\n  // values, followed by the `column` values, in the same order in which\n  // the column names appear in their respective fields, e.g.\n  //\n  // ```\n  // [\n  //   [...rowKeyValues, ...columnValues],  // row 1\n  //   [...rowKeyValues, ...columnValues],  // row 2\n  // ]\n  // ```\n  rowValues: v.array(v.array(jsonValueSchema)),\n});\n\n// Indicates that the backfill for the specified columns have\n// been successfully backfilled and can be published to clients.\nexport const backfillCompletedSchema = v.object({\n  tag: v.literal('backfill-completed'),\n\n  relation: newRelationSchema,\n\n  // The columns to be backfilled. `rowKey` columns are automatically excluded,\n  // which means that this field may be empty.\n  columns: v.array(v.string()),\n\n  // The watermark at which the backfill data was queried. Note that this\n  // generally will be different from the commit watermarks of the main change\n  // stream, and in particular, the commit watermark of the backfill change's\n  // enclosing transaction.\n  watermark: v.string(),\n});\n\nexport type MessageBegin = v.Infer<typeof beginSchema>;\nexport type MessageCommit = v.Infer<typeof commitSchema>;\nexport type MessageRollback = v.Infer<typeof rollbackSchema>;\n\nexport type MessageRelation = v.Infer<typeof relationSchema>;\nexport type MessageInsert = v.Infer<typeof insertSchema>;\nexport type MessageUpdate = v.Infer<typeof updateSchema>;\nexport type MessageDelete = v.Infer<typeof deleteSchema>;\nexport type MessageTruncate = v.Infer<typeof truncateSchema>;\n\nexport type MessageBackfill = v.Infer<typeof backfillSchema>;\n\nexport type TableCreate = v.Infer<typeof createTableSchema>;\nexport type TableRename = v.Infer<typeof renameTableSchema>;\nexport type TableUpdateMetadata = v.Infer<typeof updateTableMetadataSchema>;\nexport type ColumnAdd = v.Infer<typeof addColumnSchema>;\nexport type ColumnUpdate = v.Infer<typeof updateColumnSchema>;\nexport type ColumnDrop = v.Infer<typeof dropColumnSchema>;\nexport type TableDrop = v.Infer<typeof dropTableSchema>;\nexport type IndexCreate = v.Infer<typeof createIndexSchema>;\nexport type IndexDrop = v.Infer<typeof dropIndexSchema>;\nexport type BackfillCompleted = v.Infer<typeof backfillCompletedSchema>;\n\nexport const dataChangeSchema = v.union(\n  insertSchema,\n  updateSchema,\n  deleteSchema,\n  truncateSchema,\n  backfillSchema,\n);\n\n// Note: keep in sync or the tag tests will fail\nconst dataChangeTags = [\n  'insert',\n  'update',\n  'delete',\n  'truncate',\n  'backfill',\n] as const;\n\nconst dataChangeTagsSchema = v.literalUnion(...dataChangeTags);\n\nexport type DataChange = Satisfies<\n  JSONObject, // guarantees serialization over IPC or network\n  v.Infer<typeof dataChangeSchema>\n>;\n\nexport type DataChangeTag = v.Infer<typeof dataChangeTagsSchema>;\n\nconst schemaChanges = [\n  createTableSchema,\n  renameTableSchema,\n  updateTableMetadataSchema,\n  addColumnSchema,\n  updateColumnSchema,\n  dropColumnSchema,\n  dropTableSchema,\n  createIndexSchema,\n  dropIndexSchema,\n  backfillCompletedSchema,\n] as const;\n\n// Note: keep in sync or the tag tests will fail\nconst schemaChangeTags = [\n  'create-table',\n  'rename-table',\n  'update-table-metadata',\n  'add-column',\n  'update-column',\n  'drop-column',\n  'drop-table',\n  'create-index',\n  'drop-index',\n  'backfill-completed',\n] as const;\n\nexport const schemaChangeSchema = v.union(...schemaChanges);\n\nconst schemaChangeTagsSchema = v.literalUnion(...schemaChangeTags);\n\nexport type SchemaChange = Satisfies<\n  JSONObject,\n  v.Infer<typeof schemaChangeSchema>\n>;\n\nexport type SchemaChangeTag = v.Infer<typeof schemaChangeTagsSchema>;\n\nexport type DataOrSchemaChange = DataChange | SchemaChange;\n\nexport type Change =\n  | MessageBegin\n  | DataOrSchemaChange\n  | MessageCommit\n  | MessageRollback;\n\nexport type ChangeTag = Change['tag'];\n\nconst schemaChangeTagSet = new Set<string>(schemaChangeTags);\n\nexport function isSchemaChange(change: Change): change is SchemaChange {\n  return schemaChangeTagSet.has(change.tag);\n}\n\nconst dataChangeTagSet = new Set<string>(dataChangeTags);\n\nexport function isDataChange(change: Change): change is DataChange {\n  return dataChangeTagSet.has(change.tag);\n}\n"],"names":["v.object","v.literal","v.literalUnion","v.boolean","v.array","v.string","v.record","v.union"],"mappings":";;;;;;AAgBO,MAAM,cAAcA,OAAS;AAAA,EAClC,KAAKC,QAAU,OAAO;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAWtB,MAAMC,aAAe,KAAK,GAAG,EAAE,SAAA;AAAA;AAAA,EAG/B,SAASC,QAAE,EAAU,SAAA;AACvB,CAAC;AAEM,MAAM,eAAeH,OAAS;AAAA,EACnC,KAAKC,QAAU,QAAQ;AACzB,CAAC;AAEM,MAAM,iBAAiBD,OAAS;AAAA,EACrC,KAAKC,QAAU,UAAU;AAC3B,CAAC;AAED,MAAM,eAAeD,OAAS;AAAA;AAAA,EAE5B,SAASI,MAAQC,QAAU;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAO3B,MAAMH,aAAe,WAAW,WAAW,QAAQ,OAAO,EAAE,SAAA;AAC9D,CAAC;AAEM,MAAM,iBAAiBF,OACpB;AAAA,EACN,QAAQK,OAAE;AAAA,EACV,MAAMA,OAAE;AAAA;AAAA,EAGR,QAAQ,aAAa,SAAA;AAAA;AAAA,EAGrB,YAAYD,MAAQC,OAAE,CAAQ,EAAE,SAAA;AAAA;AAAA,EAEhC,iBAAiBH,aACD,WAAW,WAAW,QAAQ,OAAO,EAClD,SAAA;AACL,CAAC,EACA,IAAI,CAAA,QAAO;AACV,QAAM,EAAC,QAAQ,GAAG,KAAA,IAAQ;AAC1B,MAAI,QAAQ;AACV,WAAO,EAAC,GAAG,MAAM,OAAA;AAAA,EACnB;AACA,SAAO;AAAA,IACL,GAAG;AAAA,IACH,QAAQ;AAAA,MACN,SAAS,KAAK,IAAI,UAAU;AAAA,MAC5B,MAAM,IAAI;AAAA,IAAA;AAAA,EACZ;AAEJ,CAAC;AAGI,MAAM,oBAAoBF,OAAS;AAAA,EACxC,QAAQK,OAAE;AAAA,EACV,MAAMA,OAAE;AAAA,EAER,QAAQ;AACV,CAAC;AAUM,MAAM,sBAAsBL,OACzB,EAAC,QAAQM,OAAS,eAAe,EAAA,CAAE,EAC1C,KAAK,eAAe;AAIhB,MAAM,YAAYA,OAAS,eAAe;AAE1C,MAAM,eAAeN,OAAS;AAAA,EACnC,KAAKC,QAAU,QAAQ;AAAA,EACvB,UAAU;AAAA,EACV,KAAK;AACP,CAAC;AAEM,MAAM,eAAeD,OAAS;AAAA,EACnC,KAAKC,QAAU,QAAQ;AAAA,EACvB,UAAU;AAAA;AAAA;AAAA,EAGV,KAAK,UAAU,SAAA;AAAA;AAAA;AAAA;AAAA,EAIf,KAAK;AACP,CAAC;AAEM,MAAM,eAAeD,OAAS;AAAA,EACnC,KAAKC,QAAU,QAAQ;AAAA,EACvB,UAAU;AAAA;AAAA,EAEV,KAAK;AACP,CAAC;AAEM,MAAM,iBAAiBD,OAAS;AAAA,EACrC,KAAKC,QAAU,UAAU;AAAA,EACzB,WAAWG,MAAQ,cAAc;AACnC,CAAC;AAEM,MAAM,mBAAmBJ,OAAS;AAAA,EACvC,QAAQK,OAAE;AAAA,EACV,MAAMA,OAAE;AACV,CAAC;AAYM,MAAM,mBAAmB;AAIzB,MAAM,oBAAoBL,OAAS;AAAA,EACxC,KAAKC,QAAU,cAAc;AAAA,EAC7B,MAAM;AAAA;AAAA;AAAA;AAAA;AAAA,EAMN,UAAU,oBAAoB,SAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAmB9B,UAAUK,OAAS,gBAAgB,EAAE,SAAA;AACvC,CAAC;AAEM,MAAM,oBAAoBN,OAAS;AAAA,EACxC,KAAKC,QAAU,cAAc;AAAA,EAC7B,KAAK;AAAA,EACL,KAAK;AACP,CAAC;AAEM,MAAM,4BAA4BD,OAAS;AAAA,EAChD,KAAKC,QAAU,uBAAuB;AAAA,EACtC,OAAO;AAAA,EACP,KAAK;AAAA,EACL,KAAK;AACP,CAAC;AAED,MAAM,eAAeD,OAAS;AAAA,EAC5B,MAAMK,OAAE;AAAA,EACR,MAAM;AACR,CAAC;AAEM,MAAM,kBAAkBL,OAAS;AAAA,EACtC,KAAKC,QAAU,YAAY;AAAA,EAC3B,OAAO;AAAA,EACP,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA,EAMR,eAAe,oBAAoB,SAAA;AAAA;AAAA,EAGnC,UAAU,iBAAiB,SAAA;AAC7B,CAAC;AAEM,MAAM,qBAAqBD,OAAS;AAAA,EACzC,KAAKC,QAAU,eAAe;AAAA,EAC9B,OAAO;AAAA,EACP,KAAK;AAAA,EACL,KAAK;AACP,CAAC;AAEM,MAAM,mBAAmBD,OAAS;AAAA,EACvC,KAAKC,QAAU,aAAa;AAAA,EAC5B,OAAO;AAAA,EACP,QAAQI,OAAE;AACZ,CAAC;AAEM,MAAM,kBAAkBL,OAAS;AAAA,EACtC,KAAKC,QAAU,YAAY;AAAA,EAC3B,IAAI;AACN,CAAC;AAEM,MAAM,oBAAoBD,OAAS;AAAA,EACxC,KAAKC,QAAU,cAAc;AAAA,EAC7B,MAAM;AACR,CAAC;AAEM,MAAM,kBAAkBD,OAAS;AAAA,EACtC,KAAKC,QAAU,YAAY;AAAA,EAC3B,IAAI;AACN,CAAC;AAIM,MAAM,iBAAiBD,OAAS;AAAA,EACrC,KAAKC,QAAU,UAAU;AAAA,EAEzB,UAAU;AAAA;AAAA;AAAA,EAIV,SAASG,MAAQC,QAAU;AAAA;AAAA;AAAA;AAAA;AAAA,EAM3B,WAAWA,OAAE;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAYb,WAAWD,MAAQA,MAAQ,eAAe,CAAC;AAC7C,CAAC;AAIM,MAAM,0BAA0BJ,OAAS;AAAA,EAC9C,KAAKC,QAAU,oBAAoB;AAAA,EAEnC,UAAU;AAAA;AAAA;AAAA,EAIV,SAASG,MAAQC,QAAU;AAAA;AAAA;AAAA;AAAA;AAAA,EAM3B,WAAWA,OAAE;AACf,CAAC;AAyBM,MAAM,mBAAmBE;AAAAA,EAC9B;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAGA,MAAM,iBAAiB;AAAA,EACrB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAE6BL,aAAe,GAAG,cAAc;AAS7D,MAAM,gBAAgB;AAAA,EACpB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAGA,MAAM,mBAAmB;AAAA,EACvB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAEO,MAAM,qBAAqBK,MAAQ,GAAG,aAAa;AAE3BL,aAAe,GAAG,gBAAgB;AAmBjE,MAAM,qBAAqB,IAAI,IAAY,gBAAgB;AAEpD,SAAS,eAAe,QAAwC;AACrE,SAAO,mBAAmB,IAAI,OAAO,GAAG;AAC1C;AAEA,MAAM,mBAAmB,IAAI,IAAY,cAAc;AAEhD,SAAS,aAAa,QAAsC;AACjE,SAAO,iBAAiB,IAAI,OAAO,GAAG;AACxC;"}

package/out/zero-cache/src/services/change-source/protocol/current/downstream.d.ts

@@ -2,6 +2,7 @@ import * as v from '../../../../../../shared/src/valita.ts';
 declare const begin: v.TupleType<[v.Type<"begin">, v.ObjectType<{
     tag: v.Type<"begin">;
     json: v.Optional<"p" | "s">;
+    skipAck: v.Optional<boolean>;
 }, undefined>, v.ObjectType<{
     commitWatermark: v.Type<string>;
 }, undefined>]>;
@@ -224,6 +225,7 @@ export type Rollback = v.Infer<typeof rollback>;
 export declare const changeStreamDataSchema: v.UnionType<[v.TupleType<[v.Type<"begin">, v.ObjectType<{
     tag: v.Type<"begin">;
     json: v.Optional<"p" | "s">;
+    skipAck: v.Optional<boolean>;
 }, undefined>, v.ObjectType<{
     commitWatermark: v.Type<string>;
 }, undefined>]>, v.TupleType<[v.Type<"data">, v.UnionType<[v.UnionType<[v.ObjectType<{
@@ -447,6 +449,7 @@ export type ChangeStreamControl = v.Infer<typeof changeStreamControlSchema>;
 export declare const changeStreamMessageSchema: v.UnionType<[v.UnionType<[v.TupleType<[v.Type<"begin">, v.ObjectType<{
     tag: v.Type<"begin">;
     json: v.Optional<"p" | "s">;
+    skipAck: v.Optional<boolean>;
 }, undefined>, v.ObjectType<{
     commitWatermark: v.Type<string>;
 }, undefined>]>, v.TupleType<[v.Type<"data">, v.UnionType<[v.UnionType<[v.ObjectType<{

package/out/zero-cache/src/services/change-source/protocol/current/downstream.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"downstream.d.ts","sourceRoot":"","sources":["../../../../../../../../zero-cache/src/services/change-source/protocol/current/downstream.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,CAAC,MAAM,wCAAwC,CAAC;AAW5D,QAAA,MAAM,KAAK;;;;;eAIT,CAAC;AACH,QAAA,MAAM,IAAI;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;mBAGR,CAAC;AACH,QAAA,MAAM,MAAM;;;;eAIV,CAAC;AACH,QAAA,MAAM,QAAQ;;eAAmD,CAAC;AAElE,MAAM,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,KAAK,CAAC,CAAC;AAC1C,MAAM,MAAM,IAAI,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,IAAI,CAAC,CAAC;AACxC,MAAM,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,MAAM,CAAC,CAAC;AAC5C,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,QAAQ,CAAC,CAAC;AAEhD,eAAO,MAAM,sBAAsB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAAyC,CAAC;AAC7E,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,sBAAsB,CAAC,CAAC;AAEtE,eAAO,MAAM,yBAAyB;;;;eAGpC,CAAC;AACH,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC;AAE5E,4EAA4E;AAC5E,eAAO,MAAM,yBAAyB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAIrC,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC"}
+{"version":3,"file":"downstream.d.ts","sourceRoot":"","sources":["../../../../../../../../zero-cache/src/services/change-source/protocol/current/downstream.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,CAAC,MAAM,wCAAwC,CAAC;AAW5D,QAAA,MAAM,KAAK;;;;;;eAIT,CAAC;AACH,QAAA,MAAM,IAAI;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;mBAGR,CAAC;AACH,QAAA,MAAM,MAAM;;;;eAIV,CAAC;AACH,QAAA,MAAM,QAAQ;;eAAmD,CAAC;AAElE,MAAM,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,KAAK,CAAC,CAAC;AAC1C,MAAM,MAAM,IAAI,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,IAAI,CAAC,CAAC;AACxC,MAAM,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,MAAM,CAAC,CAAC;AAC5C,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,QAAQ,CAAC,CAAC;AAEhD,eAAO,MAAM,sBAAsB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAAyC,CAAC;AAC7E,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,sBAAsB,CAAC,CAAC;AAEtE,eAAO,MAAM,yBAAyB;;;;eAGpC,CAAC;AACH,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC;AAE5E,4EAA4E;AAC5E,eAAO,MAAM,yBAAyB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAIrC,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC"}

package/out/zero-cache/src/services/change-source/protocol/current/status.d.ts

@@ -13,9 +13,10 @@ export declare const downstreamStatusMessageSchema: v.TupleType<[v.Type<"status"
     watermark: v.Type<string>;
 }, undefined>]>;
 /**
- * The `zero-cache` will send the Commit payload when acknowledging a
- * completed transaction, and will echo back the downstreamStatus message
- * if `ack` is true.
+ * The `zero-cache` will send the Commit payload to acknowledge a completed
+ * transaction (unless the `skipAck` field was specified in the Begin message
+ * of the transaction), and will echo back the downstream `status` message if
+ * `ack` is true.
  */
 export declare const upstreamStatusMessageSchema: v.TupleType<[v.Type<"status">, v.UnionType<[v.ObjectType<{
     ack: v.Type<boolean>;
@@ -37,7 +38,8 @@ export declare const upstreamStatusMessageSchema: v.TupleType<[v.Type<"status">,
  *
  * The `zero-cache` sends StatusMessages to the ChangeSource:
  *
- * * when it has processed a `Commit` received from the ChangeSource
+ * * when it has processed a `Commit` received from the ChangeSource,
+ *   unless the `Begin` message specified `skipAck`.
  *
  * * when it receives a `StatusMessage` and all preceding `Commit` messages
  *   have been processed

package/out/zero-cache/src/services/change-source/protocol/current/status.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"status.d.ts","sourceRoot":"","sources":["../../../../../../../../zero-cache/src/services/change-source/protocol/current/status.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,CAAC,MAAM,wCAAwC,CAAC;AAG5D;;;GAGG;AACH,eAAO,MAAM,sBAAsB;;aAEjC,CAAC;AAEH,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,sBAAsB,CAAC,CAAC;AAEtE,eAAO,MAAM,6BAA6B;;;;eAIxC,CAAC;AAEH;;;;GAIG;AACH,eAAO,MAAM,2BAA2B;;;;;;eAItC,CAAC;AAEH;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,MAAM,MAAM,uBAAuB,GAAG,CAAC,CAAC,KAAK,CAC3C,OAAO,6BAA6B,CACrC,CAAC;AACF,MAAM,MAAM,qBAAqB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,2BAA2B,CAAC,CAAC"}
+{"version":3,"file":"status.d.ts","sourceRoot":"","sources":["../../../../../../../../zero-cache/src/services/change-source/protocol/current/status.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,CAAC,MAAM,wCAAwC,CAAC;AAG5D;;;GAGG;AACH,eAAO,MAAM,sBAAsB;;aAEjC,CAAC;AAEH,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,sBAAsB,CAAC,CAAC;AAEtE,eAAO,MAAM,6BAA6B;;;;eAIxC,CAAC;AAEH;;;;;GAKG;AACH,eAAO,MAAM,2BAA2B;;;;;;eAItC,CAAC;AAEH;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,MAAM,MAAM,uBAAuB,GAAG,CAAC,CAAC,KAAK,CAC3C,OAAO,6BAA6B,CACrC,CAAC;AACF,MAAM,MAAM,qBAAqB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,2BAA2B,CAAC,CAAC"}

package/out/zero-cache/src/services/change-source/protocol/current/status.js.map

@@ -1 +1 @@
-{"version":3,"file":"status.js","sources":["../../../../../../../../zero-cache/src/services/change-source/protocol/current/status.ts"],"sourcesContent":["import * as v from '../../../../../../shared/src/valita.ts';\nimport {commitSchema} from './data.ts';\n\n/**\n * The downstream status message indicates whether it should be echoed\n * back in an upstream status message.\n */\nexport const downstreamStatusSchema = v.object({\n  ack: v.boolean().optional(() => true),\n});\n\nexport type DownstreamStatus = v.Infer<typeof downstreamStatusSchema>;\n\nexport const downstreamStatusMessageSchema = v.tuple([\n  v.literal('status'),\n  downstreamStatusSchema,\n  v.object({watermark: v.string()}),\n]);\n\n/**\n * The `zero-cache` will send the Commit payload when acknowledging a\n * completed transaction, and will echo back the downstreamStatus message\n * if `ack` is true.\n */\nexport const upstreamStatusMessageSchema = v.tuple([\n  v.literal('status'),\n  v.union(downstreamStatusSchema, commitSchema),\n  v.object({watermark: v.string()}),\n]);\n\n/**\n * Status messages convey positional information from both the ChangeSource\n * and the `zero-cache`.\n *\n * A StatusMessage from the ChangeSource indicates a position in its change\n * log. Generally, the watermarks sent in `Commit` messages already convey\n * this information, but a StatusMessage may also be sent to indicate that the\n * log has progressed without any corresponding changes relevant to the\n * subscriber. The watermarks of commit messages and status messages must be\n * monotonic in the stream of messages from the ChangeSource.\n *\n * The `zero-cache` sends StatusMessages to the ChangeSource:\n *\n * * when it has processed a `Commit` received from the ChangeSource\n *\n * * when it receives a `StatusMessage` and all preceding `Commit` messages\n *   have been processed\n *\n * This allows the ChangeSource to clean up change log entries appropriately.\n *\n * Note that StatusMessages from the ChangeSource are optional. If a\n * ChangeSource implementation can track subscriber progress and clean up\n * its change log purely from Commit-driven StatusMessages there is no need\n * for the ChangeSource to send StatusMessages.\n */\nexport type DownstreamStatusMessage = v.Infer<\n  typeof downstreamStatusMessageSchema\n>;\nexport type UpstreamStatusMessage = v.Infer<typeof upstreamStatusMessageSchema>;\n"],"names":["v.object","v.boolean","v.tuple","v.literal","v.string","v.union"],"mappings":";;;AAOO,MAAM,yBAAyBA,OAAS;AAAA,EAC7C,KAAKC,QAAE,EAAU,SAAS,MAAM,IAAI;AACtC,CAAC;AAIM,MAAM,gCAAgCC,MAAQ;AAAA,EACnDC,QAAU,QAAQ;AAAA,EAClB;AAAA,EACAH,OAAS,EAAC,WAAWI,UAAW;AAClC,CAAC;AAOM,MAAM,8BAA8BF,MAAQ;AAAA,EACjDC,QAAU,QAAQ;AAAA,EAClBE,MAAQ,wBAAwB,YAAY;AAAA,EAC5CL,OAAS,EAAC,WAAWI,UAAW;AAClC,CAAC;"}
+{"version":3,"file":"status.js","sources":["../../../../../../../../zero-cache/src/services/change-source/protocol/current/status.ts"],"sourcesContent":["import * as v from '../../../../../../shared/src/valita.ts';\nimport {commitSchema} from './data.ts';\n\n/**\n * The downstream status message indicates whether it should be echoed\n * back in an upstream status message.\n */\nexport const downstreamStatusSchema = v.object({\n  ack: v.boolean().optional(() => true),\n});\n\nexport type DownstreamStatus = v.Infer<typeof downstreamStatusSchema>;\n\nexport const downstreamStatusMessageSchema = v.tuple([\n  v.literal('status'),\n  downstreamStatusSchema,\n  v.object({watermark: v.string()}),\n]);\n\n/**\n * The `zero-cache` will send the Commit payload to acknowledge a completed\n * transaction (unless the `skipAck` field was specified in the Begin message\n * of the transaction), and will echo back the downstream `status` message if\n * `ack` is true.\n */\nexport const upstreamStatusMessageSchema = v.tuple([\n  v.literal('status'),\n  v.union(downstreamStatusSchema, commitSchema),\n  v.object({watermark: v.string()}),\n]);\n\n/**\n * Status messages convey positional information from both the ChangeSource\n * and the `zero-cache`.\n *\n * A StatusMessage from the ChangeSource indicates a position in its change\n * log. Generally, the watermarks sent in `Commit` messages already convey\n * this information, but a StatusMessage may also be sent to indicate that the\n * log has progressed without any corresponding changes relevant to the\n * subscriber. The watermarks of commit messages and status messages must be\n * monotonic in the stream of messages from the ChangeSource.\n *\n * The `zero-cache` sends StatusMessages to the ChangeSource:\n *\n * * when it has processed a `Commit` received from the ChangeSource,\n *   unless the `Begin` message specified `skipAck`.\n *\n * * when it receives a `StatusMessage` and all preceding `Commit` messages\n *   have been processed\n *\n * This allows the ChangeSource to clean up change log entries appropriately.\n *\n * Note that StatusMessages from the ChangeSource are optional. If a\n * ChangeSource implementation can track subscriber progress and clean up\n * its change log purely from Commit-driven StatusMessages there is no need\n * for the ChangeSource to send StatusMessages.\n */\nexport type DownstreamStatusMessage = v.Infer<\n  typeof downstreamStatusMessageSchema\n>;\nexport type UpstreamStatusMessage = v.Infer<typeof upstreamStatusMessageSchema>;\n"],"names":["v.object","v.boolean","v.tuple","v.literal","v.string","v.union"],"mappings":";;;AAOO,MAAM,yBAAyBA,OAAS;AAAA,EAC7C,KAAKC,QAAE,EAAU,SAAS,MAAM,IAAI;AACtC,CAAC;AAIM,MAAM,gCAAgCC,MAAQ;AAAA,EACnDC,QAAU,QAAQ;AAAA,EAClB;AAAA,EACAH,OAAS,EAAC,WAAWI,UAAW;AAClC,CAAC;AAQM,MAAM,8BAA8BF,MAAQ;AAAA,EACjDC,QAAU,QAAQ;AAAA,EAClBE,MAAQ,wBAAwB,YAAY;AAAA,EAC5CL,OAAS,EAAC,WAAWI,UAAW;AAClC,CAAC;"}

package/out/zero-cache/src/services/change-streamer/backup-monitor.d.ts

@@ -3,13 +3,13 @@ import { Subscription } from '../../types/subscription.ts';
 import type { Service } from '../service.ts';
 import type { ChangeStreamerService } from './change-streamer.ts';
 import type { SnapshotMessage } from './snapshot.ts';
-export declare const CHECK_INTERVAL_MS: number;
+export declare const CHECK_INTERVAL_MS = 60000;
 /**
  * The BackupMonitor polls the litestream "/metrics" endpoint to track the
  * watermark (label) value of the `litestream_replica_progress` gauge and
  * schedules cleanup of change log entries that can be purged as a result.
  *
- * See: https: *github.com/rocicorp/litestream/pull/3
+ * See: https://github.com/rocicorp/litestream/pull/3
  *
  * Note that change log entries cannot simply be purged as soon as they
  * have been applied and backed up by litestream. Consider the case in which

package/out/zero-cache/src/services/change-streamer/backup-monitor.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"backup-monitor.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/backup-monitor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAGjD,OAAO,EAAC,YAAY,EAAC,MAAM,6BAA6B,CAAC;AAEzD,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,eAAe,CAAC;AAC3C,OAAO,KAAK,EAAC,qBAAqB,EAAC,MAAM,sBAAsB,CAAC;AAChE,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,eAAe,CAAC;AAEnD,eAAO,MAAM,iBAAiB,QAAY,CAAC;AAQ3C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AACH,qBAAa,aAAc,YAAW,OAAO;;IAC3C,QAAQ,CAAC,EAAE,oBAAoB;gBAe7B,EAAE,EAAE,UAAU,EACd,SAAS,EAAE,MAAM,EACjB,eAAe,EAAE,MAAM,EACvB,cAAc,EAAE,qBAAqB,EACrC,qBAAqB,EAAE,MAAM;IAgB/B,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;IAYpB,wBAAwB,CAAC,MAAM,EAAE,MAAM,GAAG,YAAY,CAAC,eAAe,CAAC;IA6BvE,cAAc,CAAC,MAAM,EAAE,MAAM,EAAE,kBAAkB,UAAO;IAoBxD,QAAQ,CAAC,iCAAiC,sBAWxC;IAgEF,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;CAYtB"}
+{"version":3,"file":"backup-monitor.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/backup-monitor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAGjD,OAAO,EAAC,YAAY,EAAC,MAAM,6BAA6B,CAAC;AAEzD,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,eAAe,CAAC;AAC3C,OAAO,KAAK,EAAC,qBAAqB,EAAC,MAAM,sBAAsB,CAAC;AAChE,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,eAAe,CAAC;AAEnD,eAAO,MAAM,iBAAiB,QAAS,CAAC;AAQxC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AACH,qBAAa,aAAc,YAAW,OAAO;;IAC3C,QAAQ,CAAC,EAAE,oBAAoB;gBAe7B,EAAE,EAAE,UAAU,EACd,SAAS,EAAE,MAAM,EACjB,eAAe,EAAE,MAAM,EACvB,cAAc,EAAE,qBAAqB,EACrC,qBAAqB,EAAE,MAAM;IAgB/B,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;IAYpB,wBAAwB,CAAC,MAAM,EAAE,MAAM,GAAG,YAAY,CAAC,eAAe,CAAC;IA6BvE,cAAc,CAAC,MAAM,EAAE,MAAM,EAAE,kBAAkB,UAAO;IAoBxD,QAAQ,CAAC,iCAAiC,sBAWxC;IAwFF,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;CAYtB"}

package/out/zero-cache/src/services/change-streamer/backup-monitor.js

@@ -2,8 +2,8 @@ import parsePrometheusTextFormat from "parse-prometheus-text-format";
 import { promiseVoid } from "../../../../shared/src/resolved-promises.js";
 import { Subscription } from "../../types/subscription.js";
 import { RunningState } from "../running-state.js";
-const CHECK_INTERVAL_MS = 60 * 1e3;
-const MIN_CLEANUP_DELAY_MS = 30 * 1e3;
+const CHECK_INTERVAL_MS = 6e4;
+const MIN_CLEANUP_DELAY_MS = 3e4;
 class BackupMonitor {
   id = "backup-monitor";
   #lc;
@@ -91,12 +91,22 @@ class BackupMonitor {
       this.#lc.warn?.(`error scheduling cleanup`, e);
     }
   };
-  async #checkWatermarks() {
-    const resp = await fetch(this.#metricsEndpoint);
+  async *#fetchWatermarks() {
+    const metricsEndpoint = this.#metricsEndpoint;
+    const signal = this.#state.signal;
+    let resp;
+    try {
+      resp = await fetch(metricsEndpoint, { signal });
+    } catch (e) {
+      if (signal.aborted) {
+        return;
+      }
+      this.#lc.warn?.(`unable to fetch metrics at ${this.#metricsEndpoint}`, e);
+      return;
+    }
     if (!resp.ok) {
       this.#lc.warn?.(
-        `unable to fetch metrics at ${this.#metricsEndpoint}`,
-        await resp.text()
+        `unable to fetch metrics at ${this.#metricsEndpoint}: ${await resp.text()}`
       );
       return;
     }
@@ -105,17 +115,25 @@ class BackupMonitor {
       if (family.type === "GAUGE" && family.name === "litestream_replica_progress") {
         for (const metric of family.metrics) {
           const watermark = metric.labels?.watermark;
-          if (watermark && watermark > this.#lastWatermark && !this.#watermarks.has(watermark)) {
-            const time = new Date(parseFloat(metric.value) * 1e3);
-            this.#lc.info?.(
-              `replicated watermark=${watermark} to ${metric.labels?.name} at ${time.toISOString()}.`
-            );
-            this.#watermarks.set(watermark, time);
+          const name = metric.labels?.name;
+          const time = new Date(parseFloat(metric.value) * 1e3);
+          if (watermark) {
+            yield { watermark, time, name };
           }
         }
       }
     }
   }
+  async #checkWatermarks() {
+    for await (const { watermark, name, time } of this.#fetchWatermarks()) {
+      if (watermark > this.#lastWatermark && !this.#watermarks.has(watermark)) {
+        this.#lc.info?.(
+          `replicated watermark=${watermark} to ${name} at ${time.toISOString()}.`
+        );
+        this.#watermarks.set(watermark, time);
+      }
+    }
+  }
   #scheduleCleanup() {
     if (this.#reservations.size > 0) {
       this.#lc.info?.(

package/out/zero-cache/src/services/change-streamer/backup-monitor.js.map

@@ -1 +1 @@
-{"version":3,"file":"backup-monitor.js","sources":["../../../../../../zero-cache/src/services/change-streamer/backup-monitor.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport parsePrometheusTextFormat from 'parse-prometheus-text-format';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport {RunningState} from '../running-state.ts';\nimport type {Service} from '../service.ts';\nimport type {ChangeStreamerService} from './change-streamer.ts';\nimport type {SnapshotMessage} from './snapshot.ts';\n\nexport const CHECK_INTERVAL_MS = 60 * 1000;\nconst MIN_CLEANUP_DELAY_MS = 30 * 1000;\n\ntype Reservation = {\n  start: Date;\n  sub: Subscription<SnapshotMessage>;\n};\n\n/**\n * The BackupMonitor polls the litestream \"/metrics\" endpoint to track the\n * watermark (label) value of the `litestream_replica_progress` gauge and\n * schedules cleanup of change log entries that can be purged as a result.\n *\n * See: https: *github.com/rocicorp/litestream/pull/3\n *\n * Note that change log entries cannot simply be purged as soon as they\n * have been applied and backed up by litestream. Consider the case in which\n * litestream backs up new wal segments every minute, but it takes 5 minutes\n * to restore a replica: if a zero-cache starts restoring a replica at\n * minute 0, and new watermarks are replicated at minutes 1, 2, 3, 4, and 5,\n * purging changelog records as soon as those watermarks are replicated would\n * result in the zero-cache not being able to catch up from minute 0 once it\n * has finished restoring the replica.\n *\n * The `/snapshot` reservation protocol is used to prevent premature change\n * log cleanup:\n * - Clients restoring a snapshot initiate a `/snapshot` request and hold that\n *   request open while it restores its snapshot, prepares it, and\n *   starts its subscription to the change stream. During this time, no\n *   cleanups are scheduled.\n * - When the subscription is started, the interval since the beginning of\n *   of the reservation is tracked to increase the background cleanup delay\n *   interval if needed. The reservation is ended (and request closed), and\n *   cleanup scheduling is resumed with the current delay interval.\n *\n * Note that the reservation request is the primary mechanism by which\n * premature change log cleanup is prevented. The cleanup delay interval is\n * a secondary safeguard.\n */\nexport class BackupMonitor implements Service {\n  readonly id = 'backup-monitor';\n  readonly #lc: LogContext;\n  readonly #backupURL: string;\n  readonly #metricsEndpoint: string;\n  readonly #changeStreamer: ChangeStreamerService;\n  readonly #state = new RunningState(this.id);\n\n  readonly #reservations = new Map<string, Reservation>();\n  readonly #watermarks = new Map<string, Date>();\n\n  #lastWatermark: string = '';\n  #cleanupDelayMs: number;\n  #checkMetricsTimer: NodeJS.Timeout | undefined;\n\n  constructor(\n    lc: LogContext,\n    backupURL: string,\n    metricsEndpoint: string,\n    changeStreamer: ChangeStreamerService,\n    initialCleanupDelayMs: number,\n  ) {\n    this.#lc = lc.withContext('component', this.id);\n    this.#backupURL = backupURL;\n    this.#metricsEndpoint = metricsEndpoint;\n    this.#changeStreamer = changeStreamer;\n    this.#cleanupDelayMs = Math.max(\n      initialCleanupDelayMs,\n      MIN_CLEANUP_DELAY_MS, // purely for peace of mind\n    );\n\n    this.#lc.info?.(\n      `backup monitor started ${initialCleanupDelayMs} ms after snapshot restore`,\n    );\n  }\n\n  run(): Promise<void> {\n    this.#lc.info?.(\n      `monitoring backups at ${this.#metricsEndpoint} with ` +\n        `${this.#cleanupDelayMs} ms cleanup delay`,\n    );\n    this.#checkMetricsTimer = setInterval(\n      this.checkWatermarksAndScheduleCleanup,\n      CHECK_INTERVAL_MS,\n    );\n    return this.#state.stopped();\n  }\n\n  startSnapshotReservation(taskID: string): Subscription<SnapshotMessage> {\n    this.#lc.info?.(`pausing change-log cleanup while ${taskID} snapshots`);\n    // In the case of retries, only track the last reservation.\n    this.#reservations.get(taskID)?.sub.cancel();\n\n    const sub = Subscription.create<SnapshotMessage>({\n      // If the reservation still exists when the connection closes\n      // (e.g. subscriber crashed), clean it up without updating the\n      // cleanup delay.\n      cleanup: () => this.endReservation(taskID, false),\n    });\n    this.#reservations.set(taskID, {start: new Date(), sub});\n    // Note: the Subscription must be returned immediately so that the\n    //       websocket can begin sending liveness pings.\n    void this.#changeStreamer\n      .getChangeLogState()\n      .then(changeLogState => {\n        sub.push([\n          'status',\n          {tag: 'status', backupURL: this.#backupURL, ...changeLogState},\n        ]);\n      })\n      .catch(e => {\n        this.#lc.warn?.(`failing snapshot reservation`, e);\n        sub.fail(e);\n      });\n    return sub;\n  }\n\n  endReservation(taskID: string, updateCleanupDelay = true) {\n    const res = this.#reservations.get(taskID);\n    if (res === undefined) {\n      return;\n    }\n    this.#reservations.delete(taskID);\n    const {start, sub} = res;\n    sub.cancel(); // closes the connection if still open\n\n    if (updateCleanupDelay) {\n      const duration = Date.now() - start.getTime();\n      this.#lc.info?.(`snapshot initialized by ${taskID} in ${duration} ms`);\n      if (duration > this.#cleanupDelayMs) {\n        this.#cleanupDelayMs = duration;\n        this.#lc.info?.(`increased cleanup delay to ${duration} ms`);\n      }\n    }\n  }\n\n  // Exported for testing\n  readonly checkWatermarksAndScheduleCleanup = async () => {\n    try {\n      await this.#checkWatermarks();\n    } catch (e) {\n      this.#lc.warn?.(`unable to fetch metrics at ${this.#metricsEndpoint}`, e);\n    }\n    try {\n      this.#scheduleCleanup();\n    } catch (e) {\n      this.#lc.warn?.(`error scheduling cleanup`, e);\n    }\n  };\n\n  async #checkWatermarks() {\n    const resp = await fetch(this.#metricsEndpoint);\n    if (!resp.ok) {\n      this.#lc.warn?.(\n        `unable to fetch metrics at ${this.#metricsEndpoint}`,\n        await resp.text(),\n      );\n      return;\n    }\n    const families = parsePrometheusTextFormat(await resp.text());\n    for (const family of families) {\n      if (\n        family.type === 'GAUGE' &&\n        family.name === 'litestream_replica_progress'\n      ) {\n        for (const metric of family.metrics) {\n          const watermark = metric.labels?.watermark;\n          if (\n            watermark &&\n            watermark > this.#lastWatermark &&\n            !this.#watermarks.has(watermark)\n          ) {\n            const time = new Date(parseFloat(metric.value) * 1000);\n            this.#lc.info?.(\n              `replicated watermark=${watermark} to ${metric.labels?.name}` +\n                ` at ${time.toISOString()}.`,\n            );\n            this.#watermarks.set(watermark, time);\n          }\n        }\n      }\n    }\n  }\n\n  #scheduleCleanup() {\n    if (this.#reservations.size > 0) {\n      this.#lc.info?.(\n        `watermark cleanup paused for snapshot(s): ${[...this.#reservations.keys()]}`,\n      );\n      return;\n    }\n    const latestCleanupTime = Date.now() - this.#cleanupDelayMs;\n    let maxWatermark = '';\n    for (const [watermark, backupTime] of this.#watermarks.entries()) {\n      if (\n        backupTime.getTime() <= latestCleanupTime &&\n        watermark > maxWatermark\n      ) {\n        maxWatermark = watermark;\n      }\n    }\n    if (maxWatermark.length) {\n      this.#changeStreamer.scheduleCleanup(maxWatermark);\n      for (const watermark of this.#watermarks.keys()) {\n        if (watermark <= maxWatermark) {\n          this.#watermarks.delete(watermark);\n        }\n      }\n      this.#lastWatermark = maxWatermark;\n    }\n  }\n\n  stop(): Promise<void> {\n    clearInterval(this.#checkMetricsTimer);\n    for (const {sub} of this.#reservations.values()) {\n      // Close any pending reservations. This commonly happens when a new\n      // replication-manager makes a `/snapshot` reservation on the existing\n      // replication-manager, and then shuts it down when it takes over the\n      // replication slot.\n      sub.cancel();\n    }\n    this.#state.stop(this.#lc);\n    return promiseVoid;\n  }\n}\n"],"names":[],"mappings":";;;;AASO,MAAM,oBAAoB,KAAK;AACtC,MAAM,uBAAuB,KAAK;AAsC3B,MAAM,cAAiC;AAAA,EACnC,KAAK;AAAA,EACL;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA,SAAS,IAAI,aAAa,KAAK,EAAE;AAAA,EAEjC,oCAAoB,IAAA;AAAA,EACpB,kCAAkB,IAAA;AAAA,EAE3B,iBAAyB;AAAA,EACzB;AAAA,EACA;AAAA,EAEA,YACE,IACA,WACA,iBACA,gBACA,uBACA;AACA,SAAK,MAAM,GAAG,YAAY,aAAa,KAAK,EAAE;AAC9C,SAAK,aAAa;AAClB,SAAK,mBAAmB;AACxB,SAAK,kBAAkB;AACvB,SAAK,kBAAkB,KAAK;AAAA,MAC1B;AAAA,MACA;AAAA;AAAA,IAAA;AAGF,SAAK,IAAI;AAAA,MACP,0BAA0B,qBAAqB;AAAA,IAAA;AAAA,EAEnD;AAAA,EAEA,MAAqB;AACnB,SAAK,IAAI;AAAA,MACP,yBAAyB,KAAK,gBAAgB,SACzC,KAAK,eAAe;AAAA,IAAA;AAE3B,SAAK,qBAAqB;AAAA,MACxB,KAAK;AAAA,MACL;AAAA,IAAA;AAEF,WAAO,KAAK,OAAO,QAAA;AAAA,EACrB;AAAA,EAEA,yBAAyB,QAA+C;AACtE,SAAK,IAAI,OAAO,oCAAoC,MAAM,YAAY;AAEtE,SAAK,cAAc,IAAI,MAAM,GAAG,IAAI,OAAA;AAEpC,UAAM,MAAM,aAAa,OAAwB;AAAA;AAAA;AAAA;AAAA,MAI/C,SAAS,MAAM,KAAK,eAAe,QAAQ,KAAK;AAAA,IAAA,CACjD;AACD,SAAK,cAAc,IAAI,QAAQ,EAAC,OAAO,oBAAI,QAAQ,KAAI;AAGvD,SAAK,KAAK,gBACP,kBAAA,EACA,KAAK,CAAA,mBAAkB;AACtB,UAAI,KAAK;AAAA,QACP;AAAA,QACA,EAAC,KAAK,UAAU,WAAW,KAAK,YAAY,GAAG,eAAA;AAAA,MAAc,CAC9D;AAAA,IACH,CAAC,EACA,MAAM,CAAA,MAAK;AACV,WAAK,IAAI,OAAO,gCAAgC,CAAC;AACjD,UAAI,KAAK,CAAC;AAAA,IACZ,CAAC;AACH,WAAO;AAAA,EACT;AAAA,EAEA,eAAe,QAAgB,qBAAqB,MAAM;AACxD,UAAM,MAAM,KAAK,cAAc,IAAI,MAAM;AACzC,QAAI,QAAQ,QAAW;AACrB;AAAA,IACF;AACA,SAAK,cAAc,OAAO,MAAM;AAChC,UAAM,EAAC,OAAO,IAAA,IAAO;AACrB,QAAI,OAAA;AAEJ,QAAI,oBAAoB;AACtB,YAAM,WAAW,KAAK,IAAA,IAAQ,MAAM,QAAA;AACpC,WAAK,IAAI,OAAO,2BAA2B,MAAM,OAAO,QAAQ,KAAK;AACrE,UAAI,WAAW,KAAK,iBAAiB;AACnC,aAAK,kBAAkB;AACvB,aAAK,IAAI,OAAO,8BAA8B,QAAQ,KAAK;AAAA,MAC7D;AAAA,IACF;AAAA,EACF;AAAA;AAAA,EAGS,oCAAoC,YAAY;AACvD,QAAI;AACF,YAAM,KAAK,iBAAA;AAAA,IACb,SAAS,GAAG;AACV,WAAK,IAAI,OAAO,8BAA8B,KAAK,gBAAgB,IAAI,CAAC;AAAA,IAC1E;AACA,QAAI;AACF,WAAK,iBAAA;AAAA,IACP,SAAS,GAAG;AACV,WAAK,IAAI,OAAO,4BAA4B,CAAC;AAAA,IAC/C;AAAA,EACF;AAAA,EAEA,MAAM,mBAAmB;AACvB,UAAM,OAAO,MAAM,MAAM,KAAK,gBAAgB;AAC9C,QAAI,CAAC,KAAK,IAAI;AACZ,WAAK,IAAI;AAAA,QACP,8BAA8B,KAAK,gBAAgB;AAAA,QACnD,MAAM,KAAK,KAAA;AAAA,MAAK;AAElB;AAAA,IACF;AACA,UAAM,WAAW,0BAA0B,MAAM,KAAK,MAAM;AAC5D,eAAW,UAAU,UAAU;AAC7B,UACE,OAAO,SAAS,WAChB,OAAO,SAAS,+BAChB;AACA,mBAAW,UAAU,OAAO,SAAS;AACnC,gBAAM,YAAY,OAAO,QAAQ;AACjC,cACE,aACA,YAAY,KAAK,kBACjB,CAAC,KAAK,YAAY,IAAI,SAAS,GAC/B;AACA,kBAAM,OAAO,IAAI,KAAK,WAAW,OAAO,KAAK,IAAI,GAAI;AACrD,iBAAK,IAAI;AAAA,cACP,wBAAwB,SAAS,OAAO,OAAO,QAAQ,IAAI,OAClD,KAAK,YAAA,CAAa;AAAA,YAAA;AAE7B,iBAAK,YAAY,IAAI,WAAW,IAAI;AAAA,UACtC;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEA,mBAAmB;AACjB,QAAI,KAAK,cAAc,OAAO,GAAG;AAC/B,WAAK,IAAI;AAAA,QACP,6CAA6C,CAAC,GAAG,KAAK,cAAc,KAAA,CAAM,CAAC;AAAA,MAAA;AAE7E;AAAA,IACF;AACA,UAAM,oBAAoB,KAAK,IAAA,IAAQ,KAAK;AAC5C,QAAI,eAAe;AACnB,eAAW,CAAC,WAAW,UAAU,KAAK,KAAK,YAAY,WAAW;AAChE,UACE,WAAW,QAAA,KAAa,qBACxB,YAAY,cACZ;AACA,uBAAe;AAAA,MACjB;AAAA,IACF;AACA,QAAI,aAAa,QAAQ;AACvB,WAAK,gBAAgB,gBAAgB,YAAY;AACjD,iBAAW,aAAa,KAAK,YAAY,KAAA,GAAQ;AAC/C,YAAI,aAAa,cAAc;AAC7B,eAAK,YAAY,OAAO,SAAS;AAAA,QACnC;AAAA,MACF;AACA,WAAK,iBAAiB;AAAA,IACxB;AAAA,EACF;AAAA,EAEA,OAAsB;AACpB,kBAAc,KAAK,kBAAkB;AACrC,eAAW,EAAC,IAAA,KAAQ,KAAK,cAAc,UAAU;AAK/C,UAAI,OAAA;AAAA,IACN;AACA,SAAK,OAAO,KAAK,KAAK,GAAG;AACzB,WAAO;AAAA,EACT;AACF;"}
+{"version":3,"file":"backup-monitor.js","sources":["../../../../../../zero-cache/src/services/change-streamer/backup-monitor.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport parsePrometheusTextFormat from 'parse-prometheus-text-format';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport {RunningState} from '../running-state.ts';\nimport type {Service} from '../service.ts';\nimport type {ChangeStreamerService} from './change-streamer.ts';\nimport type {SnapshotMessage} from './snapshot.ts';\n\nexport const CHECK_INTERVAL_MS = 60_000;\nconst MIN_CLEANUP_DELAY_MS = 30_000;\n\ntype Reservation = {\n  start: Date;\n  sub: Subscription<SnapshotMessage>;\n};\n\n/**\n * The BackupMonitor polls the litestream \"/metrics\" endpoint to track the\n * watermark (label) value of the `litestream_replica_progress` gauge and\n * schedules cleanup of change log entries that can be purged as a result.\n *\n * See: https://github.com/rocicorp/litestream/pull/3\n *\n * Note that change log entries cannot simply be purged as soon as they\n * have been applied and backed up by litestream. Consider the case in which\n * litestream backs up new wal segments every minute, but it takes 5 minutes\n * to restore a replica: if a zero-cache starts restoring a replica at\n * minute 0, and new watermarks are replicated at minutes 1, 2, 3, 4, and 5,\n * purging changelog records as soon as those watermarks are replicated would\n * result in the zero-cache not being able to catch up from minute 0 once it\n * has finished restoring the replica.\n *\n * The `/snapshot` reservation protocol is used to prevent premature change\n * log cleanup:\n * - Clients restoring a snapshot initiate a `/snapshot` request and hold that\n *   request open while it restores its snapshot, prepares it, and\n *   starts its subscription to the change stream. During this time, no\n *   cleanups are scheduled.\n * - When the subscription is started, the interval since the beginning of\n *   of the reservation is tracked to increase the background cleanup delay\n *   interval if needed. The reservation is ended (and request closed), and\n *   cleanup scheduling is resumed with the current delay interval.\n *\n * Note that the reservation request is the primary mechanism by which\n * premature change log cleanup is prevented. The cleanup delay interval is\n * a secondary safeguard.\n */\nexport class BackupMonitor implements Service {\n  readonly id = 'backup-monitor';\n  readonly #lc: LogContext;\n  readonly #backupURL: string;\n  readonly #metricsEndpoint: string;\n  readonly #changeStreamer: ChangeStreamerService;\n  readonly #state = new RunningState(this.id);\n\n  readonly #reservations = new Map<string, Reservation>();\n  readonly #watermarks = new Map<string, Date>();\n\n  #lastWatermark: string = '';\n  #cleanupDelayMs: number;\n  #checkMetricsTimer: NodeJS.Timeout | undefined;\n\n  constructor(\n    lc: LogContext,\n    backupURL: string,\n    metricsEndpoint: string,\n    changeStreamer: ChangeStreamerService,\n    initialCleanupDelayMs: number,\n  ) {\n    this.#lc = lc.withContext('component', this.id);\n    this.#backupURL = backupURL;\n    this.#metricsEndpoint = metricsEndpoint;\n    this.#changeStreamer = changeStreamer;\n    this.#cleanupDelayMs = Math.max(\n      initialCleanupDelayMs,\n      MIN_CLEANUP_DELAY_MS, // purely for peace of mind\n    );\n\n    this.#lc.info?.(\n      `backup monitor started ${initialCleanupDelayMs} ms after snapshot restore`,\n    );\n  }\n\n  run(): Promise<void> {\n    this.#lc.info?.(\n      `monitoring backups at ${this.#metricsEndpoint} with ` +\n        `${this.#cleanupDelayMs} ms cleanup delay`,\n    );\n    this.#checkMetricsTimer = setInterval(\n      this.checkWatermarksAndScheduleCleanup,\n      CHECK_INTERVAL_MS,\n    );\n    return this.#state.stopped();\n  }\n\n  startSnapshotReservation(taskID: string): Subscription<SnapshotMessage> {\n    this.#lc.info?.(`pausing change-log cleanup while ${taskID} snapshots`);\n    // In the case of retries, only track the last reservation.\n    this.#reservations.get(taskID)?.sub.cancel();\n\n    const sub = Subscription.create<SnapshotMessage>({\n      // If the reservation still exists when the connection closes\n      // (e.g. subscriber crashed), clean it up without updating the\n      // cleanup delay.\n      cleanup: () => this.endReservation(taskID, false),\n    });\n    this.#reservations.set(taskID, {start: new Date(), sub});\n    // Note: the Subscription must be returned immediately so that the\n    //       websocket can begin sending liveness pings.\n    void this.#changeStreamer\n      .getChangeLogState()\n      .then(changeLogState => {\n        sub.push([\n          'status',\n          {tag: 'status', backupURL: this.#backupURL, ...changeLogState},\n        ]);\n      })\n      .catch(e => {\n        this.#lc.warn?.(`failing snapshot reservation`, e);\n        sub.fail(e);\n      });\n    return sub;\n  }\n\n  endReservation(taskID: string, updateCleanupDelay = true) {\n    const res = this.#reservations.get(taskID);\n    if (res === undefined) {\n      return;\n    }\n    this.#reservations.delete(taskID);\n    const {start, sub} = res;\n    sub.cancel(); // closes the connection if still open\n\n    if (updateCleanupDelay) {\n      const duration = Date.now() - start.getTime();\n      this.#lc.info?.(`snapshot initialized by ${taskID} in ${duration} ms`);\n      if (duration > this.#cleanupDelayMs) {\n        this.#cleanupDelayMs = duration;\n        this.#lc.info?.(`increased cleanup delay to ${duration} ms`);\n      }\n    }\n  }\n\n  // Exported for testing\n  readonly checkWatermarksAndScheduleCleanup = async () => {\n    try {\n      await this.#checkWatermarks();\n    } catch (e) {\n      this.#lc.warn?.(`unable to fetch metrics at ${this.#metricsEndpoint}`, e);\n    }\n    try {\n      this.#scheduleCleanup();\n    } catch (e) {\n      this.#lc.warn?.(`error scheduling cleanup`, e);\n    }\n  };\n\n  async *#fetchWatermarks(): AsyncGenerator<{\n    watermark: string;\n    time: Date;\n    name?: string | undefined;\n  }> {\n    const metricsEndpoint = this.#metricsEndpoint;\n    const signal = this.#state.signal;\n    let resp;\n    try {\n      resp = await fetch(metricsEndpoint, {signal});\n    } catch (e) {\n      if (signal.aborted) {\n        // not an error.\n        return;\n      }\n      // Treat exceptions from fetch (e.g. network errors) as non-fatal, and simply\n      // log them and skip the watermark check until the next interval.\n      this.#lc.warn?.(`unable to fetch metrics at ${this.#metricsEndpoint}`, e);\n      return;\n    }\n    if (!resp.ok) {\n      this.#lc.warn?.(\n        `unable to fetch metrics at ${this.#metricsEndpoint}: ${await resp.text()}`,\n      );\n      return;\n    }\n\n    const families = parsePrometheusTextFormat(await resp.text());\n    for (const family of families) {\n      if (\n        family.type === 'GAUGE' &&\n        family.name === 'litestream_replica_progress'\n      ) {\n        for (const metric of family.metrics) {\n          const watermark = metric.labels?.watermark;\n          const name = metric.labels?.name;\n          const time = new Date(parseFloat(metric.value) * 1000);\n\n          if (watermark) {\n            yield {watermark, time, name};\n          }\n        }\n      }\n    }\n  }\n\n  async #checkWatermarks() {\n    for await (const {watermark, name, time} of this.#fetchWatermarks()) {\n      if (watermark > this.#lastWatermark && !this.#watermarks.has(watermark)) {\n        this.#lc.info?.(\n          `replicated watermark=${watermark} to ${name}` +\n            ` at ${time.toISOString()}.`,\n        );\n        this.#watermarks.set(watermark, time);\n      }\n    }\n  }\n\n  #scheduleCleanup() {\n    if (this.#reservations.size > 0) {\n      this.#lc.info?.(\n        `watermark cleanup paused for snapshot(s): ${[...this.#reservations.keys()]}`,\n      );\n      return;\n    }\n    const latestCleanupTime = Date.now() - this.#cleanupDelayMs;\n    let maxWatermark = '';\n    for (const [watermark, backupTime] of this.#watermarks.entries()) {\n      if (\n        backupTime.getTime() <= latestCleanupTime &&\n        watermark > maxWatermark\n      ) {\n        maxWatermark = watermark;\n      }\n    }\n    if (maxWatermark.length) {\n      this.#changeStreamer.scheduleCleanup(maxWatermark);\n      for (const watermark of this.#watermarks.keys()) {\n        if (watermark <= maxWatermark) {\n          this.#watermarks.delete(watermark);\n        }\n      }\n      this.#lastWatermark = maxWatermark;\n    }\n  }\n\n  stop(): Promise<void> {\n    clearInterval(this.#checkMetricsTimer);\n    for (const {sub} of this.#reservations.values()) {\n      // Close any pending reservations. This commonly happens when a new\n      // replication-manager makes a `/snapshot` reservation on the existing\n      // replication-manager, and then shuts it down when it takes over the\n      // replication slot.\n      sub.cancel();\n    }\n    this.#state.stop(this.#lc);\n    return promiseVoid;\n  }\n}\n"],"names":[],"mappings":";;;;AASO,MAAM,oBAAoB;AACjC,MAAM,uBAAuB;AAsCtB,MAAM,cAAiC;AAAA,EACnC,KAAK;AAAA,EACL;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA,SAAS,IAAI,aAAa,KAAK,EAAE;AAAA,EAEjC,oCAAoB,IAAA;AAAA,EACpB,kCAAkB,IAAA;AAAA,EAE3B,iBAAyB;AAAA,EACzB;AAAA,EACA;AAAA,EAEA,YACE,IACA,WACA,iBACA,gBACA,uBACA;AACA,SAAK,MAAM,GAAG,YAAY,aAAa,KAAK,EAAE;AAC9C,SAAK,aAAa;AAClB,SAAK,mBAAmB;AACxB,SAAK,kBAAkB;AACvB,SAAK,kBAAkB,KAAK;AAAA,MAC1B;AAAA,MACA;AAAA;AAAA,IAAA;AAGF,SAAK,IAAI;AAAA,MACP,0BAA0B,qBAAqB;AAAA,IAAA;AAAA,EAEnD;AAAA,EAEA,MAAqB;AACnB,SAAK,IAAI;AAAA,MACP,yBAAyB,KAAK,gBAAgB,SACzC,KAAK,eAAe;AAAA,IAAA;AAE3B,SAAK,qBAAqB;AAAA,MACxB,KAAK;AAAA,MACL;AAAA,IAAA;AAEF,WAAO,KAAK,OAAO,QAAA;AAAA,EACrB;AAAA,EAEA,yBAAyB,QAA+C;AACtE,SAAK,IAAI,OAAO,oCAAoC,MAAM,YAAY;AAEtE,SAAK,cAAc,IAAI,MAAM,GAAG,IAAI,OAAA;AAEpC,UAAM,MAAM,aAAa,OAAwB;AAAA;AAAA;AAAA;AAAA,MAI/C,SAAS,MAAM,KAAK,eAAe,QAAQ,KAAK;AAAA,IAAA,CACjD;AACD,SAAK,cAAc,IAAI,QAAQ,EAAC,OAAO,oBAAI,QAAQ,KAAI;AAGvD,SAAK,KAAK,gBACP,kBAAA,EACA,KAAK,CAAA,mBAAkB;AACtB,UAAI,KAAK;AAAA,QACP;AAAA,QACA,EAAC,KAAK,UAAU,WAAW,KAAK,YAAY,GAAG,eAAA;AAAA,MAAc,CAC9D;AAAA,IACH,CAAC,EACA,MAAM,CAAA,MAAK;AACV,WAAK,IAAI,OAAO,gCAAgC,CAAC;AACjD,UAAI,KAAK,CAAC;AAAA,IACZ,CAAC;AACH,WAAO;AAAA,EACT;AAAA,EAEA,eAAe,QAAgB,qBAAqB,MAAM;AACxD,UAAM,MAAM,KAAK,cAAc,IAAI,MAAM;AACzC,QAAI,QAAQ,QAAW;AACrB;AAAA,IACF;AACA,SAAK,cAAc,OAAO,MAAM;AAChC,UAAM,EAAC,OAAO,IAAA,IAAO;AACrB,QAAI,OAAA;AAEJ,QAAI,oBAAoB;AACtB,YAAM,WAAW,KAAK,IAAA,IAAQ,MAAM,QAAA;AACpC,WAAK,IAAI,OAAO,2BAA2B,MAAM,OAAO,QAAQ,KAAK;AACrE,UAAI,WAAW,KAAK,iBAAiB;AACnC,aAAK,kBAAkB;AACvB,aAAK,IAAI,OAAO,8BAA8B,QAAQ,KAAK;AAAA,MAC7D;AAAA,IACF;AAAA,EACF;AAAA;AAAA,EAGS,oCAAoC,YAAY;AACvD,QAAI;AACF,YAAM,KAAK,iBAAA;AAAA,IACb,SAAS,GAAG;AACV,WAAK,IAAI,OAAO,8BAA8B,KAAK,gBAAgB,IAAI,CAAC;AAAA,IAC1E;AACA,QAAI;AACF,WAAK,iBAAA;AAAA,IACP,SAAS,GAAG;AACV,WAAK,IAAI,OAAO,4BAA4B,CAAC;AAAA,IAC/C;AAAA,EACF;AAAA,EAEA,OAAO,mBAIJ;AACD,UAAM,kBAAkB,KAAK;AAC7B,UAAM,SAAS,KAAK,OAAO;AAC3B,QAAI;AACJ,QAAI;AACF,aAAO,MAAM,MAAM,iBAAiB,EAAC,QAAO;AAAA,IAC9C,SAAS,GAAG;AACV,UAAI,OAAO,SAAS;AAElB;AAAA,MACF;AAGA,WAAK,IAAI,OAAO,8BAA8B,KAAK,gBAAgB,IAAI,CAAC;AACxE;AAAA,IACF;AACA,QAAI,CAAC,KAAK,IAAI;AACZ,WAAK,IAAI;AAAA,QACP,8BAA8B,KAAK,gBAAgB,KAAK,MAAM,KAAK,MAAM;AAAA,MAAA;AAE3E;AAAA,IACF;AAEA,UAAM,WAAW,0BAA0B,MAAM,KAAK,MAAM;AAC5D,eAAW,UAAU,UAAU;AAC7B,UACE,OAAO,SAAS,WAChB,OAAO,SAAS,+BAChB;AACA,mBAAW,UAAU,OAAO,SAAS;AACnC,gBAAM,YAAY,OAAO,QAAQ;AACjC,gBAAM,OAAO,OAAO,QAAQ;AAC5B,gBAAM,OAAO,IAAI,KAAK,WAAW,OAAO,KAAK,IAAI,GAAI;AAErD,cAAI,WAAW;AACb,kBAAM,EAAC,WAAW,MAAM,KAAA;AAAA,UAC1B;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAM,mBAAmB;AACvB,qBAAiB,EAAC,WAAW,MAAM,UAAS,KAAK,oBAAoB;AACnE,UAAI,YAAY,KAAK,kBAAkB,CAAC,KAAK,YAAY,IAAI,SAAS,GAAG;AACvE,aAAK,IAAI;AAAA,UACP,wBAAwB,SAAS,OAAO,IAAI,OACnC,KAAK,aAAa;AAAA,QAAA;AAE7B,aAAK,YAAY,IAAI,WAAW,IAAI;AAAA,MACtC;AAAA,IACF;AAAA,EACF;AAAA,EAEA,mBAAmB;AACjB,QAAI,KAAK,cAAc,OAAO,GAAG;AAC/B,WAAK,IAAI;AAAA,QACP,6CAA6C,CAAC,GAAG,KAAK,cAAc,KAAA,CAAM,CAAC;AAAA,MAAA;AAE7E;AAAA,IACF;AACA,UAAM,oBAAoB,KAAK,IAAA,IAAQ,KAAK;AAC5C,QAAI,eAAe;AACnB,eAAW,CAAC,WAAW,UAAU,KAAK,KAAK,YAAY,WAAW;AAChE,UACE,WAAW,QAAA,KAAa,qBACxB,YAAY,cACZ;AACA,uBAAe;AAAA,MACjB;AAAA,IACF;AACA,QAAI,aAAa,QAAQ;AACvB,WAAK,gBAAgB,gBAAgB,YAAY;AACjD,iBAAW,aAAa,KAAK,YAAY,KAAA,GAAQ;AAC/C,YAAI,aAAa,cAAc;AAC7B,eAAK,YAAY,OAAO,SAAS;AAAA,QACnC;AAAA,MACF;AACA,WAAK,iBAAiB;AAAA,IACxB;AAAA,EACF;AAAA,EAEA,OAAsB;AACpB,kBAAc,KAAK,kBAAkB;AACrC,eAAW,EAAC,IAAA,KAAQ,KAAK,cAAc,UAAU;AAK/C,UAAI,OAAA;AAAA,IACN;AACA,SAAK,OAAO,KAAK,KAAK,GAAG;AACzB,WAAO;AAAA,EACT;AACF;"}

package/out/zero-cache/src/services/change-streamer/change-streamer-service.js

@@ -91,7 +91,6 @@ class ChangeStreamerImpl {
   async run() {
     this.#lc.info?.("starting change stream");
     await this.#storer.assumeOwnership();
-    this.#storer.run().then(() => this.stop()).catch((e) => this.stop(e));
     const flushBytesThreshold = getDefaultHighWaterMark(false);
     while (this.#state.shouldRun()) {
       let err;
@@ -103,6 +102,7 @@ class ChangeStreamerImpl {
           lastWatermark,
           backfillRequests
         );
+        this.#storer.run().catch((e) => stream.changes.cancel(e));
         this.#stream = stream;
         this.#state.resetBackoff();
         watermark = null;
@@ -165,7 +165,10 @@ class ChangeStreamerImpl {
           ["rollback", { tag: "rollback" }]
         ]);
       }
-      await this.#state.backoff(this.#lc, err);
+      await Promise.all([
+        this.#storer.stop(),
+        this.#state.backoff(this.#lc, err)
+      ]);
     }
     this.#lc.info?.("ChangeStreamer stopped");
   }
@@ -231,11 +234,21 @@ class ChangeStreamerImpl {
   }
   async getChangeLogState() {
     const minWatermark = await this.#storer.getMinWatermarkForCatchup();
+    if (!minWatermark) {
+      this.#lc.warn?.(
+        `Unexpected empty changeLog. Resync if "Local replica watermark" errors arise`
+      );
+    }
     return {
       replicaVersion: this.#replicaVersion,
       minWatermark: minWatermark ?? this.#replicaVersion
     };
   }
+  /**
+   * Makes a best effort to purge the change log. In the event of a database
+   * error, exceptions will be logged and swallowed, so this method is safe
+   * to run in a timeout.
+   */
   async #purgeOldChanges() {
     const initial = [...this.#initialWatermarks];
     if (initial.length === 0) {
@@ -255,10 +268,17 @@ class ChangeStreamerImpl {
           `At least one client is behind backup (${earliestCurrent} < ${earliestInitial})`
         );
       } else {
+        this.#lc.info?.(`Purging changes before ${earliestInitial} ...`);
+        const start = performance.now();
         const deleted = await this.#storer.purgeRecordsBefore(earliestInitial);
-        this.#lc.info?.(`Purged ${deleted} changes before ${earliestInitial}`);
+        const elapsed = (performance.now() - start).toFixed(2);
+        this.#lc.info?.(
+          `Purged ${deleted} changes before ${earliestInitial} (${elapsed} ms)`
+        );
         this.#initialWatermarks.delete(earliestInitial);
       }
+    } catch (e) {
+      this.#lc.warn?.(`error purging change log`, e);
     } finally {
       if (this.#initialWatermarks.size) {
         this.#state.setTimeout(() => this.#purgeOldChanges(), CLEANUP_DELAY_MS);