@rocicorp/zero 1.6.0-canary.0 → 1.6.0-canary.10

package/out/zero-cache/src/services/change-source/pg/schema/shard.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"shard.js","names":[],"sources":["../../../../../../../../zero-cache/src/services/change-source/pg/schema/shard.ts"],"sourcesContent":["import {PG_INSUFFICIENT_PRIVILEGE} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport {literal} from 'pg-format';\nimport postgres from 'postgres';\nimport {assert} from '../../../../../../shared/src/asserts.ts';\nimport {\n  jsonObjectSchema,\n  stringify,\n  type JSONObject,\n} from '../../../../../../shared/src/bigint-json.ts';\nimport * as v from '../../../../../../shared/src/valita.ts';\nimport {Default} from '../../../../db/postgres-replica-identity-enum.ts';\nimport type {PostgresDB, PostgresTransaction} from '../../../../types/pg.ts';\nimport type {AppID, ShardConfig, ShardID} from '../../../../types/shards.ts';\nimport {appSchema, check, upstreamSchema} from '../../../../types/shards.ts';\nimport {id} from '../../../../types/sql.ts';\nimport {\n  createEventFunctionStatements,\n  createEventTriggerStatements,\n} from './ddl.ts';\nimport {\n  getPublicationInfo,\n  publishedSchema,\n  type PublicationInfo,\n  type PublishedSchema,\n} from './published.ts';\nimport {validate} from './validation.ts';\n\n/**\n * PostgreSQL unquoted identifiers must start with a letter or underscore\n * and contain only letters, digits, and underscores.\n */\nconst VALID_PUBLICATION_NAME = /^[a-zA-Z_][a-zA-Z0-9_]*$/;\n\n/**\n * Validates that a publication name is a valid PostgreSQL identifier.\n * This provides defense-in-depth against SQL injection when publication\n * names are used in replication commands.\n */\nexport function validatePublicationName(name: string): void {\n  if (!VALID_PUBLICATION_NAME.test(name)) {\n    throw new Error(\n      `Invalid publication name \"${name}\". Publication names must start with a letter or underscore ` +\n        `and contain only letters, digits, and underscores.`,\n    );\n  }\n  if (name.length > 63) {\n    throw new Error(\n      `Publication name \"${name}\" exceeds PostgreSQL's 63-character identifier limit.`,\n    );\n  }\n}\n\nexport function internalPublicationPrefix({appID}: AppID) {\n  return `_${appID}_`;\n}\n\nexport function legacyReplicationSlot({appID, shardNum}: ShardID) {\n  return `${appID}_${shardNum}`;\n}\n\nexport function replicationSlotPrefix(shard: ShardID) {\n  const {appID, shardNum} = check(shard);\n  return `${appID}_${shardNum}_`;\n}\n\n/**\n * An expression used to match replication slots in the shard\n * in a Postgres `LIKE` operator.\n */\nexport function replicationSlotExpression(shard: ShardID) {\n  // Underscores have a special meaning in LIKE values\n  // so they have to be escaped.\n  return `${replicationSlotPrefix(shard)}%`.replaceAll('_', '\\\\_');\n}\n\nfunction defaultPublicationName(appID: string, shardID: string | number) {\n  return `_${appID}_public_${shardID}`;\n}\n\nexport function metadataPublicationName(\n  appID: string,\n  shardID: string | number,\n) {\n  return `_${appID}_metadata_${shardID}`;\n}\n\n// The GLOBAL_SETUP must be idempotent as it can be run multiple times for different shards.\nfunction globalSetup(appID: AppID): string {\n  const app = id(appSchema(appID));\n\n  return /*sql*/ `\n  CREATE SCHEMA IF NOT EXISTS ${app};\n\n  CREATE TABLE IF NOT EXISTS ${app}.permissions (\n    \"permissions\" JSONB,\n    \"hash\"        TEXT,\n\n    -- Ensure that there is only a single row in the table.\n    -- Application code can be agnostic to this column, and\n    -- simply invoke UPDATE statements on the version columns.\n    \"lock\" BOOL PRIMARY KEY DEFAULT true CHECK (lock)\n  );\n\n  CREATE OR REPLACE FUNCTION ${app}.set_permissions_hash()\n  RETURNS TRIGGER AS $$\n  BEGIN\n      NEW.hash = md5(NEW.permissions::text);\n      RETURN NEW;\n  END;\n  $$ LANGUAGE plpgsql;\n\n  CREATE OR REPLACE TRIGGER on_set_permissions \n    BEFORE INSERT OR UPDATE ON ${app}.permissions\n    FOR EACH ROW\n    EXECUTE FUNCTION ${app}.set_permissions_hash();\n\n  INSERT INTO ${app}.permissions (permissions) VALUES (NULL) ON CONFLICT DO NOTHING;\n`;\n}\n\nexport async function ensureGlobalTables(db: PostgresDB, appID: AppID) {\n  await db.unsafe(globalSetup(appID));\n}\n\nexport function getClientsTableDefinition(schema: string) {\n  return /*sql*/ `\n  CREATE TABLE ${schema}.\"clients\" (\n    \"clientGroupID\"  TEXT NOT NULL,\n    \"clientID\"       TEXT NOT NULL,\n    \"lastMutationID\" BIGINT NOT NULL,\n    \"userID\"         TEXT,\n    PRIMARY KEY(\"clientGroupID\", \"clientID\")\n  );`;\n}\n\n/**\n * Tracks the results of mutations.\n * 1. It is an error for the same mutation ID to be used twice.\n * 2. The result is JSONB to allow for arbitrary results.\n *\n * The tables must be cleaned up as the clients\n * receive the mutation responses and as clients are removed.\n */\nexport function getMutationsTableDefinition(schema: string) {\n  return /*sql*/ `\n  CREATE TABLE ${schema}.\"mutations\" (\n    \"clientGroupID\"  TEXT NOT NULL,\n    \"clientID\"       TEXT NOT NULL,\n    \"mutationID\"     BIGINT NOT NULL,\n    \"result\"         JSON NOT NULL,\n    PRIMARY KEY(\"clientGroupID\", \"clientID\", \"mutationID\")\n  );`;\n}\n\nexport const SHARD_CONFIG_TABLE = 'shardConfig';\n\nexport function shardSetup(\n  shardConfig: ShardConfig,\n  metadataPublication: string,\n): string {\n  const app = id(appSchema(shardConfig));\n  const shard = id(upstreamSchema(shardConfig));\n\n  const pubs = shardConfig.publications.toSorted();\n  assert(\n    pubs.includes(metadataPublication),\n    () => `Publications must include ${metadataPublication}`,\n  );\n\n  return /*sql*/ `\n  CREATE SCHEMA IF NOT EXISTS ${shard};\n\n  ${getClientsTableDefinition(shard)}\n  ${getMutationsTableDefinition(shard)}\n\n  DROP PUBLICATION IF EXISTS ${id(metadataPublication)};\n  CREATE PUBLICATION ${id(metadataPublication)}\n    FOR TABLE ${app}.\"permissions\", TABLE ${shard}.\"clients\", ${shard}.\"mutations\";\n\n  CREATE TABLE ${shard}.\"${SHARD_CONFIG_TABLE}\" (\n    \"publications\"  TEXT[] NOT NULL,\n    \"ddlDetection\"  BOOL NOT NULL,\n\n    -- Ensure that there is only a single row in the table.\n    \"lock\" BOOL PRIMARY KEY DEFAULT true CHECK (lock)\n  );\n\n  INSERT INTO ${shard}.\"${SHARD_CONFIG_TABLE}\" (\n      \"publications\",\n      \"ddlDetection\" \n    ) VALUES (\n      ARRAY[${literal(pubs)}], \n      false  -- set in SAVEPOINT with triggerSetup() statements\n    );\n\n  CREATE TABLE ${shard}.replicas (\n    -- The DEFAULT exists purely for backwards compatibility support.\n    -- New code always specifies a value based on Date.now().\n    \"id\"                 TEXT PRIMARY KEY DEFAULT replace(gen_random_uuid()::text, '-', ''),\n    \"rank\"               BIGSERIAL,\n    \"slot\"               TEXT NOT NULL,\n    \"version\"            TEXT NOT NULL,\n    \"initialSchema\"      JSON,  -- set after initial sync\n    \"initialSyncContext\" JSON,\n    \"subscriberContext\"  JSON\n  );\n  `;\n}\n\nexport function dropShard(appID: string, shardID: string | number): string {\n  const schema = `${appID}_${shardID}`;\n  const metadataPublication = metadataPublicationName(appID, shardID);\n  const defaultPublication = defaultPublicationName(appID, shardID);\n\n  // DROP SCHEMA ... CASCADE does not drop dependent PUBLICATIONS,\n  // so PUBLICATIONs must be dropped explicitly.\n  return /*sql*/ `\n    DROP PUBLICATION IF EXISTS ${id(defaultPublication)};\n    DROP PUBLICATION IF EXISTS ${id(metadataPublication)};\n    DROP SCHEMA IF EXISTS ${id(schema)} CASCADE;\n  `;\n}\n\nconst internalShardConfigSchema = v.object({\n  publications: v.array(v.string()),\n  ddlDetection: v.boolean(),\n});\n\nexport type InternalShardConfig = v.Infer<typeof internalShardConfigSchema>;\n\nconst replicaSchema = internalShardConfigSchema.extend({\n  id: v.string(),\n  rank: v.bigint(),\n  slot: v.string(),\n  version: v.string(),\n  initialSchema: publishedSchema,\n  initialSyncContext: jsonObjectSchema.nullable(),\n  subscriberContext: jsonObjectSchema.nullable(),\n});\n\nexport type Replica = v.Infer<typeof replicaSchema>;\n\n// triggerSetup is run separately in a sub-transaction (i.e. SAVEPOINT) so\n// that a failure (e.g. due to lack of superuser permissions) can be handled\n// by continuing in a degraded mode (ddlDetection = false).\nfunction triggerSetup(shard: ShardConfig): string {\n  const schema = id(upstreamSchema(shard));\n  return (\n    createEventTriggerStatements(shard) +\n    /*sql*/ `UPDATE ${schema}.\"shardConfig\" SET \"ddlDetection\" = true;`\n  );\n}\n\n/**\n * Creates a new replica to mark it as the owner of a specified `slot`.\n * This should be done with an advisory lock for replica/slot management\n * to ensure that the slot does not get dropped by concurrent cleanup\n * logic.\n *\n * Once initial sync is complete, {@link initReplica} should be called to\n * make the replica usable for incremental sync.\n */\nexport async function createReplica(\n  sql: PostgresDB,\n  shard: ShardID,\n  id: string,\n  slot: string,\n  replicaVersion: string,\n) {\n  const schema = upstreamSchema(shard);\n  const values = {id, slot, version: replicaVersion};\n  await sql`INSERT INTO ${sql(schema)}.replicas ${sql(values)}`;\n}\n\n// Called in initial-sync to store the exact schema that was initially synced.\nexport async function initReplica(\n  sql: PostgresDB,\n  shard: ShardID,\n  id: string,\n  {tables, indexes}: PublishedSchema,\n  initialSyncContext: JSONObject,\n) {\n  const schema = upstreamSchema(shard);\n  const synced: PublishedSchema = {tables, indexes};\n  const values = {initialSchema: synced, initialSyncContext};\n  await sql`UPDATE ${sql(schema)}.replicas SET ${sql(values)} WHERE id = ${id}`;\n}\n\n/**\n * Gets the latest initialized replica at the specified version.\n */\nexport async function getReplicaAtVersion(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardID,\n  replicaVersion: string,\n  context?: JSONObject,\n): Promise<Replica | null> {\n  const schema = sql(upstreamSchema(shard));\n  const result = await sql`\n    SELECT\n      replicas.\"id\",\n      replicas.\"rank\",\n      replicas.\"slot\",\n      replicas.\"version\",\n      replicas.\"initialSchema\",\n      replicas.\"initialSyncContext\",\n      replicas.\"subscriberContext\",\n      \"shardConfig\".\"publications\",\n      \"shardConfig\".\"ddlDetection\"\n    FROM ${schema}.replicas JOIN ${schema}.\"shardConfig\" ON true\n      WHERE version = ${replicaVersion} AND \"initialSyncContext\" IS NOT NULL\n      ORDER BY rank DESC LIMIT 1;\n  `;\n  if (result.length === 0) {\n    // log out all the replicas and the joined shardConfig\n    const allReplicas = await sql`\n      SELECT id, slot, version, \"initialSyncContext\", \"subscriberContext\" \n        FROM ${schema}.replicas`;\n    lc.info?.(\n      `Replica ${replicaVersion} ` +\n        (context ? `(context: ${stringify(context)}) ` : '') +\n        `not found in: ${stringify(allReplicas)}`,\n    );\n    return null;\n  }\n  return v.parse(result[0], replicaSchema, 'passthrough');\n}\n\nexport async function getInternalShardConfig(\n  sql: PostgresDB,\n  shard: ShardID,\n): Promise<InternalShardConfig> {\n  const result = await sql`\n    SELECT \"publications\", \"ddlDetection\"\n      FROM ${sql(upstreamSchema(shard))}.\"shardConfig\";\n  `;\n  assert(\n    result.length === 1,\n    () => `Expected exactly one shardConfig row, got ${result.length}`,\n  );\n  return v.parse(result[0], internalShardConfigSchema, 'passthrough');\n}\n\n/**\n * Sets up and returns all publications (including internal ones) for\n * the given shard.\n */\nexport async function setupTablesAndReplication(\n  lc: LogContext,\n  sql: PostgresTransaction,\n  requested: ShardConfig,\n) {\n  const {publications} = requested;\n  // Validate requested publications.\n  for (const pub of publications) {\n    validatePublicationName(pub);\n    if (pub.startsWith('_')) {\n      throw new Error(\n        `Publication names starting with \"_\" are reserved for internal use.\\n` +\n          `Please use a different name for publication \"${pub}\".`,\n      );\n    }\n  }\n  const allPublications: string[] = [];\n\n  // Setup application publications.\n  if (publications.length) {\n    const results = await sql<{pubname: string}[]>`\n    SELECT pubname from pg_publication WHERE pubname IN ${sql(\n      publications,\n    )}`.values();\n\n    if (results.length !== publications.length) {\n      throw new Error(\n        `Unknown or invalid publications. Specified: [${publications}]. Found: [${results.flat()}]`,\n      );\n    }\n    allPublications.push(...publications);\n  } else {\n    const defaultPublication = defaultPublicationName(\n      requested.appID,\n      requested.shardNum,\n    );\n    await sql`\n      DROP PUBLICATION IF EXISTS ${sql(defaultPublication)}`;\n    await sql`\n      CREATE PUBLICATION ${sql(defaultPublication)} \n        FOR TABLES IN SCHEMA public\n        WITH (publish_via_partition_root = true)`;\n    allPublications.push(defaultPublication);\n  }\n\n  const metadataPublication = metadataPublicationName(\n    requested.appID,\n    requested.shardNum,\n  );\n  allPublications.push(metadataPublication);\n\n  const shard = {...requested, publications: allPublications};\n\n  // Setup the global tables and shard tables / publications.\n  await sql.unsafe(globalSetup(shard) + shardSetup(shard, metadataPublication));\n\n  const pubs = await getPublicationInfo(sql, allPublications);\n  await replicaIdentitiesForTablesWithoutPrimaryKeys(pubs)?.apply(lc, sql);\n\n  await setupTriggers(lc, sql, shard);\n}\n\nexport async function setupTriggers(\n  lc: LogContext,\n  tx: PostgresTransaction,\n  shard: ShardConfig,\n) {\n  const schema = upstreamSchema(shard);\n  const [{ddlDetection}] = await tx<InternalShardConfig[]> /*sql*/ `\n    SELECT \"ddlDetection\" FROM ${tx(schema)}.\"shardConfig\"`;\n\n  // The functions invoked by event triggers are installed even if\n  // event triggers are not supported/allowed by the db provider.\n  // This allows users to manually invoke the update_schemas() function\n  // as a workaround.\n  await tx.unsafe(createEventFunctionStatements(shard));\n  try {\n    await tx.savepoint(sub => sub.unsafe(triggerSetup(shard)));\n  } catch (e) {\n    if (ddlDetection) {\n      // If ddlDetection has already been enabled, subsequent failures to\n      // upgrade the trigger should be propagated rather than swallowed.\n      throw e;\n    }\n    if (\n      !(\n        e instanceof postgres.PostgresError &&\n        e.code === PG_INSUFFICIENT_PRIVILEGE\n      )\n    ) {\n      throw e;\n    }\n    // If triggerSetup() fails, replication continues in ddlDetection=false mode.\n    lc.warn?.(\n      `Unable to create event triggers for schema change detection:\\n\\n` +\n        `\"${e.hint ?? e.message}\"\\n\\n` +\n        `Proceeding in degraded mode: schema changes will halt replication,\\n` +\n        `requiring the replica to be reset (manually or with --auto-reset).`,\n    );\n  }\n}\n\nexport function validatePublications(\n  lc: LogContext,\n  published: PublicationInfo,\n) {\n  // Verify that all publications export the proper events.\n  published.publications.forEach(pub => {\n    if (\n      !pub.pubinsert ||\n      !pub.pubupdate ||\n      !pub.pubdelete ||\n      !pub.pubtruncate\n    ) {\n      // TODO: Make APIError?\n      throw new Error(\n        `PUBLICATION ${pub.pubname} must publish insert, update, delete, and truncate`,\n      );\n    }\n  });\n\n  published.tables.forEach(table => validate(lc, table));\n}\n\ntype ReplicaIdentities = {\n  apply(lc: LogContext, db: PostgresDB): Promise<void>;\n};\n\nexport function replicaIdentitiesForTablesWithoutPrimaryKeys(\n  pubs: PublishedSchema,\n): ReplicaIdentities | undefined {\n  const replicaIdentities: {\n    schema: string;\n    tableName: string;\n    indexName: string;\n  }[] = [];\n  for (const table of pubs.tables) {\n    if (!table.primaryKey?.length && table.replicaIdentity === Default) {\n      // Look for an index that can serve as the REPLICA IDENTITY USING INDEX. It must be:\n      // - UNIQUE\n      // - NOT NULL columns\n      // - not deferrable (i.e. isImmediate)\n      // - not partial (are already filtered out)\n      //\n      // https://www.postgresql.org/docs/current/sql-altertable.html#SQL-ALTERTABLE-REPLICA-IDENTITY\n      const {schema, name: tableName} = table;\n      for (const {columns, name: indexName} of pubs.indexes.filter(\n        idx =>\n          idx.schema === schema &&\n          idx.tableName === tableName &&\n          idx.unique &&\n          idx.isImmediate,\n      )) {\n        if (Object.keys(columns).some(col => !table.columns[col].notNull)) {\n          continue; // Only indexes with all NOT NULL columns are suitable.\n        }\n        replicaIdentities.push({schema, tableName, indexName});\n        break;\n      }\n    }\n  }\n\n  if (replicaIdentities.length === 0) {\n    return undefined;\n  }\n  return {\n    apply: async (lc: LogContext, sql: PostgresDB) => {\n      for (const {schema, tableName, indexName} of replicaIdentities) {\n        lc.info?.(\n          `setting \"${indexName}\" as the REPLICA IDENTITY for \"${tableName}\"`,\n        );\n        await sql`\n        ALTER TABLE ${sql(schema)}.${sql(tableName)} \n          REPLICA IDENTITY USING INDEX ${sql(indexName)}`;\n      }\n    },\n  };\n}\n"],"mappings":";;;;;;;;;;;;;;;;AAgCA,IAAM,yBAAyB;;;;;;AAO/B,SAAgB,wBAAwB,MAAoB;AAC1D,KAAI,CAAC,uBAAuB,KAAK,KAAK,CACpC,OAAM,IAAI,MACR,6BAA6B,KAAK,gHAEnC;AAEH,KAAI,KAAK,SAAS,GAChB,OAAM,IAAI,MACR,qBAAqB,KAAK,uDAC3B;;AAIL,SAAgB,0BAA0B,EAAC,SAAe;AACxD,QAAO,IAAI,MAAM;;AAGnB,SAAgB,sBAAsB,EAAC,OAAO,YAAoB;AAChE,QAAO,GAAG,MAAM,GAAG;;AAGrB,SAAgB,sBAAsB,OAAgB;CACpD,MAAM,EAAC,OAAO,aAAY,MAAM,MAAM;AACtC,QAAO,GAAG,MAAM,GAAG,SAAS;;;;;;AAO9B,SAAgB,0BAA0B,OAAgB;AAGxD,QAAO,GAAG,sBAAsB,MAAM,CAAC,GAAG,WAAW,KAAK,MAAM;;AAGlE,SAAS,uBAAuB,OAAe,SAA0B;AACvE,QAAO,IAAI,MAAM,UAAU;;AAG7B,SAAgB,wBACd,OACA,SACA;AACA,QAAO,IAAI,MAAM,YAAY;;AAI/B,SAAS,YAAY,OAAsB;CACzC,MAAM,MAAM,GAAG,UAAU,MAAM,CAAC;AAEhC,QAAe;gCACe,IAAI;;+BAEL,IAAI;;;;;;;;;;+BAUJ,IAAI;;;;;;;;;iCASF,IAAI;;uBAEd,IAAI;;gBAEX,IAAI;;;AAIpB,eAAsB,mBAAmB,IAAgB,OAAc;AACrE,OAAM,GAAG,OAAO,YAAY,MAAM,CAAC;;AAGrC,SAAgB,0BAA0B,QAAgB;AACxD,QAAe;iBACA,OAAO;;;;;;;;;;;;;;;;AAiBxB,SAAgB,4BAA4B,QAAgB;AAC1D,QAAe;iBACA,OAAO;;;;;;;;AASxB,IAAa,qBAAqB;AAElC,SAAgB,WACd,aACA,qBACQ;CACR,MAAM,MAAM,GAAG,UAAU,YAAY,CAAC;CACtC,MAAM,QAAQ,GAAG,eAAe,YAAY,CAAC;CAE7C,MAAM,OAAO,YAAY,aAAa,UAAU;AAChD,QACE,KAAK,SAAS,oBAAoB,QAC5B,6BAA6B,sBACpC;AAED,QAAe;gCACe,MAAM;;IAElC,0BAA0B,MAAM,CAAC;IACjC,4BAA4B,MAAM,CAAC;;+BAER,GAAG,oBAAoB,CAAC;uBAChC,GAAG,oBAAoB,CAAC;gBAC/B,IAAI,wBAAwB,MAAM,cAAc,MAAM;;iBAErD,MAAM,IAAI,mBAAmB;;;;;;;;gBAQ9B,MAAM,IAAI,mBAAmB;;;;cAI/B,QAAQ,KAAK,CAAC;;;;iBAIX,MAAM;;;;;;;;;;;;;AAcvB,SAAgB,UAAU,OAAe,SAAkC;CACzE,MAAM,SAAS,GAAG,MAAM,GAAG;CAC3B,MAAM,sBAAsB,wBAAwB,OAAO,QAAQ;AAKnE,QAAe;iCACgB,GALJ,uBAAuB,OAAO,QAAQ,CAKZ,CAAC;iCACvB,GAAG,oBAAoB,CAAC;4BAC7B,GAAG,OAAO,CAAC;;;AAIvC,IAAM,4BAA4B,eAAE,OAAO;CACzC,cAAc,eAAE,MAAM,eAAE,QAAQ,CAAC;CACjC,cAAc,eAAE,SAAS;CAC1B,CAAC;AAIF,IAAM,gBAAgB,0BAA0B,OAAO;CACrD,IAAI,eAAE,QAAQ;CACd,MAAM,eAAE,QAAQ;CAChB,MAAM,eAAE,QAAQ;CAChB,SAAS,eAAE,QAAQ;CACnB,eAAe;CACf,oBAAoB,iBAAiB,UAAU;CAC/C,mBAAmB,iBAAiB,UAAU;CAC/C,CAAC;AAOF,SAAS,aAAa,OAA4B;CAChD,MAAM,SAAS,GAAG,eAAe,MAAM,CAAC;AACxC,QACE,6BAA6B,MAAM,GAC3B,UAAU,OAAO;;;;;;;;;;;AAa7B,eAAsB,cACpB,KACA,OACA,IACA,MACA,gBACA;CACA,MAAM,SAAS,eAAe,MAAM;CACpC,MAAM,SAAS;EAAC;EAAI;EAAM,SAAS;EAAe;AAClD,OAAM,GAAG,eAAe,IAAI,OAAO,CAAC,YAAY,IAAI,OAAO;;AAI7D,eAAsB,YACpB,KACA,OACA,IACA,EAAC,QAAQ,WACT,oBACA;CACA,MAAM,SAAS,eAAe,MAAM;CAEpC,MAAM,SAAS;EAAC,eADgB;GAAC;GAAQ;GAAQ;EACV;EAAmB;AAC1D,OAAM,GAAG,UAAU,IAAI,OAAO,CAAC,gBAAgB,IAAI,OAAO,CAAC,cAAc;;;;;AAM3E,eAAsB,oBACpB,IACA,KACA,OACA,gBACA,SACyB;CACzB,MAAM,SAAS,IAAI,eAAe,MAAM,CAAC;CACzC,MAAM,SAAS,MAAM,GAAG;;;;;;;;;;;WAWf,OAAO,iBAAiB,OAAO;wBAClB,eAAe;;;AAGrC,KAAI,OAAO,WAAW,GAAG;EAEvB,MAAM,cAAc,MAAM,GAAG;;eAElB,OAAO;AAClB,KAAG,OACD,WAAW,eAAe,MACvB,UAAU,aAAa,UAAU,QAAQ,CAAC,MAAM,MACjD,iBAAiB,UAAU,YAAY,GAC1C;AACD,SAAO;;AAET,QAAO,MAAQ,OAAO,IAAI,eAAe,cAAc;;AAGzD,eAAsB,uBACpB,KACA,OAC8B;CAC9B,MAAM,SAAS,MAAM,GAAG;;aAEb,IAAI,eAAe,MAAM,CAAC,CAAC;;AAEtC,QACE,OAAO,WAAW,SACZ,6CAA6C,OAAO,SAC3D;AACD,QAAO,MAAQ,OAAO,IAAI,2BAA2B,cAAc;;;;;;AAOrE,eAAsB,0BACpB,IACA,KACA,WACA;CACA,MAAM,EAAC,iBAAgB;AAEvB,MAAK,MAAM,OAAO,cAAc;AAC9B,0BAAwB,IAAI;AAC5B,MAAI,IAAI,WAAW,IAAI,CACrB,OAAM,IAAI,MACR,oHACkD,IAAI,IACvD;;CAGL,MAAM,kBAA4B,EAAE;AAGpC,KAAI,aAAa,QAAQ;EACvB,MAAM,UAAU,MAAM,GAAwB;0DACQ,IACpD,aACD,GAAG,QAAQ;AAEZ,MAAI,QAAQ,WAAW,aAAa,OAClC,OAAM,IAAI,MACR,gDAAgD,aAAa,aAAa,QAAQ,MAAM,CAAC,GAC1F;AAEH,kBAAgB,KAAK,GAAG,aAAa;QAChC;EACL,MAAM,qBAAqB,uBACzB,UAAU,OACV,UAAU,SACX;AACD,QAAM,GAAG;mCACsB,IAAI,mBAAmB;AACtD,QAAM,GAAG;2BACc,IAAI,mBAAmB,CAAC;;;AAG/C,kBAAgB,KAAK,mBAAmB;;CAG1C,MAAM,sBAAsB,wBAC1B,UAAU,OACV,UAAU,SACX;AACD,iBAAgB,KAAK,oBAAoB;CAEzC,MAAM,QAAQ;EAAC,GAAG;EAAW,cAAc;EAAgB;AAG3D,OAAM,IAAI,OAAO,YAAY,MAAM,GAAG,WAAW,OAAO,oBAAoB,CAAC;AAG7E,OAAM,6CADO,MAAM,mBAAmB,KAAK,gBAAgB,CACH,EAAE,MAAM,IAAI,IAAI;AAExE,OAAM,cAAc,IAAI,KAAK,MAAM;;AAGrC,eAAsB,cACpB,IACA,IACA,OACA;CAEA,MAAM,CAAC,EAAC,kBAAiB,MAAM,EAAkC;iCAClC,GAFhB,eAAe,MAAM,CAEK,CAAC;AAM1C,OAAM,GAAG,OAAO,8BAA8B,MAAM,CAAC;AACrD,KAAI;AACF,QAAM,GAAG,WAAU,QAAO,IAAI,OAAO,aAAa,MAAM,CAAC,CAAC;UACnD,GAAG;AACV,MAAI,aAGF,OAAM;AAER,MACE,EACE,aAAa,SAAS,iBACtB,EAAE,SAAS,2BAGb,OAAM;AAGR,KAAG,OACD,oEACM,EAAE,QAAQ,EAAE,QAAQ,6IAG3B;;;AAIL,SAAgB,qBACd,IACA,WACA;AAEA,WAAU,aAAa,SAAQ,QAAO;AACpC,MACE,CAAC,IAAI,aACL,CAAC,IAAI,aACL,CAAC,IAAI,aACL,CAAC,IAAI,YAGL,OAAM,IAAI,MACR,eAAe,IAAI,QAAQ,oDAC5B;GAEH;AAEF,WAAU,OAAO,SAAQ,UAAS,SAAS,IAAI,MAAM,CAAC;;AAOxD,SAAgB,6CACd,MAC+B;CAC/B,MAAM,oBAIA,EAAE;AACR,MAAK,MAAM,SAAS,KAAK,OACvB,KAAI,CAAC,MAAM,YAAY,UAAU,MAAM,oBAAA,KAA6B;EAQlE,MAAM,EAAC,QAAQ,MAAM,cAAa;AAClC,OAAK,MAAM,EAAC,SAAS,MAAM,eAAc,KAAK,QAAQ,QACpD,QACE,IAAI,WAAW,UACf,IAAI,cAAc,aAClB,IAAI,UACJ,IAAI,YACP,EAAE;AACD,OAAI,OAAO,KAAK,QAAQ,CAAC,MAAK,QAAO,CAAC,MAAM,QAAQ,KAAK,QAAQ,CAC/D;AAEF,qBAAkB,KAAK;IAAC;IAAQ;IAAW;IAAU,CAAC;AACtD;;;AAKN,KAAI,kBAAkB,WAAW,EAC/B;AAEF,QAAO,EACL,OAAO,OAAO,IAAgB,QAAoB;AAChD,OAAK,MAAM,EAAC,QAAQ,WAAW,eAAc,mBAAmB;AAC9D,MAAG,OACD,YAAY,UAAU,iCAAiC,UAAU,GAClE;AACD,SAAM,GAAG;sBACK,IAAI,OAAO,CAAC,GAAG,IAAI,UAAU,CAAC;yCACX,IAAI,UAAU;;IAGpD"}

package/out/zero-cache/src/services/change-source/pg/schema/validation.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"validation.js","names":[],"sources":["../../../../../../../../zero-cache/src/services/change-source/pg/schema/validation.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {\n  mapPostgresToLite,\n  warnIfDataTypeSupported,\n} from '../../../../db/pg-to-lite.ts';\nimport {\n  Default,\n  Index,\n  Nothing,\n} from '../../../../db/postgres-replica-identity-enum.ts';\nimport {ZERO_VERSION_COLUMN_NAME} from '../../../replicator/schema/constants.ts';\nimport type {PublishedTableWithReplicaIdentity} from './published.ts';\n\nexport const ALLOWED_APP_ID_CHARACTERS = /^[a-z0-9_]+$/;\n\nconst ALLOWED_TABLE_CHARS = /^[A-Za-z_]+[A-Za-z0-9_-]*$/;\n\n// Dots are allowed in column names since there is no need for\n// a schema/table delimiter when mapped to SQLite names.\nconst ALLOWED_COLUMN_CHARS = /^[A-Za-z_]+[.A-Za-z0-9_-]*$/;\n\nexport function validate(\n  lc: LogContext,\n  table: PublishedTableWithReplicaIdentity,\n) {\n  if (ZERO_VERSION_COLUMN_NAME in table.columns) {\n    throw new UnsupportedTableSchemaError(\n      `Table \"${table.name}\" uses reserved column name \"${ZERO_VERSION_COLUMN_NAME}\"`,\n    );\n  }\n  if (!table.primaryKey?.length && table.replicaIdentity === Default) {\n    lc.warn?.(\n      `\\n\\n\\n` +\n        `Table \"${table.name}\" needs a primary key in order to be synced to clients. ` +\n        `Add one with 'ALTER TABLE \"${table.name}\" ADD PRIMARY KEY (...)'.` +\n        `\\n\\n\\n`,\n    );\n  }\n  if (table.replicaIdentity === Nothing) {\n    throw new UnsupportedTableSchemaError(\n      `Table \"${table.name}\" with REPLICA IDENTITY NOTHING cannot be replicated`,\n    );\n  }\n  if (\n    table.replicaIdentity === Index &&\n    table.replicaIdentityColumns.length === 0\n  ) {\n    throw new UnsupportedTableSchemaError(\n      `Table \"${table.name}\" is missing its REPLICA IDENTITY INDEX`,\n    );\n  }\n  if (!ALLOWED_TABLE_CHARS.test(table.name)) {\n    throw new UnsupportedTableSchemaError(\n      `Table \"${table.name}\" has invalid characters.`,\n    );\n  }\n  for (const [col, spec] of Object.entries(mapPostgresToLite(table).columns)) {\n    if (!ALLOWED_COLUMN_CHARS.test(col)) {\n      throw new UnsupportedTableSchemaError(\n        `Column \"${col}\" in table \"${table.name}\" has invalid characters.`,\n      );\n    }\n    warnIfDataTypeSupported(lc, spec.dataType, table.name, col);\n  }\n}\n\nexport class UnsupportedTableSchemaError extends Error {\n  readonly name = 'UnsupportedTableSchemaError';\n\n  constructor(msg: string) {\n    super(msg);\n  }\n}\n"],"mappings":";;;AAeA,IAAM,sBAAsB;AAI5B,IAAM,uBAAuB;AAE7B,SAAgB,SACd,IACA,OACA;AACA,KAAA,gBAAgC,MAAM,QACpC,OAAM,IAAI,4BACR,UAAU,MAAM,KAAK,+BAA+B,yBAAyB,GAC9E;AAEH,KAAI,CAAC,MAAM,YAAY,UAAU,MAAM,oBAAA,IACrC,IAAG,OACD,gBACY,MAAM,KAAK,qFACS,MAAM,KAAK,iCAE5C;AAEH,KAAI,MAAM,oBAAA,IACR,OAAM,IAAI,4BACR,UAAU,MAAM,KAAK,sDACtB;AAEH,KACE,MAAM,oBAAA,OACN,MAAM,uBAAuB,WAAW,EAExC,OAAM,IAAI,4BACR,UAAU,MAAM,KAAK,yCACtB;AAEH,KAAI,CAAC,oBAAoB,KAAK,MAAM,KAAK,CACvC,OAAM,IAAI,4BACR,UAAU,MAAM,KAAK,2BACtB;AAEH,MAAK,MAAM,CAAC,KAAK,SAAS,OAAO,QAAQ,kBAAkB,MAAM,CAAC,QAAQ,EAAE;AAC1E,MAAI,CAAC,qBAAqB,KAAK,IAAI,CACjC,OAAM,IAAI,4BACR,WAAW,IAAI,cAAc,MAAM,KAAK,2BACzC;AAEH,0BAAwB,IAAI,KAAK,UAAU,MAAM,MAAM,IAAI;;;AAI/D,IAAa,8BAAb,cAAiD,MAAM;CACrD,OAAgB;CAEhB,YAAY,KAAa;AACvB,QAAM,IAAI"}

package/out/zero-cache/src/services/change-source/protocol/current/control.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"control.js","names":[],"sources":["../../../../../../../../zero-cache/src/services/change-source/protocol/current/control.ts"],"sourcesContent":["/**\n * Control plane messages communicate non-content related signals between a\n * ChangeSource and ChangeStreamer. These are not forwarded to subscribers\n * of the ChangeStreamer.\n */\nimport * as v from '../../../../../../shared/src/valita.ts';\nimport {jsonObjectSchema} from './json.ts';\n\n/**\n * Indicates that replication cannot continue and that the replica must be resynced\n * from scratch. The replication-manager will shutdown in response to this message,\n * and upon being restarted, it will wipe the current replica and resync if the\n * `--auto-reset` option is specified.\n *\n * This signal should only be used in well advertised scenarios, and is not suitable\n * as a common occurrence in production.\n */\nexport const resetRequiredSchema = v.object({\n  tag: v.literal('reset-required'),\n  message: v.string().optional(),\n\n  // errorDetails can contain any standard JSON object. This will be\n  // published in the `errorDetails` field of a replication ERROR event.\n  errorDetails: jsonObjectSchema.optional(),\n});\n"],"mappings":";;;;;;;;;;;;;;;;;AAiBA,IAAa,sBAAsB,eAAE,OAAO;CAC1C,KAAK,eAAE,QAAQ,iBAAiB;CAChC,SAAS,eAAE,QAAQ,CAAC,UAAU;CAI9B,cAAc,iBAAiB,UAAU;CAC1C,CAAC"}

package/out/zero-cache/src/services/change-source/protocol/current/data.js

@@ -1,192 +0,0 @@
-import { literalUnion, valita_exports } from "../../../../../../shared/src/valita.js";
-import { must } from "../../../../../../shared/src/must.js";
-import { jsonValueSchema } from "../../../../../../shared/src/bigint-json.js";
-import { jsonObjectSchema } from "./json.js";
-import { columnSpec, indexSpec, tableSpec } from "../../../../db/specs.js";
-//#region ../zero-cache/src/services/change-source/protocol/current/data.ts
-/**
-* Data plane messages encapsulate changes that are sent by ChangeSources,
-* forwarded / fanned out to subscribers by the ChangeStreamerService, and
-* stored in the Change DB for catchup of old subscribers.
-*/
-var beginSchema = valita_exports.object({
-	tag: valita_exports.literal("begin"),
-	json: literalUnion("p", "s").optional(),
-	skipAck: valita_exports.boolean().optional()
-});
-var commitSchema = valita_exports.object({ tag: valita_exports.literal("commit") });
-var rollbackSchema = valita_exports.object({ tag: valita_exports.literal("rollback") });
-var rowKeySchema = valita_exports.object({
-	columns: valita_exports.array(valita_exports.string()),
-	type: literalUnion("default", "nothing", "full", "index").optional()
-});
-var relationSchema = valita_exports.object({
-	schema: valita_exports.string(),
-	name: valita_exports.string(),
-	rowKey: rowKeySchema.optional(),
-	keyColumns: valita_exports.array(valita_exports.string()).optional(),
-	replicaIdentity: literalUnion("default", "nothing", "full", "index").optional()
-}).map((rel) => {
-	const { rowKey, ...rest } = rel;
-	if (rowKey) return {
-		...rest,
-		rowKey
-	};
-	return {
-		...rest,
-		rowKey: {
-			columns: must(rel.keyColumns),
-			type: rel.replicaIdentity
-		}
-	};
-});
-var newRelationSchema = valita_exports.object({
-	schema: valita_exports.string(),
-	name: valita_exports.string(),
-	rowKey: rowKeySchema
-});
-var tableMetadataSchema = valita_exports.object({ rowKey: valita_exports.record(jsonValueSchema) }).rest(jsonValueSchema);
-var rowSchema = valita_exports.record(jsonValueSchema);
-var insertSchema = valita_exports.object({
-	tag: valita_exports.literal("insert"),
-	relation: relationSchema,
-	new: rowSchema
-});
-var updateSchema = valita_exports.object({
-	tag: valita_exports.literal("update"),
-	relation: relationSchema,
-	key: rowSchema.nullable(),
-	new: rowSchema
-});
-var deleteSchema = valita_exports.object({
-	tag: valita_exports.literal("delete"),
-	relation: relationSchema,
-	key: rowSchema
-});
-var truncateSchema = valita_exports.object({
-	tag: valita_exports.literal("truncate"),
-	relations: valita_exports.array(relationSchema)
-});
-var identifierSchema = valita_exports.object({
-	schema: valita_exports.string(),
-	name: valita_exports.string()
-});
-var backfillIDSchema = jsonObjectSchema;
-var createTableSchema = valita_exports.object({
-	tag: valita_exports.literal("create-table"),
-	spec: tableSpec,
-	metadata: tableMetadataSchema.optional(),
-	backfill: valita_exports.record(backfillIDSchema).optional()
-});
-var renameTableSchema = valita_exports.object({
-	tag: valita_exports.literal("rename-table"),
-	old: identifierSchema,
-	new: identifierSchema
-});
-var updateTableMetadataSchema = valita_exports.object({
-	tag: valita_exports.literal("update-table-metadata"),
-	table: identifierSchema,
-	old: tableMetadataSchema,
-	new: tableMetadataSchema
-});
-var columnSchema = valita_exports.object({
-	name: valita_exports.string(),
-	spec: columnSpec
-});
-var addColumnSchema = valita_exports.object({
-	tag: valita_exports.literal("add-column"),
-	table: identifierSchema,
-	column: columnSchema,
-	tableMetadata: tableMetadataSchema.optional(),
-	backfill: backfillIDSchema.optional()
-});
-var updateColumnSchema = valita_exports.object({
-	tag: valita_exports.literal("update-column"),
-	table: identifierSchema,
-	old: columnSchema,
-	new: columnSchema
-});
-var dropColumnSchema = valita_exports.object({
-	tag: valita_exports.literal("drop-column"),
-	table: identifierSchema,
-	column: valita_exports.string()
-});
-var dropTableSchema = valita_exports.object({
-	tag: valita_exports.literal("drop-table"),
-	id: identifierSchema
-});
-var createIndexSchema = valita_exports.object({
-	tag: valita_exports.literal("create-index"),
-	spec: indexSpec
-});
-var dropIndexSchema = valita_exports.object({
-	tag: valita_exports.literal("drop-index"),
-	id: identifierSchema
-});
-var downloadStatusSchema = valita_exports.object({
-	rows: valita_exports.number(),
-	totalRows: valita_exports.number(),
-	totalBytes: valita_exports.number().optional()
-});
-var backfillSchema = valita_exports.object({
-	tag: valita_exports.literal("backfill"),
-	relation: newRelationSchema,
-	columns: valita_exports.array(valita_exports.string()),
-	watermark: valita_exports.string(),
-	rowValues: valita_exports.array(valita_exports.array(jsonValueSchema)),
-	status: downloadStatusSchema.optional()
-});
-var backfillCompletedSchema = valita_exports.object({
-	tag: valita_exports.literal("backfill-completed"),
-	relation: newRelationSchema,
-	columns: valita_exports.array(valita_exports.string()),
-	watermark: valita_exports.string(),
-	status: downloadStatusSchema.optional()
-});
-var dataChangeSchema = valita_exports.union(insertSchema, updateSchema, deleteSchema, truncateSchema, backfillSchema);
-var dataChangeTags = [
-	"insert",
-	"update",
-	"delete",
-	"truncate",
-	"backfill"
-];
-literalUnion(...dataChangeTags);
-var schemaChanges = [
-	createTableSchema,
-	renameTableSchema,
-	updateTableMetadataSchema,
-	addColumnSchema,
-	updateColumnSchema,
-	dropColumnSchema,
-	dropTableSchema,
-	createIndexSchema,
-	dropIndexSchema,
-	backfillCompletedSchema
-];
-var schemaChangeTags = [
-	"create-table",
-	"rename-table",
-	"update-table-metadata",
-	"add-column",
-	"update-column",
-	"drop-column",
-	"drop-table",
-	"create-index",
-	"drop-index",
-	"backfill-completed"
-];
-var schemaChangeSchema = valita_exports.union(...schemaChanges);
-literalUnion(...schemaChangeTags);
-var schemaChangeTagSet = new Set(schemaChangeTags);
-function isSchemaChange(change) {
-	return schemaChangeTagSet.has(change.tag);
-}
-var dataChangeTagSet = new Set(dataChangeTags);
-function isDataChange(change) {
-	return dataChangeTagSet.has(change.tag);
-}
-//#endregion
-export { addColumnSchema, backfillCompletedSchema, backfillIDSchema, backfillSchema, beginSchema, commitSchema, createIndexSchema, createTableSchema, dataChangeSchema, deleteSchema, downloadStatusSchema, dropColumnSchema, dropIndexSchema, dropTableSchema, identifierSchema, insertSchema, isDataChange, isSchemaChange, newRelationSchema, relationSchema, renameTableSchema, rollbackSchema, rowSchema, schemaChangeSchema, tableMetadataSchema, truncateSchema, updateColumnSchema, updateSchema, updateTableMetadataSchema };
-
-//# sourceMappingURL=data.js.map

package/out/zero-cache/src/services/change-source/protocol/current/data.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"data.js","names":[],"sources":["../../../../../../../../zero-cache/src/services/change-source/protocol/current/data.ts"],"sourcesContent":["/**\n * Data plane messages encapsulate changes that are sent by ChangeSources,\n * forwarded / fanned out to subscribers by the ChangeStreamerService, and\n * stored in the Change DB for catchup of old subscribers.\n */\n\nimport {\n  jsonValueSchema,\n  type JSONObject,\n} from '../../../../../../shared/src/bigint-json.ts';\nimport {must} from '../../../../../../shared/src/must.ts';\nimport * as v from '../../../../../../shared/src/valita.ts';\nimport {columnSpec, indexSpec, tableSpec} from '../../../../db/specs.ts';\nimport type {Satisfies} from '../../../../types/satisfies.ts';\nimport {jsonObjectSchema} from './json.ts';\n\nexport const beginSchema = v.object({\n  tag: v.literal('begin'),\n  // The format of values of \"json\"-typed columns (e.g. \"JSON\" and \"JSONB\").\n  // - 'p' is for parsed JSON, which may include JSON values or JSON objects.\n  //   These values are parsed and stringified at every process boundary\n  //   between the change-source and the replica.\n  // - 's' is for stringified JSON. These values skip the parsing and\n  //   stringification, and are directly ferried to the replica as a JSON\n  //   string. For JSON values this improves performance by 20~25% in the\n  //   change-streamer and 25~30% in the replicator.\n  //\n  // If absent, the format is assumed to be 'p' (parsed JSON objects/values).\n  json: v.literalUnion('p', 's').optional(),\n\n  // Directs the change-streamer to skip the ACK for the corresponding commit.\n  skipAck: v.boolean().optional(),\n});\n\nexport const commitSchema = v.object({\n  tag: v.literal('commit'),\n});\n\nexport const rollbackSchema = v.object({\n  tag: v.literal('rollback'),\n});\n\nconst rowKeySchema = v.object({\n  // The columns used to identify a row in insert, update, and delete changes.\n  columns: v.array(v.string()),\n\n  // An optional qualifier identifying how the key is chosen. Currently this\n  // is postgres-specific, describing the REPLICA IDENTITY, for which replica\n  // identity 'full' (FULL) is handled differently; the replicator handles\n  // these tables by extracting a row key from the full row based on the\n  // table's PRIMARY KEY or UNIQUE INDEX.\n  type: v.literalUnion('default', 'nothing', 'full', 'index').optional(),\n});\n\nexport const relationSchema = v\n  .object({\n    schema: v.string(),\n    name: v.string(),\n\n    // This will become required.\n    rowKey: rowKeySchema.optional(),\n\n    /** Deprecated: set the rowKey.columns instead. */\n    keyColumns: v.array(v.string()).optional(),\n    /** Deprecated: set the rowKey.columns instead. */\n    replicaIdentity: v\n      .literalUnion('default', 'nothing', 'full', 'index')\n      .optional(),\n  })\n  .map(rel => {\n    const {rowKey, ...rest} = rel;\n    if (rowKey) {\n      return {...rest, rowKey};\n    }\n    return {\n      ...rest,\n      rowKey: {\n        columns: must(rel.keyColumns),\n        type: rel.replicaIdentity,\n      },\n    };\n  });\n\n// The eventual fate of relationSchema\nexport const newRelationSchema = v.object({\n  schema: v.string(),\n  name: v.string(),\n\n  rowKey: rowKeySchema,\n});\n\n// TableMetadata contains table-related configuration that does not affect the\n// actual data in the table, but rather how the table's change messages are\n// handled. The is an opaque object that clients must track (and update) based\n// on `create-table`, `add-column`, and `table-update-metadata` messages, and\n// pass in BackfillRequests when there are columns to be backfilled.\n//\n// Note that the backfill-related change-source implementation does, however,\n// rely on the rowKey (columns) being specified in the message.\nexport const tableMetadataSchema = v\n  .object({rowKey: v.record(jsonValueSchema)})\n  .rest(jsonValueSchema);\n\nexport type TableMetadata = v.Infer<typeof tableMetadataSchema>;\n\nexport const rowSchema = v.record(jsonValueSchema);\n\nexport const insertSchema = v.object({\n  tag: v.literal('insert'),\n  relation: relationSchema,\n  new: rowSchema,\n});\n\nexport const updateSchema = v.object({\n  tag: v.literal('update'),\n  relation: relationSchema,\n  // `key` is present if the update changed the key of the row, or if the\n  // table's replicaIdentity === 'full'\n  key: rowSchema.nullable(),\n  // `new` is the full row (and not just the updated columns). This is\n  // necessary for \"catchup\" replication scenarios such as adding tables\n  // to a publication, or resharding.\n  new: rowSchema,\n});\n\nexport const deleteSchema = v.object({\n  tag: v.literal('delete'),\n  relation: relationSchema,\n  // key is the full row if replicaIdentity === 'full'\n  key: rowSchema,\n});\n\nexport const truncateSchema = v.object({\n  tag: v.literal('truncate'),\n  relations: v.array(relationSchema),\n});\n\nexport const identifierSchema = v.object({\n  schema: v.string(),\n  name: v.string(),\n});\n\nexport type Identifier = v.Infer<typeof identifierSchema>;\n\n// A BackfillID is an upstream specific stable identifier for a column\n// that needs backfilling. This id is used to ensure that the schema, table,\n// and column names of a requested backfill still match the original\n// underlying upstream ID.\n//\n// The change-streamer stores these IDs as opaque values while a column is\n// being backfilled, and initiates new change-source streams with the IDs\n// in order to restart backfills that did not complete in previous sessions.\nexport const backfillIDSchema = jsonObjectSchema;\n\nexport type BackfillID = v.Infer<typeof backfillIDSchema>;\n\nexport const createTableSchema = v.object({\n  tag: v.literal('create-table'),\n  spec: tableSpec,\n\n  // This must be set by change source implementations that support\n  // table/column backfill.\n  //\n  // TODO: to simplify the protocol, see if we can make this required\n  metadata: tableMetadataSchema.optional(),\n\n  // Indicate that columns of the table require backfilling. These columns\n  // should be created on the replica but not yet synced the clients.\n  //\n  // ## State Persistence\n  //\n  // To obviate the need for change-source implementations to persist state\n  // related to backfill progress, the change-source only tracks backfills\n  // **for the current session**. In the event that the session is interrupted\n  // before columns have been fully backfilled, it is the responsibility of the\n  // change-streamer to send {@link BackfillRequest}s when it reconnects.\n  //\n  // This means that the change-streamer must track and persist:\n  // * the backfill IDs of the columns requiring backfilling\n  // * the most current table metadata of the associated table(s)\n  //\n  // The change-streamer then uses this information to send backfill requests\n  // when it reconnects.\n  backfill: v.record(backfillIDSchema).optional(),\n});\n\nexport const renameTableSchema = v.object({\n  tag: v.literal('rename-table'),\n  old: identifierSchema,\n  new: identifierSchema,\n});\n\nexport const updateTableMetadataSchema = v.object({\n  tag: v.literal('update-table-metadata'),\n  table: identifierSchema,\n  old: tableMetadataSchema,\n  new: tableMetadataSchema,\n});\n\nconst columnSchema = v.object({\n  name: v.string(),\n  spec: columnSpec,\n});\n\nexport const addColumnSchema = v.object({\n  tag: v.literal('add-column'),\n  table: identifierSchema,\n  column: columnSchema,\n\n  // This must be set by change source implementations that support\n  // table/column backfill.\n  //\n  // TODO: to simplify the protocol, see if we can make this required\n  tableMetadata: tableMetadataSchema.optional(),\n\n  // See documentation for the `backfill` field of the `create-table` change.\n  backfill: backfillIDSchema.optional(),\n});\n\nexport const updateColumnSchema = v.object({\n  tag: v.literal('update-column'),\n  table: identifierSchema,\n  old: columnSchema,\n  new: columnSchema,\n});\n\nexport const dropColumnSchema = v.object({\n  tag: v.literal('drop-column'),\n  table: identifierSchema,\n  column: v.string(),\n});\n\nexport const dropTableSchema = v.object({\n  tag: v.literal('drop-table'),\n  id: identifierSchema,\n});\n\nexport const createIndexSchema = v.object({\n  tag: v.literal('create-index'),\n  spec: indexSpec,\n});\n\nexport const dropIndexSchema = v.object({\n  tag: v.literal('drop-index'),\n  id: identifierSchema,\n});\n\nexport const downloadStatusSchema = v.object({\n  rows: v.number(),\n  totalRows: v.number(),\n  totalBytes: v.number().optional(),\n});\n\nexport type DownloadStatus = v.Infer<typeof downloadStatusSchema>;\n\n// A batch of rows from a single table containing column values\n// to be backfilled.\nexport const backfillSchema = v.object({\n  tag: v.literal('backfill'),\n\n  relation: newRelationSchema,\n\n  // The columns to be backfilled. `rowKey` columns are automatically excluded,\n  // which means that this field may be empty.\n  columns: v.array(v.string()),\n\n  // The watermark at which the backfill data was queried. Note that this\n  // generally will be different from the commit watermarks of the main change\n  // stream, and in particular, the commit watermark of the backfill change's\n  // enclosing transaction.\n  watermark: v.string(),\n\n  // A batch of row values, each row consisting of the `rowKey`\n  // values, followed by the `column` values, in the same order in which\n  // the column names appear in their respective fields, e.g.\n  //\n  // ```\n  // [\n  //   [...rowKeyValues, ...columnValues],  // row 1\n  //   [...rowKeyValues, ...columnValues],  // row 2\n  // ]\n  // ```\n  rowValues: v.array(v.array(jsonValueSchema)),\n\n  // Optionally includes the progress of the backfill operation,\n  // for display purposes.\n  status: downloadStatusSchema.optional(),\n});\n\n// Indicates that the backfill for the specified columns have\n// been successfully backfilled and can be published to clients.\nexport const backfillCompletedSchema = v.object({\n  tag: v.literal('backfill-completed'),\n\n  relation: newRelationSchema,\n\n  // The columns to be backfilled. `rowKey` columns are automatically excluded,\n  // which means that this field may be empty.\n  columns: v.array(v.string()),\n\n  // The watermark at which the backfill data was queried. Note that this\n  // generally will be different from the commit watermarks of the main change\n  // stream, and in particular, the commit watermark of the backfill change's\n  // enclosing transaction.\n  watermark: v.string(),\n\n  // Optionally includes the final status of the backfill operation,\n  // for display purposes.\n  status: downloadStatusSchema.optional(),\n});\n\nexport type MessageBegin = v.Infer<typeof beginSchema>;\nexport type MessageCommit = v.Infer<typeof commitSchema>;\nexport type MessageRollback = v.Infer<typeof rollbackSchema>;\n\nexport type MessageRelation = v.Infer<typeof relationSchema>;\nexport type MessageInsert = v.Infer<typeof insertSchema>;\nexport type MessageUpdate = v.Infer<typeof updateSchema>;\nexport type MessageDelete = v.Infer<typeof deleteSchema>;\nexport type MessageTruncate = v.Infer<typeof truncateSchema>;\n\nexport type MessageBackfill = v.Infer<typeof backfillSchema>;\n\nexport type TableCreate = v.Infer<typeof createTableSchema>;\nexport type TableRename = v.Infer<typeof renameTableSchema>;\nexport type TableUpdateMetadata = v.Infer<typeof updateTableMetadataSchema>;\nexport type ColumnAdd = v.Infer<typeof addColumnSchema>;\nexport type ColumnUpdate = v.Infer<typeof updateColumnSchema>;\nexport type ColumnDrop = v.Infer<typeof dropColumnSchema>;\nexport type TableDrop = v.Infer<typeof dropTableSchema>;\nexport type IndexCreate = v.Infer<typeof createIndexSchema>;\nexport type IndexDrop = v.Infer<typeof dropIndexSchema>;\nexport type BackfillCompleted = v.Infer<typeof backfillCompletedSchema>;\n\nexport const dataChangeSchema = v.union(\n  insertSchema,\n  updateSchema,\n  deleteSchema,\n  truncateSchema,\n  backfillSchema,\n);\n\n// Note: keep in sync or the tag tests will fail\nconst dataChangeTags = [\n  'insert',\n  'update',\n  'delete',\n  'truncate',\n  'backfill',\n] as const;\n\nconst dataChangeTagsSchema = v.literalUnion(...dataChangeTags);\n\nexport type DataChange = Satisfies<\n  JSONObject, // guarantees serialization over IPC or network\n  v.Infer<typeof dataChangeSchema>\n>;\n\nexport type DataChangeTag = v.Infer<typeof dataChangeTagsSchema>;\n\nconst schemaChanges = [\n  createTableSchema,\n  renameTableSchema,\n  updateTableMetadataSchema,\n  addColumnSchema,\n  updateColumnSchema,\n  dropColumnSchema,\n  dropTableSchema,\n  createIndexSchema,\n  dropIndexSchema,\n  backfillCompletedSchema,\n] as const;\n\n// Note: keep in sync or the tag tests will fail\nconst schemaChangeTags = [\n  'create-table',\n  'rename-table',\n  'update-table-metadata',\n  'add-column',\n  'update-column',\n  'drop-column',\n  'drop-table',\n  'create-index',\n  'drop-index',\n  'backfill-completed',\n] as const;\n\nexport const schemaChangeSchema = v.union(...schemaChanges);\n\nconst schemaChangeTagsSchema = v.literalUnion(...schemaChangeTags);\n\nexport type SchemaChange = Satisfies<\n  JSONObject,\n  v.Infer<typeof schemaChangeSchema>\n>;\n\nexport type SchemaChangeTag = v.Infer<typeof schemaChangeTagsSchema>;\n\nexport type DataOrSchemaChange = DataChange | SchemaChange;\n\nexport type Change =\n  | MessageBegin\n  | DataOrSchemaChange\n  | MessageCommit\n  | MessageRollback;\n\nexport type ChangeTag = Change['tag'];\n\nconst schemaChangeTagSet = new Set<string>(schemaChangeTags);\n\nexport function isSchemaChange(change: Change): change is SchemaChange {\n  return schemaChangeTagSet.has(change.tag);\n}\n\nconst dataChangeTagSet = new Set<string>(dataChangeTags);\n\nexport function isDataChange(change: Change): change is DataChange {\n  return dataChangeTagSet.has(change.tag);\n}\n"],"mappings":";;;;;;;;;;;AAgBA,IAAa,cAAc,eAAE,OAAO;CAClC,KAAK,eAAE,QAAQ,QAAQ;CAWvB,MAAM,aAAe,KAAK,IAAI,CAAC,UAAU;CAGzC,SAAS,eAAE,SAAS,CAAC,UAAU;CAChC,CAAC;AAEF,IAAa,eAAe,eAAE,OAAO,EACnC,KAAK,eAAE,QAAQ,SAAS,EACzB,CAAC;AAEF,IAAa,iBAAiB,eAAE,OAAO,EACrC,KAAK,eAAE,QAAQ,WAAW,EAC3B,CAAC;AAEF,IAAM,eAAe,eAAE,OAAO;CAE5B,SAAS,eAAE,MAAM,eAAE,QAAQ,CAAC;CAO5B,MAAM,aAAe,WAAW,WAAW,QAAQ,QAAQ,CAAC,UAAU;CACvE,CAAC;AAEF,IAAa,iBAAiB,eAC3B,OAAO;CACN,QAAQ,eAAE,QAAQ;CAClB,MAAM,eAAE,QAAQ;CAGhB,QAAQ,aAAa,UAAU;CAG/B,YAAY,eAAE,MAAM,eAAE,QAAQ,CAAC,CAAC,UAAU;CAE1C,iBAAiB,aACD,WAAW,WAAW,QAAQ,QAAQ,CACnD,UAAU;CACd,CAAC,CACD,KAAI,QAAO;CACV,MAAM,EAAC,QAAQ,GAAG,SAAQ;AAC1B,KAAI,OACF,QAAO;EAAC,GAAG;EAAM;EAAO;AAE1B,QAAO;EACL,GAAG;EACH,QAAQ;GACN,SAAS,KAAK,IAAI,WAAW;GAC7B,MAAM,IAAI;GACX;EACF;EACD;AAGJ,IAAa,oBAAoB,eAAE,OAAO;CACxC,QAAQ,eAAE,QAAQ;CAClB,MAAM,eAAE,QAAQ;CAEhB,QAAQ;CACT,CAAC;AAUF,IAAa,sBAAsB,eAChC,OAAO,EAAC,QAAQ,eAAE,OAAO,gBAAgB,EAAC,CAAC,CAC3C,KAAK,gBAAgB;AAIxB,IAAa,YAAY,eAAE,OAAO,gBAAgB;AAElD,IAAa,eAAe,eAAE,OAAO;CACnC,KAAK,eAAE,QAAQ,SAAS;CACxB,UAAU;CACV,KAAK;CACN,CAAC;AAEF,IAAa,eAAe,eAAE,OAAO;CACnC,KAAK,eAAE,QAAQ,SAAS;CACxB,UAAU;CAGV,KAAK,UAAU,UAAU;CAIzB,KAAK;CACN,CAAC;AAEF,IAAa,eAAe,eAAE,OAAO;CACnC,KAAK,eAAE,QAAQ,SAAS;CACxB,UAAU;CAEV,KAAK;CACN,CAAC;AAEF,IAAa,iBAAiB,eAAE,OAAO;CACrC,KAAK,eAAE,QAAQ,WAAW;CAC1B,WAAW,eAAE,MAAM,eAAe;CACnC,CAAC;AAEF,IAAa,mBAAmB,eAAE,OAAO;CACvC,QAAQ,eAAE,QAAQ;CAClB,MAAM,eAAE,QAAQ;CACjB,CAAC;AAYF,IAAa,mBAAmB;AAIhC,IAAa,oBAAoB,eAAE,OAAO;CACxC,KAAK,eAAE,QAAQ,eAAe;CAC9B,MAAM;CAMN,UAAU,oBAAoB,UAAU;CAmBxC,UAAU,eAAE,OAAO,iBAAiB,CAAC,UAAU;CAChD,CAAC;AAEF,IAAa,oBAAoB,eAAE,OAAO;CACxC,KAAK,eAAE,QAAQ,eAAe;CAC9B,KAAK;CACL,KAAK;CACN,CAAC;AAEF,IAAa,4BAA4B,eAAE,OAAO;CAChD,KAAK,eAAE,QAAQ,wBAAwB;CACvC,OAAO;CACP,KAAK;CACL,KAAK;CACN,CAAC;AAEF,IAAM,eAAe,eAAE,OAAO;CAC5B,MAAM,eAAE,QAAQ;CAChB,MAAM;CACP,CAAC;AAEF,IAAa,kBAAkB,eAAE,OAAO;CACtC,KAAK,eAAE,QAAQ,aAAa;CAC5B,OAAO;CACP,QAAQ;CAMR,eAAe,oBAAoB,UAAU;CAG7C,UAAU,iBAAiB,UAAU;CACtC,CAAC;AAEF,IAAa,qBAAqB,eAAE,OAAO;CACzC,KAAK,eAAE,QAAQ,gBAAgB;CAC/B,OAAO;CACP,KAAK;CACL,KAAK;CACN,CAAC;AAEF,IAAa,mBAAmB,eAAE,OAAO;CACvC,KAAK,eAAE,QAAQ,cAAc;CAC7B,OAAO;CACP,QAAQ,eAAE,QAAQ;CACnB,CAAC;AAEF,IAAa,kBAAkB,eAAE,OAAO;CACtC,KAAK,eAAE,QAAQ,aAAa;CAC5B,IAAI;CACL,CAAC;AAEF,IAAa,oBAAoB,eAAE,OAAO;CACxC,KAAK,eAAE,QAAQ,eAAe;CAC9B,MAAM;CACP,CAAC;AAEF,IAAa,kBAAkB,eAAE,OAAO;CACtC,KAAK,eAAE,QAAQ,aAAa;CAC5B,IAAI;CACL,CAAC;AAEF,IAAa,uBAAuB,eAAE,OAAO;CAC3C,MAAM,eAAE,QAAQ;CAChB,WAAW,eAAE,QAAQ;CACrB,YAAY,eAAE,QAAQ,CAAC,UAAU;CAClC,CAAC;AAMF,IAAa,iBAAiB,eAAE,OAAO;CACrC,KAAK,eAAE,QAAQ,WAAW;CAE1B,UAAU;CAIV,SAAS,eAAE,MAAM,eAAE,QAAQ,CAAC;CAM5B,WAAW,eAAE,QAAQ;CAYrB,WAAW,eAAE,MAAM,eAAE,MAAM,gBAAgB,CAAC;CAI5C,QAAQ,qBAAqB,UAAU;CACxC,CAAC;AAIF,IAAa,0BAA0B,eAAE,OAAO;CAC9C,KAAK,eAAE,QAAQ,qBAAqB;CAEpC,UAAU;CAIV,SAAS,eAAE,MAAM,eAAE,QAAQ,CAAC;CAM5B,WAAW,eAAE,QAAQ;CAIrB,QAAQ,qBAAqB,UAAU;CACxC,CAAC;AAyBF,IAAa,mBAAmB,eAAE,MAChC,cACA,cACA,cACA,gBACA,eACD;AAGD,IAAM,iBAAiB;CACrB;CACA;CACA;CACA;CACA;CACD;AAE4B,aAAe,GAAG,eAAe;AAS9D,IAAM,gBAAgB;CACpB;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACD;AAGD,IAAM,mBAAmB;CACvB;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACD;AAED,IAAa,qBAAqB,eAAE,MAAM,GAAG,cAAc;AAE5B,aAAe,GAAG,iBAAiB;AAmBlE,IAAM,qBAAqB,IAAI,IAAY,iBAAiB;AAE5D,SAAgB,eAAe,QAAwC;AACrE,QAAO,mBAAmB,IAAI,OAAO,IAAI;;AAG3C,IAAM,mBAAmB,IAAI,IAAY,eAAe;AAExD,SAAgB,aAAa,QAAsC;AACjE,QAAO,iBAAiB,IAAI,OAAO,IAAI"}

package/out/zero-cache/src/services/change-source/protocol/current/downstream.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"downstream.js","names":[],"sources":["../../../../../../../../zero-cache/src/services/change-source/protocol/current/downstream.ts"],"sourcesContent":["import * as v from '../../../../../../shared/src/valita.ts';\nimport {resetRequiredSchema} from './control.ts';\nimport {\n  beginSchema,\n  commitSchema,\n  dataChangeSchema,\n  rollbackSchema,\n  schemaChangeSchema,\n} from './data.ts';\nimport {downstreamStatusMessageSchema} from './status.ts';\n\nconst begin = v.tuple([\n  v.literal('begin'),\n  beginSchema,\n  v.object({commitWatermark: v.string()}),\n]);\nconst data = v.tuple([\n  v.literal('data'),\n  v.union(dataChangeSchema, schemaChangeSchema),\n]);\nconst commit = v.tuple([\n  v.literal('commit'),\n  commitSchema,\n  v.object({watermark: v.string()}),\n]);\nconst rollback = v.tuple([v.literal('rollback'), rollbackSchema]);\n\nexport type Begin = v.Infer<typeof begin>;\nexport type Data = v.Infer<typeof data>;\nexport type Commit = v.Infer<typeof commit>;\nexport type Rollback = v.Infer<typeof rollback>;\n\nexport const changeStreamDataSchema = v.union(begin, data, commit, rollback);\nexport type ChangeStreamData = v.Infer<typeof changeStreamDataSchema>;\n\nexport const changeStreamControlSchema = v.tuple([\n  v.literal('control'),\n  resetRequiredSchema, // TODO: Add statusRequestedSchema\n]);\nexport type ChangeStreamControl = v.Infer<typeof changeStreamControlSchema>;\n\n/** Downstream messages consist of data plane and control plane messages. */\nexport const changeStreamMessageSchema = v.union(\n  changeStreamDataSchema,\n  changeStreamControlSchema,\n  downstreamStatusMessageSchema,\n);\n\nexport type ChangeStreamMessage = v.Infer<typeof changeStreamMessageSchema>;\n"],"mappings":";;;;;AAWA,IAAM,QAAQ,eAAE,MAAM;CACpB,eAAE,QAAQ,QAAQ;CAClB;CACA,eAAE,OAAO,EAAC,iBAAiB,eAAE,QAAQ,EAAC,CAAC;CACxC,CAAC;AACF,IAAM,OAAO,eAAE,MAAM,CACnB,eAAE,QAAQ,OAAO,EACjB,eAAE,MAAM,kBAAkB,mBAAmB,CAC9C,CAAC;AACF,IAAM,SAAS,eAAE,MAAM;CACrB,eAAE,QAAQ,SAAS;CACnB;CACA,eAAE,OAAO,EAAC,WAAW,eAAE,QAAQ,EAAC,CAAC;CAClC,CAAC;AACF,IAAM,WAAW,eAAE,MAAM,CAAC,eAAE,QAAQ,WAAW,EAAE,eAAe,CAAC;AAOjE,IAAa,yBAAyB,eAAE,MAAM,OAAO,MAAM,QAAQ,SAAS;AAG5E,IAAa,4BAA4B,eAAE,MAAM,CAC/C,eAAE,QAAQ,UAAU,EACpB,oBACD,CAAC;;AAIF,IAAa,4BAA4B,eAAE,MACzC,wBACA,2BACA,8BACD"}

package/out/zero-cache/src/services/change-source/protocol/current/json.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"json.js","names":[],"sources":["../../../../../../../../zero-cache/src/services/change-source/protocol/current/json.ts"],"sourcesContent":["import * as v from '@badrap/valita';\n\nexport type JSONValue =\n  | null\n  | string\n  | boolean\n  | number\n  | Array<JSONValue>\n  | JSONObject;\n\nexport type JSONObject = {[key: string]: JSONValue | undefined};\n\nexport const jsonValueSchema: v.Type<JSONValue> = v.lazy(() =>\n  v.union(\n    v.null(),\n    v.string(),\n    v.boolean(),\n    v.number(),\n    v.array(jsonValueSchema),\n    jsonObjectSchema,\n  ),\n);\n\nexport const jsonObjectSchema = v.record(\n  v.union(jsonValueSchema, v.undefined()),\n);\n"],"mappings":";;AAYA,IAAa,kBAAqC,EAAE,WAClD,EAAE,MACA,EAAE,MAAM,EACR,EAAE,QAAQ,EACV,EAAE,SAAS,EACX,EAAE,QAAQ,EACV,EAAE,MAAM,gBAAgB,EACxB,iBACD,CACF;AAED,IAAa,mBAAmB,EAAE,OAChC,EAAE,MAAM,iBAAiB,EAAE,WAAW,CAAC,CACxC"}

package/out/zero-cache/src/services/change-source/protocol/current/path.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"path.js","names":[],"sources":["../../../../../../../../zero-cache/src/services/change-source/protocol/current/path.ts"],"sourcesContent":["/**\n * The path required for a custom Change Source endpoint implementing the\n * Change Source protocol. The version in the path indicates the current\n * (i.e. latest) protocol version of the code, and is the only protocol\n * supported by the code.\n *\n * Eventually, when a backwards incompatible change is made to the protocol,\n * the version will be bumped to ensure that the protocol is only used for an\n * endpoint that explicitly understands it. (While the protocol is in flux\n * and being developed, a starting \"v0\" version will not follow this\n * convention.)\n *\n * Historic versions are kept in the source code (e.g. v1, v2, etc.) to\n * allow Change Source implementations to import and support multiple\n * versions simultaneously. This is necessary to seamlessly transitioning\n * from a `zero-cache` speaking one version to a `zero-cache` speaking\n * another.\n */\nexport const CHANGE_SOURCE_PATH = '/changes/v0/stream';\n"],"mappings":";;;;;;;;;;;;;;;;;;;AAkBA,IAAa,qBAAqB"}

package/out/zero-cache/src/services/change-source/protocol/current/status.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"status.js","names":[],"sources":["../../../../../../../../zero-cache/src/services/change-source/protocol/current/status.ts"],"sourcesContent":["import * as v from '../../../../../../shared/src/valita.ts';\nimport {changeSourceReportSchema} from '../../../replicator/reporter/report-schema.ts';\nimport {commitSchema} from './data.ts';\n\n/**\n * The downstream status messages contain metadata about the status\n * of the change-source. indicates whether it should be echoed\n * back in an upstream status message.\n */\nexport const downstreamStatusSchema = v.object({\n  // Indicates whether the status message should be echoed back\n  // in an upstream status message once the consumer has successfully\n  // processed/persisted all preceding changes.\n  ack: v.boolean().optional(() => true),\n\n  // Contains a lag report for recording end to end latency metrics.\n  lagReport: changeSourceReportSchema.optional(),\n});\n\nexport type DownstreamStatus = v.Infer<typeof downstreamStatusSchema>;\n\nexport const downstreamStatusMessageSchema = v.tuple([\n  v.literal('status'),\n  downstreamStatusSchema,\n  v.object({watermark: v.string()}),\n]);\n\n/**\n * The `zero-cache` will send the Commit payload to acknowledge a completed\n * transaction (unless the `skipAck` field was specified in the Begin message\n * of the transaction), and will echo back the downstream `status` message if\n * `ack` is true.\n */\nexport const upstreamStatusMessageSchema = v.tuple([\n  v.literal('status'),\n  v.union(downstreamStatusSchema, commitSchema),\n  v.object({watermark: v.string()}),\n]);\n\n/**\n * Status messages convey positional information from both the ChangeSource\n * and the `zero-cache`.\n *\n * A StatusMessage from the ChangeSource indicates a position in its change\n * log. Generally, the watermarks sent in `Commit` messages already convey\n * this information, but a StatusMessage may also be sent to indicate that the\n * log has progressed without any corresponding changes relevant to the\n * subscriber. The watermarks of commit messages and status messages must be\n * monotonic in the stream of messages from the ChangeSource.\n *\n * The `zero-cache` sends StatusMessages to the ChangeSource:\n *\n * * when it has processed a `Commit` received from the ChangeSource,\n *   unless the `Begin` message specified `skipAck`.\n *\n * * when it receives a `StatusMessage` and all preceding `Commit` messages\n *   have been processed\n *\n * This allows the ChangeSource to clean up change log entries appropriately.\n *\n * Note that StatusMessages from the ChangeSource are optional. If a\n * ChangeSource implementation can track subscriber progress and clean up\n * its change log purely from Commit-driven StatusMessages there is no need\n * for the ChangeSource to send StatusMessages.\n */\nexport type DownstreamStatusMessage = v.Infer<\n  typeof downstreamStatusMessageSchema\n>;\nexport type UpstreamStatusMessage = v.Infer<typeof upstreamStatusMessageSchema>;\n"],"mappings":";;;;;;;;;AASA,IAAa,yBAAyB,eAAE,OAAO;CAI7C,KAAK,eAAE,SAAS,CAAC,eAAe,KAAK;CAGrC,WAAW,yBAAyB,UAAU;CAC/C,CAAC;AAIF,IAAa,gCAAgC,eAAE,MAAM;CACnD,eAAE,QAAQ,SAAS;CACnB;CACA,eAAE,OAAO,EAAC,WAAW,eAAE,QAAQ,EAAC,CAAC;CAClC,CAAC;;;;;;;AAQF,IAAa,8BAA8B,eAAE,MAAM;CACjD,eAAE,QAAQ,SAAS;CACnB,eAAE,MAAM,wBAAwB,aAAa;CAC7C,eAAE,OAAO,EAAC,WAAW,eAAE,QAAQ,EAAC,CAAC;CAClC,CAAC"}

package/out/zero-cache/src/services/change-source/protocol/current/upstream.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"upstream.js","names":[],"sources":["../../../../../../../../zero-cache/src/services/change-source/protocol/current/upstream.ts"],"sourcesContent":["import * as v from '../../../../../../shared/src/valita.ts';\nimport {\n  backfillIDSchema,\n  identifierSchema,\n  tableMetadataSchema,\n} from './data.ts';\nimport {upstreamStatusMessageSchema} from './status.ts';\n\n/** At the moment, the only upstream messages are status messages.  */\nexport const changeSourceUpstreamSchema = upstreamStatusMessageSchema;\nexport type ChangeSourceUpstream = v.Infer<typeof changeSourceUpstreamSchema>;\n\n/**\n * Contains the information for requesting a backfill of columns in a table.\n * Backfills are automatically started for new tables and columns in a given\n * change stream session; however, if the session is terminated before the\n * backfill completes, it must be restarted with appropriate\n * {@link BackfillRequest}s when creating a new session.\n *\n * The `change-streamer` is responsible for tracking any changes to the table\n * name, column names, or table metadata, and constructing a BackfillRequest\n * based on the current values (which may be different from when the\n * tables/columns were originally added).\n */\nexport const backfillRequestSchema = v.object({\n  table: identifierSchema.extend({\n    // The table metadata is set to null if it is never specified by the\n    // change-source.\n    metadata: tableMetadataSchema.nullable(),\n  }),\n  columns: v.record(backfillIDSchema),\n});\n\nexport type BackfillRequest = v.Infer<typeof backfillRequestSchema>;\n"],"mappings":";;;;;AASA,IAAa,6BAA6B;;;;;;;;;;;;;AAe1C,IAAa,wBAAwB,eAAE,OAAO;CAC5C,OAAO,iBAAiB,OAAO,EAG7B,UAAU,oBAAoB,UAAU,EACzC,CAAC;CACF,SAAS,eAAE,OAAO,iBAAiB;CACpC,CAAC"}

package/out/zero-cache/src/services/change-source/protocol/current.js

@@ -1,56 +0,0 @@
-import { __exportAll } from "../../../../../_virtual/_rolldown/runtime.js";
-import { jsonObjectSchema, jsonValueSchema } from "./current/json.js";
-import { resetRequiredSchema } from "./current/control.js";
-import { addColumnSchema, backfillCompletedSchema, backfillIDSchema, backfillSchema, beginSchema, commitSchema, createIndexSchema, createTableSchema, dataChangeSchema, deleteSchema, downloadStatusSchema, dropColumnSchema, dropIndexSchema, dropTableSchema, identifierSchema, insertSchema, isDataChange, isSchemaChange, newRelationSchema, relationSchema, renameTableSchema, rollbackSchema, rowSchema, schemaChangeSchema, tableMetadataSchema, truncateSchema, updateColumnSchema, updateSchema, updateTableMetadataSchema } from "./current/data.js";
-import { downstreamStatusMessageSchema, downstreamStatusSchema, upstreamStatusMessageSchema } from "./current/status.js";
-import { changeStreamControlSchema, changeStreamDataSchema, changeStreamMessageSchema } from "./current/downstream.js";
-import { CHANGE_SOURCE_PATH } from "./current/path.js";
-import { backfillRequestSchema, changeSourceUpstreamSchema } from "./current/upstream.js";
-//#region ../zero-cache/src/services/change-source/protocol/current.ts
-var current_exports = /* @__PURE__ */ __exportAll({
-	CHANGE_SOURCE_PATH: () => CHANGE_SOURCE_PATH,
-	addColumnSchema: () => addColumnSchema,
-	backfillCompletedSchema: () => backfillCompletedSchema,
-	backfillIDSchema: () => backfillIDSchema,
-	backfillRequestSchema: () => backfillRequestSchema,
-	backfillSchema: () => backfillSchema,
-	beginSchema: () => beginSchema,
-	changeSourceUpstreamSchema: () => changeSourceUpstreamSchema,
-	changeStreamControlSchema: () => changeStreamControlSchema,
-	changeStreamDataSchema: () => changeStreamDataSchema,
-	changeStreamMessageSchema: () => changeStreamMessageSchema,
-	commitSchema: () => commitSchema,
-	createIndexSchema: () => createIndexSchema,
-	createTableSchema: () => createTableSchema,
-	dataChangeSchema: () => dataChangeSchema,
-	deleteSchema: () => deleteSchema,
-	downloadStatusSchema: () => downloadStatusSchema,
-	downstreamStatusMessageSchema: () => downstreamStatusMessageSchema,
-	downstreamStatusSchema: () => downstreamStatusSchema,
-	dropColumnSchema: () => dropColumnSchema,
-	dropIndexSchema: () => dropIndexSchema,
-	dropTableSchema: () => dropTableSchema,
-	identifierSchema: () => identifierSchema,
-	insertSchema: () => insertSchema,
-	isDataChange: () => isDataChange,
-	isSchemaChange: () => isSchemaChange,
-	jsonObjectSchema: () => jsonObjectSchema,
-	jsonValueSchema: () => jsonValueSchema,
-	newRelationSchema: () => newRelationSchema,
-	relationSchema: () => relationSchema,
-	renameTableSchema: () => renameTableSchema,
-	resetRequiredSchema: () => resetRequiredSchema,
-	rollbackSchema: () => rollbackSchema,
-	rowSchema: () => rowSchema,
-	schemaChangeSchema: () => schemaChangeSchema,
-	tableMetadataSchema: () => tableMetadataSchema,
-	truncateSchema: () => truncateSchema,
-	updateColumnSchema: () => updateColumnSchema,
-	updateSchema: () => updateSchema,
-	updateTableMetadataSchema: () => updateTableMetadataSchema,
-	upstreamStatusMessageSchema: () => upstreamStatusMessageSchema
-});
-//#endregion
-export { current_exports };
-
-//# sourceMappingURL=current.js.map

package/out/zero-cache/src/services/change-source/protocol/current.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"current.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/protocol/current.ts"],"sourcesContent":["export * from './current/control.ts';\nexport * from './current/data.ts';\nexport * from './current/downstream.ts';\nexport * from './current/json.ts';\nexport * from './current/path.ts';\nexport * from './current/status.ts';\nexport * from './current/upstream.ts';\n"],"mappings":""}

package/out/zero-cache/src/services/change-streamer/backup-monitor.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"backup-monitor.js","names":["#lc","#replicaFile","#backupURL","#metricsEndpoint","#changeStreamer","#state","#reservations","#watermarks","#cleanupDelayMs","#checkMetricsTimer","#initBackupLagMetric","#checkWatermarks","#scheduleCleanup","#fetchWatermarks","#lastWatermark","#latestBackupTime"],"sources":["../../../../../../zero-cache/src/services/change-streamer/backup-monitor.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport parsePrometheusTextFormat from 'parse-prometheus-text-format';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {getOrCreateGauge} from '../../observability/metrics.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport {RunningState} from '../running-state.ts';\nimport type {Service} from '../service.ts';\nimport type {ChangeStreamerService} from './change-streamer.ts';\nimport type {SnapshotMessage} from './snapshot.ts';\n\nexport const CHECK_INTERVAL_MS = 60_000;\nconst MIN_CLEANUP_DELAY_MS = 30_000;\n\ntype Reservation = {\n  start: Date;\n  sub: Subscription<SnapshotMessage>;\n};\n\n/**\n * The BackupMonitor polls the litestream \"/metrics\" endpoint to track the\n * watermark (label) value of the `litestream_replica_progress` gauge and\n * schedules cleanup of change log entries that can be purged as a result.\n *\n * See: https://github.com/rocicorp/litestream/pull/3\n *\n * Note that change log entries cannot simply be purged as soon as they\n * have been applied and backed up by litestream. Consider the case in which\n * litestream backs up new wal segments every minute, but it takes 5 minutes\n * to restore a replica: if a zero-cache starts restoring a replica at\n * minute 0, and new watermarks are replicated at minutes 1, 2, 3, 4, and 5,\n * purging changelog records as soon as those watermarks are replicated would\n * result in the zero-cache not being able to catch up from minute 0 once it\n * has finished restoring the replica.\n *\n * The `/snapshot` reservation protocol is used to prevent premature change\n * log cleanup:\n * - Clients restoring a snapshot initiate a `/snapshot` request and hold that\n *   request open while it restores its snapshot, prepares it, and\n *   starts its subscription to the change stream. During this time, no\n *   cleanups are scheduled.\n * - When the subscription is started, the interval since the beginning of\n *   of the reservation is tracked to increase the background cleanup delay\n *   interval if needed. The reservation is ended (and request closed), and\n *   cleanup scheduling is resumed with the current delay interval.\n *\n * Note that the reservation request is the primary mechanism by which\n * premature change log cleanup is prevented. The cleanup delay interval is\n * a secondary safeguard.\n */\nexport class BackupMonitor implements Service {\n  readonly id = 'backup-monitor';\n  readonly #lc: LogContext;\n  readonly #replicaFile: string;\n  readonly #backupURL: string;\n  readonly #metricsEndpoint: string;\n  readonly #changeStreamer: ChangeStreamerService;\n  readonly #state = new RunningState(this.id);\n\n  readonly #reservations = new Map<string, Reservation>();\n  readonly #watermarks = new Map<string, Date>();\n\n  #lastWatermark: string = '';\n  #latestBackupTime: Date | null = null;\n  #cleanupDelayMs: number;\n  #checkMetricsTimer: NodeJS.Timeout | undefined;\n\n  constructor(\n    lc: LogContext,\n    replicaFile: string,\n    backupURL: string,\n    metricsEndpoint: string,\n    changeStreamer: ChangeStreamerService,\n    initialCleanupDelayMs: number,\n  ) {\n    this.#lc = lc.withContext('component', this.id);\n    this.#replicaFile = replicaFile;\n    this.#backupURL = backupURL;\n    this.#metricsEndpoint = metricsEndpoint;\n    this.#changeStreamer = changeStreamer;\n    this.#cleanupDelayMs = Math.max(\n      initialCleanupDelayMs,\n      MIN_CLEANUP_DELAY_MS, // purely for peace of mind\n    );\n\n    this.#lc.info?.(\n      `backup monitor started ${initialCleanupDelayMs} ms after snapshot restore`,\n    );\n  }\n\n  run(): Promise<void> {\n    this.#lc.info?.(\n      `monitoring backups at ${this.#metricsEndpoint} with ` +\n        `${this.#cleanupDelayMs} ms cleanup delay`,\n    );\n    this.#checkMetricsTimer = setInterval(\n      this.checkWatermarksAndScheduleCleanup,\n      CHECK_INTERVAL_MS,\n    );\n    this.#initBackupLagMetric();\n    return this.#state.stopped();\n  }\n\n  startSnapshotReservation(taskID: string): Subscription<SnapshotMessage> {\n    this.#lc.info?.(`pausing change-log cleanup while ${taskID} snapshots`);\n    // In the case of retries, only track the last reservation.\n    this.#reservations.get(taskID)?.sub.cancel();\n\n    const sub = Subscription.create<SnapshotMessage>({\n      // If the reservation still exists when the connection closes\n      // (e.g. subscriber crashed), clean it up without updating the\n      // cleanup delay.\n      cleanup: () => this.endReservation(taskID, false),\n    });\n    this.#reservations.set(taskID, {start: new Date(), sub});\n    // Note: the Subscription must be returned immediately so that the\n    //       websocket can begin sending liveness pings.\n    void this.#changeStreamer\n      .getChangeLogState()\n      .then(changeLogState => {\n        sub.push([\n          'status',\n          {tag: 'status', backupURL: this.#backupURL, ...changeLogState},\n        ]);\n      })\n      .catch(e => {\n        this.#lc.warn?.(`failing snapshot reservation`, e);\n        sub.fail(e);\n      });\n    return sub;\n  }\n\n  endReservation(taskID: string, updateCleanupDelay = true) {\n    const res = this.#reservations.get(taskID);\n    if (res === undefined) {\n      return;\n    }\n    this.#reservations.delete(taskID);\n    const {start, sub} = res;\n    sub.cancel(); // closes the connection if still open\n\n    if (updateCleanupDelay) {\n      const duration = Date.now() - start.getTime();\n      this.#lc.info?.(`snapshot initialized by ${taskID} in ${duration} ms`);\n      if (duration > this.#cleanupDelayMs) {\n        this.#cleanupDelayMs = duration;\n        this.#lc.info?.(`increased cleanup delay to ${duration} ms`);\n      }\n    }\n  }\n\n  // Exported for testing\n  readonly checkWatermarksAndScheduleCleanup = async () => {\n    try {\n      await this.#checkWatermarks();\n    } catch (e) {\n      this.#lc.warn?.(`unable to fetch metrics at ${this.#metricsEndpoint}`, e);\n    }\n    try {\n      this.#scheduleCleanup();\n    } catch (e) {\n      this.#lc.warn?.(`error scheduling cleanup`, e);\n    }\n  };\n\n  async *#fetchWatermarks(): AsyncGenerator<{\n    watermark: string;\n    time: Date;\n    name?: string | undefined;\n  }> {\n    const metricsEndpoint = this.#metricsEndpoint;\n    const signal = this.#state.signal;\n    let resp;\n    try {\n      resp = await fetch(metricsEndpoint, {signal});\n    } catch (e) {\n      if (signal.aborted) {\n        // not an error.\n        return;\n      }\n      // Treat exceptions from fetch (e.g. network errors) as non-fatal, and simply\n      // log them and skip the watermark check until the next interval.\n      this.#lc.warn?.(`unable to fetch metrics at ${this.#metricsEndpoint}`, e);\n      return;\n    }\n    if (!resp.ok) {\n      this.#lc.warn?.(\n        `unable to fetch metrics at ${this.#metricsEndpoint}: ${await resp.text()}`,\n      );\n      return;\n    }\n\n    const families = parsePrometheusTextFormat(await resp.text());\n    for (const family of families) {\n      if (\n        family.type === 'GAUGE' &&\n        family.name === 'litestream_replica_progress'\n      ) {\n        for (const metric of family.metrics) {\n          const watermark = metric.labels?.watermark;\n          const name = metric.labels?.name;\n          const time = new Date(parseFloat(metric.value) * 1000);\n\n          if (watermark) {\n            yield {watermark, time, name};\n          }\n        }\n      }\n    }\n  }\n\n  async #checkWatermarks() {\n    for await (const {watermark, name, time} of this.#fetchWatermarks()) {\n      if (watermark > this.#lastWatermark && !this.#watermarks.has(watermark)) {\n        this.#lc.info?.(\n          `replicated watermark=${watermark} to ${name}` +\n            ` at ${time.toISOString()}.`,\n        );\n        this.#watermarks.set(watermark, time);\n        this.#latestBackupTime = time;\n      }\n    }\n    return this.#latestBackupTime;\n  }\n\n  #scheduleCleanup() {\n    if (this.#reservations.size > 0) {\n      this.#lc.info?.(\n        `watermark cleanup paused for snapshot(s): ${[...this.#reservations.keys()]}`,\n      );\n      return;\n    }\n    const latestCleanupTime = Date.now() - this.#cleanupDelayMs;\n    let maxWatermark = '';\n    for (const [watermark, backupTime] of this.#watermarks.entries()) {\n      if (\n        backupTime.getTime() <= latestCleanupTime &&\n        watermark > maxWatermark\n      ) {\n        maxWatermark = watermark;\n      }\n    }\n    if (maxWatermark.length) {\n      this.#changeStreamer.scheduleCleanup(maxWatermark);\n      for (const watermark of this.#watermarks.keys()) {\n        if (watermark <= maxWatermark) {\n          this.#watermarks.delete(watermark);\n        }\n      }\n      this.#lastWatermark = maxWatermark;\n    }\n  }\n\n  stop(): Promise<void> {\n    clearInterval(this.#checkMetricsTimer);\n    for (const {sub} of this.#reservations.values()) {\n      // Close any pending reservations. This commonly happens when a new\n      // replication-manager makes a `/snapshot` reservation on the existing\n      // replication-manager, and then shuts it down when it takes over the\n      // replication slot.\n      sub.cancel();\n    }\n    this.#state.stop(this.#lc);\n    return promiseVoid;\n  }\n\n  #initBackupLagMetric() {\n    getOrCreateGauge('replica', 'backup_lag', {\n      description:\n        'Latency from when a change is written to the replica ' +\n        'to when it is backed up to litestream. It is expected to create a saw ' +\n        'pattern from 0 to the configured ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_MINUTES.',\n      unit: 'millisecond',\n    }).addCallback(async o => {\n      // For legacy litestream, we use the watermark metric (and its associated\n      // backup time) exported by litestream metrics to determine the time of\n      // of the backed up watermark. This is technically imprecise--it would be\n      // more correct to use the committed writeTimeMs--but it is good enough\n      // in that it serves the purpose of detecting a non-functioning backup.\n      // With litestream v5, this can be made more precise by querying the\n      // _zero.replicationState row from the backup directly using an LTX-based\n      // database reader.\n      const latestBackup = await this.#checkWatermarks();\n      if (!latestBackup) {\n        this.#lc.warn?.(\n          `no backed up watermarks. unable to report replica.backup_lag`,\n        );\n        return;\n      }\n      const db = new Database(this.#lc, this.#replicaFile, {readonly: true});\n      try {\n        const {writeTimeMs} = db\n          .prepare(/*sql*/ `SELECT writeTimeMs FROM \"_zero.replicationState\"`)\n          .get<{writeTimeMs: number}>();\n        const backupLag = Math.max(0, writeTimeMs - latestBackup.getTime());\n        o.observe(backupLag);\n      } catch (e) {\n        this.#lc.warn?.(`error measuring replica.backup_lag metric`, e);\n      } finally {\n        db.close();\n      }\n    });\n  }\n}\n"],"mappings":";;;;;;;AAWA,IAAa,oBAAoB;AACjC,IAAM,uBAAuB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAsC7B,IAAa,gBAAb,MAA8C;CAC5C,KAAc;CACd;CACA;CACA;CACA;CACA;CACA,SAAkB,IAAI,aAAa,KAAK,GAAG;CAE3C,gCAAyB,IAAI,KAA0B;CACvD,8BAAuB,IAAI,KAAmB;CAE9C,iBAAyB;CACzB,oBAAiC;CACjC;CACA;CAEA,YACE,IACA,aACA,WACA,iBACA,gBACA,uBACA;AACA,QAAA,KAAW,GAAG,YAAY,aAAa,KAAK,GAAG;AAC/C,QAAA,cAAoB;AACpB,QAAA,YAAkB;AAClB,QAAA,kBAAwB;AACxB,QAAA,iBAAuB;AACvB,QAAA,iBAAuB,KAAK,IAC1B,uBACA,qBACD;AAED,QAAA,GAAS,OACP,0BAA0B,sBAAsB,4BACjD;;CAGH,MAAqB;AACnB,QAAA,GAAS,OACP,yBAAyB,MAAA,gBAAsB,QAC1C,MAAA,eAAqB,mBAC3B;AACD,QAAA,oBAA0B,YACxB,KAAK,mCACL,kBACD;AACD,QAAA,qBAA2B;AAC3B,SAAO,MAAA,MAAY,SAAS;;CAG9B,yBAAyB,QAA+C;AACtE,QAAA,GAAS,OAAO,oCAAoC,OAAO,YAAY;AAEvE,QAAA,aAAmB,IAAI,OAAO,EAAE,IAAI,QAAQ;EAE5C,MAAM,MAAM,aAAa,OAAwB,EAI/C,eAAe,KAAK,eAAe,QAAQ,MAAM,EAClD,CAAC;AACF,QAAA,aAAmB,IAAI,QAAQ;GAAC,uBAAO,IAAI,MAAM;GAAE;GAAI,CAAC;AAGnD,QAAA,eACF,mBAAmB,CACnB,MAAK,mBAAkB;AACtB,OAAI,KAAK,CACP,UACA;IAAC,KAAK;IAAU,WAAW,MAAA;IAAiB,GAAG;IAAe,CAC/D,CAAC;IACF,CACD,OAAM,MAAK;AACV,SAAA,GAAS,OAAO,gCAAgC,EAAE;AAClD,OAAI,KAAK,EAAE;IACX;AACJ,SAAO;;CAGT,eAAe,QAAgB,qBAAqB,MAAM;EACxD,MAAM,MAAM,MAAA,aAAmB,IAAI,OAAO;AAC1C,MAAI,QAAQ,KAAA,EACV;AAEF,QAAA,aAAmB,OAAO,OAAO;EACjC,MAAM,EAAC,OAAO,QAAO;AACrB,MAAI,QAAQ;AAEZ,MAAI,oBAAoB;GACtB,MAAM,WAAW,KAAK,KAAK,GAAG,MAAM,SAAS;AAC7C,SAAA,GAAS,OAAO,2BAA2B,OAAO,MAAM,SAAS,KAAK;AACtE,OAAI,WAAW,MAAA,gBAAsB;AACnC,UAAA,iBAAuB;AACvB,UAAA,GAAS,OAAO,8BAA8B,SAAS,KAAK;;;;CAMlE,oCAA6C,YAAY;AACvD,MAAI;AACF,SAAM,MAAA,iBAAuB;WACtB,GAAG;AACV,SAAA,GAAS,OAAO,8BAA8B,MAAA,mBAAyB,EAAE;;AAE3E,MAAI;AACF,SAAA,iBAAuB;WAChB,GAAG;AACV,SAAA,GAAS,OAAO,4BAA4B,EAAE;;;CAIlD,QAAA,kBAIG;EACD,MAAM,kBAAkB,MAAA;EACxB,MAAM,SAAS,MAAA,MAAY;EAC3B,IAAI;AACJ,MAAI;AACF,UAAO,MAAM,MAAM,iBAAiB,EAAC,QAAO,CAAC;WACtC,GAAG;AACV,OAAI,OAAO,QAET;AAIF,SAAA,GAAS,OAAO,8BAA8B,MAAA,mBAAyB,EAAE;AACzE;;AAEF,MAAI,CAAC,KAAK,IAAI;AACZ,SAAA,GAAS,OACP,8BAA8B,MAAA,gBAAsB,IAAI,MAAM,KAAK,MAAM,GAC1E;AACD;;EAGF,MAAM,WAAW,0BAA0B,MAAM,KAAK,MAAM,CAAC;AAC7D,OAAK,MAAM,UAAU,SACnB,KACE,OAAO,SAAS,WAChB,OAAO,SAAS,8BAEhB,MAAK,MAAM,UAAU,OAAO,SAAS;GACnC,MAAM,YAAY,OAAO,QAAQ;GACjC,MAAM,OAAO,OAAO,QAAQ;GAC5B,MAAM,uBAAO,IAAI,KAAK,WAAW,OAAO,MAAM,GAAG,IAAK;AAEtD,OAAI,UACF,OAAM;IAAC;IAAW;IAAM;IAAK;;;CAOvC,OAAA,kBAAyB;AACvB,aAAW,MAAM,EAAC,WAAW,MAAM,UAAS,MAAA,iBAAuB,CACjE,KAAI,YAAY,MAAA,iBAAuB,CAAC,MAAA,WAAiB,IAAI,UAAU,EAAE;AACvE,SAAA,GAAS,OACP,wBAAwB,UAAU,MAAM,KAAA,MAC/B,KAAK,aAAa,CAAC,GAC7B;AACD,SAAA,WAAiB,IAAI,WAAW,KAAK;AACrC,SAAA,mBAAyB;;AAG7B,SAAO,MAAA;;CAGT,mBAAmB;AACjB,MAAI,MAAA,aAAmB,OAAO,GAAG;AAC/B,SAAA,GAAS,OACP,6CAA6C,CAAC,GAAG,MAAA,aAAmB,MAAM,CAAC,GAC5E;AACD;;EAEF,MAAM,oBAAoB,KAAK,KAAK,GAAG,MAAA;EACvC,IAAI,eAAe;AACnB,OAAK,MAAM,CAAC,WAAW,eAAe,MAAA,WAAiB,SAAS,CAC9D,KACE,WAAW,SAAS,IAAI,qBACxB,YAAY,aAEZ,gBAAe;AAGnB,MAAI,aAAa,QAAQ;AACvB,SAAA,eAAqB,gBAAgB,aAAa;AAClD,QAAK,MAAM,aAAa,MAAA,WAAiB,MAAM,CAC7C,KAAI,aAAa,aACf,OAAA,WAAiB,OAAO,UAAU;AAGtC,SAAA,gBAAsB;;;CAI1B,OAAsB;AACpB,gBAAc,MAAA,kBAAwB;AACtC,OAAK,MAAM,EAAC,SAAQ,MAAA,aAAmB,QAAQ,CAK7C,KAAI,QAAQ;AAEd,QAAA,MAAY,KAAK,MAAA,GAAS;AAC1B,SAAO;;CAGT,uBAAuB;AACrB,mBAAiB,WAAW,cAAc;GACxC,aACE;GAGF,MAAM;GACP,CAAC,CAAC,YAAY,OAAM,MAAK;GASxB,MAAM,eAAe,MAAM,MAAA,iBAAuB;AAClD,OAAI,CAAC,cAAc;AACjB,UAAA,GAAS,OACP,+DACD;AACD;;GAEF,MAAM,KAAK,IAAI,SAAS,MAAA,IAAU,MAAA,aAAmB,EAAC,UAAU,MAAK,CAAC;AACtE,OAAI;IACF,MAAM,EAAC,gBAAe,GACnB,QAAgB,mDAAmD,CACnE,KAA4B;IAC/B,MAAM,YAAY,KAAK,IAAI,GAAG,cAAc,aAAa,SAAS,CAAC;AACnE,MAAE,QAAQ,UAAU;YACb,GAAG;AACV,UAAA,GAAS,OAAO,6CAA6C,EAAE;aACvD;AACR,OAAG,OAAO;;IAEZ"}

package/out/zero-cache/src/services/change-streamer/broadcast.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"broadcast.js","names":["#pending","#completed","#done","#watermark","#majority","#start","#markCompleted","#setDone","#latestCompleted","#isDone","#logWithState"],"sources":["../../../../../../zero-cache/src/services/change-streamer/broadcast.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport type {WatermarkedChange} from './change-streamer-service.ts';\nimport type {Subscriber} from './subscriber.ts';\n\n/**\n * Initiates and tracks the progress of a change broadcasted to\n * a set of subscribers.\n *\n * Creating a `Broadcast` automatically initiates the send.\n *\n * By default, {@link Broadcast.done} resolves when all subscribers\n * have acked the change. However, {@link Broadcast.checkProgress()}\n * can be called to resolve the broadcast earlier based on the flow\n * control policy.\n */\nexport class Broadcast {\n  /**\n   * Sends the change to the subscribers without the tracking machinery.\n   * This is suitable for fire-and-forget (i.e. pipelined) sends.\n   */\n  static withoutTracking(\n    subscribers: Iterable<Subscriber>,\n    change: WatermarkedChange,\n  ) {\n    for (const sub of subscribers) {\n      void sub.send(change);\n    }\n  }\n\n  readonly #pending: Set<Subscriber>;\n  readonly #completed: Completed[];\n  readonly #done = resolver();\n  #isDone = false;\n\n  readonly #watermark: string;\n  readonly #majority: number;\n\n  readonly #start = performance.now();\n  #latestCompleted = Number.MAX_VALUE;\n\n  /**\n   * Broadcasts the `change` to the `subscribers` and tracks their\n   * completion.\n   */\n  constructor(subscribers: Iterable<Subscriber>, change: WatermarkedChange) {\n    this.#pending = new Set(subscribers);\n    this.#completed = [];\n    this.#watermark = change[0];\n    this.#majority = Math.floor(this.#pending.size / 2) + 1;\n\n    for (const sub of this.#pending) {\n      const changes = sub.numPending + 1; // add one for this `change`\n      void sub\n        .send(change)\n        .catch(() => {})\n        .finally(() => this.#markCompleted(sub, changes));\n    }\n\n    // set done if there are no subscribers (mainly for tests)\n    if (this.#pending.size === 0) {\n      this.#setDone();\n    }\n  }\n\n  #markCompleted(sub: Subscriber, changes: number) {\n    const elapsed = (this.#latestCompleted = performance.now()) - this.#start;\n    this.#completed.push({sub, changes, elapsed});\n    this.#pending.delete(sub);\n    if (this.#pending.size === 0) {\n      this.#setDone();\n    }\n  }\n\n  #setDone() {\n    this.#isDone = true;\n    this.#done.resolve();\n  }\n\n  get isDone(): boolean {\n    return this.#isDone;\n  }\n\n  get done(): Promise<void> {\n    return this.#done.promise;\n  }\n\n  /**\n   * Checks for pathological situations in which flow should be reenabled\n   * before all subscribers have acked.\n   *\n   * ### Background\n   *\n   * The purpose of flow control is to pull upstream replication changes\n   * no faster than the rate as they are processed by downstream subscribers\n   * in the steady state. In the change-streamer, this is done by occasionally\n   * waiting for ACKs from subscribers before continuing; without doing so,\n   * I/O buffers fill up and cause the system to spend most of its time in GC.\n   *\n   * However, the naive algorithm of always waiting for all subscribers (e.g.\n   * `Promise.all()`) can behave poorly in scenarios where subscribers\n   * are imbalanced:\n   * * New subscribers may have a backlog of changes to catch up with.\n   *   Having all subscribers wait for the new subscriber to catch up results\n   *   in delaying the entire application.\n   * * Broken TCP connections similarly require all subscribers to wait until\n   *   connection liveness checks kick in and disconnect the subscriber.\n   *\n   * A simplistic approach is to add a limit to the amount of time waiting for\n   * subscribers, i.e. an ack timeout. However, deciding what this timeout\n   * should be is non-trivial because of the heterogeneous nature of changes;\n   * while most changes operate on single rows and are relatively predictable\n   * in terms of running time, some changes are table-wide operations and can\n   * legitimately take an arbitrary amount of time. In such scenarios, a\n   * timeout that is too short can stop progress on replication altogether.\n   *\n   * ### Consensus-based Timeout Algorithm\n   *\n   * To address these shortcomings, a \"consensus-based timeout\" algorithm is\n   * used:\n   * * Wait for more than half of the subscribers to finish. (In\n   *   case of a single node, or the case of one replication-manager\n   *   and one view-syncer, this reduces to waiting for all subscribers.)\n   * * Once more than half of the subscribers have finished, proceed after\n   *   a fixed timeout elapses (e.g. 1 second), even if not all subscribers\n   *   have finished.\n   *\n   * In other words, the subscribers themselves are used to determine the\n   * timeout of each batch of changes; the majority determines this when\n   * they complete, upon which a timeout is logically started.\n   *\n   * In the common case, the remaining subscribers finish soon afterward and\n   * the timeout never elapses. However, in pathological cases where a minority\n   * of subscribers have a disproportionate amount of load, some will still\n   * be processing (or otherwise unresponsive). These subscribers are given\n   * a bounded amount of time to catch up at each flushed batch, up to the\n   * timeout interval. This guarantees eventual catchup because the\n   * subscribers with a backlog of changes necessarily have a higher\n   * processing rate than the subscribers that finished (and are made to wait).\n   *\n   * ### Not implemented: Broken connection detection\n   *\n   * If a subscriber has not made progress for a certain interval, the\n   * algorithm could theoretically drop it preemptively, supplementing the\n   * existing websocket-level liveness checks.\n   *\n   * However, a more reliable approach would be to change the replicator\n   * to use non-blocking writes, and subsequently increase the frequency of\n   * connection-level liveness checks. The current synchronous replica writes\n   * can delay both ping responsiveness and change progress arbitrarily (e.g.\n   * a large index creation); an independently liveness check that is not\n   * delayed by synchronous writes on the subscriber would be a more failsafe\n   * solution.\n   *\n   * @returns `true` if the broadcast was already done or was marked done.\n   */\n  checkProgress(\n    lc: LogContext,\n    flowControlConsensusPaddingMs: number,\n    now: number,\n  ) {\n    if (this.#pending.size === 0) {\n      return true;\n    }\n    const elapsed = now - this.#start;\n    if (this.#completed.length < this.#majority) {\n      if (elapsed >= 1000) {\n        this.#logWithState(\n          lc,\n          `waiting for at least ${this.#majority} subscribers to finish`,\n          elapsed,\n        );\n      }\n      return false;\n    }\n    // Note: In the implementation, #latestCompleted is always updated,\n    // even after the majority is reached. This is fine and does not affect\n    // the important properties of the algorithm.\n    if (now - this.#latestCompleted >= flowControlConsensusPaddingMs) {\n      this.#logWithState(\n        lc,\n        `continuing with ${this.#pending.size} subscriber(s) still pending`,\n        elapsed,\n      );\n      this.#setDone();\n      return true;\n    }\n    return false;\n  }\n\n  #logWithState(lc: LogContext, msg: string, elapsed: number) {\n    lc.withContext('watermark', this.#watermark).info?.(\n      `${msg} (${elapsed.toFixed(3)} ms)`,\n      {\n        completed: this.#completed.map(d => ({\n          id: d.sub.id,\n          processed: d.changes,\n          elapsed: d.elapsed,\n        })),\n        pending: Array.from(this.#pending, sub => ({\n          id: sub.id,\n          ...sub.getStats(),\n        })),\n      },\n    );\n  }\n}\n\n/** Tracks the completed result of a single subscriber. */\ntype Completed = {\n  sub: Subscriber;\n  /** The number of changes processed. */\n  changes: number;\n  /** The elapsed milliseconds. */\n  elapsed: number;\n};\n"],"mappings":";;;;;;;;;;;;;AAgBA,IAAa,YAAb,MAAuB;;;;;CAKrB,OAAO,gBACL,aACA,QACA;AACA,OAAK,MAAM,OAAO,YACX,KAAI,KAAK,OAAO;;CAIzB;CACA;CACA,QAAiB,UAAU;CAC3B,UAAU;CAEV;CACA;CAEA,SAAkB,YAAY,KAAK;CACnC,mBAAmB,OAAO;;;;;CAM1B,YAAY,aAAmC,QAA2B;AACxE,QAAA,UAAgB,IAAI,IAAI,YAAY;AACpC,QAAA,YAAkB,EAAE;AACpB,QAAA,YAAkB,OAAO;AACzB,QAAA,WAAiB,KAAK,MAAM,MAAA,QAAc,OAAO,EAAE,GAAG;AAEtD,OAAK,MAAM,OAAO,MAAA,SAAe;GAC/B,MAAM,UAAU,IAAI,aAAa;AAC5B,OACF,KAAK,OAAO,CACZ,YAAY,GAAG,CACf,cAAc,MAAA,cAAoB,KAAK,QAAQ,CAAC;;AAIrD,MAAI,MAAA,QAAc,SAAS,EACzB,OAAA,SAAe;;CAInB,eAAe,KAAiB,SAAiB;EAC/C,MAAM,WAAW,MAAA,kBAAwB,YAAY,KAAK,IAAI,MAAA;AAC9D,QAAA,UAAgB,KAAK;GAAC;GAAK;GAAS;GAAQ,CAAC;AAC7C,QAAA,QAAc,OAAO,IAAI;AACzB,MAAI,MAAA,QAAc,SAAS,EACzB,OAAA,SAAe;;CAInB,WAAW;AACT,QAAA,SAAe;AACf,QAAA,KAAW,SAAS;;CAGtB,IAAI,SAAkB;AACpB,SAAO,MAAA;;CAGT,IAAI,OAAsB;AACxB,SAAO,MAAA,KAAW;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAwEpB,cACE,IACA,+BACA,KACA;AACA,MAAI,MAAA,QAAc,SAAS,EACzB,QAAO;EAET,MAAM,UAAU,MAAM,MAAA;AACtB,MAAI,MAAA,UAAgB,SAAS,MAAA,UAAgB;AAC3C,OAAI,WAAW,IACb,OAAA,aACE,IACA,wBAAwB,MAAA,SAAe,yBACvC,QACD;AAEH,UAAO;;AAKT,MAAI,MAAM,MAAA,mBAAyB,+BAA+B;AAChE,SAAA,aACE,IACA,mBAAmB,MAAA,QAAc,KAAK,+BACtC,QACD;AACD,SAAA,SAAe;AACf,UAAO;;AAET,SAAO;;CAGT,cAAc,IAAgB,KAAa,SAAiB;AAC1D,KAAG,YAAY,aAAa,MAAA,UAAgB,CAAC,OAC3C,GAAG,IAAI,IAAI,QAAQ,QAAQ,EAAE,CAAC,OAC9B;GACE,WAAW,MAAA,UAAgB,KAAI,OAAM;IACnC,IAAI,EAAE,IAAI;IACV,WAAW,EAAE;IACb,SAAS,EAAE;IACZ,EAAE;GACH,SAAS,MAAM,KAAK,MAAA,UAAe,SAAQ;IACzC,IAAI,IAAI;IACR,GAAG,IAAI,UAAU;IAClB,EAAE;GACJ,CACF"}

package/out/zero-cache/src/services/change-streamer/change-streamer-http.js

@@ -1,158 +0,0 @@
-import { assert } from "../../../../shared/src/asserts.js";
-import { must } from "../../../../shared/src/must.js";
-import "../../types/processes.js";
-import { pgClient } from "../../types/pg.js";
-import { HttpService } from "../http-service.js";
-import { PROTOCOL_ERROR, closeWithError } from "../../types/ws.js";
-import { installWebSocketReceiver } from "../../types/websocket-handoff.js";
-import { discoverChangeStreamerAddress } from "./schema/tables.js";
-import { streamIn, streamOut, streamOutStringified } from "../../types/streams.js";
-import { URLParams } from "../../types/url-params.js";
-import { downstreamSchema } from "./change-streamer.js";
-import { snapshotMessageSchema } from "./snapshot.js";
-import WebSocket from "ws";
-import websocket from "@fastify/websocket";
-//#region ../zero-cache/src/services/change-streamer/change-streamer-http.ts
-var MIN_SUPPORTED_PROTOCOL_VERSION = 1;
-var SNAPSHOT_PATH_PATTERN = "/replication/:version/snapshot";
-var CHANGES_PATH_PATTERN = "/replication/:version/changes";
-var PATH_REGEX = /\/replication\/v(?<version>\d+)\/(changes|snapshot)$/;
-var SNAPSHOT_PATH = `/replication/v6/snapshot`;
-var CHANGES_PATH = `/replication/v6/changes`;
-var ChangeStreamerHttpServer = class extends HttpService {
-	id = "change-streamer-http-server";
-	#lc;
-	#opts;
-	#changeStreamer;
-	#backupMonitor;
-	constructor(lc, opts, parent, changeStreamer, backupMonitor) {
-		super("change-streamer-http-server", lc, opts, async (fastify) => {
-			await fastify.register(websocket);
-			fastify.get(CHANGES_PATH_PATTERN, { websocket: true }, this.#subscribe);
-			fastify.get(SNAPSHOT_PATH_PATTERN, { websocket: true }, this.#reserveSnapshot);
-			installWebSocketReceiver(lc, fastify.websocketServer, this.#receiveWebsocket, parent);
-		});
-		this.#lc = lc;
-		this.#opts = opts;
-		this.#changeStreamer = changeStreamer;
-		this.#backupMonitor = backupMonitor;
-	}
-	#getBackupMonitor() {
-		return must(this.#backupMonitor, "replication-manager is not configured with a ZERO_LITESTREAM_BACKUP_URL");
-	}
-	#receiveWebsocket = (ws, action, msg) => {
-		switch (action) {
-			case "snapshot": return this.#reserveSnapshot(ws, msg);
-			case "changes": return this.#subscribe(ws, msg);
-			default:
-				closeWithError(this._lc, ws, `invalid action "${action}" received in handoff`);
-				return;
-		}
-	};
-	#reserveSnapshot = (ws, req) => {
-		try {
-			const url = new URL(req.url ?? "", req.headers.origin ?? "http://localhost");
-			checkProtocolVersion(url.pathname);
-			const taskID = url.searchParams.get("taskID");
-			if (!taskID) throw new Error("Missing taskID in snapshot request");
-			const downstream = this.#getBackupMonitor().startSnapshotReservation(taskID);
-			streamOut(this._lc, downstream, ws);
-		} catch (err) {
-			closeWithError(this._lc, ws, err, PROTOCOL_ERROR);
-		}
-	};
-	#subscribe = async (ws, req) => {
-		try {
-			const ctx = getSubscriberContext(req);
-			if (ctx.mode === "serving") this.#ensureChangeStreamerStarted("incoming subscription");
-			const downstream = await this.#changeStreamer.subscribe(ctx);
-			if (ctx.initial && ctx.taskID && this.#backupMonitor) this.#backupMonitor.endReservation(ctx.taskID);
-			streamOutStringified(this._lc, downstream, ws);
-		} catch (err) {
-			closeWithError(this._lc, ws, err, PROTOCOL_ERROR);
-		}
-	};
-	#changeStreamerStarted = false;
-	#ensureChangeStreamerStarted(reason) {
-		if (!this.#changeStreamerStarted && this._state.shouldRun()) {
-			this.#lc.info?.(`starting ChangeStreamerService: ${reason}`);
-			this.#changeStreamer.run().catch((e) => this.#lc.warn?.(`ChangeStreamerService ended with error`, e)).finally(() => this.stop());
-			this.#changeStreamerStarted = true;
-		}
-	}
-	_onStart() {
-		const { startupDelayMs } = this.#opts;
-		this._state.setTimeout(() => this.#ensureChangeStreamerStarted(`startup delay elapsed (${startupDelayMs} ms)`), startupDelayMs);
-	}
-	async _onStop() {
-		if (this.#changeStreamerStarted) await this.#changeStreamer.stop();
-	}
-};
-var ChangeStreamerHttpClient = class {
-	#lc;
-	#shardID;
-	#changeDB;
-	#changeStreamerURI;
-	constructor(lc, shardID, changeDB, changeStreamerURI) {
-		this.#lc = lc;
-		this.#shardID = shardID;
-		this.#changeDB = pgClient(lc, changeDB, "change-streamer-discovery", {
-			max: 1,
-			["idle_timeout"]: 15
-		});
-		this.#changeStreamerURI = changeStreamerURI;
-	}
-	async #resolveChangeStreamer(path) {
-		let baseURL = this.#changeStreamerURI;
-		if (!baseURL) {
-			const address = await discoverChangeStreamerAddress(this.#shardID, this.#changeDB);
-			if (!address) throw new Error(`no change-streamer is running`);
-			baseURL = address.includes("://") ? `${address}/` : `ws://${address}/`;
-		}
-		const uri = new URL(path, baseURL);
-		this.#lc.info?.(`connecting to change-streamer@${uri}`);
-		return uri;
-	}
-	async reserveSnapshot(taskID) {
-		const ws = new WebSocket(await this.#resolveChangeStreamer(SNAPSHOT_PATH) + `?${new URLSearchParams({ taskID }).toString()}`);
-		return streamIn(this.#lc, ws, snapshotMessageSchema);
-	}
-	async subscribe(ctx) {
-		const ws = new WebSocket(await this.#resolveChangeStreamer(CHANGES_PATH) + `?${getParams(ctx).toString()}`);
-		return streamIn(this.#lc, ws, downstreamSchema);
-	}
-};
-function getSubscriberContext(req) {
-	const url = new URL(req.url ?? "", req.headers.origin ?? "http://localhost");
-	const protocolVersion = checkProtocolVersion(url.pathname);
-	const params = new URLParams(url);
-	return {
-		protocolVersion,
-		id: params.get("id", true),
-		taskID: params.get("taskID", false),
-		mode: params.get("mode", false) === "backup" ? "backup" : "serving",
-		replicaVersion: params.get("replicaVersion", true),
-		watermark: params.get("watermark", true),
-		initial: params.getBoolean("initial")
-	};
-}
-function checkProtocolVersion(pathname) {
-	const match = PATH_REGEX.exec(pathname);
-	if (!match) throw new Error(`invalid path: ${pathname}`);
-	const v = Number(match.groups?.version);
-	if (Number.isNaN(v) || v > 6 || v < MIN_SUPPORTED_PROTOCOL_VERSION) throw new Error(`Cannot service client at protocol v${v}. Supported protocols: [v${MIN_SUPPORTED_PROTOCOL_VERSION} ... v6]`);
-	return v;
-}
-function getParams(ctx) {
-	const { protocolVersion, ...stringParams } = ctx;
-	assert(protocolVersion === 6, `replicator should be setting protocolVersion to 6`);
-	return new URLSearchParams({
-		...stringParams,
-		taskID: ctx.taskID ? ctx.taskID : "",
-		initial: ctx.initial ? "true" : "false"
-	});
-}
-//#endregion
-export { ChangeStreamerHttpClient, ChangeStreamerHttpServer };
-
-//# sourceMappingURL=change-streamer-http.js.map