@rocicorp/zero 1.12.0-canary.2 → 1.12.0-canary.4

package/out/zero-cache/src/services/change-source/protocol/current/data.js.map

@@ -1 +1 @@
-{"version":3,"file":"data.js","names":[],"sources":["../../../../../../../../zero-cache/src/services/change-source/protocol/current/data.ts"],"sourcesContent":["/**\n * Data plane messages encapsulate changes that are sent by ChangeSources,\n * forwarded / fanned out to subscribers by the ChangeStreamerService, and\n * stored in the Change DB for catchup of old subscribers.\n */\n\nimport {\n  jsonValueSchema,\n  type JSONObject,\n} from '../../../../../../shared/src/bigint-json.ts';\nimport {must} from '../../../../../../shared/src/must.ts';\nimport * as v from '../../../../../../shared/src/valita.ts';\nimport {columnSpec, indexSpec, tableSpec} from '../../../../db/specs.ts';\nimport type {Satisfies} from '../../../../types/satisfies.ts';\nimport {jsonObjectSchema} from './json.ts';\nimport {schemaChangeTags} from './schema-change-tags.ts';\n\nexport const beginSchema = v.object({\n  tag: v.literal('begin'),\n  // The format of values of \"json\"-typed columns (e.g. \"JSON\" and \"JSONB\").\n  // - 'p' is for parsed JSON, which may include JSON values or JSON objects.\n  //   These values are parsed and stringified at every process boundary\n  //   between the change-source and the replica.\n  // - 's' is for stringified JSON. These values skip the parsing and\n  //   stringification, and are directly ferried to the replica as a JSON\n  //   string. For JSON values this improves performance by 20~25% in the\n  //   change-streamer and 25~30% in the replicator.\n  //\n  // If absent, the format is assumed to be 'p' (parsed JSON objects/values).\n  json: v.literalUnion('p', 's').optional(),\n\n  // Directs the change-streamer to skip the ACK for the corresponding commit.\n  skipAck: v.boolean().optional(),\n});\n\nexport const commitSchema = v.object({\n  tag: v.literal('commit'),\n\n  // Millisecond epoch at which the transaction committed upstream, as reported\n  // by the upstream database (e.g. the `commitTime` of the Postgres logical\n  // replication Commit message).\n  //\n  // This is the origin timestamp of the end-to-end serving lag measurement:\n  // it is carried through the replication stream to the ViewSyncer, which\n  // records `now - commitTimeMs` once the change has been poked to clients.\n  //\n  // Optional because it is absent from changes written to the Change DB by\n  // older versions, which are replayed verbatim during catchup, and from\n  // ChangeSources that do not report a commit time.\n  commitTimeMs: v.number().optional(),\n});\n\nexport const rollbackSchema = v.object({\n  tag: v.literal('rollback'),\n});\n\nconst rowKeySchema = v.object({\n  // The columns used to identify a row in insert, update, and delete changes.\n  columns: v.array(v.string()),\n\n  // An optional qualifier identifying how the key is chosen. Currently this\n  // is postgres-specific, describing the REPLICA IDENTITY, for which replica\n  // identity 'full' (FULL) is handled differently; the replicator handles\n  // these tables by extracting a row key from the full row based on the\n  // table's PRIMARY KEY or UNIQUE INDEX.\n  type: v.literalUnion('default', 'nothing', 'full', 'index').optional(),\n});\n\nexport const relationSchema = v\n  .object({\n    schema: v.string(),\n    name: v.string(),\n\n    // This will become required.\n    rowKey: rowKeySchema.optional(),\n\n    /** Deprecated: set the rowKey.columns instead. */\n    keyColumns: v.array(v.string()).optional(),\n    /** Deprecated: set the rowKey.columns instead. */\n    replicaIdentity: v\n      .literalUnion('default', 'nothing', 'full', 'index')\n      .optional(),\n  })\n  .map(rel => {\n    const {rowKey, ...rest} = rel;\n    if (rowKey) {\n      return {...rest, rowKey};\n    }\n    return {\n      ...rest,\n      rowKey: {\n        columns: must(rel.keyColumns),\n        type: rel.replicaIdentity,\n      },\n    };\n  });\n\n// The eventual fate of relationSchema\nexport const newRelationSchema = v.object({\n  schema: v.string(),\n  name: v.string(),\n\n  rowKey: rowKeySchema,\n});\n\n// TableMetadata contains table-related configuration that does not affect the\n// actual data in the table, but rather how the table's change messages are\n// handled. The is an opaque object that clients must track (and update) based\n// on `create-table`, `add-column`, and `table-update-metadata` messages, and\n// pass in BackfillRequests when there are columns to be backfilled.\n//\n// Note that the backfill-related change-source implementation does, however,\n// rely on the rowKey (columns) being specified in the message.\nexport const tableMetadataSchema = v\n  .object({rowKey: v.record(jsonValueSchema)})\n  .rest(jsonValueSchema);\n\nexport type TableMetadata = v.Infer<typeof tableMetadataSchema>;\n\nexport const rowSchema = v.record(jsonValueSchema);\n\nexport const insertSchema = v.object({\n  tag: v.literal('insert'),\n  relation: relationSchema,\n  new: rowSchema,\n});\n\nexport const updateSchema = v.object({\n  tag: v.literal('update'),\n  relation: relationSchema,\n  // `key` is present if the update changed the key of the row, or if the\n  // table's replicaIdentity === 'full'\n  key: rowSchema.nullable(),\n  // `new` is the full row (and not just the updated columns). This is\n  // necessary for \"catchup\" replication scenarios such as adding tables\n  // to a publication, or resharding.\n  new: rowSchema,\n});\n\nexport const deleteSchema = v.object({\n  tag: v.literal('delete'),\n  relation: relationSchema,\n  // key is the full row if replicaIdentity === 'full'\n  key: rowSchema,\n});\n\nexport const truncateSchema = v.object({\n  tag: v.literal('truncate'),\n  relations: v.array(relationSchema),\n});\n\nexport const identifierSchema = v.object({\n  schema: v.string(),\n  name: v.string(),\n});\n\nexport type Identifier = v.Infer<typeof identifierSchema>;\n\n// A BackfillID is an upstream specific stable identifier for a column\n// that needs backfilling. This id is used to ensure that the schema, table,\n// and column names of a requested backfill still match the original\n// underlying upstream ID.\n//\n// The change-streamer stores these IDs as opaque values while a column is\n// being backfilled, and initiates new change-source streams with the IDs\n// in order to restart backfills that did not complete in previous sessions.\nexport const backfillIDSchema = jsonObjectSchema;\n\nexport type BackfillID = v.Infer<typeof backfillIDSchema>;\n\nexport const backfillProgressMarkSchema = v\n  .object({\n    // A lexicographically sortable string delineating the progress of a\n    // backfill. This is used to resume a backfill that did not complete.\n    progressMark: v.string(),\n\n    // Optionally, a timeline with which the progressMark is associated.\n    // A timeline is an opaque string that determines if backfill streams\n    // are \"compatible\" with each other, i.e. can be merged in a way that\n    // avoids gaps. progressMarks from different timelines are not\n    // compatible. If two timelines differ, a new backfill is started\n    // from scratch in order to get all subscribers on the same timeline.\n    //\n    // Timeline changes should be rare events (e.g. Postgres VACUUM FULL).\n    timeline: v.string().optional(),\n  })\n  // All other fields should be preserved and returned in BackfillRequests.\n  .rest(v.union(v.number(), v.string()));\n\nexport type BackfillProgressMark = v.Infer<typeof backfillProgressMarkSchema>;\n\nexport const createTableSchema = v.object({\n  tag: v.literal('create-table'),\n  spec: tableSpec,\n\n  // This must be set by change source implementations that support\n  // table/column backfill.\n  //\n  // TODO: to simplify the protocol, see if we can make this required\n  metadata: tableMetadataSchema.optional(),\n\n  // Indicate that columns of the table require backfilling. These columns\n  // should be created on the replica but not yet synced the clients.\n  //\n  // ## State Persistence\n  //\n  // To obviate the need for change-source implementations to persist state\n  // related to backfill progress, the change-source only tracks backfills\n  // **for the current session**. In the event that the session is interrupted\n  // before columns have been fully backfilled, it is the responsibility of the\n  // change-streamer to send {@link BackfillRequest}s when it reconnects.\n  //\n  // This means that the change-streamer must track and persist:\n  // * the backfill IDs of the columns requiring backfilling\n  // * the most current table metadata of the associated table(s)\n  //\n  // The change-streamer then uses this information to send backfill requests\n  // when it reconnects.\n  backfill: v.record(backfillIDSchema).optional(),\n});\n\nexport const renameTableSchema = v.object({\n  tag: v.literal('rename-table'),\n  old: identifierSchema,\n  new: identifierSchema,\n});\n\nexport const updateTableMetadataSchema = v.object({\n  tag: v.literal('update-table-metadata'),\n  table: identifierSchema,\n  old: tableMetadataSchema,\n  new: tableMetadataSchema,\n});\n\nconst columnSchema = v.object({\n  name: v.string(),\n  spec: columnSpec,\n});\n\nexport const addColumnSchema = v.object({\n  tag: v.literal('add-column'),\n  table: identifierSchema,\n  column: columnSchema,\n\n  // This must be set by change source implementations that support\n  // table/column backfill.\n  //\n  // TODO: to simplify the protocol, see if we can make this required\n  tableMetadata: tableMetadataSchema.optional(),\n\n  // See documentation for the `backfill` field of the `create-table` change.\n  backfill: backfillIDSchema.optional(),\n});\n\nexport const updateColumnSchema = v.object({\n  tag: v.literal('update-column'),\n  table: identifierSchema,\n  old: columnSchema,\n  new: columnSchema,\n});\n\nexport const dropColumnSchema = v.object({\n  tag: v.literal('drop-column'),\n  table: identifierSchema,\n  column: v.string(),\n});\n\nexport const dropTableSchema = v.object({\n  tag: v.literal('drop-table'),\n  id: identifierSchema,\n});\n\nexport const createIndexSchema = v.object({\n  tag: v.literal('create-index'),\n  spec: indexSpec,\n});\n\nexport const dropIndexSchema = v.object({\n  tag: v.literal('drop-index'),\n  id: identifierSchema,\n});\n\nexport const downloadStatusSchema = v.object({\n  rows: v.number(),\n  totalRows: v.number(),\n  totalBytes: v.number().optional(),\n});\n\nexport type DownloadStatus = v.Infer<typeof downloadStatusSchema>;\n\n// A batch of rows from a single table containing column values\n// to be backfilled.\nexport const backfillSchema = v.object({\n  tag: v.literal('backfill'),\n\n  relation: newRelationSchema,\n\n  // The columns to be backfilled. `rowKey` columns are automatically excluded,\n  // which means that this field may be empty.\n  columns: v.array(v.string()),\n\n  // The watermark at which the backfill data was queried. Note that this\n  // generally will be different from the commit watermarks of the main change\n  // stream, and in particular, the commit watermark of the backfill change's\n  // enclosing transaction.\n  watermark: v.string(),\n\n  // A batch of row values, each row consisting of the `rowKey`\n  // values, followed by the `column` values, in the same order in which\n  // the column names appear in their respective fields, e.g.\n  //\n  // ```\n  // [\n  //   [...rowKeyValues, ...columnValues],  // row 1\n  //   [...rowKeyValues, ...columnValues],  // row 2\n  // ]\n  // ```\n  rowValues: v.array(v.array(jsonValueSchema)),\n\n  // Progress marks indicating the position of this backfill message\n  // within the stream, for resumption / continuity enforcement.\n  //\n  // This should be considered non-optional as of protocol v8. It is only\n  // absent in messages from pre-v8 replication-managers (including messages\n  // persisted in a change-log by one), in which case subscribers accept the\n  // message unconditionally (i.e. the legacy behavior).\n  progressMarks: v\n    .object({\n      // The progress mark of the preceding backfill message, from which\n      // the current backfill message is a continuation. (This may be a\n      // \"logical\" continuation at which a new backfill stream is started when\n      // resuming from a previous stream.)\n      //\n      // For the first backfill message of a stream starting from scratch,\n      // this field is unset. When the field is unset, subscribers should\n      // accept the backfill message unconditionally (i.e. starting a\n      // backfill from scratch is always safe).\n      //\n      // If the field is set, subscribers should only accept the message if\n      // it is of the same timeline as that of its own tracked progressMark,\n      // and is contiguous with or earlier than its own progressMark.\n      //\n      // In other words, it should _ignore_ backfill messages from other\n      // timelines, or messages that create a gap; this can happen when\n      // receiving messages from before this subscriber's backfill requests\n      // are taken into account.\n      //\n      // Implementation note: The change-streamer applies this same policy\n      // (see `acceptBackfill()` in `protocol/backfill-progress.ts`) to track\n      // the backfill state of each subscriber, which it uses to determine the\n      // BackfillRequests with which to (re)start change streams.\n      previous: backfillProgressMarkSchema.optional(),\n\n      // The progressMark delineating the position of the last row in this\n      // message. In the steady state, this will be the `previous`\n      // progressMark of the next backfill message, unless a rewind is\n      // requested, or new timeline encountered.\n      current: backfillProgressMarkSchema,\n    })\n    .optional(),\n\n  // Optionally includes the progress of the backfill operation,\n  // for display purposes.\n  status: downloadStatusSchema.optional(),\n});\n\n// Indicates that the backfill for the specified columns have\n// been successfully backfilled and can be published to clients.\nexport const backfillCompletedSchema = v.object({\n  tag: v.literal('backfill-completed'),\n\n  relation: newRelationSchema,\n\n  // The columns to be backfilled. `rowKey` columns are automatically excluded,\n  // which means that this field may be empty.\n  columns: v.array(v.string()),\n\n  // The watermark at which the backfill data was queried. Note that this\n  // generally will be different from the commit watermarks of the main change\n  // stream, and in particular, the commit watermark of the backfill change's\n  // enclosing transaction.\n  watermark: v.string(),\n\n  // The progress mark of the last backfill message of the run, i.e. the\n  // `current` progressMark of the last `backfill` message, or the progressMark\n  // from which the run was resumed if it produced no `backfill` messages.\n  // Subscribers apply the same continuity policy as for `backfill` messages,\n  // and only consider the columns backfilled if the run is contiguous with\n  // their own progress.\n  //\n  // `previous` is only unset for a run that started from scratch and\n  // produced no rows (e.g. an empty table), which is always safe to accept.\n  //\n  // This should be considered non-optional as of protocol v8. It is only\n  // absent in messages from pre-v8 replication-managers, in which case\n  // subscribers accept the message unconditionally (i.e. legacy behavior).\n  progressMarks: v\n    .object({previous: backfillProgressMarkSchema.optional()})\n    .optional(),\n\n  // Optionally includes the final status of the backfill operation,\n  // for display purposes.\n  status: downloadStatusSchema.optional(),\n});\n\nexport type MessageBegin = v.Infer<typeof beginSchema>;\nexport type MessageCommit = v.Infer<typeof commitSchema>;\nexport type MessageRollback = v.Infer<typeof rollbackSchema>;\n\nexport type MessageRelation = v.Infer<typeof relationSchema>;\nexport type MessageInsert = v.Infer<typeof insertSchema>;\nexport type MessageUpdate = v.Infer<typeof updateSchema>;\nexport type MessageDelete = v.Infer<typeof deleteSchema>;\nexport type MessageTruncate = v.Infer<typeof truncateSchema>;\n\nexport type MessageBackfill = v.Infer<typeof backfillSchema>;\n\nexport type TableCreate = v.Infer<typeof createTableSchema>;\nexport type TableRename = v.Infer<typeof renameTableSchema>;\nexport type TableUpdateMetadata = v.Infer<typeof updateTableMetadataSchema>;\nexport type ColumnAdd = v.Infer<typeof addColumnSchema>;\nexport type ColumnUpdate = v.Infer<typeof updateColumnSchema>;\nexport type ColumnDrop = v.Infer<typeof dropColumnSchema>;\nexport type TableDrop = v.Infer<typeof dropTableSchema>;\nexport type IndexCreate = v.Infer<typeof createIndexSchema>;\nexport type IndexDrop = v.Infer<typeof dropIndexSchema>;\nexport type BackfillCompleted = v.Infer<typeof backfillCompletedSchema>;\n\nexport const dataChangeSchema = v.union(\n  insertSchema,\n  updateSchema,\n  deleteSchema,\n  truncateSchema,\n  backfillSchema,\n);\n\n// Note: keep in sync or the tag tests will fail\nconst dataChangeTags = [\n  'insert',\n  'update',\n  'delete',\n  'truncate',\n  'backfill',\n] as const;\n\nconst dataChangeTagsSchema = v.literalUnion(...dataChangeTags);\n\nexport type DataChange = Satisfies<\n  JSONObject, // guarantees serialization over IPC or network\n  v.Infer<typeof dataChangeSchema>\n>;\n\nexport type DataChangeTag = v.Infer<typeof dataChangeTagsSchema>;\n\nconst schemaChanges = [\n  createTableSchema,\n  renameTableSchema,\n  updateTableMetadataSchema,\n  addColumnSchema,\n  updateColumnSchema,\n  dropColumnSchema,\n  dropTableSchema,\n  createIndexSchema,\n  dropIndexSchema,\n  backfillCompletedSchema,\n] as const;\n\nexport const schemaChangeSchema = v.union(...schemaChanges);\n\nconst schemaChangeTagsSchema = v.literalUnion(...schemaChangeTags);\n\nexport type SchemaChange = Satisfies<\n  JSONObject,\n  v.Infer<typeof schemaChangeSchema>\n>;\n\nexport type SchemaChangeTag = v.Infer<typeof schemaChangeTagsSchema>;\n\nexport type DataOrSchemaChange = DataChange | SchemaChange;\n\nexport type Change =\n  | MessageBegin\n  | DataOrSchemaChange\n  | MessageCommit\n  | MessageRollback;\n\nexport type ChangeTag = Change['tag'];\n\nconst schemaChangeTagSet = new Set<string>(schemaChangeTags);\n\nexport function isSchemaChange(change: Change): change is SchemaChange {\n  return schemaChangeTagSet.has(change.tag);\n}\n\nconst dataChangeTagSet = new Set<string>(dataChangeTags);\n\nexport function isDataChange(change: Change): change is DataChange {\n  return dataChangeTagSet.has(change.tag);\n}\n"],"mappings":";;;;;;;;;;;;AAiBA,IAAa,cAAc,eAAE,OAAO;CAClC,KAAK,eAAE,QAAQ,OAAO;CAWtB,MAAM,aAAe,KAAK,GAAG,EAAE,SAAS;CAGxC,SAAS,eAAE,QAAQ,EAAE,SAAS;AAChC,CAAC;AAED,IAAa,eAAe,eAAE,OAAO;CACnC,KAAK,eAAE,QAAQ,QAAQ;CAavB,cAAc,eAAE,OAAO,EAAE,SAAS;AACpC,CAAC;AAED,IAAa,iBAAiB,eAAE,OAAO,EACrC,KAAK,eAAE,QAAQ,UAAU,EAC3B,CAAC;AAED,IAAM,eAAe,eAAE,OAAO;CAE5B,SAAS,eAAE,MAAM,eAAE,OAAO,CAAC;CAO3B,MAAM,aAAe,WAAW,WAAW,QAAQ,OAAO,EAAE,SAAS;AACvE,CAAC;AAED,IAAa,iBAAiB,eAC3B,OAAO;CACN,QAAQ,eAAE,OAAO;CACjB,MAAM,eAAE,OAAO;CAGf,QAAQ,aAAa,SAAS;;CAG9B,YAAY,eAAE,MAAM,eAAE,OAAO,CAAC,EAAE,SAAS;;CAEzC,iBAAiB,aACD,WAAW,WAAW,QAAQ,OAAO,EAClD,SAAS;AACd,CAAC,EACA,KAAI,QAAO;CACV,MAAM,EAAC,QAAQ,GAAG,SAAQ;CAC1B,IAAI,QACF,OAAO;EAAC,GAAG;EAAM;CAAM;CAEzB,OAAO;EACL,GAAG;EACH,QAAQ;GACN,SAAS,KAAK,IAAI,UAAU;GAC5B,MAAM,IAAI;EACZ;CACF;AACF,CAAC;AAGH,IAAa,oBAAoB,eAAE,OAAO;CACxC,QAAQ,eAAE,OAAO;CACjB,MAAM,eAAE,OAAO;CAEf,QAAQ;AACV,CAAC;AAUD,IAAa,sBAAsB,eAChC,OAAO,EAAC,QAAQ,eAAE,OAAO,eAAe,EAAC,CAAC,EAC1C,KAAK,eAAe;AAIvB,IAAa,YAAY,eAAE,OAAO,eAAe;AAEjD,IAAa,eAAe,eAAE,OAAO;CACnC,KAAK,eAAE,QAAQ,QAAQ;CACvB,UAAU;CACV,KAAK;AACP,CAAC;AAED,IAAa,eAAe,eAAE,OAAO;CACnC,KAAK,eAAE,QAAQ,QAAQ;CACvB,UAAU;CAGV,KAAK,UAAU,SAAS;CAIxB,KAAK;AACP,CAAC;AAED,IAAa,eAAe,eAAE,OAAO;CACnC,KAAK,eAAE,QAAQ,QAAQ;CACvB,UAAU;CAEV,KAAK;AACP,CAAC;AAED,IAAa,iBAAiB,eAAE,OAAO;CACrC,KAAK,eAAE,QAAQ,UAAU;CACzB,WAAW,eAAE,MAAM,cAAc;AACnC,CAAC;AAED,IAAa,mBAAmB,eAAE,OAAO;CACvC,QAAQ,eAAE,OAAO;CACjB,MAAM,eAAE,OAAO;AACjB,CAAC;AAYD,IAAa,mBAAmB;AAIhC,IAAa,6BAA6B,eACvC,OAAO;CAGN,cAAc,eAAE,OAAO;CAUvB,UAAU,eAAE,OAAO,EAAE,SAAS;AAChC,CAAC,EAEA,KAAK,eAAE,MAAM,eAAE,OAAO,GAAG,eAAE,OAAO,CAAC,CAAC;AAIvC,IAAa,oBAAoB,eAAE,OAAO;CACxC,KAAK,eAAE,QAAQ,cAAc;CAC7B,MAAM;CAMN,UAAU,oBAAoB,SAAS;CAmBvC,UAAU,eAAE,OAAO,gBAAgB,EAAE,SAAS;AAChD,CAAC;AAED,IAAa,oBAAoB,eAAE,OAAO;CACxC,KAAK,eAAE,QAAQ,cAAc;CAC7B,KAAK;CACL,KAAK;AACP,CAAC;AAED,IAAa,4BAA4B,eAAE,OAAO;CAChD,KAAK,eAAE,QAAQ,uBAAuB;CACtC,OAAO;CACP,KAAK;CACL,KAAK;AACP,CAAC;AAED,IAAM,eAAe,eAAE,OAAO;CAC5B,MAAM,eAAE,OAAO;CACf,MAAM;AACR,CAAC;AAED,IAAa,kBAAkB,eAAE,OAAO;CACtC,KAAK,eAAE,QAAQ,YAAY;CAC3B,OAAO;CACP,QAAQ;CAMR,eAAe,oBAAoB,SAAS;CAG5C,UAAU,iBAAiB,SAAS;AACtC,CAAC;AAED,IAAa,qBAAqB,eAAE,OAAO;CACzC,KAAK,eAAE,QAAQ,eAAe;CAC9B,OAAO;CACP,KAAK;CACL,KAAK;AACP,CAAC;AAED,IAAa,mBAAmB,eAAE,OAAO;CACvC,KAAK,eAAE,QAAQ,aAAa;CAC5B,OAAO;CACP,QAAQ,eAAE,OAAO;AACnB,CAAC;AAED,IAAa,kBAAkB,eAAE,OAAO;CACtC,KAAK,eAAE,QAAQ,YAAY;CAC3B,IAAI;AACN,CAAC;AAED,IAAa,oBAAoB,eAAE,OAAO;CACxC,KAAK,eAAE,QAAQ,cAAc;CAC7B,MAAM;AACR,CAAC;AAED,IAAa,kBAAkB,eAAE,OAAO;CACtC,KAAK,eAAE,QAAQ,YAAY;CAC3B,IAAI;AACN,CAAC;AAED,IAAa,uBAAuB,eAAE,OAAO;CAC3C,MAAM,eAAE,OAAO;CACf,WAAW,eAAE,OAAO;CACpB,YAAY,eAAE,OAAO,EAAE,SAAS;AAClC,CAAC;AAMD,IAAa,iBAAiB,eAAE,OAAO;CACrC,KAAK,eAAE,QAAQ,UAAU;CAEzB,UAAU;CAIV,SAAS,eAAE,MAAM,eAAE,OAAO,CAAC;CAM3B,WAAW,eAAE,OAAO;CAYpB,WAAW,eAAE,MAAM,eAAE,MAAM,eAAe,CAAC;CAS3C,eAAe,eACZ,OAAO;EAwBN,UAAU,2BAA2B,SAAS;EAM9C,SAAS;CACX,CAAC,EACA,SAAS;CAIZ,QAAQ,qBAAqB,SAAS;AACxC,CAAC;AAID,IAAa,0BAA0B,eAAE,OAAO;CAC9C,KAAK,eAAE,QAAQ,oBAAoB;CAEnC,UAAU;CAIV,SAAS,eAAE,MAAM,eAAE,OAAO,CAAC;CAM3B,WAAW,eAAE,OAAO;CAepB,eAAe,eACZ,OAAO,EAAC,UAAU,2BAA2B,SAAS,EAAC,CAAC,EACxD,SAAS;CAIZ,QAAQ,qBAAqB,SAAS;AACxC,CAAC;AAyBD,IAAa,mBAAmB,eAAE,MAChC,cACA,cACA,cACA,gBACA,cACF;AAGA,IAAM,iBAAiB;CACrB;CACA;CACA;CACA;CACA;AACF;AAE6B,aAAe,GAAG,cAAc;AAS7D,IAAM,gBAAgB;CACpB;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;AACF;AAEA,IAAa,qBAAqB,eAAE,MAAM,GAAG,aAAa;AAE3B,aAAe,GAAG,gBAAgB;AAmBjE,IAAM,qBAAqB,IAAI,IAAY,gBAAgB;AAE3D,SAAgB,eAAe,QAAwC;CACrE,OAAO,mBAAmB,IAAI,OAAO,GAAG;AAC1C;AAEA,IAAM,mBAAmB,IAAI,IAAY,cAAc;AAEvD,SAAgB,aAAa,QAAsC;CACjE,OAAO,iBAAiB,IAAI,OAAO,GAAG;AACxC"}
+{"version":3,"file":"data.js","names":["v","v.literalUnion"],"sources":["../../../../../../../../zero-cache/src/services/change-source/protocol/current/data.ts"],"sourcesContent":["/**\n * Data plane messages encapsulate changes that are sent by ChangeSources,\n * forwarded / fanned out to subscribers by the ChangeStreamerService, and\n * stored in the Change DB for catchup of old subscribers.\n */\n\nimport {\n  jsonValueSchema,\n  type JSONObject,\n} from '../../../../../../shared/src/bigint-json.ts';\nimport {must} from '../../../../../../shared/src/must.ts';\nimport * as v from '../../../../../../shared/src/valita.ts';\nimport {columnSpec, indexSpec, tableSpec} from '../../../../db/specs.ts';\nimport type {Satisfies} from '../../../../types/satisfies.ts';\nimport {jsonObjectSchema} from './json.ts';\nimport {schemaChangeTags} from './schema-change-tags.ts';\n\nexport const beginSchema = v.object({\n  tag: v.literal('begin'),\n  // The format of values of \"json\"-typed columns (e.g. \"JSON\" and \"JSONB\").\n  // - 'p' is for parsed JSON, which may include JSON values or JSON objects.\n  //   These values are parsed and stringified at every process boundary\n  //   between the change-source and the replica.\n  // - 's' is for stringified JSON. These values skip the parsing and\n  //   stringification, and are directly ferried to the replica as a JSON\n  //   string. For JSON values this improves performance by 20~25% in the\n  //   change-streamer and 25~30% in the replicator.\n  //\n  // If absent, the format is assumed to be 'p' (parsed JSON objects/values).\n  json: v.literalUnion('p', 's').optional(),\n\n  // Directs the change-streamer to skip the ACK for the corresponding commit.\n  skipAck: v.boolean().optional(),\n});\n\nexport const commitSchema = v.object({\n  tag: v.literal('commit'),\n\n  // Millisecond epoch at which the transaction committed upstream, as reported\n  // by the upstream database (e.g. the `commitTime` of the Postgres logical\n  // replication Commit message).\n  //\n  // This is the origin timestamp of the end-to-end serving lag measurement:\n  // it is carried through the replication stream to the ViewSyncer, which\n  // records `now - commitTimeMs` once the change has been poked to clients.\n  //\n  // Optional because it is absent from changes written to the Change DB by\n  // older versions, which are replayed verbatim during catchup, and from\n  // ChangeSources that do not report a commit time.\n  commitTimeMs: v.number().optional(),\n});\n\nexport const rollbackSchema = v.object({\n  tag: v.literal('rollback'),\n});\n\nconst rowKeySchema = v.object({\n  // The columns used to identify a row in insert, update, and delete changes.\n  columns: v.array(v.string()),\n\n  // An optional qualifier identifying how the key is chosen. Currently this\n  // is postgres-specific, describing the REPLICA IDENTITY, for which replica\n  // identity 'full' (FULL) is handled differently; the replicator handles\n  // these tables by extracting a row key from the full row based on the\n  // table's PRIMARY KEY or UNIQUE INDEX.\n  type: v.literalUnion('default', 'nothing', 'full', 'index').optional(),\n});\n\nexport const relationSchema = v\n  .object({\n    schema: v.string(),\n    name: v.string(),\n\n    // This will become required.\n    rowKey: rowKeySchema.optional(),\n\n    /** Deprecated: set the rowKey.columns instead. */\n    keyColumns: v.array(v.string()).optional(),\n    /** Deprecated: set the rowKey.columns instead. */\n    replicaIdentity: v\n      .literalUnion('default', 'nothing', 'full', 'index')\n      .optional(),\n  })\n  .map(rel => {\n    const {rowKey, ...rest} = rel;\n    if (rowKey) {\n      return {...rest, rowKey};\n    }\n    return {\n      ...rest,\n      rowKey: {\n        columns: must(rel.keyColumns),\n        type: rel.replicaIdentity,\n      },\n    };\n  });\n\n// The eventual fate of relationSchema\nexport const newRelationSchema = v.object({\n  schema: v.string(),\n  name: v.string(),\n\n  rowKey: rowKeySchema,\n});\n\n// TableMetadata contains table-related configuration that does not affect the\n// actual data in the table, but rather how the table's change messages are\n// handled. The is an opaque object that clients must track (and update) based\n// on `create-table`, `add-column`, and `table-update-metadata` messages, and\n// pass in BackfillRequests when there are columns to be backfilled.\n//\n// Note that the backfill-related change-source implementation does, however,\n// rely on the rowKey (columns) being specified in the message.\nexport const tableMetadataSchema = v\n  .object({rowKey: v.record(jsonValueSchema)})\n  .rest(jsonValueSchema);\n\nexport type TableMetadata = v.Infer<typeof tableMetadataSchema>;\n\nexport const rowSchema = v.record(jsonValueSchema);\n\nexport const insertSchema = v.object({\n  tag: v.literal('insert'),\n  relation: relationSchema,\n  new: rowSchema,\n});\n\nexport const updateSchema = v.object({\n  tag: v.literal('update'),\n  relation: relationSchema,\n  // `key` is present if the update changed the key of the row, or if the\n  // table's replicaIdentity === 'full'\n  key: rowSchema.nullable(),\n  // `new` is the full row (and not just the updated columns). This is\n  // necessary for \"catchup\" replication scenarios such as adding tables\n  // to a publication, or resharding.\n  new: rowSchema,\n});\n\nexport const deleteSchema = v.object({\n  tag: v.literal('delete'),\n  relation: relationSchema,\n  // key is the full row if replicaIdentity === 'full'\n  key: rowSchema,\n});\n\nexport const truncateSchema = v.object({\n  tag: v.literal('truncate'),\n  relations: v.array(relationSchema),\n});\n\nexport const identifierSchema = v.object({\n  schema: v.string(),\n  name: v.string(),\n});\n\nexport type Identifier = v.Infer<typeof identifierSchema>;\n\n// A BackfillID is an upstream specific stable identifier for a column\n// that needs backfilling. This id is used to ensure that the schema, table,\n// and column names of a requested backfill still match the original\n// underlying upstream ID.\n//\n// The change-streamer stores these IDs as opaque values while a column is\n// being backfilled, and initiates new change-source streams with the IDs\n// in order to restart backfills that did not complete in previous sessions.\nexport const backfillIDSchema = jsonObjectSchema;\n\nexport type BackfillID = v.Infer<typeof backfillIDSchema>;\n\nexport const backfillProgressMarkSchema = v\n  .object({\n    // A lexicographically sortable string delineating the progress of a\n    // backfill. This is used to resume a backfill that did not complete.\n    progressMark: v.string(),\n\n    // Optionally, a timeline with which the progressMark is associated.\n    // A timeline is an opaque string that determines if backfill streams\n    // are \"compatible\" with each other, i.e. can be merged in a way that\n    // avoids gaps. progressMarks from different timelines are not\n    // compatible. If two timelines differ, a new backfill is started\n    // from scratch in order to get all subscribers on the same timeline.\n    //\n    // Timeline changes should be rare events (e.g. Postgres VACUUM FULL).\n    timeline: v.string().optional(),\n  })\n  // All other fields should be preserved and returned in BackfillRequests.\n  .rest(v.union(v.number(), v.string()));\n\nexport type BackfillProgressMark = v.Infer<typeof backfillProgressMarkSchema>;\n\nexport const createTableSchema = v.object({\n  tag: v.literal('create-table'),\n  spec: tableSpec,\n\n  // This must be set by change source implementations that support\n  // table/column backfill.\n  //\n  // TODO: to simplify the protocol, see if we can make this required\n  metadata: tableMetadataSchema.optional(),\n\n  // Indicate that columns of the table require backfilling. These columns\n  // should be created on the replica but not yet synced the clients.\n  //\n  // ## State Persistence\n  //\n  // To obviate the need for change-source implementations to persist state\n  // related to backfill progress, the change-source only tracks backfills\n  // **for the current session**. In the event that the session is interrupted\n  // before columns have been fully backfilled, it is the responsibility of the\n  // change-streamer to send {@link BackfillRequest}s when it reconnects.\n  //\n  // This means that the change-streamer must track and persist:\n  // * the backfill IDs of the columns requiring backfilling\n  // * the most current table metadata of the associated table(s)\n  //\n  // The change-streamer then uses this information to send backfill requests\n  // when it reconnects.\n  backfill: v.record(backfillIDSchema).optional(),\n});\n\nexport const renameTableSchema = v.object({\n  tag: v.literal('rename-table'),\n  old: identifierSchema,\n  new: identifierSchema,\n});\n\nexport const updateTableMetadataSchema = v.object({\n  tag: v.literal('update-table-metadata'),\n  table: identifierSchema,\n  old: tableMetadataSchema,\n  new: tableMetadataSchema,\n});\n\nconst columnSchema = v.object({\n  name: v.string(),\n  spec: columnSpec,\n});\n\nexport const addColumnSchema = v.object({\n  tag: v.literal('add-column'),\n  table: identifierSchema,\n  column: columnSchema,\n\n  // This must be set by change source implementations that support\n  // table/column backfill.\n  //\n  // TODO: to simplify the protocol, see if we can make this required\n  tableMetadata: tableMetadataSchema.optional(),\n\n  // See documentation for the `backfill` field of the `create-table` change.\n  backfill: backfillIDSchema.optional(),\n});\n\nexport const updateColumnSchema = v.object({\n  tag: v.literal('update-column'),\n  table: identifierSchema,\n  old: columnSchema,\n  new: columnSchema,\n});\n\nexport const dropColumnSchema = v.object({\n  tag: v.literal('drop-column'),\n  table: identifierSchema,\n  column: v.string(),\n});\n\nexport const dropTableSchema = v.object({\n  tag: v.literal('drop-table'),\n  id: identifierSchema,\n});\n\nexport const createIndexSchema = v.object({\n  tag: v.literal('create-index'),\n  spec: indexSpec,\n});\n\nexport const dropIndexSchema = v.object({\n  tag: v.literal('drop-index'),\n  id: identifierSchema,\n});\n\nexport const downloadStatusSchema = v.object({\n  rows: v.number(),\n  totalRows: v.number(),\n  totalBytes: v.number().optional(),\n});\n\nexport type DownloadStatus = v.Infer<typeof downloadStatusSchema>;\n\n// A batch of rows from a single table containing column values\n// to be backfilled.\nexport const backfillSchema = v.object({\n  tag: v.literal('backfill'),\n\n  relation: newRelationSchema,\n\n  // The columns to be backfilled. `rowKey` columns are automatically excluded,\n  // which means that this field may be empty.\n  columns: v.array(v.string()),\n\n  // The watermark at which the backfill data was queried. Note that this\n  // generally will be different from the commit watermarks of the main change\n  // stream, and in particular, the commit watermark of the backfill change's\n  // enclosing transaction.\n  watermark: v.string(),\n\n  // A batch of row values, each row consisting of the `rowKey`\n  // values, followed by the `column` values, in the same order in which\n  // the column names appear in their respective fields, e.g.\n  //\n  // ```\n  // [\n  //   [...rowKeyValues, ...columnValues],  // row 1\n  //   [...rowKeyValues, ...columnValues],  // row 2\n  // ]\n  // ```\n  rowValues: v.array(v.array(jsonValueSchema)),\n\n  // Progress marks indicating the position of this backfill message\n  // within the stream, for resumption / continuity enforcement.\n  //\n  // This should be considered non-optional as of protocol v8. It is only\n  // absent in messages from pre-v8 replication-managers (including messages\n  // persisted in a change-log by one), in which case subscribers accept the\n  // message unconditionally (i.e. the legacy behavior).\n  progressMarks: v\n    .object({\n      // The progress mark of the preceding backfill message, from which\n      // the current backfill message is a continuation. (This may be a\n      // \"logical\" continuation at which a new backfill stream is started when\n      // resuming from a previous stream.)\n      //\n      // For the first backfill message of a stream starting from scratch,\n      // this field is unset. When the field is unset, subscribers should\n      // accept the backfill message unconditionally (i.e. starting a\n      // backfill from scratch is always safe).\n      //\n      // If the field is set, subscribers should only accept the message if\n      // it is of the same timeline as that of its own tracked progressMark,\n      // and is contiguous with or earlier than its own progressMark.\n      //\n      // In other words, it should _ignore_ backfill messages from other\n      // timelines, or messages that create a gap; this can happen when\n      // receiving messages from before this subscriber's backfill requests\n      // are taken into account.\n      //\n      // Implementation note: The change-streamer applies this same policy\n      // (see `acceptBackfill()` in `protocol/backfill-progress.ts`) to track\n      // the backfill state of each subscriber, which it uses to determine the\n      // BackfillRequests with which to (re)start change streams.\n      previous: backfillProgressMarkSchema.optional(),\n\n      // The progressMark delineating the position of the last row in this\n      // message. In the steady state, this will be the `previous`\n      // progressMark of the next backfill message, unless a rewind is\n      // requested, or new timeline encountered.\n      current: backfillProgressMarkSchema,\n    })\n    .optional(),\n\n  // Optionally includes the progress of the backfill operation,\n  // for display purposes.\n  status: downloadStatusSchema.optional(),\n});\n\n// Indicates that the backfill for the specified columns have\n// been successfully backfilled and can be published to clients.\nexport const backfillCompletedSchema = v.object({\n  tag: v.literal('backfill-completed'),\n\n  relation: newRelationSchema,\n\n  // The columns to be backfilled. `rowKey` columns are automatically excluded,\n  // which means that this field may be empty.\n  columns: v.array(v.string()),\n\n  // The watermark at which the backfill data was queried. Note that this\n  // generally will be different from the commit watermarks of the main change\n  // stream, and in particular, the commit watermark of the backfill change's\n  // enclosing transaction.\n  watermark: v.string(),\n\n  // The progress mark of the last backfill message of the run, i.e. the\n  // `current` progressMark of the last `backfill` message, or the progressMark\n  // from which the run was resumed if it produced no `backfill` messages.\n  // Subscribers apply the same continuity policy as for `backfill` messages,\n  // and only consider the columns backfilled if the run is contiguous with\n  // their own progress.\n  //\n  // `previous` is only unset for a run that started from scratch and\n  // produced no rows (e.g. an empty table), which is always safe to accept.\n  //\n  // This should be considered non-optional as of protocol v8. It is only\n  // absent in messages from pre-v8 replication-managers, in which case\n  // subscribers accept the message unconditionally (i.e. legacy behavior).\n  progressMarks: v\n    .object({previous: backfillProgressMarkSchema.optional()})\n    .optional(),\n\n  // Optionally includes the final status of the backfill operation,\n  // for display purposes.\n  status: downloadStatusSchema.optional(),\n});\n\nexport type MessageBegin = v.Infer<typeof beginSchema>;\nexport type MessageCommit = v.Infer<typeof commitSchema>;\nexport type MessageRollback = v.Infer<typeof rollbackSchema>;\n\nexport type MessageRelation = v.Infer<typeof relationSchema>;\nexport type MessageInsert = v.Infer<typeof insertSchema>;\nexport type MessageUpdate = v.Infer<typeof updateSchema>;\nexport type MessageDelete = v.Infer<typeof deleteSchema>;\nexport type MessageTruncate = v.Infer<typeof truncateSchema>;\n\nexport type MessageBackfill = v.Infer<typeof backfillSchema>;\n\nexport type TableCreate = v.Infer<typeof createTableSchema>;\nexport type TableRename = v.Infer<typeof renameTableSchema>;\nexport type TableUpdateMetadata = v.Infer<typeof updateTableMetadataSchema>;\nexport type ColumnAdd = v.Infer<typeof addColumnSchema>;\nexport type ColumnUpdate = v.Infer<typeof updateColumnSchema>;\nexport type ColumnDrop = v.Infer<typeof dropColumnSchema>;\nexport type TableDrop = v.Infer<typeof dropTableSchema>;\nexport type IndexCreate = v.Infer<typeof createIndexSchema>;\nexport type IndexDrop = v.Infer<typeof dropIndexSchema>;\nexport type BackfillCompleted = v.Infer<typeof backfillCompletedSchema>;\n\nexport const dataChangeSchema = v.union(\n  insertSchema,\n  updateSchema,\n  deleteSchema,\n  truncateSchema,\n  backfillSchema,\n);\n\n// Note: keep in sync or the tag tests will fail\nconst dataChangeTags = [\n  'insert',\n  'update',\n  'delete',\n  'truncate',\n  'backfill',\n] as const;\n\nconst dataChangeTagsSchema = v.literalUnion(...dataChangeTags);\n\nexport type DataChange = Satisfies<\n  JSONObject, // guarantees serialization over IPC or network\n  v.Infer<typeof dataChangeSchema>\n>;\n\nexport type DataChangeTag = v.Infer<typeof dataChangeTagsSchema>;\n\nconst schemaChanges = [\n  createTableSchema,\n  renameTableSchema,\n  updateTableMetadataSchema,\n  addColumnSchema,\n  updateColumnSchema,\n  dropColumnSchema,\n  dropTableSchema,\n  createIndexSchema,\n  dropIndexSchema,\n  backfillCompletedSchema,\n] as const;\n\nexport const schemaChangeSchema = v.union(...schemaChanges);\n\nconst schemaChangeTagsSchema = v.literalUnion(...schemaChangeTags);\n\nexport type SchemaChange = Satisfies<\n  JSONObject,\n  v.Infer<typeof schemaChangeSchema>\n>;\n\nexport type SchemaChangeTag = v.Infer<typeof schemaChangeTagsSchema>;\n\nexport type DataOrSchemaChange = DataChange | SchemaChange;\n\nexport type Change =\n  | MessageBegin\n  | DataOrSchemaChange\n  | MessageCommit\n  | MessageRollback;\n\nexport type ChangeTag = Change['tag'];\n\nconst schemaChangeTagSet = new Set<string>(schemaChangeTags);\n\nexport function isSchemaChange(change: Change): change is SchemaChange {\n  return schemaChangeTagSet.has(change.tag);\n}\n\nconst dataChangeTagSet = new Set<string>(dataChangeTags);\n\nexport function isDataChange(change: Change): change is DataChange {\n  return dataChangeTagSet.has(change.tag);\n}\n"],"mappings":";;;;;;;;;;;;AAiBA,IAAa,cAAcA,eAAE,OAAO;CAClC,KAAKA,eAAE,QAAQ,OAAO;CAWtB,MAAMC,aAAe,KAAK,GAAG,CAAC,CAAC,SAAS;CAGxC,SAASD,eAAE,QAAQ,CAAC,CAAC,SAAS;AAChC,CAAC;AAED,IAAa,eAAeA,eAAE,OAAO;CACnC,KAAKA,eAAE,QAAQ,QAAQ;CAavB,cAAcA,eAAE,OAAO,CAAC,CAAC,SAAS;AACpC,CAAC;AAED,IAAa,iBAAiBA,eAAE,OAAO,EACrC,KAAKA,eAAE,QAAQ,UAAU,EAC3B,CAAC;AAED,IAAM,eAAeA,eAAE,OAAO;CAE5B,SAASA,eAAE,MAAMA,eAAE,OAAO,CAAC;CAO3B,MAAMC,aAAe,WAAW,WAAW,QAAQ,OAAO,CAAC,CAAC,SAAS;AACvE,CAAC;AAED,IAAa,iBAAiBD,eAC3B,OAAO;CACN,QAAQA,eAAE,OAAO;CACjB,MAAMA,eAAE,OAAO;CAGf,QAAQ,aAAa,SAAS;;CAG9B,YAAYA,eAAE,MAAMA,eAAE,OAAO,CAAC,CAAC,CAAC,SAAS;;CAEzC,iBAAiBC,aACD,WAAW,WAAW,QAAQ,OAAO,CAAC,CACnD,SAAS;AACd,CAAC,CAAC,CACD,KAAI,QAAO;CACV,MAAM,EAAC,QAAQ,GAAG,SAAQ;CAC1B,IAAI,QACF,OAAO;EAAC,GAAG;EAAM;CAAM;CAEzB,OAAO;EACL,GAAG;EACH,QAAQ;GACN,SAAS,KAAK,IAAI,UAAU;GAC5B,MAAM,IAAI;EACZ;CACF;AACF,CAAC;AAGH,IAAa,oBAAoBD,eAAE,OAAO;CACxC,QAAQA,eAAE,OAAO;CACjB,MAAMA,eAAE,OAAO;CAEf,QAAQ;AACV,CAAC;AAUD,IAAa,sBAAsBA,eAChC,OAAO,EAAC,QAAQA,eAAE,OAAO,eAAe,EAAC,CAAC,CAAC,CAC3C,KAAK,eAAe;AAIvB,IAAa,YAAYA,eAAE,OAAO,eAAe;AAEjD,IAAa,eAAeA,eAAE,OAAO;CACnC,KAAKA,eAAE,QAAQ,QAAQ;CACvB,UAAU;CACV,KAAK;AACP,CAAC;AAED,IAAa,eAAeA,eAAE,OAAO;CACnC,KAAKA,eAAE,QAAQ,QAAQ;CACvB,UAAU;CAGV,KAAK,UAAU,SAAS;CAIxB,KAAK;AACP,CAAC;AAED,IAAa,eAAeA,eAAE,OAAO;CACnC,KAAKA,eAAE,QAAQ,QAAQ;CACvB,UAAU;CAEV,KAAK;AACP,CAAC;AAED,IAAa,iBAAiBA,eAAE,OAAO;CACrC,KAAKA,eAAE,QAAQ,UAAU;CACzB,WAAWA,eAAE,MAAM,cAAc;AACnC,CAAC;AAED,IAAa,mBAAmBA,eAAE,OAAO;CACvC,QAAQA,eAAE,OAAO;CACjB,MAAMA,eAAE,OAAO;AACjB,CAAC;AAYD,IAAa,mBAAmB;AAIhC,IAAa,6BAA6BA,eACvC,OAAO;CAGN,cAAcA,eAAE,OAAO;CAUvB,UAAUA,eAAE,OAAO,CAAC,CAAC,SAAS;AAChC,CAAC,CAAC,CAED,KAAKA,eAAE,MAAMA,eAAE,OAAO,GAAGA,eAAE,OAAO,CAAC,CAAC;AAIvC,IAAa,oBAAoBA,eAAE,OAAO;CACxC,KAAKA,eAAE,QAAQ,cAAc;CAC7B,MAAM;CAMN,UAAU,oBAAoB,SAAS;CAmBvC,UAAUA,eAAE,OAAO,gBAAgB,CAAC,CAAC,SAAS;AAChD,CAAC;AAED,IAAa,oBAAoBA,eAAE,OAAO;CACxC,KAAKA,eAAE,QAAQ,cAAc;CAC7B,KAAK;CACL,KAAK;AACP,CAAC;AAED,IAAa,4BAA4BA,eAAE,OAAO;CAChD,KAAKA,eAAE,QAAQ,uBAAuB;CACtC,OAAO;CACP,KAAK;CACL,KAAK;AACP,CAAC;AAED,IAAM,eAAeA,eAAE,OAAO;CAC5B,MAAMA,eAAE,OAAO;CACf,MAAM;AACR,CAAC;AAED,IAAa,kBAAkBA,eAAE,OAAO;CACtC,KAAKA,eAAE,QAAQ,YAAY;CAC3B,OAAO;CACP,QAAQ;CAMR,eAAe,oBAAoB,SAAS;CAG5C,UAAU,iBAAiB,SAAS;AACtC,CAAC;AAED,IAAa,qBAAqBA,eAAE,OAAO;CACzC,KAAKA,eAAE,QAAQ,eAAe;CAC9B,OAAO;CACP,KAAK;CACL,KAAK;AACP,CAAC;AAED,IAAa,mBAAmBA,eAAE,OAAO;CACvC,KAAKA,eAAE,QAAQ,aAAa;CAC5B,OAAO;CACP,QAAQA,eAAE,OAAO;AACnB,CAAC;AAED,IAAa,kBAAkBA,eAAE,OAAO;CACtC,KAAKA,eAAE,QAAQ,YAAY;CAC3B,IAAI;AACN,CAAC;AAED,IAAa,oBAAoBA,eAAE,OAAO;CACxC,KAAKA,eAAE,QAAQ,cAAc;CAC7B,MAAM;AACR,CAAC;AAED,IAAa,kBAAkBA,eAAE,OAAO;CACtC,KAAKA,eAAE,QAAQ,YAAY;CAC3B,IAAI;AACN,CAAC;AAED,IAAa,uBAAuBA,eAAE,OAAO;CAC3C,MAAMA,eAAE,OAAO;CACf,WAAWA,eAAE,OAAO;CACpB,YAAYA,eAAE,OAAO,CAAC,CAAC,SAAS;AAClC,CAAC;AAMD,IAAa,iBAAiBA,eAAE,OAAO;CACrC,KAAKA,eAAE,QAAQ,UAAU;CAEzB,UAAU;CAIV,SAASA,eAAE,MAAMA,eAAE,OAAO,CAAC;CAM3B,WAAWA,eAAE,OAAO;CAYpB,WAAWA,eAAE,MAAMA,eAAE,MAAM,eAAe,CAAC;CAS3C,eAAeA,eACZ,OAAO;EAwBN,UAAU,2BAA2B,SAAS;EAM9C,SAAS;CACX,CAAC,CAAC,CACD,SAAS;CAIZ,QAAQ,qBAAqB,SAAS;AACxC,CAAC;AAID,IAAa,0BAA0BA,eAAE,OAAO;CAC9C,KAAKA,eAAE,QAAQ,oBAAoB;CAEnC,UAAU;CAIV,SAASA,eAAE,MAAMA,eAAE,OAAO,CAAC;CAM3B,WAAWA,eAAE,OAAO;CAepB,eAAeA,eACZ,OAAO,EAAC,UAAU,2BAA2B,SAAS,EAAC,CAAC,CAAC,CACzD,SAAS;CAIZ,QAAQ,qBAAqB,SAAS;AACxC,CAAC;AAyBD,IAAa,mBAAmBA,eAAE,MAChC,cACA,cACA,cACA,gBACA,cACF;AAGA,IAAM,iBAAiB;CACrB;CACA;CACA;CACA;CACA;AACF;AAE6BC,aAAe,GAAG,cAAc;AAS7D,IAAM,gBAAgB;CACpB;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;AACF;AAEA,IAAa,qBAAqBD,eAAE,MAAM,GAAG,aAAa;AAE3BC,aAAe,GAAG,gBAAgB;AAmBjE,IAAM,qBAAqB,IAAI,IAAY,gBAAgB;AAE3D,SAAgB,eAAe,QAAwC;CACrE,OAAO,mBAAmB,IAAI,OAAO,GAAG;AAC1C;AAEA,IAAM,mBAAmB,IAAI,IAAY,cAAc;AAEvD,SAAgB,aAAa,QAAsC;CACjE,OAAO,iBAAiB,IAAI,OAAO,GAAG;AACxC"}

package/out/zero-cache/src/services/change-source/protocol/current/downstream.js.map

@@ -1 +1 @@
-{"version":3,"file":"downstream.js","names":[],"sources":["../../../../../../../../zero-cache/src/services/change-source/protocol/current/downstream.ts"],"sourcesContent":["import * as v from '../../../../../../shared/src/valita.ts';\nimport {resetRequiredSchema} from './control.ts';\nimport {\n  beginSchema,\n  commitSchema,\n  dataChangeSchema,\n  rollbackSchema,\n  schemaChangeSchema,\n} from './data.ts';\nimport {downstreamStatusMessageSchema} from './status.ts';\n\nconst begin = v.tuple([\n  v.literal('begin'),\n  beginSchema,\n  v.object({commitWatermark: v.string()}),\n]);\nconst data = v.tuple([\n  v.literal('data'),\n  v.union(dataChangeSchema, schemaChangeSchema),\n]);\nconst commit = v.tuple([\n  v.literal('commit'),\n  commitSchema,\n  v.object({watermark: v.string()}),\n]);\nconst rollback = v.tuple([v.literal('rollback'), rollbackSchema]);\n\nexport type Begin = v.Infer<typeof begin>;\nexport type Data = v.Infer<typeof data>;\nexport type Commit = v.Infer<typeof commit>;\nexport type Rollback = v.Infer<typeof rollback>;\n\nexport const changeStreamDataSchema = v.union(begin, data, commit, rollback);\nexport type ChangeStreamData = v.Infer<typeof changeStreamDataSchema>;\n\nexport const changeStreamControlSchema = v.tuple([\n  v.literal('control'),\n  resetRequiredSchema, // TODO: Add statusRequestedSchema\n]);\nexport type ChangeStreamControl = v.Infer<typeof changeStreamControlSchema>;\n\n/** Downstream messages consist of data plane and control plane messages. */\nexport const changeStreamMessageSchema = v.union(\n  changeStreamDataSchema,\n  changeStreamControlSchema,\n  downstreamStatusMessageSchema,\n);\n\nexport type ChangeStreamMessage = v.Infer<typeof changeStreamMessageSchema>;\n"],"mappings":";;;;;AAWA,IAAM,QAAQ,eAAE,MAAM;CACpB,eAAE,QAAQ,OAAO;CACjB;CACA,eAAE,OAAO,EAAC,iBAAiB,eAAE,OAAO,EAAC,CAAC;AACxC,CAAC;AACD,IAAM,OAAO,eAAE,MAAM,CACnB,eAAE,QAAQ,MAAM,GAChB,eAAE,MAAM,kBAAkB,kBAAkB,CAC9C,CAAC;AACD,IAAM,SAAS,eAAE,MAAM;CACrB,eAAE,QAAQ,QAAQ;CAClB;CACA,eAAE,OAAO,EAAC,WAAW,eAAE,OAAO,EAAC,CAAC;AAClC,CAAC;AACD,IAAM,WAAW,eAAE,MAAM,CAAC,eAAE,QAAQ,UAAU,GAAG,cAAc,CAAC;AAOhE,IAAa,yBAAyB,eAAE,MAAM,OAAO,MAAM,QAAQ,QAAQ;AAG3E,IAAa,4BAA4B,eAAE,MAAM,CAC/C,eAAE,QAAQ,SAAS,GACnB,mBACF,CAAC;;AAID,IAAa,4BAA4B,eAAE,MACzC,wBACA,2BACA,6BACF"}
+{"version":3,"file":"downstream.js","names":["v"],"sources":["../../../../../../../../zero-cache/src/services/change-source/protocol/current/downstream.ts"],"sourcesContent":["import * as v from '../../../../../../shared/src/valita.ts';\nimport {resetRequiredSchema} from './control.ts';\nimport {\n  beginSchema,\n  commitSchema,\n  dataChangeSchema,\n  rollbackSchema,\n  schemaChangeSchema,\n} from './data.ts';\nimport {downstreamStatusMessageSchema} from './status.ts';\n\nconst begin = v.tuple([\n  v.literal('begin'),\n  beginSchema,\n  v.object({commitWatermark: v.string()}),\n]);\nconst data = v.tuple([\n  v.literal('data'),\n  v.union(dataChangeSchema, schemaChangeSchema),\n]);\nconst commit = v.tuple([\n  v.literal('commit'),\n  commitSchema,\n  v.object({watermark: v.string()}),\n]);\nconst rollback = v.tuple([v.literal('rollback'), rollbackSchema]);\n\nexport type Begin = v.Infer<typeof begin>;\nexport type Data = v.Infer<typeof data>;\nexport type Commit = v.Infer<typeof commit>;\nexport type Rollback = v.Infer<typeof rollback>;\n\nexport const changeStreamDataSchema = v.union(begin, data, commit, rollback);\nexport type ChangeStreamData = v.Infer<typeof changeStreamDataSchema>;\n\nexport const changeStreamControlSchema = v.tuple([\n  v.literal('control'),\n  resetRequiredSchema, // TODO: Add statusRequestedSchema\n]);\nexport type ChangeStreamControl = v.Infer<typeof changeStreamControlSchema>;\n\n/** Downstream messages consist of data plane and control plane messages. */\nexport const changeStreamMessageSchema = v.union(\n  changeStreamDataSchema,\n  changeStreamControlSchema,\n  downstreamStatusMessageSchema,\n);\n\nexport type ChangeStreamMessage = v.Infer<typeof changeStreamMessageSchema>;\n"],"mappings":";;;;;AAWA,IAAM,QAAQA,eAAE,MAAM;CACpBA,eAAE,QAAQ,OAAO;CACjB;CACAA,eAAE,OAAO,EAAC,iBAAiBA,eAAE,OAAO,EAAC,CAAC;AACxC,CAAC;AACD,IAAM,OAAOA,eAAE,MAAM,CACnBA,eAAE,QAAQ,MAAM,GAChBA,eAAE,MAAM,kBAAkB,kBAAkB,CAC9C,CAAC;AACD,IAAM,SAASA,eAAE,MAAM;CACrBA,eAAE,QAAQ,QAAQ;CAClB;CACAA,eAAE,OAAO,EAAC,WAAWA,eAAE,OAAO,EAAC,CAAC;AAClC,CAAC;AACD,IAAM,WAAWA,eAAE,MAAM,CAACA,eAAE,QAAQ,UAAU,GAAG,cAAc,CAAC;AAOhE,IAAa,yBAAyBA,eAAE,MAAM,OAAO,MAAM,QAAQ,QAAQ;AAG3E,IAAa,4BAA4BA,eAAE,MAAM,CAC/CA,eAAE,QAAQ,SAAS,GACnB,mBACF,CAAC;;AAID,IAAa,4BAA4BA,eAAE,MACzC,wBACA,2BACA,6BACF"}

package/out/zero-cache/src/services/change-source/protocol/current/status.js.map

@@ -1 +1 @@
-{"version":3,"file":"status.js","names":[],"sources":["../../../../../../../../zero-cache/src/services/change-source/protocol/current/status.ts"],"sourcesContent":["import * as v from '../../../../../../shared/src/valita.ts';\nimport {changeSourceReportSchema} from '../../../replicator/reporter/report-schema.ts';\nimport {commitSchema} from './data.ts';\n\n/**\n * The downstream status messages contain metadata about the status\n * of the change-source. indicates whether it should be echoed\n * back in an upstream status message.\n */\nexport const downstreamStatusSchema = v.object({\n  // Indicates whether the status message should be echoed back\n  // in an upstream status message once the consumer has successfully\n  // processed/persisted all preceding changes.\n  ack: v.boolean().optional(() => true),\n\n  // Contains a lag report for recording end to end latency metrics.\n  lagReport: changeSourceReportSchema.optional(),\n});\n\nexport type DownstreamStatus = v.Infer<typeof downstreamStatusSchema>;\n\nexport const downstreamStatusMessageSchema = v.tuple([\n  v.literal('status'),\n  downstreamStatusSchema,\n  v.object({watermark: v.string()}),\n]);\n\n/**\n * The `zero-cache` will send the Commit payload to acknowledge a completed\n * transaction (unless the `skipAck` field was specified in the Begin message\n * of the transaction), and will echo back the downstream `status` message if\n * `ack` is true.\n */\nexport const upstreamStatusMessageSchema = v.tuple([\n  v.literal('status'),\n  v.union(downstreamStatusSchema, commitSchema),\n  v.object({watermark: v.string()}),\n]);\n\n/**\n * Status messages convey positional information from both the ChangeSource\n * and the `zero-cache`.\n *\n * A StatusMessage from the ChangeSource indicates a position in its change\n * log. Generally, the watermarks sent in `Commit` messages already convey\n * this information, but a StatusMessage may also be sent to indicate that the\n * log has progressed without any corresponding changes relevant to the\n * subscriber. The watermarks of commit messages and status messages must be\n * monotonic in the stream of messages from the ChangeSource.\n *\n * The `zero-cache` sends StatusMessages to the ChangeSource:\n *\n * * when it has processed a `Commit` received from the ChangeSource,\n *   unless the `Begin` message specified `skipAck`.\n *\n * * when it receives a `StatusMessage` and all preceding `Commit` messages\n *   have been processed\n *\n * This allows the ChangeSource to clean up change log entries appropriately.\n *\n * Note that StatusMessages from the ChangeSource are optional. If a\n * ChangeSource implementation can track subscriber progress and clean up\n * its change log purely from Commit-driven StatusMessages there is no need\n * for the ChangeSource to send StatusMessages.\n */\nexport type DownstreamStatusMessage = v.Infer<\n  typeof downstreamStatusMessageSchema\n>;\nexport type UpstreamStatusMessage = v.Infer<typeof upstreamStatusMessageSchema>;\n"],"mappings":";;;;;;;;;AASA,IAAa,yBAAyB,eAAE,OAAO;CAI7C,KAAK,eAAE,QAAQ,EAAE,eAAe,IAAI;CAGpC,WAAW,yBAAyB,SAAS;AAC/C,CAAC;AAID,IAAa,gCAAgC,eAAE,MAAM;CACnD,eAAE,QAAQ,QAAQ;CAClB;CACA,eAAE,OAAO,EAAC,WAAW,eAAE,OAAO,EAAC,CAAC;AAClC,CAAC;;;;;;;AAQD,IAAa,8BAA8B,eAAE,MAAM;CACjD,eAAE,QAAQ,QAAQ;CAClB,eAAE,MAAM,wBAAwB,YAAY;CAC5C,eAAE,OAAO,EAAC,WAAW,eAAE,OAAO,EAAC,CAAC;AAClC,CAAC"}
+{"version":3,"file":"status.js","names":["v"],"sources":["../../../../../../../../zero-cache/src/services/change-source/protocol/current/status.ts"],"sourcesContent":["import * as v from '../../../../../../shared/src/valita.ts';\nimport {changeSourceReportSchema} from '../../../replicator/reporter/report-schema.ts';\nimport {commitSchema} from './data.ts';\n\n/**\n * The downstream status messages contain metadata about the status\n * of the change-source. indicates whether it should be echoed\n * back in an upstream status message.\n */\nexport const downstreamStatusSchema = v.object({\n  // Indicates whether the status message should be echoed back\n  // in an upstream status message once the consumer has successfully\n  // processed/persisted all preceding changes.\n  ack: v.boolean().optional(() => true),\n\n  // Contains a lag report for recording end to end latency metrics.\n  lagReport: changeSourceReportSchema.optional(),\n});\n\nexport type DownstreamStatus = v.Infer<typeof downstreamStatusSchema>;\n\nexport const downstreamStatusMessageSchema = v.tuple([\n  v.literal('status'),\n  downstreamStatusSchema,\n  v.object({watermark: v.string()}),\n]);\n\n/**\n * The `zero-cache` will send the Commit payload to acknowledge a completed\n * transaction (unless the `skipAck` field was specified in the Begin message\n * of the transaction), and will echo back the downstream `status` message if\n * `ack` is true.\n */\nexport const upstreamStatusMessageSchema = v.tuple([\n  v.literal('status'),\n  v.union(downstreamStatusSchema, commitSchema),\n  v.object({watermark: v.string()}),\n]);\n\n/**\n * Status messages convey positional information from both the ChangeSource\n * and the `zero-cache`.\n *\n * A StatusMessage from the ChangeSource indicates a position in its change\n * log. Generally, the watermarks sent in `Commit` messages already convey\n * this information, but a StatusMessage may also be sent to indicate that the\n * log has progressed without any corresponding changes relevant to the\n * subscriber. The watermarks of commit messages and status messages must be\n * monotonic in the stream of messages from the ChangeSource.\n *\n * The `zero-cache` sends StatusMessages to the ChangeSource:\n *\n * * when it has processed a `Commit` received from the ChangeSource,\n *   unless the `Begin` message specified `skipAck`.\n *\n * * when it receives a `StatusMessage` and all preceding `Commit` messages\n *   have been processed\n *\n * This allows the ChangeSource to clean up change log entries appropriately.\n *\n * Note that StatusMessages from the ChangeSource are optional. If a\n * ChangeSource implementation can track subscriber progress and clean up\n * its change log purely from Commit-driven StatusMessages there is no need\n * for the ChangeSource to send StatusMessages.\n */\nexport type DownstreamStatusMessage = v.Infer<\n  typeof downstreamStatusMessageSchema\n>;\nexport type UpstreamStatusMessage = v.Infer<typeof upstreamStatusMessageSchema>;\n"],"mappings":";;;;;;;;;AASA,IAAa,yBAAyBA,eAAE,OAAO;CAI7C,KAAKA,eAAE,QAAQ,CAAC,CAAC,eAAe,IAAI;CAGpC,WAAW,yBAAyB,SAAS;AAC/C,CAAC;AAID,IAAa,gCAAgCA,eAAE,MAAM;CACnDA,eAAE,QAAQ,QAAQ;CAClB;CACAA,eAAE,OAAO,EAAC,WAAWA,eAAE,OAAO,EAAC,CAAC;AAClC,CAAC;;;;;;;AAQD,IAAa,8BAA8BA,eAAE,MAAM;CACjDA,eAAE,QAAQ,QAAQ;CAClBA,eAAE,MAAM,wBAAwB,YAAY;CAC5CA,eAAE,OAAO,EAAC,WAAWA,eAAE,OAAO,EAAC,CAAC;AAClC,CAAC"}

package/out/zero-cache/src/services/change-source/protocol/current/upstream.js.map

@@ -1 +1 @@
-{"version":3,"file":"upstream.js","names":[],"sources":["../../../../../../../../zero-cache/src/services/change-source/protocol/current/upstream.ts"],"sourcesContent":["import * as v from '../../../../../../shared/src/valita.ts';\nimport {\n  backfillIDSchema,\n  backfillProgressMarkSchema,\n  identifierSchema,\n  tableMetadataSchema,\n} from './data.ts';\nimport {upstreamStatusMessageSchema} from './status.ts';\n\n/** At the moment, the only upstream messages are status messages.  */\nexport const changeSourceUpstreamSchema = upstreamStatusMessageSchema;\nexport type ChangeSourceUpstream = v.Infer<typeof changeSourceUpstreamSchema>;\n\n/**\n * Contains the information for requesting a backfill of columns in a table.\n * Backfills are automatically started for new tables and columns in a given\n * change stream session; however, if the session is terminated before the\n * backfill completes, it must be restarted with appropriate\n * {@link BackfillRequest}s in subsequent session(s), as backfill state is\n * ephemeral, per-replication-manager, and not persisted upstream.\n *\n * All replication subscribers track their backfill state, which includes\n * tracking any changes to the table name, column names, or table metadata,\n * and present that state to the `change-streamer` when starting a\n * subscription.\n *\n * From there the `change-streamer` manages requesting from the change-source\n * the superset (and minimum progressMark) of all subscriber-specified\n * backfills. While subscribers are connected, the change-streamer also tracks\n * this per-subscriber state in memory, so that it can properly resume\n * backfills on a new change-stream if disconnected from upstream.\n */\nexport const backfillRequestSchema = v.object({\n  table: identifierSchema.extend({\n    // The table metadata is set to null if it is never specified by the\n    // change-source.\n    metadata: tableMetadataSchema.nullable(),\n  }),\n  columns: v.record(\n    v.object({\n      id: backfillIDSchema,\n      progress: backfillProgressMarkSchema.optional(),\n    }),\n  ),\n});\n\nexport type BackfillRequest = v.Infer<typeof backfillRequestSchema>;\n"],"mappings":";;;;;AAUA,IAAa,6BAA6B;;;;;;;;;;;;;;;;;;;;AAsB1C,IAAa,wBAAwB,eAAE,OAAO;CAC5C,OAAO,iBAAiB,OAAO,EAG7B,UAAU,oBAAoB,SAAS,EACzC,CAAC;CACD,SAAS,eAAE,OACT,eAAE,OAAO;EACP,IAAI;EACJ,UAAU,2BAA2B,SAAS;CAChD,CAAC,CACH;AACF,CAAC"}
+{"version":3,"file":"upstream.js","names":["v"],"sources":["../../../../../../../../zero-cache/src/services/change-source/protocol/current/upstream.ts"],"sourcesContent":["import * as v from '../../../../../../shared/src/valita.ts';\nimport {\n  backfillIDSchema,\n  backfillProgressMarkSchema,\n  identifierSchema,\n  tableMetadataSchema,\n} from './data.ts';\nimport {upstreamStatusMessageSchema} from './status.ts';\n\n/** At the moment, the only upstream messages are status messages.  */\nexport const changeSourceUpstreamSchema = upstreamStatusMessageSchema;\nexport type ChangeSourceUpstream = v.Infer<typeof changeSourceUpstreamSchema>;\n\n/**\n * Contains the information for requesting a backfill of columns in a table.\n * Backfills are automatically started for new tables and columns in a given\n * change stream session; however, if the session is terminated before the\n * backfill completes, it must be restarted with appropriate\n * {@link BackfillRequest}s in subsequent session(s), as backfill state is\n * ephemeral, per-replication-manager, and not persisted upstream.\n *\n * All replication subscribers track their backfill state, which includes\n * tracking any changes to the table name, column names, or table metadata,\n * and present that state to the `change-streamer` when starting a\n * subscription.\n *\n * From there the `change-streamer` manages requesting from the change-source\n * the superset (and minimum progressMark) of all subscriber-specified\n * backfills. While subscribers are connected, the change-streamer also tracks\n * this per-subscriber state in memory, so that it can properly resume\n * backfills on a new change-stream if disconnected from upstream.\n */\nexport const backfillRequestSchema = v.object({\n  table: identifierSchema.extend({\n    // The table metadata is set to null if it is never specified by the\n    // change-source.\n    metadata: tableMetadataSchema.nullable(),\n  }),\n  columns: v.record(\n    v.object({\n      id: backfillIDSchema,\n      progress: backfillProgressMarkSchema.optional(),\n    }),\n  ),\n});\n\nexport type BackfillRequest = v.Infer<typeof backfillRequestSchema>;\n"],"mappings":";;;;;AAUA,IAAa,6BAA6B;;;;;;;;;;;;;;;;;;;;AAsB1C,IAAa,wBAAwBA,eAAE,OAAO;CAC5C,OAAO,iBAAiB,OAAO,EAG7B,UAAU,oBAAoB,SAAS,EACzC,CAAC;CACD,SAASA,eAAE,OACTA,eAAE,OAAO;EACP,IAAI;EACJ,UAAU,2BAA2B,SAAS;CAChD,CAAC,CACH;AACF,CAAC"}

package/out/zero-cache/src/services/change-source/protocol/current.js

@@ -52,6 +52,6 @@ var current_exports = /* @__PURE__ */ __exportAll({
 	upstreamStatusMessageSchema: () => upstreamStatusMessageSchema
 });
 //#endregion
-export { current_exports };
+export { CHANGE_SOURCE_PATH, addColumnSchema, backfillCompletedSchema, backfillIDSchema, backfillProgressMarkSchema, backfillRequestSchema, backfillSchema, beginSchema, changeSourceUpstreamSchema, changeStreamControlSchema, changeStreamDataSchema, changeStreamMessageSchema, commitSchema, createIndexSchema, createTableSchema, current_exports, dataChangeSchema, deleteSchema, downloadStatusSchema, downstreamStatusMessageSchema, downstreamStatusSchema, dropColumnSchema, dropIndexSchema, dropTableSchema, identifierSchema, insertSchema, isDataChange, isSchemaChange, jsonObjectSchema, jsonValueSchema, newRelationSchema, relationSchema, renameTableSchema, resetRequiredSchema, rollbackSchema, rowSchema, schemaChangeSchema, tableMetadataSchema, truncateSchema, updateColumnSchema, updateSchema, updateTableMetadataSchema, upstreamStatusMessageSchema };
 
 //# sourceMappingURL=current.js.map

package/out/zero-cache/src/services/change-source/protocol/mod.js

@@ -1,2 +1 @@
-import "./current.js";
-export {};
+export { current_exports as v0 } from "./current.js";

package/out/zero-cache/src/services/change-streamer/backfill-state-tracker.js.map

@@ -1 +1 @@
-{"version":3,"file":"backfill-state-tracker.js","names":["#lc","#subscribers","#restarts","#unexpectedIgnores","#restartReason","#logBackfills","#sessionBackfills","#checkCoverage"],"sources":["../../../../../../zero-cache/src/services/change-streamer/backfill-state-tracker.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {must} from '../../../../shared/src/must.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport type {Subscription} from '../../types/subscription.ts';\nimport {\n  getSuperset,\n  withResumePoint,\n} from '../change-source/protocol/backfill-progress.ts';\nimport type {ChangeStreamData} from '../change-source/protocol/current/downstream.ts';\nimport type {BackfillRequest} from '../change-source/protocol/current/upstream.ts';\nimport {BackfillState} from './backfill-state.ts';\nimport type {PreSerializedBatch} from './broadcast.ts';\nimport type {SubscriberContext} from './change-streamer.ts';\nimport type {Subscriber, SubscriberOptions} from './subscriber.ts';\n\nexport type SubscriberBackfillOptions = Pick<\n  SubscriberOptions,\n  'backfills' | 'onAligned' | 'onBackfillIgnored'\n>;\n\n/**\n * Tracks the pending backfills (and their progress) of the change log, the\n * subscribers, and the current change stream, in order to determine:\n *\n * * the BackfillRequests with which to start a change stream, i.e. the\n *   superset of the pending backfills of the change log and all aligned\n *   subscribers, starting from the earliest progress, and\n * * whether (and why) the change stream must be restarted, i.e. when a\n *   newly aligned subscriber's pending backfills are not covered by the\n *   current stream.\n *\n * The change-streamer calls {@link startStream()} whenever it starts a change\n * stream, {@link track()} for every change that it forwards, and restarts\n * the stream at the next transaction boundary whenever a\n * {@link restartReason} is set.\n */\nexport class BackfillStateTracker {\n  readonly #lc: LogContext;\n\n  /**\n   * The pending backfills of the change log itself, i.e. those of a\n   * (hypothetical) subscriber that has applied every change in the log.\n   * Whenever a stream is started, it is reset from the change log's cookie\n   * jar (which is consistent with the stream's starting watermark), carrying\n   * over the progress tracked in memory (as the cookie jar is\n   * progress-agnostic). It stands in for subscribers that do not report their\n   * pending backfills (i.e. protocol < v8), and ensures that backfills that\n   * are pending in the change log continue to be requested.\n   */\n  #logBackfills = new BackfillState();\n\n  /**\n   * The backfill progress of the current change stream (i.e. \"session\"),\n   * seeded from the BackfillRequests with which the stream was started.\n   * This determines whether a (newly aligned) subscriber's pending backfills\n   * are covered by the stream.\n   */\n  #sessionBackfills = new BackfillState();\n\n  /** Subscribers that report their pending backfills (protocol v8+). */\n  readonly #subscribers = new Set<Subscriber>();\n\n  /**\n   * Set (to the reason) when the change stream must be restarted with\n   * new BackfillRequests.\n   */\n  #restartReason: string | null = null;\n\n  readonly #restarts = getOrCreateCounter(\n    'replication',\n    'backfill_stream_restarts',\n    'Count of change stream restarts initiated to rewind backfills for ' +\n      'subscribers whose pending backfills were not covered by the stream',\n  );\n  readonly #unexpectedIgnores = getOrCreateCounter(\n    'replication',\n    'backfill_unexpected_ignores',\n    'Count of backfill messages unexpectedly ignored (for some columns) by ' +\n      'an aligned subscriber. The expected count is 0.',\n  );\n\n  constructor(lc: LogContext) {\n    this.#lc = lc;\n  }\n\n  /**\n   * The reason for which the change stream must be restarted, or `null` if\n   * it need not be. The restart should happen at the next transaction\n   * boundary, i.e. upon the next `commit`, or `status` message outside of a\n   * transaction.\n   */\n  get restartReason(): string | null {\n    return this.#restartReason;\n  }\n\n  /**\n   * Computes the BackfillRequests with which to start a new change stream:\n   * the superset of the pending backfills of the change log and of all\n   * aligned subscribers, all of which are consistent with the stream's\n   * starting watermark since streams are (re)started at transaction\n   * boundaries.\n   *\n   * @param logBackfillRequests The BackfillRequests from the change log's\n   *        cookie jar, consistent with the stream's starting watermark.\n   */\n  startStream(logBackfillRequests: BackfillRequest[]): BackfillRequest[] {\n    // The cookie jar can be behind #logBackfills (e.g. if the storer failed\n    // to persist a forwarded commit), in which case the new stream\n    // re-delivers those changes.\n    this.#logBackfills = new BackfillState(\n      this.#logBackfills.withProgress(logBackfillRequests),\n    );\n\n    const aligned = [...this.#subscribers].filter(s => s.aligned);\n    const requests = getSuperset(\n      this.#logBackfills.requests(),\n      ...aligned.map(s => must(s.backfills).requests()),\n    ).map(withResumePoint);\n\n    this.#sessionBackfills = new BackfillState(requests);\n    // Aligned subscribers are accounted for in the requests.\n    this.#restartReason = null;\n    if (requests.length) {\n      this.#lc.info?.(\n        `starting change stream with ${requests.length} backfill requests ` +\n          `(from the change log and ${aligned.length} subscribers)`,\n        {requests},\n      );\n    }\n    return requests;\n  }\n\n  /**\n   * Tracks a change that is forwarded to subscribers. This must be called\n   * (synchronously) before the change is forwarded, so that the stream's\n   * state is never ahead of that of aligned subscribers.\n   */\n  track(change: ChangeStreamData) {\n    this.#sessionBackfills.apply(change);\n    this.#logBackfills.apply(change);\n  }\n\n  /**\n   * Requests a restart of the change stream (at the next transaction\n   * boundary), e.g. to start it with BackfillRequests that it was not\n   * started with.\n   */\n  requestRestart(reason: string) {\n    this.#restartReason ??= reason;\n  }\n\n  /** Logs and records a (planned) restart for the {@link restartReason}. */\n  recordRestart() {\n    this.#lc.info?.(`restarting change stream: ${this.#restartReason}`);\n    this.#restarts.add(1);\n  }\n\n  /**\n   * Returns the backfill-related {@link SubscriberOptions} for a new\n   * subscriber, which must subsequently be passed to {@link register()}.\n   */\n  subscriberOptions(\n    lc: LogContext,\n    ctx: SubscriberContext,\n  ): SubscriberBackfillOptions {\n    const backfills = newBackfillState(lc, ctx);\n    if (!backfills) {\n      return {};\n    }\n    return {\n      backfills,\n      onAligned: subscriber => {\n        if (this.#subscribers.has(subscriber)) {\n          this.#checkCoverage(subscriber);\n        }\n      },\n      onBackfillIgnored: columns => {\n        if (this.#restartReason === null) {\n          lc.warn?.(\n            `aligned subscriber ignored backfill data for ${columns.join(', ')}`,\n          );\n          this.#unexpectedIgnores.add(1);\n        }\n      },\n    };\n  }\n\n  /**\n   * Registers a subscriber (created with {@link subscriberOptions()}) until\n   * its `downstream` is closed. This is a no-op for subscribers that do not\n   * report their pending backfills.\n   */\n  register(\n    subscriber: Subscriber,\n    downstream: Subscription<string | PreSerializedBatch>,\n  ) {\n    if (subscriber.backfills) {\n      this.#subscribers.add(subscriber);\n      downstream.addCloseHandler(() => this.#subscribers.delete(subscriber));\n    }\n  }\n\n  /**\n   * Checks whether the pending backfills of a newly aligned subscriber are\n   * covered by the current stream, requesting a restart of the stream if not.\n   *\n   * This can be checked as soon as the subscriber is aligned (even in the\n   * middle of a transaction), since both the subscriber's and the stream's\n   * BackfillStates compare their committed state, and the stream's state is\n   * never updated with a change before the change is forwarded to (aligned)\n   * subscribers. The restart itself happens at the next transaction boundary.\n   */\n  #checkCoverage(subscriber: Subscriber) {\n    const uncovered = must(subscriber.backfills).uncovered(\n      this.#sessionBackfills,\n    );\n    if (uncovered.length) {\n      this.#restartReason ??=\n        `subscriber ${subscriber.id} needs backfills that are not ` +\n        `covered by the current stream: ${uncovered.join(', ')}`;\n    }\n  }\n}\n\nfunction newBackfillState(\n  lc: LogContext,\n  ctx: SubscriberContext,\n): BackfillState | undefined {\n  if (ctx.backfills === undefined) {\n    return undefined; // protocol < v8\n  }\n  try {\n    return new BackfillState(ctx.backfills);\n  } catch (e) {\n    lc.warn?.(`invalid backfills from subscriber ${ctx.id}`, e);\n    return undefined;\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AAoCA,IAAa,uBAAb,MAAkC;CAChC;;;;;;;;;;;CAYA,gBAAgB,IAAI,cAAc;;;;;;;CAQlC,oBAAoB,IAAI,cAAc;;CAGtC,+BAAwB,IAAI,IAAgB;;;;;CAM5C,iBAAgC;CAEhC,YAAqB,mBACnB,eACA,4BACA,sIAEF;CACA,qBAA8B,mBAC5B,eACA,+BACA,uHAEF;CAEA,YAAY,IAAgB;EAC1B,KAAKA,MAAM;CACb;;;;;;;CAQA,IAAI,gBAA+B;EACjC,OAAO,KAAKI;CACd;;;;;;;;;;;CAYA,YAAY,qBAA2D;EAIrE,KAAKC,gBAAgB,IAAI,cACvB,KAAKA,cAAc,aAAa,mBAAmB,CACrD;EAEA,MAAM,UAAU,CAAC,GAAG,KAAKJ,YAAY,EAAE,QAAO,MAAK,EAAE,OAAO;EAC5D,MAAM,WAAW,YACf,KAAKI,cAAc,SAAS,GAC5B,GAAG,QAAQ,KAAI,MAAK,KAAK,EAAE,SAAS,EAAE,SAAS,CAAC,CAClD,EAAE,IAAI,eAAe;EAErB,KAAKC,oBAAoB,IAAI,cAAc,QAAQ;EAEnD,KAAKF,iBAAiB;EACtB,IAAI,SAAS,QACX,KAAKJ,IAAI,OACP,+BAA+B,SAAS,OAAO,8CACjB,QAAQ,OAAO,gBAC7C,EAAC,SAAQ,CACX;EAEF,OAAO;CACT;;;;;;CAOA,MAAM,QAA0B;EAC9B,KAAKM,kBAAkB,MAAM,MAAM;EACnC,KAAKD,cAAc,MAAM,MAAM;CACjC;;;;;;CAOA,eAAe,QAAgB;EAC7B,KAAKD,mBAAmB;CAC1B;;CAGA,gBAAgB;EACd,KAAKJ,IAAI,OAAO,6BAA6B,KAAKI,gBAAgB;EAClE,KAAKF,UAAU,IAAI,CAAC;CACtB;;;;;CAMA,kBACE,IACA,KAC2B;EAC3B,MAAM,YAAY,iBAAiB,IAAI,GAAG;EAC1C,IAAI,CAAC,WACH,OAAO,CAAC;EAEV,OAAO;GACL;GACA,YAAW,eAAc;IACvB,IAAI,KAAKD,aAAa,IAAI,UAAU,GAClC,KAAKM,eAAe,UAAU;GAElC;GACA,oBAAmB,YAAW;IAC5B,IAAI,KAAKH,mBAAmB,MAAM;KAChC,GAAG,OACD,gDAAgD,QAAQ,KAAK,IAAI,GACnE;KACA,KAAKD,mBAAmB,IAAI,CAAC;IAC/B;GACF;EACF;CACF;;;;;;CAOA,SACE,YACA,YACA;EACA,IAAI,WAAW,WAAW;GACxB,KAAKF,aAAa,IAAI,UAAU;GAChC,WAAW,sBAAsB,KAAKA,aAAa,OAAO,UAAU,CAAC;EACvE;CACF;;;;;;;;;;;CAYA,eAAe,YAAwB;EACrC,MAAM,YAAY,KAAK,WAAW,SAAS,EAAE,UAC3C,KAAKK,iBACP;EACA,IAAI,UAAU,QACZ,KAAKF,mBACH,cAAc,WAAW,GAAG,+DACM,UAAU,KAAK,IAAI;CAE3D;AACF;AAEA,SAAS,iBACP,IACA,KAC2B;CAC3B,IAAI,IAAI,cAAc,KAAA,GACpB;CAEF,IAAI;EACF,OAAO,IAAI,cAAc,IAAI,SAAS;CACxC,SAAS,GAAG;EACV,GAAG,OAAO,qCAAqC,IAAI,MAAM,CAAC;EAC1D;CACF;AACF"}
+{"version":3,"file":"backfill-state-tracker.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/backfill-state-tracker.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {must} from '../../../../shared/src/must.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport type {Subscription} from '../../types/subscription.ts';\nimport {\n  getSuperset,\n  withResumePoint,\n} from '../change-source/protocol/backfill-progress.ts';\nimport type {ChangeStreamData} from '../change-source/protocol/current/downstream.ts';\nimport type {BackfillRequest} from '../change-source/protocol/current/upstream.ts';\nimport {BackfillState} from './backfill-state.ts';\nimport type {PreSerializedBatch} from './broadcast.ts';\nimport type {SubscriberContext} from './change-streamer.ts';\nimport type {Subscriber, SubscriberOptions} from './subscriber.ts';\n\nexport type SubscriberBackfillOptions = Pick<\n  SubscriberOptions,\n  'backfills' | 'onAligned' | 'onBackfillIgnored'\n>;\n\n/**\n * Tracks the pending backfills (and their progress) of the change log, the\n * subscribers, and the current change stream, in order to determine:\n *\n * * the BackfillRequests with which to start a change stream, i.e. the\n *   superset of the pending backfills of the change log and all aligned\n *   subscribers, starting from the earliest progress, and\n * * whether (and why) the change stream must be restarted, i.e. when a\n *   newly aligned subscriber's pending backfills are not covered by the\n *   current stream.\n *\n * The change-streamer calls {@link startStream()} whenever it starts a change\n * stream, {@link track()} for every change that it forwards, and restarts\n * the stream at the next transaction boundary whenever a\n * {@link restartReason} is set.\n */\nexport class BackfillStateTracker {\n  readonly #lc: LogContext;\n\n  /**\n   * The pending backfills of the change log itself, i.e. those of a\n   * (hypothetical) subscriber that has applied every change in the log.\n   * Whenever a stream is started, it is reset from the change log's cookie\n   * jar (which is consistent with the stream's starting watermark), carrying\n   * over the progress tracked in memory (as the cookie jar is\n   * progress-agnostic). It stands in for subscribers that do not report their\n   * pending backfills (i.e. protocol < v8), and ensures that backfills that\n   * are pending in the change log continue to be requested.\n   */\n  #logBackfills = new BackfillState();\n\n  /**\n   * The backfill progress of the current change stream (i.e. \"session\"),\n   * seeded from the BackfillRequests with which the stream was started.\n   * This determines whether a (newly aligned) subscriber's pending backfills\n   * are covered by the stream.\n   */\n  #sessionBackfills = new BackfillState();\n\n  /** Subscribers that report their pending backfills (protocol v8+). */\n  readonly #subscribers = new Set<Subscriber>();\n\n  /**\n   * Set (to the reason) when the change stream must be restarted with\n   * new BackfillRequests.\n   */\n  #restartReason: string | null = null;\n\n  readonly #restarts = getOrCreateCounter(\n    'replication',\n    'backfill_stream_restarts',\n    'Count of change stream restarts initiated to rewind backfills for ' +\n      'subscribers whose pending backfills were not covered by the stream',\n  );\n  readonly #unexpectedIgnores = getOrCreateCounter(\n    'replication',\n    'backfill_unexpected_ignores',\n    'Count of backfill messages unexpectedly ignored (for some columns) by ' +\n      'an aligned subscriber. The expected count is 0.',\n  );\n\n  constructor(lc: LogContext) {\n    this.#lc = lc;\n  }\n\n  /**\n   * The reason for which the change stream must be restarted, or `null` if\n   * it need not be. The restart should happen at the next transaction\n   * boundary, i.e. upon the next `commit`, or `status` message outside of a\n   * transaction.\n   */\n  get restartReason(): string | null {\n    return this.#restartReason;\n  }\n\n  /**\n   * Computes the BackfillRequests with which to start a new change stream:\n   * the superset of the pending backfills of the change log and of all\n   * aligned subscribers, all of which are consistent with the stream's\n   * starting watermark since streams are (re)started at transaction\n   * boundaries.\n   *\n   * @param logBackfillRequests The BackfillRequests from the change log's\n   *        cookie jar, consistent with the stream's starting watermark.\n   */\n  startStream(logBackfillRequests: BackfillRequest[]): BackfillRequest[] {\n    // The cookie jar can be behind #logBackfills (e.g. if the storer failed\n    // to persist a forwarded commit), in which case the new stream\n    // re-delivers those changes.\n    this.#logBackfills = new BackfillState(\n      this.#logBackfills.withProgress(logBackfillRequests),\n    );\n\n    const aligned = [...this.#subscribers].filter(s => s.aligned);\n    const requests = getSuperset(\n      this.#logBackfills.requests(),\n      ...aligned.map(s => must(s.backfills).requests()),\n    ).map(withResumePoint);\n\n    this.#sessionBackfills = new BackfillState(requests);\n    // Aligned subscribers are accounted for in the requests.\n    this.#restartReason = null;\n    if (requests.length) {\n      this.#lc.info?.(\n        `starting change stream with ${requests.length} backfill requests ` +\n          `(from the change log and ${aligned.length} subscribers)`,\n        {requests},\n      );\n    }\n    return requests;\n  }\n\n  /**\n   * Tracks a change that is forwarded to subscribers. This must be called\n   * (synchronously) before the change is forwarded, so that the stream's\n   * state is never ahead of that of aligned subscribers.\n   */\n  track(change: ChangeStreamData) {\n    this.#sessionBackfills.apply(change);\n    this.#logBackfills.apply(change);\n  }\n\n  /**\n   * Requests a restart of the change stream (at the next transaction\n   * boundary), e.g. to start it with BackfillRequests that it was not\n   * started with.\n   */\n  requestRestart(reason: string) {\n    this.#restartReason ??= reason;\n  }\n\n  /** Logs and records a (planned) restart for the {@link restartReason}. */\n  recordRestart() {\n    this.#lc.info?.(`restarting change stream: ${this.#restartReason}`);\n    this.#restarts.add(1);\n  }\n\n  /**\n   * Returns the backfill-related {@link SubscriberOptions} for a new\n   * subscriber, which must subsequently be passed to {@link register()}.\n   */\n  subscriberOptions(\n    lc: LogContext,\n    ctx: SubscriberContext,\n  ): SubscriberBackfillOptions {\n    const backfills = newBackfillState(lc, ctx);\n    if (!backfills) {\n      return {};\n    }\n    return {\n      backfills,\n      onAligned: subscriber => {\n        if (this.#subscribers.has(subscriber)) {\n          this.#checkCoverage(subscriber);\n        }\n      },\n      onBackfillIgnored: columns => {\n        if (this.#restartReason === null) {\n          lc.warn?.(\n            `aligned subscriber ignored backfill data for ${columns.join(', ')}`,\n          );\n          this.#unexpectedIgnores.add(1);\n        }\n      },\n    };\n  }\n\n  /**\n   * Registers a subscriber (created with {@link subscriberOptions()}) until\n   * its `downstream` is closed. This is a no-op for subscribers that do not\n   * report their pending backfills.\n   */\n  register(\n    subscriber: Subscriber,\n    downstream: Subscription<string | PreSerializedBatch>,\n  ) {\n    if (subscriber.backfills) {\n      this.#subscribers.add(subscriber);\n      downstream.addCloseHandler(() => this.#subscribers.delete(subscriber));\n    }\n  }\n\n  /**\n   * Checks whether the pending backfills of a newly aligned subscriber are\n   * covered by the current stream, requesting a restart of the stream if not.\n   *\n   * This can be checked as soon as the subscriber is aligned (even in the\n   * middle of a transaction), since both the subscriber's and the stream's\n   * BackfillStates compare their committed state, and the stream's state is\n   * never updated with a change before the change is forwarded to (aligned)\n   * subscribers. The restart itself happens at the next transaction boundary.\n   */\n  #checkCoverage(subscriber: Subscriber) {\n    const uncovered = must(subscriber.backfills).uncovered(\n      this.#sessionBackfills,\n    );\n    if (uncovered.length) {\n      this.#restartReason ??=\n        `subscriber ${subscriber.id} needs backfills that are not ` +\n        `covered by the current stream: ${uncovered.join(', ')}`;\n    }\n  }\n}\n\nfunction newBackfillState(\n  lc: LogContext,\n  ctx: SubscriberContext,\n): BackfillState | undefined {\n  if (ctx.backfills === undefined) {\n    return undefined; // protocol < v8\n  }\n  try {\n    return new BackfillState(ctx.backfills);\n  } catch (e) {\n    lc.warn?.(`invalid backfills from subscriber ${ctx.id}`, e);\n    return undefined;\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AAoCA,IAAa,uBAAb,MAAkC;CAChC;;;;;;;;;;;CAYA,gBAAgB,IAAI,cAAc;;;;;;;CAQlC,oBAAoB,IAAI,cAAc;;CAGtC,+BAAwB,IAAI,IAAgB;;;;;CAM5C,iBAAgC;CAEhC,YAAqB,mBACnB,eACA,4BACA,sIAEF;CACA,qBAA8B,mBAC5B,eACA,+BACA,uHAEF;CAEA,YAAY,IAAgB;EAC1B,KAAK,MAAM;CACb;;;;;;;CAQA,IAAI,gBAA+B;EACjC,OAAO,KAAK;CACd;;;;;;;;;;;CAYA,YAAY,qBAA2D;EAIrE,KAAK,gBAAgB,IAAI,cACvB,KAAK,cAAc,aAAa,mBAAmB,CACrD;EAEA,MAAM,UAAU,CAAC,GAAG,KAAK,YAAY,CAAC,CAAC,QAAO,MAAK,EAAE,OAAO;EAC5D,MAAM,WAAW,YACf,KAAK,cAAc,SAAS,GAC5B,GAAG,QAAQ,KAAI,MAAK,KAAK,EAAE,SAAS,CAAC,CAAC,SAAS,CAAC,CAClD,CAAC,CAAC,IAAI,eAAe;EAErB,KAAK,oBAAoB,IAAI,cAAc,QAAQ;EAEnD,KAAK,iBAAiB;EACtB,IAAI,SAAS,QACX,KAAK,IAAI,OACP,+BAA+B,SAAS,OAAO,8CACjB,QAAQ,OAAO,gBAC7C,EAAC,SAAQ,CACX;EAEF,OAAO;CACT;;;;;;CAOA,MAAM,QAA0B;EAC9B,KAAK,kBAAkB,MAAM,MAAM;EACnC,KAAK,cAAc,MAAM,MAAM;CACjC;;;;;;CAOA,eAAe,QAAgB;EAC7B,KAAK,mBAAmB;CAC1B;;CAGA,gBAAgB;EACd,KAAK,IAAI,OAAO,6BAA6B,KAAK,gBAAgB;EAClE,KAAK,UAAU,IAAI,CAAC;CACtB;;;;;CAMA,kBACE,IACA,KAC2B;EAC3B,MAAM,YAAY,iBAAiB,IAAI,GAAG;EAC1C,IAAI,CAAC,WACH,OAAO,CAAC;EAEV,OAAO;GACL;GACA,YAAW,eAAc;IACvB,IAAI,KAAK,aAAa,IAAI,UAAU,GAClC,KAAK,eAAe,UAAU;GAElC;GACA,oBAAmB,YAAW;IAC5B,IAAI,KAAK,mBAAmB,MAAM;KAChC,GAAG,OACD,gDAAgD,QAAQ,KAAK,IAAI,GACnE;KACA,KAAK,mBAAmB,IAAI,CAAC;IAC/B;GACF;EACF;CACF;;;;;;CAOA,SACE,YACA,YACA;EACA,IAAI,WAAW,WAAW;GACxB,KAAK,aAAa,IAAI,UAAU;GAChC,WAAW,sBAAsB,KAAK,aAAa,OAAO,UAAU,CAAC;EACvE;CACF;;;;;;;;;;;CAYA,eAAe,YAAwB;EACrC,MAAM,YAAY,KAAK,WAAW,SAAS,CAAC,CAAC,UAC3C,KAAK,iBACP;EACA,IAAI,UAAU,QACZ,KAAK,mBACH,cAAc,WAAW,GAAG,+DACM,UAAU,KAAK,IAAI;CAE3D;AACF;AAEA,SAAS,iBACP,IACA,KAC2B;CAC3B,IAAI,IAAI,cAAc,KAAA,GACpB;CAEF,IAAI;EACF,OAAO,IAAI,cAAc,IAAI,SAAS;CACxC,SAAS,GAAG;EACV,GAAG,OAAO,qCAAqC,IAAI,MAAM,CAAC;EAC1D;CACF;AACF"}

package/out/zero-cache/src/services/change-streamer/backfill-state.js

@@ -18,7 +18,7 @@ function clone(state) {
 * Tags of messages that affect the backfill state. Other messages need not be
 * parsed.
 */
-var RELEVANT_TAGS = new Set(["backfill", ...schemaChangeTags.filter((t) => t !== "create-index" && t !== "drop-index")]);
+var RELEVANT_TAGS = /* @__PURE__ */ new Set(["backfill", ...schemaChangeTags.filter((t) => t !== "create-index" && t !== "drop-index")]);
 var lastParsed;
 function parse(json) {
 	if (lastParsed?.json !== json) lastParsed = {

package/out/zero-cache/src/services/change-streamer/backfill-state.js.map

@@ -1 +1 @@
-{"version":3,"file":"backfill-state.js","names":["#state","#committed","#beforeTx","#begin","#commit","#rollback","#applyBackfill","#applySchemaChange"],"sources":["../../../../../../zero-cache/src/services/change-streamer/backfill-state.ts"],"sourcesContent":["import {unreachable} from '../../../../shared/src/asserts.ts';\nimport {BigIntJSON} from '../../../../shared/src/bigint-json.ts';\nimport {deepEqual} from '../../../../shared/src/json.ts';\nimport {\n  acceptBackfill,\n  covers,\n} from '../change-source/protocol/backfill-progress.ts';\nimport type {\n  BackfillRequest,\n  ChangeStreamData,\n  Identifier,\n  SchemaChange,\n  TableMetadata,\n} from '../change-source/protocol/current.ts';\nimport {schemaChangeTags} from '../change-source/protocol/current/schema-change-tags.ts';\nimport {cookieOps} from '../replicator/change-log-cookies.ts';\nimport type {ChangeTag, WatermarkedChange} from './change-streamer.ts';\n\nfunction tableKey({schema, name}: Identifier) {\n  return JSON.stringify([schema, name]);\n}\n\ntype Column = BackfillRequest['columns'][string];\n\n/** A mutable BackfillRequest. */\ntype Backfill = {\n  table: {schema: string; name: string; metadata: TableMetadata | null};\n  columns: Record<string, Column>;\n};\n\ntype State = Map<string, Backfill>;\n\nfunction clone(state: State): State {\n  return new Map(\n    Array.from(state, ([key, {table, columns}]) => [\n      key,\n      {table: {...table}, columns: {...columns}},\n    ]),\n  );\n}\n\n/**\n * Tags of messages that affect the backfill state. Other messages need not be\n * parsed.\n */\nconst RELEVANT_TAGS = new Set<ChangeTag>([\n  'backfill',\n  ...schemaChangeTags.filter(t => t !== 'create-index' && t !== 'drop-index'),\n]);\n\n// Serialized changes are typically forwarded (as the same string) to every\n// subscriber, so a single entry cache avoids re-parsing the same message for\n// each subscriber.\nlet lastParsed: {json: string; change: ChangeStreamData} | undefined;\n\nfunction parse(json: string): ChangeStreamData {\n  if (lastParsed?.json !== json) {\n    lastParsed = {json, change: BigIntJSON.parse(json) as ChangeStreamData};\n  }\n  return lastParsed.change;\n}\n\n/**\n * Tracks the state of the pending backfills (and their progress) of a party\n * in the replication stream (e.g. a subscriber, or the change-log itself), by\n * applying the same policies that a subscriber applies to its replica:\n * schema changes are folded with the {@link cookieOps} shared by the\n * cookie jars, and `backfill` / `backfill-completed` messages are accepted\n * or ignored with the continuity policy of {@link acceptBackfill}.\n *\n * Changes are applied transactionally: the changes of a transaction are\n * discarded if it is rolled back, and {@link requests()} only reflects\n * committed transactions.\n */\nexport class BackfillState {\n  #state: State;\n  #beforeTx: State | null = null;\n\n  constructor(requests: readonly BackfillRequest[] = []) {\n    this.#state = new Map(\n      requests\n        .filter(req => Object.keys(req.columns).length > 0)\n        .map(({table, columns}) => [\n          tableKey(table),\n          {table: {...table}, columns: {...columns}},\n        ]),\n    );\n  }\n\n  get #committed(): State {\n    return this.#beforeTx ?? this.#state;\n  }\n\n  /** The (committed) pending backfills. */\n  requests(): BackfillRequest[] {\n    return [...clone(this.#committed).values()];\n  }\n\n  /**\n   * Returns the `requests` with the (committed) progress of this state\n   * applied to the columns that are pending in both, i.e. same table, column,\n   * and backfill ID. Other columns have no progress (i.e. from scratch).\n   */\n  withProgress(requests: readonly BackfillRequest[]): BackfillRequest[] {\n    const state = this.#committed;\n    return requests.map(({table, columns}) => {\n      const tracked = state.get(tableKey(table))?.columns;\n      return {\n        table,\n        columns: Object.fromEntries(\n          Object.entries(columns).map(([name, {id}]) => {\n            const col = tracked?.[name];\n            return [\n              name,\n              col?.progress && deepEqual(col.id, id)\n                ? {id, progress: col.progress}\n                : {id},\n            ];\n          }),\n        ),\n      };\n    });\n  }\n\n  /** Whether there are no (committed) pending backfills. */\n  get empty(): boolean {\n    return this.#committed.size === 0;\n  }\n\n  /**\n   * Returns the columns (as `schema.table.column` strings) with (committed)\n   * pending backfills that are not covered by the (committed) progress of the\n   * `stream`, i.e. columns for which the `stream` will not deliver all of the\n   * necessary data.\n   */\n  uncovered(stream: BackfillState): string[] {\n    const uncovered: string[] = [];\n    const streamState = stream.#committed;\n    for (const [key, {table, columns}] of this.#committed) {\n      const streamColumns = streamState.get(key)?.columns;\n      for (const [name, {id, progress}] of Object.entries(columns)) {\n        const s = streamColumns?.[name];\n        if (!s || !deepEqual(s.id, id) || !covers(s.progress, progress)) {\n          uncovered.push(`${table.schema}.${table.name}.${name}`);\n        }\n      }\n    }\n    return uncovered;\n  }\n\n  /**\n   * Applies a serialized change, parsing it only if it is relevant.\n   *\n   * @returns The columns (as `schema.table.column` strings) of a backfill\n   *          message that were ignored due to the continuity policy.\n   */\n  applySerialized([, tag, json]: WatermarkedChange): string[] {\n    switch (tag) {\n      case 'begin':\n        this.#begin();\n        return [];\n      case 'commit':\n        this.#commit();\n        return [];\n      case 'rollback':\n        this.#rollback();\n        return [];\n    }\n    if (!RELEVANT_TAGS.has(tag)) {\n      return [];\n    }\n    return this.apply(parse(json));\n  }\n\n  /**\n   * Applies a change.\n   *\n   * @returns The columns (as `schema.table.column` strings) of a backfill\n   *          message that were ignored due to the continuity policy.\n   */\n  apply(change: ChangeStreamData): string[] {\n    switch (change[0]) {\n      case 'begin':\n        this.#begin();\n        return [];\n      case 'commit':\n        this.#commit();\n        return [];\n      case 'rollback':\n        this.#rollback();\n        return [];\n    }\n    const msg = change[1];\n    switch (msg.tag) {\n      case 'backfill': {\n        const {relation, columns, progressMarks} = msg;\n        return this.#applyBackfill(relation, columns, (progress, col) => {\n          const result = acceptBackfill(progress, progressMarks);\n          if (result.accept) {\n            col.progress = result.progress;\n            if (col.progress === undefined) {\n              delete col.progress;\n            }\n          }\n          return result.accept;\n        });\n      }\n      case 'backfill-completed': {\n        const {relation, columns, progressMarks} = msg;\n        const backfill = this.#state.get(tableKey(relation));\n        const ignored = this.#applyBackfill(\n          relation,\n          columns,\n          (progress, _, name) => {\n            const {accept} = acceptBackfill(\n              progress,\n              progressMarks && {previous: progressMarks.previous},\n            );\n            if (accept && backfill) {\n              delete backfill.columns[name];\n            }\n            return accept;\n          },\n        );\n        if (backfill && Object.keys(backfill.columns).length === 0) {\n          this.#state.delete(tableKey(relation));\n        }\n        return ignored;\n      }\n      case 'create-table':\n      case 'rename-table':\n      case 'update-table-metadata':\n      case 'add-column':\n      case 'update-column':\n      case 'drop-column':\n      case 'drop-table':\n        this.#applySchemaChange(msg);\n        return [];\n    }\n    return [];\n  }\n\n  #applyBackfill(\n    relation: {schema: string; name: string; rowKey: {columns: string[]}},\n    columns: readonly string[],\n    apply: (progress: Column['progress'], col: Column, name: string) => boolean,\n  ): string[] {\n    const backfill = this.#state.get(tableKey(relation));\n    if (!backfill) {\n      return [];\n    }\n    const ignored: string[] = [];\n    // rowKey columns are excluded from `columns` but are backfilled with them\n    // (e.g. for a new table).\n    for (const name of [...relation.rowKey.columns, ...columns]) {\n      const col = backfill.columns[name];\n      if (!col) {\n        continue;\n      }\n      // Copy-on-write, since the column object may be shared with the\n      // pre-transaction state.\n      const updated = {...col};\n      if (apply(col.progress, updated, name)) {\n        if (name in backfill.columns) {\n          backfill.columns[name] = updated;\n        }\n      } else {\n        ignored.push(`${relation.schema}.${relation.name}.${name}`);\n      }\n    }\n    return ignored;\n  }\n\n  #applySchemaChange(change: SchemaChange) {\n    // Table metadata specified in the same change (e.g. a `create-table` or\n    // `add-column`), for backfills of tables that are not yet tracked.\n    const metadata = new Map<string, TableMetadata>();\n\n    for (const op of cookieOps(change)) {\n      switch (op.op) {\n        case 'upsert-metadata': {\n          const key = tableKey(op.table);\n          metadata.set(key, op.metadata);\n          const backfill = this.#state.get(key);\n          if (backfill) {\n            backfill.table.metadata = op.metadata;\n          }\n          break;\n        }\n        case 'upsert-backfill': {\n          const key = tableKey(op.table);\n          let backfill = this.#state.get(key);\n          if (!backfill) {\n            backfill = {\n              table: {\n                schema: op.table.schema,\n                name: op.table.name,\n                metadata: metadata.get(key) ?? null,\n              },\n              columns: {},\n            };\n            this.#state.set(key, backfill);\n          }\n          // A new backfill always starts from scratch.\n          backfill.columns[op.column] = {id: op.backfill};\n          break;\n        }\n        case 'rename-table': {\n          const backfill = this.#state.get(tableKey(op.old));\n          if (backfill) {\n            this.#state.delete(tableKey(op.old));\n            backfill.table.schema = op.new.schema;\n            backfill.table.name = op.new.name;\n            this.#state.set(tableKey(op.new), backfill);\n          }\n          break;\n        }\n        case 'drop-table':\n          this.#state.delete(tableKey(op.table));\n          break;\n        case 'rename-column': {\n          const backfill = this.#state.get(tableKey(op.table));\n          const col = backfill?.columns[op.old];\n          if (backfill && col) {\n            delete backfill.columns[op.old];\n            backfill.columns[op.new] = col;\n          }\n          break;\n        }\n        case 'drop-column': {\n          const key = tableKey(op.table);\n          const backfill = this.#state.get(key);\n          if (backfill) {\n            delete backfill.columns[op.column];\n            if (Object.keys(backfill.columns).length === 0) {\n              this.#state.delete(key);\n            }\n          }\n          break;\n        }\n        case 'complete-backfill':\n          // Only produced for `backfill-completed`, which is handled\n          // separately (with the continuity policy).\n          throw new Error(`unexpected ${op.op} op`);\n        default:\n          unreachable(op);\n      }\n    }\n  }\n\n  #begin() {\n    // Tolerate a missing commit/rollback, e.g. from an interrupted stream.\n    this.#beforeTx ??= clone(this.#state);\n  }\n\n  #commit() {\n    this.#beforeTx = null;\n  }\n\n  #rollback() {\n    if (this.#beforeTx) {\n      this.#state = this.#beforeTx;\n      this.#beforeTx = null;\n    }\n  }\n}\n"],"mappings":";;;;;;;AAkBA,SAAS,SAAS,EAAC,QAAQ,QAAmB;CAC5C,OAAO,KAAK,UAAU,CAAC,QAAQ,IAAI,CAAC;AACtC;AAYA,SAAS,MAAM,OAAqB;CAClC,OAAO,IAAI,IACT,MAAM,KAAK,QAAQ,CAAC,KAAK,EAAC,OAAO,eAAc,CAC7C,KACA;EAAC,OAAO,EAAC,GAAG,MAAK;EAAG,SAAS,EAAC,GAAG,QAAO;CAAC,CAC3C,CAAC,CACH;AACF;;;;;AAMA,IAAM,gBAAgB,IAAI,IAAe,CACvC,YACA,GAAG,iBAAiB,QAAO,MAAK,MAAM,kBAAkB,MAAM,YAAY,CAC5E,CAAC;AAKD,IAAI;AAEJ,SAAS,MAAM,MAAgC;CAC7C,IAAI,YAAY,SAAS,MACvB,aAAa;EAAC;EAAM,QAAQ,WAAW,MAAM,IAAI;CAAqB;CAExE,OAAO,WAAW;AACpB;;;;;;;;;;;;;AAcA,IAAa,gBAAb,MAA2B;CACzB;CACA,YAA0B;CAE1B,YAAY,WAAuC,CAAC,GAAG;EACrD,KAAKA,SAAS,IAAI,IAChB,SACG,QAAO,QAAO,OAAO,KAAK,IAAI,OAAO,EAAE,SAAS,CAAC,EACjD,KAAK,EAAC,OAAO,cAAa,CACzB,SAAS,KAAK,GACd;GAAC,OAAO,EAAC,GAAG,MAAK;GAAG,SAAS,EAAC,GAAG,QAAO;EAAC,CAC3C,CAAC,CACL;CACF;CAEA,IAAIC,aAAoB;EACtB,OAAO,KAAKC,aAAa,KAAKF;CAChC;;CAGA,WAA8B;EAC5B,OAAO,CAAC,GAAG,MAAM,KAAKC,UAAU,EAAE,OAAO,CAAC;CAC5C;;;;;;CAOA,aAAa,UAAyD;EACpE,MAAM,QAAQ,KAAKA;EACnB,OAAO,SAAS,KAAK,EAAC,OAAO,cAAa;GACxC,MAAM,UAAU,MAAM,IAAI,SAAS,KAAK,CAAC,GAAG;GAC5C,OAAO;IACL;IACA,SAAS,OAAO,YACd,OAAO,QAAQ,OAAO,EAAE,KAAK,CAAC,MAAM,EAAC,UAAS;KAC5C,MAAM,MAAM,UAAU;KACtB,OAAO,CACL,MACA,KAAK,YAAY,UAAU,IAAI,IAAI,EAAE,IACjC;MAAC;MAAI,UAAU,IAAI;KAAQ,IAC3B,EAAC,GAAE,CACT;IACF,CAAC,CACH;GACF;EACF,CAAC;CACH;;CAGA,IAAI,QAAiB;EACnB,OAAO,KAAKA,WAAW,SAAS;CAClC;;;;;;;CAQA,UAAU,QAAiC;EACzC,MAAM,YAAsB,CAAC;EAC7B,MAAM,cAAc,OAAOA;EAC3B,KAAK,MAAM,CAAC,KAAK,EAAC,OAAO,cAAa,KAAKA,YAAY;GACrD,MAAM,gBAAgB,YAAY,IAAI,GAAG,GAAG;GAC5C,KAAK,MAAM,CAAC,MAAM,EAAC,IAAI,eAAc,OAAO,QAAQ,OAAO,GAAG;IAC5D,MAAM,IAAI,gBAAgB;IAC1B,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,IAAI,EAAE,KAAK,CAAC,OAAO,EAAE,UAAU,QAAQ,GAC5D,UAAU,KAAK,GAAG,MAAM,OAAO,GAAG,MAAM,KAAK,GAAG,MAAM;GAE1D;EACF;EACA,OAAO;CACT;;;;;;;CAQA,gBAAgB,GAAG,KAAK,OAAoC;EAC1D,QAAQ,KAAR;GACE,KAAK;IACH,KAAKE,OAAO;IACZ,OAAO,CAAC;GACV,KAAK;IACH,KAAKC,QAAQ;IACb,OAAO,CAAC;GACV,KAAK;IACH,KAAKC,UAAU;IACf,OAAO,CAAC;EACZ;EACA,IAAI,CAAC,cAAc,IAAI,GAAG,GACxB,OAAO,CAAC;EAEV,OAAO,KAAK,MAAM,MAAM,IAAI,CAAC;CAC/B;;;;;;;CAQA,MAAM,QAAoC;EACxC,QAAQ,OAAO,IAAf;GACE,KAAK;IACH,KAAKF,OAAO;IACZ,OAAO,CAAC;GACV,KAAK;IACH,KAAKC,QAAQ;IACb,OAAO,CAAC;GACV,KAAK;IACH,KAAKC,UAAU;IACf,OAAO,CAAC;EACZ;EACA,MAAM,MAAM,OAAO;EACnB,QAAQ,IAAI,KAAZ;GACE,KAAK,YAAY;IACf,MAAM,EAAC,UAAU,SAAS,kBAAiB;IAC3C,OAAO,KAAKC,eAAe,UAAU,UAAU,UAAU,QAAQ;KAC/D,MAAM,SAAS,eAAe,UAAU,aAAa;KACrD,IAAI,OAAO,QAAQ;MACjB,IAAI,WAAW,OAAO;MACtB,IAAI,IAAI,aAAa,KAAA,GACnB,OAAO,IAAI;KAEf;KACA,OAAO,OAAO;IAChB,CAAC;GACH;GACA,KAAK,sBAAsB;IACzB,MAAM,EAAC,UAAU,SAAS,kBAAiB;IAC3C,MAAM,WAAW,KAAKN,OAAO,IAAI,SAAS,QAAQ,CAAC;IACnD,MAAM,UAAU,KAAKM,eACnB,UACA,UACC,UAAU,GAAG,SAAS;KACrB,MAAM,EAAC,WAAU,eACf,UACA,iBAAiB,EAAC,UAAU,cAAc,SAAQ,CACpD;KACA,IAAI,UAAU,UACZ,OAAO,SAAS,QAAQ;KAE1B,OAAO;IACT,CACF;IACA,IAAI,YAAY,OAAO,KAAK,SAAS,OAAO,EAAE,WAAW,GACvD,KAAKN,OAAO,OAAO,SAAS,QAAQ,CAAC;IAEvC,OAAO;GACT;GACA,KAAK;GACL,KAAK;GACL,KAAK;GACL,KAAK;GACL,KAAK;GACL,KAAK;GACL,KAAK;IACH,KAAKO,mBAAmB,GAAG;IAC3B,OAAO,CAAC;EACZ;EACA,OAAO,CAAC;CACV;CAEA,eACE,UACA,SACA,OACU;EACV,MAAM,WAAW,KAAKP,OAAO,IAAI,SAAS,QAAQ,CAAC;EACnD,IAAI,CAAC,UACH,OAAO,CAAC;EAEV,MAAM,UAAoB,CAAC;EAG3B,KAAK,MAAM,QAAQ,CAAC,GAAG,SAAS,OAAO,SAAS,GAAG,OAAO,GAAG;GAC3D,MAAM,MAAM,SAAS,QAAQ;GAC7B,IAAI,CAAC,KACH;GAIF,MAAM,UAAU,EAAC,GAAG,IAAG;GACvB,IAAI,MAAM,IAAI,UAAU,SAAS,IAAI;QAC/B,QAAQ,SAAS,SACnB,SAAS,QAAQ,QAAQ;GAAA,OAG3B,QAAQ,KAAK,GAAG,SAAS,OAAO,GAAG,SAAS,KAAK,GAAG,MAAM;EAE9D;EACA,OAAO;CACT;CAEA,mBAAmB,QAAsB;EAGvC,MAAM,2BAAW,IAAI,IAA2B;EAEhD,KAAK,MAAM,MAAM,UAAU,MAAM,GAC/B,QAAQ,GAAG,IAAX;GACE,KAAK,mBAAmB;IACtB,MAAM,MAAM,SAAS,GAAG,KAAK;IAC7B,SAAS,IAAI,KAAK,GAAG,QAAQ;IAC7B,MAAM,WAAW,KAAKA,OAAO,IAAI,GAAG;IACpC,IAAI,UACF,SAAS,MAAM,WAAW,GAAG;IAE/B;GACF;GACA,KAAK,mBAAmB;IACtB,MAAM,MAAM,SAAS,GAAG,KAAK;IAC7B,IAAI,WAAW,KAAKA,OAAO,IAAI,GAAG;IAClC,IAAI,CAAC,UAAU;KACb,WAAW;MACT,OAAO;OACL,QAAQ,GAAG,MAAM;OACjB,MAAM,GAAG,MAAM;OACf,UAAU,SAAS,IAAI,GAAG,KAAK;MACjC;MACA,SAAS,CAAC;KACZ;KACA,KAAKA,OAAO,IAAI,KAAK,QAAQ;IAC/B;IAEA,SAAS,QAAQ,GAAG,UAAU,EAAC,IAAI,GAAG,SAAQ;IAC9C;GACF;GACA,KAAK,gBAAgB;IACnB,MAAM,WAAW,KAAKA,OAAO,IAAI,SAAS,GAAG,GAAG,CAAC;IACjD,IAAI,UAAU;KACZ,KAAKA,OAAO,OAAO,SAAS,GAAG,GAAG,CAAC;KACnC,SAAS,MAAM,SAAS,GAAG,IAAI;KAC/B,SAAS,MAAM,OAAO,GAAG,IAAI;KAC7B,KAAKA,OAAO,IAAI,SAAS,GAAG,GAAG,GAAG,QAAQ;IAC5C;IACA;GACF;GACA,KAAK;IACH,KAAKA,OAAO,OAAO,SAAS,GAAG,KAAK,CAAC;IACrC;GACF,KAAK,iBAAiB;IACpB,MAAM,WAAW,KAAKA,OAAO,IAAI,SAAS,GAAG,KAAK,CAAC;IACnD,MAAM,MAAM,UAAU,QAAQ,GAAG;IACjC,IAAI,YAAY,KAAK;KACnB,OAAO,SAAS,QAAQ,GAAG;KAC3B,SAAS,QAAQ,GAAG,OAAO;IAC7B;IACA;GACF;GACA,KAAK,eAAe;IAClB,MAAM,MAAM,SAAS,GAAG,KAAK;IAC7B,MAAM,WAAW,KAAKA,OAAO,IAAI,GAAG;IACpC,IAAI,UAAU;KACZ,OAAO,SAAS,QAAQ,GAAG;KAC3B,IAAI,OAAO,KAAK,SAAS,OAAO,EAAE,WAAW,GAC3C,KAAKA,OAAO,OAAO,GAAG;IAE1B;IACA;GACF;GACA,KAAK,qBAGH,MAAM,IAAI,MAAM,cAAc,GAAG,GAAG,IAAI;GAC1C,SACE,YAAY,EAAE;EAClB;CAEJ;CAEA,SAAS;EAEP,KAAKE,cAAc,MAAM,KAAKF,MAAM;CACtC;CAEA,UAAU;EACR,KAAKE,YAAY;CACnB;CAEA,YAAY;EACV,IAAI,KAAKA,WAAW;GAClB,KAAKF,SAAS,KAAKE;GACnB,KAAKA,YAAY;EACnB;CACF;AACF"}
+{"version":3,"file":"backfill-state.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/backfill-state.ts"],"sourcesContent":["import {unreachable} from '../../../../shared/src/asserts.ts';\nimport {BigIntJSON} from '../../../../shared/src/bigint-json.ts';\nimport {deepEqual} from '../../../../shared/src/json.ts';\nimport {\n  acceptBackfill,\n  covers,\n} from '../change-source/protocol/backfill-progress.ts';\nimport type {\n  BackfillRequest,\n  ChangeStreamData,\n  Identifier,\n  SchemaChange,\n  TableMetadata,\n} from '../change-source/protocol/current.ts';\nimport {schemaChangeTags} from '../change-source/protocol/current/schema-change-tags.ts';\nimport {cookieOps} from '../replicator/change-log-cookies.ts';\nimport type {ChangeTag, WatermarkedChange} from './change-streamer.ts';\n\nfunction tableKey({schema, name}: Identifier) {\n  return JSON.stringify([schema, name]);\n}\n\ntype Column = BackfillRequest['columns'][string];\n\n/** A mutable BackfillRequest. */\ntype Backfill = {\n  table: {schema: string; name: string; metadata: TableMetadata | null};\n  columns: Record<string, Column>;\n};\n\ntype State = Map<string, Backfill>;\n\nfunction clone(state: State): State {\n  return new Map(\n    Array.from(state, ([key, {table, columns}]) => [\n      key,\n      {table: {...table}, columns: {...columns}},\n    ]),\n  );\n}\n\n/**\n * Tags of messages that affect the backfill state. Other messages need not be\n * parsed.\n */\nconst RELEVANT_TAGS = new Set<ChangeTag>([\n  'backfill',\n  ...schemaChangeTags.filter(t => t !== 'create-index' && t !== 'drop-index'),\n]);\n\n// Serialized changes are typically forwarded (as the same string) to every\n// subscriber, so a single entry cache avoids re-parsing the same message for\n// each subscriber.\nlet lastParsed: {json: string; change: ChangeStreamData} | undefined;\n\nfunction parse(json: string): ChangeStreamData {\n  if (lastParsed?.json !== json) {\n    lastParsed = {json, change: BigIntJSON.parse(json) as ChangeStreamData};\n  }\n  return lastParsed.change;\n}\n\n/**\n * Tracks the state of the pending backfills (and their progress) of a party\n * in the replication stream (e.g. a subscriber, or the change-log itself), by\n * applying the same policies that a subscriber applies to its replica:\n * schema changes are folded with the {@link cookieOps} shared by the\n * cookie jars, and `backfill` / `backfill-completed` messages are accepted\n * or ignored with the continuity policy of {@link acceptBackfill}.\n *\n * Changes are applied transactionally: the changes of a transaction are\n * discarded if it is rolled back, and {@link requests()} only reflects\n * committed transactions.\n */\nexport class BackfillState {\n  #state: State;\n  #beforeTx: State | null = null;\n\n  constructor(requests: readonly BackfillRequest[] = []) {\n    this.#state = new Map(\n      requests\n        .filter(req => Object.keys(req.columns).length > 0)\n        .map(({table, columns}) => [\n          tableKey(table),\n          {table: {...table}, columns: {...columns}},\n        ]),\n    );\n  }\n\n  get #committed(): State {\n    return this.#beforeTx ?? this.#state;\n  }\n\n  /** The (committed) pending backfills. */\n  requests(): BackfillRequest[] {\n    return [...clone(this.#committed).values()];\n  }\n\n  /**\n   * Returns the `requests` with the (committed) progress of this state\n   * applied to the columns that are pending in both, i.e. same table, column,\n   * and backfill ID. Other columns have no progress (i.e. from scratch).\n   */\n  withProgress(requests: readonly BackfillRequest[]): BackfillRequest[] {\n    const state = this.#committed;\n    return requests.map(({table, columns}) => {\n      const tracked = state.get(tableKey(table))?.columns;\n      return {\n        table,\n        columns: Object.fromEntries(\n          Object.entries(columns).map(([name, {id}]) => {\n            const col = tracked?.[name];\n            return [\n              name,\n              col?.progress && deepEqual(col.id, id)\n                ? {id, progress: col.progress}\n                : {id},\n            ];\n          }),\n        ),\n      };\n    });\n  }\n\n  /** Whether there are no (committed) pending backfills. */\n  get empty(): boolean {\n    return this.#committed.size === 0;\n  }\n\n  /**\n   * Returns the columns (as `schema.table.column` strings) with (committed)\n   * pending backfills that are not covered by the (committed) progress of the\n   * `stream`, i.e. columns for which the `stream` will not deliver all of the\n   * necessary data.\n   */\n  uncovered(stream: BackfillState): string[] {\n    const uncovered: string[] = [];\n    const streamState = stream.#committed;\n    for (const [key, {table, columns}] of this.#committed) {\n      const streamColumns = streamState.get(key)?.columns;\n      for (const [name, {id, progress}] of Object.entries(columns)) {\n        const s = streamColumns?.[name];\n        if (!s || !deepEqual(s.id, id) || !covers(s.progress, progress)) {\n          uncovered.push(`${table.schema}.${table.name}.${name}`);\n        }\n      }\n    }\n    return uncovered;\n  }\n\n  /**\n   * Applies a serialized change, parsing it only if it is relevant.\n   *\n   * @returns The columns (as `schema.table.column` strings) of a backfill\n   *          message that were ignored due to the continuity policy.\n   */\n  applySerialized([, tag, json]: WatermarkedChange): string[] {\n    switch (tag) {\n      case 'begin':\n        this.#begin();\n        return [];\n      case 'commit':\n        this.#commit();\n        return [];\n      case 'rollback':\n        this.#rollback();\n        return [];\n    }\n    if (!RELEVANT_TAGS.has(tag)) {\n      return [];\n    }\n    return this.apply(parse(json));\n  }\n\n  /**\n   * Applies a change.\n   *\n   * @returns The columns (as `schema.table.column` strings) of a backfill\n   *          message that were ignored due to the continuity policy.\n   */\n  apply(change: ChangeStreamData): string[] {\n    switch (change[0]) {\n      case 'begin':\n        this.#begin();\n        return [];\n      case 'commit':\n        this.#commit();\n        return [];\n      case 'rollback':\n        this.#rollback();\n        return [];\n    }\n    const msg = change[1];\n    switch (msg.tag) {\n      case 'backfill': {\n        const {relation, columns, progressMarks} = msg;\n        return this.#applyBackfill(relation, columns, (progress, col) => {\n          const result = acceptBackfill(progress, progressMarks);\n          if (result.accept) {\n            col.progress = result.progress;\n            if (col.progress === undefined) {\n              delete col.progress;\n            }\n          }\n          return result.accept;\n        });\n      }\n      case 'backfill-completed': {\n        const {relation, columns, progressMarks} = msg;\n        const backfill = this.#state.get(tableKey(relation));\n        const ignored = this.#applyBackfill(\n          relation,\n          columns,\n          (progress, _, name) => {\n            const {accept} = acceptBackfill(\n              progress,\n              progressMarks && {previous: progressMarks.previous},\n            );\n            if (accept && backfill) {\n              delete backfill.columns[name];\n            }\n            return accept;\n          },\n        );\n        if (backfill && Object.keys(backfill.columns).length === 0) {\n          this.#state.delete(tableKey(relation));\n        }\n        return ignored;\n      }\n      case 'create-table':\n      case 'rename-table':\n      case 'update-table-metadata':\n      case 'add-column':\n      case 'update-column':\n      case 'drop-column':\n      case 'drop-table':\n        this.#applySchemaChange(msg);\n        return [];\n    }\n    return [];\n  }\n\n  #applyBackfill(\n    relation: {schema: string; name: string; rowKey: {columns: string[]}},\n    columns: readonly string[],\n    apply: (progress: Column['progress'], col: Column, name: string) => boolean,\n  ): string[] {\n    const backfill = this.#state.get(tableKey(relation));\n    if (!backfill) {\n      return [];\n    }\n    const ignored: string[] = [];\n    // rowKey columns are excluded from `columns` but are backfilled with them\n    // (e.g. for a new table).\n    for (const name of [...relation.rowKey.columns, ...columns]) {\n      const col = backfill.columns[name];\n      if (!col) {\n        continue;\n      }\n      // Copy-on-write, since the column object may be shared with the\n      // pre-transaction state.\n      const updated = {...col};\n      if (apply(col.progress, updated, name)) {\n        if (name in backfill.columns) {\n          backfill.columns[name] = updated;\n        }\n      } else {\n        ignored.push(`${relation.schema}.${relation.name}.${name}`);\n      }\n    }\n    return ignored;\n  }\n\n  #applySchemaChange(change: SchemaChange) {\n    // Table metadata specified in the same change (e.g. a `create-table` or\n    // `add-column`), for backfills of tables that are not yet tracked.\n    const metadata = new Map<string, TableMetadata>();\n\n    for (const op of cookieOps(change)) {\n      switch (op.op) {\n        case 'upsert-metadata': {\n          const key = tableKey(op.table);\n          metadata.set(key, op.metadata);\n          const backfill = this.#state.get(key);\n          if (backfill) {\n            backfill.table.metadata = op.metadata;\n          }\n          break;\n        }\n        case 'upsert-backfill': {\n          const key = tableKey(op.table);\n          let backfill = this.#state.get(key);\n          if (!backfill) {\n            backfill = {\n              table: {\n                schema: op.table.schema,\n                name: op.table.name,\n                metadata: metadata.get(key) ?? null,\n              },\n              columns: {},\n            };\n            this.#state.set(key, backfill);\n          }\n          // A new backfill always starts from scratch.\n          backfill.columns[op.column] = {id: op.backfill};\n          break;\n        }\n        case 'rename-table': {\n          const backfill = this.#state.get(tableKey(op.old));\n          if (backfill) {\n            this.#state.delete(tableKey(op.old));\n            backfill.table.schema = op.new.schema;\n            backfill.table.name = op.new.name;\n            this.#state.set(tableKey(op.new), backfill);\n          }\n          break;\n        }\n        case 'drop-table':\n          this.#state.delete(tableKey(op.table));\n          break;\n        case 'rename-column': {\n          const backfill = this.#state.get(tableKey(op.table));\n          const col = backfill?.columns[op.old];\n          if (backfill && col) {\n            delete backfill.columns[op.old];\n            backfill.columns[op.new] = col;\n          }\n          break;\n        }\n        case 'drop-column': {\n          const key = tableKey(op.table);\n          const backfill = this.#state.get(key);\n          if (backfill) {\n            delete backfill.columns[op.column];\n            if (Object.keys(backfill.columns).length === 0) {\n              this.#state.delete(key);\n            }\n          }\n          break;\n        }\n        case 'complete-backfill':\n          // Only produced for `backfill-completed`, which is handled\n          // separately (with the continuity policy).\n          throw new Error(`unexpected ${op.op} op`);\n        default:\n          unreachable(op);\n      }\n    }\n  }\n\n  #begin() {\n    // Tolerate a missing commit/rollback, e.g. from an interrupted stream.\n    this.#beforeTx ??= clone(this.#state);\n  }\n\n  #commit() {\n    this.#beforeTx = null;\n  }\n\n  #rollback() {\n    if (this.#beforeTx) {\n      this.#state = this.#beforeTx;\n      this.#beforeTx = null;\n    }\n  }\n}\n"],"mappings":";;;;;;;AAkBA,SAAS,SAAS,EAAC,QAAQ,QAAmB;CAC5C,OAAO,KAAK,UAAU,CAAC,QAAQ,IAAI,CAAC;AACtC;AAYA,SAAS,MAAM,OAAqB;CAClC,OAAO,IAAI,IACT,MAAM,KAAK,QAAQ,CAAC,KAAK,EAAC,OAAO,eAAc,CAC7C,KACA;EAAC,OAAO,EAAC,GAAG,MAAK;EAAG,SAAS,EAAC,GAAG,QAAO;CAAC,CAC3C,CAAC,CACH;AACF;;;;;AAMA,IAAM,gCAAgB,IAAI,IAAe,CACvC,YACA,GAAG,iBAAiB,QAAO,MAAK,MAAM,kBAAkB,MAAM,YAAY,CAC5E,CAAC;AAKD,IAAI;AAEJ,SAAS,MAAM,MAAgC;CAC7C,IAAI,YAAY,SAAS,MACvB,aAAa;EAAC;EAAM,QAAQ,WAAW,MAAM,IAAI;CAAqB;CAExE,OAAO,WAAW;AACpB;;;;;;;;;;;;;AAcA,IAAa,gBAAb,MAA2B;CACzB;CACA,YAA0B;CAE1B,YAAY,WAAuC,CAAC,GAAG;EACrD,KAAK,SAAS,IAAI,IAChB,SACG,QAAO,QAAO,OAAO,KAAK,IAAI,OAAO,CAAC,CAAC,SAAS,CAAC,CAAC,CAClD,KAAK,EAAC,OAAO,cAAa,CACzB,SAAS,KAAK,GACd;GAAC,OAAO,EAAC,GAAG,MAAK;GAAG,SAAS,EAAC,GAAG,QAAO;EAAC,CAC3C,CAAC,CACL;CACF;CAEA,IAAI,aAAoB;EACtB,OAAO,KAAK,aAAa,KAAK;CAChC;;CAGA,WAA8B;EAC5B,OAAO,CAAC,GAAG,MAAM,KAAK,UAAU,CAAC,CAAC,OAAO,CAAC;CAC5C;;;;;;CAOA,aAAa,UAAyD;EACpE,MAAM,QAAQ,KAAK;EACnB,OAAO,SAAS,KAAK,EAAC,OAAO,cAAa;GACxC,MAAM,UAAU,MAAM,IAAI,SAAS,KAAK,CAAC,CAAC,EAAE;GAC5C,OAAO;IACL;IACA,SAAS,OAAO,YACd,OAAO,QAAQ,OAAO,CAAC,CAAC,KAAK,CAAC,MAAM,EAAC,UAAS;KAC5C,MAAM,MAAM,UAAU;KACtB,OAAO,CACL,MACA,KAAK,YAAY,UAAU,IAAI,IAAI,EAAE,IACjC;MAAC;MAAI,UAAU,IAAI;KAAQ,IAC3B,EAAC,GAAE,CACT;IACF,CAAC,CACH;GACF;EACF,CAAC;CACH;;CAGA,IAAI,QAAiB;EACnB,OAAO,KAAK,WAAW,SAAS;CAClC;;;;;;;CAQA,UAAU,QAAiC;EACzC,MAAM,YAAsB,CAAC;EAC7B,MAAM,cAAc,OAAO;EAC3B,KAAK,MAAM,CAAC,KAAK,EAAC,OAAO,cAAa,KAAK,YAAY;GACrD,MAAM,gBAAgB,YAAY,IAAI,GAAG,CAAC,EAAE;GAC5C,KAAK,MAAM,CAAC,MAAM,EAAC,IAAI,eAAc,OAAO,QAAQ,OAAO,GAAG;IAC5D,MAAM,IAAI,gBAAgB;IAC1B,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,IAAI,EAAE,KAAK,CAAC,OAAO,EAAE,UAAU,QAAQ,GAC5D,UAAU,KAAK,GAAG,MAAM,OAAO,GAAG,MAAM,KAAK,GAAG,MAAM;GAE1D;EACF;EACA,OAAO;CACT;;;;;;;CAQA,gBAAgB,GAAG,KAAK,OAAoC;EAC1D,QAAQ,KAAR;GACE,KAAK;IACH,KAAK,OAAO;IACZ,OAAO,CAAC;GACV,KAAK;IACH,KAAK,QAAQ;IACb,OAAO,CAAC;GACV,KAAK;IACH,KAAK,UAAU;IACf,OAAO,CAAC;EACZ;EACA,IAAI,CAAC,cAAc,IAAI,GAAG,GACxB,OAAO,CAAC;EAEV,OAAO,KAAK,MAAM,MAAM,IAAI,CAAC;CAC/B;;;;;;;CAQA,MAAM,QAAoC;EACxC,QAAQ,OAAO,IAAf;GACE,KAAK;IACH,KAAK,OAAO;IACZ,OAAO,CAAC;GACV,KAAK;IACH,KAAK,QAAQ;IACb,OAAO,CAAC;GACV,KAAK;IACH,KAAK,UAAU;IACf,OAAO,CAAC;EACZ;EACA,MAAM,MAAM,OAAO;EACnB,QAAQ,IAAI,KAAZ;GACE,KAAK,YAAY;IACf,MAAM,EAAC,UAAU,SAAS,kBAAiB;IAC3C,OAAO,KAAK,eAAe,UAAU,UAAU,UAAU,QAAQ;KAC/D,MAAM,SAAS,eAAe,UAAU,aAAa;KACrD,IAAI,OAAO,QAAQ;MACjB,IAAI,WAAW,OAAO;MACtB,IAAI,IAAI,aAAa,KAAA,GACnB,OAAO,IAAI;KAEf;KACA,OAAO,OAAO;IAChB,CAAC;GACH;GACA,KAAK,sBAAsB;IACzB,MAAM,EAAC,UAAU,SAAS,kBAAiB;IAC3C,MAAM,WAAW,KAAK,OAAO,IAAI,SAAS,QAAQ,CAAC;IACnD,MAAM,UAAU,KAAK,eACnB,UACA,UACC,UAAU,GAAG,SAAS;KACrB,MAAM,EAAC,WAAU,eACf,UACA,iBAAiB,EAAC,UAAU,cAAc,SAAQ,CACpD;KACA,IAAI,UAAU,UACZ,OAAO,SAAS,QAAQ;KAE1B,OAAO;IACT,CACF;IACA,IAAI,YAAY,OAAO,KAAK,SAAS,OAAO,CAAC,CAAC,WAAW,GACvD,KAAK,OAAO,OAAO,SAAS,QAAQ,CAAC;IAEvC,OAAO;GACT;GACA,KAAK;GACL,KAAK;GACL,KAAK;GACL,KAAK;GACL,KAAK;GACL,KAAK;GACL,KAAK;IACH,KAAK,mBAAmB,GAAG;IAC3B,OAAO,CAAC;EACZ;EACA,OAAO,CAAC;CACV;CAEA,eACE,UACA,SACA,OACU;EACV,MAAM,WAAW,KAAK,OAAO,IAAI,SAAS,QAAQ,CAAC;EACnD,IAAI,CAAC,UACH,OAAO,CAAC;EAEV,MAAM,UAAoB,CAAC;EAG3B,KAAK,MAAM,QAAQ,CAAC,GAAG,SAAS,OAAO,SAAS,GAAG,OAAO,GAAG;GAC3D,MAAM,MAAM,SAAS,QAAQ;GAC7B,IAAI,CAAC,KACH;GAIF,MAAM,UAAU,EAAC,GAAG,IAAG;GACvB,IAAI,MAAM,IAAI,UAAU,SAAS,IAAI,GAC/B;QAAA,QAAQ,SAAS,SACnB,SAAS,QAAQ,QAAQ;GAAA,OAG3B,QAAQ,KAAK,GAAG,SAAS,OAAO,GAAG,SAAS,KAAK,GAAG,MAAM;EAE9D;EACA,OAAO;CACT;CAEA,mBAAmB,QAAsB;EAGvC,MAAM,2BAAW,IAAI,IAA2B;EAEhD,KAAK,MAAM,MAAM,UAAU,MAAM,GAC/B,QAAQ,GAAG,IAAX;GACE,KAAK,mBAAmB;IACtB,MAAM,MAAM,SAAS,GAAG,KAAK;IAC7B,SAAS,IAAI,KAAK,GAAG,QAAQ;IAC7B,MAAM,WAAW,KAAK,OAAO,IAAI,GAAG;IACpC,IAAI,UACF,SAAS,MAAM,WAAW,GAAG;IAE/B;GACF;GACA,KAAK,mBAAmB;IACtB,MAAM,MAAM,SAAS,GAAG,KAAK;IAC7B,IAAI,WAAW,KAAK,OAAO,IAAI,GAAG;IAClC,IAAI,CAAC,UAAU;KACb,WAAW;MACT,OAAO;OACL,QAAQ,GAAG,MAAM;OACjB,MAAM,GAAG,MAAM;OACf,UAAU,SAAS,IAAI,GAAG,KAAK;MACjC;MACA,SAAS,CAAC;KACZ;KACA,KAAK,OAAO,IAAI,KAAK,QAAQ;IAC/B;IAEA,SAAS,QAAQ,GAAG,UAAU,EAAC,IAAI,GAAG,SAAQ;IAC9C;GACF;GACA,KAAK,gBAAgB;IACnB,MAAM,WAAW,KAAK,OAAO,IAAI,SAAS,GAAG,GAAG,CAAC;IACjD,IAAI,UAAU;KACZ,KAAK,OAAO,OAAO,SAAS,GAAG,GAAG,CAAC;KACnC,SAAS,MAAM,SAAS,GAAG,IAAI;KAC/B,SAAS,MAAM,OAAO,GAAG,IAAI;KAC7B,KAAK,OAAO,IAAI,SAAS,GAAG,GAAG,GAAG,QAAQ;IAC5C;IACA;GACF;GACA,KAAK;IACH,KAAK,OAAO,OAAO,SAAS,GAAG,KAAK,CAAC;IACrC;GACF,KAAK,iBAAiB;IACpB,MAAM,WAAW,KAAK,OAAO,IAAI,SAAS,GAAG,KAAK,CAAC;IACnD,MAAM,MAAM,UAAU,QAAQ,GAAG;IACjC,IAAI,YAAY,KAAK;KACnB,OAAO,SAAS,QAAQ,GAAG;KAC3B,SAAS,QAAQ,GAAG,OAAO;IAC7B;IACA;GACF;GACA,KAAK,eAAe;IAClB,MAAM,MAAM,SAAS,GAAG,KAAK;IAC7B,MAAM,WAAW,KAAK,OAAO,IAAI,GAAG;IACpC,IAAI,UAAU;KACZ,OAAO,SAAS,QAAQ,GAAG;KAC3B,IAAI,OAAO,KAAK,SAAS,OAAO,CAAC,CAAC,WAAW,GAC3C,KAAK,OAAO,OAAO,GAAG;IAE1B;IACA;GACF;GACA,KAAK,qBAGH,MAAM,IAAI,MAAM,cAAc,GAAG,GAAG,IAAI;GAC1C,SACE,YAAY,EAAE;EAClB;CAEJ;CAEA,SAAS;EAEP,KAAK,cAAc,MAAM,KAAK,MAAM;CACtC;CAEA,UAAU;EACR,KAAK,YAAY;CACnB;CAEA,YAAY;EACV,IAAI,KAAK,WAAW;GAClB,KAAK,SAAS,KAAK;GACnB,KAAK,YAAY;EACnB;CACF;AACF"}

package/out/zero-cache/src/services/change-streamer/backup-cleanup-monitor-factory.js.map

@@ -1 +1 @@
-{"version":3,"file":"backup-cleanup-monitor-factory.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/backup-cleanup-monitor-factory.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport type {NormalizedZeroConfig} from '../../config/normalize.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {\n  backupDestinationURL,\n  getLastBackupTime,\n} from '../litestream/commands.ts';\nimport {type BackedUpWatermark, BackupMonitor} from './backup-monitor.ts';\nimport type {ChangeStreamerService} from './change-streamer.ts';\nimport {LitestreamSyncRequester} from './litestream-sync-requester.ts';\nimport {\n  type BackupStateVerifier,\n  Litestream3PrometheusPoller,\n} from './litestream3-prometheus-poller.ts';\nimport {ReplicaPoller} from './replica-poller.ts';\n\nexport type BackupCleanupMonitorFactoryOptions = {\n  lc: LogContext;\n  config: NormalizedZeroConfig;\n  replicaFile: string;\n  changeStreamer: ChangeStreamerService;\n  verifyBackupState?: BackupStateVerifier | undefined;\n};\n\nexport function createBackupCleanupMonitor({\n  lc,\n  config,\n  replicaFile,\n  changeStreamer,\n  verifyBackupState,\n}: BackupCleanupMonitorFactoryOptions): BackupMonitor {\n  const {litestream, replica} = config;\n  const backupURL = backupDestinationURL(litestream);\n\n  let stream: Source<BackedUpWatermark>;\n\n  if (!backupURL) {\n    stream = new ReplicaPoller(lc, replicaFile).start();\n  } else if (config.litestream.backupUsingV5) {\n    stream = new LitestreamSyncRequester(lc, replicaFile, {\n      intervalMs: litestream.incrementalBackupIntervalSeconds * 1000,\n    }).start();\n  } else {\n    const {port: metricsPort} = litestream;\n    stream = new Litestream3PrometheusPoller(\n      lc,\n      replicaFile,\n      backupURL,\n      `http://localhost:${metricsPort}/metrics`,\n      verifyBackupState ??\n        (() => getLastBackupTime(lc, litestream, replica.file)),\n    ).start();\n  }\n\n  return new BackupMonitor(lc, stream, changeStreamer, replicaFile);\n}\n"],"mappings":";;;;;;AAwBA,SAAgB,2BAA2B,EACzC,IACA,QACA,aACA,gBACA,qBACoD;CACpD,MAAM,EAAC,YAAY,YAAW;CAC9B,MAAM,YAAY,qBAAqB,UAAU;CAEjD,IAAI;CAEJ,IAAI,CAAC,WACH,SAAS,IAAI,cAAc,IAAI,WAAW,EAAE,MAAM;MAC7C,IAAI,OAAO,WAAW,eAC3B,SAAS,IAAI,wBAAwB,IAAI,aAAa,EACpD,YAAY,WAAW,mCAAmC,IAC5D,CAAC,EAAE,MAAM;MACJ;EACL,MAAM,EAAC,MAAM,gBAAe;EAC5B,SAAS,IAAI,4BACX,IACA,aACA,WACA,oBAAoB,YAAY,WAChC,4BACS,kBAAkB,IAAI,YAAY,QAAQ,IAAI,EACzD,EAAE,MAAM;CACV;CAEA,OAAO,IAAI,cAAc,IAAI,QAAQ,gBAAgB,WAAW;AAClE"}
+{"version":3,"file":"backup-cleanup-monitor-factory.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/backup-cleanup-monitor-factory.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport type {NormalizedZeroConfig} from '../../config/normalize.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {\n  backupDestinationURL,\n  getLastBackupTime,\n} from '../litestream/commands.ts';\nimport {type BackedUpWatermark, BackupMonitor} from './backup-monitor.ts';\nimport type {ChangeStreamerService} from './change-streamer.ts';\nimport {LitestreamSyncRequester} from './litestream-sync-requester.ts';\nimport {\n  type BackupStateVerifier,\n  Litestream3PrometheusPoller,\n} from './litestream3-prometheus-poller.ts';\nimport {ReplicaPoller} from './replica-poller.ts';\n\nexport type BackupCleanupMonitorFactoryOptions = {\n  lc: LogContext;\n  config: NormalizedZeroConfig;\n  replicaFile: string;\n  changeStreamer: ChangeStreamerService;\n  verifyBackupState?: BackupStateVerifier | undefined;\n};\n\nexport function createBackupCleanupMonitor({\n  lc,\n  config,\n  replicaFile,\n  changeStreamer,\n  verifyBackupState,\n}: BackupCleanupMonitorFactoryOptions): BackupMonitor {\n  const {litestream, replica} = config;\n  const backupURL = backupDestinationURL(litestream);\n\n  let stream: Source<BackedUpWatermark>;\n\n  if (!backupURL) {\n    stream = new ReplicaPoller(lc, replicaFile).start();\n  } else if (config.litestream.backupUsingV5) {\n    stream = new LitestreamSyncRequester(lc, replicaFile, {\n      intervalMs: litestream.incrementalBackupIntervalSeconds * 1000,\n    }).start();\n  } else {\n    const {port: metricsPort} = litestream;\n    stream = new Litestream3PrometheusPoller(\n      lc,\n      replicaFile,\n      backupURL,\n      `http://localhost:${metricsPort}/metrics`,\n      verifyBackupState ??\n        (() => getLastBackupTime(lc, litestream, replica.file)),\n    ).start();\n  }\n\n  return new BackupMonitor(lc, stream, changeStreamer, replicaFile);\n}\n"],"mappings":";;;;;;AAwBA,SAAgB,2BAA2B,EACzC,IACA,QACA,aACA,gBACA,qBACoD;CACpD,MAAM,EAAC,YAAY,YAAW;CAC9B,MAAM,YAAY,qBAAqB,UAAU;CAEjD,IAAI;CAEJ,IAAI,CAAC,WACH,SAAS,IAAI,cAAc,IAAI,WAAW,CAAC,CAAC,MAAM;MAC7C,IAAI,OAAO,WAAW,eAC3B,SAAS,IAAI,wBAAwB,IAAI,aAAa,EACpD,YAAY,WAAW,mCAAmC,IAC5D,CAAC,CAAC,CAAC,MAAM;MACJ;EACL,MAAM,EAAC,MAAM,gBAAe;EAC5B,SAAS,IAAI,4BACX,IACA,aACA,WACA,oBAAoB,YAAY,WAChC,4BACS,kBAAkB,IAAI,YAAY,QAAQ,IAAI,EACzD,CAAC,CAAC,MAAM;CACV;CAEA,OAAO,IAAI,cAAc,IAAI,QAAQ,gBAAgB,WAAW;AAClE"}

package/out/zero-cache/src/services/change-streamer/backup-monitor.js.map

@@ -1 +1 @@
-{"version":3,"file":"backup-monitor.js","names":["#lc","#watermarks","#changeStreamer","#replicaFile","#firstBackupReceived","#initBackupLagMetric","#coverageTarget","#readReplicaStateVersion","#latestBackup","#checkFirstBackupCovers","#firstBackupResolved"],"sources":["../../../../../../zero-cache/src/services/change-streamer/backup-monitor.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {getOrCreateGauge} from '../../observability/metrics.ts';\nimport type {Source} from '../../types/streams.ts';\nimport type {SingletonService} from '../service.ts';\nimport type {ChangeStreamerService} from './change-streamer.ts';\n\nexport type BackedUpWatermark = {\n  watermark: string;\n  writeTimeMs?: number | undefined; // optional for debugging; available on v5\n  backupTimeMs: number;\n};\n\nexport class BackupMonitor implements SingletonService {\n  readonly id = 'backup-monitor';\n  readonly #lc: LogContext;\n  readonly #watermarks: Source<BackedUpWatermark>;\n  readonly #changeStreamer: ChangeStreamerService;\n  readonly #replicaFile: string;\n  readonly #firstBackupReceived = resolver();\n\n  #latestBackup: BackedUpWatermark | undefined;\n  #firstBackupResolved = false;\n  /**\n   * The replica's `stateVersion` when the monitor started, i.e. the point the\n   * backup has to reach before it covers what this task is about to serve.\n   * `undefined` when the replica could not be read, which degrades the gate to\n   * \"any watermark\" rather than hanging startup forever.\n   */\n  #coverageTarget: string | undefined;\n\n  constructor(\n    lc: LogContext,\n    watermarks: Source<BackedUpWatermark>,\n    changeStreamer: ChangeStreamerService,\n    replicaFile: string,\n  ) {\n    this.#lc = lc;\n    this.#watermarks = watermarks;\n    this.#changeStreamer = changeStreamer;\n    this.#replicaFile = replicaFile;\n  }\n\n  firstBackupReceived() {\n    return this.#firstBackupReceived.promise;\n  }\n\n  async run() {\n    this.#lc.info?.('starting backup monitor');\n    this.#initBackupLagMetric();\n    this.#coverageTarget = this.#readReplicaStateVersion();\n\n    for await (const backedUp of this.#watermarks) {\n      if (this.#latestBackup) {\n        if (backedUp.watermark < this.#latestBackup.watermark) {\n          this.#lc.warn?.(\n            `ignoring earlier backup watermark ${backedUp.watermark}`,\n            {backedUp},\n          );\n          continue;\n        }\n        if (backedUp.watermark === this.#latestBackup.watermark) {\n          this.#lc.debug?.(\n            `ignoring redundant backup watermark ${backedUp.watermark}`,\n            {backedUp},\n          );\n          continue;\n        }\n      }\n      this.#lc.info?.(`received backup watermark ${backedUp.watermark}`, {\n        backedUp,\n      });\n      this.#latestBackup = backedUp;\n      this.#changeStreamer.trackBackupWatermark(backedUp.watermark);\n      this.#checkFirstBackupCovers(backedUp);\n    }\n    this.#lc.info?.('watermark stream closed. BackupMonitor stopped.');\n  }\n\n  /**\n   * Resolves {@link firstBackupReceived} once the backup actually covers the\n   * replica, rather than on the first watermark of any value.\n   *\n   * A new backup destination starts empty and is filled by re-uploading the\n   * local LTX chain, so the first watermarks read back out of it are real but\n   * far behind the replica. Releasing the readiness gate on one of those lets\n   * the task serve against a backup that does not yet cover it, which is what\n   * demotes a restoring view-syncer to Postgres catchup.\n   *\n   * The target is pinned at startup rather than compared against the replica's\n   * current position, which keeps the gate satisfiable under sustained writes.\n   */\n  #checkFirstBackupCovers(backedUp: BackedUpWatermark) {\n    if (this.#firstBackupResolved) {\n      return;\n    }\n    const target = this.#coverageTarget;\n    if (target !== undefined && backedUp.watermark < target) {\n      this.#lc.info?.(`backup does not yet cover the replica`, {\n        backedUp,\n        coverageTarget: target,\n      });\n      return;\n    }\n    this.#firstBackupResolved = true;\n    this.#firstBackupReceived.resolve();\n  }\n\n  #readReplicaStateVersion(): string | undefined {\n    let db;\n    try {\n      db = new Database(this.#lc, this.#replicaFile, {readonly: true});\n      const {stateVersion} = db\n        .prepare(/*sql*/ `SELECT stateVersion FROM \"_zero.replicationState\"`)\n        .get<{stateVersion: string}>();\n      return stateVersion;\n    } catch (e) {\n      // Without a target the gate degrades to its previous behavior. That is\n      // strictly better than blocking startup on a replica we cannot read.\n      this.#lc.warn?.(\n        `unable to read the replica's stateVersion; ` +\n          `the initial backup gate will accept the first watermark`,\n        e,\n      );\n      return undefined;\n    } finally {\n      db?.close();\n    }\n  }\n\n  stop(): Promise<void> {\n    this.#watermarks.cancel();\n    return promiseVoid;\n  }\n\n  #initBackupLagMetric() {\n    getOrCreateGauge('replica', 'backup_lag', {\n      description:\n        'Latency from when a change is written to the replica ' +\n        'to when it is backed up to litestream. It is expected to create a saw ' +\n        'pattern from 0 to the configured ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_MINUTES.',\n      unit: 'millisecond',\n    }).addCallback(o => {\n      const latestBackup = this.#latestBackup;\n      if (!latestBackup) {\n        this.#lc.warn?.(\n          `no backed up watermarks. unable to report replica.backup_lag`,\n        );\n        return;\n      }\n      const db = new Database(this.#lc, this.#replicaFile, {readonly: true});\n      try {\n        const {writeTimeMs} = db\n          .prepare(/*sql*/ `SELECT writeTimeMs FROM \"_zero.replicationState\"`)\n          .get<{writeTimeMs: number}>();\n        const backupLag = Math.max(0, writeTimeMs - latestBackup.backupTimeMs);\n        o.observe(backupLag);\n      } catch (e) {\n        this.#lc.warn?.(`error measuring replica.backup_lag metric`, e);\n      } finally {\n        db.close();\n      }\n    });\n  }\n}\n"],"mappings":";;;;;AAeA,IAAa,gBAAb,MAAuD;CACrD,KAAc;CACd;CACA;CACA;CACA;CACA,uBAAgC,SAAS;CAEzC;CACA,uBAAuB;;;;;;;CAOvB;CAEA,YACE,IACA,YACA,gBACA,aACA;EACA,KAAKA,MAAM;EACX,KAAKC,cAAc;EACnB,KAAKC,kBAAkB;EACvB,KAAKC,eAAe;CACtB;CAEA,sBAAsB;EACpB,OAAO,KAAKC,qBAAqB;CACnC;CAEA,MAAM,MAAM;EACV,KAAKJ,IAAI,OAAO,yBAAyB;EACzC,KAAKK,qBAAqB;EAC1B,KAAKC,kBAAkB,KAAKC,yBAAyB;EAErD,WAAW,MAAM,YAAY,KAAKN,aAAa;GAC7C,IAAI,KAAKO,eAAe;IACtB,IAAI,SAAS,YAAY,KAAKA,cAAc,WAAW;KACrD,KAAKR,IAAI,OACP,qCAAqC,SAAS,aAC9C,EAAC,SAAQ,CACX;KACA;IACF;IACA,IAAI,SAAS,cAAc,KAAKQ,cAAc,WAAW;KACvD,KAAKR,IAAI,QACP,uCAAuC,SAAS,aAChD,EAAC,SAAQ,CACX;KACA;IACF;GACF;GACA,KAAKA,IAAI,OAAO,6BAA6B,SAAS,aAAa,EACjE,SACF,CAAC;GACD,KAAKQ,gBAAgB;GACrB,KAAKN,gBAAgB,qBAAqB,SAAS,SAAS;GAC5D,KAAKO,wBAAwB,QAAQ;EACvC;EACA,KAAKT,IAAI,OAAO,iDAAiD;CACnE;;;;;;;;;;;;;;CAeA,wBAAwB,UAA6B;EACnD,IAAI,KAAKU,sBACP;EAEF,MAAM,SAAS,KAAKJ;EACpB,IAAI,WAAW,KAAA,KAAa,SAAS,YAAY,QAAQ;GACvD,KAAKN,IAAI,OAAO,yCAAyC;IACvD;IACA,gBAAgB;GAClB,CAAC;GACD;EACF;EACA,KAAKU,uBAAuB;EAC5B,KAAKN,qBAAqB,QAAQ;CACpC;CAEA,2BAA+C;EAC7C,IAAI;EACJ,IAAI;GACF,KAAK,IAAI,SAAS,KAAKJ,KAAK,KAAKG,cAAc,EAAC,UAAU,KAAI,CAAC;GAC/D,MAAM,EAAC,iBAAgB,GACpB,QAAgB,mDAAmD,EACnE,IAA4B;GAC/B,OAAO;EACT,SAAS,GAAG;GAGV,KAAKH,IAAI,OACP,sGAEA,CACF;GACA;EACF,UAAU;GACR,IAAI,MAAM;EACZ;CACF;CAEA,OAAsB;EACpB,KAAKC,YAAY,OAAO;EACxB,OAAO;CACT;CAEA,uBAAuB;EACrB,iBAAiB,WAAW,cAAc;GACxC,aACE;GAGF,MAAM;EACR,CAAC,EAAE,aAAY,MAAK;GAClB,MAAM,eAAe,KAAKO;GAC1B,IAAI,CAAC,cAAc;IACjB,KAAKR,IAAI,OACP,8DACF;IACA;GACF;GACA,MAAM,KAAK,IAAI,SAAS,KAAKA,KAAK,KAAKG,cAAc,EAAC,UAAU,KAAI,CAAC;GACrE,IAAI;IACF,MAAM,EAAC,gBAAe,GACnB,QAAgB,kDAAkD,EAClE,IAA2B;IAC9B,MAAM,YAAY,KAAK,IAAI,GAAG,cAAc,aAAa,YAAY;IACrE,EAAE,QAAQ,SAAS;GACrB,SAAS,GAAG;IACV,KAAKH,IAAI,OAAO,6CAA6C,CAAC;GAChE,UAAU;IACR,GAAG,MAAM;GACX;EACF,CAAC;CACH;AACF"}
+{"version":3,"file":"backup-monitor.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/backup-monitor.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {getOrCreateGauge} from '../../observability/metrics.ts';\nimport type {Source} from '../../types/streams.ts';\nimport type {SingletonService} from '../service.ts';\nimport type {ChangeStreamerService} from './change-streamer.ts';\n\nexport type BackedUpWatermark = {\n  watermark: string;\n  writeTimeMs?: number | undefined; // optional for debugging; available on v5\n  backupTimeMs: number;\n};\n\nexport class BackupMonitor implements SingletonService {\n  readonly id = 'backup-monitor';\n  readonly #lc: LogContext;\n  readonly #watermarks: Source<BackedUpWatermark>;\n  readonly #changeStreamer: ChangeStreamerService;\n  readonly #replicaFile: string;\n  readonly #firstBackupReceived = resolver();\n\n  #latestBackup: BackedUpWatermark | undefined;\n  #firstBackupResolved = false;\n  /**\n   * The replica's `stateVersion` when the monitor started, i.e. the point the\n   * backup has to reach before it covers what this task is about to serve.\n   * `undefined` when the replica could not be read, which degrades the gate to\n   * \"any watermark\" rather than hanging startup forever.\n   */\n  #coverageTarget: string | undefined;\n\n  constructor(\n    lc: LogContext,\n    watermarks: Source<BackedUpWatermark>,\n    changeStreamer: ChangeStreamerService,\n    replicaFile: string,\n  ) {\n    this.#lc = lc;\n    this.#watermarks = watermarks;\n    this.#changeStreamer = changeStreamer;\n    this.#replicaFile = replicaFile;\n  }\n\n  firstBackupReceived() {\n    return this.#firstBackupReceived.promise;\n  }\n\n  async run() {\n    this.#lc.info?.('starting backup monitor');\n    this.#initBackupLagMetric();\n    this.#coverageTarget = this.#readReplicaStateVersion();\n\n    for await (const backedUp of this.#watermarks) {\n      if (this.#latestBackup) {\n        if (backedUp.watermark < this.#latestBackup.watermark) {\n          this.#lc.warn?.(\n            `ignoring earlier backup watermark ${backedUp.watermark}`,\n            {backedUp},\n          );\n          continue;\n        }\n        if (backedUp.watermark === this.#latestBackup.watermark) {\n          this.#lc.debug?.(\n            `ignoring redundant backup watermark ${backedUp.watermark}`,\n            {backedUp},\n          );\n          continue;\n        }\n      }\n      this.#lc.info?.(`received backup watermark ${backedUp.watermark}`, {\n        backedUp,\n      });\n      this.#latestBackup = backedUp;\n      this.#changeStreamer.trackBackupWatermark(backedUp.watermark);\n      this.#checkFirstBackupCovers(backedUp);\n    }\n    this.#lc.info?.('watermark stream closed. BackupMonitor stopped.');\n  }\n\n  /**\n   * Resolves {@link firstBackupReceived} once the backup actually covers the\n   * replica, rather than on the first watermark of any value.\n   *\n   * A new backup destination starts empty and is filled by re-uploading the\n   * local LTX chain, so the first watermarks read back out of it are real but\n   * far behind the replica. Releasing the readiness gate on one of those lets\n   * the task serve against a backup that does not yet cover it, which is what\n   * demotes a restoring view-syncer to Postgres catchup.\n   *\n   * The target is pinned at startup rather than compared against the replica's\n   * current position, which keeps the gate satisfiable under sustained writes.\n   */\n  #checkFirstBackupCovers(backedUp: BackedUpWatermark) {\n    if (this.#firstBackupResolved) {\n      return;\n    }\n    const target = this.#coverageTarget;\n    if (target !== undefined && backedUp.watermark < target) {\n      this.#lc.info?.(`backup does not yet cover the replica`, {\n        backedUp,\n        coverageTarget: target,\n      });\n      return;\n    }\n    this.#firstBackupResolved = true;\n    this.#firstBackupReceived.resolve();\n  }\n\n  #readReplicaStateVersion(): string | undefined {\n    let db;\n    try {\n      db = new Database(this.#lc, this.#replicaFile, {readonly: true});\n      const {stateVersion} = db\n        .prepare(/*sql*/ `SELECT stateVersion FROM \"_zero.replicationState\"`)\n        .get<{stateVersion: string}>();\n      return stateVersion;\n    } catch (e) {\n      // Without a target the gate degrades to its previous behavior. That is\n      // strictly better than blocking startup on a replica we cannot read.\n      this.#lc.warn?.(\n        `unable to read the replica's stateVersion; ` +\n          `the initial backup gate will accept the first watermark`,\n        e,\n      );\n      return undefined;\n    } finally {\n      db?.close();\n    }\n  }\n\n  stop(): Promise<void> {\n    this.#watermarks.cancel();\n    return promiseVoid;\n  }\n\n  #initBackupLagMetric() {\n    getOrCreateGauge('replica', 'backup_lag', {\n      description:\n        'Latency from when a change is written to the replica ' +\n        'to when it is backed up to litestream. It is expected to create a saw ' +\n        'pattern from 0 to the configured ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_MINUTES.',\n      unit: 'millisecond',\n    }).addCallback(o => {\n      const latestBackup = this.#latestBackup;\n      if (!latestBackup) {\n        this.#lc.warn?.(\n          `no backed up watermarks. unable to report replica.backup_lag`,\n        );\n        return;\n      }\n      const db = new Database(this.#lc, this.#replicaFile, {readonly: true});\n      try {\n        const {writeTimeMs} = db\n          .prepare(/*sql*/ `SELECT writeTimeMs FROM \"_zero.replicationState\"`)\n          .get<{writeTimeMs: number}>();\n        const backupLag = Math.max(0, writeTimeMs - latestBackup.backupTimeMs);\n        o.observe(backupLag);\n      } catch (e) {\n        this.#lc.warn?.(`error measuring replica.backup_lag metric`, e);\n      } finally {\n        db.close();\n      }\n    });\n  }\n}\n"],"mappings":";;;;;AAeA,IAAa,gBAAb,MAAuD;CACrD,KAAc;CACd;CACA;CACA;CACA;CACA,uBAAgC,SAAS;CAEzC;CACA,uBAAuB;;;;;;;CAOvB;CAEA,YACE,IACA,YACA,gBACA,aACA;EACA,KAAK,MAAM;EACX,KAAK,cAAc;EACnB,KAAK,kBAAkB;EACvB,KAAK,eAAe;CACtB;CAEA,sBAAsB;EACpB,OAAO,KAAK,qBAAqB;CACnC;CAEA,MAAM,MAAM;EACV,KAAK,IAAI,OAAO,yBAAyB;EACzC,KAAK,qBAAqB;EAC1B,KAAK,kBAAkB,KAAK,yBAAyB;EAErD,WAAW,MAAM,YAAY,KAAK,aAAa;GAC7C,IAAI,KAAK,eAAe;IACtB,IAAI,SAAS,YAAY,KAAK,cAAc,WAAW;KACrD,KAAK,IAAI,OACP,qCAAqC,SAAS,aAC9C,EAAC,SAAQ,CACX;KACA;IACF;IACA,IAAI,SAAS,cAAc,KAAK,cAAc,WAAW;KACvD,KAAK,IAAI,QACP,uCAAuC,SAAS,aAChD,EAAC,SAAQ,CACX;KACA;IACF;GACF;GACA,KAAK,IAAI,OAAO,6BAA6B,SAAS,aAAa,EACjE,SACF,CAAC;GACD,KAAK,gBAAgB;GACrB,KAAK,gBAAgB,qBAAqB,SAAS,SAAS;GAC5D,KAAK,wBAAwB,QAAQ;EACvC;EACA,KAAK,IAAI,OAAO,iDAAiD;CACnE;;;;;;;;;;;;;;CAeA,wBAAwB,UAA6B;EACnD,IAAI,KAAK,sBACP;EAEF,MAAM,SAAS,KAAK;EACpB,IAAI,WAAW,KAAA,KAAa,SAAS,YAAY,QAAQ;GACvD,KAAK,IAAI,OAAO,yCAAyC;IACvD;IACA,gBAAgB;GAClB,CAAC;GACD;EACF;EACA,KAAK,uBAAuB;EAC5B,KAAK,qBAAqB,QAAQ;CACpC;CAEA,2BAA+C;EAC7C,IAAI;EACJ,IAAI;GACF,KAAK,IAAI,SAAS,KAAK,KAAK,KAAK,cAAc,EAAC,UAAU,KAAI,CAAC;GAC/D,MAAM,EAAC,iBAAgB,GACpB,QAAgB,mDAAmD,CAAC,CACpE,IAA4B;GAC/B,OAAO;EACT,SAAS,GAAG;GAGV,KAAK,IAAI,OACP,sGAEA,CACF;GACA;EACF,UAAU;GACR,IAAI,MAAM;EACZ;CACF;CAEA,OAAsB;EACpB,KAAK,YAAY,OAAO;EACxB,OAAO;CACT;CAEA,uBAAuB;EACrB,iBAAiB,WAAW,cAAc;GACxC,aACE;GAGF,MAAM;EACR,CAAC,CAAC,CAAC,aAAY,MAAK;GAClB,MAAM,eAAe,KAAK;GAC1B,IAAI,CAAC,cAAc;IACjB,KAAK,IAAI,OACP,8DACF;IACA;GACF;GACA,MAAM,KAAK,IAAI,SAAS,KAAK,KAAK,KAAK,cAAc,EAAC,UAAU,KAAI,CAAC;GACrE,IAAI;IACF,MAAM,EAAC,gBAAe,GACnB,QAAgB,kDAAkD,CAAC,CACnE,IAA2B;IAC9B,MAAM,YAAY,KAAK,IAAI,GAAG,cAAc,aAAa,YAAY;IACrE,EAAE,QAAQ,SAAS;GACrB,SAAS,GAAG;IACV,KAAK,IAAI,OAAO,6CAA6C,CAAC;GAChE,UAAU;IACR,GAAG,MAAM;GACX;EACF,CAAC;CACH;AACF"}

package/out/zero-cache/src/services/change-streamer/broadcast.js

@@ -182,6 +182,6 @@ function toBatch(change) {
 	return Array.isArray(change[0]) ? change : [change];
 }
 //#endregion
-export { Broadcast };
+export { Broadcast, preSerializeBatch };
 
 //# sourceMappingURL=broadcast.js.map

package/out/zero-cache/src/services/change-streamer/broadcast.js.map

@@ -1 +1 @@
-{"version":3,"file":"broadcast.js","names":["#lc","#pending","#completed","#done","#watermark","#majority","#start","#earlyReleaseTimeoutProportion","#setTimeout","#clearTimeout","#needsBackupResponse","#markCompleted","#setDone","#isDone","#earlyReleaseTimer","#logWithState","#releaseMode"],"sources":["../../../../../../zero-cache/src/services/change-streamer/broadcast.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {must} from '../../../../shared/src/must.ts';\nimport {isPreSerialized, type PreSerialized} from '../../types/streams.ts';\nimport type {WatermarkedChange} from './change-streamer.ts';\nimport type {Subscriber} from './subscriber.ts';\n\nexport type BroadcastReleaseMode = 'all-subscribers' | 'consensus-timeout';\n\nexport type PreSerializedBatch = PreSerialized & {\n  readonly changes: readonly WatermarkedChange[];\n};\n\nexport function isPreSerializedBatch(val: unknown): val is PreSerializedBatch {\n  return isPreSerialized(val) && 'changes' in val && Array.isArray(val.changes);\n}\n\nexport function preSerializeBatch(\n  changes: readonly WatermarkedChange[],\n): PreSerializedBatch {\n  const jsonList = changes.map(c => c[2]);\n  const joined = jsonList.join(',');\n  const payload = Buffer.from(\n    changes.length === 1 ? `,\"msg\":${jsonList[0]}}` : `,\"batch\":[${joined}]}`,\n    'utf8',\n  );\n  return {\n    changes,\n    payload,\n    byteLength: payload.length,\n  };\n}\n\n/**\n * Enables event-driven early release of a {@link Broadcast}. When provided and a\n * majority has acked, the broadcast schedules its own consensus-timeout release\n * proportional to how long the majority took to ack, instead of relying on the\n * Forwarder's periodic progress tick.\n */\nexport type EarlyReleaseOptions = {\n  /** Time to wait, proportional to majority completion, before releasing. */\n  consensusTimeoutProportion: number;\n  /** Injectable timer, for deterministic testing. */\n  setTimeoutFn?: typeof setTimeout;\n  /** Injectable timer cancellation, for deterministic testing. */\n  clearTimeoutFn?: typeof clearTimeout;\n};\n\n/**\n * Initiates and tracks the progress of a change broadcasted to\n * a set of subscribers, implementing **consensus-based timeouts**\n * to effect short-term graceful degradation when a minority of\n * subscribers are behind.\n *\n * ### Background\n *\n * The purpose of flow control is to pull upstream replication changes\n * no faster than the rate they are processed by downstream subscribers\n * in the steady state. In the change-streamer, this is done by occasionally\n * waiting for ACKs from subscribers before continuing; without doing so,\n * I/O buffers fill up and cause the system to spend most of its time in GC.\n *\n * However, the naive algorithm of always waiting for all subscribers (e.g.\n * `Promise.all()`) can behave poorly in scenarios where subscribers\n * are imbalanced:\n * * New subscribers may have a backlog of changes to catch up with.\n *   Having all subscribers wait for the new subscriber to catch up results\n *   in delaying the entire application.\n * * Broken TCP connections similarly require all subscribers to wait until\n *   connection liveness checks kick in and disconnect the subscriber.\n *\n * A simplistic approach is to add a limit to the amount of time waiting for\n * subscribers, i.e. an ack timeout. However, deciding what this timeout\n * should be is non-trivial because of the heterogeneous nature of changes;\n * while most changes operate on single rows and are relatively predictable\n * in terms of running time, some changes are table-wide operations and can\n * legitimately take an arbitrary amount of time. In such scenarios, a\n * timeout that is too short can stop progress on replication altogether.\n *\n * ### Consensus-based Timeout Algorithm\n *\n * To address these shortcomings, a \"consensus-based timeout\" algorithm is\n * used:\n * * Wait for more than half of the subscribers to finish. (In\n *   case of a single node, or the case of one replication-manager\n *   and one view-syncer, this reduces to waiting for all subscribers.)\n * * Once more than half of the subscribers have finished, proceed after\n *   the configured timeout, proportional to the time it took for the\n *   majority of subscribers to ack, even if not all subscribers have\n *   finished.\n *\n * In other words, the subscribers themselves are used to determine the\n * timeout of each batch of changes; the majority determines this when\n * they complete, upon which a timeout is logically started.\n *\n * In the common case, the remaining subscribers finish soon afterward and\n * the timeout never elapses. However, in pathological cases where a minority\n * of subscribers have a disproportionate amount of load, some will still\n * be processing (or otherwise unresponsive). These subscribers are given\n * a bounded amount of time to catch up at each flushed batch, up to the\n * timeout interval. This guarantees eventual catchup because the\n * subscribers with a backlog of changes necessarily have a higher\n * processing rate than the subscribers that finished (and are made to wait).\n */\nexport class Broadcast {\n  /**\n   * Sends the change to the subscribers without the tracking machinery.\n   * This is suitable for fire-and-forget (i.e. pipelined) sends.\n   */\n  static withoutTracking(\n    subscribers: Iterable<Subscriber>,\n    change: WatermarkedChange | readonly WatermarkedChange[],\n  ) {\n    const changes = toBatch(change);\n    const preSerialized = preSerializeBatch(changes);\n    for (const sub of subscribers) {\n      void sub.sendBatch(changes, preSerialized);\n    }\n  }\n\n  readonly #lc: LogContext;\n  readonly #pending: Set<Subscriber>;\n  readonly #completed: Completed[];\n  readonly #done = resolver();\n  #isDone = false;\n  #releaseMode: BroadcastReleaseMode | undefined;\n\n  readonly #watermark: string;\n  readonly #majority: number;\n\n  readonly #start = performance.now();\n\n  // When set, the broadcast schedules its own consensus-timeout release instead\n  // of waiting for the Forwarder's periodic progress tick.\n  readonly #earlyReleaseTimeoutProportion: number | undefined;\n  readonly #setTimeout: typeof setTimeout;\n  readonly #clearTimeout: typeof clearTimeout;\n  #earlyReleaseTimer: ReturnType<typeof setTimeout> | undefined;\n\n  // Tracks whether the backup-replicator responded. It is a required member\n  // of the consensus timeout since it cannot be dropped as a laggard (since\n  // that would shutdown the replication-manager itself).\n  #needsBackupResponse = false;\n\n  /**\n   * Broadcasts the `change` to the `subscribers` and tracks their\n   * completion.\n   */\n  constructor(\n    lc: LogContext,\n    subscribers: Iterable<Subscriber>,\n    change: WatermarkedChange | readonly WatermarkedChange[],\n    earlyRelease?: EarlyReleaseOptions,\n  ) {\n    const changes = toBatch(change);\n    const preSerialized = preSerializeBatch(changes);\n    this.#lc = lc;\n    this.#pending = new Set(subscribers);\n    this.#completed = [];\n    this.#watermark = must(changes.at(-1))[0];\n    this.#majority = Math.floor(this.#pending.size / 2) + 1;\n    this.#earlyReleaseTimeoutProportion =\n      earlyRelease?.consensusTimeoutProportion;\n    this.#setTimeout = earlyRelease?.setTimeoutFn ?? setTimeout;\n    this.#clearTimeout = earlyRelease?.clearTimeoutFn ?? clearTimeout;\n\n    for (const sub of this.#pending) {\n      const totalChanges = sub.numPending + changes.length;\n      if (sub.mode === 'backup') {\n        // Only gate consensus on the backup-replicator after it has finished\n        // its initial catchup.\n        this.#needsBackupResponse ||= !sub.isBacklogged();\n      }\n      void sub\n        .sendBatch(changes, preSerialized)\n        .catch(() => {})\n        .finally(() => this.#markCompleted(sub, totalChanges));\n    }\n\n    // set done if there are no subscribers (mainly for tests)\n    if (this.#pending.size === 0) {\n      this.#setDone('all-subscribers');\n    }\n  }\n\n  #markCompleted(sub: Subscriber, changes: number) {\n    if (this.#isDone) {\n      return;\n    }\n    const elapsed = performance.now() - this.#start;\n    sub.trackResponseResult('on-time');\n    if (sub.mode === 'backup') {\n      this.#needsBackupResponse = false;\n    }\n    this.#completed.push({sub, changes, elapsed});\n    this.#pending.delete(sub);\n    if (this.#pending.size === 0) {\n      this.#setDone('all-subscribers');\n      return;\n    }\n    // Event-driven consensus-timeout: the moment a majority acks, and it\n    // includes the backup-replicator, arm a single release timer\n    // proportional to how long the majority took to ack. The broadcast\n    // resolves when it fires, unless all subscribers ack first.\n    if (\n      this.#earlyReleaseTimeoutProportion !== undefined &&\n      this.#earlyReleaseTimer === undefined &&\n      this.#completed.length >= this.#majority &&\n      !this.#needsBackupResponse\n    ) {\n      const timeoutMs = Math.ceil(\n        elapsed * this.#earlyReleaseTimeoutProportion,\n      );\n      this.#earlyReleaseTimer = this.#setTimeout(\n        () => this.#setDone('consensus-timeout'),\n        timeoutMs,\n      );\n    }\n  }\n\n  #setDone(releaseMode: BroadcastReleaseMode) {\n    if (this.#isDone) {\n      return;\n    }\n    if (releaseMode === 'consensus-timeout') {\n      const elapsed = performance.now() - this.#start;\n      this.#logWithState(\n        `continuing with ${this.#pending.size} subscriber(s) still pending`,\n        elapsed,\n      );\n    }\n    this.#pending.forEach(sub => sub.trackResponseResult('timed-out'));\n    this.#isDone = true;\n    this.#releaseMode = releaseMode;\n    this.#clearTimeout(this.#earlyReleaseTimer);\n    this.#earlyReleaseTimer = undefined;\n    this.#done.resolve();\n  }\n\n  get isDone(): boolean {\n    return this.#isDone;\n  }\n\n  get done(): Promise<void> {\n    return this.#done.promise;\n  }\n\n  get releaseMode(): BroadcastReleaseMode {\n    return this.#releaseMode ?? 'all-subscribers';\n  }\n\n  logSlowRequests(now: number) {\n    if (this.#isDone || this.#pending.size === 0) {\n      return;\n    }\n    const elapsed = now - this.#start;\n    if (this.#completed.length < this.#majority) {\n      if (elapsed >= 1000) {\n        this.#logWithState(\n          `waiting for at least ${this.#majority} subscribers to finish`,\n          elapsed,\n        );\n      }\n    }\n  }\n\n  #logWithState(msg: string, elapsed: number) {\n    this.#lc\n      .withContext('watermark', this.#watermark)\n      .info?.(`${msg} (${elapsed.toFixed(3)} ms)`, {\n        completed: this.#completed.map(d => ({\n          id: d.sub.id,\n          processed: d.changes,\n          elapsed: d.elapsed,\n        })),\n        pending: Array.from(this.#pending, sub => ({\n          id: sub.id,\n          ...sub.getStats(),\n        })),\n      });\n  }\n}\n\n/** Tracks the completed result of a single subscriber. */\ntype Completed = {\n  sub: Subscriber;\n  /** The number of changes processed. */\n  changes: number;\n  /** The elapsed milliseconds. */\n  elapsed: number;\n};\n\nfunction toBatch(\n  change: WatermarkedChange | readonly WatermarkedChange[],\n): readonly WatermarkedChange[] {\n  if (change.length === 0) {\n    return [];\n  }\n  return Array.isArray(change[0])\n    ? (change as readonly WatermarkedChange[])\n    : [change as WatermarkedChange];\n}\n"],"mappings":";;;;AAiBA,SAAgB,kBACd,SACoB;CACpB,MAAM,WAAW,QAAQ,KAAI,MAAK,EAAE,EAAE;CACtC,MAAM,SAAS,SAAS,KAAK,GAAG;CAChC,MAAM,UAAU,OAAO,KACrB,QAAQ,WAAW,IAAI,UAAU,SAAS,GAAG,KAAK,aAAa,OAAO,KACtE,MACF;CACA,OAAO;EACL;EACA;EACA,YAAY,QAAQ;CACtB;AACF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAyEA,IAAa,YAAb,MAAuB;;;;;CAKrB,OAAO,gBACL,aACA,QACA;EACA,MAAM,UAAU,QAAQ,MAAM;EAC9B,MAAM,gBAAgB,kBAAkB,OAAO;EAC/C,KAAK,MAAM,OAAO,aAChB,IAAS,UAAU,SAAS,aAAa;CAE7C;CAEA;CACA;CACA;CACA,QAAiB,SAAS;CAC1B,UAAU;CACV;CAEA;CACA;CAEA,SAAkB,YAAY,IAAI;CAIlC;CACA;CACA;CACA;CAKA,uBAAuB;;;;;CAMvB,YACE,IACA,aACA,QACA,cACA;EACA,MAAM,UAAU,QAAQ,MAAM;EAC9B,MAAM,gBAAgB,kBAAkB,OAAO;EAC/C,KAAKA,MAAM;EACX,KAAKC,WAAW,IAAI,IAAI,WAAW;EACnC,KAAKC,aAAa,CAAC;EACnB,KAAKE,aAAa,KAAK,QAAQ,GAAG,EAAE,CAAC,EAAE;EACvC,KAAKC,YAAY,KAAK,MAAM,KAAKJ,SAAS,OAAO,CAAC,IAAI;EACtD,KAAKM,iCACH,cAAc;EAChB,KAAKC,cAAc,cAAc,gBAAgB;EACjD,KAAKC,gBAAgB,cAAc,kBAAkB;EAErD,KAAK,MAAM,OAAO,KAAKR,UAAU;GAC/B,MAAM,eAAe,IAAI,aAAa,QAAQ;GAC9C,IAAI,IAAI,SAAS,UAGf,KAAKS,yBAAyB,CAAC,IAAI,aAAa;GAElD,IACG,UAAU,SAAS,aAAa,EAChC,YAAY,CAAC,CAAC,EACd,cAAc,KAAKC,eAAe,KAAK,YAAY,CAAC;EACzD;EAGA,IAAI,KAAKV,SAAS,SAAS,GACzB,KAAKW,SAAS,iBAAiB;CAEnC;CAEA,eAAe,KAAiB,SAAiB;EAC/C,IAAI,KAAKC,SACP;EAEF,MAAM,UAAU,YAAY,IAAI,IAAI,KAAKP;EACzC,IAAI,oBAAoB,SAAS;EACjC,IAAI,IAAI,SAAS,UACf,KAAKI,uBAAuB;EAE9B,KAAKR,WAAW,KAAK;GAAC;GAAK;GAAS;EAAO,CAAC;EAC5C,KAAKD,SAAS,OAAO,GAAG;EACxB,IAAI,KAAKA,SAAS,SAAS,GAAG;GAC5B,KAAKW,SAAS,iBAAiB;GAC/B;EACF;EAKA,IACE,KAAKL,mCAAmC,KAAA,KACxC,KAAKO,uBAAuB,KAAA,KAC5B,KAAKZ,WAAW,UAAU,KAAKG,aAC/B,CAAC,KAAKK,sBACN;GACA,MAAM,YAAY,KAAK,KACrB,UAAU,KAAKH,8BACjB;GACA,KAAKO,qBAAqB,KAAKN,kBACvB,KAAKI,SAAS,mBAAmB,GACvC,SACF;EACF;CACF;CAEA,SAAS,aAAmC;EAC1C,IAAI,KAAKC,SACP;EAEF,IAAI,gBAAgB,qBAAqB;GACvC,MAAM,UAAU,YAAY,IAAI,IAAI,KAAKP;GACzC,KAAKS,cACH,mBAAmB,KAAKd,SAAS,KAAK,+BACtC,OACF;EACF;EACA,KAAKA,SAAS,SAAQ,QAAO,IAAI,oBAAoB,WAAW,CAAC;EACjE,KAAKY,UAAU;EACf,KAAKG,eAAe;EACpB,KAAKP,cAAc,KAAKK,kBAAkB;EAC1C,KAAKA,qBAAqB,KAAA;EAC1B,KAAKX,MAAM,QAAQ;CACrB;CAEA,IAAI,SAAkB;EACpB,OAAO,KAAKU;CACd;CAEA,IAAI,OAAsB;EACxB,OAAO,KAAKV,MAAM;CACpB;CAEA,IAAI,cAAoC;EACtC,OAAO,KAAKa,gBAAgB;CAC9B;CAEA,gBAAgB,KAAa;EAC3B,IAAI,KAAKH,WAAW,KAAKZ,SAAS,SAAS,GACzC;EAEF,MAAM,UAAU,MAAM,KAAKK;EAC3B,IAAI,KAAKJ,WAAW,SAAS,KAAKG;OAC5B,WAAW,KACb,KAAKU,cACH,wBAAwB,KAAKV,UAAU,yBACvC,OACF;EAAA;CAGN;CAEA,cAAc,KAAa,SAAiB;EAC1C,KAAKL,IACF,YAAY,aAAa,KAAKI,UAAU,EACxC,OAAO,GAAG,IAAI,IAAI,QAAQ,QAAQ,CAAC,EAAE,OAAO;GAC3C,WAAW,KAAKF,WAAW,KAAI,OAAM;IACnC,IAAI,EAAE,IAAI;IACV,WAAW,EAAE;IACb,SAAS,EAAE;GACb,EAAE;GACF,SAAS,MAAM,KAAK,KAAKD,WAAU,SAAQ;IACzC,IAAI,IAAI;IACR,GAAG,IAAI,SAAS;GAClB,EAAE;EACJ,CAAC;CACL;AACF;AAWA,SAAS,QACP,QAC8B;CAC9B,IAAI,OAAO,WAAW,GACpB,OAAO,CAAC;CAEV,OAAO,MAAM,QAAQ,OAAO,EAAE,IACzB,SACD,CAAC,MAA2B;AAClC"}
+{"version":3,"file":"broadcast.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/broadcast.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {must} from '../../../../shared/src/must.ts';\nimport {isPreSerialized, type PreSerialized} from '../../types/streams.ts';\nimport type {WatermarkedChange} from './change-streamer.ts';\nimport type {Subscriber} from './subscriber.ts';\n\nexport type BroadcastReleaseMode = 'all-subscribers' | 'consensus-timeout';\n\nexport type PreSerializedBatch = PreSerialized & {\n  readonly changes: readonly WatermarkedChange[];\n};\n\nexport function isPreSerializedBatch(val: unknown): val is PreSerializedBatch {\n  return isPreSerialized(val) && 'changes' in val && Array.isArray(val.changes);\n}\n\nexport function preSerializeBatch(\n  changes: readonly WatermarkedChange[],\n): PreSerializedBatch {\n  const jsonList = changes.map(c => c[2]);\n  const joined = jsonList.join(',');\n  const payload = Buffer.from(\n    changes.length === 1 ? `,\"msg\":${jsonList[0]}}` : `,\"batch\":[${joined}]}`,\n    'utf8',\n  );\n  return {\n    changes,\n    payload,\n    byteLength: payload.length,\n  };\n}\n\n/**\n * Enables event-driven early release of a {@link Broadcast}. When provided and a\n * majority has acked, the broadcast schedules its own consensus-timeout release\n * proportional to how long the majority took to ack, instead of relying on the\n * Forwarder's periodic progress tick.\n */\nexport type EarlyReleaseOptions = {\n  /** Time to wait, proportional to majority completion, before releasing. */\n  consensusTimeoutProportion: number;\n  /** Injectable timer, for deterministic testing. */\n  setTimeoutFn?: typeof setTimeout;\n  /** Injectable timer cancellation, for deterministic testing. */\n  clearTimeoutFn?: typeof clearTimeout;\n};\n\n/**\n * Initiates and tracks the progress of a change broadcasted to\n * a set of subscribers, implementing **consensus-based timeouts**\n * to effect short-term graceful degradation when a minority of\n * subscribers are behind.\n *\n * ### Background\n *\n * The purpose of flow control is to pull upstream replication changes\n * no faster than the rate they are processed by downstream subscribers\n * in the steady state. In the change-streamer, this is done by occasionally\n * waiting for ACKs from subscribers before continuing; without doing so,\n * I/O buffers fill up and cause the system to spend most of its time in GC.\n *\n * However, the naive algorithm of always waiting for all subscribers (e.g.\n * `Promise.all()`) can behave poorly in scenarios where subscribers\n * are imbalanced:\n * * New subscribers may have a backlog of changes to catch up with.\n *   Having all subscribers wait for the new subscriber to catch up results\n *   in delaying the entire application.\n * * Broken TCP connections similarly require all subscribers to wait until\n *   connection liveness checks kick in and disconnect the subscriber.\n *\n * A simplistic approach is to add a limit to the amount of time waiting for\n * subscribers, i.e. an ack timeout. However, deciding what this timeout\n * should be is non-trivial because of the heterogeneous nature of changes;\n * while most changes operate on single rows and are relatively predictable\n * in terms of running time, some changes are table-wide operations and can\n * legitimately take an arbitrary amount of time. In such scenarios, a\n * timeout that is too short can stop progress on replication altogether.\n *\n * ### Consensus-based Timeout Algorithm\n *\n * To address these shortcomings, a \"consensus-based timeout\" algorithm is\n * used:\n * * Wait for more than half of the subscribers to finish. (In\n *   case of a single node, or the case of one replication-manager\n *   and one view-syncer, this reduces to waiting for all subscribers.)\n * * Once more than half of the subscribers have finished, proceed after\n *   the configured timeout, proportional to the time it took for the\n *   majority of subscribers to ack, even if not all subscribers have\n *   finished.\n *\n * In other words, the subscribers themselves are used to determine the\n * timeout of each batch of changes; the majority determines this when\n * they complete, upon which a timeout is logically started.\n *\n * In the common case, the remaining subscribers finish soon afterward and\n * the timeout never elapses. However, in pathological cases where a minority\n * of subscribers have a disproportionate amount of load, some will still\n * be processing (or otherwise unresponsive). These subscribers are given\n * a bounded amount of time to catch up at each flushed batch, up to the\n * timeout interval. This guarantees eventual catchup because the\n * subscribers with a backlog of changes necessarily have a higher\n * processing rate than the subscribers that finished (and are made to wait).\n */\nexport class Broadcast {\n  /**\n   * Sends the change to the subscribers without the tracking machinery.\n   * This is suitable for fire-and-forget (i.e. pipelined) sends.\n   */\n  static withoutTracking(\n    subscribers: Iterable<Subscriber>,\n    change: WatermarkedChange | readonly WatermarkedChange[],\n  ) {\n    const changes = toBatch(change);\n    const preSerialized = preSerializeBatch(changes);\n    for (const sub of subscribers) {\n      void sub.sendBatch(changes, preSerialized);\n    }\n  }\n\n  readonly #lc: LogContext;\n  readonly #pending: Set<Subscriber>;\n  readonly #completed: Completed[];\n  readonly #done = resolver();\n  #isDone = false;\n  #releaseMode: BroadcastReleaseMode | undefined;\n\n  readonly #watermark: string;\n  readonly #majority: number;\n\n  readonly #start = performance.now();\n\n  // When set, the broadcast schedules its own consensus-timeout release instead\n  // of waiting for the Forwarder's periodic progress tick.\n  readonly #earlyReleaseTimeoutProportion: number | undefined;\n  readonly #setTimeout: typeof setTimeout;\n  readonly #clearTimeout: typeof clearTimeout;\n  #earlyReleaseTimer: ReturnType<typeof setTimeout> | undefined;\n\n  // Tracks whether the backup-replicator responded. It is a required member\n  // of the consensus timeout since it cannot be dropped as a laggard (since\n  // that would shutdown the replication-manager itself).\n  #needsBackupResponse = false;\n\n  /**\n   * Broadcasts the `change` to the `subscribers` and tracks their\n   * completion.\n   */\n  constructor(\n    lc: LogContext,\n    subscribers: Iterable<Subscriber>,\n    change: WatermarkedChange | readonly WatermarkedChange[],\n    earlyRelease?: EarlyReleaseOptions,\n  ) {\n    const changes = toBatch(change);\n    const preSerialized = preSerializeBatch(changes);\n    this.#lc = lc;\n    this.#pending = new Set(subscribers);\n    this.#completed = [];\n    this.#watermark = must(changes.at(-1))[0];\n    this.#majority = Math.floor(this.#pending.size / 2) + 1;\n    this.#earlyReleaseTimeoutProportion =\n      earlyRelease?.consensusTimeoutProportion;\n    this.#setTimeout = earlyRelease?.setTimeoutFn ?? setTimeout;\n    this.#clearTimeout = earlyRelease?.clearTimeoutFn ?? clearTimeout;\n\n    for (const sub of this.#pending) {\n      const totalChanges = sub.numPending + changes.length;\n      if (sub.mode === 'backup') {\n        // Only gate consensus on the backup-replicator after it has finished\n        // its initial catchup.\n        this.#needsBackupResponse ||= !sub.isBacklogged();\n      }\n      void sub\n        .sendBatch(changes, preSerialized)\n        .catch(() => {})\n        .finally(() => this.#markCompleted(sub, totalChanges));\n    }\n\n    // set done if there are no subscribers (mainly for tests)\n    if (this.#pending.size === 0) {\n      this.#setDone('all-subscribers');\n    }\n  }\n\n  #markCompleted(sub: Subscriber, changes: number) {\n    if (this.#isDone) {\n      return;\n    }\n    const elapsed = performance.now() - this.#start;\n    sub.trackResponseResult('on-time');\n    if (sub.mode === 'backup') {\n      this.#needsBackupResponse = false;\n    }\n    this.#completed.push({sub, changes, elapsed});\n    this.#pending.delete(sub);\n    if (this.#pending.size === 0) {\n      this.#setDone('all-subscribers');\n      return;\n    }\n    // Event-driven consensus-timeout: the moment a majority acks, and it\n    // includes the backup-replicator, arm a single release timer\n    // proportional to how long the majority took to ack. The broadcast\n    // resolves when it fires, unless all subscribers ack first.\n    if (\n      this.#earlyReleaseTimeoutProportion !== undefined &&\n      this.#earlyReleaseTimer === undefined &&\n      this.#completed.length >= this.#majority &&\n      !this.#needsBackupResponse\n    ) {\n      const timeoutMs = Math.ceil(\n        elapsed * this.#earlyReleaseTimeoutProportion,\n      );\n      this.#earlyReleaseTimer = this.#setTimeout(\n        () => this.#setDone('consensus-timeout'),\n        timeoutMs,\n      );\n    }\n  }\n\n  #setDone(releaseMode: BroadcastReleaseMode) {\n    if (this.#isDone) {\n      return;\n    }\n    if (releaseMode === 'consensus-timeout') {\n      const elapsed = performance.now() - this.#start;\n      this.#logWithState(\n        `continuing with ${this.#pending.size} subscriber(s) still pending`,\n        elapsed,\n      );\n    }\n    this.#pending.forEach(sub => sub.trackResponseResult('timed-out'));\n    this.#isDone = true;\n    this.#releaseMode = releaseMode;\n    this.#clearTimeout(this.#earlyReleaseTimer);\n    this.#earlyReleaseTimer = undefined;\n    this.#done.resolve();\n  }\n\n  get isDone(): boolean {\n    return this.#isDone;\n  }\n\n  get done(): Promise<void> {\n    return this.#done.promise;\n  }\n\n  get releaseMode(): BroadcastReleaseMode {\n    return this.#releaseMode ?? 'all-subscribers';\n  }\n\n  logSlowRequests(now: number) {\n    if (this.#isDone || this.#pending.size === 0) {\n      return;\n    }\n    const elapsed = now - this.#start;\n    if (this.#completed.length < this.#majority) {\n      if (elapsed >= 1000) {\n        this.#logWithState(\n          `waiting for at least ${this.#majority} subscribers to finish`,\n          elapsed,\n        );\n      }\n    }\n  }\n\n  #logWithState(msg: string, elapsed: number) {\n    this.#lc\n      .withContext('watermark', this.#watermark)\n      .info?.(`${msg} (${elapsed.toFixed(3)} ms)`, {\n        completed: this.#completed.map(d => ({\n          id: d.sub.id,\n          processed: d.changes,\n          elapsed: d.elapsed,\n        })),\n        pending: Array.from(this.#pending, sub => ({\n          id: sub.id,\n          ...sub.getStats(),\n        })),\n      });\n  }\n}\n\n/** Tracks the completed result of a single subscriber. */\ntype Completed = {\n  sub: Subscriber;\n  /** The number of changes processed. */\n  changes: number;\n  /** The elapsed milliseconds. */\n  elapsed: number;\n};\n\nfunction toBatch(\n  change: WatermarkedChange | readonly WatermarkedChange[],\n): readonly WatermarkedChange[] {\n  if (change.length === 0) {\n    return [];\n  }\n  return Array.isArray(change[0])\n    ? (change as readonly WatermarkedChange[])\n    : [change as WatermarkedChange];\n}\n"],"mappings":";;;;AAiBA,SAAgB,kBACd,SACoB;CACpB,MAAM,WAAW,QAAQ,KAAI,MAAK,EAAE,EAAE;CACtC,MAAM,SAAS,SAAS,KAAK,GAAG;CAChC,MAAM,UAAU,OAAO,KACrB,QAAQ,WAAW,IAAI,UAAU,SAAS,GAAG,KAAK,aAAa,OAAO,KACtE,MACF;CACA,OAAO;EACL;EACA;EACA,YAAY,QAAQ;CACtB;AACF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAyEA,IAAa,YAAb,MAAuB;;;;;CAKrB,OAAO,gBACL,aACA,QACA;EACA,MAAM,UAAU,QAAQ,MAAM;EAC9B,MAAM,gBAAgB,kBAAkB,OAAO;EAC/C,KAAK,MAAM,OAAO,aAChB,IAAS,UAAU,SAAS,aAAa;CAE7C;CAEA;CACA;CACA;CACA,QAAiB,SAAS;CAC1B,UAAU;CACV;CAEA;CACA;CAEA,SAAkB,YAAY,IAAI;CAIlC;CACA;CACA;CACA;CAKA,uBAAuB;;;;;CAMvB,YACE,IACA,aACA,QACA,cACA;EACA,MAAM,UAAU,QAAQ,MAAM;EAC9B,MAAM,gBAAgB,kBAAkB,OAAO;EAC/C,KAAK,MAAM;EACX,KAAK,WAAW,IAAI,IAAI,WAAW;EACnC,KAAK,aAAa,CAAC;EACnB,KAAK,aAAa,KAAK,QAAQ,GAAG,EAAE,CAAC,CAAC,CAAC;EACvC,KAAK,YAAY,KAAK,MAAM,KAAK,SAAS,OAAO,CAAC,IAAI;EACtD,KAAK,iCACH,cAAc;EAChB,KAAK,cAAc,cAAc,gBAAgB;EACjD,KAAK,gBAAgB,cAAc,kBAAkB;EAErD,KAAK,MAAM,OAAO,KAAK,UAAU;GAC/B,MAAM,eAAe,IAAI,aAAa,QAAQ;GAC9C,IAAI,IAAI,SAAS,UAGf,KAAK,yBAAyB,CAAC,IAAI,aAAa;GAElD,IACG,UAAU,SAAS,aAAa,CAAC,CACjC,YAAY,CAAC,CAAC,CAAC,CACf,cAAc,KAAK,eAAe,KAAK,YAAY,CAAC;EACzD;EAGA,IAAI,KAAK,SAAS,SAAS,GACzB,KAAK,SAAS,iBAAiB;CAEnC;CAEA,eAAe,KAAiB,SAAiB;EAC/C,IAAI,KAAK,SACP;EAEF,MAAM,UAAU,YAAY,IAAI,IAAI,KAAK;EACzC,IAAI,oBAAoB,SAAS;EACjC,IAAI,IAAI,SAAS,UACf,KAAK,uBAAuB;EAE9B,KAAK,WAAW,KAAK;GAAC;GAAK;GAAS;EAAO,CAAC;EAC5C,KAAK,SAAS,OAAO,GAAG;EACxB,IAAI,KAAK,SAAS,SAAS,GAAG;GAC5B,KAAK,SAAS,iBAAiB;GAC/B;EACF;EAKA,IACE,KAAK,mCAAmC,KAAA,KACxC,KAAK,uBAAuB,KAAA,KAC5B,KAAK,WAAW,UAAU,KAAK,aAC/B,CAAC,KAAK,sBACN;GACA,MAAM,YAAY,KAAK,KACrB,UAAU,KAAK,8BACjB;GACA,KAAK,qBAAqB,KAAK,kBACvB,KAAK,SAAS,mBAAmB,GACvC,SACF;EACF;CACF;CAEA,SAAS,aAAmC;EAC1C,IAAI,KAAK,SACP;EAEF,IAAI,gBAAgB,qBAAqB;GACvC,MAAM,UAAU,YAAY,IAAI,IAAI,KAAK;GACzC,KAAK,cACH,mBAAmB,KAAK,SAAS,KAAK,+BACtC,OACF;EACF;EACA,KAAK,SAAS,SAAQ,QAAO,IAAI,oBAAoB,WAAW,CAAC;EACjE,KAAK,UAAU;EACf,KAAK,eAAe;EACpB,KAAK,cAAc,KAAK,kBAAkB;EAC1C,KAAK,qBAAqB,KAAA;EAC1B,KAAK,MAAM,QAAQ;CACrB;CAEA,IAAI,SAAkB;EACpB,OAAO,KAAK;CACd;CAEA,IAAI,OAAsB;EACxB,OAAO,KAAK,MAAM;CACpB;CAEA,IAAI,cAAoC;EACtC,OAAO,KAAK,gBAAgB;CAC9B;CAEA,gBAAgB,KAAa;EAC3B,IAAI,KAAK,WAAW,KAAK,SAAS,SAAS,GACzC;EAEF,MAAM,UAAU,MAAM,KAAK;EAC3B,IAAI,KAAK,WAAW,SAAS,KAAK,WAC5B;OAAA,WAAW,KACb,KAAK,cACH,wBAAwB,KAAK,UAAU,yBACvC,OACF;EAAA;CAGN;CAEA,cAAc,KAAa,SAAiB;EAC1C,KAAK,IACF,YAAY,aAAa,KAAK,UAAU,CAAC,CACzC,OAAO,GAAG,IAAI,IAAI,QAAQ,QAAQ,CAAC,EAAE,OAAO;GAC3C,WAAW,KAAK,WAAW,KAAI,OAAM;IACnC,IAAI,EAAE,IAAI;IACV,WAAW,EAAE;IACb,SAAS,EAAE;GACb,EAAE;GACF,SAAS,MAAM,KAAK,KAAK,WAAU,SAAQ;IACzC,IAAI,IAAI;IACR,GAAG,IAAI,SAAS;GAClB,EAAE;EACJ,CAAC;CACL;AACF;AAWA,SAAS,QACP,QAC8B;CAC9B,IAAI,OAAO,WAAW,GACpB,OAAO,CAAC;CAEV,OAAO,MAAM,QAAQ,OAAO,EAAE,IACzB,SACD,CAAC,MAA2B;AAClC"}

package/out/zero-cache/src/services/change-streamer/change-log-codec.d.ts

@@ -30,5 +30,5 @@ export declare function serializeChangeStreamData(data: ChangeStreamData): strin
  */
 export declare function extractChangeSubstring(streamMessageJSON: string, tag: ChangeTag | undefined): string;
 /** Reconstructs the canonical downstream JSON around a stored change. */
-export declare function reconstructWatermarkedChange(entry: ChangeLogEntry): WatermarkedChange;
+export declare function reconstructWatermarkedChange(watermark: string, entryTag: string, change: string): WatermarkedChange;
 //# sourceMappingURL=change-log-codec.d.ts.map

package/out/zero-cache/src/services/change-streamer/change-log-codec.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"change-log-codec.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/change-log-codec.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,iDAAiD,CAAC;AACtF,OAAO,KAAK,EAAC,SAAS,EAAE,iBAAiB,EAAC,MAAM,sBAAsB,CAAC;AAEvE,MAAM,MAAM,cAAc,GAAG;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,MAAM,MAAM,0BAA0B,GAAG;IACvC,wEAAwE;IACxE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,yEAAyE;IACzE,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB,CAAC;AAEF;;;;;;;GAOG;AACH,wBAAgB,mCAAmC,CACjD,IAAI,EAAE,gBAAgB,GACrB,0BAA0B,CAc5B;AAED,6EAA6E;AAC7E,wBAAgB,yBAAyB,CAAC,IAAI,EAAE,gBAAgB,GAAG,MAAM,CAExE;AAED;;;;;GAKG;AACH,wBAAgB,sBAAsB,CACpC,iBAAiB,EAAE,MAAM,EACzB,GAAG,EAAE,SAAS,GAAG,SAAS,GACzB,MAAM,CAkBR;AAED,yEAAyE;AACzE,wBAAgB,4BAA4B,CAC1C,KAAK,EAAE,cAAc,GACpB,iBAAiB,CAqBnB"}
+{"version":3,"file":"change-log-codec.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/change-log-codec.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,iDAAiD,CAAC;AACtF,OAAO,KAAK,EAAC,SAAS,EAAE,iBAAiB,EAAC,MAAM,sBAAsB,CAAC;AAEvE,MAAM,MAAM,cAAc,GAAG;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,MAAM,MAAM,0BAA0B,GAAG;IACvC,wEAAwE;IACxE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,yEAAyE;IACzE,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB,CAAC;AAEF;;;;;;;GAOG;AACH,wBAAgB,mCAAmC,CACjD,IAAI,EAAE,gBAAgB,GACrB,0BAA0B,CAc5B;AAED,6EAA6E;AAC7E,wBAAgB,yBAAyB,CAAC,IAAI,EAAE,gBAAgB,GAAG,MAAM,CAExE;AAED;;;;;GAKG;AACH,wBAAgB,sBAAsB,CACpC,iBAAiB,EAAE,MAAM,EACzB,GAAG,EAAE,SAAS,GAAG,SAAS,GACzB,MAAM,CAkBR;AAED,yEAAyE;AACzE,wBAAgB,4BAA4B,CAC1C,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,MAAM,GACb,iBAAiB,CAoBnB"}

package/out/zero-cache/src/services/change-streamer/change-log-codec.js

@@ -38,9 +38,8 @@ function extractChangeSubstring(streamMessageJSON, tag) {
 	}
 }
 /** Reconstructs the canonical downstream JSON around a stored change. */
-function reconstructWatermarkedChange(entry) {
-	const { watermark, change } = entry;
-	const tag = entry.tag;
+function reconstructWatermarkedChange(watermark, entryTag, change) {
+	const tag = entryTag;
 	switch (tag) {
 		case "begin": return [
 			watermark,

package/out/zero-cache/src/services/change-streamer/change-log-codec.js.map

@@ -1 +1 @@
-{"version":3,"file":"change-log-codec.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-log-codec.ts"],"sourcesContent":["import {BigIntJSON} from '../../../../shared/src/bigint-json.ts';\nimport type {ChangeStreamData} from '../change-source/protocol/current/downstream.ts';\nimport type {ChangeTag, WatermarkedChange} from './change-streamer.ts';\n\nexport type ChangeLogEntry = {\n  watermark: string;\n  tag: string;\n  change: string;\n};\n\nexport type SerializedChangeStreamData = {\n  /** The canonical downstream representation forwarded to subscribers. */\n  readonly json: string;\n  /** The already-serialized change tuple element stored in change logs. */\n  readonly change: string;\n};\n\n/**\n * Serializes a data-plane message and its stored change payload in one pass.\n *\n * Change logs retain only the second tuple element for backwards\n * compatibility. Serializing that element first avoids stringifying the full\n * message and then copying the payload back out of the resulting string on the\n * replication hot path.\n */\nexport function serializeChangeStreamDataWithChange(\n  data: ChangeStreamData,\n): SerializedChangeStreamData {\n  const change = BigIntJSON.stringify(data[1]);\n  const type = data[0];\n  switch (type) {\n    case 'begin':\n    case 'commit':\n      return {\n        json: `[${JSON.stringify(type)},${change},${BigIntJSON.stringify(data[2])}]`,\n        change,\n      };\n    case 'data':\n    case 'rollback':\n      return {json: `[${JSON.stringify(type)},${change}]`, change};\n  }\n}\n\n/** Serializes a data-plane message to the canonical downstream JSON form. */\nexport function serializeChangeStreamData(data: ChangeStreamData): string {\n  return serializeChangeStreamDataWithChange(data).json;\n}\n\n/**\n * Extracts the second tuple element for callers that only have the canonical\n * downstream JSON. The live write path uses\n * {@link serializeChangeStreamDataWithChange} and does not scan or copy the\n * completed message.\n */\nexport function extractChangeSubstring(\n  streamMessageJSON: string,\n  tag: ChangeTag | undefined,\n): string {\n  switch (tag) {\n    case 'begin':\n    case 'commit':\n      // e.g.\n      // [\"begin\",<message-json>,{\"commitWatermark\":\"92fj2d0s\"}]\n      // [\"commit\",<message-json>,{\"watermark\":\"92fj2d0s\"}]\n      return streamMessageJSON.substring(\n        streamMessageJSON.indexOf(',') + 1,\n        streamMessageJSON.lastIndexOf(','),\n      );\n    default:\n      // [\"data\",<message-json>]\n      return streamMessageJSON.substring(\n        streamMessageJSON.indexOf(',') + 1,\n        streamMessageJSON.lastIndexOf(']'),\n      );\n  }\n}\n\n/** Reconstructs the canonical downstream JSON around a stored change. */\nexport function reconstructWatermarkedChange(\n  entry: ChangeLogEntry,\n): WatermarkedChange {\n  const {watermark, change} = entry;\n  const tag = entry.tag as ChangeTag;\n  switch (tag) {\n    case 'begin':\n      return [\n        watermark,\n        tag,\n        `[\"begin\",${change},{\"commitWatermark\":\"${watermark}\"}]`,\n      ];\n    case 'commit':\n      return [\n        watermark,\n        tag,\n        `[\"commit\",${change},{\"watermark\":\"${watermark}\"}]`,\n      ];\n    case 'rollback':\n      return [watermark, tag, `[\"rollback\",${change}]`];\n    default:\n      return [watermark, tag, `[\"data\",${change}]`];\n  }\n}\n"],"mappings":";;;;;;;;;;AAyBA,SAAgB,oCACd,MAC4B;CAC5B,MAAM,SAAS,WAAW,UAAU,KAAK,EAAE;CAC3C,MAAM,OAAO,KAAK;CAClB,QAAQ,MAAR;EACE,KAAK;EACL,KAAK,UACH,OAAO;GACL,MAAM,IAAI,KAAK,UAAU,IAAI,EAAE,GAAG,OAAO,GAAG,WAAW,UAAU,KAAK,EAAE,EAAE;GAC1E;EACF;EACF,KAAK;EACL,KAAK,YACH,OAAO;GAAC,MAAM,IAAI,KAAK,UAAU,IAAI,EAAE,GAAG,OAAO;GAAI;EAAM;CAC/D;AACF;;;;;;;AAaA,SAAgB,uBACd,mBACA,KACQ;CACR,QAAQ,KAAR;EACE,KAAK;EACL,KAAK,UAIH,OAAO,kBAAkB,UACvB,kBAAkB,QAAQ,GAAG,IAAI,GACjC,kBAAkB,YAAY,GAAG,CACnC;EACF,SAEE,OAAO,kBAAkB,UACvB,kBAAkB,QAAQ,GAAG,IAAI,GACjC,kBAAkB,YAAY,GAAG,CACnC;CACJ;AACF;;AAGA,SAAgB,6BACd,OACmB;CACnB,MAAM,EAAC,WAAW,WAAU;CAC5B,MAAM,MAAM,MAAM;CAClB,QAAQ,KAAR;EACE,KAAK,SACH,OAAO;GACL;GACA;GACA,YAAY,OAAO,uBAAuB,UAAU;EACtD;EACF,KAAK,UACH,OAAO;GACL;GACA;GACA,aAAa,OAAO,iBAAiB,UAAU;EACjD;EACF,KAAK,YACH,OAAO;GAAC;GAAW;GAAK,eAAe,OAAO;EAAE;EAClD,SACE,OAAO;GAAC;GAAW;GAAK,WAAW,OAAO;EAAE;CAChD;AACF"}
+{"version":3,"file":"change-log-codec.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-log-codec.ts"],"sourcesContent":["import {BigIntJSON} from '../../../../shared/src/bigint-json.ts';\nimport type {ChangeStreamData} from '../change-source/protocol/current/downstream.ts';\nimport type {ChangeTag, WatermarkedChange} from './change-streamer.ts';\n\nexport type ChangeLogEntry = {\n  watermark: string;\n  tag: string;\n  change: string;\n};\n\nexport type SerializedChangeStreamData = {\n  /** The canonical downstream representation forwarded to subscribers. */\n  readonly json: string;\n  /** The already-serialized change tuple element stored in change logs. */\n  readonly change: string;\n};\n\n/**\n * Serializes a data-plane message and its stored change payload in one pass.\n *\n * Change logs retain only the second tuple element for backwards\n * compatibility. Serializing that element first avoids stringifying the full\n * message and then copying the payload back out of the resulting string on the\n * replication hot path.\n */\nexport function serializeChangeStreamDataWithChange(\n  data: ChangeStreamData,\n): SerializedChangeStreamData {\n  const change = BigIntJSON.stringify(data[1]);\n  const type = data[0];\n  switch (type) {\n    case 'begin':\n    case 'commit':\n      return {\n        json: `[${JSON.stringify(type)},${change},${BigIntJSON.stringify(data[2])}]`,\n        change,\n      };\n    case 'data':\n    case 'rollback':\n      return {json: `[${JSON.stringify(type)},${change}]`, change};\n  }\n}\n\n/** Serializes a data-plane message to the canonical downstream JSON form. */\nexport function serializeChangeStreamData(data: ChangeStreamData): string {\n  return serializeChangeStreamDataWithChange(data).json;\n}\n\n/**\n * Extracts the second tuple element for callers that only have the canonical\n * downstream JSON. The live write path uses\n * {@link serializeChangeStreamDataWithChange} and does not scan or copy the\n * completed message.\n */\nexport function extractChangeSubstring(\n  streamMessageJSON: string,\n  tag: ChangeTag | undefined,\n): string {\n  switch (tag) {\n    case 'begin':\n    case 'commit':\n      // e.g.\n      // [\"begin\",<message-json>,{\"commitWatermark\":\"92fj2d0s\"}]\n      // [\"commit\",<message-json>,{\"watermark\":\"92fj2d0s\"}]\n      return streamMessageJSON.substring(\n        streamMessageJSON.indexOf(',') + 1,\n        streamMessageJSON.lastIndexOf(','),\n      );\n    default:\n      // [\"data\",<message-json>]\n      return streamMessageJSON.substring(\n        streamMessageJSON.indexOf(',') + 1,\n        streamMessageJSON.lastIndexOf(']'),\n      );\n  }\n}\n\n/** Reconstructs the canonical downstream JSON around a stored change. */\nexport function reconstructWatermarkedChange(\n  watermark: string,\n  entryTag: string,\n  change: string,\n): WatermarkedChange {\n  const tag = entryTag as ChangeTag;\n  switch (tag) {\n    case 'begin':\n      return [\n        watermark,\n        tag,\n        `[\"begin\",${change},{\"commitWatermark\":\"${watermark}\"}]`,\n      ];\n    case 'commit':\n      return [\n        watermark,\n        tag,\n        `[\"commit\",${change},{\"watermark\":\"${watermark}\"}]`,\n      ];\n    case 'rollback':\n      return [watermark, tag, `[\"rollback\",${change}]`];\n    default:\n      return [watermark, tag, `[\"data\",${change}]`];\n  }\n}\n"],"mappings":";;;;;;;;;;AAyBA,SAAgB,oCACd,MAC4B;CAC5B,MAAM,SAAS,WAAW,UAAU,KAAK,EAAE;CAC3C,MAAM,OAAO,KAAK;CAClB,QAAQ,MAAR;EACE,KAAK;EACL,KAAK,UACH,OAAO;GACL,MAAM,IAAI,KAAK,UAAU,IAAI,EAAE,GAAG,OAAO,GAAG,WAAW,UAAU,KAAK,EAAE,EAAE;GAC1E;EACF;EACF,KAAK;EACL,KAAK,YACH,OAAO;GAAC,MAAM,IAAI,KAAK,UAAU,IAAI,EAAE,GAAG,OAAO;GAAI;EAAM;CAC/D;AACF;;;;;;;AAaA,SAAgB,uBACd,mBACA,KACQ;CACR,QAAQ,KAAR;EACE,KAAK;EACL,KAAK,UAIH,OAAO,kBAAkB,UACvB,kBAAkB,QAAQ,GAAG,IAAI,GACjC,kBAAkB,YAAY,GAAG,CACnC;EACF,SAEE,OAAO,kBAAkB,UACvB,kBAAkB,QAAQ,GAAG,IAAI,GACjC,kBAAkB,YAAY,GAAG,CACnC;CACJ;AACF;;AAGA,SAAgB,6BACd,WACA,UACA,QACmB;CACnB,MAAM,MAAM;CACZ,QAAQ,KAAR;EACE,KAAK,SACH,OAAO;GACL;GACA;GACA,YAAY,OAAO,uBAAuB,UAAU;EACtD;EACF,KAAK,UACH,OAAO;GACL;GACA;GACA,aAAa,OAAO,iBAAiB,UAAU;EACjD;EACF,KAAK,YACH,OAAO;GAAC;GAAW;GAAK,eAAe,OAAO;EAAE;EAClD,SACE,OAAO;GAAC;GAAW;GAAK,WAAW,OAAO;EAAE;CAChD;AACF"}

package/out/zero-cache/src/services/change-streamer/change-log-initializer.js

@@ -251,7 +251,7 @@ function readSchemaChanges(db, after, through) {
       SELECT count(*) AS "rows" FROM (
         SELECT 1 FROM "${CHANGE_LOG_STREAM_TABLE}"
           WHERE "watermark" > ? AND "watermark" <= ?
-          LIMIT ${MAX_FOLD_SCAN_ROWS + 1}
+          LIMIT 10001
       )
     `).get(after, through);
 	if (rows > MAX_FOLD_SCAN_ROWS) return;
@@ -308,6 +308,6 @@ function sortKeys(value) {
 	return Object.fromEntries(Object.entries(value).sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0).map(([k, v]) => [k, sortKeys(v)]));
 }
 //#endregion
-export { ChangeLogInitializer, replicaInitializationSource };
+export { ChangeLogInitializer, readReplicaInitializationParameters, replicaInitializationSource };
 
 //# sourceMappingURL=change-log-initializer.js.map