npm - @rocicorp/zero - Versions diffs - 1.10.0-canary.0 → 1.10.0-canary.10 - Mend

@rocicorp/zero 1.10.0-canary.0 → 1.10.0-canary.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (484) hide show

package/out/zero-cache/src/services/change-source/protocol/current/data.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"data.js","names":[],"sources":["../../../../../../../../zero-cache/src/services/change-source/protocol/current/data.ts"],"sourcesContent":["/*\n Data plane messages encapsulate changes that are sent by ChangeSources,\n * forwarded / fanned out to subscribers by the ChangeStreamerService, and\n * stored in the Change DB for catchup of old subscribers.\n /\n\nimport {\n jsonValueSchema,\n type JSONObject,\n} from '../../../../../../shared/src/bigint-json.ts';\nimport {must} from '../../../../../../shared/src/must.ts';\nimport as v from '../../../../../../shared/src/valita.ts';\nimport {columnSpec, indexSpec, tableSpec} from '../../../../db/specs.ts';\nimport type {Satisfies} from '../../../../types/satisfies.ts';\nimport {jsonObjectSchema} from './json.ts';\n\nexport const beginSchema = v.object({\n tag: v.literal('begin'),\n // The format of values of \"json\"-typed columns (e.g. \"JSON\" and \"JSONB\").\n // - 'p' is for parsed JSON, which may include JSON values or JSON objects.\n // These values are parsed and stringified at every process boundary\n // between the change-source and the replica.\n // - 's' is for stringified JSON. These values skip the parsing and\n // stringification, and are directly ferried to the replica as a JSON\n // string. For JSON values this improves performance by 20~25% in the\n // change-streamer and 25~30% in the replicator.\n //\n // If absent, the format is assumed to be 'p' (parsed JSON objects/values).\n json: v.literalUnion('p', 's').optional(),\n\n // Directs the change-streamer to skip the ACK for the corresponding commit.\n skipAck: v.boolean().optional(),\n});\n\nexport const commitSchema = v.object({\n tag: v.literal('commit'),\n});\n\nexport const rollbackSchema = v.object({\n tag: v.literal('rollback'),\n});\n\nconst rowKeySchema = v.object({\n // The columns used to identify a row in insert, update, and delete changes.\n columns: v.array(v.string()),\n\n // An optional qualifier identifying how the key is chosen. Currently this\n // is postgres-specific, describing the REPLICA IDENTITY, for which replica\n // identity 'full' (FULL) is handled differently; the replicator handles\n // these tables by extracting a row key from the full row based on the\n // table's PRIMARY KEY or UNIQUE INDEX.\n type: v.literalUnion('default', 'nothing', 'full', 'index').optional(),\n});\n\nexport const relationSchema = v\n .object({\n schema: v.string(),\n name: v.string(),\n\n // This will become required.\n rowKey: rowKeySchema.optional(),\n\n /** Deprecated: set the rowKey.columns instead. /\n keyColumns: v.array(v.string()).optional(),\n /* Deprecated: set the rowKey.columns instead. /\n replicaIdentity: v\n .literalUnion('default', 'nothing', 'full', 'index')\n .optional(),\n })\n .map(rel => {\n const {rowKey, ...rest} = rel;\n if (rowKey) {\n return {...rest, rowKey};\n }\n return {\n ...rest,\n rowKey: {\n columns: must(rel.keyColumns),\n type: rel.replicaIdentity,\n },\n };\n });\n\n// The eventual fate of relationSchema\nexport const newRelationSchema = v.object({\n schema: v.string(),\n name: v.string(),\n\n rowKey: rowKeySchema,\n});\n\n// TableMetadata contains table-related configuration that does not affect the\n// actual data in the table, but rather how the table's change messages are\n// handled. The is an opaque object that clients must track (and update) based\n// on `create-table`, `add-column`, and `table-update-metadata` messages, and\n// pass in BackfillRequests when there are columns to be backfilled.\n//\n// Note that the backfill-related change-source implementation does, however,\n// rely on the rowKey (columns) being specified in the message.\nexport const tableMetadataSchema = v\n .object({rowKey: v.record(jsonValueSchema)})\n .rest(jsonValueSchema);\n\nexport type TableMetadata = v.Infer<typeof tableMetadataSchema>;\n\nexport const rowSchema = v.record(jsonValueSchema);\n\nexport const insertSchema = v.object({\n tag: v.literal('insert'),\n relation: relationSchema,\n new: rowSchema,\n});\n\nexport const updateSchema = v.object({\n tag: v.literal('update'),\n relation: relationSchema,\n // `key` is present if the update changed the key of the row, or if the\n // table's replicaIdentity === 'full'\n key: rowSchema.nullable(),\n // `new` is the full row (and not just the updated columns). This is\n // necessary for \"catchup\" replication scenarios such as adding tables\n // to a publication, or resharding.\n new: rowSchema,\n});\n\nexport const deleteSchema = v.object({\n tag: v.literal('delete'),\n relation: relationSchema,\n // key is the full row if replicaIdentity === 'full'\n key: rowSchema,\n});\n\nexport const truncateSchema = v.object({\n tag: v.literal('truncate'),\n relations: v.array(relationSchema),\n});\n\nexport const identifierSchema = v.object({\n schema: v.string(),\n name: v.string(),\n});\n\nexport type Identifier = v.Infer<typeof identifierSchema>;\n\n// A BackfillID is an upstream specific stable identifier for a column\n// that needs backfilling. This id is used to ensure that the schema, table,\n// and column names of a requested backfill still match the original\n// underlying upstream ID.\n//\n// The change-streamer stores these IDs as opaque values while a column is\n// being backfilled, and initiates new change-source streams with the IDs\n// in order to restart backfills that did not complete in previous sessions.\nexport const backfillIDSchema = jsonObjectSchema;\n\nexport type BackfillID = v.Infer<typeof backfillIDSchema>;\n\nexport const createTableSchema = v.object({\n tag: v.literal('create-table'),\n spec: tableSpec,\n\n // This must be set by change source implementations that support\n // table/column backfill.\n //\n // TODO: to simplify the protocol, see if we can make this required\n metadata: tableMetadataSchema.optional(),\n\n // Indicate that columns of the table require backfilling. These columns\n // should be created on the replica but not yet synced the clients.\n //\n // ## State Persistence\n //\n // To obviate the need for change-source implementations to persist state\n // related to backfill progress, the change-source only tracks backfills\n // for the current session. In the event that the session is interrupted\n // before columns have been fully backfilled, it is the responsibility of the\n // change-streamer to send {@link BackfillRequest}s when it reconnects.\n //\n // This means that the change-streamer must track and persist:\n // the backfill IDs of the columns requiring backfilling\n // * the most current table metadata of the associated table(s)\n //\n // The change-streamer then uses this information to send backfill requests\n // when it reconnects.\n backfill: v.record(backfillIDSchema).optional(),\n});\n\nexport const renameTableSchema = v.object({\n tag: v.literal('rename-table'),\n old: identifierSchema,\n new: identifierSchema,\n});\n\nexport const updateTableMetadataSchema = v.object({\n tag: v.literal('update-table-metadata'),\n table: identifierSchema,\n old: tableMetadataSchema,\n new: tableMetadataSchema,\n});\n\nconst columnSchema = v.object({\n name: v.string(),\n spec: columnSpec,\n});\n\nexport const addColumnSchema = v.object({\n tag: v.literal('add-column'),\n table: identifierSchema,\n column: columnSchema,\n\n // This must be set by change source implementations that support\n // table/column backfill.\n //\n // TODO: to simplify the protocol, see if we can make this required\n tableMetadata: tableMetadataSchema.optional(),\n\n // See documentation for the `backfill` field of the `create-table` change.\n backfill: backfillIDSchema.optional(),\n});\n\nexport const updateColumnSchema = v.object({\n tag: v.literal('update-column'),\n table: identifierSchema,\n old: columnSchema,\n new: columnSchema,\n});\n\nexport const dropColumnSchema = v.object({\n tag: v.literal('drop-column'),\n table: identifierSchema,\n column: v.string(),\n});\n\nexport const dropTableSchema = v.object({\n tag: v.literal('drop-table'),\n id: identifierSchema,\n});\n\nexport const createIndexSchema = v.object({\n tag: v.literal('create-index'),\n spec: indexSpec,\n});\n\nexport const dropIndexSchema = v.object({\n tag: v.literal('drop-index'),\n id: identifierSchema,\n});\n\nexport const downloadStatusSchema = v.object({\n rows: v.number(),\n totalRows: v.number(),\n totalBytes: v.number().optional(),\n});\n\nexport type DownloadStatus = v.Infer<typeof downloadStatusSchema>;\n\n// A batch of rows from a single table containing column values\n// to be backfilled.\nexport const backfillSchema = v.object({\n tag: v.literal('backfill'),\n\n relation: newRelationSchema,\n\n // The columns to be backfilled. `rowKey` columns are automatically excluded,\n // which means that this field may be empty.\n columns: v.array(v.string()),\n\n // The watermark at which the backfill data was queried. Note that this\n // generally will be different from the commit watermarks of the main change\n // stream, and in particular, the commit watermark of the backfill change's\n // enclosing transaction.\n watermark: v.string(),\n\n // A batch of row values, each row consisting of the `rowKey`\n // values, followed by the `column` values, in the same order in which\n // the column names appear in their respective fields, e.g.\n //\n // ```\n // [\n // [...rowKeyValues, ...columnValues], // row 1\n // [...rowKeyValues, ...columnValues], // row 2\n // ]\n // ```\n rowValues: v.array(v.array(jsonValueSchema)),\n\n // Optionally includes the progress of the backfill operation,\n // for display purposes.\n status: downloadStatusSchema.optional(),\n});\n\n// Indicates that the backfill for the specified columns have\n// been successfully backfilled and can be published to clients.\nexport const backfillCompletedSchema = v.object({\n tag: v.literal('backfill-completed'),\n\n relation: newRelationSchema,\n\n // The columns to be backfilled. `rowKey` columns are automatically excluded,\n // which means that this field may be empty.\n columns: v.array(v.string()),\n\n // The watermark at which the backfill data was queried. Note that this\n // generally will be different from the commit watermarks of the main change\n // stream, and in particular, the commit watermark of the backfill change's\n // enclosing transaction.\n watermark: v.string(),\n\n // Optionally includes the final status of the backfill operation,\n // for display purposes.\n status: downloadStatusSchema.optional(),\n});\n\nexport type MessageBegin = v.Infer<typeof beginSchema>;\nexport type MessageCommit = v.Infer<typeof commitSchema>;\nexport type MessageRollback = v.Infer<typeof rollbackSchema>;\n\nexport type MessageRelation = v.Infer<typeof relationSchema>;\nexport type MessageInsert = v.Infer<typeof insertSchema>;\nexport type MessageUpdate = v.Infer<typeof updateSchema>;\nexport type MessageDelete = v.Infer<typeof deleteSchema>;\nexport type MessageTruncate = v.Infer<typeof truncateSchema>;\n\nexport type MessageBackfill = v.Infer<typeof backfillSchema>;\n\nexport type TableCreate = v.Infer<typeof createTableSchema>;\nexport type TableRename = v.Infer<typeof renameTableSchema>;\nexport type TableUpdateMetadata = v.Infer<typeof updateTableMetadataSchema>;\nexport type ColumnAdd = v.Infer<typeof addColumnSchema>;\nexport type ColumnUpdate = v.Infer<typeof updateColumnSchema>;\nexport type ColumnDrop = v.Infer<typeof dropColumnSchema>;\nexport type TableDrop = v.Infer<typeof dropTableSchema>;\nexport type IndexCreate = v.Infer<typeof createIndexSchema>;\nexport type IndexDrop = v.Infer<typeof dropIndexSchema>;\nexport type BackfillCompleted = v.Infer<typeof backfillCompletedSchema>;\n\nexport const dataChangeSchema = v.union(\n insertSchema,\n updateSchema,\n deleteSchema,\n truncateSchema,\n backfillSchema,\n);\n\n// Note: keep in sync or the tag tests will fail\nconst dataChangeTags = [\n 'insert',\n 'update',\n 'delete',\n 'truncate',\n 'backfill',\n] as const;\n\nconst dataChangeTagsSchema = v.literalUnion(...dataChangeTags);\n\nexport type DataChange = Satisfies<\n JSONObject, // guarantees serialization over IPC or network\n v.Infer<typeof dataChangeSchema>\n>;\n\nexport type DataChangeTag = v.Infer<typeof dataChangeTagsSchema>;\n\nconst schemaChanges = [\n createTableSchema,\n renameTableSchema,\n updateTableMetadataSchema,\n addColumnSchema,\n updateColumnSchema,\n dropColumnSchema,\n dropTableSchema,\n createIndexSchema,\n dropIndexSchema,\n backfillCompletedSchema,\n] as const;\n\n// Note: keep in sync or the tag tests will fail\nconst schemaChangeTags = [\n 'create-table',\n 'rename-table',\n 'update-table-metadata',\n 'add-column',\n 'update-column',\n 'drop-column',\n 'drop-table',\n 'create-index',\n 'drop-index',\n 'backfill-completed',\n] as const;\n\nexport const schemaChangeSchema = v.union(...schemaChanges);\n\nconst schemaChangeTagsSchema = v.literalUnion(...schemaChangeTags);\n\nexport type SchemaChange = Satisfies<\n JSONObject,\n v.Infer<typeof schemaChangeSchema>\n>;\n\nexport type SchemaChangeTag = v.Infer<typeof schemaChangeTagsSchema>;\n\nexport type DataOrSchemaChange = DataChange \| SchemaChange;\n\nexport type Change =\n \| MessageBegin\n \| DataOrSchemaChange\n \| MessageCommit\n \| MessageRollback;\n\nexport type ChangeTag = Change['tag'];\n\nconst schemaChangeTagSet = new Set<string>(schemaChangeTags);\n\nexport function isSchemaChange(change: Change): change is SchemaChange {\n return schemaChangeTagSet.has(change.tag);\n}\n\nconst dataChangeTagSet = new Set<string>(dataChangeTags);\n\nexport function isDataChange(change: Change): change is DataChange {\n return dataChangeTagSet.has(change.tag);\n}\n"],"mappings":";;;;;;;;;;;AAgBA,IAAa,cAAc,eAAE,OAAO;CAClC,KAAK,eAAE,QAAQ,OAAO;CAWtB,MAAM,aAAe,KAAK,GAAG,EAAE,SAAS;CAGxC,SAAS,eAAE,QAAQ,EAAE,SAAS;AAChC,CAAC;AAED,IAAa,eAAe,eAAE,OAAO,EACnC,KAAK,eAAE,QAAQ,QAAQ,EACzB,CAAC;AAED,IAAa,iBAAiB,eAAE,OAAO,EACrC,KAAK,eAAE,QAAQ,UAAU,EAC3B,CAAC;AAED,IAAM,eAAe,eAAE,OAAO;CAE5B,SAAS,eAAE,MAAM,eAAE,OAAO,CAAC;CAO3B,MAAM,aAAe,WAAW,WAAW,QAAQ,OAAO,EAAE,SAAS;AACvE,CAAC;AAED,IAAa,iBAAiB,eAC3B,OAAO;CACN,QAAQ,eAAE,OAAO;CACjB,MAAM,eAAE,OAAO;CAGf,QAAQ,aAAa,SAAS;;CAG9B,YAAY,eAAE,MAAM,eAAE,OAAO,CAAC,EAAE,SAAS;;CAEzC,iBAAiB,aACD,WAAW,WAAW,QAAQ,OAAO,EAClD,SAAS;AACd,CAAC,EACA,KAAI,QAAO;CACV,MAAM,EAAC,QAAQ,GAAG,SAAQ;CAC1B,IAAI,QACF,OAAO;EAAC,GAAG;EAAM;CAAM;CAEzB,OAAO;EACL,GAAG;EACH,QAAQ;GACN,SAAS,KAAK,IAAI,UAAU;GAC5B,MAAM,IAAI;EACZ;CACF;AACF,CAAC;AAGH,IAAa,oBAAoB,eAAE,OAAO;CACxC,QAAQ,eAAE,OAAO;CACjB,MAAM,eAAE,OAAO;CAEf,QAAQ;AACV,CAAC;AAUD,IAAa,sBAAsB,eAChC,OAAO,EAAC,QAAQ,eAAE,OAAO,eAAe,EAAC,CAAC,EAC1C,KAAK,eAAe;AAIvB,IAAa,YAAY,eAAE,OAAO,eAAe;AAEjD,IAAa,eAAe,eAAE,OAAO;CACnC,KAAK,eAAE,QAAQ,QAAQ;CACvB,UAAU;CACV,KAAK;AACP,CAAC;AAED,IAAa,eAAe,eAAE,OAAO;CACnC,KAAK,eAAE,QAAQ,QAAQ;CACvB,UAAU;CAGV,KAAK,UAAU,SAAS;CAIxB,KAAK;AACP,CAAC;AAED,IAAa,eAAe,eAAE,OAAO;CACnC,KAAK,eAAE,QAAQ,QAAQ;CACvB,UAAU;CAEV,KAAK;AACP,CAAC;AAED,IAAa,iBAAiB,eAAE,OAAO;CACrC,KAAK,eAAE,QAAQ,UAAU;CACzB,WAAW,eAAE,MAAM,cAAc;AACnC,CAAC;AAED,IAAa,mBAAmB,eAAE,OAAO;CACvC,QAAQ,eAAE,OAAO;CACjB,MAAM,eAAE,OAAO;AACjB,CAAC;AAYD,IAAa,mBAAmB;AAIhC,IAAa,oBAAoB,eAAE,OAAO;CACxC,KAAK,eAAE,QAAQ,cAAc;CAC7B,MAAM;CAMN,UAAU,oBAAoB,SAAS;CAmBvC,UAAU,eAAE,OAAO,gBAAgB,EAAE,SAAS;AAChD,CAAC;AAED,IAAa,oBAAoB,eAAE,OAAO;CACxC,KAAK,eAAE,QAAQ,cAAc;CAC7B,KAAK;CACL,KAAK;AACP,CAAC;AAED,IAAa,4BAA4B,eAAE,OAAO;CAChD,KAAK,eAAE,QAAQ,uBAAuB;CACtC,OAAO;CACP,KAAK;CACL,KAAK;AACP,CAAC;AAED,IAAM,eAAe,eAAE,OAAO;CAC5B,MAAM,eAAE,OAAO;CACf,MAAM;AACR,CAAC;AAED,IAAa,kBAAkB,eAAE,OAAO;CACtC,KAAK,eAAE,QAAQ,YAAY;CAC3B,OAAO;CACP,QAAQ;CAMR,eAAe,oBAAoB,SAAS;CAG5C,UAAU,iBAAiB,SAAS;AACtC,CAAC;AAED,IAAa,qBAAqB,eAAE,OAAO;CACzC,KAAK,eAAE,QAAQ,eAAe;CAC9B,OAAO;CACP,KAAK;CACL,KAAK;AACP,CAAC;AAED,IAAa,mBAAmB,eAAE,OAAO;CACvC,KAAK,eAAE,QAAQ,aAAa;CAC5B,OAAO;CACP,QAAQ,eAAE,OAAO;AACnB,CAAC;AAED,IAAa,kBAAkB,eAAE,OAAO;CACtC,KAAK,eAAE,QAAQ,YAAY;CAC3B,IAAI;AACN,CAAC;AAED,IAAa,oBAAoB,eAAE,OAAO;CACxC,KAAK,eAAE,QAAQ,cAAc;CAC7B,MAAM;AACR,CAAC;AAED,IAAa,kBAAkB,eAAE,OAAO;CACtC,KAAK,eAAE,QAAQ,YAAY;CAC3B,IAAI;AACN,CAAC;AAED,IAAa,uBAAuB,eAAE,OAAO;CAC3C,MAAM,eAAE,OAAO;CACf,WAAW,eAAE,OAAO;CACpB,YAAY,eAAE,OAAO,EAAE,SAAS;AAClC,CAAC;AAMD,IAAa,iBAAiB,eAAE,OAAO;CACrC,KAAK,eAAE,QAAQ,UAAU;CAEzB,UAAU;CAIV,SAAS,eAAE,MAAM,eAAE,OAAO,CAAC;CAM3B,WAAW,eAAE,OAAO;CAYpB,WAAW,eAAE,MAAM,eAAE,MAAM,eAAe,CAAC;CAI3C,QAAQ,qBAAqB,SAAS;AACxC,CAAC;AAID,IAAa,0BAA0B,eAAE,OAAO;CAC9C,KAAK,eAAE,QAAQ,oBAAoB;CAEnC,UAAU;CAIV,SAAS,eAAE,MAAM,eAAE,OAAO,CAAC;CAM3B,WAAW,eAAE,OAAO;CAIpB,QAAQ,qBAAqB,SAAS;AACxC,CAAC;AAyBD,IAAa,mBAAmB,eAAE,MAChC,cACA,cACA,cACA,gBACA,cACF;AAGA,IAAM,iBAAiB;CACrB;CACA;CACA;CACA;CACA;AACF;AAE6B,aAAe,GAAG,cAAc;AAS7D,IAAM,gBAAgB;CACpB;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;AACF;AAGA,IAAM,mBAAmB;CACvB;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;AACF;AAEA,IAAa,qBAAqB,eAAE,MAAM,GAAG,aAAa;AAE3B,aAAe,GAAG,gBAAgB;AAmBjE,IAAM,qBAAqB,IAAI,IAAY,gBAAgB;AAE3D,SAAgB,eAAe,QAAwC;CACrE,OAAO,mBAAmB,IAAI,OAAO,GAAG;AAC1C;AAEA,IAAM,mBAAmB,IAAI,IAAY,cAAc;AAEvD,SAAgB,aAAa,QAAsC;CACjE,OAAO,iBAAiB,IAAI,OAAO,GAAG;AACxC"}
1	+ {"version":3,"file":"data.js","names":[],"sources":["../../../../../../../../zero-cache/src/services/change-source/protocol/current/data.ts"],"sourcesContent":["/*\n Data plane messages encapsulate changes that are sent by ChangeSources,\n * forwarded / fanned out to subscribers by the ChangeStreamerService, and\n * stored in the Change DB for catchup of old subscribers.\n /\n\nimport {\n jsonValueSchema,\n type JSONObject,\n} from '../../../../../../shared/src/bigint-json.ts';\nimport {must} from '../../../../../../shared/src/must.ts';\nimport as v from '../../../../../../shared/src/valita.ts';\nimport {columnSpec, indexSpec, tableSpec} from '../../../../db/specs.ts';\nimport type {Satisfies} from '../../../../types/satisfies.ts';\nimport {jsonObjectSchema} from './json.ts';\nimport {schemaChangeTags} from './schema-change-tags.ts';\n\nexport const beginSchema = v.object({\n tag: v.literal('begin'),\n // The format of values of \"json\"-typed columns (e.g. \"JSON\" and \"JSONB\").\n // - 'p' is for parsed JSON, which may include JSON values or JSON objects.\n // These values are parsed and stringified at every process boundary\n // between the change-source and the replica.\n // - 's' is for stringified JSON. These values skip the parsing and\n // stringification, and are directly ferried to the replica as a JSON\n // string. For JSON values this improves performance by 20~25% in the\n // change-streamer and 25~30% in the replicator.\n //\n // If absent, the format is assumed to be 'p' (parsed JSON objects/values).\n json: v.literalUnion('p', 's').optional(),\n\n // Directs the change-streamer to skip the ACK for the corresponding commit.\n skipAck: v.boolean().optional(),\n});\n\nexport const commitSchema = v.object({\n tag: v.literal('commit'),\n\n // Millisecond epoch at which the transaction committed upstream, as reported\n // by the upstream database (e.g. the `commitTime` of the Postgres logical\n // replication Commit message).\n //\n // This is the origin timestamp of the end-to-end serving lag measurement:\n // it is carried through the replication stream to the ViewSyncer, which\n // records `now - commitTimeMs` once the change has been poked to clients.\n //\n // Optional because it is absent from changes written to the Change DB by\n // older versions, which are replayed verbatim during catchup, and from\n // ChangeSources that do not report a commit time.\n commitTimeMs: v.number().optional(),\n});\n\nexport const rollbackSchema = v.object({\n tag: v.literal('rollback'),\n});\n\nconst rowKeySchema = v.object({\n // The columns used to identify a row in insert, update, and delete changes.\n columns: v.array(v.string()),\n\n // An optional qualifier identifying how the key is chosen. Currently this\n // is postgres-specific, describing the REPLICA IDENTITY, for which replica\n // identity 'full' (FULL) is handled differently; the replicator handles\n // these tables by extracting a row key from the full row based on the\n // table's PRIMARY KEY or UNIQUE INDEX.\n type: v.literalUnion('default', 'nothing', 'full', 'index').optional(),\n});\n\nexport const relationSchema = v\n .object({\n schema: v.string(),\n name: v.string(),\n\n // This will become required.\n rowKey: rowKeySchema.optional(),\n\n /** Deprecated: set the rowKey.columns instead. /\n keyColumns: v.array(v.string()).optional(),\n /* Deprecated: set the rowKey.columns instead. /\n replicaIdentity: v\n .literalUnion('default', 'nothing', 'full', 'index')\n .optional(),\n })\n .map(rel => {\n const {rowKey, ...rest} = rel;\n if (rowKey) {\n return {...rest, rowKey};\n }\n return {\n ...rest,\n rowKey: {\n columns: must(rel.keyColumns),\n type: rel.replicaIdentity,\n },\n };\n });\n\n// The eventual fate of relationSchema\nexport const newRelationSchema = v.object({\n schema: v.string(),\n name: v.string(),\n\n rowKey: rowKeySchema,\n});\n\n// TableMetadata contains table-related configuration that does not affect the\n// actual data in the table, but rather how the table's change messages are\n// handled. The is an opaque object that clients must track (and update) based\n// on `create-table`, `add-column`, and `table-update-metadata` messages, and\n// pass in BackfillRequests when there are columns to be backfilled.\n//\n// Note that the backfill-related change-source implementation does, however,\n// rely on the rowKey (columns) being specified in the message.\nexport const tableMetadataSchema = v\n .object({rowKey: v.record(jsonValueSchema)})\n .rest(jsonValueSchema);\n\nexport type TableMetadata = v.Infer<typeof tableMetadataSchema>;\n\nexport const rowSchema = v.record(jsonValueSchema);\n\nexport const insertSchema = v.object({\n tag: v.literal('insert'),\n relation: relationSchema,\n new: rowSchema,\n});\n\nexport const updateSchema = v.object({\n tag: v.literal('update'),\n relation: relationSchema,\n // `key` is present if the update changed the key of the row, or if the\n // table's replicaIdentity === 'full'\n key: rowSchema.nullable(),\n // `new` is the full row (and not just the updated columns). This is\n // necessary for \"catchup\" replication scenarios such as adding tables\n // to a publication, or resharding.\n new: rowSchema,\n});\n\nexport const deleteSchema = v.object({\n tag: v.literal('delete'),\n relation: relationSchema,\n // key is the full row if replicaIdentity === 'full'\n key: rowSchema,\n});\n\nexport const truncateSchema = v.object({\n tag: v.literal('truncate'),\n relations: v.array(relationSchema),\n});\n\nexport const identifierSchema = v.object({\n schema: v.string(),\n name: v.string(),\n});\n\nexport type Identifier = v.Infer<typeof identifierSchema>;\n\n// A BackfillID is an upstream specific stable identifier for a column\n// that needs backfilling. This id is used to ensure that the schema, table,\n// and column names of a requested backfill still match the original\n// underlying upstream ID.\n//\n// The change-streamer stores these IDs as opaque values while a column is\n// being backfilled, and initiates new change-source streams with the IDs\n// in order to restart backfills that did not complete in previous sessions.\nexport const backfillIDSchema = jsonObjectSchema;\n\nexport type BackfillID = v.Infer<typeof backfillIDSchema>;\n\nexport const createTableSchema = v.object({\n tag: v.literal('create-table'),\n spec: tableSpec,\n\n // This must be set by change source implementations that support\n // table/column backfill.\n //\n // TODO: to simplify the protocol, see if we can make this required\n metadata: tableMetadataSchema.optional(),\n\n // Indicate that columns of the table require backfilling. These columns\n // should be created on the replica but not yet synced the clients.\n //\n // ## State Persistence\n //\n // To obviate the need for change-source implementations to persist state\n // related to backfill progress, the change-source only tracks backfills\n // for the current session. In the event that the session is interrupted\n // before columns have been fully backfilled, it is the responsibility of the\n // change-streamer to send {@link BackfillRequest}s when it reconnects.\n //\n // This means that the change-streamer must track and persist:\n // the backfill IDs of the columns requiring backfilling\n // * the most current table metadata of the associated table(s)\n //\n // The change-streamer then uses this information to send backfill requests\n // when it reconnects.\n backfill: v.record(backfillIDSchema).optional(),\n});\n\nexport const renameTableSchema = v.object({\n tag: v.literal('rename-table'),\n old: identifierSchema,\n new: identifierSchema,\n});\n\nexport const updateTableMetadataSchema = v.object({\n tag: v.literal('update-table-metadata'),\n table: identifierSchema,\n old: tableMetadataSchema,\n new: tableMetadataSchema,\n});\n\nconst columnSchema = v.object({\n name: v.string(),\n spec: columnSpec,\n});\n\nexport const addColumnSchema = v.object({\n tag: v.literal('add-column'),\n table: identifierSchema,\n column: columnSchema,\n\n // This must be set by change source implementations that support\n // table/column backfill.\n //\n // TODO: to simplify the protocol, see if we can make this required\n tableMetadata: tableMetadataSchema.optional(),\n\n // See documentation for the `backfill` field of the `create-table` change.\n backfill: backfillIDSchema.optional(),\n});\n\nexport const updateColumnSchema = v.object({\n tag: v.literal('update-column'),\n table: identifierSchema,\n old: columnSchema,\n new: columnSchema,\n});\n\nexport const dropColumnSchema = v.object({\n tag: v.literal('drop-column'),\n table: identifierSchema,\n column: v.string(),\n});\n\nexport const dropTableSchema = v.object({\n tag: v.literal('drop-table'),\n id: identifierSchema,\n});\n\nexport const createIndexSchema = v.object({\n tag: v.literal('create-index'),\n spec: indexSpec,\n});\n\nexport const dropIndexSchema = v.object({\n tag: v.literal('drop-index'),\n id: identifierSchema,\n});\n\nexport const downloadStatusSchema = v.object({\n rows: v.number(),\n totalRows: v.number(),\n totalBytes: v.number().optional(),\n});\n\nexport type DownloadStatus = v.Infer<typeof downloadStatusSchema>;\n\n// A batch of rows from a single table containing column values\n// to be backfilled.\nexport const backfillSchema = v.object({\n tag: v.literal('backfill'),\n\n relation: newRelationSchema,\n\n // The columns to be backfilled. `rowKey` columns are automatically excluded,\n // which means that this field may be empty.\n columns: v.array(v.string()),\n\n // The watermark at which the backfill data was queried. Note that this\n // generally will be different from the commit watermarks of the main change\n // stream, and in particular, the commit watermark of the backfill change's\n // enclosing transaction.\n watermark: v.string(),\n\n // A batch of row values, each row consisting of the `rowKey`\n // values, followed by the `column` values, in the same order in which\n // the column names appear in their respective fields, e.g.\n //\n // ```\n // [\n // [...rowKeyValues, ...columnValues], // row 1\n // [...rowKeyValues, ...columnValues], // row 2\n // ]\n // ```\n rowValues: v.array(v.array(jsonValueSchema)),\n\n // Optionally includes the progress of the backfill operation,\n // for display purposes.\n status: downloadStatusSchema.optional(),\n});\n\n// Indicates that the backfill for the specified columns have\n// been successfully backfilled and can be published to clients.\nexport const backfillCompletedSchema = v.object({\n tag: v.literal('backfill-completed'),\n\n relation: newRelationSchema,\n\n // The columns to be backfilled. `rowKey` columns are automatically excluded,\n // which means that this field may be empty.\n columns: v.array(v.string()),\n\n // The watermark at which the backfill data was queried. Note that this\n // generally will be different from the commit watermarks of the main change\n // stream, and in particular, the commit watermark of the backfill change's\n // enclosing transaction.\n watermark: v.string(),\n\n // Optionally includes the final status of the backfill operation,\n // for display purposes.\n status: downloadStatusSchema.optional(),\n});\n\nexport type MessageBegin = v.Infer<typeof beginSchema>;\nexport type MessageCommit = v.Infer<typeof commitSchema>;\nexport type MessageRollback = v.Infer<typeof rollbackSchema>;\n\nexport type MessageRelation = v.Infer<typeof relationSchema>;\nexport type MessageInsert = v.Infer<typeof insertSchema>;\nexport type MessageUpdate = v.Infer<typeof updateSchema>;\nexport type MessageDelete = v.Infer<typeof deleteSchema>;\nexport type MessageTruncate = v.Infer<typeof truncateSchema>;\n\nexport type MessageBackfill = v.Infer<typeof backfillSchema>;\n\nexport type TableCreate = v.Infer<typeof createTableSchema>;\nexport type TableRename = v.Infer<typeof renameTableSchema>;\nexport type TableUpdateMetadata = v.Infer<typeof updateTableMetadataSchema>;\nexport type ColumnAdd = v.Infer<typeof addColumnSchema>;\nexport type ColumnUpdate = v.Infer<typeof updateColumnSchema>;\nexport type ColumnDrop = v.Infer<typeof dropColumnSchema>;\nexport type TableDrop = v.Infer<typeof dropTableSchema>;\nexport type IndexCreate = v.Infer<typeof createIndexSchema>;\nexport type IndexDrop = v.Infer<typeof dropIndexSchema>;\nexport type BackfillCompleted = v.Infer<typeof backfillCompletedSchema>;\n\nexport const dataChangeSchema = v.union(\n insertSchema,\n updateSchema,\n deleteSchema,\n truncateSchema,\n backfillSchema,\n);\n\n// Note: keep in sync or the tag tests will fail\nconst dataChangeTags = [\n 'insert',\n 'update',\n 'delete',\n 'truncate',\n 'backfill',\n] as const;\n\nconst dataChangeTagsSchema = v.literalUnion(...dataChangeTags);\n\nexport type DataChange = Satisfies<\n JSONObject, // guarantees serialization over IPC or network\n v.Infer<typeof dataChangeSchema>\n>;\n\nexport type DataChangeTag = v.Infer<typeof dataChangeTagsSchema>;\n\nconst schemaChanges = [\n createTableSchema,\n renameTableSchema,\n updateTableMetadataSchema,\n addColumnSchema,\n updateColumnSchema,\n dropColumnSchema,\n dropTableSchema,\n createIndexSchema,\n dropIndexSchema,\n backfillCompletedSchema,\n] as const;\n\nexport const schemaChangeSchema = v.union(...schemaChanges);\n\nconst schemaChangeTagsSchema = v.literalUnion(...schemaChangeTags);\n\nexport type SchemaChange = Satisfies<\n JSONObject,\n v.Infer<typeof schemaChangeSchema>\n>;\n\nexport type SchemaChangeTag = v.Infer<typeof schemaChangeTagsSchema>;\n\nexport type DataOrSchemaChange = DataChange \| SchemaChange;\n\nexport type Change =\n \| MessageBegin\n \| DataOrSchemaChange\n \| MessageCommit\n \| MessageRollback;\n\nexport type ChangeTag = Change['tag'];\n\nconst schemaChangeTagSet = new Set<string>(schemaChangeTags);\n\nexport function isSchemaChange(change: Change): change is SchemaChange {\n return schemaChangeTagSet.has(change.tag);\n}\n\nconst dataChangeTagSet = new Set<string>(dataChangeTags);\n\nexport function isDataChange(change: Change): change is DataChange {\n return dataChangeTagSet.has(change.tag);\n}\n"],"mappings":";;;;;;;;;;;;AAiBA,IAAa,cAAc,eAAE,OAAO;CAClC,KAAK,eAAE,QAAQ,OAAO;CAWtB,MAAM,aAAe,KAAK,GAAG,EAAE,SAAS;CAGxC,SAAS,eAAE,QAAQ,EAAE,SAAS;AAChC,CAAC;AAED,IAAa,eAAe,eAAE,OAAO;CACnC,KAAK,eAAE,QAAQ,QAAQ;CAavB,cAAc,eAAE,OAAO,EAAE,SAAS;AACpC,CAAC;AAED,IAAa,iBAAiB,eAAE,OAAO,EACrC,KAAK,eAAE,QAAQ,UAAU,EAC3B,CAAC;AAED,IAAM,eAAe,eAAE,OAAO;CAE5B,SAAS,eAAE,MAAM,eAAE,OAAO,CAAC;CAO3B,MAAM,aAAe,WAAW,WAAW,QAAQ,OAAO,EAAE,SAAS;AACvE,CAAC;AAED,IAAa,iBAAiB,eAC3B,OAAO;CACN,QAAQ,eAAE,OAAO;CACjB,MAAM,eAAE,OAAO;CAGf,QAAQ,aAAa,SAAS;;CAG9B,YAAY,eAAE,MAAM,eAAE,OAAO,CAAC,EAAE,SAAS;;CAEzC,iBAAiB,aACD,WAAW,WAAW,QAAQ,OAAO,EAClD,SAAS;AACd,CAAC,EACA,KAAI,QAAO;CACV,MAAM,EAAC,QAAQ,GAAG,SAAQ;CAC1B,IAAI,QACF,OAAO;EAAC,GAAG;EAAM;CAAM;CAEzB,OAAO;EACL,GAAG;EACH,QAAQ;GACN,SAAS,KAAK,IAAI,UAAU;GAC5B,MAAM,IAAI;EACZ;CACF;AACF,CAAC;AAGH,IAAa,oBAAoB,eAAE,OAAO;CACxC,QAAQ,eAAE,OAAO;CACjB,MAAM,eAAE,OAAO;CAEf,QAAQ;AACV,CAAC;AAUD,IAAa,sBAAsB,eAChC,OAAO,EAAC,QAAQ,eAAE,OAAO,eAAe,EAAC,CAAC,EAC1C,KAAK,eAAe;AAIvB,IAAa,YAAY,eAAE,OAAO,eAAe;AAEjD,IAAa,eAAe,eAAE,OAAO;CACnC,KAAK,eAAE,QAAQ,QAAQ;CACvB,UAAU;CACV,KAAK;AACP,CAAC;AAED,IAAa,eAAe,eAAE,OAAO;CACnC,KAAK,eAAE,QAAQ,QAAQ;CACvB,UAAU;CAGV,KAAK,UAAU,SAAS;CAIxB,KAAK;AACP,CAAC;AAED,IAAa,eAAe,eAAE,OAAO;CACnC,KAAK,eAAE,QAAQ,QAAQ;CACvB,UAAU;CAEV,KAAK;AACP,CAAC;AAED,IAAa,iBAAiB,eAAE,OAAO;CACrC,KAAK,eAAE,QAAQ,UAAU;CACzB,WAAW,eAAE,MAAM,cAAc;AACnC,CAAC;AAED,IAAa,mBAAmB,eAAE,OAAO;CACvC,QAAQ,eAAE,OAAO;CACjB,MAAM,eAAE,OAAO;AACjB,CAAC;AAYD,IAAa,mBAAmB;AAIhC,IAAa,oBAAoB,eAAE,OAAO;CACxC,KAAK,eAAE,QAAQ,cAAc;CAC7B,MAAM;CAMN,UAAU,oBAAoB,SAAS;CAmBvC,UAAU,eAAE,OAAO,gBAAgB,EAAE,SAAS;AAChD,CAAC;AAED,IAAa,oBAAoB,eAAE,OAAO;CACxC,KAAK,eAAE,QAAQ,cAAc;CAC7B,KAAK;CACL,KAAK;AACP,CAAC;AAED,IAAa,4BAA4B,eAAE,OAAO;CAChD,KAAK,eAAE,QAAQ,uBAAuB;CACtC,OAAO;CACP,KAAK;CACL,KAAK;AACP,CAAC;AAED,IAAM,eAAe,eAAE,OAAO;CAC5B,MAAM,eAAE,OAAO;CACf,MAAM;AACR,CAAC;AAED,IAAa,kBAAkB,eAAE,OAAO;CACtC,KAAK,eAAE,QAAQ,YAAY;CAC3B,OAAO;CACP,QAAQ;CAMR,eAAe,oBAAoB,SAAS;CAG5C,UAAU,iBAAiB,SAAS;AACtC,CAAC;AAED,IAAa,qBAAqB,eAAE,OAAO;CACzC,KAAK,eAAE,QAAQ,eAAe;CAC9B,OAAO;CACP,KAAK;CACL,KAAK;AACP,CAAC;AAED,IAAa,mBAAmB,eAAE,OAAO;CACvC,KAAK,eAAE,QAAQ,aAAa;CAC5B,OAAO;CACP,QAAQ,eAAE,OAAO;AACnB,CAAC;AAED,IAAa,kBAAkB,eAAE,OAAO;CACtC,KAAK,eAAE,QAAQ,YAAY;CAC3B,IAAI;AACN,CAAC;AAED,IAAa,oBAAoB,eAAE,OAAO;CACxC,KAAK,eAAE,QAAQ,cAAc;CAC7B,MAAM;AACR,CAAC;AAED,IAAa,kBAAkB,eAAE,OAAO;CACtC,KAAK,eAAE,QAAQ,YAAY;CAC3B,IAAI;AACN,CAAC;AAED,IAAa,uBAAuB,eAAE,OAAO;CAC3C,MAAM,eAAE,OAAO;CACf,WAAW,eAAE,OAAO;CACpB,YAAY,eAAE,OAAO,EAAE,SAAS;AAClC,CAAC;AAMD,IAAa,iBAAiB,eAAE,OAAO;CACrC,KAAK,eAAE,QAAQ,UAAU;CAEzB,UAAU;CAIV,SAAS,eAAE,MAAM,eAAE,OAAO,CAAC;CAM3B,WAAW,eAAE,OAAO;CAYpB,WAAW,eAAE,MAAM,eAAE,MAAM,eAAe,CAAC;CAI3C,QAAQ,qBAAqB,SAAS;AACxC,CAAC;AAID,IAAa,0BAA0B,eAAE,OAAO;CAC9C,KAAK,eAAE,QAAQ,oBAAoB;CAEnC,UAAU;CAIV,SAAS,eAAE,MAAM,eAAE,OAAO,CAAC;CAM3B,WAAW,eAAE,OAAO;CAIpB,QAAQ,qBAAqB,SAAS;AACxC,CAAC;AAyBD,IAAa,mBAAmB,eAAE,MAChC,cACA,cACA,cACA,gBACA,cACF;AAGA,IAAM,iBAAiB;CACrB;CACA;CACA;CACA;CACA;AACF;AAE6B,aAAe,GAAG,cAAc;AAS7D,IAAM,gBAAgB;CACpB;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;AACF;AAEA,IAAa,qBAAqB,eAAE,MAAM,GAAG,aAAa;AAE3B,aAAe,GAAG,gBAAgB;AAmBjE,IAAM,qBAAqB,IAAI,IAAY,gBAAgB;AAE3D,SAAgB,eAAe,QAAwC;CACrE,OAAO,mBAAmB,IAAI,OAAO,GAAG;AAC1C;AAEA,IAAM,mBAAmB,IAAI,IAAY,cAAc;AAEvD,SAAgB,aAAa,QAAsC;CACjE,OAAO,iBAAiB,IAAI,OAAO,GAAG;AACxC"}

package/out/zero-cache/src/services/change-source/protocol/current/downstream.d.ts CHANGED Viewed

@@ -1,7 +1,7 @@
 import * as v from '../../../../../../shared/src/valita.ts';
 declare const begin: v.TupleType<[v.Type<"begin">, v.ObjectType<{
     tag: v.Type<"begin">;
-    json: v.Optional<"p" | "s">;
+    json: v.Optional<"s" | "p">;
     skipAck: v.Optional<boolean>;
 }, undefined>, v.ObjectType<{
     commitWatermark: v.Type<string>;
@@ -222,6 +222,7 @@ declare const data: v.TupleType<[v.Type<"data">, v.UnionType<[v.UnionType<[v.Obj
 }, undefined>]>]>]>;
 declare const commit: v.TupleType<[v.Type<"commit">, v.ObjectType<{
     tag: v.Type<"commit">;
+    commitTimeMs: v.Optional<number>;
 }, undefined>, v.ObjectType<{
     watermark: v.Type<string>;
 }, undefined>]>;
@@ -234,7 +235,7 @@ export type Commit = v.Infer<typeof commit>;
 export type Rollback = v.Infer<typeof rollback>;
 export declare const changeStreamDataSchema: v.UnionType<[v.TupleType<[v.Type<"begin">, v.ObjectType<{
     tag: v.Type<"begin">;
-    json: v.Optional<"p" | "s">;
+    json: v.Optional<"s" | "p">;
     skipAck: v.Optional<boolean>;
 }, undefined>, v.ObjectType<{
     commitWatermark: v.Type<string>;
@@ -453,6 +454,7 @@ export declare const changeStreamDataSchema: v.UnionType<[v.TupleType<[v.Type<"b
     }>;
 }, undefined>]>]>]>, v.TupleType<[v.Type<"commit">, v.ObjectType<{
     tag: v.Type<"commit">;
+    commitTimeMs: v.Optional<number>;
 }, undefined>, v.ObjectType<{
     watermark: v.Type<string>;
 }, undefined>]>, v.TupleType<[v.Type<"rollback">, v.ObjectType<{
@@ -468,7 +470,7 @@ export type ChangeStreamControl = v.Infer<typeof changeStreamControlSchema>;
 /** Downstream messages consist of data plane and control plane messages. */
 export declare const changeStreamMessageSchema: v.UnionType<[v.UnionType<[v.TupleType<[v.Type<"begin">, v.ObjectType<{
     tag: v.Type<"begin">;
-    json: v.Optional<"p" | "s">;
+    json: v.Optional<"s" | "p">;
     skipAck: v.Optional<boolean>;
 }, undefined>, v.ObjectType<{
     commitWatermark: v.Type<string>;
@@ -687,6 +689,7 @@ export declare const changeStreamMessageSchema: v.UnionType<[v.UnionType<[v.Tupl
     }>;
 }, undefined>]>]>]>, v.TupleType<[v.Type<"commit">, v.ObjectType<{
     tag: v.Type<"commit">;
+    commitTimeMs: v.Optional<number>;
 }, undefined>, v.ObjectType<{
     watermark: v.Type<string>;
 }, undefined>]>, v.TupleType<[v.Type<"rollback">, v.ObjectType<{

package/out/zero-cache/src/services/change-source/protocol/current/downstream.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"downstream.d.ts","sourceRoot":"","sources":["../../../../../../../../zero-cache/src/services/change-source/protocol/current/downstream.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,CAAC,MAAM,wCAAwC,CAAC;AAW5D,QAAA,MAAM,KAAK;;;;;;eAIT,CAAC;AACH,QAAA,MAAM,IAAI;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;mBAGR,CAAC;AACH,QAAA,MAAM,MAAM~~;;;;~~eAIV,CAAC;AACH,QAAA,MAAM,QAAQ;;eAAmD,CAAC;AAElE,MAAM,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,KAAK,CAAC,CAAC;AAC1C,MAAM,MAAM,IAAI,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,IAAI,CAAC,CAAC;AACxC,MAAM,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,MAAM,CAAC,CAAC;AAC5C,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,QAAQ,CAAC,CAAC;AAEhD,eAAO,MAAM,sBAAsB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAAyC,CAAC;AAC7E,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,sBAAsB,CAAC,CAAC;AAEtE,eAAO,MAAM,yBAAyB;;;;eAGpC,CAAC;AACH,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC;AAE5E,4EAA4E;AAC5E,eAAO,MAAM,yBAAyB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAIrC,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC"}
1	+ {"version":3,"file":"downstream.d.ts","sourceRoot":"","sources":["../../../../../../../../zero-cache/src/services/change-source/protocol/current/downstream.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,CAAC,MAAM,wCAAwC,CAAC;AAW5D,QAAA,MAAM,KAAK;;;;;;eAIT,CAAC;AACH,QAAA,MAAM,IAAI;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;mBAGR,CAAC;AACH,QAAA,MAAM,MAAM;;;;;eAIV,CAAC;AACH,QAAA,MAAM,QAAQ;;eAAmD,CAAC;AAElE,MAAM,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,KAAK,CAAC,CAAC;AAC1C,MAAM,MAAM,IAAI,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,IAAI,CAAC,CAAC;AACxC,MAAM,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,MAAM,CAAC,CAAC;AAC5C,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,QAAQ,CAAC,CAAC;AAEhD,eAAO,MAAM,sBAAsB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAAyC,CAAC;AAC7E,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,sBAAsB,CAAC,CAAC;AAEtE,eAAO,MAAM,yBAAyB;;;;eAGpC,CAAC;AACH,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC;AAE5E,4EAA4E;AAC5E,eAAO,MAAM,yBAAyB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAIrC,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC"}

package/out/zero-cache/src/services/change-source/protocol/current/schema-change-tags.d.ts ADDED Viewed

@@ -0,0 +1,6 @@
+/**
+ * Kept in a module that is not re-exported by the public change protocol so
+ * internal consumers can share the tag list without expanding the public API.
+ */
+export declare const schemaChangeTags: readonly ["create-table", "rename-table", "update-table-metadata", "add-column", "update-column", "drop-column", "drop-table", "create-index", "drop-index", "backfill-completed"];
+//# sourceMappingURL=schema-change-tags.d.ts.map

package/out/zero-cache/src/services/change-source/protocol/current/schema-change-tags.d.ts.map ADDED Viewed

	@@ -0,0 +1 @@
1	+ {"version":3,"file":"schema-change-tags.d.ts","sourceRoot":"","sources":["../../../../../../../../zero-cache/src/services/change-source/protocol/current/schema-change-tags.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,eAAO,MAAM,gBAAgB,oLAWnB,CAAC"}

package/out/zero-cache/src/services/change-source/protocol/current/schema-change-tags.js ADDED Viewed

@@ -0,0 +1,21 @@
+//#region ../zero-cache/src/services/change-source/protocol/current/schema-change-tags.ts
+/**
+* Kept in a module that is not re-exported by the public change protocol so
+* internal consumers can share the tag list without expanding the public API.
+*/
+var schemaChangeTags = [
+	"create-table",
+	"rename-table",
+	"update-table-metadata",
+	"add-column",
+	"update-column",
+	"drop-column",
+	"drop-table",
+	"create-index",
+	"drop-index",
+	"backfill-completed"
+];
+//#endregion
+export { schemaChangeTags };
+//# sourceMappingURL=schema-change-tags.js.map

package/out/zero-cache/src/services/change-source/protocol/current/schema-change-tags.js.map ADDED Viewed

@@ -0,0 +1 @@

+ {"version":3,"file":"schema-change-tags.js","names":[],"sources":["../../../../../../../../zero-cache/src/services/change-source/protocol/current/schema-change-tags.ts"],"sourcesContent":["/**\n * Kept in a module that is not re-exported by the public change protocol so\n * internal consumers can share the tag list without expanding the public API.\n */\nexport const schemaChangeTags = [\n 'create-table',\n 'rename-table',\n 'update-table-metadata',\n 'add-column',\n 'update-column',\n 'drop-column',\n 'drop-table',\n 'create-index',\n 'drop-index',\n 'backfill-completed',\n] as const;\n"],"mappings":";;;;;AAIA,IAAa,mBAAmB;CAC9B;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;AACF"}

package/out/zero-cache/src/services/change-source/protocol/current/status.d.ts CHANGED Viewed

@@ -47,6 +47,7 @@ export declare const upstreamStatusMessageSchema: v.TupleType<[v.Type<"status">,
     }>;
 }, undefined>, v.ObjectType<{
     tag: v.Type<"commit">;
+    commitTimeMs: v.Optional<number>;
 }, undefined>]>, v.ObjectType<{
     watermark: v.Type<string>;
 }, undefined>]>;

package/out/zero-cache/src/services/change-source/protocol/current/status.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"status.d.ts","sourceRoot":"","sources":["../../../../../../../../zero-cache/src/services/change-source/protocol/current/status.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,CAAC,MAAM,wCAAwC,CAAC;AAI5D;;;;GAIG;AACH,eAAO,MAAM,sBAAsB;;;;;;;;;;aAQjC,CAAC;AAEH,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,sBAAsB,CAAC,CAAC;AAEtE,eAAO,MAAM,6BAA6B;;;;;;;;;;;;eAIxC,CAAC;AAEH;;;;;GAKG;AACH,eAAO,MAAM,2BAA2B~~;;;;;;;;;;;;;;~~eAItC,CAAC;AAEH;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,MAAM,MAAM,uBAAuB,GAAG,CAAC,CAAC,KAAK,CAC3C,OAAO,6BAA6B,CACrC,CAAC;AACF,MAAM,MAAM,qBAAqB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,2BAA2B,CAAC,CAAC"}
1	+ {"version":3,"file":"status.d.ts","sourceRoot":"","sources":["../../../../../../../../zero-cache/src/services/change-source/protocol/current/status.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,CAAC,MAAM,wCAAwC,CAAC;AAI5D;;;;GAIG;AACH,eAAO,MAAM,sBAAsB;;;;;;;;;;aAQjC,CAAC;AAEH,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,sBAAsB,CAAC,CAAC;AAEtE,eAAO,MAAM,6BAA6B;;;;;;;;;;;;eAIxC,CAAC;AAEH;;;;;GAKG;AACH,eAAO,MAAM,2BAA2B;;;;;;;;;;;;;;;eAItC,CAAC;AAEH;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,MAAM,MAAM,uBAAuB,GAAG,CAAC,CAAC,KAAK,CAC3C,OAAO,6BAA6B,CACrC,CAAC;AACF,MAAM,MAAM,qBAAqB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,2BAA2B,CAAC,CAAC"}

package/out/zero-cache/src/services/change-source/protocol/current/upstream.d.ts CHANGED Viewed

@@ -12,6 +12,7 @@ export declare const changeSourceUpstreamSchema: v.TupleType<[v.Type<"status">,
     }>;
 }, undefined>, v.ObjectType<{
     tag: v.Type<"commit">;
+    commitTimeMs: v.Optional<number>;
 }, undefined>]>, v.ObjectType<{
     watermark: v.Type<string>;
 }, undefined>]>;

package/out/zero-cache/src/services/change-source/protocol/current/upstream.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"upstream.d.ts","sourceRoot":"","sources":["../../../../../../../../zero-cache/src/services/change-source/protocol/current/upstream.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,CAAC,MAAM,wCAAwC,CAAC;AAQ5D,sEAAsE;AACtE,eAAO,MAAM,0BAA0B~~;;;;;;;;;;;;;;~~eAA8B,CAAC;AACtE,MAAM,MAAM,oBAAoB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,0BAA0B,CAAC,CAAC;AAE9E;;;;;;;;;;;GAWG;AACH,eAAO,MAAM,qBAAqB;;;;;;;;;;;aAOhC,CAAC;AAEH,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,qBAAqB,CAAC,CAAC"}
1	+ {"version":3,"file":"upstream.d.ts","sourceRoot":"","sources":["../../../../../../../../zero-cache/src/services/change-source/protocol/current/upstream.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,CAAC,MAAM,wCAAwC,CAAC;AAQ5D,sEAAsE;AACtE,eAAO,MAAM,0BAA0B;;;;;;;;;;;;;;;eAA8B,CAAC;AACtE,MAAM,MAAM,oBAAoB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,0BAA0B,CAAC,CAAC;AAE9E;;;;;;;;;;;GAWG;AACH,eAAO,MAAM,qBAAqB;;;;;;;;;;;aAOhC,CAAC;AAEH,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,qBAAqB,CAAC,CAAC"}

package/out/zero-cache/src/services/change-streamer/backup-cleanup-monitor-factory.d.ts CHANGED Viewed

@@ -1,18 +1,14 @@
 import type { LogContext } from '@rocicorp/logger';
 import type { NormalizedZeroConfig } from '../../config/normalize.ts';
-import type { BackupMonitor } from './backup-monitor.ts';
+import { BackupMonitor } from './backup-monitor.ts';
 import type { ChangeStreamerService } from './change-streamer.ts';
-import { type BackupStateVerifier } from './litestream3-backup-monitor.ts';
-import { type VfsBackupWatermarkSource } from './vfs-backup-monitor.ts';
+import { type BackupStateVerifier } from './litestream3-prometheus-poller.ts';
 export type BackupCleanupMonitorFactoryOptions = {
     lc: LogContext;
     config: NormalizedZeroConfig;
     replicaFile: string;
     changeStreamer: ChangeStreamerService;
-    initialCleanupDelayMs: number;
     verifyBackupState?: BackupStateVerifier | undefined;
-    vfsBackupWatermarkSource?: VfsBackupWatermarkSource | undefined;
-    env?: NodeJS.ProcessEnv | undefined;
 };
-export declare function createBackupCleanupMonitor({ lc, config, replicaFile, changeStreamer, initialCleanupDelayMs, verifyBackupState, vfsBackupWatermarkSource, env, }: BackupCleanupMonitorFactoryOptions): BackupMonitor | null;
+export declare function createBackupCleanupMonitor({ lc, config, replicaFile, changeStreamer, verifyBackupState, }: BackupCleanupMonitorFactoryOptions): BackupMonitor;
 //# sourceMappingURL=backup-cleanup-monitor-factory.d.ts.map

package/out/zero-cache/src/services/change-streamer/backup-cleanup-monitor-factory.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"backup-cleanup-monitor-factory.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/backup-cleanup-monitor-factory.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;~~AACjD~~,OAAO,KAAK,EAAC,oBAAoB,EAAC,MAAM,2BAA2B,CAAC;~~AAKpE~~,OAAO,~~KAAK~~,~~EAAC,~~aAAa,EAAC,MAAM,qBAAqB,CAAC;~~AACvD~~,OAAO,KAAK,EAAC,qBAAqB,EAAC,MAAM,sBAAsB,CAAC;AAChE,OAAO,~~EAEL~~,KAAK,mBAAmB,~~EACzB~~,MAAM,~~iCAAiC~~,CAAC;~~AACzC~~,~~OAAO,EAEL,KAAK,wBAAwB,EAC9B,~~MAAM,~~yBAAyB,CAAC;AAEjC,~~MAAM,~~MAAM,~~kCAAkC,GAAG;IAC/C,EAAE,EAAE,UAAU,CAAC;IACf,MAAM,EAAE,oBAAoB,CAAC;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,EAAE,qBAAqB,CAAC;IACtC,~~qBAAqB,EAAE,MAAM,CAAC;IAC9B,~~iBAAiB,CAAC,EAAE,mBAAmB,GAAG,SAAS,CAAC;~~IACpD~~,~~wBAAwB,~~CAAC~~,EAAE,wBAAwB,GAAG,SAAS,CAAC~~;~~IAChE,GAAG,CAAC,EAAE,MAAM,CAAC,UAAU,GAAG,SAAS,CAAC;CACrC,CAAC;~~AAEF,wBAAgB,0BAA0B,CAAC,EACzC,EAAE,EACF,MAAM,EACN,WAAW,EACX,cAAc,EACd,~~qBAAqB,EACrB,~~iBAAiB,~~EACjB~~,~~wBAAwB,EACxB,GAAG,GACJ,~~EAAE,kCAAkC,GAAG,aAAa,~~GAAG,IAAI,CAgC3D~~"}
1	+ {"version":3,"file":"backup-cleanup-monitor-factory.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/backup-cleanup-monitor-factory.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,oBAAoB,EAAC,MAAM,2BAA2B,CAAC;AAGpE,OAAO,EAAyB,aAAa,EAAC,MAAM,qBAAqB,CAAC;AAC1E,OAAO,KAAK,EAAC,qBAAqB,EAAC,MAAM,sBAAsB,CAAC;AAChE,OAAO,EACL,KAAK,mBAAmB,EAEzB,MAAM,oCAAoC,CAAC;AAI5C,MAAM,MAAM,kCAAkC,GAAG;IAC/C,EAAE,EAAE,UAAU,CAAC;IACf,MAAM,EAAE,oBAAoB,CAAC;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,EAAE,qBAAqB,CAAC;IACtC,iBAAiB,CAAC,EAAE,mBAAmB,GAAG,SAAS,CAAC;CACrD,CAAC;AAEF,wBAAgB,0BAA0B,CAAC,EACzC,EAAE,EACF,MAAM,EACN,WAAW,EACX,cAAc,EACd,iBAAiB,GAClB,EAAE,kCAAkC,GAAG,aAAa,CAyCpD"}

package/out/zero-cache/src/services/change-streamer/backup-cleanup-monitor-factory.js CHANGED Viewed

@@ -1,15 +1,31 @@
-import { forkChildWorker } from "../../types/processes.js";
-import { BACKUP_WATERMARK_READER_URL } from "../../server/worker-urls.js";
+import { must } from "../../../../shared/src/must.js";
 import { getLastBackupTime } from "../litestream/commands.js";
-import { VfsBackupWatermarkWorkerSource } from "../litestream/vfs-watermark-worker-source.js";
-import { Litestream3BackupMonitor } from "./litestream3-backup-monitor.js";
-import { VfsBackupMonitor } from "./vfs-backup-monitor.js";
+import { BackupMonitor } from "./backup-monitor.js";
+import { Litestream3PrometheusPoller } from "./litestream3-prometheus-poller.js";
+import { ReplicaPoller } from "./replica-poller.js";
+import { VfsWatermarkPoller } from "./vfs-watermark-poller.js";
 //#region ../zero-cache/src/services/change-streamer/backup-cleanup-monitor-factory.ts
-function createBackupCleanupMonitor({ lc, config, replicaFile, changeStreamer, initialCleanupDelayMs, verifyBackupState, vfsBackupWatermarkSource, env }) {
-	const { backupURL, port: metricsPort } = config.litestream;
-	if (!backupURL) return null;
-	if (config.litestream.backupUsingV5) return new VfsBackupMonitor(lc, backupURL, changeStreamer, initialCleanupDelayMs, config.litestream.vfsProbeIntervalMs, vfsBackupWatermarkSource ?? new VfsBackupWatermarkWorkerSource(lc, () => forkChildWorker(BACKUP_WATERMARK_READER_URL, env ?? process.env), config.litestream.vfsProbeTimeoutMs));
-	return new Litestream3BackupMonitor(lc, replicaFile, backupURL, `http://localhost:${metricsPort}/metrics`, changeStreamer, initialCleanupDelayMs, verifyBackupState ?? (() => getLastBackupTime(lc, config)));
+function createBackupCleanupMonitor({ lc, config, replicaFile, changeStreamer, verifyBackupState }) {
+	const { log, litestream, replica } = config;
+	const { backupURL } = litestream;
+	let stream;
+	if (!backupURL) stream = new ReplicaPoller(lc, replicaFile).start();
+	else if (config.litestream.backupUsingV5) {
+		const { logLevel, endpoint, region, vfsQueryExecutable, vfsPollIntervalMs: remotePollIntervalMs } = litestream;
+		stream = new VfsWatermarkPoller(lc, replicaFile, {
+			executable: must(vfsQueryExecutable, `litestream-vfs-query-executable must be defined`),
+			remotePollIntervalMs,
+			backupURL,
+			region,
+			endpoint,
+			logLevel,
+			logFormat: log.format
+		}).start();
+	} else {
+		const { port: metricsPort } = litestream;
+		stream = new Litestream3PrometheusPoller(lc, replicaFile, backupURL, `http://localhost:${metricsPort}/metrics`, verifyBackupState ?? (() => getLastBackupTime(lc, litestream, replica.file))).start();
+	}
+	return new BackupMonitor(lc, stream, changeStreamer, replicaFile);
 }
 //#endregion
 export { createBackupCleanupMonitor };

package/out/zero-cache/src/services/change-streamer/backup-cleanup-monitor-factory.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"backup-cleanup-monitor-factory.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/backup-cleanup-monitor-factory.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport ~~type~~ {~~NormalizedZeroConfig~~} from '~~../../config~~/~~normalize~~.ts';\nimport {~~BACKUP_WATERMARK_READER_URL~~} from '../../~~server~~/~~worker-urls~~.ts';\nimport {~~forkChildWorker~~} from '../../types/~~processes~~.ts';\nimport {getLastBackupTime} from '../litestream/commands.ts';\nimport {~~VfsBackupWatermarkWorkerSource}~~ ~~from~~ ~~'../litestream/vfs-watermark-worker-source.ts';\nimport type {~~BackupMonitor} from './backup-monitor.ts';\nimport type {ChangeStreamerService} from './change-streamer.ts';\nimport {\n ~~Litestream3BackupMonitor,\n~~ type BackupStateVerifier,\n} from './litestream3-~~backup~~-~~monitor~~.ts';\nimport {~~\n VfsBackupMonitor,\n type~~ ~~VfsBackupWatermarkSource,\n~~} from './vfs-~~backup~~-~~monitor~~.ts';\n\nexport type BackupCleanupMonitorFactoryOptions = {\n lc: LogContext;\n config: NormalizedZeroConfig;\n replicaFile: string;\n changeStreamer: ChangeStreamerService;\n ~~initialCleanupDelayMs: number;\n~~ verifyBackupState?: BackupStateVerifier \| undefined;\n ~~vfsBackupWatermarkSource?: VfsBackupWatermarkSource \| undefined;\n env?: NodeJS.ProcessEnv \| undefined;\n~~};\n\nexport function createBackupCleanupMonitor({\n lc,\n config,\n replicaFile,\n changeStreamer,\n ~~initialCleanupDelayMs,\n~~ verifyBackupState,\n ~~vfsBackupWatermarkSource,\n env,\n~~}: BackupCleanupMonitorFactoryOptions): BackupMonitor ~~\| null~~ {\n const {~~backupURL~~, ~~port:~~ ~~metricsPort~~} = config.litestream;\n if (!backupURL) {\n ~~return~~ ~~null~~;\n }~~\n\n~~ if (config.litestream.backupUsingV5) {\n ~~return~~ ~~new VfsBackupMonitor(~~\n lc,\n ~~backupURL~~,\n ~~changeStreamer~~,\n ~~initialCleanupDelayMs~~,\n ~~config.litestream.vfsProbeIntervalMs~~,\n ~~vfsBackupWatermarkSource~~ ~~??\~~n new ~~VfsBackupWatermarkWorkerSource~~(\n lc,\n () ~~=>\n forkChildWorker(BACKUP_WATERMARK_READER_URL,~~ ~~env~~ ~~?? process.env~~),\n ~~config.litestream.vfsProbeTimeoutMs~~,\n ),\n );\n }\n\n ~~return~~ new ~~Litestream3BackupMonitor~~(\n lc,\n replicaFile,\n backupURL,\n `http://localhost:${metricsPort}/metrics`,\n ~~changeStreamer,\n initialCleanupDelayMs,\n~~ verifyBackupState ?? (() => getLastBackupTime(lc, ~~config~~)),\n );\n}\n"],"mappings":";;;;;;;~~AA4BA~~,SAAgB,2BAA2B,EACzC,IACA,QACA,aACA,gBACA,~~uBACA,mBACA,0BACA,OAC2D~~;~~CAC3D~~,MAAM,EAAC,~~WAAW~~,MAAM,~~gBAAe~~,~~OAAO~~;~~CAC9C~~,IAAI,CAAC,WACH,~~OAAO~~;~~CAGT~~,IAAI,OAAO,WAAW,~~eACpB~~,~~OAAO~~,~~IAAI~~,~~iBACT~~,~~IACA~~,~~WACA~~,~~gBACA~~,~~uBACA~~,~~OAAO~~,~~WAAW~~,~~oBAClB~~,~~4BACE~~,IAAI~~,+BACF~~,~~UAEE~~,~~gBAAgB~~,~~6BAA6B~~,~~OAAO~~,~~QAAQ~~,~~GAAG~~,~~GACjE~~,OAAO,~~WAAW~~,~~iBACpB~~,~~CACJ~~;~~CAGF~~,~~OAAO~~,IAAI,~~yBACT~~,IACA,aACA,WACA,oBAAoB,YAAY,WAChC,~~gBACA~~,~~uBACA~~,~~4BAA4B~~,~~kBAAkB~~,IAAI,MAAM,~~EAC1D~~;~~AACF~~"}
1	+ {"version":3,"file":"backup-cleanup-monitor-factory.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/backup-cleanup-monitor-factory.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {must} from '../../../../shared/src/must.ts';\nimport type {NormalizedZeroConfig} from '../../config/normalize.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {getLastBackupTime} from '../litestream/commands.ts';\nimport {type BackedUpWatermark, BackupMonitor} from './backup-monitor.ts';\nimport type {ChangeStreamerService} from './change-streamer.ts';\nimport {\n type BackupStateVerifier,\n Litestream3PrometheusPoller,\n} from './litestream3-prometheus-poller.ts';\nimport {ReplicaPoller} from './replica-poller.ts';\nimport {VfsWatermarkPoller} from './vfs-watermark-poller.ts';\n\nexport type BackupCleanupMonitorFactoryOptions = {\n lc: LogContext;\n config: NormalizedZeroConfig;\n replicaFile: string;\n changeStreamer: ChangeStreamerService;\n verifyBackupState?: BackupStateVerifier \| undefined;\n};\n\nexport function createBackupCleanupMonitor({\n lc,\n config,\n replicaFile,\n changeStreamer,\n verifyBackupState,\n}: BackupCleanupMonitorFactoryOptions): BackupMonitor {\n const {log, litestream, replica} = config;\n const {backupURL} = litestream;\n\n let stream: Source<BackedUpWatermark>;\n\n if (!backupURL) {\n stream = new ReplicaPoller(lc, replicaFile).start();\n } else if (config.litestream.backupUsingV5) {\n const {\n logLevel,\n endpoint,\n region,\n vfsQueryExecutable,\n vfsPollIntervalMs: remotePollIntervalMs,\n } = litestream;\n stream = new VfsWatermarkPoller(lc, replicaFile, {\n executable: must(\n vfsQueryExecutable,\n `litestream-vfs-query-executable must be defined`,\n ),\n remotePollIntervalMs,\n backupURL,\n region,\n endpoint,\n logLevel,\n logFormat: log.format,\n }).start();\n } else {\n const {port: metricsPort} = litestream;\n stream = new Litestream3PrometheusPoller(\n lc,\n replicaFile,\n backupURL,\n `http://localhost:${metricsPort}/metrics`,\n verifyBackupState ??\n (() => getLastBackupTime(lc, litestream, replica.file)),\n ).start();\n }\n\n return new BackupMonitor(lc, stream, changeStreamer, replicaFile);\n}\n"],"mappings":";;;;;;;AAsBA,SAAgB,2BAA2B,EACzC,IACA,QACA,aACA,gBACA,qBACoD;CACpD,MAAM,EAAC,KAAK,YAAY,YAAW;CACnC,MAAM,EAAC,cAAa;CAEpB,IAAI;CAEJ,IAAI,CAAC,WACH,SAAS,IAAI,cAAc,IAAI,WAAW,EAAE,MAAM;MAC7C,IAAI,OAAO,WAAW,eAAe;EAC1C,MAAM,EACJ,UACA,UACA,QACA,oBACA,mBAAmB,yBACjB;EACJ,SAAS,IAAI,mBAAmB,IAAI,aAAa;GAC/C,YAAY,KACV,oBACA,iDACF;GACA;GACA;GACA;GACA;GACA;GACA,WAAW,IAAI;EACjB,CAAC,EAAE,MAAM;CACX,OAAO;EACL,MAAM,EAAC,MAAM,gBAAe;EAC5B,SAAS,IAAI,4BACX,IACA,aACA,WACA,oBAAoB,YAAY,WAChC,4BACS,kBAAkB,IAAI,YAAY,QAAQ,IAAI,EACzD,EAAE,MAAM;CACV;CAEA,OAAO,IAAI,cAAc,IAAI,QAAQ,gBAAgB,WAAW;AAClE"}

package/out/zero-cache/src/services/change-streamer/backup-monitor.d.ts CHANGED Viewed

@@ -1,8 +1,18 @@
-import type { Subscription } from '../../types/subscription.ts';
+import type { LogContext } from '@rocicorp/logger';
+import type { Source } from '../../types/streams.ts';
 import type { SingletonService } from '../service.ts';
-import type { SnapshotMessage } from './snapshot.ts';
-export interface BackupMonitor extends SingletonService {
-    startSnapshotReservation(taskID: string): Subscription<SnapshotMessage>;
-    endReservation(taskID: string, updateCleanupDelay?: boolean | undefined): void;
+import type { ChangeStreamerService } from './change-streamer.ts';
+export type BackedUpWatermark = {
+    watermark: string;
+    writeTimeMs?: number | undefined;
+    backupTimeMs: number;
+};
+export declare class BackupMonitor implements SingletonService {
+    #private;
+    readonly id = "backup-monitor";
+    constructor(lc: LogContext, watermarks: Source<BackedUpWatermark>, changeStreamer: ChangeStreamerService, replicaFile: string);
+    firstBackupReceived(): Promise<void>;
+    run(): Promise<void>;
+    stop(): Promise<void>;
 }
 //# sourceMappingURL=backup-monitor.d.ts.map

package/out/zero-cache/src/services/change-streamer/backup-monitor.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"backup-monitor.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/backup-monitor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,~~YAAY~~,EAAC,MAAM,~~6BAA6B~~,CAAC;~~AAC9D~~,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,eAAe,CAAC;AACpD,OAAO,KAAK,EAAC,~~eAAe~~,EAAC,MAAM,~~eAAe~~,CAAC;~~AAEnD~~,MAAM,~~WAAW~~,~~aAAc~~,~~SAAQ~~,~~gBAAgB~~;~~IACrD~~,~~wBAAwB~~,CAAC,~~MAAM,~~EAAE,MAAM,GAAG,~~YAAY~~,CAAC,~~eAAe~~,CAAC,CAAC;~~IACxE~~,~~cAAc~~,~~CACZ~~,~~MAAM~~,EAAE,~~MAAM~~,EACd,~~kBAAkB~~,CAAC,EAAE,~~OAAO~~,GAAG,~~SAAS~~,~~GACvC~~,IAAI,CAAC;~~CACT~~"}
1	+ {"version":3,"file":"backup-monitor.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/backup-monitor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAKjD,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,wBAAwB,CAAC;AACnD,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,eAAe,CAAC;AACpD,OAAO,KAAK,EAAC,qBAAqB,EAAC,MAAM,sBAAsB,CAAC;AAEhE,MAAM,MAAM,iBAAiB,GAAG;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACjC,YAAY,EAAE,MAAM,CAAC;CACtB,CAAC;AAEF,qBAAa,aAAc,YAAW,gBAAgB;;IACpD,QAAQ,CAAC,EAAE,oBAAoB;gBAU7B,EAAE,EAAE,UAAU,EACd,UAAU,EAAE,MAAM,CAAC,iBAAiB,CAAC,EACrC,cAAc,EAAE,qBAAqB,EACrC,WAAW,EAAE,MAAM;IAQrB,mBAAmB;IAIb,GAAG;IAqBT,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;CAkCtB"}

package/out/zero-cache/src/services/change-streamer/backup-monitor.js ADDED Viewed

@@ -0,0 +1,71 @@
+import { promiseVoid } from "../../../../shared/src/resolved-promises.js";
+import { Database } from "../../../../zqlite/src/db.js";
+import { getOrCreateGauge } from "../../observability/metrics.js";
+import { resolver } from "@rocicorp/resolver";
+//#region ../zero-cache/src/services/change-streamer/backup-monitor.ts
+var BackupMonitor = class {
+	id = "backup-monitor";
+	#lc;
+	#watermarks;
+	#changeStreamer;
+	#replicaFile;
+	#firstBackupReceived = resolver();
+	#latestBackup;
+	constructor(lc, watermarks, changeStreamer, replicaFile) {
+		this.#lc = lc;
+		this.#watermarks = watermarks;
+		this.#changeStreamer = changeStreamer;
+		this.#replicaFile = replicaFile;
+	}
+	firstBackupReceived() {
+		return this.#firstBackupReceived.promise;
+	}
+	async run() {
+		this.#lc.info?.("starting backup monitor");
+		this.#initBackupLagMetric();
+		for await (const backedUp of this.#watermarks) {
+			if (!this.#latestBackup) this.#firstBackupReceived.resolve();
+			else if (backedUp.watermark < this.#latestBackup.watermark) {
+				this.#lc.warn?.(`ignoring earlier backup watermark`, { backedUp });
+				continue;
+			} else if (backedUp.watermark === this.#latestBackup.watermark) {
+				this.#lc.debug?.(`ignoring redundant backup watermark`, { backedUp });
+				continue;
+			}
+			this.#lc.info?.(`received backup watermark`, { backedUp });
+			this.#latestBackup = backedUp;
+			this.#changeStreamer.trackBackupWatermark(backedUp.watermark);
+		}
+		this.#lc.info?.("watermark stream closed. BackupMonitor stopped.");
+	}
+	stop() {
+		this.#watermarks.cancel();
+		return promiseVoid;
+	}
+	#initBackupLagMetric() {
+		getOrCreateGauge("replica", "backup_lag", {
+			description: "Latency from when a change is written to the replica to when it is backed up to litestream. It is expected to create a saw pattern from 0 to the configured ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_MINUTES.",
+			unit: "millisecond"
+		}).addCallback((o) => {
+			const latestBackup = this.#latestBackup;
+			if (!latestBackup) {
+				this.#lc.warn?.(`no backed up watermarks. unable to report replica.backup_lag`);
+				return;
+			}
+			const db = new Database(this.#lc, this.#replicaFile, { readonly: true });
+			try {
+				const { writeTimeMs } = db.prepare(`SELECT writeTimeMs FROM "_zero.replicationState"`).get();
+				const backupLag = Math.max(0, writeTimeMs - latestBackup.backupTimeMs);
+				o.observe(backupLag);
+			} catch (e) {
+				this.#lc.warn?.(`error measuring replica.backup_lag metric`, e);
+			} finally {
+				db.close();
+			}
+		});
+	}
+};
+//#endregion
+export { BackupMonitor };
+//# sourceMappingURL=backup-monitor.js.map

package/out/zero-cache/src/services/change-streamer/backup-monitor.js.map ADDED Viewed

@@ -0,0 +1 @@

+ {"version":3,"file":"backup-monitor.js","names":["#lc","#watermarks","#changeStreamer","#replicaFile","#firstBackupReceived","#initBackupLagMetric","#latestBackup"],"sources":["../../../../../../zero-cache/src/services/change-streamer/backup-monitor.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {getOrCreateGauge} from '../../observability/metrics.ts';\nimport type {Source} from '../../types/streams.ts';\nimport type {SingletonService} from '../service.ts';\nimport type {ChangeStreamerService} from './change-streamer.ts';\n\nexport type BackedUpWatermark = {\n watermark: string;\n writeTimeMs?: number | undefined; // optional for debuggin; available on v5\n backupTimeMs: number;\n};\n\nexport class BackupMonitor implements SingletonService {\n readonly id = 'backup-monitor';\n readonly #lc: LogContext;\n readonly #watermarks: Source<BackedUpWatermark>;\n readonly #changeStreamer: ChangeStreamerService;\n readonly #replicaFile: string;\n readonly #firstBackupReceived = resolver();\n\n #latestBackup: BackedUpWatermark | undefined;\n\n constructor(\n lc: LogContext,\n watermarks: Source<BackedUpWatermark>,\n changeStreamer: ChangeStreamerService,\n replicaFile: string,\n ) {\n this.#lc = lc;\n this.#watermarks = watermarks;\n this.#changeStreamer = changeStreamer;\n this.#replicaFile = replicaFile;\n }\n\n firstBackupReceived() {\n return this.#firstBackupReceived.promise;\n }\n\n async run() {\n this.#lc.info?.('starting backup monitor');\n this.#initBackupLagMetric();\n\n for await (const backedUp of this.#watermarks) {\n if (!this.#latestBackup) {\n this.#firstBackupReceived.resolve();\n } else if (backedUp.watermark < this.#latestBackup.watermark) {\n this.#lc.warn?.(`ignoring earlier backup watermark`, {backedUp});\n continue;\n } else if (backedUp.watermark === this.#latestBackup.watermark) {\n this.#lc.debug?.(`ignoring redundant backup watermark`, {backedUp});\n continue;\n }\n this.#lc.info?.(`received backup watermark`, {backedUp});\n this.#latestBackup = backedUp;\n this.#changeStreamer.trackBackupWatermark(backedUp.watermark);\n }\n this.#lc.info?.('watermark stream closed. BackupMonitor stopped.');\n }\n\n stop(): Promise<void> {\n this.#watermarks.cancel();\n return promiseVoid;\n }\n\n #initBackupLagMetric() {\n getOrCreateGauge('replica', 'backup_lag', {\n description:\n 'Latency from when a change is written to the replica ' +\n 'to when it is backed up to litestream. It is expected to create a saw ' +\n 'pattern from 0 to the configured ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_MINUTES.',\n unit: 'millisecond',\n }).addCallback(o => {\n const latestBackup = this.#latestBackup;\n if (!latestBackup) {\n this.#lc.warn?.(\n `no backed up watermarks. unable to report replica.backup_lag`,\n );\n return;\n }\n const db = new Database(this.#lc, this.#replicaFile, {readonly: true});\n try {\n const {writeTimeMs} = db\n .prepare(/*sql*/ `SELECT writeTimeMs FROM \"_zero.replicationState\"`)\n .get<{writeTimeMs: number}>();\n const backupLag = Math.max(0, writeTimeMs - latestBackup.backupTimeMs);\n o.observe(backupLag);\n } catch (e) {\n this.#lc.warn?.(`error measuring replica.backup_lag metric`, e);\n } finally {\n db.close();\n }\n });\n }\n}\n"],"mappings":";;;;;AAeA,IAAa,gBAAb,MAAuD;CACrD,KAAc;CACd;CACA;CACA;CACA;CACA,uBAAgC,SAAS;CAEzC;CAEA,YACE,IACA,YACA,gBACA,aACA;EACA,KAAKA,MAAM;EACX,KAAKC,cAAc;EACnB,KAAKC,kBAAkB;EACvB,KAAKC,eAAe;CACtB;CAEA,sBAAsB;EACpB,OAAO,KAAKC,qBAAqB;CACnC;CAEA,MAAM,MAAM;EACV,KAAKJ,IAAI,OAAO,yBAAyB;EACzC,KAAKK,qBAAqB;EAE1B,WAAW,MAAM,YAAY,KAAKJ,aAAa;GAC7C,IAAI,CAAC,KAAKK,eACR,KAAKF,qBAAqB,QAAQ;QAC7B,IAAI,SAAS,YAAY,KAAKE,cAAc,WAAW;IAC5D,KAAKN,IAAI,OAAO,qCAAqC,EAAC,SAAQ,CAAC;IAC/D;GACF,OAAO,IAAI,SAAS,cAAc,KAAKM,cAAc,WAAW;IAC9D,KAAKN,IAAI,QAAQ,uCAAuC,EAAC,SAAQ,CAAC;IAClE;GACF;GACA,KAAKA,IAAI,OAAO,6BAA6B,EAAC,SAAQ,CAAC;GACvD,KAAKM,gBAAgB;GACrB,KAAKJ,gBAAgB,qBAAqB,SAAS,SAAS;EAC9D;EACA,KAAKF,IAAI,OAAO,iDAAiD;CACnE;CAEA,OAAsB;EACpB,KAAKC,YAAY,OAAO;EACxB,OAAO;CACT;CAEA,uBAAuB;EACrB,iBAAiB,WAAW,cAAc;GACxC,aACE;GAGF,MAAM;EACR,CAAC,EAAE,aAAY,MAAK;GAClB,MAAM,eAAe,KAAKK;GAC1B,IAAI,CAAC,cAAc;IACjB,KAAKN,IAAI,OACP,8DACF;IACA;GACF;GACA,MAAM,KAAK,IAAI,SAAS,KAAKA,KAAK,KAAKG,cAAc,EAAC,UAAU,KAAI,CAAC;GACrE,IAAI;IACF,MAAM,EAAC,gBAAe,GACnB,QAAgB,kDAAkD,EAClE,IAA2B;IAC9B,MAAM,YAAY,KAAK,IAAI,GAAG,cAAc,aAAa,YAAY;IACrE,EAAE,QAAQ,SAAS;GACrB,SAAS,GAAG;IACV,KAAKH,IAAI,OAAO,6CAA6C,CAAC;GAChE,UAAU;IACR,GAAG,MAAM;GACX;EACF,CAAC;CACH;AACF"}

package/out/zero-cache/src/services/change-streamer/broadcast.d.ts CHANGED Viewed

@@ -1,17 +1,76 @@
 import type { LogContext } from '@rocicorp/logger';
-import type { WatermarkedChange } from './change-streamer-service.ts';
+import type { WatermarkedChange } from './change-streamer.ts';
 import type { Subscriber } from './subscriber.ts';
 export type BroadcastReleaseMode = 'all-subscribers' | 'consensus-timeout';
+/**
+ * Enables event-driven early release of a {@link Broadcast}. When provided and a
+ * majority has acked, the broadcast schedules its own consensus-timeout release
+ * proportional to how long the majority took to ack, instead of relying on the
+ * Forwarder's periodic progress tick.
+ */
+export type EarlyReleaseOptions = {
+    /** Time to wait, proportional to majority completion, before releasing. */
+    consensusTimeoutProportion: number;
+    /** Injectable timer, for deterministic testing. */
+    setTimeoutFn?: typeof setTimeout;
+    /** Injectable timer cancellation, for deterministic testing. */
+    clearTimeoutFn?: typeof clearTimeout;
+};
 /**
  * Initiates and tracks the progress of a change broadcasted to
- * a set of subscribers.
+ * a set of subscribers, implementing **consensus-based timeouts**
+ * to effect short-term graceful degradation when a minority of
+ * subscribers are behind.
+ *
+ * ### Background
+ *
+ * The purpose of flow control is to pull upstream replication changes
+ * no faster than the rate they are processed by downstream subscribers
+ * in the steady state. In the change-streamer, this is done by occasionally
+ * waiting for ACKs from subscribers before continuing; without doing so,
+ * I/O buffers fill up and cause the system to spend most of its time in GC.
+ *
+ * However, the naive algorithm of always waiting for all subscribers (e.g.
+ * `Promise.all()`) can behave poorly in scenarios where subscribers
+ * are imbalanced:
+ * * New subscribers may have a backlog of changes to catch up with.
+ *   Having all subscribers wait for the new subscriber to catch up results
+ *   in delaying the entire application.
+ * * Broken TCP connections similarly require all subscribers to wait until
+ *   connection liveness checks kick in and disconnect the subscriber.
  *
- * Creating a `Broadcast` automatically initiates the send.
+ * A simplistic approach is to add a limit to the amount of time waiting for
+ * subscribers, i.e. an ack timeout. However, deciding what this timeout
+ * should be is non-trivial because of the heterogeneous nature of changes;
+ * while most changes operate on single rows and are relatively predictable
+ * in terms of running time, some changes are table-wide operations and can
+ * legitimately take an arbitrary amount of time. In such scenarios, a
+ * timeout that is too short can stop progress on replication altogether.
  *
- * By default, {@link Broadcast.done} resolves when all subscribers
- * have acked the change. However, {@link Broadcast.checkProgress()}
- * can be called to resolve the broadcast earlier based on the flow
- * control policy.
+ * ### Consensus-based Timeout Algorithm
+ *
+ * To address these shortcomings, a "consensus-based timeout" algorithm is
+ * used:
+ * * Wait for more than half of the subscribers to finish. (In
+ *   case of a single node, or the case of one replication-manager
+ *   and one view-syncer, this reduces to waiting for all subscribers.)
+ * * Once more than half of the subscribers have finished, proceed after
+ *   the configured timeout, proportional to the time it took for the
+ *   majority of subscribers to ack, even if not all subscribers have
+ *   finished.
+ *
+ * In other words, the subscribers themselves are used to determine the
+ * timeout of each batch of changes; the majority determines this when
+ * they complete, upon which a timeout is logically started.
+ *
+ * In the common case, the remaining subscribers finish soon afterward and
+ * the timeout never elapses. However, in pathological cases where a minority
+ * of subscribers have a disproportionate amount of load, some will still
+ * be processing (or otherwise unresponsive). These subscribers are given
+ * a bounded amount of time to catch up at each flushed batch, up to the
+ * timeout interval. This guarantees eventual catchup because the
+ * subscribers with a backlog of changes necessarily have a higher
+ * processing rate than the subscribers that finished (and are made to wait).
  */
 export declare class Broadcast {
     #private;
@@ -24,79 +83,10 @@ export declare class Broadcast {
      * Broadcasts the `change` to the `subscribers` and tracks their
      * completion.
      */
-    constructor(subscribers: Iterable<Subscriber>, change: WatermarkedChange);
+    constructor(lc: LogContext, subscribers: Iterable<Subscriber>, change: WatermarkedChange, earlyRelease?: EarlyReleaseOptions);
     get isDone(): boolean;
     get done(): Promise<void>;
     get releaseMode(): BroadcastReleaseMode;
-    /**
-     * Checks for pathological situations in which flow should be reenabled
-     * before all subscribers have acked.
-     *
-     * ### Background
-     *
-     * The purpose of flow control is to pull upstream replication changes
-     * no faster than the rate as they are processed by downstream subscribers
-     * in the steady state. In the change-streamer, this is done by occasionally
-     * waiting for ACKs from subscribers before continuing; without doing so,
-     * I/O buffers fill up and cause the system to spend most of its time in GC.
-     *
-     * However, the naive algorithm of always waiting for all subscribers (e.g.
-     * `Promise.all()`) can behave poorly in scenarios where subscribers
-     * are imbalanced:
-     * * New subscribers may have a backlog of changes to catch up with.
-     *   Having all subscribers wait for the new subscriber to catch up results
-     *   in delaying the entire application.
-     * * Broken TCP connections similarly require all subscribers to wait until
-     *   connection liveness checks kick in and disconnect the subscriber.
-     *
-     * A simplistic approach is to add a limit to the amount of time waiting for
-     * subscribers, i.e. an ack timeout. However, deciding what this timeout
-     * should be is non-trivial because of the heterogeneous nature of changes;
-     * while most changes operate on single rows and are relatively predictable
-     * in terms of running time, some changes are table-wide operations and can
-     * legitimately take an arbitrary amount of time. In such scenarios, a
-     * timeout that is too short can stop progress on replication altogether.
-     *
-     * ### Consensus-based Timeout Algorithm
-     *
-     * To address these shortcomings, a "consensus-based timeout" algorithm is
-     * used:
-     * * Wait for more than half of the subscribers to finish. (In
-     *   case of a single node, or the case of one replication-manager
-     *   and one view-syncer, this reduces to waiting for all subscribers.)
-     * * Once more than half of the subscribers have finished, proceed after
-     *   a fixed timeout elapses (e.g. 1 second), even if not all subscribers
-     *   have finished.
-     *
-     * In other words, the subscribers themselves are used to determine the
-     * timeout of each batch of changes; the majority determines this when
-     * they complete, upon which a timeout is logically started.
-     *
-     * In the common case, the remaining subscribers finish soon afterward and
-     * the timeout never elapses. However, in pathological cases where a minority
-     * of subscribers have a disproportionate amount of load, some will still
-     * be processing (or otherwise unresponsive). These subscribers are given
-     * a bounded amount of time to catch up at each flushed batch, up to the
-     * timeout interval. This guarantees eventual catchup because the
-     * subscribers with a backlog of changes necessarily have a higher
-     * processing rate than the subscribers that finished (and are made to wait).
-     *
-     * ### Not implemented: Broken connection detection
-     *
-     * If a subscriber has not made progress for a certain interval, the
-     * algorithm could theoretically drop it preemptively, supplementing the
-     * existing websocket-level liveness checks.
-     *
-     * However, a more reliable approach would be to change the replicator
-     * to use non-blocking writes, and subsequently increase the frequency of
-     * connection-level liveness checks. The current synchronous replica writes
-     * can delay both ping responsiveness and change progress arbitrarily (e.g.
-     * a large index creation); an independently liveness check that is not
-     * delayed by synchronous writes on the subscriber would be a more failsafe
-     * solution.
-     *
-     * @returns `true` if the broadcast was already done or was marked done.
-     */
-    checkProgress(lc: LogContext, flowControlConsensusPaddingMs: number, now: number): boolean;
+    logSlowRequests(now: number): void;
 }
 //# sourceMappingURL=broadcast.d.ts.map

package/out/zero-cache/src/services/change-streamer/broadcast.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"broadcast.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/broadcast.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,~~8BAA8B~~,CAAC;~~AACpE~~,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,iBAAiB,CAAC;AAEhD,MAAM,MAAM,oBAAoB,GAAG,iBAAiB,GAAG,mBAAmB,CAAC;AAE3E~~;;;;;;;;;;GAUG~~;AACH,qBAAa,SAAS;;IACpB;;;OAGG;IACH,MAAM,CAAC,eAAe,CACpB,WAAW,EAAE,QAAQ,CAAC,UAAU,CAAC,EACjC,MAAM,EAAE,iBAAiB;~~IAmB3B~~;;;OAGG;~~gBACS~~,WAAW,EAAE,QAAQ,CAAC,UAAU,CAAC,~~EAAE~~,MAAM,EAAE,iBAAiB;~~IAsCxE~~,IAAI,MAAM,IAAI,OAAO,CAEpB;IAED,IAAI,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAExB;IAED,IAAI,WAAW,IAAI,oBAAoB,CAEtC;IAED~~;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAoEG;IACH~~,~~aAAa~~,~~CACX~~,~~EAAE,EAAE,UAAU,EACd,6BAA6B,EAAE,MAAM,EACrC,~~GAAG,EAAE,MAAM;~~CAkDd~~"}
1	+ {"version":3,"file":"broadcast.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/broadcast.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,sBAAsB,CAAC;AAC5D,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,iBAAiB,CAAC;AAEhD,MAAM,MAAM,oBAAoB,GAAG,iBAAiB,GAAG,mBAAmB,CAAC;AAE3E;;;;;GAKG;AACH,MAAM,MAAM,mBAAmB,GAAG;IAChC,2EAA2E;IAC3E,0BAA0B,EAAE,MAAM,CAAC;IACnC,mDAAmD;IACnD,YAAY,CAAC,EAAE,OAAO,UAAU,CAAC;IACjC,gEAAgE;IAChE,cAAc,CAAC,EAAE,OAAO,YAAY,CAAC;CACtC,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuDG;AACH,qBAAa,SAAS;;IACpB;;;OAGG;IACH,MAAM,CAAC,eAAe,CACpB,WAAW,EAAE,QAAQ,CAAC,UAAU,CAAC,EACjC,MAAM,EAAE,iBAAiB;IA0B3B;;;OAGG;gBAED,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,QAAQ,CAAC,UAAU,CAAC,EACjC,MAAM,EAAE,iBAAiB,EACzB,YAAY,CAAC,EAAE,mBAAmB;IA4EpC,IAAI,MAAM,IAAI,OAAO,CAEpB;IAED,IAAI,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAExB;IAED,IAAI,WAAW,IAAI,oBAAoB,CAEtC;IAED,eAAe,CAAC,GAAG,EAAE,MAAM;CA8B5B"}