@rocicorp/zero 1.10.0-canary.0 → 1.10.0-canary.10

package/out/zero-cache/src/config/zero-config.js

@@ -273,6 +273,27 @@ var zeroOptions = {
 				``,
 				`(Note that this option has no effect for Postgres versions before 17.)`
 			]
+		},
+		pgStreamInboundTimeoutMs: {
+			type: valita_exports.number().optional(),
+			desc: [
+				`The time (in milliseconds) without any inbound message from the upstream`,
+				`wal sender after which the replication stream is considered unresponsive`,
+				`and torn down to force a reconnect.`,
+				``,
+				`Defaults to 2x the server's {bold wal_sender_timeout}. That suits idle`,
+				`streams, but a busy wal sender can be legitimately silent for longer —`,
+				`e.g. while decoding through WAL from unpublished tables, or assembling a`,
+				`large transaction that is only sent at commit. If the server's`,
+				`{bold wal_sender_timeout} is aggressive (some managed environments`,
+				`default it as low as 5 seconds), the resulting teardown aborts and`,
+				`replays the in-flight transaction, which can prevent replication from`,
+				`ever catching up on a large backlog. Set this option to widen the`,
+				`client-side threshold without changing the server setting.`,
+				``,
+				`(This option has no effect when {bold wal_sender_timeout} is 0, which`,
+				`disables inbound liveness detection entirely.)`
+			]
 		}
 	},
 	/** @deprecated */
@@ -347,6 +368,11 @@ var zeroOptions = {
 			`This is useful for debugging and performance tuning.`
 		]
 	},
+	sqliteCorruptionChecks: {
+		type: valita_exports.boolean().default(false),
+		desc: [`Run SQLite quick_check and integrity_check when corruption is detected.`, `These checks scan the replica and can take a long time on large databases.`],
+		hidden: true
+	},
 	enableQueryPlanner: {
 		type: valita_exports.boolean().default(true),
 		desc: [
@@ -396,12 +422,12 @@ var zeroOptions = {
 			]
 		},
 		statementTimeoutMs: {
-			type: valita_exports.number().default(2e4),
+			type: valita_exports.number().default(3e4),
 			desc: [
-				`Fail change-log transactions if a statement response from postgres is not received within`,
-				`the specified timeout. This differs from a postgres {bold statement_timeout} in that`,
-				`it is implemented to handle a pathological case in which Postgres does not return a`,
-				`response but otherwise believes the transaction to be idle.`
+				`Fail change-log db operations that make no progress within the specified timeout. This`,
+				`differs from a postgres {bold statement_timeout} in that it is implemented to handle a`,
+				`pathological case in which Postgres does not return a response but otherwise believes the`,
+				`connection to be idle (e.g. a half-open connection).`
 			],
 			hidden: true
 		},
@@ -513,6 +539,60 @@ var zeroOptions = {
 				`immediately, since the incoming request indicates that the task is registered as a target.`
 			]
 		},
+		sqliteChangeLogMode: {
+			type: literalUnion("off", "write", "compare", "serve").default("off"),
+			desc: [`Controls the staged SQLite change-log rollout. Modes are cumulative:`, `{bold off}, {bold write}, {bold compare}, and {bold serve}.`],
+			hidden: true
+		},
+		sqliteChangeLogReadPercent: {
+			type: valita_exports.number().default(0),
+			desc: [`The stable percentage of eligible catchup subscriptions served from SQLite.`],
+			hidden: true
+		},
+		sqliteChangeLogColdReadPercent: {
+			type: valita_exports.number().default(0),
+			desc: [
+				`The stable percentage of eligible catchup subscriptions served from a`,
+				`SQLite change log that is younger than its retention window, i.e. one`,
+				`seeded less than {bold --change-streamer-sqlite-change-log-retention-ms}`,
+				`ago. Selection uses the same hash as`,
+				`{bold --change-streamer-sqlite-change-log-read-percent}, so a task that`,
+				`serves cold is always one that would serve warm.`
+			],
+			hidden: true
+		},
+		sqliteChangeLogComparePercent: {
+			type: valita_exports.number().default(1),
+			desc: [`The stable percentage of committed transactions whose catchup output is`, `compared between the Postgres and SQLite change logs in {bold compare} mode.`],
+			hidden: true
+		},
+		sqliteChangeLogRetentionMs: {
+			type: valita_exports.number().default(6e4),
+			desc: [`The minimum time window retained in the SQLite change log.`],
+			hidden: true
+		},
+		sqliteChangeLogReadBatchRows: {
+			type: valita_exports.number().default(1e3),
+			desc: [`The target number of rows in each SQLite catchup read batch.`],
+			hidden: true
+		},
+		sqliteChangeLogPurgeBatchRows: {
+			type: valita_exports.number().default(1e3),
+			desc: [`The target number of rows in each SQLite purge batch.`],
+			hidden: true
+		},
+		sqliteChangeLogBarrierTimeoutMs: {
+			type: valita_exports.number().default(3e5),
+			desc: [
+				`The maximum wait for the SQLite required-head barrier. This is a`,
+				`backstop for a wedged replica on an idle shard, where waiting costs`,
+				`nothing and would otherwise go unnoticed. A shard with traffic is`,
+				`bounded well before this by the subscriber's backlog reaching its`,
+				`high water mark, which is the point at which waiting starts holding`,
+				`up replication.`
+			],
+			hidden: true
+		},
 		backPressureLimitHeapProportion: {
 			type: valita_exports.number().default(.04),
 			desc: [
@@ -536,22 +616,58 @@ var zeroOptions = {
 				`throughput.`
 			]
 		},
-		flowControlConsensusPaddingSeconds: {
-			type: valita_exports.number().default(1),
+		flowControlConsensusTimeoutProportion: {
+			type: valita_exports.number().default(2),
 			desc: [
-				`During periodic flow control checks (every 64kb), the amount of time to wait after the`,
-				`majority of subscribers have acked, after which replication will continue even if`,
-				`some subscribers have yet to ack. (Note that this is not a timeout for the {italic entire} send,`,
-				`but a timeout that starts {italic after} the majority of receivers have acked.)`,
+				`During periodic flow control checks (every 64kb), the amount of time to wait after the majority`,
+				`of subscribers have acked, proportional to that interval, after which replication will continue`,
+				`even if some subscribers have yet to ack.`,
 				``,
 				`This allows a bounded amount of time for backlogged subscribers to catch up on each flush`,
 				`without forcing all subscribers to wait for the entire backlog to be processed. It is also`,
 				`useful for mitigating the effect of unresponsive subscribers due to severed websocket`,
-				`connections (until liveness checks disconnect them).`,
+				`connections or pathological zombie situations (until liveness checks or laggard detection`,
+				`disconnects them).`,
+				``,
+				`For example, if the majority of subscribers ack a message in 2.5ms, a padding proportion of`,
+				`1.0 instructs replication to continue after an additional 2.5ms; for a proportion of 2.0, an`,
+				`additional 5.0ms, etc. The default value of 2.0 allows for a subscriber to be 3x slower than the`,
+				`majority in the steady state, while similarly bounding the extent to which a temporarily lagging`,
+				`subscriber (e.g. due to catchup) slows down the fleet.`,
+				``,
+				`Note that subscribers that continually exceed the timeout will eventually be disconnected and`,
+				`instructed to shutdown in order to protect the healthy subscribers. It is thus important that`,
+				`the proportion account for expected variance in processing speed across subscribers.`,
 				``,
 				`Set this to a negative number to disable early flow control releases. (Not recommended, but`,
 				`available as an emergency measure.)`
 			]
+		},
+		flowControlSlowSubscriberGracePeriodSeconds: {
+			type: valita_exports.number().default(30),
+			desc: [
+				`The period of time after which a lagging subscriber is disconnected and instructed to`,
+				`restart. A subscriber is considered lagging if it {italic continuously} (1) exceeds the`,
+				`consensus timeout and (2) fails to exceed the change rate of healthy subscribers. These`,
+				`conditions distinguish expected, temporary periods of slowness from pathological`,
+				`scenarios, such as zombie tasks, in which the subscriber is unlikely to recover.`,
+				``,
+				`In particular, the second condition excludes subscribers that are performing initial`,
+				`catchup, as the rate of change for catchup must eventually exceed the rate of upstream`,
+				`changes. Note, however, that catchup rate can be legitimately slow during (rare) expensive`,
+				`operations such as index creation.`,
+				``,
+				`Thus, this grace period caps the amount of time that a lagging subscriber degrades overall`,
+				`throughput, but should be long enough to allow for legitimate intervals of slowness.`,
+				``,
+				`Note that in the pathological case where a catchup operation exceeds this grace period,`,
+				`the system will eventually recover after the next backup/restore cycle, as the expensive`,
+				`operation (e.g. index creation) will have been applied to the restored replica and no longer`,
+				`executed during catchup.`,
+				``,
+				`Set this to 0 or a negative number to disable laggard detection. (Not recommended, but`,
+				`available as an emergency measure.)`
+			]
 		}
 	},
 	taskID: {
@@ -663,15 +779,20 @@ var zeroOptions = {
 			]
 		},
 		restoreUsingV5: {
-			type: valita_exports.boolean().default(false),
-			desc: [`Restores the backup using the {bold ZERO_LITESTREAM_EXECUTABLE_V5} if specified.`]
+			type: valita_exports.boolean().default(true),
+			desc: [
+				`Restores the backup using the {bold ZERO_LITESTREAM_EXECUTABLE_V5} if specified.`,
+				`This provides a recovery path if rolling back from {bold ZERO_LITESTREAM_BACKUP_USING_V5}`,
+				`as v5 restores from both v3 and v5 backups (whichever is more recent).`
+			]
 		},
 		backupUsingV5: {
 			type: valita_exports.boolean().default(false),
 			desc: [
 				`Backs up the replica using Litestream v0.5.x and monitors cleanup`,
 				`watermarks by reading the backup through the Litestream SQLite VFS.`,
-				`This requires {bold ZERO_LITESTREAM_RESTORE_USING_V5}.`
+				`This requires {bold ZERO_LITESTREAM_RESTORE_USING_V5} and`,
+				`{bold ZERO_LITESTREAM_VFS_QUERY_EXECUTABLE}`
 			]
 		},
 		configPath: {
@@ -686,21 +807,35 @@ var zeroOptions = {
 				`* {bold ZERO_LOG_FORMAT} for the log type`
 			]
 		},
-		vfsExtensionPath: {
-			type: valita_exports.string().default("/usr/local/lib/litestream-vfs.so"),
-			desc: [`Path to the Litestream v0.5.x SQLite VFS loadable extension used by`, `the backup watermark reader to query the backup directly.`]
+		configPathV5: {
+			type: valita_exports.string().default("./src/services/litestream/config-v5.yml"),
+			desc: [
+				`Path to the litestream v5 yaml config file. zero-cache will run this with its`,
+				`environment variables, which can be referenced in the file via $\\{ENV\\}`,
+				`substitution, for example:`,
+				`* {bold ZERO_REPLICA_FILE} for the db path`,
+				`* {bold ZERO_LITESTREAM_BACKUP_LOCATION} for the db replica url`,
+				`* {bold ZERO_LITESTREAM_LOG_LEVEL} for the log level`,
+				`* {bold ZERO_LOG_FORMAT} for the log type`
+			]
+		},
+		vfsQueryExecutable: {
+			type: valita_exports.string().optional(),
+			desc: [`Path to the rocicorp vfs-query executable that runs the VFS-based`, `polling of backup watermark. This is required when backing up with V5.`]
 		},
-		vfsProbeIntervalMs: {
-			type: valita_exports.number().default(30 * 1e3),
+		vfsPollIntervalMs: {
+			type: valita_exports.number().default(15 * 1e3),
 			desc: [
-				`Interval in milliseconds at which the standalone backup watermark reader`,
-				`logs the watermark when it is run without a parent worker. The integrated`,
-				`backup monitor requests watermarks on demand.`
+				`Interval in milliseconds litestream vfs extension polls the backup store (e.g. s3)`,
+				`to determine the most recent backup.`,
+				``,
+				`This, in turn, influences how quickly new backups are confirmed, allowing the`,
+				`change-streamer to ack the upstream change-source (e.g. replication slot).`
 			]
 		},
-		vfsProbeTimeoutMs: {
-			type: valita_exports.number().default(30 * 1e3),
-			desc: [`Timeout in milliseconds for requests to the Litestream VFS backup`, `watermark reader worker.`]
+		vfsPollTimeoutMs: {
+			type: valita_exports.number().default(10 * 1e3),
+			desc: [`Timeout in milliseconds for requests to the Litestream VFS poller.`]
 		},
 		vfsLogFile: {
 			type: valita_exports.string().optional(),
@@ -741,9 +876,12 @@ var zeroOptions = {
 			desc: [
 				`The size of the WAL file at which to perform an SQlite checkpoint to apply`,
 				`the writes in the WAL to the main database file. Each checkpoint creates`,
-				`a new WAL segment file that will be backed up by litestream. Smaller thresholds`,
+				`a new WAL segment file that will be backed up by litestream (v3). Smaller thresholds`,
 				`may improve read performance, at the expense of creating more files to download`,
-				`when restoring the replica from the backup.`
+				`when restoring the replica from the backup.`,
+				``,
+				`This setting is only relevant when replicating with litestream v3, and is ignored`,
+				`when replicating with litestream v5.`
 			]
 		},
 		minCheckpointPageCount: {
@@ -751,7 +889,10 @@ var zeroOptions = {
 			desc: [
 				`The WAL page count at which SQLite attempts a PASSIVE checkpoint, which`,
 				`transfers pages to the main database file without blocking writers.`,
-				`Defaults to {bold checkpointThresholdMB * 250} (since SQLite page size is 4KB).`
+				`Defaults to {bold checkpointThresholdMB * 250} (since SQLite page size is 4KB).`,
+				``,
+				`This setting is only relevant when replicating with litestream v3, and is ignored`,
+				`when replicating with litestream v5.`
 			]
 		},
 		maxCheckpointPageCount: {
@@ -759,26 +900,52 @@ var zeroOptions = {
 			desc: [
 				`The WAL page count at which SQLite performs a RESTART checkpoint, which`,
 				`blocks writers until complete. Defaults to {bold minCheckpointPageCount * 10}.`,
-				`Set to {bold 0} to disable RESTART checkpoints entirely.`
+				`Set to {bold 0} to disable RESTART checkpoints entirely.`,
+				``,
+				`This setting is only relevant when replicating with litestream v3, and is ignored`,
+				`when replicating with litestream v5.`
 			]
 		},
 		incrementalBackupIntervalMinutes: {
-			type: valita_exports.number().default(15),
+			type: valita_exports.number().default(5),
 			desc: [
-				`The interval between incremental backups of the replica. Shorter intervals`,
+				`The interval between incremental v3 backups of the replica. Shorter intervals`,
 				`reduce the amount of change history that needs to be replayed when catching`,
 				`up a new view-syncer, at the expense of increasing the number of files needed`,
-				`to download for the initial litestream restore.`
+				`to download for the initial litestream restore.`,
+				``,
+				`This option only applies to litestream v3 backups and will be deprecated/removed`,
+				`once the zero-cache is transitioned to litestream v5. For configuring v5 backup`,
+				`frequency, use {bold ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_SECONDS}.`
+			]
+		},
+		incrementalBackupIntervalSeconds: {
+			type: valita_exports.number().default(15),
+			desc: [
+				`The interval between incremental v5 backups of the replica. With litestream v5`,
+				`the upstream change source is not ACKed until the corresponding changes have been`,
+				`applied to the replica and backed up by litestream. As such, shorter intervals`,
+				`incur a higher number of backup storage writes and files managed (e.g. in s3),`,
+				`while longer intervals result requiring a larger buffer for changes upstream `,
+				`(e.g. per-replication slot wal records). The default value of 15 seconds targets`,
+				`an s3 API cost of ~$1/month (not counting storage costs).`,
+				``,
+				`This option only applies to litestream v5 backups. For v3 backups, use`,
+				`{bold ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_MINUTES}.`
 			]
 		},
 		snapshotBackupIntervalHours: {
-			type: valita_exports.number().default(12),
+			type: valita_exports.number().default(4),
 			desc: [
 				`The interval between snapshot backups of the replica. Snapshot backups`,
 				`make a full copy of the database to a new litestream generation. This`,
 				`improves restore time at the expense of bandwidth. Applications with a`,
 				`large database and low write rate can increase this interval to reduce`,
-				`network usage for backups (litestream defaults to 24 hours).`
+				`network usage for backups (litestream defaults to 24 hours).`,
+				``,
+				`This setting is applied when replicating with either litestream v3 or v5.`,
+				`Note, however, that snapshots are generally not needed to improve restore time`,
+				`with v5, and so a longer interval (e.g. the litestream default of 24h) is fine.`
 			]
 		},
 		restoreParallelism: {