@rocicorp/zero 1.11.0-canary.2 → 1.11.0-canary.21

package/out/zero-cache/src/config/zero-config.js

@@ -274,6 +274,57 @@ var zeroOptions = {
 				`(Note that this option has no effect for Postgres versions before 17.)`
 			]
 		},
+		pgReplicationSlotPerReplica: {
+			type: valita_exports.boolean().default(false),
+			desc: [`Allocates a dedicated replication slot per replication-manager, allowing multiple tasks`, `to run in tandem for high-availability.`],
+			hidden: true
+		},
+		pgHighAvailabilityReplication: {
+			type: valita_exports.boolean().optional(),
+			desc: [
+				`Runs replication-managers in high-availability mode, allowing multiple tasks to`,
+				`run concurrently, each using their own replication slot and maintaining a local`,
+				`change-log in sqlite.`,
+				``,
+				`In this configuration, the PG change-streamer db is not used or maintained. Consequently`,
+				`after running servers in high-availability mode, rolling back to a legacy singleton-mode task`,
+				`will trigger a resync in order to re-intialize the PG change-streamer db.`
+			],
+			hidden: true
+		},
+		pgResumeOrphanedSlotGracePeriodMs: {
+			type: valita_exports.number().default(2e4),
+			desc: [
+				`The grace period to wait before taking over an inactive replication slot, "resuming"`,
+				`the replica. With high availability replication, a new replica is generally "forked"`,
+				`from a running, active replica. Resumption happens as a last resort when no there are no`,
+				`active replicas, as it carries the risk of stealing the slot from a task that is attempting`,
+				`to reconnect to its slot.`,
+				``,
+				`In local development, where it is often expected for slots to be inactive, it is reasonable to set`,
+				`this to value to 0.`
+			],
+			hidden: true
+		},
+		pgReplicationEpoch: {
+			type: valita_exports.number().default(0),
+			desc: [
+				`Replication managers sharing the same epoch attempt to converge on a single`,
+				`generation (i.e. the result of an initial-sync); if multiple tasks are started`,
+				`simultaneously, they coordinate with each other such that one performs the`,
+				`initial-sync while the others wait for it.`,
+				``,
+				`Incrementing the epoch is the way to reliably force a resync; a replication manager`,
+				`on a new epoch will ignore any replicas on other epochs.`,
+				``,
+				`Note that changing the {bold ZERO_LITESTREAM_BACKUP_URL} generally achieves a resync`,
+				`as well, but with high available replication managers, changing only the backup url can`,
+				`lead to confusion if an old replication manager is performing initial sync, causing the`,
+				`new replication manager to wait for it. Incrementing the replication epoch is recommended`,
+				`instead.`
+			],
+			hidden: true
+		},
 		pgPartialIndexTriggers: {
 			type: valita_exports.boolean().default(false),
 			hidden: true
@@ -388,6 +439,19 @@ var zeroOptions = {
 		desc: [`Run SQLite quick_check and integrity_check when corruption is detected.`, `These checks scan the replica and can take a long time on large databases.`],
 		hidden: true
 	},
+	snapshotRowCacheSize: {
+		type: valita_exports.number().default(5e4),
+		desc: [
+			`The maximum number of entries in the per-worker cache of replica rows`,
+			`read while advancing IVM pipelines. Every client group on a sync worker`,
+			`reads the new and previous value of each replicated row change; the`,
+			`cache lets the client groups on a worker share those reads. A replicated`,
+			`transaction of N rows produces up to 2N entries.`,
+			``,
+			`Setting this to 0 disables the cache.`
+		],
+		hidden: true
+	},
 	enableQueryPlanner: {
 		type: valita_exports.boolean().default(true),
 		desc: [
@@ -399,6 +463,28 @@ var zeroOptions = {
 			`You can disable the planner if it is picking bad strategies.`
 		]
 	},
+	enableCorrelatedPredicatePushdown: {
+		type: valita_exports.boolean().default(true),
+		desc: [
+			`Copy a query's equality conditions on a relationship's join columns`,
+			`into the related subquery. This reduces the rows that are read when a`,
+			`change is pushed through the query.`,
+			``,
+			`This is a kill switch. It will be removed.`
+		],
+		hidden: true
+	},
+	enablePlannerAwarePushdown: {
+		type: valita_exports.boolean().default(true),
+		desc: [
+			`Copy the conditions of correlated predicate pushdown before the query`,
+			`planner runs instead of after, so that the planner can use them. This`,
+			`can change the plans that the planner picks.`,
+			``,
+			`Has no effect when correlated predicate pushdown is off.`
+		],
+		hidden: true
+	},
 	enableQueryCovering: {
 		type: valita_exports.boolean().default(true),
 		desc: [
@@ -410,6 +496,42 @@ var zeroOptions = {
 			`You can disable this if covering detection adds too much CPU overhead.`
 		]
 	},
+	deferIvmWrites: {
+		type: valita_exports.boolean().default(true),
+		desc: [
+			`Derive IVM advancements without writing to the replica snapshot.`,
+			``,
+			`A view-syncer normally makes each change durable in the SQLite snapshot`,
+			`it is reading from, because the next change has to read a base that`,
+			`includes the previous ones. Those writes are always rolled back -- the`,
+			`replicator is the only committer -- so they exist only to serve reads`,
+			`for the rest of the advancement.`,
+			``,
+			`When enabled (the default), the changes are held in memory and merged`,
+			`into each leaf scan instead, leaving the snapshot read-only, within the`,
+			`budget set by {bold deferIvmWritesHeapProportion}. Disable it to write`,
+			`every advancement through to the snapshot. The two modes are required`,
+			`to be indistinguishable to the pipelines above the source.`
+		],
+		hidden: true
+	},
+	deferIvmWritesHeapProportion: {
+		type: valita_exports.number().assert((value) => value > 0 && value <= 1, "must be greater than 0 and at most 1").default(.25),
+		desc: [
+			`With {bold deferIvmWrites}, the proportion of {bold --max-old-space-size}`,
+			`that the client groups of one sync worker may hold in memory at once.`,
+			`Each client group holds its own copy of the changes it is advancing`,
+			`through. Before an advancement starts, the number of rows it will`,
+			`change is reserved from this budget, at an assumed 1 KiB per row; an`,
+			`advancement that does not fit is written`,
+			`through to the replica snapshot instead, as when {bold deferIvmWrites}`,
+			`is off. The bytes the rows are estimated to hold are also added up as`,
+			`the client groups go, and a client group that takes the total past the`,
+			`budget writes the rows it holds through to the replica snapshot, and`,
+			`the rest of its changes after them.`
+		],
+		hidden: true
+	},
 	yieldThresholdMs: {
 		type: valita_exports.number().default(10),
 		desc: [
@@ -419,6 +541,30 @@ var zeroOptions = {
 			`the cost of reduced throughput.`
 		]
 	},
+	viewSyncerHydrationBudgetMs: {
+		type: valita_exports.number().assert((value) => Number.isSafeInteger(value) && value >= 0, "must be a nonnegative integer").default(0),
+		desc: [
+			`The soft time budget in milliseconds for hydrating inactive queries`,
+			`during a view-syncer hydration pass. Active and internal queries always`,
+			`finish, and time spent in custom-query transform round trips is not`,
+			`charged to the budget. An inactive query not reached before the budget`,
+			`is spent is evicted: its CVR record and the remaining TTL that would`,
+			`have kept it warm are both dropped. A value of 0 disables`,
+			`hydration-budget eviction.`
+		]
+	},
+	viewSyncerQueryHydrationTimeoutMs: {
+		type: valita_exports.number().assert((value) => Number.isSafeInteger(value) && value >= 0, "must be a nonnegative integer").default(0),
+		desc: [
+			`The maximum processing time in milliseconds that a view-syncer spends`,
+			`hydrating a single client query. Time spent yielding to other work is`,
+			`not counted. A query whose hydration exceeds this limit is aborted and`,
+			`removed from the client's view, and affected clients receive an error`,
+			`for the query. The query is then rejected without being run again for`,
+			`a cooldown period, after which a retry is allowed. Internal queries are`,
+			`never aborted. A value of 0 disables the limit.`
+		]
+	},
 	change: {
 		db: {
 			type: valita_exports.string().optional(),
@@ -557,6 +703,15 @@ var zeroOptions = {
 			desc: [`Controls the staged SQLite change-log rollout. Modes are cumulative:`, `{bold off}, {bold write}, {bold compare}, and {bold serve}.`],
 			hidden: true
 		},
+		pgChangeLogEnabled: {
+			type: valita_exports.boolean().default(true),
+			desc: [
+				`Whether the legacy Postgres change log remains authoritative for`,
+				`stream initialization, persistence, catchup, and upstream ACKs.`,
+				`Disabling it requires SQLite serve mode at 100 percent and a v5 backup.`
+			],
+			hidden: true
+		},
 		sqliteChangeLogReadPercent: {
 			type: valita_exports.number().default(0),
 			desc: [`The stable percentage of eligible catchup subscriptions served from SQLite.`],
@@ -630,7 +785,7 @@ var zeroOptions = {
 			]
 		},
 		flowControlConsensusTimeoutProportion: {
-			type: valita_exports.number().default(2),
+			type: valita_exports.number().default(4),
 			desc: [
 				`During periodic flow control checks (every 64kb), the amount of time to wait after the majority`,
 				`of subscribers have acked, proportional to that interval, after which replication will continue`,
@@ -644,7 +799,7 @@ var zeroOptions = {
 				``,
 				`For example, if the majority of subscribers ack a message in 2.5ms, a padding proportion of`,
 				`1.0 instructs replication to continue after an additional 2.5ms; for a proportion of 2.0, an`,
-				`additional 5.0ms, etc. The default value of 2.0 allows for a subscriber to be 3x slower than the`,
+				`additional 5.0ms, etc. The default value of 4.0 allows for a subscriber to be 5x slower than the`,
 				`majority in the steady state, while similarly bounding the extent to which a temporarily lagging`,
 				`subscriber (e.g. due to catchup) slows down the fleet.`,
 				``,
@@ -737,7 +892,32 @@ var zeroOptions = {
 			`A password used to administer zero-cache server, for example to access the`,
 			`/statz endpoint.`,
 			"",
-			"A password is optional in development mode but {bold required in production} mode."
+			"A password is optional in development mode but {bold required in production} mode.",
+			"",
+			"The admin password also grants access to everything that the",
+			"{bold operator-password} does."
+		]
+	},
+	operatorPassword: {
+		type: valita_exports.string().optional(),
+		desc: [
+			`A password that grants access only to the diagnostic endpoints that cannot`,
+			`expose application data, so that people who operate zero-cache can`,
+			`diagnose it without being able to read the data it serves. These are:`,
+			"",
+			`* {bold /profz} and {bold /profrmz}: CPU profiles. Profiles served for the`,
+			`  operator password have the source of every regular expression redacted,`,
+			`  since those can contain LIKE patterns from client queries.`,
+			`* {bold /statz}: counts and sizes.`,
+			`* {bold /plannerz}: the replica's schema and planner statistics.`,
+			"",
+			`All other protected endpoints (the inspector and {bold /heapz}) still`,
+			`require the {bold admin-password}.`,
+			"",
+			`For {bold /profrmz} in a multi-node deployment, the replication-manager`,
+			`must be configured with the same operator password.`,
+			"",
+			`If it is the same as the {bold admin-password}, it grants admin access.`
 		]
 	},
 	websocketCompression: {
@@ -832,6 +1012,19 @@ var zeroOptions = {
 				`* {bold ZERO_LOG_FORMAT} for the log type`
 			]
 		},
+		syncRequesterEnabled: {
+			type: valita_exports.boolean().default(true),
+			desc: [
+				`When backing up with {bold ZERO_LITESTREAM_BACKUP_USING_V5}, drives litestream`,
+				`backups via its control socket and determines backed-up watermarks from the`,
+				`completed syncs, instead of polling the backup with the vfs-query executable.`,
+				``,
+				`Syncs are requested every {bold ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_SECONDS}`,
+				`(a no-op when there are no changes to back up), and litestream's own monitor`,
+				`interval is relaxed to a less frequent backstop.`
+			],
+			hidden: true
+		},
 		vfsQueryExecutable: {
 			type: valita_exports.string().optional(),
 			desc: [`Path to the rocicorp vfs-query executable that runs the VFS-based`, `polling of backup watermark. This is required when backing up with V5.`]
@@ -912,7 +1105,7 @@ var zeroOptions = {
 			]
 		},
 		forceCheckpointThresholdMB: {
-			type: valita_exports.number().default(0),
+			type: valita_exports.number().default(256),
 			desc: [
 				`The size of the WAL file at which to pause writes and explicitly initiate a`,
 				`local litestream sync. This is a safeguard for the situation in which litestream's`,
@@ -931,8 +1124,7 @@ var zeroOptions = {
 				`emergency break.`,
 				``,
 				`This feature is only enabled with {bold ZERO_LITESTREAM_BACKUP_USING_V5}.`
-			],
-			hidden: true
+			]
 		},
 		maxWalSizeMB: {
 			type: valita_exports.number().default(10240),
@@ -949,11 +1141,10 @@ var zeroOptions = {
 				`normal operation. Set to {bold 0} to disable.`,
 				``,
 				`This feature is only enabled with {bold ZERO_LITESTREAM_BACKUP_USING_V5}.`
-			],
-			hidden: true
+			]
 		},
 		incrementalBackupIntervalMinutes: {
-			type: valita_exports.number().default(5),
+			type: valita_exports.number().default(15),
 			desc: [
 				`The interval between incremental v3 backups of the replica. Shorter intervals`,
 				`reduce the amount of change history that needs to be replayed when catching`,
@@ -966,17 +1157,21 @@ var zeroOptions = {
 			]
 		},
 		snapshotBackupIntervalHours: {
-			type: valita_exports.number().default(4),
+			type: valita_exports.number().default(12),
 			desc: [
-				`The interval between snapshot backups of the replica. Snapshot backups`,
-				`make a full copy of the database to a new litestream generation. This`,
-				`improves restore time at the expense of bandwidth. Applications with a`,
-				`large database and low write rate can increase this interval to reduce`,
-				`network usage for backups (litestream defaults to 24 hours).`,
+				`The interval between snapshot backups of the replica. Applies to both`,
+				`litestream v3 and v5 backups.`,
 				``,
-				`This option only applies to litestream v3 backups and will be deprecated/removed`,
-				`once the zero-cache is transitioned to litestream v5. For configuring v5 backup`,
-				`frequency, use {bold ZERO_LITESTREAM_SNAPSHOT_BACKUP_INTERVAL_HOURS_V5}.`
+				`For v3, each snapshot makes a full copy of the database to a new litestream`,
+				`generation. This improves restore time at the expense of bandwidth.`,
+				`Applications with a large database and low write rate can increase this`,
+				`interval to reduce network usage for backups (litestream defaults to 24 hours).`,
+				``,
+				`For v5, continuous compaction already keeps restore times low, so snapshots`,
+				`serve a different purpose: they keep every file a restore needs within about`,
+				`one interval of age, so that lifecycle policies can clean up old backups.`,
+				`v5 snapshots are built by compacting the existing backup files rather than`,
+				`reading the replica, so they do not block WAL checkpoints.`
 			]
 		},
 		incrementalBackupIntervalSeconds: {
@@ -994,30 +1189,6 @@ var zeroOptions = {
 				`{bold ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_MINUTES}.`
 			]
 		},
-		snapshotBackupIntervalHoursV5: {
-			type: valita_exports.number().default(720),
-			desc: [
-				`The interval between snapshot backups of the replica when`,
-				`{bold ZERO_LITESTREAM_BACKUP_USING_V5} is enabled.`,
-				``,
-				`By default, snapshots are effectively disabled (i.e. every 30 days)`,
-				`because v5 compaction fulfills the role that snapshots played in v3`,
-				`(i.e. because of litestream v5 compaction, restores will generally involve`,
-				`O(db-size) bytes.`,
-				``,
-				`Snapshots are disabled by default because they hold a read-lock on`,
-				`the database and prevent wal checkpoints, introducing the risk of large wal`,
-				`files for large databases with a high write rate.`,
-				``,
-				`If configuring the zero-cache to actually perform v5 snapshots, the`,
-				`{bold ZERO_LITESTREAM_MAX_WAL_SIZE_MB} option can be used to pause replication`,
-				`if the wal reaches a certain size and cannot be checkpointed because of an`,
-				`in-progress snapshot.`,
-				``,
-				`This option only applies to litestream v5 backups. For v3 backups, use`,
-				`{bold ZERO_LITESTREAM_SNAPSHOT_BACKUP_INTERVAL_HOURS}.`
-			]
-		},
 		restoreParallelism: {
 			type: valita_exports.number().default(48),
 			desc: [`The number of WAL files to download in parallel when performing the`, `initial restore of the replica from the backup.`]
@@ -1186,28 +1357,54 @@ function getNormalizedZeroConfig(opts = {}) {
 function getServerVersion(config) {
 	return config?.serverVersion ?? package_default.version;
 }
+/**
+* Returns whether `password` is the admin password. Endpoints that can expose
+* application data must use this, which never accepts the operator password.
+*/
 function isAdminPasswordValid(lc, config, password) {
-	if (!password && !config.adminPassword && isDevelopmentMode()) {
+	return getAccess(lc, config, void 0, password) === "admin";
+}
+/**
+* Returns the access that `password` grants, accepting either the admin
+* password or the operator password, or `undefined` if it grants none.
+*
+* Only endpoints that cannot expose application data may use this, and they
+* must not serve anything to `operator` access that `--operator-password`
+* documents as withheld.
+*/
+function getOperatorAccess(lc, config, password) {
+	return getAccess(lc, config, config.operatorPassword, password);
+}
+function getAccess(lc, { adminPassword }, operatorPassword, password) {
+	if (!password && !adminPassword && isDevelopmentMode()) {
 		warnOnce(lc, "No admin password set; allowing access in development mode only");
-		return true;
+		return "admin";
 	}
-	if (!config.adminPassword) {
+	if (!adminPassword && !operatorPassword) {
 		lc.warn?.("No admin password set; denying access");
-		return false;
+		return;
 	}
-	const passwordBuffer = Buffer.from(password ?? "");
-	const configBuffer = Buffer.from(config.adminPassword);
-	if (passwordBuffer.length !== configBuffer.length) {
-		timingSafeEqual(configBuffer, configBuffer);
-		lc.warn?.("Invalid admin password");
-		return false;
+	const isAdmin = passwordMatches(password, adminPassword);
+	const isOperator = passwordMatches(password, operatorPassword);
+	if (isAdmin) {
+		lc.debug?.("Admin password accepted");
+		return "admin";
 	}
-	if (!timingSafeEqual(passwordBuffer, configBuffer)) {
-		lc.warn?.("Invalid admin password");
+	if (isOperator) {
+		lc.debug?.("Operator password accepted");
+		return "operator";
+	}
+	lc.warn?.(operatorPassword ? "Invalid admin or operator password" : "Invalid admin password");
+}
+function passwordMatches(password, expected) {
+	if (!expected) return false;
+	const passwordBuffer = Buffer.from(password ?? "");
+	const expectedBuffer = Buffer.from(expected);
+	if (passwordBuffer.length !== expectedBuffer.length) {
+		timingSafeEqual(expectedBuffer, expectedBuffer);
 		return false;
 	}
-	lc.debug?.("Admin password accepted");
-	return true;
+	return timingSafeEqual(passwordBuffer, expectedBuffer);
 }
 var hasWarned = false;
 function warnOnce(lc, msg) {
@@ -1217,6 +1414,6 @@ function warnOnce(lc, msg) {
 	}
 }
 //#endregion
-export { ZERO_ENV_VAR_PREFIX, appOptions, getNormalizedZeroConfig, getServerVersion, getZeroConfig, isAdminPasswordValid, shardOptions, zeroOptions };
+export { ZERO_ENV_VAR_PREFIX, appOptions, getNormalizedZeroConfig, getOperatorAccess, getServerVersion, getZeroConfig, isAdminPasswordValid, shardOptions, zeroOptions };
 
 //# sourceMappingURL=zero-config.js.map